script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from subprocess import call, check_call, check_output, Popen, PIPE, CalledProcessError
   7 import os
   8 import tarfile
   9 import sys
  10 import time
  11 import random
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from sysconfig import get_path
  20 import platform
  21
  22 import logging
  23
  24 try:
  25     from waflib.Build import CACHE_SUFFIX
  26 except ImportError:
  27     sys.path.insert(0, "./third_party/waf")
  28     from waflib.Build import CACHE_SUFFIX
  29
  30 logging.basicConfig(format='%(asctime)s %(message)s')
  31 logger = logging.getLogger('autobuild')
  32 logger.setLevel(logging.INFO)
  33
  34 os.environ["PYTHONUNBUFFERED"] = "1"
  35
  36 # This speeds up testing remarkably.
  37 os.environ['TDB_NO_FSYNC'] = '1'
  38
  39 # allow autobuild to run within git rebase -i
  40 if "GIT_DIR" in os.environ:
  41     del os.environ["GIT_DIR"]
  42 if "GIT_WORK_TREE" in os.environ:
  43     del os.environ["GIT_WORK_TREE"]
  44
  45 def find_git_root():
  46     '''get to the top of the git repo'''
  47     p = os.getcwd()
  48     while p != '/':
  49         if os.path.exists(os.path.join(p, ".git")):
  50             return p
  51         p = os.path.abspath(os.path.join(p, '..'))
  52     return None
  53
  54
  55 gitroot = find_git_root()
  56 if gitroot is None:
  57     raise Exception("Failed to find git root")
  58
  59
  60 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  61
  62 parser = OptionParser()
  63 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  64 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  65 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  66 parser.add_option("--skip-dependencies", help="skip to run task dependency tasks", default=False, action="store_true")
  67 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  68                   default=def_testbase)
  69 parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase,
  70                   default=None)
  71 parser.add_option("--passcmd", help="command to run on success", default=None)
  72 parser.add_option("--verbose", help="show all commands as they are run",
  73                   default=False, action="store_true")
  74 parser.add_option("--rebase", help="rebase on the given tree before testing",
  75                   default=None, type='str')
  76 parser.add_option("--pushto", help="push to a git url on success",
  77                   default=None, type='str')
  78 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  79                   default=False, action="store_true")
  80 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  81                   default=False, action="store_true")
  82 parser.add_option("--retry", help="automatically retry if master changes",
  83                   default=False, action="store_true")
  84 parser.add_option("--email", help="send email to the given address on failure",
  85                   type='str', default=None)
  86 parser.add_option("--email-from", help="send email from the given address",
  87                   type='str', default="autobuild@samba.org")
  88 parser.add_option("--email-server", help="send email via the given server",
  89                   type='str', default='localhost')
  90 parser.add_option("--always-email", help="always send email, even on success",
  91                   action="store_true")
  92 parser.add_option("--daemon", help="daemonize after initial setup",
  93                   action="store_true")
  94 parser.add_option("--branch", help="the branch to work on (default=master)",
  95                   default="master", type='str')
  96 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  97                   default=gitroot, type='str')
  98 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  99                   default=False, action="store_true")
 100 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
 101                   default='')
 102 parser.add_option("--enable-coverage", dest='enable_coverage',
 103                   action="store_const", const='--enable-coverage', default='',
 104                   help="Add --enable-coverage option while configure")
 105
 106 (options, args) = parser.parse_args()
 107
 108 if options.retry:
 109     if options.rebase is None:
 110         raise Exception('You can only use --retry if you also rebase')
 111
 112 if options.verbose:
 113     logger.setLevel(logging.DEBUG)
 114
 115 if options.full_testbase is not None:
 116     testbase = options.full_testbase
 117 else:
 118     testbase = "%s/b%u" % (options.testbase, os.getpid())
 119 test_master = "%s/master" % testbase
 120 test_prefix = "%s/prefix" % testbase
 121 test_tmpdir = "%s/tmp" % testbase
 122 os.environ['TMPDIR'] = test_tmpdir
 123
 124 if options.enable_coverage:
 125     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 126 else:
 127     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 128
 129 if options.enable_coverage:
 130     PUBLISH_DOCS = "mkdir -p ${LOG_BASE}/public && mv output/htmldocs ${LOG_BASE}/public/htmldocs"
 131 else:
 132     PUBLISH_DOCS = 'echo "HTML documentation publishing skipped since no --enable-coverage specified"'
 133
 134 CLEAN_SOURCE_TREE_CMD = "cd ${TEST_SOURCE_DIR} && script/clean-source-tree.sh"
 135
 136
 137 def check_symbols(sofile, expected_symbols=""):
 138     return "objdump --dynamic-syms " + sofile + " | " + \
 139            "awk \'$0 !~ /" + expected_symbols + "/ {if ($2 == \"g\" && $3 ~ /D(F|O)/ && $4 ~ /(.bss|.text)/ && $7 !~ /(__gcov_|mangle_path)/) exit 1}\'"
 140
 141 if args:
 142     # If we are only running specific test,
 143     # do not sleep randomly to wait for it to start
 144     def random_sleep(low, high):
 145         return 'sleep 1'
 146 else:
 147     def random_sleep(low, high):
 148         return 'sleep {}'.format(random.randint(low, high))
 149
 150 cleanup_list = []
 151
 152 builddirs = {
 153     "ctdb": "ctdb",
 154     "tdb": "lib/tdb",
 155     "talloc": "lib/talloc",
 156     "replace": "lib/replace",
 157     "tevent": "lib/tevent",
 158     "pidl": "pidl",
 159     "docs-xml": "docs-xml"
 160 }
 161
 162 ctdb_configure_params = " --enable-developer ${PREFIX}"
 163 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 164
 165 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 166 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 167 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 168 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 169 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 170 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!tevent,!pytevent,!popt"
 171 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 172
 173
 174 def format_option(name, value=None):
 175     """Format option as str list."""
 176     if value is None:  # boolean option
 177         return [name]
 178     if not isinstance(value, list):  # single value option
 179         value = [value]
 180     # repeatable option
 181     return ['{}={}'.format(name, item) for item in value]
 182
 183
 184 def make_test(
 185         cmd='make testonly',
 186         INJECT_SELFTEST_PREFIX=1,
 187         TESTS='',
 188         include_envs=None,
 189         exclude_envs=None):
 190
 191     test_options = []
 192     if include_envs:
 193         test_options = format_option('--include-env', include_envs)
 194     if exclude_envs:
 195         test_options = format_option('--exclude-env', exclude_envs)
 196     if test_options:
 197         # join envs options to original test options
 198         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 199
 200     _options = []
 201
 202     # Allow getting a full CI with
 203     # git push -o ci.variable='AUTOBUILD_FAIL_IMMEDIATELY=0'
 204
 205     FAIL_IMMEDIATELY = os.getenv("AUTOBUILD_FAIL_IMMEDIATELY", "1")
 206
 207     if int(FAIL_IMMEDIATELY):
 208         _options.append('FAIL_IMMEDIATELY=1')
 209     if TESTS:
 210         _options.append("TESTS='{}'".format(TESTS))
 211
 212     if INJECT_SELFTEST_PREFIX:
 213         _options.append("TEST_OPTIONS='--with-selftest-prefix={}'".format("${SELFTEST_PREFIX}"))
 214         _options.append("--directory='{}'".format("${TEST_SOURCE_DIR}"))
 215
 216     return ' '.join([cmd] + _options)
 217
 218
 219 # When updating this list, also update .gitlab-ci.yml to add the job
 220 # and to make it a dependency of 'page' for the coverage report.
 221
 222 tasks = {
 223     "ctdb": {
 224         "sequence": [
 225             ("random-sleep", random_sleep(300, 900)),
 226             ("configure", "./configure " + ctdb_configure_params),
 227             ("make", "make all"),
 228             ("install", "make install"),
 229             ("test", "make autotest"),
 230             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 231             ("clean", "make clean"),
 232         ],
 233     },
 234     "docs-xml": {
 235         "sequence": [
 236             ("random-sleep", random_sleep(300, 900)),
 237             ("autoconf", "autoconf"),
 238             ("configure", "./configure"),
 239             ("make", "make html htmlman"),
 240             ("publish-docs", PUBLISH_DOCS),
 241             ("clean", "make clean"),
 242         ],
 243     },
 244
 245     "samba-def-build": {
 246         "git-clone-required": True,
 247         "sequence": [
 248             ("configure", "./configure.developer" + samba_configure_params),
 249             ("make", "make -j"),
 250             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 251             ("chmod-R-a-w", "chmod -R a-w ."),
 252         ],
 253     },
 254
 255     "samba-mit-build": {
 256         "git-clone-required": True,
 257         "sequence": [
 258             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 259             ("make", "make -j"),
 260             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 261             ("chmod-R-a-w", "chmod -R a-w ."),
 262         ],
 263     },
 264
 265     "samba-nt4-build": {
 266         "git-clone-required": True,
 267         "sequence": [
 268             ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json" + samba_configure_params),
 269             ("make", "make -j"),
 270             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 271             ("chmod-R-a-w", "chmod -R a-w ."),
 272         ],
 273     },
 274
 275     "samba-h5l-build": {
 276         "git-clone-required": True,
 277         "sequence": [
 278             ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5" + samba_configure_params),
 279             ("make", "make -j"),
 280             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 281             ("chmod-R-a-w", "chmod -R a-w ."),
 282         ],
 283     },
 284
 285     "samba-without-smb1-build": {
 286         "git-clone-required": True,
 287         "sequence": [
 288             ("configure", "./configure.developer --without-smb1-server --without-ad-dc" + samba_configure_params),
 289             ("make", "make -j"),
 290             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 291             ("chmod-R-a-w", "chmod -R a-w ."),
 292         ],
 293     },
 294
 295     "samba-no-opath-build": {
 296         "git-clone-required": True,
 297         "sequence": [
 298             ("configure", "ADDITIONAL_CFLAGS='-DDISABLE_OPATH=1 -DDISABLE_VFS_OPEN_HOW_RESOLVE_NO_SYMLINKS=1 -DDISABLE_PROC_FDS=1' ./configure.developer --without-ad-dc " + samba_configure_params),
 299             ("make", "make -j"),
 300             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 301             ("chmod-R-a-w", "chmod -R a-w ."),
 302         ],
 303     },
 304
 305     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 306     "samba": {
 307         "sequence": [
 308             ("random-sleep", random_sleep(300, 900)),
 309             ("configure", "./configure.developer" + samba_configure_params),
 310             ("make", "make -j"),
 311             ("test", make_test(exclude_envs=[
 312             "none",
 313             "nt4_dc",
 314             "nt4_dc_smb1",
 315             "nt4_dc_smb1_done",
 316             "nt4_dc_schannel",
 317             "nt4_member",
 318             "ad_dc",
 319             "ad_dc_smb1",
 320             "ad_dc_smb1_done",
 321             "ad_dc_backup",
 322             "ad_dc_ntvfs",
 323             "ad_dc_default",
 324             "ad_dc_default_smb1",
 325             "ad_dc_slowtests",
 326             "ad_dc_no_nss",
 327             "ad_dc_no_ntlm",
 328             "fl2003dc",
 329             "fl2008dc",
 330             "fl2008r2dc",
 331             "ad_member",
 332             "ad_member_idmap_rid",
 333             "admem_idmap_autorid",
 334             "ad_member_idmap_ad",
 335             "ad_member_rfc2307",
 336             "ad_member_idmap_nss",
 337             "ad_member_oneway",
 338             "chgdcpass",
 339             "vampire_2000_dc",
 340             "fl2000dc",
 341             "fileserver",
 342             "fileserver_smb1",
 343             "fileserver_smb1_done",
 344             "maptoguest",
 345             "simpleserver",
 346             "backupfromdc",
 347             "restoredc",
 348             "renamedc",
 349             "offlinebackupdc",
 350             "labdc",
 351             "preforkrestartdc",
 352             "proclimitdc",
 353             "promoted_dc",
 354             "vampire_dc",
 355             "rodc",
 356             "ad_dc_default",
 357             "ad_dc_default_smb1",
 358             "ad_dc_default_smb1_done",
 359             "ad_dc_slowtests",
 360             "schema_pair_dc",
 361             "schema_dc",
 362             "clusteredmember",
 363             "ad_dc_fips",
 364             "ad_member_fips",
 365             ])),
 366             ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
 367             ("lcov", LCOV_CMD),
 368             ("install", "make install"),
 369             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 370             ("clean", "make clean"),
 371         ],
 372     },
 373
 374     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 375     "samba-mitkrb5": {
 376         "sequence": [
 377             ("random-sleep", random_sleep(300, 900)),
 378             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 379             ("make", "make -j"),
 380             ("test", make_test(exclude_envs=[
 381             "none",
 382             "nt4_dc",
 383             "nt4_dc_smb1",
 384             "nt4_dc_smb1_done",
 385             "nt4_dc_schannel",
 386             "nt4_member",
 387             "ad_dc",
 388             "ad_dc_smb1",
 389             "ad_dc_smb1_done",
 390             "ad_dc_backup",
 391             "ad_dc_ntvfs",
 392             "ad_dc_default",
 393             "ad_dc_default_smb1",
 394             "ad_dc_default_smb1_done",
 395             "ad_dc_slowtests",
 396             "ad_dc_no_nss",
 397             "ad_dc_no_ntlm",
 398             "fl2003dc",
 399             "fl2008dc",
 400             "fl2008r2dc",
 401             "ad_member",
 402             "ad_member_idmap_rid",
 403             "admem_idmap_autorid",
 404             "ad_member_idmap_ad",
 405             "ad_member_rfc2307",
 406             "ad_member_idmap_nss",
 407             "ad_member_oneway",
 408             "chgdcpass",
 409             "vampire_2000_dc",
 410             "fl2000dc",
 411             "fileserver",
 412             "fileserver_smb1",
 413             "fileserver_smb1_done",
 414             "maptoguest",
 415             "simpleserver",
 416             "backupfromdc",
 417             "restoredc",
 418             "renamedc",
 419             "offlinebackupdc",
 420             "labdc",
 421             "preforkrestartdc",
 422             "proclimitdc",
 423             "promoted_dc",
 424             "vampire_dc",
 425             "rodc",
 426             "ad_dc_default",
 427             "ad_dc_default_smb1",
 428             "ad_dc_default_smb1_done",
 429             "ad_dc_slowtests",
 430             "schema_pair_dc",
 431             "schema_dc",
 432             "clusteredmember",
 433             "ad_dc_fips",
 434             "ad_member_fips",
 435             ])),
 436             ("lcov", LCOV_CMD),
 437             ("install", "make install"),
 438             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 439             ("clean", "make clean"),
 440         ],
 441     },
 442
 443     "samba-nt4": {
 444         "dependency": "samba-nt4-build",
 445         "sequence": [
 446             ("random-sleep", random_sleep(300, 900)),
 447             ("test", make_test(include_envs=[
 448             "nt4_dc",
 449             "nt4_dc_smb1",
 450             "nt4_dc_smb1_done",
 451             "nt4_dc_schannel",
 452             "nt4_member",
 453             "simpleserver",
 454             ])),
 455             ("lcov", LCOV_CMD),
 456             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 457         ],
 458     },
 459
 460     "samba-fileserver": {
 461         "dependency": "samba-h5l-build",
 462         "sequence": [
 463             ("random-sleep", random_sleep(300, 900)),
 464             ("test", make_test(include_envs=[
 465             "fileserver",
 466             "fileserver_smb1",
 467             "fileserver_smb1_done",
 468             "maptoguest",
 469             "ktest", # ktest is also tested in samba-ktest-mit samba
 470                      # and samba-mitkrb5 but is tested here against
 471                      # a system Heimdal
 472             ])),
 473             ("lcov", LCOV_CMD),
 474             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 475         ],
 476     },
 477
 478     "samba-fileserver-without-smb1": {
 479         "dependency": "samba-without-smb1-build",
 480         "sequence": [
 481             ("random-sleep", random_sleep(300, 900)),
 482             ("test", make_test(include_envs=["fileserver"])),
 483             ("lcov", LCOV_CMD),
 484             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 485         ],
 486     },
 487
 488     # This is a full build without the AD DC so we test the build with
 489     # MIT Kerberos from the current system.  Runtime behaviour is
 490     # confirmed via the ktest (static ccache and keytab) environment
 491
 492     # This environment also used to confirm we can still build with --with-libunwind
 493     "samba-ktest-mit": {
 494         "sequence": [
 495             ("random-sleep", random_sleep(300, 900)),
 496             ("configure", "./configure.developer --without-ad-dc --with-libunwind --with-system-mitkrb5 " + samba_configure_params),
 497             ("make", "make -j"),
 498             ("test", make_test(include_envs=[
 499             "ktest", # ktest is also tested in fileserver, samba and
 500                      # samba-mitkrb5 but is tested here against a
 501                      # system MIT krb5
 502             ])),
 503             ("lcov", LCOV_CMD),
 504             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 505         ],
 506     },
 507
 508     "samba-admem": {
 509         "dependency": "samba-def-build",
 510         "sequence": [
 511             ("random-sleep", random_sleep(300, 900)),
 512             ("test", make_test(include_envs=[
 513             "ad_member",
 514             "ad_member_idmap_rid",
 515             "admem_idmap_autorid",
 516             "ad_member_idmap_ad",
 517             "ad_member_rfc2307",
 518             "ad_member_idmap_nss",
 519             "ad_member_offlogon",
 520             ])),
 521             ("lcov", LCOV_CMD),
 522             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 523         ],
 524     },
 525
 526     "samba-no-opath1": {
 527         "dependency": "samba-no-opath-build",
 528         "sequence": [
 529             ("random-sleep", random_sleep(300, 900)),
 530             ("test", make_test(
 531                 cmd="make testonly DISABLE_OPATH=1",
 532                 include_envs=[
 533                 "nt4_dc",
 534                 "nt4_dc_smb1",
 535                 "nt4_dc_smb1_done",
 536                 "nt4_dc_schannel",
 537                 "nt4_member",
 538                 "simpleserver",
 539                 ])),
 540             ("lcov", LCOV_CMD),
 541             ("check-clean-tree", "script/clean-source-tree.sh"),
 542         ],
 543     },
 544
 545     "samba-no-opath2": {
 546         "dependency": "samba-no-opath-build",
 547         "sequence": [
 548             ("random-sleep", random_sleep(300, 900)),
 549             ("test", make_test(
 550                 cmd="make testonly DISABLE_OPATH=1",
 551                 include_envs=[
 552                 "fileserver",
 553                 "fileserver_smb1",
 554                 "fileserver_smb1_done",
 555                 ])),
 556             ("lcov", LCOV_CMD),
 557             ("check-clean-tree", "script/clean-source-tree.sh"),
 558         ],
 559     },
 560
 561     "samba-ad-dc-1": {
 562         "dependency": "samba-def-build",
 563         "sequence": [
 564             ("random-sleep", random_sleep(1, 1)),
 565             ("test", make_test(include_envs=[
 566             "ad_dc",
 567             "ad_dc_smb1",
 568             "ad_dc_smb1_done",
 569             "ad_dc_no_nss",
 570             "ad_dc_no_ntlm",
 571             ])),
 572             ("lcov", LCOV_CMD),
 573             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 574         ],
 575     },
 576
 577     "samba-ad-dc-2": {
 578         "dependency": "samba-def-build",
 579         "sequence": [
 580             ("random-sleep", random_sleep(1, 1)),
 581             ("test", make_test(include_envs=[
 582             "vampire_dc",
 583             "vampire_2000_dc",
 584             "rodc",
 585             ])),
 586             ("lcov", LCOV_CMD),
 587             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 588         ],
 589     },
 590
 591     "samba-ad-dc-3": {
 592         "dependency": "samba-def-build",
 593         "sequence": [
 594             ("random-sleep", random_sleep(1, 1)),
 595             ("test", make_test(include_envs=[
 596             "promoted_dc",
 597             "chgdcpass",
 598             "preforkrestartdc",
 599             "proclimitdc",
 600             ])),
 601             ("lcov", LCOV_CMD),
 602             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 603         ],
 604     },
 605
 606     "samba-ad-dc-4a": {
 607         "dependency": "samba-def-build",
 608         "sequence": [
 609             ("random-sleep", random_sleep(1, 1)),
 610             ("test", make_test(include_envs=[
 611             "fl2000dc",
 612             "ad_member_oneway",
 613             "fl2003dc",
 614             ])),
 615             ("lcov", LCOV_CMD),
 616             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 617         ],
 618     },
 619     "samba-ad-dc-4b": {
 620         "dependency": "samba-def-build",
 621         "sequence": [
 622             ("random-sleep", random_sleep(1, 1)),
 623             ("test", make_test(include_envs=[
 624             "fl2008dc",
 625             "fl2008r2dc",
 626             ])),
 627             ("lcov", LCOV_CMD),
 628             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 629         ],
 630     },
 631
 632     "samba-ad-dc-5": {
 633         "dependency": "samba-def-build",
 634         "sequence": [
 635             ("random-sleep", random_sleep(1, 1)),
 636             ("test", make_test(include_envs=[
 637             "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
 638             ("lcov", LCOV_CMD),
 639             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 640         ],
 641     },
 642
 643     "samba-ad-dc-6": {
 644         "dependency": "samba-def-build",
 645         "sequence": [
 646             ("random-sleep", random_sleep(1, 1)),
 647             ("test", make_test(include_envs=["ad_dc_slowtests", "ad_dc_backup"])),
 648             ("lcov", LCOV_CMD),
 649             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 650         ],
 651     },
 652
 653     "samba-schemaupgrade": {
 654         "dependency": "samba-def-build",
 655         "sequence": [
 656             ("random-sleep", random_sleep(1, 1)),
 657             ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 658             ("lcov", LCOV_CMD),
 659             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 660         ],
 661     },
 662
 663     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 664     # This is currently the longest task, so we don't randomly delay it.
 665     "samba-ad-dc-ntvfs": {
 666         "dependency": "samba-def-build",
 667         "sequence": [
 668             ("random-sleep", random_sleep(1, 1)),
 669             ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 670             ("lcov", LCOV_CMD),
 671             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 672         ],
 673     },
 674
 675     # Test fips compliance
 676     "samba-fips": {
 677         "dependency": "samba-mit-build",
 678         "sequence": [
 679             ("random-sleep", random_sleep(1, 1)),
 680             ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
 681             # TODO: This seems to generate only an empty samba-fips.info ("lcov", LCOV_CMD),
 682             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 683         ],
 684     },
 685
 686     # run the backup/restore testenvs separately as they're fairly standalone
 687     # (and CI seems to max out at ~3 different DCs running at once)
 688     "samba-ad-back1": {
 689         "dependency": "samba-def-build",
 690         "sequence": [
 691             ("random-sleep", random_sleep(300, 900)),
 692             ("test", make_test(include_envs=[
 693             "backupfromdc",
 694             "restoredc",
 695             "renamedc",
 696             ])),
 697             ("lcov", LCOV_CMD),
 698             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 699         ],
 700     },
 701     "samba-ad-back2": {
 702         "dependency": "samba-def-build",
 703         "sequence": [
 704             ("random-sleep", random_sleep(300, 900)),
 705             ("test", make_test(include_envs=[
 706             "backupfromdc",
 707             "offlinebackupdc",
 708             "labdc",
 709             ])),
 710             ("lcov", LCOV_CMD),
 711             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 712         ],
 713     },
 714
 715     "samba-admem-mit": {
 716         "dependency": "samba-mit-build",
 717         "sequence": [
 718             ("random-sleep", random_sleep(1, 1)),
 719             ("test", make_test(include_envs=[
 720             "ad_member",
 721             "ad_member_idmap_rid",
 722             "admem_idmap_autorid",
 723             "ad_member_idmap_ad",
 724             "ad_member_rfc2307",
 725             "ad_member_idmap_nss",
 726             "ad_member_offlogon",
 727             ])),
 728             ("lcov", LCOV_CMD),
 729             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 730         ],
 731     },
 732
 733     "samba-addc-mit-1": {
 734         "dependency": "samba-mit-build",
 735         "sequence": [
 736             ("random-sleep", random_sleep(1, 1)),
 737             ("test", make_test(include_envs=[
 738             "ad_dc",
 739             "ad_dc_smb1",
 740             "ad_dc_smb1_done",
 741             "ad_dc_no_nss",
 742             "ad_dc_no_ntlm",
 743             ])),
 744             ("lcov", LCOV_CMD),
 745             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 746         ],
 747     },
 748
 749     "samba-addc-mit-4a": {
 750         "dependency": "samba-mit-build",
 751         "sequence": [
 752             ("random-sleep", random_sleep(1, 1)),
 753             ("test", make_test(include_envs=[
 754             "fl2000dc",
 755             "ad_member_oneway",
 756             "fl2003dc",
 757             ])),
 758             ("lcov", LCOV_CMD),
 759             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 760         ],
 761     },
 762     "samba-addc-mit-4b": {
 763         "dependency": "samba-mit-build",
 764         "sequence": [
 765             ("random-sleep", random_sleep(1, 1)),
 766             ("test", make_test(include_envs=[
 767             "fl2008dc",
 768             "fl2008r2dc",
 769             ])),
 770             ("lcov", LCOV_CMD),
 771             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 772         ],
 773     },
 774
 775     "samba-test-only": {
 776         "sequence": [
 777             ("configure", "./configure.developer  --abi-check-disable" + samba_configure_params),
 778             ("make", "make -j"),
 779             ("test", make_test(TESTS="${TESTS}")),
 780             ("lcov", LCOV_CMD),
 781         ],
 782     },
 783
 784     # Test cross-compile infrastructure
 785     "samba-xc": {
 786         "sequence": [
 787             ("random-sleep", random_sleep(900, 1500)),
 788             ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 789             ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 790             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 791             ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
 792             ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 793             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 794             ("compare-results", "script/compare_cc_results.py "
 795             "./bin/c4che/default{} "
 796             "./bin-xe/c4che/default{} "
 797             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 798             ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
 799             ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
 800             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
 801             ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
 802             " = \"'1234'\"".format(CACHE_SUFFIX)),
 803             ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
 804             ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
 805             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
 806             " ; test $? -ne 0"),
 807         ],
 808     },
 809
 810     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 811     "samba-o3": {
 812         "sequence": [
 813             ("random-sleep", random_sleep(300, 900)),
 814             ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --abi-check-disable" + samba_configure_params),
 815             ("make", "make -j"),
 816             ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 817             ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 818             ("lcov", LCOV_CMD),
 819             ("install", "make install"),
 820             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 821             ("clean", "make clean"),
 822         ],
 823     },
 824
 825     "samba-32bit": {
 826         "sequence": [
 827             ("random-sleep", random_sleep(300, 900)),
 828             ("configure", "./configure.developer --abi-check-disable --disable-warnings-as-errors" + samba_configure_params),
 829             ("make", "make -j"),
 830             ("nonetest", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 831             ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 832             ("ktest", make_test(cmd='make test', include_envs=["ktest"])),
 833             ("install", "make install"),
 834             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 835             ("clean", "make clean"),
 836         ],
 837     },
 838
 839     "samba-ctdb": {
 840         "sequence": [
 841             ("random-sleep", random_sleep(900, 1500)),
 842
 843         # make sure we have tdb around:
 844             ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 845             ("tdb-make", "cd lib/tdb && make"),
 846             ("tdb-install", "cd lib/tdb && make install"),
 847
 848         # build samba with cluster support (also building ctdb):
 849             ("samba-configure",
 850          "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
 851          "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
 852          "./configure.developer ${PREFIX} "
 853          "--with-selftest-prefix=./bin/ab "
 854          "--with-cluster-support "
 855          "--without-ad-dc "
 856          "--bundled-libraries=!tdb"),
 857             ("samba-make", "make"),
 858             ("samba-check", "./bin/smbd --configfile=/dev/null -b | grep CLUSTER_SUPPORT"),
 859             ("samba-install", "make install"),
 860             ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 861
 862             ("test", make_test(
 863                 cmd='PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH make test',
 864                 INJECT_SELFTEST_PREFIX=0,
 865                 include_envs=["clusteredmember"])
 866             ),
 867
 868         # clean up:
 869             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 870             ("clean", "make clean"),
 871             ("ctdb-clean", "cd ./ctdb && make clean"),
 872         ],
 873     },
 874
 875     "samba-libs": {
 876         "sequence": [
 877             ("random-sleep", random_sleep(300, 900)),
 878             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 879             ("talloc-make", "cd lib/talloc && make"),
 880             ("talloc-install", "cd lib/talloc && make install"),
 881
 882             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 883             ("tdb-make", "cd lib/tdb && make"),
 884             ("tdb-install", "cd lib/tdb && make install"),
 885
 886             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 887             ("tevent-make", "cd lib/tevent && make"),
 888             ("tevent-install", "cd lib/tevent && make install"),
 889
 890             ("nondevel-configure", samba_libs_envvars + " ./configure --private-libraries='!ldb' ${PREFIX}"),
 891             ("nondevel-make", "make -j"),
 892             ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 893             ("nondevel-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
 894             ("nondevel-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
 895             ("nondevel-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
 896             ("nondevel-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 897             ("nondevel-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 898             ("nondevel-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 899             ("nondevel-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 900             ("nondevel-no-public-nss_winbind",
 901                 check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
 902             ("nondevel-no-public-nss_wins",
 903                 check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
 904             ("nondevel-no-public-libwbclient",
 905                 check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
 906             ("nondevel-no-public-pam_winbind",
 907                 check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
 908             ("nondevel-no-public-winbind_krb5_locator",
 909                 check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
 910             ("nondevel-no-public-async_dns_krb5_locator",
 911                 check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
 912             ("nondevel-install", "make -j install"),
 913             ("nondevel-dist", "make dist"),
 914
 915             ("prefix-no-private-libtalloc", "find ${PREFIX_DIR} | grep -v 'libtalloc-report' | grep 'private.*libtalloc' && exit 1; exit 0"),
 916             ("prefix-no-private-libtdb", "find ${PREFIX_DIR} | grep -v 'libtdb-wrap' | grep 'private.*libtdb' && exit 1; exit 0"),
 917             ("prefix-no-private-libtevent", "find ${PREFIX_DIR} | grep -v 'libtevent-util' | grep 'private.*libtevent' && exit 1; exit 0"),
 918             ("prefix-no-private-libldb", "find ${PREFIX_DIR} | grep -v 'module' | grep -v 'libldbsamba' | grep 'private.*libldb.so' && exit 1; exit 0"),
 919             ("prefix-public-libldb", "find ${PREFIX_DIR} | grep 'lib/libldb.so' && exit 0; exit 1"),
 920             ("prefix-no-samba-nss_winbind", "ldd ${PREFIX_DIR}/lib/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 921             ("prefix-no-samba-nss_wins", "ldd ${PREFIX_DIR}/lib/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 922             ("prefix-no-samba-libwbclient", "ldd ${PREFIX_DIR}/lib/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 923             ("prefix-no-samba-pam_winbind", "ldd ${PREFIX_DIR}/lib/security/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 924             ("prefix-no-public-nss_winbind",
 925                 check_symbols("${PREFIX_DIR}/lib/libnss_winbind.so.2", "_nss_winbind_")),
 926             ("prefix-no-public-nss_wins",
 927                 check_symbols("${PREFIX_DIR}/lib/libnss_wins.so.2", "_nss_wins_")),
 928             ("prefix-no-public-libwbclient",
 929                 check_symbols("${PREFIX_DIR}/lib/libwbclient.so.0", "wbc")),
 930             ("prefix-no-public-pam_winbind",
 931                 check_symbols("${PREFIX_DIR}/lib/security/pam_winbind.so", "pam_sm_")),
 932             ("prefix-no-public-winbind_krb5_locator",
 933                 check_symbols("${PREFIX_DIR}/lib/krb5/winbind_krb5_locator.so",
 934                               "service_locator")),
 935             ("prefix-no-public-async_dns_krb5_locator",
 936                 check_symbols("${PREFIX_DIR}/lib/krb5/async_dns_krb5_locator.so",
 937                               "service_locator")),
 938
 939             # retry with all modules shared
 940             ("allshared-distclean", "make distclean"),
 941             ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 942             ("allshared-make", "make -j"),
 943             ("allshared-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
 944             ("allshared-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
 945             ("allshared-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
 946             ("allshared-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 947             ("allshared-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 948             ("allshared-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 949             ("allshared-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 950             ("allshared-no-public-nss_winbind",
 951                 check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
 952             ("allshared-no-public-nss_wins",
 953                 check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
 954             ("allshared-no-public-libwbclient",
 955                 check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
 956             ("allshared-no-public-pam_winbind",
 957                 check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
 958             ("allshared-no-public-winbind_krb5_locator",
 959                 check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
 960             ("allshared-no-public-async_dns_krb5_locator",
 961                 check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
 962         ],
 963     },
 964
 965     "samba-fuzz": {
 966         "sequence": [
 967         # build the fuzzers (static) via the oss-fuzz script
 968             ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
 969             ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ADDITIONAL_LDFLAGS='-fuse-ld=bfd' ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl-fuzzer"),
 970         ],
 971     },
 972
 973     # * Test smbd and smbtorture can build semi-static
 974     #
 975     # * Test Samba without python still builds.
 976     #
 977     # When this test fails due to more use of Python, the expectations
 978     # is that the newly failing part of the code should be disabled
 979     # when --disable-python is set (rather than major work being done
 980     # to support this environment).
 981     #
 982     # The target here is for vendors shipping a minimal smbd.
 983     "samba-minimal-smbd": {
 984         "sequence": [
 985             ("random-sleep", random_sleep(300, 900)),
 986
 987         # build with all modules static
 988             ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
 989             ("allstatic-make", "make -j"),
 990             ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 991             ("allstatic-lcov", LCOV_CMD),
 992             ("allstatic-def-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 993             ("allstatic-def-clean", "make clean"),
 994
 995         # force all libraries as private
 996             ("allprivate-def-distclean", "make distclean"),
 997             ("allprivate-def-configure", "./configure.developer " + samba_configure_params + " --private-libraries=ALL"),
 998             ("allprivate-def-make", "make -j"),
 999             # note wrapper libraries need to be public
1000             ("allprivate-def-no-public", "ls ./bin/shared | egrep -v '^private$|lib[nprsu][saeoi][smscd].*-wrapper.so$|pam_set_items.so' | wc -l | grep -q '^0'"),
1001             ("allprivate-def-only-private-ext", "ls ./bin/shared/private | egrep 'private-samba' | wc -l | grep -q '^0' && exit 1; exit 0"),
1002             ("allprivate-def-no-non-private-ext", "ls ./bin/shared/private | egrep -v 'private-samba|^libpypamtest.so$' | wc -l | grep -q '^0'"),
1003             ("allprivate-def-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
1004             ("allprivate-def-lcov", LCOV_CMD),
1005             ("allprivate-def-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1006             ("allprivate-def-clean", "make clean"),
1007
1008         # force all libraries as private with a non default
1009         # extension and 2 exceptions
1010             ("allprivate-ext-distclean", "make distclean"),
1011             ("allprivate-ext-configure", "./configure.developer " + samba_configure_params + " --private-libraries=ALL --private-library-extension=private-library --private-extension-exception=pac,ndr"),
1012             ("allprivate-ext-make", "make -j"),
1013             # note wrapper libraries need to be public
1014             ("allprivate-ext-no-public", "ls ./bin/shared | egrep -v '^private$|lib[nprsu][saeoi][smscd].*-wrapper.so$|pam_set_items.so' | wc -l | grep -q '^0'"),
1015             ("allprivate-ext-no-private-default-ext", "ls ./bin/shared/private | grep 'private-samba' | wc -l | grep -q '^0'"),
1016             ("allprivate-ext-has-private-ext", "ls ./bin/shared/private | grep 'private-library' | wc -l | grep -q '^0' && exit 1; exit 0"),
1017             ("allprivate-ext-libndr-no-private-ext", "ls ./bin/shared/private | grep -v 'private-library' | grep 'libndr' | wc -l | grep -q '^1'"),
1018             ("allprivate-ext-libpac-no-private-ext", "ls ./bin/shared/private | grep -v 'private-library' | grep 'libpac' | wc -l | grep -q '^1'"),
1019             ("allprivate-ext-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
1020             ("allprivate-ext-lcov", LCOV_CMD),
1021             ("allprivate-ext-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1022             ("allprivate-ext-clean", "make clean"),
1023
1024         # retry with nonshared smbd and smbtorture
1025             ("nonshared-distclean", "make distclean"),
1026             ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
1027             ("nonshared-make", "make -j"),
1028             ("nonshared-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
1029             ("nonshared-lcov", LCOV_CMD),
1030             ("nonshared-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1031             ("nonshared-clean", "make clean"),
1032         ],
1033     },
1034
1035     "samba-nopython": {
1036         "sequence": [
1037             ("random-sleep", random_sleep(300, 900)),
1038
1039             ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
1040             ("make", "make -j"),
1041             ("find-python", "script/find_python.sh ${PREFIX}"),
1042             ("test", "make test-nopython"),
1043             ("lcov", LCOV_CMD),
1044             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1045             ("clean", "make clean"),
1046
1047             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1048             ("talloc-make", "cd lib/talloc && make"),
1049             ("talloc-install", "cd lib/talloc && make install"),
1050
1051             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1052             ("tdb-make", "cd lib/tdb && make"),
1053             ("tdb-install", "cd lib/tdb && make install"),
1054
1055             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1056             ("tevent-make", "cd lib/tevent && make"),
1057             ("tevent-install", "cd lib/tevent && make install"),
1058
1059         # retry against installed library packages, but no required modules
1060             ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc  --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
1061             ("libs-make", "make -j"),
1062             ("libs-install", "make install"),
1063             ("libs-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1064             ("libs-clean", "make clean"),
1065
1066         ],
1067     },
1068
1069     "samba-codecheck": {
1070         "sequence": [
1071             ("run", "script/check-shell-scripts.sh ."),
1072             ("run", "script/codespell.sh ."),
1073         ],
1074     },
1075
1076     "tdb": {
1077         "sequence": [
1078             ("random-sleep", random_sleep(60, 600)),
1079             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1080             ("make", "make"),
1081             ("install", "make install"),
1082             ("test", "make test"),
1083             ("lcov", LCOV_CMD),
1084             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1085             ("distcheck", "make distcheck"),
1086             ("clean", "make clean"),
1087         ],
1088     },
1089
1090     "talloc": {
1091         "sequence": [
1092             ("random-sleep", random_sleep(60, 600)),
1093             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1094             ("make", "make"),
1095             ("install", "make install"),
1096             ("test", "make test"),
1097             ("lcov", LCOV_CMD),
1098             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1099             ("distcheck", "make distcheck"),
1100             ("clean", "make clean"),
1101         ],
1102     },
1103
1104     "replace": {
1105         "sequence": [
1106             ("random-sleep", random_sleep(60, 600)),
1107             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1108             ("make", "make"),
1109             ("install", "make install"),
1110             ("test", "make test"),
1111             ("lcov", LCOV_CMD),
1112             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1113             ("distcheck", "make distcheck"),
1114             ("clean", "make clean"),
1115         ],
1116     },
1117
1118     "tevent": {
1119         "sequence": [
1120             ("random-sleep", random_sleep(60, 600)),
1121             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1122             ("make", "make"),
1123             ("install", "make install"),
1124             ("test", "make test"),
1125             ("lcov", LCOV_CMD),
1126             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1127             ("distcheck", "make distcheck"),
1128             ("clean", "make clean"),
1129         ],
1130     },
1131
1132     "pidl": {
1133         "git-clone-required": True,
1134         "sequence": [
1135             ("random-sleep", random_sleep(60, 600)),
1136             ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
1137             ("touch", "touch *.yp"),
1138             ("make", "make"),
1139             ("test", "make test"),
1140             ("install", "make install"),
1141             ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
1142             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1143             ("clean", "make clean"),
1144         ],
1145     },
1146
1147     # these are useful for debugging autobuild
1148     "pass": {
1149         "sequence": [
1150             ("pass", 'echo passing && /bin/true'),
1151         ],
1152     },
1153     "fail": {
1154         "sequence": [
1155             ("fail", 'echo failing && /bin/false'),
1156         ],
1157     },
1158 }
1159
1160 defaulttasks = list(tasks.keys())
1161
1162 defaulttasks.remove("pass")
1163 defaulttasks.remove("fail")
1164
1165 # The build tasks will be brought in by the test tasks as needed
1166 defaulttasks.remove("samba-def-build")
1167 defaulttasks.remove("samba-nt4-build")
1168 defaulttasks.remove("samba-mit-build")
1169 defaulttasks.remove("samba-h5l-build")
1170 defaulttasks.remove("samba-no-opath-build")
1171
1172 # This is not a normal test, but a task to support manually running
1173 # one test under autobuild
1174 defaulttasks.remove("samba-test-only")
1175
1176 # Only built on GitLab CI and not in the default autobuild because it
1177 # uses too much space (4GB of semi-static binaries)
1178 defaulttasks.remove("samba-fuzz")
1179
1180 # The FIPS build runs only in GitLab CI on a current Fedora Docker
1181 # container where a simulated FIPS mode is possible.
1182 defaulttasks.remove("samba-fips")
1183
1184 # The MIT build runs on a current Fedora where an up to date MIT KDC
1185 # is already packaged.  This avoids needing to backport a current MIT
1186 # to the default Ubuntu 18.04, particularly during development, and
1187 # the need to install on the shared sn-devel-184.
1188
1189 defaulttasks.remove("samba-mitkrb5")
1190 defaulttasks.remove("samba-admem-mit")
1191 defaulttasks.remove("samba-addc-mit-1")
1192 defaulttasks.remove("samba-addc-mit-4a")
1193 defaulttasks.remove("samba-addc-mit-4b")
1194
1195 defaulttasks.remove("samba-32bit")
1196
1197 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
1198     defaulttasks.remove("samba-o3")
1199
1200
1201 def do_print(msg):
1202     logger.info(msg)
1203     sys.stdout.flush()
1204     sys.stderr.flush()
1205
1206 def do_debug(msg):
1207     logger.debug(msg)
1208     sys.stdout.flush()
1209     sys.stderr.flush()
1210
1211
1212 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
1213     if show is None:
1214         do_debug("Running: '%s' in '%s'" % (cmd, dir))
1215     elif show:
1216         do_print("Running: '%s' in '%s'" % (cmd, dir))
1217
1218     if output:
1219         out = check_output([cmd], shell=True, cwd=dir)
1220         return out.decode(encoding='utf-8', errors='backslashreplace')
1221     elif checkfail:
1222         return check_call(cmd, shell=True, cwd=dir)
1223     else:
1224         return call(cmd, shell=True, cwd=dir)
1225
1226 def rmdir_force(dirname, re_raise=True):
1227     try:
1228         run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % (
1229                 dirname, dirname, dirname), output=True, show=True)
1230     except CalledProcessError as e:
1231         do_print("Failed: '%s'" % (str(e)))
1232         run_cmd("tree %s" % dirname, output=True, show=True)
1233         if re_raise:
1234             raise
1235         return False
1236     return True
1237
1238 class builder(object):
1239     '''handle build of one directory'''
1240
1241     def __init__(self, name, definition):
1242         self.name = name
1243         self.dir = builddirs.get(name, '.')
1244         self.tag = self.name.replace('/', '_')
1245         self.definition = definition
1246         self.sequence = definition["sequence"]
1247         self.git_clone_required = False
1248         if "git-clone-required" in definition:
1249             self.git_clone_required = bool(definition["git-clone-required"])
1250         self.proc = None
1251         self.done = False
1252         self.next = 0
1253         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
1254         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
1255         do_debug("stdout for %s in %s" % (self.name, self.stdout_path))
1256         do_debug("stderr for %s in %s" % (self.name, self.stderr_path))
1257         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
1258         self.stdout = open(self.stdout_path, 'w')
1259         self.stderr = open(self.stderr_path, 'w')
1260         self.stdin  = open("/dev/null", 'r')
1261         self.builder_dir = "%s/%s" % (testbase, self.tag)
1262         self.test_source_dir = self.builder_dir
1263         self.cwd = "%s/%s" % (self.builder_dir, self.dir)
1264         self.selftest_prefix = "%s/bin/ab" % (self.cwd)
1265         self.prefix = "%s/%s" % (test_prefix, self.tag)
1266         self.consumers = []
1267         self.producer = None
1268
1269         if self.git_clone_required:
1270             assert "dependency" not in definition
1271
1272     def mark_existing(self):
1273         do_debug('%s: Mark as existing dependency' % self.name)
1274         self.next = len(self.sequence)
1275         self.done = True
1276
1277     def add_consumer(self, consumer):
1278         do_debug("%s: add consumer: %s" % (self.name, consumer.name))
1279         consumer.producer = self
1280         consumer.test_source_dir = self.test_source_dir
1281         self.consumers.append(consumer)
1282
1283     def start_next(self):
1284         if self.producer is not None:
1285             if not self.producer.done:
1286                 do_debug("%s: Waiting for producer: %s" % (self.name, self.producer.name))
1287                 return
1288
1289         if self.next == 0:
1290             rmdir_force(self.builder_dir)
1291             rmdir_force(self.prefix)
1292             if self.producer is not None:
1293                 run_cmd("mkdir %s" % (self.builder_dir), dir=test_master, show=True)
1294             elif not self.git_clone_required:
1295                 run_cmd("cp -R -a -l %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1296             else:
1297                 run_cmd("git clone --recursive --shared %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1298
1299         if self.next == len(self.sequence):
1300             if not self.done:
1301                 do_print('%s: Completed OK' % self.name)
1302                 self.done = True
1303             if not options.nocleanup and len(self.consumers) == 0:
1304                 do_print('%s: Cleaning up' % self.name)
1305                 rmdir_force(self.builder_dir)
1306                 rmdir_force(self.prefix)
1307             for consumer in self.consumers:
1308                 if consumer.next != 0:
1309                     continue
1310                 do_print('%s: Starting consumer %s' % (self.name, consumer.name))
1311                 consumer.start_next()
1312             if self.producer is not None:
1313                 self.producer.consumers.remove(self)
1314                 assert self.producer.done
1315                 self.producer.start_next()
1316             do_print('%s: Remaining consumers %u' % (self.name, len(self.consumers)))
1317             return
1318         (self.stage, self.cmd) = self.sequence[self.next]
1319         self.cmd = self.cmd.replace("${PYTHON_PREFIX}",
1320                                     get_path(name='platlib',
1321                                              scheme="posix_prefix",
1322                                              vars={"base": self.prefix,
1323                                                    "platbase": self.prefix}))
1324         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
1325         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
1326         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
1327         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
1328         self.cmd = self.cmd.replace("${SELFTEST_PREFIX}", self.selftest_prefix)
1329         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
1330         self.cmd = self.cmd.replace("${NAME}", self.name)
1331         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
1332         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
1333         self.proc = Popen(self.cmd, shell=True,
1334                           close_fds=True, cwd=self.cwd,
1335                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
1336         self.next += 1
1337
1338 def expand_dependencies(n):
1339     deps = list()
1340     if "dependency" in tasks[n]:
1341         depname = tasks[n]["dependency"]
1342         assert depname in tasks
1343         sdeps = expand_dependencies(depname)
1344         assert n not in sdeps
1345         for sdep in sdeps:
1346             deps.append(sdep)
1347         deps.append(depname)
1348     return deps
1349
1350
1351 class buildlist(object):
1352     '''handle build of multiple directories'''
1353
1354     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
1355         self.tail_proc = None
1356         self.retry = None
1357         if not tasknames:
1358             if options.restrict_tests:
1359                 tasknames = ["samba-test-only"]
1360             else:
1361                 tasknames = defaulttasks
1362
1363         given_tasknames = tasknames.copy()
1364         implicit_tasknames = []
1365         for n in given_tasknames:
1366             deps = expand_dependencies(n)
1367             for dep in deps:
1368                 if dep in given_tasknames:
1369                     continue
1370                 if dep in implicit_tasknames:
1371                     continue
1372                 implicit_tasknames.append(dep)
1373
1374         tasknames = implicit_tasknames.copy()
1375         tasknames.extend(given_tasknames)
1376         do_debug("given_tasknames: %s" % given_tasknames)
1377         do_debug("implicit_tasknames: %s" % implicit_tasknames)
1378         do_debug("tasknames: %s" % tasknames)
1379         self.tlist = [builder(n, tasks[n]) for n in tasknames]
1380
1381         if options.retry:
1382             rebase_remote = "rebaseon"
1383             retry_task = {
1384                     "git-clone-required": True,
1385                     "sequence": [
1386                             ("retry",
1387                             '''set -e
1388                             git remote add -t %s %s %s
1389                             git fetch %s
1390                             while :; do
1391                               sleep 60
1392                               git describe %s/%s > old_remote_branch.desc
1393                               git fetch %s
1394                               git describe %s/%s > remote_branch.desc
1395                               diff old_remote_branch.desc remote_branch.desc
1396                             done
1397                            ''' % (
1398                                rebase_branch, rebase_remote, rebase_url,
1399                                rebase_remote,
1400                                rebase_remote, rebase_branch,
1401                                rebase_remote,
1402                                rebase_remote, rebase_branch
1403                             ))]}
1404
1405             self.retry = builder('retry', retry_task)
1406             self.need_retry = False
1407
1408         if options.skip_dependencies:
1409             for b in self.tlist:
1410                 if b.name in implicit_tasknames:
1411                     b.mark_existing()
1412
1413         for b in self.tlist:
1414             do_debug("b.name=%s" % b.name)
1415             if "dependency" not in b.definition:
1416                 continue
1417             depname = b.definition["dependency"]
1418             do_debug("b.name=%s: dependency:%s" % (b.name, depname))
1419             for p in self.tlist:
1420                 if p.name == depname:
1421                     p.add_consumer(b)
1422
1423     def kill_kids(self):
1424         if self.tail_proc is not None:
1425             self.tail_proc.terminate()
1426             self.tail_proc.wait()
1427             self.tail_proc = None
1428         if self.retry is not None:
1429             self.retry.proc.terminate()
1430             self.retry.proc.wait()
1431             self.retry = None
1432         for b in self.tlist:
1433             if b.proc is not None:
1434                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
1435                 b.proc.terminate()
1436                 b.proc.wait()
1437                 b.proc = None
1438
1439     def wait_one(self):
1440         while True:
1441             none_running = True
1442             for b in self.tlist:
1443                 if b.proc is None:
1444                     continue
1445                 none_running = False
1446                 b.status = b.proc.poll()
1447                 if b.status is None:
1448                     continue
1449                 b.proc = None
1450                 return b
1451             if options.retry:
1452                 ret = self.retry.proc.poll()
1453                 if ret is not None:
1454                     self.need_retry = True
1455                     self.retry = None
1456                     return None
1457             if none_running:
1458                 return None
1459             time.sleep(0.1)
1460
1461     def run(self):
1462         for b in self.tlist:
1463             b.start_next()
1464         if options.retry:
1465             self.retry.start_next()
1466         while True:
1467             b = self.wait_one()
1468             if options.retry and self.need_retry:
1469                 self.kill_kids()
1470                 do_print("retry needed")
1471                 return (0, None, None, None, "retry")
1472             if b is None:
1473                 break
1474             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
1475                 self.kill_kids()
1476                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
1477             b.start_next()
1478         self.kill_kids()
1479         return (0, None, None, None, "All OK")
1480
1481     def write_system_info(self, filename):
1482         with open(filename, 'w') as f:
1483             for cmd in ['uname -a',
1484                         'lsb_release -a',
1485                         'free',
1486                         'mount',
1487                         'cat /proc/cpuinfo',
1488                         'cc --version',
1489                         'df -m .',
1490                         'df -m %s' % testbase]:
1491                 try:
1492                     out = run_cmd(cmd, output=True, checkfail=False)
1493                 except CalledProcessError as e:
1494                     out = "<failed: %s>" % str(e)
1495                 print('### %s' % cmd, file=f)
1496                 print(out, file=f)
1497                 print(file=f)
1498
1499     def tarlogs(self, fname):
1500         with tarfile.open(fname, "w:gz") as tar:
1501             for b in self.tlist:
1502                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
1503                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
1504             if os.path.exists("autobuild.log"):
1505                 tar.add("autobuild.log")
1506             filename = 'system-info.txt'
1507             self.write_system_info(filename)
1508             tar.add(filename)
1509
1510     def remove_logs(self):
1511         for b in self.tlist:
1512             os.unlink(b.stdout_path)
1513             os.unlink(b.stderr_path)
1514
1515     def start_tail(self):
1516         cmd = ["tail", "-f"]
1517         for b in self.tlist:
1518             cmd.append(b.stdout_path)
1519             cmd.append(b.stderr_path)
1520         self.tail_proc = Popen(cmd, close_fds=True)
1521
1522
1523 def cleanup(do_raise=False):
1524     if options.nocleanup:
1525         return
1526     run_cmd("stat %s || true" % test_tmpdir, show=True)
1527     run_cmd("stat %s" % testbase, show=True)
1528     do_print("Cleaning up %r" % cleanup_list)
1529     for d in cleanup_list:
1530         ok = rmdir_force(d, re_raise=False)
1531         if ok:
1532             continue
1533         if os.path.isdir(d):
1534             do_print("Killing, waiting and retry")
1535             run_cmd("killbysubdir %s > /dev/null 2>&1" % d, checkfail=False)
1536         else:
1537             do_print("Waiting and retry")
1538         time.sleep(1)
1539         rmdir_force(d, re_raise=do_raise)
1540
1541
1542 def daemonize(logfile):
1543     pid = os.fork()
1544     if pid == 0:  # Parent
1545         os.setsid()
1546         pid = os.fork()
1547         if pid != 0:  # Actual daemon
1548             os._exit(0)
1549     else:  # Grandparent
1550         os._exit(0)
1551
1552     import resource      # Resource usage information.
1553     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1554     if maxfd == resource.RLIM_INFINITY:
1555         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1556     for fd in range(0, maxfd):
1557         try:
1558             os.close(fd)
1559         except OSError:
1560             pass
1561     os.open(logfile, os.O_RDWR | os.O_CREAT)
1562     os.dup2(0, 1)
1563     os.dup2(0, 2)
1564
1565
1566 def write_pidfile(fname):
1567     '''write a pid file, cleanup on exit'''
1568     with open(fname, mode='w') as f:
1569         f.write("%u\n" % os.getpid())
1570
1571
1572 def rebase_tree(rebase_url, rebase_branch="master"):
1573     rebase_remote = "rebaseon"
1574     do_print("Rebasing on %s" % rebase_url)
1575     run_cmd("git describe HEAD", show=True, dir=test_master)
1576     run_cmd("git remote add -t %s %s %s" %
1577             (rebase_branch, rebase_remote, rebase_url),
1578             show=True, dir=test_master)
1579     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1580     if options.fix_whitespace:
1581         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1582                 (rebase_remote, rebase_branch),
1583                 show=True, dir=test_master)
1584     else:
1585         run_cmd("git rebase --force-rebase %s/%s" %
1586                 (rebase_remote, rebase_branch),
1587                 show=True, dir=test_master)
1588     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1589                    (rebase_remote, rebase_branch),
1590                    dir=test_master, output=True)
1591     if diff == '':
1592         do_print("No differences between HEAD and %s/%s - exiting" %
1593                  (rebase_remote, rebase_branch))
1594         sys.exit(0)
1595     run_cmd("git describe %s/%s" %
1596             (rebase_remote, rebase_branch),
1597             show=True, dir=test_master)
1598     run_cmd("git describe HEAD", show=True, dir=test_master)
1599     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1600             (rebase_remote, rebase_branch),
1601             show=True, dir=test_master)
1602
1603
1604 def push_to(push_url, push_branch="master"):
1605     push_remote = "pushto"
1606     do_print("Pushing to %s" % push_url)
1607     if options.mark:
1608         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1609         run_cmd("git commit --amend -c HEAD", dir=test_master)
1610         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1611         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1612     run_cmd("git remote add -t %s %s %s" %
1613             (push_branch, push_remote, push_url),
1614             show=True, dir=test_master)
1615     run_cmd("git push %s +HEAD:%s" %
1616             (push_remote, push_branch),
1617             show=True, dir=test_master)
1618
1619
1620 def send_email(subject, text, log_tar):
1621     if options.email is None:
1622         do_print("not sending email because the recipient is not set")
1623         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1624                  (subject, text))
1625         return
1626     outer = MIMEMultipart()
1627     outer['Subject'] = subject
1628     outer['To'] = options.email
1629     outer['From'] = options.email_from
1630     outer['Date'] = email.utils.formatdate(localtime=True)
1631     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1632     outer.attach(MIMEText(text, 'plain', 'utf-8'))
1633     if options.attach_logs:
1634         with open(log_tar, 'rb') as fp:
1635             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1636         # Set the filename parameter
1637         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1638         outer.attach(msg)
1639     content = outer.as_string()
1640     s = smtplib.SMTP(options.email_server)
1641     email_user = os.getenv('SMTP_USERNAME')
1642     email_password = os.getenv('SMTP_PASSWORD')
1643     if email_user is not None:
1644         s.starttls()
1645         s.login(email_user, email_password)
1646
1647     s.sendmail(options.email_from, [options.email], content)
1648     s.set_debuglevel(1)
1649     s.quit()
1650
1651
1652 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1653                   elapsed_time, log_base=None, add_log_tail=True):
1654     '''send an email to options.email about the failure'''
1655     elapsed_minutes = elapsed_time / 60.0
1656     if log_base is None:
1657         log_base = gitroot
1658     text = '''
1659 Dear Developer,
1660
1661 Your autobuild on %s failed after %.1f minutes
1662 when trying to test %s with the following error:
1663
1664    %s
1665
1666 the autobuild has been abandoned. Please fix the error and resubmit.
1667
1668 A summary of the autobuild process is here:
1669
1670   %s/autobuild.log
1671 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1672
1673     if options.restrict_tests:
1674         text += """
1675 The build was restricted to tests matching %s\n""" % options.restrict_tests
1676
1677     if failed_task != 'rebase':
1678         text += '''
1679 You can see logs of the failed task here:
1680
1681   %s/%s.stdout
1682   %s/%s.stderr
1683
1684 or you can get full logs of all tasks in this job here:
1685
1686   %s/logs.tar.gz
1687
1688 The top commit for the tree that was built was:
1689
1690 %s
1691
1692 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1693
1694     log_stdout = "%s/%s.stdout" % (gitroot, failed_tag)
1695     if add_log_tail and os.access(log_stdout, os.R_OK):
1696         f = open(log_stdout, 'r')
1697         lines = f.readlines()
1698         log_tail = "".join(lines[-50:])
1699         num_lines = len(lines)
1700         log_stderr = "%s/%s.stderr" % (gitroot, failed_tag)
1701         if num_lines < 50 and os.access(log_stderr, os.R_OK):
1702             # Also include stderr (compile failures) if < 50 lines of stdout
1703             f = open(log_stderr, 'r')
1704             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1705
1706         text += '''
1707 The last 50 lines of log messages:
1708
1709 %s
1710     ''' % log_tail
1711         f.close()
1712
1713     logs = os.path.join(gitroot, 'logs.tar.gz')
1714     send_email('autobuild[%s] failure on %s for task %s during %s'
1715                % (options.branch, platform.node(), failed_task, failed_stage),
1716                text, logs)
1717
1718
1719 def email_success(elapsed_time, log_base=None):
1720     '''send an email to options.email about a successful build'''
1721     if log_base is None:
1722         log_base = gitroot
1723     text = '''
1724 Dear Developer,
1725
1726 Your autobuild on %s has succeeded after %.1f minutes.
1727
1728 ''' % (platform.node(), elapsed_time / 60.)
1729
1730     if options.restrict_tests:
1731         text += """
1732 The build was restricted to tests matching %s\n""" % options.restrict_tests
1733
1734     if options.keeplogs:
1735         text += '''
1736
1737 you can get full logs of all tasks in this job here:
1738
1739   %s/logs.tar.gz
1740
1741 ''' % log_base
1742
1743     text += '''
1744 The top commit for the tree that was built was:
1745
1746 %s
1747 ''' % top_commit_msg
1748
1749     logs = os.path.join(gitroot, 'logs.tar.gz')
1750     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1751                text, logs)
1752
1753
1754 # get the top commit message, for emails
1755 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1756
1757 try:
1758     if options.skip_dependencies:
1759         run_cmd("stat %s" % testbase, dir=testbase, output=True)
1760     else:
1761         os.makedirs(testbase)
1762 except Exception as reason:
1763     raise Exception("Unable to create %s : %s" % (testbase, reason))
1764 cleanup_list.append(testbase)
1765
1766 if options.daemon:
1767     logfile = os.path.join(testbase, "log")
1768     do_print("Forking into the background, writing progress to %s" % logfile)
1769     daemonize(logfile)
1770
1771 write_pidfile(gitroot + "/autobuild.pid")
1772
1773 start_time = time.time()
1774
1775 while True:
1776     try:
1777         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1778         os.makedirs(test_tmpdir)
1779         # The waf uninstall code removes empty directories all the way
1780         # up the tree.  Creating a file in test_tmpdir stops it from
1781         # being removed.
1782         run_cmd("touch %s" % os.path.join(test_tmpdir,
1783                                           ".directory-is-not-empty"), show=True)
1784         run_cmd("stat %s" % test_tmpdir, show=True)
1785         run_cmd("stat %s" % testbase, show=True)
1786         if options.skip_dependencies:
1787             run_cmd("stat %s" % test_master, dir=testbase, output=True)
1788         else:
1789             run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1790     except Exception:
1791         cleanup()
1792         raise
1793
1794     try:
1795         if options.rebase is not None:
1796             rebase_tree(options.rebase, rebase_branch=options.branch)
1797     except Exception:
1798         cleanup_list.append(gitroot + "/autobuild.pid")
1799         cleanup()
1800         elapsed_time = time.time() - start_time
1801         email_failure(-1, 'rebase', 'rebase', 'rebase',
1802                       'rebase on %s failed' % options.branch,
1803                       elapsed_time, log_base=options.log_base)
1804         sys.exit(1)
1805
1806     try:
1807         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1808         if options.tail:
1809             blist.start_tail()
1810         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1811         if status != 0 or errstr != "retry":
1812             break
1813         cleanup(do_raise=True)
1814     except Exception:
1815         cleanup()
1816         raise
1817
1818 cleanup_list.append(gitroot + "/autobuild.pid")
1819
1820 do_print(errstr)
1821
1822 blist.kill_kids()
1823 if options.tail:
1824     do_print("waiting for tail to flush")
1825     time.sleep(1)
1826
1827 elapsed_time = time.time() - start_time
1828 if status == 0:
1829     if options.passcmd is not None:
1830         do_print("Running passcmd: %s" % options.passcmd)
1831         run_cmd(options.passcmd, dir=test_master)
1832     if options.pushto is not None:
1833         push_to(options.pushto, push_branch=options.branch)
1834     if options.keeplogs or options.attach_logs:
1835         blist.tarlogs("logs.tar.gz")
1836         do_print("Logs in logs.tar.gz")
1837     if options.always_email:
1838         email_success(elapsed_time, log_base=options.log_base)
1839     blist.remove_logs()
1840     cleanup()
1841     do_print(errstr)
1842     sys.exit(0)
1843
1844 # something failed, gather a tar of the logs
1845 blist.tarlogs("logs.tar.gz")
1846
1847 if options.email is not None:
1848     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1849                   elapsed_time, log_base=options.log_base)
1850 else:
1851     elapsed_minutes = elapsed_time / 60.0
1852     print('''
1853
1854 ####################################################################
1855
1856 AUTOBUILD FAILURE
1857
1858 Your autobuild[%s] on %s failed after %.1f minutes
1859 when trying to test %s with the following error:
1860
1861    %s
1862
1863 the autobuild has been abandoned. Please fix the error and resubmit.
1864
1865 ####################################################################
1866
1867 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1868
1869 cleanup()
1870 do_print(errstr)
1871 do_print("Logs in logs.tar.gz")
1872 sys.exit(status)