script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from subprocess import call, check_call, check_output, Popen, PIPE, CalledProcessError
   7 import os
   8 import tarfile
   9 import sys
  10 import time
  11 import random
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from sysconfig import get_path
  20 import platform
  21
  22 import logging
  23
  24 try:
  25     from waflib.Build import CACHE_SUFFIX
  26 except ImportError:
  27     sys.path.insert(0, "./third_party/waf")
  28     from waflib.Build import CACHE_SUFFIX
  29
  30 logging.basicConfig(format='%(asctime)s %(message)s')
  31 logger = logging.getLogger('autobuild')
  32 logger.setLevel(logging.INFO)
  33
  34 os.environ["PYTHONUNBUFFERED"] = "1"
  35
  36 # This speeds up testing remarkably.
  37 os.environ['TDB_NO_FSYNC'] = '1'
  38
  39 # allow autobuild to run within git rebase -i
  40 if "GIT_DIR" in os.environ:
  41     del os.environ["GIT_DIR"]
  42 if "GIT_WORK_TREE" in os.environ:
  43     del os.environ["GIT_WORK_TREE"]
  44
  45 def find_git_root():
  46     '''get to the top of the git repo'''
  47     p = os.getcwd()
  48     while p != '/':
  49         if os.path.exists(os.path.join(p, ".git")):
  50             return p
  51         p = os.path.abspath(os.path.join(p, '..'))
  52     return None
  53
  54
  55 gitroot = find_git_root()
  56 if gitroot is None:
  57     raise Exception("Failed to find git root")
  58
  59
  60 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  61
  62 parser = OptionParser()
  63 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  64 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  65 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  66 parser.add_option("--skip-dependencies", help="skip to run task dependency tasks", default=False, action="store_true")
  67 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  68                   default=def_testbase)
  69 parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase,
  70                   default=None)
  71 parser.add_option("--passcmd", help="command to run on success", default=None)
  72 parser.add_option("--verbose", help="show all commands as they are run",
  73                   default=False, action="store_true")
  74 parser.add_option("--rebase", help="rebase on the given tree before testing",
  75                   default=None, type='str')
  76 parser.add_option("--pushto", help="push to a git url on success",
  77                   default=None, type='str')
  78 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  79                   default=False, action="store_true")
  80 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  81                   default=False, action="store_true")
  82 parser.add_option("--retry", help="automatically retry if master changes",
  83                   default=False, action="store_true")
  84 parser.add_option("--email", help="send email to the given address on failure",
  85                   type='str', default=None)
  86 parser.add_option("--email-from", help="send email from the given address",
  87                   type='str', default="autobuild@samba.org")
  88 parser.add_option("--email-server", help="send email via the given server",
  89                   type='str', default='localhost')
  90 parser.add_option("--always-email", help="always send email, even on success",
  91                   action="store_true")
  92 parser.add_option("--daemon", help="daemonize after initial setup",
  93                   action="store_true")
  94 parser.add_option("--branch", help="the branch to work on (default=master)",
  95                   default="master", type='str')
  96 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  97                   default=gitroot, type='str')
  98 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  99                   default=False, action="store_true")
 100 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
 101                   default='')
 102 parser.add_option("--enable-coverage", dest='enable_coverage',
 103                   action="store_const", const='--enable-coverage', default='',
 104                   help="Add --enable-coverage option while configure")
 105
 106 (options, args) = parser.parse_args()
 107
 108 if options.retry:
 109     if options.rebase is None:
 110         raise Exception('You can only use --retry if you also rebase')
 111
 112 if options.verbose:
 113     logger.setLevel(logging.DEBUG)
 114
 115 if options.full_testbase is not None:
 116     testbase = options.full_testbase
 117 else:
 118     testbase = "%s/b%u" % (options.testbase, os.getpid())
 119 test_master = "%s/master" % testbase
 120 test_prefix = "%s/prefix" % testbase
 121 test_tmpdir = "%s/tmp" % testbase
 122 os.environ['TMPDIR'] = test_tmpdir
 123
 124 if options.enable_coverage:
 125     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 126 else:
 127     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 128
 129 if options.enable_coverage:
 130     PUBLISH_DOCS = "mkdir -p ${LOG_BASE}/public && mv output/htmldocs ${LOG_BASE}/public/htmldocs"
 131 else:
 132     PUBLISH_DOCS = 'echo "HTML documentation publishing skipped since no --enable-coverage specified"'
 133
 134 CLEAN_SOURCE_TREE_CMD = "cd ${TEST_SOURCE_DIR} && script/clean-source-tree.sh"
 135
 136
 137 def check_symbols(sofile, expected_symbols=""):
 138     return "objdump --dynamic-syms " + sofile + " | " + \
 139            "awk \'$0 !~ /" + expected_symbols + "/ {if ($2 == \"g\" && $3 ~ /D(F|O)/ && $4 ~ /(.bss|.text)/ && $7 !~ /(__gcov_|mangle_path)/) exit 1}\'"
 140
 141 if args:
 142     # If we are only running specific test,
 143     # do not sleep randomly to wait for it to start
 144     def random_sleep(low, high):
 145         return 'sleep 1'
 146 else:
 147     def random_sleep(low, high):
 148         return 'sleep {}'.format(random.randint(low, high))
 149
 150 cleanup_list = []
 151
 152 builddirs = {
 153     "ctdb": "ctdb",
 154     "ldb": "lib/ldb",
 155     "tdb": "lib/tdb",
 156     "talloc": "lib/talloc",
 157     "replace": "lib/replace",
 158     "tevent": "lib/tevent",
 159     "pidl": "pidl",
 160     "docs-xml": "docs-xml"
 161 }
 162
 163 ctdb_configure_params = " --enable-developer ${PREFIX}"
 164 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 165
 166 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 167 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 168 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 169 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 170 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 171 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 172 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 173
 174
 175 def format_option(name, value=None):
 176     """Format option as str list."""
 177     if value is None:  # boolean option
 178         return [name]
 179     if not isinstance(value, list):  # single value option
 180         value = [value]
 181     # repeatable option
 182     return ['{}={}'.format(name, item) for item in value]
 183
 184
 185 def make_test(
 186         cmd='make testonly',
 187         INJECT_SELFTEST_PREFIX=1,
 188         TESTS='',
 189         include_envs=None,
 190         exclude_envs=None):
 191
 192     test_options = []
 193     if include_envs:
 194         test_options = format_option('--include-env', include_envs)
 195     if exclude_envs:
 196         test_options = format_option('--exclude-env', exclude_envs)
 197     if test_options:
 198         # join envs options to original test options
 199         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 200
 201     _options = []
 202
 203     # Allow getting a full CI with
 204     # git push -o ci.variable='AUTOBUILD_FAIL_IMMEDIATELY=0'
 205
 206     FAIL_IMMEDIATELY = os.getenv("AUTOBUILD_FAIL_IMMEDIATELY", "1")
 207
 208     if int(FAIL_IMMEDIATELY):
 209         _options.append('FAIL_IMMEDIATELY=1')
 210     if TESTS:
 211         _options.append("TESTS='{}'".format(TESTS))
 212
 213     if INJECT_SELFTEST_PREFIX:
 214         _options.append("TEST_OPTIONS='--with-selftest-prefix={}'".format("${SELFTEST_PREFIX}"))
 215         _options.append("--directory='{}'".format("${TEST_SOURCE_DIR}"))
 216
 217     return ' '.join([cmd] + _options)
 218
 219
 220 # When updating this list, also update .gitlab-ci.yml to add the job
 221 # and to make it a dependency of 'page' for the coverage report.
 222
 223 tasks = {
 224     "ctdb": {
 225         "sequence": [
 226             ("random-sleep", random_sleep(300, 900)),
 227             ("configure", "./configure " + ctdb_configure_params),
 228             ("make", "make all"),
 229             ("install", "make install"),
 230             ("test", "make autotest"),
 231             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 232             ("clean", "make clean"),
 233         ],
 234     },
 235     "docs-xml": {
 236         "sequence": [
 237             ("random-sleep", random_sleep(300, 900)),
 238             ("autoconf", "autoconf"),
 239             ("configure", "./configure"),
 240             ("make", "make html htmlman"),
 241             ("publish-docs", PUBLISH_DOCS),
 242             ("clean", "make clean"),
 243         ],
 244     },
 245
 246     "samba-def-build": {
 247         "git-clone-required": True,
 248         "sequence": [
 249             ("configure", "./configure.developer" + samba_configure_params),
 250             ("make", "make -j"),
 251             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 252             ("chmod-R-a-w", "chmod -R a-w ."),
 253         ],
 254     },
 255
 256     "samba-mit-build": {
 257         "git-clone-required": True,
 258         "sequence": [
 259             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 260             ("make", "make -j"),
 261             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 262             ("chmod-R-a-w", "chmod -R a-w ."),
 263         ],
 264     },
 265
 266     "samba-nt4-build": {
 267         "git-clone-required": True,
 268         "sequence": [
 269             ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json" + samba_configure_params),
 270             ("make", "make -j"),
 271             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 272             ("chmod-R-a-w", "chmod -R a-w ."),
 273         ],
 274     },
 275
 276     "samba-h5l-build": {
 277         "git-clone-required": True,
 278         "sequence": [
 279             ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5" + samba_configure_params),
 280             ("make", "make -j"),
 281             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 282             ("chmod-R-a-w", "chmod -R a-w ."),
 283         ],
 284     },
 285
 286     "samba-without-smb1-build": {
 287         "git-clone-required": True,
 288         "sequence": [
 289             ("configure", "./configure.developer --without-smb1-server --without-ad-dc" + samba_configure_params),
 290             ("make", "make -j"),
 291             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 292             ("chmod-R-a-w", "chmod -R a-w ."),
 293         ],
 294     },
 295
 296     "samba-no-opath-build": {
 297         "git-clone-required": True,
 298         "sequence": [
 299             ("configure", "ADDITIONAL_CFLAGS='-DDISABLE_OPATH=1 -DDISABLE_VFS_OPEN_HOW_RESOLVE_NO_SYMLINKS=1 -DDISABLE_PROC_FDS=1' ./configure.developer --without-ad-dc " + samba_configure_params),
 300             ("make", "make -j"),
 301             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 302             ("chmod-R-a-w", "chmod -R a-w ."),
 303         ],
 304     },
 305
 306     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 307     "samba": {
 308         "sequence": [
 309             ("random-sleep", random_sleep(300, 900)),
 310             ("configure", "./configure.developer" + samba_configure_params),
 311             ("make", "make -j"),
 312             ("test", make_test(exclude_envs=[
 313             "none",
 314             "nt4_dc",
 315             "nt4_dc_smb1",
 316             "nt4_dc_smb1_done",
 317             "nt4_dc_schannel",
 318             "nt4_member",
 319             "ad_dc",
 320             "ad_dc_smb1",
 321             "ad_dc_smb1_done",
 322             "ad_dc_backup",
 323             "ad_dc_ntvfs",
 324             "ad_dc_default",
 325             "ad_dc_default_smb1",
 326             "ad_dc_slowtests",
 327             "ad_dc_no_nss",
 328             "ad_dc_no_ntlm",
 329             "fl2003dc",
 330             "fl2008dc",
 331             "fl2008r2dc",
 332             "ad_member",
 333             "ad_member_idmap_rid",
 334             "admem_idmap_autorid",
 335             "ad_member_idmap_ad",
 336             "ad_member_rfc2307",
 337             "ad_member_idmap_nss",
 338             "ad_member_oneway",
 339             "chgdcpass",
 340             "vampire_2000_dc",
 341             "fl2000dc",
 342             "fileserver",
 343             "fileserver_smb1",
 344             "fileserver_smb1_done",
 345             "maptoguest",
 346             "simpleserver",
 347             "backupfromdc",
 348             "restoredc",
 349             "renamedc",
 350             "offlinebackupdc",
 351             "labdc",
 352             "preforkrestartdc",
 353             "proclimitdc",
 354             "promoted_dc",
 355             "vampire_dc",
 356             "rodc",
 357             "ad_dc_default",
 358             "ad_dc_default_smb1",
 359             "ad_dc_default_smb1_done",
 360             "ad_dc_slowtests",
 361             "schema_pair_dc",
 362             "schema_dc",
 363             "clusteredmember",
 364             "ad_dc_fips",
 365             "ad_member_fips",
 366             ])),
 367             ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
 368             ("lcov", LCOV_CMD),
 369             ("install", "make install"),
 370             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 371             ("clean", "make clean"),
 372         ],
 373     },
 374
 375     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 376     "samba-mitkrb5": {
 377         "sequence": [
 378             ("random-sleep", random_sleep(300, 900)),
 379             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 380             ("make", "make -j"),
 381             ("test", make_test(exclude_envs=[
 382             "none",
 383             "nt4_dc",
 384             "nt4_dc_smb1",
 385             "nt4_dc_smb1_done",
 386             "nt4_dc_schannel",
 387             "nt4_member",
 388             "ad_dc",
 389             "ad_dc_smb1",
 390             "ad_dc_smb1_done",
 391             "ad_dc_backup",
 392             "ad_dc_ntvfs",
 393             "ad_dc_default",
 394             "ad_dc_default_smb1",
 395             "ad_dc_default_smb1_done",
 396             "ad_dc_slowtests",
 397             "ad_dc_no_nss",
 398             "ad_dc_no_ntlm",
 399             "fl2003dc",
 400             "fl2008dc",
 401             "fl2008r2dc",
 402             "ad_member",
 403             "ad_member_idmap_rid",
 404             "admem_idmap_autorid",
 405             "ad_member_idmap_ad",
 406             "ad_member_rfc2307",
 407             "ad_member_idmap_nss",
 408             "ad_member_oneway",
 409             "chgdcpass",
 410             "vampire_2000_dc",
 411             "fl2000dc",
 412             "fileserver",
 413             "fileserver_smb1",
 414             "fileserver_smb1_done",
 415             "maptoguest",
 416             "simpleserver",
 417             "backupfromdc",
 418             "restoredc",
 419             "renamedc",
 420             "offlinebackupdc",
 421             "labdc",
 422             "preforkrestartdc",
 423             "proclimitdc",
 424             "promoted_dc",
 425             "vampire_dc",
 426             "rodc",
 427             "ad_dc_default",
 428             "ad_dc_default_smb1",
 429             "ad_dc_default_smb1_done",
 430             "ad_dc_slowtests",
 431             "schema_pair_dc",
 432             "schema_dc",
 433             "clusteredmember",
 434             "ad_dc_fips",
 435             "ad_member_fips",
 436             ])),
 437             ("lcov", LCOV_CMD),
 438             ("install", "make install"),
 439             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 440             ("clean", "make clean"),
 441         ],
 442     },
 443
 444     "samba-nt4": {
 445         "dependency": "samba-nt4-build",
 446         "sequence": [
 447             ("random-sleep", random_sleep(300, 900)),
 448             ("test", make_test(include_envs=[
 449             "nt4_dc",
 450             "nt4_dc_smb1",
 451             "nt4_dc_smb1_done",
 452             "nt4_dc_schannel",
 453             "nt4_member",
 454             "simpleserver",
 455             ])),
 456             ("lcov", LCOV_CMD),
 457             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 458         ],
 459     },
 460
 461     "samba-fileserver": {
 462         "dependency": "samba-h5l-build",
 463         "sequence": [
 464             ("random-sleep", random_sleep(300, 900)),
 465             ("test", make_test(include_envs=[
 466             "fileserver",
 467             "fileserver_smb1",
 468             "fileserver_smb1_done",
 469             "maptoguest",
 470             "ktest", # ktest is also tested in samba-ktest-mit samba
 471                      # and samba-mitkrb5 but is tested here against
 472                      # a system Heimdal
 473             ])),
 474             ("lcov", LCOV_CMD),
 475             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 476         ],
 477     },
 478
 479     "samba-fileserver-without-smb1": {
 480         "dependency": "samba-without-smb1-build",
 481         "sequence": [
 482             ("random-sleep", random_sleep(300, 900)),
 483             ("test", make_test(include_envs=["fileserver"])),
 484             ("lcov", LCOV_CMD),
 485             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 486         ],
 487     },
 488
 489     # This is a full build without the AD DC so we test the build with
 490     # MIT Kerberos from the current system.  Runtime behaviour is
 491     # confirmed via the ktest (static ccache and keytab) environment
 492
 493     # This environment also used to confirm we can still build with --with-libunwind
 494     "samba-ktest-mit": {
 495         "sequence": [
 496             ("random-sleep", random_sleep(300, 900)),
 497             ("configure", "./configure.developer --without-ad-dc --with-libunwind --with-system-mitkrb5 " + samba_configure_params),
 498             ("make", "make -j"),
 499             ("test", make_test(include_envs=[
 500             "ktest", # ktest is also tested in fileserver, samba and
 501                      # samba-mitkrb5 but is tested here against a
 502                      # system MIT krb5
 503             ])),
 504             ("lcov", LCOV_CMD),
 505             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 506         ],
 507     },
 508
 509     "samba-admem": {
 510         "dependency": "samba-def-build",
 511         "sequence": [
 512             ("random-sleep", random_sleep(300, 900)),
 513             ("test", make_test(include_envs=[
 514             "ad_member",
 515             "ad_member_idmap_rid",
 516             "admem_idmap_autorid",
 517             "ad_member_idmap_ad",
 518             "ad_member_rfc2307",
 519             "ad_member_idmap_nss",
 520             "ad_member_offlogon",
 521             ])),
 522             ("lcov", LCOV_CMD),
 523             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 524         ],
 525     },
 526
 527     "samba-no-opath1": {
 528         "dependency": "samba-no-opath-build",
 529         "sequence": [
 530             ("random-sleep", random_sleep(300, 900)),
 531             ("test", make_test(
 532                 cmd="make testonly DISABLE_OPATH=1",
 533                 include_envs=[
 534                 "nt4_dc",
 535                 "nt4_dc_smb1",
 536                 "nt4_dc_smb1_done",
 537                 "nt4_dc_schannel",
 538                 "nt4_member",
 539                 "simpleserver",
 540                 ])),
 541             ("lcov", LCOV_CMD),
 542             ("check-clean-tree", "script/clean-source-tree.sh"),
 543         ],
 544     },
 545
 546     "samba-no-opath2": {
 547         "dependency": "samba-no-opath-build",
 548         "sequence": [
 549             ("random-sleep", random_sleep(300, 900)),
 550             ("test", make_test(
 551                 cmd="make testonly DISABLE_OPATH=1",
 552                 include_envs=[
 553                 "fileserver",
 554                 "fileserver_smb1",
 555                 "fileserver_smb1_done",
 556                 ])),
 557             ("lcov", LCOV_CMD),
 558             ("check-clean-tree", "script/clean-source-tree.sh"),
 559         ],
 560     },
 561
 562     "samba-ad-dc-1": {
 563         "dependency": "samba-def-build",
 564         "sequence": [
 565             ("random-sleep", random_sleep(1, 1)),
 566             ("test", make_test(include_envs=[
 567             "ad_dc",
 568             "ad_dc_smb1",
 569             "ad_dc_smb1_done",
 570             "ad_dc_no_nss",
 571             "ad_dc_no_ntlm",
 572             ])),
 573             ("lcov", LCOV_CMD),
 574             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 575         ],
 576     },
 577
 578     "samba-ad-dc-2": {
 579         "dependency": "samba-def-build",
 580         "sequence": [
 581             ("random-sleep", random_sleep(1, 1)),
 582             ("test", make_test(include_envs=[
 583             "vampire_dc",
 584             "vampire_2000_dc",
 585             "rodc",
 586             ])),
 587             ("lcov", LCOV_CMD),
 588             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 589         ],
 590     },
 591
 592     "samba-ad-dc-3": {
 593         "dependency": "samba-def-build",
 594         "sequence": [
 595             ("random-sleep", random_sleep(1, 1)),
 596             ("test", make_test(include_envs=[
 597             "promoted_dc",
 598             "chgdcpass",
 599             "preforkrestartdc",
 600             "proclimitdc",
 601             ])),
 602             ("lcov", LCOV_CMD),
 603             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 604         ],
 605     },
 606
 607     "samba-ad-dc-4a": {
 608         "dependency": "samba-def-build",
 609         "sequence": [
 610             ("random-sleep", random_sleep(1, 1)),
 611             ("test", make_test(include_envs=[
 612             "fl2000dc",
 613             "ad_member_oneway",
 614             "fl2003dc",
 615             ])),
 616             ("lcov", LCOV_CMD),
 617             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 618         ],
 619     },
 620     "samba-ad-dc-4b": {
 621         "dependency": "samba-def-build",
 622         "sequence": [
 623             ("random-sleep", random_sleep(1, 1)),
 624             ("test", make_test(include_envs=[
 625             "fl2008dc",
 626             "fl2008r2dc",
 627             ])),
 628             ("lcov", LCOV_CMD),
 629             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 630         ],
 631     },
 632
 633     "samba-ad-dc-5": {
 634         "dependency": "samba-def-build",
 635         "sequence": [
 636             ("random-sleep", random_sleep(1, 1)),
 637             ("test", make_test(include_envs=[
 638             "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
 639             ("lcov", LCOV_CMD),
 640             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 641         ],
 642     },
 643
 644     "samba-ad-dc-6": {
 645         "dependency": "samba-def-build",
 646         "sequence": [
 647             ("random-sleep", random_sleep(1, 1)),
 648             ("test", make_test(include_envs=["ad_dc_slowtests", "ad_dc_backup"])),
 649             ("lcov", LCOV_CMD),
 650             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 651         ],
 652     },
 653
 654     "samba-schemaupgrade": {
 655         "dependency": "samba-def-build",
 656         "sequence": [
 657             ("random-sleep", random_sleep(1, 1)),
 658             ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 659             ("lcov", LCOV_CMD),
 660             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 661         ],
 662     },
 663
 664     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 665     # This is currently the longest task, so we don't randomly delay it.
 666     "samba-ad-dc-ntvfs": {
 667         "dependency": "samba-def-build",
 668         "sequence": [
 669             ("random-sleep", random_sleep(1, 1)),
 670             ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 671             ("lcov", LCOV_CMD),
 672             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 673         ],
 674     },
 675
 676     # Test fips compliance
 677     "samba-fips": {
 678         "dependency": "samba-mit-build",
 679         "sequence": [
 680             ("random-sleep", random_sleep(1, 1)),
 681             ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
 682             # TODO: This seems to generate only an empty samba-fips.info ("lcov", LCOV_CMD),
 683             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 684         ],
 685     },
 686
 687     # run the backup/restore testenvs separately as they're fairly standalone
 688     # (and CI seems to max out at ~3 different DCs running at once)
 689     "samba-ad-back1": {
 690         "dependency": "samba-def-build",
 691         "sequence": [
 692             ("random-sleep", random_sleep(300, 900)),
 693             ("test", make_test(include_envs=[
 694             "backupfromdc",
 695             "restoredc",
 696             "renamedc",
 697             ])),
 698             ("lcov", LCOV_CMD),
 699             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 700         ],
 701     },
 702     "samba-ad-back2": {
 703         "dependency": "samba-def-build",
 704         "sequence": [
 705             ("random-sleep", random_sleep(300, 900)),
 706             ("test", make_test(include_envs=[
 707             "backupfromdc",
 708             "offlinebackupdc",
 709             "labdc",
 710             ])),
 711             ("lcov", LCOV_CMD),
 712             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 713         ],
 714     },
 715
 716     "samba-admem-mit": {
 717         "dependency": "samba-mit-build",
 718         "sequence": [
 719             ("random-sleep", random_sleep(1, 1)),
 720             ("test", make_test(include_envs=[
 721             "ad_member",
 722             "ad_member_idmap_rid",
 723             "admem_idmap_autorid",
 724             "ad_member_idmap_ad",
 725             "ad_member_rfc2307",
 726             "ad_member_idmap_nss",
 727             "ad_member_offlogon",
 728             ])),
 729             ("lcov", LCOV_CMD),
 730             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 731         ],
 732     },
 733
 734     "samba-addc-mit-1": {
 735         "dependency": "samba-mit-build",
 736         "sequence": [
 737             ("random-sleep", random_sleep(1, 1)),
 738             ("test", make_test(include_envs=[
 739             "ad_dc",
 740             "ad_dc_smb1",
 741             "ad_dc_smb1_done",
 742             "ad_dc_no_nss",
 743             "ad_dc_no_ntlm",
 744             ])),
 745             ("lcov", LCOV_CMD),
 746             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 747         ],
 748     },
 749
 750     "samba-addc-mit-4a": {
 751         "dependency": "samba-mit-build",
 752         "sequence": [
 753             ("random-sleep", random_sleep(1, 1)),
 754             ("test", make_test(include_envs=[
 755             "fl2000dc",
 756             "ad_member_oneway",
 757             "fl2003dc",
 758             ])),
 759             ("lcov", LCOV_CMD),
 760             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 761         ],
 762     },
 763     "samba-addc-mit-4b": {
 764         "dependency": "samba-mit-build",
 765         "sequence": [
 766             ("random-sleep", random_sleep(1, 1)),
 767             ("test", make_test(include_envs=[
 768             "fl2008dc",
 769             "fl2008r2dc",
 770             ])),
 771             ("lcov", LCOV_CMD),
 772             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 773         ],
 774     },
 775
 776     "samba-test-only": {
 777         "sequence": [
 778             ("configure", "./configure.developer  --abi-check-disable" + samba_configure_params),
 779             ("make", "make -j"),
 780             ("test", make_test(TESTS="${TESTS}")),
 781             ("lcov", LCOV_CMD),
 782         ],
 783     },
 784
 785     # Test cross-compile infrastructure
 786     "samba-xc": {
 787         "sequence": [
 788             ("random-sleep", random_sleep(900, 1500)),
 789             ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 790             ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 791             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 792             ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
 793             ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 794             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 795             ("compare-results", "script/compare_cc_results.py "
 796             "./bin/c4che/default{} "
 797             "./bin-xe/c4che/default{} "
 798             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 799             ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
 800             ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
 801             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
 802             ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
 803             " = \"'1234'\"".format(CACHE_SUFFIX)),
 804             ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
 805             ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
 806             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
 807             " ; test $? -ne 0"),
 808         ],
 809     },
 810
 811     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 812     "samba-o3": {
 813         "sequence": [
 814             ("random-sleep", random_sleep(300, 900)),
 815             ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --abi-check-disable" + samba_configure_params),
 816             ("make", "make -j"),
 817             ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 818             ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 819             ("lcov", LCOV_CMD),
 820             ("install", "make install"),
 821             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 822             ("clean", "make clean"),
 823         ],
 824     },
 825
 826     "samba-32bit": {
 827         "sequence": [
 828             ("random-sleep", random_sleep(300, 900)),
 829             ("configure", "./configure.developer --abi-check-disable --disable-warnings-as-errors" + samba_configure_params),
 830             ("make", "make -j"),
 831             ("nonetest", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 832             ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 833             ("ktest", make_test(cmd='make test', include_envs=["ktest"])),
 834             ("install", "make install"),
 835             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 836             ("clean", "make clean"),
 837         ],
 838     },
 839
 840     "samba-ctdb": {
 841         "sequence": [
 842             ("random-sleep", random_sleep(900, 1500)),
 843
 844         # make sure we have tdb around:
 845             ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 846             ("tdb-make", "cd lib/tdb && make"),
 847             ("tdb-install", "cd lib/tdb && make install"),
 848
 849         # build samba with cluster support (also building ctdb):
 850             ("samba-configure",
 851          "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
 852          "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
 853          "./configure.developer ${PREFIX} "
 854          "--with-selftest-prefix=./bin/ab "
 855          "--with-cluster-support "
 856          "--without-ad-dc "
 857          "--bundled-libraries=!tdb"),
 858             ("samba-make", "make"),
 859             ("samba-check", "./bin/smbd --configfile=/dev/null -b | grep CLUSTER_SUPPORT"),
 860             ("samba-install", "make install"),
 861             ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 862
 863             ("test", make_test(
 864                 cmd='PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH make test',
 865                 INJECT_SELFTEST_PREFIX=0,
 866                 include_envs=["clusteredmember"])
 867             ),
 868
 869         # clean up:
 870             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 871             ("clean", "make clean"),
 872             ("ctdb-clean", "cd ./ctdb && make clean"),
 873         ],
 874     },
 875
 876     "samba-libs": {
 877         "sequence": [
 878             ("random-sleep", random_sleep(300, 900)),
 879             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 880             ("talloc-make", "cd lib/talloc && make"),
 881             ("talloc-install", "cd lib/talloc && make install"),
 882
 883             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 884             ("tdb-make", "cd lib/tdb && make"),
 885             ("tdb-install", "cd lib/tdb && make install"),
 886
 887             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 888             ("tevent-make", "cd lib/tevent && make"),
 889             ("tevent-install", "cd lib/tevent && make install"),
 890
 891             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
 892             ("ldb-make", "cd lib/ldb && make"),
 893             ("ldb-install", "cd lib/ldb && make install"),
 894
 895             ("nondevel-configure", samba_libs_envvars + " ./configure ${PREFIX}"),
 896             ("nondevel-make", "make -j"),
 897             ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 898             ("nondevel-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
 899             ("nondevel-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
 900             ("nondevel-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
 901             ("nondevel-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"),
 902             ("nondevel-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 903             ("nondevel-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 904             ("nondevel-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 905             ("nondevel-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 906             ("nondevel-no-public-nss_winbind",
 907                 check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
 908             ("nondevel-no-public-nss_wins",
 909                 check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
 910             ("nondevel-no-public-libwbclient",
 911                 check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
 912             ("nondevel-no-public-pam_winbind",
 913                 check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
 914             ("nondevel-no-public-winbind_krb5_locator",
 915                 check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
 916             ("nondevel-no-public-async_dns_krb5_locator",
 917                 check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
 918             ("nondevel-install", "make -j install"),
 919             ("nondevel-dist", "make dist"),
 920
 921             ("prefix-no-private-libtalloc", "find ${PREFIX_DIR} | grep -v 'libtalloc-report' | grep 'private.*libtalloc' && exit 1; exit 0"),
 922             ("prefix-no-private-libtdb", "find ${PREFIX_DIR} | grep -v 'libtdb-wrap' | grep 'private.*libtdb' && exit 1; exit 0"),
 923             ("prefix-no-private-libtevent", "find ${PREFIX_DIR} | grep -v 'libtevent-util' | grep 'private.*libtevent' && exit 1; exit 0"),
 924             ("prefix-no-private-libldb", "find ${PREFIX_DIR} | grep -v 'module' | grep -v 'libldbsamba' | grep 'private.*libldb' && exit 1; exit 0"),
 925             ("prefix-no-samba-nss_winbind", "ldd ${PREFIX_DIR}/lib/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 926             ("prefix-no-samba-nss_wins", "ldd ${PREFIX_DIR}/lib/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 927             ("prefix-no-samba-libwbclient", "ldd ${PREFIX_DIR}/lib/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 928             ("prefix-no-samba-pam_winbind", "ldd ${PREFIX_DIR}/lib/security/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 929             ("prefix-no-public-nss_winbind",
 930                 check_symbols("${PREFIX_DIR}/lib/libnss_winbind.so.2", "_nss_winbind_")),
 931             ("prefix-no-public-nss_wins",
 932                 check_symbols("${PREFIX_DIR}/lib/libnss_wins.so.2", "_nss_wins_")),
 933             ("prefix-no-public-libwbclient",
 934                 check_symbols("${PREFIX_DIR}/lib/libwbclient.so.0", "wbc")),
 935             ("prefix-no-public-pam_winbind",
 936                 check_symbols("${PREFIX_DIR}/lib/security/pam_winbind.so", "pam_sm_")),
 937             ("prefix-no-public-winbind_krb5_locator",
 938                 check_symbols("${PREFIX_DIR}/lib/krb5/winbind_krb5_locator.so",
 939                               "service_locator")),
 940             ("prefix-no-public-async_dns_krb5_locator",
 941                 check_symbols("${PREFIX_DIR}/lib/krb5/async_dns_krb5_locator.so",
 942                               "service_locator")),
 943
 944             # retry with all modules shared
 945             ("allshared-distclean", "make distclean"),
 946             ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 947             ("allshared-make", "make -j"),
 948             ("allshared-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
 949             ("allshared-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
 950             ("allshared-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
 951             ("allshared-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"),
 952             ("allshared-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 953             ("allshared-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 954             ("allshared-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 955             ("allshared-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 956             ("allshared-no-public-nss_winbind",
 957                 check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
 958             ("allshared-no-public-nss_wins",
 959                 check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
 960             ("allshared-no-public-libwbclient",
 961                 check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
 962             ("allshared-no-public-pam_winbind",
 963                 check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
 964             ("allshared-no-public-winbind_krb5_locator",
 965                 check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
 966             ("allshared-no-public-async_dns_krb5_locator",
 967                 check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
 968         ],
 969     },
 970
 971     "samba-fuzz": {
 972         "sequence": [
 973         # build the fuzzers (static) via the oss-fuzz script
 974             ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
 975             ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ADDITIONAL_LDFLAGS='-fuse-ld=bfd' ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl-fuzzer"),
 976         ],
 977     },
 978
 979     # * Test smbd and smbtorture can build semi-static
 980     #
 981     # * Test Samba without python still builds.
 982     #
 983     # When this test fails due to more use of Python, the expectations
 984     # is that the newly failing part of the code should be disabled
 985     # when --disable-python is set (rather than major work being done
 986     # to support this environment).
 987     #
 988     # The target here is for vendors shipping a minimal smbd.
 989     "samba-minimal-smbd": {
 990         "sequence": [
 991             ("random-sleep", random_sleep(300, 900)),
 992
 993         # build with all modules static
 994             ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
 995             ("allstatic-make", "make -j"),
 996             ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 997             ("allstatic-lcov", LCOV_CMD),
 998             ("allstatic-def-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 999             ("allstatic-def-clean", "make clean"),
1000
1001         # force all libraries as private
1002             ("allprivate-def-distclean", "make distclean"),
1003             ("allprivate-def-configure", "./configure.developer " + samba_configure_params + " --private-libraries=ALL"),
1004             ("allprivate-def-make", "make -j"),
1005             # note wrapper libraries need to be public
1006             ("allprivate-def-no-public", "ls ./bin/shared | egrep -v '^private$|lib[nprsu][saeoi][smscd].*-wrapper.so$|pam_set_items.so' | wc -l | grep -q '^0'"),
1007             ("allprivate-def-only-private-ext", "ls ./bin/shared/private | egrep 'private-samba' | wc -l | grep -q '^0' && exit 1; exit 0"),
1008             ("allprivate-def-no-non-private-ext", "ls ./bin/shared/private | egrep -v 'private-samba|^libpypamtest.so$' | wc -l | grep -q '^0'"),
1009             ("allprivate-def-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
1010             ("allprivate-def-lcov", LCOV_CMD),
1011             ("allprivate-def-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1012             ("allprivate-def-clean", "make clean"),
1013
1014         # force all libraries as private with a non default
1015         # extension and 2 exceptions
1016             ("allprivate-ext-distclean", "make distclean"),
1017             ("allprivate-ext-configure", "./configure.developer " + samba_configure_params + " --private-libraries=ALL --private-library-extension=private-library --private-extension-exception=pac,ndr"),
1018             ("allprivate-ext-make", "make -j"),
1019             # note wrapper libraries need to be public
1020             ("allprivate-ext-no-public", "ls ./bin/shared | egrep -v '^private$|lib[nprsu][saeoi][smscd].*-wrapper.so$|pam_set_items.so' | wc -l | grep -q '^0'"),
1021             ("allprivate-ext-no-private-default-ext", "ls ./bin/shared/private | grep 'private-samba' | wc -l | grep -q '^0'"),
1022             ("allprivate-ext-has-private-ext", "ls ./bin/shared/private | grep 'private-library' | wc -l | grep -q '^0' && exit 1; exit 0"),
1023             ("allprivate-ext-libndr-no-private-ext", "ls ./bin/shared/private | grep -v 'private-library' | grep 'libndr' | wc -l | grep -q '^1'"),
1024             ("allprivate-ext-libpac-no-private-ext", "ls ./bin/shared/private | grep -v 'private-library' | grep 'libpac' | wc -l | grep -q '^1'"),
1025             ("allprivate-ext-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
1026             ("allprivate-ext-lcov", LCOV_CMD),
1027             ("allprivate-ext-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1028             ("allprivate-ext-clean", "make clean"),
1029
1030         # retry with nonshared smbd and smbtorture
1031             ("nonshared-distclean", "make distclean"),
1032             ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
1033             ("nonshared-make", "make -j"),
1034             ("nonshared-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
1035             ("nonshared-lcov", LCOV_CMD),
1036             ("nonshared-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1037             ("nonshared-clean", "make clean"),
1038         ],
1039     },
1040
1041     "samba-nopython": {
1042         "sequence": [
1043             ("random-sleep", random_sleep(300, 900)),
1044
1045             ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
1046             ("make", "make -j"),
1047             ("find-python", "script/find_python.sh ${PREFIX}"),
1048             ("test", "make test-nopython"),
1049             ("lcov", LCOV_CMD),
1050             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1051             ("clean", "make clean"),
1052
1053             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1054             ("talloc-make", "cd lib/talloc && make"),
1055             ("talloc-install", "cd lib/talloc && make install"),
1056
1057             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1058             ("tdb-make", "cd lib/tdb && make"),
1059             ("tdb-install", "cd lib/tdb && make install"),
1060
1061             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1062             ("tevent-make", "cd lib/tevent && make"),
1063             ("tevent-install", "cd lib/tevent && make install"),
1064
1065             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1066             ("ldb-make", "cd lib/ldb && make"),
1067             ("ldb-install", "cd lib/ldb && make install"),
1068
1069         # retry against installed library packages, but no required modules
1070             ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc  --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
1071             ("libs-make", "make -j"),
1072             ("libs-install", "make install"),
1073             ("libs-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1074             ("libs-clean", "make clean"),
1075
1076         ],
1077     },
1078
1079     "samba-codecheck": {
1080         "sequence": [
1081             ("run", "script/check-shell-scripts.sh ."),
1082             ("run", "script/codespell.sh ."),
1083         ],
1084     },
1085
1086     "ldb": {
1087         "sequence": [
1088             ("random-sleep", random_sleep(60, 600)),
1089             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1090             ("make", "make"),
1091             ("install", "make install"),
1092             ("test", "make test"),
1093             ("lcov", LCOV_CMD),
1094             ("clean", "make clean"),
1095             ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
1096             ("make-no-lmdb", "make"),
1097             ("test-no-lmdb", "make test"),
1098             ("lcov-no-lmdb", LCOV_CMD),
1099             ("install-no-lmdb", "make install"),
1100             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1101             ("distcheck", "make distcheck"),
1102             ("clean", "make clean"),
1103         ],
1104     },
1105
1106     "tdb": {
1107         "sequence": [
1108             ("random-sleep", random_sleep(60, 600)),
1109             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1110             ("make", "make"),
1111             ("install", "make install"),
1112             ("test", "make test"),
1113             ("lcov", LCOV_CMD),
1114             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1115             ("distcheck", "make distcheck"),
1116             ("clean", "make clean"),
1117         ],
1118     },
1119
1120     "talloc": {
1121         "sequence": [
1122             ("random-sleep", random_sleep(60, 600)),
1123             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1124             ("make", "make"),
1125             ("install", "make install"),
1126             ("test", "make test"),
1127             ("lcov", LCOV_CMD),
1128             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1129             ("distcheck", "make distcheck"),
1130             ("clean", "make clean"),
1131         ],
1132     },
1133
1134     "replace": {
1135         "sequence": [
1136             ("random-sleep", random_sleep(60, 600)),
1137             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1138             ("make", "make"),
1139             ("install", "make install"),
1140             ("test", "make test"),
1141             ("lcov", LCOV_CMD),
1142             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1143             ("distcheck", "make distcheck"),
1144             ("clean", "make clean"),
1145         ],
1146     },
1147
1148     "tevent": {
1149         "sequence": [
1150             ("random-sleep", random_sleep(60, 600)),
1151             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1152             ("make", "make"),
1153             ("install", "make install"),
1154             ("test", "make test"),
1155             ("lcov", LCOV_CMD),
1156             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1157             ("distcheck", "make distcheck"),
1158             ("clean", "make clean"),
1159         ],
1160     },
1161
1162     "pidl": {
1163         "git-clone-required": True,
1164         "sequence": [
1165             ("random-sleep", random_sleep(60, 600)),
1166             ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
1167             ("touch", "touch *.yp"),
1168             ("make", "make"),
1169             ("test", "make test"),
1170             ("install", "make install"),
1171             ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
1172             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1173             ("clean", "make clean"),
1174         ],
1175     },
1176
1177     # these are useful for debugging autobuild
1178     "pass": {
1179         "sequence": [
1180             ("pass", 'echo passing && /bin/true'),
1181         ],
1182     },
1183     "fail": {
1184         "sequence": [
1185             ("fail", 'echo failing && /bin/false'),
1186         ],
1187     },
1188 }
1189
1190 defaulttasks = list(tasks.keys())
1191
1192 defaulttasks.remove("pass")
1193 defaulttasks.remove("fail")
1194
1195 # The build tasks will be brought in by the test tasks as needed
1196 defaulttasks.remove("samba-def-build")
1197 defaulttasks.remove("samba-nt4-build")
1198 defaulttasks.remove("samba-mit-build")
1199 defaulttasks.remove("samba-h5l-build")
1200 defaulttasks.remove("samba-no-opath-build")
1201
1202 # This is not a normal test, but a task to support manually running
1203 # one test under autobuild
1204 defaulttasks.remove("samba-test-only")
1205
1206 # Only built on GitLab CI and not in the default autobuild because it
1207 # uses too much space (4GB of semi-static binaries)
1208 defaulttasks.remove("samba-fuzz")
1209
1210 # The FIPS build runs only in GitLab CI on a current Fedora Docker
1211 # container where a simulated FIPS mode is possible.
1212 defaulttasks.remove("samba-fips")
1213
1214 # The MIT build runs on a current Fedora where an up to date MIT KDC
1215 # is already packaged.  This avoids needing to backport a current MIT
1216 # to the default Ubuntu 18.04, particularly during development, and
1217 # the need to install on the shared sn-devel-184.
1218
1219 defaulttasks.remove("samba-mitkrb5")
1220 defaulttasks.remove("samba-admem-mit")
1221 defaulttasks.remove("samba-addc-mit-1")
1222 defaulttasks.remove("samba-addc-mit-4a")
1223 defaulttasks.remove("samba-addc-mit-4b")
1224
1225 defaulttasks.remove("samba-32bit")
1226
1227 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
1228     defaulttasks.remove("samba-o3")
1229
1230
1231 def do_print(msg):
1232     logger.info(msg)
1233     sys.stdout.flush()
1234     sys.stderr.flush()
1235
1236 def do_debug(msg):
1237     logger.debug(msg)
1238     sys.stdout.flush()
1239     sys.stderr.flush()
1240
1241
1242 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
1243     if show is None:
1244         do_debug("Running: '%s' in '%s'" % (cmd, dir))
1245     elif show:
1246         do_print("Running: '%s' in '%s'" % (cmd, dir))
1247
1248     if output:
1249         out = check_output([cmd], shell=True, cwd=dir)
1250         return out.decode(encoding='utf-8', errors='backslashreplace')
1251     elif checkfail:
1252         return check_call(cmd, shell=True, cwd=dir)
1253     else:
1254         return call(cmd, shell=True, cwd=dir)
1255
1256 def rmdir_force(dirname, re_raise=True):
1257     try:
1258         run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % (
1259                 dirname, dirname, dirname), output=True, show=True)
1260     except CalledProcessError as e:
1261         do_print("Failed: '%s'" % (str(e)))
1262         run_cmd("tree %s" % dirname, output=True, show=True)
1263         if re_raise:
1264             raise
1265         return False
1266     return True
1267
1268 class builder(object):
1269     '''handle build of one directory'''
1270
1271     def __init__(self, name, definition):
1272         self.name = name
1273         self.dir = builddirs.get(name, '.')
1274         self.tag = self.name.replace('/', '_')
1275         self.definition = definition
1276         self.sequence = definition["sequence"]
1277         self.git_clone_required = False
1278         if "git-clone-required" in definition:
1279             self.git_clone_required = bool(definition["git-clone-required"])
1280         self.proc = None
1281         self.done = False
1282         self.next = 0
1283         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
1284         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
1285         do_debug("stdout for %s in %s" % (self.name, self.stdout_path))
1286         do_debug("stderr for %s in %s" % (self.name, self.stderr_path))
1287         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
1288         self.stdout = open(self.stdout_path, 'w')
1289         self.stderr = open(self.stderr_path, 'w')
1290         self.stdin  = open("/dev/null", 'r')
1291         self.builder_dir = "%s/%s" % (testbase, self.tag)
1292         self.test_source_dir = self.builder_dir
1293         self.cwd = "%s/%s" % (self.builder_dir, self.dir)
1294         self.selftest_prefix = "%s/bin/ab" % (self.cwd)
1295         self.prefix = "%s/%s" % (test_prefix, self.tag)
1296         self.consumers = []
1297         self.producer = None
1298
1299         if self.git_clone_required:
1300             assert "dependency" not in definition
1301
1302     def mark_existing(self):
1303         do_debug('%s: Mark as existing dependency' % self.name)
1304         self.next = len(self.sequence)
1305         self.done = True
1306
1307     def add_consumer(self, consumer):
1308         do_debug("%s: add consumer: %s" % (self.name, consumer.name))
1309         consumer.producer = self
1310         consumer.test_source_dir = self.test_source_dir
1311         self.consumers.append(consumer)
1312
1313     def start_next(self):
1314         if self.producer is not None:
1315             if not self.producer.done:
1316                 do_debug("%s: Waiting for producer: %s" % (self.name, self.producer.name))
1317                 return
1318
1319         if self.next == 0:
1320             rmdir_force(self.builder_dir)
1321             rmdir_force(self.prefix)
1322             if self.producer is not None:
1323                 run_cmd("mkdir %s" % (self.builder_dir), dir=test_master, show=True)
1324             elif not self.git_clone_required:
1325                 run_cmd("cp -R -a -l %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1326             else:
1327                 run_cmd("git clone --recursive --shared %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1328
1329         if self.next == len(self.sequence):
1330             if not self.done:
1331                 do_print('%s: Completed OK' % self.name)
1332                 self.done = True
1333             if not options.nocleanup and len(self.consumers) == 0:
1334                 do_print('%s: Cleaning up' % self.name)
1335                 rmdir_force(self.builder_dir)
1336                 rmdir_force(self.prefix)
1337             for consumer in self.consumers:
1338                 if consumer.next != 0:
1339                     continue
1340                 do_print('%s: Starting consumer %s' % (self.name, consumer.name))
1341                 consumer.start_next()
1342             if self.producer is not None:
1343                 self.producer.consumers.remove(self)
1344                 assert self.producer.done
1345                 self.producer.start_next()
1346             do_print('%s: Remaining consumers %u' % (self.name, len(self.consumers)))
1347             return
1348         (self.stage, self.cmd) = self.sequence[self.next]
1349         self.cmd = self.cmd.replace("${PYTHON_PREFIX}",
1350                                     get_path(name='platlib',
1351                                              scheme="posix_prefix",
1352                                              vars={"base": self.prefix,
1353                                                    "platbase": self.prefix}))
1354         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
1355         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
1356         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
1357         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
1358         self.cmd = self.cmd.replace("${SELFTEST_PREFIX}", self.selftest_prefix)
1359         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
1360         self.cmd = self.cmd.replace("${NAME}", self.name)
1361         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
1362         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
1363         self.proc = Popen(self.cmd, shell=True,
1364                           close_fds=True, cwd=self.cwd,
1365                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
1366         self.next += 1
1367
1368 def expand_dependencies(n):
1369     deps = list()
1370     if "dependency" in tasks[n]:
1371         depname = tasks[n]["dependency"]
1372         assert depname in tasks
1373         sdeps = expand_dependencies(depname)
1374         assert n not in sdeps
1375         for sdep in sdeps:
1376             deps.append(sdep)
1377         deps.append(depname)
1378     return deps
1379
1380
1381 class buildlist(object):
1382     '''handle build of multiple directories'''
1383
1384     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
1385         self.tail_proc = None
1386         self.retry = None
1387         if not tasknames:
1388             if options.restrict_tests:
1389                 tasknames = ["samba-test-only"]
1390             else:
1391                 tasknames = defaulttasks
1392
1393         given_tasknames = tasknames.copy()
1394         implicit_tasknames = []
1395         for n in given_tasknames:
1396             deps = expand_dependencies(n)
1397             for dep in deps:
1398                 if dep in given_tasknames:
1399                     continue
1400                 if dep in implicit_tasknames:
1401                     continue
1402                 implicit_tasknames.append(dep)
1403
1404         tasknames = implicit_tasknames.copy()
1405         tasknames.extend(given_tasknames)
1406         do_debug("given_tasknames: %s" % given_tasknames)
1407         do_debug("implicit_tasknames: %s" % implicit_tasknames)
1408         do_debug("tasknames: %s" % tasknames)
1409         self.tlist = [builder(n, tasks[n]) for n in tasknames]
1410
1411         if options.retry:
1412             rebase_remote = "rebaseon"
1413             retry_task = {
1414                     "git-clone-required": True,
1415                     "sequence": [
1416                             ("retry",
1417                             '''set -e
1418                             git remote add -t %s %s %s
1419                             git fetch %s
1420                             while :; do
1421                               sleep 60
1422                               git describe %s/%s > old_remote_branch.desc
1423                               git fetch %s
1424                               git describe %s/%s > remote_branch.desc
1425                               diff old_remote_branch.desc remote_branch.desc
1426                             done
1427                            ''' % (
1428                                rebase_branch, rebase_remote, rebase_url,
1429                                rebase_remote,
1430                                rebase_remote, rebase_branch,
1431                                rebase_remote,
1432                                rebase_remote, rebase_branch
1433                             ))]}
1434
1435             self.retry = builder('retry', retry_task)
1436             self.need_retry = False
1437
1438         if options.skip_dependencies:
1439             for b in self.tlist:
1440                 if b.name in implicit_tasknames:
1441                     b.mark_existing()
1442
1443         for b in self.tlist:
1444             do_debug("b.name=%s" % b.name)
1445             if "dependency" not in b.definition:
1446                 continue
1447             depname = b.definition["dependency"]
1448             do_debug("b.name=%s: dependency:%s" % (b.name, depname))
1449             for p in self.tlist:
1450                 if p.name == depname:
1451                     p.add_consumer(b)
1452
1453     def kill_kids(self):
1454         if self.tail_proc is not None:
1455             self.tail_proc.terminate()
1456             self.tail_proc.wait()
1457             self.tail_proc = None
1458         if self.retry is not None:
1459             self.retry.proc.terminate()
1460             self.retry.proc.wait()
1461             self.retry = None
1462         for b in self.tlist:
1463             if b.proc is not None:
1464                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
1465                 b.proc.terminate()
1466                 b.proc.wait()
1467                 b.proc = None
1468
1469     def wait_one(self):
1470         while True:
1471             none_running = True
1472             for b in self.tlist:
1473                 if b.proc is None:
1474                     continue
1475                 none_running = False
1476                 b.status = b.proc.poll()
1477                 if b.status is None:
1478                     continue
1479                 b.proc = None
1480                 return b
1481             if options.retry:
1482                 ret = self.retry.proc.poll()
1483                 if ret is not None:
1484                     self.need_retry = True
1485                     self.retry = None
1486                     return None
1487             if none_running:
1488                 return None
1489             time.sleep(0.1)
1490
1491     def run(self):
1492         for b in self.tlist:
1493             b.start_next()
1494         if options.retry:
1495             self.retry.start_next()
1496         while True:
1497             b = self.wait_one()
1498             if options.retry and self.need_retry:
1499                 self.kill_kids()
1500                 do_print("retry needed")
1501                 return (0, None, None, None, "retry")
1502             if b is None:
1503                 break
1504             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
1505                 self.kill_kids()
1506                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
1507             b.start_next()
1508         self.kill_kids()
1509         return (0, None, None, None, "All OK")
1510
1511     def write_system_info(self, filename):
1512         with open(filename, 'w') as f:
1513             for cmd in ['uname -a',
1514                         'lsb_release -a',
1515                         'free',
1516                         'mount',
1517                         'cat /proc/cpuinfo',
1518                         'cc --version',
1519                         'df -m .',
1520                         'df -m %s' % testbase]:
1521                 try:
1522                     out = run_cmd(cmd, output=True, checkfail=False)
1523                 except CalledProcessError as e:
1524                     out = "<failed: %s>" % str(e)
1525                 print('### %s' % cmd, file=f)
1526                 print(out, file=f)
1527                 print(file=f)
1528
1529     def tarlogs(self, fname):
1530         with tarfile.open(fname, "w:gz") as tar:
1531             for b in self.tlist:
1532                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
1533                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
1534             if os.path.exists("autobuild.log"):
1535                 tar.add("autobuild.log")
1536             filename = 'system-info.txt'
1537             self.write_system_info(filename)
1538             tar.add(filename)
1539
1540     def remove_logs(self):
1541         for b in self.tlist:
1542             os.unlink(b.stdout_path)
1543             os.unlink(b.stderr_path)
1544
1545     def start_tail(self):
1546         cmd = ["tail", "-f"]
1547         for b in self.tlist:
1548             cmd.append(b.stdout_path)
1549             cmd.append(b.stderr_path)
1550         self.tail_proc = Popen(cmd, close_fds=True)
1551
1552
1553 def cleanup(do_raise=False):
1554     if options.nocleanup:
1555         return
1556     run_cmd("stat %s || true" % test_tmpdir, show=True)
1557     run_cmd("stat %s" % testbase, show=True)
1558     do_print("Cleaning up %r" % cleanup_list)
1559     for d in cleanup_list:
1560         ok = rmdir_force(d, re_raise=False)
1561         if ok:
1562             continue
1563         if os.path.isdir(d):
1564             do_print("Killing, waiting and retry")
1565             run_cmd("killbysubdir %s > /dev/null 2>&1" % d, checkfail=False)
1566         else:
1567             do_print("Waiting and retry")
1568         time.sleep(1)
1569         rmdir_force(d, re_raise=do_raise)
1570
1571
1572 def daemonize(logfile):
1573     pid = os.fork()
1574     if pid == 0:  # Parent
1575         os.setsid()
1576         pid = os.fork()
1577         if pid != 0:  # Actual daemon
1578             os._exit(0)
1579     else:  # Grandparent
1580         os._exit(0)
1581
1582     import resource      # Resource usage information.
1583     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1584     if maxfd == resource.RLIM_INFINITY:
1585         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1586     for fd in range(0, maxfd):
1587         try:
1588             os.close(fd)
1589         except OSError:
1590             pass
1591     os.open(logfile, os.O_RDWR | os.O_CREAT)
1592     os.dup2(0, 1)
1593     os.dup2(0, 2)
1594
1595
1596 def write_pidfile(fname):
1597     '''write a pid file, cleanup on exit'''
1598     with open(fname, mode='w') as f:
1599         f.write("%u\n" % os.getpid())
1600
1601
1602 def rebase_tree(rebase_url, rebase_branch="master"):
1603     rebase_remote = "rebaseon"
1604     do_print("Rebasing on %s" % rebase_url)
1605     run_cmd("git describe HEAD", show=True, dir=test_master)
1606     run_cmd("git remote add -t %s %s %s" %
1607             (rebase_branch, rebase_remote, rebase_url),
1608             show=True, dir=test_master)
1609     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1610     if options.fix_whitespace:
1611         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1612                 (rebase_remote, rebase_branch),
1613                 show=True, dir=test_master)
1614     else:
1615         run_cmd("git rebase --force-rebase %s/%s" %
1616                 (rebase_remote, rebase_branch),
1617                 show=True, dir=test_master)
1618     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1619                    (rebase_remote, rebase_branch),
1620                    dir=test_master, output=True)
1621     if diff == '':
1622         do_print("No differences between HEAD and %s/%s - exiting" %
1623                  (rebase_remote, rebase_branch))
1624         sys.exit(0)
1625     run_cmd("git describe %s/%s" %
1626             (rebase_remote, rebase_branch),
1627             show=True, dir=test_master)
1628     run_cmd("git describe HEAD", show=True, dir=test_master)
1629     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1630             (rebase_remote, rebase_branch),
1631             show=True, dir=test_master)
1632
1633
1634 def push_to(push_url, push_branch="master"):
1635     push_remote = "pushto"
1636     do_print("Pushing to %s" % push_url)
1637     if options.mark:
1638         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1639         run_cmd("git commit --amend -c HEAD", dir=test_master)
1640         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1641         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1642     run_cmd("git remote add -t %s %s %s" %
1643             (push_branch, push_remote, push_url),
1644             show=True, dir=test_master)
1645     run_cmd("git push %s +HEAD:%s" %
1646             (push_remote, push_branch),
1647             show=True, dir=test_master)
1648
1649
1650 def send_email(subject, text, log_tar):
1651     if options.email is None:
1652         do_print("not sending email because the recipient is not set")
1653         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1654                  (subject, text))
1655         return
1656     outer = MIMEMultipart()
1657     outer['Subject'] = subject
1658     outer['To'] = options.email
1659     outer['From'] = options.email_from
1660     outer['Date'] = email.utils.formatdate(localtime=True)
1661     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1662     outer.attach(MIMEText(text, 'plain', 'utf-8'))
1663     if options.attach_logs:
1664         with open(log_tar, 'rb') as fp:
1665             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1666         # Set the filename parameter
1667         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1668         outer.attach(msg)
1669     content = outer.as_string()
1670     s = smtplib.SMTP(options.email_server)
1671     email_user = os.getenv('SMTP_USERNAME')
1672     email_password = os.getenv('SMTP_PASSWORD')
1673     if email_user is not None:
1674         s.starttls()
1675         s.login(email_user, email_password)
1676
1677     s.sendmail(options.email_from, [options.email], content)
1678     s.set_debuglevel(1)
1679     s.quit()
1680
1681
1682 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1683                   elapsed_time, log_base=None, add_log_tail=True):
1684     '''send an email to options.email about the failure'''
1685     elapsed_minutes = elapsed_time / 60.0
1686     if log_base is None:
1687         log_base = gitroot
1688     text = '''
1689 Dear Developer,
1690
1691 Your autobuild on %s failed after %.1f minutes
1692 when trying to test %s with the following error:
1693
1694    %s
1695
1696 the autobuild has been abandoned. Please fix the error and resubmit.
1697
1698 A summary of the autobuild process is here:
1699
1700   %s/autobuild.log
1701 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1702
1703     if options.restrict_tests:
1704         text += """
1705 The build was restricted to tests matching %s\n""" % options.restrict_tests
1706
1707     if failed_task != 'rebase':
1708         text += '''
1709 You can see logs of the failed task here:
1710
1711   %s/%s.stdout
1712   %s/%s.stderr
1713
1714 or you can get full logs of all tasks in this job here:
1715
1716   %s/logs.tar.gz
1717
1718 The top commit for the tree that was built was:
1719
1720 %s
1721
1722 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1723
1724     log_stdout = "%s/%s.stdout" % (gitroot, failed_tag)
1725     if add_log_tail and os.access(log_stdout, os.R_OK):
1726         f = open(log_stdout, 'r')
1727         lines = f.readlines()
1728         log_tail = "".join(lines[-50:])
1729         num_lines = len(lines)
1730         log_stderr = "%s/%s.stderr" % (gitroot, failed_tag)
1731         if num_lines < 50 and os.access(log_stderr, os.R_OK):
1732             # Also include stderr (compile failures) if < 50 lines of stdout
1733             f = open(log_stderr, 'r')
1734             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1735
1736         text += '''
1737 The last 50 lines of log messages:
1738
1739 %s
1740     ''' % log_tail
1741         f.close()
1742
1743     logs = os.path.join(gitroot, 'logs.tar.gz')
1744     send_email('autobuild[%s] failure on %s for task %s during %s'
1745                % (options.branch, platform.node(), failed_task, failed_stage),
1746                text, logs)
1747
1748
1749 def email_success(elapsed_time, log_base=None):
1750     '''send an email to options.email about a successful build'''
1751     if log_base is None:
1752         log_base = gitroot
1753     text = '''
1754 Dear Developer,
1755
1756 Your autobuild on %s has succeeded after %.1f minutes.
1757
1758 ''' % (platform.node(), elapsed_time / 60.)
1759
1760     if options.restrict_tests:
1761         text += """
1762 The build was restricted to tests matching %s\n""" % options.restrict_tests
1763
1764     if options.keeplogs:
1765         text += '''
1766
1767 you can get full logs of all tasks in this job here:
1768
1769   %s/logs.tar.gz
1770
1771 ''' % log_base
1772
1773     text += '''
1774 The top commit for the tree that was built was:
1775
1776 %s
1777 ''' % top_commit_msg
1778
1779     logs = os.path.join(gitroot, 'logs.tar.gz')
1780     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1781                text, logs)
1782
1783
1784 # get the top commit message, for emails
1785 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1786
1787 try:
1788     if options.skip_dependencies:
1789         run_cmd("stat %s" % testbase, dir=testbase, output=True)
1790     else:
1791         os.makedirs(testbase)
1792 except Exception as reason:
1793     raise Exception("Unable to create %s : %s" % (testbase, reason))
1794 cleanup_list.append(testbase)
1795
1796 if options.daemon:
1797     logfile = os.path.join(testbase, "log")
1798     do_print("Forking into the background, writing progress to %s" % logfile)
1799     daemonize(logfile)
1800
1801 write_pidfile(gitroot + "/autobuild.pid")
1802
1803 start_time = time.time()
1804
1805 while True:
1806     try:
1807         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1808         os.makedirs(test_tmpdir)
1809         # The waf uninstall code removes empty directories all the way
1810         # up the tree.  Creating a file in test_tmpdir stops it from
1811         # being removed.
1812         run_cmd("touch %s" % os.path.join(test_tmpdir,
1813                                           ".directory-is-not-empty"), show=True)
1814         run_cmd("stat %s" % test_tmpdir, show=True)
1815         run_cmd("stat %s" % testbase, show=True)
1816         if options.skip_dependencies:
1817             run_cmd("stat %s" % test_master, dir=testbase, output=True)
1818         else:
1819             run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1820     except Exception:
1821         cleanup()
1822         raise
1823
1824     try:
1825         if options.rebase is not None:
1826             rebase_tree(options.rebase, rebase_branch=options.branch)
1827     except Exception:
1828         cleanup_list.append(gitroot + "/autobuild.pid")
1829         cleanup()
1830         elapsed_time = time.time() - start_time
1831         email_failure(-1, 'rebase', 'rebase', 'rebase',
1832                       'rebase on %s failed' % options.branch,
1833                       elapsed_time, log_base=options.log_base)
1834         sys.exit(1)
1835
1836     try:
1837         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1838         if options.tail:
1839             blist.start_tail()
1840         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1841         if status != 0 or errstr != "retry":
1842             break
1843         cleanup(do_raise=True)
1844     except Exception:
1845         cleanup()
1846         raise
1847
1848 cleanup_list.append(gitroot + "/autobuild.pid")
1849
1850 do_print(errstr)
1851
1852 blist.kill_kids()
1853 if options.tail:
1854     do_print("waiting for tail to flush")
1855     time.sleep(1)
1856
1857 elapsed_time = time.time() - start_time
1858 if status == 0:
1859     if options.passcmd is not None:
1860         do_print("Running passcmd: %s" % options.passcmd)
1861         run_cmd(options.passcmd, dir=test_master)
1862     if options.pushto is not None:
1863         push_to(options.pushto, push_branch=options.branch)
1864     if options.keeplogs or options.attach_logs:
1865         blist.tarlogs("logs.tar.gz")
1866         do_print("Logs in logs.tar.gz")
1867     if options.always_email:
1868         email_success(elapsed_time, log_base=options.log_base)
1869     blist.remove_logs()
1870     cleanup()
1871     do_print(errstr)
1872     sys.exit(0)
1873
1874 # something failed, gather a tar of the logs
1875 blist.tarlogs("logs.tar.gz")
1876
1877 if options.email is not None:
1878     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1879                   elapsed_time, log_base=options.log_base)
1880 else:
1881     elapsed_minutes = elapsed_time / 60.0
1882     print('''
1883
1884 ####################################################################
1885
1886 AUTOBUILD FAILURE
1887
1888 Your autobuild[%s] on %s failed after %.1f minutes
1889 when trying to test %s with the following error:
1890
1891    %s
1892
1893 the autobuild has been abandoned. Please fix the error and resubmit.
1894
1895 ####################################################################
1896
1897 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1898
1899 cleanup()
1900 do_print(errstr)
1901 do_print("Logs in logs.tar.gz")
1902 sys.exit(status)