script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, check_output, Popen, PIPE, CalledProcessError
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 import random
  13 from optparse import OptionParser
  14 import smtplib
  15 import email
  16 from email.mime.text import MIMEText
  17 from email.mime.base import MIMEBase
  18 from email.mime.application import MIMEApplication
  19 from email.mime.multipart import MIMEMultipart
  20 from distutils.sysconfig import get_python_lib
  21 import platform
  22
  23 try:
  24     from waflib.Build import CACHE_SUFFIX
  25 except ImportError:
  26     sys.path.insert(0, "./third_party/waf")
  27     from waflib.Build import CACHE_SUFFIX
  28
  29
  30 os.environ["PYTHONUNBUFFERED"] = "1"
  31
  32 # This speeds up testing remarkably.
  33 os.environ['TDB_NO_FSYNC'] = '1'
  34
  35
  36 def find_git_root():
  37     '''get to the top of the git repo'''
  38     p = os.getcwd()
  39     while p != '/':
  40         if os.path.exists(os.path.join(p, ".git")):
  41             return p
  42         p = os.path.abspath(os.path.join(p, '..'))
  43     return None
  44
  45
  46 gitroot = find_git_root()
  47 if gitroot is None:
  48     raise Exception("Failed to find git root")
  49
  50
  51 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  52
  53 parser = OptionParser()
  54 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  55 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  56 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  57 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  58                   default=def_testbase)
  59 parser.add_option("--passcmd", help="command to run on success", default=None)
  60 parser.add_option("--verbose", help="show all commands as they are run",
  61                   default=False, action="store_true")
  62 parser.add_option("--rebase", help="rebase on the given tree before testing",
  63                   default=None, type='str')
  64 parser.add_option("--pushto", help="push to a git url on success",
  65                   default=None, type='str')
  66 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  67                   default=False, action="store_true")
  68 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  69                   default=False, action="store_true")
  70 parser.add_option("--retry", help="automatically retry if master changes",
  71                   default=False, action="store_true")
  72 parser.add_option("--email", help="send email to the given address on failure",
  73                   type='str', default=None)
  74 parser.add_option("--email-from", help="send email from the given address",
  75                   type='str', default="autobuild@samba.org")
  76 parser.add_option("--email-server", help="send email via the given server",
  77                   type='str', default='localhost')
  78 parser.add_option("--always-email", help="always send email, even on success",
  79                   action="store_true")
  80 parser.add_option("--daemon", help="daemonize after initial setup",
  81                   action="store_true")
  82 parser.add_option("--branch", help="the branch to work on (default=master)",
  83                   default="master", type='str')
  84 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  85                   default=gitroot, type='str')
  86 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  87                   default=False, action="store_true")
  88 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
  89                   default='')
  90 parser.add_option("--enable-coverage", dest='enable_coverage',
  91                   action="store_const", const='--enable-coverage', default='',
  92                   help="Add --enable-coverage option while configure")
  93
  94 (options, args) = parser.parse_args()
  95
  96 if options.retry:
  97     if options.rebase is None:
  98         raise Exception('You can only use --retry if you also rebase')
  99
 100 testbase = "%s/b%u" % (options.testbase, os.getpid())
 101 test_master = "%s/master" % testbase
 102 test_prefix = "%s/prefix" % testbase
 103 test_tmpdir = "%s/tmp" % testbase
 104 os.environ['TMPDIR'] = test_tmpdir
 105
 106 if options.enable_coverage:
 107     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 108 else:
 109     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 110
 111 if args:
 112     # If we are only running specific test,
 113     # do not sleep randomly to wait for it to start
 114     def random_sleep(low, high):
 115         return 'sleep 1'
 116 else:
 117     def random_sleep(low, high):
 118         return 'sleep {}'.format(random.randint(low, high))
 119
 120 cleanup_list = []
 121
 122 builddirs = {
 123     "ctdb": "ctdb",
 124     "ldb": "lib/ldb",
 125     "tdb": "lib/tdb",
 126     "talloc": "lib/talloc",
 127     "replace": "lib/replace",
 128     "tevent": "lib/tevent",
 129     "pidl": "pidl",
 130     "docs-xml": "docs-xml"
 131 }
 132
 133 ctdb_configure_params = " --enable-developer ${PREFIX}"
 134 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 135
 136 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 137 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 138 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 139 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 140 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 141 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 142 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 143
 144
 145 def format_option(name, value=None):
 146     """Format option as str list."""
 147     if value is None:  # boolean option
 148         return [name]
 149     if not isinstance(value, list):  # single value option
 150         value = [value]
 151     # repeatable option
 152     return ['{}={}'.format(name, item) for item in value]
 153
 154
 155 def make_test(
 156         cmd='make test',
 157         TESTS='',
 158         include_envs=None,
 159         exclude_envs=None):
 160
 161     test_options = []
 162     if include_envs:
 163         test_options = format_option('--include-env', include_envs)
 164     if exclude_envs:
 165         test_options = format_option('--exclude-env', exclude_envs)
 166     if test_options:
 167         # join envs options to original test options
 168         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 169
 170     _options = []
 171
 172     # Allow getting a full CI with
 173     # git push -o ci.variable='AUTOBUILD_FAIL_IMMEDIATELY=0'
 174
 175     FAIL_IMMEDIATELY = os.getenv("AUTOBUILD_FAIL_IMMEDIATELY", "1")
 176
 177     if int(FAIL_IMMEDIATELY):
 178         _options.append('FAIL_IMMEDIATELY=1')
 179     if TESTS:
 180         _options.append("TESTS='{}'".format(TESTS))
 181
 182     return ' '.join([cmd] + _options)
 183
 184
 185 # When updating this list, also update .gitlab-ci.yml to add the job
 186 # and to make it a dependency of 'page' for the coverage report.
 187
 188 tasks = {
 189     "ctdb": [
 190         ("random-sleep", random_sleep(300, 900)),
 191         ("configure", "./configure " + ctdb_configure_params),
 192         ("make", "make all"),
 193         ("install", "make install"),
 194         ("test", "make autotest"),
 195         ("check-clean-tree", "../script/clean-source-tree.sh"),
 196         ("clean", "make clean"),
 197         ],
 198
 199     "docs-xml": [
 200         ("random-sleep", random_sleep(300, 900)),
 201         ("autoconf", "autoconf"),
 202         ("configure", "./configure"),
 203         ("make", "make html htmlman"),
 204         ("clean", "make clean"),
 205         ],
 206
 207     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 208     "samba": [
 209         ("random-sleep", random_sleep(300, 900)),
 210         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 211         ("make", "make -j"),
 212         ("test", make_test(exclude_envs=[
 213             "none",
 214             "nt4_dc",
 215             "nt4_dc_smb1",
 216             "nt4_dc_smb1_done",
 217             "nt4_dc_schannel",
 218             "nt4_member",
 219             "ad_dc",
 220             "ad_dc_smb1",
 221             "ad_dc_smb1_done",
 222             "ad_dc_backup",
 223             "ad_dc_ntvfs",
 224             "ad_dc_default",
 225             "ad_dc_default_smb1",
 226             "ad_dc_slowtests",
 227             "ad_dc_no_nss",
 228             "ad_dc_no_ntlm",
 229             "fl2003dc",
 230             "fl2008dc",
 231             "fl2008r2dc",
 232             "ad_member",
 233             "ad_member_idmap_rid",
 234             "ad_member_idmap_ad",
 235             "ad_member_rfc2307",
 236             "chgdcpass",
 237             "vampire_2000_dc",
 238             "fl2000dc",
 239             "fileserver",
 240             "fileserver_smb1",
 241             "fileserver_smb1_done",
 242             "maptoguest",
 243             "simpleserver",
 244             "backupfromdc",
 245             "restoredc",
 246             "renamedc",
 247             "offlinebackupdc",
 248             "labdc",
 249             "preforkrestartdc",
 250             "proclimitdc",
 251             "promoted_dc",
 252             "vampire_dc",
 253             "rodc",
 254             "ad_dc_default",
 255             "ad_dc_default_smb1",
 256             "ad_dc_default_smb1_done",
 257             "ad_dc_slowtests",
 258             "schema_pair_dc",
 259             "schema_dc",
 260             "clusteredmember_smb1",
 261             ])),
 262         ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
 263         ("lcov", LCOV_CMD),
 264         ("install", "make install"),
 265         ("check-clean-tree", "script/clean-source-tree.sh"),
 266         ("clean", "make clean"),
 267         ],
 268
 269     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 270     "samba-mitkrb5": [
 271         ("random-sleep", random_sleep(300, 900)),
 272         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 273         ("make", "make -j"),
 274         ("test", make_test(exclude_envs=[
 275             "none",
 276             "nt4_dc",
 277             "nt4_dc_smb1",
 278             "nt4_dc_smb1_done",
 279             "nt4_dc_schannel",
 280             "nt4_member",
 281             "ad_dc",
 282             "ad_dc_smb1",
 283             "ad_dc_smb1_done",
 284             "ad_dc_backup",
 285             "ad_dc_ntvfs",
 286             "ad_dc_default",
 287             "ad_dc_default_smb1",
 288             "ad_dc_default_smb1_done",
 289             "ad_dc_slowtests",
 290             "ad_dc_no_nss",
 291             "ad_dc_no_ntlm",
 292             "fl2003dc",
 293             "fl2008dc",
 294             "fl2008r2dc",
 295             "ad_member",
 296             "ad_member_idmap_rid",
 297             "ad_member_idmap_ad",
 298             "ad_member_rfc2307",
 299             "chgdcpass",
 300             "vampire_2000_dc",
 301             "fl2000dc",
 302             "fileserver",
 303             "fileserver_smb1",
 304             "fileserver_smb1_done",
 305             "maptoguest",
 306             "simpleserver",
 307             "backupfromdc",
 308             "restoredc",
 309             "renamedc",
 310             "offlinebackupdc",
 311             "labdc",
 312             "preforkrestartdc",
 313             "proclimitdc",
 314             "promoted_dc",
 315             "vampire_dc",
 316             "rodc",
 317             "ad_dc_default",
 318             "ad_dc_default_smb1",
 319             "ad_dc_default_smb1_done",
 320             "ad_dc_slowtests",
 321             "schema_pair_dc",
 322             "schema_dc",
 323             "clusteredmember_smb1",
 324             ])),
 325         ("lcov", LCOV_CMD),
 326         ("install", "make install"),
 327         ("check-clean-tree", "script/clean-source-tree.sh"),
 328         ("clean", "make clean"),
 329         ],
 330
 331     "samba-nt4": [
 332         ("random-sleep", random_sleep(300, 900)),
 333         ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params),
 334         ("make", "make -j"),
 335         ("test", make_test(include_envs=[
 336             "nt4_dc",
 337             "nt4_dc_smb1",
 338             "nt4_dc_smb1_done",
 339             "nt4_dc_schannel",
 340             "nt4_member",
 341             "simpleserver",
 342             ])),
 343         ("lcov", LCOV_CMD),
 344         ("install", "make install"),
 345         ("check-clean-tree", "script/clean-source-tree.sh"),
 346         ("clean", "make clean"),
 347         ],
 348
 349     "samba-fileserver": [
 350         ("random-sleep", random_sleep(300, 900)),
 351         ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5 --with-selftest-prefix=./bin/ab" + samba_configure_params),
 352         ("make", "make -j"),
 353         ("test", make_test(include_envs=[
 354             "fileserver",
 355             "fileserver_smb1",
 356             "fileserver_smb1_done",
 357             "maptoguest",
 358             "ktest", # ktest is also tested in samba and samba-mitkrb5
 359                      # but is tested here against a system Heimdal
 360             ])),
 361         ("lcov", LCOV_CMD),
 362         ("check-clean-tree", "script/clean-source-tree.sh"),
 363         ],
 364
 365     "samba-admem": [
 366         ("random-sleep", random_sleep(300, 900)),
 367         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 368         ("make", "make -j"),
 369         ("test", make_test(include_envs=[
 370             "ad_member",
 371             "ad_member_idmap_rid",
 372             "ad_member_idmap_ad",
 373             "ad_member_rfc2307",
 374             ])),
 375         ("lcov", LCOV_CMD),
 376         ("check-clean-tree", "script/clean-source-tree.sh"),
 377         ],
 378
 379     "samba-ad-dc-1": [
 380         ("random-sleep", random_sleep(1, 1)),
 381         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 382         ("make", "make -j"),
 383         ("test", make_test(include_envs=[
 384             "ad_dc",
 385             "ad_dc_smb1",
 386             "ad_dc_smb1_done",
 387             "ad_dc_no_nss",
 388             "ad_dc_no_ntlm",
 389             ])),
 390         ("lcov", LCOV_CMD),
 391         ("check-clean-tree", "script/clean-source-tree.sh"),
 392         ],
 393
 394     "samba-ad-dc-2": [
 395         ("random-sleep", random_sleep(1, 1)),
 396         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 397         ("make", "make -j"),
 398         ("test", make_test(include_envs=[
 399             "vampire_dc",
 400             "vampire_2000_dc",
 401             "rodc",
 402             ])),
 403         ("lcov", LCOV_CMD),
 404         ("check-clean-tree", "script/clean-source-tree.sh"),
 405         ],
 406
 407     "samba-ad-dc-3": [
 408         ("random-sleep", random_sleep(1, 1)),
 409         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 410         ("make", "make -j"),
 411         ("test", make_test(include_envs=[
 412             "promoted_dc",
 413             "chgdcpass",
 414             "preforkrestartdc",
 415             "proclimitdc",
 416             ])),
 417         ("lcov", LCOV_CMD),
 418         ("check-clean-tree", "script/clean-source-tree.sh"),
 419         ],
 420
 421     "samba-ad-dc-4": [
 422         ("random-sleep", random_sleep(1, 1)),
 423         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 424         ("make", "make -j"),
 425         ("test", make_test(include_envs=[
 426             "fl2000dc",
 427             "fl2003dc",
 428             "fl2008dc",
 429             "fl2008r2dc",
 430             ])),
 431         ("lcov", LCOV_CMD),
 432         ("check-clean-tree", "script/clean-source-tree.sh"),
 433         ],
 434
 435     "samba-ad-dc-5": [
 436         ("random-sleep", random_sleep(1, 1)),
 437         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 438         ("make", "make -j"),
 439         ("test", make_test(include_envs=[
 440             "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
 441         ("lcov", LCOV_CMD),
 442         ("check-clean-tree", "script/clean-source-tree.sh"),
 443         ],
 444
 445     "samba-ad-dc-6": [
 446         ("random-sleep", random_sleep(1, 1)),
 447         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 448         ("make", "make -j"),
 449         ("test", make_test(include_envs=["ad_dc_slowtests"])),
 450         ("lcov", LCOV_CMD),
 451         ("check-clean-tree", "script/clean-source-tree.sh"),
 452         ],
 453
 454     "samba-schemaupgrade": [
 455         ("random-sleep", random_sleep(1, 1)),
 456         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 457         ("make", "make -j"),
 458         ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 459         ("lcov", LCOV_CMD),
 460         ("check-clean-tree", "script/clean-source-tree.sh"),
 461         ],
 462
 463     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 464     # This is currently the longest task, so we don't randomly delay it.
 465     "samba-ad-dc-ntvfs": [
 466         ("random-sleep", random_sleep(1, 1)),
 467         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 468         ("make", "make -j"),
 469         ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 470         ("lcov", LCOV_CMD),
 471         ("check-clean-tree", "script/clean-source-tree.sh"),
 472         ],
 473
 474     # Test fips compliance
 475     "samba-fips": [
 476         ("random-sleep", random_sleep(100, 500)),
 477         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 478         ("make", "make -j"),
 479         ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
 480         ("lcov", LCOV_CMD),
 481         ("check-clean-tree", "script/clean-source-tree.sh"),
 482         ],
 483
 484     # run the backup/restore testenvs separately as they're fairly standalone
 485     # (and CI seems to max out at ~8 different DCs running at once)
 486     "samba-ad-dc-backup": [
 487         ("random-sleep", random_sleep(300, 900)),
 488         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 489         ("make", "make -j"),
 490         ("test", make_test(include_envs=[
 491             "backupfromdc",
 492             "restoredc",
 493             "renamedc",
 494             "offlinebackupdc",
 495             "labdc",
 496             "ad_dc_backup",
 497             ])),
 498         ("lcov", LCOV_CMD),
 499         ("check-clean-tree", "script/clean-source-tree.sh"),
 500         ],
 501
 502     "samba-admem-mit": [
 503         ("random-sleep", random_sleep(1, 1)),
 504         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 505         ("make", "make -j"),
 506         ("test", make_test(include_envs=[
 507             "ad_member",
 508             "ad_member_idmap_rid",
 509             "ad_member_idmap_ad",
 510             "ad_member_rfc2307",
 511             ])),
 512         ("lcov", LCOV_CMD),
 513         ("check-clean-tree", "script/clean-source-tree.sh"),
 514         ],
 515
 516     "samba-ad-dc-1-mitkrb5": [
 517         ("random-sleep", random_sleep(1, 1)),
 518         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 519         ("make", "make -j"),
 520         ("test", make_test(include_envs=[
 521             "ad_dc",
 522             "ad_dc_smb1",
 523             "ad_dc_smb1_done",
 524             "ad_dc_no_nss",
 525             "ad_dc_no_ntlm",
 526             ])),
 527         ("lcov", LCOV_CMD),
 528         ("check-clean-tree", "script/clean-source-tree.sh"),
 529         ],
 530
 531     "samba-ad-dc-4-mitkrb5": [
 532         ("random-sleep", random_sleep(1, 1)),
 533         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 534         ("make", "make -j"),
 535         ("test", make_test(include_envs=[
 536             "fl2000dc",
 537             "fl2003dc",
 538             "fl2008dc",
 539             "fl2008r2dc",
 540             ])),
 541         ("lcov", LCOV_CMD),
 542         ("check-clean-tree", "script/clean-source-tree.sh"),
 543         ],
 544
 545     "samba-test-only": [
 546         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params),
 547         ("make", "make -j"),
 548         ("test", make_test(TESTS="${TESTS}")),
 549         ("lcov", LCOV_CMD),
 550         ],
 551
 552     # Test cross-compile infrastructure
 553     "samba-xc": [
 554         ("random-sleep", random_sleep(900, 1500)),
 555         ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 556         ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 557             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 558         ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
 559         ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 560             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 561         ("compare-results", "script/compare_cc_results.py "
 562             "./bin/c4che/default{} "
 563             "./bin-xe/c4che/default{} "
 564             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 565         ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
 566         ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
 567             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
 568         ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
 569             " = \"'1234'\"".format(CACHE_SUFFIX)),
 570         ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
 571         ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
 572             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
 573             " ; test $? -ne 0"),
 574         ],
 575
 576     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 577     "samba-o3": [
 578         ("random-sleep", random_sleep(300, 900)),
 579         ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params),
 580         ("make", "make -j"),
 581         ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 582         ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 583         ("lcov", LCOV_CMD),
 584         ("install", "make install"),
 585         ("check-clean-tree", "script/clean-source-tree.sh"),
 586         ("clean", "make clean"),
 587         ],
 588
 589     "samba-ctdb": [
 590         ("random-sleep", random_sleep(900, 1500)),
 591
 592         # make sure we have tdb around:
 593         ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 594         ("tdb-make", "cd lib/tdb && make"),
 595         ("tdb-install", "cd lib/tdb && make install"),
 596
 597         # build samba with cluster support (also building ctdb):
 598         ("samba-configure",
 599          "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
 600          "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
 601          "./configure.developer ${PREFIX} "
 602          "--with-selftest-prefix=./bin/ab "
 603          "--with-cluster-support "
 604          "--without-ad-dc "
 605          "--bundled-libraries=!tdb"),
 606         ("samba-make", "make"),
 607         ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT"),
 608         ("samba-install", "make install"),
 609         ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 610
 611         ("test",
 612          make_test(cmd='make test',
 613                    include_envs=["clusteredmember_smb1"])
 614         ),
 615
 616         # clean up:
 617         ("check-clean-tree", "script/clean-source-tree.sh"),
 618         ("clean", "make clean"),
 619         ("ctdb-clean", "cd ./ctdb && make clean"),
 620         ],
 621
 622     "samba-libs": [
 623         ("random-sleep", random_sleep(300, 900)),
 624         ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 625         ("talloc-make", "cd lib/talloc && make"),
 626         ("talloc-install", "cd lib/talloc && make install"),
 627
 628         ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 629         ("tdb-make", "cd lib/tdb && make"),
 630         ("tdb-install", "cd lib/tdb && make install"),
 631
 632         ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 633         ("tevent-make", "cd lib/tevent && make"),
 634         ("tevent-install", "cd lib/tevent && make install"),
 635
 636         ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
 637         ("ldb-make", "cd lib/ldb && make"),
 638         ("ldb-install", "cd lib/ldb && make install"),
 639
 640         ("nondevel-configure", "./configure ${PREFIX}"),
 641         ("nondevel-make", "make -j"),
 642         ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 643         ("nondevel-install", "make install"),
 644         ("nondevel-dist", "make dist"),
 645
 646         # retry with all modules shared
 647         ("allshared-distclean", "make distclean"),
 648         ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 649         ("allshared-make", "make -j"),
 650         ],
 651
 652     "samba-static": [
 653         ("random-sleep", random_sleep(1, 1)),
 654         # build with all modules static
 655         ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
 656         ("allstatic-make", "make -j"),
 657         ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 658         ("lcov", LCOV_CMD),
 659
 660         # retry without any required modules
 661         ("none-distclean", "make distclean"),
 662         ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
 663         ("none-make", "make -j"),
 664
 665         # retry with nonshared smbd and smbtorture
 666         ("nonshared-distclean", "make distclean"),
 667         ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
 668         ("nonshared-make", "make -j")
 669         ],
 670
 671     "samba-fuzz": [
 672         # build the fuzzers (static) via the oss-fuzz script
 673         ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
 674         ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl"),
 675         ("fuzzers-check", "./lib/fuzzing/oss-fuzz/check_build.sh ${PREFIX_DIR}")
 676         ],
 677
 678     # Test Samba without python still builds.  When this test fails
 679     # due to more use of Python, the expectations is that the newly
 680     # failing part of the code should be disabled when
 681     # --disable-python is set (rather than major work being done to
 682     # support this environment).  The target here is for vendors
 683     # shipping a minimal smbd.
 684     "samba-nopython": [
 685         ("random-sleep", random_sleep(300, 900)),
 686         ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
 687         ("make", "make -j"),
 688         ("install", "make install"),
 689         ("find-python", "script/find_python.sh ${PREFIX}"),
 690         ("test", "make test-nopython"),
 691         ("lcov", LCOV_CMD),
 692         ("check-clean-tree", "script/clean-source-tree.sh"),
 693         ("clean", "make clean"),
 694
 695         ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 696         ("talloc-make", "cd lib/talloc && make"),
 697         ("talloc-install", "cd lib/talloc && make install"),
 698
 699         ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 700         ("tdb-make", "cd lib/tdb && make"),
 701         ("tdb-install", "cd lib/tdb && make install"),
 702
 703         ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 704         ("tevent-make", "cd lib/tevent && make"),
 705         ("tevent-install", "cd lib/tevent && make install"),
 706
 707         ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 708         ("ldb-make", "cd lib/ldb && make"),
 709         ("ldb-install", "cd lib/ldb && make install"),
 710
 711         # retry against installed library packages
 712         ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc"),
 713         ("libs-make", "make -j"),
 714         ("libs-install", "make install"),
 715         ("libs-check-clean-tree", "script/clean-source-tree.sh"),
 716         ("libs-clean", "make clean"),
 717         ],
 718
 719     # check we can do the same thing using python2
 720     "samba-nopython-py2": [
 721         ("random-sleep", random_sleep(300, 900)),
 722         ("configure", "PYTHON=python2 ./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
 723         ("make", "PYTHON=python2 make -j"),
 724         ("install", "PYTHON=python2 make install"),
 725         ("find-python", "script/find_python.sh ${PREFIX}"),
 726         ("test", "make test-nopython"),
 727         ("lcov", LCOV_CMD),
 728         ("check-clean-tree", "script/clean-source-tree.sh"),
 729         ("clean", "PYTHON=python2 make clean"),
 730
 731         ("talloc-configure", "cd lib/talloc && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 732         ("talloc-make", "cd lib/talloc && PYTHON=python2 make"),
 733         ("talloc-install", "cd lib/talloc && PYTHON=python2 make install"),
 734
 735         ("tdb-configure", "cd lib/tdb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 736         ("tdb-make", "cd lib/tdb && PYTHON=python2 make"),
 737         ("tdb-install", "cd lib/tdb && PYTHON=python2 make install"),
 738
 739         ("tevent-configure", "cd lib/tevent && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 740         ("tevent-make", "cd lib/tevent && PYTHON=python2 make"),
 741         ("tevent-install", "cd lib/tevent && PYTHON=python2 make install"),
 742
 743         ("ldb-configure", "cd lib/ldb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 744         ("ldb-make", "cd lib/ldb && PYTHON=python2 make"),
 745         ("ldb-install", "cd lib/ldb && PYTHON=python2 make install"),
 746
 747         # retry against installed library packages
 748         ("libs-configure", "PYTHON=python2 " + samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc"),
 749         ("libs-make", "PYTHON=python2 make -j"),
 750         ("libs-install", "PYTHON=python2 make install"),
 751         ("libs-check-clean-tree", "script/clean-source-tree.sh"),
 752         ("libs-clean", "PYTHON=python2 make clean"),
 753         ],
 754
 755     "ldb": [
 756         ("random-sleep", random_sleep(60, 600)),
 757         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 758         ("make", "make"),
 759         ("install", "make install"),
 760         ("test", "make test"),
 761         ("lcov", LCOV_CMD),
 762         ("clean", "make clean"),
 763         ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
 764         ("make-no-lmdb", "make"),
 765         ("test-no-lmdb", "make test"),
 766         ("lcov-no-lmdb", LCOV_CMD),
 767         ("install-no-lmdb", "make install"),
 768         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 769         ("distcheck", "make distcheck"),
 770         ("clean", "make clean"),
 771         ],
 772
 773     "tdb": [
 774         ("random-sleep", random_sleep(60, 600)),
 775         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 776         ("make", "make"),
 777         ("install", "make install"),
 778         ("test", "make test"),
 779         ("lcov", LCOV_CMD),
 780         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 781         ("distcheck", "make distcheck"),
 782         ("clean", "make clean"),
 783         ],
 784
 785     "talloc": [
 786         ("random-sleep", random_sleep(60, 600)),
 787         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 788         ("make", "make"),
 789         ("install", "make install"),
 790         ("test", "make test"),
 791         ("lcov", LCOV_CMD),
 792         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 793         ("distcheck", "make distcheck"),
 794         ("clean", "make clean"),
 795         ],
 796
 797     "replace": [
 798         ("random-sleep", random_sleep(60, 600)),
 799         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 800         ("make", "make"),
 801         ("install", "make install"),
 802         ("test", "make test"),
 803         ("lcov", LCOV_CMD),
 804         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 805         ("distcheck", "make distcheck"),
 806         ("clean", "make clean"),
 807         ],
 808
 809     "tevent": [
 810         ("random-sleep", random_sleep(60, 600)),
 811         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 812         ("make", "make"),
 813         ("install", "make install"),
 814         ("test", "make test"),
 815         ("lcov", LCOV_CMD),
 816         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 817         ("distcheck", "make distcheck"),
 818         ("clean", "make clean"),
 819         ],
 820
 821     "pidl": [
 822         ("random-sleep", random_sleep(60, 600)),
 823         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
 824         ("touch", "touch *.yp"),
 825         ("make", "make"),
 826         ("test", "make test"),
 827         ("install", "make install"),
 828         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
 829         ("check-clean-tree", "../script/clean-source-tree.sh"),
 830         ("clean", "make clean"),
 831         ],
 832
 833     # these are useful for debugging autobuild
 834     'pass': [("pass", 'echo passing && /bin/true')],
 835     'fail': [("fail", 'echo failing && /bin/false')],
 836 }
 837
 838 defaulttasks = list(tasks.keys())
 839
 840 defaulttasks.remove("pass")
 841 defaulttasks.remove("fail")
 842 defaulttasks.remove("samba-test-only")
 843 defaulttasks.remove("samba-fuzz")
 844 defaulttasks.remove("samba-fips")
 845 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
 846     defaulttasks.remove("samba-o3")
 847
 848
 849 def do_print(msg):
 850     print("%s" % msg)
 851     sys.stdout.flush()
 852     sys.stderr.flush()
 853
 854
 855 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 856     if show is None:
 857         show = options.verbose
 858     if show:
 859         do_print("Running: '%s' in '%s'" % (cmd, dir))
 860     if output:
 861         out = check_output([cmd], shell=True, cwd=dir)
 862         return out.decode(encoding='utf-8', errors='backslashreplace')
 863     elif checkfail:
 864         return check_call(cmd, shell=True, cwd=dir)
 865     else:
 866         return call(cmd, shell=True, cwd=dir)
 867
 868 def rmdir_force(dirname, re_raise=True):
 869     try:
 870         run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % (
 871                 dirname, dirname, dirname), output=True, show=True)
 872     except CalledProcessError as e:
 873         do_print("Failed: '%s'" % (str(e)))
 874         run_cmd("tree %s" % dirname, output=True, show=True)
 875         if re_raise:
 876             raise
 877         return False
 878     return True
 879
 880 class builder(object):
 881     '''handle build of one directory'''
 882
 883     def __init__(self, name, sequence, cp=True):
 884         self.name = name
 885         self.dir = builddirs.get(name, '.')
 886         self.tag = self.name.replace('/', '_')
 887         self.sequence = sequence
 888         self.next = 0
 889         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 890         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 891         if options.verbose:
 892             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 893             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 894         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 895         self.stdout = open(self.stdout_path, 'w')
 896         self.stderr = open(self.stderr_path, 'w')
 897         self.stdin  = open("/dev/null", 'r')
 898         self.test_source_dir = "%s/%s" % (testbase, self.tag)
 899         self.cwd = "%s/%s" % (self.test_source_dir, self.dir)
 900         self.prefix = "%s/%s" % (test_prefix, self.tag)
 901         rmdir_force(self.test_source_dir)
 902         rmdir_force(self.prefix)
 903         if cp:
 904             run_cmd("cp -R -a -l %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
 905         else:
 906             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
 907         self.start_next()
 908
 909     def start_next(self):
 910         if self.next == len(self.sequence):
 911             if not options.nocleanup:
 912                 rmdir_force(self.test_source_dir)
 913                 rmdir_force(self.prefix)
 914             do_print('%s: Completed OK' % self.name)
 915             self.done = True
 916             return
 917         (self.stage, self.cmd) = self.sequence[self.next]
 918         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
 919         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 920         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 921         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 922         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
 923         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
 924         self.cmd = self.cmd.replace("${NAME}", self.name)
 925         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
 926         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
 927         self.proc = Popen(self.cmd, shell=True,
 928                           close_fds=True, cwd=self.cwd,
 929                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 930         self.next += 1
 931
 932
 933 class buildlist(object):
 934     '''handle build of multiple directories'''
 935
 936     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 937         self.tail_proc = None
 938         self.retry = None
 939         if not tasknames:
 940             if options.restrict_tests:
 941                 tasknames = ["samba-test-only"]
 942             else:
 943                 tasknames = defaulttasks
 944
 945         self.tlist = [builder(n, tasks[n], cp=(n != "pidl")) for n in tasknames]
 946
 947         if options.retry:
 948             rebase_remote = "rebaseon"
 949             retry_task = [("retry",
 950                             '''set -e
 951                             git remote add -t %s %s %s
 952                             git fetch %s
 953                             while :; do
 954                               sleep 60
 955                               git describe %s/%s > old_remote_branch.desc
 956                               git fetch %s
 957                               git describe %s/%s > remote_branch.desc
 958                               diff old_remote_branch.desc remote_branch.desc
 959                             done
 960                            ''' % (
 961                                rebase_branch, rebase_remote, rebase_url,
 962                                rebase_remote,
 963                                rebase_remote, rebase_branch,
 964                                rebase_remote,
 965                                rebase_remote, rebase_branch
 966                             ))]
 967
 968             self.retry = builder('retry', retry_task, cp=False)
 969             self.need_retry = False
 970
 971     def kill_kids(self):
 972         if self.tail_proc is not None:
 973             self.tail_proc.terminate()
 974             self.tail_proc.wait()
 975             self.tail_proc = None
 976         if self.retry is not None:
 977             self.retry.proc.terminate()
 978             self.retry.proc.wait()
 979             self.retry = None
 980         for b in self.tlist:
 981             if b.proc is not None:
 982                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
 983                 b.proc.terminate()
 984                 b.proc.wait()
 985                 b.proc = None
 986
 987     def wait_one(self):
 988         while True:
 989             none_running = True
 990             for b in self.tlist:
 991                 if b.proc is None:
 992                     continue
 993                 none_running = False
 994                 b.status = b.proc.poll()
 995                 if b.status is None:
 996                     continue
 997                 b.proc = None
 998                 return b
 999             if options.retry:
1000                 ret = self.retry.proc.poll()
1001                 if ret is not None:
1002                     self.need_retry = True
1003                     self.retry = None
1004                     return None
1005             if none_running:
1006                 return None
1007             time.sleep(0.1)
1008
1009     def run(self):
1010         while True:
1011             b = self.wait_one()
1012             if options.retry and self.need_retry:
1013                 self.kill_kids()
1014                 do_print("retry needed")
1015                 return (0, None, None, None, "retry")
1016             if b is None:
1017                 break
1018             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
1019                 self.kill_kids()
1020                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
1021             b.start_next()
1022         self.kill_kids()
1023         return (0, None, None, None, "All OK")
1024
1025     def write_system_info(self, filename):
1026         with open(filename, 'w') as f:
1027             for cmd in ['uname -a',
1028                         'lsb_release -a',
1029                         'free',
1030                         'mount',
1031                         'cat /proc/cpuinfo',
1032                         'cc --version',
1033                         'df -m .',
1034                         'df -m %s' % testbase]:
1035                 try:
1036                     out = run_cmd(cmd, output=True, checkfail=False)
1037                 except CalledProcessError as e:
1038                     out = "<failed: %s>" % str(e)
1039                 print('### %s' % cmd, file=f)
1040                 print(out, file=f)
1041                 print(file=f)
1042
1043     def tarlogs(self, fname):
1044         with tarfile.open(fname, "w:gz") as tar:
1045             for b in self.tlist:
1046                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
1047                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
1048             if os.path.exists("autobuild.log"):
1049                 tar.add("autobuild.log")
1050             filename = 'system-info.txt'
1051             self.write_system_info(filename)
1052             tar.add(filename)
1053
1054     def remove_logs(self):
1055         for b in self.tlist:
1056             os.unlink(b.stdout_path)
1057             os.unlink(b.stderr_path)
1058
1059     def start_tail(self):
1060         cmd = ["tail", "-f"]
1061         for b in self.tlist:
1062             cmd.append(b.stdout_path)
1063             cmd.append(b.stderr_path)
1064         self.tail_proc = Popen(cmd, close_fds=True)
1065
1066
1067 def cleanup(do_raise=False):
1068     if options.nocleanup:
1069         return
1070     run_cmd("stat %s || true" % test_tmpdir, show=True)
1071     run_cmd("stat %s" % testbase, show=True)
1072     do_print("Cleaning up %r" % cleanup_list)
1073     for d in cleanup_list:
1074         ok = rmdir_force(d, re_raise=False)
1075         if ok:
1076             continue
1077         if os.path.isdir(d):
1078             do_print("Killing, waiting and retry")
1079             run_cmd("killbysubdir %s > /dev/null 2>&1" % d, checkfail=False)
1080         else:
1081             do_print("Waiting and retry")
1082         time.sleep(1)
1083         rmdir_force(d, re_raise=do_raise)
1084
1085
1086 def daemonize(logfile):
1087     pid = os.fork()
1088     if pid == 0:  # Parent
1089         os.setsid()
1090         pid = os.fork()
1091         if pid != 0:  # Actual daemon
1092             os._exit(0)
1093     else:  # Grandparent
1094         os._exit(0)
1095
1096     import resource      # Resource usage information.
1097     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1098     if maxfd == resource.RLIM_INFINITY:
1099         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1100     for fd in range(0, maxfd):
1101         try:
1102             os.close(fd)
1103         except OSError:
1104             pass
1105     os.open(logfile, os.O_RDWR | os.O_CREAT)
1106     os.dup2(0, 1)
1107     os.dup2(0, 2)
1108
1109
1110 def write_pidfile(fname):
1111     '''write a pid file, cleanup on exit'''
1112     with open(fname, mode='w') as f:
1113         f.write("%u\n" % os.getpid())
1114
1115
1116 def rebase_tree(rebase_url, rebase_branch="master"):
1117     rebase_remote = "rebaseon"
1118     do_print("Rebasing on %s" % rebase_url)
1119     run_cmd("git describe HEAD", show=True, dir=test_master)
1120     run_cmd("git remote add -t %s %s %s" %
1121             (rebase_branch, rebase_remote, rebase_url),
1122             show=True, dir=test_master)
1123     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1124     if options.fix_whitespace:
1125         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1126                 (rebase_remote, rebase_branch),
1127                 show=True, dir=test_master)
1128     else:
1129         run_cmd("git rebase --force-rebase %s/%s" %
1130                 (rebase_remote, rebase_branch),
1131                 show=True, dir=test_master)
1132     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1133                    (rebase_remote, rebase_branch),
1134                    dir=test_master, output=True)
1135     if diff == '':
1136         do_print("No differences between HEAD and %s/%s - exiting" %
1137                  (rebase_remote, rebase_branch))
1138         sys.exit(0)
1139     run_cmd("git describe %s/%s" %
1140             (rebase_remote, rebase_branch),
1141             show=True, dir=test_master)
1142     run_cmd("git describe HEAD", show=True, dir=test_master)
1143     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1144             (rebase_remote, rebase_branch),
1145             show=True, dir=test_master)
1146
1147
1148 def push_to(push_url, push_branch="master"):
1149     push_remote = "pushto"
1150     do_print("Pushing to %s" % push_url)
1151     if options.mark:
1152         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1153         run_cmd("git commit --amend -c HEAD", dir=test_master)
1154         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1155         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1156     run_cmd("git remote add -t %s %s %s" %
1157             (push_branch, push_remote, push_url),
1158             show=True, dir=test_master)
1159     run_cmd("git push %s +HEAD:%s" %
1160             (push_remote, push_branch),
1161             show=True, dir=test_master)
1162
1163
1164 def send_email(subject, text, log_tar):
1165     if options.email is None:
1166         do_print("not sending email because the recipient is not set")
1167         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1168                  (subject, text))
1169         return
1170     outer = MIMEMultipart()
1171     outer['Subject'] = subject
1172     outer['To'] = options.email
1173     outer['From'] = options.email_from
1174     outer['Date'] = email.utils.formatdate(localtime=True)
1175     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1176     outer.attach(MIMEText(text, 'plain'))
1177     if options.attach_logs:
1178         with open(log_tar, 'rb') as fp:
1179             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1180         # Set the filename parameter
1181         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1182         outer.attach(msg)
1183     content = outer.as_string()
1184     s = smtplib.SMTP(options.email_server)
1185     email_user = os.getenv('SMTP_USERNAME')
1186     email_password = os.getenv('SMTP_PASSWORD')
1187     if email_user is not None:
1188         s.starttls()
1189         s.login(email_user, email_password)
1190
1191     s.sendmail(options.email_from, [options.email], content)
1192     s.set_debuglevel(1)
1193     s.quit()
1194
1195
1196 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1197                   elapsed_time, log_base=None, add_log_tail=True):
1198     '''send an email to options.email about the failure'''
1199     elapsed_minutes = elapsed_time / 60.0
1200     if log_base is None:
1201         log_base = gitroot
1202     text = '''
1203 Dear Developer,
1204
1205 Your autobuild on %s failed after %.1f minutes
1206 when trying to test %s with the following error:
1207
1208    %s
1209
1210 the autobuild has been abandoned. Please fix the error and resubmit.
1211
1212 A summary of the autobuild process is here:
1213
1214   %s/autobuild.log
1215 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1216
1217     if options.restrict_tests:
1218         text += """
1219 The build was restricted to tests matching %s\n""" % options.restrict_tests
1220
1221     if failed_task != 'rebase':
1222         text += '''
1223 You can see logs of the failed task here:
1224
1225   %s/%s.stdout
1226   %s/%s.stderr
1227
1228 or you can get full logs of all tasks in this job here:
1229
1230   %s/logs.tar.gz
1231
1232 The top commit for the tree that was built was:
1233
1234 %s
1235
1236 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1237
1238     if add_log_tail:
1239         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1240         lines = f.readlines()
1241         log_tail = "".join(lines[-50:])
1242         num_lines = len(lines)
1243         if num_lines < 50:
1244             # Also include stderr (compile failures) if < 50 lines of stdout
1245             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1246             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1247
1248         text += '''
1249 The last 50 lines of log messages:
1250
1251 %s
1252     ''' % log_tail
1253         f.close()
1254
1255     logs = os.path.join(gitroot, 'logs.tar.gz')
1256     send_email('autobuild[%s] failure on %s for task %s during %s'
1257                % (options.branch, platform.node(), failed_task, failed_stage),
1258                text, logs)
1259
1260
1261 def email_success(elapsed_time, log_base=None):
1262     '''send an email to options.email about a successful build'''
1263     if log_base is None:
1264         log_base = gitroot
1265     text = '''
1266 Dear Developer,
1267
1268 Your autobuild on %s has succeeded after %.1f minutes.
1269
1270 ''' % (platform.node(), elapsed_time / 60.)
1271
1272     if options.restrict_tests:
1273         text += """
1274 The build was restricted to tests matching %s\n""" % options.restrict_tests
1275
1276     if options.keeplogs:
1277         text += '''
1278
1279 you can get full logs of all tasks in this job here:
1280
1281   %s/logs.tar.gz
1282
1283 ''' % log_base
1284
1285     text += '''
1286 The top commit for the tree that was built was:
1287
1288 %s
1289 ''' % top_commit_msg
1290
1291     logs = os.path.join(gitroot, 'logs.tar.gz')
1292     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1293                text, logs)
1294
1295
1296 # get the top commit message, for emails
1297 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1298
1299 try:
1300     os.makedirs(testbase)
1301 except Exception as reason:
1302     raise Exception("Unable to create %s : %s" % (testbase, reason))
1303 cleanup_list.append(testbase)
1304
1305 if options.daemon:
1306     logfile = os.path.join(testbase, "log")
1307     do_print("Forking into the background, writing progress to %s" % logfile)
1308     daemonize(logfile)
1309
1310 write_pidfile(gitroot + "/autobuild.pid")
1311
1312 start_time = time.time()
1313
1314 while True:
1315     try:
1316         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1317         os.makedirs(test_tmpdir)
1318         # The waf uninstall code removes empty directories all the way
1319         # up the tree.  Creating a file in test_tmpdir stops it from
1320         # being removed.
1321         run_cmd("touch %s" % os.path.join(test_tmpdir,
1322                                           ".directory-is-not-empty"), show=True)
1323         run_cmd("stat %s" % test_tmpdir, show=True)
1324         run_cmd("stat %s" % testbase, show=True)
1325         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1326     except Exception:
1327         cleanup()
1328         raise
1329
1330     try:
1331         if options.rebase is not None:
1332             rebase_tree(options.rebase, rebase_branch=options.branch)
1333     except Exception:
1334         cleanup_list.append(gitroot + "/autobuild.pid")
1335         cleanup()
1336         elapsed_time = time.time() - start_time
1337         email_failure(-1, 'rebase', 'rebase', 'rebase',
1338                       'rebase on %s failed' % options.branch,
1339                       elapsed_time, log_base=options.log_base)
1340         sys.exit(1)
1341
1342     try:
1343         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1344         if options.tail:
1345             blist.start_tail()
1346         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1347         if status != 0 or errstr != "retry":
1348             break
1349         cleanup(do_raise=True)
1350     except Exception:
1351         cleanup()
1352         raise
1353
1354 cleanup_list.append(gitroot + "/autobuild.pid")
1355
1356 do_print(errstr)
1357
1358 blist.kill_kids()
1359 if options.tail:
1360     do_print("waiting for tail to flush")
1361     time.sleep(1)
1362
1363 elapsed_time = time.time() - start_time
1364 if status == 0:
1365     if options.passcmd is not None:
1366         do_print("Running passcmd: %s" % options.passcmd)
1367         run_cmd(options.passcmd, dir=test_master)
1368     if options.pushto is not None:
1369         push_to(options.pushto, push_branch=options.branch)
1370     if options.keeplogs or options.attach_logs:
1371         blist.tarlogs("logs.tar.gz")
1372         do_print("Logs in logs.tar.gz")
1373     if options.always_email:
1374         email_success(elapsed_time, log_base=options.log_base)
1375     blist.remove_logs()
1376     cleanup()
1377     do_print(errstr)
1378     sys.exit(0)
1379
1380 # something failed, gather a tar of the logs
1381 blist.tarlogs("logs.tar.gz")
1382
1383 if options.email is not None:
1384     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1385                   elapsed_time, log_base=options.log_base)
1386 else:
1387     elapsed_minutes = elapsed_time / 60.0
1388     print('''
1389
1390 ####################################################################
1391
1392 AUTOBUILD FAILURE
1393
1394 Your autobuild[%s] on %s failed after %.1f minutes
1395 when trying to test %s with the following error:
1396
1397    %s
1398
1399 the autobuild has been abandoned. Please fix the error and resubmit.
1400
1401 ####################################################################
1402
1403 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1404
1405 cleanup()
1406 do_print(errstr)
1407 do_print("Logs in logs.tar.gz")
1408 sys.exit(status)