script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, check_output, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 import random
  13 from optparse import OptionParser
  14 import smtplib
  15 import email
  16 from email.mime.text import MIMEText
  17 from email.mime.base import MIMEBase
  18 from email.mime.application import MIMEApplication
  19 from email.mime.multipart import MIMEMultipart
  20 from distutils.sysconfig import get_python_lib
  21 import platform
  22
  23 try:
  24     from waflib.Build import CACHE_SUFFIX
  25 except ImportError:
  26     sys.path.insert(0, "./third_party/waf")
  27     from waflib.Build import CACHE_SUFFIX
  28
  29
  30 os.environ["PYTHONUNBUFFERED"] = "1"
  31
  32 # This speeds up testing remarkably.
  33 os.environ['TDB_NO_FSYNC'] = '1'
  34
  35
  36 def find_git_root():
  37     '''get to the top of the git repo'''
  38     p = os.getcwd()
  39     while p != '/':
  40         if os.path.exists(os.path.join(p, ".git")):
  41             return p
  42         p = os.path.abspath(os.path.join(p, '..'))
  43     return None
  44
  45
  46 gitroot = find_git_root()
  47 if gitroot is None:
  48     raise Exception("Failed to find git root")
  49
  50
  51 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  52
  53 parser = OptionParser()
  54 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  55 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  56 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  57 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  58                   default=def_testbase)
  59 parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase,
  60                   default=None)
  61 parser.add_option("--passcmd", help="command to run on success", default=None)
  62 parser.add_option("--verbose", help="show all commands as they are run",
  63                   default=False, action="store_true")
  64 parser.add_option("--rebase", help="rebase on the given tree before testing",
  65                   default=None, type='str')
  66 parser.add_option("--pushto", help="push to a git url on success",
  67                   default=None, type='str')
  68 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  69                   default=False, action="store_true")
  70 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  71                   default=False, action="store_true")
  72 parser.add_option("--retry", help="automatically retry if master changes",
  73                   default=False, action="store_true")
  74 parser.add_option("--email", help="send email to the given address on failure",
  75                   type='str', default=None)
  76 parser.add_option("--email-from", help="send email from the given address",
  77                   type='str', default="autobuild@samba.org")
  78 parser.add_option("--email-server", help="send email via the given server",
  79                   type='str', default='localhost')
  80 parser.add_option("--always-email", help="always send email, even on success",
  81                   action="store_true")
  82 parser.add_option("--daemon", help="daemonize after initial setup",
  83                   action="store_true")
  84 parser.add_option("--branch", help="the branch to work on (default=master)",
  85                   default="master", type='str')
  86 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  87                   default=gitroot, type='str')
  88 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  89                   default=False, action="store_true")
  90 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
  91                   default='')
  92 parser.add_option("--enable-coverage", dest='enable_coverage',
  93                   action="store_const", const='--enable-coverage', default='',
  94                   help="Add --enable-coverage option while configure")
  95
  96 (options, args) = parser.parse_args()
  97
  98 if options.retry:
  99     if options.rebase is None:
 100         raise Exception('You can only use --retry if you also rebase')
 101
 102 if options.full_testbase is not None:
 103     testbase = options.full_testbase
 104 else:
 105     testbase = "%s/b%u" % (options.testbase, os.getpid())
 106 test_master = "%s/master" % testbase
 107 test_prefix = "%s/prefix" % testbase
 108 test_tmpdir = "%s/tmp" % testbase
 109 os.environ['TMPDIR'] = test_tmpdir
 110
 111 if options.enable_coverage:
 112     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 113 else:
 114     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 115
 116 if args:
 117     # If we are only running specific test,
 118     # do not sleep randomly to wait for it to start
 119     def random_sleep(low, high):
 120         return 'sleep 1'
 121 else:
 122     def random_sleep(low, high):
 123         return 'sleep {}'.format(random.randint(low, high))
 124
 125 cleanup_list = []
 126
 127 builddirs = {
 128     "ctdb": "ctdb",
 129     "ldb": "lib/ldb",
 130     "tdb": "lib/tdb",
 131     "talloc": "lib/talloc",
 132     "replace": "lib/replace",
 133     "tevent": "lib/tevent",
 134     "pidl": "pidl",
 135     "docs-xml": "docs-xml"
 136 }
 137
 138 ctdb_configure_params = " --enable-developer ${PREFIX}"
 139 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 140
 141 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 142 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 143 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 144 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 145 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 146 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 147 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 148
 149
 150 def format_option(name, value=None):
 151     """Format option as str list."""
 152     if value is None:  # boolean option
 153         return [name]
 154     if not isinstance(value, list):  # single value option
 155         value = [value]
 156     # repeatable option
 157     return ['{}={}'.format(name, item) for item in value]
 158
 159
 160 def make_test(
 161         cmd='make test',
 162         FAIL_IMMEDIATELY=1,
 163         TESTS='',
 164         include_envs=None,
 165         exclude_envs=None):
 166
 167     test_options = []
 168     if include_envs:
 169         test_options = format_option('--include-env', include_envs)
 170     if exclude_envs:
 171         test_options = format_option('--exclude-env', exclude_envs)
 172     if test_options:
 173         # join envs options to original test options
 174         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 175
 176     _options = []
 177     if FAIL_IMMEDIATELY:
 178         _options.append('FAIL_IMMEDIATELY=1')
 179     if TESTS:
 180         _options.append("TESTS='{}'".format(TESTS))
 181
 182     return ' '.join([cmd] + _options)
 183
 184
 185 # When updating this list, also update .gitlab-ci.yml to add the job
 186 # and to make it a dependency of 'page' for the coverage report.
 187
 188 tasks = {
 189     "ctdb": [
 190         ("random-sleep", random_sleep(300, 900)),
 191         ("configure", "./configure " + ctdb_configure_params),
 192         ("make", "make all"),
 193         ("install", "make install"),
 194         ("test", "make autotest"),
 195         ("check-clean-tree", "../script/clean-source-tree.sh"),
 196         ("clean", "make clean"),
 197         ],
 198
 199     "docs-xml": [
 200         ("random-sleep", random_sleep(300, 900)),
 201         ("autoconf", "autoconf"),
 202         ("configure", "./configure"),
 203         ("make", "make html htmlman"),
 204         ("clean", "make clean"),
 205         ],
 206
 207     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 208     "samba": [
 209         ("random-sleep", random_sleep(300, 900)),
 210         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 211         ("make", "make -j"),
 212         ("test", make_test(exclude_envs=[
 213             "none",
 214             "nt4_dc",
 215             "nt4_dc_smb1",
 216             "nt4_dc_smb1_done",
 217             "nt4_dc_schannel",
 218             "nt4_member",
 219             "ad_dc",
 220             "ad_dc_smb1",
 221             "ad_dc_smb1_done",
 222             "ad_dc_backup",
 223             "ad_dc_ntvfs",
 224             "ad_dc_default",
 225             "ad_dc_default_smb1",
 226             "ad_dc_slowtests",
 227             "ad_dc_no_nss",
 228             "ad_dc_no_ntlm",
 229             "fl2003dc",
 230             "fl2008dc",
 231             "fl2008r2dc",
 232             "ad_member",
 233             "ad_member_idmap_rid",
 234             "ad_member_idmap_ad",
 235             "ad_member_rfc2307",
 236             "chgdcpass",
 237             "vampire_2000_dc",
 238             "fl2000dc",
 239             "fileserver",
 240             "fileserver_smb1",
 241             "fileserver_smb1_done",
 242             "maptoguest",
 243             "simpleserver",
 244             "backupfromdc",
 245             "restoredc",
 246             "renamedc",
 247             "offlinebackupdc",
 248             "labdc",
 249             "preforkrestartdc",
 250             "proclimitdc",
 251             "promoted_dc",
 252             "vampire_dc",
 253             "rodc",
 254             "ad_dc_default",
 255             "ad_dc_default_smb1",
 256             "ad_dc_default_smb1_done",
 257             "ad_dc_slowtests",
 258             "schema_pair_dc",
 259             "schema_dc",
 260             "clusteredmember",
 261             ])),
 262         ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
 263         ("lcov", LCOV_CMD),
 264         ("install", "make install"),
 265         ("check-clean-tree", "script/clean-source-tree.sh"),
 266         ("clean", "make clean"),
 267         ],
 268
 269     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 270     "samba-mitkrb5": [
 271         ("random-sleep", random_sleep(300, 900)),
 272         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 273         ("make", "make -j"),
 274         ("test", make_test(exclude_envs=[
 275             "none",
 276             "nt4_dc",
 277             "nt4_dc_smb1",
 278             "nt4_dc_smb1_done",
 279             "nt4_dc_schannel",
 280             "nt4_member",
 281             "ad_dc",
 282             "ad_dc_smb1",
 283             "ad_dc_smb1_done",
 284             "ad_dc_backup",
 285             "ad_dc_ntvfs",
 286             "ad_dc_default",
 287             "ad_dc_default_smb1",
 288             "ad_dc_default_smb1_done",
 289             "ad_dc_slowtests",
 290             "ad_dc_no_nss",
 291             "ad_dc_no_ntlm",
 292             "fl2003dc",
 293             "fl2008dc",
 294             "fl2008r2dc",
 295             "ad_member",
 296             "ad_member_idmap_rid",
 297             "ad_member_idmap_ad",
 298             "ad_member_rfc2307",
 299             "chgdcpass",
 300             "vampire_2000_dc",
 301             "fl2000dc",
 302             "fileserver",
 303             "fileserver_smb1",
 304             "fileserver_smb1_done",
 305             "maptoguest",
 306             "simpleserver",
 307             "backupfromdc",
 308             "restoredc",
 309             "renamedc",
 310             "offlinebackupdc",
 311             "labdc",
 312             "preforkrestartdc",
 313             "proclimitdc",
 314             "promoted_dc",
 315             "vampire_dc",
 316             "rodc",
 317             "ad_dc_default",
 318             "ad_dc_default_smb1",
 319             "ad_dc_default_smb1_done",
 320             "ad_dc_slowtests",
 321             "schema_pair_dc",
 322             "schema_dc",
 323             "clusteredmember",
 324             ])),
 325         ("lcov", LCOV_CMD),
 326         ("install", "make install"),
 327         ("check-clean-tree", "script/clean-source-tree.sh"),
 328         ("clean", "make clean"),
 329         ],
 330
 331     "samba-nt4": [
 332         ("random-sleep", random_sleep(300, 900)),
 333         ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params),
 334         ("make", "make -j"),
 335         ("test", make_test(include_envs=[
 336             "nt4_dc",
 337             "nt4_dc_smb1",
 338             "nt4_dc_smb1_done",
 339             "nt4_dc_schannel",
 340             "nt4_member",
 341             "simpleserver",
 342             ])),
 343         ("lcov", LCOV_CMD),
 344         ("check-clean-tree", "script/clean-source-tree.sh"),
 345         ("clean", "make clean"),
 346         ],
 347
 348     "samba-fileserver": [
 349         ("random-sleep", random_sleep(300, 900)),
 350         ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5 --with-selftest-prefix=./bin/ab" + samba_configure_params),
 351         ("make", "make -j"),
 352         ("test", make_test(include_envs=[
 353             "fileserver",
 354             "fileserver_smb1",
 355             "fileserver_smb1_done",
 356             "maptoguest",
 357             "ktest", # ktest is also tested in samba and samba-mitkrb5
 358                      # but is tested here against a system Heimdal
 359             ])),
 360         ("lcov", LCOV_CMD),
 361         ("check-clean-tree", "script/clean-source-tree.sh"),
 362         ],
 363
 364     "samba-admem": [
 365         ("random-sleep", random_sleep(300, 900)),
 366         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 367         ("make", "make -j"),
 368         ("test", make_test(include_envs=[
 369             "ad_member",
 370             "ad_member_idmap_rid",
 371             "ad_member_idmap_ad",
 372             "ad_member_rfc2307",
 373             ])),
 374         ("lcov", LCOV_CMD),
 375         ("check-clean-tree", "script/clean-source-tree.sh"),
 376         ],
 377
 378     "samba-no-opath": [
 379         ("random-sleep", random_sleep(300, 900)),
 380         ("configure", "ADDITIONAL_CFLAGS='-DDISABLE_OPATH=1' ./configure.developer --without-ad-dc --with-selftest-prefix=./bin/ab" + samba_configure_params),
 381         ("make", "make -j"),
 382         ("test", make_test(
 383             cmd="make test DISABLE_OPATH=1",
 384             include_envs=[
 385             "nt4_dc",
 386             "nt4_dc_smb1",
 387             "nt4_dc_smb1_done",
 388             "nt4_dc_schannel",
 389             "nt4_member",
 390             "simpleserver",
 391             "fileserver",
 392             "fileserver_smb1",
 393             "fileserver_smb1_done",
 394             ])),
 395         ("lcov", LCOV_CMD),
 396         ("check-clean-tree", "script/clean-source-tree.sh"),
 397         ],
 398
 399     "samba-ad-dc-1": [
 400         ("random-sleep", random_sleep(1, 1)),
 401         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 402         ("make", "make -j"),
 403         ("test", make_test(include_envs=[
 404             "ad_dc",
 405             "ad_dc_smb1",
 406             "ad_dc_smb1_done",
 407             "ad_dc_no_nss",
 408             "ad_dc_no_ntlm",
 409             ])),
 410         ("lcov", LCOV_CMD),
 411         ("check-clean-tree", "script/clean-source-tree.sh"),
 412         ],
 413
 414     "samba-ad-dc-2": [
 415         ("random-sleep", random_sleep(1, 1)),
 416         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 417         ("make", "make -j"),
 418         ("test", make_test(include_envs=[
 419             "vampire_dc",
 420             "vampire_2000_dc",
 421             "rodc",
 422             ])),
 423         ("lcov", LCOV_CMD),
 424         ("check-clean-tree", "script/clean-source-tree.sh"),
 425         ],
 426
 427     "samba-ad-dc-3": [
 428         ("random-sleep", random_sleep(1, 1)),
 429         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 430         ("make", "make -j"),
 431         ("test", make_test(include_envs=[
 432             "promoted_dc",
 433             "chgdcpass",
 434             "preforkrestartdc",
 435             "proclimitdc",
 436             ])),
 437         ("lcov", LCOV_CMD),
 438         ("check-clean-tree", "script/clean-source-tree.sh"),
 439         ],
 440
 441     "samba-ad-dc-4": [
 442         ("random-sleep", random_sleep(1, 1)),
 443         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 444         ("make", "make -j"),
 445         ("test", make_test(include_envs=[
 446             "fl2000dc",
 447             "fl2003dc",
 448             "fl2008dc",
 449             "fl2008r2dc",
 450             ])),
 451         ("lcov", LCOV_CMD),
 452         ("check-clean-tree", "script/clean-source-tree.sh"),
 453         ],
 454
 455     "samba-ad-dc-5": [
 456         ("random-sleep", random_sleep(1, 1)),
 457         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 458         ("make", "make -j"),
 459         ("test", make_test(include_envs=[
 460             "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
 461         ("lcov", LCOV_CMD),
 462         ("check-clean-tree", "script/clean-source-tree.sh"),
 463         ],
 464
 465     "samba-ad-dc-6": [
 466         ("random-sleep", random_sleep(1, 1)),
 467         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 468         ("make", "make -j"),
 469         ("test", make_test(include_envs=["ad_dc_slowtests"])),
 470         ("lcov", LCOV_CMD),
 471         ("check-clean-tree", "script/clean-source-tree.sh"),
 472         ],
 473
 474     "samba-schemaupgrade": [
 475         ("random-sleep", random_sleep(1, 1)),
 476         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 477         ("make", "make -j"),
 478         ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 479         ("lcov", LCOV_CMD),
 480         ("check-clean-tree", "script/clean-source-tree.sh"),
 481         ],
 482
 483     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 484     # This is currently the longest task, so we don't randomly delay it.
 485     "samba-ad-dc-ntvfs": [
 486         ("random-sleep", random_sleep(1, 1)),
 487         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 488         ("make", "make -j"),
 489         ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 490         ("lcov", LCOV_CMD),
 491         ("check-clean-tree", "script/clean-source-tree.sh"),
 492         ],
 493
 494     # Test fips compliance
 495     "samba-fips": [
 496         ("random-sleep", random_sleep(100, 500)),
 497         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 498         ("make", "make -j"),
 499         ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
 500         ("lcov", LCOV_CMD),
 501         ("check-clean-tree", "script/clean-source-tree.sh"),
 502         ],
 503
 504     # run the backup/restore testenvs separately as they're fairly standalone
 505     # (and CI seems to max out at ~8 different DCs running at once)
 506     "samba-ad-dc-backup": [
 507         ("random-sleep", random_sleep(300, 900)),
 508         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 509         ("make", "make -j"),
 510         ("test", make_test(include_envs=[
 511             "backupfromdc",
 512             "restoredc",
 513             "renamedc",
 514             "offlinebackupdc",
 515             "labdc",
 516             "ad_dc_backup",
 517             ])),
 518         ("lcov", LCOV_CMD),
 519         ("check-clean-tree", "script/clean-source-tree.sh"),
 520         ],
 521
 522     "samba-admem-mit": [
 523         ("random-sleep", random_sleep(1, 1)),
 524         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 525         ("make", "make -j"),
 526         ("test", make_test(include_envs=[
 527             "ad_member",
 528             "ad_member_idmap_rid",
 529             "ad_member_idmap_ad",
 530             "ad_member_rfc2307",
 531             ])),
 532         ("lcov", LCOV_CMD),
 533         ("check-clean-tree", "script/clean-source-tree.sh"),
 534         ],
 535
 536     "samba-ad-dc-1-mitkrb5": [
 537         ("random-sleep", random_sleep(1, 1)),
 538         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 539         ("make", "make -j"),
 540         ("test", make_test(include_envs=[
 541             "ad_dc",
 542             "ad_dc_smb1",
 543             "ad_dc_smb1_done",
 544             "ad_dc_no_nss",
 545             "ad_dc_no_ntlm",
 546             ])),
 547         ("lcov", LCOV_CMD),
 548         ("check-clean-tree", "script/clean-source-tree.sh"),
 549         ],
 550
 551     "samba-ad-dc-4-mitkrb5": [
 552         ("random-sleep", random_sleep(1, 1)),
 553         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 554         ("make", "make -j"),
 555         ("test", make_test(include_envs=[
 556             "fl2000dc",
 557             "fl2003dc",
 558             "fl2008dc",
 559             "fl2008r2dc",
 560             ])),
 561         ("lcov", LCOV_CMD),
 562         ("check-clean-tree", "script/clean-source-tree.sh"),
 563         ],
 564
 565     "samba-test-only": [
 566         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params),
 567         ("make", "make -j"),
 568         ("test", make_test(TESTS="${TESTS}")),
 569         ("lcov", LCOV_CMD),
 570         ],
 571
 572     # Test cross-compile infrastructure
 573     "samba-xc": [
 574         ("random-sleep", random_sleep(900, 1500)),
 575         ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 576         ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 577             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 578         ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
 579         ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 580             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 581         ("compare-results", "script/compare_cc_results.py "
 582             "./bin/c4che/default{} "
 583             "./bin-xe/c4che/default{} "
 584             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 585         ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
 586         ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
 587             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
 588         ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
 589             " = \"'1234'\"".format(CACHE_SUFFIX)),
 590         ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
 591         ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
 592             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
 593             " ; test $? -ne 0"),
 594         ],
 595
 596     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 597     "samba-o3": [
 598         ("random-sleep", random_sleep(300, 900)),
 599         ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params),
 600         ("make", "make -j"),
 601         ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 602         ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 603         ("lcov", LCOV_CMD),
 604         ("install", "make install"),
 605         ("check-clean-tree", "script/clean-source-tree.sh"),
 606         ("clean", "make clean"),
 607         ],
 608
 609     "samba-ctdb": [
 610         ("random-sleep", random_sleep(900, 1500)),
 611
 612         # make sure we have tdb around:
 613         ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 614         ("tdb-make", "cd lib/tdb && make"),
 615         ("tdb-install", "cd lib/tdb && make install"),
 616
 617         # build samba with cluster support (also building ctdb):
 618         ("samba-configure",
 619          "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
 620          "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
 621          "./configure.developer ${PREFIX} "
 622          "--with-selftest-prefix=./bin/ab "
 623          "--enable-clangdb "
 624          "--with-cluster-support "
 625          "--without-ad-dc "
 626          "--bundled-libraries=!tdb"),
 627         ("samba-make", "make"),
 628         ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT"),
 629         ("samba-install", "make install"),
 630         ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 631
 632         ("test",
 633          make_test(cmd='make test',
 634                    include_envs=["clusteredmember"])
 635         ),
 636
 637         # clean up:
 638         ("check-clean-tree", "script/clean-source-tree.sh"),
 639         ("clean", "make clean"),
 640         ("ctdb-clean", "cd ./ctdb && make clean"),
 641         ],
 642
 643     "samba-libs": [
 644         ("random-sleep", random_sleep(300, 900)),
 645         ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 646         ("talloc-make", "cd lib/talloc && make"),
 647         ("talloc-install", "cd lib/talloc && make install"),
 648
 649         ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 650         ("tdb-make", "cd lib/tdb && make"),
 651         ("tdb-install", "cd lib/tdb && make install"),
 652
 653         ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 654         ("tevent-make", "cd lib/tevent && make"),
 655         ("tevent-install", "cd lib/tevent && make install"),
 656
 657         ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
 658         ("ldb-make", "cd lib/ldb && make"),
 659         ("ldb-install", "cd lib/ldb && make install"),
 660
 661         ("nondevel-configure", "./configure ${PREFIX}"),
 662         ("nondevel-make", "make -j"),
 663         ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 664         ("nondevel-install", "make install"),
 665         ("nondevel-dist", "make dist"),
 666
 667         # retry with all modules shared
 668         ("allshared-distclean", "make distclean"),
 669         ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 670         ("allshared-make", "make -j"),
 671         ],
 672
 673     "samba-fuzz": [
 674         # build the fuzzers (static) via the oss-fuzz script
 675         ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
 676         ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ADDITIONAL_LDFLAGS='-fuse-ld=bfd' ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl"),
 677         ],
 678
 679     # * Test smbd and smbtorture can build semi-static
 680     #
 681     # * Test Samba without python still builds.
 682     #
 683     # When this test fails due to more use of Python, the expectations
 684     # is that the newly failing part of the code should be disabled
 685     # when --disable-python is set (rather than major work being done
 686     # to support this environment).
 687     #
 688     # The target here is for vendors shipping a minimal smbd.
 689     "samba-minimal-smbd": [
 690         ("random-sleep", random_sleep(300, 900)),
 691
 692         # build with all modules static
 693         ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
 694         ("allstatic-make", "make -j"),
 695         ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 696         ("lcov", LCOV_CMD),
 697
 698         # retry with nonshared smbd and smbtorture
 699         ("nonshared-distclean", "make distclean"),
 700         ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
 701         ("nonshared-make", "make -j"),
 702
 703         ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
 704         ("make", "make -j"),
 705         ("find-python", "script/find_python.sh ${PREFIX}"),
 706         ("test", "make test-nopython"),
 707         ("lcov", LCOV_CMD),
 708         ("check-clean-tree", "script/clean-source-tree.sh"),
 709         ("clean", "make clean"),
 710
 711         ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 712         ("talloc-make", "cd lib/talloc && make"),
 713         ("talloc-install", "cd lib/talloc && make install"),
 714
 715         ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 716         ("tdb-make", "cd lib/tdb && make"),
 717         ("tdb-install", "cd lib/tdb && make install"),
 718
 719         ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 720         ("tevent-make", "cd lib/tevent && make"),
 721         ("tevent-install", "cd lib/tevent && make install"),
 722
 723         ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 724         ("ldb-make", "cd lib/ldb && make"),
 725         ("ldb-install", "cd lib/ldb && make install"),
 726
 727         # retry against installed library packages, but no required modules
 728         ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc  --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
 729         ("libs-make", "make -j"),
 730         ("libs-install", "make install"),
 731         ("libs-check-clean-tree", "script/clean-source-tree.sh"),
 732         ("libs-clean", "make clean"),
 733
 734         ],
 735
 736     "ldb": [
 737         ("random-sleep", random_sleep(60, 600)),
 738         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 739         ("make", "make"),
 740         ("install", "make install"),
 741         ("test", "make test"),
 742         ("lcov", LCOV_CMD),
 743         ("clean", "make clean"),
 744         ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
 745         ("make-no-lmdb", "make"),
 746         ("test-no-lmdb", "make test"),
 747         ("lcov-no-lmdb", LCOV_CMD),
 748         ("install-no-lmdb", "make install"),
 749         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 750         ("distcheck", "make distcheck"),
 751         ("clean", "make clean"),
 752         ],
 753
 754     "tdb": [
 755         ("random-sleep", random_sleep(60, 600)),
 756         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 757         ("make", "make"),
 758         ("install", "make install"),
 759         ("test", "make test"),
 760         ("lcov", LCOV_CMD),
 761         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 762         ("distcheck", "make distcheck"),
 763         ("clean", "make clean"),
 764         ],
 765
 766     "talloc": [
 767         ("random-sleep", random_sleep(60, 600)),
 768         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 769         ("make", "make"),
 770         ("install", "make install"),
 771         ("test", "make test"),
 772         ("lcov", LCOV_CMD),
 773         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 774         ("distcheck", "make distcheck"),
 775         ("clean", "make clean"),
 776         ],
 777
 778     "replace": [
 779         ("random-sleep", random_sleep(60, 600)),
 780         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 781         ("make", "make"),
 782         ("install", "make install"),
 783         ("test", "make test"),
 784         ("lcov", LCOV_CMD),
 785         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 786         ("distcheck", "make distcheck"),
 787         ("clean", "make clean"),
 788         ],
 789
 790     "tevent": [
 791         ("random-sleep", random_sleep(60, 600)),
 792         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 793         ("make", "make"),
 794         ("install", "make install"),
 795         ("test", "make test"),
 796         ("lcov", LCOV_CMD),
 797         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 798         ("distcheck", "make distcheck"),
 799         ("clean", "make clean"),
 800         ],
 801
 802     "pidl": [
 803         ("random-sleep", random_sleep(60, 600)),
 804         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
 805         ("touch", "touch *.yp"),
 806         ("make", "make"),
 807         ("test", "make test"),
 808         ("install", "make install"),
 809         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
 810         ("check-clean-tree", "../script/clean-source-tree.sh"),
 811         ("clean", "make clean"),
 812         ],
 813
 814     # these are useful for debugging autobuild
 815     'pass': [("pass", 'echo passing && /bin/true')],
 816     'fail': [("fail", 'echo failing && /bin/false')],
 817 }
 818
 819 defaulttasks = list(tasks.keys())
 820
 821 defaulttasks.remove("pass")
 822 defaulttasks.remove("fail")
 823 defaulttasks.remove("samba-test-only")
 824 defaulttasks.remove("samba-fuzz")
 825 defaulttasks.remove("samba-fips")
 826 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
 827     defaulttasks.remove("samba-o3")
 828
 829
 830 def do_print(msg):
 831     print("%s" % msg)
 832     sys.stdout.flush()
 833     sys.stderr.flush()
 834
 835
 836 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 837     if show is None:
 838         show = options.verbose
 839     if show:
 840         do_print("Running: '%s' in '%s'" % (cmd, dir))
 841     if output:
 842         out = check_output([cmd], shell=True, cwd=dir)
 843         return out.decode(encoding='utf-8', errors='backslashreplace')
 844     elif checkfail:
 845         return check_call(cmd, shell=True, cwd=dir)
 846     else:
 847         return call(cmd, shell=True, cwd=dir)
 848
 849
 850 class builder(object):
 851     '''handle build of one directory'''
 852
 853     def __init__(self, name, sequence, cp=True):
 854         self.name = name
 855         self.dir = builddirs.get(name, '.')
 856         self.tag = self.name.replace('/', '_')
 857         self.sequence = sequence
 858         self.next = 0
 859         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 860         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 861         if options.verbose:
 862             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 863             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 864         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 865         self.stdout = open(self.stdout_path, 'w')
 866         self.stderr = open(self.stderr_path, 'w')
 867         self.stdin  = open("/dev/null", 'r')
 868         self.test_source_dir = "%s/%s" % (testbase, self.tag)
 869         self.cwd = "%s/%s" % (self.test_source_dir, self.dir)
 870         self.prefix = "%s/%s" % (test_prefix, self.tag)
 871         run_cmd("rm -rf %s" % self.test_source_dir)
 872         run_cmd("rm -rf %s" % self.prefix)
 873         if cp:
 874             run_cmd("cp -R -a -l %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
 875         else:
 876             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
 877         self.start_next()
 878
 879     def start_next(self):
 880         if self.next == len(self.sequence):
 881             if not options.nocleanup:
 882                 run_cmd("rm -rf %s" % self.test_source_dir)
 883                 run_cmd("rm -rf %s" % self.prefix)
 884             do_print('%s: Completed OK' % self.name)
 885             self.done = True
 886             return
 887         (self.stage, self.cmd) = self.sequence[self.next]
 888         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
 889         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 890         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 891         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 892         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
 893         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
 894         self.cmd = self.cmd.replace("${NAME}", self.name)
 895         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
 896         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
 897         self.proc = Popen(self.cmd, shell=True,
 898                           close_fds=True, cwd=self.cwd,
 899                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 900         self.next += 1
 901
 902
 903 class buildlist(object):
 904     '''handle build of multiple directories'''
 905
 906     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 907         self.tail_proc = None
 908         self.retry = None
 909         if not tasknames:
 910             if options.restrict_tests:
 911                 tasknames = ["samba-test-only"]
 912             else:
 913                 tasknames = defaulttasks
 914
 915         self.tlist = [builder(n, tasks[n], cp=(n != "pidl")) for n in tasknames]
 916
 917         if options.retry:
 918             rebase_remote = "rebaseon"
 919             retry_task = [("retry",
 920                             '''set -e
 921                             git remote add -t %s %s %s
 922                             git fetch %s
 923                             while :; do
 924                               sleep 60
 925                               git describe %s/%s > old_remote_branch.desc
 926                               git fetch %s
 927                               git describe %s/%s > remote_branch.desc
 928                               diff old_remote_branch.desc remote_branch.desc
 929                             done
 930                            ''' % (
 931                                rebase_branch, rebase_remote, rebase_url,
 932                                rebase_remote,
 933                                rebase_remote, rebase_branch,
 934                                rebase_remote,
 935                                rebase_remote, rebase_branch
 936                             ))]
 937
 938             self.retry = builder('retry', retry_task, cp=False)
 939             self.need_retry = False
 940
 941     def kill_kids(self):
 942         if self.tail_proc is not None:
 943             self.tail_proc.terminate()
 944             self.tail_proc.wait()
 945             self.tail_proc = None
 946         if self.retry is not None:
 947             self.retry.proc.terminate()
 948             self.retry.proc.wait()
 949             self.retry = None
 950         for b in self.tlist:
 951             if b.proc is not None:
 952                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
 953                 b.proc.terminate()
 954                 b.proc.wait()
 955                 b.proc = None
 956
 957     def wait_one(self):
 958         while True:
 959             none_running = True
 960             for b in self.tlist:
 961                 if b.proc is None:
 962                     continue
 963                 none_running = False
 964                 b.status = b.proc.poll()
 965                 if b.status is None:
 966                     continue
 967                 b.proc = None
 968                 return b
 969             if options.retry:
 970                 ret = self.retry.proc.poll()
 971                 if ret is not None:
 972                     self.need_retry = True
 973                     self.retry = None
 974                     return None
 975             if none_running:
 976                 return None
 977             time.sleep(0.1)
 978
 979     def run(self):
 980         while True:
 981             b = self.wait_one()
 982             if options.retry and self.need_retry:
 983                 self.kill_kids()
 984                 do_print("retry needed")
 985                 return (0, None, None, None, "retry")
 986             if b is None:
 987                 break
 988             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 989                 self.kill_kids()
 990                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 991             b.start_next()
 992         self.kill_kids()
 993         return (0, None, None, None, "All OK")
 994
 995     def write_system_info(self, filename):
 996         with open(filename, 'w') as f:
 997             for cmd in ['uname -a',
 998                         'lsb_release -a',
 999                         'free',
1000                         'mount',
1001                         'cat /proc/cpuinfo',
1002                         'cc --version',
1003                         'df -m .',
1004                         'df -m %s' % testbase]:
1005                 try:
1006                     out = run_cmd(cmd, output=True, checkfail=False)
1007                 except subprocess.CalledProcessError as e:
1008                     out = "<failed: %s>" % str(e)
1009                 print('### %s' % cmd, file=f)
1010                 print(out, file=f)
1011                 print(file=f)
1012
1013     def tarlogs(self, fname):
1014         with tarfile.open(fname, "w:gz") as tar:
1015             for b in self.tlist:
1016                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
1017                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
1018             if os.path.exists("autobuild.log"):
1019                 tar.add("autobuild.log")
1020             filename = 'system-info.txt'
1021             self.write_system_info(filename)
1022             tar.add(filename)
1023
1024     def remove_logs(self):
1025         for b in self.tlist:
1026             os.unlink(b.stdout_path)
1027             os.unlink(b.stderr_path)
1028
1029     def start_tail(self):
1030         cmd = ["tail", "-f"]
1031         for b in self.tlist:
1032             cmd.append(b.stdout_path)
1033             cmd.append(b.stderr_path)
1034         self.tail_proc = Popen(cmd, close_fds=True)
1035
1036
1037 def cleanup():
1038     if options.nocleanup:
1039         return
1040     run_cmd("stat %s || true" % test_tmpdir, show=True)
1041     run_cmd("stat %s" % testbase, show=True)
1042     do_print("Cleaning up %r" % cleanup_list)
1043     for d in cleanup_list:
1044         run_cmd("rm -rf %s" % d)
1045
1046
1047 def daemonize(logfile):
1048     pid = os.fork()
1049     if pid == 0:  # Parent
1050         os.setsid()
1051         pid = os.fork()
1052         if pid != 0:  # Actual daemon
1053             os._exit(0)
1054     else:  # Grandparent
1055         os._exit(0)
1056
1057     import resource      # Resource usage information.
1058     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1059     if maxfd == resource.RLIM_INFINITY:
1060         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1061     for fd in range(0, maxfd):
1062         try:
1063             os.close(fd)
1064         except OSError:
1065             pass
1066     os.open(logfile, os.O_RDWR | os.O_CREAT)
1067     os.dup2(0, 1)
1068     os.dup2(0, 2)
1069
1070
1071 def write_pidfile(fname):
1072     '''write a pid file, cleanup on exit'''
1073     with open(fname, mode='w') as f:
1074         f.write("%u\n" % os.getpid())
1075
1076
1077 def rebase_tree(rebase_url, rebase_branch="master"):
1078     rebase_remote = "rebaseon"
1079     do_print("Rebasing on %s" % rebase_url)
1080     run_cmd("git describe HEAD", show=True, dir=test_master)
1081     run_cmd("git remote add -t %s %s %s" %
1082             (rebase_branch, rebase_remote, rebase_url),
1083             show=True, dir=test_master)
1084     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1085     if options.fix_whitespace:
1086         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1087                 (rebase_remote, rebase_branch),
1088                 show=True, dir=test_master)
1089     else:
1090         run_cmd("git rebase --force-rebase %s/%s" %
1091                 (rebase_remote, rebase_branch),
1092                 show=True, dir=test_master)
1093     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1094                    (rebase_remote, rebase_branch),
1095                    dir=test_master, output=True)
1096     if diff == '':
1097         do_print("No differences between HEAD and %s/%s - exiting" %
1098                  (rebase_remote, rebase_branch))
1099         sys.exit(0)
1100     run_cmd("git describe %s/%s" %
1101             (rebase_remote, rebase_branch),
1102             show=True, dir=test_master)
1103     run_cmd("git describe HEAD", show=True, dir=test_master)
1104     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1105             (rebase_remote, rebase_branch),
1106             show=True, dir=test_master)
1107
1108
1109 def push_to(push_url, push_branch="master"):
1110     push_remote = "pushto"
1111     do_print("Pushing to %s" % push_url)
1112     if options.mark:
1113         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1114         run_cmd("git commit --amend -c HEAD", dir=test_master)
1115         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1116         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1117     run_cmd("git remote add -t %s %s %s" %
1118             (push_branch, push_remote, push_url),
1119             show=True, dir=test_master)
1120     run_cmd("git push %s +HEAD:%s" %
1121             (push_remote, push_branch),
1122             show=True, dir=test_master)
1123
1124
1125 def send_email(subject, text, log_tar):
1126     if options.email is None:
1127         do_print("not sending email because the recipient is not set")
1128         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1129                  (subject, text))
1130         return
1131     outer = MIMEMultipart()
1132     outer['Subject'] = subject
1133     outer['To'] = options.email
1134     outer['From'] = options.email_from
1135     outer['Date'] = email.utils.formatdate(localtime=True)
1136     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1137     outer.attach(MIMEText(text, 'plain', 'utf-8'))
1138     if options.attach_logs:
1139         with open(log_tar, 'rb') as fp:
1140             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1141         # Set the filename parameter
1142         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1143         outer.attach(msg)
1144     content = outer.as_string()
1145     s = smtplib.SMTP(options.email_server)
1146     email_user = os.getenv('SMTP_USERNAME')
1147     email_password = os.getenv('SMTP_PASSWORD')
1148     if email_user is not None:
1149         s.starttls()
1150         s.login(email_user, email_password)
1151
1152     s.sendmail(options.email_from, [options.email], content)
1153     s.set_debuglevel(1)
1154     s.quit()
1155
1156
1157 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1158                   elapsed_time, log_base=None, add_log_tail=True):
1159     '''send an email to options.email about the failure'''
1160     elapsed_minutes = elapsed_time / 60.0
1161     if log_base is None:
1162         log_base = gitroot
1163     text = '''
1164 Dear Developer,
1165
1166 Your autobuild on %s failed after %.1f minutes
1167 when trying to test %s with the following error:
1168
1169    %s
1170
1171 the autobuild has been abandoned. Please fix the error and resubmit.
1172
1173 A summary of the autobuild process is here:
1174
1175   %s/autobuild.log
1176 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1177
1178     if options.restrict_tests:
1179         text += """
1180 The build was restricted to tests matching %s\n""" % options.restrict_tests
1181
1182     if failed_task != 'rebase':
1183         text += '''
1184 You can see logs of the failed task here:
1185
1186   %s/%s.stdout
1187   %s/%s.stderr
1188
1189 or you can get full logs of all tasks in this job here:
1190
1191   %s/logs.tar.gz
1192
1193 The top commit for the tree that was built was:
1194
1195 %s
1196
1197 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1198
1199     if add_log_tail:
1200         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1201         lines = f.readlines()
1202         log_tail = "".join(lines[-50:])
1203         num_lines = len(lines)
1204         if num_lines < 50:
1205             # Also include stderr (compile failures) if < 50 lines of stdout
1206             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1207             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1208
1209         text += '''
1210 The last 50 lines of log messages:
1211
1212 %s
1213     ''' % log_tail
1214         f.close()
1215
1216     logs = os.path.join(gitroot, 'logs.tar.gz')
1217     send_email('autobuild[%s] failure on %s for task %s during %s'
1218                % (options.branch, platform.node(), failed_task, failed_stage),
1219                text, logs)
1220
1221
1222 def email_success(elapsed_time, log_base=None):
1223     '''send an email to options.email about a successful build'''
1224     if log_base is None:
1225         log_base = gitroot
1226     text = '''
1227 Dear Developer,
1228
1229 Your autobuild on %s has succeeded after %.1f minutes.
1230
1231 ''' % (platform.node(), elapsed_time / 60.)
1232
1233     if options.restrict_tests:
1234         text += """
1235 The build was restricted to tests matching %s\n""" % options.restrict_tests
1236
1237     if options.keeplogs:
1238         text += '''
1239
1240 you can get full logs of all tasks in this job here:
1241
1242   %s/logs.tar.gz
1243
1244 ''' % log_base
1245
1246     text += '''
1247 The top commit for the tree that was built was:
1248
1249 %s
1250 ''' % top_commit_msg
1251
1252     logs = os.path.join(gitroot, 'logs.tar.gz')
1253     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1254                text, logs)
1255
1256
1257 # get the top commit message, for emails
1258 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1259
1260 try:
1261     os.makedirs(testbase)
1262 except Exception as reason:
1263     raise Exception("Unable to create %s : %s" % (testbase, reason))
1264 cleanup_list.append(testbase)
1265
1266 if options.daemon:
1267     logfile = os.path.join(testbase, "log")
1268     do_print("Forking into the background, writing progress to %s" % logfile)
1269     daemonize(logfile)
1270
1271 write_pidfile(gitroot + "/autobuild.pid")
1272
1273 start_time = time.time()
1274
1275 while True:
1276     try:
1277         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1278         os.makedirs(test_tmpdir)
1279         # The waf uninstall code removes empty directories all the way
1280         # up the tree.  Creating a file in test_tmpdir stops it from
1281         # being removed.
1282         run_cmd("touch %s" % os.path.join(test_tmpdir,
1283                                           ".directory-is-not-empty"), show=True)
1284         run_cmd("stat %s" % test_tmpdir, show=True)
1285         run_cmd("stat %s" % testbase, show=True)
1286         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1287     except Exception:
1288         cleanup()
1289         raise
1290
1291     try:
1292         if options.rebase is not None:
1293             rebase_tree(options.rebase, rebase_branch=options.branch)
1294     except Exception:
1295         cleanup_list.append(gitroot + "/autobuild.pid")
1296         cleanup()
1297         elapsed_time = time.time() - start_time
1298         email_failure(-1, 'rebase', 'rebase', 'rebase',
1299                       'rebase on %s failed' % options.branch,
1300                       elapsed_time, log_base=options.log_base)
1301         sys.exit(1)
1302
1303     try:
1304         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1305         if options.tail:
1306             blist.start_tail()
1307         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1308         if status != 0 or errstr != "retry":
1309             break
1310         cleanup()
1311     except Exception:
1312         cleanup()
1313         raise
1314
1315 cleanup_list.append(gitroot + "/autobuild.pid")
1316
1317 do_print(errstr)
1318
1319 blist.kill_kids()
1320 if options.tail:
1321     do_print("waiting for tail to flush")
1322     time.sleep(1)
1323
1324 elapsed_time = time.time() - start_time
1325 if status == 0:
1326     if options.passcmd is not None:
1327         do_print("Running passcmd: %s" % options.passcmd)
1328         run_cmd(options.passcmd, dir=test_master)
1329     if options.pushto is not None:
1330         push_to(options.pushto, push_branch=options.branch)
1331     if options.keeplogs or options.attach_logs:
1332         blist.tarlogs("logs.tar.gz")
1333         do_print("Logs in logs.tar.gz")
1334     if options.always_email:
1335         email_success(elapsed_time, log_base=options.log_base)
1336     blist.remove_logs()
1337     cleanup()
1338     do_print(errstr)
1339     sys.exit(0)
1340
1341 # something failed, gather a tar of the logs
1342 blist.tarlogs("logs.tar.gz")
1343
1344 if options.email is not None:
1345     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1346                   elapsed_time, log_base=options.log_base)
1347 else:
1348     elapsed_minutes = elapsed_time / 60.0
1349     print('''
1350
1351 ####################################################################
1352
1353 AUTOBUILD FAILURE
1354
1355 Your autobuild[%s] on %s failed after %.1f minutes
1356 when trying to test %s with the following error:
1357
1358    %s
1359
1360 the autobuild has been abandoned. Please fix the error and resubmit.
1361
1362 ####################################################################
1363
1364 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1365
1366 cleanup()
1367 do_print(errstr)
1368 do_print("Logs in logs.tar.gz")
1369 sys.exit(status)