script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, check_output, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 import random
  13 from optparse import OptionParser
  14 import smtplib
  15 import email
  16 from email.mime.text import MIMEText
  17 from email.mime.base import MIMEBase
  18 from email.mime.application import MIMEApplication
  19 from email.mime.multipart import MIMEMultipart
  20 from distutils.sysconfig import get_python_lib
  21 import platform
  22
  23 try:
  24     from waflib.Build import CACHE_SUFFIX
  25 except ImportError:
  26     sys.path.insert(0, "./third_party/waf")
  27     from waflib.Build import CACHE_SUFFIX
  28
  29
  30 os.environ["PYTHONUNBUFFERED"] = "1"
  31
  32 # This speeds up testing remarkably.
  33 os.environ['TDB_NO_FSYNC'] = '1'
  34
  35
  36 def find_git_root():
  37     '''get to the top of the git repo'''
  38     p = os.getcwd()
  39     while p != '/':
  40         if os.path.exists(os.path.join(p, ".git")):
  41             return p
  42         p = os.path.abspath(os.path.join(p, '..'))
  43     return None
  44
  45
  46 gitroot = find_git_root()
  47 if gitroot is None:
  48     raise Exception("Failed to find git root")
  49
  50
  51 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  52
  53 parser = OptionParser()
  54 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  55 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  56 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  57 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  58                   default=def_testbase)
  59 parser.add_option("--passcmd", help="command to run on success", default=None)
  60 parser.add_option("--verbose", help="show all commands as they are run",
  61                   default=False, action="store_true")
  62 parser.add_option("--rebase", help="rebase on the given tree before testing",
  63                   default=None, type='str')
  64 parser.add_option("--pushto", help="push to a git url on success",
  65                   default=None, type='str')
  66 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  67                   default=False, action="store_true")
  68 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  69                   default=False, action="store_true")
  70 parser.add_option("--retry", help="automatically retry if master changes",
  71                   default=False, action="store_true")
  72 parser.add_option("--email", help="send email to the given address on failure",
  73                   type='str', default=None)
  74 parser.add_option("--email-from", help="send email from the given address",
  75                   type='str', default="autobuild@samba.org")
  76 parser.add_option("--email-server", help="send email via the given server",
  77                   type='str', default='localhost')
  78 parser.add_option("--always-email", help="always send email, even on success",
  79                   action="store_true")
  80 parser.add_option("--daemon", help="daemonize after initial setup",
  81                   action="store_true")
  82 parser.add_option("--branch", help="the branch to work on (default=master)",
  83                   default="master", type='str')
  84 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  85                   default=gitroot, type='str')
  86 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  87                   default=False, action="store_true")
  88 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
  89                   default='')
  90 parser.add_option("--enable-coverage", dest='enable_coverage',
  91                   action="store_const", const='--enable-coverage', default='',
  92                   help="Add --enable-coverage option while configure")
  93
  94 (options, args) = parser.parse_args()
  95
  96 if options.retry:
  97     if options.rebase is None:
  98         raise Exception('You can only use --retry if you also rebase')
  99
 100 testbase = "%s/b%u" % (options.testbase, os.getpid())
 101 test_master = "%s/master" % testbase
 102 test_prefix = "%s/prefix" % testbase
 103 test_tmpdir = "%s/tmp" % testbase
 104 os.environ['TMPDIR'] = test_tmpdir
 105
 106 if options.enable_coverage:
 107     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 108 else:
 109     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 110
 111 if args:
 112     # If we are only running specific test,
 113     # do not sleep randomly to wait for it to start
 114     def random_sleep(low, high):
 115         return 'sleep 1'
 116 else:
 117     def random_sleep(low, high):
 118         return 'sleep {}'.format(random.randint(low, high))
 119
 120 cleanup_list = []
 121
 122 builddirs = {
 123     "ctdb": "ctdb",
 124     "ldb": "lib/ldb",
 125     "tdb": "lib/tdb",
 126     "talloc": "lib/talloc",
 127     "replace": "lib/replace",
 128     "tevent": "lib/tevent",
 129     "pidl": "pidl",
 130     "docs-xml": "docs-xml"
 131 }
 132
 133 ctdb_configure_params = " --enable-developer ${PREFIX}"
 134 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 135
 136 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 137 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 138 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 139 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 140 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 141 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 142 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 143
 144
 145 def format_option(name, value=None):
 146     """Format option as str list."""
 147     if value is None:  # boolean option
 148         return [name]
 149     if not isinstance(value, list):  # single value option
 150         value = [value]
 151     # repeatable option
 152     return ['{}={}'.format(name, item) for item in value]
 153
 154
 155 def make_test(
 156         cmd='make test',
 157         FAIL_IMMEDIATELY=1,
 158         TESTS='',
 159         include_envs=None,
 160         exclude_envs=None):
 161
 162     test_options = []
 163     if include_envs:
 164         test_options = format_option('--include-env', include_envs)
 165     if exclude_envs:
 166         test_options = format_option('--exclude-env', exclude_envs)
 167     if test_options:
 168         # join envs options to original test options
 169         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 170
 171     _options = []
 172     if FAIL_IMMEDIATELY:
 173         _options.append('FAIL_IMMEDIATELY=1')
 174     if TESTS:
 175         _options.append("TESTS='{}'".format(TESTS))
 176
 177     return ' '.join([cmd] + _options)
 178
 179
 180 # When updating this list, also update .gitlab-ci.yml to add the job
 181 # and to make it a dependency of 'page' for the coverage report.
 182
 183 tasks = {
 184     "ctdb": [
 185         ("random-sleep", random_sleep(300, 900)),
 186         ("configure", "./configure " + ctdb_configure_params),
 187         ("make", "make all"),
 188         ("install", "make install"),
 189         ("test", "make autotest"),
 190         ("check-clean-tree", "../script/clean-source-tree.sh"),
 191         ("clean", "make clean"),
 192         ],
 193
 194     "docs-xml": [
 195         ("random-sleep", random_sleep(300, 900)),
 196         ("autoconf", "autoconf"),
 197         ("configure", "./configure"),
 198         ("make", "make html htmlman"),
 199         ("clean", "make clean"),
 200         ],
 201
 202     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 203     "samba": [
 204         ("random-sleep", random_sleep(300, 900)),
 205         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 206         ("make", "make -j"),
 207         ("test", make_test(exclude_envs=[
 208             "none",
 209             "nt4_dc",
 210             "nt4_dc_smb1",
 211             "nt4_dc_smb1_done",
 212             "nt4_dc_schannel",
 213             "nt4_member",
 214             "ad_dc",
 215             "ad_dc_smb1",
 216             "ad_dc_smb1_done",
 217             "ad_dc_backup",
 218             "ad_dc_ntvfs",
 219             "ad_dc_default",
 220             "ad_dc_default_smb1",
 221             "ad_dc_slowtests",
 222             "ad_dc_no_nss",
 223             "ad_dc_no_ntlm",
 224             "fl2003dc",
 225             "fl2008dc",
 226             "fl2008r2dc",
 227             "ad_member",
 228             "ad_member_idmap_rid",
 229             "ad_member_idmap_ad",
 230             "ad_member_rfc2307",
 231             "chgdcpass",
 232             "vampire_2000_dc",
 233             "fl2000dc",
 234             "fileserver",
 235             "fileserver_smb1",
 236             "fileserver_smb1_done",
 237             "maptoguest",
 238             "simpleserver",
 239             "backupfromdc",
 240             "restoredc",
 241             "renamedc",
 242             "offlinebackupdc",
 243             "labdc",
 244             "preforkrestartdc",
 245             "proclimitdc",
 246             "promoted_dc",
 247             "vampire_dc",
 248             "rodc",
 249             "ad_dc_default",
 250             "ad_dc_default_smb1",
 251             "ad_dc_default_smb1_done",
 252             "ad_dc_slowtests",
 253             "schema_pair_dc",
 254             "schema_dc",
 255             "clusteredmember_smb1",
 256             ])),
 257         ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
 258         ("lcov", LCOV_CMD),
 259         ("install", "make install"),
 260         ("check-clean-tree", "script/clean-source-tree.sh"),
 261         ("clean", "make clean"),
 262         ],
 263
 264     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 265     "samba-mitkrb5": [
 266         ("random-sleep", random_sleep(300, 900)),
 267         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 268         ("make", "make -j"),
 269         ("test", make_test(exclude_envs=[
 270             "none",
 271             "nt4_dc",
 272             "nt4_dc_smb1",
 273             "nt4_dc_smb1_done",
 274             "nt4_dc_schannel",
 275             "nt4_member",
 276             "ad_dc",
 277             "ad_dc_smb1",
 278             "ad_dc_smb1_done",
 279             "ad_dc_backup",
 280             "ad_dc_ntvfs",
 281             "ad_dc_default",
 282             "ad_dc_default_smb1",
 283             "ad_dc_default_smb1_done",
 284             "ad_dc_slowtests",
 285             "ad_dc_no_nss",
 286             "ad_dc_no_ntlm",
 287             "fl2003dc",
 288             "fl2008dc",
 289             "fl2008r2dc",
 290             "ad_member",
 291             "ad_member_idmap_rid",
 292             "ad_member_idmap_ad",
 293             "ad_member_rfc2307",
 294             "chgdcpass",
 295             "vampire_2000_dc",
 296             "fl2000dc",
 297             "fileserver",
 298             "fileserver_smb1",
 299             "fileserver_smb1_done",
 300             "maptoguest",
 301             "simpleserver",
 302             "backupfromdc",
 303             "restoredc",
 304             "renamedc",
 305             "offlinebackupdc",
 306             "labdc",
 307             "preforkrestartdc",
 308             "proclimitdc",
 309             "promoted_dc",
 310             "vampire_dc",
 311             "rodc",
 312             "ad_dc_default",
 313             "ad_dc_default_smb1",
 314             "ad_dc_default_smb1_done",
 315             "ad_dc_slowtests",
 316             "schema_pair_dc",
 317             "schema_dc",
 318             "clusteredmember_smb1",
 319             ])),
 320         ("lcov", LCOV_CMD),
 321         ("install", "make install"),
 322         ("check-clean-tree", "script/clean-source-tree.sh"),
 323         ("clean", "make clean"),
 324         ],
 325
 326     "samba-nt4": [
 327         ("random-sleep", random_sleep(300, 900)),
 328         ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params),
 329         ("make", "make -j"),
 330         ("test", make_test(include_envs=[
 331             "nt4_dc",
 332             "nt4_dc_smb1",
 333             "nt4_dc_smb1_done",
 334             "nt4_dc_schannel",
 335             "nt4_member",
 336             "simpleserver",
 337             ])),
 338         ("lcov", LCOV_CMD),
 339         ("install", "make install"),
 340         ("check-clean-tree", "script/clean-source-tree.sh"),
 341         ("clean", "make clean"),
 342         ],
 343
 344     "samba-fileserver": [
 345         ("random-sleep", random_sleep(300, 900)),
 346         ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5 --with-selftest-prefix=./bin/ab" + samba_configure_params),
 347         ("make", "make -j"),
 348         ("test", make_test(include_envs=[
 349             "fileserver",
 350             "fileserver_smb1",
 351             "fileserver_smb1_done",
 352             "maptoguest",
 353             "ktest", # ktest is also tested in samba and samba-mitkrb5
 354                      # but is tested here against a system Heimdal
 355             ])),
 356         ("lcov", LCOV_CMD),
 357         ("check-clean-tree", "script/clean-source-tree.sh"),
 358         ],
 359
 360     "samba-admem": [
 361         ("random-sleep", random_sleep(300, 900)),
 362         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 363         ("make", "make -j"),
 364         ("test", make_test(include_envs=[
 365             "ad_member",
 366             "ad_member_idmap_rid",
 367             "ad_member_idmap_ad",
 368             "ad_member_rfc2307",
 369             ])),
 370         ("lcov", LCOV_CMD),
 371         ("check-clean-tree", "script/clean-source-tree.sh"),
 372         ],
 373
 374     "samba-ad-dc-1": [
 375         ("random-sleep", random_sleep(1, 1)),
 376         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 377         ("make", "make -j"),
 378         ("test", make_test(include_envs=[
 379             "ad_dc",
 380             "ad_dc_smb1",
 381             "ad_dc_smb1_done",
 382             "ad_dc_no_nss",
 383             "ad_dc_no_ntlm",
 384             ])),
 385         ("lcov", LCOV_CMD),
 386         ("check-clean-tree", "script/clean-source-tree.sh"),
 387         ],
 388
 389     "samba-ad-dc-2": [
 390         ("random-sleep", random_sleep(1, 1)),
 391         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 392         ("make", "make -j"),
 393         ("test", make_test(include_envs=[
 394             "vampire_dc",
 395             "vampire_2000_dc",
 396             "rodc",
 397             ])),
 398         ("lcov", LCOV_CMD),
 399         ("check-clean-tree", "script/clean-source-tree.sh"),
 400         ],
 401
 402     "samba-ad-dc-3": [
 403         ("random-sleep", random_sleep(1, 1)),
 404         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 405         ("make", "make -j"),
 406         ("test", make_test(include_envs=[
 407             "promoted_dc",
 408             "chgdcpass",
 409             "preforkrestartdc",
 410             "proclimitdc",
 411             ])),
 412         ("lcov", LCOV_CMD),
 413         ("check-clean-tree", "script/clean-source-tree.sh"),
 414         ],
 415
 416     "samba-ad-dc-4": [
 417         ("random-sleep", random_sleep(1, 1)),
 418         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 419         ("make", "make -j"),
 420         ("test", make_test(include_envs=[
 421             "fl2000dc",
 422             "fl2003dc",
 423             "fl2008dc",
 424             "fl2008r2dc",
 425             ])),
 426         ("lcov", LCOV_CMD),
 427         ("check-clean-tree", "script/clean-source-tree.sh"),
 428         ],
 429
 430     "samba-ad-dc-5": [
 431         ("random-sleep", random_sleep(1, 1)),
 432         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 433         ("make", "make -j"),
 434         ("test", make_test(include_envs=[
 435             "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
 436         ("lcov", LCOV_CMD),
 437         ("check-clean-tree", "script/clean-source-tree.sh"),
 438         ],
 439
 440     "samba-ad-dc-6": [
 441         ("random-sleep", random_sleep(1, 1)),
 442         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 443         ("make", "make -j"),
 444         ("test", make_test(include_envs=["ad_dc_slowtests"])),
 445         ("lcov", LCOV_CMD),
 446         ("check-clean-tree", "script/clean-source-tree.sh"),
 447         ],
 448
 449     "samba-schemaupgrade": [
 450         ("random-sleep", random_sleep(1, 1)),
 451         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 452         ("make", "make -j"),
 453         ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 454         ("lcov", LCOV_CMD),
 455         ("check-clean-tree", "script/clean-source-tree.sh"),
 456         ],
 457
 458     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 459     # This is currently the longest task, so we don't randomly delay it.
 460     "samba-ad-dc-ntvfs": [
 461         ("random-sleep", random_sleep(1, 1)),
 462         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 463         ("make", "make -j"),
 464         ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 465         ("lcov", LCOV_CMD),
 466         ("check-clean-tree", "script/clean-source-tree.sh"),
 467         ],
 468
 469     # Test fips compliance
 470     "samba-fips": [
 471         ("random-sleep", random_sleep(100, 500)),
 472         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 473         ("make", "make -j"),
 474         ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
 475         ("lcov", LCOV_CMD),
 476         ("check-clean-tree", "script/clean-source-tree.sh"),
 477         ],
 478
 479     # run the backup/restore testenvs separately as they're fairly standalone
 480     # (and CI seems to max out at ~8 different DCs running at once)
 481     "samba-ad-dc-backup": [
 482         ("random-sleep", random_sleep(300, 900)),
 483         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 484         ("make", "make -j"),
 485         ("test", make_test(include_envs=[
 486             "backupfromdc",
 487             "restoredc",
 488             "renamedc",
 489             "offlinebackupdc",
 490             "labdc",
 491             "ad_dc_backup",
 492             ])),
 493         ("lcov", LCOV_CMD),
 494         ("check-clean-tree", "script/clean-source-tree.sh"),
 495         ],
 496
 497     "samba-admem-mit": [
 498         ("random-sleep", random_sleep(1, 1)),
 499         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 500         ("make", "make -j"),
 501         ("test", make_test(include_envs=[
 502             "ad_member",
 503             "ad_member_idmap_rid",
 504             "ad_member_idmap_ad",
 505             "ad_member_rfc2307",
 506             ])),
 507         ("lcov", LCOV_CMD),
 508         ("check-clean-tree", "script/clean-source-tree.sh"),
 509         ],
 510
 511     "samba-ad-dc-1-mitkrb5": [
 512         ("random-sleep", random_sleep(1, 1)),
 513         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 514         ("make", "make -j"),
 515         ("test", make_test(include_envs=[
 516             "ad_dc",
 517             "ad_dc_smb1",
 518             "ad_dc_smb1_done",
 519             "ad_dc_no_nss",
 520             "ad_dc_no_ntlm",
 521             ])),
 522         ("lcov", LCOV_CMD),
 523         ("check-clean-tree", "script/clean-source-tree.sh"),
 524         ],
 525
 526     "samba-ad-dc-4-mitkrb5": [
 527         ("random-sleep", random_sleep(1, 1)),
 528         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 529         ("make", "make -j"),
 530         ("test", make_test(include_envs=[
 531             "fl2000dc",
 532             "fl2003dc",
 533             "fl2008dc",
 534             "fl2008r2dc",
 535             ])),
 536         ("lcov", LCOV_CMD),
 537         ("check-clean-tree", "script/clean-source-tree.sh"),
 538         ],
 539
 540     "samba-test-only": [
 541         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params),
 542         ("make", "make -j"),
 543         ("test", make_test(TESTS="${TESTS}")),
 544         ("lcov", LCOV_CMD),
 545         ],
 546
 547     # Test cross-compile infrastructure
 548     "samba-xc": [
 549         ("random-sleep", random_sleep(900, 1500)),
 550         ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 551         ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 552             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 553         ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
 554         ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 555             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 556         ("compare-results", "script/compare_cc_results.py "
 557             "./bin/c4che/default{} "
 558             "./bin-xe/c4che/default{} "
 559             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 560         ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
 561         ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
 562             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
 563         ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
 564             " = \"'1234'\"".format(CACHE_SUFFIX)),
 565         ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
 566         ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
 567             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
 568             " ; test $? -ne 0"),
 569         ],
 570
 571     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 572     "samba-o3": [
 573         ("random-sleep", random_sleep(300, 900)),
 574         ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params),
 575         ("make", "make -j"),
 576         ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 577         ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 578         ("lcov", LCOV_CMD),
 579         ("install", "make install"),
 580         ("check-clean-tree", "script/clean-source-tree.sh"),
 581         ("clean", "make clean"),
 582         ],
 583
 584     "samba-ctdb": [
 585         ("random-sleep", random_sleep(900, 1500)),
 586
 587         # make sure we have tdb around:
 588         ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 589         ("tdb-make", "cd lib/tdb && make"),
 590         ("tdb-install", "cd lib/tdb && make install"),
 591
 592         # build samba with cluster support (also building ctdb):
 593         ("samba-configure",
 594          "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
 595          "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
 596          "./configure.developer ${PREFIX} "
 597          "--with-selftest-prefix=./bin/ab "
 598          "--with-cluster-support "
 599          "--without-ad-dc "
 600          "--bundled-libraries=!tdb"),
 601         ("samba-make", "make"),
 602         ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT"),
 603         ("samba-install", "make install"),
 604         ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 605
 606         ("test",
 607          make_test(cmd='make test',
 608                    include_envs=["clusteredmember_smb1"])
 609         ),
 610
 611         # clean up:
 612         ("check-clean-tree", "script/clean-source-tree.sh"),
 613         ("clean", "make clean"),
 614         ("ctdb-clean", "cd ./ctdb && make clean"),
 615         ],
 616
 617     "samba-libs": [
 618         ("random-sleep", random_sleep(300, 900)),
 619         ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 620         ("talloc-make", "cd lib/talloc && make"),
 621         ("talloc-install", "cd lib/talloc && make install"),
 622
 623         ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 624         ("tdb-make", "cd lib/tdb && make"),
 625         ("tdb-install", "cd lib/tdb && make install"),
 626
 627         ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 628         ("tevent-make", "cd lib/tevent && make"),
 629         ("tevent-install", "cd lib/tevent && make install"),
 630
 631         ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
 632         ("ldb-make", "cd lib/ldb && make"),
 633         ("ldb-install", "cd lib/ldb && make install"),
 634
 635         ("nondevel-configure", "./configure ${PREFIX}"),
 636         ("nondevel-make", "make -j"),
 637         ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 638         ("nondevel-install", "make install"),
 639         ("nondevel-dist", "make dist"),
 640
 641         # retry with all modules shared
 642         ("allshared-distclean", "make distclean"),
 643         ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 644         ("allshared-make", "make -j"),
 645         ],
 646
 647     "samba-static": [
 648         ("random-sleep", random_sleep(1, 1)),
 649         # build with all modules static
 650         ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
 651         ("allstatic-make", "make -j"),
 652         ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 653         ("lcov", LCOV_CMD),
 654
 655         # retry without any required modules
 656         ("none-distclean", "make distclean"),
 657         ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
 658         ("none-make", "make -j"),
 659
 660         # retry with nonshared smbd and smbtorture
 661         ("nonshared-distclean", "make distclean"),
 662         ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
 663         ("nonshared-make", "make -j")
 664         ],
 665
 666     "samba-fuzz": [
 667         # build the fuzzers (static) via the oss-fuzz script
 668         ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
 669         ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl"),
 670         ("fuzzers-check", "./lib/fuzzing/oss-fuzz/check_build.sh ${PREFIX_DIR}")
 671         ],
 672
 673     # Test Samba without python still builds.  When this test fails
 674     # due to more use of Python, the expectations is that the newly
 675     # failing part of the code should be disabled when
 676     # --disable-python is set (rather than major work being done to
 677     # support this environment).  The target here is for vendors
 678     # shipping a minimal smbd.
 679     "samba-nopython": [
 680         ("random-sleep", random_sleep(300, 900)),
 681         ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
 682         ("make", "make -j"),
 683         ("install", "make install"),
 684         ("find-python", "script/find_python.sh ${PREFIX}"),
 685         ("test", "make test-nopython"),
 686         ("lcov", LCOV_CMD),
 687         ("check-clean-tree", "script/clean-source-tree.sh"),
 688         ("clean", "make clean"),
 689
 690         ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 691         ("talloc-make", "cd lib/talloc && make"),
 692         ("talloc-install", "cd lib/talloc && make install"),
 693
 694         ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 695         ("tdb-make", "cd lib/tdb && make"),
 696         ("tdb-install", "cd lib/tdb && make install"),
 697
 698         ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 699         ("tevent-make", "cd lib/tevent && make"),
 700         ("tevent-install", "cd lib/tevent && make install"),
 701
 702         ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 703         ("ldb-make", "cd lib/ldb && make"),
 704         ("ldb-install", "cd lib/ldb && make install"),
 705
 706         # retry against installed library packages
 707         ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc"),
 708         ("libs-make", "make -j"),
 709         ("libs-install", "make install"),
 710         ("libs-check-clean-tree", "script/clean-source-tree.sh"),
 711         ("libs-clean", "make clean"),
 712         ],
 713
 714     # check we can do the same thing using python2
 715     "samba-nopython-py2": [
 716         ("random-sleep", random_sleep(300, 900)),
 717         ("configure", "PYTHON=python2 ./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
 718         ("make", "PYTHON=python2 make -j"),
 719         ("install", "PYTHON=python2 make install"),
 720         ("find-python", "script/find_python.sh ${PREFIX}"),
 721         ("test", "make test-nopython"),
 722         ("lcov", LCOV_CMD),
 723         ("check-clean-tree", "script/clean-source-tree.sh"),
 724         ("clean", "PYTHON=python2 make clean"),
 725
 726         ("talloc-configure", "cd lib/talloc && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 727         ("talloc-make", "cd lib/talloc && PYTHON=python2 make"),
 728         ("talloc-install", "cd lib/talloc && PYTHON=python2 make install"),
 729
 730         ("tdb-configure", "cd lib/tdb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 731         ("tdb-make", "cd lib/tdb && PYTHON=python2 make"),
 732         ("tdb-install", "cd lib/tdb && PYTHON=python2 make install"),
 733
 734         ("tevent-configure", "cd lib/tevent && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 735         ("tevent-make", "cd lib/tevent && PYTHON=python2 make"),
 736         ("tevent-install", "cd lib/tevent && PYTHON=python2 make install"),
 737
 738         ("ldb-configure", "cd lib/ldb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 739         ("ldb-make", "cd lib/ldb && PYTHON=python2 make"),
 740         ("ldb-install", "cd lib/ldb && PYTHON=python2 make install"),
 741
 742         # retry against installed library packages
 743         ("libs-configure", "PYTHON=python2 " + samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc"),
 744         ("libs-make", "PYTHON=python2 make -j"),
 745         ("libs-install", "PYTHON=python2 make install"),
 746         ("libs-check-clean-tree", "script/clean-source-tree.sh"),
 747         ("libs-clean", "PYTHON=python2 make clean"),
 748         ],
 749
 750     "ldb": [
 751         ("random-sleep", random_sleep(60, 600)),
 752         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 753         ("make", "make"),
 754         ("install", "make install"),
 755         ("test", "make test"),
 756         ("lcov", LCOV_CMD),
 757         ("clean", "make clean"),
 758         ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
 759         ("make-no-lmdb", "make"),
 760         ("test-no-lmdb", "make test"),
 761         ("lcov-no-lmdb", LCOV_CMD),
 762         ("install-no-lmdb", "make install"),
 763         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 764         ("distcheck", "make distcheck"),
 765         ("clean", "make clean"),
 766         ],
 767
 768     "tdb": [
 769         ("random-sleep", random_sleep(60, 600)),
 770         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 771         ("make", "make"),
 772         ("install", "make install"),
 773         ("test", "make test"),
 774         ("lcov", LCOV_CMD),
 775         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 776         ("distcheck", "make distcheck"),
 777         ("clean", "make clean"),
 778         ],
 779
 780     "talloc": [
 781         ("random-sleep", random_sleep(60, 600)),
 782         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 783         ("make", "make"),
 784         ("install", "make install"),
 785         ("test", "make test"),
 786         ("lcov", LCOV_CMD),
 787         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 788         ("distcheck", "make distcheck"),
 789         ("clean", "make clean"),
 790         ],
 791
 792     "replace": [
 793         ("random-sleep", random_sleep(60, 600)),
 794         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 795         ("make", "make"),
 796         ("install", "make install"),
 797         ("test", "make test"),
 798         ("lcov", LCOV_CMD),
 799         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 800         ("distcheck", "make distcheck"),
 801         ("clean", "make clean"),
 802         ],
 803
 804     "tevent": [
 805         ("random-sleep", random_sleep(60, 600)),
 806         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 807         ("make", "make"),
 808         ("install", "make install"),
 809         ("test", "make test"),
 810         ("lcov", LCOV_CMD),
 811         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 812         ("distcheck", "make distcheck"),
 813         ("clean", "make clean"),
 814         ],
 815
 816     "pidl": [
 817         ("random-sleep", random_sleep(60, 600)),
 818         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
 819         ("touch", "touch *.yp"),
 820         ("make", "make"),
 821         ("test", "make test"),
 822         ("install", "make install"),
 823         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
 824         ("check-clean-tree", "../script/clean-source-tree.sh"),
 825         ("clean", "make clean"),
 826         ],
 827
 828     # these are useful for debugging autobuild
 829     'pass': [("pass", 'echo passing && /bin/true')],
 830     'fail': [("fail", 'echo failing && /bin/false')],
 831 }
 832
 833 defaulttasks = list(tasks.keys())
 834
 835 defaulttasks.remove("pass")
 836 defaulttasks.remove("fail")
 837 defaulttasks.remove("samba-test-only")
 838 defaulttasks.remove("samba-fuzz")
 839 defaulttasks.remove("samba-fips")
 840 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
 841     defaulttasks.remove("samba-o3")
 842
 843
 844 def do_print(msg):
 845     print("%s" % msg)
 846     sys.stdout.flush()
 847     sys.stderr.flush()
 848
 849
 850 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 851     if show is None:
 852         show = options.verbose
 853     if show:
 854         do_print("Running: '%s' in '%s'" % (cmd, dir))
 855     if output:
 856         out = check_output([cmd], shell=True, cwd=dir)
 857         return out.decode(encoding='utf-8', errors='backslashreplace')
 858     elif checkfail:
 859         return check_call(cmd, shell=True, cwd=dir)
 860     else:
 861         return call(cmd, shell=True, cwd=dir)
 862
 863
 864 class builder(object):
 865     '''handle build of one directory'''
 866
 867     def __init__(self, name, sequence, cp=True):
 868         self.name = name
 869         self.dir = builddirs.get(name, '.')
 870         self.tag = self.name.replace('/', '_')
 871         self.sequence = sequence
 872         self.next = 0
 873         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 874         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 875         if options.verbose:
 876             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 877             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 878         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 879         self.stdout = open(self.stdout_path, 'w')
 880         self.stderr = open(self.stderr_path, 'w')
 881         self.stdin  = open("/dev/null", 'r')
 882         self.test_source_dir = "%s/%s" % (testbase, self.tag)
 883         self.cwd = "%s/%s" % (self.test_source_dir, self.dir)
 884         self.prefix = "%s/%s" % (test_prefix, self.tag)
 885         run_cmd("rm -rf %s" % self.test_source_dir)
 886         run_cmd("rm -rf %s" % self.prefix)
 887         if cp:
 888             run_cmd("cp -R -a -l %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
 889         else:
 890             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
 891         self.start_next()
 892
 893     def start_next(self):
 894         if self.next == len(self.sequence):
 895             if not options.nocleanup:
 896                 run_cmd("rm -rf %s" % self.test_source_dir)
 897                 run_cmd("rm -rf %s" % self.prefix)
 898             do_print('%s: Completed OK' % self.name)
 899             self.done = True
 900             return
 901         (self.stage, self.cmd) = self.sequence[self.next]
 902         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
 903         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 904         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 905         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 906         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
 907         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
 908         self.cmd = self.cmd.replace("${NAME}", self.name)
 909         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
 910         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
 911         self.proc = Popen(self.cmd, shell=True,
 912                           close_fds=True, cwd=self.cwd,
 913                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 914         self.next += 1
 915
 916
 917 class buildlist(object):
 918     '''handle build of multiple directories'''
 919
 920     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 921         self.tail_proc = None
 922         self.retry = None
 923         if not tasknames:
 924             if options.restrict_tests:
 925                 tasknames = ["samba-test-only"]
 926             else:
 927                 tasknames = defaulttasks
 928
 929         self.tlist = [builder(n, tasks[n], cp=(n != "pidl")) for n in tasknames]
 930
 931         if options.retry:
 932             rebase_remote = "rebaseon"
 933             retry_task = [("retry",
 934                             '''set -e
 935                             git remote add -t %s %s %s
 936                             git fetch %s
 937                             while :; do
 938                               sleep 60
 939                               git describe %s/%s > old_remote_branch.desc
 940                               git fetch %s
 941                               git describe %s/%s > remote_branch.desc
 942                               diff old_remote_branch.desc remote_branch.desc
 943                             done
 944                            ''' % (
 945                                rebase_branch, rebase_remote, rebase_url,
 946                                rebase_remote,
 947                                rebase_remote, rebase_branch,
 948                                rebase_remote,
 949                                rebase_remote, rebase_branch
 950                             ))]
 951
 952             self.retry = builder('retry', retry_task, cp=False)
 953             self.need_retry = False
 954
 955     def kill_kids(self):
 956         if self.tail_proc is not None:
 957             self.tail_proc.terminate()
 958             self.tail_proc.wait()
 959             self.tail_proc = None
 960         if self.retry is not None:
 961             self.retry.proc.terminate()
 962             self.retry.proc.wait()
 963             self.retry = None
 964         for b in self.tlist:
 965             if b.proc is not None:
 966                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
 967                 b.proc.terminate()
 968                 b.proc.wait()
 969                 b.proc = None
 970
 971     def wait_one(self):
 972         while True:
 973             none_running = True
 974             for b in self.tlist:
 975                 if b.proc is None:
 976                     continue
 977                 none_running = False
 978                 b.status = b.proc.poll()
 979                 if b.status is None:
 980                     continue
 981                 b.proc = None
 982                 return b
 983             if options.retry:
 984                 ret = self.retry.proc.poll()
 985                 if ret is not None:
 986                     self.need_retry = True
 987                     self.retry = None
 988                     return None
 989             if none_running:
 990                 return None
 991             time.sleep(0.1)
 992
 993     def run(self):
 994         while True:
 995             b = self.wait_one()
 996             if options.retry and self.need_retry:
 997                 self.kill_kids()
 998                 do_print("retry needed")
 999                 return (0, None, None, None, "retry")
1000             if b is None:
1001                 break
1002             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
1003                 self.kill_kids()
1004                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
1005             b.start_next()
1006         self.kill_kids()
1007         return (0, None, None, None, "All OK")
1008
1009     def write_system_info(self, filename):
1010         with open(filename, 'w') as f:
1011             for cmd in ['uname -a',
1012                         'lsb_release -a',
1013                         'free',
1014                         'mount',
1015                         'cat /proc/cpuinfo',
1016                         'cc --version',
1017                         'df -m .',
1018                         'df -m %s' % testbase]:
1019                 try:
1020                     out = run_cmd(cmd, output=True, checkfail=False)
1021                 except subprocess.CalledProcessError as e:
1022                     out = "<failed: %s>" % str(e)
1023                 print('### %s' % cmd, file=f)
1024                 print(out, file=f)
1025                 print(file=f)
1026
1027     def tarlogs(self, fname):
1028         with tarfile.open(fname, "w:gz") as tar:
1029             for b in self.tlist:
1030                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
1031                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
1032             if os.path.exists("autobuild.log"):
1033                 tar.add("autobuild.log")
1034             filename = 'system-info.txt'
1035             self.write_system_info(filename)
1036             tar.add(filename)
1037
1038     def remove_logs(self):
1039         for b in self.tlist:
1040             os.unlink(b.stdout_path)
1041             os.unlink(b.stderr_path)
1042
1043     def start_tail(self):
1044         cmd = ["tail", "-f"]
1045         for b in self.tlist:
1046             cmd.append(b.stdout_path)
1047             cmd.append(b.stderr_path)
1048         self.tail_proc = Popen(cmd, close_fds=True)
1049
1050
1051 def cleanup():
1052     if options.nocleanup:
1053         return
1054     run_cmd("stat %s || true" % test_tmpdir, show=True)
1055     run_cmd("stat %s" % testbase, show=True)
1056     do_print("Cleaning up %r" % cleanup_list)
1057     for d in cleanup_list:
1058         run_cmd("rm -rf %s" % d)
1059
1060
1061 def daemonize(logfile):
1062     pid = os.fork()
1063     if pid == 0:  # Parent
1064         os.setsid()
1065         pid = os.fork()
1066         if pid != 0:  # Actual daemon
1067             os._exit(0)
1068     else:  # Grandparent
1069         os._exit(0)
1070
1071     import resource      # Resource usage information.
1072     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1073     if maxfd == resource.RLIM_INFINITY:
1074         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1075     for fd in range(0, maxfd):
1076         try:
1077             os.close(fd)
1078         except OSError:
1079             pass
1080     os.open(logfile, os.O_RDWR | os.O_CREAT)
1081     os.dup2(0, 1)
1082     os.dup2(0, 2)
1083
1084
1085 def write_pidfile(fname):
1086     '''write a pid file, cleanup on exit'''
1087     with open(fname, mode='w') as f:
1088         f.write("%u\n" % os.getpid())
1089
1090
1091 def rebase_tree(rebase_url, rebase_branch="master"):
1092     rebase_remote = "rebaseon"
1093     do_print("Rebasing on %s" % rebase_url)
1094     run_cmd("git describe HEAD", show=True, dir=test_master)
1095     run_cmd("git remote add -t %s %s %s" %
1096             (rebase_branch, rebase_remote, rebase_url),
1097             show=True, dir=test_master)
1098     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1099     if options.fix_whitespace:
1100         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1101                 (rebase_remote, rebase_branch),
1102                 show=True, dir=test_master)
1103     else:
1104         run_cmd("git rebase --force-rebase %s/%s" %
1105                 (rebase_remote, rebase_branch),
1106                 show=True, dir=test_master)
1107     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1108                    (rebase_remote, rebase_branch),
1109                    dir=test_master, output=True)
1110     if diff == '':
1111         do_print("No differences between HEAD and %s/%s - exiting" %
1112                  (rebase_remote, rebase_branch))
1113         sys.exit(0)
1114     run_cmd("git describe %s/%s" %
1115             (rebase_remote, rebase_branch),
1116             show=True, dir=test_master)
1117     run_cmd("git describe HEAD", show=True, dir=test_master)
1118     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1119             (rebase_remote, rebase_branch),
1120             show=True, dir=test_master)
1121
1122
1123 def push_to(push_url, push_branch="master"):
1124     push_remote = "pushto"
1125     do_print("Pushing to %s" % push_url)
1126     if options.mark:
1127         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1128         run_cmd("git commit --amend -c HEAD", dir=test_master)
1129         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1130         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1131     run_cmd("git remote add -t %s %s %s" %
1132             (push_branch, push_remote, push_url),
1133             show=True, dir=test_master)
1134     run_cmd("git push %s +HEAD:%s" %
1135             (push_remote, push_branch),
1136             show=True, dir=test_master)
1137
1138
1139 def send_email(subject, text, log_tar):
1140     if options.email is None:
1141         do_print("not sending email because the recipient is not set")
1142         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1143                  (subject, text))
1144         return
1145     outer = MIMEMultipart()
1146     outer['Subject'] = subject
1147     outer['To'] = options.email
1148     outer['From'] = options.email_from
1149     outer['Date'] = email.utils.formatdate(localtime=True)
1150     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1151     outer.attach(MIMEText(text, 'plain'))
1152     if options.attach_logs:
1153         with open(log_tar, 'rb') as fp:
1154             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1155         # Set the filename parameter
1156         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1157         outer.attach(msg)
1158     content = outer.as_string()
1159     s = smtplib.SMTP(options.email_server)
1160     email_user = os.getenv('SMTP_USERNAME')
1161     email_password = os.getenv('SMTP_PASSWORD')
1162     if email_user is not None:
1163         s.starttls()
1164         s.login(email_user, email_password)
1165
1166     s.sendmail(options.email_from, [options.email], content)
1167     s.set_debuglevel(1)
1168     s.quit()
1169
1170
1171 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1172                   elapsed_time, log_base=None, add_log_tail=True):
1173     '''send an email to options.email about the failure'''
1174     elapsed_minutes = elapsed_time / 60.0
1175     if log_base is None:
1176         log_base = gitroot
1177     text = '''
1178 Dear Developer,
1179
1180 Your autobuild on %s failed after %.1f minutes
1181 when trying to test %s with the following error:
1182
1183    %s
1184
1185 the autobuild has been abandoned. Please fix the error and resubmit.
1186
1187 A summary of the autobuild process is here:
1188
1189   %s/autobuild.log
1190 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1191
1192     if options.restrict_tests:
1193         text += """
1194 The build was restricted to tests matching %s\n""" % options.restrict_tests
1195
1196     if failed_task != 'rebase':
1197         text += '''
1198 You can see logs of the failed task here:
1199
1200   %s/%s.stdout
1201   %s/%s.stderr
1202
1203 or you can get full logs of all tasks in this job here:
1204
1205   %s/logs.tar.gz
1206
1207 The top commit for the tree that was built was:
1208
1209 %s
1210
1211 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1212
1213     if add_log_tail:
1214         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1215         lines = f.readlines()
1216         log_tail = "".join(lines[-50:])
1217         num_lines = len(lines)
1218         if num_lines < 50:
1219             # Also include stderr (compile failures) if < 50 lines of stdout
1220             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1221             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1222
1223         text += '''
1224 The last 50 lines of log messages:
1225
1226 %s
1227     ''' % log_tail
1228         f.close()
1229
1230     logs = os.path.join(gitroot, 'logs.tar.gz')
1231     send_email('autobuild[%s] failure on %s for task %s during %s'
1232                % (options.branch, platform.node(), failed_task, failed_stage),
1233                text, logs)
1234
1235
1236 def email_success(elapsed_time, log_base=None):
1237     '''send an email to options.email about a successful build'''
1238     if log_base is None:
1239         log_base = gitroot
1240     text = '''
1241 Dear Developer,
1242
1243 Your autobuild on %s has succeeded after %.1f minutes.
1244
1245 ''' % (platform.node(), elapsed_time / 60.)
1246
1247     if options.restrict_tests:
1248         text += """
1249 The build was restricted to tests matching %s\n""" % options.restrict_tests
1250
1251     if options.keeplogs:
1252         text += '''
1253
1254 you can get full logs of all tasks in this job here:
1255
1256   %s/logs.tar.gz
1257
1258 ''' % log_base
1259
1260     text += '''
1261 The top commit for the tree that was built was:
1262
1263 %s
1264 ''' % top_commit_msg
1265
1266     logs = os.path.join(gitroot, 'logs.tar.gz')
1267     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1268                text, logs)
1269
1270
1271 # get the top commit message, for emails
1272 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1273
1274 try:
1275     os.makedirs(testbase)
1276 except Exception as reason:
1277     raise Exception("Unable to create %s : %s" % (testbase, reason))
1278 cleanup_list.append(testbase)
1279
1280 if options.daemon:
1281     logfile = os.path.join(testbase, "log")
1282     do_print("Forking into the background, writing progress to %s" % logfile)
1283     daemonize(logfile)
1284
1285 write_pidfile(gitroot + "/autobuild.pid")
1286
1287 start_time = time.time()
1288
1289 while True:
1290     try:
1291         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1292         os.makedirs(test_tmpdir)
1293         # The waf uninstall code removes empty directories all the way
1294         # up the tree.  Creating a file in test_tmpdir stops it from
1295         # being removed.
1296         run_cmd("touch %s" % os.path.join(test_tmpdir,
1297                                           ".directory-is-not-empty"), show=True)
1298         run_cmd("stat %s" % test_tmpdir, show=True)
1299         run_cmd("stat %s" % testbase, show=True)
1300         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1301     except Exception:
1302         cleanup()
1303         raise
1304
1305     try:
1306         if options.rebase is not None:
1307             rebase_tree(options.rebase, rebase_branch=options.branch)
1308     except Exception:
1309         cleanup_list.append(gitroot + "/autobuild.pid")
1310         cleanup()
1311         elapsed_time = time.time() - start_time
1312         email_failure(-1, 'rebase', 'rebase', 'rebase',
1313                       'rebase on %s failed' % options.branch,
1314                       elapsed_time, log_base=options.log_base)
1315         sys.exit(1)
1316
1317     try:
1318         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1319         if options.tail:
1320             blist.start_tail()
1321         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1322         if status != 0 or errstr != "retry":
1323             break
1324         cleanup()
1325     except Exception:
1326         cleanup()
1327         raise
1328
1329 cleanup_list.append(gitroot + "/autobuild.pid")
1330
1331 do_print(errstr)
1332
1333 blist.kill_kids()
1334 if options.tail:
1335     do_print("waiting for tail to flush")
1336     time.sleep(1)
1337
1338 elapsed_time = time.time() - start_time
1339 if status == 0:
1340     if options.passcmd is not None:
1341         do_print("Running passcmd: %s" % options.passcmd)
1342         run_cmd(options.passcmd, dir=test_master)
1343     if options.pushto is not None:
1344         push_to(options.pushto, push_branch=options.branch)
1345     if options.keeplogs or options.attach_logs:
1346         blist.tarlogs("logs.tar.gz")
1347         do_print("Logs in logs.tar.gz")
1348     if options.always_email:
1349         email_success(elapsed_time, log_base=options.log_base)
1350     blist.remove_logs()
1351     cleanup()
1352     do_print(errstr)
1353     sys.exit(0)
1354
1355 # something failed, gather a tar of the logs
1356 blist.tarlogs("logs.tar.gz")
1357
1358 if options.email is not None:
1359     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1360                   elapsed_time, log_base=options.log_base)
1361 else:
1362     elapsed_minutes = elapsed_time / 60.0
1363     print('''
1364
1365 ####################################################################
1366
1367 AUTOBUILD FAILURE
1368
1369 Your autobuild[%s] on %s failed after %.1f minutes
1370 when trying to test %s with the following error:
1371
1372    %s
1373
1374 the autobuild has been abandoned. Please fix the error and resubmit.
1375
1376 ####################################################################
1377
1378 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1379
1380 cleanup()
1381 do_print(errstr)
1382 do_print("Logs in logs.tar.gz")
1383 sys.exit(status)