script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, check_output, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 import random
  13 from optparse import OptionParser
  14 import smtplib
  15 import email
  16 from email.mime.text import MIMEText
  17 from email.mime.base import MIMEBase
  18 from email.mime.application import MIMEApplication
  19 from email.mime.multipart import MIMEMultipart
  20 from distutils.sysconfig import get_python_lib
  21 import platform
  22
  23 try:
  24     from waflib.Build import CACHE_SUFFIX
  25 except ImportError:
  26     sys.path.insert(0, "./third_party/waf")
  27     from waflib.Build import CACHE_SUFFIX
  28
  29
  30 os.environ["PYTHONUNBUFFERED"] = "1"
  31
  32 # This speeds up testing remarkably.
  33 os.environ['TDB_NO_FSYNC'] = '1'
  34
  35
  36 def find_git_root():
  37     '''get to the top of the git repo'''
  38     p = os.getcwd()
  39     while p != '/':
  40         if os.path.exists(os.path.join(p, ".git")):
  41             return p
  42         p = os.path.abspath(os.path.join(p, '..'))
  43     return None
  44
  45
  46 gitroot = find_git_root()
  47 if gitroot is None:
  48     raise Exception("Failed to find git root")
  49
  50
  51 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  52
  53 parser = OptionParser()
  54 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  55 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  56 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  57 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  58                   default=def_testbase)
  59 parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase,
  60                   default=None)
  61 parser.add_option("--passcmd", help="command to run on success", default=None)
  62 parser.add_option("--verbose", help="show all commands as they are run",
  63                   default=False, action="store_true")
  64 parser.add_option("--rebase", help="rebase on the given tree before testing",
  65                   default=None, type='str')
  66 parser.add_option("--pushto", help="push to a git url on success",
  67                   default=None, type='str')
  68 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  69                   default=False, action="store_true")
  70 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  71                   default=False, action="store_true")
  72 parser.add_option("--retry", help="automatically retry if master changes",
  73                   default=False, action="store_true")
  74 parser.add_option("--email", help="send email to the given address on failure",
  75                   type='str', default=None)
  76 parser.add_option("--email-from", help="send email from the given address",
  77                   type='str', default="autobuild@samba.org")
  78 parser.add_option("--email-server", help="send email via the given server",
  79                   type='str', default='localhost')
  80 parser.add_option("--always-email", help="always send email, even on success",
  81                   action="store_true")
  82 parser.add_option("--daemon", help="daemonize after initial setup",
  83                   action="store_true")
  84 parser.add_option("--branch", help="the branch to work on (default=master)",
  85                   default="master", type='str')
  86 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  87                   default=gitroot, type='str')
  88 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  89                   default=False, action="store_true")
  90 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
  91                   default='')
  92 parser.add_option("--enable-coverage", dest='enable_coverage',
  93                   action="store_const", const='--enable-coverage', default='',
  94                   help="Add --enable-coverage option while configure")
  95
  96 (options, args) = parser.parse_args()
  97
  98 if options.retry:
  99     if options.rebase is None:
 100         raise Exception('You can only use --retry if you also rebase')
 101
 102 if options.full_testbase is not None:
 103     testbase = options.full_testbase
 104 else:
 105     testbase = "%s/b%u" % (options.testbase, os.getpid())
 106 test_master = "%s/master" % testbase
 107 test_prefix = "%s/prefix" % testbase
 108 test_tmpdir = "%s/tmp" % testbase
 109 os.environ['TMPDIR'] = test_tmpdir
 110
 111 if options.enable_coverage:
 112     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 113 else:
 114     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 115
 116 if args:
 117     # If we are only running specific test,
 118     # do not sleep randomly to wait for it to start
 119     def random_sleep(low, high):
 120         return 'sleep 1'
 121 else:
 122     def random_sleep(low, high):
 123         return 'sleep {}'.format(random.randint(low, high))
 124
 125 cleanup_list = []
 126
 127 builddirs = {
 128     "ctdb": "ctdb",
 129     "ldb": "lib/ldb",
 130     "tdb": "lib/tdb",
 131     "talloc": "lib/talloc",
 132     "replace": "lib/replace",
 133     "tevent": "lib/tevent",
 134     "pidl": "pidl",
 135     "docs-xml": "docs-xml"
 136 }
 137
 138 ctdb_configure_params = " --enable-developer ${PREFIX}"
 139 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 140
 141 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 142 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 143 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 144 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 145 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 146 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 147 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 148
 149
 150 def format_option(name, value=None):
 151     """Format option as str list."""
 152     if value is None:  # boolean option
 153         return [name]
 154     if not isinstance(value, list):  # single value option
 155         value = [value]
 156     # repeatable option
 157     return ['{}={}'.format(name, item) for item in value]
 158
 159
 160 def make_test(
 161         cmd='make test',
 162         FAIL_IMMEDIATELY=1,
 163         TESTS='',
 164         include_envs=None,
 165         exclude_envs=None):
 166
 167     test_options = []
 168     if include_envs:
 169         test_options = format_option('--include-env', include_envs)
 170     if exclude_envs:
 171         test_options = format_option('--exclude-env', exclude_envs)
 172     if test_options:
 173         # join envs options to original test options
 174         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 175
 176     _options = []
 177     if FAIL_IMMEDIATELY:
 178         _options.append('FAIL_IMMEDIATELY=1')
 179     if TESTS:
 180         _options.append("TESTS='{}'".format(TESTS))
 181
 182     return ' '.join([cmd] + _options)
 183
 184
 185 # When updating this list, also update .gitlab-ci.yml to add the job
 186 # and to make it a dependency of 'page' for the coverage report.
 187
 188 tasks = {
 189     "ctdb": [
 190         ("random-sleep", random_sleep(300, 900)),
 191         ("configure", "./configure " + ctdb_configure_params),
 192         ("make", "make all"),
 193         ("install", "make install"),
 194         ("test", "make autotest"),
 195         ("check-clean-tree", "../script/clean-source-tree.sh"),
 196         ("clean", "make clean"),
 197         ],
 198
 199     "docs-xml": [
 200         ("random-sleep", random_sleep(300, 900)),
 201         ("autoconf", "autoconf"),
 202         ("configure", "./configure"),
 203         ("make", "make html htmlman"),
 204         ("clean", "make clean"),
 205         ],
 206
 207     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 208     "samba": [
 209         ("random-sleep", random_sleep(300, 900)),
 210         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 211         ("make", "make -j"),
 212         ("test", make_test(exclude_envs=[
 213             "none",
 214             "nt4_dc",
 215             "nt4_dc_smb1",
 216             "nt4_dc_smb1_done",
 217             "nt4_dc_schannel",
 218             "nt4_member",
 219             "ad_dc",
 220             "ad_dc_smb1",
 221             "ad_dc_smb1_done",
 222             "ad_dc_backup",
 223             "ad_dc_ntvfs",
 224             "ad_dc_default",
 225             "ad_dc_default_smb1",
 226             "ad_dc_slowtests",
 227             "ad_dc_no_nss",
 228             "ad_dc_no_ntlm",
 229             "fl2003dc",
 230             "fl2008dc",
 231             "fl2008r2dc",
 232             "ad_member",
 233             "ad_member_idmap_rid",
 234             "ad_member_idmap_ad",
 235             "ad_member_rfc2307",
 236             "chgdcpass",
 237             "vampire_2000_dc",
 238             "fl2000dc",
 239             "fileserver",
 240             "fileserver_smb1",
 241             "fileserver_smb1_done",
 242             "maptoguest",
 243             "simpleserver",
 244             "backupfromdc",
 245             "restoredc",
 246             "renamedc",
 247             "offlinebackupdc",
 248             "labdc",
 249             "preforkrestartdc",
 250             "proclimitdc",
 251             "promoted_dc",
 252             "vampire_dc",
 253             "rodc",
 254             "ad_dc_default",
 255             "ad_dc_default_smb1",
 256             "ad_dc_default_smb1_done",
 257             "ad_dc_slowtests",
 258             "schema_pair_dc",
 259             "schema_dc",
 260             "clusteredmember_smb1",
 261             ])),
 262         ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
 263         ("lcov", LCOV_CMD),
 264         ("install", "make install"),
 265         ("check-clean-tree", "script/clean-source-tree.sh"),
 266         ("clean", "make clean"),
 267         ],
 268
 269     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 270     "samba-mitkrb5": [
 271         ("random-sleep", random_sleep(300, 900)),
 272         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 273         ("make", "make -j"),
 274         ("test", make_test(exclude_envs=[
 275             "none",
 276             "nt4_dc",
 277             "nt4_dc_smb1",
 278             "nt4_dc_smb1_done",
 279             "nt4_dc_schannel",
 280             "nt4_member",
 281             "ad_dc",
 282             "ad_dc_smb1",
 283             "ad_dc_smb1_done",
 284             "ad_dc_backup",
 285             "ad_dc_ntvfs",
 286             "ad_dc_default",
 287             "ad_dc_default_smb1",
 288             "ad_dc_default_smb1_done",
 289             "ad_dc_slowtests",
 290             "ad_dc_no_nss",
 291             "ad_dc_no_ntlm",
 292             "fl2003dc",
 293             "fl2008dc",
 294             "fl2008r2dc",
 295             "ad_member",
 296             "ad_member_idmap_rid",
 297             "ad_member_idmap_ad",
 298             "ad_member_rfc2307",
 299             "chgdcpass",
 300             "vampire_2000_dc",
 301             "fl2000dc",
 302             "fileserver",
 303             "fileserver_smb1",
 304             "fileserver_smb1_done",
 305             "maptoguest",
 306             "simpleserver",
 307             "backupfromdc",
 308             "restoredc",
 309             "renamedc",
 310             "offlinebackupdc",
 311             "labdc",
 312             "preforkrestartdc",
 313             "proclimitdc",
 314             "promoted_dc",
 315             "vampire_dc",
 316             "rodc",
 317             "ad_dc_default",
 318             "ad_dc_default_smb1",
 319             "ad_dc_default_smb1_done",
 320             "ad_dc_slowtests",
 321             "schema_pair_dc",
 322             "schema_dc",
 323             "clusteredmember_smb1",
 324             ])),
 325         ("lcov", LCOV_CMD),
 326         ("install", "make install"),
 327         ("check-clean-tree", "script/clean-source-tree.sh"),
 328         ("clean", "make clean"),
 329         ],
 330
 331     "samba-nt4": [
 332         ("random-sleep", random_sleep(300, 900)),
 333         ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params),
 334         ("make", "make -j"),
 335         ("test", make_test(include_envs=[
 336             "nt4_dc",
 337             "nt4_dc_smb1",
 338             "nt4_dc_smb1_done",
 339             "nt4_dc_schannel",
 340             "nt4_member",
 341             "simpleserver",
 342             ])),
 343         ("lcov", LCOV_CMD),
 344         ("check-clean-tree", "script/clean-source-tree.sh"),
 345         ("clean", "make clean"),
 346         ],
 347
 348     "samba-fileserver": [
 349         ("random-sleep", random_sleep(300, 900)),
 350         ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5 --with-selftest-prefix=./bin/ab" + samba_configure_params),
 351         ("make", "make -j"),
 352         ("test", make_test(include_envs=[
 353             "fileserver",
 354             "fileserver_smb1",
 355             "fileserver_smb1_done",
 356             "maptoguest",
 357             "ktest", # ktest is also tested in samba and samba-mitkrb5
 358                      # but is tested here against a system Heimdal
 359             ])),
 360         ("lcov", LCOV_CMD),
 361         ("check-clean-tree", "script/clean-source-tree.sh"),
 362         ],
 363
 364     "samba-admem": [
 365         ("random-sleep", random_sleep(300, 900)),
 366         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 367         ("make", "make -j"),
 368         ("test", make_test(include_envs=[
 369             "ad_member",
 370             "ad_member_idmap_rid",
 371             "ad_member_idmap_ad",
 372             "ad_member_rfc2307",
 373             ])),
 374         ("lcov", LCOV_CMD),
 375         ("check-clean-tree", "script/clean-source-tree.sh"),
 376         ],
 377
 378     "samba-ad-dc-1": [
 379         ("random-sleep", random_sleep(1, 1)),
 380         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 381         ("make", "make -j"),
 382         ("test", make_test(include_envs=[
 383             "ad_dc",
 384             "ad_dc_smb1",
 385             "ad_dc_smb1_done",
 386             "ad_dc_no_nss",
 387             "ad_dc_no_ntlm",
 388             ])),
 389         ("lcov", LCOV_CMD),
 390         ("check-clean-tree", "script/clean-source-tree.sh"),
 391         ],
 392
 393     "samba-ad-dc-2": [
 394         ("random-sleep", random_sleep(1, 1)),
 395         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 396         ("make", "make -j"),
 397         ("test", make_test(include_envs=[
 398             "vampire_dc",
 399             "vampire_2000_dc",
 400             "rodc",
 401             ])),
 402         ("lcov", LCOV_CMD),
 403         ("check-clean-tree", "script/clean-source-tree.sh"),
 404         ],
 405
 406     "samba-ad-dc-3": [
 407         ("random-sleep", random_sleep(1, 1)),
 408         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 409         ("make", "make -j"),
 410         ("test", make_test(include_envs=[
 411             "promoted_dc",
 412             "chgdcpass",
 413             "preforkrestartdc",
 414             "proclimitdc",
 415             ])),
 416         ("lcov", LCOV_CMD),
 417         ("check-clean-tree", "script/clean-source-tree.sh"),
 418         ],
 419
 420     "samba-ad-dc-4": [
 421         ("random-sleep", random_sleep(1, 1)),
 422         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 423         ("make", "make -j"),
 424         ("test", make_test(include_envs=[
 425             "fl2000dc",
 426             "fl2003dc",
 427             "fl2008dc",
 428             "fl2008r2dc",
 429             ])),
 430         ("lcov", LCOV_CMD),
 431         ("check-clean-tree", "script/clean-source-tree.sh"),
 432         ],
 433
 434     "samba-ad-dc-5": [
 435         ("random-sleep", random_sleep(1, 1)),
 436         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 437         ("make", "make -j"),
 438         ("test", make_test(include_envs=[
 439             "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
 440         ("lcov", LCOV_CMD),
 441         ("check-clean-tree", "script/clean-source-tree.sh"),
 442         ],
 443
 444     "samba-ad-dc-6": [
 445         ("random-sleep", random_sleep(1, 1)),
 446         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 447         ("make", "make -j"),
 448         ("test", make_test(include_envs=["ad_dc_slowtests"])),
 449         ("lcov", LCOV_CMD),
 450         ("check-clean-tree", "script/clean-source-tree.sh"),
 451         ],
 452
 453     "samba-schemaupgrade": [
 454         ("random-sleep", random_sleep(1, 1)),
 455         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 456         ("make", "make -j"),
 457         ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 458         ("lcov", LCOV_CMD),
 459         ("check-clean-tree", "script/clean-source-tree.sh"),
 460         ],
 461
 462     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 463     # This is currently the longest task, so we don't randomly delay it.
 464     "samba-ad-dc-ntvfs": [
 465         ("random-sleep", random_sleep(1, 1)),
 466         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 467         ("make", "make -j"),
 468         ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 469         ("lcov", LCOV_CMD),
 470         ("check-clean-tree", "script/clean-source-tree.sh"),
 471         ],
 472
 473     # Test fips compliance
 474     "samba-fips": [
 475         ("random-sleep", random_sleep(100, 500)),
 476         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 477         ("make", "make -j"),
 478         ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
 479         ("lcov", LCOV_CMD),
 480         ("check-clean-tree", "script/clean-source-tree.sh"),
 481         ],
 482
 483     # run the backup/restore testenvs separately as they're fairly standalone
 484     # (and CI seems to max out at ~8 different DCs running at once)
 485     "samba-ad-dc-backup": [
 486         ("random-sleep", random_sleep(300, 900)),
 487         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 488         ("make", "make -j"),
 489         ("test", make_test(include_envs=[
 490             "backupfromdc",
 491             "restoredc",
 492             "renamedc",
 493             "offlinebackupdc",
 494             "labdc",
 495             "ad_dc_backup",
 496             ])),
 497         ("lcov", LCOV_CMD),
 498         ("check-clean-tree", "script/clean-source-tree.sh"),
 499         ],
 500
 501     "samba-admem-mit": [
 502         ("random-sleep", random_sleep(1, 1)),
 503         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 504         ("make", "make -j"),
 505         ("test", make_test(include_envs=[
 506             "ad_member",
 507             "ad_member_idmap_rid",
 508             "ad_member_idmap_ad",
 509             "ad_member_rfc2307",
 510             ])),
 511         ("lcov", LCOV_CMD),
 512         ("check-clean-tree", "script/clean-source-tree.sh"),
 513         ],
 514
 515     "samba-ad-dc-1-mitkrb5": [
 516         ("random-sleep", random_sleep(1, 1)),
 517         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 518         ("make", "make -j"),
 519         ("test", make_test(include_envs=[
 520             "ad_dc",
 521             "ad_dc_smb1",
 522             "ad_dc_smb1_done",
 523             "ad_dc_no_nss",
 524             "ad_dc_no_ntlm",
 525             ])),
 526         ("lcov", LCOV_CMD),
 527         ("check-clean-tree", "script/clean-source-tree.sh"),
 528         ],
 529
 530     "samba-ad-dc-4-mitkrb5": [
 531         ("random-sleep", random_sleep(1, 1)),
 532         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 533         ("make", "make -j"),
 534         ("test", make_test(include_envs=[
 535             "fl2000dc",
 536             "fl2003dc",
 537             "fl2008dc",
 538             "fl2008r2dc",
 539             ])),
 540         ("lcov", LCOV_CMD),
 541         ("check-clean-tree", "script/clean-source-tree.sh"),
 542         ],
 543
 544     "samba-test-only": [
 545         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params),
 546         ("make", "make -j"),
 547         ("test", make_test(TESTS="${TESTS}")),
 548         ("lcov", LCOV_CMD),
 549         ],
 550
 551     # Test cross-compile infrastructure
 552     "samba-xc": [
 553         ("random-sleep", random_sleep(900, 1500)),
 554         ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 555         ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 556             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 557         ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
 558         ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 559             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 560         ("compare-results", "script/compare_cc_results.py "
 561             "./bin/c4che/default{} "
 562             "./bin-xe/c4che/default{} "
 563             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 564         ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
 565         ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
 566             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
 567         ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
 568             " = \"'1234'\"".format(CACHE_SUFFIX)),
 569         ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
 570         ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
 571             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
 572             " ; test $? -ne 0"),
 573         ],
 574
 575     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 576     "samba-o3": [
 577         ("random-sleep", random_sleep(300, 900)),
 578         ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params),
 579         ("make", "make -j"),
 580         ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 581         ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 582         ("lcov", LCOV_CMD),
 583         ("install", "make install"),
 584         ("check-clean-tree", "script/clean-source-tree.sh"),
 585         ("clean", "make clean"),
 586         ],
 587
 588     "samba-ctdb": [
 589         ("random-sleep", random_sleep(900, 1500)),
 590
 591         # make sure we have tdb around:
 592         ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 593         ("tdb-make", "cd lib/tdb && make"),
 594         ("tdb-install", "cd lib/tdb && make install"),
 595
 596         # build samba with cluster support (also building ctdb):
 597         ("samba-configure",
 598          "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
 599          "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
 600          "./configure.developer ${PREFIX} "
 601          "--with-selftest-prefix=./bin/ab "
 602          "--with-cluster-support "
 603          "--without-ad-dc "
 604          "--bundled-libraries=!tdb"),
 605         ("samba-make", "make"),
 606         ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT"),
 607         ("samba-install", "make install"),
 608         ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 609
 610         ("test",
 611          make_test(cmd='make test',
 612                    include_envs=["clusteredmember_smb1"])
 613         ),
 614
 615         # clean up:
 616         ("check-clean-tree", "script/clean-source-tree.sh"),
 617         ("clean", "make clean"),
 618         ("ctdb-clean", "cd ./ctdb && make clean"),
 619         ],
 620
 621     "samba-libs": [
 622         ("random-sleep", random_sleep(300, 900)),
 623         ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 624         ("talloc-make", "cd lib/talloc && make"),
 625         ("talloc-install", "cd lib/talloc && make install"),
 626
 627         ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 628         ("tdb-make", "cd lib/tdb && make"),
 629         ("tdb-install", "cd lib/tdb && make install"),
 630
 631         ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 632         ("tevent-make", "cd lib/tevent && make"),
 633         ("tevent-install", "cd lib/tevent && make install"),
 634
 635         ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
 636         ("ldb-make", "cd lib/ldb && make"),
 637         ("ldb-install", "cd lib/ldb && make install"),
 638
 639         ("nondevel-configure", "./configure ${PREFIX}"),
 640         ("nondevel-make", "make -j"),
 641         ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 642         ("nondevel-install", "make install"),
 643         ("nondevel-dist", "make dist"),
 644
 645         # retry with all modules shared
 646         ("allshared-distclean", "make distclean"),
 647         ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 648         ("allshared-make", "make -j"),
 649         ],
 650
 651     "samba-fuzz": [
 652         # build the fuzzers (static) via the oss-fuzz script
 653         ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
 654         ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl"),
 655         ("fuzzers-check", "./lib/fuzzing/oss-fuzz/check_build.sh ${PREFIX_DIR}")
 656         ],
 657
 658     # * Test smbd and smbtorture can build semi-static
 659     #
 660     # * Test Samba without python still builds.
 661     #
 662     # When this test fails due to more use of Python, the expectations
 663     # is that the newly failing part of the code should be disabled
 664     # when --disable-python is set (rather than major work being done
 665     # to support this environment).
 666     #
 667     # The target here is for vendors shipping a minimal smbd.
 668     "samba-minimal-smbd": [
 669         ("random-sleep", random_sleep(300, 900)),
 670
 671         # build with all modules static
 672         ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
 673         ("allstatic-make", "make -j"),
 674         ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 675         ("lcov", LCOV_CMD),
 676
 677         # retry with nonshared smbd and smbtorture
 678         ("nonshared-distclean", "make distclean"),
 679         ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
 680         ("nonshared-make", "make -j"),
 681
 682         ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
 683         ("make", "make -j"),
 684         ("find-python", "script/find_python.sh ${PREFIX}"),
 685         ("test", "make test-nopython"),
 686         ("lcov", LCOV_CMD),
 687         ("check-clean-tree", "script/clean-source-tree.sh"),
 688         ("clean", "make clean"),
 689
 690         ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 691         ("talloc-make", "cd lib/talloc && make"),
 692         ("talloc-install", "cd lib/talloc && make install"),
 693
 694         ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 695         ("tdb-make", "cd lib/tdb && make"),
 696         ("tdb-install", "cd lib/tdb && make install"),
 697
 698         ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 699         ("tevent-make", "cd lib/tevent && make"),
 700         ("tevent-install", "cd lib/tevent && make install"),
 701
 702         ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 703         ("ldb-make", "cd lib/ldb && make"),
 704         ("ldb-install", "cd lib/ldb && make install"),
 705
 706         # retry against installed library packages, but no required modules
 707         ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc  --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
 708         ("libs-make", "make -j"),
 709         ("libs-install", "make install"),
 710         ("libs-check-clean-tree", "script/clean-source-tree.sh"),
 711         ("libs-clean", "make clean"),
 712
 713         ],
 714
 715     "ldb": [
 716         ("random-sleep", random_sleep(60, 600)),
 717         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 718         ("make", "make"),
 719         ("install", "make install"),
 720         ("test", "make test"),
 721         ("lcov", LCOV_CMD),
 722         ("clean", "make clean"),
 723         ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
 724         ("make-no-lmdb", "make"),
 725         ("test-no-lmdb", "make test"),
 726         ("lcov-no-lmdb", LCOV_CMD),
 727         ("install-no-lmdb", "make install"),
 728         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 729         ("distcheck", "make distcheck"),
 730         ("clean", "make clean"),
 731         ],
 732
 733     "tdb": [
 734         ("random-sleep", random_sleep(60, 600)),
 735         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 736         ("make", "make"),
 737         ("install", "make install"),
 738         ("test", "make test"),
 739         ("lcov", LCOV_CMD),
 740         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 741         ("distcheck", "make distcheck"),
 742         ("clean", "make clean"),
 743         ],
 744
 745     "talloc": [
 746         ("random-sleep", random_sleep(60, 600)),
 747         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 748         ("make", "make"),
 749         ("install", "make install"),
 750         ("test", "make test"),
 751         ("lcov", LCOV_CMD),
 752         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 753         ("distcheck", "make distcheck"),
 754         ("clean", "make clean"),
 755         ],
 756
 757     "replace": [
 758         ("random-sleep", random_sleep(60, 600)),
 759         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 760         ("make", "make"),
 761         ("install", "make install"),
 762         ("test", "make test"),
 763         ("lcov", LCOV_CMD),
 764         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 765         ("distcheck", "make distcheck"),
 766         ("clean", "make clean"),
 767         ],
 768
 769     "tevent": [
 770         ("random-sleep", random_sleep(60, 600)),
 771         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 772         ("make", "make"),
 773         ("install", "make install"),
 774         ("test", "make test"),
 775         ("lcov", LCOV_CMD),
 776         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 777         ("distcheck", "make distcheck"),
 778         ("clean", "make clean"),
 779         ],
 780
 781     "pidl": [
 782         ("random-sleep", random_sleep(60, 600)),
 783         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
 784         ("touch", "touch *.yp"),
 785         ("make", "make"),
 786         ("test", "make test"),
 787         ("install", "make install"),
 788         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
 789         ("check-clean-tree", "../script/clean-source-tree.sh"),
 790         ("clean", "make clean"),
 791         ],
 792
 793     # these are useful for debugging autobuild
 794     'pass': [("pass", 'echo passing && /bin/true')],
 795     'fail': [("fail", 'echo failing && /bin/false')],
 796 }
 797
 798 defaulttasks = list(tasks.keys())
 799
 800 defaulttasks.remove("pass")
 801 defaulttasks.remove("fail")
 802 defaulttasks.remove("samba-test-only")
 803 defaulttasks.remove("samba-fuzz")
 804 defaulttasks.remove("samba-fips")
 805 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
 806     defaulttasks.remove("samba-o3")
 807
 808
 809 def do_print(msg):
 810     print("%s" % msg)
 811     sys.stdout.flush()
 812     sys.stderr.flush()
 813
 814
 815 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 816     if show is None:
 817         show = options.verbose
 818     if show:
 819         do_print("Running: '%s' in '%s'" % (cmd, dir))
 820     if output:
 821         out = check_output([cmd], shell=True, cwd=dir)
 822         return out.decode(encoding='utf-8', errors='backslashreplace')
 823     elif checkfail:
 824         return check_call(cmd, shell=True, cwd=dir)
 825     else:
 826         return call(cmd, shell=True, cwd=dir)
 827
 828
 829 class builder(object):
 830     '''handle build of one directory'''
 831
 832     def __init__(self, name, sequence, cp=True):
 833         self.name = name
 834         self.dir = builddirs.get(name, '.')
 835         self.tag = self.name.replace('/', '_')
 836         self.sequence = sequence
 837         self.next = 0
 838         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 839         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 840         if options.verbose:
 841             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 842             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 843         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 844         self.stdout = open(self.stdout_path, 'w')
 845         self.stderr = open(self.stderr_path, 'w')
 846         self.stdin  = open("/dev/null", 'r')
 847         self.test_source_dir = "%s/%s" % (testbase, self.tag)
 848         self.cwd = "%s/%s" % (self.test_source_dir, self.dir)
 849         self.prefix = "%s/%s" % (test_prefix, self.tag)
 850         run_cmd("rm -rf %s" % self.test_source_dir)
 851         run_cmd("rm -rf %s" % self.prefix)
 852         if cp:
 853             run_cmd("cp -R -a -l %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
 854         else:
 855             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
 856         self.start_next()
 857
 858     def start_next(self):
 859         if self.next == len(self.sequence):
 860             if not options.nocleanup:
 861                 run_cmd("rm -rf %s" % self.test_source_dir)
 862                 run_cmd("rm -rf %s" % self.prefix)
 863             do_print('%s: Completed OK' % self.name)
 864             self.done = True
 865             return
 866         (self.stage, self.cmd) = self.sequence[self.next]
 867         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
 868         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 869         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 870         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 871         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
 872         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
 873         self.cmd = self.cmd.replace("${NAME}", self.name)
 874         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
 875         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
 876         self.proc = Popen(self.cmd, shell=True,
 877                           close_fds=True, cwd=self.cwd,
 878                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 879         self.next += 1
 880
 881
 882 class buildlist(object):
 883     '''handle build of multiple directories'''
 884
 885     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 886         self.tail_proc = None
 887         self.retry = None
 888         if not tasknames:
 889             if options.restrict_tests:
 890                 tasknames = ["samba-test-only"]
 891             else:
 892                 tasknames = defaulttasks
 893
 894         self.tlist = [builder(n, tasks[n], cp=(n != "pidl")) for n in tasknames]
 895
 896         if options.retry:
 897             rebase_remote = "rebaseon"
 898             retry_task = [("retry",
 899                             '''set -e
 900                             git remote add -t %s %s %s
 901                             git fetch %s
 902                             while :; do
 903                               sleep 60
 904                               git describe %s/%s > old_remote_branch.desc
 905                               git fetch %s
 906                               git describe %s/%s > remote_branch.desc
 907                               diff old_remote_branch.desc remote_branch.desc
 908                             done
 909                            ''' % (
 910                                rebase_branch, rebase_remote, rebase_url,
 911                                rebase_remote,
 912                                rebase_remote, rebase_branch,
 913                                rebase_remote,
 914                                rebase_remote, rebase_branch
 915                             ))]
 916
 917             self.retry = builder('retry', retry_task, cp=False)
 918             self.need_retry = False
 919
 920     def kill_kids(self):
 921         if self.tail_proc is not None:
 922             self.tail_proc.terminate()
 923             self.tail_proc.wait()
 924             self.tail_proc = None
 925         if self.retry is not None:
 926             self.retry.proc.terminate()
 927             self.retry.proc.wait()
 928             self.retry = None
 929         for b in self.tlist:
 930             if b.proc is not None:
 931                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
 932                 b.proc.terminate()
 933                 b.proc.wait()
 934                 b.proc = None
 935
 936     def wait_one(self):
 937         while True:
 938             none_running = True
 939             for b in self.tlist:
 940                 if b.proc is None:
 941                     continue
 942                 none_running = False
 943                 b.status = b.proc.poll()
 944                 if b.status is None:
 945                     continue
 946                 b.proc = None
 947                 return b
 948             if options.retry:
 949                 ret = self.retry.proc.poll()
 950                 if ret is not None:
 951                     self.need_retry = True
 952                     self.retry = None
 953                     return None
 954             if none_running:
 955                 return None
 956             time.sleep(0.1)
 957
 958     def run(self):
 959         while True:
 960             b = self.wait_one()
 961             if options.retry and self.need_retry:
 962                 self.kill_kids()
 963                 do_print("retry needed")
 964                 return (0, None, None, None, "retry")
 965             if b is None:
 966                 break
 967             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 968                 self.kill_kids()
 969                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 970             b.start_next()
 971         self.kill_kids()
 972         return (0, None, None, None, "All OK")
 973
 974     def write_system_info(self, filename):
 975         with open(filename, 'w') as f:
 976             for cmd in ['uname -a',
 977                         'lsb_release -a',
 978                         'free',
 979                         'mount',
 980                         'cat /proc/cpuinfo',
 981                         'cc --version',
 982                         'df -m .',
 983                         'df -m %s' % testbase]:
 984                 try:
 985                     out = run_cmd(cmd, output=True, checkfail=False)
 986                 except subprocess.CalledProcessError as e:
 987                     out = "<failed: %s>" % str(e)
 988                 print('### %s' % cmd, file=f)
 989                 print(out, file=f)
 990                 print(file=f)
 991
 992     def tarlogs(self, fname):
 993         with tarfile.open(fname, "w:gz") as tar:
 994             for b in self.tlist:
 995                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 996                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 997             if os.path.exists("autobuild.log"):
 998                 tar.add("autobuild.log")
 999             filename = 'system-info.txt'
1000             self.write_system_info(filename)
1001             tar.add(filename)
1002
1003     def remove_logs(self):
1004         for b in self.tlist:
1005             os.unlink(b.stdout_path)
1006             os.unlink(b.stderr_path)
1007
1008     def start_tail(self):
1009         cmd = ["tail", "-f"]
1010         for b in self.tlist:
1011             cmd.append(b.stdout_path)
1012             cmd.append(b.stderr_path)
1013         self.tail_proc = Popen(cmd, close_fds=True)
1014
1015
1016 def cleanup():
1017     if options.nocleanup:
1018         return
1019     run_cmd("stat %s || true" % test_tmpdir, show=True)
1020     run_cmd("stat %s" % testbase, show=True)
1021     do_print("Cleaning up %r" % cleanup_list)
1022     for d in cleanup_list:
1023         run_cmd("rm -rf %s" % d)
1024
1025
1026 def daemonize(logfile):
1027     pid = os.fork()
1028     if pid == 0:  # Parent
1029         os.setsid()
1030         pid = os.fork()
1031         if pid != 0:  # Actual daemon
1032             os._exit(0)
1033     else:  # Grandparent
1034         os._exit(0)
1035
1036     import resource      # Resource usage information.
1037     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1038     if maxfd == resource.RLIM_INFINITY:
1039         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1040     for fd in range(0, maxfd):
1041         try:
1042             os.close(fd)
1043         except OSError:
1044             pass
1045     os.open(logfile, os.O_RDWR | os.O_CREAT)
1046     os.dup2(0, 1)
1047     os.dup2(0, 2)
1048
1049
1050 def write_pidfile(fname):
1051     '''write a pid file, cleanup on exit'''
1052     with open(fname, mode='w') as f:
1053         f.write("%u\n" % os.getpid())
1054
1055
1056 def rebase_tree(rebase_url, rebase_branch="master"):
1057     rebase_remote = "rebaseon"
1058     do_print("Rebasing on %s" % rebase_url)
1059     run_cmd("git describe HEAD", show=True, dir=test_master)
1060     run_cmd("git remote add -t %s %s %s" %
1061             (rebase_branch, rebase_remote, rebase_url),
1062             show=True, dir=test_master)
1063     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1064     if options.fix_whitespace:
1065         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1066                 (rebase_remote, rebase_branch),
1067                 show=True, dir=test_master)
1068     else:
1069         run_cmd("git rebase --force-rebase %s/%s" %
1070                 (rebase_remote, rebase_branch),
1071                 show=True, dir=test_master)
1072     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1073                    (rebase_remote, rebase_branch),
1074                    dir=test_master, output=True)
1075     if diff == '':
1076         do_print("No differences between HEAD and %s/%s - exiting" %
1077                  (rebase_remote, rebase_branch))
1078         sys.exit(0)
1079     run_cmd("git describe %s/%s" %
1080             (rebase_remote, rebase_branch),
1081             show=True, dir=test_master)
1082     run_cmd("git describe HEAD", show=True, dir=test_master)
1083     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1084             (rebase_remote, rebase_branch),
1085             show=True, dir=test_master)
1086
1087
1088 def push_to(push_url, push_branch="master"):
1089     push_remote = "pushto"
1090     do_print("Pushing to %s" % push_url)
1091     if options.mark:
1092         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1093         run_cmd("git commit --amend -c HEAD", dir=test_master)
1094         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1095         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1096     run_cmd("git remote add -t %s %s %s" %
1097             (push_branch, push_remote, push_url),
1098             show=True, dir=test_master)
1099     run_cmd("git push %s +HEAD:%s" %
1100             (push_remote, push_branch),
1101             show=True, dir=test_master)
1102
1103
1104 def send_email(subject, text, log_tar):
1105     if options.email is None:
1106         do_print("not sending email because the recipient is not set")
1107         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1108                  (subject, text))
1109         return
1110     outer = MIMEMultipart()
1111     outer['Subject'] = subject
1112     outer['To'] = options.email
1113     outer['From'] = options.email_from
1114     outer['Date'] = email.utils.formatdate(localtime=True)
1115     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1116     outer.attach(MIMEText(text, 'plain'))
1117     if options.attach_logs:
1118         with open(log_tar, 'rb') as fp:
1119             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1120         # Set the filename parameter
1121         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1122         outer.attach(msg)
1123     content = outer.as_string()
1124     s = smtplib.SMTP(options.email_server)
1125     email_user = os.getenv('SMTP_USERNAME')
1126     email_password = os.getenv('SMTP_PASSWORD')
1127     if email_user is not None:
1128         s.starttls()
1129         s.login(email_user, email_password)
1130
1131     s.sendmail(options.email_from, [options.email], content)
1132     s.set_debuglevel(1)
1133     s.quit()
1134
1135
1136 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1137                   elapsed_time, log_base=None, add_log_tail=True):
1138     '''send an email to options.email about the failure'''
1139     elapsed_minutes = elapsed_time / 60.0
1140     if log_base is None:
1141         log_base = gitroot
1142     text = '''
1143 Dear Developer,
1144
1145 Your autobuild on %s failed after %.1f minutes
1146 when trying to test %s with the following error:
1147
1148    %s
1149
1150 the autobuild has been abandoned. Please fix the error and resubmit.
1151
1152 A summary of the autobuild process is here:
1153
1154   %s/autobuild.log
1155 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1156
1157     if options.restrict_tests:
1158         text += """
1159 The build was restricted to tests matching %s\n""" % options.restrict_tests
1160
1161     if failed_task != 'rebase':
1162         text += '''
1163 You can see logs of the failed task here:
1164
1165   %s/%s.stdout
1166   %s/%s.stderr
1167
1168 or you can get full logs of all tasks in this job here:
1169
1170   %s/logs.tar.gz
1171
1172 The top commit for the tree that was built was:
1173
1174 %s
1175
1176 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1177
1178     if add_log_tail:
1179         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1180         lines = f.readlines()
1181         log_tail = "".join(lines[-50:])
1182         num_lines = len(lines)
1183         if num_lines < 50:
1184             # Also include stderr (compile failures) if < 50 lines of stdout
1185             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1186             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1187
1188         text += '''
1189 The last 50 lines of log messages:
1190
1191 %s
1192     ''' % log_tail
1193         f.close()
1194
1195     logs = os.path.join(gitroot, 'logs.tar.gz')
1196     send_email('autobuild[%s] failure on %s for task %s during %s'
1197                % (options.branch, platform.node(), failed_task, failed_stage),
1198                text, logs)
1199
1200
1201 def email_success(elapsed_time, log_base=None):
1202     '''send an email to options.email about a successful build'''
1203     if log_base is None:
1204         log_base = gitroot
1205     text = '''
1206 Dear Developer,
1207
1208 Your autobuild on %s has succeeded after %.1f minutes.
1209
1210 ''' % (platform.node(), elapsed_time / 60.)
1211
1212     if options.restrict_tests:
1213         text += """
1214 The build was restricted to tests matching %s\n""" % options.restrict_tests
1215
1216     if options.keeplogs:
1217         text += '''
1218
1219 you can get full logs of all tasks in this job here:
1220
1221   %s/logs.tar.gz
1222
1223 ''' % log_base
1224
1225     text += '''
1226 The top commit for the tree that was built was:
1227
1228 %s
1229 ''' % top_commit_msg
1230
1231     logs = os.path.join(gitroot, 'logs.tar.gz')
1232     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1233                text, logs)
1234
1235
1236 # get the top commit message, for emails
1237 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1238
1239 try:
1240     os.makedirs(testbase)
1241 except Exception as reason:
1242     raise Exception("Unable to create %s : %s" % (testbase, reason))
1243 cleanup_list.append(testbase)
1244
1245 if options.daemon:
1246     logfile = os.path.join(testbase, "log")
1247     do_print("Forking into the background, writing progress to %s" % logfile)
1248     daemonize(logfile)
1249
1250 write_pidfile(gitroot + "/autobuild.pid")
1251
1252 start_time = time.time()
1253
1254 while True:
1255     try:
1256         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1257         os.makedirs(test_tmpdir)
1258         # The waf uninstall code removes empty directories all the way
1259         # up the tree.  Creating a file in test_tmpdir stops it from
1260         # being removed.
1261         run_cmd("touch %s" % os.path.join(test_tmpdir,
1262                                           ".directory-is-not-empty"), show=True)
1263         run_cmd("stat %s" % test_tmpdir, show=True)
1264         run_cmd("stat %s" % testbase, show=True)
1265         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1266     except Exception:
1267         cleanup()
1268         raise
1269
1270     try:
1271         if options.rebase is not None:
1272             rebase_tree(options.rebase, rebase_branch=options.branch)
1273     except Exception:
1274         cleanup_list.append(gitroot + "/autobuild.pid")
1275         cleanup()
1276         elapsed_time = time.time() - start_time
1277         email_failure(-1, 'rebase', 'rebase', 'rebase',
1278                       'rebase on %s failed' % options.branch,
1279                       elapsed_time, log_base=options.log_base)
1280         sys.exit(1)
1281
1282     try:
1283         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1284         if options.tail:
1285             blist.start_tail()
1286         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1287         if status != 0 or errstr != "retry":
1288             break
1289         cleanup()
1290     except Exception:
1291         cleanup()
1292         raise
1293
1294 cleanup_list.append(gitroot + "/autobuild.pid")
1295
1296 do_print(errstr)
1297
1298 blist.kill_kids()
1299 if options.tail:
1300     do_print("waiting for tail to flush")
1301     time.sleep(1)
1302
1303 elapsed_time = time.time() - start_time
1304 if status == 0:
1305     if options.passcmd is not None:
1306         do_print("Running passcmd: %s" % options.passcmd)
1307         run_cmd(options.passcmd, dir=test_master)
1308     if options.pushto is not None:
1309         push_to(options.pushto, push_branch=options.branch)
1310     if options.keeplogs or options.attach_logs:
1311         blist.tarlogs("logs.tar.gz")
1312         do_print("Logs in logs.tar.gz")
1313     if options.always_email:
1314         email_success(elapsed_time, log_base=options.log_base)
1315     blist.remove_logs()
1316     cleanup()
1317     do_print(errstr)
1318     sys.exit(0)
1319
1320 # something failed, gather a tar of the logs
1321 blist.tarlogs("logs.tar.gz")
1322
1323 if options.email is not None:
1324     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1325                   elapsed_time, log_base=options.log_base)
1326 else:
1327     elapsed_minutes = elapsed_time / 60.0
1328     print('''
1329
1330 ####################################################################
1331
1332 AUTOBUILD FAILURE
1333
1334 Your autobuild[%s] on %s failed after %.1f minutes
1335 when trying to test %s with the following error:
1336
1337    %s
1338
1339 the autobuild has been abandoned. Please fix the error and resubmit.
1340
1341 ####################################################################
1342
1343 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1344
1345 cleanup()
1346 do_print(errstr)
1347 do_print("Logs in logs.tar.gz")
1348 sys.exit(status)