script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, check_output, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 import random
  13 from optparse import OptionParser
  14 import smtplib
  15 import email
  16 from email.mime.text import MIMEText
  17 from email.mime.base import MIMEBase
  18 from email.mime.application import MIMEApplication
  19 from email.mime.multipart import MIMEMultipart
  20 from distutils.sysconfig import get_python_lib
  21 import platform
  22
  23 try:
  24     from waflib.Build import CACHE_SUFFIX
  25 except ImportError:
  26     sys.path.insert(0, "./third_party/waf")
  27     from waflib.Build import CACHE_SUFFIX
  28
  29
  30 os.environ["PYTHONUNBUFFERED"] = "1"
  31
  32 # This speeds up testing remarkably.
  33 os.environ['TDB_NO_FSYNC'] = '1'
  34
  35
  36 def find_git_root():
  37     '''get to the top of the git repo'''
  38     p = os.getcwd()
  39     while p != '/':
  40         if os.path.exists(os.path.join(p, ".git")):
  41             return p
  42         p = os.path.abspath(os.path.join(p, '..'))
  43     return None
  44
  45
  46 gitroot = find_git_root()
  47 if gitroot is None:
  48     raise Exception("Failed to find git root")
  49
  50
  51 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  52
  53 parser = OptionParser()
  54 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  55 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  56 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  57 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  58                   default=def_testbase)
  59 parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase,
  60                   default=None)
  61 parser.add_option("--passcmd", help="command to run on success", default=None)
  62 parser.add_option("--verbose", help="show all commands as they are run",
  63                   default=False, action="store_true")
  64 parser.add_option("--rebase", help="rebase on the given tree before testing",
  65                   default=None, type='str')
  66 parser.add_option("--pushto", help="push to a git url on success",
  67                   default=None, type='str')
  68 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  69                   default=False, action="store_true")
  70 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  71                   default=False, action="store_true")
  72 parser.add_option("--retry", help="automatically retry if master changes",
  73                   default=False, action="store_true")
  74 parser.add_option("--email", help="send email to the given address on failure",
  75                   type='str', default=None)
  76 parser.add_option("--email-from", help="send email from the given address",
  77                   type='str', default="autobuild@samba.org")
  78 parser.add_option("--email-server", help="send email via the given server",
  79                   type='str', default='localhost')
  80 parser.add_option("--always-email", help="always send email, even on success",
  81                   action="store_true")
  82 parser.add_option("--daemon", help="daemonize after initial setup",
  83                   action="store_true")
  84 parser.add_option("--branch", help="the branch to work on (default=master)",
  85                   default="master", type='str')
  86 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  87                   default=gitroot, type='str')
  88 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  89                   default=False, action="store_true")
  90 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
  91                   default='')
  92 parser.add_option("--enable-coverage", dest='enable_coverage',
  93                   action="store_const", const='--enable-coverage', default='',
  94                   help="Add --enable-coverage option while configure")
  95
  96 (options, args) = parser.parse_args()
  97
  98 if options.retry:
  99     if options.rebase is None:
 100         raise Exception('You can only use --retry if you also rebase')
 101
 102 if options.full_testbase is not None:
 103     testbase = options.full_testbase
 104 else:
 105     testbase = "%s/b%u" % (options.testbase, os.getpid())
 106 test_master = "%s/master" % testbase
 107 test_prefix = "%s/prefix" % testbase
 108 test_tmpdir = "%s/tmp" % testbase
 109 os.environ['TMPDIR'] = test_tmpdir
 110
 111 if options.enable_coverage:
 112     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 113 else:
 114     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 115
 116 if args:
 117     # If we are only running specific test,
 118     # do not sleep randomly to wait for it to start
 119     def random_sleep(low, high):
 120         return 'sleep 1'
 121 else:
 122     def random_sleep(low, high):
 123         return 'sleep {}'.format(random.randint(low, high))
 124
 125 cleanup_list = []
 126
 127 builddirs = {
 128     "ctdb": "ctdb",
 129     "ldb": "lib/ldb",
 130     "tdb": "lib/tdb",
 131     "talloc": "lib/talloc",
 132     "replace": "lib/replace",
 133     "tevent": "lib/tevent",
 134     "pidl": "pidl",
 135     "docs-xml": "docs-xml"
 136 }
 137
 138 ctdb_configure_params = " --enable-developer ${PREFIX}"
 139 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 140
 141 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 142 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 143 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 144 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 145 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 146 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 147 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 148
 149
 150 def format_option(name, value=None):
 151     """Format option as str list."""
 152     if value is None:  # boolean option
 153         return [name]
 154     if not isinstance(value, list):  # single value option
 155         value = [value]
 156     # repeatable option
 157     return ['{}={}'.format(name, item) for item in value]
 158
 159
 160 def make_test(
 161         cmd='make test',
 162         FAIL_IMMEDIATELY=1,
 163         TESTS='',
 164         include_envs=None,
 165         exclude_envs=None):
 166
 167     test_options = []
 168     if include_envs:
 169         test_options = format_option('--include-env', include_envs)
 170     if exclude_envs:
 171         test_options = format_option('--exclude-env', exclude_envs)
 172     if test_options:
 173         # join envs options to original test options
 174         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 175
 176     _options = []
 177     if FAIL_IMMEDIATELY:
 178         _options.append('FAIL_IMMEDIATELY=1')
 179     if TESTS:
 180         _options.append("TESTS='{}'".format(TESTS))
 181
 182     return ' '.join([cmd] + _options)
 183
 184
 185 # When updating this list, also update .gitlab-ci.yml to add the job
 186 # and to make it a dependency of 'page' for the coverage report.
 187
 188 tasks = {
 189     "ctdb": [
 190         ("random-sleep", random_sleep(300, 900)),
 191         ("configure", "./configure " + ctdb_configure_params),
 192         ("make", "make all"),
 193         ("install", "make install"),
 194         ("test", "make autotest"),
 195         ("check-clean-tree", "../script/clean-source-tree.sh"),
 196         ("clean", "make clean"),
 197         ],
 198
 199     "docs-xml": [
 200         ("random-sleep", random_sleep(300, 900)),
 201         ("autoconf", "autoconf"),
 202         ("configure", "./configure"),
 203         ("make", "make html htmlman"),
 204         ("clean", "make clean"),
 205         ],
 206
 207     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 208     "samba": [
 209         ("random-sleep", random_sleep(300, 900)),
 210         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 211         ("make", "make -j"),
 212         ("test", make_test(exclude_envs=[
 213             "none",
 214             "nt4_dc",
 215             "nt4_dc_smb1",
 216             "nt4_dc_smb1_done",
 217             "nt4_dc_schannel",
 218             "nt4_member",
 219             "ad_dc",
 220             "ad_dc_smb1",
 221             "ad_dc_smb1_done",
 222             "ad_dc_backup",
 223             "ad_dc_ntvfs",
 224             "ad_dc_default",
 225             "ad_dc_default_smb1",
 226             "ad_dc_slowtests",
 227             "ad_dc_no_nss",
 228             "ad_dc_no_ntlm",
 229             "fl2003dc",
 230             "fl2008dc",
 231             "fl2008r2dc",
 232             "ad_member",
 233             "ad_member_idmap_rid",
 234             "ad_member_idmap_ad",
 235             "ad_member_rfc2307",
 236             "chgdcpass",
 237             "vampire_2000_dc",
 238             "fl2000dc",
 239             "fileserver",
 240             "fileserver_smb1",
 241             "fileserver_smb1_done",
 242             "maptoguest",
 243             "simpleserver",
 244             "backupfromdc",
 245             "restoredc",
 246             "renamedc",
 247             "offlinebackupdc",
 248             "labdc",
 249             "preforkrestartdc",
 250             "proclimitdc",
 251             "promoted_dc",
 252             "vampire_dc",
 253             "rodc",
 254             "ad_dc_default",
 255             "ad_dc_default_smb1",
 256             "ad_dc_default_smb1_done",
 257             "ad_dc_slowtests",
 258             "schema_pair_dc",
 259             "schema_dc",
 260             "clusteredmember",
 261             ])),
 262         ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
 263         ("lcov", LCOV_CMD),
 264         ("install", "make install"),
 265         ("check-clean-tree", "script/clean-source-tree.sh"),
 266         ("clean", "make clean"),
 267         ],
 268
 269     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 270     "samba-mitkrb5": [
 271         ("random-sleep", random_sleep(300, 900)),
 272         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 273         ("make", "make -j"),
 274         ("test", make_test(exclude_envs=[
 275             "none",
 276             "nt4_dc",
 277             "nt4_dc_smb1",
 278             "nt4_dc_smb1_done",
 279             "nt4_dc_schannel",
 280             "nt4_member",
 281             "ad_dc",
 282             "ad_dc_smb1",
 283             "ad_dc_smb1_done",
 284             "ad_dc_backup",
 285             "ad_dc_ntvfs",
 286             "ad_dc_default",
 287             "ad_dc_default_smb1",
 288             "ad_dc_default_smb1_done",
 289             "ad_dc_slowtests",
 290             "ad_dc_no_nss",
 291             "ad_dc_no_ntlm",
 292             "fl2003dc",
 293             "fl2008dc",
 294             "fl2008r2dc",
 295             "ad_member",
 296             "ad_member_idmap_rid",
 297             "ad_member_idmap_ad",
 298             "ad_member_rfc2307",
 299             "chgdcpass",
 300             "vampire_2000_dc",
 301             "fl2000dc",
 302             "fileserver",
 303             "fileserver_smb1",
 304             "fileserver_smb1_done",
 305             "maptoguest",
 306             "simpleserver",
 307             "backupfromdc",
 308             "restoredc",
 309             "renamedc",
 310             "offlinebackupdc",
 311             "labdc",
 312             "preforkrestartdc",
 313             "proclimitdc",
 314             "promoted_dc",
 315             "vampire_dc",
 316             "rodc",
 317             "ad_dc_default",
 318             "ad_dc_default_smb1",
 319             "ad_dc_default_smb1_done",
 320             "ad_dc_slowtests",
 321             "schema_pair_dc",
 322             "schema_dc",
 323             "clusteredmember",
 324             ])),
 325         ("lcov", LCOV_CMD),
 326         ("install", "make install"),
 327         ("check-clean-tree", "script/clean-source-tree.sh"),
 328         ("clean", "make clean"),
 329         ],
 330
 331     "samba-nt4": [
 332         ("random-sleep", random_sleep(300, 900)),
 333         ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params),
 334         ("make", "make -j"),
 335         ("test", make_test(include_envs=[
 336             "nt4_dc",
 337             "nt4_dc_smb1",
 338             "nt4_dc_smb1_done",
 339             "nt4_dc_schannel",
 340             "nt4_member",
 341             "simpleserver",
 342             ])),
 343         ("lcov", LCOV_CMD),
 344         ("check-clean-tree", "script/clean-source-tree.sh"),
 345         ("clean", "make clean"),
 346         ],
 347
 348     "samba-fileserver": [
 349         ("random-sleep", random_sleep(300, 900)),
 350         ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5 --with-selftest-prefix=./bin/ab" + samba_configure_params),
 351         ("make", "make -j"),
 352         ("test", make_test(include_envs=[
 353             "fileserver",
 354             "fileserver_smb1",
 355             "fileserver_smb1_done",
 356             "maptoguest",
 357             "ktest", # ktest is also tested in samba and samba-mitkrb5
 358                      # but is tested here against a system Heimdal
 359             ])),
 360         ("lcov", LCOV_CMD),
 361         ("check-clean-tree", "script/clean-source-tree.sh"),
 362         ],
 363
 364     "samba-admem": [
 365         ("random-sleep", random_sleep(300, 900)),
 366         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 367         ("make", "make -j"),
 368         ("test", make_test(include_envs=[
 369             "ad_member",
 370             "ad_member_idmap_rid",
 371             "ad_member_idmap_ad",
 372             "ad_member_rfc2307",
 373             ])),
 374         ("lcov", LCOV_CMD),
 375         ("check-clean-tree", "script/clean-source-tree.sh"),
 376         ],
 377
 378     "samba-ad-dc-1": [
 379         ("random-sleep", random_sleep(1, 1)),
 380         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 381         ("make", "make -j"),
 382         ("test", make_test(include_envs=[
 383             "ad_dc",
 384             "ad_dc_smb1",
 385             "ad_dc_smb1_done",
 386             "ad_dc_no_nss",
 387             "ad_dc_no_ntlm",
 388             ])),
 389         ("lcov", LCOV_CMD),
 390         ("check-clean-tree", "script/clean-source-tree.sh"),
 391         ],
 392
 393     "samba-ad-dc-2": [
 394         ("random-sleep", random_sleep(1, 1)),
 395         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 396         ("make", "make -j"),
 397         ("test", make_test(include_envs=[
 398             "vampire_dc",
 399             "vampire_2000_dc",
 400             "rodc",
 401             ])),
 402         ("lcov", LCOV_CMD),
 403         ("check-clean-tree", "script/clean-source-tree.sh"),
 404         ],
 405
 406     "samba-ad-dc-3": [
 407         ("random-sleep", random_sleep(1, 1)),
 408         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 409         ("make", "make -j"),
 410         ("test", make_test(include_envs=[
 411             "promoted_dc",
 412             "chgdcpass",
 413             "preforkrestartdc",
 414             "proclimitdc",
 415             ])),
 416         ("lcov", LCOV_CMD),
 417         ("check-clean-tree", "script/clean-source-tree.sh"),
 418         ],
 419
 420     "samba-ad-dc-4": [
 421         ("random-sleep", random_sleep(1, 1)),
 422         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 423         ("make", "make -j"),
 424         ("test", make_test(include_envs=[
 425             "fl2000dc",
 426             "fl2003dc",
 427             "fl2008dc",
 428             "fl2008r2dc",
 429             ])),
 430         ("lcov", LCOV_CMD),
 431         ("check-clean-tree", "script/clean-source-tree.sh"),
 432         ],
 433
 434     "samba-ad-dc-5": [
 435         ("random-sleep", random_sleep(1, 1)),
 436         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 437         ("make", "make -j"),
 438         ("test", make_test(include_envs=[
 439             "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
 440         ("lcov", LCOV_CMD),
 441         ("check-clean-tree", "script/clean-source-tree.sh"),
 442         ],
 443
 444     "samba-ad-dc-6": [
 445         ("random-sleep", random_sleep(1, 1)),
 446         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 447         ("make", "make -j"),
 448         ("test", make_test(include_envs=["ad_dc_slowtests"])),
 449         ("lcov", LCOV_CMD),
 450         ("check-clean-tree", "script/clean-source-tree.sh"),
 451         ],
 452
 453     "samba-schemaupgrade": [
 454         ("random-sleep", random_sleep(1, 1)),
 455         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 456         ("make", "make -j"),
 457         ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 458         ("lcov", LCOV_CMD),
 459         ("check-clean-tree", "script/clean-source-tree.sh"),
 460         ],
 461
 462     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 463     # This is currently the longest task, so we don't randomly delay it.
 464     "samba-ad-dc-ntvfs": [
 465         ("random-sleep", random_sleep(1, 1)),
 466         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 467         ("make", "make -j"),
 468         ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 469         ("lcov", LCOV_CMD),
 470         ("check-clean-tree", "script/clean-source-tree.sh"),
 471         ],
 472
 473     # Test fips compliance
 474     "samba-fips": [
 475         ("random-sleep", random_sleep(100, 500)),
 476         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 477         ("make", "make -j"),
 478         ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
 479         ("lcov", LCOV_CMD),
 480         ("check-clean-tree", "script/clean-source-tree.sh"),
 481         ],
 482
 483     # run the backup/restore testenvs separately as they're fairly standalone
 484     # (and CI seems to max out at ~8 different DCs running at once)
 485     "samba-ad-dc-backup": [
 486         ("random-sleep", random_sleep(300, 900)),
 487         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 488         ("make", "make -j"),
 489         ("test", make_test(include_envs=[
 490             "backupfromdc",
 491             "restoredc",
 492             "renamedc",
 493             "offlinebackupdc",
 494             "labdc",
 495             "ad_dc_backup",
 496             ])),
 497         ("lcov", LCOV_CMD),
 498         ("check-clean-tree", "script/clean-source-tree.sh"),
 499         ],
 500
 501     "samba-admem-mit": [
 502         ("random-sleep", random_sleep(1, 1)),
 503         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 504         ("make", "make -j"),
 505         ("test", make_test(include_envs=[
 506             "ad_member",
 507             "ad_member_idmap_rid",
 508             "ad_member_idmap_ad",
 509             "ad_member_rfc2307",
 510             ])),
 511         ("lcov", LCOV_CMD),
 512         ("check-clean-tree", "script/clean-source-tree.sh"),
 513         ],
 514
 515     "samba-ad-dc-1-mitkrb5": [
 516         ("random-sleep", random_sleep(1, 1)),
 517         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 518         ("make", "make -j"),
 519         ("test", make_test(include_envs=[
 520             "ad_dc",
 521             "ad_dc_smb1",
 522             "ad_dc_smb1_done",
 523             "ad_dc_no_nss",
 524             "ad_dc_no_ntlm",
 525             ])),
 526         ("lcov", LCOV_CMD),
 527         ("check-clean-tree", "script/clean-source-tree.sh"),
 528         ],
 529
 530     "samba-ad-dc-4-mitkrb5": [
 531         ("random-sleep", random_sleep(1, 1)),
 532         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 533         ("make", "make -j"),
 534         ("test", make_test(include_envs=[
 535             "fl2000dc",
 536             "fl2003dc",
 537             "fl2008dc",
 538             "fl2008r2dc",
 539             ])),
 540         ("lcov", LCOV_CMD),
 541         ("check-clean-tree", "script/clean-source-tree.sh"),
 542         ],
 543
 544     "samba-test-only": [
 545         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params),
 546         ("make", "make -j"),
 547         ("test", make_test(TESTS="${TESTS}")),
 548         ("lcov", LCOV_CMD),
 549         ],
 550
 551     # Test cross-compile infrastructure
 552     "samba-xc": [
 553         ("random-sleep", random_sleep(900, 1500)),
 554         ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 555         ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 556             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 557         ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
 558         ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 559             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 560         ("compare-results", "script/compare_cc_results.py "
 561             "./bin/c4che/default{} "
 562             "./bin-xe/c4che/default{} "
 563             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 564         ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
 565         ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
 566             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
 567         ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
 568             " = \"'1234'\"".format(CACHE_SUFFIX)),
 569         ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
 570         ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
 571             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
 572             " ; test $? -ne 0"),
 573         ],
 574
 575     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 576     "samba-o3": [
 577         ("random-sleep", random_sleep(300, 900)),
 578         ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params),
 579         ("make", "make -j"),
 580         ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 581         ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 582         ("lcov", LCOV_CMD),
 583         ("install", "make install"),
 584         ("check-clean-tree", "script/clean-source-tree.sh"),
 585         ("clean", "make clean"),
 586         ],
 587
 588     "samba-ctdb": [
 589         ("random-sleep", random_sleep(900, 1500)),
 590
 591         # make sure we have tdb around:
 592         ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 593         ("tdb-make", "cd lib/tdb && make"),
 594         ("tdb-install", "cd lib/tdb && make install"),
 595
 596         # build samba with cluster support (also building ctdb):
 597         ("samba-configure",
 598          "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
 599          "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
 600          "./configure.developer ${PREFIX} "
 601          "--with-selftest-prefix=./bin/ab "
 602          "--with-cluster-support "
 603          "--without-ad-dc "
 604          "--bundled-libraries=!tdb"),
 605         ("samba-make", "make"),
 606         ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT"),
 607         ("samba-install", "make install"),
 608         ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 609
 610         ("test",
 611          make_test(cmd='make test',
 612                    include_envs=["clusteredmember"])
 613         ),
 614
 615         # clean up:
 616         ("check-clean-tree", "script/clean-source-tree.sh"),
 617         ("clean", "make clean"),
 618         ("ctdb-clean", "cd ./ctdb && make clean"),
 619         ],
 620
 621     "samba-libs": [
 622         ("random-sleep", random_sleep(300, 900)),
 623         ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 624         ("talloc-make", "cd lib/talloc && make"),
 625         ("talloc-install", "cd lib/talloc && make install"),
 626
 627         ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 628         ("tdb-make", "cd lib/tdb && make"),
 629         ("tdb-install", "cd lib/tdb && make install"),
 630
 631         ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 632         ("tevent-make", "cd lib/tevent && make"),
 633         ("tevent-install", "cd lib/tevent && make install"),
 634
 635         ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
 636         ("ldb-make", "cd lib/ldb && make"),
 637         ("ldb-install", "cd lib/ldb && make install"),
 638
 639         ("nondevel-configure", "./configure ${PREFIX}"),
 640         ("nondevel-make", "make -j"),
 641         ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 642         ("nondevel-install", "make install"),
 643         ("nondevel-dist", "make dist"),
 644
 645         # retry with all modules shared
 646         ("allshared-distclean", "make distclean"),
 647         ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 648         ("allshared-make", "make -j"),
 649         ],
 650
 651     "samba-fuzz": [
 652         # build the fuzzers (static) via the oss-fuzz script
 653         ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
 654         ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ADDITIONAL_LDFLAGS='-fuse-ld=bfd' ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl"),
 655         ],
 656
 657     # * Test smbd and smbtorture can build semi-static
 658     #
 659     # * Test Samba without python still builds.
 660     #
 661     # When this test fails due to more use of Python, the expectations
 662     # is that the newly failing part of the code should be disabled
 663     # when --disable-python is set (rather than major work being done
 664     # to support this environment).
 665     #
 666     # The target here is for vendors shipping a minimal smbd.
 667     "samba-minimal-smbd": [
 668         ("random-sleep", random_sleep(300, 900)),
 669
 670         # build with all modules static
 671         ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
 672         ("allstatic-make", "make -j"),
 673         ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 674         ("lcov", LCOV_CMD),
 675
 676         # retry with nonshared smbd and smbtorture
 677         ("nonshared-distclean", "make distclean"),
 678         ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
 679         ("nonshared-make", "make -j"),
 680
 681         ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
 682         ("make", "make -j"),
 683         ("find-python", "script/find_python.sh ${PREFIX}"),
 684         ("test", "make test-nopython"),
 685         ("lcov", LCOV_CMD),
 686         ("check-clean-tree", "script/clean-source-tree.sh"),
 687         ("clean", "make clean"),
 688
 689         ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 690         ("talloc-make", "cd lib/talloc && make"),
 691         ("talloc-install", "cd lib/talloc && make install"),
 692
 693         ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 694         ("tdb-make", "cd lib/tdb && make"),
 695         ("tdb-install", "cd lib/tdb && make install"),
 696
 697         ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 698         ("tevent-make", "cd lib/tevent && make"),
 699         ("tevent-install", "cd lib/tevent && make install"),
 700
 701         ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 702         ("ldb-make", "cd lib/ldb && make"),
 703         ("ldb-install", "cd lib/ldb && make install"),
 704
 705         # retry against installed library packages, but no required modules
 706         ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc  --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
 707         ("libs-make", "make -j"),
 708         ("libs-install", "make install"),
 709         ("libs-check-clean-tree", "script/clean-source-tree.sh"),
 710         ("libs-clean", "make clean"),
 711
 712         ],
 713
 714     "ldb": [
 715         ("random-sleep", random_sleep(60, 600)),
 716         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 717         ("make", "make"),
 718         ("install", "make install"),
 719         ("test", "make test"),
 720         ("lcov", LCOV_CMD),
 721         ("clean", "make clean"),
 722         ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
 723         ("make-no-lmdb", "make"),
 724         ("test-no-lmdb", "make test"),
 725         ("lcov-no-lmdb", LCOV_CMD),
 726         ("install-no-lmdb", "make install"),
 727         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 728         ("distcheck", "make distcheck"),
 729         ("clean", "make clean"),
 730         ],
 731
 732     "tdb": [
 733         ("random-sleep", random_sleep(60, 600)),
 734         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 735         ("make", "make"),
 736         ("install", "make install"),
 737         ("test", "make test"),
 738         ("lcov", LCOV_CMD),
 739         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 740         ("distcheck", "make distcheck"),
 741         ("clean", "make clean"),
 742         ],
 743
 744     "talloc": [
 745         ("random-sleep", random_sleep(60, 600)),
 746         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 747         ("make", "make"),
 748         ("install", "make install"),
 749         ("test", "make test"),
 750         ("lcov", LCOV_CMD),
 751         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 752         ("distcheck", "make distcheck"),
 753         ("clean", "make clean"),
 754         ],
 755
 756     "replace": [
 757         ("random-sleep", random_sleep(60, 600)),
 758         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 759         ("make", "make"),
 760         ("install", "make install"),
 761         ("test", "make test"),
 762         ("lcov", LCOV_CMD),
 763         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 764         ("distcheck", "make distcheck"),
 765         ("clean", "make clean"),
 766         ],
 767
 768     "tevent": [
 769         ("random-sleep", random_sleep(60, 600)),
 770         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 771         ("make", "make"),
 772         ("install", "make install"),
 773         ("test", "make test"),
 774         ("lcov", LCOV_CMD),
 775         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 776         ("distcheck", "make distcheck"),
 777         ("clean", "make clean"),
 778         ],
 779
 780     "pidl": [
 781         ("random-sleep", random_sleep(60, 600)),
 782         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
 783         ("touch", "touch *.yp"),
 784         ("make", "make"),
 785         ("test", "make test"),
 786         ("install", "make install"),
 787         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
 788         ("check-clean-tree", "../script/clean-source-tree.sh"),
 789         ("clean", "make clean"),
 790         ],
 791
 792     # these are useful for debugging autobuild
 793     'pass': [("pass", 'echo passing && /bin/true')],
 794     'fail': [("fail", 'echo failing && /bin/false')],
 795 }
 796
 797 defaulttasks = list(tasks.keys())
 798
 799 defaulttasks.remove("pass")
 800 defaulttasks.remove("fail")
 801 defaulttasks.remove("samba-test-only")
 802 defaulttasks.remove("samba-fuzz")
 803 defaulttasks.remove("samba-fips")
 804 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
 805     defaulttasks.remove("samba-o3")
 806
 807
 808 def do_print(msg):
 809     print("%s" % msg)
 810     sys.stdout.flush()
 811     sys.stderr.flush()
 812
 813
 814 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 815     if show is None:
 816         show = options.verbose
 817     if show:
 818         do_print("Running: '%s' in '%s'" % (cmd, dir))
 819     if output:
 820         out = check_output([cmd], shell=True, cwd=dir)
 821         return out.decode(encoding='utf-8', errors='backslashreplace')
 822     elif checkfail:
 823         return check_call(cmd, shell=True, cwd=dir)
 824     else:
 825         return call(cmd, shell=True, cwd=dir)
 826
 827
 828 class builder(object):
 829     '''handle build of one directory'''
 830
 831     def __init__(self, name, sequence, cp=True):
 832         self.name = name
 833         self.dir = builddirs.get(name, '.')
 834         self.tag = self.name.replace('/', '_')
 835         self.sequence = sequence
 836         self.next = 0
 837         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 838         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 839         if options.verbose:
 840             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 841             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 842         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 843         self.stdout = open(self.stdout_path, 'w')
 844         self.stderr = open(self.stderr_path, 'w')
 845         self.stdin  = open("/dev/null", 'r')
 846         self.test_source_dir = "%s/%s" % (testbase, self.tag)
 847         self.cwd = "%s/%s" % (self.test_source_dir, self.dir)
 848         self.prefix = "%s/%s" % (test_prefix, self.tag)
 849         run_cmd("rm -rf %s" % self.test_source_dir)
 850         run_cmd("rm -rf %s" % self.prefix)
 851         if cp:
 852             run_cmd("cp -R -a -l %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
 853         else:
 854             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
 855         self.start_next()
 856
 857     def start_next(self):
 858         if self.next == len(self.sequence):
 859             if not options.nocleanup:
 860                 run_cmd("rm -rf %s" % self.test_source_dir)
 861                 run_cmd("rm -rf %s" % self.prefix)
 862             do_print('%s: Completed OK' % self.name)
 863             self.done = True
 864             return
 865         (self.stage, self.cmd) = self.sequence[self.next]
 866         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
 867         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 868         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 869         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 870         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
 871         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
 872         self.cmd = self.cmd.replace("${NAME}", self.name)
 873         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
 874         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
 875         self.proc = Popen(self.cmd, shell=True,
 876                           close_fds=True, cwd=self.cwd,
 877                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 878         self.next += 1
 879
 880
 881 class buildlist(object):
 882     '''handle build of multiple directories'''
 883
 884     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 885         self.tail_proc = None
 886         self.retry = None
 887         if not tasknames:
 888             if options.restrict_tests:
 889                 tasknames = ["samba-test-only"]
 890             else:
 891                 tasknames = defaulttasks
 892
 893         self.tlist = [builder(n, tasks[n], cp=(n != "pidl")) for n in tasknames]
 894
 895         if options.retry:
 896             rebase_remote = "rebaseon"
 897             retry_task = [("retry",
 898                             '''set -e
 899                             git remote add -t %s %s %s
 900                             git fetch %s
 901                             while :; do
 902                               sleep 60
 903                               git describe %s/%s > old_remote_branch.desc
 904                               git fetch %s
 905                               git describe %s/%s > remote_branch.desc
 906                               diff old_remote_branch.desc remote_branch.desc
 907                             done
 908                            ''' % (
 909                                rebase_branch, rebase_remote, rebase_url,
 910                                rebase_remote,
 911                                rebase_remote, rebase_branch,
 912                                rebase_remote,
 913                                rebase_remote, rebase_branch
 914                             ))]
 915
 916             self.retry = builder('retry', retry_task, cp=False)
 917             self.need_retry = False
 918
 919     def kill_kids(self):
 920         if self.tail_proc is not None:
 921             self.tail_proc.terminate()
 922             self.tail_proc.wait()
 923             self.tail_proc = None
 924         if self.retry is not None:
 925             self.retry.proc.terminate()
 926             self.retry.proc.wait()
 927             self.retry = None
 928         for b in self.tlist:
 929             if b.proc is not None:
 930                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
 931                 b.proc.terminate()
 932                 b.proc.wait()
 933                 b.proc = None
 934
 935     def wait_one(self):
 936         while True:
 937             none_running = True
 938             for b in self.tlist:
 939                 if b.proc is None:
 940                     continue
 941                 none_running = False
 942                 b.status = b.proc.poll()
 943                 if b.status is None:
 944                     continue
 945                 b.proc = None
 946                 return b
 947             if options.retry:
 948                 ret = self.retry.proc.poll()
 949                 if ret is not None:
 950                     self.need_retry = True
 951                     self.retry = None
 952                     return None
 953             if none_running:
 954                 return None
 955             time.sleep(0.1)
 956
 957     def run(self):
 958         while True:
 959             b = self.wait_one()
 960             if options.retry and self.need_retry:
 961                 self.kill_kids()
 962                 do_print("retry needed")
 963                 return (0, None, None, None, "retry")
 964             if b is None:
 965                 break
 966             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 967                 self.kill_kids()
 968                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 969             b.start_next()
 970         self.kill_kids()
 971         return (0, None, None, None, "All OK")
 972
 973     def write_system_info(self, filename):
 974         with open(filename, 'w') as f:
 975             for cmd in ['uname -a',
 976                         'lsb_release -a',
 977                         'free',
 978                         'mount',
 979                         'cat /proc/cpuinfo',
 980                         'cc --version',
 981                         'df -m .',
 982                         'df -m %s' % testbase]:
 983                 try:
 984                     out = run_cmd(cmd, output=True, checkfail=False)
 985                 except subprocess.CalledProcessError as e:
 986                     out = "<failed: %s>" % str(e)
 987                 print('### %s' % cmd, file=f)
 988                 print(out, file=f)
 989                 print(file=f)
 990
 991     def tarlogs(self, fname):
 992         with tarfile.open(fname, "w:gz") as tar:
 993             for b in self.tlist:
 994                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 995                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 996             if os.path.exists("autobuild.log"):
 997                 tar.add("autobuild.log")
 998             filename = 'system-info.txt'
 999             self.write_system_info(filename)
1000             tar.add(filename)
1001
1002     def remove_logs(self):
1003         for b in self.tlist:
1004             os.unlink(b.stdout_path)
1005             os.unlink(b.stderr_path)
1006
1007     def start_tail(self):
1008         cmd = ["tail", "-f"]
1009         for b in self.tlist:
1010             cmd.append(b.stdout_path)
1011             cmd.append(b.stderr_path)
1012         self.tail_proc = Popen(cmd, close_fds=True)
1013
1014
1015 def cleanup():
1016     if options.nocleanup:
1017         return
1018     run_cmd("stat %s || true" % test_tmpdir, show=True)
1019     run_cmd("stat %s" % testbase, show=True)
1020     do_print("Cleaning up %r" % cleanup_list)
1021     for d in cleanup_list:
1022         run_cmd("rm -rf %s" % d)
1023
1024
1025 def daemonize(logfile):
1026     pid = os.fork()
1027     if pid == 0:  # Parent
1028         os.setsid()
1029         pid = os.fork()
1030         if pid != 0:  # Actual daemon
1031             os._exit(0)
1032     else:  # Grandparent
1033         os._exit(0)
1034
1035     import resource      # Resource usage information.
1036     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1037     if maxfd == resource.RLIM_INFINITY:
1038         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1039     for fd in range(0, maxfd):
1040         try:
1041             os.close(fd)
1042         except OSError:
1043             pass
1044     os.open(logfile, os.O_RDWR | os.O_CREAT)
1045     os.dup2(0, 1)
1046     os.dup2(0, 2)
1047
1048
1049 def write_pidfile(fname):
1050     '''write a pid file, cleanup on exit'''
1051     with open(fname, mode='w') as f:
1052         f.write("%u\n" % os.getpid())
1053
1054
1055 def rebase_tree(rebase_url, rebase_branch="master"):
1056     rebase_remote = "rebaseon"
1057     do_print("Rebasing on %s" % rebase_url)
1058     run_cmd("git describe HEAD", show=True, dir=test_master)
1059     run_cmd("git remote add -t %s %s %s" %
1060             (rebase_branch, rebase_remote, rebase_url),
1061             show=True, dir=test_master)
1062     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1063     if options.fix_whitespace:
1064         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1065                 (rebase_remote, rebase_branch),
1066                 show=True, dir=test_master)
1067     else:
1068         run_cmd("git rebase --force-rebase %s/%s" %
1069                 (rebase_remote, rebase_branch),
1070                 show=True, dir=test_master)
1071     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1072                    (rebase_remote, rebase_branch),
1073                    dir=test_master, output=True)
1074     if diff == '':
1075         do_print("No differences between HEAD and %s/%s - exiting" %
1076                  (rebase_remote, rebase_branch))
1077         sys.exit(0)
1078     run_cmd("git describe %s/%s" %
1079             (rebase_remote, rebase_branch),
1080             show=True, dir=test_master)
1081     run_cmd("git describe HEAD", show=True, dir=test_master)
1082     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1083             (rebase_remote, rebase_branch),
1084             show=True, dir=test_master)
1085
1086
1087 def push_to(push_url, push_branch="master"):
1088     push_remote = "pushto"
1089     do_print("Pushing to %s" % push_url)
1090     if options.mark:
1091         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1092         run_cmd("git commit --amend -c HEAD", dir=test_master)
1093         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1094         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1095     run_cmd("git remote add -t %s %s %s" %
1096             (push_branch, push_remote, push_url),
1097             show=True, dir=test_master)
1098     run_cmd("git push %s +HEAD:%s" %
1099             (push_remote, push_branch),
1100             show=True, dir=test_master)
1101
1102
1103 def send_email(subject, text, log_tar):
1104     if options.email is None:
1105         do_print("not sending email because the recipient is not set")
1106         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1107                  (subject, text))
1108         return
1109     outer = MIMEMultipart()
1110     outer['Subject'] = subject
1111     outer['To'] = options.email
1112     outer['From'] = options.email_from
1113     outer['Date'] = email.utils.formatdate(localtime=True)
1114     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1115     outer.attach(MIMEText(text, 'plain'))
1116     if options.attach_logs:
1117         with open(log_tar, 'rb') as fp:
1118             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1119         # Set the filename parameter
1120         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1121         outer.attach(msg)
1122     content = outer.as_string()
1123     s = smtplib.SMTP(options.email_server)
1124     email_user = os.getenv('SMTP_USERNAME')
1125     email_password = os.getenv('SMTP_PASSWORD')
1126     if email_user is not None:
1127         s.starttls()
1128         s.login(email_user, email_password)
1129
1130     s.sendmail(options.email_from, [options.email], content)
1131     s.set_debuglevel(1)
1132     s.quit()
1133
1134
1135 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1136                   elapsed_time, log_base=None, add_log_tail=True):
1137     '''send an email to options.email about the failure'''
1138     elapsed_minutes = elapsed_time / 60.0
1139     if log_base is None:
1140         log_base = gitroot
1141     text = '''
1142 Dear Developer,
1143
1144 Your autobuild on %s failed after %.1f minutes
1145 when trying to test %s with the following error:
1146
1147    %s
1148
1149 the autobuild has been abandoned. Please fix the error and resubmit.
1150
1151 A summary of the autobuild process is here:
1152
1153   %s/autobuild.log
1154 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1155
1156     if options.restrict_tests:
1157         text += """
1158 The build was restricted to tests matching %s\n""" % options.restrict_tests
1159
1160     if failed_task != 'rebase':
1161         text += '''
1162 You can see logs of the failed task here:
1163
1164   %s/%s.stdout
1165   %s/%s.stderr
1166
1167 or you can get full logs of all tasks in this job here:
1168
1169   %s/logs.tar.gz
1170
1171 The top commit for the tree that was built was:
1172
1173 %s
1174
1175 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1176
1177     if add_log_tail:
1178         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1179         lines = f.readlines()
1180         log_tail = "".join(lines[-50:])
1181         num_lines = len(lines)
1182         if num_lines < 50:
1183             # Also include stderr (compile failures) if < 50 lines of stdout
1184             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1185             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1186
1187         text += '''
1188 The last 50 lines of log messages:
1189
1190 %s
1191     ''' % log_tail
1192         f.close()
1193
1194     logs = os.path.join(gitroot, 'logs.tar.gz')
1195     send_email('autobuild[%s] failure on %s for task %s during %s'
1196                % (options.branch, platform.node(), failed_task, failed_stage),
1197                text, logs)
1198
1199
1200 def email_success(elapsed_time, log_base=None):
1201     '''send an email to options.email about a successful build'''
1202     if log_base is None:
1203         log_base = gitroot
1204     text = '''
1205 Dear Developer,
1206
1207 Your autobuild on %s has succeeded after %.1f minutes.
1208
1209 ''' % (platform.node(), elapsed_time / 60.)
1210
1211     if options.restrict_tests:
1212         text += """
1213 The build was restricted to tests matching %s\n""" % options.restrict_tests
1214
1215     if options.keeplogs:
1216         text += '''
1217
1218 you can get full logs of all tasks in this job here:
1219
1220   %s/logs.tar.gz
1221
1222 ''' % log_base
1223
1224     text += '''
1225 The top commit for the tree that was built was:
1226
1227 %s
1228 ''' % top_commit_msg
1229
1230     logs = os.path.join(gitroot, 'logs.tar.gz')
1231     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1232                text, logs)
1233
1234
1235 # get the top commit message, for emails
1236 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1237
1238 try:
1239     os.makedirs(testbase)
1240 except Exception as reason:
1241     raise Exception("Unable to create %s : %s" % (testbase, reason))
1242 cleanup_list.append(testbase)
1243
1244 if options.daemon:
1245     logfile = os.path.join(testbase, "log")
1246     do_print("Forking into the background, writing progress to %s" % logfile)
1247     daemonize(logfile)
1248
1249 write_pidfile(gitroot + "/autobuild.pid")
1250
1251 start_time = time.time()
1252
1253 while True:
1254     try:
1255         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1256         os.makedirs(test_tmpdir)
1257         # The waf uninstall code removes empty directories all the way
1258         # up the tree.  Creating a file in test_tmpdir stops it from
1259         # being removed.
1260         run_cmd("touch %s" % os.path.join(test_tmpdir,
1261                                           ".directory-is-not-empty"), show=True)
1262         run_cmd("stat %s" % test_tmpdir, show=True)
1263         run_cmd("stat %s" % testbase, show=True)
1264         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1265     except Exception:
1266         cleanup()
1267         raise
1268
1269     try:
1270         if options.rebase is not None:
1271             rebase_tree(options.rebase, rebase_branch=options.branch)
1272     except Exception:
1273         cleanup_list.append(gitroot + "/autobuild.pid")
1274         cleanup()
1275         elapsed_time = time.time() - start_time
1276         email_failure(-1, 'rebase', 'rebase', 'rebase',
1277                       'rebase on %s failed' % options.branch,
1278                       elapsed_time, log_base=options.log_base)
1279         sys.exit(1)
1280
1281     try:
1282         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1283         if options.tail:
1284             blist.start_tail()
1285         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1286         if status != 0 or errstr != "retry":
1287             break
1288         cleanup()
1289     except Exception:
1290         cleanup()
1291         raise
1292
1293 cleanup_list.append(gitroot + "/autobuild.pid")
1294
1295 do_print(errstr)
1296
1297 blist.kill_kids()
1298 if options.tail:
1299     do_print("waiting for tail to flush")
1300     time.sleep(1)
1301
1302 elapsed_time = time.time() - start_time
1303 if status == 0:
1304     if options.passcmd is not None:
1305         do_print("Running passcmd: %s" % options.passcmd)
1306         run_cmd(options.passcmd, dir=test_master)
1307     if options.pushto is not None:
1308         push_to(options.pushto, push_branch=options.branch)
1309     if options.keeplogs or options.attach_logs:
1310         blist.tarlogs("logs.tar.gz")
1311         do_print("Logs in logs.tar.gz")
1312     if options.always_email:
1313         email_success(elapsed_time, log_base=options.log_base)
1314     blist.remove_logs()
1315     cleanup()
1316     do_print(errstr)
1317     sys.exit(0)
1318
1319 # something failed, gather a tar of the logs
1320 blist.tarlogs("logs.tar.gz")
1321
1322 if options.email is not None:
1323     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1324                   elapsed_time, log_base=options.log_base)
1325 else:
1326     elapsed_minutes = elapsed_time / 60.0
1327     print('''
1328
1329 ####################################################################
1330
1331 AUTOBUILD FAILURE
1332
1333 Your autobuild[%s] on %s failed after %.1f minutes
1334 when trying to test %s with the following error:
1335
1336    %s
1337
1338 the autobuild has been abandoned. Please fix the error and resubmit.
1339
1340 ####################################################################
1341
1342 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1343
1344 cleanup()
1345 do_print(errstr)
1346 do_print("Logs in logs.tar.gz")
1347 sys.exit(status)