script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from subprocess import call, check_call, check_output, Popen, PIPE, CalledProcessError
   7 import os
   8 import tarfile
   9 import sys
  10 import time
  11 import random
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34
  35 def find_git_root():
  36     '''get to the top of the git repo'''
  37     p = os.getcwd()
  38     while p != '/':
  39         if os.path.exists(os.path.join(p, ".git")):
  40             return p
  41         p = os.path.abspath(os.path.join(p, '..'))
  42     return None
  43
  44
  45 gitroot = find_git_root()
  46 if gitroot is None:
  47     raise Exception("Failed to find git root")
  48
  49
  50 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  51
  52 parser = OptionParser()
  53 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  54 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  55 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  56 parser.add_option("--skip-dependencies", help="skip to run task dependency tasks", default=False, action="store_true")
  57 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  58                   default=def_testbase)
  59 parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase,
  60                   default=None)
  61 parser.add_option("--passcmd", help="command to run on success", default=None)
  62 parser.add_option("--verbose", help="show all commands as they are run",
  63                   default=False, action="store_true")
  64 parser.add_option("--rebase", help="rebase on the given tree before testing",
  65                   default=None, type='str')
  66 parser.add_option("--pushto", help="push to a git url on success",
  67                   default=None, type='str')
  68 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  69                   default=False, action="store_true")
  70 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  71                   default=False, action="store_true")
  72 parser.add_option("--retry", help="automatically retry if master changes",
  73                   default=False, action="store_true")
  74 parser.add_option("--email", help="send email to the given address on failure",
  75                   type='str', default=None)
  76 parser.add_option("--email-from", help="send email from the given address",
  77                   type='str', default="autobuild@samba.org")
  78 parser.add_option("--email-server", help="send email via the given server",
  79                   type='str', default='localhost')
  80 parser.add_option("--always-email", help="always send email, even on success",
  81                   action="store_true")
  82 parser.add_option("--daemon", help="daemonize after initial setup",
  83                   action="store_true")
  84 parser.add_option("--branch", help="the branch to work on (default=master)",
  85                   default="master", type='str')
  86 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  87                   default=gitroot, type='str')
  88 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  89                   default=False, action="store_true")
  90 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
  91                   default='')
  92 parser.add_option("--enable-coverage", dest='enable_coverage',
  93                   action="store_const", const='--enable-coverage', default='',
  94                   help="Add --enable-coverage option while configure")
  95
  96 (options, args) = parser.parse_args()
  97
  98 if options.retry:
  99     if options.rebase is None:
 100         raise Exception('You can only use --retry if you also rebase')
 101
 102 if options.full_testbase is not None:
 103     testbase = options.full_testbase
 104 else:
 105     testbase = "%s/b%u" % (options.testbase, os.getpid())
 106 test_master = "%s/master" % testbase
 107 test_prefix = "%s/prefix" % testbase
 108 test_tmpdir = "%s/tmp" % testbase
 109 os.environ['TMPDIR'] = test_tmpdir
 110
 111 if options.enable_coverage:
 112     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 113 else:
 114     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 115
 116 if options.enable_coverage:
 117     PUBLISH_DOCS = "mkdir -p ${LOG_BASE}/public && mv output/htmldocs ${LOG_BASE}/public/htmldocs"
 118 else:
 119     PUBLISH_DOCS = 'echo "HTML documentation publishing skipped since no --enable-coverage specified"'
 120
 121 CLEAN_SOURCE_TREE_CMD = "cd ${TEST_SOURCE_DIR} && script/clean-source-tree.sh"
 122
 123 if args:
 124     # If we are only running specific test,
 125     # do not sleep randomly to wait for it to start
 126     def random_sleep(low, high):
 127         return 'sleep 1'
 128 else:
 129     def random_sleep(low, high):
 130         return 'sleep {}'.format(random.randint(low, high))
 131
 132 cleanup_list = []
 133
 134 builddirs = {
 135     "ctdb": "ctdb",
 136     "ldb": "lib/ldb",
 137     "tdb": "lib/tdb",
 138     "talloc": "lib/talloc",
 139     "replace": "lib/replace",
 140     "tevent": "lib/tevent",
 141     "pidl": "pidl",
 142     "docs-xml": "docs-xml"
 143 }
 144
 145 ctdb_configure_params = " --enable-developer ${PREFIX}"
 146 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 147
 148 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 149 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 150 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 151 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 152 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 153 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 154 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 155
 156
 157 def format_option(name, value=None):
 158     """Format option as str list."""
 159     if value is None:  # boolean option
 160         return [name]
 161     if not isinstance(value, list):  # single value option
 162         value = [value]
 163     # repeatable option
 164     return ['{}={}'.format(name, item) for item in value]
 165
 166
 167 def make_test(
 168         cmd='make testonly',
 169         FAIL_IMMEDIATELY=1,
 170         INJECT_SELFTEST_PREFIX=1,
 171         TESTS='',
 172         include_envs=None,
 173         exclude_envs=None):
 174
 175     test_options = []
 176     if include_envs:
 177         test_options = format_option('--include-env', include_envs)
 178     if exclude_envs:
 179         test_options = format_option('--exclude-env', exclude_envs)
 180     if test_options:
 181         # join envs options to original test options
 182         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 183
 184     _options = []
 185     if FAIL_IMMEDIATELY:
 186         _options.append('FAIL_IMMEDIATELY=1')
 187     if TESTS:
 188         _options.append("TESTS='{}'".format(TESTS))
 189
 190     if INJECT_SELFTEST_PREFIX:
 191         _options.append("TEST_OPTIONS='--with-selftest-prefix={}'".format("${SELFTEST_PREFIX}"))
 192         _options.append("--directory='{}'".format("${TEST_SOURCE_DIR}"))
 193
 194     return ' '.join([cmd] + _options)
 195
 196
 197 # When updating this list, also update .gitlab-ci.yml to add the job
 198 # and to make it a dependency of 'page' for the coverage report.
 199
 200 tasks = {
 201     "ctdb": {
 202         "sequence": [
 203             ("random-sleep", random_sleep(300, 900)),
 204             ("configure", "./configure " + ctdb_configure_params),
 205             ("make", "make all"),
 206             ("install", "make install"),
 207             ("test", "make autotest"),
 208             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 209             ("clean", "make clean"),
 210         ],
 211     },
 212     "docs-xml": {
 213         "sequence": [
 214             ("random-sleep", random_sleep(300, 900)),
 215             ("autoconf", "autoconf"),
 216             ("configure", "./configure"),
 217             ("make", "make html htmlman"),
 218             ("publish-docs", PUBLISH_DOCS),
 219             ("clean", "make clean"),
 220         ],
 221     },
 222
 223     "samba-def-build": {
 224         "git-clone-required": True,
 225         "sequence": [
 226             ("configure", "./configure.developer" + samba_configure_params),
 227             ("make", "make -j"),
 228             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 229             ("chmod-R-a-w", "chmod -R a-w ."),
 230         ],
 231     },
 232
 233     "samba-mit-build": {
 234         "git-clone-required": True,
 235         "sequence": [
 236             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 237             ("make", "make -j"),
 238             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 239             ("chmod-R-a-w", "chmod -R a-w ."),
 240         ],
 241     },
 242
 243     "samba-nt4-build": {
 244         "git-clone-required": True,
 245         "sequence": [
 246             ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json" + samba_configure_params),
 247             ("make", "make -j"),
 248             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 249             ("chmod-R-a-w", "chmod -R a-w ."),
 250         ],
 251     },
 252
 253     "samba-h5l-build": {
 254         "git-clone-required": True,
 255         "sequence": [
 256             ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5" + samba_configure_params),
 257             ("make", "make -j"),
 258             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 259             ("chmod-R-a-w", "chmod -R a-w ."),
 260         ],
 261     },
 262
 263     "samba-no-opath-build": {
 264         "git-clone-required": True,
 265         "sequence": [
 266             ("configure", "ADDITIONAL_CFLAGS='-DDISABLE_OPATH=1' ./configure.developer --without-ad-dc " + samba_configure_params),
 267             ("make", "make -j"),
 268             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 269             ("chmod-R-a-w", "chmod -R a-w ."),
 270         ],
 271     },
 272
 273     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 274     "samba": {
 275         "sequence": [
 276             ("random-sleep", random_sleep(300, 900)),
 277             ("configure", "./configure.developer" + samba_configure_params),
 278             ("make", "make -j"),
 279             ("test", make_test(exclude_envs=[
 280             "none",
 281             "nt4_dc",
 282             "nt4_dc_smb1",
 283             "nt4_dc_smb1_done",
 284             "nt4_dc_schannel",
 285             "nt4_member",
 286             "ad_dc",
 287             "ad_dc_smb1",
 288             "ad_dc_smb1_done",
 289             "ad_dc_backup",
 290             "ad_dc_ntvfs",
 291             "ad_dc_default",
 292             "ad_dc_default_smb1",
 293             "ad_dc_slowtests",
 294             "ad_dc_no_nss",
 295             "ad_dc_no_ntlm",
 296             "fl2003dc",
 297             "fl2008dc",
 298             "fl2008r2dc",
 299             "ad_member",
 300             "ad_member_idmap_rid",
 301             "ad_member_idmap_ad",
 302             "ad_member_rfc2307",
 303             "chgdcpass",
 304             "vampire_2000_dc",
 305             "fl2000dc",
 306             "fileserver",
 307             "fileserver_smb1",
 308             "fileserver_smb1_done",
 309             "maptoguest",
 310             "simpleserver",
 311             "backupfromdc",
 312             "restoredc",
 313             "renamedc",
 314             "offlinebackupdc",
 315             "labdc",
 316             "preforkrestartdc",
 317             "proclimitdc",
 318             "promoted_dc",
 319             "vampire_dc",
 320             "rodc",
 321             "ad_dc_default",
 322             "ad_dc_default_smb1",
 323             "ad_dc_default_smb1_done",
 324             "ad_dc_slowtests",
 325             "schema_pair_dc",
 326             "schema_dc",
 327             "clusteredmember",
 328             ])),
 329             ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
 330             ("lcov", LCOV_CMD),
 331             ("install", "make install"),
 332             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 333             ("clean", "make clean"),
 334         ],
 335     },
 336
 337     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 338     "samba-mitkrb5": {
 339         "sequence": [
 340             ("random-sleep", random_sleep(300, 900)),
 341             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 342             ("make", "make -j"),
 343             ("test", make_test(exclude_envs=[
 344             "none",
 345             "nt4_dc",
 346             "nt4_dc_smb1",
 347             "nt4_dc_smb1_done",
 348             "nt4_dc_schannel",
 349             "nt4_member",
 350             "ad_dc",
 351             "ad_dc_smb1",
 352             "ad_dc_smb1_done",
 353             "ad_dc_backup",
 354             "ad_dc_ntvfs",
 355             "ad_dc_default",
 356             "ad_dc_default_smb1",
 357             "ad_dc_default_smb1_done",
 358             "ad_dc_slowtests",
 359             "ad_dc_no_nss",
 360             "ad_dc_no_ntlm",
 361             "fl2003dc",
 362             "fl2008dc",
 363             "fl2008r2dc",
 364             "ad_member",
 365             "ad_member_idmap_rid",
 366             "ad_member_idmap_ad",
 367             "ad_member_rfc2307",
 368             "chgdcpass",
 369             "vampire_2000_dc",
 370             "fl2000dc",
 371             "fileserver",
 372             "fileserver_smb1",
 373             "fileserver_smb1_done",
 374             "maptoguest",
 375             "simpleserver",
 376             "backupfromdc",
 377             "restoredc",
 378             "renamedc",
 379             "offlinebackupdc",
 380             "labdc",
 381             "preforkrestartdc",
 382             "proclimitdc",
 383             "promoted_dc",
 384             "vampire_dc",
 385             "rodc",
 386             "ad_dc_default",
 387             "ad_dc_default_smb1",
 388             "ad_dc_default_smb1_done",
 389             "ad_dc_slowtests",
 390             "schema_pair_dc",
 391             "schema_dc",
 392             "clusteredmember",
 393             ])),
 394             ("lcov", LCOV_CMD),
 395             ("install", "make install"),
 396             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 397             ("clean", "make clean"),
 398         ],
 399     },
 400
 401     "samba-nt4": {
 402         "dependency": "samba-nt4-build",
 403         "sequence": [
 404             ("random-sleep", random_sleep(300, 900)),
 405             ("test", make_test(include_envs=[
 406             "nt4_dc",
 407             "nt4_dc_smb1",
 408             "nt4_dc_smb1_done",
 409             "nt4_dc_schannel",
 410             "nt4_member",
 411             "simpleserver",
 412             ])),
 413             ("lcov", LCOV_CMD),
 414             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 415         ],
 416     },
 417
 418     "samba-fileserver": {
 419         "dependency": "samba-h5l-build",
 420         "sequence": [
 421             ("random-sleep", random_sleep(300, 900)),
 422             ("test", make_test(include_envs=[
 423             "fileserver",
 424             "fileserver_smb1",
 425             "fileserver_smb1_done",
 426             "maptoguest",
 427             "ktest", # ktest is also tested in samba and samba-mitkrb5
 428                      # but is tested here against a system Heimdal
 429             ])),
 430             ("lcov", LCOV_CMD),
 431             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 432         ],
 433     },
 434
 435     "samba-admem": {
 436         "dependency": "samba-def-build",
 437         "sequence": [
 438             ("random-sleep", random_sleep(300, 900)),
 439             ("test", make_test(include_envs=[
 440             "ad_member",
 441             "ad_member_idmap_rid",
 442             "ad_member_idmap_ad",
 443             "ad_member_rfc2307",
 444             "ad_member_offline_logon",
 445             ])),
 446             ("lcov", LCOV_CMD),
 447             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 448         ],
 449     },
 450
 451     "samba-no-opath1": {
 452         "dependency": "samba-no-opath-build",
 453         "sequence": [
 454             ("random-sleep", random_sleep(300, 900)),
 455             ("test", make_test(
 456                 cmd="make testonly DISABLE_OPATH=1",
 457                 include_envs=[
 458                 "nt4_dc",
 459                 "nt4_dc_smb1",
 460                 "nt4_dc_smb1_done",
 461                 "nt4_dc_schannel",
 462                 "nt4_member",
 463                 "simpleserver",
 464                 ])),
 465             ("lcov", LCOV_CMD),
 466             ("check-clean-tree", "script/clean-source-tree.sh"),
 467         ],
 468     },
 469
 470     "samba-no-opath2": {
 471         "dependency": "samba-no-opath-build",
 472         "sequence": [
 473             ("random-sleep", random_sleep(300, 900)),
 474             ("test", make_test(
 475                 cmd="make testonly DISABLE_OPATH=1",
 476                 include_envs=[
 477                 "fileserver",
 478                 "fileserver_smb1",
 479                 "fileserver_smb1_done",
 480                 ])),
 481             ("lcov", LCOV_CMD),
 482             ("check-clean-tree", "script/clean-source-tree.sh"),
 483         ],
 484     },
 485
 486     "samba-ad-dc-1": {
 487         "dependency": "samba-def-build",
 488         "sequence": [
 489             ("random-sleep", random_sleep(1, 1)),
 490             ("test", make_test(include_envs=[
 491             "ad_dc",
 492             "ad_dc_smb1",
 493             "ad_dc_smb1_done",
 494             "ad_dc_no_nss",
 495             "ad_dc_no_ntlm",
 496             ])),
 497             ("lcov", LCOV_CMD),
 498             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 499         ],
 500     },
 501
 502     "samba-ad-dc-2": {
 503         "dependency": "samba-def-build",
 504         "sequence": [
 505             ("random-sleep", random_sleep(1, 1)),
 506             ("test", make_test(include_envs=[
 507             "vampire_dc",
 508             "vampire_2000_dc",
 509             "rodc",
 510             ])),
 511             ("lcov", LCOV_CMD),
 512             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 513         ],
 514     },
 515
 516     "samba-ad-dc-3": {
 517         "dependency": "samba-def-build",
 518         "sequence": [
 519             ("random-sleep", random_sleep(1, 1)),
 520             ("test", make_test(include_envs=[
 521             "promoted_dc",
 522             "chgdcpass",
 523             "preforkrestartdc",
 524             "proclimitdc",
 525             ])),
 526             ("lcov", LCOV_CMD),
 527             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 528         ],
 529     },
 530
 531     "samba-ad-dc-4a": {
 532         "dependency": "samba-def-build",
 533         "sequence": [
 534             ("random-sleep", random_sleep(1, 1)),
 535             ("test", make_test(include_envs=[
 536             "fl2000dc",
 537             "fl2003dc",
 538             ])),
 539             ("lcov", LCOV_CMD),
 540             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 541         ],
 542     },
 543     "samba-ad-dc-4b": {
 544         "dependency": "samba-def-build",
 545         "sequence": [
 546             ("random-sleep", random_sleep(1, 1)),
 547             ("test", make_test(include_envs=[
 548             "fl2008dc",
 549             "fl2008r2dc",
 550             ])),
 551             ("lcov", LCOV_CMD),
 552             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 553         ],
 554     },
 555
 556     "samba-ad-dc-5": {
 557         "dependency": "samba-def-build",
 558         "sequence": [
 559             ("random-sleep", random_sleep(1, 1)),
 560             ("test", make_test(include_envs=[
 561             "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
 562             ("lcov", LCOV_CMD),
 563             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 564         ],
 565     },
 566
 567     "samba-ad-dc-6": {
 568         "dependency": "samba-def-build",
 569         "sequence": [
 570             ("random-sleep", random_sleep(1, 1)),
 571             ("test", make_test(include_envs=["ad_dc_slowtests", "ad_dc_backup"])),
 572             ("lcov", LCOV_CMD),
 573             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 574         ],
 575     },
 576
 577     "samba-schemaupgrade": {
 578         "dependency": "samba-def-build",
 579         "sequence": [
 580             ("random-sleep", random_sleep(1, 1)),
 581             ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 582             ("lcov", LCOV_CMD),
 583             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 584         ],
 585     },
 586
 587     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 588     # This is currently the longest task, so we don't randomly delay it.
 589     "samba-ad-dc-ntvfs": {
 590         "dependency": "samba-def-build",
 591         "sequence": [
 592             ("random-sleep", random_sleep(1, 1)),
 593             ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 594             ("lcov", LCOV_CMD),
 595             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 596         ],
 597     },
 598
 599     # Test fips compliance
 600     "samba-fips": {
 601         "dependency": "samba-mit-build",
 602         "sequence": [
 603             ("random-sleep", random_sleep(1, 1)),
 604             ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
 605             # TODO: This seems to generate only an empty samba-fips.info ("lcov", LCOV_CMD),
 606             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 607         ],
 608     },
 609
 610     # run the backup/restore testenvs separately as they're fairly standalone
 611     # (and CI seems to max out at ~3 different DCs running at once)
 612     "samba-ad-back1": {
 613         "dependency": "samba-def-build",
 614         "sequence": [
 615             ("random-sleep", random_sleep(300, 900)),
 616             ("test", make_test(include_envs=[
 617             "backupfromdc",
 618             "restoredc",
 619             "renamedc",
 620             ])),
 621             ("lcov", LCOV_CMD),
 622             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 623         ],
 624     },
 625     "samba-ad-back2": {
 626         "dependency": "samba-def-build",
 627         "sequence": [
 628             ("random-sleep", random_sleep(300, 900)),
 629             ("test", make_test(include_envs=[
 630             "backupfromdc",
 631             "offlinebackupdc",
 632             "labdc",
 633             ])),
 634             ("lcov", LCOV_CMD),
 635             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 636         ],
 637     },
 638
 639     "samba-admem-mit": {
 640         "dependency": "samba-mit-build",
 641         "sequence": [
 642             ("random-sleep", random_sleep(1, 1)),
 643             ("test", make_test(include_envs=[
 644             "ad_member",
 645             "ad_member_idmap_rid",
 646             "ad_member_idmap_ad",
 647             "ad_member_rfc2307",
 648             "ad_member_offline_logon",
 649             ])),
 650             ("lcov", LCOV_CMD),
 651             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 652         ],
 653     },
 654
 655     "samba-ad-dc-1-mitkrb5": {
 656         "dependency": "samba-mit-build",
 657         "sequence": [
 658             ("random-sleep", random_sleep(1, 1)),
 659             ("test", make_test(include_envs=[
 660             "ad_dc",
 661             "ad_dc_smb1",
 662             "ad_dc_smb1_done",
 663             "ad_dc_no_nss",
 664             "ad_dc_no_ntlm",
 665             ])),
 666             ("lcov", LCOV_CMD),
 667             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 668         ],
 669     },
 670
 671     "samba-ad-dc-4a-mitkrb5": {
 672         "dependency": "samba-mit-build",
 673         "sequence": [
 674             ("random-sleep", random_sleep(1, 1)),
 675             ("test", make_test(include_envs=[
 676             "fl2000dc",
 677             "fl2003dc",
 678             ])),
 679             ("lcov", LCOV_CMD),
 680             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 681         ],
 682     },
 683     "samba-ad-dc-4b-mitkrb5": {
 684         "dependency": "samba-mit-build",
 685         "sequence": [
 686             ("random-sleep", random_sleep(1, 1)),
 687             ("test", make_test(include_envs=[
 688             "fl2000dc",
 689             "fl2003dc",
 690             ])),
 691             ("lcov", LCOV_CMD),
 692             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 693         ],
 694     },
 695
 696     "samba-test-only": {
 697         "sequence": [
 698             ("configure", "./configure.developer  --abi-check-disable" + samba_configure_params),
 699             ("make", "make -j"),
 700             ("test", make_test(TESTS="${TESTS}")),
 701             ("lcov", LCOV_CMD),
 702         ],
 703     },
 704
 705     # Test cross-compile infrastructure
 706     "samba-xc": {
 707         "sequence": [
 708             ("random-sleep", random_sleep(900, 1500)),
 709             ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 710             ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 711             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 712             ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
 713             ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 714             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 715             ("compare-results", "script/compare_cc_results.py "
 716             "./bin/c4che/default{} "
 717             "./bin-xe/c4che/default{} "
 718             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 719             ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
 720             ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
 721             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
 722             ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
 723             " = \"'1234'\"".format(CACHE_SUFFIX)),
 724             ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
 725             ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
 726             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
 727             " ; test $? -ne 0"),
 728         ],
 729     },
 730
 731     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 732     "samba-o3": {
 733         "sequence": [
 734             ("random-sleep", random_sleep(300, 900)),
 735             ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --abi-check-disable" + samba_configure_params),
 736             ("make", "make -j"),
 737             ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 738             ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 739             ("lcov", LCOV_CMD),
 740             ("install", "make install"),
 741             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 742             ("clean", "make clean"),
 743         ],
 744     },
 745
 746     "samba-ctdb": {
 747         "sequence": [
 748             ("random-sleep", random_sleep(900, 1500)),
 749
 750         # make sure we have tdb around:
 751             ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 752             ("tdb-make", "cd lib/tdb && make"),
 753             ("tdb-install", "cd lib/tdb && make install"),
 754
 755         # build samba with cluster support (also building ctdb):
 756             ("samba-configure",
 757          "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
 758          "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
 759          "./configure.developer ${PREFIX} "
 760          "--with-selftest-prefix=./bin/ab "
 761          "--enable-clangdb "
 762          "--with-cluster-support "
 763          "--without-ad-dc "
 764          "--bundled-libraries=!tdb"),
 765             ("samba-make", "make"),
 766             ("samba-check", "./bin/smbd --configfile=/dev/null -b | grep CLUSTER_SUPPORT"),
 767             ("samba-install", "make install"),
 768             ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 769
 770             ("test", make_test(
 771                 cmd='make test',
 772                 INJECT_SELFTEST_PREFIX=0,
 773                 include_envs=["clusteredmember"])
 774             ),
 775
 776         # clean up:
 777             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 778             ("clean", "make clean"),
 779             ("ctdb-clean", "cd ./ctdb && make clean"),
 780         ],
 781     },
 782
 783     "samba-libs": {
 784         "sequence": [
 785             ("random-sleep", random_sleep(300, 900)),
 786             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 787             ("talloc-make", "cd lib/talloc && make"),
 788             ("talloc-install", "cd lib/talloc && make install"),
 789
 790             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 791             ("tdb-make", "cd lib/tdb && make"),
 792             ("tdb-install", "cd lib/tdb && make install"),
 793
 794             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 795             ("tevent-make", "cd lib/tevent && make"),
 796             ("tevent-install", "cd lib/tevent && make install"),
 797
 798             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
 799             ("ldb-make", "cd lib/ldb && make"),
 800             ("ldb-install", "cd lib/ldb && make install"),
 801
 802             ("nondevel-configure", "./configure ${PREFIX}"),
 803             ("nondevel-make", "make -j"),
 804             ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 805             ("nondevel-install", "make install"),
 806             ("nondevel-dist", "make dist"),
 807
 808         # retry with all modules shared
 809             ("allshared-distclean", "make distclean"),
 810             ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 811             ("allshared-make", "make -j"),
 812         ],
 813     },
 814
 815     "samba-fuzz": {
 816         "sequence": [
 817         # build the fuzzers (static) via the oss-fuzz script
 818             ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
 819             ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ADDITIONAL_LDFLAGS='-fuse-ld=bfd' ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl-fuzzer"),
 820         ],
 821     },
 822
 823     # * Test smbd and smbtorture can build semi-static
 824     #
 825     # * Test Samba without python still builds.
 826     #
 827     # When this test fails due to more use of Python, the expectations
 828     # is that the newly failing part of the code should be disabled
 829     # when --disable-python is set (rather than major work being done
 830     # to support this environment).
 831     #
 832     # The target here is for vendors shipping a minimal smbd.
 833     "samba-minimal-smbd": {
 834         "sequence": [
 835             ("random-sleep", random_sleep(300, 900)),
 836
 837         # build with all modules static
 838             ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
 839             ("allstatic-make", "make -j"),
 840             ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 841             ("allstatic-lcov", LCOV_CMD),
 842
 843         # retry with nonshared smbd and smbtorture
 844             ("nonshared-distclean", "make distclean"),
 845             ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
 846             ("nonshared-make", "make -j"),
 847             # TODO ("nonshared-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 848             # TODO ("nonshared-lcov", LCOV_CMD),
 849
 850             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 851             ("clean", "make clean"),
 852         ],
 853     },
 854
 855     "samba-nopython": {
 856         "sequence": [
 857             ("random-sleep", random_sleep(300, 900)),
 858
 859             ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
 860             ("make", "make -j"),
 861             ("find-python", "script/find_python.sh ${PREFIX}"),
 862             ("test", "make test-nopython"),
 863             ("lcov", LCOV_CMD),
 864             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 865             ("clean", "make clean"),
 866
 867             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 868             ("talloc-make", "cd lib/talloc && make"),
 869             ("talloc-install", "cd lib/talloc && make install"),
 870
 871             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 872             ("tdb-make", "cd lib/tdb && make"),
 873             ("tdb-install", "cd lib/tdb && make install"),
 874
 875             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 876             ("tevent-make", "cd lib/tevent && make"),
 877             ("tevent-install", "cd lib/tevent && make install"),
 878
 879             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 880             ("ldb-make", "cd lib/ldb && make"),
 881             ("ldb-install", "cd lib/ldb && make install"),
 882
 883         # retry against installed library packages, but no required modules
 884             ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc  --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
 885             ("libs-make", "make -j"),
 886             ("libs-install", "make install"),
 887             ("libs-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 888             ("libs-clean", "make clean"),
 889
 890         ],
 891     },
 892
 893     "ldb": {
 894         "sequence": [
 895             ("random-sleep", random_sleep(60, 600)),
 896             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 897             ("make", "make"),
 898             ("install", "make install"),
 899             ("test", "make test"),
 900             ("lcov", LCOV_CMD),
 901             ("clean", "make clean"),
 902             ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
 903             ("make-no-lmdb", "make"),
 904             ("test-no-lmdb", "make test"),
 905             ("lcov-no-lmdb", LCOV_CMD),
 906             ("install-no-lmdb", "make install"),
 907             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 908             ("distcheck", "make distcheck"),
 909             ("clean", "make clean"),
 910         ],
 911     },
 912
 913     "tdb": {
 914         "sequence": [
 915             ("random-sleep", random_sleep(60, 600)),
 916             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 917             ("make", "make"),
 918             ("install", "make install"),
 919             ("test", "make test"),
 920             ("lcov", LCOV_CMD),
 921             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 922             ("distcheck", "make distcheck"),
 923             ("clean", "make clean"),
 924         ],
 925     },
 926
 927     "talloc": {
 928         "sequence": [
 929             ("random-sleep", random_sleep(60, 600)),
 930             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 931             ("make", "make"),
 932             ("install", "make install"),
 933             ("test", "make test"),
 934             ("lcov", LCOV_CMD),
 935             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 936             ("distcheck", "make distcheck"),
 937             ("clean", "make clean"),
 938         ],
 939     },
 940
 941     "replace": {
 942         "sequence": [
 943             ("random-sleep", random_sleep(60, 600)),
 944             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 945             ("make", "make"),
 946             ("install", "make install"),
 947             ("test", "make test"),
 948             ("lcov", LCOV_CMD),
 949             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 950             ("distcheck", "make distcheck"),
 951             ("clean", "make clean"),
 952         ],
 953     },
 954
 955     "tevent": {
 956         "sequence": [
 957             ("random-sleep", random_sleep(60, 600)),
 958             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 959             ("make", "make"),
 960             ("install", "make install"),
 961             ("test", "make test"),
 962             ("lcov", LCOV_CMD),
 963             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 964             ("distcheck", "make distcheck"),
 965             ("clean", "make clean"),
 966         ],
 967     },
 968
 969     "pidl": {
 970         "git-clone-required": True,
 971         "sequence": [
 972             ("random-sleep", random_sleep(60, 600)),
 973             ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
 974             ("touch", "touch *.yp"),
 975             ("make", "make"),
 976             ("test", "make test"),
 977             ("install", "make install"),
 978             ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
 979             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 980             ("clean", "make clean"),
 981         ],
 982     },
 983
 984     # these are useful for debugging autobuild
 985     "pass": {
 986         "sequence": [
 987             ("pass", 'echo passing && /bin/true'),
 988         ],
 989     },
 990     "fail": {
 991         "sequence": [
 992             ("fail", 'echo failing && /bin/false'),
 993         ],
 994     },
 995 }
 996
 997 defaulttasks = list(tasks.keys())
 998
 999 defaulttasks.remove("pass")
1000 defaulttasks.remove("fail")
1001 defaulttasks.remove("samba-def-build")
1002 defaulttasks.remove("samba-nt4-build")
1003 defaulttasks.remove("samba-mit-build")
1004 defaulttasks.remove("samba-h5l-build")
1005 defaulttasks.remove("samba-no-opath-build")
1006 defaulttasks.remove("samba-test-only")
1007 defaulttasks.remove("samba-fuzz")
1008 defaulttasks.remove("samba-fips")
1009 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
1010     defaulttasks.remove("samba-o3")
1011
1012
1013 def do_print(msg):
1014     print("%s" % msg)
1015     sys.stdout.flush()
1016     sys.stderr.flush()
1017
1018
1019 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
1020     if show is None:
1021         show = options.verbose
1022     if show:
1023         do_print("Running: '%s' in '%s'" % (cmd, dir))
1024     if output:
1025         out = check_output([cmd], shell=True, cwd=dir)
1026         return out.decode(encoding='utf-8', errors='backslashreplace')
1027     elif checkfail:
1028         return check_call(cmd, shell=True, cwd=dir)
1029     else:
1030         return call(cmd, shell=True, cwd=dir)
1031
1032 def rmdir_force(dirname, re_raise=True):
1033     try:
1034         run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % (
1035                 dirname, dirname, dirname), output=True, show=True)
1036     except CalledProcessError as e:
1037         do_print("Failed: '%s'" % (str(e)))
1038         run_cmd("tree %s" % dirname, output=True, show=True)
1039         if re_raise:
1040             raise
1041         return False
1042     return True
1043
1044 class builder(object):
1045     '''handle build of one directory'''
1046
1047     def __init__(self, name, definition):
1048         self.name = name
1049         self.dir = builddirs.get(name, '.')
1050         self.tag = self.name.replace('/', '_')
1051         self.definition = definition
1052         self.sequence = definition["sequence"]
1053         self.git_clone_required = False
1054         if "git-clone-required" in definition:
1055             self.git_clone_required = bool(definition["git-clone-required"])
1056         self.proc = None
1057         self.done = False
1058         self.next = 0
1059         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
1060         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
1061         if options.verbose:
1062             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
1063             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
1064         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
1065         self.stdout = open(self.stdout_path, 'w')
1066         self.stderr = open(self.stderr_path, 'w')
1067         self.stdin  = open("/dev/null", 'r')
1068         self.builder_dir = "%s/%s" % (testbase, self.tag)
1069         self.test_source_dir = self.builder_dir
1070         self.cwd = "%s/%s" % (self.builder_dir, self.dir)
1071         self.selftest_prefix = "%s/bin/ab" % (self.cwd)
1072         self.prefix = "%s/%s" % (test_prefix, self.tag)
1073         self.consumers = []
1074         self.producer = None
1075
1076         if self.git_clone_required:
1077             assert "dependency" not in definition
1078
1079     def mark_existing(self):
1080         do_print('%s: Mark as existing dependency' % self.name)
1081         self.next = len(self.sequence)
1082         self.done = True
1083
1084     def add_consumer(self, consumer):
1085         do_print("%s: add consumer: %s" % (self.name, consumer.name))
1086         consumer.producer = self
1087         consumer.test_source_dir = self.test_source_dir
1088         self.consumers.append(consumer)
1089
1090     def start_next(self):
1091         if self.producer is not None:
1092             if not self.producer.done:
1093                 do_print("%s: Waiting for producer: %s" % (self.name, self.producer.name))
1094                 return
1095
1096         if self.next == 0:
1097             rmdir_force(self.builder_dir)
1098             rmdir_force(self.prefix)
1099             if self.producer is not None:
1100                 run_cmd("mkdir %s" % (self.builder_dir), dir=test_master, show=True)
1101             elif not self.git_clone_required:
1102                 run_cmd("cp -R -a -l %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1103             else:
1104                 run_cmd("git clone --recursive --shared %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1105
1106         if self.next == len(self.sequence):
1107             if not self.done:
1108                 do_print('%s: Completed OK' % self.name)
1109                 self.done = True
1110             if not options.nocleanup and len(self.consumers) == 0:
1111                 do_print('%s: Cleaning up' % self.name)
1112                 rmdir_force(self.builder_dir)
1113                 rmdir_force(self.prefix)
1114             for consumer in self.consumers:
1115                 if consumer.next != 0:
1116                     continue
1117                 do_print('%s: Starting consumer %s' % (self.name, consumer.name))
1118                 consumer.start_next()
1119             if self.producer is not None:
1120                 self.producer.consumers.remove(self)
1121                 assert self.producer.done
1122                 self.producer.start_next()
1123             do_print('%s: Remaining consumers %u' % (self.name, len(self.consumers)))
1124             return
1125         (self.stage, self.cmd) = self.sequence[self.next]
1126         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
1127         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
1128         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
1129         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
1130         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
1131         self.cmd = self.cmd.replace("${SELFTEST_PREFIX}", self.selftest_prefix)
1132         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
1133         self.cmd = self.cmd.replace("${NAME}", self.name)
1134         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
1135         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
1136         self.proc = Popen(self.cmd, shell=True,
1137                           close_fds=True, cwd=self.cwd,
1138                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
1139         self.next += 1
1140
1141 def expand_dependencies(n):
1142     deps = list()
1143     if "dependency" in tasks[n]:
1144         depname = tasks[n]["dependency"]
1145         assert depname in tasks
1146         sdeps = expand_dependencies(depname)
1147         assert n not in sdeps
1148         for sdep in sdeps:
1149             deps.append(sdep)
1150         deps.append(depname)
1151     return deps
1152
1153
1154 class buildlist(object):
1155     '''handle build of multiple directories'''
1156
1157     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
1158         self.tail_proc = None
1159         self.retry = None
1160         if not tasknames:
1161             if options.restrict_tests:
1162                 tasknames = ["samba-test-only"]
1163             else:
1164                 tasknames = defaulttasks
1165
1166         given_tasknames = tasknames.copy()
1167         implicit_tasknames = []
1168         for n in given_tasknames:
1169             deps = expand_dependencies(n)
1170             for dep in deps:
1171                 if dep in given_tasknames:
1172                     continue
1173                 if dep in implicit_tasknames:
1174                     continue
1175                 implicit_tasknames.append(dep)
1176
1177         tasknames = implicit_tasknames.copy()
1178         tasknames.extend(given_tasknames)
1179         do_print("given_tasknames: %s" % given_tasknames)
1180         do_print("implicit_tasknames: %s" % implicit_tasknames)
1181         do_print("tasknames: %s" % tasknames)
1182         self.tlist = [builder(n, tasks[n]) for n in tasknames]
1183
1184         if options.retry:
1185             rebase_remote = "rebaseon"
1186             retry_task = {
1187                     "git-clone-required": True,
1188                     "sequence": [
1189                             ("retry",
1190                             '''set -e
1191                             git remote add -t %s %s %s
1192                             git fetch %s
1193                             while :; do
1194                               sleep 60
1195                               git describe %s/%s > old_remote_branch.desc
1196                               git fetch %s
1197                               git describe %s/%s > remote_branch.desc
1198                               diff old_remote_branch.desc remote_branch.desc
1199                             done
1200                            ''' % (
1201                                rebase_branch, rebase_remote, rebase_url,
1202                                rebase_remote,
1203                                rebase_remote, rebase_branch,
1204                                rebase_remote,
1205                                rebase_remote, rebase_branch
1206                             ))]}
1207
1208             self.retry = builder('retry', retry_task)
1209             self.need_retry = False
1210
1211         if options.skip_dependencies:
1212             for b in self.tlist:
1213                 if b.name in implicit_tasknames:
1214                     b.mark_existing()
1215
1216         for b in self.tlist:
1217             do_print("b.name=%s" % b.name)
1218             if "dependency" not in b.definition:
1219                 continue
1220             depname = b.definition["dependency"]
1221             do_print("b.name=%s: dependency:%s" % (b.name, depname))
1222             for p in self.tlist:
1223                 if p.name == depname:
1224                     p.add_consumer(b)
1225
1226     def kill_kids(self):
1227         if self.tail_proc is not None:
1228             self.tail_proc.terminate()
1229             self.tail_proc.wait()
1230             self.tail_proc = None
1231         if self.retry is not None:
1232             self.retry.proc.terminate()
1233             self.retry.proc.wait()
1234             self.retry = None
1235         for b in self.tlist:
1236             if b.proc is not None:
1237                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
1238                 b.proc.terminate()
1239                 b.proc.wait()
1240                 b.proc = None
1241
1242     def wait_one(self):
1243         while True:
1244             none_running = True
1245             for b in self.tlist:
1246                 if b.proc is None:
1247                     continue
1248                 none_running = False
1249                 b.status = b.proc.poll()
1250                 if b.status is None:
1251                     continue
1252                 b.proc = None
1253                 return b
1254             if options.retry:
1255                 ret = self.retry.proc.poll()
1256                 if ret is not None:
1257                     self.need_retry = True
1258                     self.retry = None
1259                     return None
1260             if none_running:
1261                 return None
1262             time.sleep(0.1)
1263
1264     def run(self):
1265         for b in self.tlist:
1266             b.start_next()
1267         if options.retry:
1268             self.retry.start_next()
1269         while True:
1270             b = self.wait_one()
1271             if options.retry and self.need_retry:
1272                 self.kill_kids()
1273                 do_print("retry needed")
1274                 return (0, None, None, None, "retry")
1275             if b is None:
1276                 break
1277             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
1278                 self.kill_kids()
1279                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
1280             b.start_next()
1281         self.kill_kids()
1282         return (0, None, None, None, "All OK")
1283
1284     def write_system_info(self, filename):
1285         with open(filename, 'w') as f:
1286             for cmd in ['uname -a',
1287                         'lsb_release -a',
1288                         'free',
1289                         'mount',
1290                         'cat /proc/cpuinfo',
1291                         'cc --version',
1292                         'df -m .',
1293                         'df -m %s' % testbase]:
1294                 try:
1295                     out = run_cmd(cmd, output=True, checkfail=False)
1296                 except CalledProcessError as e:
1297                     out = "<failed: %s>" % str(e)
1298                 print('### %s' % cmd, file=f)
1299                 print(out, file=f)
1300                 print(file=f)
1301
1302     def tarlogs(self, fname):
1303         with tarfile.open(fname, "w:gz") as tar:
1304             for b in self.tlist:
1305                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
1306                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
1307             if os.path.exists("autobuild.log"):
1308                 tar.add("autobuild.log")
1309             filename = 'system-info.txt'
1310             self.write_system_info(filename)
1311             tar.add(filename)
1312
1313     def remove_logs(self):
1314         for b in self.tlist:
1315             os.unlink(b.stdout_path)
1316             os.unlink(b.stderr_path)
1317
1318     def start_tail(self):
1319         cmd = ["tail", "-f"]
1320         for b in self.tlist:
1321             cmd.append(b.stdout_path)
1322             cmd.append(b.stderr_path)
1323         self.tail_proc = Popen(cmd, close_fds=True)
1324
1325
1326 def cleanup(do_raise=False):
1327     if options.nocleanup:
1328         return
1329     run_cmd("stat %s || true" % test_tmpdir, show=True)
1330     run_cmd("stat %s" % testbase, show=True)
1331     do_print("Cleaning up %r" % cleanup_list)
1332     for d in cleanup_list:
1333         ok = rmdir_force(d, re_raise=False)
1334         if ok:
1335             continue
1336         if os.path.isdir(d):
1337             do_print("Killing, waiting and retry")
1338             run_cmd("killbysubdir %s > /dev/null 2>&1" % d, checkfail=False)
1339         else:
1340             do_print("Waiting and retry")
1341         time.sleep(1)
1342         rmdir_force(d, re_raise=do_raise)
1343
1344
1345 def daemonize(logfile):
1346     pid = os.fork()
1347     if pid == 0:  # Parent
1348         os.setsid()
1349         pid = os.fork()
1350         if pid != 0:  # Actual daemon
1351             os._exit(0)
1352     else:  # Grandparent
1353         os._exit(0)
1354
1355     import resource      # Resource usage information.
1356     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1357     if maxfd == resource.RLIM_INFINITY:
1358         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1359     for fd in range(0, maxfd):
1360         try:
1361             os.close(fd)
1362         except OSError:
1363             pass
1364     os.open(logfile, os.O_RDWR | os.O_CREAT)
1365     os.dup2(0, 1)
1366     os.dup2(0, 2)
1367
1368
1369 def write_pidfile(fname):
1370     '''write a pid file, cleanup on exit'''
1371     with open(fname, mode='w') as f:
1372         f.write("%u\n" % os.getpid())
1373
1374
1375 def rebase_tree(rebase_url, rebase_branch="master"):
1376     rebase_remote = "rebaseon"
1377     do_print("Rebasing on %s" % rebase_url)
1378     run_cmd("git describe HEAD", show=True, dir=test_master)
1379     run_cmd("git remote add -t %s %s %s" %
1380             (rebase_branch, rebase_remote, rebase_url),
1381             show=True, dir=test_master)
1382     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1383     if options.fix_whitespace:
1384         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1385                 (rebase_remote, rebase_branch),
1386                 show=True, dir=test_master)
1387     else:
1388         run_cmd("git rebase --force-rebase %s/%s" %
1389                 (rebase_remote, rebase_branch),
1390                 show=True, dir=test_master)
1391     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1392                    (rebase_remote, rebase_branch),
1393                    dir=test_master, output=True)
1394     if diff == '':
1395         do_print("No differences between HEAD and %s/%s - exiting" %
1396                  (rebase_remote, rebase_branch))
1397         sys.exit(0)
1398     run_cmd("git describe %s/%s" %
1399             (rebase_remote, rebase_branch),
1400             show=True, dir=test_master)
1401     run_cmd("git describe HEAD", show=True, dir=test_master)
1402     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1403             (rebase_remote, rebase_branch),
1404             show=True, dir=test_master)
1405
1406
1407 def push_to(push_url, push_branch="master"):
1408     push_remote = "pushto"
1409     do_print("Pushing to %s" % push_url)
1410     if options.mark:
1411         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1412         run_cmd("git commit --amend -c HEAD", dir=test_master)
1413         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1414         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1415     run_cmd("git remote add -t %s %s %s" %
1416             (push_branch, push_remote, push_url),
1417             show=True, dir=test_master)
1418     run_cmd("git push %s +HEAD:%s" %
1419             (push_remote, push_branch),
1420             show=True, dir=test_master)
1421
1422
1423 def send_email(subject, text, log_tar):
1424     if options.email is None:
1425         do_print("not sending email because the recipient is not set")
1426         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1427                  (subject, text))
1428         return
1429     outer = MIMEMultipart()
1430     outer['Subject'] = subject
1431     outer['To'] = options.email
1432     outer['From'] = options.email_from
1433     outer['Date'] = email.utils.formatdate(localtime=True)
1434     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1435     outer.attach(MIMEText(text, 'plain', 'utf-8'))
1436     if options.attach_logs:
1437         with open(log_tar, 'rb') as fp:
1438             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1439         # Set the filename parameter
1440         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1441         outer.attach(msg)
1442     content = outer.as_string()
1443     s = smtplib.SMTP(options.email_server)
1444     email_user = os.getenv('SMTP_USERNAME')
1445     email_password = os.getenv('SMTP_PASSWORD')
1446     if email_user is not None:
1447         s.starttls()
1448         s.login(email_user, email_password)
1449
1450     s.sendmail(options.email_from, [options.email], content)
1451     s.set_debuglevel(1)
1452     s.quit()
1453
1454
1455 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1456                   elapsed_time, log_base=None, add_log_tail=True):
1457     '''send an email to options.email about the failure'''
1458     elapsed_minutes = elapsed_time / 60.0
1459     if log_base is None:
1460         log_base = gitroot
1461     text = '''
1462 Dear Developer,
1463
1464 Your autobuild on %s failed after %.1f minutes
1465 when trying to test %s with the following error:
1466
1467    %s
1468
1469 the autobuild has been abandoned. Please fix the error and resubmit.
1470
1471 A summary of the autobuild process is here:
1472
1473   %s/autobuild.log
1474 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1475
1476     if options.restrict_tests:
1477         text += """
1478 The build was restricted to tests matching %s\n""" % options.restrict_tests
1479
1480     if failed_task != 'rebase':
1481         text += '''
1482 You can see logs of the failed task here:
1483
1484   %s/%s.stdout
1485   %s/%s.stderr
1486
1487 or you can get full logs of all tasks in this job here:
1488
1489   %s/logs.tar.gz
1490
1491 The top commit for the tree that was built was:
1492
1493 %s
1494
1495 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1496
1497     if add_log_tail:
1498         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1499         lines = f.readlines()
1500         log_tail = "".join(lines[-50:])
1501         num_lines = len(lines)
1502         if num_lines < 50:
1503             # Also include stderr (compile failures) if < 50 lines of stdout
1504             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1505             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1506
1507         text += '''
1508 The last 50 lines of log messages:
1509
1510 %s
1511     ''' % log_tail
1512         f.close()
1513
1514     logs = os.path.join(gitroot, 'logs.tar.gz')
1515     send_email('autobuild[%s] failure on %s for task %s during %s'
1516                % (options.branch, platform.node(), failed_task, failed_stage),
1517                text, logs)
1518
1519
1520 def email_success(elapsed_time, log_base=None):
1521     '''send an email to options.email about a successful build'''
1522     if log_base is None:
1523         log_base = gitroot
1524     text = '''
1525 Dear Developer,
1526
1527 Your autobuild on %s has succeeded after %.1f minutes.
1528
1529 ''' % (platform.node(), elapsed_time / 60.)
1530
1531     if options.restrict_tests:
1532         text += """
1533 The build was restricted to tests matching %s\n""" % options.restrict_tests
1534
1535     if options.keeplogs:
1536         text += '''
1537
1538 you can get full logs of all tasks in this job here:
1539
1540   %s/logs.tar.gz
1541
1542 ''' % log_base
1543
1544     text += '''
1545 The top commit for the tree that was built was:
1546
1547 %s
1548 ''' % top_commit_msg
1549
1550     logs = os.path.join(gitroot, 'logs.tar.gz')
1551     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1552                text, logs)
1553
1554
1555 # get the top commit message, for emails
1556 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1557
1558 try:
1559     if options.skip_dependencies:
1560         run_cmd("stat %s" % testbase, dir=testbase, output=True)
1561     else:
1562         os.makedirs(testbase)
1563 except Exception as reason:
1564     raise Exception("Unable to create %s : %s" % (testbase, reason))
1565 cleanup_list.append(testbase)
1566
1567 if options.daemon:
1568     logfile = os.path.join(testbase, "log")
1569     do_print("Forking into the background, writing progress to %s" % logfile)
1570     daemonize(logfile)
1571
1572 write_pidfile(gitroot + "/autobuild.pid")
1573
1574 start_time = time.time()
1575
1576 while True:
1577     try:
1578         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1579         os.makedirs(test_tmpdir)
1580         # The waf uninstall code removes empty directories all the way
1581         # up the tree.  Creating a file in test_tmpdir stops it from
1582         # being removed.
1583         run_cmd("touch %s" % os.path.join(test_tmpdir,
1584                                           ".directory-is-not-empty"), show=True)
1585         run_cmd("stat %s" % test_tmpdir, show=True)
1586         run_cmd("stat %s" % testbase, show=True)
1587         if options.skip_dependencies:
1588             run_cmd("stat %s" % test_master, dir=testbase, output=True)
1589         else:
1590             run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1591     except Exception:
1592         cleanup()
1593         raise
1594
1595     try:
1596         if options.rebase is not None:
1597             rebase_tree(options.rebase, rebase_branch=options.branch)
1598     except Exception:
1599         cleanup_list.append(gitroot + "/autobuild.pid")
1600         cleanup()
1601         elapsed_time = time.time() - start_time
1602         email_failure(-1, 'rebase', 'rebase', 'rebase',
1603                       'rebase on %s failed' % options.branch,
1604                       elapsed_time, log_base=options.log_base)
1605         sys.exit(1)
1606
1607     try:
1608         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1609         if options.tail:
1610             blist.start_tail()
1611         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1612         if status != 0 or errstr != "retry":
1613             break
1614         cleanup(do_raise=True)
1615     except Exception:
1616         cleanup()
1617         raise
1618
1619 cleanup_list.append(gitroot + "/autobuild.pid")
1620
1621 do_print(errstr)
1622
1623 blist.kill_kids()
1624 if options.tail:
1625     do_print("waiting for tail to flush")
1626     time.sleep(1)
1627
1628 elapsed_time = time.time() - start_time
1629 if status == 0:
1630     if options.passcmd is not None:
1631         do_print("Running passcmd: %s" % options.passcmd)
1632         run_cmd(options.passcmd, dir=test_master)
1633     if options.pushto is not None:
1634         push_to(options.pushto, push_branch=options.branch)
1635     if options.keeplogs or options.attach_logs:
1636         blist.tarlogs("logs.tar.gz")
1637         do_print("Logs in logs.tar.gz")
1638     if options.always_email:
1639         email_success(elapsed_time, log_base=options.log_base)
1640     blist.remove_logs()
1641     cleanup()
1642     do_print(errstr)
1643     sys.exit(0)
1644
1645 # something failed, gather a tar of the logs
1646 blist.tarlogs("logs.tar.gz")
1647
1648 if options.email is not None:
1649     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1650                   elapsed_time, log_base=options.log_base)
1651 else:
1652     elapsed_minutes = elapsed_time / 60.0
1653     print('''
1654
1655 ####################################################################
1656
1657 AUTOBUILD FAILURE
1658
1659 Your autobuild[%s] on %s failed after %.1f minutes
1660 when trying to test %s with the following error:
1661
1662    %s
1663
1664 the autobuild has been abandoned. Please fix the error and resubmit.
1665
1666 ####################################################################
1667
1668 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1669
1670 cleanup()
1671 do_print(errstr)
1672 do_print("Logs in logs.tar.gz")
1673 sys.exit(status)