script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, check_output, Popen, PIPE, CalledProcessError
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 import random
  13 from optparse import OptionParser
  14 import smtplib
  15 import email
  16 from email.mime.text import MIMEText
  17 from email.mime.base import MIMEBase
  18 from email.mime.application import MIMEApplication
  19 from email.mime.multipart import MIMEMultipart
  20 from distutils.sysconfig import get_python_lib
  21 import platform
  22
  23 try:
  24     from waflib.Build import CACHE_SUFFIX
  25 except ImportError:
  26     sys.path.insert(0, "./third_party/waf")
  27     from waflib.Build import CACHE_SUFFIX
  28
  29
  30 os.environ["PYTHONUNBUFFERED"] = "1"
  31
  32 # This speeds up testing remarkably.
  33 os.environ['TDB_NO_FSYNC'] = '1'
  34
  35
  36 def find_git_root():
  37     '''get to the top of the git repo'''
  38     p = os.getcwd()
  39     while p != '/':
  40         if os.path.exists(os.path.join(p, ".git")):
  41             return p
  42         p = os.path.abspath(os.path.join(p, '..'))
  43     return None
  44
  45
  46 gitroot = find_git_root()
  47 if gitroot is None:
  48     raise Exception("Failed to find git root")
  49
  50
  51 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  52
  53 parser = OptionParser()
  54 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  55 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  56 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  57 parser.add_option("--skip-dependencies", help="skip to run task dependency tasks", default=False, action="store_true")
  58 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  59                   default=def_testbase)
  60 parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase,
  61                   default=None)
  62 parser.add_option("--passcmd", help="command to run on success", default=None)
  63 parser.add_option("--verbose", help="show all commands as they are run",
  64                   default=False, action="store_true")
  65 parser.add_option("--rebase", help="rebase on the given tree before testing",
  66                   default=None, type='str')
  67 parser.add_option("--pushto", help="push to a git url on success",
  68                   default=None, type='str')
  69 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  70                   default=False, action="store_true")
  71 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  72                   default=False, action="store_true")
  73 parser.add_option("--retry", help="automatically retry if master changes",
  74                   default=False, action="store_true")
  75 parser.add_option("--email", help="send email to the given address on failure",
  76                   type='str', default=None)
  77 parser.add_option("--email-from", help="send email from the given address",
  78                   type='str', default="autobuild@samba.org")
  79 parser.add_option("--email-server", help="send email via the given server",
  80                   type='str', default='localhost')
  81 parser.add_option("--always-email", help="always send email, even on success",
  82                   action="store_true")
  83 parser.add_option("--daemon", help="daemonize after initial setup",
  84                   action="store_true")
  85 parser.add_option("--branch", help="the branch to work on (default=master)",
  86                   default="master", type='str')
  87 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  88                   default=gitroot, type='str')
  89 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  90                   default=False, action="store_true")
  91 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
  92                   default='')
  93 parser.add_option("--enable-coverage", dest='enable_coverage',
  94                   action="store_const", const='--enable-coverage', default='',
  95                   help="Add --enable-coverage option while configure")
  96
  97 (options, args) = parser.parse_args()
  98
  99 if options.retry:
 100     if options.rebase is None:
 101         raise Exception('You can only use --retry if you also rebase')
 102
 103 if options.full_testbase is not None:
 104     testbase = options.full_testbase
 105 else:
 106     testbase = "%s/b%u" % (options.testbase, os.getpid())
 107 test_master = "%s/master" % testbase
 108 test_prefix = "%s/prefix" % testbase
 109 test_tmpdir = "%s/tmp" % testbase
 110 os.environ['TMPDIR'] = test_tmpdir
 111
 112 if options.enable_coverage:
 113     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 114 else:
 115     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 116
 117 CLEAN_SOURCE_TREE_CMD = "cd ${TEST_SOURCE_DIR} && script/clean-source-tree.sh"
 118
 119 if args:
 120     # If we are only running specific test,
 121     # do not sleep randomly to wait for it to start
 122     def random_sleep(low, high):
 123         return 'sleep 1'
 124 else:
 125     def random_sleep(low, high):
 126         return 'sleep {}'.format(random.randint(low, high))
 127
 128 cleanup_list = []
 129
 130 builddirs = {
 131     "ctdb": "ctdb",
 132     "ldb": "lib/ldb",
 133     "tdb": "lib/tdb",
 134     "talloc": "lib/talloc",
 135     "replace": "lib/replace",
 136     "tevent": "lib/tevent",
 137     "pidl": "pidl",
 138     "docs-xml": "docs-xml"
 139 }
 140
 141 ctdb_configure_params = " --enable-developer ${PREFIX}"
 142 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 143
 144 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 145 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 146 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 147 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 148 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 149 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 150 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 151
 152
 153 def format_option(name, value=None):
 154     """Format option as str list."""
 155     if value is None:  # boolean option
 156         return [name]
 157     if not isinstance(value, list):  # single value option
 158         value = [value]
 159     # repeatable option
 160     return ['{}={}'.format(name, item) for item in value]
 161
 162
 163 def make_test(
 164         cmd='make testonly',
 165         FAIL_IMMEDIATELY=1,
 166         INJECT_SELFTEST_PREFIX=1,
 167         TESTS='',
 168         include_envs=None,
 169         exclude_envs=None):
 170
 171     test_options = []
 172     if include_envs:
 173         test_options = format_option('--include-env', include_envs)
 174     if exclude_envs:
 175         test_options = format_option('--exclude-env', exclude_envs)
 176     if test_options:
 177         # join envs options to original test options
 178         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 179
 180     _options = []
 181     if FAIL_IMMEDIATELY:
 182         _options.append('FAIL_IMMEDIATELY=1')
 183     if TESTS:
 184         _options.append("TESTS='{}'".format(TESTS))
 185
 186     if INJECT_SELFTEST_PREFIX:
 187         _options.append("TEST_OPTIONS='--with-selftest-prefix={}'".format("${SELFTEST_PREFIX}"))
 188         _options.append("--directory='{}'".format("${TEST_SOURCE_DIR}"))
 189
 190     return ' '.join([cmd] + _options)
 191
 192
 193 # When updating this list, also update .gitlab-ci.yml to add the job
 194 # and to make it a dependency of 'page' for the coverage report.
 195
 196 tasks = {
 197     "ctdb": {
 198         "sequence": [
 199             ("random-sleep", random_sleep(300, 900)),
 200             ("configure", "./configure " + ctdb_configure_params),
 201             ("make", "make all"),
 202             ("install", "make install"),
 203             ("test", "make autotest"),
 204             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 205             ("clean", "make clean"),
 206         ],
 207     },
 208     "docs-xml": {
 209         "sequence": [
 210             ("random-sleep", random_sleep(300, 900)),
 211             ("autoconf", "autoconf"),
 212             ("configure", "./configure"),
 213             ("make", "make html htmlman"),
 214             ("clean", "make clean"),
 215         ],
 216     },
 217
 218     "samba-def-build": {
 219         "git-clone-required": True,
 220         "sequence": [
 221             ("configure", "./configure.developer" + samba_configure_params),
 222             ("make", "make -j"),
 223             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 224             ("chmod-R-a-w", "chmod -R a-w ."),
 225         ],
 226     },
 227
 228     "samba-mit-build": {
 229         "git-clone-required": True,
 230         "sequence": [
 231             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 232             ("make", "make -j"),
 233             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 234             ("chmod-R-a-w", "chmod -R a-w ."),
 235         ],
 236     },
 237
 238     "samba-nt4-build": {
 239         "git-clone-required": True,
 240         "sequence": [
 241             ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json" + samba_configure_params),
 242             ("make", "make -j"),
 243             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 244             ("chmod-R-a-w", "chmod -R a-w ."),
 245         ],
 246     },
 247
 248     "samba-h5l-build": {
 249         "git-clone-required": True,
 250         "sequence": [
 251             ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5" + samba_configure_params),
 252             ("make", "make -j"),
 253             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 254             ("chmod-R-a-w", "chmod -R a-w ."),
 255         ],
 256     },
 257
 258     "samba-no-opath-build": {
 259         "git-clone-required": True,
 260         "sequence": [
 261             ("configure", "ADDITIONAL_CFLAGS='-DDISABLE_OPATH=1' ./configure.developer --without-ad-dc " + samba_configure_params),
 262             ("make", "make -j"),
 263             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 264             ("chmod-R-a-w", "chmod -R a-w ."),
 265         ],
 266     },
 267
 268     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 269     "samba": {
 270         "sequence": [
 271             ("random-sleep", random_sleep(300, 900)),
 272             ("configure", "./configure.developer" + samba_configure_params),
 273             ("make", "make -j"),
 274             ("test", make_test(exclude_envs=[
 275             "none",
 276             "nt4_dc",
 277             "nt4_dc_smb1",
 278             "nt4_dc_smb1_done",
 279             "nt4_dc_schannel",
 280             "nt4_member",
 281             "ad_dc",
 282             "ad_dc_smb1",
 283             "ad_dc_smb1_done",
 284             "ad_dc_backup",
 285             "ad_dc_ntvfs",
 286             "ad_dc_default",
 287             "ad_dc_default_smb1",
 288             "ad_dc_slowtests",
 289             "ad_dc_no_nss",
 290             "ad_dc_no_ntlm",
 291             "fl2003dc",
 292             "fl2008dc",
 293             "fl2008r2dc",
 294             "ad_member",
 295             "ad_member_idmap_rid",
 296             "ad_member_idmap_ad",
 297             "ad_member_rfc2307",
 298             "chgdcpass",
 299             "vampire_2000_dc",
 300             "fl2000dc",
 301             "fileserver",
 302             "fileserver_smb1",
 303             "fileserver_smb1_done",
 304             "maptoguest",
 305             "simpleserver",
 306             "backupfromdc",
 307             "restoredc",
 308             "renamedc",
 309             "offlinebackupdc",
 310             "labdc",
 311             "preforkrestartdc",
 312             "proclimitdc",
 313             "promoted_dc",
 314             "vampire_dc",
 315             "rodc",
 316             "ad_dc_default",
 317             "ad_dc_default_smb1",
 318             "ad_dc_default_smb1_done",
 319             "ad_dc_slowtests",
 320             "schema_pair_dc",
 321             "schema_dc",
 322             "clusteredmember",
 323             ])),
 324             ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
 325             ("lcov", LCOV_CMD),
 326             ("install", "make install"),
 327             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 328             ("clean", "make clean"),
 329         ],
 330     },
 331
 332     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 333     "samba-mitkrb5": {
 334         "sequence": [
 335             ("random-sleep", random_sleep(300, 900)),
 336             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 337             ("make", "make -j"),
 338             ("test", make_test(exclude_envs=[
 339             "none",
 340             "nt4_dc",
 341             "nt4_dc_smb1",
 342             "nt4_dc_smb1_done",
 343             "nt4_dc_schannel",
 344             "nt4_member",
 345             "ad_dc",
 346             "ad_dc_smb1",
 347             "ad_dc_smb1_done",
 348             "ad_dc_backup",
 349             "ad_dc_ntvfs",
 350             "ad_dc_default",
 351             "ad_dc_default_smb1",
 352             "ad_dc_default_smb1_done",
 353             "ad_dc_slowtests",
 354             "ad_dc_no_nss",
 355             "ad_dc_no_ntlm",
 356             "fl2003dc",
 357             "fl2008dc",
 358             "fl2008r2dc",
 359             "ad_member",
 360             "ad_member_idmap_rid",
 361             "ad_member_idmap_ad",
 362             "ad_member_rfc2307",
 363             "chgdcpass",
 364             "vampire_2000_dc",
 365             "fl2000dc",
 366             "fileserver",
 367             "fileserver_smb1",
 368             "fileserver_smb1_done",
 369             "maptoguest",
 370             "simpleserver",
 371             "backupfromdc",
 372             "restoredc",
 373             "renamedc",
 374             "offlinebackupdc",
 375             "labdc",
 376             "preforkrestartdc",
 377             "proclimitdc",
 378             "promoted_dc",
 379             "vampire_dc",
 380             "rodc",
 381             "ad_dc_default",
 382             "ad_dc_default_smb1",
 383             "ad_dc_default_smb1_done",
 384             "ad_dc_slowtests",
 385             "schema_pair_dc",
 386             "schema_dc",
 387             "clusteredmember",
 388             ])),
 389             ("lcov", LCOV_CMD),
 390             ("install", "make install"),
 391             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 392             ("clean", "make clean"),
 393         ],
 394     },
 395
 396     "samba-nt4": {
 397         "dependency": "samba-nt4-build",
 398         "sequence": [
 399             ("random-sleep", random_sleep(300, 900)),
 400             ("test", make_test(include_envs=[
 401             "nt4_dc",
 402             "nt4_dc_smb1",
 403             "nt4_dc_smb1_done",
 404             "nt4_dc_schannel",
 405             "nt4_member",
 406             "simpleserver",
 407             ])),
 408             ("lcov", LCOV_CMD),
 409             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 410         ],
 411     },
 412
 413     "samba-fileserver": {
 414         "dependency": "samba-h5l-build",
 415         "sequence": [
 416             ("random-sleep", random_sleep(300, 900)),
 417             ("test", make_test(include_envs=[
 418             "fileserver",
 419             "fileserver_smb1",
 420             "fileserver_smb1_done",
 421             "maptoguest",
 422             "ktest", # ktest is also tested in samba and samba-mitkrb5
 423                      # but is tested here against a system Heimdal
 424             ])),
 425             ("lcov", LCOV_CMD),
 426             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 427         ],
 428     },
 429
 430     "samba-admem": {
 431         "dependency": "samba-def-build",
 432         "sequence": [
 433             ("random-sleep", random_sleep(300, 900)),
 434             ("test", make_test(include_envs=[
 435             "ad_member",
 436             "ad_member_idmap_rid",
 437             "ad_member_idmap_ad",
 438             "ad_member_rfc2307",
 439             ])),
 440             ("lcov", LCOV_CMD),
 441             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 442         ],
 443     },
 444
 445     "samba-no-opath": {
 446         "dependency": "samba-no-opath-build",
 447         "sequence": [
 448             ("random-sleep", random_sleep(300, 900)),
 449             ("test", make_test(
 450                 cmd="make test DISABLE_OPATH=1",
 451                 include_envs=[
 452                 "nt4_dc",
 453                 "nt4_dc_smb1",
 454                 "nt4_dc_smb1_done",
 455                 "nt4_dc_schannel",
 456                 "nt4_member",
 457                 "simpleserver",
 458                 "fileserver",
 459                 "fileserver_smb1",
 460                 "fileserver_smb1_done",
 461                 ])),
 462             ("lcov", LCOV_CMD),
 463             ("check-clean-tree", "script/clean-source-tree.sh"),
 464         ],
 465     },
 466
 467     "samba-ad-dc-1": {
 468         "dependency": "samba-def-build",
 469         "sequence": [
 470             ("random-sleep", random_sleep(1, 1)),
 471             ("test", make_test(include_envs=[
 472             "ad_dc",
 473             "ad_dc_smb1",
 474             "ad_dc_smb1_done",
 475             "ad_dc_no_nss",
 476             "ad_dc_no_ntlm",
 477             ])),
 478             ("lcov", LCOV_CMD),
 479             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 480         ],
 481     },
 482
 483     "samba-ad-dc-2": {
 484         "dependency": "samba-def-build",
 485         "sequence": [
 486             ("random-sleep", random_sleep(1, 1)),
 487             ("test", make_test(include_envs=[
 488             "vampire_dc",
 489             "vampire_2000_dc",
 490             "rodc",
 491             ])),
 492             ("lcov", LCOV_CMD),
 493             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 494         ],
 495     },
 496
 497     "samba-ad-dc-3": {
 498         "dependency": "samba-def-build",
 499         "sequence": [
 500             ("random-sleep", random_sleep(1, 1)),
 501             ("test", make_test(include_envs=[
 502             "promoted_dc",
 503             "chgdcpass",
 504             "preforkrestartdc",
 505             "proclimitdc",
 506             ])),
 507             ("lcov", LCOV_CMD),
 508             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 509         ],
 510     },
 511
 512     "samba-ad-dc-4": {
 513         "dependency": "samba-def-build",
 514         "sequence": [
 515             ("random-sleep", random_sleep(1, 1)),
 516             ("test", make_test(include_envs=[
 517             "fl2000dc",
 518             "fl2003dc",
 519             "fl2008dc",
 520             "fl2008r2dc",
 521             ])),
 522             ("lcov", LCOV_CMD),
 523             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 524         ],
 525     },
 526
 527     "samba-ad-dc-5": {
 528         "dependency": "samba-def-build",
 529         "sequence": [
 530             ("random-sleep", random_sleep(1, 1)),
 531             ("test", make_test(include_envs=[
 532             "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
 533             ("lcov", LCOV_CMD),
 534             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 535         ],
 536     },
 537
 538     "samba-ad-dc-6": {
 539         "dependency": "samba-def-build",
 540         "sequence": [
 541             ("random-sleep", random_sleep(1, 1)),
 542             ("test", make_test(include_envs=["ad_dc_slowtests", "ad_dc_backup"])),
 543             ("lcov", LCOV_CMD),
 544             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 545         ],
 546     },
 547
 548     "samba-schemaupgrade": {
 549         "dependency": "samba-def-build",
 550         "sequence": [
 551             ("random-sleep", random_sleep(1, 1)),
 552             ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 553             ("lcov", LCOV_CMD),
 554             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 555         ],
 556     },
 557
 558     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 559     # This is currently the longest task, so we don't randomly delay it.
 560     "samba-ad-dc-ntvfs": {
 561         "dependency": "samba-def-build",
 562         "sequence": [
 563             ("random-sleep", random_sleep(1, 1)),
 564             ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 565             ("lcov", LCOV_CMD),
 566             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 567         ],
 568     },
 569
 570     # Test fips compliance
 571     "samba-fips": {
 572         "dependency": "samba-mit-build",
 573         "sequence": [
 574             ("random-sleep", random_sleep(1, 1)),
 575             ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
 576             # TODO: This seems to generate only an empty samba-fips.info ("lcov", LCOV_CMD),
 577             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 578         ],
 579     },
 580
 581     # run the backup/restore testenvs separately as they're fairly standalone
 582     # (and CI seems to max out at ~3 different DCs running at once)
 583     "samba-ad-back1": {
 584         "dependency": "samba-def-build",
 585         "sequence": [
 586             ("random-sleep", random_sleep(300, 900)),
 587             ("test", make_test(include_envs=[
 588             "backupfromdc",
 589             "restoredc",
 590             "renamedc",
 591             ])),
 592             ("lcov", LCOV_CMD),
 593             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 594         ],
 595     },
 596     "samba-ad-back2": {
 597         "dependency": "samba-def-build",
 598         "sequence": [
 599             ("random-sleep", random_sleep(300, 900)),
 600             ("test", make_test(include_envs=[
 601             "backupfromdc",
 602             "offlinebackupdc",
 603             "labdc",
 604             ])),
 605             ("lcov", LCOV_CMD),
 606             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 607         ],
 608     },
 609
 610     "samba-admem-mit": {
 611         "dependency": "samba-mit-build",
 612         "sequence": [
 613             ("random-sleep", random_sleep(1, 1)),
 614             ("test", make_test(include_envs=[
 615             "ad_member",
 616             "ad_member_idmap_rid",
 617             "ad_member_idmap_ad",
 618             "ad_member_rfc2307",
 619             ])),
 620             ("lcov", LCOV_CMD),
 621             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 622         ],
 623     },
 624
 625     "samba-ad-dc-1-mitkrb5": {
 626         "dependency": "samba-mit-build",
 627         "sequence": [
 628             ("random-sleep", random_sleep(1, 1)),
 629             ("test", make_test(include_envs=[
 630             "ad_dc",
 631             "ad_dc_smb1",
 632             "ad_dc_smb1_done",
 633             "ad_dc_no_nss",
 634             "ad_dc_no_ntlm",
 635             ])),
 636             ("lcov", LCOV_CMD),
 637             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 638         ],
 639     },
 640
 641     "samba-ad-dc-4-mitkrb5": {
 642         "dependency": "samba-mit-build",
 643         "sequence": [
 644             ("random-sleep", random_sleep(1, 1)),
 645             ("test", make_test(include_envs=[
 646             "fl2000dc",
 647             "fl2003dc",
 648             "fl2008dc",
 649             "fl2008r2dc",
 650             ])),
 651             ("lcov", LCOV_CMD),
 652             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 653         ],
 654     },
 655
 656     "samba-test-only": {
 657         "sequence": [
 658             ("configure", "./configure.developer  --abi-check-disable" + samba_configure_params),
 659             ("make", "make -j"),
 660             ("test", make_test(TESTS="${TESTS}")),
 661             ("lcov", LCOV_CMD),
 662         ],
 663     },
 664
 665     # Test cross-compile infrastructure
 666     "samba-xc": {
 667         "sequence": [
 668             ("random-sleep", random_sleep(900, 1500)),
 669             ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 670             ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 671             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 672             ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
 673             ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 674             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 675             ("compare-results", "script/compare_cc_results.py "
 676             "./bin/c4che/default{} "
 677             "./bin-xe/c4che/default{} "
 678             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 679             ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
 680             ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
 681             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
 682             ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
 683             " = \"'1234'\"".format(CACHE_SUFFIX)),
 684             ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
 685             ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
 686             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
 687             " ; test $? -ne 0"),
 688         ],
 689     },
 690
 691     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 692     "samba-o3": {
 693         "sequence": [
 694             ("random-sleep", random_sleep(300, 900)),
 695             ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --abi-check-disable" + samba_configure_params),
 696             ("make", "make -j"),
 697             ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 698             ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 699             ("lcov", LCOV_CMD),
 700             ("install", "make install"),
 701             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 702             ("clean", "make clean"),
 703         ],
 704     },
 705
 706     "samba-ctdb": {
 707         "sequence": [
 708             ("random-sleep", random_sleep(900, 1500)),
 709
 710         # make sure we have tdb around:
 711             ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 712             ("tdb-make", "cd lib/tdb && make"),
 713             ("tdb-install", "cd lib/tdb && make install"),
 714
 715         # build samba with cluster support (also building ctdb):
 716             ("samba-configure",
 717          "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
 718          "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
 719          "./configure.developer ${PREFIX} "
 720          "--with-selftest-prefix=./bin/ab "
 721          "--enable-clangdb "
 722          "--with-cluster-support "
 723          "--without-ad-dc "
 724          "--bundled-libraries=!tdb"),
 725             ("samba-make", "make"),
 726             ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT"),
 727             ("samba-install", "make install"),
 728             ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 729
 730             ("test", make_test(
 731                 cmd='make test',
 732                 INJECT_SELFTEST_PREFIX=0,
 733                 include_envs=["clusteredmember"])
 734             ),
 735
 736         # clean up:
 737             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 738             ("clean", "make clean"),
 739             ("ctdb-clean", "cd ./ctdb && make clean"),
 740         ],
 741     },
 742
 743     "samba-libs": {
 744         "sequence": [
 745             ("random-sleep", random_sleep(300, 900)),
 746             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 747             ("talloc-make", "cd lib/talloc && make"),
 748             ("talloc-install", "cd lib/talloc && make install"),
 749
 750             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 751             ("tdb-make", "cd lib/tdb && make"),
 752             ("tdb-install", "cd lib/tdb && make install"),
 753
 754             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 755             ("tevent-make", "cd lib/tevent && make"),
 756             ("tevent-install", "cd lib/tevent && make install"),
 757
 758             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
 759             ("ldb-make", "cd lib/ldb && make"),
 760             ("ldb-install", "cd lib/ldb && make install"),
 761
 762             ("nondevel-configure", "./configure ${PREFIX}"),
 763             ("nondevel-make", "make -j"),
 764             ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 765             ("nondevel-install", "make install"),
 766             ("nondevel-dist", "make dist"),
 767
 768         # retry with all modules shared
 769             ("allshared-distclean", "make distclean"),
 770             ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 771             ("allshared-make", "make -j"),
 772         ],
 773     },
 774
 775     "samba-fuzz": {
 776         "sequence": [
 777         # build the fuzzers (static) via the oss-fuzz script
 778             ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
 779             ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ADDITIONAL_LDFLAGS='-fuse-ld=bfd' ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl-fuzzer"),
 780         ],
 781     },
 782
 783     # * Test smbd and smbtorture can build semi-static
 784     #
 785     # * Test Samba without python still builds.
 786     #
 787     # When this test fails due to more use of Python, the expectations
 788     # is that the newly failing part of the code should be disabled
 789     # when --disable-python is set (rather than major work being done
 790     # to support this environment).
 791     #
 792     # The target here is for vendors shipping a minimal smbd.
 793     "samba-minimal-smbd": {
 794         "sequence": [
 795             ("random-sleep", random_sleep(300, 900)),
 796
 797         # build with all modules static
 798             ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
 799             ("allstatic-make", "make -j"),
 800             ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 801             ("lcov", LCOV_CMD),
 802
 803         # retry with nonshared smbd and smbtorture
 804             ("nonshared-distclean", "make distclean"),
 805             ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
 806             ("nonshared-make", "make -j"),
 807
 808             ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
 809             ("make", "make -j"),
 810             ("find-python", "script/find_python.sh ${PREFIX}"),
 811             ("test", "make test-nopython"),
 812             ("lcov", LCOV_CMD),
 813             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 814             ("clean", "make clean"),
 815
 816             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 817             ("talloc-make", "cd lib/talloc && make"),
 818             ("talloc-install", "cd lib/talloc && make install"),
 819
 820             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 821             ("tdb-make", "cd lib/tdb && make"),
 822             ("tdb-install", "cd lib/tdb && make install"),
 823
 824             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 825             ("tevent-make", "cd lib/tevent && make"),
 826             ("tevent-install", "cd lib/tevent && make install"),
 827
 828             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 829             ("ldb-make", "cd lib/ldb && make"),
 830             ("ldb-install", "cd lib/ldb && make install"),
 831
 832         # retry against installed library packages, but no required modules
 833             ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc  --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
 834             ("libs-make", "make -j"),
 835             ("libs-install", "make install"),
 836             ("libs-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 837             ("libs-clean", "make clean"),
 838
 839         ],
 840     },
 841
 842     "ldb": {
 843         "sequence": [
 844             ("random-sleep", random_sleep(60, 600)),
 845             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 846             ("make", "make"),
 847             ("install", "make install"),
 848             ("test", "make test"),
 849             ("lcov", LCOV_CMD),
 850             ("clean", "make clean"),
 851             ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
 852             ("make-no-lmdb", "make"),
 853             ("test-no-lmdb", "make test"),
 854             ("lcov-no-lmdb", LCOV_CMD),
 855             ("install-no-lmdb", "make install"),
 856             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 857             ("distcheck", "make distcheck"),
 858             ("clean", "make clean"),
 859         ],
 860     },
 861
 862     "tdb": {
 863         "sequence": [
 864             ("random-sleep", random_sleep(60, 600)),
 865             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 866             ("make", "make"),
 867             ("install", "make install"),
 868             ("test", "make test"),
 869             ("lcov", LCOV_CMD),
 870             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 871             ("distcheck", "make distcheck"),
 872             ("clean", "make clean"),
 873         ],
 874     },
 875
 876     "talloc": {
 877         "sequence": [
 878             ("random-sleep", random_sleep(60, 600)),
 879             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 880             ("make", "make"),
 881             ("install", "make install"),
 882             ("test", "make test"),
 883             ("lcov", LCOV_CMD),
 884             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 885             ("distcheck", "make distcheck"),
 886             ("clean", "make clean"),
 887         ],
 888     },
 889
 890     "replace": {
 891         "sequence": [
 892             ("random-sleep", random_sleep(60, 600)),
 893             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 894             ("make", "make"),
 895             ("install", "make install"),
 896             ("test", "make test"),
 897             ("lcov", LCOV_CMD),
 898             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 899             ("distcheck", "make distcheck"),
 900             ("clean", "make clean"),
 901         ],
 902     },
 903
 904     "tevent": {
 905         "sequence": [
 906             ("random-sleep", random_sleep(60, 600)),
 907             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 908             ("make", "make"),
 909             ("install", "make install"),
 910             ("test", "make test"),
 911             ("lcov", LCOV_CMD),
 912             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 913             ("distcheck", "make distcheck"),
 914             ("clean", "make clean"),
 915         ],
 916     },
 917
 918     "pidl": {
 919         "git-clone-required": True,
 920         "sequence": [
 921             ("random-sleep", random_sleep(60, 600)),
 922             ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
 923             ("touch", "touch *.yp"),
 924             ("make", "make"),
 925             ("test", "make test"),
 926             ("install", "make install"),
 927             ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
 928             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 929             ("clean", "make clean"),
 930         ],
 931     },
 932
 933     # these are useful for debugging autobuild
 934     "pass": {
 935         "sequence": [
 936             ("pass", 'echo passing && /bin/true'),
 937         ],
 938     },
 939     "fail": {
 940         "sequence": [
 941             ("fail", 'echo failing && /bin/false'),
 942         ],
 943     },
 944 }
 945
 946 defaulttasks = list(tasks.keys())
 947
 948 defaulttasks.remove("pass")
 949 defaulttasks.remove("fail")
 950 defaulttasks.remove("samba-def-build")
 951 defaulttasks.remove("samba-nt4-build")
 952 defaulttasks.remove("samba-mit-build")
 953 defaulttasks.remove("samba-h5l-build")
 954 defaulttasks.remove("samba-no-opath-build")
 955 defaulttasks.remove("samba-test-only")
 956 defaulttasks.remove("samba-fuzz")
 957 defaulttasks.remove("samba-fips")
 958 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
 959     defaulttasks.remove("samba-o3")
 960
 961
 962 def do_print(msg):
 963     print("%s" % msg)
 964     sys.stdout.flush()
 965     sys.stderr.flush()
 966
 967
 968 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 969     if show is None:
 970         show = options.verbose
 971     if show:
 972         do_print("Running: '%s' in '%s'" % (cmd, dir))
 973     if output:
 974         out = check_output([cmd], shell=True, cwd=dir)
 975         return out.decode(encoding='utf-8', errors='backslashreplace')
 976     elif checkfail:
 977         return check_call(cmd, shell=True, cwd=dir)
 978     else:
 979         return call(cmd, shell=True, cwd=dir)
 980
 981 def rmdir_force(dirname, re_raise=True):
 982     try:
 983         run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % (
 984                 dirname, dirname, dirname), output=True, show=True)
 985     except CalledProcessError as e:
 986         do_print("Failed: '%s'" % (str(e)))
 987         run_cmd("tree %s" % dirname, output=True, show=True)
 988         if re_raise:
 989             raise
 990         return False
 991     return True
 992
 993 class builder(object):
 994     '''handle build of one directory'''
 995
 996     def __init__(self, name, definition):
 997         self.name = name
 998         self.dir = builddirs.get(name, '.')
 999         self.tag = self.name.replace('/', '_')
1000         self.definition = definition
1001         self.sequence = definition["sequence"]
1002         self.git_clone_required = False
1003         if "git-clone-required" in definition:
1004             self.git_clone_required = bool(definition["git-clone-required"])
1005         self.proc = None
1006         self.done = False
1007         self.next = 0
1008         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
1009         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
1010         if options.verbose:
1011             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
1012             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
1013         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
1014         self.stdout = open(self.stdout_path, 'w')
1015         self.stderr = open(self.stderr_path, 'w')
1016         self.stdin  = open("/dev/null", 'r')
1017         self.builder_dir = "%s/%s" % (testbase, self.tag)
1018         self.test_source_dir = self.builder_dir
1019         self.cwd = "%s/%s" % (self.builder_dir, self.dir)
1020         self.selftest_prefix = "%s/bin/ab" % (self.cwd)
1021         self.prefix = "%s/%s" % (test_prefix, self.tag)
1022         self.consumers = []
1023         self.producer = None
1024
1025         if self.git_clone_required:
1026             assert "dependency" not in definition
1027
1028     def mark_existing(self):
1029         do_print('%s: Mark as existing dependency' % self.name)
1030         self.next = len(self.sequence)
1031         self.done = True
1032
1033     def add_consumer(self, consumer):
1034         do_print("%s: add consumer: %s" % (self.name, consumer.name))
1035         consumer.producer = self
1036         consumer.test_source_dir = self.test_source_dir
1037         self.consumers.append(consumer)
1038
1039     def start_next(self):
1040         if self.producer is not None:
1041             if not self.producer.done:
1042                 do_print("%s: Waiting for producer: %s" % (self.name, self.producer.name))
1043                 return
1044
1045         if self.next == 0:
1046             rmdir_force(self.builder_dir)
1047             rmdir_force(self.prefix)
1048             if self.producer is not None:
1049                 run_cmd("mkdir %s" % (self.builder_dir), dir=test_master, show=True)
1050             elif not self.git_clone_required:
1051                 run_cmd("cp -R -a -l %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1052             else:
1053                 run_cmd("git clone --recursive --shared %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1054
1055         if self.next == len(self.sequence):
1056             if not self.done:
1057                 do_print('%s: Completed OK' % self.name)
1058                 self.done = True
1059             if not options.nocleanup and len(self.consumers) == 0:
1060                 do_print('%s: Cleaning up' % self.name)
1061                 rmdir_force(self.builder_dir)
1062                 rmdir_force(self.prefix)
1063             for consumer in self.consumers:
1064                 if consumer.next != 0:
1065                     continue
1066                 do_print('%s: Starting consumer %s' % (self.name, consumer.name))
1067                 consumer.start_next()
1068             if self.producer is not None:
1069                 self.producer.consumers.remove(self)
1070                 assert self.producer.done
1071                 self.producer.start_next()
1072             do_print('%s: Remaining consumers %u' % (self.name, len(self.consumers)))
1073             return
1074         (self.stage, self.cmd) = self.sequence[self.next]
1075         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
1076         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
1077         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
1078         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
1079         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
1080         self.cmd = self.cmd.replace("${SELFTEST_PREFIX}", self.selftest_prefix)
1081         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
1082         self.cmd = self.cmd.replace("${NAME}", self.name)
1083         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
1084         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
1085         self.proc = Popen(self.cmd, shell=True,
1086                           close_fds=True, cwd=self.cwd,
1087                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
1088         self.next += 1
1089
1090 def expand_dependencies(n):
1091     deps = list()
1092     if "dependency" in tasks[n]:
1093         depname = tasks[n]["dependency"]
1094         assert depname in tasks
1095         sdeps = expand_dependencies(depname)
1096         assert n not in sdeps
1097         for sdep in sdeps:
1098             deps.append(sdep)
1099         deps.append(depname)
1100     return deps
1101
1102
1103 class buildlist(object):
1104     '''handle build of multiple directories'''
1105
1106     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
1107         self.tail_proc = None
1108         self.retry = None
1109         if not tasknames:
1110             if options.restrict_tests:
1111                 tasknames = ["samba-test-only"]
1112             else:
1113                 tasknames = defaulttasks
1114
1115         given_tasknames = tasknames.copy()
1116         implicit_tasknames = []
1117         for n in given_tasknames:
1118             deps = expand_dependencies(n)
1119             for dep in deps:
1120                 if dep in given_tasknames:
1121                     continue
1122                 if dep in implicit_tasknames:
1123                     continue
1124                 implicit_tasknames.append(dep)
1125
1126         tasknames = implicit_tasknames.copy()
1127         tasknames.extend(given_tasknames)
1128         do_print("given_tasknames: %s" % given_tasknames)
1129         do_print("implicit_tasknames: %s" % implicit_tasknames)
1130         do_print("tasknames: %s" % tasknames)
1131         self.tlist = [builder(n, tasks[n]) for n in tasknames]
1132
1133         if options.retry:
1134             rebase_remote = "rebaseon"
1135             retry_task = {
1136                     "git-clone-required": True,
1137                     "sequence": [
1138                             ("retry",
1139                             '''set -e
1140                             git remote add -t %s %s %s
1141                             git fetch %s
1142                             while :; do
1143                               sleep 60
1144                               git describe %s/%s > old_remote_branch.desc
1145                               git fetch %s
1146                               git describe %s/%s > remote_branch.desc
1147                               diff old_remote_branch.desc remote_branch.desc
1148                             done
1149                            ''' % (
1150                                rebase_branch, rebase_remote, rebase_url,
1151                                rebase_remote,
1152                                rebase_remote, rebase_branch,
1153                                rebase_remote,
1154                                rebase_remote, rebase_branch
1155                             ))]}
1156
1157             self.retry = builder('retry', retry_task)
1158             self.need_retry = False
1159
1160         if options.skip_dependencies:
1161             for b in self.tlist:
1162                 if b.name in implicit_tasknames:
1163                     b.mark_existing()
1164
1165         for b in self.tlist:
1166             do_print("b.name=%s" % b.name)
1167             if "dependency" not in b.definition:
1168                 continue
1169             depname = b.definition["dependency"]
1170             do_print("b.name=%s: dependency:%s" % (b.name, depname))
1171             for p in self.tlist:
1172                 if p.name == depname:
1173                     p.add_consumer(b)
1174
1175     def kill_kids(self):
1176         if self.tail_proc is not None:
1177             self.tail_proc.terminate()
1178             self.tail_proc.wait()
1179             self.tail_proc = None
1180         if self.retry is not None:
1181             self.retry.proc.terminate()
1182             self.retry.proc.wait()
1183             self.retry = None
1184         for b in self.tlist:
1185             if b.proc is not None:
1186                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
1187                 b.proc.terminate()
1188                 b.proc.wait()
1189                 b.proc = None
1190
1191     def wait_one(self):
1192         while True:
1193             none_running = True
1194             for b in self.tlist:
1195                 if b.proc is None:
1196                     continue
1197                 none_running = False
1198                 b.status = b.proc.poll()
1199                 if b.status is None:
1200                     continue
1201                 b.proc = None
1202                 return b
1203             if options.retry:
1204                 ret = self.retry.proc.poll()
1205                 if ret is not None:
1206                     self.need_retry = True
1207                     self.retry = None
1208                     return None
1209             if none_running:
1210                 return None
1211             time.sleep(0.1)
1212
1213     def run(self):
1214         for b in self.tlist:
1215             b.start_next()
1216         if options.retry:
1217             self.retry.start_next()
1218         while True:
1219             b = self.wait_one()
1220             if options.retry and self.need_retry:
1221                 self.kill_kids()
1222                 do_print("retry needed")
1223                 return (0, None, None, None, "retry")
1224             if b is None:
1225                 break
1226             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
1227                 self.kill_kids()
1228                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
1229             b.start_next()
1230         self.kill_kids()
1231         return (0, None, None, None, "All OK")
1232
1233     def write_system_info(self, filename):
1234         with open(filename, 'w') as f:
1235             for cmd in ['uname -a',
1236                         'lsb_release -a',
1237                         'free',
1238                         'mount',
1239                         'cat /proc/cpuinfo',
1240                         'cc --version',
1241                         'df -m .',
1242                         'df -m %s' % testbase]:
1243                 try:
1244                     out = run_cmd(cmd, output=True, checkfail=False)
1245                 except CalledProcessError as e:
1246                     out = "<failed: %s>" % str(e)
1247                 print('### %s' % cmd, file=f)
1248                 print(out, file=f)
1249                 print(file=f)
1250
1251     def tarlogs(self, fname):
1252         with tarfile.open(fname, "w:gz") as tar:
1253             for b in self.tlist:
1254                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
1255                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
1256             if os.path.exists("autobuild.log"):
1257                 tar.add("autobuild.log")
1258             filename = 'system-info.txt'
1259             self.write_system_info(filename)
1260             tar.add(filename)
1261
1262     def remove_logs(self):
1263         for b in self.tlist:
1264             os.unlink(b.stdout_path)
1265             os.unlink(b.stderr_path)
1266
1267     def start_tail(self):
1268         cmd = ["tail", "-f"]
1269         for b in self.tlist:
1270             cmd.append(b.stdout_path)
1271             cmd.append(b.stderr_path)
1272         self.tail_proc = Popen(cmd, close_fds=True)
1273
1274
1275 def cleanup(do_raise=False):
1276     if options.nocleanup:
1277         return
1278     run_cmd("stat %s || true" % test_tmpdir, show=True)
1279     run_cmd("stat %s" % testbase, show=True)
1280     do_print("Cleaning up %r" % cleanup_list)
1281     for d in cleanup_list:
1282         ok = rmdir_force(d, re_raise=False)
1283         if ok:
1284             continue
1285         if os.path.isdir(d):
1286             do_print("Killing, waiting and retry")
1287             run_cmd("killbysubdir %s > /dev/null 2>&1" % d, checkfail=False)
1288         else:
1289             do_print("Waiting and retry")
1290         time.sleep(1)
1291         rmdir_force(d, re_raise=do_raise)
1292
1293
1294 def daemonize(logfile):
1295     pid = os.fork()
1296     if pid == 0:  # Parent
1297         os.setsid()
1298         pid = os.fork()
1299         if pid != 0:  # Actual daemon
1300             os._exit(0)
1301     else:  # Grandparent
1302         os._exit(0)
1303
1304     import resource      # Resource usage information.
1305     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1306     if maxfd == resource.RLIM_INFINITY:
1307         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1308     for fd in range(0, maxfd):
1309         try:
1310             os.close(fd)
1311         except OSError:
1312             pass
1313     os.open(logfile, os.O_RDWR | os.O_CREAT)
1314     os.dup2(0, 1)
1315     os.dup2(0, 2)
1316
1317
1318 def write_pidfile(fname):
1319     '''write a pid file, cleanup on exit'''
1320     with open(fname, mode='w') as f:
1321         f.write("%u\n" % os.getpid())
1322
1323
1324 def rebase_tree(rebase_url, rebase_branch="master"):
1325     rebase_remote = "rebaseon"
1326     do_print("Rebasing on %s" % rebase_url)
1327     run_cmd("git describe HEAD", show=True, dir=test_master)
1328     run_cmd("git remote add -t %s %s %s" %
1329             (rebase_branch, rebase_remote, rebase_url),
1330             show=True, dir=test_master)
1331     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1332     if options.fix_whitespace:
1333         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1334                 (rebase_remote, rebase_branch),
1335                 show=True, dir=test_master)
1336     else:
1337         run_cmd("git rebase --force-rebase %s/%s" %
1338                 (rebase_remote, rebase_branch),
1339                 show=True, dir=test_master)
1340     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1341                    (rebase_remote, rebase_branch),
1342                    dir=test_master, output=True)
1343     if diff == '':
1344         do_print("No differences between HEAD and %s/%s - exiting" %
1345                  (rebase_remote, rebase_branch))
1346         sys.exit(0)
1347     run_cmd("git describe %s/%s" %
1348             (rebase_remote, rebase_branch),
1349             show=True, dir=test_master)
1350     run_cmd("git describe HEAD", show=True, dir=test_master)
1351     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1352             (rebase_remote, rebase_branch),
1353             show=True, dir=test_master)
1354
1355
1356 def push_to(push_url, push_branch="master"):
1357     push_remote = "pushto"
1358     do_print("Pushing to %s" % push_url)
1359     if options.mark:
1360         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1361         run_cmd("git commit --amend -c HEAD", dir=test_master)
1362         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1363         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1364     run_cmd("git remote add -t %s %s %s" %
1365             (push_branch, push_remote, push_url),
1366             show=True, dir=test_master)
1367     run_cmd("git push %s +HEAD:%s" %
1368             (push_remote, push_branch),
1369             show=True, dir=test_master)
1370
1371
1372 def send_email(subject, text, log_tar):
1373     if options.email is None:
1374         do_print("not sending email because the recipient is not set")
1375         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1376                  (subject, text))
1377         return
1378     outer = MIMEMultipart()
1379     outer['Subject'] = subject
1380     outer['To'] = options.email
1381     outer['From'] = options.email_from
1382     outer['Date'] = email.utils.formatdate(localtime=True)
1383     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1384     outer.attach(MIMEText(text, 'plain', 'utf-8'))
1385     if options.attach_logs:
1386         with open(log_tar, 'rb') as fp:
1387             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1388         # Set the filename parameter
1389         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1390         outer.attach(msg)
1391     content = outer.as_string()
1392     s = smtplib.SMTP(options.email_server)
1393     email_user = os.getenv('SMTP_USERNAME')
1394     email_password = os.getenv('SMTP_PASSWORD')
1395     if email_user is not None:
1396         s.starttls()
1397         s.login(email_user, email_password)
1398
1399     s.sendmail(options.email_from, [options.email], content)
1400     s.set_debuglevel(1)
1401     s.quit()
1402
1403
1404 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1405                   elapsed_time, log_base=None, add_log_tail=True):
1406     '''send an email to options.email about the failure'''
1407     elapsed_minutes = elapsed_time / 60.0
1408     if log_base is None:
1409         log_base = gitroot
1410     text = '''
1411 Dear Developer,
1412
1413 Your autobuild on %s failed after %.1f minutes
1414 when trying to test %s with the following error:
1415
1416    %s
1417
1418 the autobuild has been abandoned. Please fix the error and resubmit.
1419
1420 A summary of the autobuild process is here:
1421
1422   %s/autobuild.log
1423 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1424
1425     if options.restrict_tests:
1426         text += """
1427 The build was restricted to tests matching %s\n""" % options.restrict_tests
1428
1429     if failed_task != 'rebase':
1430         text += '''
1431 You can see logs of the failed task here:
1432
1433   %s/%s.stdout
1434   %s/%s.stderr
1435
1436 or you can get full logs of all tasks in this job here:
1437
1438   %s/logs.tar.gz
1439
1440 The top commit for the tree that was built was:
1441
1442 %s
1443
1444 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1445
1446     if add_log_tail:
1447         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1448         lines = f.readlines()
1449         log_tail = "".join(lines[-50:])
1450         num_lines = len(lines)
1451         if num_lines < 50:
1452             # Also include stderr (compile failures) if < 50 lines of stdout
1453             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1454             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1455
1456         text += '''
1457 The last 50 lines of log messages:
1458
1459 %s
1460     ''' % log_tail
1461         f.close()
1462
1463     logs = os.path.join(gitroot, 'logs.tar.gz')
1464     send_email('autobuild[%s] failure on %s for task %s during %s'
1465                % (options.branch, platform.node(), failed_task, failed_stage),
1466                text, logs)
1467
1468
1469 def email_success(elapsed_time, log_base=None):
1470     '''send an email to options.email about a successful build'''
1471     if log_base is None:
1472         log_base = gitroot
1473     text = '''
1474 Dear Developer,
1475
1476 Your autobuild on %s has succeeded after %.1f minutes.
1477
1478 ''' % (platform.node(), elapsed_time / 60.)
1479
1480     if options.restrict_tests:
1481         text += """
1482 The build was restricted to tests matching %s\n""" % options.restrict_tests
1483
1484     if options.keeplogs:
1485         text += '''
1486
1487 you can get full logs of all tasks in this job here:
1488
1489   %s/logs.tar.gz
1490
1491 ''' % log_base
1492
1493     text += '''
1494 The top commit for the tree that was built was:
1495
1496 %s
1497 ''' % top_commit_msg
1498
1499     logs = os.path.join(gitroot, 'logs.tar.gz')
1500     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1501                text, logs)
1502
1503
1504 # get the top commit message, for emails
1505 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1506
1507 try:
1508     if options.skip_dependencies:
1509         run_cmd("stat %s" % testbase, dir=testbase, output=True)
1510     else:
1511         os.makedirs(testbase)
1512 except Exception as reason:
1513     raise Exception("Unable to create %s : %s" % (testbase, reason))
1514 cleanup_list.append(testbase)
1515
1516 if options.daemon:
1517     logfile = os.path.join(testbase, "log")
1518     do_print("Forking into the background, writing progress to %s" % logfile)
1519     daemonize(logfile)
1520
1521 write_pidfile(gitroot + "/autobuild.pid")
1522
1523 start_time = time.time()
1524
1525 while True:
1526     try:
1527         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1528         os.makedirs(test_tmpdir)
1529         # The waf uninstall code removes empty directories all the way
1530         # up the tree.  Creating a file in test_tmpdir stops it from
1531         # being removed.
1532         run_cmd("touch %s" % os.path.join(test_tmpdir,
1533                                           ".directory-is-not-empty"), show=True)
1534         run_cmd("stat %s" % test_tmpdir, show=True)
1535         run_cmd("stat %s" % testbase, show=True)
1536         if options.skip_dependencies:
1537             run_cmd("stat %s" % test_master, dir=testbase, output=True)
1538         else:
1539             run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1540     except Exception:
1541         cleanup()
1542         raise
1543
1544     try:
1545         if options.rebase is not None:
1546             rebase_tree(options.rebase, rebase_branch=options.branch)
1547     except Exception:
1548         cleanup_list.append(gitroot + "/autobuild.pid")
1549         cleanup()
1550         elapsed_time = time.time() - start_time
1551         email_failure(-1, 'rebase', 'rebase', 'rebase',
1552                       'rebase on %s failed' % options.branch,
1553                       elapsed_time, log_base=options.log_base)
1554         sys.exit(1)
1555
1556     try:
1557         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1558         if options.tail:
1559             blist.start_tail()
1560         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1561         if status != 0 or errstr != "retry":
1562             break
1563         cleanup(do_raise=True)
1564     except Exception:
1565         cleanup()
1566         raise
1567
1568 cleanup_list.append(gitroot + "/autobuild.pid")
1569
1570 do_print(errstr)
1571
1572 blist.kill_kids()
1573 if options.tail:
1574     do_print("waiting for tail to flush")
1575     time.sleep(1)
1576
1577 elapsed_time = time.time() - start_time
1578 if status == 0:
1579     if options.passcmd is not None:
1580         do_print("Running passcmd: %s" % options.passcmd)
1581         run_cmd(options.passcmd, dir=test_master)
1582     if options.pushto is not None:
1583         push_to(options.pushto, push_branch=options.branch)
1584     if options.keeplogs or options.attach_logs:
1585         blist.tarlogs("logs.tar.gz")
1586         do_print("Logs in logs.tar.gz")
1587     if options.always_email:
1588         email_success(elapsed_time, log_base=options.log_base)
1589     blist.remove_logs()
1590     cleanup()
1591     do_print(errstr)
1592     sys.exit(0)
1593
1594 # something failed, gather a tar of the logs
1595 blist.tarlogs("logs.tar.gz")
1596
1597 if options.email is not None:
1598     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1599                   elapsed_time, log_base=options.log_base)
1600 else:
1601     elapsed_minutes = elapsed_time / 60.0
1602     print('''
1603
1604 ####################################################################
1605
1606 AUTOBUILD FAILURE
1607
1608 Your autobuild[%s] on %s failed after %.1f minutes
1609 when trying to test %s with the following error:
1610
1611    %s
1612
1613 the autobuild has been abandoned. Please fix the error and resubmit.
1614
1615 ####################################################################
1616
1617 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1618
1619 cleanup()
1620 do_print(errstr)
1621 do_print("Logs in logs.tar.gz")
1622 sys.exit(status)