script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from subprocess import call, check_call, check_output, Popen, PIPE, CalledProcessError
   7 import os
   8 import tarfile
   9 import sys
  10 import time
  11 import random
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34
  35 def find_git_root():
  36     '''get to the top of the git repo'''
  37     p = os.getcwd()
  38     while p != '/':
  39         if os.path.exists(os.path.join(p, ".git")):
  40             return p
  41         p = os.path.abspath(os.path.join(p, '..'))
  42     return None
  43
  44
  45 gitroot = find_git_root()
  46 if gitroot is None:
  47     raise Exception("Failed to find git root")
  48
  49
  50 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  51
  52 parser = OptionParser()
  53 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  54 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  55 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  56 parser.add_option("--skip-dependencies", help="skip to run task dependency tasks", default=False, action="store_true")
  57 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  58                   default=def_testbase)
  59 parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase,
  60                   default=None)
  61 parser.add_option("--passcmd", help="command to run on success", default=None)
  62 parser.add_option("--verbose", help="show all commands as they are run",
  63                   default=False, action="store_true")
  64 parser.add_option("--rebase", help="rebase on the given tree before testing",
  65                   default=None, type='str')
  66 parser.add_option("--pushto", help="push to a git url on success",
  67                   default=None, type='str')
  68 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  69                   default=False, action="store_true")
  70 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  71                   default=False, action="store_true")
  72 parser.add_option("--retry", help="automatically retry if master changes",
  73                   default=False, action="store_true")
  74 parser.add_option("--email", help="send email to the given address on failure",
  75                   type='str', default=None)
  76 parser.add_option("--email-from", help="send email from the given address",
  77                   type='str', default="autobuild@samba.org")
  78 parser.add_option("--email-server", help="send email via the given server",
  79                   type='str', default='localhost')
  80 parser.add_option("--always-email", help="always send email, even on success",
  81                   action="store_true")
  82 parser.add_option("--daemon", help="daemonize after initial setup",
  83                   action="store_true")
  84 parser.add_option("--branch", help="the branch to work on (default=master)",
  85                   default="master", type='str')
  86 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  87                   default=gitroot, type='str')
  88 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  89                   default=False, action="store_true")
  90 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
  91                   default='')
  92 parser.add_option("--enable-coverage", dest='enable_coverage',
  93                   action="store_const", const='--enable-coverage', default='',
  94                   help="Add --enable-coverage option while configure")
  95
  96 (options, args) = parser.parse_args()
  97
  98 if options.retry:
  99     if options.rebase is None:
 100         raise Exception('You can only use --retry if you also rebase')
 101
 102 if options.full_testbase is not None:
 103     testbase = options.full_testbase
 104 else:
 105     testbase = "%s/b%u" % (options.testbase, os.getpid())
 106 test_master = "%s/master" % testbase
 107 test_prefix = "%s/prefix" % testbase
 108 test_tmpdir = "%s/tmp" % testbase
 109 os.environ['TMPDIR'] = test_tmpdir
 110
 111 if options.enable_coverage:
 112     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 113 else:
 114     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 115
 116 if options.enable_coverage:
 117     PUBLISH_DOCS = "mkdir -p ${LOG_BASE}/public && mv output/htmldocs ${LOG_BASE}/public/htmldocs"
 118 else:
 119     PUBLISH_DOCS = 'echo "HTML documentation publishing skipped since no --enable-coverage specified"'
 120
 121 CLEAN_SOURCE_TREE_CMD = "cd ${TEST_SOURCE_DIR} && script/clean-source-tree.sh"
 122
 123 if args:
 124     # If we are only running specific test,
 125     # do not sleep randomly to wait for it to start
 126     def random_sleep(low, high):
 127         return 'sleep 1'
 128 else:
 129     def random_sleep(low, high):
 130         return 'sleep {}'.format(random.randint(low, high))
 131
 132 cleanup_list = []
 133
 134 builddirs = {
 135     "ctdb": "ctdb",
 136     "ldb": "lib/ldb",
 137     "tdb": "lib/tdb",
 138     "talloc": "lib/talloc",
 139     "replace": "lib/replace",
 140     "tevent": "lib/tevent",
 141     "pidl": "pidl",
 142     "docs-xml": "docs-xml"
 143 }
 144
 145 ctdb_configure_params = " --enable-developer ${PREFIX}"
 146 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 147
 148 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 149 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 150 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 151 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 152 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 153 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 154 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 155
 156
 157 def format_option(name, value=None):
 158     """Format option as str list."""
 159     if value is None:  # boolean option
 160         return [name]
 161     if not isinstance(value, list):  # single value option
 162         value = [value]
 163     # repeatable option
 164     return ['{}={}'.format(name, item) for item in value]
 165
 166
 167 def make_test(
 168         cmd='make testonly',
 169         FAIL_IMMEDIATELY=1,
 170         INJECT_SELFTEST_PREFIX=1,
 171         TESTS='',
 172         include_envs=None,
 173         exclude_envs=None):
 174
 175     test_options = []
 176     if include_envs:
 177         test_options = format_option('--include-env', include_envs)
 178     if exclude_envs:
 179         test_options = format_option('--exclude-env', exclude_envs)
 180     if test_options:
 181         # join envs options to original test options
 182         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 183
 184     _options = []
 185     if FAIL_IMMEDIATELY:
 186         _options.append('FAIL_IMMEDIATELY=1')
 187     if TESTS:
 188         _options.append("TESTS='{}'".format(TESTS))
 189
 190     if INJECT_SELFTEST_PREFIX:
 191         _options.append("TEST_OPTIONS='--with-selftest-prefix={}'".format("${SELFTEST_PREFIX}"))
 192         _options.append("--directory='{}'".format("${TEST_SOURCE_DIR}"))
 193
 194     return ' '.join([cmd] + _options)
 195
 196
 197 # When updating this list, also update .gitlab-ci.yml to add the job
 198 # and to make it a dependency of 'page' for the coverage report.
 199
 200 tasks = {
 201     "ctdb": {
 202         "sequence": [
 203             ("random-sleep", random_sleep(300, 900)),
 204             ("configure", "./configure " + ctdb_configure_params),
 205             ("make", "make all"),
 206             ("install", "make install"),
 207             ("test", "make autotest"),
 208             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 209             ("clean", "make clean"),
 210         ],
 211     },
 212     "docs-xml": {
 213         "sequence": [
 214             ("random-sleep", random_sleep(300, 900)),
 215             ("autoconf", "autoconf"),
 216             ("configure", "./configure"),
 217             ("make", "make html htmlman"),
 218             ("publish-docs", PUBLISH_DOCS),
 219             ("clean", "make clean"),
 220         ],
 221     },
 222
 223     "samba-def-build": {
 224         "git-clone-required": True,
 225         "sequence": [
 226             ("configure", "./configure.developer" + samba_configure_params),
 227             ("make", "make -j"),
 228             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 229             ("chmod-R-a-w", "chmod -R a-w ."),
 230         ],
 231     },
 232
 233     "samba-mit-build": {
 234         "git-clone-required": True,
 235         "sequence": [
 236             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 237             ("make", "make -j"),
 238             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 239             ("chmod-R-a-w", "chmod -R a-w ."),
 240         ],
 241     },
 242
 243     "samba-nt4-build": {
 244         "git-clone-required": True,
 245         "sequence": [
 246             ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json" + samba_configure_params),
 247             ("make", "make -j"),
 248             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 249             ("chmod-R-a-w", "chmod -R a-w ."),
 250         ],
 251     },
 252
 253     "samba-h5l-build": {
 254         "git-clone-required": True,
 255         "sequence": [
 256             ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5" + samba_configure_params),
 257             ("make", "make -j"),
 258             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 259             ("chmod-R-a-w", "chmod -R a-w ."),
 260         ],
 261     },
 262
 263     "samba-no-opath-build": {
 264         "git-clone-required": True,
 265         "sequence": [
 266             ("configure", "ADDITIONAL_CFLAGS='-DDISABLE_OPATH=1' ./configure.developer --without-ad-dc " + samba_configure_params),
 267             ("make", "make -j"),
 268             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 269             ("chmod-R-a-w", "chmod -R a-w ."),
 270         ],
 271     },
 272
 273     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 274     "samba": {
 275         "sequence": [
 276             ("random-sleep", random_sleep(300, 900)),
 277             ("configure", "./configure.developer" + samba_configure_params),
 278             ("make", "make -j"),
 279             ("test", make_test(exclude_envs=[
 280             "none",
 281             "nt4_dc",
 282             "nt4_dc_smb1",
 283             "nt4_dc_smb1_done",
 284             "nt4_dc_schannel",
 285             "nt4_member",
 286             "ad_dc",
 287             "ad_dc_smb1",
 288             "ad_dc_smb1_done",
 289             "ad_dc_backup",
 290             "ad_dc_ntvfs",
 291             "ad_dc_default",
 292             "ad_dc_default_smb1",
 293             "ad_dc_slowtests",
 294             "ad_dc_no_nss",
 295             "ad_dc_no_ntlm",
 296             "fl2003dc",
 297             "fl2008dc",
 298             "fl2008r2dc",
 299             "ad_member",
 300             "ad_member_idmap_rid",
 301             "ad_member_idmap_ad",
 302             "ad_member_rfc2307",
 303             "ad_member_oneway",
 304             "chgdcpass",
 305             "vampire_2000_dc",
 306             "fl2000dc",
 307             "fileserver",
 308             "fileserver_smb1",
 309             "fileserver_smb1_done",
 310             "maptoguest",
 311             "simpleserver",
 312             "backupfromdc",
 313             "restoredc",
 314             "renamedc",
 315             "offlinebackupdc",
 316             "labdc",
 317             "preforkrestartdc",
 318             "proclimitdc",
 319             "promoted_dc",
 320             "vampire_dc",
 321             "rodc",
 322             "ad_dc_default",
 323             "ad_dc_default_smb1",
 324             "ad_dc_default_smb1_done",
 325             "ad_dc_slowtests",
 326             "schema_pair_dc",
 327             "schema_dc",
 328             "clusteredmember",
 329             ])),
 330             ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
 331             ("lcov", LCOV_CMD),
 332             ("install", "make install"),
 333             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 334             ("clean", "make clean"),
 335         ],
 336     },
 337
 338     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 339     "samba-mitkrb5": {
 340         "sequence": [
 341             ("random-sleep", random_sleep(300, 900)),
 342             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 343             ("make", "make -j"),
 344             ("test", make_test(exclude_envs=[
 345             "none",
 346             "nt4_dc",
 347             "nt4_dc_smb1",
 348             "nt4_dc_smb1_done",
 349             "nt4_dc_schannel",
 350             "nt4_member",
 351             "ad_dc",
 352             "ad_dc_smb1",
 353             "ad_dc_smb1_done",
 354             "ad_dc_backup",
 355             "ad_dc_ntvfs",
 356             "ad_dc_default",
 357             "ad_dc_default_smb1",
 358             "ad_dc_default_smb1_done",
 359             "ad_dc_slowtests",
 360             "ad_dc_no_nss",
 361             "ad_dc_no_ntlm",
 362             "fl2003dc",
 363             "fl2008dc",
 364             "fl2008r2dc",
 365             "ad_member",
 366             "ad_member_idmap_rid",
 367             "ad_member_idmap_ad",
 368             "ad_member_rfc2307",
 369             "ad_member_oneway",
 370             "chgdcpass",
 371             "vampire_2000_dc",
 372             "fl2000dc",
 373             "fileserver",
 374             "fileserver_smb1",
 375             "fileserver_smb1_done",
 376             "maptoguest",
 377             "simpleserver",
 378             "backupfromdc",
 379             "restoredc",
 380             "renamedc",
 381             "offlinebackupdc",
 382             "labdc",
 383             "preforkrestartdc",
 384             "proclimitdc",
 385             "promoted_dc",
 386             "vampire_dc",
 387             "rodc",
 388             "ad_dc_default",
 389             "ad_dc_default_smb1",
 390             "ad_dc_default_smb1_done",
 391             "ad_dc_slowtests",
 392             "schema_pair_dc",
 393             "schema_dc",
 394             "clusteredmember",
 395             ])),
 396             ("lcov", LCOV_CMD),
 397             ("install", "make install"),
 398             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 399             ("clean", "make clean"),
 400         ],
 401     },
 402
 403     "samba-nt4": {
 404         "dependency": "samba-nt4-build",
 405         "sequence": [
 406             ("random-sleep", random_sleep(300, 900)),
 407             ("test", make_test(include_envs=[
 408             "nt4_dc",
 409             "nt4_dc_smb1",
 410             "nt4_dc_smb1_done",
 411             "nt4_dc_schannel",
 412             "nt4_member",
 413             "simpleserver",
 414             ])),
 415             ("lcov", LCOV_CMD),
 416             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 417         ],
 418     },
 419
 420     "samba-fileserver": {
 421         "dependency": "samba-h5l-build",
 422         "sequence": [
 423             ("random-sleep", random_sleep(300, 900)),
 424             ("test", make_test(include_envs=[
 425             "fileserver",
 426             "fileserver_smb1",
 427             "fileserver_smb1_done",
 428             "maptoguest",
 429             "ktest", # ktest is also tested in samba and samba-mitkrb5
 430                      # but is tested here against a system Heimdal
 431             ])),
 432             ("lcov", LCOV_CMD),
 433             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 434         ],
 435     },
 436
 437     "samba-admem": {
 438         "dependency": "samba-def-build",
 439         "sequence": [
 440             ("random-sleep", random_sleep(300, 900)),
 441             ("test", make_test(include_envs=[
 442             "ad_member",
 443             "ad_member_idmap_rid",
 444             "ad_member_idmap_ad",
 445             "ad_member_rfc2307",
 446             "ad_member_offlogon",
 447             ])),
 448             ("lcov", LCOV_CMD),
 449             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 450         ],
 451     },
 452
 453     "samba-no-opath1": {
 454         "dependency": "samba-no-opath-build",
 455         "sequence": [
 456             ("random-sleep", random_sleep(300, 900)),
 457             ("test", make_test(
 458                 cmd="make testonly DISABLE_OPATH=1",
 459                 include_envs=[
 460                 "nt4_dc",
 461                 "nt4_dc_smb1",
 462                 "nt4_dc_smb1_done",
 463                 "nt4_dc_schannel",
 464                 "nt4_member",
 465                 "simpleserver",
 466                 ])),
 467             ("lcov", LCOV_CMD),
 468             ("check-clean-tree", "script/clean-source-tree.sh"),
 469         ],
 470     },
 471
 472     "samba-no-opath2": {
 473         "dependency": "samba-no-opath-build",
 474         "sequence": [
 475             ("random-sleep", random_sleep(300, 900)),
 476             ("test", make_test(
 477                 cmd="make testonly DISABLE_OPATH=1",
 478                 include_envs=[
 479                 "fileserver",
 480                 "fileserver_smb1",
 481                 "fileserver_smb1_done",
 482                 ])),
 483             ("lcov", LCOV_CMD),
 484             ("check-clean-tree", "script/clean-source-tree.sh"),
 485         ],
 486     },
 487
 488     "samba-ad-dc-1": {
 489         "dependency": "samba-def-build",
 490         "sequence": [
 491             ("random-sleep", random_sleep(1, 1)),
 492             ("test", make_test(include_envs=[
 493             "ad_dc",
 494             "ad_dc_smb1",
 495             "ad_dc_smb1_done",
 496             "ad_dc_no_nss",
 497             "ad_dc_no_ntlm",
 498             ])),
 499             ("lcov", LCOV_CMD),
 500             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 501         ],
 502     },
 503
 504     "samba-ad-dc-2": {
 505         "dependency": "samba-def-build",
 506         "sequence": [
 507             ("random-sleep", random_sleep(1, 1)),
 508             ("test", make_test(include_envs=[
 509             "vampire_dc",
 510             "vampire_2000_dc",
 511             "rodc",
 512             ])),
 513             ("lcov", LCOV_CMD),
 514             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 515         ],
 516     },
 517
 518     "samba-ad-dc-3": {
 519         "dependency": "samba-def-build",
 520         "sequence": [
 521             ("random-sleep", random_sleep(1, 1)),
 522             ("test", make_test(include_envs=[
 523             "promoted_dc",
 524             "chgdcpass",
 525             "preforkrestartdc",
 526             "proclimitdc",
 527             ])),
 528             ("lcov", LCOV_CMD),
 529             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 530         ],
 531     },
 532
 533     "samba-ad-dc-4a": {
 534         "dependency": "samba-def-build",
 535         "sequence": [
 536             ("random-sleep", random_sleep(1, 1)),
 537             ("test", make_test(include_envs=[
 538             "fl2000dc",
 539             "ad_member_oneway",
 540             "fl2003dc",
 541             ])),
 542             ("lcov", LCOV_CMD),
 543             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 544         ],
 545     },
 546     "samba-ad-dc-4b": {
 547         "dependency": "samba-def-build",
 548         "sequence": [
 549             ("random-sleep", random_sleep(1, 1)),
 550             ("test", make_test(include_envs=[
 551             "fl2008dc",
 552             "fl2008r2dc",
 553             ])),
 554             ("lcov", LCOV_CMD),
 555             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 556         ],
 557     },
 558
 559     "samba-ad-dc-5": {
 560         "dependency": "samba-def-build",
 561         "sequence": [
 562             ("random-sleep", random_sleep(1, 1)),
 563             ("test", make_test(include_envs=[
 564             "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
 565             ("lcov", LCOV_CMD),
 566             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 567         ],
 568     },
 569
 570     "samba-ad-dc-6": {
 571         "dependency": "samba-def-build",
 572         "sequence": [
 573             ("random-sleep", random_sleep(1, 1)),
 574             ("test", make_test(include_envs=["ad_dc_slowtests", "ad_dc_backup"])),
 575             ("lcov", LCOV_CMD),
 576             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 577         ],
 578     },
 579
 580     "samba-schemaupgrade": {
 581         "dependency": "samba-def-build",
 582         "sequence": [
 583             ("random-sleep", random_sleep(1, 1)),
 584             ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 585             ("lcov", LCOV_CMD),
 586             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 587         ],
 588     },
 589
 590     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 591     # This is currently the longest task, so we don't randomly delay it.
 592     "samba-ad-dc-ntvfs": {
 593         "dependency": "samba-def-build",
 594         "sequence": [
 595             ("random-sleep", random_sleep(1, 1)),
 596             ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 597             ("lcov", LCOV_CMD),
 598             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 599         ],
 600     },
 601
 602     # Test fips compliance
 603     "samba-fips": {
 604         "dependency": "samba-mit-build",
 605         "sequence": [
 606             ("random-sleep", random_sleep(1, 1)),
 607             ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
 608             # TODO: This seems to generate only an empty samba-fips.info ("lcov", LCOV_CMD),
 609             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 610         ],
 611     },
 612
 613     # run the backup/restore testenvs separately as they're fairly standalone
 614     # (and CI seems to max out at ~3 different DCs running at once)
 615     "samba-ad-back1": {
 616         "dependency": "samba-def-build",
 617         "sequence": [
 618             ("random-sleep", random_sleep(300, 900)),
 619             ("test", make_test(include_envs=[
 620             "backupfromdc",
 621             "restoredc",
 622             "renamedc",
 623             ])),
 624             ("lcov", LCOV_CMD),
 625             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 626         ],
 627     },
 628     "samba-ad-back2": {
 629         "dependency": "samba-def-build",
 630         "sequence": [
 631             ("random-sleep", random_sleep(300, 900)),
 632             ("test", make_test(include_envs=[
 633             "backupfromdc",
 634             "offlinebackupdc",
 635             "labdc",
 636             ])),
 637             ("lcov", LCOV_CMD),
 638             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 639         ],
 640     },
 641
 642     "samba-admem-mit": {
 643         "dependency": "samba-mit-build",
 644         "sequence": [
 645             ("random-sleep", random_sleep(1, 1)),
 646             ("test", make_test(include_envs=[
 647             "ad_member",
 648             "ad_member_idmap_rid",
 649             "ad_member_idmap_ad",
 650             "ad_member_rfc2307",
 651             "ad_member_offlogon",
 652             ])),
 653             ("lcov", LCOV_CMD),
 654             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 655         ],
 656     },
 657
 658     "samba-ad-dc-1-mitkrb5": {
 659         "dependency": "samba-mit-build",
 660         "sequence": [
 661             ("random-sleep", random_sleep(1, 1)),
 662             ("test", make_test(include_envs=[
 663             "ad_dc",
 664             "ad_dc_smb1",
 665             "ad_dc_smb1_done",
 666             "ad_dc_no_nss",
 667             "ad_dc_no_ntlm",
 668             ])),
 669             ("lcov", LCOV_CMD),
 670             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 671         ],
 672     },
 673
 674     "samba-ad-dc-4a-mitkrb5": {
 675         "dependency": "samba-mit-build",
 676         "sequence": [
 677             ("random-sleep", random_sleep(1, 1)),
 678             ("test", make_test(include_envs=[
 679             "fl2000dc",
 680             "ad_member_oneway",
 681             "fl2003dc",
 682             ])),
 683             ("lcov", LCOV_CMD),
 684             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 685         ],
 686     },
 687     "samba-ad-dc-4b-mitkrb5": {
 688         "dependency": "samba-mit-build",
 689         "sequence": [
 690             ("random-sleep", random_sleep(1, 1)),
 691             ("test", make_test(include_envs=[
 692             "fl2000dc",
 693             "fl2003dc",
 694             ])),
 695             ("lcov", LCOV_CMD),
 696             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 697         ],
 698     },
 699
 700     "samba-test-only": {
 701         "sequence": [
 702             ("configure", "./configure.developer  --abi-check-disable" + samba_configure_params),
 703             ("make", "make -j"),
 704             ("test", make_test(TESTS="${TESTS}")),
 705             ("lcov", LCOV_CMD),
 706         ],
 707     },
 708
 709     # Test cross-compile infrastructure
 710     "samba-xc": {
 711         "sequence": [
 712             ("random-sleep", random_sleep(900, 1500)),
 713             ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 714             ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 715             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 716             ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
 717             ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 718             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 719             ("compare-results", "script/compare_cc_results.py "
 720             "./bin/c4che/default{} "
 721             "./bin-xe/c4che/default{} "
 722             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 723             ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
 724             ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
 725             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
 726             ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
 727             " = \"'1234'\"".format(CACHE_SUFFIX)),
 728             ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
 729             ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
 730             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
 731             " ; test $? -ne 0"),
 732         ],
 733     },
 734
 735     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 736     "samba-o3": {
 737         "sequence": [
 738             ("random-sleep", random_sleep(300, 900)),
 739             ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --abi-check-disable" + samba_configure_params),
 740             ("make", "make -j"),
 741             ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 742             ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 743             ("lcov", LCOV_CMD),
 744             ("install", "make install"),
 745             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 746             ("clean", "make clean"),
 747         ],
 748     },
 749
 750     "samba-ctdb": {
 751         "sequence": [
 752             ("random-sleep", random_sleep(900, 1500)),
 753
 754         # make sure we have tdb around:
 755             ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 756             ("tdb-make", "cd lib/tdb && make"),
 757             ("tdb-install", "cd lib/tdb && make install"),
 758
 759         # build samba with cluster support (also building ctdb):
 760             ("samba-configure",
 761          "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
 762          "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
 763          "./configure.developer ${PREFIX} "
 764          "--with-selftest-prefix=./bin/ab "
 765          "--enable-clangdb "
 766          "--with-cluster-support "
 767          "--without-ad-dc "
 768          "--bundled-libraries=!tdb"),
 769             ("samba-make", "make"),
 770             ("samba-check", "./bin/smbd --configfile=/dev/null -b | grep CLUSTER_SUPPORT"),
 771             ("samba-install", "make install"),
 772             ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 773
 774             ("test", make_test(
 775                 cmd='make test',
 776                 INJECT_SELFTEST_PREFIX=0,
 777                 include_envs=["clusteredmember"])
 778             ),
 779
 780         # clean up:
 781             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 782             ("clean", "make clean"),
 783             ("ctdb-clean", "cd ./ctdb && make clean"),
 784         ],
 785     },
 786
 787     "samba-libs": {
 788         "sequence": [
 789             ("random-sleep", random_sleep(300, 900)),
 790             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 791             ("talloc-make", "cd lib/talloc && make"),
 792             ("talloc-install", "cd lib/talloc && make install"),
 793
 794             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 795             ("tdb-make", "cd lib/tdb && make"),
 796             ("tdb-install", "cd lib/tdb && make install"),
 797
 798             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 799             ("tevent-make", "cd lib/tevent && make"),
 800             ("tevent-install", "cd lib/tevent && make install"),
 801
 802             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
 803             ("ldb-make", "cd lib/ldb && make"),
 804             ("ldb-install", "cd lib/ldb && make install"),
 805
 806             ("nondevel-configure", "./configure ${PREFIX}"),
 807             ("nondevel-make", "make -j"),
 808             ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 809             ("nondevel-install", "make install"),
 810             ("nondevel-dist", "make dist"),
 811
 812         # retry with all modules shared
 813             ("allshared-distclean", "make distclean"),
 814             ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 815             ("allshared-make", "make -j"),
 816         ],
 817     },
 818
 819     "samba-fuzz": {
 820         "sequence": [
 821         # build the fuzzers (static) via the oss-fuzz script
 822             ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
 823             ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ADDITIONAL_LDFLAGS='-fuse-ld=bfd' ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl-fuzzer"),
 824         ],
 825     },
 826
 827     # * Test smbd and smbtorture can build semi-static
 828     #
 829     # * Test Samba without python still builds.
 830     #
 831     # When this test fails due to more use of Python, the expectations
 832     # is that the newly failing part of the code should be disabled
 833     # when --disable-python is set (rather than major work being done
 834     # to support this environment).
 835     #
 836     # The target here is for vendors shipping a minimal smbd.
 837     "samba-minimal-smbd": {
 838         "sequence": [
 839             ("random-sleep", random_sleep(300, 900)),
 840
 841         # build with all modules static
 842             ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
 843             ("allstatic-make", "make -j"),
 844             ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 845             ("allstatic-lcov", LCOV_CMD),
 846
 847         # retry with nonshared smbd and smbtorture
 848             ("nonshared-distclean", "make distclean"),
 849             ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
 850             ("nonshared-make", "make -j"),
 851             # TODO ("nonshared-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 852             # TODO ("nonshared-lcov", LCOV_CMD),
 853
 854             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 855             ("clean", "make clean"),
 856         ],
 857     },
 858
 859     "samba-nopython": {
 860         "sequence": [
 861             ("random-sleep", random_sleep(300, 900)),
 862
 863             ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
 864             ("make", "make -j"),
 865             ("find-python", "script/find_python.sh ${PREFIX}"),
 866             ("test", "make test-nopython"),
 867             ("lcov", LCOV_CMD),
 868             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 869             ("clean", "make clean"),
 870
 871             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 872             ("talloc-make", "cd lib/talloc && make"),
 873             ("talloc-install", "cd lib/talloc && make install"),
 874
 875             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 876             ("tdb-make", "cd lib/tdb && make"),
 877             ("tdb-install", "cd lib/tdb && make install"),
 878
 879             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 880             ("tevent-make", "cd lib/tevent && make"),
 881             ("tevent-install", "cd lib/tevent && make install"),
 882
 883             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 884             ("ldb-make", "cd lib/ldb && make"),
 885             ("ldb-install", "cd lib/ldb && make install"),
 886
 887         # retry against installed library packages, but no required modules
 888             ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc  --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
 889             ("libs-make", "make -j"),
 890             ("libs-install", "make install"),
 891             ("libs-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 892             ("libs-clean", "make clean"),
 893
 894         ],
 895     },
 896
 897     "ldb": {
 898         "sequence": [
 899             ("random-sleep", random_sleep(60, 600)),
 900             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 901             ("make", "make"),
 902             ("install", "make install"),
 903             ("test", "make test"),
 904             ("lcov", LCOV_CMD),
 905             ("clean", "make clean"),
 906             ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
 907             ("make-no-lmdb", "make"),
 908             ("test-no-lmdb", "make test"),
 909             ("lcov-no-lmdb", LCOV_CMD),
 910             ("install-no-lmdb", "make install"),
 911             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 912             ("distcheck", "make distcheck"),
 913             ("clean", "make clean"),
 914         ],
 915     },
 916
 917     "tdb": {
 918         "sequence": [
 919             ("random-sleep", random_sleep(60, 600)),
 920             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 921             ("make", "make"),
 922             ("install", "make install"),
 923             ("test", "make test"),
 924             ("lcov", LCOV_CMD),
 925             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 926             ("distcheck", "make distcheck"),
 927             ("clean", "make clean"),
 928         ],
 929     },
 930
 931     "talloc": {
 932         "sequence": [
 933             ("random-sleep", random_sleep(60, 600)),
 934             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 935             ("make", "make"),
 936             ("install", "make install"),
 937             ("test", "make test"),
 938             ("lcov", LCOV_CMD),
 939             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 940             ("distcheck", "make distcheck"),
 941             ("clean", "make clean"),
 942         ],
 943     },
 944
 945     "replace": {
 946         "sequence": [
 947             ("random-sleep", random_sleep(60, 600)),
 948             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 949             ("make", "make"),
 950             ("install", "make install"),
 951             ("test", "make test"),
 952             ("lcov", LCOV_CMD),
 953             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 954             ("distcheck", "make distcheck"),
 955             ("clean", "make clean"),
 956         ],
 957     },
 958
 959     "tevent": {
 960         "sequence": [
 961             ("random-sleep", random_sleep(60, 600)),
 962             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 963             ("make", "make"),
 964             ("install", "make install"),
 965             ("test", "make test"),
 966             ("lcov", LCOV_CMD),
 967             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 968             ("distcheck", "make distcheck"),
 969             ("clean", "make clean"),
 970         ],
 971     },
 972
 973     "pidl": {
 974         "git-clone-required": True,
 975         "sequence": [
 976             ("random-sleep", random_sleep(60, 600)),
 977             ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
 978             ("touch", "touch *.yp"),
 979             ("make", "make"),
 980             ("test", "make test"),
 981             ("install", "make install"),
 982             ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
 983             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 984             ("clean", "make clean"),
 985         ],
 986     },
 987
 988     # these are useful for debugging autobuild
 989     "pass": {
 990         "sequence": [
 991             ("pass", 'echo passing && /bin/true'),
 992         ],
 993     },
 994     "fail": {
 995         "sequence": [
 996             ("fail", 'echo failing && /bin/false'),
 997         ],
 998     },
 999 }
1000
1001 defaulttasks = list(tasks.keys())
1002
1003 defaulttasks.remove("pass")
1004 defaulttasks.remove("fail")
1005 defaulttasks.remove("samba-def-build")
1006 defaulttasks.remove("samba-nt4-build")
1007 defaulttasks.remove("samba-mit-build")
1008 defaulttasks.remove("samba-h5l-build")
1009 defaulttasks.remove("samba-no-opath-build")
1010 defaulttasks.remove("samba-test-only")
1011 defaulttasks.remove("samba-fuzz")
1012 defaulttasks.remove("samba-fips")
1013 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
1014     defaulttasks.remove("samba-o3")
1015
1016
1017 def do_print(msg):
1018     print("%s" % msg)
1019     sys.stdout.flush()
1020     sys.stderr.flush()
1021
1022
1023 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
1024     if show is None:
1025         show = options.verbose
1026     if show:
1027         do_print("Running: '%s' in '%s'" % (cmd, dir))
1028     if output:
1029         out = check_output([cmd], shell=True, cwd=dir)
1030         return out.decode(encoding='utf-8', errors='backslashreplace')
1031     elif checkfail:
1032         return check_call(cmd, shell=True, cwd=dir)
1033     else:
1034         return call(cmd, shell=True, cwd=dir)
1035
1036 def rmdir_force(dirname, re_raise=True):
1037     try:
1038         run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % (
1039                 dirname, dirname, dirname), output=True, show=True)
1040     except CalledProcessError as e:
1041         do_print("Failed: '%s'" % (str(e)))
1042         run_cmd("tree %s" % dirname, output=True, show=True)
1043         if re_raise:
1044             raise
1045         return False
1046     return True
1047
1048 class builder(object):
1049     '''handle build of one directory'''
1050
1051     def __init__(self, name, definition):
1052         self.name = name
1053         self.dir = builddirs.get(name, '.')
1054         self.tag = self.name.replace('/', '_')
1055         self.definition = definition
1056         self.sequence = definition["sequence"]
1057         self.git_clone_required = False
1058         if "git-clone-required" in definition:
1059             self.git_clone_required = bool(definition["git-clone-required"])
1060         self.proc = None
1061         self.done = False
1062         self.next = 0
1063         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
1064         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
1065         if options.verbose:
1066             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
1067             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
1068         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
1069         self.stdout = open(self.stdout_path, 'w')
1070         self.stderr = open(self.stderr_path, 'w')
1071         self.stdin  = open("/dev/null", 'r')
1072         self.builder_dir = "%s/%s" % (testbase, self.tag)
1073         self.test_source_dir = self.builder_dir
1074         self.cwd = "%s/%s" % (self.builder_dir, self.dir)
1075         self.selftest_prefix = "%s/bin/ab" % (self.cwd)
1076         self.prefix = "%s/%s" % (test_prefix, self.tag)
1077         self.consumers = []
1078         self.producer = None
1079
1080         if self.git_clone_required:
1081             assert "dependency" not in definition
1082
1083     def mark_existing(self):
1084         do_print('%s: Mark as existing dependency' % self.name)
1085         self.next = len(self.sequence)
1086         self.done = True
1087
1088     def add_consumer(self, consumer):
1089         do_print("%s: add consumer: %s" % (self.name, consumer.name))
1090         consumer.producer = self
1091         consumer.test_source_dir = self.test_source_dir
1092         self.consumers.append(consumer)
1093
1094     def start_next(self):
1095         if self.producer is not None:
1096             if not self.producer.done:
1097                 do_print("%s: Waiting for producer: %s" % (self.name, self.producer.name))
1098                 return
1099
1100         if self.next == 0:
1101             rmdir_force(self.builder_dir)
1102             rmdir_force(self.prefix)
1103             if self.producer is not None:
1104                 run_cmd("mkdir %s" % (self.builder_dir), dir=test_master, show=True)
1105             elif not self.git_clone_required:
1106                 run_cmd("cp -R -a -l %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1107             else:
1108                 run_cmd("git clone --recursive --shared %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1109
1110         if self.next == len(self.sequence):
1111             if not self.done:
1112                 do_print('%s: Completed OK' % self.name)
1113                 self.done = True
1114             if not options.nocleanup and len(self.consumers) == 0:
1115                 do_print('%s: Cleaning up' % self.name)
1116                 rmdir_force(self.builder_dir)
1117                 rmdir_force(self.prefix)
1118             for consumer in self.consumers:
1119                 if consumer.next != 0:
1120                     continue
1121                 do_print('%s: Starting consumer %s' % (self.name, consumer.name))
1122                 consumer.start_next()
1123             if self.producer is not None:
1124                 self.producer.consumers.remove(self)
1125                 assert self.producer.done
1126                 self.producer.start_next()
1127             do_print('%s: Remaining consumers %u' % (self.name, len(self.consumers)))
1128             return
1129         (self.stage, self.cmd) = self.sequence[self.next]
1130         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
1131         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
1132         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
1133         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
1134         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
1135         self.cmd = self.cmd.replace("${SELFTEST_PREFIX}", self.selftest_prefix)
1136         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
1137         self.cmd = self.cmd.replace("${NAME}", self.name)
1138         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
1139         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
1140         self.proc = Popen(self.cmd, shell=True,
1141                           close_fds=True, cwd=self.cwd,
1142                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
1143         self.next += 1
1144
1145 def expand_dependencies(n):
1146     deps = list()
1147     if "dependency" in tasks[n]:
1148         depname = tasks[n]["dependency"]
1149         assert depname in tasks
1150         sdeps = expand_dependencies(depname)
1151         assert n not in sdeps
1152         for sdep in sdeps:
1153             deps.append(sdep)
1154         deps.append(depname)
1155     return deps
1156
1157
1158 class buildlist(object):
1159     '''handle build of multiple directories'''
1160
1161     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
1162         self.tail_proc = None
1163         self.retry = None
1164         if not tasknames:
1165             if options.restrict_tests:
1166                 tasknames = ["samba-test-only"]
1167             else:
1168                 tasknames = defaulttasks
1169
1170         given_tasknames = tasknames.copy()
1171         implicit_tasknames = []
1172         for n in given_tasknames:
1173             deps = expand_dependencies(n)
1174             for dep in deps:
1175                 if dep in given_tasknames:
1176                     continue
1177                 if dep in implicit_tasknames:
1178                     continue
1179                 implicit_tasknames.append(dep)
1180
1181         tasknames = implicit_tasknames.copy()
1182         tasknames.extend(given_tasknames)
1183         do_print("given_tasknames: %s" % given_tasknames)
1184         do_print("implicit_tasknames: %s" % implicit_tasknames)
1185         do_print("tasknames: %s" % tasknames)
1186         self.tlist = [builder(n, tasks[n]) for n in tasknames]
1187
1188         if options.retry:
1189             rebase_remote = "rebaseon"
1190             retry_task = {
1191                     "git-clone-required": True,
1192                     "sequence": [
1193                             ("retry",
1194                             '''set -e
1195                             git remote add -t %s %s %s
1196                             git fetch %s
1197                             while :; do
1198                               sleep 60
1199                               git describe %s/%s > old_remote_branch.desc
1200                               git fetch %s
1201                               git describe %s/%s > remote_branch.desc
1202                               diff old_remote_branch.desc remote_branch.desc
1203                             done
1204                            ''' % (
1205                                rebase_branch, rebase_remote, rebase_url,
1206                                rebase_remote,
1207                                rebase_remote, rebase_branch,
1208                                rebase_remote,
1209                                rebase_remote, rebase_branch
1210                             ))]}
1211
1212             self.retry = builder('retry', retry_task)
1213             self.need_retry = False
1214
1215         if options.skip_dependencies:
1216             for b in self.tlist:
1217                 if b.name in implicit_tasknames:
1218                     b.mark_existing()
1219
1220         for b in self.tlist:
1221             do_print("b.name=%s" % b.name)
1222             if "dependency" not in b.definition:
1223                 continue
1224             depname = b.definition["dependency"]
1225             do_print("b.name=%s: dependency:%s" % (b.name, depname))
1226             for p in self.tlist:
1227                 if p.name == depname:
1228                     p.add_consumer(b)
1229
1230     def kill_kids(self):
1231         if self.tail_proc is not None:
1232             self.tail_proc.terminate()
1233             self.tail_proc.wait()
1234             self.tail_proc = None
1235         if self.retry is not None:
1236             self.retry.proc.terminate()
1237             self.retry.proc.wait()
1238             self.retry = None
1239         for b in self.tlist:
1240             if b.proc is not None:
1241                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
1242                 b.proc.terminate()
1243                 b.proc.wait()
1244                 b.proc = None
1245
1246     def wait_one(self):
1247         while True:
1248             none_running = True
1249             for b in self.tlist:
1250                 if b.proc is None:
1251                     continue
1252                 none_running = False
1253                 b.status = b.proc.poll()
1254                 if b.status is None:
1255                     continue
1256                 b.proc = None
1257                 return b
1258             if options.retry:
1259                 ret = self.retry.proc.poll()
1260                 if ret is not None:
1261                     self.need_retry = True
1262                     self.retry = None
1263                     return None
1264             if none_running:
1265                 return None
1266             time.sleep(0.1)
1267
1268     def run(self):
1269         for b in self.tlist:
1270             b.start_next()
1271         if options.retry:
1272             self.retry.start_next()
1273         while True:
1274             b = self.wait_one()
1275             if options.retry and self.need_retry:
1276                 self.kill_kids()
1277                 do_print("retry needed")
1278                 return (0, None, None, None, "retry")
1279             if b is None:
1280                 break
1281             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
1282                 self.kill_kids()
1283                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
1284             b.start_next()
1285         self.kill_kids()
1286         return (0, None, None, None, "All OK")
1287
1288     def write_system_info(self, filename):
1289         with open(filename, 'w') as f:
1290             for cmd in ['uname -a',
1291                         'lsb_release -a',
1292                         'free',
1293                         'mount',
1294                         'cat /proc/cpuinfo',
1295                         'cc --version',
1296                         'df -m .',
1297                         'df -m %s' % testbase]:
1298                 try:
1299                     out = run_cmd(cmd, output=True, checkfail=False)
1300                 except CalledProcessError as e:
1301                     out = "<failed: %s>" % str(e)
1302                 print('### %s' % cmd, file=f)
1303                 print(out, file=f)
1304                 print(file=f)
1305
1306     def tarlogs(self, fname):
1307         with tarfile.open(fname, "w:gz") as tar:
1308             for b in self.tlist:
1309                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
1310                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
1311             if os.path.exists("autobuild.log"):
1312                 tar.add("autobuild.log")
1313             filename = 'system-info.txt'
1314             self.write_system_info(filename)
1315             tar.add(filename)
1316
1317     def remove_logs(self):
1318         for b in self.tlist:
1319             os.unlink(b.stdout_path)
1320             os.unlink(b.stderr_path)
1321
1322     def start_tail(self):
1323         cmd = ["tail", "-f"]
1324         for b in self.tlist:
1325             cmd.append(b.stdout_path)
1326             cmd.append(b.stderr_path)
1327         self.tail_proc = Popen(cmd, close_fds=True)
1328
1329
1330 def cleanup(do_raise=False):
1331     if options.nocleanup:
1332         return
1333     run_cmd("stat %s || true" % test_tmpdir, show=True)
1334     run_cmd("stat %s" % testbase, show=True)
1335     do_print("Cleaning up %r" % cleanup_list)
1336     for d in cleanup_list:
1337         ok = rmdir_force(d, re_raise=False)
1338         if ok:
1339             continue
1340         if os.path.isdir(d):
1341             do_print("Killing, waiting and retry")
1342             run_cmd("killbysubdir %s > /dev/null 2>&1" % d, checkfail=False)
1343         else:
1344             do_print("Waiting and retry")
1345         time.sleep(1)
1346         rmdir_force(d, re_raise=do_raise)
1347
1348
1349 def daemonize(logfile):
1350     pid = os.fork()
1351     if pid == 0:  # Parent
1352         os.setsid()
1353         pid = os.fork()
1354         if pid != 0:  # Actual daemon
1355             os._exit(0)
1356     else:  # Grandparent
1357         os._exit(0)
1358
1359     import resource      # Resource usage information.
1360     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1361     if maxfd == resource.RLIM_INFINITY:
1362         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1363     for fd in range(0, maxfd):
1364         try:
1365             os.close(fd)
1366         except OSError:
1367             pass
1368     os.open(logfile, os.O_RDWR | os.O_CREAT)
1369     os.dup2(0, 1)
1370     os.dup2(0, 2)
1371
1372
1373 def write_pidfile(fname):
1374     '''write a pid file, cleanup on exit'''
1375     with open(fname, mode='w') as f:
1376         f.write("%u\n" % os.getpid())
1377
1378
1379 def rebase_tree(rebase_url, rebase_branch="master"):
1380     rebase_remote = "rebaseon"
1381     do_print("Rebasing on %s" % rebase_url)
1382     run_cmd("git describe HEAD", show=True, dir=test_master)
1383     run_cmd("git remote add -t %s %s %s" %
1384             (rebase_branch, rebase_remote, rebase_url),
1385             show=True, dir=test_master)
1386     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1387     if options.fix_whitespace:
1388         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1389                 (rebase_remote, rebase_branch),
1390                 show=True, dir=test_master)
1391     else:
1392         run_cmd("git rebase --force-rebase %s/%s" %
1393                 (rebase_remote, rebase_branch),
1394                 show=True, dir=test_master)
1395     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1396                    (rebase_remote, rebase_branch),
1397                    dir=test_master, output=True)
1398     if diff == '':
1399         do_print("No differences between HEAD and %s/%s - exiting" %
1400                  (rebase_remote, rebase_branch))
1401         sys.exit(0)
1402     run_cmd("git describe %s/%s" %
1403             (rebase_remote, rebase_branch),
1404             show=True, dir=test_master)
1405     run_cmd("git describe HEAD", show=True, dir=test_master)
1406     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1407             (rebase_remote, rebase_branch),
1408             show=True, dir=test_master)
1409
1410
1411 def push_to(push_url, push_branch="master"):
1412     push_remote = "pushto"
1413     do_print("Pushing to %s" % push_url)
1414     if options.mark:
1415         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1416         run_cmd("git commit --amend -c HEAD", dir=test_master)
1417         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1418         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1419     run_cmd("git remote add -t %s %s %s" %
1420             (push_branch, push_remote, push_url),
1421             show=True, dir=test_master)
1422     run_cmd("git push %s +HEAD:%s" %
1423             (push_remote, push_branch),
1424             show=True, dir=test_master)
1425
1426
1427 def send_email(subject, text, log_tar):
1428     if options.email is None:
1429         do_print("not sending email because the recipient is not set")
1430         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1431                  (subject, text))
1432         return
1433     outer = MIMEMultipart()
1434     outer['Subject'] = subject
1435     outer['To'] = options.email
1436     outer['From'] = options.email_from
1437     outer['Date'] = email.utils.formatdate(localtime=True)
1438     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1439     outer.attach(MIMEText(text, 'plain', 'utf-8'))
1440     if options.attach_logs:
1441         with open(log_tar, 'rb') as fp:
1442             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1443         # Set the filename parameter
1444         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1445         outer.attach(msg)
1446     content = outer.as_string()
1447     s = smtplib.SMTP(options.email_server)
1448     email_user = os.getenv('SMTP_USERNAME')
1449     email_password = os.getenv('SMTP_PASSWORD')
1450     if email_user is not None:
1451         s.starttls()
1452         s.login(email_user, email_password)
1453
1454     s.sendmail(options.email_from, [options.email], content)
1455     s.set_debuglevel(1)
1456     s.quit()
1457
1458
1459 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1460                   elapsed_time, log_base=None, add_log_tail=True):
1461     '''send an email to options.email about the failure'''
1462     elapsed_minutes = elapsed_time / 60.0
1463     if log_base is None:
1464         log_base = gitroot
1465     text = '''
1466 Dear Developer,
1467
1468 Your autobuild on %s failed after %.1f minutes
1469 when trying to test %s with the following error:
1470
1471    %s
1472
1473 the autobuild has been abandoned. Please fix the error and resubmit.
1474
1475 A summary of the autobuild process is here:
1476
1477   %s/autobuild.log
1478 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1479
1480     if options.restrict_tests:
1481         text += """
1482 The build was restricted to tests matching %s\n""" % options.restrict_tests
1483
1484     if failed_task != 'rebase':
1485         text += '''
1486 You can see logs of the failed task here:
1487
1488   %s/%s.stdout
1489   %s/%s.stderr
1490
1491 or you can get full logs of all tasks in this job here:
1492
1493   %s/logs.tar.gz
1494
1495 The top commit for the tree that was built was:
1496
1497 %s
1498
1499 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1500
1501     if add_log_tail:
1502         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1503         lines = f.readlines()
1504         log_tail = "".join(lines[-50:])
1505         num_lines = len(lines)
1506         if num_lines < 50:
1507             # Also include stderr (compile failures) if < 50 lines of stdout
1508             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1509             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1510
1511         text += '''
1512 The last 50 lines of log messages:
1513
1514 %s
1515     ''' % log_tail
1516         f.close()
1517
1518     logs = os.path.join(gitroot, 'logs.tar.gz')
1519     send_email('autobuild[%s] failure on %s for task %s during %s'
1520                % (options.branch, platform.node(), failed_task, failed_stage),
1521                text, logs)
1522
1523
1524 def email_success(elapsed_time, log_base=None):
1525     '''send an email to options.email about a successful build'''
1526     if log_base is None:
1527         log_base = gitroot
1528     text = '''
1529 Dear Developer,
1530
1531 Your autobuild on %s has succeeded after %.1f minutes.
1532
1533 ''' % (platform.node(), elapsed_time / 60.)
1534
1535     if options.restrict_tests:
1536         text += """
1537 The build was restricted to tests matching %s\n""" % options.restrict_tests
1538
1539     if options.keeplogs:
1540         text += '''
1541
1542 you can get full logs of all tasks in this job here:
1543
1544   %s/logs.tar.gz
1545
1546 ''' % log_base
1547
1548     text += '''
1549 The top commit for the tree that was built was:
1550
1551 %s
1552 ''' % top_commit_msg
1553
1554     logs = os.path.join(gitroot, 'logs.tar.gz')
1555     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1556                text, logs)
1557
1558
1559 # get the top commit message, for emails
1560 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1561
1562 try:
1563     if options.skip_dependencies:
1564         run_cmd("stat %s" % testbase, dir=testbase, output=True)
1565     else:
1566         os.makedirs(testbase)
1567 except Exception as reason:
1568     raise Exception("Unable to create %s : %s" % (testbase, reason))
1569 cleanup_list.append(testbase)
1570
1571 if options.daemon:
1572     logfile = os.path.join(testbase, "log")
1573     do_print("Forking into the background, writing progress to %s" % logfile)
1574     daemonize(logfile)
1575
1576 write_pidfile(gitroot + "/autobuild.pid")
1577
1578 start_time = time.time()
1579
1580 while True:
1581     try:
1582         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1583         os.makedirs(test_tmpdir)
1584         # The waf uninstall code removes empty directories all the way
1585         # up the tree.  Creating a file in test_tmpdir stops it from
1586         # being removed.
1587         run_cmd("touch %s" % os.path.join(test_tmpdir,
1588                                           ".directory-is-not-empty"), show=True)
1589         run_cmd("stat %s" % test_tmpdir, show=True)
1590         run_cmd("stat %s" % testbase, show=True)
1591         if options.skip_dependencies:
1592             run_cmd("stat %s" % test_master, dir=testbase, output=True)
1593         else:
1594             run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1595     except Exception:
1596         cleanup()
1597         raise
1598
1599     try:
1600         if options.rebase is not None:
1601             rebase_tree(options.rebase, rebase_branch=options.branch)
1602     except Exception:
1603         cleanup_list.append(gitroot + "/autobuild.pid")
1604         cleanup()
1605         elapsed_time = time.time() - start_time
1606         email_failure(-1, 'rebase', 'rebase', 'rebase',
1607                       'rebase on %s failed' % options.branch,
1608                       elapsed_time, log_base=options.log_base)
1609         sys.exit(1)
1610
1611     try:
1612         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1613         if options.tail:
1614             blist.start_tail()
1615         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1616         if status != 0 or errstr != "retry":
1617             break
1618         cleanup(do_raise=True)
1619     except Exception:
1620         cleanup()
1621         raise
1622
1623 cleanup_list.append(gitroot + "/autobuild.pid")
1624
1625 do_print(errstr)
1626
1627 blist.kill_kids()
1628 if options.tail:
1629     do_print("waiting for tail to flush")
1630     time.sleep(1)
1631
1632 elapsed_time = time.time() - start_time
1633 if status == 0:
1634     if options.passcmd is not None:
1635         do_print("Running passcmd: %s" % options.passcmd)
1636         run_cmd(options.passcmd, dir=test_master)
1637     if options.pushto is not None:
1638         push_to(options.pushto, push_branch=options.branch)
1639     if options.keeplogs or options.attach_logs:
1640         blist.tarlogs("logs.tar.gz")
1641         do_print("Logs in logs.tar.gz")
1642     if options.always_email:
1643         email_success(elapsed_time, log_base=options.log_base)
1644     blist.remove_logs()
1645     cleanup()
1646     do_print(errstr)
1647     sys.exit(0)
1648
1649 # something failed, gather a tar of the logs
1650 blist.tarlogs("logs.tar.gz")
1651
1652 if options.email is not None:
1653     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1654                   elapsed_time, log_base=options.log_base)
1655 else:
1656     elapsed_minutes = elapsed_time / 60.0
1657     print('''
1658
1659 ####################################################################
1660
1661 AUTOBUILD FAILURE
1662
1663 Your autobuild[%s] on %s failed after %.1f minutes
1664 when trying to test %s with the following error:
1665
1666    %s
1667
1668 the autobuild has been abandoned. Please fix the error and resubmit.
1669
1670 ####################################################################
1671
1672 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1673
1674 cleanup()
1675 do_print(errstr)
1676 do_print("Logs in logs.tar.gz")
1677 sys.exit(status)