script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34
  35 def find_git_root():
  36     '''get to the top of the git repo'''
  37     p = os.getcwd()
  38     while p != '/':
  39         if os.path.isdir(os.path.join(p, ".git")):
  40             return p
  41         p = os.path.abspath(os.path.join(p, '..'))
  42     return None
  43
  44
  45 gitroot = find_git_root()
  46 if gitroot is None:
  47     raise Exception("Failed to find git root")
  48
  49
  50 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  51
  52 parser = OptionParser()
  53 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
  54 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
  55 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
  56 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  57                   default=def_testbase)
  58 parser.add_option("", "--passcmd", help="command to run on success", default=None)
  59 parser.add_option("", "--verbose", help="show all commands as they are run",
  60                   default=False, action="store_true")
  61 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
  62                   default=None, type='str')
  63 parser.add_option("", "--pushto", help="push to a git url on success",
  64                   default=None, type='str')
  65 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
  66                   default=False, action="store_true")
  67 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
  68                   default=False, action="store_true")
  69 parser.add_option("", "--retry", help="automatically retry if master changes",
  70                   default=False, action="store_true")
  71 parser.add_option("", "--email", help="send email to the given address on failure",
  72                   type='str', default=None)
  73 parser.add_option("", "--email-from", help="send email from the given address",
  74                   type='str', default="autobuild@samba.org")
  75 parser.add_option("", "--email-server", help="send email via the given server",
  76                   type='str', default='localhost')
  77 parser.add_option("", "--always-email", help="always send email, even on success",
  78                   action="store_true")
  79 parser.add_option("", "--daemon", help="daemonize after initial setup",
  80                   action="store_true")
  81 parser.add_option("", "--branch", help="the branch to work on (default=master)",
  82                   default="master", type='str')
  83 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
  84                   default=gitroot, type='str')
  85 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
  86                   default=False, action="store_true")
  87 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
  88                   default='')
  89 parser.add_option("--enable-coverage", dest='enable_coverage',
  90                   action="store_const", const='--enable-coverage', default='',
  91                   help="Add --enable-coverage option while configure")
  92
  93 (options, args) = parser.parse_args()
  94
  95 if options.retry:
  96     if options.rebase is None:
  97         raise Exception('You can only use --retry if you also rebase')
  98
  99 testbase = "%s/b%u" % (options.testbase, os.getpid())
 100 test_master = "%s/master" % testbase
 101 test_prefix = "%s/prefix" % testbase
 102 test_tmpdir = "%s/tmp" % testbase
 103 os.environ['TMPDIR'] = test_tmpdir
 104
 105 if options.enable_coverage:
 106     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 107 else:
 108     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 109
 110 cleanup_list = []
 111
 112 builddirs = {
 113     "ctdb": "ctdb",
 114     "samba": ".",
 115     "samba-nt4": ".",
 116     "samba-fileserver": ".",
 117     "samba-ad-member": ".",
 118     "samba-xc": ".",
 119     "samba-o3": ".",
 120     "samba-ctdb": ".",
 121     "samba-libs": ".",
 122     "samba-static": ".",
 123     "samba-none-env": ".",
 124     "samba-ad-dc-1": ".",
 125     "samba-ad-dc-2": ".",
 126     "samba-ad-dc-3": ".",
 127     "samba-ad-dc-4": ".",
 128     "samba-ad-dc-5": ".",
 129     "samba-ad-dc-6": ".",
 130     "samba-ad-dc-ntvfs": ".",
 131     "samba-ad-dc-backup": ".",
 132     "samba-systemkrb5": ".",
 133     "samba-nopython": ".",
 134     "samba-nopython-py2": ".",
 135     "samba-schemaupgrade": ".",
 136     "ldb": "lib/ldb",
 137     "tdb": "lib/tdb",
 138     "talloc": "lib/talloc",
 139     "replace": "lib/replace",
 140     "tevent": "lib/tevent",
 141     "pidl": "pidl"
 142 }
 143
 144 defaulttasks = builddirs.keys()
 145
 146 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
 147     defaulttasks.remove("samba-o3")
 148
 149 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
 150 samba_configure_params = " ${ENABLE_COVERAGE} --picky-developer ${PREFIX} --with-profiling-data"
 151
 152 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 153 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 154 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 155 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug --picky-developer -C ${PREFIX}"
 156 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 157 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 158 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 159
 160 tasks = {
 161     "ctdb": [("random-sleep", "../script/random-sleep.sh 300 900", "text/plain"),
 162                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
 163                ("make", "make all", "text/plain"),
 164                ("install", "make install", "text/plain"),
 165                ("test", "make autotest", "text/plain"),
 166                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 167                ("clean", "make clean", "text/plain")],
 168
 169     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 170     "samba": [
 171                 ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 172                 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 173                 ("make", "make -j", "text/plain"),
 174                 ("test", "make test FAIL_IMMEDIATELY=1 "
 175                  "TESTS='--exclude-env=none "
 176                  "--exclude-env=nt4_dc "
 177                  "--exclude-env=nt4_dc_schannel "
 178                  "--exclude-env=nt4_member "
 179                  "--exclude-env=ad_dc "
 180                  "--exclude-env=ad_dc_backup "
 181                  "--exclude-env=ad_dc_ntvfs "
 182                  "--exclude-env=ad_dc_default "
 183                  "--exclude-env=ad_dc_slowtests "
 184                  "--exclude-env=ad_dc_no_nss "
 185                  "--exclude-env=ad_dc_no_ntlm "
 186                  "--exclude-env=fl2003dc "
 187                  "--exclude-env=fl2008dc "
 188                  "--exclude-env=fl2008r2dc "
 189                  "--exclude-env=ad_member "
 190                  "--exclude-env=ad_member_idmap_rid "
 191                  "--exclude-env=ad_member_idmap_ad "
 192                  "--exclude-env=ad_member_rfc2307 "
 193                  "--exclude-env=chgdcpass "
 194                  "--exclude-env=vampire_2000_dc "
 195                  "--exclude-env=fl2000dc "
 196                  "--exclude-env=fileserver "
 197                  "--exclude-env=maptoguest "
 198                  "--exclude-env=simpleserver "
 199                  "--exclude-env=backupfromdc "
 200                  "--exclude-env=restoredc "
 201                  "--exclude-env=renamedc "
 202                  "--exclude-env=offlinebackupdc "
 203                  "--exclude-env=labdc "
 204                  "--exclude-env=preforkrestartdc "
 205                  "--exclude-env=proclimitdc "
 206                  "--exclude-env=promoted_dc "
 207                  "--exclude-env=vampire_dc "
 208                  "--exclude-env=rodc "
 209                  "--exclude-env=ad_dc_default "
 210                  "--exclude-env=ad_dc_slowtests "
 211                  "--exclude-env=schema_pair_dc "
 212                  "--exclude-env=schema_dc "
 213                  "'",
 214                  "text/plain"),
 215                 ("lcov", LCOV_CMD, "text/plain"),
 216                 ("install", "make install", "text/plain"),
 217                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 218                 ("clean", "make clean", "text/plain")],
 219
 220     "samba-nt4": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 221                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 222                     ("make", "make -j", "text/plain"),
 223                     ("test", "make test FAIL_IMMEDIATELY=1 "
 224                      "TESTS='"
 225                      "--include-env=nt4_dc "
 226                      "--include-env=nt4_dc_schannel "
 227                      "--include-env=nt4_member "
 228                      "'", "text/plain"),
 229                     ("lcov", LCOV_CMD, "text/plain"),
 230                     ("install", "make install", "text/plain"),
 231                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 232                     ("clean", "make clean", "text/plain")],
 233
 234     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 235                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 236                            ("make", "make -j", "text/plain"),
 237                            ("test", "make test FAIL_IMMEDIATELY=1 "
 238                             "TESTS='"
 239                             "--include-env=fileserver "
 240                             "--include-env=maptoguest "
 241                             "--include-env=simpleserver "
 242                             "'", "text/plain"),
 243                            ("lcov", LCOV_CMD, "text/plain"),
 244                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 245
 246     "samba-ad-member": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 247                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 248                         ("make", "make -j", "text/plain"),
 249                         ("test", "make test FAIL_IMMEDIATELY=1 "
 250                          "TESTS='"
 251                          "--include-env=ad_member "
 252                          "--include-env=ad_member_idmap_rid "
 253                          "--include-env=ad_member_idmap_ad "
 254                          "--include-env=ad_member_rfc2307 "
 255                          "'", "text/plain"),
 256                         ("lcov", LCOV_CMD, "text/plain"),
 257                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 258
 259     "samba-ad-dc-1": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 260                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 261                       ("make", "make -j", "text/plain"),
 262                       ("test", "make test FAIL_IMMEDIATELY=1 "
 263                        "TESTS='--include-env=ad_dc "
 264                        "--include-env=ad_dc_no_nss "
 265                        "--include-env=ad_dc_no_ntlm "
 266                        "'", "text/plain"),
 267                       ("lcov", LCOV_CMD, "text/plain"),
 268                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 269
 270     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 271                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 272                         ("make", "make -j", "text/plain"),
 273                         ("test", "make test FAIL_IMMEDIATELY=1 "
 274                          "TESTS='"
 275                          "--include-env=vampire_dc "
 276                          "--include-env=vampire_2000_dc "
 277                          "--include-env=rodc "
 278                          "'", "text/plain"),
 279                         ("lcov", LCOV_CMD, "text/plain"),
 280                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 281
 282     "samba-ad-dc-3": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 283                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 284                         ("make", "make -j", "text/plain"),
 285                         ("test", "make test FAIL_IMMEDIATELY=1 "
 286                          "TESTS='"
 287                          "--include-env=promoted_dc "
 288                          "--include-env=chgdcpass "
 289                          "--include-env=preforkrestartdc "
 290                          "--include-env=proclimitdc "
 291                          "'", "text/plain"),
 292                         ("lcov", LCOV_CMD, "text/plain"),
 293                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 294
 295     "samba-ad-dc-4": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 296                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 297                         ("make", "make -j", "text/plain"),
 298                         ("test", "make test FAIL_IMMEDIATELY=1 "
 299                          "TESTS='"
 300                          "--include-env=fl2000dc "
 301                          "--include-env=fl2003dc "
 302                          "--include-env=fl2008dc "
 303                          "--include-env=fl2008r2dc "
 304                          "'", "text/plain"),
 305                         ("lcov", LCOV_CMD, "text/plain"),
 306                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 307
 308     "samba-ad-dc-5": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 309                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 310                         ("make", "make -j", "text/plain"),
 311                         ("test", "make test FAIL_IMMEDIATELY=1 "
 312                          "TESTS='"
 313                          "--include-env=ad_dc_default "
 314                          "'", "text/plain"),
 315                         ("lcov", LCOV_CMD, "text/plain"),
 316                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 317
 318     "samba-ad-dc-6": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 319                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 320                         ("make", "make -j", "text/plain"),
 321                         ("test", "make test FAIL_IMMEDIATELY=1 "
 322                          "TESTS='"
 323                          "--include-env=ad_dc_slowtests "
 324                          "'", "text/plain"),
 325                         ("lcov", LCOV_CMD, "text/plain"),
 326                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 327
 328
 329     "samba-schemaupgrade": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 330                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 331                         ("make", "make -j", "text/plain"),
 332                         ("test", "make test FAIL_IMMEDIATELY=1 "
 333                          "TESTS='"
 334                          "--include-env=schema_dc "
 335                          "--include-env=schema_pair_dc "
 336                          "'", "text/plain"),
 337                         ("lcov", LCOV_CMD, "text/plain"),
 338                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 339
 340     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 341     # This is currently the longest task, so we don't randomly delay it.
 342     "samba-ad-dc-ntvfs": [
 343                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 344                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 345                       ("make", "make -j", "text/plain"),
 346                       ("test", "make test FAIL_IMMEDIATELY=1 "
 347                        "TESTS='"
 348                        "--include-env=ad_dc_ntvfs "
 349                        "'", "text/plain"),
 350                       ("lcov", LCOV_CMD, "text/plain"),
 351                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 352
 353     # run the backup/restore testenvs separately as they're fairly standalone
 354     # (and CI seems to max out at ~8 different DCs running at once)
 355     "samba-ad-dc-backup": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 356                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 357                         ("make", "make -j", "text/plain"),
 358                         ("test", "make test FAIL_IMMEDIATELY=1 "
 359                          "TESTS='--include-env=backupfromdc "
 360                          "--include-env=restoredc "
 361                          "--include-env=renamedc "
 362                          "--include-env=offlinebackupdc "
 363                          "--include-env=labdc "
 364                          "--include-env=ad_dc_backup "
 365                          "'", "text/plain"),
 366                         ("lcov", LCOV_CMD, "text/plain"),
 367                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 368
 369     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 370                           ("make", "make -j", "text/plain"),
 371                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain"),
 372                           ("lcov", LCOV_CMD, "text/plain"),
 373                         ],
 374
 375     # Test cross-compile infrastructure
 376     "samba-xc": [("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
 377                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 378                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 379                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 380                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 381                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 382                    ("compare-results", "script/compare_cc_results.py "
 383                     "./bin/c4che/default{} "
 384                     "./bin-xe/c4che/default{} "
 385                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
 386
 387     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 388     "samba-o3": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 389                    ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 390                    ("make", "make -j", "text/plain"),
 391                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
 392                     "TESTS='--include-env=ad_dc'", "text/plain"),
 393                    ("lcov", LCOV_CMD, "text/plain"),
 394                    ("install", "make install", "text/plain"),
 395                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 396                    ("clean", "make clean", "text/plain")],
 397
 398     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
 399
 400                      # make sure we have tdb around:
 401                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 402                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 403                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 404
 405
 406                      # build samba with cluster support (also building ctdb):
 407                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 408                      ("samba-make", "make", "text/plain"),
 409                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 410                      ("samba-install", "make install", "text/plain"),
 411                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 412
 413                      # clean up:
 414                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 415                      ("clean", "make clean", "text/plain"),
 416                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
 417
 418     "samba-libs": [
 419                       ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 420                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 421                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 422                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 423
 424                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 425                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 426                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 427
 428                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 429                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 430                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 431
 432                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 433                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 434                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 435
 436                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 437                       ("nondevel-make", "make -j", "text/plain"),
 438                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 439                       ("nondevel-install", "make install", "text/plain"),
 440                       ("nondevel-dist", "make dist", "text/plain"),
 441
 442                       # retry with all modules shared
 443                       ("allshared-distclean", "make distclean", "text/plain"),
 444                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 445                       ("allshared-make", "make -j", "text/plain")],
 446
 447     "samba-none-env": [
 448                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 449                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 450                       ("make", "make -j", "text/plain"),
 451                       ("test", "make test "
 452                        "FAIL_IMMEDIATELY=1 "
 453                        "TESTS='--include-env=none'",
 454                        "text/plain"),
 455                       ("lcov", LCOV_CMD, "text/plain")],
 456
 457     "samba-static": [
 458                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 459                       # build with all modules static
 460                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 461                       ("allstatic-make", "make -j", "text/plain"),
 462                       ("allstatic-test", "make test "
 463                        "FAIL_IMMEDIATELY=1 "
 464                        "TESTS='samba3.smb2.create.*nt4_dc'",
 465                        "text/plain"),
 466                       ("lcov", LCOV_CMD, "text/plain"),
 467
 468                       # retry without any required modules
 469                       ("none-distclean", "make distclean", "text/plain"),
 470                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 471                       ("none-make", "make -j", "text/plain"),
 472
 473                       # retry with nonshared smbd and smbtorture
 474                       ("nonshared-distclean", "make distclean", "text/plain"),
 475                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 476                       ("nonshared-make", "make -j", "text/plain")],
 477
 478     "samba-systemkrb5": [
 479                       ("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
 480                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --with-experimental-mit-ad-dc", "text/plain"),
 481                       ("make", "make -j", "text/plain"),
 482                       # we currently cannot run a full make test, a limited list of tests could be run
 483                       # via "make test TESTS=sometests"
 484                       ("test", "make test FAIL_IMMEDIATELY=1 "
 485                        "TESTS='--include-env=ktest'", "text/plain"),
 486                       ("lcov", LCOV_CMD, "text/plain"),
 487                       ("install", "make install", "text/plain"),
 488                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 489                       ("clean", "make clean", "text/plain")
 490                       ],
 491
 492     # Test Samba without python still builds.  When this test fails
 493     # due to more use of Python, the expectations is that the newly
 494     # failing part of the code should be disabled when
 495     # --disable-python is set (rather than major work being done to
 496     # support this environment).  The target here is for vendors
 497     # shipping a minimal smbd.
 498     "samba-nopython": [
 499                       ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 500                       ("configure", "./configure.developer ${ENABLE_COVERAGE} --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 501                       ("make", "make -j", "text/plain"),
 502                       ("install", "make install", "text/plain"),
 503                       ("find-python", "script/find_python.sh ${PREFIX}", "text/plain"),
 504                       ("test", "make test-nopython", "text/plain"),
 505                       ("lcov", LCOV_CMD, "text/plain"),
 506                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 507                       ("clean", "make clean", "text/plain"),
 508
 509                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 510                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 511                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 512
 513                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 514                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 515                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 516
 517                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 518                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 519                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 520
 521                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 522                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 523                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 524
 525                       # retry against installed library packages
 526                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 527                       ("libs-make", "make -j", "text/plain"),
 528                       ("libs-install", "make install", "text/plain"),
 529                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 530                       ("libs-clean", "make clean", "text/plain")
 531                       ],
 532
 533     # check we can do the same thing using python2
 534     "samba-nopython-py2": [
 535                       ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 536                       ("configure", "PYTHON=python2 ./configure.developer ${ENABLE_COVERAGE} --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 537                       ("make", "PYTHON=python2 make -j", "text/plain"),
 538                       ("install", "PYTHON=python2 make install", "text/plain"),
 539                       ("find-python", "script/find_python.sh ${PREFIX}", "text/plain"),
 540                       ("test", "make test-nopython", "text/plain"),
 541                       ("lcov", LCOV_CMD, "text/plain"),
 542                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 543                       ("clean", "PYTHON=python2 make clean", "text/plain"),
 544
 545                       ("talloc-configure", "cd lib/talloc && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 546                       ("talloc-make", "cd lib/talloc && PYTHON=python2 make", "text/plain"),
 547                       ("talloc-install", "cd lib/talloc && PYTHON=python2 make install", "text/plain"),
 548
 549                       ("tdb-configure", "cd lib/tdb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 550                       ("tdb-make", "cd lib/tdb && PYTHON=python2 make", "text/plain"),
 551                       ("tdb-install", "cd lib/tdb && PYTHON=python2 make install", "text/plain"),
 552
 553                       ("tevent-configure", "cd lib/tevent && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 554                       ("tevent-make", "cd lib/tevent && PYTHON=python2 make", "text/plain"),
 555                       ("tevent-install", "cd lib/tevent && PYTHON=python2 make install", "text/plain"),
 556
 557                       ("ldb-configure", "cd lib/ldb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 558                       ("ldb-make", "cd lib/ldb && PYTHON=python2 make", "text/plain"),
 559                       ("ldb-install", "cd lib/ldb && PYTHON=python2 make install", "text/plain"),
 560
 561                       # retry against installed library packages
 562                       ("libs-configure", "PYTHON=python2 " + samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 563                       ("libs-make", "PYTHON=python2 make -j", "text/plain"),
 564                       ("libs-install", "PYTHON=python2 make install", "text/plain"),
 565                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 566                       ("libs-clean", "PYTHON=python2 make clean", "text/plain")
 567                       ],
 568
 569     "ldb": [
 570               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 571               ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}", "text/plain"),
 572               ("make", "make", "text/plain"),
 573               ("install", "make install", "text/plain"),
 574               ("test", "make test", "text/plain"),
 575               ("lcov", LCOV_CMD, "text/plain"),
 576               ("clean", "make clean", "text/plain"),
 577               ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}", "text/plain"),
 578               ("make-no-lmdb", "make", "text/plain"),
 579               ("test-no-lmdb", "make test", "text/plain"),
 580               ("lcov-no-lmdb", LCOV_CMD, "text/plain"),
 581               ("install-no-lmdb", "make install", "text/plain"),
 582               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 583               ("distcheck", "make distcheck", "text/plain"),
 584               ("clean", "make clean", "text/plain")],
 585
 586     "tdb": [
 587               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 588               ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}", "text/plain"),
 589               ("make", "make", "text/plain"),
 590               ("install", "make install", "text/plain"),
 591               ("test", "make test", "text/plain"),
 592               ("lcov", LCOV_CMD, "text/plain"),
 593               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 594               ("distcheck", "make distcheck", "text/plain"),
 595               ("clean", "make clean", "text/plain")],
 596
 597     "talloc": [
 598                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 599                  ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}", "text/plain"),
 600                  ("make", "make", "text/plain"),
 601                  ("install", "make install", "text/plain"),
 602                  ("test", "make test", "text/plain"),
 603                  ("lcov", LCOV_CMD, "text/plain"),
 604                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 605                  ("distcheck", "make distcheck", "text/plain"),
 606                  ("clean", "make clean", "text/plain")],
 607
 608     "replace": [
 609                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 610                   ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}", "text/plain"),
 611                   ("make", "make", "text/plain"),
 612                   ("install", "make install", "text/plain"),
 613                   ("test", "make test", "text/plain"),
 614                   ("lcov", LCOV_CMD, "text/plain"),
 615                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 616                   ("distcheck", "make distcheck", "text/plain"),
 617                   ("clean", "make clean", "text/plain")],
 618
 619     "tevent": [
 620                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 621                  ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}", "text/plain"),
 622                  ("make", "make", "text/plain"),
 623                  ("install", "make install", "text/plain"),
 624                  ("test", "make test", "text/plain"),
 625                  ("lcov", LCOV_CMD, "text/plain"),
 626                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 627                  ("distcheck", "make distcheck", "text/plain"),
 628                  ("clean", "make clean", "text/plain")],
 629
 630     "pidl": [
 631         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 632         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 633         ("touch", "touch *.yp", "text/plain"),
 634         ("make", "make", "text/plain"),
 635         ("test", "make test", "text/plain"),
 636         ("install", "make install", "text/plain"),
 637         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 638         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 639         ("clean", "make clean", "text/plain")],
 640
 641
 642     # these are useful for debugging autobuild
 643     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
 644     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
 645
 646
 647 }
 648
 649
 650 def do_print(msg):
 651     print("%s" % msg)
 652     sys.stdout.flush()
 653     sys.stderr.flush()
 654
 655
 656 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 657     if show is None:
 658         show = options.verbose
 659     if show:
 660         do_print("Running: '%s' in '%s'" % (cmd, dir))
 661     if output:
 662         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
 663     elif checkfail:
 664         return check_call(cmd, shell=True, cwd=dir)
 665     else:
 666         return call(cmd, shell=True, cwd=dir)
 667
 668
 669 class builder(object):
 670     '''handle build of one directory'''
 671
 672     def __init__(self, name, sequence, cp=True):
 673         self.name = name
 674         if name in builddirs:
 675             self.dir = builddirs[name]
 676         else:
 677             self.dir = "."
 678
 679         self.tag = self.name.replace('/', '_')
 680         self.sequence = sequence
 681         self.next = 0
 682         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 683         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 684         if options.verbose:
 685             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 686             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 687         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 688         self.stdout = open(self.stdout_path, 'w')
 689         self.stderr = open(self.stderr_path, 'w')
 690         self.stdin  = open("/dev/null", 'r')
 691         self.test_source_dir = "%s/%s" % (testbase, self.tag)
 692         self.prefix = "%s/%s" % (test_prefix, self.tag)
 693         run_cmd("rm -rf %s" % self.test_source_dir)
 694         run_cmd("rm -rf %s" % self.prefix)
 695         if cp:
 696             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
 697         else:
 698             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
 699         self.start_next()
 700
 701     def start_next(self):
 702         if self.next == len(self.sequence):
 703             if not options.nocleanup:
 704                 run_cmd("rm -rf %s" % self.test_source_dir)
 705                 run_cmd("rm -rf %s" % self.prefix)
 706             do_print('%s: Completed OK' % self.name)
 707             self.done = True
 708             return
 709         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 710         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
 711         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 712         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 713         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 714         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
 715         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
 716         self.cmd = self.cmd.replace("${NAME}", self.name)
 717         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
 718 #        if self.output_mime_type == "text/x-subunit":
 719 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 720         cwd = os.getcwd()
 721         os.chdir("%s/%s" % (self.test_source_dir, self.dir))
 722         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
 723         self.proc = Popen(self.cmd, shell=True,
 724                           close_fds=True,
 725                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 726         os.chdir(cwd)
 727         self.next += 1
 728
 729
 730 class buildlist(object):
 731     '''handle build of multiple directories'''
 732
 733     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 734         global tasks
 735         self.tlist = []
 736         self.tail_proc = None
 737         self.retry = None
 738         if tasknames == []:
 739             if options.restrict_tests:
 740                 tasknames = ["samba-test-only"]
 741             else:
 742                 tasknames = defaulttasks
 743         else:
 744             # If we are only running one test,
 745             # do not sleep randomly to wait for it to start
 746             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 747
 748         for n in tasknames:
 749             b = builder(n, tasks[n], cp=n is not "pidl")
 750             self.tlist.append(b)
 751         if options.retry:
 752             rebase_remote = "rebaseon"
 753             retry_task = [("retry",
 754                             '''set -e
 755                             git remote add -t %s %s %s
 756                             git fetch %s
 757                             while :; do
 758                               sleep 60
 759                               git describe %s/%s > old_remote_branch.desc
 760                               git fetch %s
 761                               git describe %s/%s > remote_branch.desc
 762                               diff old_remote_branch.desc remote_branch.desc
 763                             done
 764                            ''' % (
 765                                 rebase_branch, rebase_remote, rebase_url,
 766                                rebase_remote,
 767                                rebase_remote, rebase_branch,
 768                                rebase_remote,
 769                                rebase_remote, rebase_branch
 770                             ),
 771                             "test/plain")]
 772
 773             self.retry = builder('retry', retry_task, cp=False)
 774             self.need_retry = False
 775
 776     def kill_kids(self):
 777         if self.tail_proc is not None:
 778             self.tail_proc.terminate()
 779             self.tail_proc.wait()
 780             self.tail_proc = None
 781         if self.retry is not None:
 782             self.retry.proc.terminate()
 783             self.retry.proc.wait()
 784             self.retry = None
 785         for b in self.tlist:
 786             if b.proc is not None:
 787                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
 788                 b.proc.terminate()
 789                 b.proc.wait()
 790                 b.proc = None
 791
 792     def wait_one(self):
 793         while True:
 794             none_running = True
 795             for b in self.tlist:
 796                 if b.proc is None:
 797                     continue
 798                 none_running = False
 799                 b.status = b.proc.poll()
 800                 if b.status is None:
 801                     continue
 802                 b.proc = None
 803                 return b
 804             if options.retry:
 805                 ret = self.retry.proc.poll()
 806                 if ret is not None:
 807                     self.need_retry = True
 808                     self.retry = None
 809                     return None
 810             if none_running:
 811                 return None
 812             time.sleep(0.1)
 813
 814     def run(self):
 815         while True:
 816             b = self.wait_one()
 817             if options.retry and self.need_retry:
 818                 self.kill_kids()
 819                 do_print("retry needed")
 820                 return (0, None, None, None, "retry")
 821             if b is None:
 822                 break
 823             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 824                 self.kill_kids()
 825                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 826             b.start_next()
 827         self.kill_kids()
 828         return (0, None, None, None, "All OK")
 829
 830     def write_system_info(self):
 831         filename = 'system-info.txt'
 832         f = open(filename, 'w')
 833         for cmd in ['uname -a',
 834                     'lsb_release -a',
 835                     'free',
 836                     'mount',
 837                     'cat /proc/cpuinfo',
 838                     'cc --version',
 839                     'df -m .',
 840                     'df -m %s' % testbase]:
 841             out = run_cmd(cmd, output=True, checkfail=False)
 842             print('### %s' % cmd, file=f)
 843             print(out.decode('utf8', 'backslashreplace'), file=f)
 844             print(file=f)
 845         f.close()
 846         return filename
 847
 848     def tarlogs(self, fname):
 849         tar = tarfile.open(fname, "w:gz")
 850         for b in self.tlist:
 851             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 852             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 853         if os.path.exists("autobuild.log"):
 854             tar.add("autobuild.log")
 855         sys_info = self.write_system_info()
 856         tar.add(sys_info)
 857         tar.close()
 858
 859     def remove_logs(self):
 860         for b in self.tlist:
 861             os.unlink(b.stdout_path)
 862             os.unlink(b.stderr_path)
 863
 864     def start_tail(self):
 865         cmd = ["tail", "-f"]
 866         for b in self.tlist:
 867             cmd.append(b.stdout_path)
 868             cmd.append(b.stderr_path)
 869         self.tail_proc = Popen(cmd, close_fds=True)
 870
 871
 872 def cleanup():
 873     if options.nocleanup:
 874         return
 875     run_cmd("stat %s || true" % test_tmpdir, show=True)
 876     run_cmd("stat %s" % testbase, show=True)
 877     do_print("Cleaning up %r" % cleanup_list)
 878     for d in cleanup_list:
 879         run_cmd("rm -rf %s" % d)
 880
 881
 882 def daemonize(logfile):
 883     pid = os.fork()
 884     if pid == 0:  # Parent
 885         os.setsid()
 886         pid = os.fork()
 887         if pid != 0:  # Actual daemon
 888             os._exit(0)
 889     else:  # Grandparent
 890         os._exit(0)
 891
 892     import resource      # Resource usage information.
 893     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 894     if maxfd == resource.RLIM_INFINITY:
 895         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 896     for fd in range(0, maxfd):
 897         try:
 898             os.close(fd)
 899         except OSError:
 900             pass
 901     os.open(logfile, os.O_RDWR | os.O_CREAT)
 902     os.dup2(0, 1)
 903     os.dup2(0, 2)
 904
 905
 906 def write_pidfile(fname):
 907     '''write a pid file, cleanup on exit'''
 908     f = open(fname, mode='w')
 909     f.write("%u\n" % os.getpid())
 910     f.close()
 911
 912
 913 def rebase_tree(rebase_url, rebase_branch="master"):
 914     rebase_remote = "rebaseon"
 915     do_print("Rebasing on %s" % rebase_url)
 916     run_cmd("git describe HEAD", show=True, dir=test_master)
 917     run_cmd("git remote add -t %s %s %s" %
 918             (rebase_branch, rebase_remote, rebase_url),
 919             show=True, dir=test_master)
 920     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 921     if options.fix_whitespace:
 922         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 923                 (rebase_remote, rebase_branch),
 924                 show=True, dir=test_master)
 925     else:
 926         run_cmd("git rebase --force-rebase %s/%s" %
 927                 (rebase_remote, rebase_branch),
 928                 show=True, dir=test_master)
 929     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 930                    (rebase_remote, rebase_branch),
 931                    dir=test_master, output=True)
 932     if diff == '':
 933         do_print("No differences between HEAD and %s/%s - exiting" %
 934                  (rebase_remote, rebase_branch))
 935         sys.exit(0)
 936     run_cmd("git describe %s/%s" %
 937             (rebase_remote, rebase_branch),
 938             show=True, dir=test_master)
 939     run_cmd("git describe HEAD", show=True, dir=test_master)
 940     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 941             (rebase_remote, rebase_branch),
 942             show=True, dir=test_master)
 943
 944
 945 def push_to(push_url, push_branch="master"):
 946     push_remote = "pushto"
 947     do_print("Pushing to %s" % push_url)
 948     if options.mark:
 949         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 950         run_cmd("git commit --amend -c HEAD", dir=test_master)
 951         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 952         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 953     run_cmd("git remote add -t %s %s %s" %
 954             (push_branch, push_remote, push_url),
 955             show=True, dir=test_master)
 956     run_cmd("git push %s +HEAD:%s" %
 957             (push_remote, push_branch),
 958             show=True, dir=test_master)
 959
 960
 961 def send_email(subject, text, log_tar):
 962     if options.email is None:
 963         do_print("not sending email because the recipient is not set")
 964         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
 965                  (subject, text))
 966         return
 967     outer = MIMEMultipart()
 968     outer['Subject'] = subject
 969     outer['To'] = options.email
 970     outer['From'] = options.email_from
 971     outer['Date'] = email.utils.formatdate(localtime=True)
 972     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 973     outer.attach(MIMEText(text, 'plain'))
 974     if options.attach_logs:
 975         fp = open(log_tar, 'rb')
 976         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 977         fp.close()
 978         # Set the filename parameter
 979         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 980         outer.attach(msg)
 981     content = outer.as_string()
 982     s = smtplib.SMTP(options.email_server)
 983     email_user = os.getenv('SMTP_USERNAME')
 984     email_password = os.getenv('SMTP_PASSWORD')
 985     if email_user is not None:
 986         s.starttls()
 987         s.login(email_user, email_password)
 988
 989     s.sendmail(options.email_from, [options.email], content)
 990     s.set_debuglevel(1)
 991     s.quit()
 992
 993
 994 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 995                   elapsed_time, log_base=None, add_log_tail=True):
 996     '''send an email to options.email about the failure'''
 997     elapsed_minutes = elapsed_time / 60.0
 998     if log_base is None:
 999         log_base = gitroot
1000     text = '''
1001 Dear Developer,
1002
1003 Your autobuild on %s failed after %.1f minutes
1004 when trying to test %s with the following error:
1005
1006    %s
1007
1008 the autobuild has been abandoned. Please fix the error and resubmit.
1009
1010 A summary of the autobuild process is here:
1011
1012   %s/autobuild.log
1013 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1014
1015     if options.restrict_tests:
1016         text += """
1017 The build was restricted to tests matching %s\n""" % options.restrict_tests
1018
1019     if failed_task != 'rebase':
1020         text += '''
1021 You can see logs of the failed task here:
1022
1023   %s/%s.stdout
1024   %s/%s.stderr
1025
1026 or you can get full logs of all tasks in this job here:
1027
1028   %s/logs.tar.gz
1029
1030 The top commit for the tree that was built was:
1031
1032 %s
1033
1034 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1035
1036     if add_log_tail:
1037         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1038         lines = f.readlines()
1039         log_tail = "".join(lines[-50:])
1040         num_lines = len(lines)
1041         if num_lines < 50:
1042             # Also include stderr (compile failures) if < 50 lines of stdout
1043             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1044             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1045
1046         text += '''
1047 The last 50 lines of log messages:
1048
1049 %s
1050     ''' % log_tail
1051         f.close()
1052
1053     logs = os.path.join(gitroot, 'logs.tar.gz')
1054     send_email('autobuild[%s] failure on %s for task %s during %s'
1055                % (options.branch, platform.node(), failed_task, failed_stage),
1056                text, logs)
1057
1058
1059 def email_success(elapsed_time, log_base=None):
1060     '''send an email to options.email about a successful build'''
1061     if log_base is None:
1062         log_base = gitroot
1063     text = '''
1064 Dear Developer,
1065
1066 Your autobuild on %s has succeeded after %.1f minutes.
1067
1068 ''' % (platform.node(), elapsed_time / 60.)
1069
1070     if options.restrict_tests:
1071         text += """
1072 The build was restricted to tests matching %s\n""" % options.restrict_tests
1073
1074     if options.keeplogs:
1075         text += '''
1076
1077 you can get full logs of all tasks in this job here:
1078
1079   %s/logs.tar.gz
1080
1081 ''' % log_base
1082
1083     text += '''
1084 The top commit for the tree that was built was:
1085
1086 %s
1087 ''' % top_commit_msg
1088
1089     logs = os.path.join(gitroot, 'logs.tar.gz')
1090     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1091                text, logs)
1092
1093
1094 # get the top commit message, for emails
1095 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1096 top_commit_msg = top_commit_msg.decode('utf-8', 'backslashreplace')
1097
1098 try:
1099     os.makedirs(testbase)
1100 except Exception as reason:
1101     raise Exception("Unable to create %s : %s" % (testbase, reason))
1102 cleanup_list.append(testbase)
1103
1104 if options.daemon:
1105     logfile = os.path.join(testbase, "log")
1106     do_print("Forking into the background, writing progress to %s" % logfile)
1107     daemonize(logfile)
1108
1109 write_pidfile(gitroot + "/autobuild.pid")
1110
1111 start_time = time.time()
1112
1113 while True:
1114     try:
1115         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1116         os.makedirs(test_tmpdir)
1117         # The waf uninstall code removes empty directories all the way
1118         # up the tree.  Creating a file in test_tmpdir stops it from
1119         # being removed.
1120         run_cmd("touch %s" % os.path.join(test_tmpdir,
1121                                           ".directory-is-not-empty"), show=True)
1122         run_cmd("stat %s" % test_tmpdir, show=True)
1123         run_cmd("stat %s" % testbase, show=True)
1124         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1125     except Exception:
1126         cleanup()
1127         raise
1128
1129     try:
1130         try:
1131             if options.rebase is not None:
1132                 rebase_tree(options.rebase, rebase_branch=options.branch)
1133         except Exception:
1134             cleanup_list.append(gitroot + "/autobuild.pid")
1135             cleanup()
1136             elapsed_time = time.time() - start_time
1137             email_failure(-1, 'rebase', 'rebase', 'rebase',
1138                           'rebase on %s failed' % options.branch,
1139                           elapsed_time, log_base=options.log_base)
1140             sys.exit(1)
1141         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1142         if options.tail:
1143             blist.start_tail()
1144         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1145         if status != 0 or errstr != "retry":
1146             break
1147         cleanup()
1148     except Exception:
1149         cleanup()
1150         raise
1151
1152 cleanup_list.append(gitroot + "/autobuild.pid")
1153
1154 do_print(errstr)
1155
1156 blist.kill_kids()
1157 if options.tail:
1158     do_print("waiting for tail to flush")
1159     time.sleep(1)
1160
1161 elapsed_time = time.time() - start_time
1162 if status == 0:
1163     if options.passcmd is not None:
1164         do_print("Running passcmd: %s" % options.passcmd)
1165         run_cmd(options.passcmd, dir=test_master)
1166     if options.pushto is not None:
1167         push_to(options.pushto, push_branch=options.branch)
1168     if options.keeplogs or options.attach_logs:
1169         blist.tarlogs("logs.tar.gz")
1170         do_print("Logs in logs.tar.gz")
1171     if options.always_email:
1172         email_success(elapsed_time, log_base=options.log_base)
1173     blist.remove_logs()
1174     cleanup()
1175     do_print(errstr)
1176     sys.exit(0)
1177
1178 # something failed, gather a tar of the logs
1179 blist.tarlogs("logs.tar.gz")
1180
1181 if options.email is not None:
1182     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1183                   elapsed_time, log_base=options.log_base)
1184 else:
1185     elapsed_minutes = elapsed_time / 60.0
1186     print('''
1187
1188 ####################################################################
1189
1190 AUTOBUILD FAILURE
1191
1192 Your autobuild[%s] on %s failed after %.1f minutes
1193 when trying to test %s with the following error:
1194
1195    %s
1196
1197 the autobuild has been abandoned. Please fix the error and resubmit.
1198
1199 ####################################################################
1200
1201 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1202
1203 cleanup()
1204 do_print(errstr)
1205 do_print("Logs in logs.tar.gz")
1206 sys.exit(status)