script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34 cleanup_list = []
  35
  36 builddirs = {
  37     "ctdb": "ctdb",
  38     "samba": ".",
  39     "samba-py2": ".",
  40     "samba-nt4": ".",
  41     "samba-nt4-py2": ".",
  42     "samba-fileserver": ".",
  43     "samba-xc": ".",
  44     "samba-o3": ".",
  45     "samba-ctdb": ".",
  46     "samba-libs": ".",
  47     "samba-libs-py2": ".",
  48     "samba-static": ".",
  49     "samba-none-env": ".",
  50     "samba-ad-dc": ".",
  51     "samba-ad-dc-py2": ".",
  52     "samba-ad-dc-2": ".",
  53     "samba-ad-dc-2-py2": ".",
  54     "samba-ad-dc-backup": ".",
  55     "samba-systemkrb5": ".",
  56     "samba-nopython": ".",
  57     "samba-buildpy2-only": ".",
  58     "ldb": "lib/ldb",
  59     "tdb": "lib/tdb",
  60     "talloc": "lib/talloc",
  61     "replace": "lib/replace",
  62     "tevent": "lib/tevent",
  63     "pidl": "pidl"
  64 }
  65
  66 defaulttasks = builddirs.keys()
  67
  68 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  69     defaulttasks.remove("samba-o3")
  70
  71 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  72 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
  73
  74 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
  75 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  76 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  77 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
  78 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE ${EXTRA_PYTHON}"
  79 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
  80 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs + " ${EXTRA_PYTHON}"
  81
  82 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
  83     extra_python = ""
  84 else:
  85     extra_python = "--extra-python=/usr/bin/python2"
  86
  87 tasks = {
  88     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  89                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  90                ("make", "make all", "text/plain"),
  91                ("install", "make install", "text/plain"),
  92                ("test", "make autotest", "text/plain"),
  93                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  94                ("clean", "make clean", "text/plain")],
  95
  96     # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)'
  97     "samba": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
  98                 ("make", "make -j", "text/plain"),
  99                 ("test", "make test FAIL_IMMEDIATELY=1 "
 100                  "TESTS='${PY3_ONLY}"
 101                  "--exclude-env=none "
 102                  "--exclude-env=nt4_dc "
 103                  "--exclude-env=nt4_member "
 104                  "--exclude-env=ad_dc "
 105                  "--exclude-env=ad_dc_no_nss "
 106                  "--exclude-env=fl2003dc "
 107                  "--exclude-env=fl2008r2dc "
 108                  "--exclude-env=ad_member "
 109                  "--exclude-env=ad_member_idmap_rid "
 110                  "--exclude-env=ad_member_idmap_ad "
 111                  "--exclude-env=chgdcpass "
 112                  "--exclude-env=vampire_2000_dc "
 113                  "--exclude-env=fl2000dc "
 114                  "--exclude-env=fileserver "
 115                  "--exclude-env=backupfromdc "
 116                  "--exclude-env=restoredc "
 117                  "--exclude-env=renamedc "
 118                  "--exclude-env=offlinebackupdc "
 119                  "--exclude-env=labdc "
 120                  "'",
 121                  "text/plain"),
 122                 ("install", "make install", "text/plain"),
 123                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 124                 ("clean", "make clean", "text/plain")],
 125
 126     # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long)
 127     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 128                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 129                     ("make", "make -j", "text/plain"),
 130                     ("test", "make test FAIL_IMMEDIATELY=1 "
 131                      "TESTS='${PY3_ONLY}"
 132                      "--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 133                     ("install", "make install", "text/plain"),
 134                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 135                     ("clean", "make clean", "text/plain")],
 136
 137     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 138     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 139                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 140                            ("make", "make -j", "text/plain"),
 141                            ("test", "make test FAIL_IMMEDIATELY=1 "
 142                             "TESTS='${PY3_ONLY}"
 143                             "--include-env=fileserver'", "text/plain"),
 144                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 145
 146     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 147     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 148                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 149                       ("make", "make -j", "text/plain"),
 150                       ("test", "make test FAIL_IMMEDIATELY=1 "
 151                        "TESTS='${PY3_ONLY}"
 152                        "--include-env=ad_dc "
 153                        "--include-env=fl2003dc "
 154                        "--include-env=fl2008r2dc "
 155                        "--include-env=ad_member "
 156                        "--include-env=ad_member_idmap_rid "
 157                        "--include-env=ad_member_idmap_ad'", "text/plain"),
 158                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 159
 160     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 161     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 162                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 163                         ("make", "make -j", "text/plain"),
 164                         ("test", "make test FAIL_IMMEDIATELY=1 "
 165                          "TESTS='${PY3_ONLY}"
 166                          "--include-env=chgdcpass "
 167                          "--include-env=vampire_2000_dc "
 168                          "--include-env=fl2000dc "
 169                          "--include-env=ad_dc_no_nss "
 170                          "'",
 171                          "text/plain"),
 172                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 173
 174     # run the backup/restore testenvs separately as they're fairly standalone
 175     # (and CI seems to max out at ~8 different DCs running at once)
 176     "samba-ad-dc-backup": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 177                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 178                         ("make", "make -j", "text/plain"),
 179                         ("test", "make test FAIL_IMMEDIATELY=1 "
 180                          "TESTS='${PY3_ONLY}"
 181                          "--include-env=backupfromdc "
 182                          "--include-env=restoredc "
 183                          "--include-env=renamedc "
 184                          "--include-env=offlinebackupdc "
 185                          "--include-env=labdc "
 186                          "'",
 187                          "text/plain"),
 188                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 189
 190     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 191                           ("make", "make -j", "text/plain"),
 192                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
 193
 194     # Test cross-compile infrastructure
 195     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 196                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 197                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 198                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 199                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 200                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 201                    ("compare-results", "script/compare_cc_results.py "
 202                     "./bin/c4che/default{} "
 203                     "./bin-xe/c4che/default{} "
 204                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
 205
 206     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 207     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 208                    ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 209                    ("make", "make -j", "text/plain"),
 210                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
 211                     "TESTS='${PY3_ONLY}"
 212                     "--include-env=ad_dc'", "text/plain"),
 213                    ("install", "make install", "text/plain"),
 214                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 215                    ("clean", "make clean", "text/plain")],
 216
 217     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 218
 219                      # make sure we have tdb around:
 220                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 221                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 222                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 223
 224
 225                      # build samba with cluster support (also building ctdb):
 226                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 227                      ("samba-make", "make", "text/plain"),
 228                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 229                      ("samba-install", "make install", "text/plain"),
 230                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 231
 232                      # clean up:
 233                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 234                      ("clean", "make clean", "text/plain"),
 235                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
 236
 237     "samba-libs": [
 238                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 239                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 240                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 241                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 242
 243                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 244                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 245                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 246
 247                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 248                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 249                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 250
 251                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 252                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 253                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 254
 255                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 256                       ("nondevel-make", "make -j", "text/plain"),
 257                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 258                       ("nondevel-install", "make install", "text/plain"),
 259                       ("nondevel-dist", "make dist", "text/plain"),
 260
 261                       # retry with all modules shared
 262                       ("allshared-distclean", "make distclean", "text/plain"),
 263                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 264                       ("allshared-make", "make -j", "text/plain")],
 265
 266     "samba-none-env": [
 267                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 268                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 269                       ("make", "make -j", "text/plain"),
 270                       ("test", "make test "
 271                        "FAIL_IMMEDIATELY=1 "
 272                        "TESTS='${PY3_ONLY}"
 273                        "--include-env=none'",
 274                        "text/plain")],
 275
 276     "samba-static": [
 277                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 278                       # build with all modules static
 279                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 280                       ("allstatic-make", "make -j", "text/plain"),
 281                       ("allstatic-test", "make test "
 282                        "FAIL_IMMEDIATELY=1 "
 283                        "TESTS='samba3.smb2.create.*nt4_dc'",
 284                        "text/plain"),
 285
 286                       # retry without any required modules
 287                       ("none-distclean", "make distclean", "text/plain"),
 288                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 289                       ("none-make", "make -j", "text/plain"),
 290
 291                       # retry with nonshared smbd and smbtorture
 292                       ("nonshared-distclean", "make distclean", "text/plain"),
 293                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 294                       ("nonshared-make", "make -j", "text/plain")],
 295
 296     "samba-systemkrb5": [
 297                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 298                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 299                       ("make", "make -j", "text/plain"),
 300                       # we currently cannot run a full make test, a limited list of tests could be run
 301                       # via "make test TESTS=sometests"
 302                       ("test", "make test FAIL_IMMEDIATELY=1 "
 303                        "TESTS='${PY3_ONLY}"
 304                        "--include-env=ktest'", "text/plain"),
 305                       ("install", "make install", "text/plain"),
 306                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 307                       ("clean", "make clean", "text/plain")
 308                       ],
 309
 310     # Test Samba without python still builds.  When this test fails
 311     # due to more use of Python, the expectations is that the newly
 312     # failing part of the code should be disabled when
 313     # --disable-python is set (rather than major work being done to
 314     # support this environment).  The target here is for vendors
 315     # shipping a minimal smbd.
 316     "samba-nopython": [
 317                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 318                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 319                       ("make", "make -j", "text/plain"),
 320                       ("install", "make install", "text/plain"),
 321                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 322                       ("clean", "make clean", "text/plain"),
 323
 324                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 325                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 326                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 327
 328                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 329                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 330                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 331
 332                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 333                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 334                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 335
 336                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 337                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 338                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 339
 340                       # retry against installed library packages
 341                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 342                       ("libs-make", "make -j", "text/plain"),
 343                       ("libs-install", "make install", "text/plain"),
 344                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 345                       ("libs-clean", "make clean", "text/plain")
 346                       ],
 347
 348
 349
 350     "ldb": [
 351               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 352               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 353               ("make", "make", "text/plain"),
 354               ("install", "make install", "text/plain"),
 355               ("test", "make test", "text/plain"),
 356               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 357               ("make-no-lmdb", "make", "text/plain"),
 358               ("install-no-lmdb", "make install", "text/plain"),
 359               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 360               ("distcheck", "make distcheck", "text/plain"),
 361               ("clean", "make clean", "text/plain")],
 362
 363     "tdb": [
 364               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 365               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 366               ("make", "make", "text/plain"),
 367               ("install", "make install", "text/plain"),
 368               ("test", "make test", "text/plain"),
 369               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 370               ("distcheck", "make distcheck", "text/plain"),
 371               ("clean", "make clean", "text/plain")],
 372
 373     "talloc": [
 374                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 375                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 376                  ("make", "make", "text/plain"),
 377                  ("install", "make install", "text/plain"),
 378                  ("test", "make test", "text/plain"),
 379                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 380                  ("distcheck", "make distcheck", "text/plain"),
 381                  ("clean", "make clean", "text/plain")],
 382
 383     "replace": [
 384                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 385                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 386                   ("make", "make", "text/plain"),
 387                   ("install", "make install", "text/plain"),
 388                   ("test", "make test", "text/plain"),
 389                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 390                   ("distcheck", "make distcheck", "text/plain"),
 391                   ("clean", "make clean", "text/plain")],
 392
 393     "tevent": [
 394                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 395                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 396                  ("make", "make", "text/plain"),
 397                  ("install", "make install", "text/plain"),
 398                  ("test", "make test", "text/plain"),
 399                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 400                  ("distcheck", "make distcheck", "text/plain"),
 401                  ("clean", "make clean", "text/plain")],
 402
 403     "pidl": [
 404         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 405         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 406         ("touch", "touch *.yp", "text/plain"),
 407         ("make", "make", "text/plain"),
 408         ("test", "make test", "text/plain"),
 409         ("install", "make install", "text/plain"),
 410         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 411         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 412         ("clean", "make clean", "text/plain")],
 413
 414     "samba-buildpy2-only": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 415                    ("configure", "PYTHON='python' ./configure.developer --with-selftest-prefix=./bin/ab " + samba_configure_params, "text/plain"),
 416                    ("make", "PYTHON='python' make -j", "text/plain"),
 417                    ("install", "PYTHON='python' make install", "text/plain"),
 418                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 419                    ("clean", "PYTHON='python' make clean", "text/plain")],
 420
 421
 422     # these are useful for debugging autobuild
 423     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
 424     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
 425
 426
 427 }
 428
 429
 430 def do_print(msg):
 431     print("%s" % msg)
 432     sys.stdout.flush()
 433     sys.stderr.flush()
 434
 435
 436 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 437     if show is None:
 438         show = options.verbose
 439     if show:
 440         do_print("Running: '%s' in '%s'" % (cmd, dir))
 441     if output:
 442         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
 443     elif checkfail:
 444         return check_call(cmd, shell=True, cwd=dir)
 445     else:
 446         return call(cmd, shell=True, cwd=dir)
 447
 448
 449 class builder(object):
 450     '''handle build of one directory'''
 451
 452     def __init__(self, name, sequence, cp=True, py3=False):
 453         self.name = name
 454         self.py3 = py3
 455         if name in builddirs:
 456             self.dir = builddirs[name]
 457         else:
 458             self.dir = "."
 459
 460         self.tag = self.name.replace('/', '_')
 461         self.sequence = sequence
 462         self.next = 0
 463         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 464         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 465         if options.verbose:
 466             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 467             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 468         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 469         self.stdout = open(self.stdout_path, 'w')
 470         self.stderr = open(self.stderr_path, 'w')
 471         self.stdin  = open("/dev/null", 'r')
 472         self.sdir = "%s/%s" % (testbase, self.tag)
 473         self.prefix = "%s/%s" % (test_prefix, self.tag)
 474         run_cmd("rm -rf %s" % self.sdir)
 475         run_cmd("rm -rf %s" % self.prefix)
 476         if cp:
 477             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 478         else:
 479             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 480         self.start_next()
 481
 482     def start_next(self):
 483         if self.next == len(self.sequence):
 484             if not options.nocleanup:
 485                 run_cmd("rm -rf %s" % self.sdir)
 486                 run_cmd("rm -rf %s" % self.prefix)
 487             do_print('%s: Completed OK' % self.name)
 488             self.done = True
 489             return
 490         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 491         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
 492         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 493         if self.py3:
 494             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
 495             # The trailing space is important
 496             self.cmd = self.cmd.replace("${PY3_ONLY}", "python2 ")
 497         else:
 498             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "")
 499             self.cmd = self.cmd.replace("${PY3_ONLY}", "")
 500         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 501         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 502 #        if self.output_mime_type == "text/x-subunit":
 503 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 504         cwd = os.getcwd()
 505         os.chdir("%s/%s" % (self.sdir, self.dir))
 506         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
 507         self.proc = Popen(self.cmd, shell=True,
 508                           close_fds=True,
 509                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 510         os.chdir(cwd)
 511         self.next += 1
 512
 513
 514 class buildlist(object):
 515     '''handle build of multiple directories'''
 516
 517     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 518         global tasks
 519         self.tlist = []
 520         self.tail_proc = None
 521         self.retry = None
 522         if tasknames == []:
 523             if options.restrict_tests:
 524                 tasknames = ["samba-test-only"]
 525             else:
 526                 tasknames = defaulttasks
 527         else:
 528             # If we are only running one test,
 529             # do not sleep randomly to wait for it to start
 530             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 531
 532         for n in tasknames:
 533             if n not in tasks and n.endswith("-py2"):
 534                 b = builder(n,
 535                             tasks[n[:-4]],
 536                             cp=n is not "pidl",
 537                             py3=True)
 538             else:
 539                 b = builder(n, tasks[n], cp=n is not "pidl")
 540             self.tlist.append(b)
 541         if options.retry:
 542             rebase_remote = "rebaseon"
 543             retry_task = [("retry",
 544                             '''set -e
 545                             git remote add -t %s %s %s
 546                             git fetch %s
 547                             while :; do
 548                               sleep 60
 549                               git describe %s/%s > old_remote_branch.desc
 550                               git fetch %s
 551                               git describe %s/%s > remote_branch.desc
 552                               diff old_remote_branch.desc remote_branch.desc
 553                             done
 554                            ''' % (
 555                                 rebase_branch, rebase_remote, rebase_url,
 556                                rebase_remote,
 557                                rebase_remote, rebase_branch,
 558                                rebase_remote,
 559                                rebase_remote, rebase_branch
 560                             ),
 561                             "test/plain")]
 562
 563             self.retry = builder('retry', retry_task, cp=False)
 564             self.need_retry = False
 565
 566     def kill_kids(self):
 567         if self.tail_proc is not None:
 568             self.tail_proc.terminate()
 569             self.tail_proc.wait()
 570             self.tail_proc = None
 571         if self.retry is not None:
 572             self.retry.proc.terminate()
 573             self.retry.proc.wait()
 574             self.retry = None
 575         for b in self.tlist:
 576             if b.proc is not None:
 577                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 578                 b.proc.terminate()
 579                 b.proc.wait()
 580                 b.proc = None
 581
 582     def wait_one(self):
 583         while True:
 584             none_running = True
 585             for b in self.tlist:
 586                 if b.proc is None:
 587                     continue
 588                 none_running = False
 589                 b.status = b.proc.poll()
 590                 if b.status is None:
 591                     continue
 592                 b.proc = None
 593                 return b
 594             if options.retry:
 595                 ret = self.retry.proc.poll()
 596                 if ret is not None:
 597                     self.need_retry = True
 598                     self.retry = None
 599                     return None
 600             if none_running:
 601                 return None
 602             time.sleep(0.1)
 603
 604     def run(self):
 605         while True:
 606             b = self.wait_one()
 607             if options.retry and self.need_retry:
 608                 self.kill_kids()
 609                 do_print("retry needed")
 610                 return (0, None, None, None, "retry")
 611             if b is None:
 612                 break
 613             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 614                 self.kill_kids()
 615                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 616             b.start_next()
 617         self.kill_kids()
 618         return (0, None, None, None, "All OK")
 619
 620     def write_system_info(self):
 621         filename = 'system-info.txt'
 622         f = open(filename, 'w')
 623         for cmd in ['uname -a',
 624                     'free',
 625                     'cat /proc/cpuinfo',
 626                     'cc --version',
 627                     'df -m .',
 628                     'df -m %s' % testbase]:
 629             out = run_cmd(cmd, output=True, checkfail=False)
 630             print('### %s' % cmd, file=f)
 631             print(out.decode('utf8', 'backslashreplace'), file=f)
 632             print(file=f)
 633         f.close()
 634         return filename
 635
 636     def tarlogs(self, fname):
 637         tar = tarfile.open(fname, "w:gz")
 638         for b in self.tlist:
 639             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 640             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 641         if os.path.exists("autobuild.log"):
 642             tar.add("autobuild.log")
 643         sys_info = self.write_system_info()
 644         tar.add(sys_info)
 645         tar.close()
 646
 647     def remove_logs(self):
 648         for b in self.tlist:
 649             os.unlink(b.stdout_path)
 650             os.unlink(b.stderr_path)
 651
 652     def start_tail(self):
 653         cmd = ["tail", "-f"]
 654         for b in self.tlist:
 655             cmd.append(b.stdout_path)
 656             cmd.append(b.stderr_path)
 657         self.tail_proc = Popen(cmd, close_fds=True)
 658
 659
 660 def cleanup():
 661     if options.nocleanup:
 662         return
 663     run_cmd("stat %s || true" % test_tmpdir, show=True)
 664     run_cmd("stat %s" % testbase, show=True)
 665     do_print("Cleaning up %r" % cleanup_list)
 666     for d in cleanup_list:
 667         run_cmd("rm -rf %s" % d)
 668
 669
 670 def find_git_root():
 671     '''get to the top of the git repo'''
 672     p = os.getcwd()
 673     while p != '/':
 674         if os.path.isdir(os.path.join(p, ".git")):
 675             return p
 676         p = os.path.abspath(os.path.join(p, '..'))
 677     return None
 678
 679
 680 def daemonize(logfile):
 681     pid = os.fork()
 682     if pid == 0:  # Parent
 683         os.setsid()
 684         pid = os.fork()
 685         if pid != 0:  # Actual daemon
 686             os._exit(0)
 687     else:  # Grandparent
 688         os._exit(0)
 689
 690     import resource      # Resource usage information.
 691     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 692     if maxfd == resource.RLIM_INFINITY:
 693         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 694     for fd in range(0, maxfd):
 695         try:
 696             os.close(fd)
 697         except OSError:
 698             pass
 699     os.open(logfile, os.O_RDWR | os.O_CREAT)
 700     os.dup2(0, 1)
 701     os.dup2(0, 2)
 702
 703
 704 def write_pidfile(fname):
 705     '''write a pid file, cleanup on exit'''
 706     f = open(fname, mode='w')
 707     f.write("%u\n" % os.getpid())
 708     f.close()
 709
 710
 711 def rebase_tree(rebase_url, rebase_branch="master"):
 712     rebase_remote = "rebaseon"
 713     do_print("Rebasing on %s" % rebase_url)
 714     run_cmd("git describe HEAD", show=True, dir=test_master)
 715     run_cmd("git remote add -t %s %s %s" %
 716             (rebase_branch, rebase_remote, rebase_url),
 717             show=True, dir=test_master)
 718     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 719     if options.fix_whitespace:
 720         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 721                 (rebase_remote, rebase_branch),
 722                 show=True, dir=test_master)
 723     else:
 724         run_cmd("git rebase --force-rebase %s/%s" %
 725                 (rebase_remote, rebase_branch),
 726                 show=True, dir=test_master)
 727     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 728                    (rebase_remote, rebase_branch),
 729                    dir=test_master, output=True)
 730     if diff == '':
 731         do_print("No differences between HEAD and %s/%s - exiting" %
 732                  (rebase_remote, rebase_branch))
 733         sys.exit(0)
 734     run_cmd("git describe %s/%s" %
 735             (rebase_remote, rebase_branch),
 736             show=True, dir=test_master)
 737     run_cmd("git describe HEAD", show=True, dir=test_master)
 738     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 739             (rebase_remote, rebase_branch),
 740             show=True, dir=test_master)
 741
 742
 743 def push_to(push_url, push_branch="master"):
 744     push_remote = "pushto"
 745     do_print("Pushing to %s" % push_url)
 746     if options.mark:
 747         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 748         run_cmd("git commit --amend -c HEAD", dir=test_master)
 749         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 750         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 751     run_cmd("git remote add -t %s %s %s" %
 752             (push_branch, push_remote, push_url),
 753             show=True, dir=test_master)
 754     run_cmd("git push %s +HEAD:%s" %
 755             (push_remote, push_branch),
 756             show=True, dir=test_master)
 757
 758
 759 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 760
 761 gitroot = find_git_root()
 762 if gitroot is None:
 763     raise Exception("Failed to find git root")
 764
 765 parser = OptionParser()
 766 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 767 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 768 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 769 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 770                   default=def_testbase)
 771 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 772 parser.add_option("", "--verbose", help="show all commands as they are run",
 773                   default=False, action="store_true")
 774 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 775                   default=None, type='str')
 776 parser.add_option("", "--pushto", help="push to a git url on success",
 777                   default=None, type='str')
 778 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 779                   default=False, action="store_true")
 780 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 781                   default=False, action="store_true")
 782 parser.add_option("", "--retry", help="automatically retry if master changes",
 783                   default=False, action="store_true")
 784 parser.add_option("", "--email", help="send email to the given address on failure",
 785                   type='str', default=None)
 786 parser.add_option("", "--email-from", help="send email from the given address",
 787                   type='str', default="autobuild@samba.org")
 788 parser.add_option("", "--email-server", help="send email via the given server",
 789                   type='str', default='localhost')
 790 parser.add_option("", "--always-email", help="always send email, even on success",
 791                   action="store_true")
 792 parser.add_option("", "--daemon", help="daemonize after initial setup",
 793                   action="store_true")
 794 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 795                   default="master", type='str')
 796 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 797                   default=gitroot, type='str')
 798 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 799                   default=False, action="store_true")
 800 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 801                   default='')
 802
 803
 804 def send_email(subject, text, log_tar):
 805     if options.email is None:
 806         do_print("not sending email because the recipient is not set")
 807         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
 808                  (subject, text))
 809         return
 810     outer = MIMEMultipart()
 811     outer['Subject'] = subject
 812     outer['To'] = options.email
 813     outer['From'] = options.email_from
 814     outer['Date'] = email.utils.formatdate(localtime=True)
 815     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 816     outer.attach(MIMEText(text, 'plain'))
 817     if options.attach_logs:
 818         fp = open(log_tar, 'rb')
 819         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 820         fp.close()
 821         # Set the filename parameter
 822         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 823         outer.attach(msg)
 824     content = outer.as_string()
 825     s = smtplib.SMTP(options.email_server)
 826     s.sendmail(options.email_from, [options.email], content)
 827     s.set_debuglevel(1)
 828     s.quit()
 829
 830
 831 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 832                   elapsed_time, log_base=None, add_log_tail=True):
 833     '''send an email to options.email about the failure'''
 834     elapsed_minutes = elapsed_time / 60.0
 835     if log_base is None:
 836         log_base = gitroot
 837     text = '''
 838 Dear Developer,
 839
 840 Your autobuild on %s failed after %.1f minutes
 841 when trying to test %s with the following error:
 842
 843    %s
 844
 845 the autobuild has been abandoned. Please fix the error and resubmit.
 846
 847 A summary of the autobuild process is here:
 848
 849   %s/autobuild.log
 850 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 851
 852     if options.restrict_tests:
 853         text += """
 854 The build was restricted to tests matching %s\n""" % options.restrict_tests
 855
 856     if failed_task != 'rebase':
 857         text += '''
 858 You can see logs of the failed task here:
 859
 860   %s/%s.stdout
 861   %s/%s.stderr
 862
 863 or you can get full logs of all tasks in this job here:
 864
 865   %s/logs.tar.gz
 866
 867 The top commit for the tree that was built was:
 868
 869 %s
 870
 871 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 872
 873     if add_log_tail:
 874         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 875         lines = f.readlines()
 876         log_tail = "".join(lines[-50:])
 877         num_lines = len(lines)
 878         if num_lines < 50:
 879             # Also include stderr (compile failures) if < 50 lines of stdout
 880             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 881             log_tail += "".join(f.readlines()[-(50 - num_lines):])
 882
 883         text += '''
 884 The last 50 lines of log messages:
 885
 886 %s
 887     ''' % log_tail
 888         f.close()
 889
 890     logs = os.path.join(gitroot, 'logs.tar.gz')
 891     send_email('autobuild[%s] failure on %s for task %s during %s'
 892                % (options.branch, platform.node(), failed_task, failed_stage),
 893                text, logs)
 894
 895
 896 def email_success(elapsed_time, log_base=None):
 897     '''send an email to options.email about a successful build'''
 898     if log_base is None:
 899         log_base = gitroot
 900     text = '''
 901 Dear Developer,
 902
 903 Your autobuild on %s has succeeded after %.1f minutes.
 904
 905 ''' % (platform.node(), elapsed_time / 60.)
 906
 907     if options.restrict_tests:
 908         text += """
 909 The build was restricted to tests matching %s\n""" % options.restrict_tests
 910
 911     if options.keeplogs:
 912         text += '''
 913
 914 you can get full logs of all tasks in this job here:
 915
 916   %s/logs.tar.gz
 917
 918 ''' % log_base
 919
 920     text += '''
 921 The top commit for the tree that was built was:
 922
 923 %s
 924 ''' % top_commit_msg
 925
 926     logs = os.path.join(gitroot, 'logs.tar.gz')
 927     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 928                text, logs)
 929
 930
 931 (options, args) = parser.parse_args()
 932
 933 if options.retry:
 934     if options.rebase is None:
 935         raise Exception('You can only use --retry if you also rebase')
 936
 937 testbase = "%s/b%u" % (options.testbase, os.getpid())
 938 test_master = "%s/master" % testbase
 939 test_prefix = "%s/prefix" % testbase
 940 test_tmpdir = "%s/tmp" % testbase
 941 os.environ['TMPDIR'] = test_tmpdir
 942
 943 # get the top commit message, for emails
 944 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 945 top_commit_msg = top_commit_msg.decode('utf-8', 'backslashreplace')
 946
 947 try:
 948     os.makedirs(testbase)
 949 except Exception as reason:
 950     raise Exception("Unable to create %s : %s" % (testbase, reason))
 951 cleanup_list.append(testbase)
 952
 953 if options.daemon:
 954     logfile = os.path.join(testbase, "log")
 955     do_print("Forking into the background, writing progress to %s" % logfile)
 956     daemonize(logfile)
 957
 958 write_pidfile(gitroot + "/autobuild.pid")
 959
 960 start_time = time.time()
 961
 962 while True:
 963     try:
 964         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 965         os.makedirs(test_tmpdir)
 966         # The waf uninstall code removes empty directories all the way
 967         # up the tree.  Creating a file in test_tmpdir stops it from
 968         # being removed.
 969         run_cmd("touch %s" % os.path.join(test_tmpdir,
 970                                           ".directory-is-not-empty"), show=True)
 971         run_cmd("stat %s" % test_tmpdir, show=True)
 972         run_cmd("stat %s" % testbase, show=True)
 973         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 974     except Exception:
 975         cleanup()
 976         raise
 977
 978     try:
 979         try:
 980             if options.rebase is not None:
 981                 rebase_tree(options.rebase, rebase_branch=options.branch)
 982         except Exception:
 983             cleanup_list.append(gitroot + "/autobuild.pid")
 984             cleanup()
 985             elapsed_time = time.time() - start_time
 986             email_failure(-1, 'rebase', 'rebase', 'rebase',
 987                           'rebase on %s failed' % options.branch,
 988                           elapsed_time, log_base=options.log_base)
 989             sys.exit(1)
 990         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
 991         if options.tail:
 992             blist.start_tail()
 993         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
 994         if status != 0 or errstr != "retry":
 995             break
 996         cleanup()
 997     except Exception:
 998         cleanup()
 999         raise
1000
1001 cleanup_list.append(gitroot + "/autobuild.pid")
1002
1003 do_print(errstr)
1004
1005 blist.kill_kids()
1006 if options.tail:
1007     do_print("waiting for tail to flush")
1008     time.sleep(1)
1009
1010 elapsed_time = time.time() - start_time
1011 if status == 0:
1012     if options.passcmd is not None:
1013         do_print("Running passcmd: %s" % options.passcmd)
1014         run_cmd(options.passcmd, dir=test_master)
1015     if options.pushto is not None:
1016         push_to(options.pushto, push_branch=options.branch)
1017     if options.keeplogs or options.attach_logs:
1018         blist.tarlogs("logs.tar.gz")
1019         do_print("Logs in logs.tar.gz")
1020     if options.always_email:
1021         email_success(elapsed_time, log_base=options.log_base)
1022     blist.remove_logs()
1023     cleanup()
1024     do_print(errstr)
1025     sys.exit(0)
1026
1027 # something failed, gather a tar of the logs
1028 blist.tarlogs("logs.tar.gz")
1029
1030 if options.email is not None:
1031     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1032                   elapsed_time, log_base=options.log_base)
1033 else:
1034     elapsed_minutes = elapsed_time / 60.0
1035     print('''
1036
1037 ####################################################################
1038
1039 AUTOBUILD FAILURE
1040
1041 Your autobuild[%s] on %s failed after %.1f minutes
1042 when trying to test %s with the following error:
1043
1044    %s
1045
1046 the autobuild has been abandoned. Please fix the error and resubmit.
1047
1048 ####################################################################
1049
1050 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1051
1052 cleanup()
1053 do_print(errstr)
1054 do_print("Logs in logs.tar.gz")
1055 sys.exit(status)