script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34 cleanup_list = []
  35
  36 builddirs = {
  37     "ctdb": "ctdb",
  38     "samba": ".",
  39     "samba-py2": ".",
  40     "samba-nt4": ".",
  41     "samba-nt4-py2": ".",
  42     "samba-fileserver": ".",
  43     "samba-xc": ".",
  44     "samba-o3": ".",
  45     "samba-ctdb": ".",
  46     "samba-libs": ".",
  47     "samba-libs-py2": ".",
  48     "samba-static": ".",
  49     "samba-none-env": ".",
  50     "samba-ad-dc": ".",
  51     "samba-ad-dc-py2": ".",
  52     "samba-ad-dc-2": ".",
  53     "samba-ad-dc-2-py2": ".",
  54     "samba-ad-dc-backup": ".",
  55     "samba-ad-dc-backup-py2": ".",
  56     "samba-systemkrb5": ".",
  57     "samba-nopython": ".",
  58     "samba-buildpy2-only": ".",
  59     "ldb": "lib/ldb",
  60     "tdb": "lib/tdb",
  61     "talloc": "lib/talloc",
  62     "replace": "lib/replace",
  63     "tevent": "lib/tevent",
  64     "pidl": "pidl"
  65 }
  66
  67 defaulttasks = builddirs.keys()
  68
  69 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  70     defaulttasks.remove("samba-o3")
  71
  72 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  73 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
  74
  75 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
  76 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  77 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  78 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
  79 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE ${EXTRA_PYTHON}"
  80 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
  81 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs + " ${EXTRA_PYTHON}"
  82
  83 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
  84     extra_python = ""
  85 else:
  86     extra_python = "--extra-python=/usr/bin/python2"
  87
  88 tasks = {
  89     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  90                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  91                ("make", "make all", "text/plain"),
  92                ("install", "make install", "text/plain"),
  93                ("test", "make autotest", "text/plain"),
  94                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  95                ("clean", "make clean", "text/plain")],
  96
  97     # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)'
  98     "samba": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
  99                 ("make", "make -j", "text/plain"),
 100                 ("test", "make test FAIL_IMMEDIATELY=1 "
 101                  "TESTS='${PY3_ONLY}"
 102                  "--exclude-env=none "
 103                  "--exclude-env=nt4_dc "
 104                  "--exclude-env=nt4_member "
 105                  "--exclude-env=ad_dc "
 106                  "--exclude-env=ad_dc_no_nss "
 107                  "--exclude-env=fl2003dc "
 108                  "--exclude-env=fl2008r2dc "
 109                  "--exclude-env=ad_member "
 110                  "--exclude-env=ad_member_idmap_rid "
 111                  "--exclude-env=ad_member_idmap_ad "
 112                  "--exclude-env=chgdcpass "
 113                  "--exclude-env=vampire_2000_dc "
 114                  "--exclude-env=fl2000dc "
 115                  "--exclude-env=fileserver "
 116                  "--exclude-env=backupfromdc "
 117                  "--exclude-env=restoredc "
 118                  "--exclude-env=renamedc "
 119                  "--exclude-env=offlinebackupdc "
 120                  "--exclude-env=labdc "
 121                  "'",
 122                  "text/plain"),
 123                 ("install", "make install", "text/plain"),
 124                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 125                 ("clean", "make clean", "text/plain")],
 126
 127     # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long)
 128     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 129                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 130                     ("make", "make -j", "text/plain"),
 131                     ("test", "make test FAIL_IMMEDIATELY=1 "
 132                      "TESTS='${PY3_ONLY}"
 133                      "--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 134                     ("install", "make install", "text/plain"),
 135                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 136                     ("clean", "make clean", "text/plain")],
 137
 138     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 139     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 140                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 141                            ("make", "make -j", "text/plain"),
 142                            ("test", "make test FAIL_IMMEDIATELY=1 "
 143                             "TESTS='${PY3_ONLY}"
 144                             "--include-env=fileserver'", "text/plain"),
 145                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 146
 147     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 148     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 149                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 150                       ("make", "make -j", "text/plain"),
 151                       ("test", "make test FAIL_IMMEDIATELY=1 "
 152                        "TESTS='${PY3_ONLY}"
 153                        "--include-env=ad_dc "
 154                        "--include-env=fl2003dc "
 155                        "--include-env=fl2008r2dc "
 156                        "--include-env=ad_member "
 157                        "--include-env=ad_member_idmap_rid "
 158                        "--include-env=ad_member_idmap_ad'", "text/plain"),
 159                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 160
 161     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 162     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 163                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 164                         ("make", "make -j", "text/plain"),
 165                         ("test", "make test FAIL_IMMEDIATELY=1 "
 166                          "TESTS='${PY3_ONLY}"
 167                          "--include-env=chgdcpass "
 168                          "--include-env=vampire_2000_dc "
 169                          "--include-env=fl2000dc "
 170                          "--include-env=ad_dc_no_nss "
 171                          "'",
 172                          "text/plain"),
 173                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 174
 175     # run the backup/restore testenvs separately as they're fairly standalone
 176     # (and CI seems to max out at ~8 different DCs running at once)
 177     "samba-ad-dc-backup": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 178                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 179                         ("make", "make -j", "text/plain"),
 180                         ("test", "make test FAIL_IMMEDIATELY=1 "
 181                          "TESTS='${PY3_ONLY}"
 182                          "--include-env=backupfromdc "
 183                          "--include-env=restoredc "
 184                          "--include-env=renamedc "
 185                          "--include-env=offlinebackupdc "
 186                          "--include-env=labdc "
 187                          "'",
 188                          "text/plain"),
 189                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 190
 191     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 192                           ("make", "make -j", "text/plain"),
 193                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
 194
 195     # Test cross-compile infrastructure
 196     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 197                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 198                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 199                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 200                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 201                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 202                    ("compare-results", "script/compare_cc_results.py "
 203                     "./bin/c4che/default{} "
 204                     "./bin-xe/c4che/default{} "
 205                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
 206
 207     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 208     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 209                    ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 210                    ("make", "make -j", "text/plain"),
 211                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
 212                     "TESTS='${PY3_ONLY}"
 213                     "--include-env=ad_dc'", "text/plain"),
 214                    ("install", "make install", "text/plain"),
 215                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 216                    ("clean", "make clean", "text/plain")],
 217
 218     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 219
 220                      # make sure we have tdb around:
 221                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 222                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 223                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 224
 225
 226                      # build samba with cluster support (also building ctdb):
 227                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 228                      ("samba-make", "make", "text/plain"),
 229                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 230                      ("samba-install", "make install", "text/plain"),
 231                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 232
 233                      # clean up:
 234                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 235                      ("clean", "make clean", "text/plain"),
 236                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
 237
 238     "samba-libs": [
 239                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 240                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 241                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 242                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 243
 244                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 245                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 246                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 247
 248                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 249                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 250                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 251
 252                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 253                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 254                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 255
 256                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 257                       ("nondevel-make", "make -j", "text/plain"),
 258                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 259                       ("nondevel-install", "make install", "text/plain"),
 260                       ("nondevel-dist", "make dist", "text/plain"),
 261
 262                       # retry with all modules shared
 263                       ("allshared-distclean", "make distclean", "text/plain"),
 264                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 265                       ("allshared-make", "make -j", "text/plain")],
 266
 267     "samba-none-env": [
 268                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 269                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 270                       ("make", "make -j", "text/plain"),
 271                       ("test", "make test "
 272                        "FAIL_IMMEDIATELY=1 "
 273                        "TESTS='${PY3_ONLY}"
 274                        "--include-env=none'",
 275                        "text/plain")],
 276
 277     "samba-static": [
 278                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 279                       # build with all modules static
 280                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 281                       ("allstatic-make", "make -j", "text/plain"),
 282                       ("allstatic-test", "make test "
 283                        "FAIL_IMMEDIATELY=1 "
 284                        "TESTS='samba3.smb2.create.*nt4_dc'",
 285                        "text/plain"),
 286
 287                       # retry without any required modules
 288                       ("none-distclean", "make distclean", "text/plain"),
 289                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 290                       ("none-make", "make -j", "text/plain"),
 291
 292                       # retry with nonshared smbd and smbtorture
 293                       ("nonshared-distclean", "make distclean", "text/plain"),
 294                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 295                       ("nonshared-make", "make -j", "text/plain")],
 296
 297     "samba-systemkrb5": [
 298                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 299                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 300                       ("make", "make -j", "text/plain"),
 301                       # we currently cannot run a full make test, a limited list of tests could be run
 302                       # via "make test TESTS=sometests"
 303                       ("test", "make test FAIL_IMMEDIATELY=1 "
 304                        "TESTS='${PY3_ONLY}"
 305                        "--include-env=ktest'", "text/plain"),
 306                       ("install", "make install", "text/plain"),
 307                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 308                       ("clean", "make clean", "text/plain")
 309                       ],
 310
 311     # Test Samba without python still builds.  When this test fails
 312     # due to more use of Python, the expectations is that the newly
 313     # failing part of the code should be disabled when
 314     # --disable-python is set (rather than major work being done to
 315     # support this environment).  The target here is for vendors
 316     # shipping a minimal smbd.
 317     "samba-nopython": [
 318                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 319                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 320                       ("make", "make -j", "text/plain"),
 321                       ("install", "make install", "text/plain"),
 322                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 323                       ("clean", "make clean", "text/plain"),
 324
 325                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 326                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 327                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 328
 329                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 330                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 331                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 332
 333                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 334                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 335                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 336
 337                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 338                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 339                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 340
 341                       # retry against installed library packages
 342                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 343                       ("libs-make", "make -j", "text/plain"),
 344                       ("libs-install", "make install", "text/plain"),
 345                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 346                       ("libs-clean", "make clean", "text/plain")
 347                       ],
 348
 349
 350
 351     "ldb": [
 352               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 353               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 354               ("make", "make", "text/plain"),
 355               ("install", "make install", "text/plain"),
 356               ("test", "make test", "text/plain"),
 357               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 358               ("make-no-lmdb", "make", "text/plain"),
 359               ("install-no-lmdb", "make install", "text/plain"),
 360               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 361               ("distcheck", "make distcheck", "text/plain"),
 362               ("clean", "make clean", "text/plain")],
 363
 364     "tdb": [
 365               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 366               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 367               ("make", "make", "text/plain"),
 368               ("install", "make install", "text/plain"),
 369               ("test", "make test", "text/plain"),
 370               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 371               ("distcheck", "make distcheck", "text/plain"),
 372               ("clean", "make clean", "text/plain")],
 373
 374     "talloc": [
 375                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 376                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 377                  ("make", "make", "text/plain"),
 378                  ("install", "make install", "text/plain"),
 379                  ("test", "make test", "text/plain"),
 380                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 381                  ("distcheck", "make distcheck", "text/plain"),
 382                  ("clean", "make clean", "text/plain")],
 383
 384     "replace": [
 385                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 386                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 387                   ("make", "make", "text/plain"),
 388                   ("install", "make install", "text/plain"),
 389                   ("test", "make test", "text/plain"),
 390                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 391                   ("distcheck", "make distcheck", "text/plain"),
 392                   ("clean", "make clean", "text/plain")],
 393
 394     "tevent": [
 395                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 396                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 397                  ("make", "make", "text/plain"),
 398                  ("install", "make install", "text/plain"),
 399                  ("test", "make test", "text/plain"),
 400                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 401                  ("distcheck", "make distcheck", "text/plain"),
 402                  ("clean", "make clean", "text/plain")],
 403
 404     "pidl": [
 405         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 406         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 407         ("touch", "touch *.yp", "text/plain"),
 408         ("make", "make", "text/plain"),
 409         ("test", "make test", "text/plain"),
 410         ("install", "make install", "text/plain"),
 411         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 412         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 413         ("clean", "make clean", "text/plain")],
 414
 415     "samba-buildpy2-only": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 416                    ("configure", "PYTHON='python' ./configure.developer --with-selftest-prefix=./bin/ab " + samba_configure_params, "text/plain"),
 417                    ("make", "PYTHON='python' make -j", "text/plain"),
 418                    ("install", "PYTHON='python' make install", "text/plain"),
 419                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 420                    ("clean", "PYTHON='python' make clean", "text/plain")],
 421
 422
 423     # these are useful for debugging autobuild
 424     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
 425     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
 426
 427
 428 }
 429
 430
 431 def do_print(msg):
 432     print("%s" % msg)
 433     sys.stdout.flush()
 434     sys.stderr.flush()
 435
 436
 437 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 438     if show is None:
 439         show = options.verbose
 440     if show:
 441         do_print("Running: '%s' in '%s'" % (cmd, dir))
 442     if output:
 443         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
 444     elif checkfail:
 445         return check_call(cmd, shell=True, cwd=dir)
 446     else:
 447         return call(cmd, shell=True, cwd=dir)
 448
 449
 450 class builder(object):
 451     '''handle build of one directory'''
 452
 453     def __init__(self, name, sequence, cp=True, py3=False):
 454         self.name = name
 455         self.py3 = py3
 456         if name in builddirs:
 457             self.dir = builddirs[name]
 458         else:
 459             self.dir = "."
 460
 461         self.tag = self.name.replace('/', '_')
 462         self.sequence = sequence
 463         self.next = 0
 464         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 465         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 466         if options.verbose:
 467             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 468             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 469         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 470         self.stdout = open(self.stdout_path, 'w')
 471         self.stderr = open(self.stderr_path, 'w')
 472         self.stdin  = open("/dev/null", 'r')
 473         self.sdir = "%s/%s" % (testbase, self.tag)
 474         self.prefix = "%s/%s" % (test_prefix, self.tag)
 475         run_cmd("rm -rf %s" % self.sdir)
 476         run_cmd("rm -rf %s" % self.prefix)
 477         if cp:
 478             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 479         else:
 480             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 481         self.start_next()
 482
 483     def start_next(self):
 484         if self.next == len(self.sequence):
 485             if not options.nocleanup:
 486                 run_cmd("rm -rf %s" % self.sdir)
 487                 run_cmd("rm -rf %s" % self.prefix)
 488             do_print('%s: Completed OK' % self.name)
 489             self.done = True
 490             return
 491         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 492         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
 493         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 494         if self.py3:
 495             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
 496             # The trailing space is important
 497             self.cmd = self.cmd.replace("${PY3_ONLY}", "python2 ")
 498         else:
 499             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "")
 500             self.cmd = self.cmd.replace("${PY3_ONLY}", "")
 501         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 502         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 503 #        if self.output_mime_type == "text/x-subunit":
 504 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 505         cwd = os.getcwd()
 506         os.chdir("%s/%s" % (self.sdir, self.dir))
 507         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
 508         self.proc = Popen(self.cmd, shell=True,
 509                           close_fds=True,
 510                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 511         os.chdir(cwd)
 512         self.next += 1
 513
 514
 515 class buildlist(object):
 516     '''handle build of multiple directories'''
 517
 518     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 519         global tasks
 520         self.tlist = []
 521         self.tail_proc = None
 522         self.retry = None
 523         if tasknames == []:
 524             if options.restrict_tests:
 525                 tasknames = ["samba-test-only"]
 526             else:
 527                 tasknames = defaulttasks
 528         else:
 529             # If we are only running one test,
 530             # do not sleep randomly to wait for it to start
 531             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 532
 533         for n in tasknames:
 534             if n not in tasks and n.endswith("-py2"):
 535                 b = builder(n,
 536                             tasks[n[:-4]],
 537                             cp=n is not "pidl",
 538                             py3=True)
 539             else:
 540                 b = builder(n, tasks[n], cp=n is not "pidl")
 541             self.tlist.append(b)
 542         if options.retry:
 543             rebase_remote = "rebaseon"
 544             retry_task = [("retry",
 545                             '''set -e
 546                             git remote add -t %s %s %s
 547                             git fetch %s
 548                             while :; do
 549                               sleep 60
 550                               git describe %s/%s > old_remote_branch.desc
 551                               git fetch %s
 552                               git describe %s/%s > remote_branch.desc
 553                               diff old_remote_branch.desc remote_branch.desc
 554                             done
 555                            ''' % (
 556                                 rebase_branch, rebase_remote, rebase_url,
 557                                rebase_remote,
 558                                rebase_remote, rebase_branch,
 559                                rebase_remote,
 560                                rebase_remote, rebase_branch
 561                             ),
 562                             "test/plain")]
 563
 564             self.retry = builder('retry', retry_task, cp=False)
 565             self.need_retry = False
 566
 567     def kill_kids(self):
 568         if self.tail_proc is not None:
 569             self.tail_proc.terminate()
 570             self.tail_proc.wait()
 571             self.tail_proc = None
 572         if self.retry is not None:
 573             self.retry.proc.terminate()
 574             self.retry.proc.wait()
 575             self.retry = None
 576         for b in self.tlist:
 577             if b.proc is not None:
 578                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 579                 b.proc.terminate()
 580                 b.proc.wait()
 581                 b.proc = None
 582
 583     def wait_one(self):
 584         while True:
 585             none_running = True
 586             for b in self.tlist:
 587                 if b.proc is None:
 588                     continue
 589                 none_running = False
 590                 b.status = b.proc.poll()
 591                 if b.status is None:
 592                     continue
 593                 b.proc = None
 594                 return b
 595             if options.retry:
 596                 ret = self.retry.proc.poll()
 597                 if ret is not None:
 598                     self.need_retry = True
 599                     self.retry = None
 600                     return None
 601             if none_running:
 602                 return None
 603             time.sleep(0.1)
 604
 605     def run(self):
 606         while True:
 607             b = self.wait_one()
 608             if options.retry and self.need_retry:
 609                 self.kill_kids()
 610                 do_print("retry needed")
 611                 return (0, None, None, None, "retry")
 612             if b is None:
 613                 break
 614             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 615                 self.kill_kids()
 616                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 617             b.start_next()
 618         self.kill_kids()
 619         return (0, None, None, None, "All OK")
 620
 621     def write_system_info(self):
 622         filename = 'system-info.txt'
 623         f = open(filename, 'w')
 624         for cmd in ['uname -a',
 625                     'free',
 626                     'cat /proc/cpuinfo',
 627                     'cc --version',
 628                     'df -m .',
 629                     'df -m %s' % testbase]:
 630             out = run_cmd(cmd, output=True, checkfail=False)
 631             print('### %s' % cmd, file=f)
 632             print(out.decode('utf8', 'backslashreplace'), file=f)
 633             print(file=f)
 634         f.close()
 635         return filename
 636
 637     def tarlogs(self, fname):
 638         tar = tarfile.open(fname, "w:gz")
 639         for b in self.tlist:
 640             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 641             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 642         if os.path.exists("autobuild.log"):
 643             tar.add("autobuild.log")
 644         sys_info = self.write_system_info()
 645         tar.add(sys_info)
 646         tar.close()
 647
 648     def remove_logs(self):
 649         for b in self.tlist:
 650             os.unlink(b.stdout_path)
 651             os.unlink(b.stderr_path)
 652
 653     def start_tail(self):
 654         cmd = ["tail", "-f"]
 655         for b in self.tlist:
 656             cmd.append(b.stdout_path)
 657             cmd.append(b.stderr_path)
 658         self.tail_proc = Popen(cmd, close_fds=True)
 659
 660
 661 def cleanup():
 662     if options.nocleanup:
 663         return
 664     run_cmd("stat %s || true" % test_tmpdir, show=True)
 665     run_cmd("stat %s" % testbase, show=True)
 666     do_print("Cleaning up %r" % cleanup_list)
 667     for d in cleanup_list:
 668         run_cmd("rm -rf %s" % d)
 669
 670
 671 def find_git_root():
 672     '''get to the top of the git repo'''
 673     p = os.getcwd()
 674     while p != '/':
 675         if os.path.isdir(os.path.join(p, ".git")):
 676             return p
 677         p = os.path.abspath(os.path.join(p, '..'))
 678     return None
 679
 680
 681 def daemonize(logfile):
 682     pid = os.fork()
 683     if pid == 0:  # Parent
 684         os.setsid()
 685         pid = os.fork()
 686         if pid != 0:  # Actual daemon
 687             os._exit(0)
 688     else:  # Grandparent
 689         os._exit(0)
 690
 691     import resource      # Resource usage information.
 692     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 693     if maxfd == resource.RLIM_INFINITY:
 694         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 695     for fd in range(0, maxfd):
 696         try:
 697             os.close(fd)
 698         except OSError:
 699             pass
 700     os.open(logfile, os.O_RDWR | os.O_CREAT)
 701     os.dup2(0, 1)
 702     os.dup2(0, 2)
 703
 704
 705 def write_pidfile(fname):
 706     '''write a pid file, cleanup on exit'''
 707     f = open(fname, mode='w')
 708     f.write("%u\n" % os.getpid())
 709     f.close()
 710
 711
 712 def rebase_tree(rebase_url, rebase_branch="master"):
 713     rebase_remote = "rebaseon"
 714     do_print("Rebasing on %s" % rebase_url)
 715     run_cmd("git describe HEAD", show=True, dir=test_master)
 716     run_cmd("git remote add -t %s %s %s" %
 717             (rebase_branch, rebase_remote, rebase_url),
 718             show=True, dir=test_master)
 719     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 720     if options.fix_whitespace:
 721         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 722                 (rebase_remote, rebase_branch),
 723                 show=True, dir=test_master)
 724     else:
 725         run_cmd("git rebase --force-rebase %s/%s" %
 726                 (rebase_remote, rebase_branch),
 727                 show=True, dir=test_master)
 728     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 729                    (rebase_remote, rebase_branch),
 730                    dir=test_master, output=True)
 731     if diff == '':
 732         do_print("No differences between HEAD and %s/%s - exiting" %
 733                  (rebase_remote, rebase_branch))
 734         sys.exit(0)
 735     run_cmd("git describe %s/%s" %
 736             (rebase_remote, rebase_branch),
 737             show=True, dir=test_master)
 738     run_cmd("git describe HEAD", show=True, dir=test_master)
 739     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 740             (rebase_remote, rebase_branch),
 741             show=True, dir=test_master)
 742
 743
 744 def push_to(push_url, push_branch="master"):
 745     push_remote = "pushto"
 746     do_print("Pushing to %s" % push_url)
 747     if options.mark:
 748         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 749         run_cmd("git commit --amend -c HEAD", dir=test_master)
 750         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 751         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 752     run_cmd("git remote add -t %s %s %s" %
 753             (push_branch, push_remote, push_url),
 754             show=True, dir=test_master)
 755     run_cmd("git push %s +HEAD:%s" %
 756             (push_remote, push_branch),
 757             show=True, dir=test_master)
 758
 759
 760 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 761
 762 gitroot = find_git_root()
 763 if gitroot is None:
 764     raise Exception("Failed to find git root")
 765
 766 parser = OptionParser()
 767 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 768 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 769 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 770 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 771                   default=def_testbase)
 772 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 773 parser.add_option("", "--verbose", help="show all commands as they are run",
 774                   default=False, action="store_true")
 775 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 776                   default=None, type='str')
 777 parser.add_option("", "--pushto", help="push to a git url on success",
 778                   default=None, type='str')
 779 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 780                   default=False, action="store_true")
 781 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 782                   default=False, action="store_true")
 783 parser.add_option("", "--retry", help="automatically retry if master changes",
 784                   default=False, action="store_true")
 785 parser.add_option("", "--email", help="send email to the given address on failure",
 786                   type='str', default=None)
 787 parser.add_option("", "--email-from", help="send email from the given address",
 788                   type='str', default="autobuild@samba.org")
 789 parser.add_option("", "--email-server", help="send email via the given server",
 790                   type='str', default='localhost')
 791 parser.add_option("", "--always-email", help="always send email, even on success",
 792                   action="store_true")
 793 parser.add_option("", "--daemon", help="daemonize after initial setup",
 794                   action="store_true")
 795 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 796                   default="master", type='str')
 797 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 798                   default=gitroot, type='str')
 799 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 800                   default=False, action="store_true")
 801 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 802                   default='')
 803
 804
 805 def send_email(subject, text, log_tar):
 806     if options.email is None:
 807         do_print("not sending email because the recipient is not set")
 808         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
 809                  (subject, text))
 810         return
 811     outer = MIMEMultipart()
 812     outer['Subject'] = subject
 813     outer['To'] = options.email
 814     outer['From'] = options.email_from
 815     outer['Date'] = email.utils.formatdate(localtime=True)
 816     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 817     outer.attach(MIMEText(text, 'plain'))
 818     if options.attach_logs:
 819         fp = open(log_tar, 'rb')
 820         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 821         fp.close()
 822         # Set the filename parameter
 823         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 824         outer.attach(msg)
 825     content = outer.as_string()
 826     s = smtplib.SMTP(options.email_server)
 827     s.sendmail(options.email_from, [options.email], content)
 828     s.set_debuglevel(1)
 829     s.quit()
 830
 831
 832 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 833                   elapsed_time, log_base=None, add_log_tail=True):
 834     '''send an email to options.email about the failure'''
 835     elapsed_minutes = elapsed_time / 60.0
 836     if log_base is None:
 837         log_base = gitroot
 838     text = '''
 839 Dear Developer,
 840
 841 Your autobuild on %s failed after %.1f minutes
 842 when trying to test %s with the following error:
 843
 844    %s
 845
 846 the autobuild has been abandoned. Please fix the error and resubmit.
 847
 848 A summary of the autobuild process is here:
 849
 850   %s/autobuild.log
 851 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 852
 853     if options.restrict_tests:
 854         text += """
 855 The build was restricted to tests matching %s\n""" % options.restrict_tests
 856
 857     if failed_task != 'rebase':
 858         text += '''
 859 You can see logs of the failed task here:
 860
 861   %s/%s.stdout
 862   %s/%s.stderr
 863
 864 or you can get full logs of all tasks in this job here:
 865
 866   %s/logs.tar.gz
 867
 868 The top commit for the tree that was built was:
 869
 870 %s
 871
 872 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 873
 874     if add_log_tail:
 875         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 876         lines = f.readlines()
 877         log_tail = "".join(lines[-50:])
 878         num_lines = len(lines)
 879         if num_lines < 50:
 880             # Also include stderr (compile failures) if < 50 lines of stdout
 881             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 882             log_tail += "".join(f.readlines()[-(50 - num_lines):])
 883
 884         text += '''
 885 The last 50 lines of log messages:
 886
 887 %s
 888     ''' % log_tail
 889         f.close()
 890
 891     logs = os.path.join(gitroot, 'logs.tar.gz')
 892     send_email('autobuild[%s] failure on %s for task %s during %s'
 893                % (options.branch, platform.node(), failed_task, failed_stage),
 894                text, logs)
 895
 896
 897 def email_success(elapsed_time, log_base=None):
 898     '''send an email to options.email about a successful build'''
 899     if log_base is None:
 900         log_base = gitroot
 901     text = '''
 902 Dear Developer,
 903
 904 Your autobuild on %s has succeeded after %.1f minutes.
 905
 906 ''' % (platform.node(), elapsed_time / 60.)
 907
 908     if options.restrict_tests:
 909         text += """
 910 The build was restricted to tests matching %s\n""" % options.restrict_tests
 911
 912     if options.keeplogs:
 913         text += '''
 914
 915 you can get full logs of all tasks in this job here:
 916
 917   %s/logs.tar.gz
 918
 919 ''' % log_base
 920
 921     text += '''
 922 The top commit for the tree that was built was:
 923
 924 %s
 925 ''' % top_commit_msg
 926
 927     logs = os.path.join(gitroot, 'logs.tar.gz')
 928     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 929                text, logs)
 930
 931
 932 (options, args) = parser.parse_args()
 933
 934 if options.retry:
 935     if options.rebase is None:
 936         raise Exception('You can only use --retry if you also rebase')
 937
 938 testbase = "%s/b%u" % (options.testbase, os.getpid())
 939 test_master = "%s/master" % testbase
 940 test_prefix = "%s/prefix" % testbase
 941 test_tmpdir = "%s/tmp" % testbase
 942 os.environ['TMPDIR'] = test_tmpdir
 943
 944 # get the top commit message, for emails
 945 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 946 top_commit_msg = top_commit_msg.decode('utf-8', 'backslashreplace')
 947
 948 try:
 949     os.makedirs(testbase)
 950 except Exception as reason:
 951     raise Exception("Unable to create %s : %s" % (testbase, reason))
 952 cleanup_list.append(testbase)
 953
 954 if options.daemon:
 955     logfile = os.path.join(testbase, "log")
 956     do_print("Forking into the background, writing progress to %s" % logfile)
 957     daemonize(logfile)
 958
 959 write_pidfile(gitroot + "/autobuild.pid")
 960
 961 start_time = time.time()
 962
 963 while True:
 964     try:
 965         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 966         os.makedirs(test_tmpdir)
 967         # The waf uninstall code removes empty directories all the way
 968         # up the tree.  Creating a file in test_tmpdir stops it from
 969         # being removed.
 970         run_cmd("touch %s" % os.path.join(test_tmpdir,
 971                                           ".directory-is-not-empty"), show=True)
 972         run_cmd("stat %s" % test_tmpdir, show=True)
 973         run_cmd("stat %s" % testbase, show=True)
 974         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 975     except Exception:
 976         cleanup()
 977         raise
 978
 979     try:
 980         try:
 981             if options.rebase is not None:
 982                 rebase_tree(options.rebase, rebase_branch=options.branch)
 983         except Exception:
 984             cleanup_list.append(gitroot + "/autobuild.pid")
 985             cleanup()
 986             elapsed_time = time.time() - start_time
 987             email_failure(-1, 'rebase', 'rebase', 'rebase',
 988                           'rebase on %s failed' % options.branch,
 989                           elapsed_time, log_base=options.log_base)
 990             sys.exit(1)
 991         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
 992         if options.tail:
 993             blist.start_tail()
 994         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
 995         if status != 0 or errstr != "retry":
 996             break
 997         cleanup()
 998     except Exception:
 999         cleanup()
1000         raise
1001
1002 cleanup_list.append(gitroot + "/autobuild.pid")
1003
1004 do_print(errstr)
1005
1006 blist.kill_kids()
1007 if options.tail:
1008     do_print("waiting for tail to flush")
1009     time.sleep(1)
1010
1011 elapsed_time = time.time() - start_time
1012 if status == 0:
1013     if options.passcmd is not None:
1014         do_print("Running passcmd: %s" % options.passcmd)
1015         run_cmd(options.passcmd, dir=test_master)
1016     if options.pushto is not None:
1017         push_to(options.pushto, push_branch=options.branch)
1018     if options.keeplogs or options.attach_logs:
1019         blist.tarlogs("logs.tar.gz")
1020         do_print("Logs in logs.tar.gz")
1021     if options.always_email:
1022         email_success(elapsed_time, log_base=options.log_base)
1023     blist.remove_logs()
1024     cleanup()
1025     do_print(errstr)
1026     sys.exit(0)
1027
1028 # something failed, gather a tar of the logs
1029 blist.tarlogs("logs.tar.gz")
1030
1031 if options.email is not None:
1032     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1033                   elapsed_time, log_base=options.log_base)
1034 else:
1035     elapsed_minutes = elapsed_time / 60.0
1036     print('''
1037
1038 ####################################################################
1039
1040 AUTOBUILD FAILURE
1041
1042 Your autobuild[%s] on %s failed after %.1f minutes
1043 when trying to test %s with the following error:
1044
1045    %s
1046
1047 the autobuild has been abandoned. Please fix the error and resubmit.
1048
1049 ####################################################################
1050
1051 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1052
1053 cleanup()
1054 do_print(errstr)
1055 do_print("Logs in logs.tar.gz")
1056 sys.exit(status)