script/autobuild.py

   1 #!/usr/bin/env python
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34 cleanup_list = []
  35
  36 builddirs = {
  37     "ctdb": "ctdb",
  38     "samba": ".",
  39     "samba-py3": ".",
  40     "samba-nt4": ".",
  41     "samba-nt4-py3": ".",
  42     "samba-fileserver": ".",
  43     "samba-xc": ".",
  44     "samba-o3": ".",
  45     "samba-ctdb": ".",
  46     "samba-libs": ".",
  47     "samba-libs-py3": ".",
  48     "samba-static": ".",
  49     "samba-none-env": ".",
  50     "samba-ad-dc": ".",
  51     "samba-ad-dc-py3": ".",
  52     "samba-ad-dc-2": ".",
  53     "samba-ad-dc-2-py3": ".",
  54     "samba-systemkrb5": ".",
  55     "samba-nopython": ".",
  56     "samba-buildpy3-only": ".",
  57     "samba-purepy3-none-env": ".",
  58     "samba-purepy3-ad-dc-2": ".",
  59     "ldb": "lib/ldb",
  60     "tdb": "lib/tdb",
  61     "talloc": "lib/talloc",
  62     "replace": "lib/replace",
  63     "tevent": "lib/tevent",
  64     "pidl": "pidl"
  65 }
  66
  67 defaulttasks = builddirs.keys()
  68
  69 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  70     defaulttasks.remove("samba-o3")
  71
  72 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  73 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
  74
  75 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH"
  76 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  77 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  78 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
  79 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE ${EXTRA_PYTHON}"
  80 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
  81 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs + " ${EXTRA_PYTHON}"
  82
  83 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
  84     extra_python = ""
  85 else:
  86     extra_python = "--extra-python=/usr/bin/python3"
  87
  88 tasks = {
  89     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  90                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  91                ("make", "make all", "text/plain"),
  92                ("install", "make install", "text/plain"),
  93                ("test", "make autotest", "text/plain"),
  94                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  95                ("clean", "make clean", "text/plain")],
  96
  97     # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)'
  98     "samba": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
  99                 ("make", "make -j", "text/plain"),
 100                 ("test", "make test FAIL_IMMEDIATELY=1 "
 101                  "TESTS='${PY3_ONLY}"
 102                  "--exclude-env=none "
 103                  "--exclude-env=nt4_dc "
 104                  "--exclude-env=nt4_member "
 105                  "--exclude-env=ad_dc "
 106                  "--exclude-env=ad_dc_no_nss "
 107                  "--exclude-env=fl2003dc "
 108                  "--exclude-env=fl2008r2dc "
 109                  "--exclude-env=ad_member "
 110                  "--exclude-env=ad_member_idmap_rid "
 111                  "--exclude-env=ad_member_idmap_ad "
 112                  "--exclude-env=chgdcpass "
 113                  "--exclude-env=vampire_2000_dc "
 114                  "--exclude-env=fl2000dc "
 115                  "--exclude-env=fileserver "
 116                  "--exclude-env=backupfromdc "
 117                  "--exclude-env=restoredc "
 118                  "--exclude-env=renamedc "
 119                  "--exclude-env=offlinebackupdc "
 120                  "--exclude-env=labdc "
 121                  "'",
 122                  "text/plain"),
 123                 ("install", "make install", "text/plain"),
 124                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 125                 ("clean", "make clean", "text/plain")],
 126
 127     # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long)
 128     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 129                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 130                     ("make", "make -j", "text/plain"),
 131                     ("test", "make test FAIL_IMMEDIATELY=1 "
 132                      "TESTS='${PY3_ONLY}"
 133                      "--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 134                     ("install", "make install", "text/plain"),
 135                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 136                     ("clean", "make clean", "text/plain")],
 137
 138     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 139     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 140                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 141                            ("make", "make -j", "text/plain"),
 142                            ("test", "make test FAIL_IMMEDIATELY=1 "
 143                             "TESTS='${PY3_ONLY}"
 144                             "--include-env=fileserver'", "text/plain"),
 145                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 146
 147     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 148     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 149                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 150                       ("make", "make -j", "text/plain"),
 151                       ("test", "make test FAIL_IMMEDIATELY=1 "
 152                        "TESTS='${PY3_ONLY}"
 153                        "--include-env=ad_dc "
 154                        "--include-env=fl2003dc "
 155                        "--include-env=fl2008r2dc "
 156                        "--include-env=ad_member "
 157                        "--include-env=ad_member_idmap_rid "
 158                        "--include-env=ad_member_idmap_ad'", "text/plain"),
 159                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 160
 161     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 162     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 163                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 164                         ("make", "make -j", "text/plain"),
 165                         ("test", "make test FAIL_IMMEDIATELY=1 "
 166                          "TESTS='${PY3_ONLY}"
 167                          "--include-env=chgdcpass "
 168                          "--include-env=vampire_2000_dc "
 169                          "--include-env=fl2000dc "
 170                          "--include-env=ad_dc_no_nss "
 171                          "--include-env=backupfromdc "
 172                          "--include-env=restoredc "
 173                          "--include-env=renamedc "
 174                          "--include-env=offlinebackupdc "
 175                          "--include-env=labdc "
 176                          "'",
 177                          "text/plain"),
 178                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 179
 180     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 181                           ("make", "make -j", "text/plain"),
 182                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
 183
 184     # Test cross-compile infrastructure
 185     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 186                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 187                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 188                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 189                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 190                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 191                    ("compare-results", "script/compare_cc_results.py "
 192                     "./bin/c4che/default{} "
 193                     "./bin-xe/c4che/default{} "
 194                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
 195
 196     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 197     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 198                    ("configure", "ADDITIONAL_CFLAGS='-O3' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 199                    ("make", "make -j", "text/plain"),
 200                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
 201                     "TESTS='${PY3_ONLY}"
 202                     "--include-env=ad_dc'", "text/plain"),
 203                    ("install", "make install", "text/plain"),
 204                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 205                    ("clean", "make clean", "text/plain")],
 206
 207     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 208
 209                      # make sure we have tdb around:
 210                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 211                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 212                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 213
 214
 215                      # build samba with cluster support (also building ctdb):
 216                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 217                      ("samba-make", "make", "text/plain"),
 218                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 219                      ("samba-install", "make install", "text/plain"),
 220                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 221
 222                      # clean up:
 223                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 224                      ("clean", "make clean", "text/plain"),
 225                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
 226
 227     "samba-libs": [
 228                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 229                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 230                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 231                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 232
 233                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 234                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 235                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 236
 237                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 238                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 239                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 240
 241                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 242                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 243                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 244
 245                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 246                       ("nondevel-make", "make -j", "text/plain"),
 247                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 248                       ("nondevel-install", "make install", "text/plain"),
 249                       ("nondevel-dist", "make dist", "text/plain"),
 250
 251                       # retry with all modules shared
 252                       ("allshared-distclean", "make distclean", "text/plain"),
 253                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 254                       ("allshared-make", "make -j", "text/plain")],
 255
 256     "samba-none-env": [
 257                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 258                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 259                       ("make", "make -j", "text/plain"),
 260                       ("test", "make test "
 261                        "FAIL_IMMEDIATELY=1 "
 262                        "TESTS='${PY3_ONLY}"
 263                        "--include-env=none'",
 264                        "text/plain")],
 265
 266     "samba-static": [
 267                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 268                       # build with all modules static
 269                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 270                       ("allstatic-make", "make -j", "text/plain"),
 271                       ("allstatic-test", "make test "
 272                        "FAIL_IMMEDIATELY=1 "
 273                        "TESTS='samba3.smb2.create.*nt4_dc'",
 274                        "text/plain"),
 275
 276                       # retry without any required modules
 277                       ("none-distclean", "make distclean", "text/plain"),
 278                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 279                       ("none-make", "make -j", "text/plain"),
 280
 281                       # retry with nonshared smbd and smbtorture
 282                       ("nonshared-distclean", "make distclean", "text/plain"),
 283                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 284                       ("nonshared-make", "make -j", "text/plain")],
 285
 286     "samba-systemkrb5": [
 287                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 288                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 289                       ("make", "make -j", "text/plain"),
 290                       # we currently cannot run a full make test, a limited list of tests could be run
 291                       # via "make test TESTS=sometests"
 292                       ("test", "make test FAIL_IMMEDIATELY=1 "
 293                        "TESTS='${PY3_ONLY}"
 294                        "--include-env=ktest'", "text/plain"),
 295                       ("install", "make install", "text/plain"),
 296                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 297                       ("clean", "make clean", "text/plain")
 298                       ],
 299
 300     # Test Samba without python still builds.  When this test fails
 301     # due to more use of Python, the expectations is that the newly
 302     # failing part of the code should be disabled when
 303     # --disable-python is set (rather than major work being done to
 304     # support this environment).  The target here is for vendors
 305     # shipping a minimal smbd.
 306     "samba-nopython": [
 307                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 308                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 309                       ("make", "make -j", "text/plain"),
 310                       ("install", "make install", "text/plain"),
 311                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 312                       ("clean", "make clean", "text/plain"),
 313
 314                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 315                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 316                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 317
 318                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 319                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 320                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 321
 322                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 323                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 324                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 325
 326                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 327                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 328                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 329
 330                       # retry against installed library packages
 331                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 332                       ("libs-make", "make -j", "text/plain"),
 333                       ("libs-install", "make install", "text/plain"),
 334                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 335                       ("libs-clean", "make clean", "text/plain")
 336                       ],
 337
 338
 339
 340     "ldb": [
 341               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 342               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 343               ("make", "make", "text/plain"),
 344               ("install", "make install", "text/plain"),
 345               ("test", "make test", "text/plain"),
 346               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 347               ("make-no-lmdb", "make", "text/plain"),
 348               ("install-no-lmdb", "make install", "text/plain"),
 349               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 350               ("distcheck", "make distcheck", "text/plain"),
 351               ("clean", "make clean", "text/plain")],
 352
 353     "tdb": [
 354               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 355               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 356               ("make", "make", "text/plain"),
 357               ("install", "make install", "text/plain"),
 358               ("test", "make test", "text/plain"),
 359               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 360               ("distcheck", "make distcheck", "text/plain"),
 361               ("clean", "make clean", "text/plain")],
 362
 363     "talloc": [
 364                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 365                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 366                  ("make", "make", "text/plain"),
 367                  ("install", "make install", "text/plain"),
 368                  ("test", "make test", "text/plain"),
 369                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 370                  ("distcheck", "make distcheck", "text/plain"),
 371                  ("clean", "make clean", "text/plain")],
 372
 373     "replace": [
 374                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 375                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 376                   ("make", "make", "text/plain"),
 377                   ("install", "make install", "text/plain"),
 378                   ("test", "make test", "text/plain"),
 379                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 380                   ("distcheck", "make distcheck", "text/plain"),
 381                   ("clean", "make clean", "text/plain")],
 382
 383     "tevent": [
 384                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 385                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 386                  ("make", "make", "text/plain"),
 387                  ("install", "make install", "text/plain"),
 388                  ("test", "make test", "text/plain"),
 389                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 390                  ("distcheck", "make distcheck", "text/plain"),
 391                  ("clean", "make clean", "text/plain")],
 392
 393     "pidl": [
 394         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 395         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 396         ("touch", "touch *.yp", "text/plain"),
 397         ("make", "make", "text/plain"),
 398         ("test", "make test", "text/plain"),
 399         ("install", "make install", "text/plain"),
 400         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 401         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 402         ("clean", "make clean", "text/plain")],
 403
 404     "samba-buildpy3-only": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 405                    ("configure", "PYTHON='python3' ./configure.developer --with-selftest-prefix=./bin/ab " + samba_configure_params, "text/plain"),
 406                    ("make", "PYTHON='python3' make -j", "text/plain"),
 407                    ("install", "PYTHON='python3' make install", "text/plain"),
 408                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 409                    ("clean", "PYTHON='python3' make clean", "text/plain")],
 410
 411     "samba-purepy3-none-env": [
 412                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 413                       ("configure", "PYTHON='python3' ./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 414                       ("make", "PYTHON='python3' make -j", "text/plain"),
 415                       ("test", "PYTHON='python3' make test "
 416                        "FAIL_IMMEDIATELY=1 "
 417                        "TESTS='${PY3_ONLY}"
 418                        "--include-env=none'",
 419                        "text/plain")],
 420     "samba-purepy3-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 421                         ("configure", "PYTHON='python3' ./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 422                         ("make", "PYTHON='python3' make -j", "text/plain"),
 423                         ("test", "PYTHON='python3' make test FAIL_IMMEDIATELY=1 "
 424                          "TESTS='${PY3_ONLY}"
 425                          "--include-env=chgdcpass "
 426                          "--include-env=vampire_2000_dc "
 427                          "--include-env=fl2000dc "
 428                          "--include-env=ad_dc_no_nss "
 429                          "--include-env=backupfromdc "
 430                          "--include-env=restoredc "
 431                          "--include-env=renamedc "
 432                          "--include-env=offlinebackupdc "
 433                          "--include-env=labdc "
 434                          "'",
 435                          "text/plain"),
 436                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 437
 438     # these are useful for debugging autobuild
 439     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
 440     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
 441
 442
 443 }
 444
 445
 446 def do_print(msg):
 447     print("%s" % msg)
 448     sys.stdout.flush()
 449     sys.stderr.flush()
 450
 451
 452 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 453     if show is None:
 454         show = options.verbose
 455     if show:
 456         do_print("Running: '%s' in '%s'" % (cmd, dir))
 457     if output:
 458         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
 459     elif checkfail:
 460         return check_call(cmd, shell=True, cwd=dir)
 461     else:
 462         return call(cmd, shell=True, cwd=dir)
 463
 464
 465 class builder(object):
 466     '''handle build of one directory'''
 467
 468     def __init__(self, name, sequence, cp=True, py3=False):
 469         self.name = name
 470         self.py3 = py3
 471         if name in builddirs:
 472             self.dir = builddirs[name]
 473         else:
 474             self.dir = "."
 475
 476         self.tag = self.name.replace('/', '_')
 477         self.sequence = sequence
 478         self.next = 0
 479         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 480         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 481         if options.verbose:
 482             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 483             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 484         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 485         self.stdout = open(self.stdout_path, 'w')
 486         self.stderr = open(self.stderr_path, 'w')
 487         self.stdin  = open("/dev/null", 'r')
 488         self.sdir = "%s/%s" % (testbase, self.tag)
 489         self.prefix = "%s/%s" % (test_prefix, self.tag)
 490         run_cmd("rm -rf %s" % self.sdir)
 491         run_cmd("rm -rf %s" % self.prefix)
 492         if cp:
 493             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 494         else:
 495             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 496         self.start_next()
 497
 498     def start_next(self):
 499         if self.next == len(self.sequence):
 500             if not options.nocleanup:
 501                 run_cmd("rm -rf %s" % self.sdir)
 502                 run_cmd("rm -rf %s" % self.prefix)
 503             do_print('%s: Completed OK' % self.name)
 504             self.done = True
 505             return
 506         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 507         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix))
 508         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 509         if self.py3:
 510             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
 511             # The trailing space is important
 512             self.cmd = self.cmd.replace("${PY3_ONLY}", "python3 ")
 513         else:
 514             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "")
 515             self.cmd = self.cmd.replace("${PY3_ONLY}", "")
 516         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 517         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 518 #        if self.output_mime_type == "text/x-subunit":
 519 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 520         cwd = os.getcwd()
 521         os.chdir("%s/%s" % (self.sdir, self.dir))
 522         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
 523         self.proc = Popen(self.cmd, shell=True,
 524                           close_fds=True,
 525                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 526         os.chdir(cwd)
 527         self.next += 1
 528
 529
 530 class buildlist(object):
 531     '''handle build of multiple directories'''
 532
 533     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 534         global tasks
 535         self.tlist = []
 536         self.tail_proc = None
 537         self.retry = None
 538         if tasknames == []:
 539             if options.restrict_tests:
 540                 tasknames = ["samba-test-only"]
 541             else:
 542                 tasknames = defaulttasks
 543         else:
 544             # If we are only running one test,
 545             # do not sleep randomly to wait for it to start
 546             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 547
 548         for n in tasknames:
 549             if n not in tasks and n.endswith("-py3"):
 550                 b = builder(n,
 551                             tasks[n[:-4]],
 552                             cp=n is not "pidl",
 553                             py3=True)
 554             else:
 555                 b = builder(n, tasks[n], cp=n is not "pidl")
 556             self.tlist.append(b)
 557         if options.retry:
 558             rebase_remote = "rebaseon"
 559             retry_task = [("retry",
 560                             '''set -e
 561                             git remote add -t %s %s %s
 562                             git fetch %s
 563                             while :; do
 564                               sleep 60
 565                               git describe %s/%s > old_remote_branch.desc
 566                               git fetch %s
 567                               git describe %s/%s > remote_branch.desc
 568                               diff old_remote_branch.desc remote_branch.desc
 569                             done
 570                            ''' % (
 571                                 rebase_branch, rebase_remote, rebase_url,
 572                                rebase_remote,
 573                                rebase_remote, rebase_branch,
 574                                rebase_remote,
 575                                rebase_remote, rebase_branch
 576                             ),
 577                             "test/plain")]
 578
 579             self.retry = builder('retry', retry_task, cp=False)
 580             self.need_retry = False
 581
 582     def kill_kids(self):
 583         if self.tail_proc is not None:
 584             self.tail_proc.terminate()
 585             self.tail_proc.wait()
 586             self.tail_proc = None
 587         if self.retry is not None:
 588             self.retry.proc.terminate()
 589             self.retry.proc.wait()
 590             self.retry = None
 591         for b in self.tlist:
 592             if b.proc is not None:
 593                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 594                 b.proc.terminate()
 595                 b.proc.wait()
 596                 b.proc = None
 597
 598     def wait_one(self):
 599         while True:
 600             none_running = True
 601             for b in self.tlist:
 602                 if b.proc is None:
 603                     continue
 604                 none_running = False
 605                 b.status = b.proc.poll()
 606                 if b.status is None:
 607                     continue
 608                 b.proc = None
 609                 return b
 610             if options.retry:
 611                 ret = self.retry.proc.poll()
 612                 if ret is not None:
 613                     self.need_retry = True
 614                     self.retry = None
 615                     return None
 616             if none_running:
 617                 return None
 618             time.sleep(0.1)
 619
 620     def run(self):
 621         while True:
 622             b = self.wait_one()
 623             if options.retry and self.need_retry:
 624                 self.kill_kids()
 625                 do_print("retry needed")
 626                 return (0, None, None, None, "retry")
 627             if b is None:
 628                 break
 629             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 630                 self.kill_kids()
 631                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 632             b.start_next()
 633         self.kill_kids()
 634         return (0, None, None, None, "All OK")
 635
 636     def write_system_info(self):
 637         filename = 'system-info.txt'
 638         f = open(filename, 'w')
 639         for cmd in ['uname -a', 'free', 'cat /proc/cpuinfo',
 640                     'cc --version', 'df -m .', 'df -m %s' % testbase]:
 641             print('### %s' % cmd, file=f)
 642             print(run_cmd(cmd, output=True, checkfail=False), file=f)
 643             print(file=f)
 644         f.close()
 645         return filename
 646
 647     def tarlogs(self, fname):
 648         tar = tarfile.open(fname, "w:gz")
 649         for b in self.tlist:
 650             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 651             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 652         if os.path.exists("autobuild.log"):
 653             tar.add("autobuild.log")
 654         sys_info = self.write_system_info()
 655         tar.add(sys_info)
 656         tar.close()
 657
 658     def remove_logs(self):
 659         for b in self.tlist:
 660             os.unlink(b.stdout_path)
 661             os.unlink(b.stderr_path)
 662
 663     def start_tail(self):
 664         cmd = ["tail", "-f"]
 665         for b in self.tlist:
 666             cmd.append(b.stdout_path)
 667             cmd.append(b.stderr_path)
 668         self.tail_proc = Popen(cmd, close_fds=True)
 669
 670
 671 def cleanup():
 672     if options.nocleanup:
 673         return
 674     run_cmd("stat %s || true" % test_tmpdir, show=True)
 675     run_cmd("stat %s" % testbase, show=True)
 676     do_print("Cleaning up %r" % cleanup_list)
 677     for d in cleanup_list:
 678         run_cmd("rm -rf %s" % d)
 679
 680
 681 def find_git_root():
 682     '''get to the top of the git repo'''
 683     p = os.getcwd()
 684     while p != '/':
 685         if os.path.isdir(os.path.join(p, ".git")):
 686             return p
 687         p = os.path.abspath(os.path.join(p, '..'))
 688     return None
 689
 690
 691 def daemonize(logfile):
 692     pid = os.fork()
 693     if pid == 0:  # Parent
 694         os.setsid()
 695         pid = os.fork()
 696         if pid != 0:  # Actual daemon
 697             os._exit(0)
 698     else:  # Grandparent
 699         os._exit(0)
 700
 701     import resource      # Resource usage information.
 702     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 703     if maxfd == resource.RLIM_INFINITY:
 704         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 705     for fd in range(0, maxfd):
 706         try:
 707             os.close(fd)
 708         except OSError:
 709             pass
 710     os.open(logfile, os.O_RDWR | os.O_CREAT)
 711     os.dup2(0, 1)
 712     os.dup2(0, 2)
 713
 714
 715 def write_pidfile(fname):
 716     '''write a pid file, cleanup on exit'''
 717     f = open(fname, mode='w')
 718     f.write("%u\n" % os.getpid())
 719     f.close()
 720
 721
 722 def rebase_tree(rebase_url, rebase_branch="master"):
 723     rebase_remote = "rebaseon"
 724     do_print("Rebasing on %s" % rebase_url)
 725     run_cmd("git describe HEAD", show=True, dir=test_master)
 726     run_cmd("git remote add -t %s %s %s" %
 727             (rebase_branch, rebase_remote, rebase_url),
 728             show=True, dir=test_master)
 729     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 730     if options.fix_whitespace:
 731         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 732                 (rebase_remote, rebase_branch),
 733                 show=True, dir=test_master)
 734     else:
 735         run_cmd("git rebase --force-rebase %s/%s" %
 736                 (rebase_remote, rebase_branch),
 737                 show=True, dir=test_master)
 738     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 739                    (rebase_remote, rebase_branch),
 740                    dir=test_master, output=True)
 741     if diff == '':
 742         do_print("No differences between HEAD and %s/%s - exiting" %
 743                  (rebase_remote, rebase_branch))
 744         sys.exit(0)
 745     run_cmd("git describe %s/%s" %
 746             (rebase_remote, rebase_branch),
 747             show=True, dir=test_master)
 748     run_cmd("git describe HEAD", show=True, dir=test_master)
 749     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 750             (rebase_remote, rebase_branch),
 751             show=True, dir=test_master)
 752
 753
 754 def push_to(push_url, push_branch="master"):
 755     push_remote = "pushto"
 756     do_print("Pushing to %s" % push_url)
 757     if options.mark:
 758         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 759         run_cmd("git commit --amend -c HEAD", dir=test_master)
 760         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 761         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 762     run_cmd("git remote add -t %s %s %s" %
 763             (push_branch, push_remote, push_url),
 764             show=True, dir=test_master)
 765     run_cmd("git push %s +HEAD:%s" %
 766             (push_remote, push_branch),
 767             show=True, dir=test_master)
 768
 769
 770 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 771
 772 gitroot = find_git_root()
 773 if gitroot is None:
 774     raise Exception("Failed to find git root")
 775
 776 parser = OptionParser()
 777 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 778 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 779 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 780 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 781                   default=def_testbase)
 782 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 783 parser.add_option("", "--verbose", help="show all commands as they are run",
 784                   default=False, action="store_true")
 785 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 786                   default=None, type='str')
 787 parser.add_option("", "--pushto", help="push to a git url on success",
 788                   default=None, type='str')
 789 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 790                   default=False, action="store_true")
 791 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 792                   default=False, action="store_true")
 793 parser.add_option("", "--retry", help="automatically retry if master changes",
 794                   default=False, action="store_true")
 795 parser.add_option("", "--email", help="send email to the given address on failure",
 796                   type='str', default=None)
 797 parser.add_option("", "--email-from", help="send email from the given address",
 798                   type='str', default="autobuild@samba.org")
 799 parser.add_option("", "--email-server", help="send email via the given server",
 800                   type='str', default='localhost')
 801 parser.add_option("", "--always-email", help="always send email, even on success",
 802                   action="store_true")
 803 parser.add_option("", "--daemon", help="daemonize after initial setup",
 804                   action="store_true")
 805 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 806                   default="master", type='str')
 807 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 808                   default=gitroot, type='str')
 809 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 810                   default=False, action="store_true")
 811 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 812                   default='')
 813
 814
 815 def send_email(subject, text, log_tar):
 816     if options.email is None:
 817         do_print("not sending email because the recipient is not set")
 818         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
 819                  (subject, text))
 820         return
 821     outer = MIMEMultipart()
 822     outer['Subject'] = subject
 823     outer['To'] = options.email
 824     outer['From'] = options.email_from
 825     outer['Date'] = email.utils.formatdate(localtime=True)
 826     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 827     outer.attach(MIMEText(text, 'plain'))
 828     if options.attach_logs:
 829         fp = open(log_tar, 'rb')
 830         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 831         fp.close()
 832         # Set the filename parameter
 833         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 834         outer.attach(msg)
 835     content = outer.as_string()
 836     s = smtplib.SMTP(options.email_server)
 837     s.sendmail(options.email_from, [options.email], content)
 838     s.set_debuglevel(1)
 839     s.quit()
 840
 841
 842 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 843                   elapsed_time, log_base=None, add_log_tail=True):
 844     '''send an email to options.email about the failure'''
 845     elapsed_minutes = elapsed_time / 60.0
 846     if log_base is None:
 847         log_base = gitroot
 848     text = '''
 849 Dear Developer,
 850
 851 Your autobuild on %s failed after %.1f minutes
 852 when trying to test %s with the following error:
 853
 854    %s
 855
 856 the autobuild has been abandoned. Please fix the error and resubmit.
 857
 858 A summary of the autobuild process is here:
 859
 860   %s/autobuild.log
 861 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 862
 863     if options.restrict_tests:
 864         text += """
 865 The build was restricted to tests matching %s\n""" % options.restrict_tests
 866
 867     if failed_task != 'rebase':
 868         text += '''
 869 You can see logs of the failed task here:
 870
 871   %s/%s.stdout
 872   %s/%s.stderr
 873
 874 or you can get full logs of all tasks in this job here:
 875
 876   %s/logs.tar.gz
 877
 878 The top commit for the tree that was built was:
 879
 880 %s
 881
 882 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 883
 884     if add_log_tail:
 885         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 886         lines = f.readlines()
 887         log_tail = "".join(lines[-50:])
 888         num_lines = len(lines)
 889         if num_lines < 50:
 890             # Also include stderr (compile failures) if < 50 lines of stdout
 891             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 892             log_tail += "".join(f.readlines()[-(50 - num_lines):])
 893
 894         text += '''
 895 The last 50 lines of log messages:
 896
 897 %s
 898     ''' % log_tail
 899         f.close()
 900
 901     logs = os.path.join(gitroot, 'logs.tar.gz')
 902     send_email('autobuild[%s] failure on %s for task %s during %s'
 903                % (options.branch, platform.node(), failed_task, failed_stage),
 904                text, logs)
 905
 906
 907 def email_success(elapsed_time, log_base=None):
 908     '''send an email to options.email about a successful build'''
 909     if log_base is None:
 910         log_base = gitroot
 911     text = '''
 912 Dear Developer,
 913
 914 Your autobuild on %s has succeeded after %.1f minutes.
 915
 916 ''' % (platform.node(), elapsed_time / 60.)
 917
 918     if options.restrict_tests:
 919         text += """
 920 The build was restricted to tests matching %s\n""" % options.restrict_tests
 921
 922     if options.keeplogs:
 923         text += '''
 924
 925 you can get full logs of all tasks in this job here:
 926
 927   %s/logs.tar.gz
 928
 929 ''' % log_base
 930
 931     text += '''
 932 The top commit for the tree that was built was:
 933
 934 %s
 935 ''' % top_commit_msg
 936
 937     logs = os.path.join(gitroot, 'logs.tar.gz')
 938     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 939                text, logs)
 940
 941
 942 (options, args) = parser.parse_args()
 943
 944 if options.retry:
 945     if options.rebase is None:
 946         raise Exception('You can only use --retry if you also rebase')
 947
 948 testbase = "%s/b%u" % (options.testbase, os.getpid())
 949 test_master = "%s/master" % testbase
 950 test_prefix = "%s/prefix" % testbase
 951 test_tmpdir = "%s/tmp" % testbase
 952 os.environ['TMPDIR'] = test_tmpdir
 953
 954 # get the top commit message, for emails
 955 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 956
 957 try:
 958     os.makedirs(testbase)
 959 except Exception as reason:
 960     raise Exception("Unable to create %s : %s" % (testbase, reason))
 961 cleanup_list.append(testbase)
 962
 963 if options.daemon:
 964     logfile = os.path.join(testbase, "log")
 965     do_print("Forking into the background, writing progress to %s" % logfile)
 966     daemonize(logfile)
 967
 968 write_pidfile(gitroot + "/autobuild.pid")
 969
 970 start_time = time.time()
 971
 972 while True:
 973     try:
 974         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 975         os.makedirs(test_tmpdir)
 976         # The waf uninstall code removes empty directories all the way
 977         # up the tree.  Creating a file in test_tmpdir stops it from
 978         # being removed.
 979         run_cmd("touch %s" % os.path.join(test_tmpdir,
 980                                           ".directory-is-not-empty"), show=True)
 981         run_cmd("stat %s" % test_tmpdir, show=True)
 982         run_cmd("stat %s" % testbase, show=True)
 983         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 984     except Exception:
 985         cleanup()
 986         raise
 987
 988     try:
 989         try:
 990             if options.rebase is not None:
 991                 rebase_tree(options.rebase, rebase_branch=options.branch)
 992         except Exception:
 993             cleanup_list.append(gitroot + "/autobuild.pid")
 994             cleanup()
 995             elapsed_time = time.time() - start_time
 996             email_failure(-1, 'rebase', 'rebase', 'rebase',
 997                           'rebase on %s failed' % options.branch,
 998                           elapsed_time, log_base=options.log_base)
 999             sys.exit(1)
1000         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1001         if options.tail:
1002             blist.start_tail()
1003         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1004         if status != 0 or errstr != "retry":
1005             break
1006         cleanup()
1007     except Exception:
1008         cleanup()
1009         raise
1010
1011 cleanup_list.append(gitroot + "/autobuild.pid")
1012
1013 do_print(errstr)
1014
1015 blist.kill_kids()
1016 if options.tail:
1017     do_print("waiting for tail to flush")
1018     time.sleep(1)
1019
1020 elapsed_time = time.time() - start_time
1021 if status == 0:
1022     if options.passcmd is not None:
1023         do_print("Running passcmd: %s" % options.passcmd)
1024         run_cmd(options.passcmd, dir=test_master)
1025     if options.pushto is not None:
1026         push_to(options.pushto, push_branch=options.branch)
1027     if options.keeplogs or options.attach_logs:
1028         blist.tarlogs("logs.tar.gz")
1029         do_print("Logs in logs.tar.gz")
1030     if options.always_email:
1031         email_success(elapsed_time, log_base=options.log_base)
1032     blist.remove_logs()
1033     cleanup()
1034     do_print(errstr)
1035     sys.exit(0)
1036
1037 # something failed, gather a tar of the logs
1038 blist.tarlogs("logs.tar.gz")
1039
1040 if options.email is not None:
1041     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1042                   elapsed_time, log_base=options.log_base)
1043 else:
1044     elapsed_minutes = elapsed_time / 60.0
1045     print('''
1046
1047 ####################################################################
1048
1049 AUTOBUILD FAILURE
1050
1051 Your autobuild[%s] on %s failed after %.1f minutes
1052 when trying to test %s with the following error:
1053
1054    %s
1055
1056 the autobuild has been abandoned. Please fix the error and resubmit.
1057
1058 ####################################################################
1059
1060 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1061
1062 cleanup()
1063 do_print(errstr)
1064 do_print("Logs in logs.tar.gz")
1065 sys.exit(status)