script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34 cleanup_list = []
  35
  36 builddirs = {
  37     "ctdb": "ctdb",
  38     "samba": ".",
  39     "samba-py2": ".",
  40     "samba-nt4": ".",
  41     "samba-nt4-py2": ".",
  42     "samba-fileserver": ".",
  43     "samba-xc": ".",
  44     "samba-o3": ".",
  45     "samba-ctdb": ".",
  46     "samba-libs": ".",
  47     "samba-libs-py2": ".",
  48     "samba-static": ".",
  49     "samba-none-env": ".",
  50     "samba-ad-dc": ".",
  51     "samba-ad-dc-py2": ".",
  52     "samba-ad-dc-2": ".",
  53     "samba-ad-dc-2-py2": ".",
  54     "samba-systemkrb5": ".",
  55     "samba-nopython": ".",
  56     "samba-buildpy2-only": ".",
  57     "ldb": "lib/ldb",
  58     "tdb": "lib/tdb",
  59     "talloc": "lib/talloc",
  60     "replace": "lib/replace",
  61     "tevent": "lib/tevent",
  62     "pidl": "pidl"
  63 }
  64
  65 defaulttasks = builddirs.keys()
  66
  67 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  68     defaulttasks.remove("samba-o3")
  69
  70 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  71 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
  72
  73 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
  74 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  75 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  76 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
  77 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE ${EXTRA_PYTHON}"
  78 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
  79 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs + " ${EXTRA_PYTHON}"
  80
  81 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
  82     extra_python = ""
  83 else:
  84     extra_python = "--extra-python=/usr/bin/python2"
  85
  86 tasks = {
  87     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  88                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  89                ("make", "make all", "text/plain"),
  90                ("install", "make install", "text/plain"),
  91                ("test", "make autotest", "text/plain"),
  92                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  93                ("clean", "make clean", "text/plain")],
  94
  95     # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)'
  96     "samba": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
  97                 ("make", "make -j", "text/plain"),
  98                 ("test", "make test FAIL_IMMEDIATELY=1 "
  99                  "TESTS='${PY3_ONLY}"
 100                  "--exclude-env=none "
 101                  "--exclude-env=nt4_dc "
 102                  "--exclude-env=nt4_member "
 103                  "--exclude-env=ad_dc "
 104                  "--exclude-env=ad_dc_no_nss "
 105                  "--exclude-env=fl2003dc "
 106                  "--exclude-env=fl2008r2dc "
 107                  "--exclude-env=ad_member "
 108                  "--exclude-env=ad_member_idmap_rid "
 109                  "--exclude-env=ad_member_idmap_ad "
 110                  "--exclude-env=chgdcpass "
 111                  "--exclude-env=vampire_2000_dc "
 112                  "--exclude-env=fl2000dc "
 113                  "--exclude-env=fileserver "
 114                  "--exclude-env=backupfromdc "
 115                  "--exclude-env=restoredc "
 116                  "--exclude-env=renamedc "
 117                  "--exclude-env=offlinebackupdc "
 118                  "--exclude-env=labdc "
 119                  "'",
 120                  "text/plain"),
 121                 ("install", "make install", "text/plain"),
 122                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 123                 ("clean", "make clean", "text/plain")],
 124
 125     # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long)
 126     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 127                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 128                     ("make", "make -j", "text/plain"),
 129                     ("test", "make test FAIL_IMMEDIATELY=1 "
 130                      "TESTS='${PY3_ONLY}"
 131                      "--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 132                     ("install", "make install", "text/plain"),
 133                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 134                     ("clean", "make clean", "text/plain")],
 135
 136     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 137     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 138                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 139                            ("make", "make -j", "text/plain"),
 140                            ("test", "make test FAIL_IMMEDIATELY=1 "
 141                             "TESTS='${PY3_ONLY}"
 142                             "--include-env=fileserver'", "text/plain"),
 143                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 144
 145     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 146     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 147                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 148                       ("make", "make -j", "text/plain"),
 149                       ("test", "make test FAIL_IMMEDIATELY=1 "
 150                        "TESTS='${PY3_ONLY}"
 151                        "--include-env=ad_dc "
 152                        "--include-env=fl2003dc "
 153                        "--include-env=fl2008r2dc "
 154                        "--include-env=ad_member "
 155                        "--include-env=ad_member_idmap_rid "
 156                        "--include-env=ad_member_idmap_ad'", "text/plain"),
 157                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 158
 159     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 160     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 161                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 162                         ("make", "make -j", "text/plain"),
 163                         ("test", "make test FAIL_IMMEDIATELY=1 "
 164                          "TESTS='${PY3_ONLY}"
 165                          "--include-env=chgdcpass "
 166                          "--include-env=vampire_2000_dc "
 167                          "--include-env=fl2000dc "
 168                          "--include-env=ad_dc_no_nss "
 169                          "--include-env=backupfromdc "
 170                          "--include-env=restoredc "
 171                          "--include-env=renamedc "
 172                          "--include-env=offlinebackupdc "
 173                          "--include-env=labdc "
 174                          "'",
 175                          "text/plain"),
 176                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 177
 178     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 179                           ("make", "make -j", "text/plain"),
 180                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
 181
 182     # Test cross-compile infrastructure
 183     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 184                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 185                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 186                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 187                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 188                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 189                    ("compare-results", "script/compare_cc_results.py "
 190                     "./bin/c4che/default{} "
 191                     "./bin-xe/c4che/default{} "
 192                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
 193
 194     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 195     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 196                    ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 197                    ("make", "make -j", "text/plain"),
 198                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
 199                     "TESTS='${PY3_ONLY}"
 200                     "--include-env=ad_dc'", "text/plain"),
 201                    ("install", "make install", "text/plain"),
 202                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 203                    ("clean", "make clean", "text/plain")],
 204
 205     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 206
 207                      # make sure we have tdb around:
 208                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 209                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 210                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 211
 212
 213                      # build samba with cluster support (also building ctdb):
 214                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 215                      ("samba-make", "make", "text/plain"),
 216                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 217                      ("samba-install", "make install", "text/plain"),
 218                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 219
 220                      # clean up:
 221                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 222                      ("clean", "make clean", "text/plain"),
 223                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
 224
 225     "samba-libs": [
 226                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 227                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 228                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 229                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 230
 231                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 232                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 233                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 234
 235                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 236                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 237                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 238
 239                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 240                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 241                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 242
 243                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 244                       ("nondevel-make", "make -j", "text/plain"),
 245                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 246                       ("nondevel-install", "make install", "text/plain"),
 247                       ("nondevel-dist", "make dist", "text/plain"),
 248
 249                       # retry with all modules shared
 250                       ("allshared-distclean", "make distclean", "text/plain"),
 251                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 252                       ("allshared-make", "make -j", "text/plain")],
 253
 254     "samba-none-env": [
 255                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 256                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 257                       ("make", "make -j", "text/plain"),
 258                       ("test", "make test "
 259                        "FAIL_IMMEDIATELY=1 "
 260                        "TESTS='${PY3_ONLY}"
 261                        "--include-env=none'",
 262                        "text/plain")],
 263
 264     "samba-static": [
 265                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 266                       # build with all modules static
 267                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 268                       ("allstatic-make", "make -j", "text/plain"),
 269                       ("allstatic-test", "make test "
 270                        "FAIL_IMMEDIATELY=1 "
 271                        "TESTS='samba3.smb2.create.*nt4_dc'",
 272                        "text/plain"),
 273
 274                       # retry without any required modules
 275                       ("none-distclean", "make distclean", "text/plain"),
 276                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 277                       ("none-make", "make -j", "text/plain"),
 278
 279                       # retry with nonshared smbd and smbtorture
 280                       ("nonshared-distclean", "make distclean", "text/plain"),
 281                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 282                       ("nonshared-make", "make -j", "text/plain")],
 283
 284     "samba-systemkrb5": [
 285                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 286                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 287                       ("make", "make -j", "text/plain"),
 288                       # we currently cannot run a full make test, a limited list of tests could be run
 289                       # via "make test TESTS=sometests"
 290                       ("test", "make test FAIL_IMMEDIATELY=1 "
 291                        "TESTS='${PY3_ONLY}"
 292                        "--include-env=ktest'", "text/plain"),
 293                       ("install", "make install", "text/plain"),
 294                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 295                       ("clean", "make clean", "text/plain")
 296                       ],
 297
 298     # Test Samba without python still builds.  When this test fails
 299     # due to more use of Python, the expectations is that the newly
 300     # failing part of the code should be disabled when
 301     # --disable-python is set (rather than major work being done to
 302     # support this environment).  The target here is for vendors
 303     # shipping a minimal smbd.
 304     "samba-nopython": [
 305                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 306                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 307                       ("make", "make -j", "text/plain"),
 308                       ("install", "make install", "text/plain"),
 309                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 310                       ("clean", "make clean", "text/plain"),
 311
 312                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 313                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 314                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 315
 316                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 317                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 318                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 319
 320                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 321                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 322                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 323
 324                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 325                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 326                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 327
 328                       # retry against installed library packages
 329                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 330                       ("libs-make", "make -j", "text/plain"),
 331                       ("libs-install", "make install", "text/plain"),
 332                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 333                       ("libs-clean", "make clean", "text/plain")
 334                       ],
 335
 336
 337
 338     "ldb": [
 339               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 340               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 341               ("make", "make", "text/plain"),
 342               ("install", "make install", "text/plain"),
 343               ("test", "make test", "text/plain"),
 344               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 345               ("make-no-lmdb", "make", "text/plain"),
 346               ("install-no-lmdb", "make install", "text/plain"),
 347               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 348               ("distcheck", "make distcheck", "text/plain"),
 349               ("clean", "make clean", "text/plain")],
 350
 351     "tdb": [
 352               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 353               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 354               ("make", "make", "text/plain"),
 355               ("install", "make install", "text/plain"),
 356               ("test", "make test", "text/plain"),
 357               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 358               ("distcheck", "make distcheck", "text/plain"),
 359               ("clean", "make clean", "text/plain")],
 360
 361     "talloc": [
 362                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 363                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 364                  ("make", "make", "text/plain"),
 365                  ("install", "make install", "text/plain"),
 366                  ("test", "make test", "text/plain"),
 367                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 368                  ("distcheck", "make distcheck", "text/plain"),
 369                  ("clean", "make clean", "text/plain")],
 370
 371     "replace": [
 372                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 373                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 374                   ("make", "make", "text/plain"),
 375                   ("install", "make install", "text/plain"),
 376                   ("test", "make test", "text/plain"),
 377                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 378                   ("distcheck", "make distcheck", "text/plain"),
 379                   ("clean", "make clean", "text/plain")],
 380
 381     "tevent": [
 382                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 383                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 384                  ("make", "make", "text/plain"),
 385                  ("install", "make install", "text/plain"),
 386                  ("test", "make test", "text/plain"),
 387                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 388                  ("distcheck", "make distcheck", "text/plain"),
 389                  ("clean", "make clean", "text/plain")],
 390
 391     "pidl": [
 392         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 393         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 394         ("touch", "touch *.yp", "text/plain"),
 395         ("make", "make", "text/plain"),
 396         ("test", "make test", "text/plain"),
 397         ("install", "make install", "text/plain"),
 398         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 399         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 400         ("clean", "make clean", "text/plain")],
 401
 402     "samba-buildpy2-only": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 403                    ("configure", "PYTHON='python' ./configure.developer --with-selftest-prefix=./bin/ab " + samba_configure_params, "text/plain"),
 404                    ("make", "PYTHON='python' make -j", "text/plain"),
 405                    ("install", "PYTHON='python' make install", "text/plain"),
 406                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 407                    ("clean", "PYTHON='python' make clean", "text/plain")],
 408
 409
 410     # these are useful for debugging autobuild
 411     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
 412     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
 413
 414
 415 }
 416
 417
 418 def do_print(msg):
 419     print("%s" % msg)
 420     sys.stdout.flush()
 421     sys.stderr.flush()
 422
 423
 424 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 425     if show is None:
 426         show = options.verbose
 427     if show:
 428         do_print("Running: '%s' in '%s'" % (cmd, dir))
 429     if output:
 430         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
 431     elif checkfail:
 432         return check_call(cmd, shell=True, cwd=dir)
 433     else:
 434         return call(cmd, shell=True, cwd=dir)
 435
 436
 437 class builder(object):
 438     '''handle build of one directory'''
 439
 440     def __init__(self, name, sequence, cp=True, py3=False):
 441         self.name = name
 442         self.py3 = py3
 443         if name in builddirs:
 444             self.dir = builddirs[name]
 445         else:
 446             self.dir = "."
 447
 448         self.tag = self.name.replace('/', '_')
 449         self.sequence = sequence
 450         self.next = 0
 451         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 452         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 453         if options.verbose:
 454             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 455             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 456         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 457         self.stdout = open(self.stdout_path, 'w')
 458         self.stderr = open(self.stderr_path, 'w')
 459         self.stdin  = open("/dev/null", 'r')
 460         self.sdir = "%s/%s" % (testbase, self.tag)
 461         self.prefix = "%s/%s" % (test_prefix, self.tag)
 462         run_cmd("rm -rf %s" % self.sdir)
 463         run_cmd("rm -rf %s" % self.prefix)
 464         if cp:
 465             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 466         else:
 467             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 468         self.start_next()
 469
 470     def start_next(self):
 471         if self.next == len(self.sequence):
 472             if not options.nocleanup:
 473                 run_cmd("rm -rf %s" % self.sdir)
 474                 run_cmd("rm -rf %s" % self.prefix)
 475             do_print('%s: Completed OK' % self.name)
 476             self.done = True
 477             return
 478         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 479         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
 480         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 481         if self.py3:
 482             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
 483             # The trailing space is important
 484             self.cmd = self.cmd.replace("${PY3_ONLY}", "python2 ")
 485         else:
 486             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "")
 487             self.cmd = self.cmd.replace("${PY3_ONLY}", "")
 488         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 489         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 490 #        if self.output_mime_type == "text/x-subunit":
 491 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 492         cwd = os.getcwd()
 493         os.chdir("%s/%s" % (self.sdir, self.dir))
 494         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
 495         self.proc = Popen(self.cmd, shell=True,
 496                           close_fds=True,
 497                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 498         os.chdir(cwd)
 499         self.next += 1
 500
 501
 502 class buildlist(object):
 503     '''handle build of multiple directories'''
 504
 505     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 506         global tasks
 507         self.tlist = []
 508         self.tail_proc = None
 509         self.retry = None
 510         if tasknames == []:
 511             if options.restrict_tests:
 512                 tasknames = ["samba-test-only"]
 513             else:
 514                 tasknames = defaulttasks
 515         else:
 516             # If we are only running one test,
 517             # do not sleep randomly to wait for it to start
 518             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 519
 520         for n in tasknames:
 521             if n not in tasks and n.endswith("-py2"):
 522                 b = builder(n,
 523                             tasks[n[:-4]],
 524                             cp=n is not "pidl",
 525                             py3=True)
 526             else:
 527                 b = builder(n, tasks[n], cp=n is not "pidl")
 528             self.tlist.append(b)
 529         if options.retry:
 530             rebase_remote = "rebaseon"
 531             retry_task = [("retry",
 532                             '''set -e
 533                             git remote add -t %s %s %s
 534                             git fetch %s
 535                             while :; do
 536                               sleep 60
 537                               git describe %s/%s > old_remote_branch.desc
 538                               git fetch %s
 539                               git describe %s/%s > remote_branch.desc
 540                               diff old_remote_branch.desc remote_branch.desc
 541                             done
 542                            ''' % (
 543                                 rebase_branch, rebase_remote, rebase_url,
 544                                rebase_remote,
 545                                rebase_remote, rebase_branch,
 546                                rebase_remote,
 547                                rebase_remote, rebase_branch
 548                             ),
 549                             "test/plain")]
 550
 551             self.retry = builder('retry', retry_task, cp=False)
 552             self.need_retry = False
 553
 554     def kill_kids(self):
 555         if self.tail_proc is not None:
 556             self.tail_proc.terminate()
 557             self.tail_proc.wait()
 558             self.tail_proc = None
 559         if self.retry is not None:
 560             self.retry.proc.terminate()
 561             self.retry.proc.wait()
 562             self.retry = None
 563         for b in self.tlist:
 564             if b.proc is not None:
 565                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 566                 b.proc.terminate()
 567                 b.proc.wait()
 568                 b.proc = None
 569
 570     def wait_one(self):
 571         while True:
 572             none_running = True
 573             for b in self.tlist:
 574                 if b.proc is None:
 575                     continue
 576                 none_running = False
 577                 b.status = b.proc.poll()
 578                 if b.status is None:
 579                     continue
 580                 b.proc = None
 581                 return b
 582             if options.retry:
 583                 ret = self.retry.proc.poll()
 584                 if ret is not None:
 585                     self.need_retry = True
 586                     self.retry = None
 587                     return None
 588             if none_running:
 589                 return None
 590             time.sleep(0.1)
 591
 592     def run(self):
 593         while True:
 594             b = self.wait_one()
 595             if options.retry and self.need_retry:
 596                 self.kill_kids()
 597                 do_print("retry needed")
 598                 return (0, None, None, None, "retry")
 599             if b is None:
 600                 break
 601             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 602                 self.kill_kids()
 603                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 604             b.start_next()
 605         self.kill_kids()
 606         return (0, None, None, None, "All OK")
 607
 608     def write_system_info(self):
 609         filename = 'system-info.txt'
 610         f = open(filename, 'w')
 611         for cmd in ['uname -a',
 612                     'free',
 613                     'cat /proc/cpuinfo',
 614                     'cc --version',
 615                     'df -m .',
 616                     'df -m %s' % testbase]:
 617             out = run_cmd(cmd, output=True, checkfail=False)
 618             print('### %s' % cmd, file=f)
 619             print(out.decode('utf8', 'backslashreplace'), file=f)
 620             print(file=f)
 621         f.close()
 622         return filename
 623
 624     def tarlogs(self, fname):
 625         tar = tarfile.open(fname, "w:gz")
 626         for b in self.tlist:
 627             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 628             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 629         if os.path.exists("autobuild.log"):
 630             tar.add("autobuild.log")
 631         sys_info = self.write_system_info()
 632         tar.add(sys_info)
 633         tar.close()
 634
 635     def remove_logs(self):
 636         for b in self.tlist:
 637             os.unlink(b.stdout_path)
 638             os.unlink(b.stderr_path)
 639
 640     def start_tail(self):
 641         cmd = ["tail", "-f"]
 642         for b in self.tlist:
 643             cmd.append(b.stdout_path)
 644             cmd.append(b.stderr_path)
 645         self.tail_proc = Popen(cmd, close_fds=True)
 646
 647
 648 def cleanup():
 649     if options.nocleanup:
 650         return
 651     run_cmd("stat %s || true" % test_tmpdir, show=True)
 652     run_cmd("stat %s" % testbase, show=True)
 653     do_print("Cleaning up %r" % cleanup_list)
 654     for d in cleanup_list:
 655         run_cmd("rm -rf %s" % d)
 656
 657
 658 def find_git_root():
 659     '''get to the top of the git repo'''
 660     p = os.getcwd()
 661     while p != '/':
 662         if os.path.isdir(os.path.join(p, ".git")):
 663             return p
 664         p = os.path.abspath(os.path.join(p, '..'))
 665     return None
 666
 667
 668 def daemonize(logfile):
 669     pid = os.fork()
 670     if pid == 0:  # Parent
 671         os.setsid()
 672         pid = os.fork()
 673         if pid != 0:  # Actual daemon
 674             os._exit(0)
 675     else:  # Grandparent
 676         os._exit(0)
 677
 678     import resource      # Resource usage information.
 679     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 680     if maxfd == resource.RLIM_INFINITY:
 681         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 682     for fd in range(0, maxfd):
 683         try:
 684             os.close(fd)
 685         except OSError:
 686             pass
 687     os.open(logfile, os.O_RDWR | os.O_CREAT)
 688     os.dup2(0, 1)
 689     os.dup2(0, 2)
 690
 691
 692 def write_pidfile(fname):
 693     '''write a pid file, cleanup on exit'''
 694     f = open(fname, mode='w')
 695     f.write("%u\n" % os.getpid())
 696     f.close()
 697
 698
 699 def rebase_tree(rebase_url, rebase_branch="master"):
 700     rebase_remote = "rebaseon"
 701     do_print("Rebasing on %s" % rebase_url)
 702     run_cmd("git describe HEAD", show=True, dir=test_master)
 703     run_cmd("git remote add -t %s %s %s" %
 704             (rebase_branch, rebase_remote, rebase_url),
 705             show=True, dir=test_master)
 706     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 707     if options.fix_whitespace:
 708         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 709                 (rebase_remote, rebase_branch),
 710                 show=True, dir=test_master)
 711     else:
 712         run_cmd("git rebase --force-rebase %s/%s" %
 713                 (rebase_remote, rebase_branch),
 714                 show=True, dir=test_master)
 715     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 716                    (rebase_remote, rebase_branch),
 717                    dir=test_master, output=True)
 718     if diff == '':
 719         do_print("No differences between HEAD and %s/%s - exiting" %
 720                  (rebase_remote, rebase_branch))
 721         sys.exit(0)
 722     run_cmd("git describe %s/%s" %
 723             (rebase_remote, rebase_branch),
 724             show=True, dir=test_master)
 725     run_cmd("git describe HEAD", show=True, dir=test_master)
 726     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 727             (rebase_remote, rebase_branch),
 728             show=True, dir=test_master)
 729
 730
 731 def push_to(push_url, push_branch="master"):
 732     push_remote = "pushto"
 733     do_print("Pushing to %s" % push_url)
 734     if options.mark:
 735         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 736         run_cmd("git commit --amend -c HEAD", dir=test_master)
 737         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 738         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 739     run_cmd("git remote add -t %s %s %s" %
 740             (push_branch, push_remote, push_url),
 741             show=True, dir=test_master)
 742     run_cmd("git push %s +HEAD:%s" %
 743             (push_remote, push_branch),
 744             show=True, dir=test_master)
 745
 746
 747 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 748
 749 gitroot = find_git_root()
 750 if gitroot is None:
 751     raise Exception("Failed to find git root")
 752
 753 parser = OptionParser()
 754 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 755 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 756 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 757 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 758                   default=def_testbase)
 759 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 760 parser.add_option("", "--verbose", help="show all commands as they are run",
 761                   default=False, action="store_true")
 762 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 763                   default=None, type='str')
 764 parser.add_option("", "--pushto", help="push to a git url on success",
 765                   default=None, type='str')
 766 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 767                   default=False, action="store_true")
 768 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 769                   default=False, action="store_true")
 770 parser.add_option("", "--retry", help="automatically retry if master changes",
 771                   default=False, action="store_true")
 772 parser.add_option("", "--email", help="send email to the given address on failure",
 773                   type='str', default=None)
 774 parser.add_option("", "--email-from", help="send email from the given address",
 775                   type='str', default="autobuild@samba.org")
 776 parser.add_option("", "--email-server", help="send email via the given server",
 777                   type='str', default='localhost')
 778 parser.add_option("", "--always-email", help="always send email, even on success",
 779                   action="store_true")
 780 parser.add_option("", "--daemon", help="daemonize after initial setup",
 781                   action="store_true")
 782 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 783                   default="master", type='str')
 784 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 785                   default=gitroot, type='str')
 786 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 787                   default=False, action="store_true")
 788 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 789                   default='')
 790
 791
 792 def send_email(subject, text, log_tar):
 793     if options.email is None:
 794         do_print("not sending email because the recipient is not set")
 795         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
 796                  (subject, text))
 797         return
 798     outer = MIMEMultipart()
 799     outer['Subject'] = subject
 800     outer['To'] = options.email
 801     outer['From'] = options.email_from
 802     outer['Date'] = email.utils.formatdate(localtime=True)
 803     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 804     outer.attach(MIMEText(text, 'plain'))
 805     if options.attach_logs:
 806         fp = open(log_tar, 'rb')
 807         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 808         fp.close()
 809         # Set the filename parameter
 810         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 811         outer.attach(msg)
 812     content = outer.as_string()
 813     s = smtplib.SMTP(options.email_server)
 814     s.sendmail(options.email_from, [options.email], content)
 815     s.set_debuglevel(1)
 816     s.quit()
 817
 818
 819 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 820                   elapsed_time, log_base=None, add_log_tail=True):
 821     '''send an email to options.email about the failure'''
 822     elapsed_minutes = elapsed_time / 60.0
 823     if log_base is None:
 824         log_base = gitroot
 825     text = '''
 826 Dear Developer,
 827
 828 Your autobuild on %s failed after %.1f minutes
 829 when trying to test %s with the following error:
 830
 831    %s
 832
 833 the autobuild has been abandoned. Please fix the error and resubmit.
 834
 835 A summary of the autobuild process is here:
 836
 837   %s/autobuild.log
 838 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 839
 840     if options.restrict_tests:
 841         text += """
 842 The build was restricted to tests matching %s\n""" % options.restrict_tests
 843
 844     if failed_task != 'rebase':
 845         text += '''
 846 You can see logs of the failed task here:
 847
 848   %s/%s.stdout
 849   %s/%s.stderr
 850
 851 or you can get full logs of all tasks in this job here:
 852
 853   %s/logs.tar.gz
 854
 855 The top commit for the tree that was built was:
 856
 857 %s
 858
 859 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 860
 861     if add_log_tail:
 862         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 863         lines = f.readlines()
 864         log_tail = "".join(lines[-50:])
 865         num_lines = len(lines)
 866         if num_lines < 50:
 867             # Also include stderr (compile failures) if < 50 lines of stdout
 868             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 869             log_tail += "".join(f.readlines()[-(50 - num_lines):])
 870
 871         text += '''
 872 The last 50 lines of log messages:
 873
 874 %s
 875     ''' % log_tail
 876         f.close()
 877
 878     logs = os.path.join(gitroot, 'logs.tar.gz')
 879     send_email('autobuild[%s] failure on %s for task %s during %s'
 880                % (options.branch, platform.node(), failed_task, failed_stage),
 881                text, logs)
 882
 883
 884 def email_success(elapsed_time, log_base=None):
 885     '''send an email to options.email about a successful build'''
 886     if log_base is None:
 887         log_base = gitroot
 888     text = '''
 889 Dear Developer,
 890
 891 Your autobuild on %s has succeeded after %.1f minutes.
 892
 893 ''' % (platform.node(), elapsed_time / 60.)
 894
 895     if options.restrict_tests:
 896         text += """
 897 The build was restricted to tests matching %s\n""" % options.restrict_tests
 898
 899     if options.keeplogs:
 900         text += '''
 901
 902 you can get full logs of all tasks in this job here:
 903
 904   %s/logs.tar.gz
 905
 906 ''' % log_base
 907
 908     text += '''
 909 The top commit for the tree that was built was:
 910
 911 %s
 912 ''' % top_commit_msg
 913
 914     logs = os.path.join(gitroot, 'logs.tar.gz')
 915     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 916                text, logs)
 917
 918
 919 (options, args) = parser.parse_args()
 920
 921 if options.retry:
 922     if options.rebase is None:
 923         raise Exception('You can only use --retry if you also rebase')
 924
 925 testbase = "%s/b%u" % (options.testbase, os.getpid())
 926 test_master = "%s/master" % testbase
 927 test_prefix = "%s/prefix" % testbase
 928 test_tmpdir = "%s/tmp" % testbase
 929 os.environ['TMPDIR'] = test_tmpdir
 930
 931 # get the top commit message, for emails
 932 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 933 top_commit_msg = top_commit_msg.decode('utf-8', 'backslashreplace')
 934
 935 try:
 936     os.makedirs(testbase)
 937 except Exception as reason:
 938     raise Exception("Unable to create %s : %s" % (testbase, reason))
 939 cleanup_list.append(testbase)
 940
 941 if options.daemon:
 942     logfile = os.path.join(testbase, "log")
 943     do_print("Forking into the background, writing progress to %s" % logfile)
 944     daemonize(logfile)
 945
 946 write_pidfile(gitroot + "/autobuild.pid")
 947
 948 start_time = time.time()
 949
 950 while True:
 951     try:
 952         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 953         os.makedirs(test_tmpdir)
 954         # The waf uninstall code removes empty directories all the way
 955         # up the tree.  Creating a file in test_tmpdir stops it from
 956         # being removed.
 957         run_cmd("touch %s" % os.path.join(test_tmpdir,
 958                                           ".directory-is-not-empty"), show=True)
 959         run_cmd("stat %s" % test_tmpdir, show=True)
 960         run_cmd("stat %s" % testbase, show=True)
 961         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 962     except Exception:
 963         cleanup()
 964         raise
 965
 966     try:
 967         try:
 968             if options.rebase is not None:
 969                 rebase_tree(options.rebase, rebase_branch=options.branch)
 970         except Exception:
 971             cleanup_list.append(gitroot + "/autobuild.pid")
 972             cleanup()
 973             elapsed_time = time.time() - start_time
 974             email_failure(-1, 'rebase', 'rebase', 'rebase',
 975                           'rebase on %s failed' % options.branch,
 976                           elapsed_time, log_base=options.log_base)
 977             sys.exit(1)
 978         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
 979         if options.tail:
 980             blist.start_tail()
 981         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
 982         if status != 0 or errstr != "retry":
 983             break
 984         cleanup()
 985     except Exception:
 986         cleanup()
 987         raise
 988
 989 cleanup_list.append(gitroot + "/autobuild.pid")
 990
 991 do_print(errstr)
 992
 993 blist.kill_kids()
 994 if options.tail:
 995     do_print("waiting for tail to flush")
 996     time.sleep(1)
 997
 998 elapsed_time = time.time() - start_time
 999 if status == 0:
1000     if options.passcmd is not None:
1001         do_print("Running passcmd: %s" % options.passcmd)
1002         run_cmd(options.passcmd, dir=test_master)
1003     if options.pushto is not None:
1004         push_to(options.pushto, push_branch=options.branch)
1005     if options.keeplogs or options.attach_logs:
1006         blist.tarlogs("logs.tar.gz")
1007         do_print("Logs in logs.tar.gz")
1008     if options.always_email:
1009         email_success(elapsed_time, log_base=options.log_base)
1010     blist.remove_logs()
1011     cleanup()
1012     do_print(errstr)
1013     sys.exit(0)
1014
1015 # something failed, gather a tar of the logs
1016 blist.tarlogs("logs.tar.gz")
1017
1018 if options.email is not None:
1019     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1020                   elapsed_time, log_base=options.log_base)
1021 else:
1022     elapsed_minutes = elapsed_time / 60.0
1023     print('''
1024
1025 ####################################################################
1026
1027 AUTOBUILD FAILURE
1028
1029 Your autobuild[%s] on %s failed after %.1f minutes
1030 when trying to test %s with the following error:
1031
1032    %s
1033
1034 the autobuild has been abandoned. Please fix the error and resubmit.
1035
1036 ####################################################################
1037
1038 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1039
1040 cleanup()
1041 do_print(errstr)
1042 do_print("Logs in logs.tar.gz")
1043 sys.exit(status)