script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34 cleanup_list = []
  35
  36 builddirs = {
  37     "ctdb": "ctdb",
  38     "samba": ".",
  39     "samba-nt4": ".",
  40     "samba-fileserver": ".",
  41     "samba-xc": ".",
  42     "samba-o3": ".",
  43     "samba-ctdb": ".",
  44     "samba-libs": ".",
  45     "samba-static": ".",
  46     "samba-none-env": ".",
  47     "samba-ad-dc": ".",
  48     "samba-ad-dc-ntvfs": ".",
  49     "samba-ad-dc-2": ".",
  50     "samba-ad-dc-backup": ".",
  51     "samba-systemkrb5": ".",
  52     "samba-nopython": ".",
  53     "samba-nopython-py2": ".",
  54     "ldb": "lib/ldb",
  55     "tdb": "lib/tdb",
  56     "talloc": "lib/talloc",
  57     "replace": "lib/replace",
  58     "tevent": "lib/tevent",
  59     "pidl": "pidl"
  60 }
  61
  62 defaulttasks = builddirs.keys()
  63
  64 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  65     defaulttasks.remove("samba-o3")
  66
  67 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  68 samba_configure_params = " --picky-developer ${PREFIX} --with-profiling-data"
  69
  70 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
  71 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  72 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  73 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
  74 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
  75 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
  76 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
  77
  78 tasks = {
  79     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  80                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  81                ("make", "make all", "text/plain"),
  82                ("install", "make install", "text/plain"),
  83                ("test", "make autotest", "text/plain"),
  84                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  85                ("clean", "make clean", "text/plain")],
  86
  87     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
  88     "samba": [
  89                 ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
  90                 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
  91                 ("make", "make -j", "text/plain"),
  92                 ("test", "make test FAIL_IMMEDIATELY=1 "
  93                  "TESTS='--exclude-env=none "
  94                  "--exclude-env=nt4_dc "
  95                  "--exclude-env=nt4_dc_schannel "
  96                  "--exclude-env=nt4_member "
  97                  "--exclude-env=ad_dc "
  98                  "--exclude-env=ad_dc_backup "
  99                  "--exclude-env=ad_dc_ntvfs "
 100                  "--exclude-env=ad_dc_default "
 101                  "--exclude-env=ad_dc_slowtests "
 102                  "--exclude-env=ad_dc_no_nss "
 103                  "--exclude-env=fl2003dc "
 104                  "--exclude-env=fl2008dc "
 105                  "--exclude-env=fl2008r2dc "
 106                  "--exclude-env=ad_member "
 107                  "--exclude-env=ad_member_idmap_rid "
 108                  "--exclude-env=ad_member_idmap_ad "
 109                  "--exclude-env=chgdcpass "
 110                  "--exclude-env=vampire_2000_dc "
 111                  "--exclude-env=fl2000dc "
 112                  "--exclude-env=fileserver "
 113                  "--exclude-env=backupfromdc "
 114                  "--exclude-env=restoredc "
 115                  "--exclude-env=renamedc "
 116                  "--exclude-env=offlinebackupdc "
 117                  "--exclude-env=labdc "
 118                  "'",
 119                  "text/plain"),
 120                 ("install", "make install", "text/plain"),
 121                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 122                 ("clean", "make clean", "text/plain")],
 123
 124     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 125                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 126                     ("make", "make -j", "text/plain"),
 127                     ("test", "make test FAIL_IMMEDIATELY=1 "
 128                      "TESTS='"
 129                      "--include-env=nt4_dc "
 130                      "--include-env=nt4_dc_schannel "
 131                      "--include-env=nt4_member "
 132                      "'", "text/plain"),
 133                     ("install", "make install", "text/plain"),
 134                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 135                     ("clean", "make clean", "text/plain")],
 136
 137     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 138                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 139                            ("make", "make -j", "text/plain"),
 140                            ("test", "make test FAIL_IMMEDIATELY=1 "
 141                             "TESTS='--include-env=fileserver'", "text/plain"),
 142                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 143
 144     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 145                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 146                       ("make", "make -j", "text/plain"),
 147                       ("test", "make test FAIL_IMMEDIATELY=1 "
 148                        "TESTS='--include-env=ad_dc "
 149                        "--include-env=ad_dc_backup "
 150                        "--include-env=fl2003dc "
 151                        "--include-env=fl2008r2dc "
 152                        "--include-env=ad_member "
 153                        "--include-env=ad_member_idmap_rid "
 154                        "--include-env=ad_member_idmap_ad'", "text/plain"),
 155                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 156
 157     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 158                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 159                         ("make", "make -j", "text/plain"),
 160                         ("test", "make test FAIL_IMMEDIATELY=1 "
 161                          "TESTS='--include-env=chgdcpass "
 162                          "--include-env=vampire_2000_dc "
 163                          "--include-env=fl2000dc "
 164                          "--include-env=ad_dc_no_nss "
 165                          "'",
 166                          "text/plain"),
 167                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 168
 169     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 170     # This is currently the longest task, so we don't randomly delay it.
 171     "samba-ad-dc-ntvfs": [
 172                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 173                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 174                       ("make", "make -j", "text/plain"),
 175                       ("test", "make test FAIL_IMMEDIATELY=1 "
 176                        "TESTS='"
 177                        "--include-env=ad_dc_ntvfs "
 178                        "--include-env=fl2008dc "
 179                        "--include-env=ad_dc_default "
 180                        "--include-env=ad_dc_slowtests "
 181                        "'", "text/plain"),
 182                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 183
 184     # run the backup/restore testenvs separately as they're fairly standalone
 185     # (and CI seems to max out at ~8 different DCs running at once)
 186     "samba-ad-dc-backup": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 187                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 188                         ("make", "make -j", "text/plain"),
 189                         ("test", "make test FAIL_IMMEDIATELY=1 "
 190                          "TESTS='--include-env=backupfromdc "
 191                          "--include-env=restoredc "
 192                          "--include-env=renamedc "
 193                          "--include-env=offlinebackupdc "
 194                          "--include-env=labdc "
 195                          "'",
 196                          "text/plain"),
 197                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 198
 199     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 200                           ("make", "make -j", "text/plain"),
 201                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
 202
 203     # Test cross-compile infrastructure
 204     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 205                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 206                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 207                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 208                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 209                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 210                    ("compare-results", "script/compare_cc_results.py "
 211                     "./bin/c4che/default{} "
 212                     "./bin-xe/c4che/default{} "
 213                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
 214
 215     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 216     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 217                    ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 218                    ("make", "make -j", "text/plain"),
 219                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
 220                     "TESTS='--include-env=ad_dc'", "text/plain"),
 221                    ("install", "make install", "text/plain"),
 222                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 223                    ("clean", "make clean", "text/plain")],
 224
 225     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 226
 227                      # make sure we have tdb around:
 228                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 229                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 230                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 231
 232
 233                      # build samba with cluster support (also building ctdb):
 234                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 235                      ("samba-make", "make", "text/plain"),
 236                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 237                      ("samba-install", "make install", "text/plain"),
 238                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 239
 240                      # clean up:
 241                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 242                      ("clean", "make clean", "text/plain"),
 243                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
 244
 245     "samba-libs": [
 246                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 247                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 248                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 249                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 250
 251                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 252                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 253                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 254
 255                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 256                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 257                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 258
 259                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 260                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 261                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 262
 263                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 264                       ("nondevel-make", "make -j", "text/plain"),
 265                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 266                       ("nondevel-install", "make install", "text/plain"),
 267                       ("nondevel-dist", "make dist", "text/plain"),
 268
 269                       # retry with all modules shared
 270                       ("allshared-distclean", "make distclean", "text/plain"),
 271                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 272                       ("allshared-make", "make -j", "text/plain")],
 273
 274     "samba-none-env": [
 275                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 276                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 277                       ("make", "make -j", "text/plain"),
 278                       ("test", "make test "
 279                        "FAIL_IMMEDIATELY=1 "
 280                        "TESTS='--include-env=none'",
 281                        "text/plain")],
 282
 283     "samba-static": [
 284                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 285                       # build with all modules static
 286                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 287                       ("allstatic-make", "make -j", "text/plain"),
 288                       ("allstatic-test", "make test "
 289                        "FAIL_IMMEDIATELY=1 "
 290                        "TESTS='samba3.smb2.create.*nt4_dc'",
 291                        "text/plain"),
 292
 293                       # retry without any required modules
 294                       ("none-distclean", "make distclean", "text/plain"),
 295                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 296                       ("none-make", "make -j", "text/plain"),
 297
 298                       # retry with nonshared smbd and smbtorture
 299                       ("nonshared-distclean", "make distclean", "text/plain"),
 300                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 301                       ("nonshared-make", "make -j", "text/plain")],
 302
 303     "samba-systemkrb5": [
 304                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 305                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 306                       ("make", "make -j", "text/plain"),
 307                       # we currently cannot run a full make test, a limited list of tests could be run
 308                       # via "make test TESTS=sometests"
 309                       ("test", "make test FAIL_IMMEDIATELY=1 "
 310                        "TESTS='--include-env=ktest'", "text/plain"),
 311                       ("install", "make install", "text/plain"),
 312                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 313                       ("clean", "make clean", "text/plain")
 314                       ],
 315
 316     # Test Samba without python still builds.  When this test fails
 317     # due to more use of Python, the expectations is that the newly
 318     # failing part of the code should be disabled when
 319     # --disable-python is set (rather than major work being done to
 320     # support this environment).  The target here is for vendors
 321     # shipping a minimal smbd.
 322     "samba-nopython": [
 323                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 324                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 325                       ("make", "make -j", "text/plain"),
 326                       ("install", "make install", "text/plain"),
 327                       ("test", "make test-nopython", "text/plain"),
 328                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 329                       ("clean", "make clean", "text/plain"),
 330
 331                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 332                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 333                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 334
 335                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 336                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 337                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 338
 339                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 340                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 341                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 342
 343                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 344                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 345                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 346
 347                       # retry against installed library packages
 348                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 349                       ("libs-make", "make -j", "text/plain"),
 350                       ("libs-install", "make install", "text/plain"),
 351                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 352                       ("libs-clean", "make clean", "text/plain")
 353                       ],
 354
 355     # check we can do the same thing using python2
 356     "samba-nopython-py2": [
 357                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 358                       ("configure", "PYTHON=python2 ./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 359                       ("make", "PYTHON=python2 make -j", "text/plain"),
 360                       ("install", "PYTHON=python2 make install", "text/plain"),
 361                       ("test", "make test-nopython", "text/plain"),
 362                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 363                       ("clean", "PYTHON=python2 make clean", "text/plain"),
 364
 365                       ("talloc-configure", "cd lib/talloc && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 366                       ("talloc-make", "cd lib/talloc && PYTHON=python2 make", "text/plain"),
 367                       ("talloc-install", "cd lib/talloc && PYTHON=python2 make install", "text/plain"),
 368
 369                       ("tdb-configure", "cd lib/tdb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 370                       ("tdb-make", "cd lib/tdb && PYTHON=python2 make", "text/plain"),
 371                       ("tdb-install", "cd lib/tdb && PYTHON=python2 make install", "text/plain"),
 372
 373                       ("tevent-configure", "cd lib/tevent && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 374                       ("tevent-make", "cd lib/tevent && PYTHON=python2 make", "text/plain"),
 375                       ("tevent-install", "cd lib/tevent && PYTHON=python2 make install", "text/plain"),
 376
 377                       ("ldb-configure", "cd lib/ldb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 378                       ("ldb-make", "cd lib/ldb && PYTHON=python2 make", "text/plain"),
 379                       ("ldb-install", "cd lib/ldb && PYTHON=python2 make install", "text/plain"),
 380
 381                       # retry against installed library packages
 382                       ("libs-configure", "PYTHON=python2 " + samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 383                       ("libs-make", "PYTHON=python2 make -j", "text/plain"),
 384                       ("libs-install", "PYTHON=python2 make install", "text/plain"),
 385                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 386                       ("libs-clean", "PYTHON=python2 make clean", "text/plain")
 387                       ],
 388
 389     "ldb": [
 390               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 391               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 392               ("make", "make", "text/plain"),
 393               ("install", "make install", "text/plain"),
 394               ("test", "make test", "text/plain"),
 395               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX}", "text/plain"),
 396               ("make-no-lmdb", "make", "text/plain"),
 397               ("install-no-lmdb", "make install", "text/plain"),
 398               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 399               ("distcheck", "make distcheck", "text/plain"),
 400               ("clean", "make clean", "text/plain")],
 401
 402     "tdb": [
 403               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 404               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 405               ("make", "make", "text/plain"),
 406               ("install", "make install", "text/plain"),
 407               ("test", "make test", "text/plain"),
 408               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 409               ("distcheck", "make distcheck", "text/plain"),
 410               ("clean", "make clean", "text/plain")],
 411
 412     "talloc": [
 413                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 414                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 415                  ("make", "make", "text/plain"),
 416                  ("install", "make install", "text/plain"),
 417                  ("test", "make test", "text/plain"),
 418                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 419                  ("distcheck", "make distcheck", "text/plain"),
 420                  ("clean", "make clean", "text/plain")],
 421
 422     "replace": [
 423                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 424                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 425                   ("make", "make", "text/plain"),
 426                   ("install", "make install", "text/plain"),
 427                   ("test", "make test", "text/plain"),
 428                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 429                   ("distcheck", "make distcheck", "text/plain"),
 430                   ("clean", "make clean", "text/plain")],
 431
 432     "tevent": [
 433                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 434                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 435                  ("make", "make", "text/plain"),
 436                  ("install", "make install", "text/plain"),
 437                  ("test", "make test", "text/plain"),
 438                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 439                  ("distcheck", "make distcheck", "text/plain"),
 440                  ("clean", "make clean", "text/plain")],
 441
 442     "pidl": [
 443         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 444         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 445         ("touch", "touch *.yp", "text/plain"),
 446         ("make", "make", "text/plain"),
 447         ("test", "make test", "text/plain"),
 448         ("install", "make install", "text/plain"),
 449         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 450         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 451         ("clean", "make clean", "text/plain")],
 452
 453
 454     # these are useful for debugging autobuild
 455     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
 456     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
 457
 458
 459 }
 460
 461
 462 def do_print(msg):
 463     print("%s" % msg)
 464     sys.stdout.flush()
 465     sys.stderr.flush()
 466
 467
 468 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 469     if show is None:
 470         show = options.verbose
 471     if show:
 472         do_print("Running: '%s' in '%s'" % (cmd, dir))
 473     if output:
 474         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
 475     elif checkfail:
 476         return check_call(cmd, shell=True, cwd=dir)
 477     else:
 478         return call(cmd, shell=True, cwd=dir)
 479
 480
 481 class builder(object):
 482     '''handle build of one directory'''
 483
 484     def __init__(self, name, sequence, cp=True):
 485         self.name = name
 486         if name in builddirs:
 487             self.dir = builddirs[name]
 488         else:
 489             self.dir = "."
 490
 491         self.tag = self.name.replace('/', '_')
 492         self.sequence = sequence
 493         self.next = 0
 494         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 495         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 496         if options.verbose:
 497             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 498             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 499         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 500         self.stdout = open(self.stdout_path, 'w')
 501         self.stderr = open(self.stderr_path, 'w')
 502         self.stdin  = open("/dev/null", 'r')
 503         self.sdir = "%s/%s" % (testbase, self.tag)
 504         self.prefix = "%s/%s" % (test_prefix, self.tag)
 505         run_cmd("rm -rf %s" % self.sdir)
 506         run_cmd("rm -rf %s" % self.prefix)
 507         if cp:
 508             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 509         else:
 510             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 511         self.start_next()
 512
 513     def start_next(self):
 514         if self.next == len(self.sequence):
 515             if not options.nocleanup:
 516                 run_cmd("rm -rf %s" % self.sdir)
 517                 run_cmd("rm -rf %s" % self.prefix)
 518             do_print('%s: Completed OK' % self.name)
 519             self.done = True
 520             return
 521         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 522         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
 523         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 524         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 525         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 526 #        if self.output_mime_type == "text/x-subunit":
 527 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 528         cwd = os.getcwd()
 529         os.chdir("%s/%s" % (self.sdir, self.dir))
 530         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
 531         self.proc = Popen(self.cmd, shell=True,
 532                           close_fds=True,
 533                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 534         os.chdir(cwd)
 535         self.next += 1
 536
 537
 538 class buildlist(object):
 539     '''handle build of multiple directories'''
 540
 541     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 542         global tasks
 543         self.tlist = []
 544         self.tail_proc = None
 545         self.retry = None
 546         if tasknames == []:
 547             if options.restrict_tests:
 548                 tasknames = ["samba-test-only"]
 549             else:
 550                 tasknames = defaulttasks
 551         else:
 552             # If we are only running one test,
 553             # do not sleep randomly to wait for it to start
 554             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 555
 556         for n in tasknames:
 557             b = builder(n, tasks[n], cp=n is not "pidl")
 558             self.tlist.append(b)
 559         if options.retry:
 560             rebase_remote = "rebaseon"
 561             retry_task = [("retry",
 562                             '''set -e
 563                             git remote add -t %s %s %s
 564                             git fetch %s
 565                             while :; do
 566                               sleep 60
 567                               git describe %s/%s > old_remote_branch.desc
 568                               git fetch %s
 569                               git describe %s/%s > remote_branch.desc
 570                               diff old_remote_branch.desc remote_branch.desc
 571                             done
 572                            ''' % (
 573                                 rebase_branch, rebase_remote, rebase_url,
 574                                rebase_remote,
 575                                rebase_remote, rebase_branch,
 576                                rebase_remote,
 577                                rebase_remote, rebase_branch
 578                             ),
 579                             "test/plain")]
 580
 581             self.retry = builder('retry', retry_task, cp=False)
 582             self.need_retry = False
 583
 584     def kill_kids(self):
 585         if self.tail_proc is not None:
 586             self.tail_proc.terminate()
 587             self.tail_proc.wait()
 588             self.tail_proc = None
 589         if self.retry is not None:
 590             self.retry.proc.terminate()
 591             self.retry.proc.wait()
 592             self.retry = None
 593         for b in self.tlist:
 594             if b.proc is not None:
 595                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 596                 b.proc.terminate()
 597                 b.proc.wait()
 598                 b.proc = None
 599
 600     def wait_one(self):
 601         while True:
 602             none_running = True
 603             for b in self.tlist:
 604                 if b.proc is None:
 605                     continue
 606                 none_running = False
 607                 b.status = b.proc.poll()
 608                 if b.status is None:
 609                     continue
 610                 b.proc = None
 611                 return b
 612             if options.retry:
 613                 ret = self.retry.proc.poll()
 614                 if ret is not None:
 615                     self.need_retry = True
 616                     self.retry = None
 617                     return None
 618             if none_running:
 619                 return None
 620             time.sleep(0.1)
 621
 622     def run(self):
 623         while True:
 624             b = self.wait_one()
 625             if options.retry and self.need_retry:
 626                 self.kill_kids()
 627                 do_print("retry needed")
 628                 return (0, None, None, None, "retry")
 629             if b is None:
 630                 break
 631             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 632                 self.kill_kids()
 633                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 634             b.start_next()
 635         self.kill_kids()
 636         return (0, None, None, None, "All OK")
 637
 638     def write_system_info(self):
 639         filename = 'system-info.txt'
 640         f = open(filename, 'w')
 641         for cmd in ['uname -a',
 642                     'lsb_release -a',
 643                     'free',
 644                     'mount',
 645                     'cat /proc/cpuinfo',
 646                     'cc --version',
 647                     'df -m .',
 648                     'df -m %s' % testbase]:
 649             out = run_cmd(cmd, output=True, checkfail=False)
 650             print('### %s' % cmd, file=f)
 651             print(out.decode('utf8', 'backslashreplace'), file=f)
 652             print(file=f)
 653         f.close()
 654         return filename
 655
 656     def tarlogs(self, fname):
 657         tar = tarfile.open(fname, "w:gz")
 658         for b in self.tlist:
 659             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 660             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 661         if os.path.exists("autobuild.log"):
 662             tar.add("autobuild.log")
 663         sys_info = self.write_system_info()
 664         tar.add(sys_info)
 665         tar.close()
 666
 667     def remove_logs(self):
 668         for b in self.tlist:
 669             os.unlink(b.stdout_path)
 670             os.unlink(b.stderr_path)
 671
 672     def start_tail(self):
 673         cmd = ["tail", "-f"]
 674         for b in self.tlist:
 675             cmd.append(b.stdout_path)
 676             cmd.append(b.stderr_path)
 677         self.tail_proc = Popen(cmd, close_fds=True)
 678
 679
 680 def cleanup():
 681     if options.nocleanup:
 682         return
 683     run_cmd("stat %s || true" % test_tmpdir, show=True)
 684     run_cmd("stat %s" % testbase, show=True)
 685     do_print("Cleaning up %r" % cleanup_list)
 686     for d in cleanup_list:
 687         run_cmd("rm -rf %s" % d)
 688
 689
 690 def find_git_root():
 691     '''get to the top of the git repo'''
 692     p = os.getcwd()
 693     while p != '/':
 694         if os.path.isdir(os.path.join(p, ".git")):
 695             return p
 696         p = os.path.abspath(os.path.join(p, '..'))
 697     return None
 698
 699
 700 def daemonize(logfile):
 701     pid = os.fork()
 702     if pid == 0:  # Parent
 703         os.setsid()
 704         pid = os.fork()
 705         if pid != 0:  # Actual daemon
 706             os._exit(0)
 707     else:  # Grandparent
 708         os._exit(0)
 709
 710     import resource      # Resource usage information.
 711     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 712     if maxfd == resource.RLIM_INFINITY:
 713         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 714     for fd in range(0, maxfd):
 715         try:
 716             os.close(fd)
 717         except OSError:
 718             pass
 719     os.open(logfile, os.O_RDWR | os.O_CREAT)
 720     os.dup2(0, 1)
 721     os.dup2(0, 2)
 722
 723
 724 def write_pidfile(fname):
 725     '''write a pid file, cleanup on exit'''
 726     f = open(fname, mode='w')
 727     f.write("%u\n" % os.getpid())
 728     f.close()
 729
 730
 731 def rebase_tree(rebase_url, rebase_branch="master"):
 732     rebase_remote = "rebaseon"
 733     do_print("Rebasing on %s" % rebase_url)
 734     run_cmd("git describe HEAD", show=True, dir=test_master)
 735     run_cmd("git remote add -t %s %s %s" %
 736             (rebase_branch, rebase_remote, rebase_url),
 737             show=True, dir=test_master)
 738     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 739     if options.fix_whitespace:
 740         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 741                 (rebase_remote, rebase_branch),
 742                 show=True, dir=test_master)
 743     else:
 744         run_cmd("git rebase --force-rebase %s/%s" %
 745                 (rebase_remote, rebase_branch),
 746                 show=True, dir=test_master)
 747     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 748                    (rebase_remote, rebase_branch),
 749                    dir=test_master, output=True)
 750     if diff == '':
 751         do_print("No differences between HEAD and %s/%s - exiting" %
 752                  (rebase_remote, rebase_branch))
 753         sys.exit(0)
 754     run_cmd("git describe %s/%s" %
 755             (rebase_remote, rebase_branch),
 756             show=True, dir=test_master)
 757     run_cmd("git describe HEAD", show=True, dir=test_master)
 758     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 759             (rebase_remote, rebase_branch),
 760             show=True, dir=test_master)
 761
 762
 763 def push_to(push_url, push_branch="master"):
 764     push_remote = "pushto"
 765     do_print("Pushing to %s" % push_url)
 766     if options.mark:
 767         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 768         run_cmd("git commit --amend -c HEAD", dir=test_master)
 769         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 770         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 771     run_cmd("git remote add -t %s %s %s" %
 772             (push_branch, push_remote, push_url),
 773             show=True, dir=test_master)
 774     run_cmd("git push %s +HEAD:%s" %
 775             (push_remote, push_branch),
 776             show=True, dir=test_master)
 777
 778
 779 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 780
 781 gitroot = find_git_root()
 782 if gitroot is None:
 783     raise Exception("Failed to find git root")
 784
 785 parser = OptionParser()
 786 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 787 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 788 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 789 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 790                   default=def_testbase)
 791 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 792 parser.add_option("", "--verbose", help="show all commands as they are run",
 793                   default=False, action="store_true")
 794 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 795                   default=None, type='str')
 796 parser.add_option("", "--pushto", help="push to a git url on success",
 797                   default=None, type='str')
 798 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 799                   default=False, action="store_true")
 800 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 801                   default=False, action="store_true")
 802 parser.add_option("", "--retry", help="automatically retry if master changes",
 803                   default=False, action="store_true")
 804 parser.add_option("", "--email", help="send email to the given address on failure",
 805                   type='str', default=None)
 806 parser.add_option("", "--email-from", help="send email from the given address",
 807                   type='str', default="autobuild@samba.org")
 808 parser.add_option("", "--email-server", help="send email via the given server",
 809                   type='str', default='localhost')
 810 parser.add_option("", "--always-email", help="always send email, even on success",
 811                   action="store_true")
 812 parser.add_option("", "--daemon", help="daemonize after initial setup",
 813                   action="store_true")
 814 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 815                   default="master", type='str')
 816 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 817                   default=gitroot, type='str')
 818 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 819                   default=False, action="store_true")
 820 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 821                   default='')
 822
 823
 824 def send_email(subject, text, log_tar):
 825     if options.email is None:
 826         do_print("not sending email because the recipient is not set")
 827         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
 828                  (subject, text))
 829         return
 830     outer = MIMEMultipart()
 831     outer['Subject'] = subject
 832     outer['To'] = options.email
 833     outer['From'] = options.email_from
 834     outer['Date'] = email.utils.formatdate(localtime=True)
 835     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 836     outer.attach(MIMEText(text, 'plain'))
 837     if options.attach_logs:
 838         fp = open(log_tar, 'rb')
 839         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 840         fp.close()
 841         # Set the filename parameter
 842         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 843         outer.attach(msg)
 844     content = outer.as_string()
 845     s = smtplib.SMTP(options.email_server)
 846     s.sendmail(options.email_from, [options.email], content)
 847     s.set_debuglevel(1)
 848     s.quit()
 849
 850
 851 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 852                   elapsed_time, log_base=None, add_log_tail=True):
 853     '''send an email to options.email about the failure'''
 854     elapsed_minutes = elapsed_time / 60.0
 855     if log_base is None:
 856         log_base = gitroot
 857     text = '''
 858 Dear Developer,
 859
 860 Your autobuild on %s failed after %.1f minutes
 861 when trying to test %s with the following error:
 862
 863    %s
 864
 865 the autobuild has been abandoned. Please fix the error and resubmit.
 866
 867 A summary of the autobuild process is here:
 868
 869   %s/autobuild.log
 870 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 871
 872     if options.restrict_tests:
 873         text += """
 874 The build was restricted to tests matching %s\n""" % options.restrict_tests
 875
 876     if failed_task != 'rebase':
 877         text += '''
 878 You can see logs of the failed task here:
 879
 880   %s/%s.stdout
 881   %s/%s.stderr
 882
 883 or you can get full logs of all tasks in this job here:
 884
 885   %s/logs.tar.gz
 886
 887 The top commit for the tree that was built was:
 888
 889 %s
 890
 891 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 892
 893     if add_log_tail:
 894         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 895         lines = f.readlines()
 896         log_tail = "".join(lines[-50:])
 897         num_lines = len(lines)
 898         if num_lines < 50:
 899             # Also include stderr (compile failures) if < 50 lines of stdout
 900             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 901             log_tail += "".join(f.readlines()[-(50 - num_lines):])
 902
 903         text += '''
 904 The last 50 lines of log messages:
 905
 906 %s
 907     ''' % log_tail
 908         f.close()
 909
 910     logs = os.path.join(gitroot, 'logs.tar.gz')
 911     send_email('autobuild[%s] failure on %s for task %s during %s'
 912                % (options.branch, platform.node(), failed_task, failed_stage),
 913                text, logs)
 914
 915
 916 def email_success(elapsed_time, log_base=None):
 917     '''send an email to options.email about a successful build'''
 918     if log_base is None:
 919         log_base = gitroot
 920     text = '''
 921 Dear Developer,
 922
 923 Your autobuild on %s has succeeded after %.1f minutes.
 924
 925 ''' % (platform.node(), elapsed_time / 60.)
 926
 927     if options.restrict_tests:
 928         text += """
 929 The build was restricted to tests matching %s\n""" % options.restrict_tests
 930
 931     if options.keeplogs:
 932         text += '''
 933
 934 you can get full logs of all tasks in this job here:
 935
 936   %s/logs.tar.gz
 937
 938 ''' % log_base
 939
 940     text += '''
 941 The top commit for the tree that was built was:
 942
 943 %s
 944 ''' % top_commit_msg
 945
 946     logs = os.path.join(gitroot, 'logs.tar.gz')
 947     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 948                text, logs)
 949
 950
 951 (options, args) = parser.parse_args()
 952
 953 if options.retry:
 954     if options.rebase is None:
 955         raise Exception('You can only use --retry if you also rebase')
 956
 957 testbase = "%s/b%u" % (options.testbase, os.getpid())
 958 test_master = "%s/master" % testbase
 959 test_prefix = "%s/prefix" % testbase
 960 test_tmpdir = "%s/tmp" % testbase
 961 os.environ['TMPDIR'] = test_tmpdir
 962
 963 # get the top commit message, for emails
 964 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 965 top_commit_msg = top_commit_msg.decode('utf-8', 'backslashreplace')
 966
 967 try:
 968     os.makedirs(testbase)
 969 except Exception as reason:
 970     raise Exception("Unable to create %s : %s" % (testbase, reason))
 971 cleanup_list.append(testbase)
 972
 973 if options.daemon:
 974     logfile = os.path.join(testbase, "log")
 975     do_print("Forking into the background, writing progress to %s" % logfile)
 976     daemonize(logfile)
 977
 978 write_pidfile(gitroot + "/autobuild.pid")
 979
 980 start_time = time.time()
 981
 982 while True:
 983     try:
 984         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 985         os.makedirs(test_tmpdir)
 986         # The waf uninstall code removes empty directories all the way
 987         # up the tree.  Creating a file in test_tmpdir stops it from
 988         # being removed.
 989         run_cmd("touch %s" % os.path.join(test_tmpdir,
 990                                           ".directory-is-not-empty"), show=True)
 991         run_cmd("stat %s" % test_tmpdir, show=True)
 992         run_cmd("stat %s" % testbase, show=True)
 993         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 994     except Exception:
 995         cleanup()
 996         raise
 997
 998     try:
 999         try:
1000             if options.rebase is not None:
1001                 rebase_tree(options.rebase, rebase_branch=options.branch)
1002         except Exception:
1003             cleanup_list.append(gitroot + "/autobuild.pid")
1004             cleanup()
1005             elapsed_time = time.time() - start_time
1006             email_failure(-1, 'rebase', 'rebase', 'rebase',
1007                           'rebase on %s failed' % options.branch,
1008                           elapsed_time, log_base=options.log_base)
1009             sys.exit(1)
1010         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1011         if options.tail:
1012             blist.start_tail()
1013         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1014         if status != 0 or errstr != "retry":
1015             break
1016         cleanup()
1017     except Exception:
1018         cleanup()
1019         raise
1020
1021 cleanup_list.append(gitroot + "/autobuild.pid")
1022
1023 do_print(errstr)
1024
1025 blist.kill_kids()
1026 if options.tail:
1027     do_print("waiting for tail to flush")
1028     time.sleep(1)
1029
1030 elapsed_time = time.time() - start_time
1031 if status == 0:
1032     if options.passcmd is not None:
1033         do_print("Running passcmd: %s" % options.passcmd)
1034         run_cmd(options.passcmd, dir=test_master)
1035     if options.pushto is not None:
1036         push_to(options.pushto, push_branch=options.branch)
1037     if options.keeplogs or options.attach_logs:
1038         blist.tarlogs("logs.tar.gz")
1039         do_print("Logs in logs.tar.gz")
1040     if options.always_email:
1041         email_success(elapsed_time, log_base=options.log_base)
1042     blist.remove_logs()
1043     cleanup()
1044     do_print(errstr)
1045     sys.exit(0)
1046
1047 # something failed, gather a tar of the logs
1048 blist.tarlogs("logs.tar.gz")
1049
1050 if options.email is not None:
1051     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1052                   elapsed_time, log_base=options.log_base)
1053 else:
1054     elapsed_minutes = elapsed_time / 60.0
1055     print('''
1056
1057 ####################################################################
1058
1059 AUTOBUILD FAILURE
1060
1061 Your autobuild[%s] on %s failed after %.1f minutes
1062 when trying to test %s with the following error:
1063
1064    %s
1065
1066 the autobuild has been abandoned. Please fix the error and resubmit.
1067
1068 ####################################################################
1069
1070 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1071
1072 cleanup()
1073 do_print(errstr)
1074 do_print("Logs in logs.tar.gz")
1075 sys.exit(status)