script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34 cleanup_list = []
  35
  36 builddirs = {
  37     "ctdb": "ctdb",
  38     "samba": ".",
  39     "samba-nt4": ".",
  40     "samba-fileserver": ".",
  41     "samba-ad-member": ".",
  42     "samba-xc": ".",
  43     "samba-o3": ".",
  44     "samba-ctdb": ".",
  45     "samba-libs": ".",
  46     "samba-static": ".",
  47     "samba-none-env": ".",
  48     "samba-ad-dc-1": ".",
  49     "samba-ad-dc-2": ".",
  50     "samba-ad-dc-3": ".",
  51     "samba-ad-dc-4": ".",
  52     "samba-ad-dc-5": ".",
  53     "samba-ad-dc-6": ".",
  54     "samba-ad-dc-ntvfs": ".",
  55     "samba-ad-dc-backup": ".",
  56     "samba-systemkrb5": ".",
  57     "samba-nopython": ".",
  58     "samba-nopython-py2": ".",
  59     "ldb": "lib/ldb",
  60     "tdb": "lib/tdb",
  61     "talloc": "lib/talloc",
  62     "replace": "lib/replace",
  63     "tevent": "lib/tevent",
  64     "pidl": "pidl"
  65 }
  66
  67 defaulttasks = builddirs.keys()
  68
  69 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  70     defaulttasks.remove("samba-o3")
  71
  72 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  73 samba_configure_params = " --picky-developer ${PREFIX} --with-profiling-data"
  74
  75 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
  76 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  77 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  78 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
  79 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
  80 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
  81 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
  82
  83 tasks = {
  84     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  85                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  86                ("make", "make all", "text/plain"),
  87                ("install", "make install", "text/plain"),
  88                ("test", "make autotest", "text/plain"),
  89                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  90                ("clean", "make clean", "text/plain")],
  91
  92     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
  93     "samba": [
  94                 ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
  95                 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
  96                 ("make", "make -j", "text/plain"),
  97                 ("test", "make test FAIL_IMMEDIATELY=1 "
  98                  "TESTS='--exclude-env=none "
  99                  "--exclude-env=nt4_dc "
 100                  "--exclude-env=nt4_dc_schannel "
 101                  "--exclude-env=nt4_member "
 102                  "--exclude-env=ad_dc "
 103                  "--exclude-env=ad_dc_backup "
 104                  "--exclude-env=ad_dc_ntvfs "
 105                  "--exclude-env=ad_dc_default "
 106                  "--exclude-env=ad_dc_slowtests "
 107                  "--exclude-env=ad_dc_no_nss "
 108                  "--exclude-env=ad_dc_no_ntlm "
 109                  "--exclude-env=fl2003dc "
 110                  "--exclude-env=fl2008dc "
 111                  "--exclude-env=fl2008r2dc "
 112                  "--exclude-env=ad_member "
 113                  "--exclude-env=ad_member_idmap_rid "
 114                  "--exclude-env=ad_member_idmap_ad "
 115                  "--exclude-env=ad_member_rfc2307 "
 116                  "--exclude-env=chgdcpass "
 117                  "--exclude-env=vampire_2000_dc "
 118                  "--exclude-env=fl2000dc "
 119                  "--exclude-env=fileserver "
 120                  "--exclude-env=maptoguest "
 121                  "--exclude-env=simpleserver "
 122                  "--exclude-env=backupfromdc "
 123                  "--exclude-env=restoredc "
 124                  "--exclude-env=renamedc "
 125                  "--exclude-env=offlinebackupdc "
 126                  "--exclude-env=labdc "
 127                  "--exclude-env=preforkrestartdc "
 128                  "--exclude-env=proclimitdc "
 129                  "--exclude-env=promoted_dc "
 130                  "--exclude-env=vampire_dc "
 131                  "--exclude-env=rodc "
 132                  "--exclude-env=ad_dc_default "
 133                  "--exclude-env=ad_dc_slowtests "
 134                  "'",
 135                  "text/plain"),
 136                 ("install", "make install", "text/plain"),
 137                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 138                 ("clean", "make clean", "text/plain")],
 139
 140     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 141                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 142                     ("make", "make -j", "text/plain"),
 143                     ("test", "make test FAIL_IMMEDIATELY=1 "
 144                      "TESTS='"
 145                      "--include-env=nt4_dc "
 146                      "--include-env=nt4_dc_schannel "
 147                      "--include-env=nt4_member "
 148                      "'", "text/plain"),
 149                     ("install", "make install", "text/plain"),
 150                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 151                     ("clean", "make clean", "text/plain")],
 152
 153     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 154                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 155                            ("make", "make -j", "text/plain"),
 156                            ("test", "make test FAIL_IMMEDIATELY=1 "
 157                             "TESTS='"
 158                             "--include-env=fileserver "
 159                             "--include-env=maptoguest "
 160                             "--include-env=simpleserver "
 161                             "'", "text/plain"),
 162                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 163
 164     "samba-ad-member": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 165                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 166                         ("make", "make -j", "text/plain"),
 167                         ("test", "make test FAIL_IMMEDIATELY=1 "
 168                          "TESTS='"
 169                          "--include-env=ad_member "
 170                          "--include-env=ad_member_idmap_rid "
 171                          "--include-env=ad_member_idmap_ad "
 172                          "--include-env=ad_member_rfc2307 "
 173                          "'", "text/plain"),
 174                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 175
 176     "samba-ad-dc-1": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 177                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 178                       ("make", "make -j", "text/plain"),
 179                       ("test", "make test FAIL_IMMEDIATELY=1 "
 180                        "TESTS='--include-env=ad_dc "
 181                        "--include-env=ad_dc_no_nss "
 182                        "--include-env=ad_dc_no_ntlm "
 183                        "'", "text/plain"),
 184                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 185
 186     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 187                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 188                         ("make", "make -j", "text/plain"),
 189                         ("test", "make test FAIL_IMMEDIATELY=1 "
 190                          "TESTS='"
 191                          "--include-env=vampire_dc "
 192                          "--include-env=vampire_2000_dc "
 193                          "--include-env=rodc "
 194                          "'", "text/plain"),
 195                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 196
 197     "samba-ad-dc-3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 198                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 199                         ("make", "make -j", "text/plain"),
 200                         ("test", "make test FAIL_IMMEDIATELY=1 "
 201                          "TESTS='"
 202                          "--include-env=promoted_dc "
 203                          "--include-env=chgdcpass "
 204                          "--include-env=preforkrestartdc "
 205                          "--include-env=proclimitdc "
 206                          "'", "text/plain"),
 207                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 208
 209     "samba-ad-dc-4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 210                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 211                         ("make", "make -j", "text/plain"),
 212                         ("test", "make test FAIL_IMMEDIATELY=1 "
 213                          "TESTS='"
 214                          "--include-env=fl2000dc "
 215                          "--include-env=fl2003dc "
 216                          "--include-env=fl2008dc "
 217                          "--include-env=fl2008r2dc "
 218                          "'", "text/plain"),
 219                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 220
 221     "samba-ad-dc-5": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 222                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 223                         ("make", "make -j", "text/plain"),
 224                         ("test", "make test FAIL_IMMEDIATELY=1 "
 225                          "TESTS='"
 226                          "--include-env=ad_dc_default "
 227                          "'", "text/plain"),
 228                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 229
 230     "samba-ad-dc-6": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 231                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 232                         ("make", "make -j", "text/plain"),
 233                         ("test", "make test FAIL_IMMEDIATELY=1 "
 234                          "TESTS='"
 235                          "--include-env=ad_dc_slowtests "
 236                          "'", "text/plain"),
 237                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 238
 239     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 240     # This is currently the longest task, so we don't randomly delay it.
 241     "samba-ad-dc-ntvfs": [
 242                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 243                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 244                       ("make", "make -j", "text/plain"),
 245                       ("test", "make test FAIL_IMMEDIATELY=1 "
 246                        "TESTS='"
 247                        "--include-env=ad_dc_ntvfs "
 248                        "'", "text/plain"),
 249                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 250
 251     # run the backup/restore testenvs separately as they're fairly standalone
 252     # (and CI seems to max out at ~8 different DCs running at once)
 253     "samba-ad-dc-backup": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 254                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 255                         ("make", "make -j", "text/plain"),
 256                         ("test", "make test FAIL_IMMEDIATELY=1 "
 257                          "TESTS='--include-env=backupfromdc "
 258                          "--include-env=restoredc "
 259                          "--include-env=renamedc "
 260                          "--include-env=offlinebackupdc "
 261                          "--include-env=labdc "
 262                          "--include-env=ad_dc_backup "
 263                          "'", "text/plain"),
 264                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 265
 266     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 267                           ("make", "make -j", "text/plain"),
 268                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
 269
 270     # Test cross-compile infrastructure
 271     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 272                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 273                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 274                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 275                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 276                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 277                    ("compare-results", "script/compare_cc_results.py "
 278                     "./bin/c4che/default{} "
 279                     "./bin-xe/c4che/default{} "
 280                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
 281
 282     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 283     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 284                    ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 285                    ("make", "make -j", "text/plain"),
 286                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
 287                     "TESTS='--include-env=ad_dc'", "text/plain"),
 288                    ("install", "make install", "text/plain"),
 289                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 290                    ("clean", "make clean", "text/plain")],
 291
 292     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 293
 294                      # make sure we have tdb around:
 295                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 296                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 297                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 298
 299
 300                      # build samba with cluster support (also building ctdb):
 301                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 302                      ("samba-make", "make", "text/plain"),
 303                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 304                      ("samba-install", "make install", "text/plain"),
 305                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 306
 307                      # clean up:
 308                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 309                      ("clean", "make clean", "text/plain"),
 310                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
 311
 312     "samba-libs": [
 313                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 314                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 315                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 316                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 317
 318                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 319                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 320                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 321
 322                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 323                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 324                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 325
 326                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 327                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 328                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 329
 330                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 331                       ("nondevel-make", "make -j", "text/plain"),
 332                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 333                       ("nondevel-install", "make install", "text/plain"),
 334                       ("nondevel-dist", "make dist", "text/plain"),
 335
 336                       # retry with all modules shared
 337                       ("allshared-distclean", "make distclean", "text/plain"),
 338                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 339                       ("allshared-make", "make -j", "text/plain")],
 340
 341     "samba-none-env": [
 342                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 343                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 344                       ("make", "make -j", "text/plain"),
 345                       ("test", "make test "
 346                        "FAIL_IMMEDIATELY=1 "
 347                        "TESTS='--include-env=none'",
 348                        "text/plain")],
 349
 350     "samba-static": [
 351                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 352                       # build with all modules static
 353                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 354                       ("allstatic-make", "make -j", "text/plain"),
 355                       ("allstatic-test", "make test "
 356                        "FAIL_IMMEDIATELY=1 "
 357                        "TESTS='samba3.smb2.create.*nt4_dc'",
 358                        "text/plain"),
 359
 360                       # retry without any required modules
 361                       ("none-distclean", "make distclean", "text/plain"),
 362                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 363                       ("none-make", "make -j", "text/plain"),
 364
 365                       # retry with nonshared smbd and smbtorture
 366                       ("nonshared-distclean", "make distclean", "text/plain"),
 367                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 368                       ("nonshared-make", "make -j", "text/plain")],
 369
 370     "samba-systemkrb5": [
 371                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 372                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 373                       ("make", "make -j", "text/plain"),
 374                       # we currently cannot run a full make test, a limited list of tests could be run
 375                       # via "make test TESTS=sometests"
 376                       ("test", "make test FAIL_IMMEDIATELY=1 "
 377                        "TESTS='--include-env=ktest'", "text/plain"),
 378                       ("install", "make install", "text/plain"),
 379                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 380                       ("clean", "make clean", "text/plain")
 381                       ],
 382
 383     # Test Samba without python still builds.  When this test fails
 384     # due to more use of Python, the expectations is that the newly
 385     # failing part of the code should be disabled when
 386     # --disable-python is set (rather than major work being done to
 387     # support this environment).  The target here is for vendors
 388     # shipping a minimal smbd.
 389     "samba-nopython": [
 390                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 391                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 392                       ("make", "make -j", "text/plain"),
 393                       ("install", "make install", "text/plain"),
 394                       ("test", "make test-nopython", "text/plain"),
 395                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 396                       ("clean", "make clean", "text/plain"),
 397
 398                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 399                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 400                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 401
 402                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 403                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 404                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 405
 406                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 407                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 408                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 409
 410                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 411                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 412                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 413
 414                       # retry against installed library packages
 415                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 416                       ("libs-make", "make -j", "text/plain"),
 417                       ("libs-install", "make install", "text/plain"),
 418                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 419                       ("libs-clean", "make clean", "text/plain")
 420                       ],
 421
 422     # check we can do the same thing using python2
 423     "samba-nopython-py2": [
 424                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 425                       ("configure", "PYTHON=python2 ./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 426                       ("make", "PYTHON=python2 make -j", "text/plain"),
 427                       ("install", "PYTHON=python2 make install", "text/plain"),
 428                       ("test", "make test-nopython", "text/plain"),
 429                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 430                       ("clean", "PYTHON=python2 make clean", "text/plain"),
 431
 432                       ("talloc-configure", "cd lib/talloc && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 433                       ("talloc-make", "cd lib/talloc && PYTHON=python2 make", "text/plain"),
 434                       ("talloc-install", "cd lib/talloc && PYTHON=python2 make install", "text/plain"),
 435
 436                       ("tdb-configure", "cd lib/tdb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 437                       ("tdb-make", "cd lib/tdb && PYTHON=python2 make", "text/plain"),
 438                       ("tdb-install", "cd lib/tdb && PYTHON=python2 make install", "text/plain"),
 439
 440                       ("tevent-configure", "cd lib/tevent && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 441                       ("tevent-make", "cd lib/tevent && PYTHON=python2 make", "text/plain"),
 442                       ("tevent-install", "cd lib/tevent && PYTHON=python2 make install", "text/plain"),
 443
 444                       ("ldb-configure", "cd lib/ldb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 445                       ("ldb-make", "cd lib/ldb && PYTHON=python2 make", "text/plain"),
 446                       ("ldb-install", "cd lib/ldb && PYTHON=python2 make install", "text/plain"),
 447
 448                       # retry against installed library packages
 449                       ("libs-configure", "PYTHON=python2 " + samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 450                       ("libs-make", "PYTHON=python2 make -j", "text/plain"),
 451                       ("libs-install", "PYTHON=python2 make install", "text/plain"),
 452                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 453                       ("libs-clean", "PYTHON=python2 make clean", "text/plain")
 454                       ],
 455
 456     "ldb": [
 457               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 458               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 459               ("make", "make", "text/plain"),
 460               ("install", "make install", "text/plain"),
 461               ("test", "make test", "text/plain"),
 462               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX}", "text/plain"),
 463               ("make-no-lmdb", "make", "text/plain"),
 464               ("install-no-lmdb", "make install", "text/plain"),
 465               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 466               ("distcheck", "make distcheck", "text/plain"),
 467               ("clean", "make clean", "text/plain")],
 468
 469     "tdb": [
 470               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 471               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 472               ("make", "make", "text/plain"),
 473               ("install", "make install", "text/plain"),
 474               ("test", "make test", "text/plain"),
 475               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 476               ("distcheck", "make distcheck", "text/plain"),
 477               ("clean", "make clean", "text/plain")],
 478
 479     "talloc": [
 480                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 481                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 482                  ("make", "make", "text/plain"),
 483                  ("install", "make install", "text/plain"),
 484                  ("test", "make test", "text/plain"),
 485                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 486                  ("distcheck", "make distcheck", "text/plain"),
 487                  ("clean", "make clean", "text/plain")],
 488
 489     "replace": [
 490                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 491                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 492                   ("make", "make", "text/plain"),
 493                   ("install", "make install", "text/plain"),
 494                   ("test", "make test", "text/plain"),
 495                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 496                   ("distcheck", "make distcheck", "text/plain"),
 497                   ("clean", "make clean", "text/plain")],
 498
 499     "tevent": [
 500                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 501                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 502                  ("make", "make", "text/plain"),
 503                  ("install", "make install", "text/plain"),
 504                  ("test", "make test", "text/plain"),
 505                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 506                  ("distcheck", "make distcheck", "text/plain"),
 507                  ("clean", "make clean", "text/plain")],
 508
 509     "pidl": [
 510         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 511         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 512         ("touch", "touch *.yp", "text/plain"),
 513         ("make", "make", "text/plain"),
 514         ("test", "make test", "text/plain"),
 515         ("install", "make install", "text/plain"),
 516         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 517         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 518         ("clean", "make clean", "text/plain")],
 519
 520
 521     # these are useful for debugging autobuild
 522     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
 523     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
 524
 525
 526 }
 527
 528
 529 def do_print(msg):
 530     print("%s" % msg)
 531     sys.stdout.flush()
 532     sys.stderr.flush()
 533
 534
 535 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 536     if show is None:
 537         show = options.verbose
 538     if show:
 539         do_print("Running: '%s' in '%s'" % (cmd, dir))
 540     if output:
 541         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
 542     elif checkfail:
 543         return check_call(cmd, shell=True, cwd=dir)
 544     else:
 545         return call(cmd, shell=True, cwd=dir)
 546
 547
 548 class builder(object):
 549     '''handle build of one directory'''
 550
 551     def __init__(self, name, sequence, cp=True):
 552         self.name = name
 553         if name in builddirs:
 554             self.dir = builddirs[name]
 555         else:
 556             self.dir = "."
 557
 558         self.tag = self.name.replace('/', '_')
 559         self.sequence = sequence
 560         self.next = 0
 561         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 562         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 563         if options.verbose:
 564             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 565             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 566         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 567         self.stdout = open(self.stdout_path, 'w')
 568         self.stderr = open(self.stderr_path, 'w')
 569         self.stdin  = open("/dev/null", 'r')
 570         self.sdir = "%s/%s" % (testbase, self.tag)
 571         self.prefix = "%s/%s" % (test_prefix, self.tag)
 572         run_cmd("rm -rf %s" % self.sdir)
 573         run_cmd("rm -rf %s" % self.prefix)
 574         if cp:
 575             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 576         else:
 577             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 578         self.start_next()
 579
 580     def start_next(self):
 581         if self.next == len(self.sequence):
 582             if not options.nocleanup:
 583                 run_cmd("rm -rf %s" % self.sdir)
 584                 run_cmd("rm -rf %s" % self.prefix)
 585             do_print('%s: Completed OK' % self.name)
 586             self.done = True
 587             return
 588         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 589         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
 590         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 591         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 592         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 593 #        if self.output_mime_type == "text/x-subunit":
 594 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 595         cwd = os.getcwd()
 596         os.chdir("%s/%s" % (self.sdir, self.dir))
 597         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
 598         self.proc = Popen(self.cmd, shell=True,
 599                           close_fds=True,
 600                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 601         os.chdir(cwd)
 602         self.next += 1
 603
 604
 605 class buildlist(object):
 606     '''handle build of multiple directories'''
 607
 608     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 609         global tasks
 610         self.tlist = []
 611         self.tail_proc = None
 612         self.retry = None
 613         if tasknames == []:
 614             if options.restrict_tests:
 615                 tasknames = ["samba-test-only"]
 616             else:
 617                 tasknames = defaulttasks
 618         else:
 619             # If we are only running one test,
 620             # do not sleep randomly to wait for it to start
 621             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 622
 623         for n in tasknames:
 624             b = builder(n, tasks[n], cp=n is not "pidl")
 625             self.tlist.append(b)
 626         if options.retry:
 627             rebase_remote = "rebaseon"
 628             retry_task = [("retry",
 629                             '''set -e
 630                             git remote add -t %s %s %s
 631                             git fetch %s
 632                             while :; do
 633                               sleep 60
 634                               git describe %s/%s > old_remote_branch.desc
 635                               git fetch %s
 636                               git describe %s/%s > remote_branch.desc
 637                               diff old_remote_branch.desc remote_branch.desc
 638                             done
 639                            ''' % (
 640                                 rebase_branch, rebase_remote, rebase_url,
 641                                rebase_remote,
 642                                rebase_remote, rebase_branch,
 643                                rebase_remote,
 644                                rebase_remote, rebase_branch
 645                             ),
 646                             "test/plain")]
 647
 648             self.retry = builder('retry', retry_task, cp=False)
 649             self.need_retry = False
 650
 651     def kill_kids(self):
 652         if self.tail_proc is not None:
 653             self.tail_proc.terminate()
 654             self.tail_proc.wait()
 655             self.tail_proc = None
 656         if self.retry is not None:
 657             self.retry.proc.terminate()
 658             self.retry.proc.wait()
 659             self.retry = None
 660         for b in self.tlist:
 661             if b.proc is not None:
 662                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 663                 b.proc.terminate()
 664                 b.proc.wait()
 665                 b.proc = None
 666
 667     def wait_one(self):
 668         while True:
 669             none_running = True
 670             for b in self.tlist:
 671                 if b.proc is None:
 672                     continue
 673                 none_running = False
 674                 b.status = b.proc.poll()
 675                 if b.status is None:
 676                     continue
 677                 b.proc = None
 678                 return b
 679             if options.retry:
 680                 ret = self.retry.proc.poll()
 681                 if ret is not None:
 682                     self.need_retry = True
 683                     self.retry = None
 684                     return None
 685             if none_running:
 686                 return None
 687             time.sleep(0.1)
 688
 689     def run(self):
 690         while True:
 691             b = self.wait_one()
 692             if options.retry and self.need_retry:
 693                 self.kill_kids()
 694                 do_print("retry needed")
 695                 return (0, None, None, None, "retry")
 696             if b is None:
 697                 break
 698             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 699                 self.kill_kids()
 700                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 701             b.start_next()
 702         self.kill_kids()
 703         return (0, None, None, None, "All OK")
 704
 705     def write_system_info(self):
 706         filename = 'system-info.txt'
 707         f = open(filename, 'w')
 708         for cmd in ['uname -a',
 709                     'lsb_release -a',
 710                     'free',
 711                     'mount',
 712                     'cat /proc/cpuinfo',
 713                     'cc --version',
 714                     'df -m .',
 715                     'df -m %s' % testbase]:
 716             out = run_cmd(cmd, output=True, checkfail=False)
 717             print('### %s' % cmd, file=f)
 718             print(out.decode('utf8', 'backslashreplace'), file=f)
 719             print(file=f)
 720         f.close()
 721         return filename
 722
 723     def tarlogs(self, fname):
 724         tar = tarfile.open(fname, "w:gz")
 725         for b in self.tlist:
 726             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 727             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 728         if os.path.exists("autobuild.log"):
 729             tar.add("autobuild.log")
 730         sys_info = self.write_system_info()
 731         tar.add(sys_info)
 732         tar.close()
 733
 734     def remove_logs(self):
 735         for b in self.tlist:
 736             os.unlink(b.stdout_path)
 737             os.unlink(b.stderr_path)
 738
 739     def start_tail(self):
 740         cmd = ["tail", "-f"]
 741         for b in self.tlist:
 742             cmd.append(b.stdout_path)
 743             cmd.append(b.stderr_path)
 744         self.tail_proc = Popen(cmd, close_fds=True)
 745
 746
 747 def cleanup():
 748     if options.nocleanup:
 749         return
 750     run_cmd("stat %s || true" % test_tmpdir, show=True)
 751     run_cmd("stat %s" % testbase, show=True)
 752     do_print("Cleaning up %r" % cleanup_list)
 753     for d in cleanup_list:
 754         run_cmd("rm -rf %s" % d)
 755
 756
 757 def find_git_root():
 758     '''get to the top of the git repo'''
 759     p = os.getcwd()
 760     while p != '/':
 761         if os.path.isdir(os.path.join(p, ".git")):
 762             return p
 763         p = os.path.abspath(os.path.join(p, '..'))
 764     return None
 765
 766
 767 def daemonize(logfile):
 768     pid = os.fork()
 769     if pid == 0:  # Parent
 770         os.setsid()
 771         pid = os.fork()
 772         if pid != 0:  # Actual daemon
 773             os._exit(0)
 774     else:  # Grandparent
 775         os._exit(0)
 776
 777     import resource      # Resource usage information.
 778     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 779     if maxfd == resource.RLIM_INFINITY:
 780         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 781     for fd in range(0, maxfd):
 782         try:
 783             os.close(fd)
 784         except OSError:
 785             pass
 786     os.open(logfile, os.O_RDWR | os.O_CREAT)
 787     os.dup2(0, 1)
 788     os.dup2(0, 2)
 789
 790
 791 def write_pidfile(fname):
 792     '''write a pid file, cleanup on exit'''
 793     f = open(fname, mode='w')
 794     f.write("%u\n" % os.getpid())
 795     f.close()
 796
 797
 798 def rebase_tree(rebase_url, rebase_branch="master"):
 799     rebase_remote = "rebaseon"
 800     do_print("Rebasing on %s" % rebase_url)
 801     run_cmd("git describe HEAD", show=True, dir=test_master)
 802     run_cmd("git remote add -t %s %s %s" %
 803             (rebase_branch, rebase_remote, rebase_url),
 804             show=True, dir=test_master)
 805     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 806     if options.fix_whitespace:
 807         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 808                 (rebase_remote, rebase_branch),
 809                 show=True, dir=test_master)
 810     else:
 811         run_cmd("git rebase --force-rebase %s/%s" %
 812                 (rebase_remote, rebase_branch),
 813                 show=True, dir=test_master)
 814     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 815                    (rebase_remote, rebase_branch),
 816                    dir=test_master, output=True)
 817     if diff == '':
 818         do_print("No differences between HEAD and %s/%s - exiting" %
 819                  (rebase_remote, rebase_branch))
 820         sys.exit(0)
 821     run_cmd("git describe %s/%s" %
 822             (rebase_remote, rebase_branch),
 823             show=True, dir=test_master)
 824     run_cmd("git describe HEAD", show=True, dir=test_master)
 825     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 826             (rebase_remote, rebase_branch),
 827             show=True, dir=test_master)
 828
 829
 830 def push_to(push_url, push_branch="master"):
 831     push_remote = "pushto"
 832     do_print("Pushing to %s" % push_url)
 833     if options.mark:
 834         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 835         run_cmd("git commit --amend -c HEAD", dir=test_master)
 836         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 837         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 838     run_cmd("git remote add -t %s %s %s" %
 839             (push_branch, push_remote, push_url),
 840             show=True, dir=test_master)
 841     run_cmd("git push %s +HEAD:%s" %
 842             (push_remote, push_branch),
 843             show=True, dir=test_master)
 844
 845
 846 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 847
 848 gitroot = find_git_root()
 849 if gitroot is None:
 850     raise Exception("Failed to find git root")
 851
 852 parser = OptionParser()
 853 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 854 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 855 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 856 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 857                   default=def_testbase)
 858 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 859 parser.add_option("", "--verbose", help="show all commands as they are run",
 860                   default=False, action="store_true")
 861 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 862                   default=None, type='str')
 863 parser.add_option("", "--pushto", help="push to a git url on success",
 864                   default=None, type='str')
 865 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 866                   default=False, action="store_true")
 867 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 868                   default=False, action="store_true")
 869 parser.add_option("", "--retry", help="automatically retry if master changes",
 870                   default=False, action="store_true")
 871 parser.add_option("", "--email", help="send email to the given address on failure",
 872                   type='str', default=None)
 873 parser.add_option("", "--email-from", help="send email from the given address",
 874                   type='str', default="autobuild@samba.org")
 875 parser.add_option("", "--email-server", help="send email via the given server",
 876                   type='str', default='localhost')
 877 parser.add_option("", "--always-email", help="always send email, even on success",
 878                   action="store_true")
 879 parser.add_option("", "--daemon", help="daemonize after initial setup",
 880                   action="store_true")
 881 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 882                   default="master", type='str')
 883 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 884                   default=gitroot, type='str')
 885 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 886                   default=False, action="store_true")
 887 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 888                   default='')
 889
 890
 891 def send_email(subject, text, log_tar):
 892     if options.email is None:
 893         do_print("not sending email because the recipient is not set")
 894         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
 895                  (subject, text))
 896         return
 897     outer = MIMEMultipart()
 898     outer['Subject'] = subject
 899     outer['To'] = options.email
 900     outer['From'] = options.email_from
 901     outer['Date'] = email.utils.formatdate(localtime=True)
 902     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 903     outer.attach(MIMEText(text, 'plain'))
 904     if options.attach_logs:
 905         fp = open(log_tar, 'rb')
 906         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 907         fp.close()
 908         # Set the filename parameter
 909         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 910         outer.attach(msg)
 911     content = outer.as_string()
 912     s = smtplib.SMTP(options.email_server)
 913     s.sendmail(options.email_from, [options.email], content)
 914     s.set_debuglevel(1)
 915     s.quit()
 916
 917
 918 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 919                   elapsed_time, log_base=None, add_log_tail=True):
 920     '''send an email to options.email about the failure'''
 921     elapsed_minutes = elapsed_time / 60.0
 922     if log_base is None:
 923         log_base = gitroot
 924     text = '''
 925 Dear Developer,
 926
 927 Your autobuild on %s failed after %.1f minutes
 928 when trying to test %s with the following error:
 929
 930    %s
 931
 932 the autobuild has been abandoned. Please fix the error and resubmit.
 933
 934 A summary of the autobuild process is here:
 935
 936   %s/autobuild.log
 937 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 938
 939     if options.restrict_tests:
 940         text += """
 941 The build was restricted to tests matching %s\n""" % options.restrict_tests
 942
 943     if failed_task != 'rebase':
 944         text += '''
 945 You can see logs of the failed task here:
 946
 947   %s/%s.stdout
 948   %s/%s.stderr
 949
 950 or you can get full logs of all tasks in this job here:
 951
 952   %s/logs.tar.gz
 953
 954 The top commit for the tree that was built was:
 955
 956 %s
 957
 958 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 959
 960     if add_log_tail:
 961         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 962         lines = f.readlines()
 963         log_tail = "".join(lines[-50:])
 964         num_lines = len(lines)
 965         if num_lines < 50:
 966             # Also include stderr (compile failures) if < 50 lines of stdout
 967             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 968             log_tail += "".join(f.readlines()[-(50 - num_lines):])
 969
 970         text += '''
 971 The last 50 lines of log messages:
 972
 973 %s
 974     ''' % log_tail
 975         f.close()
 976
 977     logs = os.path.join(gitroot, 'logs.tar.gz')
 978     send_email('autobuild[%s] failure on %s for task %s during %s'
 979                % (options.branch, platform.node(), failed_task, failed_stage),
 980                text, logs)
 981
 982
 983 def email_success(elapsed_time, log_base=None):
 984     '''send an email to options.email about a successful build'''
 985     if log_base is None:
 986         log_base = gitroot
 987     text = '''
 988 Dear Developer,
 989
 990 Your autobuild on %s has succeeded after %.1f minutes.
 991
 992 ''' % (platform.node(), elapsed_time / 60.)
 993
 994     if options.restrict_tests:
 995         text += """
 996 The build was restricted to tests matching %s\n""" % options.restrict_tests
 997
 998     if options.keeplogs:
 999         text += '''
1000
1001 you can get full logs of all tasks in this job here:
1002
1003   %s/logs.tar.gz
1004
1005 ''' % log_base
1006
1007     text += '''
1008 The top commit for the tree that was built was:
1009
1010 %s
1011 ''' % top_commit_msg
1012
1013     logs = os.path.join(gitroot, 'logs.tar.gz')
1014     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1015                text, logs)
1016
1017
1018 (options, args) = parser.parse_args()
1019
1020 if options.retry:
1021     if options.rebase is None:
1022         raise Exception('You can only use --retry if you also rebase')
1023
1024 testbase = "%s/b%u" % (options.testbase, os.getpid())
1025 test_master = "%s/master" % testbase
1026 test_prefix = "%s/prefix" % testbase
1027 test_tmpdir = "%s/tmp" % testbase
1028 os.environ['TMPDIR'] = test_tmpdir
1029
1030 # get the top commit message, for emails
1031 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1032 top_commit_msg = top_commit_msg.decode('utf-8', 'backslashreplace')
1033
1034 try:
1035     os.makedirs(testbase)
1036 except Exception as reason:
1037     raise Exception("Unable to create %s : %s" % (testbase, reason))
1038 cleanup_list.append(testbase)
1039
1040 if options.daemon:
1041     logfile = os.path.join(testbase, "log")
1042     do_print("Forking into the background, writing progress to %s" % logfile)
1043     daemonize(logfile)
1044
1045 write_pidfile(gitroot + "/autobuild.pid")
1046
1047 start_time = time.time()
1048
1049 while True:
1050     try:
1051         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1052         os.makedirs(test_tmpdir)
1053         # The waf uninstall code removes empty directories all the way
1054         # up the tree.  Creating a file in test_tmpdir stops it from
1055         # being removed.
1056         run_cmd("touch %s" % os.path.join(test_tmpdir,
1057                                           ".directory-is-not-empty"), show=True)
1058         run_cmd("stat %s" % test_tmpdir, show=True)
1059         run_cmd("stat %s" % testbase, show=True)
1060         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1061     except Exception:
1062         cleanup()
1063         raise
1064
1065     try:
1066         try:
1067             if options.rebase is not None:
1068                 rebase_tree(options.rebase, rebase_branch=options.branch)
1069         except Exception:
1070             cleanup_list.append(gitroot + "/autobuild.pid")
1071             cleanup()
1072             elapsed_time = time.time() - start_time
1073             email_failure(-1, 'rebase', 'rebase', 'rebase',
1074                           'rebase on %s failed' % options.branch,
1075                           elapsed_time, log_base=options.log_base)
1076             sys.exit(1)
1077         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1078         if options.tail:
1079             blist.start_tail()
1080         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1081         if status != 0 or errstr != "retry":
1082             break
1083         cleanup()
1084     except Exception:
1085         cleanup()
1086         raise
1087
1088 cleanup_list.append(gitroot + "/autobuild.pid")
1089
1090 do_print(errstr)
1091
1092 blist.kill_kids()
1093 if options.tail:
1094     do_print("waiting for tail to flush")
1095     time.sleep(1)
1096
1097 elapsed_time = time.time() - start_time
1098 if status == 0:
1099     if options.passcmd is not None:
1100         do_print("Running passcmd: %s" % options.passcmd)
1101         run_cmd(options.passcmd, dir=test_master)
1102     if options.pushto is not None:
1103         push_to(options.pushto, push_branch=options.branch)
1104     if options.keeplogs or options.attach_logs:
1105         blist.tarlogs("logs.tar.gz")
1106         do_print("Logs in logs.tar.gz")
1107     if options.always_email:
1108         email_success(elapsed_time, log_base=options.log_base)
1109     blist.remove_logs()
1110     cleanup()
1111     do_print(errstr)
1112     sys.exit(0)
1113
1114 # something failed, gather a tar of the logs
1115 blist.tarlogs("logs.tar.gz")
1116
1117 if options.email is not None:
1118     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1119                   elapsed_time, log_base=options.log_base)
1120 else:
1121     elapsed_minutes = elapsed_time / 60.0
1122     print('''
1123
1124 ####################################################################
1125
1126 AUTOBUILD FAILURE
1127
1128 Your autobuild[%s] on %s failed after %.1f minutes
1129 when trying to test %s with the following error:
1130
1131    %s
1132
1133 the autobuild has been abandoned. Please fix the error and resubmit.
1134
1135 ####################################################################
1136
1137 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1138
1139 cleanup()
1140 do_print(errstr)
1141 do_print("Logs in logs.tar.gz")
1142 sys.exit(status)