script/autobuild.py

   1 #!/usr/bin/env python
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34 cleanup_list = []
  35
  36 builddirs = {
  37     "ctdb": "ctdb",
  38     "samba": ".",
  39     "samba-py3": ".",
  40     "samba-nt4": ".",
  41     "samba-nt4-py3": ".",
  42     "samba-fileserver": ".",
  43     "samba-xc": ".",
  44     "samba-o3": ".",
  45     "samba-ctdb": ".",
  46     "samba-libs": ".",
  47     "samba-libs-py3": ".",
  48     "samba-static": ".",
  49     "samba-none-env": ".",
  50     "samba-none-env-py3": ".",
  51     "samba-ad-dc": ".",
  52     "samba-ad-dc-py3": ".",
  53     "samba-ad-dc-2": ".",
  54     "samba-ad-dc-2-py3": ".",
  55     "samba-systemkrb5": ".",
  56     "samba-nopython": ".",
  57     "ldb": "lib/ldb",
  58     "tdb": "lib/tdb",
  59     "talloc": "lib/talloc",
  60     "replace": "lib/replace",
  61     "tevent": "lib/tevent",
  62     "pidl": "pidl"
  63 }
  64
  65 defaulttasks = builddirs.keys()
  66
  67 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  68     defaulttasks.remove("samba-o3")
  69
  70 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  71 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
  72
  73 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH"
  74 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  75 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  76 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
  77 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE ${EXTRA_PYTHON}"
  78 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
  79 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs + " ${EXTRA_PYTHON}"
  80
  81 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
  82     extra_python = ""
  83 else:
  84     extra_python = "--extra-python=/usr/bin/python3"
  85
  86 tasks = {
  87     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  88                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  89                ("make", "make all", "text/plain"),
  90                ("install", "make install", "text/plain"),
  91                ("test", "make autotest", "text/plain"),
  92                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  93                ("clean", "make clean", "text/plain")],
  94
  95     # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)'
  96     "samba": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
  97                 ("make", "make -j", "text/plain"),
  98                 ("test", "make test FAIL_IMMEDIATELY=1 "
  99                  "TESTS='${PY3_ONLY}"
 100                  "--exclude-env=none "
 101                  "--exclude-env=nt4_dc "
 102                  "--exclude-env=nt4_member "
 103                  "--exclude-env=ad_dc "
 104                  "--exclude-env=ad_dc_no_nss "
 105                  "--exclude-env=fl2003dc "
 106                  "--exclude-env=fl2008r2dc "
 107                  "--exclude-env=ad_member "
 108                  "--exclude-env=ad_member_idmap_rid "
 109                  "--exclude-env=ad_member_idmap_ad "
 110                  "--exclude-env=chgdcpass "
 111                  "--exclude-env=vampire_2000_dc "
 112                  "--exclude-env=fl2000dc "
 113                  "--exclude-env=fileserver "
 114                  "--exclude-env=backupfromdc "
 115                  "--exclude-env=restoredc "
 116                  "--exclude-env=renamedc "
 117                  "--exclude-env=offlinebackupdc "
 118                  "--exclude-env=labdc "
 119                  "'",
 120                  "text/plain"),
 121                 ("install", "make install", "text/plain"),
 122                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 123                 ("clean", "make clean", "text/plain")],
 124
 125     # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long)
 126     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 127                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 128                     ("make", "make -j", "text/plain"),
 129                     ("test", "make test FAIL_IMMEDIATELY=1 "
 130                      "TESTS='${PY3_ONLY}"
 131                      "--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 132                     ("install", "make install", "text/plain"),
 133                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 134                     ("clean", "make clean", "text/plain")],
 135
 136     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 137     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 138                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json-audit --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 139                            ("make", "make -j", "text/plain"),
 140                            ("test", "make test FAIL_IMMEDIATELY=1 "
 141                             "TESTS='${PY3_ONLY}"
 142                             "--include-env=fileserver'", "text/plain"),
 143                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 144
 145     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 146     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 147                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 148                       ("make", "make -j", "text/plain"),
 149                       ("test", "make test FAIL_IMMEDIATELY=1 "
 150                        "TESTS='${PY3_ONLY}"
 151                        "--include-env=ad_dc "
 152                        "--include-env=fl2003dc "
 153                        "--include-env=fl2008r2dc "
 154                        "--include-env=ad_member "
 155                        "--include-env=ad_member_idmap_rid "
 156                        "--include-env=ad_member_idmap_ad'", "text/plain"),
 157                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 158
 159     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 160     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 161                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 162                         ("make", "make -j", "text/plain"),
 163                         ("test", "make test FAIL_IMMEDIATELY=1 "
 164                          "TESTS='${PY3_ONLY}"
 165                          "--include-env=chgdcpass "
 166                          "--include-env=vampire_2000_dc "
 167                          "--include-env=fl2000dc "
 168                          "--include-env=ad_dc_no_nss "
 169                          "--include-env=backupfromdc "
 170                          "--include-env=restoredc "
 171                          "--include-env=renamedc "
 172                          "--include-env=offlinebackupdc "
 173                          "--include-env=labdc "
 174                          "'",
 175                          "text/plain"),
 176                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 177
 178     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 179                           ("make", "make -j", "text/plain"),
 180                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
 181
 182     # Test cross-compile infrastructure
 183     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 184                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 185                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 186                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 187                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 188                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 189                    ("compare-results", "script/compare_cc_results.py "
 190                     "./bin/c4che/default{} "
 191                     "./bin-xe/c4che/default{} "
 192                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
 193
 194     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 195     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 196                    ("configure", "ADDITIONAL_CFLAGS='-O3' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 197                    ("make", "make -j", "text/plain"),
 198                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
 199                     "TESTS='${PY3_ONLY}"
 200                     "--include-env=ad_dc'", "text/plain"),
 201                    ("install", "make install", "text/plain"),
 202                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 203                    ("clean", "make clean", "text/plain")],
 204
 205     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 206
 207                      # make sure we have tdb around:
 208                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 209                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 210                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 211
 212
 213                      # build samba with cluster support (also building ctdb):
 214                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 215                      ("samba-make", "make", "text/plain"),
 216                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 217                      ("samba-install", "make install", "text/plain"),
 218                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 219
 220                      # clean up:
 221                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 222                      ("clean", "make clean", "text/plain"),
 223                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
 224
 225     "samba-libs": [
 226                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 227                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 228                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 229                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 230
 231                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 232                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 233                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 234
 235                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 236                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 237                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 238
 239                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 240                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 241                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 242
 243                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 244                       ("nondevel-make", "make -j", "text/plain"),
 245                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 246                       ("nondevel-install", "make install", "text/plain"),
 247                       ("nondevel-dist", "make dist", "text/plain"),
 248
 249                       # retry with all modules shared
 250                       ("allshared-distclean", "make distclean", "text/plain"),
 251                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 252                       ("allshared-make", "make -j", "text/plain")],
 253
 254     "samba-none-env": [
 255                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 256                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 257                       ("make", "make -j", "text/plain"),
 258                       ("test", "make test "
 259                        "FAIL_IMMEDIATELY=1 "
 260                        "TESTS='${PY3_ONLY}"
 261                        "--include-env=none'",
 262                        "text/plain")],
 263
 264     "samba-static": [
 265                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 266                       # build with all modules static
 267                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 268                       ("allstatic-make", "make -j", "text/plain"),
 269                       ("allstatic-test", "make test "
 270                        "FAIL_IMMEDIATELY=1 "
 271                        "TESTS='samba3.smb2.create.*nt4_dc'",
 272                        "text/plain"),
 273
 274                       # retry without any required modules
 275                       ("none-distclean", "make distclean", "text/plain"),
 276                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 277                       ("none-make", "make -j", "text/plain"),
 278
 279                       # retry with nonshared smbd and smbtorture
 280                       ("nonshared-distclean", "make distclean", "text/plain"),
 281                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 282                       ("nonshared-make", "make -j", "text/plain")],
 283
 284     "samba-systemkrb5": [
 285                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 286                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 287                       ("make", "make -j", "text/plain"),
 288                       # we currently cannot run a full make test, a limited list of tests could be run
 289                       # via "make test TESTS=sometests"
 290                       ("test", "make test FAIL_IMMEDIATELY=1 "
 291                        "TESTS='${PY3_ONLY}"
 292                        "--include-env=ktest'", "text/plain"),
 293                       ("install", "make install", "text/plain"),
 294                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 295                       ("clean", "make clean", "text/plain")
 296                       ],
 297
 298     # Test Samba without python still builds.  When this test fails
 299     # due to more use of Python, the expectations is that the newly
 300     # failing part of the code should be disabled when
 301     # --disable-python is set (rather than major work being done to
 302     # support this environment).  The target here is for vendors
 303     # shipping a minimal smbd.
 304     "samba-nopython": [
 305                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 306                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 307                       ("make", "make -j", "text/plain"),
 308                       ("install", "make install", "text/plain"),
 309                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 310                       ("clean", "make clean", "text/plain"),
 311
 312                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 313                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 314                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 315
 316                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 317                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 318                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 319
 320                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 321                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 322                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 323
 324                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 325                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 326                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 327
 328                       # retry against installed library packages
 329                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 330                       ("libs-make", "make -j", "text/plain"),
 331                       ("libs-install", "make install", "text/plain"),
 332                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 333                       ("libs-clean", "make clean", "text/plain")
 334                       ],
 335
 336
 337
 338     "ldb": [
 339               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 340               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 341               ("make", "make", "text/plain"),
 342               ("install", "make install", "text/plain"),
 343               ("test", "make test", "text/plain"),
 344               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 345               ("make-no-lmdb", "make", "text/plain"),
 346               ("install-no-lmdb", "make install", "text/plain"),
 347               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 348               ("distcheck", "make distcheck", "text/plain"),
 349               ("clean", "make clean", "text/plain")],
 350
 351     "tdb": [
 352               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 353               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 354               ("make", "make", "text/plain"),
 355               ("install", "make install", "text/plain"),
 356               ("test", "make test", "text/plain"),
 357               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 358               ("distcheck", "make distcheck", "text/plain"),
 359               ("clean", "make clean", "text/plain")],
 360
 361     "talloc": [
 362                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 363                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 364                  ("make", "make", "text/plain"),
 365                  ("install", "make install", "text/plain"),
 366                  ("test", "make test", "text/plain"),
 367                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 368                  ("distcheck", "make distcheck", "text/plain"),
 369                  ("clean", "make clean", "text/plain")],
 370
 371     "replace": [
 372                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 373                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 374                   ("make", "make", "text/plain"),
 375                   ("install", "make install", "text/plain"),
 376                   ("test", "make test", "text/plain"),
 377                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 378                   ("distcheck", "make distcheck", "text/plain"),
 379                   ("clean", "make clean", "text/plain")],
 380
 381     "tevent": [
 382                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 383                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 384                  ("make", "make", "text/plain"),
 385                  ("install", "make install", "text/plain"),
 386                  ("test", "make test", "text/plain"),
 387                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 388                  ("distcheck", "make distcheck", "text/plain"),
 389                  ("clean", "make clean", "text/plain")],
 390
 391     "pidl": [
 392         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 393         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 394         ("touch", "touch *.yp", "text/plain"),
 395         ("make", "make", "text/plain"),
 396         ("test", "make test", "text/plain"),
 397         ("install", "make install", "text/plain"),
 398         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 399         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 400         ("clean", "make clean", "text/plain")],
 401
 402     # these are useful for debugging autobuild
 403     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
 404     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
 405 }
 406
 407
 408 def do_print(msg):
 409     print("%s" % msg)
 410     sys.stdout.flush()
 411     sys.stderr.flush()
 412
 413
 414 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 415     if show is None:
 416         show = options.verbose
 417     if show:
 418         do_print("Running: '%s' in '%s'" % (cmd, dir))
 419     if output:
 420         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
 421     elif checkfail:
 422         return check_call(cmd, shell=True, cwd=dir)
 423     else:
 424         return call(cmd, shell=True, cwd=dir)
 425
 426
 427 class builder(object):
 428     '''handle build of one directory'''
 429
 430     def __init__(self, name, sequence, cp=True, py3=False):
 431         self.name = name
 432         self.py3 = py3
 433         if name in builddirs:
 434             self.dir = builddirs[name]
 435         else:
 436             self.dir = "."
 437
 438         self.tag = self.name.replace('/', '_')
 439         self.sequence = sequence
 440         self.next = 0
 441         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 442         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 443         if options.verbose:
 444             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 445             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 446         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 447         self.stdout = open(self.stdout_path, 'w')
 448         self.stderr = open(self.stderr_path, 'w')
 449         self.stdin  = open("/dev/null", 'r')
 450         self.sdir = "%s/%s" % (testbase, self.tag)
 451         self.prefix = "%s/%s" % (test_prefix, self.tag)
 452         run_cmd("rm -rf %s" % self.sdir)
 453         run_cmd("rm -rf %s" % self.prefix)
 454         if cp:
 455             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 456         else:
 457             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 458         self.start_next()
 459
 460     def start_next(self):
 461         if self.next == len(self.sequence):
 462             if not options.nocleanup:
 463                 run_cmd("rm -rf %s" % self.sdir)
 464                 run_cmd("rm -rf %s" % self.prefix)
 465             do_print('%s: Completed OK' % self.name)
 466             self.done = True
 467             return
 468         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 469         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix))
 470         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 471         if self.py3:
 472             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
 473             # The trailing space is important
 474             self.cmd = self.cmd.replace("${PY3_ONLY}", "python3 ")
 475         else:
 476             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "")
 477             self.cmd = self.cmd.replace("${PY3_ONLY}", "")
 478         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 479         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 480 #        if self.output_mime_type == "text/x-subunit":
 481 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 482         cwd = os.getcwd()
 483         os.chdir("%s/%s" % (self.sdir, self.dir))
 484         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
 485         self.proc = Popen(self.cmd, shell=True,
 486                           close_fds=True,
 487                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 488         os.chdir(cwd)
 489         self.next += 1
 490
 491
 492 class buildlist(object):
 493     '''handle build of multiple directories'''
 494
 495     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 496         global tasks
 497         self.tlist = []
 498         self.tail_proc = None
 499         self.retry = None
 500         if tasknames == []:
 501             if options.restrict_tests:
 502                 tasknames = ["samba-test-only"]
 503             else:
 504                 tasknames = defaulttasks
 505         else:
 506             # If we are only running one test,
 507             # do not sleep randomly to wait for it to start
 508             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 509
 510         for n in tasknames:
 511             if n not in tasks and n.endswith("-py3"):
 512                 b = builder(n,
 513                             tasks[n[:-4]],
 514                             cp=n is not "pidl",
 515                             py3=True)
 516             else:
 517                 b = builder(n, tasks[n], cp=n is not "pidl")
 518             self.tlist.append(b)
 519         if options.retry:
 520             rebase_remote = "rebaseon"
 521             retry_task = [("retry",
 522                             '''set -e
 523                             git remote add -t %s %s %s
 524                             git fetch %s
 525                             while :; do
 526                               sleep 60
 527                               git describe %s/%s > old_remote_branch.desc
 528                               git fetch %s
 529                               git describe %s/%s > remote_branch.desc
 530                               diff old_remote_branch.desc remote_branch.desc
 531                             done
 532                            ''' % (
 533                                 rebase_branch, rebase_remote, rebase_url,
 534                                rebase_remote,
 535                                rebase_remote, rebase_branch,
 536                                rebase_remote,
 537                                rebase_remote, rebase_branch
 538                             ),
 539                             "test/plain")]
 540
 541             self.retry = builder('retry', retry_task, cp=False)
 542             self.need_retry = False
 543
 544     def kill_kids(self):
 545         if self.tail_proc is not None:
 546             self.tail_proc.terminate()
 547             self.tail_proc.wait()
 548             self.tail_proc = None
 549         if self.retry is not None:
 550             self.retry.proc.terminate()
 551             self.retry.proc.wait()
 552             self.retry = None
 553         for b in self.tlist:
 554             if b.proc is not None:
 555                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 556                 b.proc.terminate()
 557                 b.proc.wait()
 558                 b.proc = None
 559
 560     def wait_one(self):
 561         while True:
 562             none_running = True
 563             for b in self.tlist:
 564                 if b.proc is None:
 565                     continue
 566                 none_running = False
 567                 b.status = b.proc.poll()
 568                 if b.status is None:
 569                     continue
 570                 b.proc = None
 571                 return b
 572             if options.retry:
 573                 ret = self.retry.proc.poll()
 574                 if ret is not None:
 575                     self.need_retry = True
 576                     self.retry = None
 577                     return None
 578             if none_running:
 579                 return None
 580             time.sleep(0.1)
 581
 582     def run(self):
 583         while True:
 584             b = self.wait_one()
 585             if options.retry and self.need_retry:
 586                 self.kill_kids()
 587                 do_print("retry needed")
 588                 return (0, None, None, None, "retry")
 589             if b is None:
 590                 break
 591             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 592                 self.kill_kids()
 593                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 594             b.start_next()
 595         self.kill_kids()
 596         return (0, None, None, None, "All OK")
 597
 598     def write_system_info(self):
 599         filename = 'system-info.txt'
 600         f = open(filename, 'w')
 601         for cmd in ['uname -a', 'free', 'cat /proc/cpuinfo',
 602                     'cc --version', 'df -m .', 'df -m %s' % testbase]:
 603             print('### %s' % cmd, file=f)
 604             print(run_cmd(cmd, output=True, checkfail=False), file=f)
 605             print(file=f)
 606         f.close()
 607         return filename
 608
 609     def tarlogs(self, fname):
 610         tar = tarfile.open(fname, "w:gz")
 611         for b in self.tlist:
 612             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 613             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 614         if os.path.exists("autobuild.log"):
 615             tar.add("autobuild.log")
 616         sys_info = self.write_system_info()
 617         tar.add(sys_info)
 618         tar.close()
 619
 620     def remove_logs(self):
 621         for b in self.tlist:
 622             os.unlink(b.stdout_path)
 623             os.unlink(b.stderr_path)
 624
 625     def start_tail(self):
 626         cmd = ["tail", "-f"]
 627         for b in self.tlist:
 628             cmd.append(b.stdout_path)
 629             cmd.append(b.stderr_path)
 630         self.tail_proc = Popen(cmd, close_fds=True)
 631
 632
 633 def cleanup():
 634     if options.nocleanup:
 635         return
 636     run_cmd("stat %s || true" % test_tmpdir, show=True)
 637     run_cmd("stat %s" % testbase, show=True)
 638     do_print("Cleaning up %r" % cleanup_list)
 639     for d in cleanup_list:
 640         run_cmd("rm -rf %s" % d)
 641
 642
 643 def find_git_root():
 644     '''get to the top of the git repo'''
 645     p = os.getcwd()
 646     while p != '/':
 647         if os.path.isdir(os.path.join(p, ".git")):
 648             return p
 649         p = os.path.abspath(os.path.join(p, '..'))
 650     return None
 651
 652
 653 def daemonize(logfile):
 654     pid = os.fork()
 655     if pid == 0:  # Parent
 656         os.setsid()
 657         pid = os.fork()
 658         if pid != 0:  # Actual daemon
 659             os._exit(0)
 660     else:  # Grandparent
 661         os._exit(0)
 662
 663     import resource      # Resource usage information.
 664     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 665     if maxfd == resource.RLIM_INFINITY:
 666         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 667     for fd in range(0, maxfd):
 668         try:
 669             os.close(fd)
 670         except OSError:
 671             pass
 672     os.open(logfile, os.O_RDWR | os.O_CREAT)
 673     os.dup2(0, 1)
 674     os.dup2(0, 2)
 675
 676
 677 def write_pidfile(fname):
 678     '''write a pid file, cleanup on exit'''
 679     f = open(fname, mode='w')
 680     f.write("%u\n" % os.getpid())
 681     f.close()
 682
 683
 684 def rebase_tree(rebase_url, rebase_branch="master"):
 685     rebase_remote = "rebaseon"
 686     do_print("Rebasing on %s" % rebase_url)
 687     run_cmd("git describe HEAD", show=True, dir=test_master)
 688     run_cmd("git remote add -t %s %s %s" %
 689             (rebase_branch, rebase_remote, rebase_url),
 690             show=True, dir=test_master)
 691     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 692     if options.fix_whitespace:
 693         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 694                 (rebase_remote, rebase_branch),
 695                 show=True, dir=test_master)
 696     else:
 697         run_cmd("git rebase --force-rebase %s/%s" %
 698                 (rebase_remote, rebase_branch),
 699                 show=True, dir=test_master)
 700     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 701                    (rebase_remote, rebase_branch),
 702                    dir=test_master, output=True)
 703     if diff == '':
 704         do_print("No differences between HEAD and %s/%s - exiting" %
 705                  (rebase_remote, rebase_branch))
 706         sys.exit(0)
 707     run_cmd("git describe %s/%s" %
 708             (rebase_remote, rebase_branch),
 709             show=True, dir=test_master)
 710     run_cmd("git describe HEAD", show=True, dir=test_master)
 711     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 712             (rebase_remote, rebase_branch),
 713             show=True, dir=test_master)
 714
 715
 716 def push_to(push_url, push_branch="master"):
 717     push_remote = "pushto"
 718     do_print("Pushing to %s" % push_url)
 719     if options.mark:
 720         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 721         run_cmd("git commit --amend -c HEAD", dir=test_master)
 722         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 723         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 724     run_cmd("git remote add -t %s %s %s" %
 725             (push_branch, push_remote, push_url),
 726             show=True, dir=test_master)
 727     run_cmd("git push %s +HEAD:%s" %
 728             (push_remote, push_branch),
 729             show=True, dir=test_master)
 730
 731
 732 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 733
 734 gitroot = find_git_root()
 735 if gitroot is None:
 736     raise Exception("Failed to find git root")
 737
 738 parser = OptionParser()
 739 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 740 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 741 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 742 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 743                   default=def_testbase)
 744 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 745 parser.add_option("", "--verbose", help="show all commands as they are run",
 746                   default=False, action="store_true")
 747 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 748                   default=None, type='str')
 749 parser.add_option("", "--pushto", help="push to a git url on success",
 750                   default=None, type='str')
 751 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 752                   default=False, action="store_true")
 753 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 754                   default=False, action="store_true")
 755 parser.add_option("", "--retry", help="automatically retry if master changes",
 756                   default=False, action="store_true")
 757 parser.add_option("", "--email", help="send email to the given address on failure",
 758                   type='str', default=None)
 759 parser.add_option("", "--email-from", help="send email from the given address",
 760                   type='str', default="autobuild@samba.org")
 761 parser.add_option("", "--email-server", help="send email via the given server",
 762                   type='str', default='localhost')
 763 parser.add_option("", "--always-email", help="always send email, even on success",
 764                   action="store_true")
 765 parser.add_option("", "--daemon", help="daemonize after initial setup",
 766                   action="store_true")
 767 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 768                   default="master", type='str')
 769 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 770                   default=gitroot, type='str')
 771 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 772                   default=False, action="store_true")
 773 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 774                   default='')
 775
 776
 777 def send_email(subject, text, log_tar):
 778     if options.email is None:
 779         do_print("not sending email because the recipient is not set")
 780         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
 781                  (subject, text))
 782         return
 783     outer = MIMEMultipart()
 784     outer['Subject'] = subject
 785     outer['To'] = options.email
 786     outer['From'] = options.email_from
 787     outer['Date'] = email.utils.formatdate(localtime=True)
 788     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 789     outer.attach(MIMEText(text, 'plain'))
 790     if options.attach_logs:
 791         fp = open(log_tar, 'rb')
 792         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 793         fp.close()
 794         # Set the filename parameter
 795         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 796         outer.attach(msg)
 797     content = outer.as_string()
 798     s = smtplib.SMTP(options.email_server)
 799     s.sendmail(options.email_from, [options.email], content)
 800     s.set_debuglevel(1)
 801     s.quit()
 802
 803
 804 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 805                   elapsed_time, log_base=None, add_log_tail=True):
 806     '''send an email to options.email about the failure'''
 807     elapsed_minutes = elapsed_time / 60.0
 808     user = os.getenv("USER")
 809     if log_base is None:
 810         log_base = gitroot
 811     text = '''
 812 Dear Developer,
 813
 814 Your autobuild on %s failed after %.1f minutes
 815 when trying to test %s with the following error:
 816
 817    %s
 818
 819 the autobuild has been abandoned. Please fix the error and resubmit.
 820
 821 A summary of the autobuild process is here:
 822
 823   %s/autobuild.log
 824 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 825
 826     if options.restrict_tests:
 827         text += """
 828 The build was restricted to tests matching %s\n""" % options.restrict_tests
 829
 830     if failed_task != 'rebase':
 831         text += '''
 832 You can see logs of the failed task here:
 833
 834   %s/%s.stdout
 835   %s/%s.stderr
 836
 837 or you can get full logs of all tasks in this job here:
 838
 839   %s/logs.tar.gz
 840
 841 The top commit for the tree that was built was:
 842
 843 %s
 844
 845 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 846
 847     if add_log_tail:
 848         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 849         lines = f.readlines()
 850         log_tail = "".join(lines[-50:])
 851         num_lines = len(lines)
 852         if num_lines < 50:
 853             # Also include stderr (compile failures) if < 50 lines of stdout
 854             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 855             log_tail += "".join(f.readlines()[-(50 - num_lines):])
 856
 857         text += '''
 858 The last 50 lines of log messages:
 859
 860 %s
 861     ''' % log_tail
 862         f.close()
 863
 864     logs = os.path.join(gitroot, 'logs.tar.gz')
 865     send_email('autobuild[%s] failure on %s for task %s during %s'
 866                % (options.branch, platform.node(), failed_task, failed_stage),
 867                text, logs)
 868
 869
 870 def email_success(elapsed_time, log_base=None):
 871     '''send an email to options.email about a successful build'''
 872     user = os.getenv("USER")
 873     if log_base is None:
 874         log_base = gitroot
 875     text = '''
 876 Dear Developer,
 877
 878 Your autobuild on %s has succeeded after %.1f minutes.
 879
 880 ''' % (platform.node(), elapsed_time / 60.)
 881
 882     if options.restrict_tests:
 883         text += """
 884 The build was restricted to tests matching %s\n""" % options.restrict_tests
 885
 886     if options.keeplogs:
 887         text += '''
 888
 889 you can get full logs of all tasks in this job here:
 890
 891   %s/logs.tar.gz
 892
 893 ''' % log_base
 894
 895     text += '''
 896 The top commit for the tree that was built was:
 897
 898 %s
 899 ''' % top_commit_msg
 900
 901     logs = os.path.join(gitroot, 'logs.tar.gz')
 902     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 903                text, logs)
 904
 905
 906 (options, args) = parser.parse_args()
 907
 908 if options.retry:
 909     if options.rebase is None:
 910         raise Exception('You can only use --retry if you also rebase')
 911
 912 testbase = "%s/b%u" % (options.testbase, os.getpid())
 913 test_master = "%s/master" % testbase
 914 test_prefix = "%s/prefix" % testbase
 915 test_tmpdir = "%s/tmp" % testbase
 916 os.environ['TMPDIR'] = test_tmpdir
 917
 918 # get the top commit message, for emails
 919 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 920
 921 try:
 922     os.makedirs(testbase)
 923 except Exception as reason:
 924     raise Exception("Unable to create %s : %s" % (testbase, reason))
 925 cleanup_list.append(testbase)
 926
 927 if options.daemon:
 928     logfile = os.path.join(testbase, "log")
 929     do_print("Forking into the background, writing progress to %s" % logfile)
 930     daemonize(logfile)
 931
 932 write_pidfile(gitroot + "/autobuild.pid")
 933
 934 start_time = time.time()
 935
 936 while True:
 937     try:
 938         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 939         os.makedirs(test_tmpdir)
 940         # The waf uninstall code removes empty directories all the way
 941         # up the tree.  Creating a file in test_tmpdir stops it from
 942         # being removed.
 943         run_cmd("touch %s" % os.path.join(test_tmpdir,
 944                                           ".directory-is-not-empty"), show=True)
 945         run_cmd("stat %s" % test_tmpdir, show=True)
 946         run_cmd("stat %s" % testbase, show=True)
 947         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 948     except Exception:
 949         cleanup()
 950         raise
 951
 952     try:
 953         try:
 954             if options.rebase is not None:
 955                 rebase_tree(options.rebase, rebase_branch=options.branch)
 956         except Exception:
 957             cleanup_list.append(gitroot + "/autobuild.pid")
 958             cleanup()
 959             elapsed_time = time.time() - start_time
 960             email_failure(-1, 'rebase', 'rebase', 'rebase',
 961                           'rebase on %s failed' % options.branch,
 962                           elapsed_time, log_base=options.log_base)
 963             sys.exit(1)
 964         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
 965         if options.tail:
 966             blist.start_tail()
 967         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
 968         if status != 0 or errstr != "retry":
 969             break
 970         cleanup()
 971     except Exception:
 972         cleanup()
 973         raise
 974
 975 cleanup_list.append(gitroot + "/autobuild.pid")
 976
 977 do_print(errstr)
 978
 979 blist.kill_kids()
 980 if options.tail:
 981     do_print("waiting for tail to flush")
 982     time.sleep(1)
 983
 984 elapsed_time = time.time() - start_time
 985 if status == 0:
 986     if options.passcmd is not None:
 987         do_print("Running passcmd: %s" % options.passcmd)
 988         run_cmd(options.passcmd, dir=test_master)
 989     if options.pushto is not None:
 990         push_to(options.pushto, push_branch=options.branch)
 991     if options.keeplogs or options.attach_logs:
 992         blist.tarlogs("logs.tar.gz")
 993         do_print("Logs in logs.tar.gz")
 994     if options.always_email:
 995         email_success(elapsed_time, log_base=options.log_base)
 996     blist.remove_logs()
 997     cleanup()
 998     do_print(errstr)
 999     sys.exit(0)
1000
1001 # something failed, gather a tar of the logs
1002 blist.tarlogs("logs.tar.gz")
1003
1004 if options.email is not None:
1005     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1006                   elapsed_time, log_base=options.log_base)
1007 else:
1008     elapsed_minutes = elapsed_time / 60.0
1009     print('''
1010
1011 ####################################################################
1012
1013 AUTOBUILD FAILURE
1014
1015 Your autobuild[%s] on %s failed after %.1f minutes
1016 when trying to test %s with the following error:
1017
1018    %s
1019
1020 the autobuild has been abandoned. Please fix the error and resubmit.
1021
1022 ####################################################################
1023
1024 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1025
1026 cleanup()
1027 do_print(errstr)
1028 do_print("Logs in logs.tar.gz")
1029 sys.exit(status)