script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34 cleanup_list = []
  35
  36 builddirs = {
  37     "ctdb": "ctdb",
  38     "samba": ".",
  39     "samba-nt4": ".",
  40     "samba-fileserver": ".",
  41     "samba-ad-member": ".",
  42     "samba-xc": ".",
  43     "samba-o3": ".",
  44     "samba-ctdb": ".",
  45     "samba-libs": ".",
  46     "samba-static": ".",
  47     "samba-none-env": ".",
  48     "samba-ad-dc-1": ".",
  49     "samba-ad-dc-2": ".",
  50     "samba-ad-dc-3": ".",
  51     "samba-ad-dc-4": ".",
  52     "samba-ad-dc-5": ".",
  53     "samba-ad-dc-6": ".",
  54     "samba-ad-dc-ntvfs": ".",
  55     "samba-ad-dc-backup": ".",
  56     "samba-systemkrb5": ".",
  57     "samba-nopython": ".",
  58     "samba-nopython-py2": ".",
  59     "samba-schemaupgrade": ".",
  60     "ldb": "lib/ldb",
  61     "tdb": "lib/tdb",
  62     "talloc": "lib/talloc",
  63     "replace": "lib/replace",
  64     "tevent": "lib/tevent",
  65     "pidl": "pidl"
  66 }
  67
  68 defaulttasks = builddirs.keys()
  69
  70 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  71     defaulttasks.remove("samba-o3")
  72
  73 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  74 samba_configure_params = " --picky-developer ${PREFIX} --with-profiling-data"
  75
  76 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
  77 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  78 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  79 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
  80 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
  81 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
  82 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
  83
  84 tasks = {
  85     "ctdb": [("random-sleep", "../script/random-sleep.sh 300 900", "text/plain"),
  86                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  87                ("make", "make all", "text/plain"),
  88                ("install", "make install", "text/plain"),
  89                ("test", "make autotest", "text/plain"),
  90                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  91                ("clean", "make clean", "text/plain")],
  92
  93     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
  94     "samba": [
  95                 ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
  96                 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
  97                 ("make", "make -j", "text/plain"),
  98                 ("test", "make test FAIL_IMMEDIATELY=1 "
  99                  "TESTS='--exclude-env=none "
 100                  "--exclude-env=nt4_dc "
 101                  "--exclude-env=nt4_dc_schannel "
 102                  "--exclude-env=nt4_member "
 103                  "--exclude-env=ad_dc "
 104                  "--exclude-env=ad_dc_backup "
 105                  "--exclude-env=ad_dc_ntvfs "
 106                  "--exclude-env=ad_dc_default "
 107                  "--exclude-env=ad_dc_slowtests "
 108                  "--exclude-env=ad_dc_no_nss "
 109                  "--exclude-env=ad_dc_no_ntlm "
 110                  "--exclude-env=fl2003dc "
 111                  "--exclude-env=fl2008dc "
 112                  "--exclude-env=fl2008r2dc "
 113                  "--exclude-env=ad_member "
 114                  "--exclude-env=ad_member_idmap_rid "
 115                  "--exclude-env=ad_member_idmap_ad "
 116                  "--exclude-env=ad_member_rfc2307 "
 117                  "--exclude-env=chgdcpass "
 118                  "--exclude-env=vampire_2000_dc "
 119                  "--exclude-env=fl2000dc "
 120                  "--exclude-env=fileserver "
 121                  "--exclude-env=maptoguest "
 122                  "--exclude-env=simpleserver "
 123                  "--exclude-env=backupfromdc "
 124                  "--exclude-env=restoredc "
 125                  "--exclude-env=renamedc "
 126                  "--exclude-env=offlinebackupdc "
 127                  "--exclude-env=labdc "
 128                  "--exclude-env=preforkrestartdc "
 129                  "--exclude-env=proclimitdc "
 130                  "--exclude-env=promoted_dc "
 131                  "--exclude-env=vampire_dc "
 132                  "--exclude-env=rodc "
 133                  "--exclude-env=ad_dc_default "
 134                  "--exclude-env=ad_dc_slowtests "
 135                  "--exclude-env=schema_pair_dc "
 136                  "--exclude-env=schema_dc "
 137                  "'",
 138                  "text/plain"),
 139                 ("install", "make install", "text/plain"),
 140                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 141                 ("clean", "make clean", "text/plain")],
 142
 143     "samba-nt4": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 144                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 145                     ("make", "make -j", "text/plain"),
 146                     ("test", "make test FAIL_IMMEDIATELY=1 "
 147                      "TESTS='"
 148                      "--include-env=nt4_dc "
 149                      "--include-env=nt4_dc_schannel "
 150                      "--include-env=nt4_member "
 151                      "'", "text/plain"),
 152                     ("install", "make install", "text/plain"),
 153                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 154                     ("clean", "make clean", "text/plain")],
 155
 156     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 157                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 158                            ("make", "make -j", "text/plain"),
 159                            ("test", "make test FAIL_IMMEDIATELY=1 "
 160                             "TESTS='"
 161                             "--include-env=fileserver "
 162                             "--include-env=maptoguest "
 163                             "--include-env=simpleserver "
 164                             "'", "text/plain"),
 165                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 166
 167     "samba-ad-member": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 168                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 169                         ("make", "make -j", "text/plain"),
 170                         ("test", "make test FAIL_IMMEDIATELY=1 "
 171                          "TESTS='"
 172                          "--include-env=ad_member "
 173                          "--include-env=ad_member_idmap_rid "
 174                          "--include-env=ad_member_idmap_ad "
 175                          "--include-env=ad_member_rfc2307 "
 176                          "'", "text/plain"),
 177                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 178
 179     "samba-ad-dc-1": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 180                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 181                       ("make", "make -j", "text/plain"),
 182                       ("test", "make test FAIL_IMMEDIATELY=1 "
 183                        "TESTS='--include-env=ad_dc "
 184                        "--include-env=ad_dc_no_nss "
 185                        "--include-env=ad_dc_no_ntlm "
 186                        "'", "text/plain"),
 187                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 188
 189     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 190                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 191                         ("make", "make -j", "text/plain"),
 192                         ("test", "make test FAIL_IMMEDIATELY=1 "
 193                          "TESTS='"
 194                          "--include-env=vampire_dc "
 195                          "--include-env=vampire_2000_dc "
 196                          "--include-env=rodc "
 197                          "'", "text/plain"),
 198                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 199
 200     "samba-ad-dc-3": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 201                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 202                         ("make", "make -j", "text/plain"),
 203                         ("test", "make test FAIL_IMMEDIATELY=1 "
 204                          "TESTS='"
 205                          "--include-env=promoted_dc "
 206                          "--include-env=chgdcpass "
 207                          "--include-env=preforkrestartdc "
 208                          "--include-env=proclimitdc "
 209                          "'", "text/plain"),
 210                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 211
 212     "samba-ad-dc-4": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 213                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 214                         ("make", "make -j", "text/plain"),
 215                         ("test", "make test FAIL_IMMEDIATELY=1 "
 216                          "TESTS='"
 217                          "--include-env=fl2000dc "
 218                          "--include-env=fl2003dc "
 219                          "--include-env=fl2008dc "
 220                          "--include-env=fl2008r2dc "
 221                          "'", "text/plain"),
 222                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 223
 224     "samba-ad-dc-5": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 225                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 226                         ("make", "make -j", "text/plain"),
 227                         ("test", "make test FAIL_IMMEDIATELY=1 "
 228                          "TESTS='"
 229                          "--include-env=ad_dc_default "
 230                          "'", "text/plain"),
 231                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 232
 233     "samba-ad-dc-6": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 234                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 235                         ("make", "make -j", "text/plain"),
 236                         ("test", "make test FAIL_IMMEDIATELY=1 "
 237                          "TESTS='"
 238                          "--include-env=ad_dc_slowtests "
 239                          "'", "text/plain"),
 240                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 241
 242
 243     "samba-schemaupgrade": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 244                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 245                         ("make", "make -j", "text/plain"),
 246                         ("test", "make test FAIL_IMMEDIATELY=1 "
 247                          "TESTS='"
 248                          "--include-env=schema_dc "
 249                          "--include-env=schema_pair_dc "
 250                          "'", "text/plain"),
 251                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 252
 253     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 254     # This is currently the longest task, so we don't randomly delay it.
 255     "samba-ad-dc-ntvfs": [
 256                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 257                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 258                       ("make", "make -j", "text/plain"),
 259                       ("test", "make test FAIL_IMMEDIATELY=1 "
 260                        "TESTS='"
 261                        "--include-env=ad_dc_ntvfs "
 262                        "'", "text/plain"),
 263                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 264
 265     # run the backup/restore testenvs separately as they're fairly standalone
 266     # (and CI seems to max out at ~8 different DCs running at once)
 267     "samba-ad-dc-backup": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 268                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 269                         ("make", "make -j", "text/plain"),
 270                         ("test", "make test FAIL_IMMEDIATELY=1 "
 271                          "TESTS='--include-env=backupfromdc "
 272                          "--include-env=restoredc "
 273                          "--include-env=renamedc "
 274                          "--include-env=offlinebackupdc "
 275                          "--include-env=labdc "
 276                          "--include-env=ad_dc_backup "
 277                          "'", "text/plain"),
 278                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 279
 280     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 281                           ("make", "make -j", "text/plain"),
 282                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
 283
 284     # Test cross-compile infrastructure
 285     "samba-xc": [("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
 286                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 287                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 288                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 289                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 290                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 291                    ("compare-results", "script/compare_cc_results.py "
 292                     "./bin/c4che/default{} "
 293                     "./bin-xe/c4che/default{} "
 294                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
 295
 296     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 297     "samba-o3": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 298                    ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 299                    ("make", "make -j", "text/plain"),
 300                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
 301                     "TESTS='--include-env=ad_dc'", "text/plain"),
 302                    ("install", "make install", "text/plain"),
 303                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 304                    ("clean", "make clean", "text/plain")],
 305
 306     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
 307
 308                      # make sure we have tdb around:
 309                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 310                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 311                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 312
 313
 314                      # build samba with cluster support (also building ctdb):
 315                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 316                      ("samba-make", "make", "text/plain"),
 317                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 318                      ("samba-install", "make install", "text/plain"),
 319                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 320
 321                      # clean up:
 322                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 323                      ("clean", "make clean", "text/plain"),
 324                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
 325
 326     "samba-libs": [
 327                       ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 328                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 329                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 330                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 331
 332                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 333                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 334                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 335
 336                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 337                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 338                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 339
 340                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 341                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 342                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 343
 344                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 345                       ("nondevel-make", "make -j", "text/plain"),
 346                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 347                       ("nondevel-install", "make install", "text/plain"),
 348                       ("nondevel-dist", "make dist", "text/plain"),
 349
 350                       # retry with all modules shared
 351                       ("allshared-distclean", "make distclean", "text/plain"),
 352                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 353                       ("allshared-make", "make -j", "text/plain")],
 354
 355     "samba-none-env": [
 356                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 357                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 358                       ("make", "make -j", "text/plain"),
 359                       ("test", "make test "
 360                        "FAIL_IMMEDIATELY=1 "
 361                        "TESTS='--include-env=none'",
 362                        "text/plain")],
 363
 364     "samba-static": [
 365                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 366                       # build with all modules static
 367                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 368                       ("allstatic-make", "make -j", "text/plain"),
 369                       ("allstatic-test", "make test "
 370                        "FAIL_IMMEDIATELY=1 "
 371                        "TESTS='samba3.smb2.create.*nt4_dc'",
 372                        "text/plain"),
 373
 374                       # retry without any required modules
 375                       ("none-distclean", "make distclean", "text/plain"),
 376                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 377                       ("none-make", "make -j", "text/plain"),
 378
 379                       # retry with nonshared smbd and smbtorture
 380                       ("nonshared-distclean", "make distclean", "text/plain"),
 381                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 382                       ("nonshared-make", "make -j", "text/plain")],
 383
 384     "samba-systemkrb5": [
 385                       ("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
 386                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --with-experimental-mit-ad-dc", "text/plain"),
 387                       ("make", "make -j", "text/plain"),
 388                       # we currently cannot run a full make test, a limited list of tests could be run
 389                       # via "make test TESTS=sometests"
 390                       ("test", "make test FAIL_IMMEDIATELY=1 "
 391                        "TESTS='--include-env=ktest'", "text/plain"),
 392                       ("install", "make install", "text/plain"),
 393                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 394                       ("clean", "make clean", "text/plain")
 395                       ],
 396
 397     # Test Samba without python still builds.  When this test fails
 398     # due to more use of Python, the expectations is that the newly
 399     # failing part of the code should be disabled when
 400     # --disable-python is set (rather than major work being done to
 401     # support this environment).  The target here is for vendors
 402     # shipping a minimal smbd.
 403     "samba-nopython": [
 404                       ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 405                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 406                       ("make", "make -j", "text/plain"),
 407                       ("install", "make install", "text/plain"),
 408                       ("find-python", "script/find_python.sh ${PREFIX}", "text/plain"),
 409                       ("test", "make test-nopython", "text/plain"),
 410                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 411                       ("clean", "make clean", "text/plain"),
 412
 413                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 414                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 415                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 416
 417                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 418                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 419                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 420
 421                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 422                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 423                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 424
 425                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 426                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 427                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 428
 429                       # retry against installed library packages
 430                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 431                       ("libs-make", "make -j", "text/plain"),
 432                       ("libs-install", "make install", "text/plain"),
 433                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 434                       ("libs-clean", "make clean", "text/plain")
 435                       ],
 436
 437     # check we can do the same thing using python2
 438     "samba-nopython-py2": [
 439                       ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 440                       ("configure", "PYTHON=python2 ./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 441                       ("make", "PYTHON=python2 make -j", "text/plain"),
 442                       ("install", "PYTHON=python2 make install", "text/plain"),
 443                       ("find-python", "script/find_python.sh ${PREFIX}", "text/plain"),
 444                       ("test", "make test-nopython", "text/plain"),
 445                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 446                       ("clean", "PYTHON=python2 make clean", "text/plain"),
 447
 448                       ("talloc-configure", "cd lib/talloc && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 449                       ("talloc-make", "cd lib/talloc && PYTHON=python2 make", "text/plain"),
 450                       ("talloc-install", "cd lib/talloc && PYTHON=python2 make install", "text/plain"),
 451
 452                       ("tdb-configure", "cd lib/tdb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 453                       ("tdb-make", "cd lib/tdb && PYTHON=python2 make", "text/plain"),
 454                       ("tdb-install", "cd lib/tdb && PYTHON=python2 make install", "text/plain"),
 455
 456                       ("tevent-configure", "cd lib/tevent && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 457                       ("tevent-make", "cd lib/tevent && PYTHON=python2 make", "text/plain"),
 458                       ("tevent-install", "cd lib/tevent && PYTHON=python2 make install", "text/plain"),
 459
 460                       ("ldb-configure", "cd lib/ldb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 461                       ("ldb-make", "cd lib/ldb && PYTHON=python2 make", "text/plain"),
 462                       ("ldb-install", "cd lib/ldb && PYTHON=python2 make install", "text/plain"),
 463
 464                       # retry against installed library packages
 465                       ("libs-configure", "PYTHON=python2 " + samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 466                       ("libs-make", "PYTHON=python2 make -j", "text/plain"),
 467                       ("libs-install", "PYTHON=python2 make install", "text/plain"),
 468                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 469                       ("libs-clean", "PYTHON=python2 make clean", "text/plain")
 470                       ],
 471
 472     "ldb": [
 473               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 474               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 475               ("make", "make", "text/plain"),
 476               ("install", "make install", "text/plain"),
 477               ("test", "make test", "text/plain"),
 478               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX}", "text/plain"),
 479               ("make-no-lmdb", "make", "text/plain"),
 480               ("install-no-lmdb", "make install", "text/plain"),
 481               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 482               ("distcheck", "make distcheck", "text/plain"),
 483               ("clean", "make clean", "text/plain")],
 484
 485     "tdb": [
 486               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 487               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 488               ("make", "make", "text/plain"),
 489               ("install", "make install", "text/plain"),
 490               ("test", "make test", "text/plain"),
 491               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 492               ("distcheck", "make distcheck", "text/plain"),
 493               ("clean", "make clean", "text/plain")],
 494
 495     "talloc": [
 496                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 497                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 498                  ("make", "make", "text/plain"),
 499                  ("install", "make install", "text/plain"),
 500                  ("test", "make test", "text/plain"),
 501                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 502                  ("distcheck", "make distcheck", "text/plain"),
 503                  ("clean", "make clean", "text/plain")],
 504
 505     "replace": [
 506                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 507                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 508                   ("make", "make", "text/plain"),
 509                   ("install", "make install", "text/plain"),
 510                   ("test", "make test", "text/plain"),
 511                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 512                   ("distcheck", "make distcheck", "text/plain"),
 513                   ("clean", "make clean", "text/plain")],
 514
 515     "tevent": [
 516                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 517                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 518                  ("make", "make", "text/plain"),
 519                  ("install", "make install", "text/plain"),
 520                  ("test", "make test", "text/plain"),
 521                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 522                  ("distcheck", "make distcheck", "text/plain"),
 523                  ("clean", "make clean", "text/plain")],
 524
 525     "pidl": [
 526         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 527         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 528         ("touch", "touch *.yp", "text/plain"),
 529         ("make", "make", "text/plain"),
 530         ("test", "make test", "text/plain"),
 531         ("install", "make install", "text/plain"),
 532         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 533         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 534         ("clean", "make clean", "text/plain")],
 535
 536
 537     # these are useful for debugging autobuild
 538     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
 539     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
 540
 541
 542 }
 543
 544
 545 def do_print(msg):
 546     print("%s" % msg)
 547     sys.stdout.flush()
 548     sys.stderr.flush()
 549
 550
 551 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 552     if show is None:
 553         show = options.verbose
 554     if show:
 555         do_print("Running: '%s' in '%s'" % (cmd, dir))
 556     if output:
 557         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
 558     elif checkfail:
 559         return check_call(cmd, shell=True, cwd=dir)
 560     else:
 561         return call(cmd, shell=True, cwd=dir)
 562
 563
 564 class builder(object):
 565     '''handle build of one directory'''
 566
 567     def __init__(self, name, sequence, cp=True):
 568         self.name = name
 569         if name in builddirs:
 570             self.dir = builddirs[name]
 571         else:
 572             self.dir = "."
 573
 574         self.tag = self.name.replace('/', '_')
 575         self.sequence = sequence
 576         self.next = 0
 577         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 578         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 579         if options.verbose:
 580             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 581             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 582         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 583         self.stdout = open(self.stdout_path, 'w')
 584         self.stderr = open(self.stderr_path, 'w')
 585         self.stdin  = open("/dev/null", 'r')
 586         self.sdir = "%s/%s" % (testbase, self.tag)
 587         self.prefix = "%s/%s" % (test_prefix, self.tag)
 588         run_cmd("rm -rf %s" % self.sdir)
 589         run_cmd("rm -rf %s" % self.prefix)
 590         if cp:
 591             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 592         else:
 593             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 594         self.start_next()
 595
 596     def start_next(self):
 597         if self.next == len(self.sequence):
 598             if not options.nocleanup:
 599                 run_cmd("rm -rf %s" % self.sdir)
 600                 run_cmd("rm -rf %s" % self.prefix)
 601             do_print('%s: Completed OK' % self.name)
 602             self.done = True
 603             return
 604         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 605         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
 606         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 607         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 608         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 609 #        if self.output_mime_type == "text/x-subunit":
 610 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 611         cwd = os.getcwd()
 612         os.chdir("%s/%s" % (self.sdir, self.dir))
 613         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
 614         self.proc = Popen(self.cmd, shell=True,
 615                           close_fds=True,
 616                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 617         os.chdir(cwd)
 618         self.next += 1
 619
 620
 621 class buildlist(object):
 622     '''handle build of multiple directories'''
 623
 624     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 625         global tasks
 626         self.tlist = []
 627         self.tail_proc = None
 628         self.retry = None
 629         if tasknames == []:
 630             if options.restrict_tests:
 631                 tasknames = ["samba-test-only"]
 632             else:
 633                 tasknames = defaulttasks
 634         else:
 635             # If we are only running one test,
 636             # do not sleep randomly to wait for it to start
 637             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 638
 639         for n in tasknames:
 640             b = builder(n, tasks[n], cp=n is not "pidl")
 641             self.tlist.append(b)
 642         if options.retry:
 643             rebase_remote = "rebaseon"
 644             retry_task = [("retry",
 645                             '''set -e
 646                             git remote add -t %s %s %s
 647                             git fetch %s
 648                             while :; do
 649                               sleep 60
 650                               git describe %s/%s > old_remote_branch.desc
 651                               git fetch %s
 652                               git describe %s/%s > remote_branch.desc
 653                               diff old_remote_branch.desc remote_branch.desc
 654                             done
 655                            ''' % (
 656                                 rebase_branch, rebase_remote, rebase_url,
 657                                rebase_remote,
 658                                rebase_remote, rebase_branch,
 659                                rebase_remote,
 660                                rebase_remote, rebase_branch
 661                             ),
 662                             "test/plain")]
 663
 664             self.retry = builder('retry', retry_task, cp=False)
 665             self.need_retry = False
 666
 667     def kill_kids(self):
 668         if self.tail_proc is not None:
 669             self.tail_proc.terminate()
 670             self.tail_proc.wait()
 671             self.tail_proc = None
 672         if self.retry is not None:
 673             self.retry.proc.terminate()
 674             self.retry.proc.wait()
 675             self.retry = None
 676         for b in self.tlist:
 677             if b.proc is not None:
 678                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 679                 b.proc.terminate()
 680                 b.proc.wait()
 681                 b.proc = None
 682
 683     def wait_one(self):
 684         while True:
 685             none_running = True
 686             for b in self.tlist:
 687                 if b.proc is None:
 688                     continue
 689                 none_running = False
 690                 b.status = b.proc.poll()
 691                 if b.status is None:
 692                     continue
 693                 b.proc = None
 694                 return b
 695             if options.retry:
 696                 ret = self.retry.proc.poll()
 697                 if ret is not None:
 698                     self.need_retry = True
 699                     self.retry = None
 700                     return None
 701             if none_running:
 702                 return None
 703             time.sleep(0.1)
 704
 705     def run(self):
 706         while True:
 707             b = self.wait_one()
 708             if options.retry and self.need_retry:
 709                 self.kill_kids()
 710                 do_print("retry needed")
 711                 return (0, None, None, None, "retry")
 712             if b is None:
 713                 break
 714             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 715                 self.kill_kids()
 716                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 717             b.start_next()
 718         self.kill_kids()
 719         return (0, None, None, None, "All OK")
 720
 721     def write_system_info(self):
 722         filename = 'system-info.txt'
 723         f = open(filename, 'w')
 724         for cmd in ['uname -a',
 725                     'lsb_release -a',
 726                     'free',
 727                     'mount',
 728                     'cat /proc/cpuinfo',
 729                     'cc --version',
 730                     'df -m .',
 731                     'df -m %s' % testbase]:
 732             out = run_cmd(cmd, output=True, checkfail=False)
 733             print('### %s' % cmd, file=f)
 734             print(out.decode('utf8', 'backslashreplace'), file=f)
 735             print(file=f)
 736         f.close()
 737         return filename
 738
 739     def tarlogs(self, fname):
 740         tar = tarfile.open(fname, "w:gz")
 741         for b in self.tlist:
 742             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 743             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 744         if os.path.exists("autobuild.log"):
 745             tar.add("autobuild.log")
 746         sys_info = self.write_system_info()
 747         tar.add(sys_info)
 748         tar.close()
 749
 750     def remove_logs(self):
 751         for b in self.tlist:
 752             os.unlink(b.stdout_path)
 753             os.unlink(b.stderr_path)
 754
 755     def start_tail(self):
 756         cmd = ["tail", "-f"]
 757         for b in self.tlist:
 758             cmd.append(b.stdout_path)
 759             cmd.append(b.stderr_path)
 760         self.tail_proc = Popen(cmd, close_fds=True)
 761
 762
 763 def cleanup():
 764     if options.nocleanup:
 765         return
 766     run_cmd("stat %s || true" % test_tmpdir, show=True)
 767     run_cmd("stat %s" % testbase, show=True)
 768     do_print("Cleaning up %r" % cleanup_list)
 769     for d in cleanup_list:
 770         run_cmd("rm -rf %s" % d)
 771
 772
 773 def find_git_root():
 774     '''get to the top of the git repo'''
 775     p = os.getcwd()
 776     while p != '/':
 777         if os.path.isdir(os.path.join(p, ".git")):
 778             return p
 779         p = os.path.abspath(os.path.join(p, '..'))
 780     return None
 781
 782
 783 def daemonize(logfile):
 784     pid = os.fork()
 785     if pid == 0:  # Parent
 786         os.setsid()
 787         pid = os.fork()
 788         if pid != 0:  # Actual daemon
 789             os._exit(0)
 790     else:  # Grandparent
 791         os._exit(0)
 792
 793     import resource      # Resource usage information.
 794     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 795     if maxfd == resource.RLIM_INFINITY:
 796         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 797     for fd in range(0, maxfd):
 798         try:
 799             os.close(fd)
 800         except OSError:
 801             pass
 802     os.open(logfile, os.O_RDWR | os.O_CREAT)
 803     os.dup2(0, 1)
 804     os.dup2(0, 2)
 805
 806
 807 def write_pidfile(fname):
 808     '''write a pid file, cleanup on exit'''
 809     f = open(fname, mode='w')
 810     f.write("%u\n" % os.getpid())
 811     f.close()
 812
 813
 814 def rebase_tree(rebase_url, rebase_branch="master"):
 815     rebase_remote = "rebaseon"
 816     do_print("Rebasing on %s" % rebase_url)
 817     run_cmd("git describe HEAD", show=True, dir=test_master)
 818     run_cmd("git remote add -t %s %s %s" %
 819             (rebase_branch, rebase_remote, rebase_url),
 820             show=True, dir=test_master)
 821     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 822     if options.fix_whitespace:
 823         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 824                 (rebase_remote, rebase_branch),
 825                 show=True, dir=test_master)
 826     else:
 827         run_cmd("git rebase --force-rebase %s/%s" %
 828                 (rebase_remote, rebase_branch),
 829                 show=True, dir=test_master)
 830     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 831                    (rebase_remote, rebase_branch),
 832                    dir=test_master, output=True)
 833     if diff == '':
 834         do_print("No differences between HEAD and %s/%s - exiting" %
 835                  (rebase_remote, rebase_branch))
 836         sys.exit(0)
 837     run_cmd("git describe %s/%s" %
 838             (rebase_remote, rebase_branch),
 839             show=True, dir=test_master)
 840     run_cmd("git describe HEAD", show=True, dir=test_master)
 841     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 842             (rebase_remote, rebase_branch),
 843             show=True, dir=test_master)
 844
 845
 846 def push_to(push_url, push_branch="master"):
 847     push_remote = "pushto"
 848     do_print("Pushing to %s" % push_url)
 849     if options.mark:
 850         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 851         run_cmd("git commit --amend -c HEAD", dir=test_master)
 852         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 853         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 854     run_cmd("git remote add -t %s %s %s" %
 855             (push_branch, push_remote, push_url),
 856             show=True, dir=test_master)
 857     run_cmd("git push %s +HEAD:%s" %
 858             (push_remote, push_branch),
 859             show=True, dir=test_master)
 860
 861
 862 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 863
 864 gitroot = find_git_root()
 865 if gitroot is None:
 866     raise Exception("Failed to find git root")
 867
 868 parser = OptionParser()
 869 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 870 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 871 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 872 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 873                   default=def_testbase)
 874 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 875 parser.add_option("", "--verbose", help="show all commands as they are run",
 876                   default=False, action="store_true")
 877 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 878                   default=None, type='str')
 879 parser.add_option("", "--pushto", help="push to a git url on success",
 880                   default=None, type='str')
 881 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 882                   default=False, action="store_true")
 883 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 884                   default=False, action="store_true")
 885 parser.add_option("", "--retry", help="automatically retry if master changes",
 886                   default=False, action="store_true")
 887 parser.add_option("", "--email", help="send email to the given address on failure",
 888                   type='str', default=None)
 889 parser.add_option("", "--email-from", help="send email from the given address",
 890                   type='str', default="autobuild@samba.org")
 891 parser.add_option("", "--email-server", help="send email via the given server",
 892                   type='str', default='localhost')
 893 parser.add_option("", "--always-email", help="always send email, even on success",
 894                   action="store_true")
 895 parser.add_option("", "--daemon", help="daemonize after initial setup",
 896                   action="store_true")
 897 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 898                   default="master", type='str')
 899 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 900                   default=gitroot, type='str')
 901 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 902                   default=False, action="store_true")
 903 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 904                   default='')
 905
 906
 907 def send_email(subject, text, log_tar):
 908     if options.email is None:
 909         do_print("not sending email because the recipient is not set")
 910         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
 911                  (subject, text))
 912         return
 913     outer = MIMEMultipart()
 914     outer['Subject'] = subject
 915     outer['To'] = options.email
 916     outer['From'] = options.email_from
 917     outer['Date'] = email.utils.formatdate(localtime=True)
 918     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 919     outer.attach(MIMEText(text, 'plain'))
 920     if options.attach_logs:
 921         fp = open(log_tar, 'rb')
 922         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 923         fp.close()
 924         # Set the filename parameter
 925         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 926         outer.attach(msg)
 927     content = outer.as_string()
 928     s = smtplib.SMTP(options.email_server)
 929     email_user = os.getenv('SMTP_USERNAME')
 930     email_password = os.getenv('SMTP_PASSWORD')
 931     if email_user is not None:
 932         s.starttls()
 933         s.login(email_user, email_password)
 934
 935     s.sendmail(options.email_from, [options.email], content)
 936     s.set_debuglevel(1)
 937     s.quit()
 938
 939
 940 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 941                   elapsed_time, log_base=None, add_log_tail=True):
 942     '''send an email to options.email about the failure'''
 943     elapsed_minutes = elapsed_time / 60.0
 944     if log_base is None:
 945         log_base = gitroot
 946     text = '''
 947 Dear Developer,
 948
 949 Your autobuild on %s failed after %.1f minutes
 950 when trying to test %s with the following error:
 951
 952    %s
 953
 954 the autobuild has been abandoned. Please fix the error and resubmit.
 955
 956 A summary of the autobuild process is here:
 957
 958   %s/autobuild.log
 959 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 960
 961     if options.restrict_tests:
 962         text += """
 963 The build was restricted to tests matching %s\n""" % options.restrict_tests
 964
 965     if failed_task != 'rebase':
 966         text += '''
 967 You can see logs of the failed task here:
 968
 969   %s/%s.stdout
 970   %s/%s.stderr
 971
 972 or you can get full logs of all tasks in this job here:
 973
 974   %s/logs.tar.gz
 975
 976 The top commit for the tree that was built was:
 977
 978 %s
 979
 980 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 981
 982     if add_log_tail:
 983         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 984         lines = f.readlines()
 985         log_tail = "".join(lines[-50:])
 986         num_lines = len(lines)
 987         if num_lines < 50:
 988             # Also include stderr (compile failures) if < 50 lines of stdout
 989             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 990             log_tail += "".join(f.readlines()[-(50 - num_lines):])
 991
 992         text += '''
 993 The last 50 lines of log messages:
 994
 995 %s
 996     ''' % log_tail
 997         f.close()
 998
 999     logs = os.path.join(gitroot, 'logs.tar.gz')
1000     send_email('autobuild[%s] failure on %s for task %s during %s'
1001                % (options.branch, platform.node(), failed_task, failed_stage),
1002                text, logs)
1003
1004
1005 def email_success(elapsed_time, log_base=None):
1006     '''send an email to options.email about a successful build'''
1007     if log_base is None:
1008         log_base = gitroot
1009     text = '''
1010 Dear Developer,
1011
1012 Your autobuild on %s has succeeded after %.1f minutes.
1013
1014 ''' % (platform.node(), elapsed_time / 60.)
1015
1016     if options.restrict_tests:
1017         text += """
1018 The build was restricted to tests matching %s\n""" % options.restrict_tests
1019
1020     if options.keeplogs:
1021         text += '''
1022
1023 you can get full logs of all tasks in this job here:
1024
1025   %s/logs.tar.gz
1026
1027 ''' % log_base
1028
1029     text += '''
1030 The top commit for the tree that was built was:
1031
1032 %s
1033 ''' % top_commit_msg
1034
1035     logs = os.path.join(gitroot, 'logs.tar.gz')
1036     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1037                text, logs)
1038
1039
1040 (options, args) = parser.parse_args()
1041
1042 if options.retry:
1043     if options.rebase is None:
1044         raise Exception('You can only use --retry if you also rebase')
1045
1046 testbase = "%s/b%u" % (options.testbase, os.getpid())
1047 test_master = "%s/master" % testbase
1048 test_prefix = "%s/prefix" % testbase
1049 test_tmpdir = "%s/tmp" % testbase
1050 os.environ['TMPDIR'] = test_tmpdir
1051
1052 # get the top commit message, for emails
1053 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1054 top_commit_msg = top_commit_msg.decode('utf-8', 'backslashreplace')
1055
1056 try:
1057     os.makedirs(testbase)
1058 except Exception as reason:
1059     raise Exception("Unable to create %s : %s" % (testbase, reason))
1060 cleanup_list.append(testbase)
1061
1062 if options.daemon:
1063     logfile = os.path.join(testbase, "log")
1064     do_print("Forking into the background, writing progress to %s" % logfile)
1065     daemonize(logfile)
1066
1067 write_pidfile(gitroot + "/autobuild.pid")
1068
1069 start_time = time.time()
1070
1071 while True:
1072     try:
1073         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1074         os.makedirs(test_tmpdir)
1075         # The waf uninstall code removes empty directories all the way
1076         # up the tree.  Creating a file in test_tmpdir stops it from
1077         # being removed.
1078         run_cmd("touch %s" % os.path.join(test_tmpdir,
1079                                           ".directory-is-not-empty"), show=True)
1080         run_cmd("stat %s" % test_tmpdir, show=True)
1081         run_cmd("stat %s" % testbase, show=True)
1082         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1083     except Exception:
1084         cleanup()
1085         raise
1086
1087     try:
1088         try:
1089             if options.rebase is not None:
1090                 rebase_tree(options.rebase, rebase_branch=options.branch)
1091         except Exception:
1092             cleanup_list.append(gitroot + "/autobuild.pid")
1093             cleanup()
1094             elapsed_time = time.time() - start_time
1095             email_failure(-1, 'rebase', 'rebase', 'rebase',
1096                           'rebase on %s failed' % options.branch,
1097                           elapsed_time, log_base=options.log_base)
1098             sys.exit(1)
1099         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1100         if options.tail:
1101             blist.start_tail()
1102         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1103         if status != 0 or errstr != "retry":
1104             break
1105         cleanup()
1106     except Exception:
1107         cleanup()
1108         raise
1109
1110 cleanup_list.append(gitroot + "/autobuild.pid")
1111
1112 do_print(errstr)
1113
1114 blist.kill_kids()
1115 if options.tail:
1116     do_print("waiting for tail to flush")
1117     time.sleep(1)
1118
1119 elapsed_time = time.time() - start_time
1120 if status == 0:
1121     if options.passcmd is not None:
1122         do_print("Running passcmd: %s" % options.passcmd)
1123         run_cmd(options.passcmd, dir=test_master)
1124     if options.pushto is not None:
1125         push_to(options.pushto, push_branch=options.branch)
1126     if options.keeplogs or options.attach_logs:
1127         blist.tarlogs("logs.tar.gz")
1128         do_print("Logs in logs.tar.gz")
1129     if options.always_email:
1130         email_success(elapsed_time, log_base=options.log_base)
1131     blist.remove_logs()
1132     cleanup()
1133     do_print(errstr)
1134     sys.exit(0)
1135
1136 # something failed, gather a tar of the logs
1137 blist.tarlogs("logs.tar.gz")
1138
1139 if options.email is not None:
1140     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1141                   elapsed_time, log_base=options.log_base)
1142 else:
1143     elapsed_minutes = elapsed_time / 60.0
1144     print('''
1145
1146 ####################################################################
1147
1148 AUTOBUILD FAILURE
1149
1150 Your autobuild[%s] on %s failed after %.1f minutes
1151 when trying to test %s with the following error:
1152
1153    %s
1154
1155 the autobuild has been abandoned. Please fix the error and resubmit.
1156
1157 ####################################################################
1158
1159 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1160
1161 cleanup()
1162 do_print(errstr)
1163 do_print("Logs in logs.tar.gz")
1164 sys.exit(status)