script/autobuild.py

   1 #!/usr/bin/env python
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call,Popen, PIPE
   8 import os, tarfile, sys, time
   9 from optparse import OptionParser
  10 import smtplib
  11 import email
  12 from email.mime.text import MIMEText
  13 from email.mime.base import MIMEBase
  14 from email.mime.application import MIMEApplication
  15 from email.mime.multipart import MIMEMultipart
  16 from distutils.sysconfig import get_python_lib
  17 import platform
  18
  19 os.environ["PYTHONUNBUFFERED"] = "1"
  20
  21 # This speeds up testing remarkably.
  22 os.environ['TDB_NO_FSYNC'] = '1'
  23
  24 cleanup_list = []
  25
  26 builddirs = {
  27     "ctdb"    : "ctdb",
  28     "samba"  : ".",
  29     "samba-nt4"  : ".",
  30     "samba-fileserver"  : ".",
  31     "samba-xc" : ".",
  32     "samba-o3" : ".",
  33     "samba-ctdb" : ".",
  34     "samba-libs"  : ".",
  35     "samba-static"  : ".",
  36     "samba-test-only"  : ".",
  37     "samba-none-env"  : ".",
  38     "samba-ad-dc"  : ".",
  39     "samba-ad-dc-2"  : ".",
  40     "samba-systemkrb5"  : ".",
  41     "samba-nopython"  : ".",
  42     "ldb"     : "lib/ldb",
  43     "tdb"     : "lib/tdb",
  44     "talloc"  : "lib/talloc",
  45     "replace" : "lib/replace",
  46     "tevent"  : "lib/tevent",
  47     "pidl"    : "pidl",
  48     "pass"    : ".",
  49     "fail"    : ".",
  50     "retry"   : "."
  51     }
  52
  53 defaulttasks = [ "ctdb",
  54                  "samba",
  55                  "samba-nt4",
  56                  "samba-fileserver",
  57                  "samba-xc",
  58                  "samba-o3",
  59                  "samba-ctdb",
  60                  "samba-libs",
  61                  "samba-static",
  62                  "samba-none-env",
  63                  "samba-ad-dc",
  64                  "samba-ad-dc-2",
  65                  "samba-systemkrb5",
  66                  "samba-nopython",
  67                  "ldb",
  68                  "tdb",
  69                  "talloc",
  70                  "replace",
  71                  "tevent",
  72                  "pidl" ]
  73
  74 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  75     defaulttasks.remove("samba-o3")
  76
  77 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  78 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
  79
  80 samba_libs_envvars =  "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH"
  81 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  82 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  83 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
  84 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,NONE ${EXTRA_PYTHON}"
  85 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent"
  86 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs + " ${EXTRA_PYTHON}"
  87
  88 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
  89     extra_python = ""
  90 else:
  91     extra_python = "--extra-python=/usr/bin/python3"
  92
  93 tasks = {
  94     "ctdb" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  95                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  96                ("make", "make all", "text/plain"),
  97                ("install", "make install", "text/plain"),
  98                ("test", "make autotest", "text/plain"),
  99                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 100                ("clean", "make clean", "text/plain") ],
 101
 102     # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)'
 103     "samba" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 104                 ("make", "make -j", "text/plain"),
 105                 ("test", "make test FAIL_IMMEDIATELY=1 "
 106                  "TESTS='--exclude-env=none "
 107                  "--exclude-env=nt4_dc "
 108                  "--exclude-env=nt4_member "
 109                  "--exclude-env=ad_dc "
 110                  "--exclude-env=fl2003dc "
 111                  "--exclude-env=fl2008r2dc "
 112                  "--exclude-env=ad_member "
 113                  "--exclude-env=ad_member_idmap_rid "
 114                  "--exclude-env=ad_member_idmap_ad "
 115                  "--exclude-env=chgdcpass "
 116                  "--exclude-env=vampire_2000_dc "
 117                  "--exclude-env=fl2000dc "
 118                  "--exclude-env=fileserver'",
 119                  "text/plain"),
 120                 ("install", "make install", "text/plain"),
 121                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 122                 ("clean", "make clean", "text/plain") ],
 123
 124     # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long)
 125     "samba-nt4" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 126                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 127                     ("make", "make -j", "text/plain"),
 128                     ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 129                     ("install", "make install", "text/plain"),
 130                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 131                     ("clean", "make clean", "text/plain") ],
 132
 133     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 134     "samba-fileserver" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 135                       ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json-audit --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 136                       ("make", "make -j", "text/plain"),
 137                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=fileserver'", "text/plain"),
 138                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 139
 140     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 141     "samba-ad-dc" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 142                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 143                       ("make", "make -j", "text/plain"),
 144                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='"
 145                        "--include-env=ad_dc "
 146                        "--include-env=fl2003dc "
 147                        "--include-env=fl2008r2dc "
 148                        "--include-env=ad_member "
 149                        "--include-env=ad_member_idmap_rid "
 150                        "--include-env=ad_member_idmap_ad'", "text/plain"),
 151                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 152
 153     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 154     "samba-ad-dc-2" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 155                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 156                       ("make", "make -j", "text/plain"),
 157                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=chgdcpass --include-env=vampire_2000_dc --include-env=fl2000dc'", "text/plain"),
 158                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 159
 160     "samba-test-only" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 161                           ("make", "make -j", "text/plain"),
 162                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"',"text/plain") ],
 163
 164     # Test cross-compile infrastructure
 165     "samba-xc" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 166                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 167                    ("configure-cross-execute", "./configure.developer -b ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 168                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 169                    ("configure-cross-answers", "./configure.developer -b ./bin-xa --cross-compile" \
 170                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 171                    ("compare-results", "script/compare_cc_results.py ./bin/c4che/default.cache.py ./bin-xe/c4che/default.cache.py ./bin-xa/c4che/default.cache.py", "text/plain")],
 172
 173     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 174     "samba-o3" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 175                    ("configure", "ADDITIONAL_CFLAGS='-O3' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 176                    ("make", "make -j", "text/plain"),
 177                    ("test", "make quicktest FAIL_IMMEDIATELY=1 TESTS='--include-env=ad_dc'", "text/plain"),
 178                    ("install", "make install", "text/plain"),
 179                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 180                    ("clean", "make clean", "text/plain") ],
 181
 182     "samba-ctdb" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 183
 184                      # make sure we have tdb around:
 185                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 186                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 187                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 188
 189
 190                      # build samba with cluster support (also building ctdb):
 191                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 192                      ("samba-make", "make", "text/plain"),
 193                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 194                      ("samba-install", "make install", "text/plain"),
 195                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 196
 197                      # clean up:
 198                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 199                      ("clean", "make clean", "text/plain"),
 200                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain") ],
 201
 202     "samba-libs" : [
 203                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 204                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 205                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 206                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 207
 208                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 209                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 210                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 211
 212                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 213                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 214                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 215
 216                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 217                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 218                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 219
 220                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 221                       ("nondevel-make", "make -j", "text/plain"),
 222                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 223                       ("nondevel-install", "make install", "text/plain"),
 224                       ("nondevel-dist", "make dist", "text/plain"),
 225
 226                       # retry with all modules shared
 227                       ("allshared-distclean", "make distclean", "text/plain"),
 228                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 229                       ("allshared-make", "make -j", "text/plain")],
 230
 231     "samba-none-env" : [
 232                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 233                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 234                       ("make", "make -j", "text/plain"),
 235                       ("test", "make test "
 236                        "FAIL_IMMEDIATELY=1 "
 237                        "TESTS='--include-env=none'",
 238                        "text/plain")],
 239
 240     "samba-static" : [
 241                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 242                       # build with all modules static
 243                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 244                       ("allstatic-make", "make -j", "text/plain"),
 245
 246                       # retry without any required modules
 247                       ("none-distclean", "make distclean", "text/plain"),
 248                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 249                       ("none-make", "make -j", "text/plain"),
 250
 251                       # retry with nonshared smbd and smbtorture
 252                       ("nonshared-distclean", "make distclean", "text/plain"),
 253                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 254                       ("nonshared-make", "make -j", "text/plain")],
 255
 256     "samba-systemkrb5" : [
 257                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 258                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 259                       ("make", "make -j", "text/plain"),
 260                       # we currently cannot run a full make test, a limited list of tests could be run
 261                       # via "make test TESTS=sometests"
 262                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=ktest'", "text/plain"),
 263                       ("install", "make install", "text/plain"),
 264                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 265                       ("clean", "make clean", "text/plain")
 266                       ],
 267
 268     # Test Samba without python still builds.  When this test fails
 269     # due to more use of Python, the expectations is that the newly
 270     # failing part of the code should be disabled when
 271     # --disable-python is set (rather than major work being done to
 272     # support this environment).  The target here is for vendors
 273     # shipping a minimal smbd.
 274     "samba-nopython" : [
 275                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 276                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 277                       ("make", "make -j", "text/plain"),
 278                       ("install", "make install", "text/plain"),
 279                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 280                       ("clean", "make clean", "text/plain"),
 281
 282                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 283                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 284                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 285
 286                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 287                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 288                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 289
 290                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 291                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 292                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 293
 294                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 295                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 296                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 297
 298                       # retry against installed library packages
 299                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 300                       ("libs-make", "make -j", "text/plain"),
 301                       ("libs-install", "make install", "text/plain"),
 302                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 303                       ("libs-clean", "make clean", "text/plain")
 304                       ],
 305
 306
 307
 308     "ldb" : [
 309               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 310               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 311               ("make", "make", "text/plain"),
 312               ("install", "make install", "text/plain"),
 313               ("test", "make test", "text/plain"),
 314               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 315               ("make-no-lmdb", "make", "text/plain"),
 316               ("install-no-lmdb", "make install", "text/plain"),
 317               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 318               ("distcheck", "make distcheck", "text/plain"),
 319               ("clean", "make clean", "text/plain") ],
 320
 321     "tdb" : [
 322               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 323               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 324               ("make", "make", "text/plain"),
 325               ("install", "make install", "text/plain"),
 326               ("test", "make test", "text/plain"),
 327               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 328               ("distcheck", "make distcheck", "text/plain"),
 329               ("clean", "make clean", "text/plain") ],
 330
 331     "talloc" : [
 332                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 333                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 334                  ("make", "make", "text/plain"),
 335                  ("install", "make install", "text/plain"),
 336                  ("test", "make test", "text/plain"),
 337                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 338                  ("distcheck", "make distcheck", "text/plain"),
 339                  ("clean", "make clean", "text/plain") ],
 340
 341     "replace" : [
 342                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 343                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 344                   ("make", "make", "text/plain"),
 345                   ("install", "make install", "text/plain"),
 346                   ("test", "make test", "text/plain"),
 347                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 348                   ("distcheck", "make distcheck", "text/plain"),
 349                   ("clean", "make clean", "text/plain") ],
 350
 351     "tevent" : [
 352                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 353                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 354                  ("make", "make", "text/plain"),
 355                  ("install", "make install", "text/plain"),
 356                  ("test", "make test", "text/plain"),
 357                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 358                  ("distcheck", "make distcheck", "text/plain"),
 359                  ("clean", "make clean", "text/plain") ],
 360
 361     "pidl" : [
 362                ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 363                ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 364                ("touch", "touch *.yp", "text/plain"),
 365                ("make", "make", "text/plain"),
 366                ("test", "make test", "text/plain"),
 367                ("install", "make install", "text/plain"),
 368                ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 369                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 370                ("clean", "make clean", "text/plain") ],
 371
 372     # these are useful for debugging autobuild
 373     'pass' : [ ("pass", 'echo passing && /bin/true', "text/plain") ],
 374     'fail' : [ ("fail", 'echo failing && /bin/false', "text/plain") ]
 375 }
 376
 377 def do_print(msg):
 378     print("%s" % msg)
 379     sys.stdout.flush()
 380     sys.stderr.flush()
 381
 382 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 383     if show is None:
 384         show = options.verbose
 385     if show:
 386         do_print("Running: '%s' in '%s'" % (cmd, dir))
 387     if output:
 388         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir).communicate()[0]
 389     elif checkfail:
 390         return check_call(cmd, shell=True, cwd=dir)
 391     else:
 392         return call(cmd, shell=True, cwd=dir)
 393
 394
 395 class builder(object):
 396     '''handle build of one directory'''
 397
 398     def __init__(self, name, sequence, cp=True):
 399         self.name = name
 400         self.dir = builddirs[name]
 401
 402         self.tag = self.name.replace('/', '_')
 403         self.sequence = sequence
 404         self.next = 0
 405         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 406         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 407         if options.verbose:
 408             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 409             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 410         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 411         self.stdout = open(self.stdout_path, 'w')
 412         self.stderr = open(self.stderr_path, 'w')
 413         self.stdin  = open("/dev/null", 'r')
 414         self.sdir = "%s/%s" % (testbase, self.tag)
 415         self.prefix = "%s/%s" % (test_prefix, self.tag)
 416         run_cmd("rm -rf %s" % self.sdir)
 417         run_cmd("rm -rf %s" % self.prefix)
 418         if cp:
 419             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 420         else:
 421             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 422         self.start_next()
 423
 424     def start_next(self):
 425         if self.next == len(self.sequence):
 426             if not options.nocleanup:
 427                 run_cmd("rm -rf %s" % self.sdir)
 428                 run_cmd("rm -rf %s" % self.prefix)
 429             do_print('%s: Completed OK' % self.name)
 430             self.done = True
 431             return
 432         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 433         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix))
 434         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 435         self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
 436         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 437         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 438 #        if self.output_mime_type == "text/x-subunit":
 439 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 440         do_print('%s: [%s] Running %s' % (self.name, self.stage, self.cmd))
 441         cwd = os.getcwd()
 442         os.chdir("%s/%s" % (self.sdir, self.dir))
 443         self.proc = Popen(self.cmd, shell=True,
 444                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 445         os.chdir(cwd)
 446         self.next += 1
 447
 448
 449 class buildlist(object):
 450     '''handle build of multiple directories'''
 451
 452     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 453         global tasks
 454         self.tlist = []
 455         self.tail_proc = None
 456         self.retry = None
 457         if tasknames == []:
 458             if options.restrict_tests:
 459                 tasknames = ["samba-test-only"]
 460             else:
 461                 tasknames = defaulttasks
 462         else:
 463             # If we are only running one test,
 464             # do not sleep randomly to wait for it to start
 465             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 466
 467         for n in tasknames:
 468             b = builder(n, tasks[n], cp=n is not "pidl")
 469             self.tlist.append(b)
 470         if options.retry:
 471             rebase_remote = "rebaseon"
 472             retry_task = [ ("retry",
 473                             '''set -e
 474                             git remote add -t %s %s %s
 475                             git fetch %s
 476                             while :; do
 477                               sleep 60
 478                               git describe %s/%s > old_remote_branch.desc
 479                               git fetch %s
 480                               git describe %s/%s > remote_branch.desc
 481                               diff old_remote_branch.desc remote_branch.desc
 482                             done
 483                            ''' % (
 484                                rebase_branch, rebase_remote, rebase_url,
 485                                rebase_remote,
 486                                rebase_remote, rebase_branch,
 487                                rebase_remote,
 488                                rebase_remote, rebase_branch
 489                            ),
 490                            "test/plain" ) ]
 491
 492             self.retry = builder('retry', retry_task, cp=False)
 493             self.need_retry = False
 494
 495     def kill_kids(self):
 496         if self.tail_proc is not None:
 497             self.tail_proc.terminate()
 498             self.tail_proc.wait()
 499             self.tail_proc = None
 500         if self.retry is not None:
 501             self.retry.proc.terminate()
 502             self.retry.proc.wait()
 503             self.retry = None
 504         for b in self.tlist:
 505             if b.proc is not None:
 506                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 507                 b.proc.terminate()
 508                 b.proc.wait()
 509                 b.proc = None
 510
 511     def wait_one(self):
 512         while True:
 513             none_running = True
 514             for b in self.tlist:
 515                 if b.proc is None:
 516                     continue
 517                 none_running = False
 518                 b.status = b.proc.poll()
 519                 if b.status is None:
 520                     continue
 521                 b.proc = None
 522                 return b
 523             if options.retry:
 524                 ret = self.retry.proc.poll()
 525                 if ret is not None:
 526                     self.need_retry = True
 527                     self.retry = None
 528                     return None
 529             if none_running:
 530                 return None
 531             time.sleep(0.1)
 532
 533     def run(self):
 534         while True:
 535             b = self.wait_one()
 536             if options.retry and self.need_retry:
 537                 self.kill_kids()
 538                 do_print("retry needed")
 539                 return (0, None, None, None, "retry")
 540             if b is None:
 541                 break
 542             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 543                 self.kill_kids()
 544                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 545             b.start_next()
 546         self.kill_kids()
 547         return (0, None, None, None, "All OK")
 548
 549     def write_system_info(self):
 550         filename = 'system-info.txt'
 551         f = open(filename, 'w')
 552         for cmd in ['uname -a', 'free', 'cat /proc/cpuinfo',
 553                     'cc --version', 'df -m .', 'df -m %s' % testbase]:
 554             print('### %s' % cmd, file=f)
 555             print(run_cmd(cmd, output=True, checkfail=False), file=f)
 556             print(file=f)
 557         f.close()
 558         return filename
 559
 560     def tarlogs(self, fname):
 561         tar = tarfile.open(fname, "w:gz")
 562         for b in self.tlist:
 563             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 564             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 565         if os.path.exists("autobuild.log"):
 566             tar.add("autobuild.log")
 567         sys_info = self.write_system_info()
 568         tar.add(sys_info)
 569         tar.close()
 570
 571     def remove_logs(self):
 572         for b in self.tlist:
 573             os.unlink(b.stdout_path)
 574             os.unlink(b.stderr_path)
 575
 576     def start_tail(self):
 577         cwd = os.getcwd()
 578         cmd = "tail -f *.stdout *.stderr"
 579         os.chdir(gitroot)
 580         self.tail_proc = Popen(cmd, shell=True)
 581         os.chdir(cwd)
 582
 583
 584 def cleanup():
 585     if options.nocleanup:
 586         return
 587     run_cmd("stat %s || true" % test_tmpdir, show=True)
 588     run_cmd("stat %s" % testbase, show=True)
 589     do_print("Cleaning up ....")
 590     for d in cleanup_list:
 591         run_cmd("rm -rf %s" % d)
 592
 593
 594 def find_git_root():
 595     '''get to the top of the git repo'''
 596     p=os.getcwd()
 597     while p != '/':
 598         if os.path.isdir(os.path.join(p, ".git")):
 599             return p
 600         p = os.path.abspath(os.path.join(p, '..'))
 601     return None
 602
 603
 604 def daemonize(logfile):
 605     pid = os.fork()
 606     if pid == 0: # Parent
 607         os.setsid()
 608         pid = os.fork()
 609         if pid != 0: # Actual daemon
 610             os._exit(0)
 611     else: # Grandparent
 612         os._exit(0)
 613
 614     import resource      # Resource usage information.
 615     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 616     if maxfd == resource.RLIM_INFINITY:
 617         maxfd = 1024 # Rough guess at maximum number of open file descriptors.
 618     for fd in range(0, maxfd):
 619         try:
 620             os.close(fd)
 621         except OSError:
 622             pass
 623     os.open(logfile, os.O_RDWR | os.O_CREAT)
 624     os.dup2(0, 1)
 625     os.dup2(0, 2)
 626
 627 def write_pidfile(fname):
 628     '''write a pid file, cleanup on exit'''
 629     f = open(fname, mode='w')
 630     f.write("%u\n" % os.getpid())
 631     f.close()
 632
 633
 634 def rebase_tree(rebase_url, rebase_branch = "master"):
 635     rebase_remote = "rebaseon"
 636     do_print("Rebasing on %s" % rebase_url)
 637     run_cmd("git describe HEAD", show=True, dir=test_master)
 638     run_cmd("git remote add -t %s %s %s" %
 639             (rebase_branch, rebase_remote, rebase_url),
 640             show=True, dir=test_master)
 641     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 642     if options.fix_whitespace:
 643         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 644                 (rebase_remote, rebase_branch),
 645                 show=True, dir=test_master)
 646     else:
 647         run_cmd("git rebase --force-rebase %s/%s" %
 648                 (rebase_remote, rebase_branch),
 649                 show=True, dir=test_master)
 650     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 651                    (rebase_remote, rebase_branch),
 652                    dir=test_master, output=True)
 653     if diff == '':
 654         do_print("No differences between HEAD and %s/%s - exiting" %
 655               (rebase_remote, rebase_branch))
 656         sys.exit(0)
 657     run_cmd("git describe %s/%s" %
 658             (rebase_remote, rebase_branch),
 659             show=True, dir=test_master)
 660     run_cmd("git describe HEAD", show=True, dir=test_master)
 661     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 662             (rebase_remote, rebase_branch),
 663             show=True, dir=test_master)
 664
 665 def push_to(push_url, push_branch = "master"):
 666     push_remote = "pushto"
 667     do_print("Pushing to %s" % push_url)
 668     if options.mark:
 669         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 670         run_cmd("git commit --amend -c HEAD", dir=test_master)
 671         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 672         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 673     run_cmd("git remote add -t %s %s %s" %
 674             (push_branch, push_remote, push_url),
 675             show=True, dir=test_master)
 676     run_cmd("git push %s +HEAD:%s" %
 677             (push_remote, push_branch),
 678             show=True, dir=test_master)
 679
 680 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 681
 682 gitroot = find_git_root()
 683 if gitroot is None:
 684     raise Exception("Failed to find git root")
 685
 686 parser = OptionParser()
 687 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 688 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 689 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 690 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 691                   default=def_testbase)
 692 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 693 parser.add_option("", "--verbose", help="show all commands as they are run",
 694                   default=False, action="store_true")
 695 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 696                   default=None, type='str')
 697 parser.add_option("", "--pushto", help="push to a git url on success",
 698                   default=None, type='str')
 699 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 700                   default=False, action="store_true")
 701 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 702                   default=False, action="store_true")
 703 parser.add_option("", "--retry", help="automatically retry if master changes",
 704                   default=False, action="store_true")
 705 parser.add_option("", "--email", help="send email to the given address on failure",
 706                   type='str', default=None)
 707 parser.add_option("", "--email-from", help="send email from the given address",
 708                   type='str', default="autobuild@samba.org")
 709 parser.add_option("", "--email-server", help="send email via the given server",
 710                   type='str', default='localhost')
 711 parser.add_option("", "--always-email", help="always send email, even on success",
 712                   action="store_true")
 713 parser.add_option("", "--daemon", help="daemonize after initial setup",
 714                   action="store_true")
 715 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 716                   default="master", type='str')
 717 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 718                   default=gitroot, type='str')
 719 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 720                   default=False, action="store_true")
 721 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 722                   default='')
 723
 724 def send_email(subject, text, log_tar):
 725     if options.email is None:
 726         do_print("not sending email because the recipient is not set")
 727         do_print("the text content would have been:\n\nSubject: %s\n\nTs" %
 728                  (subject, text))
 729         return
 730     outer = MIMEMultipart()
 731     outer['Subject'] = subject
 732     outer['To'] = options.email
 733     outer['From'] = options.email_from
 734     outer['Date'] = email.utils.formatdate(localtime = True)
 735     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 736     outer.attach(MIMEText(text, 'plain'))
 737     if options.attach_logs:
 738         fp = open(log_tar, 'rb')
 739         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 740         fp.close()
 741         # Set the filename parameter
 742         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 743         outer.attach(msg)
 744     content = outer.as_string()
 745     s = smtplib.SMTP(options.email_server)
 746     s.sendmail(options.email_from, [options.email], content)
 747     s.set_debuglevel(1)
 748     s.quit()
 749
 750 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 751                   elapsed_time, log_base=None, add_log_tail=True):
 752     '''send an email to options.email about the failure'''
 753     elapsed_minutes = elapsed_time / 60.0
 754     user = os.getenv("USER")
 755     if log_base is None:
 756         log_base = gitroot
 757     text = '''
 758 Dear Developer,
 759
 760 Your autobuild on %s failed after %.1f minutes
 761 when trying to test %s with the following error:
 762
 763    %s
 764
 765 the autobuild has been abandoned. Please fix the error and resubmit.
 766
 767 A summary of the autobuild process is here:
 768
 769   %s/autobuild.log
 770 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 771
 772     if options.restrict_tests:
 773         text += """
 774 The build was restricted to tests matching %s\n""" % options.restrict_tests
 775
 776     if failed_task != 'rebase':
 777         text += '''
 778 You can see logs of the failed task here:
 779
 780   %s/%s.stdout
 781   %s/%s.stderr
 782
 783 or you can get full logs of all tasks in this job here:
 784
 785   %s/logs.tar.gz
 786
 787 The top commit for the tree that was built was:
 788
 789 %s
 790
 791 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 792
 793     if add_log_tail:
 794         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 795         lines = f.readlines()
 796         log_tail = "".join(lines[-50:])
 797         num_lines = len(lines)
 798         if num_lines < 50:
 799             # Also include stderr (compile failures) if < 50 lines of stdout
 800             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 801             log_tail += "".join(f.readlines()[-(50-num_lines):])
 802
 803         text += '''
 804 The last 50 lines of log messages:
 805
 806 %s
 807     ''' % log_tail
 808         f.close()
 809
 810     logs = os.path.join(gitroot, 'logs.tar.gz')
 811     send_email('autobuild[%s] failure on %s for task %s during %s'
 812                % (options.branch, platform.node(), failed_task, failed_stage),
 813                text, logs)
 814
 815 def email_success(elapsed_time, log_base=None):
 816     '''send an email to options.email about a successful build'''
 817     user = os.getenv("USER")
 818     if log_base is None:
 819         log_base = gitroot
 820     text = '''
 821 Dear Developer,
 822
 823 Your autobuild on %s has succeeded after %.1f minutes.
 824
 825 ''' % (platform.node(), elapsed_time / 60.)
 826
 827     if options.restrict_tests:
 828         text += """
 829 The build was restricted to tests matching %s\n""" % options.restrict_tests
 830
 831     if options.keeplogs:
 832         text += '''
 833
 834 you can get full logs of all tasks in this job here:
 835
 836   %s/logs.tar.gz
 837
 838 ''' % log_base
 839
 840     text += '''
 841 The top commit for the tree that was built was:
 842
 843 %s
 844 ''' % top_commit_msg
 845
 846     logs = os.path.join(gitroot, 'logs.tar.gz')
 847     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 848                text, logs)
 849
 850
 851 (options, args) = parser.parse_args()
 852
 853 if options.retry:
 854     if options.rebase is None:
 855         raise Exception('You can only use --retry if you also rebase')
 856
 857 testbase = "%s/b%u" % (options.testbase, os.getpid())
 858 test_master = "%s/master" % testbase
 859 test_prefix = "%s/prefix" % testbase
 860 test_tmpdir = "%s/tmp" % testbase
 861 os.environ['TMPDIR'] = test_tmpdir
 862
 863 # get the top commit message, for emails
 864 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 865
 866 try:
 867     os.makedirs(testbase)
 868 except Exception as reason:
 869     raise Exception("Unable to create %s : %s" % (testbase, reason))
 870 cleanup_list.append(testbase)
 871
 872 if options.daemon:
 873     logfile = os.path.join(testbase, "log")
 874     do_print("Forking into the background, writing progress to %s" % logfile)
 875     daemonize(logfile)
 876
 877 write_pidfile(gitroot + "/autobuild.pid")
 878
 879 start_time = time.time()
 880
 881 while True:
 882     try:
 883         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 884         os.makedirs(test_tmpdir)
 885         # The waf uninstall code removes empty directories all the way
 886         # up the tree.  Creating a file in test_tmpdir stops it from
 887         # being removed.
 888         run_cmd("touch %s" % os.path.join(test_tmpdir,
 889                                           ".directory-is-not-empty"), show=True)
 890         run_cmd("stat %s" % test_tmpdir, show=True)
 891         run_cmd("stat %s" % testbase, show=True)
 892         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 893     except Exception:
 894         cleanup()
 895         raise
 896
 897     try:
 898         try:
 899             if options.rebase is not None:
 900                 rebase_tree(options.rebase, rebase_branch=options.branch)
 901         except Exception:
 902             cleanup_list.append(gitroot + "/autobuild.pid")
 903             cleanup()
 904             elapsed_time = time.time() - start_time
 905             email_failure(-1, 'rebase', 'rebase', 'rebase',
 906                           'rebase on %s failed' % options.branch,
 907                           elapsed_time, log_base=options.log_base)
 908             sys.exit(1)
 909         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
 910         if options.tail:
 911             blist.start_tail()
 912         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
 913         if status != 0 or errstr != "retry":
 914             break
 915         cleanup()
 916     except Exception:
 917         cleanup()
 918         raise
 919
 920 cleanup_list.append(gitroot + "/autobuild.pid")
 921
 922 do_print(errstr)
 923
 924 blist.kill_kids()
 925 if options.tail:
 926     do_print("waiting for tail to flush")
 927     time.sleep(1)
 928
 929 elapsed_time = time.time() - start_time
 930 if status == 0:
 931     if options.passcmd is not None:
 932         do_print("Running passcmd: %s" % options.passcmd)
 933         run_cmd(options.passcmd, dir=test_master)
 934     if options.pushto is not None:
 935         push_to(options.pushto, push_branch=options.branch)
 936     if options.keeplogs or options.attach_logs:
 937         blist.tarlogs("logs.tar.gz")
 938         do_print("Logs in logs.tar.gz")
 939     if options.always_email:
 940         email_success(elapsed_time, log_base=options.log_base)
 941     blist.remove_logs()
 942     cleanup()
 943     do_print(errstr)
 944     sys.exit(0)
 945
 946 # something failed, gather a tar of the logs
 947 blist.tarlogs("logs.tar.gz")
 948
 949 if options.email is not None:
 950     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 951                   elapsed_time, log_base=options.log_base)
 952 else:
 953     elapsed_minutes = elapsed_time / 60.0
 954     print('''
 955
 956 ####################################################################
 957
 958 AUTOBUILD FAILURE
 959
 960 Your autobuild[%s] on %s failed after %.1f minutes
 961 when trying to test %s with the following error:
 962
 963    %s
 964
 965 the autobuild has been abandoned. Please fix the error and resubmit.
 966
 967 ####################################################################
 968
 969 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
 970
 971 cleanup()
 972 do_print(errstr)
 973 do_print("Logs in logs.tar.gz")
 974 sys.exit(status)