script/autobuild.py

   1 #!/usr/bin/env python
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 os.environ["PYTHONUNBUFFERED"] = "1"
  23
  24 # This speeds up testing remarkably.
  25 os.environ['TDB_NO_FSYNC'] = '1'
  26
  27 cleanup_list = []
  28
  29 builddirs = {
  30     "ctdb": "ctdb",
  31     "samba": ".",
  32     "samba-nt4": ".",
  33     "samba-fileserver": ".",
  34     "samba-xc": ".",
  35     "samba-o3": ".",
  36     "samba-ctdb": ".",
  37     "samba-libs": ".",
  38     "samba-static": ".",
  39     "samba-test-only": ".",
  40     "samba-none-env": ".",
  41     "samba-ad-dc": ".",
  42     "samba-ad-dc-2": ".",
  43     "samba-systemkrb5": ".",
  44     "samba-nopython": ".",
  45     "ldb": "lib/ldb",
  46     "tdb": "lib/tdb",
  47     "talloc": "lib/talloc",
  48     "replace": "lib/replace",
  49     "tevent": "lib/tevent",
  50     "pidl": "pidl",
  51     "pass": ".",
  52     "fail": ".",
  53     "retry": "."
  54 }
  55
  56 defaulttasks = ["ctdb",
  57                  "samba",
  58                  "samba-nt4",
  59                  "samba-fileserver",
  60                  "samba-xc",
  61                  "samba-o3",
  62                  "samba-ctdb",
  63                  "samba-libs",
  64                  "samba-static",
  65                  "samba-none-env",
  66                  "samba-ad-dc",
  67                  "samba-ad-dc-2",
  68                  "samba-systemkrb5",
  69                  "samba-nopython",
  70                  "ldb",
  71                  "tdb",
  72                  "talloc",
  73                  "replace",
  74                  "tevent",
  75                  "pidl"]
  76
  77 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  78     defaulttasks.remove("samba-o3")
  79
  80 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  81 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
  82
  83 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH"
  84 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  85 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  86 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
  87 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE ${EXTRA_PYTHON}"
  88 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
  89 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs + " ${EXTRA_PYTHON}"
  90
  91 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
  92     extra_python = ""
  93 else:
  94     extra_python = "--extra-python=/usr/bin/python3"
  95
  96 tasks = {
  97     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  98                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  99                ("make", "make all", "text/plain"),
 100                ("install", "make install", "text/plain"),
 101                ("test", "make autotest", "text/plain"),
 102                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 103                ("clean", "make clean", "text/plain")],
 104
 105     # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)'
 106     "samba": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 107                 ("make", "make -j", "text/plain"),
 108                 ("test", "make test FAIL_IMMEDIATELY=1 "
 109                  "TESTS='--exclude-env=none "
 110                  "--exclude-env=nt4_dc "
 111                  "--exclude-env=nt4_member "
 112                  "--exclude-env=ad_dc "
 113                  "--exclude-env=fl2003dc "
 114                  "--exclude-env=fl2008r2dc "
 115                  "--exclude-env=ad_member "
 116                  "--exclude-env=ad_member_idmap_rid "
 117                  "--exclude-env=ad_member_idmap_ad "
 118                  "--exclude-env=chgdcpass "
 119                  "--exclude-env=vampire_2000_dc "
 120                  "--exclude-env=fl2000dc "
 121                  "--exclude-env=fileserver'",
 122                  "text/plain"),
 123                 ("install", "make install", "text/plain"),
 124                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 125                 ("clean", "make clean", "text/plain")],
 126
 127     # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long)
 128     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 129                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 130                     ("make", "make -j", "text/plain"),
 131                     ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 132                     ("install", "make install", "text/plain"),
 133                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 134                     ("clean", "make clean", "text/plain")],
 135
 136     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 137     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 138                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json-audit --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 139                            ("make", "make -j", "text/plain"),
 140                            ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=fileserver'", "text/plain"),
 141                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 142
 143     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 144     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 145                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 146                       ("make", "make -j", "text/plain"),
 147                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='"
 148                        "--include-env=ad_dc "
 149                        "--include-env=fl2003dc "
 150                        "--include-env=fl2008r2dc "
 151                        "--include-env=ad_member "
 152                        "--include-env=ad_member_idmap_rid "
 153                        "--include-env=ad_member_idmap_ad'", "text/plain"),
 154                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 155
 156     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 157     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 158                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 159                         ("make", "make -j", "text/plain"),
 160                         ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=chgdcpass --include-env=vampire_2000_dc --include-env=fl2000dc'", "text/plain"),
 161                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 162
 163     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 164                           ("make", "make -j", "text/plain"),
 165                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
 166
 167     # Test cross-compile infrastructure
 168     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 169                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 170                    ("configure-cross-execute", "./configure.developer -b ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 171                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 172                    ("configure-cross-answers", "./configure.developer -b ./bin-xa --cross-compile" \
 173                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 174                    ("compare-results", "script/compare_cc_results.py ./bin/c4che/default.cache.py ./bin-xe/c4che/default.cache.py ./bin-xa/c4che/default.cache.py", "text/plain")],
 175
 176     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 177     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 178                    ("configure", "ADDITIONAL_CFLAGS='-O3' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 179                    ("make", "make -j", "text/plain"),
 180                    ("test", "make quicktest FAIL_IMMEDIATELY=1 TESTS='--include-env=ad_dc'", "text/plain"),
 181                    ("install", "make install", "text/plain"),
 182                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 183                    ("clean", "make clean", "text/plain")],
 184
 185     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 186
 187                      # make sure we have tdb around:
 188                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 189                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 190                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 191
 192
 193                      # build samba with cluster support (also building ctdb):
 194                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 195                      ("samba-make", "make", "text/plain"),
 196                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 197                      ("samba-install", "make install", "text/plain"),
 198                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 199
 200                      # clean up:
 201                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 202                      ("clean", "make clean", "text/plain"),
 203                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
 204
 205     "samba-libs": [
 206                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 207                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 208                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 209                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 210
 211                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 212                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 213                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 214
 215                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 216                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 217                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 218
 219                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 220                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 221                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 222
 223                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 224                       ("nondevel-make", "make -j", "text/plain"),
 225                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 226                       ("nondevel-install", "make install", "text/plain"),
 227                       ("nondevel-dist", "make dist", "text/plain"),
 228
 229                       # retry with all modules shared
 230                       ("allshared-distclean", "make distclean", "text/plain"),
 231                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 232                       ("allshared-make", "make -j", "text/plain")],
 233
 234     "samba-none-env": [
 235                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 236                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 237                       ("make", "make -j", "text/plain"),
 238                       ("test", "make test "
 239                        "FAIL_IMMEDIATELY=1 "
 240                        "TESTS='--include-env=none'",
 241                        "text/plain")],
 242
 243     "samba-static": [
 244                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 245                       # build with all modules static
 246                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 247                       ("allstatic-make", "make -j", "text/plain"),
 248                       ("allstatic-test", "make test "
 249                        "FAIL_IMMEDIATELY=1 "
 250                        "TESTS='samba3.smb2.create.*nt4_dc'",
 251                        "text/plain"),
 252
 253                       # retry without any required modules
 254                       ("none-distclean", "make distclean", "text/plain"),
 255                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 256                       ("none-make", "make -j", "text/plain"),
 257
 258                       # retry with nonshared smbd and smbtorture
 259                       ("nonshared-distclean", "make distclean", "text/plain"),
 260                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 261                       ("nonshared-make", "make -j", "text/plain")],
 262
 263     "samba-systemkrb5": [
 264                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 265                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 266                       ("make", "make -j", "text/plain"),
 267                       # we currently cannot run a full make test, a limited list of tests could be run
 268                       # via "make test TESTS=sometests"
 269                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=ktest'", "text/plain"),
 270                       ("install", "make install", "text/plain"),
 271                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 272                       ("clean", "make clean", "text/plain")
 273                       ],
 274
 275     # Test Samba without python still builds.  When this test fails
 276     # due to more use of Python, the expectations is that the newly
 277     # failing part of the code should be disabled when
 278     # --disable-python is set (rather than major work being done to
 279     # support this environment).  The target here is for vendors
 280     # shipping a minimal smbd.
 281     "samba-nopython": [
 282                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 283                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 284                       ("make", "make -j", "text/plain"),
 285                       ("install", "make install", "text/plain"),
 286                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 287                       ("clean", "make clean", "text/plain"),
 288
 289                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 290                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 291                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 292
 293                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 294                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 295                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 296
 297                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 298                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 299                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 300
 301                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 302                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 303                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 304
 305                       # retry against installed library packages
 306                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 307                       ("libs-make", "make -j", "text/plain"),
 308                       ("libs-install", "make install", "text/plain"),
 309                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 310                       ("libs-clean", "make clean", "text/plain")
 311                       ],
 312
 313
 314
 315     "ldb": [
 316               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 317               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 318               ("make", "make", "text/plain"),
 319               ("install", "make install", "text/plain"),
 320               ("test", "make test", "text/plain"),
 321               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 322               ("make-no-lmdb", "make", "text/plain"),
 323               ("install-no-lmdb", "make install", "text/plain"),
 324               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 325               ("distcheck", "make distcheck", "text/plain"),
 326               ("clean", "make clean", "text/plain")],
 327
 328     "tdb": [
 329               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 330               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 331               ("make", "make", "text/plain"),
 332               ("install", "make install", "text/plain"),
 333               ("test", "make test", "text/plain"),
 334               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 335               ("distcheck", "make distcheck", "text/plain"),
 336               ("clean", "make clean", "text/plain")],
 337
 338     "talloc": [
 339                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 340                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 341                  ("make", "make", "text/plain"),
 342                  ("install", "make install", "text/plain"),
 343                  ("test", "make test", "text/plain"),
 344                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 345                  ("distcheck", "make distcheck", "text/plain"),
 346                  ("clean", "make clean", "text/plain")],
 347
 348     "replace": [
 349                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 350                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 351                   ("make", "make", "text/plain"),
 352                   ("install", "make install", "text/plain"),
 353                   ("test", "make test", "text/plain"),
 354                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 355                   ("distcheck", "make distcheck", "text/plain"),
 356                   ("clean", "make clean", "text/plain")],
 357
 358     "tevent": [
 359                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 360                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 361                  ("make", "make", "text/plain"),
 362                  ("install", "make install", "text/plain"),
 363                  ("test", "make test", "text/plain"),
 364                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 365                  ("distcheck", "make distcheck", "text/plain"),
 366                  ("clean", "make clean", "text/plain")],
 367
 368     "pidl": [
 369         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 370         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 371         ("touch", "touch *.yp", "text/plain"),
 372         ("make", "make", "text/plain"),
 373         ("test", "make test", "text/plain"),
 374         ("install", "make install", "text/plain"),
 375         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 376         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 377         ("clean", "make clean", "text/plain")],
 378
 379     # these are useful for debugging autobuild
 380     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
 381     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
 382 }
 383
 384
 385 def do_print(msg):
 386     print("%s" % msg)
 387     sys.stdout.flush()
 388     sys.stderr.flush()
 389
 390
 391 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 392     if show is None:
 393         show = options.verbose
 394     if show:
 395         do_print("Running: '%s' in '%s'" % (cmd, dir))
 396     if output:
 397         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir).communicate()[0]
 398     elif checkfail:
 399         return check_call(cmd, shell=True, cwd=dir)
 400     else:
 401         return call(cmd, shell=True, cwd=dir)
 402
 403
 404 class builder(object):
 405     '''handle build of one directory'''
 406
 407     def __init__(self, name, sequence, cp=True, py3=False):
 408         self.name = name
 409         self.py3 = py3
 410         self.dir = builddirs[name]
 411
 412         self.tag = self.name.replace('/', '_')
 413         self.sequence = sequence
 414         self.next = 0
 415         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 416         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 417         if options.verbose:
 418             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 419             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 420         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 421         self.stdout = open(self.stdout_path, 'w')
 422         self.stderr = open(self.stderr_path, 'w')
 423         self.stdin  = open("/dev/null", 'r')
 424         self.sdir = "%s/%s" % (testbase, self.tag)
 425         self.prefix = "%s/%s" % (test_prefix, self.tag)
 426         run_cmd("rm -rf %s" % self.sdir)
 427         run_cmd("rm -rf %s" % self.prefix)
 428         if cp:
 429             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 430         else:
 431             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 432         self.start_next()
 433
 434     def start_next(self):
 435         if self.next == len(self.sequence):
 436             if not options.nocleanup:
 437                 run_cmd("rm -rf %s" % self.sdir)
 438                 run_cmd("rm -rf %s" % self.prefix)
 439             do_print('%s: Completed OK' % self.name)
 440             self.done = True
 441             return
 442         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 443         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix))
 444         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 445         self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
 446         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 447         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 448 #        if self.output_mime_type == "text/x-subunit":
 449 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 450         do_print('%s: [%s] Running %s' % (self.name, self.stage, self.cmd))
 451         cwd = os.getcwd()
 452         os.chdir("%s/%s" % (self.sdir, self.dir))
 453         self.proc = Popen(self.cmd, shell=True,
 454                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 455         os.chdir(cwd)
 456         self.next += 1
 457
 458
 459 class buildlist(object):
 460     '''handle build of multiple directories'''
 461
 462     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 463         global tasks
 464         self.tlist = []
 465         self.tail_proc = None
 466         self.retry = None
 467         if tasknames == []:
 468             if options.restrict_tests:
 469                 tasknames = ["samba-test-only"]
 470             else:
 471                 tasknames = defaulttasks
 472         else:
 473             # If we are only running one test,
 474             # do not sleep randomly to wait for it to start
 475             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 476
 477         for n in tasknames:
 478             if n not in tasks and n.endswith("-py3"):
 479                 b = builder(n,
 480                             tasks[n[:-4]],
 481                             cp=n is not "pidl",
 482                             py3=True)
 483             else:
 484                 b = builder(n, tasks[n], cp=n is not "pidl")
 485             self.tlist.append(b)
 486         if options.retry:
 487             rebase_remote = "rebaseon"
 488             retry_task = [("retry",
 489                             '''set -e
 490                             git remote add -t %s %s %s
 491                             git fetch %s
 492                             while :; do
 493                               sleep 60
 494                               git describe %s/%s > old_remote_branch.desc
 495                               git fetch %s
 496                               git describe %s/%s > remote_branch.desc
 497                               diff old_remote_branch.desc remote_branch.desc
 498                             done
 499                            ''' % (
 500                                 rebase_branch, rebase_remote, rebase_url,
 501                                rebase_remote,
 502                                rebase_remote, rebase_branch,
 503                                rebase_remote,
 504                                rebase_remote, rebase_branch
 505                             ),
 506                             "test/plain")]
 507
 508             self.retry = builder('retry', retry_task, cp=False)
 509             self.need_retry = False
 510
 511     def kill_kids(self):
 512         if self.tail_proc is not None:
 513             self.tail_proc.terminate()
 514             self.tail_proc.wait()
 515             self.tail_proc = None
 516         if self.retry is not None:
 517             self.retry.proc.terminate()
 518             self.retry.proc.wait()
 519             self.retry = None
 520         for b in self.tlist:
 521             if b.proc is not None:
 522                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 523                 b.proc.terminate()
 524                 b.proc.wait()
 525                 b.proc = None
 526
 527     def wait_one(self):
 528         while True:
 529             none_running = True
 530             for b in self.tlist:
 531                 if b.proc is None:
 532                     continue
 533                 none_running = False
 534                 b.status = b.proc.poll()
 535                 if b.status is None:
 536                     continue
 537                 b.proc = None
 538                 return b
 539             if options.retry:
 540                 ret = self.retry.proc.poll()
 541                 if ret is not None:
 542                     self.need_retry = True
 543                     self.retry = None
 544                     return None
 545             if none_running:
 546                 return None
 547             time.sleep(0.1)
 548
 549     def run(self):
 550         while True:
 551             b = self.wait_one()
 552             if options.retry and self.need_retry:
 553                 self.kill_kids()
 554                 do_print("retry needed")
 555                 return (0, None, None, None, "retry")
 556             if b is None:
 557                 break
 558             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 559                 self.kill_kids()
 560                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 561             b.start_next()
 562         self.kill_kids()
 563         return (0, None, None, None, "All OK")
 564
 565     def write_system_info(self):
 566         filename = 'system-info.txt'
 567         f = open(filename, 'w')
 568         for cmd in ['uname -a', 'free', 'cat /proc/cpuinfo',
 569                     'cc --version', 'df -m .', 'df -m %s' % testbase]:
 570             print('### %s' % cmd, file=f)
 571             print(run_cmd(cmd, output=True, checkfail=False), file=f)
 572             print(file=f)
 573         f.close()
 574         return filename
 575
 576     def tarlogs(self, fname):
 577         tar = tarfile.open(fname, "w:gz")
 578         for b in self.tlist:
 579             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 580             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 581         if os.path.exists("autobuild.log"):
 582             tar.add("autobuild.log")
 583         sys_info = self.write_system_info()
 584         tar.add(sys_info)
 585         tar.close()
 586
 587     def remove_logs(self):
 588         for b in self.tlist:
 589             os.unlink(b.stdout_path)
 590             os.unlink(b.stderr_path)
 591
 592     def start_tail(self):
 593         cwd = os.getcwd()
 594         cmd = "tail -f *.stdout *.stderr"
 595         os.chdir(gitroot)
 596         self.tail_proc = Popen(cmd, shell=True)
 597         os.chdir(cwd)
 598
 599
 600 def cleanup():
 601     if options.nocleanup:
 602         return
 603     run_cmd("stat %s || true" % test_tmpdir, show=True)
 604     run_cmd("stat %s" % testbase, show=True)
 605     do_print("Cleaning up ....")
 606     for d in cleanup_list:
 607         run_cmd("rm -rf %s" % d)
 608
 609
 610 def find_git_root():
 611     '''get to the top of the git repo'''
 612     p = os.getcwd()
 613     while p != '/':
 614         if os.path.isdir(os.path.join(p, ".git")):
 615             return p
 616         p = os.path.abspath(os.path.join(p, '..'))
 617     return None
 618
 619
 620 def daemonize(logfile):
 621     pid = os.fork()
 622     if pid == 0:  # Parent
 623         os.setsid()
 624         pid = os.fork()
 625         if pid != 0:  # Actual daemon
 626             os._exit(0)
 627     else:  # Grandparent
 628         os._exit(0)
 629
 630     import resource      # Resource usage information.
 631     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 632     if maxfd == resource.RLIM_INFINITY:
 633         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 634     for fd in range(0, maxfd):
 635         try:
 636             os.close(fd)
 637         except OSError:
 638             pass
 639     os.open(logfile, os.O_RDWR | os.O_CREAT)
 640     os.dup2(0, 1)
 641     os.dup2(0, 2)
 642
 643
 644 def write_pidfile(fname):
 645     '''write a pid file, cleanup on exit'''
 646     f = open(fname, mode='w')
 647     f.write("%u\n" % os.getpid())
 648     f.close()
 649
 650
 651 def rebase_tree(rebase_url, rebase_branch="master"):
 652     rebase_remote = "rebaseon"
 653     do_print("Rebasing on %s" % rebase_url)
 654     run_cmd("git describe HEAD", show=True, dir=test_master)
 655     run_cmd("git remote add -t %s %s %s" %
 656             (rebase_branch, rebase_remote, rebase_url),
 657             show=True, dir=test_master)
 658     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 659     if options.fix_whitespace:
 660         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 661                 (rebase_remote, rebase_branch),
 662                 show=True, dir=test_master)
 663     else:
 664         run_cmd("git rebase --force-rebase %s/%s" %
 665                 (rebase_remote, rebase_branch),
 666                 show=True, dir=test_master)
 667     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 668                    (rebase_remote, rebase_branch),
 669                    dir=test_master, output=True)
 670     if diff == '':
 671         do_print("No differences between HEAD and %s/%s - exiting" %
 672                  (rebase_remote, rebase_branch))
 673         sys.exit(0)
 674     run_cmd("git describe %s/%s" %
 675             (rebase_remote, rebase_branch),
 676             show=True, dir=test_master)
 677     run_cmd("git describe HEAD", show=True, dir=test_master)
 678     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 679             (rebase_remote, rebase_branch),
 680             show=True, dir=test_master)
 681
 682
 683 def push_to(push_url, push_branch="master"):
 684     push_remote = "pushto"
 685     do_print("Pushing to %s" % push_url)
 686     if options.mark:
 687         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 688         run_cmd("git commit --amend -c HEAD", dir=test_master)
 689         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 690         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 691     run_cmd("git remote add -t %s %s %s" %
 692             (push_branch, push_remote, push_url),
 693             show=True, dir=test_master)
 694     run_cmd("git push %s +HEAD:%s" %
 695             (push_remote, push_branch),
 696             show=True, dir=test_master)
 697
 698
 699 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 700
 701 gitroot = find_git_root()
 702 if gitroot is None:
 703     raise Exception("Failed to find git root")
 704
 705 parser = OptionParser()
 706 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 707 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 708 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 709 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 710                   default=def_testbase)
 711 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 712 parser.add_option("", "--verbose", help="show all commands as they are run",
 713                   default=False, action="store_true")
 714 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 715                   default=None, type='str')
 716 parser.add_option("", "--pushto", help="push to a git url on success",
 717                   default=None, type='str')
 718 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 719                   default=False, action="store_true")
 720 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 721                   default=False, action="store_true")
 722 parser.add_option("", "--retry", help="automatically retry if master changes",
 723                   default=False, action="store_true")
 724 parser.add_option("", "--email", help="send email to the given address on failure",
 725                   type='str', default=None)
 726 parser.add_option("", "--email-from", help="send email from the given address",
 727                   type='str', default="autobuild@samba.org")
 728 parser.add_option("", "--email-server", help="send email via the given server",
 729                   type='str', default='localhost')
 730 parser.add_option("", "--always-email", help="always send email, even on success",
 731                   action="store_true")
 732 parser.add_option("", "--daemon", help="daemonize after initial setup",
 733                   action="store_true")
 734 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 735                   default="master", type='str')
 736 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 737                   default=gitroot, type='str')
 738 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 739                   default=False, action="store_true")
 740 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 741                   default='')
 742
 743
 744 def send_email(subject, text, log_tar):
 745     if options.email is None:
 746         do_print("not sending email because the recipient is not set")
 747         do_print("the text content would have been:\n\nSubject: %s\n\nTs" %
 748                  (subject, text))
 749         return
 750     outer = MIMEMultipart()
 751     outer['Subject'] = subject
 752     outer['To'] = options.email
 753     outer['From'] = options.email_from
 754     outer['Date'] = email.utils.formatdate(localtime=True)
 755     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 756     outer.attach(MIMEText(text, 'plain'))
 757     if options.attach_logs:
 758         fp = open(log_tar, 'rb')
 759         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 760         fp.close()
 761         # Set the filename parameter
 762         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 763         outer.attach(msg)
 764     content = outer.as_string()
 765     s = smtplib.SMTP(options.email_server)
 766     s.sendmail(options.email_from, [options.email], content)
 767     s.set_debuglevel(1)
 768     s.quit()
 769
 770
 771 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 772                   elapsed_time, log_base=None, add_log_tail=True):
 773     '''send an email to options.email about the failure'''
 774     elapsed_minutes = elapsed_time / 60.0
 775     user = os.getenv("USER")
 776     if log_base is None:
 777         log_base = gitroot
 778     text = '''
 779 Dear Developer,
 780
 781 Your autobuild on %s failed after %.1f minutes
 782 when trying to test %s with the following error:
 783
 784    %s
 785
 786 the autobuild has been abandoned. Please fix the error and resubmit.
 787
 788 A summary of the autobuild process is here:
 789
 790   %s/autobuild.log
 791 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 792
 793     if options.restrict_tests:
 794         text += """
 795 The build was restricted to tests matching %s\n""" % options.restrict_tests
 796
 797     if failed_task != 'rebase':
 798         text += '''
 799 You can see logs of the failed task here:
 800
 801   %s/%s.stdout
 802   %s/%s.stderr
 803
 804 or you can get full logs of all tasks in this job here:
 805
 806   %s/logs.tar.gz
 807
 808 The top commit for the tree that was built was:
 809
 810 %s
 811
 812 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 813
 814     if add_log_tail:
 815         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 816         lines = f.readlines()
 817         log_tail = "".join(lines[-50:])
 818         num_lines = len(lines)
 819         if num_lines < 50:
 820             # Also include stderr (compile failures) if < 50 lines of stdout
 821             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 822             log_tail += "".join(f.readlines()[-(50 - num_lines):])
 823
 824         text += '''
 825 The last 50 lines of log messages:
 826
 827 %s
 828     ''' % log_tail
 829         f.close()
 830
 831     logs = os.path.join(gitroot, 'logs.tar.gz')
 832     send_email('autobuild[%s] failure on %s for task %s during %s'
 833                % (options.branch, platform.node(), failed_task, failed_stage),
 834                text, logs)
 835
 836
 837 def email_success(elapsed_time, log_base=None):
 838     '''send an email to options.email about a successful build'''
 839     user = os.getenv("USER")
 840     if log_base is None:
 841         log_base = gitroot
 842     text = '''
 843 Dear Developer,
 844
 845 Your autobuild on %s has succeeded after %.1f minutes.
 846
 847 ''' % (platform.node(), elapsed_time / 60.)
 848
 849     if options.restrict_tests:
 850         text += """
 851 The build was restricted to tests matching %s\n""" % options.restrict_tests
 852
 853     if options.keeplogs:
 854         text += '''
 855
 856 you can get full logs of all tasks in this job here:
 857
 858   %s/logs.tar.gz
 859
 860 ''' % log_base
 861
 862     text += '''
 863 The top commit for the tree that was built was:
 864
 865 %s
 866 ''' % top_commit_msg
 867
 868     logs = os.path.join(gitroot, 'logs.tar.gz')
 869     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 870                text, logs)
 871
 872
 873 (options, args) = parser.parse_args()
 874
 875 if options.retry:
 876     if options.rebase is None:
 877         raise Exception('You can only use --retry if you also rebase')
 878
 879 testbase = "%s/b%u" % (options.testbase, os.getpid())
 880 test_master = "%s/master" % testbase
 881 test_prefix = "%s/prefix" % testbase
 882 test_tmpdir = "%s/tmp" % testbase
 883 os.environ['TMPDIR'] = test_tmpdir
 884
 885 # get the top commit message, for emails
 886 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 887
 888 try:
 889     os.makedirs(testbase)
 890 except Exception as reason:
 891     raise Exception("Unable to create %s : %s" % (testbase, reason))
 892 cleanup_list.append(testbase)
 893
 894 if options.daemon:
 895     logfile = os.path.join(testbase, "log")
 896     do_print("Forking into the background, writing progress to %s" % logfile)
 897     daemonize(logfile)
 898
 899 write_pidfile(gitroot + "/autobuild.pid")
 900
 901 start_time = time.time()
 902
 903 while True:
 904     try:
 905         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 906         os.makedirs(test_tmpdir)
 907         # The waf uninstall code removes empty directories all the way
 908         # up the tree.  Creating a file in test_tmpdir stops it from
 909         # being removed.
 910         run_cmd("touch %s" % os.path.join(test_tmpdir,
 911                                           ".directory-is-not-empty"), show=True)
 912         run_cmd("stat %s" % test_tmpdir, show=True)
 913         run_cmd("stat %s" % testbase, show=True)
 914         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 915     except Exception:
 916         cleanup()
 917         raise
 918
 919     try:
 920         try:
 921             if options.rebase is not None:
 922                 rebase_tree(options.rebase, rebase_branch=options.branch)
 923         except Exception:
 924             cleanup_list.append(gitroot + "/autobuild.pid")
 925             cleanup()
 926             elapsed_time = time.time() - start_time
 927             email_failure(-1, 'rebase', 'rebase', 'rebase',
 928                           'rebase on %s failed' % options.branch,
 929                           elapsed_time, log_base=options.log_base)
 930             sys.exit(1)
 931         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
 932         if options.tail:
 933             blist.start_tail()
 934         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
 935         if status != 0 or errstr != "retry":
 936             break
 937         cleanup()
 938     except Exception:
 939         cleanup()
 940         raise
 941
 942 cleanup_list.append(gitroot + "/autobuild.pid")
 943
 944 do_print(errstr)
 945
 946 blist.kill_kids()
 947 if options.tail:
 948     do_print("waiting for tail to flush")
 949     time.sleep(1)
 950
 951 elapsed_time = time.time() - start_time
 952 if status == 0:
 953     if options.passcmd is not None:
 954         do_print("Running passcmd: %s" % options.passcmd)
 955         run_cmd(options.passcmd, dir=test_master)
 956     if options.pushto is not None:
 957         push_to(options.pushto, push_branch=options.branch)
 958     if options.keeplogs or options.attach_logs:
 959         blist.tarlogs("logs.tar.gz")
 960         do_print("Logs in logs.tar.gz")
 961     if options.always_email:
 962         email_success(elapsed_time, log_base=options.log_base)
 963     blist.remove_logs()
 964     cleanup()
 965     do_print(errstr)
 966     sys.exit(0)
 967
 968 # something failed, gather a tar of the logs
 969 blist.tarlogs("logs.tar.gz")
 970
 971 if options.email is not None:
 972     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 973                   elapsed_time, log_base=options.log_base)
 974 else:
 975     elapsed_minutes = elapsed_time / 60.0
 976     print('''
 977
 978 ####################################################################
 979
 980 AUTOBUILD FAILURE
 981
 982 Your autobuild[%s] on %s failed after %.1f minutes
 983 when trying to test %s with the following error:
 984
 985    %s
 986
 987 the autobuild has been abandoned. Please fix the error and resubmit.
 988
 989 ####################################################################
 990
 991 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
 992
 993 cleanup()
 994 do_print(errstr)
 995 do_print("Logs in logs.tar.gz")
 996 sys.exit(status)