2 # run tests on all Samba subprojects and push to a git tree on success
3 # Copyright Andrew Tridgell 2010
4 # released under GNU GPL v3 or later
6 from __future__
import print_function
7 from subprocess
import call
, check_call
,Popen
, PIPE
8 import os
, tarfile
, sys
, time
9 from optparse
import OptionParser
12 from email
.mime
.text
import MIMEText
13 from email
.mime
.base
import MIMEBase
14 from email
.mime
.application
import MIMEApplication
15 from email
.mime
.multipart
import MIMEMultipart
16 from distutils
.sysconfig
import get_python_lib
19 os
.environ
["PYTHONUNBUFFERED"] = "1"
21 # This speeds up testing remarkably.
22 os
.environ
['TDB_NO_FSYNC'] = '1'
30 "samba-fileserver" : ".",
36 "samba-test-only" : ".",
37 "samba-none-env" : ".",
39 "samba-ad-dc-2" : ".",
40 "samba-systemkrb5" : ".",
41 "samba-nopython" : ".",
44 "talloc" : "lib/talloc",
45 "replace" : "lib/replace",
46 "tevent" : "lib/tevent",
53 defaulttasks
= [ "ctdb",
74 if os
.environ
.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
75 defaulttasks
.remove("samba-o3")
77 ctdb_configure_params
= " --enable-developer --picky-developer ${PREFIX}"
78 samba_configure_params
= " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
80 samba_libs_envvars
= "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH"
81 samba_libs_envvars
+= " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
82 samba_libs_envvars
+= " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
83 samba_libs_configure_base
= samba_libs_envvars
+ " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
84 samba_libs_configure_libs
= samba_libs_configure_base
+ " --bundled-libraries=cmocka,NONE ${EXTRA_PYTHON}"
85 samba_libs_configure_bundled_libs
= " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent"
86 samba_libs_configure_samba
= samba_libs_configure_base
+ samba_libs_configure_bundled_libs
+ " ${EXTRA_PYTHON}"
88 if os
.environ
.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
91 extra_python
= "--extra-python=/usr/bin/python3"
94 "ctdb" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
95 ("configure", "./configure " + ctdb_configure_params
, "text/plain"),
96 ("make", "make all", "text/plain"),
97 ("install", "make install", "text/plain"),
98 ("test", "make autotest", "text/plain"),
99 ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
100 ("clean", "make clean", "text/plain") ],
102 # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)'
103 "samba" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params
, "text/plain"),
104 ("make", "make -j", "text/plain"),
105 ("test", "make test FAIL_IMMEDIATELY=1 "
106 "TESTS='--exclude-env=none "
107 "--exclude-env=nt4_dc "
108 "--exclude-env=nt4_member "
109 "--exclude-env=ad_dc "
110 "--exclude-env=fl2003dc "
111 "--exclude-env=fl2008r2dc "
112 "--exclude-env=ad_member "
113 "--exclude-env=ad_member_idmap_rid "
114 "--exclude-env=ad_member_idmap_ad "
115 "--exclude-env=chgdcpass "
116 "--exclude-env=vampire_2000_dc "
117 "--exclude-env=fl2000dc "
118 "--exclude-env=fileserver'",
120 ("install", "make install", "text/plain"),
121 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
122 ("clean", "make clean", "text/plain") ],
124 # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long)
125 "samba-nt4" : [ ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params
, "text/plain"),
126 ("make", "make -j", "text/plain"),
127 ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
128 ("install", "make install", "text/plain"),
129 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
130 ("clean", "make clean", "text/plain") ],
132 # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
133 "samba-fileserver" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
134 ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params
, "text/plain"),
135 ("make", "make -j", "text/plain"),
136 ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=fileserver'", "text/plain"),
137 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
139 # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
140 "samba-ad-dc" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
141 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params
, "text/plain"),
142 ("make", "make -j", "text/plain"),
143 ("test", "make test FAIL_IMMEDIATELY=1 TESTS='"
144 "--include-env=ad_dc "
145 "--include-env=fl2003dc "
146 "--include-env=fl2008r2dc "
147 "--include-env=ad_member "
148 "--include-env=ad_member_idmap_rid "
149 "--include-env=ad_member_idmap_ad'", "text/plain"),
150 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
152 # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
153 "samba-ad-dc-2" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
154 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params
, "text/plain"),
155 ("make", "make -j", "text/plain"),
156 ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=chgdcpass --include-env=vampire_2000_dc --include-env=fl2000dc'", "text/plain"),
157 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
159 "samba-test-only" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params
, "text/plain"),
160 ("make", "make -j", "text/plain"),
161 ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"',"text/plain") ],
163 # Test cross-compile infrastructure
164 "samba-xc" : [ ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params
, "text/plain"),
165 ("configure-cross-execute", "./configure.developer -b ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
166 " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params
, "text/plain"),
167 ("configure-cross-answers", "./configure.developer -b ./bin-xa --cross-compile" \
168 " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params
, "text/plain"),
169 ("compare-results", "script/compare_cc_results.py ./bin/c4che/default.cache.py ./bin-xe/c4che/default.cache.py ./bin-xa/c4che/default.cache.py", "text/plain")],
171 # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
172 "samba-o3" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
173 ("configure", "ADDITIONAL_CFLAGS='-O3' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params
, "text/plain"),
174 ("make", "make -j", "text/plain"),
175 ("test", "make quicktest FAIL_IMMEDIATELY=1 TESTS='--include-env=ad_dc'", "text/plain"),
176 ("install", "make install", "text/plain"),
177 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
178 ("clean", "make clean", "text/plain") ],
180 "samba-ctdb" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
182 # make sure we have tdb around:
183 ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
184 ("tdb-make", "cd lib/tdb && make", "text/plain"),
185 ("tdb-install", "cd lib/tdb && make install", "text/plain"),
188 # build samba with cluster support (also building ctdb):
189 ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
190 ("samba-make", "make", "text/plain"),
191 ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
192 ("samba-install", "make install", "text/plain"),
193 ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
196 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
197 ("clean", "make clean", "text/plain"),
198 ("ctdb-clean", "cd ./ctdb && make clean", "text/plain") ],
201 ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
202 ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs
, "text/plain"),
203 ("talloc-make", "cd lib/talloc && make", "text/plain"),
204 ("talloc-install", "cd lib/talloc && make install", "text/plain"),
206 ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs
, "text/plain"),
207 ("tdb-make", "cd lib/tdb && make", "text/plain"),
208 ("tdb-install", "cd lib/tdb && make install", "text/plain"),
210 ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs
, "text/plain"),
211 ("tevent-make", "cd lib/tevent && make", "text/plain"),
212 ("tevent-install", "cd lib/tevent && make install", "text/plain"),
214 ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs
, "text/plain"),
215 ("ldb-make", "cd lib/ldb && make", "text/plain"),
216 ("ldb-install", "cd lib/ldb && make install", "text/plain"),
218 ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
219 ("nondevel-make", "make -j", "text/plain"),
220 ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
221 ("nondevel-install", "make install", "text/plain"),
222 ("nondevel-dist", "make dist", "text/plain"),
224 # retry with all modules shared
225 ("allshared-distclean", "make distclean", "text/plain"),
226 ("allshared-configure", samba_libs_configure_samba
+ " --with-shared-modules=ALL", "text/plain"),
227 ("allshared-make", "make -j", "text/plain")],
230 ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
231 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params
, "text/plain"),
232 ("make", "make -j", "text/plain"),
233 ("test", "make test "
234 "FAIL_IMMEDIATELY=1 "
235 "TESTS='--include-env=none'",
239 ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
240 # build with all modules static
241 ("allstatic-configure", "./configure.developer " + samba_configure_params
+ " --with-static-modules=ALL", "text/plain"),
242 ("allstatic-make", "make -j", "text/plain"),
244 # retry without any required modules
245 ("none-distclean", "make distclean", "text/plain"),
246 ("none-configure", "./configure.developer " + samba_configure_params
+ " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
247 ("none-make", "make -j", "text/plain"),
249 # retry with nonshared smbd and smbtorture
250 ("nonshared-distclean", "make distclean", "text/plain"),
251 ("nonshared-configure", "./configure.developer " + samba_configure_params
+ " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
252 ("nonshared-make", "make -j", "text/plain")],
254 "samba-systemkrb5" : [
255 ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
256 ("configure", "./configure.developer " + samba_configure_params
+ " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
257 ("make", "make -j", "text/plain"),
258 # we currently cannot run a full make test, a limited list of tests could be run
259 # via "make test TESTS=sometests"
260 ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=ktest'", "text/plain"),
261 ("install", "make install", "text/plain"),
262 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
263 ("clean", "make clean", "text/plain")
266 # Test Samba without python still builds. When this test fails
267 # due to more use of Python, the expectations is that the newly
268 # failing part of the code should be disabled when
269 # --disable-python is set (rather than major work being done to
270 # support this environment). The target here is for vendors
271 # shipping a minimal smbd.
273 ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
274 ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
275 ("make", "make -j", "text/plain"),
276 ("install", "make install", "text/plain"),
277 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
278 ("clean", "make clean", "text/plain"),
280 ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base
+ " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
281 ("talloc-make", "cd lib/talloc && make", "text/plain"),
282 ("talloc-install", "cd lib/talloc && make install", "text/plain"),
284 ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base
+ " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
285 ("tdb-make", "cd lib/tdb && make", "text/plain"),
286 ("tdb-install", "cd lib/tdb && make install", "text/plain"),
288 ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base
+ " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
289 ("tevent-make", "cd lib/tevent && make", "text/plain"),
290 ("tevent-install", "cd lib/tevent && make install", "text/plain"),
292 ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base
+ " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
293 ("ldb-make", "cd lib/ldb && make", "text/plain"),
294 ("ldb-install", "cd lib/ldb && make install", "text/plain"),
296 # retry against installed library packages
297 ("libs-configure", samba_libs_configure_base
+ samba_libs_configure_bundled_libs
+ " --disable-python --without-ad-dc", "text/plain"),
298 ("libs-make", "make -j", "text/plain"),
299 ("libs-install", "make install", "text/plain"),
300 ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
301 ("libs-clean", "make clean", "text/plain")
307 ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
308 ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
309 ("make", "make", "text/plain"),
310 ("install", "make install", "text/plain"),
311 ("test", "make test", "text/plain"),
312 ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
313 ("make-no-lmdb", "make", "text/plain"),
314 ("install-no-lmdb", "make install", "text/plain"),
315 ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
316 ("distcheck", "make distcheck", "text/plain"),
317 ("clean", "make clean", "text/plain") ],
320 ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
321 ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
322 ("make", "make", "text/plain"),
323 ("install", "make install", "text/plain"),
324 ("test", "make test", "text/plain"),
325 ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
326 ("distcheck", "make distcheck", "text/plain"),
327 ("clean", "make clean", "text/plain") ],
330 ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
331 ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
332 ("make", "make", "text/plain"),
333 ("install", "make install", "text/plain"),
334 ("test", "make test", "text/plain"),
335 ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
336 ("distcheck", "make distcheck", "text/plain"),
337 ("clean", "make clean", "text/plain") ],
340 ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
341 ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
342 ("make", "make", "text/plain"),
343 ("install", "make install", "text/plain"),
344 ("test", "make test", "text/plain"),
345 ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
346 ("distcheck", "make distcheck", "text/plain"),
347 ("clean", "make clean", "text/plain") ],
350 ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
351 ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
352 ("make", "make", "text/plain"),
353 ("install", "make install", "text/plain"),
354 ("test", "make test", "text/plain"),
355 ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
356 ("distcheck", "make distcheck", "text/plain"),
357 ("clean", "make clean", "text/plain") ],
360 ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
361 ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
362 ("touch", "touch *.yp", "text/plain"),
363 ("make", "make", "text/plain"),
364 ("test", "make test", "text/plain"),
365 ("install", "make install", "text/plain"),
366 ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
367 ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
368 ("clean", "make clean", "text/plain") ],
370 # these are useful for debugging autobuild
371 'pass' : [ ("pass", 'echo passing && /bin/true', "text/plain") ],
372 'fail' : [ ("fail", 'echo failing && /bin/false', "text/plain") ]
380 def run_cmd(cmd
, dir=".", show
=None, output
=False, checkfail
=True):
382 show
= options
.verbose
384 do_print("Running: '%s' in '%s'" % (cmd
, dir))
386 return Popen([cmd
], shell
=True, stdout
=PIPE
, cwd
=dir).communicate()[0]
388 return check_call(cmd
, shell
=True, cwd
=dir)
390 return call(cmd
, shell
=True, cwd
=dir)
393 class builder(object):
394 '''handle build of one directory'''
396 def __init__(self
, name
, sequence
, cp
=True):
398 self
.dir = builddirs
[name
]
400 self
.tag
= self
.name
.replace('/', '_')
401 self
.sequence
= sequence
403 self
.stdout_path
= "%s/%s.stdout" % (gitroot
, self
.tag
)
404 self
.stderr_path
= "%s/%s.stderr" % (gitroot
, self
.tag
)
406 do_print("stdout for %s in %s" % (self
.name
, self
.stdout_path
))
407 do_print("stderr for %s in %s" % (self
.name
, self
.stderr_path
))
408 run_cmd("rm -f %s %s" % (self
.stdout_path
, self
.stderr_path
))
409 self
.stdout
= open(self
.stdout_path
, 'w')
410 self
.stderr
= open(self
.stderr_path
, 'w')
411 self
.stdin
= open("/dev/null", 'r')
412 self
.sdir
= "%s/%s" % (testbase
, self
.tag
)
413 self
.prefix
= "%s/%s" % (test_prefix
, self
.tag
)
414 run_cmd("rm -rf %s" % self
.sdir
)
415 run_cmd("rm -rf %s" % self
.prefix
)
417 run_cmd("cp --recursive --link --archive %s %s" % (test_master
, self
.sdir
), dir=test_master
, show
=True)
419 run_cmd("git clone --recursive --shared %s %s" % (test_master
, self
.sdir
), dir=test_master
, show
=True)
422 def start_next(self
):
423 if self
.next
== len(self
.sequence
):
424 if not options
.nocleanup
:
425 run_cmd("rm -rf %s" % self
.sdir
)
426 run_cmd("rm -rf %s" % self
.prefix
)
427 do_print('%s: Completed OK' % self
.name
)
430 (self
.stage
, self
.cmd
, self
.output_mime_type
) = self
.sequence
[self
.next
]
431 self
.cmd
= self
.cmd
.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib
=1, prefix
=self
.prefix
))
432 self
.cmd
= self
.cmd
.replace("${PREFIX}", "--prefix=%s" % self
.prefix
)
433 self
.cmd
= self
.cmd
.replace("${EXTRA_PYTHON}", "%s" % extra_python
)
434 self
.cmd
= self
.cmd
.replace("${PREFIX_DIR}", "%s" % self
.prefix
)
435 self
.cmd
= self
.cmd
.replace("${TESTS}", options
.restrict_tests
)
436 # if self.output_mime_type == "text/x-subunit":
437 # self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
438 do_print('%s: [%s] Running %s' % (self
.name
, self
.stage
, self
.cmd
))
440 os
.chdir("%s/%s" % (self
.sdir
, self
.dir))
441 self
.proc
= Popen(self
.cmd
, shell
=True,
442 stdout
=self
.stdout
, stderr
=self
.stderr
, stdin
=self
.stdin
)
447 class buildlist(object):
448 '''handle build of multiple directories'''
450 def __init__(self
, tasknames
, rebase_url
, rebase_branch
="master"):
453 self
.tail_proc
= None
456 if options
.restrict_tests
:
457 tasknames
= ["samba-test-only"]
459 tasknames
= defaulttasks
461 # If we are only running one test,
462 # do not sleep randomly to wait for it to start
463 os
.environ
['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
466 b
= builder(n
, tasks
[n
], cp
=n
is not "pidl")
469 rebase_remote
= "rebaseon"
470 retry_task
= [ ("retry",
472 git remote add -t %s %s %s
476 git describe %s/%s > old_remote_branch.desc
478 git describe %s/%s > remote_branch.desc
479 diff old_remote_branch.desc remote_branch.desc
482 rebase_branch
, rebase_remote
, rebase_url
,
484 rebase_remote
, rebase_branch
,
486 rebase_remote
, rebase_branch
490 self
.retry
= builder('retry', retry_task
, cp
=False)
491 self
.need_retry
= False
494 if self
.tail_proc
is not None:
495 self
.tail_proc
.terminate()
496 self
.tail_proc
.wait()
497 self
.tail_proc
= None
498 if self
.retry
is not None:
499 self
.retry
.proc
.terminate()
500 self
.retry
.proc
.wait()
503 if b
.proc
is not None:
504 run_cmd("killbysubdir %s > /dev/null 2>&1" % b
.sdir
, checkfail
=False)
516 b
.status
= b
.proc
.poll()
522 ret
= self
.retry
.proc
.poll()
524 self
.need_retry
= True
534 if options
.retry
and self
.need_retry
:
536 do_print("retry needed")
537 return (0, None, None, None, "retry")
540 if os
.WIFSIGNALED(b
.status
) or os
.WEXITSTATUS(b
.status
) != 0:
542 return (b
.status
, b
.name
, b
.stage
, b
.tag
, "%s: [%s] failed '%s' with status %d" % (b
.name
, b
.stage
, b
.cmd
, b
.status
))
545 return (0, None, None, None, "All OK")
547 def write_system_info(self
):
548 filename
= 'system-info.txt'
549 f
= open(filename
, 'w')
550 for cmd
in ['uname -a', 'free', 'cat /proc/cpuinfo',
551 'cc --version', 'df -m .', 'df -m %s' % testbase
]:
552 print('### %s' % cmd
, file=f
)
553 print(run_cmd(cmd
, output
=True, checkfail
=False), file=f
)
558 def tarlogs(self
, fname
):
559 tar
= tarfile
.open(fname
, "w:gz")
561 tar
.add(b
.stdout_path
, arcname
="%s.stdout" % b
.tag
)
562 tar
.add(b
.stderr_path
, arcname
="%s.stderr" % b
.tag
)
563 if os
.path
.exists("autobuild.log"):
564 tar
.add("autobuild.log")
565 sys_info
= self
.write_system_info()
569 def remove_logs(self
):
571 os
.unlink(b
.stdout_path
)
572 os
.unlink(b
.stderr_path
)
574 def start_tail(self
):
576 cmd
= "tail -f *.stdout *.stderr"
578 self
.tail_proc
= Popen(cmd
, shell
=True)
583 if options
.nocleanup
:
585 run_cmd("stat %s || true" % test_tmpdir
, show
=True)
586 run_cmd("stat %s" % testbase
, show
=True)
587 do_print("Cleaning up ....")
588 for d
in cleanup_list
:
589 run_cmd("rm -rf %s" % d
)
593 '''get to the top of the git repo'''
596 if os
.path
.isdir(os
.path
.join(p
, ".git")):
598 p
= os
.path
.abspath(os
.path
.join(p
, '..'))
602 def daemonize(logfile
):
604 if pid
== 0: # Parent
607 if pid
!= 0: # Actual daemon
612 import resource
# Resource usage information.
613 maxfd
= resource
.getrlimit(resource
.RLIMIT_NOFILE
)[1]
614 if maxfd
== resource
.RLIM_INFINITY
:
615 maxfd
= 1024 # Rough guess at maximum number of open file descriptors.
616 for fd
in range(0, maxfd
):
621 os
.open(logfile
, os
.O_RDWR | os
.O_CREAT
)
625 def write_pidfile(fname
):
626 '''write a pid file, cleanup on exit'''
627 f
= open(fname
, mode
='w')
628 f
.write("%u\n" % os
.getpid())
632 def rebase_tree(rebase_url
, rebase_branch
= "master"):
633 rebase_remote
= "rebaseon"
634 do_print("Rebasing on %s" % rebase_url
)
635 run_cmd("git describe HEAD", show
=True, dir=test_master
)
636 run_cmd("git remote add -t %s %s %s" %
637 (rebase_branch
, rebase_remote
, rebase_url
),
638 show
=True, dir=test_master
)
639 run_cmd("git fetch %s" % rebase_remote
, show
=True, dir=test_master
)
640 if options
.fix_whitespace
:
641 run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
642 (rebase_remote
, rebase_branch
),
643 show
=True, dir=test_master
)
645 run_cmd("git rebase --force-rebase %s/%s" %
646 (rebase_remote
, rebase_branch
),
647 show
=True, dir=test_master
)
648 diff
= run_cmd("git --no-pager diff HEAD %s/%s" %
649 (rebase_remote
, rebase_branch
),
650 dir=test_master
, output
=True)
652 do_print("No differences between HEAD and %s/%s - exiting" %
653 (rebase_remote
, rebase_branch
))
655 run_cmd("git describe %s/%s" %
656 (rebase_remote
, rebase_branch
),
657 show
=True, dir=test_master
)
658 run_cmd("git describe HEAD", show
=True, dir=test_master
)
659 run_cmd("git --no-pager diff --stat HEAD %s/%s" %
660 (rebase_remote
, rebase_branch
),
661 show
=True, dir=test_master
)
663 def push_to(push_url
, push_branch
= "master"):
664 push_remote
= "pushto"
665 do_print("Pushing to %s" % push_url
)
667 run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master
)
668 run_cmd("git commit --amend -c HEAD", dir=test_master
)
669 # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
670 # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
671 run_cmd("git remote add -t %s %s %s" %
672 (push_branch
, push_remote
, push_url
),
673 show
=True, dir=test_master
)
674 run_cmd("git push %s +HEAD:%s" %
675 (push_remote
, push_branch
),
676 show
=True, dir=test_master
)
678 def_testbase
= os
.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os
.getenv('USER'))
680 gitroot
= find_git_root()
682 raise Exception("Failed to find git root")
684 parser
= OptionParser()
685 parser
.add_option("", "--tail", help="show output while running", default
=False, action
="store_true")
686 parser
.add_option("", "--keeplogs", help="keep logs", default
=False, action
="store_true")
687 parser
.add_option("", "--nocleanup", help="don't remove test tree", default
=False, action
="store_true")
688 parser
.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase
,
689 default
=def_testbase
)
690 parser
.add_option("", "--passcmd", help="command to run on success", default
=None)
691 parser
.add_option("", "--verbose", help="show all commands as they are run",
692 default
=False, action
="store_true")
693 parser
.add_option("", "--rebase", help="rebase on the given tree before testing",
694 default
=None, type='str')
695 parser
.add_option("", "--pushto", help="push to a git url on success",
696 default
=None, type='str')
697 parser
.add_option("", "--mark", help="add a Tested-By signoff before pushing",
698 default
=False, action
="store_true")
699 parser
.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
700 default
=False, action
="store_true")
701 parser
.add_option("", "--retry", help="automatically retry if master changes",
702 default
=False, action
="store_true")
703 parser
.add_option("", "--email", help="send email to the given address on failure",
704 type='str', default
=None)
705 parser
.add_option("", "--email-from", help="send email from the given address",
706 type='str', default
="autobuild@samba.org")
707 parser
.add_option("", "--email-server", help="send email via the given server",
708 type='str', default
='localhost')
709 parser
.add_option("", "--always-email", help="always send email, even on success",
711 parser
.add_option("", "--daemon", help="daemonize after initial setup",
713 parser
.add_option("", "--branch", help="the branch to work on (default=master)",
714 default
="master", type='str')
715 parser
.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
716 default
=gitroot
, type='str')
717 parser
.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
718 default
=False, action
="store_true")
719 parser
.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
722 def send_email(subject
, text
, log_tar
):
723 if options
.email
is None:
724 do_print("not sending email because the recipient is not set")
725 do_print("the text content would have been:\n\nSubject: %s\n\nTs" %
728 outer
= MIMEMultipart()
729 outer
['Subject'] = subject
730 outer
['To'] = options
.email
731 outer
['From'] = options
.email_from
732 outer
['Date'] = email
.utils
.formatdate(localtime
= True)
733 outer
.preamble
= 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
734 outer
.attach(MIMEText(text
, 'plain'))
735 if options
.attach_logs
:
736 fp
= open(log_tar
, 'rb')
737 msg
= MIMEApplication(fp
.read(), 'gzip', email
.encoders
.encode_base64
)
739 # Set the filename parameter
740 msg
.add_header('Content-Disposition', 'attachment', filename
=os
.path
.basename(log_tar
))
742 content
= outer
.as_string()
743 s
= smtplib
.SMTP(options
.email_server
)
744 s
.sendmail(options
.email_from
, [options
.email
], content
)
748 def email_failure(status
, failed_task
, failed_stage
, failed_tag
, errstr
,
749 elapsed_time
, log_base
=None, add_log_tail
=True):
750 '''send an email to options.email about the failure'''
751 elapsed_minutes
= elapsed_time
/ 60.0
752 user
= os
.getenv("USER")
758 Your autobuild on %s failed after %.1f minutes
759 when trying to test %s with the following error:
763 the autobuild has been abandoned. Please fix the error and resubmit.
765 A summary of the autobuild process is here:
768 ''' % (platform
.node(), elapsed_minutes
, failed_task
, errstr
, log_base
)
770 if options
.restrict_tests
:
772 The build was restricted to tests matching %s\n""" % options
.restrict_tests
774 if failed_task
!= 'rebase':
776 You can see logs of the failed task here:
781 or you can get full logs of all tasks in this job here:
785 The top commit for the tree that was built was:
789 ''' % (log_base
, failed_tag
, log_base
, failed_tag
, log_base
, top_commit_msg
)
792 f
= open("%s/%s.stdout" % (gitroot
, failed_tag
), 'r')
793 lines
= f
.readlines()
794 log_tail
= "".join(lines
[-50:])
795 num_lines
= len(lines
)
797 # Also include stderr (compile failures) if < 50 lines of stdout
798 f
= open("%s/%s.stderr" % (gitroot
, failed_tag
), 'r')
799 log_tail
+= "".join(f
.readlines()[-(50-num_lines
):])
802 The last 50 lines of log messages:
808 logs
= os
.path
.join(gitroot
, 'logs.tar.gz')
809 send_email('autobuild[%s] failure on %s for task %s during %s'
810 % (options
.branch
, platform
.node(), failed_task
, failed_stage
),
813 def email_success(elapsed_time
, log_base
=None):
814 '''send an email to options.email about a successful build'''
815 user
= os
.getenv("USER")
821 Your autobuild on %s has succeeded after %.1f minutes.
823 ''' % (platform
.node(), elapsed_time
/ 60.)
825 if options
.restrict_tests
:
827 The build was restricted to tests matching %s\n""" % options
.restrict_tests
832 you can get full logs of all tasks in this job here:
839 The top commit for the tree that was built was:
844 logs
= os
.path
.join(gitroot
, 'logs.tar.gz')
845 send_email('autobuild[%s] success on %s' % (options
.branch
, platform
.node()),
849 (options
, args
) = parser
.parse_args()
852 if options
.rebase
is None:
853 raise Exception('You can only use --retry if you also rebase')
855 testbase
= "%s/b%u" % (options
.testbase
, os
.getpid())
856 test_master
= "%s/master" % testbase
857 test_prefix
= "%s/prefix" % testbase
858 test_tmpdir
= "%s/tmp" % testbase
859 os
.environ
['TMPDIR'] = test_tmpdir
861 # get the top commit message, for emails
862 top_commit_msg
= run_cmd("git log -1", dir=gitroot
, output
=True)
865 os
.makedirs(testbase
)
866 except Exception as reason
:
867 raise Exception("Unable to create %s : %s" % (testbase
, reason
))
868 cleanup_list
.append(testbase
)
871 logfile
= os
.path
.join(testbase
, "log")
872 do_print("Forking into the background, writing progress to %s" % logfile
)
875 write_pidfile(gitroot
+ "/autobuild.pid")
877 start_time
= time
.time()
881 run_cmd("rm -rf %s" % test_tmpdir
, show
=True)
882 os
.makedirs(test_tmpdir
)
883 # The waf uninstall code removes empty directories all the way
884 # up the tree. Creating a file in test_tmpdir stops it from
886 run_cmd("touch %s" % os
.path
.join(test_tmpdir
,
887 ".directory-is-not-empty"), show
=True)
888 run_cmd("stat %s" % test_tmpdir
, show
=True)
889 run_cmd("stat %s" % testbase
, show
=True)
890 run_cmd("git clone --recursive --shared %s %s" % (gitroot
, test_master
), show
=True, dir=gitroot
)
897 if options
.rebase
is not None:
898 rebase_tree(options
.rebase
, rebase_branch
=options
.branch
)
900 cleanup_list
.append(gitroot
+ "/autobuild.pid")
902 elapsed_time
= time
.time() - start_time
903 email_failure(-1, 'rebase', 'rebase', 'rebase',
904 'rebase on %s failed' % options
.branch
,
905 elapsed_time
, log_base
=options
.log_base
)
907 blist
= buildlist(args
, options
.rebase
, rebase_branch
=options
.branch
)
910 (status
, failed_task
, failed_stage
, failed_tag
, errstr
) = blist
.run()
911 if status
!= 0 or errstr
!= "retry":
918 cleanup_list
.append(gitroot
+ "/autobuild.pid")
924 do_print("waiting for tail to flush")
927 elapsed_time
= time
.time() - start_time
929 if options
.passcmd
is not None:
930 do_print("Running passcmd: %s" % options
.passcmd
)
931 run_cmd(options
.passcmd
, dir=test_master
)
932 if options
.pushto
is not None:
933 push_to(options
.pushto
, push_branch
=options
.branch
)
934 if options
.keeplogs
or options
.attach_logs
:
935 blist
.tarlogs("logs.tar.gz")
936 do_print("Logs in logs.tar.gz")
937 if options
.always_email
:
938 email_success(elapsed_time
, log_base
=options
.log_base
)
944 # something failed, gather a tar of the logs
945 blist
.tarlogs("logs.tar.gz")
947 if options
.email
is not None:
948 email_failure(status
, failed_task
, failed_stage
, failed_tag
, errstr
,
949 elapsed_time
, log_base
=options
.log_base
)
951 elapsed_minutes
= elapsed_time
/ 60.0
954 ####################################################################
958 Your autobuild[%s] on %s failed after %.1f minutes
959 when trying to test %s with the following error:
963 the autobuild has been abandoned. Please fix the error and resubmit.
965 ####################################################################
967 ''' % (options
.branch
, platform
.node(), elapsed_minutes
, failed_task
, errstr
))
971 do_print("Logs in logs.tar.gz")