2 # run tests on all Samba subprojects and push to a git tree on success
3 # Copyright Andrew Tridgell 2010
4 # Copyright Jelmer Vernooij 2010
5 # released under GNU GPL v3 or later
7 from cStringIO
import StringIO
9 from subprocess
import call
, check_call
, Popen
, PIPE
10 import os
, tarfile
, sys
, time
11 from optparse
import OptionParser
13 sys
.path
.insert(0, os
.path
.join(os
.path
.dirname(__file__
), "../selftest"))
14 sys
.path
.insert(0, os
.path
.join(os
.path
.dirname(__file__
), "../lib/testtools"))
15 sys
.path
.insert(0, os
.path
.join(os
.path
.dirname(__file__
), "../lib/subunit/python"))
19 from email
.mime
.application
import MIMEApplication
20 from email
.mime
.text
import MIMEText
21 from email
.mime
.multipart
import MIMEMultipart
23 samba_master
= os
.getenv('SAMBA_MASTER', 'git://git.samba.org/samba.git')
24 samba_master_ssh
= os
.getenv('SAMBA_MASTER_SSH', 'git+ssh://git.samba.org/data/git/samba.git')
28 os
.putenv('CC', "ccache gcc")
31 "source3" : [ ("autogen", "./autogen.sh", "text/plain"),
32 ("configure", "./configure.developer ${PREFIX}", "text/plain"),
33 ("make basics", "make basics", "text/plain"),
34 ("make", "make -j 4 everything", "text/plain"), # don't use too many processes
35 ("install", "make install", "text/plain"),
36 ("test", "TDB_NO_FSYNC=1 make subunit-test", "text/x-subunit") ],
38 "source4" : [ ("configure", "./configure.developer ${PREFIX}", "text/plain"),
39 ("make", "make -j", "text/plain"),
40 ("install", "make install", "text/plain"),
41 ("test", "TDB_NO_FSYNC=1 make subunit-test", "text/x-subunit") ],
43 "source4/lib/ldb" : [ ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
44 ("make", "make -j", "text/plain"),
45 ("install", "make install", "text/plain"),
46 ("test", "make test", "text/plain") ],
48 "lib/tdb" : [ ("autogen", "./autogen-waf.sh", "text/plain"),
49 ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
50 ("make", "make -j", "text/plain"),
51 ("install", "make install", "text/plain"),
52 ("test", "make test", "text/plain") ],
54 "lib/talloc" : [ ("autogen", "./autogen-waf.sh", "text/plain"),
55 ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
56 ("make", "make -j", "text/plain"),
57 ("install", "make install", "text/plain"),
58 ("test", "make test", "text/x-subunit"), ],
60 "lib/replace" : [ ("autogen", "./autogen-waf.sh", "text/plain"),
61 ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
62 ("make", "make -j", "text/plain"),
63 ("install", "make install", "text/plain"),
64 ("test", "make test", "text/plain"), ],
66 "lib/tevent" : [ ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
67 ("make", "make -j", "text/plain"),
68 ("install", "make install", "text/plain"),
69 ("test", "make test", "text/plain"), ],
73 def run_cmd(cmd
, dir=None, show
=None, output
=False, checkfail
=True, shell
=False):
75 show
= options
.verbose
77 print("Running: '%s' in '%s'" % (cmd
, dir))
79 return Popen(cmd
, stdout
=PIPE
, cwd
=dir, shell
=shell
).communicate()[0]
81 return check_call(cmd
, cwd
=dir, shell
=shell
)
83 return call(cmd
, cwd
=dir, shell
=shell
)
86 def clone_gitroot(test_master
, revision
="HEAD"):
87 run_cmd(["git", "clone", "--shared", gitroot
, test_master
])
88 if revision
!= "HEAD":
89 run_cmd(["git", "checkout", revision
])
92 class RetryChecker(object):
93 """Check whether it is necessary to retry."""
95 def __init__(self
, dir):
96 run_cmd(["git", "remote", "add", "-t", "master", "master", samba_master
])
97 run_cmd(["git", "fetch", "master"])
101 git describe master/master > old_master.desc
103 git describe master/master > master.desc
104 diff old_master.desc master.desc
107 self
.proc
= Popen(cmd
, shell
=True, cwd
=self
.dir)
110 return self
.proc
.poll()
113 self
.proc
.terminate()
115 self
.retry
.proc
= None
118 class TreeStageBuilder(object):
119 """Handle building of a particular stage for a tree.
122 def __init__(self
, tree
, name
, command
, fail_quickly
=False):
125 self
.command
= command
126 self
.fail_quickly
= fail_quickly
128 self
.stdin
= open(os
.devnull
, 'r')
131 raise NotImplementedError(self
.start
)
134 self
.exitcode
= self
.proc
.poll()
138 if self
.proc
is not None:
140 run_cmd(["killbysubdir", self
.tree
.sdir
], checkfail
=False)
142 # killbysubdir doesn't exist ?
144 self
.proc
.terminate()
149 def failure_reason(self
):
150 raise NotImplementedError(self
.failure_reason
)
154 return (self
.exitcode
!= 0)
157 class PlainTreeStageBuilder(TreeStageBuilder
):
160 print '%s: [%s] Running %s' % (self
.name
, self
.name
, self
.command
)
161 self
.proc
= Popen(self
.command
, shell
=True, cwd
=self
.tree
.dir,
162 stdout
=self
.tree
.stdout
, stderr
=self
.tree
.stderr
,
166 def failure_reason(self
):
167 return "failed '%s' with exit code %d" % (self
.command
, self
.exitcode
)
170 class AbortingTestResult(subunithelper
.TestsuiteEnabledTestResult
):
172 def __init__(self
, stage
):
173 super(AbortingTestResult
, self
).__init
__()
176 def addError(self
, test
, details
=None):
177 self
.stage
.proc
.terminate()
179 def addFailure(self
, test
, details
=None):
180 self
.stage
.proc
.terminate()
183 class FailureTrackingTestResult(subunithelper
.TestsuiteEnabledTestResult
):
185 def __init__(self
, stage
):
186 super(FailureTrackingTestResult
, self
).__init
__()
189 def addError(self
, test
, details
=None):
190 if self
.stage
.failed_test
is None:
191 self
.stage
.failed_test
= ("error", test
)
193 def addFailure(self
, test
, details
=None):
194 if self
.stage
.failed_test
is None:
195 self
.stage
.failed_test
= ("failure", test
)
198 class SubunitTreeStageBuilder(TreeStageBuilder
):
200 def __init__(self
, tree
, name
, command
, fail_quickly
=False):
201 super(SubunitTreeStageBuilder
, self
).__init
__(tree
, name
, command
,
203 self
.failed_test
= None
204 self
.subunit_path
= os
.path
.join(gitroot
,
205 "%s.%s.subunit" % (self
.tree
.tag
, self
.name
))
206 self
.tree
.logfiles
.append(
207 (self
.subunit_path
, os
.path
.basename(self
.subunit_path
),
209 self
.subunit
= open(self
.subunit_path
, 'w')
211 formatter
= subunithelper
.PlainFormatter(False, True, {})
212 clients
= [formatter
, subunit
.TestProtocolClient(self
.subunit
),
213 FailureTrackingTestResult(self
)]
215 clients
.append(AbortingTestResult(self
))
216 self
.subunit_server
= subunit
.TestProtocolServer(
217 testtools
.MultiTestResult(*clients
),
222 print '%s: [%s] Running' % (self
.tree
.name
, self
.name
)
223 self
.proc
= Popen(self
.command
, shell
=True, cwd
=self
.tree
.dir,
224 stdout
=PIPE
, stderr
=self
.tree
.stderr
, stdin
=self
.stdin
)
225 fd
= self
.proc
.stdout
.fileno()
226 fl
= fcntl
.fcntl(fd
, fcntl
.F_GETFL
)
227 fcntl
.fcntl(fd
, fcntl
.F_SETFL
, fl | os
.O_NONBLOCK
)
231 data
= self
.proc
.stdout
.read()
235 self
.buffered
+= data
237 for l
in self
.buffered
.splitlines(True):
239 self
.subunit_server
.lineReceived(l
)
242 self
.buffered
= buffered
243 self
.exitcode
= self
.proc
.poll()
244 if self
.exitcode
is not None:
249 def failure_reason(self
):
251 return "failed '%s' with %s in test %s" (self
.command
, self
.failed_test
[0], self
.failed_test
[1])
253 return "failed '%s' with exit code %d in unknown test" % (self
.command
, self
.exitcode
)
256 class TreeBuilder(object):
257 '''handle build of one directory'''
259 def __init__(self
, name
, sequence
, fail_quickly
=False):
261 self
.fail_quickly
= fail_quickly
263 self
.tag
= self
.name
.replace('/', '_')
264 self
.sequence
= sequence
267 self
.stdout_path
= os
.path
.join(gitroot
, "%s.stdout" % (self
.tag
, ))
268 self
.stderr_path
= os
.path
.join(gitroot
, "%s.stderr" % (self
.tag
, ))
270 (self
.stdout_path
, os
.path
.basename(self
.stdout_path
), "text/plain"),
271 (self
.stderr_path
, os
.path
.basename(self
.stderr_path
), "text/plain"),
274 print("stdout for %s in %s" % (self
.name
, self
.stdout_path
))
275 print("stderr for %s in %s" % (self
.name
, self
.stderr_path
))
276 if os
.path
.exists(self
.stdout_path
):
277 os
.unlink(self
.stdout_path
)
278 if os
.path
.exists(self
.stderr_path
):
279 os
.unlink(self
.stderr_path
)
280 self
.stdout
= open(self
.stdout_path
, 'w')
281 self
.stderr
= open(self
.stderr_path
, 'w')
282 self
.sdir
= os
.path
.join(testbase
, self
.tag
)
283 if name
in ['pass', 'fail', 'retry']:
286 self
.dir = os
.path
.join(self
.sdir
, self
.name
)
287 self
.prefix
= os
.path
.join(testbase
, "prefix", self
.tag
)
288 run_cmd(["rm", "-rf", self
.sdir
])
289 cleanup_list
.append(self
.sdir
)
290 cleanup_list
.append(self
.prefix
)
291 os
.makedirs(self
.sdir
)
292 run_cmd(["rm", "-rf", self
.sdir
])
293 clone_gitroot(self
.sdir
, revision
)
297 def start_next(self
):
298 if self
.next
== len(self
.sequence
):
299 print '%s: Completed OK' % self
.name
304 (stage_name
, cmd
, output_mime_type
) = self
.sequence
[self
.next
]
305 cmd
= cmd
.replace("${PREFIX}", "--prefix=%s" % self
.prefix
)
306 if output_mime_type
== "text/plain":
307 self
.current_stage
= PlainTreeStageBuilder(self
, stage_name
, cmd
,
309 elif output_mime_type
== "text/x-subunit":
310 self
.current_stage
= SubunitTreeStageBuilder(self
, stage_name
, cmd
,
313 raise Exception("Unknown output mime type %s" % output_mime_type
)
314 self
.stages
.append(self
.current_stage
)
315 self
.current_stage
.start()
318 def remove_logs(self
):
319 for path
, name
, mime_type
in self
.logfiles
:
323 self
.exitcode
= self
.current_stage
.poll()
324 if self
.exitcode
is not None:
325 self
.current_stage
= None
329 if self
.current_stage
is not None:
330 self
.current_stage
.kill()
331 self
.current_stage
= None
335 return any([s
.failed
for s
in self
.stages
])
338 def failed_stage(self
):
339 for s
in self
.stages
:
345 def failure_reason(self
):
346 return "%s: [%s] %s" % (self
.name
, self
.failed_stage
.name
,
347 self
.failed_stage
.failure_reason
)
350 class BuildList(object):
351 '''handle build of multiple directories'''
353 def __init__(self
, tasklist
, tasknames
):
356 self
.tail_proc
= None
358 if tasknames
== ['pass']:
359 tasks
= { 'pass' : [ ("pass", '/bin/true', "text/plain") ]}
360 if tasknames
== ['fail']:
361 tasks
= { 'fail' : [ ("fail", '/bin/false', "text/plain") ]}
365 b
= TreeBuilder(n
, tasks
[n
], not options
.fail_slowly
)
368 self
.retry
= RetryChecker(self
.sdir
)
369 self
.need_retry
= False
372 if self
.tail_proc
is not None:
373 self
.tail_proc
.terminate()
374 self
.tail_proc
.wait()
375 self
.tail_proc
= None
376 if self
.retry
is not None:
385 if b
.current_stage
is None:
392 ret
= self
.retry
.poll()
394 self
.need_retry
= True
404 if options
.retry
and self
.need_retry
:
406 print("retry needed")
407 return (0, None, None, None, "retry")
412 return (b
.exitcode
, b
.name
, b
.failed_stage
, b
.tag
, b
.failure_reason
)
415 return (0, None, None, None, "All OK")
417 def tarlogs(self
, name
=None, fileobj
=None):
418 tar
= tarfile
.open(name
=name
, fileobj
=fileobj
, mode
="w:gz")
420 for (path
, name
, mime_type
) in b
.logfiles
:
421 tar
.add(path
, arcname
=name
)
422 if os
.path
.exists("autobuild.log"):
423 tar
.add("autobuild.log")
426 def attach_logs(self
, outer
):
428 self
.tarlogs(fileobj
=f
)
429 msg
= MIMEApplication(f
.getvalue(), "x-gzip")
430 msg
.add_header('Content-Disposition', 'attachment',
431 filename
="logs.tar.gz")
434 def remove_logs(self
):
438 def start_tail(self
):
439 cmd
= "tail -f *.stdout *.stderr"
440 self
.tail_proc
= Popen(cmd
, shell
=True, cwd
=gitroot
)
444 if options
.nocleanup
:
446 print("Cleaning up ....")
447 for d
in cleanup_list
:
448 run_cmd(["rm", "-rf", d
])
451 def find_git_root(p
):
452 '''get to the top of the git repo'''
454 if os
.path
.isdir(os
.path
.join(p
, ".git")):
456 p
= os
.path
.abspath(os
.path
.join(p
, '..'))
460 def daemonize(logfile
):
462 if pid
== 0: # Parent
465 if pid
!= 0: # Actual daemon
470 import resource
# Resource usage information.
471 maxfd
= resource
.getrlimit(resource
.RLIMIT_NOFILE
)[1]
472 if maxfd
== resource
.RLIM_INFINITY
:
473 maxfd
= 1024 # Rough guess at maximum number of open file descriptors.
474 for fd
in range(0, maxfd
):
479 os
.open(logfile
, os
.O_RDWR | os
.O_CREAT
)
484 def rebase_tree(url
):
485 print("Rebasing on %s" % url
)
486 run_cmd(["git", "remote", "add", "-t", "master", "master", url
], show
=True,
488 run_cmd(["git", "fetch", "master"], show
=True, dir=test_master
)
489 if options
.fix_whitespace
:
490 run_cmd(["git", "rebase", "--whitespace=fix", "master/master"],
491 show
=True, dir=test_master
)
493 run_cmd(["git", "rebase", "master/master"], show
=True, dir=test_master
)
494 diff
= run_cmd(["git", "--no-pager", "diff", "HEAD", "master/master"],
495 dir=test_master
, output
=True)
497 print("No differences between HEAD and master/master - exiting")
501 print("Pushing to %s" % url
)
503 run_cmd("EDITOR=script/commit_mark.sh git commit --amend -c HEAD",
504 dir=test_master
, shell
=True)
505 # the notes method doesn't work yet, as metze hasn't allowed
506 # refs/notes/* in master
507 # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD",
509 run_cmd(["git", "remote", "add", "-t", "master", "pushto", url
], show
=True,
511 run_cmd(["git", "push", "pushto", "+HEAD:master"], show
=True,
514 def_testbase
= os
.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os
.getenv('USER'))
516 parser
= OptionParser()
517 parser
.add_option("--repository", help="repository to run tests for", default
=None, type=str)
518 parser
.add_option("--revision", help="revision to compile if not HEAD", default
=None, type=str)
519 parser
.add_option("--tail", help="show output while running", default
=False, action
="store_true")
520 parser
.add_option("--keeplogs", help="keep logs", default
=False, action
="store_true")
521 parser
.add_option("--nocleanup", help="don't remove test tree", default
=False, action
="store_true")
522 parser
.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase
,
523 default
=def_testbase
)
524 parser
.add_option("--passcmd", help="command to run on success", default
=None)
525 parser
.add_option("--verbose", help="show all commands as they are run",
526 default
=False, action
="store_true")
527 parser
.add_option("--rebase", help="rebase on the given tree before testing",
528 default
=None, type='str')
529 parser
.add_option("--rebase-master", help="rebase on %s before testing" % samba_master
,
530 default
=False, action
='store_true')
531 parser
.add_option("--pushto", help="push to a git url on success",
532 default
=None, type='str')
533 parser
.add_option("--push-master", help="push to %s on success" % samba_master_ssh
,
534 default
=False, action
='store_true')
535 parser
.add_option("--mark", help="add a Tested-By signoff before pushing",
536 default
=False, action
="store_true")
537 parser
.add_option("--fix-whitespace", help="fix whitespace on rebase",
538 default
=False, action
="store_true")
539 parser
.add_option("--retry", help="automatically retry if master changes",
540 default
=False, action
="store_true")
541 parser
.add_option("--email", help="send email to the given address on failure",
542 type='str', default
=None)
543 parser
.add_option("--always-email", help="always send email, even on success",
545 parser
.add_option("--daemon", help="daemonize after initial setup",
547 parser
.add_option("--fail-slowly", help="continue running tests even after one has already failed",
551 def email_failure(blist
, exitcode
, failed_task
, failed_stage
, failed_tag
, errstr
):
552 '''send an email to options.email about the failure'''
553 user
= os
.getenv("USER")
557 Your autobuild failed when trying to test %s with the following error:
560 the autobuild has been abandoned. Please fix the error and resubmit.
562 You can see logs of the failed task here:
564 http://git.samba.org/%s/samba-autobuild/%s.stdout
565 http://git.samba.org/%s/samba-autobuild/%s.stderr
567 A summary of the autobuild process is here:
569 http://git.samba.org/%s/samba-autobuild/autobuild.log
571 or you can get full logs of all tasks in this job here:
573 http://git.samba.org/%s/samba-autobuild/logs.tar.gz
575 The top commit for the tree that was built was:
579 ''' % (failed_task
, errstr
, user
, failed_tag
, user
, failed_tag
, user
, user
,
580 get_top_commit_msg(test_master
))
582 msg
= MIMEMultipart()
583 msg
['Subject'] = 'autobuild failure for task %s during %s' % (
584 failed_task
, failed_stage
.name
)
585 msg
['From'] = 'autobuild@samba.org'
586 msg
['To'] = options
.email
588 main
= MIMEText(text
)
591 blist
.attach_logs(msg
)
595 s
.sendmail(msg
['From'], [msg
['To']], msg
.as_string())
598 def email_success(blist
):
599 '''send an email to options.email about a successful build'''
600 user
= os
.getenv("USER")
604 Your autobuild has succeeded.
611 you can get full logs of all tasks in this job here:
613 http://git.samba.org/%s/samba-autobuild/logs.tar.gz
618 The top commit for the tree that was built was:
621 ''' % (get_top_commit_msg(test_master
),)
623 msg
= MIMEMultipart()
624 msg
['Subject'] = 'autobuild success'
625 msg
['From'] = 'autobuild@samba.org'
626 msg
['To'] = options
.email
628 main
= MIMEText(text
, 'plain')
631 blist
.attach_logs(msg
)
635 s
.sendmail(msg
['From'], [msg
['To']], msg
.as_string())
639 (options
, args
) = parser
.parse_args()
642 if not options
.rebase_master
and options
.rebase
is None:
643 raise Exception('You can only use --retry if you also rebase')
645 testbase
= os
.path
.join(options
.testbase
, "b%u" % (os
.getpid(),))
646 test_master
= os
.path
.join(testbase
, "master")
648 if options
.repository
is not None:
649 repository
= options
.repository
651 repository
= os
.getcwd()
653 gitroot
= find_git_root(repository
)
655 raise Exception("Failed to find git root under %s" % repository
)
657 # get the top commit message, for emails
658 if options
.revision
is not None:
659 revision
= options
.revision
663 def get_top_commit_msg(reporoot
):
664 return run_cmd(["git", "log", "-1"], dir=reporoot
, output
=True)
667 os
.makedirs(testbase
)
668 except Exception, reason
:
669 raise Exception("Unable to create %s : %s" % (testbase
, reason
))
670 cleanup_list
.append(testbase
)
673 logfile
= os
.path
.join(testbase
, "log")
674 print "Forking into the background, writing progress to %s" % logfile
679 run_cmd(["rm", "-rf", test_master
])
680 cleanup_list
.append(test_master
)
681 clone_gitroot(test_master
, revision
)
687 if options
.rebase
is not None:
688 rebase_tree(options
.rebase
)
689 elif options
.rebase_master
:
690 rebase_tree(samba_master
)
691 blist
= BuildList(tasks
, args
)
694 (exitcode
, failed_task
, failed_stage
, failed_tag
, errstr
) = blist
.run()
695 if exitcode
!= 0 or errstr
!= "retry":
704 print("waiting for tail to flush")
709 if options
.passcmd
is not None:
710 print("Running passcmd: %s" % options
.passcmd
)
711 run_cmd(options
.passcmd
, dir=test_master
, shell
=True)
712 if options
.pushto
is not None:
713 push_to(options
.pushto
)
714 elif options
.push_master
:
715 push_to(samba_master_ssh
)
717 blist
.tarlogs("logs.tar.gz")
718 print("Logs in logs.tar.gz")
719 if options
.always_email
:
725 # something failed, gather a tar of the logs
726 blist
.tarlogs("logs.tar.gz")
728 if options
.email
is not None:
729 email_failure(blist
, exitcode
, failed_task
, failed_stage
, failed_tag
,
734 print("Logs in logs.tar.gz")