enhance 'buildbot sighup' to show all related twistd.log lines. Also rename it to...
[buildbot.git] / buildbot / scripts / runner.py
blob5bcbfea605964e0f27c625dd0ee1a1771ac04a76
1 # -*- test-case-name: buildbot.test.test_runner -*-
3 # N.B.: don't import anything that might pull in a reactor yet. Some of our
4 # subcommands want to load modules that need the gtk reactor.
5 import os, sys, stat, re, time
6 from twisted.python import usage, util, runtime
8 # this is mostly just a front-end for mktap, twistd, and kill(1), but in the
9 # future it will also provide an interface to some developer tools that talk
10 # directly to a remote buildmaster (like 'try' and a status client)
12 # the create/start/stop commands should all be run as the same user,
13 # preferably a separate 'buildbot' account.
15 class MakerBase(usage.Options):
16 optFlags = [
17 ['help', 'h', "Display this message"],
18 ["quiet", "q", "Do not emit the commands being run"],
21 #["basedir", "d", None, "Base directory for the buildmaster"],
22 opt_h = usage.Options.opt_help
24 def parseArgs(self, *args):
25 if len(args) > 0:
26 self['basedir'] = args[0]
27 else:
28 self['basedir'] = None
29 if len(args) > 1:
30 raise usage.UsageError("I wasn't expecting so many arguments")
32 def postOptions(self):
33 if self['basedir'] is None:
34 raise usage.UsageError("<basedir> parameter is required")
35 self['basedir'] = os.path.abspath(self['basedir'])
37 makefile_sample = """# -*- makefile -*-
39 # This is a simple makefile which lives in a buildmaster/buildslave
40 # directory (next to the buildbot.tac file). It allows you to start/stop the
41 # master or slave by doing 'make start' or 'make stop'.
43 # The 'reconfig' target will tell a buildmaster to reload its config file.
45 start:
46 twistd --no_save -y buildbot.tac
48 stop:
49 kill `cat twistd.pid`
51 reconfig:
52 kill -HUP `cat twistd.pid`
54 log:
55 tail -f twistd.log
56 """
58 class Maker:
59 def __init__(self, config):
60 self.config = config
61 self.basedir = config['basedir']
62 self.force = config['force']
63 self.quiet = config['quiet']
65 def mkdir(self):
66 if os.path.exists(self.basedir):
67 if not self.quiet:
68 print "updating existing installation"
69 return
70 if not self.quiet: print "mkdir", self.basedir
71 os.mkdir(self.basedir)
73 def mkinfo(self):
74 path = os.path.join(self.basedir, "info")
75 if not os.path.exists(path):
76 if not self.quiet: print "mkdir", path
77 os.mkdir(path)
78 created = False
79 admin = os.path.join(path, "admin")
80 if not os.path.exists(admin):
81 if not self.quiet:
82 print "Creating info/admin, you need to edit it appropriately"
83 f = open(admin, "wt")
84 f.write("Your Name Here <admin@youraddress.invalid>\n")
85 f.close()
86 created = True
87 host = os.path.join(path, "host")
88 if not os.path.exists(host):
89 if not self.quiet:
90 print "Creating info/host, you need to edit it appropriately"
91 f = open(host, "wt")
92 f.write("Please put a description of this build host here\n")
93 f.close()
94 created = True
95 if created and not self.quiet:
96 print "Please edit the files in %s appropriately." % path
98 def chdir(self):
99 if not self.quiet: print "chdir", self.basedir
100 os.chdir(self.basedir)
102 def makeTAC(self, contents, secret=False):
103 tacfile = "buildbot.tac"
104 if os.path.exists(tacfile):
105 oldcontents = open(tacfile, "rt").read()
106 if oldcontents == contents:
107 if not self.quiet:
108 print "buildbot.tac already exists and is correct"
109 return
110 if not self.quiet:
111 print "not touching existing buildbot.tac"
112 print "creating buildbot.tac.new instead"
113 tacfile = "buildbot.tac.new"
114 f = open(tacfile, "wt")
115 f.write(contents)
116 f.close()
117 if secret:
118 os.chmod(tacfile, 0600)
120 def makefile(self):
121 target = "Makefile.sample"
122 if os.path.exists(target):
123 oldcontents = open(target, "rt").read()
124 if oldcontents == makefile_sample:
125 if not self.quiet:
126 print "Makefile.sample already exists and is correct"
127 return
128 if not self.quiet:
129 print "replacing Makefile.sample"
130 else:
131 if not self.quiet:
132 print "creating Makefile.sample"
133 f = open(target, "wt")
134 f.write(makefile_sample)
135 f.close()
137 def sampleconfig(self, source):
138 target = "master.cfg.sample"
139 config_sample = open(source, "rt").read()
140 if os.path.exists(target):
141 oldcontents = open(target, "rt").read()
142 if oldcontents == config_sample:
143 if not self.quiet:
144 print "master.cfg.sample already exists and is up-to-date"
145 return
146 if not self.quiet:
147 print "replacing master.cfg.sample"
148 else:
149 if not self.quiet:
150 print "creating master.cfg.sample"
151 f = open(target, "wt")
152 f.write(config_sample)
153 f.close()
154 os.chmod(target, 0600)
156 class MasterOptions(MakerBase):
157 optFlags = [
158 ["force", "f",
159 "Re-use an existing directory (will not overwrite master.cfg file)"],
161 optParameters = [
162 ["config", "c", "master.cfg", "name of the buildmaster config file"],
164 def getSynopsis(self):
165 return "Usage: buildbot create-master [options] <basedir>"
167 longdesc = """
168 This command creates a buildmaster working directory and buildbot.tac
169 file. The master will live in <dir> and create various files there.
171 At runtime, the master will read a configuration file (named
172 'master.cfg' by default) in its basedir. This file should contain python
173 code which eventually defines a dictionary named 'BuildmasterConfig'.
174 The elements of this dictionary are used to configure the Buildmaster.
175 See doc/config.xhtml for details about what can be controlled through
176 this interface."""
178 masterTAC = """
179 from twisted.application import service
180 from buildbot.master import BuildMaster
182 basedir = r'%(basedir)s'
183 configfile = r'%(config)s'
185 application = service.Application('buildmaster')
186 BuildMaster(basedir, configfile).setServiceParent(application)
190 def createMaster(config):
191 m = Maker(config)
192 m.mkdir()
193 m.chdir()
194 contents = masterTAC % config
195 m.makeTAC(contents)
196 m.sampleconfig(util.sibpath(__file__, "sample.cfg"))
197 m.makefile()
199 if not m.quiet: print "buildmaster configured in %s" % m.basedir
201 class SlaveOptions(MakerBase):
202 optFlags = [
203 ["force", "f", "Re-use an existing directory"],
205 optParameters = [
206 # ["name", "n", None, "Name for this build slave"],
207 # ["passwd", "p", None, "Password for this build slave"],
208 # ["basedir", "d", ".", "Base directory to use"],
209 # ["master", "m", "localhost:8007",
210 # "Location of the buildmaster (host:port)"],
212 ["keepalive", "k", 600,
213 "Interval at which keepalives should be sent (in seconds)"],
214 ["usepty", None, 1,
215 "(1 or 0) child processes should be run in a pty"],
216 ["umask", None, "None",
217 "controls permissions of generated files. Use --umask=022 to be world-readable"],
220 longdesc = """
221 This command creates a buildslave working directory and buildbot.tac
222 file. The bot will use the <name> and <passwd> arguments to authenticate
223 itself when connecting to the master. All commands are run in a
224 build-specific subdirectory of <basedir>. <master> is a string of the
225 form 'hostname:port', and specifies where the buildmaster can be reached.
227 <name>, <passwd>, and <master> will be provided by the buildmaster
228 administrator for your bot. You must choose <basedir> yourself.
231 def getSynopsis(self):
232 return "Usage: buildbot create-slave [options] <basedir> <master> <name> <passwd>"
234 def parseArgs(self, *args):
235 if len(args) < 4:
236 raise usage.UsageError("command needs more arguments")
237 basedir, master, name, passwd = args
238 self['basedir'] = basedir
239 self['master'] = master
240 self['name'] = name
241 self['passwd'] = passwd
243 def postOptions(self):
244 MakerBase.postOptions(self)
245 self['usepty'] = int(self['usepty'])
246 self['keepalive'] = int(self['keepalive'])
247 if self['master'].find(":") == -1:
248 raise usage.UsageError("--master must be in the form host:portnum")
250 slaveTAC = """
251 from twisted.application import service
252 from buildbot.slave.bot import BuildSlave
254 basedir = r'%(basedir)s'
255 host = '%(host)s'
256 port = %(port)d
257 slavename = '%(name)s'
258 passwd = '%(passwd)s'
259 keepalive = %(keepalive)d
260 usepty = %(usepty)d
261 umask = %(umask)s
263 application = service.Application('buildslave')
264 s = BuildSlave(host, port, slavename, passwd, basedir, keepalive, usepty,
265 umask=umask)
266 s.setServiceParent(application)
270 def createSlave(config):
271 m = Maker(config)
272 m.mkdir()
273 m.chdir()
274 try:
275 master = config['master']
276 host, port = re.search(r'(.+):(\d+)', master).groups()
277 config['host'] = host
278 config['port'] = int(port)
279 except:
280 print "unparseable master location '%s'" % master
281 print " expecting something more like localhost:8007"
282 raise
283 contents = slaveTAC % config
285 m.makeTAC(contents, secret=True)
287 m.makefile()
288 m.mkinfo()
290 if not m.quiet: print "buildslave configured in %s" % m.basedir
293 def start(config):
294 basedir = config['basedir']
295 quiet = config['quiet']
296 os.chdir(basedir)
297 sys.path.insert(0, os.path.abspath(os.getcwd()))
298 if os.path.exists("/usr/bin/make") and os.path.exists("Makefile.buildbot"):
299 # Preferring the Makefile lets slave admins do useful things like set
300 # up environment variables for the buildslave.
301 cmd = "make -f Makefile.buildbot start"
302 if not quiet: print cmd
303 os.system(cmd)
304 else:
305 # see if we can launch the application without actually having to
306 # spawn twistd, since spawning processes correctly is a real hassle
307 # on windows.
308 from twisted.python.runtime import platformType
309 argv = ["twistd",
310 "--no_save",
311 "--logfile=twistd.log", # windows doesn't use the same default
312 "--python=buildbot.tac"]
313 if platformType == "win32":
314 argv.append("--reactor=win32")
315 sys.argv = argv
317 # this is copied from bin/twistd. twisted-1.3.0 uses twistw, while
318 # twisted-2.0.0 uses _twistw.
319 if platformType == "win32":
320 try:
321 from twisted.scripts._twistw import run
322 except ImportError:
323 from twisted.scripts.twistw import run
324 else:
325 from twisted.scripts.twistd import run
326 run()
329 def stop(config, signame="TERM", wait=False):
330 import signal
331 basedir = config['basedir']
332 quiet = config['quiet']
333 os.chdir(basedir)
334 f = open("twistd.pid", "rt")
335 pid = int(f.read().strip())
336 signum = getattr(signal, "SIG"+signame)
337 timer = 0
338 os.kill(pid, signum)
339 if not wait:
340 print "sent SIG%s to process" % signame
341 return
342 time.sleep(0.1)
343 while timer < 5:
344 # poll once per second until twistd.pid goes away, up to 5 seconds
345 try:
346 os.kill(pid, 0)
347 except OSError:
348 print "buildbot process %d is dead" % pid
349 return
350 timer += 1
351 time.sleep(1)
352 print "never saw process go away"
354 def restart(config):
355 stop(config, wait=True)
356 print "now restarting buildbot process.."
357 start(config)
358 # this next line might not be printed, if start() ended up running twistd
359 # inline
360 print "buildbot process has been restarted"
363 def loadOptions(filename="options", here=None, home=None):
364 """Find the .buildbot/FILENAME file. Crawl from the current directory up
365 towards the root, and also look in ~/.buildbot . The first directory
366 that's owned by the user and has the file we're looking for wins. Windows
367 skips the owned-by-user test.
369 @rtype: dict
370 @return: a dictionary of names defined in the options file. If no options
371 file was found, return an empty dict.
374 if here is None:
375 here = os.getcwd()
376 here = os.path.abspath(here)
378 if home is None:
379 if runtime.platformType == 'win32':
380 home = os.path.join(os.environ['APPDATA'], "buildbot")
381 else:
382 home = os.path.expanduser("~/.buildbot")
384 searchpath = []
385 toomany = 20
386 while True:
387 searchpath.append(os.path.join(here, ".buildbot"))
388 next = os.path.dirname(here)
389 if next == here:
390 break # we've hit the root
391 here = next
392 toomany -= 1 # just in case
393 if toomany == 0:
394 raise ValueError("Hey, I seem to have wandered up into the "
395 "infinite glories of the heavens. Oops.")
396 searchpath.append(home)
398 localDict = {}
400 for d in searchpath:
401 if os.path.isdir(d):
402 if runtime.platformType != 'win32':
403 if os.stat(d)[stat.ST_UID] != os.getuid():
404 print "skipping %s because you don't own it" % d
405 continue # security, skip other people's directories
406 optfile = os.path.join(d, filename)
407 if os.path.exists(optfile):
408 try:
409 f = open(optfile, "r")
410 options = f.read()
411 exec options in localDict
412 except:
413 print "error while reading %s" % optfile
414 raise
415 break
417 for k in localDict.keys():
418 if k.startswith("__"):
419 del localDict[k]
420 return localDict
422 class StartOptions(MakerBase):
423 def getSynopsis(self):
424 return "Usage: buildbot start <basedir>"
426 class StopOptions(MakerBase):
427 def getSynopsis(self):
428 return "Usage: buildbot stop <basedir>"
430 class ReconfigOptions(MakerBase):
431 optFlags = [
432 ['quiet', 'q', "Don't display log messages about reconfiguration"],
434 def getSynopsis(self):
435 return "Usage: buildbot reconfig <basedir>"
439 class RestartOptions(MakerBase):
440 def getSynopsis(self):
441 return "Usage: buildbot restart <basedir>"
443 class DebugClientOptions(usage.Options):
444 optFlags = [
445 ['help', 'h', "Display this message"],
447 optParameters = [
448 ["master", "m", None,
449 "Location of the buildmaster's slaveport (host:port)"],
450 ["passwd", "p", None, "Debug password to use"],
453 def parseArgs(self, *args):
454 if len(args) > 0:
455 self['master'] = args[0]
456 if len(args) > 1:
457 self['passwd'] = args[1]
458 if len(args) > 2:
459 raise usage.UsageError("I wasn't expecting so many arguments")
461 def debugclient(config):
462 from buildbot.clients import debug
463 opts = loadOptions()
465 master = config.get('master')
466 if not master:
467 master = opts.get('master')
468 if master is None:
469 raise usage.UsageError("master must be specified: on the command "
470 "line or in ~/.buildbot/options")
472 passwd = config.get('passwd')
473 if not passwd:
474 passwd = opts.get('debugPassword')
475 if passwd is None:
476 raise usage.UsageError("passwd must be specified: on the command "
477 "line or in ~/.buildbot/options")
479 d = debug.DebugWidget(master, passwd)
480 d.run()
482 class StatusClientOptions(usage.Options):
483 optFlags = [
484 ['help', 'h', "Display this message"],
486 optParameters = [
487 ["master", "m", None,
488 "Location of the buildmaster's status port (host:port)"],
491 def parseArgs(self, *args):
492 if len(args) > 0:
493 self['master'] = args[0]
494 if len(args) > 1:
495 raise usage.UsageError("I wasn't expecting so many arguments")
497 def statuslog(config):
498 from buildbot.clients import base
499 opts = loadOptions()
500 master = config.get('master')
501 if not master:
502 master = opts.get('masterstatus')
503 if master is None:
504 raise usage.UsageError("master must be specified: on the command "
505 "line or in ~/.buildbot/options")
506 c = base.TextClient(master)
507 c.run()
509 def statusgui(config):
510 from buildbot.clients import gtkPanes
511 opts = loadOptions()
512 master = config.get('master')
513 if not master:
514 master = opts.get('masterstatus')
515 if master is None:
516 raise usage.UsageError("master must be specified: on the command "
517 "line or in ~/.buildbot/options")
518 c = gtkPanes.GtkClient(master)
519 c.run()
521 class SendChangeOptions(usage.Options):
522 optParameters = [
523 ("master", "m", None,
524 "Location of the buildmaster's PBListener (host:port)"),
525 ("username", "u", None, "Username performing the commit"),
526 ("branch", "b", None, "Branch specifier"),
527 ("revision", "r", None, "Revision specifier (string)"),
528 ("revision_number", "n", None, "Revision specifier (integer)"),
529 ("revision_file", None, None, "Filename containing revision spec"),
530 ("comments", "m", None, "log message"),
531 ("logfile", "F", None,
532 "Read the log messages from this file (- for stdin)"),
534 def getSynopsis(self):
535 return "Usage: buildbot sendchange [options] filenames.."
536 def parseArgs(self, *args):
537 self['files'] = args
540 def sendchange(config, runReactor=False):
541 """Send a single change to the buildmaster's PBChangeSource. The
542 connection will be drpoped as soon as the Change has been sent."""
543 from buildbot.clients.sendchange import Sender
545 opts = loadOptions()
546 user = config.get('username', opts.get('username'))
547 master = config.get('master', opts.get('master'))
548 branch = config.get('branch', opts.get('branch'))
549 revision = config.get('revision')
550 # SVN and P4 use numeric revisions
551 if config.get("revision_number"):
552 revision = int(config['revision_number'])
553 if config.get("revision_file"):
554 revision = open(config["revision_file"],"r").read()
556 comments = config.get('comments')
557 if not comments and config.get('logfile'):
558 if config['logfile'] == "-":
559 f = sys.stdin
560 else:
561 f = open(config['logfile'], "rt")
562 comments = f.read()
563 if comments is None:
564 comments = ""
566 files = config.get('files', [])
568 assert user, "you must provide a username"
569 assert master, "you must provide the master location"
571 s = Sender(master, user)
572 d = s.send(branch, revision, comments, files)
573 if runReactor:
574 d.addCallbacks(s.printSuccess, s.printFailure)
575 d.addCallback(s.stop)
576 s.run()
577 return d
580 class ForceOptions(usage.Options):
581 optParameters = [
582 ["builder", None, None, "which Builder to start"],
583 ["branch", None, None, "which branch to build"],
584 ["revision", None, None, "which revision to build"],
585 ["reason", None, None, "the reason for starting the build"],
588 def parseArgs(self, *args):
589 args = list(args)
590 if len(args) > 0:
591 if self['builder'] is not None:
592 raise usage.UsageError("--builder provided in two ways")
593 self['builder'] = args.pop(0)
594 if len(args) > 0:
595 if self['reason'] is not None:
596 raise usage.UsageError("--reason provided in two ways")
597 self['reason'] = " ".join(args)
600 class TryOptions(usage.Options):
601 optParameters = [
602 ["connect", "c", None,
603 "how to reach the buildmaster, either 'ssh' or 'pb'"],
604 # for ssh, use --tryhost, --username, and --trydir
605 ["tryhost", None, None,
606 "the hostname (used by ssh) for the buildmaster"],
607 ["trydir", None, None,
608 "the directory (on the tryhost) where tryjobs are deposited"],
609 ["username", "u", None, "Username performing the trial build"],
610 # for PB, use --master, --username, and --passwd
611 ["master", "m", None,
612 "Location of the buildmaster's PBListener (host:port)"],
613 ["passwd", None, None, "password for PB authentication"],
615 ["vc", None, None,
616 "The VC system in use, one of: cvs,svn,tla,baz,darcs"],
617 ["branch", None, None,
618 "The branch in use, for VC systems that can't figure it out"
619 " themselves"],
621 ["builder", "b", None,
622 "Run the trial build on this Builder. Can be used multiple times."],
625 optFlags = [
626 ["wait", None, "wait until the builds have finished"],
629 def __init__(self):
630 super(TryOptions, self).__init__()
631 self['builders'] = []
633 def opt_builder(self, option):
634 self['builders'].append(option)
636 def getSynopsis(self):
637 return "Usage: buildbot try [options]"
639 def doTry(config):
640 from buildbot.scripts import tryclient
641 t = tryclient.Try(config)
642 t.run()
644 class TryServerOptions(usage.Options):
645 optParameters = [
646 ["jobdir", None, None, "the jobdir (maildir) for submitting jobs"],
649 def doTryServer(config):
650 import md5
651 jobdir = os.path.expanduser(config["jobdir"])
652 job = sys.stdin.read()
653 # now do a 'safecat'-style write to jobdir/tmp, then move atomically to
654 # jobdir/new . Rather than come up with a unique name randomly, I'm just
655 # going to MD5 the contents and prepend a timestamp.
656 timestring = "%d" % time.time()
657 jobhash = md5.new(job).hexdigest()
658 fn = "%s-%s" % (timestring, jobhash)
659 tmpfile = os.path.join(jobdir, "tmp", fn)
660 newfile = os.path.join(jobdir, "new", fn)
661 f = open(tmpfile, "w")
662 f.write(job)
663 f.close()
664 os.rename(tmpfile, newfile)
667 class Options(usage.Options):
668 synopsis = "Usage: buildbot <command> [command options]"
670 subCommands = [
671 # the following are all admin commands
672 ['create-master', None, MasterOptions,
673 "Create and populate a directory for a new buildmaster"],
674 ['create-slave', None, SlaveOptions,
675 "Create and populate a directory for a new buildslave"],
676 ['start', None, StartOptions, "Start a buildmaster or buildslave"],
677 ['stop', None, StopOptions, "Stop a buildmaster or buildslave"],
678 ['restart', None, RestartOptions,
679 "Restart a buildmaster or buildslave"],
681 ['reconfig', None, ReconfigOptions,
682 "SIGHUP a buildmaster to make it re-read the config file"],
683 ['sighup', None, ReconfigOptions,
684 "SIGHUP a buildmaster to make it re-read the config file"],
686 ['sendchange', None, SendChangeOptions,
687 "Send a change to the buildmaster"],
689 ['debugclient', None, DebugClientOptions,
690 "Launch a small debug panel GUI"],
692 ['statuslog', None, StatusClientOptions,
693 "Emit current builder status to stdout"],
694 ['statusgui', None, StatusClientOptions,
695 "Display a small window showing current builder status"],
697 #['force', None, ForceOptions, "Run a build"],
698 ['try', None, TryOptions, "Run a build with your local changes"],
700 ['tryserver', None, TryServerOptions,
701 "buildmaster-side 'try' support function, not for users"],
703 # TODO: 'watch'
706 def opt_version(self):
707 import buildbot
708 print "Buildbot version: %s" % buildbot.version
709 usage.Options.opt_version(self)
711 def opt_verbose(self):
712 from twisted.python import log
713 log.startLogging(sys.stderr)
715 def postOptions(self):
716 if not hasattr(self, 'subOptions'):
717 raise usage.UsageError("must specify a command")
720 def run():
721 config = Options()
722 try:
723 config.parseOptions()
724 except usage.error, e:
725 print "%s: %s" % (sys.argv[0], e)
726 print
727 c = getattr(config, 'subOptions', config)
728 print str(c)
729 sys.exit(1)
731 command = config.subCommand
732 so = config.subOptions
734 if command == "create-master":
735 createMaster(so)
736 elif command == "create-slave":
737 createSlave(so)
738 elif command == "start":
739 start(so)
740 elif command == "stop":
741 stop(so, wait=True)
742 elif command == "restart":
743 restart(so)
744 elif command == "sighup":
745 from buildbot.scripts.reconfig import Reconfigurator
746 Reconfigurator().run(so)
747 elif command == "sendchange":
748 sendchange(so, True)
749 elif command == "debugclient":
750 debugclient(so)
751 elif command == "statuslog":
752 statuslog(so)
753 elif command == "statusgui":
754 statusgui(so)
755 elif command == "try":
756 doTry(so)
757 elif command == "tryserver":
758 doTryServer(so)