update addLogObserver example to use 0.7.5 buildbot.steps.* names
[buildbot.git] / buildbot / scripts / runner.py
blob2769f91e5f6734923d7306665297af7fc47e138a
1 # -*- test-case-name: buildbot.test.test_runner -*-
3 # N.B.: don't import anything that might pull in a reactor yet. Some of our
4 # subcommands want to load modules that need the gtk reactor.
5 import os, sys, stat, re, time
6 from twisted.python import usage, util, runtime
8 # this is mostly just a front-end for mktap, twistd, and kill(1), but in the
9 # future it will also provide an interface to some developer tools that talk
10 # directly to a remote buildmaster (like 'try' and a status client)
12 # the create/start/stop commands should all be run as the same user,
13 # preferably a separate 'buildbot' account.
15 class MakerBase(usage.Options):
16 optFlags = [
17 ['help', 'h', "Display this message"],
18 ["quiet", "q", "Do not emit the commands being run"],
21 #["basedir", "d", None, "Base directory for the buildmaster"],
22 opt_h = usage.Options.opt_help
24 def parseArgs(self, *args):
25 if len(args) > 0:
26 self['basedir'] = args[0]
27 else:
28 self['basedir'] = None
29 if len(args) > 1:
30 raise usage.UsageError("I wasn't expecting so many arguments")
32 def postOptions(self):
33 if self['basedir'] is None:
34 raise usage.UsageError("<basedir> parameter is required")
35 self['basedir'] = os.path.abspath(self['basedir'])
37 makefile_sample = """# -*- makefile -*-
39 # This is a simple makefile which lives in a buildmaster/buildslave
40 # directory (next to the buildbot.tac file). It allows you to start/stop the
41 # master or slave by doing 'make start' or 'make stop'.
43 # The 'reconfig' target will tell a buildmaster to reload its config file.
45 start:
46 twistd --no_save -y buildbot.tac
48 stop:
49 kill `cat twistd.pid`
51 reconfig:
52 kill -HUP `cat twistd.pid`
54 log:
55 tail -f twistd.log
56 """
58 class Maker:
59 def __init__(self, config):
60 self.config = config
61 self.basedir = config['basedir']
62 self.force = config['force']
63 self.quiet = config['quiet']
65 def mkdir(self):
66 if os.path.exists(self.basedir):
67 if not self.quiet:
68 print "updating existing installation"
69 return
70 if not self.quiet: print "mkdir", self.basedir
71 os.mkdir(self.basedir)
73 def mkinfo(self):
74 path = os.path.join(self.basedir, "info")
75 if not os.path.exists(path):
76 if not self.quiet: print "mkdir", path
77 os.mkdir(path)
78 created = False
79 admin = os.path.join(path, "admin")
80 if not os.path.exists(admin):
81 if not self.quiet:
82 print "Creating info/admin, you need to edit it appropriately"
83 f = open(admin, "wt")
84 f.write("Your Name Here <admin@youraddress.invalid>\n")
85 f.close()
86 created = True
87 host = os.path.join(path, "host")
88 if not os.path.exists(host):
89 if not self.quiet:
90 print "Creating info/host, you need to edit it appropriately"
91 f = open(host, "wt")
92 f.write("Please put a description of this build host here\n")
93 f.close()
94 created = True
95 if created and not self.quiet:
96 print "Please edit the files in %s appropriately." % path
98 def chdir(self):
99 if not self.quiet: print "chdir", self.basedir
100 os.chdir(self.basedir)
102 def makeTAC(self, contents, secret=False):
103 tacfile = "buildbot.tac"
104 if os.path.exists(tacfile):
105 oldcontents = open(tacfile, "rt").read()
106 if oldcontents == contents:
107 if not self.quiet:
108 print "buildbot.tac already exists and is correct"
109 return
110 if not self.quiet:
111 print "not touching existing buildbot.tac"
112 print "creating buildbot.tac.new instead"
113 tacfile = "buildbot.tac.new"
114 f = open(tacfile, "wt")
115 f.write(contents)
116 f.close()
117 if secret:
118 os.chmod(tacfile, 0600)
120 def makefile(self):
121 target = "Makefile.sample"
122 if os.path.exists(target):
123 oldcontents = open(target, "rt").read()
124 if oldcontents == makefile_sample:
125 if not self.quiet:
126 print "Makefile.sample already exists and is correct"
127 return
128 if not self.quiet:
129 print "replacing Makefile.sample"
130 else:
131 if not self.quiet:
132 print "creating Makefile.sample"
133 f = open(target, "wt")
134 f.write(makefile_sample)
135 f.close()
137 def sampleconfig(self, source):
138 target = "master.cfg.sample"
139 config_sample = open(source, "rt").read()
140 if os.path.exists(target):
141 oldcontents = open(target, "rt").read()
142 if oldcontents == config_sample:
143 if not self.quiet:
144 print "master.cfg.sample already exists and is up-to-date"
145 return
146 if not self.quiet:
147 print "replacing master.cfg.sample"
148 else:
149 if not self.quiet:
150 print "creating master.cfg.sample"
151 f = open(target, "wt")
152 f.write(config_sample)
153 f.close()
154 os.chmod(target, 0600)
156 class MasterOptions(MakerBase):
157 optFlags = [
158 ["force", "f",
159 "Re-use an existing directory (will not overwrite master.cfg file)"],
161 optParameters = [
162 ["config", "c", "master.cfg", "name of the buildmaster config file"],
164 def getSynopsis(self):
165 return "Usage: buildbot create-master [options] <basedir>"
167 longdesc = """
168 This command creates a buildmaster working directory and buildbot.tac
169 file. The master will live in <dir> and create various files there.
171 At runtime, the master will read a configuration file (named
172 'master.cfg' by default) in its basedir. This file should contain python
173 code which eventually defines a dictionary named 'BuildmasterConfig'.
174 The elements of this dictionary are used to configure the Buildmaster.
175 See doc/config.xhtml for details about what can be controlled through
176 this interface."""
178 masterTAC = """
179 from twisted.application import service
180 from buildbot.master import BuildMaster
182 basedir = r'%(basedir)s'
183 configfile = r'%(config)s'
185 application = service.Application('buildmaster')
186 BuildMaster(basedir, configfile).setServiceParent(application)
190 def createMaster(config):
191 m = Maker(config)
192 m.mkdir()
193 m.chdir()
194 contents = masterTAC % config
195 m.makeTAC(contents)
196 m.sampleconfig(util.sibpath(__file__, "sample.cfg"))
197 m.makefile()
199 if not m.quiet: print "buildmaster configured in %s" % m.basedir
201 class SlaveOptions(MakerBase):
202 optFlags = [
203 ["force", "f", "Re-use an existing directory"],
205 optParameters = [
206 # ["name", "n", None, "Name for this build slave"],
207 # ["passwd", "p", None, "Password for this build slave"],
208 # ["basedir", "d", ".", "Base directory to use"],
209 # ["master", "m", "localhost:8007",
210 # "Location of the buildmaster (host:port)"],
212 ["keepalive", "k", 600,
213 "Interval at which keepalives should be sent (in seconds)"],
214 ["usepty", None, 1,
215 "(1 or 0) child processes should be run in a pty"],
216 ["umask", None, "None",
217 "controls permissions of generated files. Use --umask=022 to be world-readable"],
220 longdesc = """
221 This command creates a buildslave working directory and buildbot.tac
222 file. The bot will use the <name> and <passwd> arguments to authenticate
223 itself when connecting to the master. All commands are run in a
224 build-specific subdirectory of <basedir>. <master> is a string of the
225 form 'hostname:port', and specifies where the buildmaster can be reached.
227 <name>, <passwd>, and <master> will be provided by the buildmaster
228 administrator for your bot. You must choose <basedir> yourself.
231 def getSynopsis(self):
232 return "Usage: buildbot create-slave [options] <basedir> <master> <name> <passwd>"
234 def parseArgs(self, *args):
235 if len(args) < 4:
236 raise usage.UsageError("command needs more arguments")
237 basedir, master, name, passwd = args
238 self['basedir'] = basedir
239 self['master'] = master
240 self['name'] = name
241 self['passwd'] = passwd
243 def postOptions(self):
244 MakerBase.postOptions(self)
245 self['usepty'] = int(self['usepty'])
246 self['keepalive'] = int(self['keepalive'])
247 if self['master'].find(":") == -1:
248 raise usage.UsageError("--master must be in the form host:portnum")
250 slaveTAC = """
251 from twisted.application import service
252 from buildbot.slave.bot import BuildSlave
254 basedir = r'%(basedir)s'
255 host = '%(host)s'
256 port = %(port)d
257 slavename = '%(name)s'
258 passwd = '%(passwd)s'
259 keepalive = %(keepalive)d
260 usepty = %(usepty)d
261 umask = %(umask)s
263 application = service.Application('buildslave')
264 s = BuildSlave(host, port, slavename, passwd, basedir, keepalive, usepty,
265 umask=umask)
266 s.setServiceParent(application)
270 def createSlave(config):
271 m = Maker(config)
272 m.mkdir()
273 m.chdir()
274 try:
275 master = config['master']
276 host, port = re.search(r'(.+):(\d+)', master).groups()
277 config['host'] = host
278 config['port'] = int(port)
279 except:
280 print "unparseable master location '%s'" % master
281 print " expecting something more like localhost:8007"
282 raise
283 contents = slaveTAC % config
285 m.makeTAC(contents, secret=True)
287 m.makefile()
288 m.mkinfo()
290 if not m.quiet: print "buildslave configured in %s" % m.basedir
294 def stop(config, signame="TERM", wait=False):
295 import signal
296 basedir = config['basedir']
297 quiet = config['quiet']
298 os.chdir(basedir)
299 f = open("twistd.pid", "rt")
300 pid = int(f.read().strip())
301 signum = getattr(signal, "SIG"+signame)
302 timer = 0
303 os.kill(pid, signum)
304 if not wait:
305 if not quiet:
306 print "sent SIG%s to process" % signame
307 return
308 time.sleep(0.1)
309 while timer < 5:
310 # poll once per second until twistd.pid goes away, up to 5 seconds
311 try:
312 os.kill(pid, 0)
313 except OSError:
314 if not quiet:
315 print "buildbot process %d is dead" % pid
316 return
317 timer += 1
318 time.sleep(1)
319 if not quiet:
320 print "never saw process go away"
322 def restart(config):
323 quiet = config['quiet']
324 from buildbot.scripts.startup import start
325 stop(config, wait=True)
326 if not quiet:
327 print "now restarting buildbot process.."
328 start(config)
331 def loadOptions(filename="options", here=None, home=None):
332 """Find the .buildbot/FILENAME file. Crawl from the current directory up
333 towards the root, and also look in ~/.buildbot . The first directory
334 that's owned by the user and has the file we're looking for wins. Windows
335 skips the owned-by-user test.
337 @rtype: dict
338 @return: a dictionary of names defined in the options file. If no options
339 file was found, return an empty dict.
342 if here is None:
343 here = os.getcwd()
344 here = os.path.abspath(here)
346 if home is None:
347 if runtime.platformType == 'win32':
348 home = os.path.join(os.environ['APPDATA'], "buildbot")
349 else:
350 home = os.path.expanduser("~/.buildbot")
352 searchpath = []
353 toomany = 20
354 while True:
355 searchpath.append(os.path.join(here, ".buildbot"))
356 next = os.path.dirname(here)
357 if next == here:
358 break # we've hit the root
359 here = next
360 toomany -= 1 # just in case
361 if toomany == 0:
362 raise ValueError("Hey, I seem to have wandered up into the "
363 "infinite glories of the heavens. Oops.")
364 searchpath.append(home)
366 localDict = {}
368 for d in searchpath:
369 if os.path.isdir(d):
370 if runtime.platformType != 'win32':
371 if os.stat(d)[stat.ST_UID] != os.getuid():
372 print "skipping %s because you don't own it" % d
373 continue # security, skip other people's directories
374 optfile = os.path.join(d, filename)
375 if os.path.exists(optfile):
376 try:
377 f = open(optfile, "r")
378 options = f.read()
379 exec options in localDict
380 except:
381 print "error while reading %s" % optfile
382 raise
383 break
385 for k in localDict.keys():
386 if k.startswith("__"):
387 del localDict[k]
388 return localDict
390 class StartOptions(MakerBase):
391 optFlags = [
392 ['quiet', 'q', "Don't display startup log messages"],
394 def getSynopsis(self):
395 return "Usage: buildbot start <basedir>"
397 class StopOptions(MakerBase):
398 def getSynopsis(self):
399 return "Usage: buildbot stop <basedir>"
401 class ReconfigOptions(MakerBase):
402 optFlags = [
403 ['quiet', 'q', "Don't display log messages about reconfiguration"],
405 def getSynopsis(self):
406 return "Usage: buildbot reconfig <basedir>"
410 class RestartOptions(MakerBase):
411 optFlags = [
412 ['quiet', 'q', "Don't display startup log messages"],
414 def getSynopsis(self):
415 return "Usage: buildbot restart <basedir>"
417 class DebugClientOptions(usage.Options):
418 optFlags = [
419 ['help', 'h', "Display this message"],
421 optParameters = [
422 ["master", "m", None,
423 "Location of the buildmaster's slaveport (host:port)"],
424 ["passwd", "p", None, "Debug password to use"],
427 def parseArgs(self, *args):
428 if len(args) > 0:
429 self['master'] = args[0]
430 if len(args) > 1:
431 self['passwd'] = args[1]
432 if len(args) > 2:
433 raise usage.UsageError("I wasn't expecting so many arguments")
435 def debugclient(config):
436 from buildbot.clients import debug
437 opts = loadOptions()
439 master = config.get('master')
440 if not master:
441 master = opts.get('master')
442 if master is None:
443 raise usage.UsageError("master must be specified: on the command "
444 "line or in ~/.buildbot/options")
446 passwd = config.get('passwd')
447 if not passwd:
448 passwd = opts.get('debugPassword')
449 if passwd is None:
450 raise usage.UsageError("passwd must be specified: on the command "
451 "line or in ~/.buildbot/options")
453 d = debug.DebugWidget(master, passwd)
454 d.run()
456 class StatusClientOptions(usage.Options):
457 optFlags = [
458 ['help', 'h', "Display this message"],
460 optParameters = [
461 ["master", "m", None,
462 "Location of the buildmaster's status port (host:port)"],
465 def parseArgs(self, *args):
466 if len(args) > 0:
467 self['master'] = args[0]
468 if len(args) > 1:
469 raise usage.UsageError("I wasn't expecting so many arguments")
471 def statuslog(config):
472 from buildbot.clients import base
473 opts = loadOptions()
474 master = config.get('master')
475 if not master:
476 master = opts.get('masterstatus')
477 if master is None:
478 raise usage.UsageError("master must be specified: on the command "
479 "line or in ~/.buildbot/options")
480 c = base.TextClient(master)
481 c.run()
483 def statusgui(config):
484 from buildbot.clients import gtkPanes
485 opts = loadOptions()
486 master = config.get('master')
487 if not master:
488 master = opts.get('masterstatus')
489 if master is None:
490 raise usage.UsageError("master must be specified: on the command "
491 "line or in ~/.buildbot/options")
492 c = gtkPanes.GtkClient(master)
493 c.run()
495 class SendChangeOptions(usage.Options):
496 optParameters = [
497 ("master", "m", None,
498 "Location of the buildmaster's PBListener (host:port)"),
499 ("username", "u", None, "Username performing the commit"),
500 ("branch", "b", None, "Branch specifier"),
501 ("revision", "r", None, "Revision specifier (string)"),
502 ("revision_number", "n", None, "Revision specifier (integer)"),
503 ("revision_file", None, None, "Filename containing revision spec"),
504 ("comments", "m", None, "log message"),
505 ("logfile", "F", None,
506 "Read the log messages from this file (- for stdin)"),
508 def getSynopsis(self):
509 return "Usage: buildbot sendchange [options] filenames.."
510 def parseArgs(self, *args):
511 self['files'] = args
514 def sendchange(config, runReactor=False):
515 """Send a single change to the buildmaster's PBChangeSource. The
516 connection will be drpoped as soon as the Change has been sent."""
517 from buildbot.clients.sendchange import Sender
519 opts = loadOptions()
520 user = config.get('username', opts.get('username'))
521 master = config.get('master', opts.get('master'))
522 branch = config.get('branch', opts.get('branch'))
523 revision = config.get('revision')
524 # SVN and P4 use numeric revisions
525 if config.get("revision_number"):
526 revision = int(config['revision_number'])
527 if config.get("revision_file"):
528 revision = open(config["revision_file"],"r").read()
530 comments = config.get('comments')
531 if not comments and config.get('logfile'):
532 if config['logfile'] == "-":
533 f = sys.stdin
534 else:
535 f = open(config['logfile'], "rt")
536 comments = f.read()
537 if comments is None:
538 comments = ""
540 files = config.get('files', [])
542 assert user, "you must provide a username"
543 assert master, "you must provide the master location"
545 s = Sender(master, user)
546 d = s.send(branch, revision, comments, files)
547 if runReactor:
548 d.addCallbacks(s.printSuccess, s.printFailure)
549 d.addCallback(s.stop)
550 s.run()
551 return d
554 class ForceOptions(usage.Options):
555 optParameters = [
556 ["builder", None, None, "which Builder to start"],
557 ["branch", None, None, "which branch to build"],
558 ["revision", None, None, "which revision to build"],
559 ["reason", None, None, "the reason for starting the build"],
562 def parseArgs(self, *args):
563 args = list(args)
564 if len(args) > 0:
565 if self['builder'] is not None:
566 raise usage.UsageError("--builder provided in two ways")
567 self['builder'] = args.pop(0)
568 if len(args) > 0:
569 if self['reason'] is not None:
570 raise usage.UsageError("--reason provided in two ways")
571 self['reason'] = " ".join(args)
574 class TryOptions(usage.Options):
575 optParameters = [
576 ["connect", "c", None,
577 "how to reach the buildmaster, either 'ssh' or 'pb'"],
578 # for ssh, use --tryhost, --username, and --trydir
579 ["tryhost", None, None,
580 "the hostname (used by ssh) for the buildmaster"],
581 ["trydir", None, None,
582 "the directory (on the tryhost) where tryjobs are deposited"],
583 ["username", "u", None, "Username performing the trial build"],
584 # for PB, use --master, --username, and --passwd
585 ["master", "m", None,
586 "Location of the buildmaster's PBListener (host:port)"],
587 ["passwd", None, None, "password for PB authentication"],
589 ["vc", None, None,
590 "The VC system in use, one of: cvs,svn,tla,baz,darcs"],
591 ["branch", None, None,
592 "The branch in use, for VC systems that can't figure it out"
593 " themselves"],
595 ["builder", "b", None,
596 "Run the trial build on this Builder. Can be used multiple times."],
599 optFlags = [
600 ["wait", None, "wait until the builds have finished"],
603 def __init__(self):
604 super(TryOptions, self).__init__()
605 self['builders'] = []
607 def opt_builder(self, option):
608 self['builders'].append(option)
610 def getSynopsis(self):
611 return "Usage: buildbot try [options]"
613 def doTry(config):
614 from buildbot.scripts import tryclient
615 t = tryclient.Try(config)
616 t.run()
618 class TryServerOptions(usage.Options):
619 optParameters = [
620 ["jobdir", None, None, "the jobdir (maildir) for submitting jobs"],
623 def doTryServer(config):
624 import md5
625 jobdir = os.path.expanduser(config["jobdir"])
626 job = sys.stdin.read()
627 # now do a 'safecat'-style write to jobdir/tmp, then move atomically to
628 # jobdir/new . Rather than come up with a unique name randomly, I'm just
629 # going to MD5 the contents and prepend a timestamp.
630 timestring = "%d" % time.time()
631 jobhash = md5.new(job).hexdigest()
632 fn = "%s-%s" % (timestring, jobhash)
633 tmpfile = os.path.join(jobdir, "tmp", fn)
634 newfile = os.path.join(jobdir, "new", fn)
635 f = open(tmpfile, "w")
636 f.write(job)
637 f.close()
638 os.rename(tmpfile, newfile)
641 class Options(usage.Options):
642 synopsis = "Usage: buildbot <command> [command options]"
644 subCommands = [
645 # the following are all admin commands
646 ['create-master', None, MasterOptions,
647 "Create and populate a directory for a new buildmaster"],
648 ['create-slave', None, SlaveOptions,
649 "Create and populate a directory for a new buildslave"],
650 ['start', None, StartOptions, "Start a buildmaster or buildslave"],
651 ['stop', None, StopOptions, "Stop a buildmaster or buildslave"],
652 ['restart', None, RestartOptions,
653 "Restart a buildmaster or buildslave"],
655 ['reconfig', None, ReconfigOptions,
656 "SIGHUP a buildmaster to make it re-read the config file"],
657 ['sighup', None, ReconfigOptions,
658 "SIGHUP a buildmaster to make it re-read the config file"],
660 ['sendchange', None, SendChangeOptions,
661 "Send a change to the buildmaster"],
663 ['debugclient', None, DebugClientOptions,
664 "Launch a small debug panel GUI"],
666 ['statuslog', None, StatusClientOptions,
667 "Emit current builder status to stdout"],
668 ['statusgui', None, StatusClientOptions,
669 "Display a small window showing current builder status"],
671 #['force', None, ForceOptions, "Run a build"],
672 ['try', None, TryOptions, "Run a build with your local changes"],
674 ['tryserver', None, TryServerOptions,
675 "buildmaster-side 'try' support function, not for users"],
677 # TODO: 'watch'
680 def opt_version(self):
681 import buildbot
682 print "Buildbot version: %s" % buildbot.version
683 usage.Options.opt_version(self)
685 def opt_verbose(self):
686 from twisted.python import log
687 log.startLogging(sys.stderr)
689 def postOptions(self):
690 if not hasattr(self, 'subOptions'):
691 raise usage.UsageError("must specify a command")
694 def run():
695 config = Options()
696 try:
697 config.parseOptions()
698 except usage.error, e:
699 print "%s: %s" % (sys.argv[0], e)
700 print
701 c = getattr(config, 'subOptions', config)
702 print str(c)
703 sys.exit(1)
705 command = config.subCommand
706 so = config.subOptions
708 if command == "create-master":
709 createMaster(so)
710 elif command == "create-slave":
711 createSlave(so)
712 elif command == "start":
713 from buildbot.scripts.startup import start
714 start(so)
715 elif command == "stop":
716 stop(so, wait=True)
717 elif command == "restart":
718 restart(so)
719 elif command == "reconfig" or command == "sighup":
720 from buildbot.scripts.reconfig import Reconfigurator
721 Reconfigurator().run(so)
722 elif command == "sendchange":
723 sendchange(so, True)
724 elif command == "debugclient":
725 debugclient(so)
726 elif command == "statuslog":
727 statuslog(so)
728 elif command == "statusgui":
729 statusgui(so)
730 elif command == "try":
731 doTry(so)
732 elif command == "tryserver":
733 doTryServer(so)