Allow users to specify the remote git branch.
[buildbot.git] / buildbot / scripts / tryclient.py
blobb1b7658f0762febf7e5ed78236208be35c8ef5ea
1 # -*- test-case-name: buildbot.test.test_scheduler,buildbot.test.test_vc -*-
3 import sys, os, re, time, random
4 from twisted.internet import utils, protocol, defer, reactor, task
5 from twisted.spread import pb
6 from twisted.cred import credentials
7 from twisted.python import log
8 from twisted.python.procutils import which
10 from buildbot.sourcestamp import SourceStamp
11 from buildbot.scripts import runner
12 from buildbot.util import now
13 from buildbot.status import builder
15 class SourceStampExtractor:
17 def __init__(self, treetop, branch):
18 self.treetop = treetop
19 self.branch = branch
20 self.exe = which(self.vcexe)[0]
22 def dovc(self, cmd):
23 """This accepts the arguments of a command, without the actual
24 command itself."""
25 env = os.environ.copy()
26 env['LC_ALL'] = "C"
27 d = utils.getProcessOutputAndValue(self.exe, cmd, env=env,
28 path=self.treetop)
29 d.addCallback(self._didvc, cmd)
30 return d
31 def _didvc(self, res, cmd):
32 (stdout, stderr, code) = res
33 # 'bzr diff' sets rc=1 if there were any differences. tla, baz, and
34 # cvs do something similar, so don't bother requring rc=0.
35 return stdout
37 def get(self):
38 """Return a Deferred that fires with a SourceStamp instance."""
39 d = self.getBaseRevision()
40 d.addCallback(self.getPatch)
41 d.addCallback(self.done)
42 return d
43 def readPatch(self, res, patchlevel):
44 self.patch = (patchlevel, res)
45 def done(self, res):
46 # TODO: figure out the branch too
47 ss = SourceStamp(self.branch, self.baserev, self.patch)
48 return ss
50 class CVSExtractor(SourceStampExtractor):
51 patchlevel = 0
52 vcexe = "cvs"
53 def getBaseRevision(self):
54 # this depends upon our local clock and the repository's clock being
55 # reasonably synchronized with each other. We express everything in
56 # UTC because the '%z' format specifier for strftime doesn't always
57 # work.
58 self.baserev = time.strftime("%Y-%m-%d %H:%M:%S +0000",
59 time.gmtime(now()))
60 return defer.succeed(None)
62 def getPatch(self, res):
63 # the -q tells CVS to not announce each directory as it works
64 if self.branch is not None:
65 # 'cvs diff' won't take both -r and -D at the same time (it
66 # ignores the -r). As best I can tell, there is no way to make
67 # cvs give you a diff relative to a timestamp on the non-trunk
68 # branch. A bare 'cvs diff' will tell you about the changes
69 # relative to your checked-out versions, but I know of no way to
70 # find out what those checked-out versions are.
71 raise RuntimeError("Sorry, CVS 'try' builds don't work with "
72 "branches")
73 args = ['-q', 'diff', '-u', '-D', self.baserev]
74 d = self.dovc(args)
75 d.addCallback(self.readPatch, self.patchlevel)
76 return d
78 class SVNExtractor(SourceStampExtractor):
79 patchlevel = 0
80 vcexe = "svn"
82 def getBaseRevision(self):
83 d = self.dovc(["status", "-u"])
84 d.addCallback(self.parseStatus)
85 return d
86 def parseStatus(self, res):
87 # svn shows the base revision for each file that has been modified or
88 # which needs an update. You can update each file to a different
89 # version, so each file is displayed with its individual base
90 # revision. It also shows the repository-wide latest revision number
91 # on the last line ("Status against revision: \d+").
93 # for our purposes, we use the latest revision number as the "base"
94 # revision, and get a diff against that. This means we will get
95 # reverse-diffs for local files that need updating, but the resulting
96 # tree will still be correct. The only weirdness is that the baserev
97 # that we emit may be different than the version of the tree that we
98 # first checked out.
100 # to do this differently would probably involve scanning the revision
101 # numbers to find the max (or perhaps the min) revision, and then
102 # using that as a base.
104 for line in res.split("\n"):
105 m = re.search(r'^Status against revision:\s+(\d+)', line)
106 if m:
107 self.baserev = int(m.group(1))
108 return
109 raise IndexError("Could not find 'Status against revision' in "
110 "SVN output: %s" % res)
111 def getPatch(self, res):
112 d = self.dovc(["diff", "-r%d" % self.baserev])
113 d.addCallback(self.readPatch, self.patchlevel)
114 return d
116 class BazExtractor(SourceStampExtractor):
117 patchlevel = 1
118 vcexe = "baz"
119 def getBaseRevision(self):
120 d = self.dovc(["tree-id"])
121 d.addCallback(self.parseStatus)
122 return d
123 def parseStatus(self, res):
124 tid = res.strip()
125 slash = tid.index("/")
126 dd = tid.rindex("--")
127 self.branch = tid[slash+1:dd]
128 self.baserev = tid[dd+2:]
129 def getPatch(self, res):
130 d = self.dovc(["diff"])
131 d.addCallback(self.readPatch, self.patchlevel)
132 return d
134 class TlaExtractor(SourceStampExtractor):
135 patchlevel = 1
136 vcexe = "tla"
137 def getBaseRevision(self):
138 # 'tla logs --full' gives us ARCHIVE/BRANCH--REVISION
139 # 'tla logs' gives us REVISION
140 d = self.dovc(["logs", "--full", "--reverse"])
141 d.addCallback(self.parseStatus)
142 return d
143 def parseStatus(self, res):
144 tid = res.split("\n")[0].strip()
145 slash = tid.index("/")
146 dd = tid.rindex("--")
147 self.branch = tid[slash+1:dd]
148 self.baserev = tid[dd+2:]
150 def getPatch(self, res):
151 d = self.dovc(["changes", "--diffs"])
152 d.addCallback(self.readPatch, self.patchlevel)
153 return d
155 class BzrExtractor(SourceStampExtractor):
156 patchlevel = 0
157 vcexe = "bzr"
158 def getBaseRevision(self):
159 d = self.dovc(["version-info"])
160 d.addCallback(self.get_revision_number)
161 return d
162 def get_revision_number(self, out):
163 for line in out.split("\n"):
164 colon = line.find(":")
165 if colon != -1:
166 key, value = line[:colon], line[colon+2:]
167 if key == "revno":
168 self.baserev = int(value)
169 return
170 raise ValueError("unable to find revno: in bzr output: '%s'" % out)
172 def getPatch(self, res):
173 d = self.dovc(["diff"])
174 d.addCallback(self.readPatch, self.patchlevel)
175 return d
177 class MercurialExtractor(SourceStampExtractor):
178 patchlevel = 1
179 vcexe = "hg"
180 def getBaseRevision(self):
181 d = self.dovc(["identify"])
182 d.addCallback(self.parseStatus)
183 return d
184 def parseStatus(self, output):
185 m = re.search(r'^(\w+)', output)
186 self.baserev = m.group(0)
187 def getPatch(self, res):
188 d = self.dovc(["diff"])
189 d.addCallback(self.readPatch, self.patchlevel)
190 return d
192 class DarcsExtractor(SourceStampExtractor):
193 patchlevel = 1
194 vcexe = "darcs"
195 def getBaseRevision(self):
196 d = self.dovc(["changes", "--context"])
197 d.addCallback(self.parseStatus)
198 return d
199 def parseStatus(self, res):
200 self.baserev = res # the whole context file
201 def getPatch(self, res):
202 d = self.dovc(["diff", "-u"])
203 d.addCallback(self.readPatch, self.patchlevel)
204 return d
206 class GitExtractor(SourceStampExtractor):
207 patchlevel = 1
208 vcexe = "git"
210 def getBaseRevision(self):
211 d = self.dovc(["branch", "--no-color", "-v", "--no-abbrev"])
212 d.addCallback(self.parseStatus)
213 return d
215 def readConfig(self):
216 d = self.dovc(["config", "-l"])
217 d.addCallback(self.parseConfig)
218 return d
220 def parseConfig(self, res):
221 git_config = {}
222 for l in res.split("\n"):
223 if l.strip():
224 parts = l.strip().split("=", 2)
225 git_config[parts[0]] = parts[1]
227 # If we're tracking a remote, consider that the base.
228 remote = git_config.get("branch." + self.branch + ".remote")
229 ref = git_config.get("branch." + self.branch + ".merge")
230 if remote and ref:
231 remote_branch = ref.split("/", 3)[-1]
232 d = self.dovc(["rev-parse", remote + "/" + remote_branch])
233 d.addCallback(self.override_baserev)
234 return d
236 def override_baserev(self, res):
237 self.baserev = res.strip()
239 def parseStatus(self, res):
240 # The current branch is marked by '*' at the start of the
241 # line, followed by the branch name and the SHA1.
243 # Branch names may contain pretty much anything but whitespace.
244 m = re.search(r'^\* (\S+)\s+([0-9a-f]{40})', res, re.MULTILINE)
245 if m:
246 self.baserev = m.group(2)
247 # If a branch is specified, parse out the rev it points to
248 # and extract the local name (assuming it has a slash).
249 # This may break if someone specifies the name of a local
250 # branch that has a slash in it and has no corresponding
251 # remote branch (or something similarly contrived).
252 if self.branch:
253 d = self.dovc(["rev-parse", self.branch])
254 if '/' in self.branch:
255 self.branch = self.branch.split('/', 1)[1]
256 d.addCallback(self.override_baserev)
257 return d
258 else:
259 self.branch = m.group(1)
260 return self.readConfig()
261 raise IndexError("Could not find current GIT branch: %s" % res)
263 def getPatch(self, res):
264 d = self.dovc(["diff", self.baserev])
265 d.addCallback(self.readPatch, self.patchlevel)
266 return d
268 def getSourceStamp(vctype, treetop, branch=None):
269 if vctype == "cvs":
270 e = CVSExtractor(treetop, branch)
271 elif vctype == "svn":
272 e = SVNExtractor(treetop, branch)
273 elif vctype == "baz":
274 e = BazExtractor(treetop, branch)
275 elif vctype == "bzr":
276 e = BzrExtractor(treetop, branch)
277 elif vctype == "tla":
278 e = TlaExtractor(treetop, branch)
279 elif vctype == "hg":
280 e = MercurialExtractor(treetop, branch)
281 elif vctype == "darcs":
282 e = DarcsExtractor(treetop, branch)
283 elif vctype == "git":
284 e = GitExtractor(treetop, branch)
285 else:
286 raise KeyError("unknown vctype '%s'" % vctype)
287 return e.get()
290 def ns(s):
291 return "%d:%s," % (len(s), s)
293 def createJobfile(bsid, branch, baserev, patchlevel, diff, builderNames):
294 job = ""
295 job += ns("1")
296 job += ns(bsid)
297 job += ns(branch)
298 job += ns(str(baserev))
299 job += ns("%d" % patchlevel)
300 job += ns(diff)
301 for bn in builderNames:
302 job += ns(bn)
303 return job
305 def getTopdir(topfile, start=None):
306 """walk upwards from the current directory until we find this topfile"""
307 if not start:
308 start = os.getcwd()
309 here = start
310 toomany = 20
311 while toomany > 0:
312 if os.path.exists(os.path.join(here, topfile)):
313 return here
314 next = os.path.dirname(here)
315 if next == here:
316 break # we've hit the root
317 here = next
318 toomany -= 1
319 raise ValueError("Unable to find topfile '%s' anywhere from %s upwards"
320 % (topfile, start))
322 class RemoteTryPP(protocol.ProcessProtocol):
323 def __init__(self, job):
324 self.job = job
325 self.d = defer.Deferred()
326 def connectionMade(self):
327 self.transport.write(self.job)
328 self.transport.closeStdin()
329 def outReceived(self, data):
330 sys.stdout.write(data)
331 def errReceived(self, data):
332 sys.stderr.write(data)
333 def processEnded(self, status_object):
334 sig = status_object.value.signal
335 rc = status_object.value.exitCode
336 if sig != None or rc != 0:
337 self.d.errback(RuntimeError("remote 'buildbot tryserver' failed"
338 ": sig=%s, rc=%s" % (sig, rc)))
339 return
340 self.d.callback((sig, rc))
342 class BuildSetStatusGrabber:
343 retryCount = 5 # how many times to we try to grab the BuildSetStatus?
344 retryDelay = 3 # seconds to wait between attempts
346 def __init__(self, status, bsid):
347 self.status = status
348 self.bsid = bsid
350 def grab(self):
351 # return a Deferred that either fires with the BuildSetStatus
352 # reference or errbacks because we were unable to grab it
353 self.d = defer.Deferred()
354 # wait a second before querying to give the master's maildir watcher
355 # a chance to see the job
356 reactor.callLater(1, self.go)
357 return self.d
359 def go(self, dummy=None):
360 if self.retryCount == 0:
361 raise RuntimeError("couldn't find matching buildset")
362 self.retryCount -= 1
363 d = self.status.callRemote("getBuildSets")
364 d.addCallback(self._gotSets)
366 def _gotSets(self, buildsets):
367 for bs,bsid in buildsets:
368 if bsid == self.bsid:
369 # got it
370 self.d.callback(bs)
371 return
372 d = defer.Deferred()
373 d.addCallback(self.go)
374 reactor.callLater(self.retryDelay, d.callback, None)
377 class Try(pb.Referenceable):
378 buildsetStatus = None
379 quiet = False
381 def __init__(self, config):
382 self.config = config
383 self.opts = runner.loadOptions()
384 self.connect = self.getopt('connect', 'try_connect')
385 assert self.connect, "you must specify a connect style: ssh or pb"
386 self.builderNames = self.getopt('builders', 'try_builders')
388 def getopt(self, config_name, options_name, default=None):
389 value = self.config.get(config_name)
390 if value is None or value == []:
391 value = self.opts.get(options_name)
392 if value is None or value == []:
393 value = default
394 return value
396 def createJob(self):
397 # returns a Deferred which fires when the job parameters have been
398 # created
399 opts = self.opts
400 # generate a random (unique) string. It would make sense to add a
401 # hostname and process ID here, but a) I suspect that would cause
402 # windows portability problems, and b) really this is good enough
403 self.bsid = "%d-%s" % (time.time(), random.randint(0, 1000000))
405 # common options
406 branch = self.getopt("branch", "try_branch")
408 difffile = self.config.get("diff")
409 if difffile:
410 baserev = self.config.get("baserev")
411 if difffile == "-":
412 diff = sys.stdin.read()
413 else:
414 diff = open(difffile,"r").read()
415 patch = (self.config['patchlevel'], diff)
416 ss = SourceStamp(branch, baserev, patch)
417 d = defer.succeed(ss)
418 else:
419 vc = self.getopt("vc", "try_vc")
420 if vc in ("cvs", "svn"):
421 # we need to find the tree-top
422 topdir = self.getopt("try_topdir", "try_topdir")
423 if topdir:
424 treedir = os.path.expanduser(topdir)
425 else:
426 topfile = self.getopt("try-topfile", "try_topfile")
427 treedir = getTopdir(topfile)
428 else:
429 treedir = os.getcwd()
430 d = getSourceStamp(vc, treedir, branch)
431 d.addCallback(self._createJob_1)
432 return d
434 def _createJob_1(self, ss):
435 self.sourcestamp = ss
436 if self.connect == "ssh":
437 patchlevel, diff = ss.patch
438 revspec = ss.revision
439 if revspec is None:
440 revspec = ""
441 self.jobfile = createJobfile(self.bsid,
442 ss.branch or "", revspec,
443 patchlevel, diff,
444 self.builderNames)
446 def fakeDeliverJob(self):
447 # Display the job to be delivered, but don't perform delivery.
448 ss = self.sourcestamp
449 print ("Job:\n\tBranch: %s\n\tRevision: %s\n\tBuilders: %s\n%s"
450 % (ss.branch,
451 ss.revision,
452 self.builderNames,
453 ss.patch[1]))
454 d = defer.Deferred()
455 d.callback(True)
456 return d
458 def deliverJob(self):
459 # returns a Deferred that fires when the job has been delivered
460 opts = self.opts
462 if self.connect == "ssh":
463 tryhost = self.getopt("tryhost", "try_host")
464 tryuser = self.getopt("username", "try_username")
465 trydir = self.getopt("trydir", "try_dir")
467 argv = ["ssh", "-l", tryuser, tryhost,
468 "buildbot", "tryserver", "--jobdir", trydir]
469 # now run this command and feed the contents of 'job' into stdin
471 pp = RemoteTryPP(self.jobfile)
472 p = reactor.spawnProcess(pp, argv[0], argv, os.environ)
473 d = pp.d
474 return d
475 if self.connect == "pb":
476 user = self.getopt("username", "try_username")
477 passwd = self.getopt("passwd", "try_password")
478 master = self.getopt("master", "try_master")
479 tryhost, tryport = master.split(":")
480 tryport = int(tryport)
481 f = pb.PBClientFactory()
482 d = f.login(credentials.UsernamePassword(user, passwd))
483 reactor.connectTCP(tryhost, tryport, f)
484 d.addCallback(self._deliverJob_pb)
485 return d
486 raise RuntimeError("unknown connecttype '%s', should be 'ssh' or 'pb'"
487 % self.connect)
489 def _deliverJob_pb(self, remote):
490 ss = self.sourcestamp
492 d = remote.callRemote("try",
493 ss.branch,
494 ss.revision,
495 ss.patch,
496 self.builderNames,
497 self.config.get('properties', {}))
498 d.addCallback(self._deliverJob_pb2)
499 return d
500 def _deliverJob_pb2(self, status):
501 self.buildsetStatus = status
502 return status
504 def getStatus(self):
505 # returns a Deferred that fires when the builds have finished, and
506 # may emit status messages while we wait
507 wait = bool(self.getopt("wait", "try_wait", False))
508 if not wait:
509 # TODO: emit the URL where they can follow the builds. This
510 # requires contacting the Status server over PB and doing
511 # getURLForThing() on the BuildSetStatus. To get URLs for
512 # individual builds would require we wait for the builds to
513 # start.
514 print "not waiting for builds to finish"
515 return
516 d = self.running = defer.Deferred()
517 if self.buildsetStatus:
518 self._getStatus_1()
519 # contact the status port
520 # we're probably using the ssh style
521 master = self.getopt("master", "masterstatus")
522 host, port = master.split(":")
523 port = int(port)
524 self.announce("contacting the status port at %s:%d" % (host, port))
525 f = pb.PBClientFactory()
526 creds = credentials.UsernamePassword("statusClient", "clientpw")
527 d = f.login(creds)
528 reactor.connectTCP(host, port, f)
529 d.addCallback(self._getStatus_ssh_1)
530 return self.running
532 def _getStatus_ssh_1(self, remote):
533 # find a remotereference to the corresponding BuildSetStatus object
534 self.announce("waiting for job to be accepted")
535 g = BuildSetStatusGrabber(remote, self.bsid)
536 d = g.grab()
537 d.addCallback(self._getStatus_1)
538 return d
540 def _getStatus_1(self, res=None):
541 if res:
542 self.buildsetStatus = res
543 # gather the set of BuildRequests
544 d = self.buildsetStatus.callRemote("getBuildRequests")
545 d.addCallback(self._getStatus_2)
547 def _getStatus_2(self, brs):
548 self.builderNames = []
549 self.buildRequests = {}
551 # self.builds holds the current BuildStatus object for each one
552 self.builds = {}
554 # self.outstanding holds the list of builderNames which haven't
555 # finished yet
556 self.outstanding = []
558 # self.results holds the list of build results. It holds a tuple of
559 # (result, text)
560 self.results = {}
562 # self.currentStep holds the name of the Step that each build is
563 # currently running
564 self.currentStep = {}
566 # self.ETA holds the expected finishing time (absolute time since
567 # epoch)
568 self.ETA = {}
570 for n,br in brs:
571 self.builderNames.append(n)
572 self.buildRequests[n] = br
573 self.builds[n] = None
574 self.outstanding.append(n)
575 self.results[n] = [None,None]
576 self.currentStep[n] = None
577 self.ETA[n] = None
578 # get new Builds for this buildrequest. We follow each one until
579 # it finishes or is interrupted.
580 br.callRemote("subscribe", self)
582 # now that those queries are in transit, we can start the
583 # display-status-every-30-seconds loop
584 self.printloop = task.LoopingCall(self.printStatus)
585 self.printloop.start(3, now=False)
588 # these methods are invoked by the status objects we've subscribed to
590 def remote_newbuild(self, bs, builderName):
591 if self.builds[builderName]:
592 self.builds[builderName].callRemote("unsubscribe", self)
593 self.builds[builderName] = bs
594 bs.callRemote("subscribe", self, 20)
595 d = bs.callRemote("waitUntilFinished")
596 d.addCallback(self._build_finished, builderName)
598 def remote_stepStarted(self, buildername, build, stepname, step):
599 self.currentStep[buildername] = stepname
601 def remote_stepFinished(self, buildername, build, stepname, step, results):
602 pass
604 def remote_buildETAUpdate(self, buildername, build, eta):
605 self.ETA[buildername] = now() + eta
607 def _build_finished(self, bs, builderName):
608 # we need to collect status from the newly-finished build. We don't
609 # remove the build from self.outstanding until we've collected
610 # everything we want.
611 self.builds[builderName] = None
612 self.ETA[builderName] = None
613 self.currentStep[builderName] = "finished"
614 d = bs.callRemote("getResults")
615 d.addCallback(self._build_finished_2, bs, builderName)
616 return d
617 def _build_finished_2(self, results, bs, builderName):
618 self.results[builderName][0] = results
619 d = bs.callRemote("getText")
620 d.addCallback(self._build_finished_3, builderName)
621 return d
622 def _build_finished_3(self, text, builderName):
623 self.results[builderName][1] = text
625 self.outstanding.remove(builderName)
626 if not self.outstanding:
627 # all done
628 return self.statusDone()
630 def printStatus(self):
631 names = self.buildRequests.keys()
632 names.sort()
633 for n in names:
634 if n not in self.outstanding:
635 # the build is finished, and we have results
636 code,text = self.results[n]
637 t = builder.Results[code]
638 if text:
639 t += " (%s)" % " ".join(text)
640 elif self.builds[n]:
641 t = self.currentStep[n] or "building"
642 if self.ETA[n]:
643 t += " [ETA %ds]" % (self.ETA[n] - now())
644 else:
645 t = "no build"
646 self.announce("%s: %s" % (n, t))
647 self.announce("")
649 def statusDone(self):
650 self.printloop.stop()
651 print "All Builds Complete"
652 # TODO: include a URL for all failing builds
653 names = self.buildRequests.keys()
654 names.sort()
655 happy = True
656 for n in names:
657 code,text = self.results[n]
658 t = "%s: %s" % (n, builder.Results[code])
659 if text:
660 t += " (%s)" % " ".join(text)
661 print t
662 if self.results[n] != builder.SUCCESS:
663 happy = False
665 if happy:
666 self.exitcode = 0
667 else:
668 self.exitcode = 1
669 self.running.callback(self.exitcode)
671 def announce(self, message):
672 if not self.quiet:
673 print message
675 def run(self):
676 # we can't do spawnProcess until we're inside reactor.run(), so get
677 # funky
678 print "using '%s' connect method" % self.connect
679 self.exitcode = 0
680 d = defer.Deferred()
681 d.addCallback(lambda res: self.createJob())
682 d.addCallback(lambda res: self.announce("job created"))
683 deliver = self.deliverJob
684 if bool(self.config.get("dryrun")):
685 deliver = self.fakeDeliverJob
686 d.addCallback(lambda res: deliver())
687 d.addCallback(lambda res: self.announce("job has been delivered"))
688 d.addCallback(lambda res: self.getStatus())
689 d.addErrback(log.err)
690 d.addCallback(self.cleanup)
691 d.addCallback(lambda res: reactor.stop())
693 reactor.callLater(0, d.callback, None)
694 reactor.run()
695 sys.exit(self.exitcode)
697 def logErr(self, why):
698 log.err(why)
699 print "error during 'try' processing"
700 print why
702 def cleanup(self, res=None):
703 if self.buildsetStatus:
704 self.buildsetStatus.broker.transport.loseConnection()