buildbot/test/test_web.py

   1 # -*- test-case-name: buildbot.test.test_web -*-
   2
   3 import os, time, shutil
   4 from HTMLParser import HTMLParser
   5 from twisted.python import components
   6
   7 from twisted.trial import unittest
   8 from buildbot.test.runutils import RunMixin
   9
  10 from twisted.internet import reactor, defer, protocol
  11 from twisted.internet.interfaces import IReactorUNIX
  12 from twisted.web import client
  13
  14 from buildbot import master, interfaces, sourcestamp
  15 from buildbot.status import html, builder
  16 from buildbot.status.web import waterfall
  17 from buildbot.changes.changes import Change
  18 from buildbot.process import base
  19 from buildbot.process.buildstep import BuildStep
  20 from buildbot.test.runutils import setupBuildStepStatus
  21
  22 class ConfiguredMaster(master.BuildMaster):
  23     """This BuildMaster variant has a static config file, provided as a
  24     string when it is created."""
  25
  26     def __init__(self, basedir, config):
  27         self.config = config
  28         master.BuildMaster.__init__(self, basedir)
  29
  30     def loadTheConfigFile(self):
  31         self.loadConfig(self.config)
  32
  33 components.registerAdapter(master.Control, ConfiguredMaster,
  34                            interfaces.IControl)
  35
  36
  37 base_config = """
  38 from buildbot.changes.pb import PBChangeSource
  39 from buildbot.status import html
  40 from buildbot.buildslave import BuildSlave
  41 from buildbot.scheduler import Scheduler
  42 from buildbot.process.factory import BuildFactory
  43
  44 BuildmasterConfig = c = {
  45     'change_source': PBChangeSource(),
  46     'slaves': [BuildSlave('bot1name', 'bot1passwd')],
  47     'schedulers': [Scheduler('name', None, 60, ['builder1'])],
  48     'builders': [{'name': 'builder1', 'slavename': 'bot1name',
  49                   'builddir': 'builder1', 'factory': BuildFactory()}],
  50     'slavePortnum': 0,
  51     }
  52 """
  53
  54
  55
  56 class DistribUNIX:
  57     def __init__(self, unixpath):
  58         from twisted.web import server, resource, distrib
  59         root = resource.Resource()
  60         self.r = r = distrib.ResourceSubscription("unix", unixpath)
  61         root.putChild('remote', r)
  62         self.p = p = reactor.listenTCP(0, server.Site(root))
  63         self.portnum = p.getHost().port
  64     def shutdown(self):
  65         d = defer.maybeDeferred(self.p.stopListening)
  66         return d
  67
  68 class DistribTCP:
  69     def __init__(self, port):
  70         from twisted.web import server, resource, distrib
  71         root = resource.Resource()
  72         self.r = r = distrib.ResourceSubscription("localhost", port)
  73         root.putChild('remote', r)
  74         self.p = p = reactor.listenTCP(0, server.Site(root))
  75         self.portnum = p.getHost().port
  76     def shutdown(self):
  77         d = defer.maybeDeferred(self.p.stopListening)
  78         d.addCallback(self._shutdown_1)
  79         return d
  80     def _shutdown_1(self, res):
  81         return self.r.publisher.broker.transport.loseConnection()
  82
  83 class SlowReader(protocol.Protocol):
  84     didPause = False
  85     count = 0
  86     data = ""
  87     def __init__(self, req):
  88         self.req = req
  89         self.d = defer.Deferred()
  90     def connectionMade(self):
  91         self.transport.write(self.req)
  92     def dataReceived(self, data):
  93         self.data += data
  94         self.count += len(data)
  95         if not self.didPause and self.count > 10*1000:
  96             self.didPause = True
  97             self.transport.pauseProducing()
  98             reactor.callLater(2, self.resume)
  99     def resume(self):
 100         self.transport.resumeProducing()
 101     def connectionLost(self, why):
 102         self.d.callback(None)
 103
 104 class CFactory(protocol.ClientFactory):
 105     def __init__(self, p):
 106         self.p = p
 107     def buildProtocol(self, addr):
 108         self.p.factory = self
 109         return self.p
 110
 111 def stopHTTPLog():
 112     # grr.
 113     from twisted.web import http
 114     http._logDateTimeStop()
 115
 116 class BaseWeb:
 117     master = None
 118
 119     def failUnlessIn(self, substr, string, note=None):
 120         self.failUnless(string.find(substr) != -1, note)
 121
 122     def tearDown(self):
 123         stopHTTPLog()
 124         if self.master:
 125             d = self.master.stopService()
 126             return d
 127
 128     def find_webstatus(self, master):
 129         for child in list(master):
 130             if isinstance(child, html.WebStatus):
 131                 return child
 132
 133     def find_waterfall(self, master):
 134         for child in list(master):
 135             if isinstance(child, html.Waterfall):
 136                 return child
 137
 138 class Ports(BaseWeb, unittest.TestCase):
 139
 140     def test_webPortnum(self):
 141         # run a regular web server on a TCP socket
 142         config = base_config + "c['status'] = [html.WebStatus(http_port=0)]\n"
 143         os.mkdir("test_web1")
 144         self.master = m = ConfiguredMaster("test_web1", config)
 145         m.startService()
 146         # hack to find out what randomly-assigned port it is listening on
 147         port = self.find_webstatus(m).getPortnum()
 148
 149         d = client.getPage("http://localhost:%d/waterfall" % port)
 150         def _check(page):
 151             #print page
 152             self.failUnless(page)
 153         d.addCallback(_check)
 154         return d
 155     test_webPortnum.timeout = 10
 156
 157     def test_webPathname(self):
 158         # running a t.web.distrib server over a UNIX socket
 159         if not IReactorUNIX.providedBy(reactor):
 160             raise unittest.SkipTest("UNIX sockets not supported here")
 161         config = (base_config +
 162                   "c['status'] = [html.WebStatus(distrib_port='.web-pb')]\n")
 163         os.mkdir("test_web2")
 164         self.master = m = ConfiguredMaster("test_web2", config)
 165         m.startService()
 166
 167         p = DistribUNIX("test_web2/.web-pb")
 168
 169         d = client.getPage("http://localhost:%d/remote/waterfall" % p.portnum)
 170         def _check(page):
 171             self.failUnless(page)
 172         d.addCallback(_check)
 173         def _done(res):
 174             d1 = p.shutdown()
 175             d1.addCallback(lambda x: res)
 176             return d1
 177         d.addBoth(_done)
 178         return d
 179     test_webPathname.timeout = 10
 180
 181
 182     def test_webPathname_port(self):
 183         # running a t.web.distrib server over TCP
 184         config = (base_config +
 185                   "c['status'] = [html.WebStatus(distrib_port=0)]\n")
 186         os.mkdir("test_web3")
 187         self.master = m = ConfiguredMaster("test_web3", config)
 188         m.startService()
 189         dport = self.find_webstatus(m).getPortnum()
 190
 191         p = DistribTCP(dport)
 192
 193         d = client.getPage("http://localhost:%d/remote/waterfall" % p.portnum)
 194         def _check(page):
 195             self.failUnlessIn("BuildBot", page)
 196         d.addCallback(_check)
 197         def _done(res):
 198             d1 = p.shutdown()
 199             d1.addCallback(lambda x: res)
 200             return d1
 201         d.addBoth(_done)
 202         return d
 203     test_webPathname_port.timeout = 10
 204
 205
 206 class Waterfall(BaseWeb, unittest.TestCase):
 207     def test_waterfall(self):
 208         os.mkdir("test_web4")
 209         os.mkdir("my-maildir"); os.mkdir("my-maildir/new")
 210         self.robots_txt = os.path.abspath(os.path.join("test_web4",
 211                                                        "robots.txt"))
 212         self.robots_txt_contents = "User-agent: *\nDisallow: /\n"
 213         f = open(self.robots_txt, "w")
 214         f.write(self.robots_txt_contents)
 215         f.close()
 216         # this is the right way to configure the Waterfall status
 217         config1 = base_config + """
 218 from buildbot.changes import mail
 219 c['change_source'] = mail.SyncmailMaildirSource('my-maildir')
 220 c['status'] = [html.Waterfall(http_port=0, robots_txt=%s)]
 221 """ % repr(self.robots_txt)
 222
 223         self.master = m = ConfiguredMaster("test_web4", config1)
 224         m.startService()
 225         port = self.find_waterfall(m).getPortnum()
 226         self.port = port
 227         # insert an event
 228         m.change_svc.addChange(Change("user", ["foo.c"], "comments"))
 229
 230         d = client.getPage("http://localhost:%d/" % port)
 231
 232         def _check1(page):
 233             self.failUnless(page)
 234             self.failUnlessIn("current activity", page)
 235             self.failUnlessIn("<html", page)
 236             TZ = time.tzname[time.localtime()[-1]]
 237             self.failUnlessIn("time (%s)" % TZ, page)
 238
 239             # phase=0 is really for debugging the waterfall layout
 240             return client.getPage("http://localhost:%d/?phase=0" % self.port)
 241         d.addCallback(_check1)
 242
 243         def _check2(page):
 244             self.failUnless(page)
 245             self.failUnlessIn("<html", page)
 246
 247             return client.getPage("http://localhost:%d/changes" % self.port)
 248         d.addCallback(_check2)
 249
 250         def _check3(changes):
 251             self.failUnlessIn("<li>Syncmail mailing list in maildir " +
 252                               "my-maildir</li>", changes)
 253
 254             return client.getPage("http://localhost:%d/robots.txt" % self.port)
 255         d.addCallback(_check3)
 256
 257         def _check4(robotstxt):
 258             self.failUnless(robotstxt == self.robots_txt_contents)
 259         d.addCallback(_check4)
 260
 261         return d
 262
 263     test_waterfall.timeout = 10
 264
 265 class WaterfallSteps(unittest.TestCase):
 266
 267     # failUnlessSubstring copied from twisted-2.1.0, because this helps us
 268     # maintain compatibility with python2.2.
 269     def failUnlessSubstring(self, substring, astring, msg=None):
 270         """a python2.2 friendly test to assert that substring is found in
 271         astring parameters follow the semantics of failUnlessIn
 272         """
 273         if astring.find(substring) == -1:
 274             raise self.failureException(msg or "%r not found in %r"
 275                                         % (substring, astring))
 276         return substring
 277     assertSubstring = failUnlessSubstring
 278
 279     def test_urls(self):
 280         s = setupBuildStepStatus("test_web.test_urls")
 281         s.addURL("coverage", "http://coverage.example.org/target")
 282         s.addURL("icon", "http://coverage.example.org/icon.png")
 283         class FakeRequest:
 284             prepath = []
 285             postpath = []
 286             def childLink(self, name):
 287                 return name
 288         req = FakeRequest()
 289         box = waterfall.IBox(s).getBox(req)
 290         td = box.td()
 291         e1 = '[<a href="http://coverage.example.org/target" class="BuildStep external">coverage</a>]'
 292         self.failUnlessSubstring(e1, td)
 293         e2 = '[<a href="http://coverage.example.org/icon.png" class="BuildStep external">icon</a>]'
 294         self.failUnlessSubstring(e2, td)
 295
 296
 297
 298 geturl_config = """
 299 from buildbot.status import html
 300 from buildbot.changes import mail
 301 from buildbot.process import factory
 302 from buildbot.steps import dummy
 303 from buildbot.scheduler import Scheduler
 304 from buildbot.changes.base import ChangeSource
 305 from buildbot.buildslave import BuildSlave
 306 s = factory.s
 307
 308 class DiscardScheduler(Scheduler):
 309     def addChange(self, change):
 310         pass
 311 class DummyChangeSource(ChangeSource):
 312     pass
 313
 314 BuildmasterConfig = c = {}
 315 c['slaves'] = [BuildSlave('bot1', 'sekrit'), BuildSlave('bot2', 'sekrit')]
 316 c['change_source'] = DummyChangeSource()
 317 c['schedulers'] = [DiscardScheduler('discard', None, 60, ['b1'])]
 318 c['slavePortnum'] = 0
 319 c['status'] = [html.Waterfall(http_port=0)]
 320
 321 f = factory.BuildFactory([s(dummy.RemoteDummy, timeout=1)])
 322
 323 c['builders'] = [
 324     {'name': 'b1', 'slavenames': ['bot1','bot2'],
 325      'builddir': 'b1', 'factory': f},
 326     ]
 327 c['buildbotURL'] = 'http://dummy.example.org:8010/'
 328
 329 """
 330
 331 class GetURL(RunMixin, unittest.TestCase):
 332
 333     def setUp(self):
 334         RunMixin.setUp(self)
 335         self.master.loadConfig(geturl_config)
 336         self.master.startService()
 337         d = self.connectSlave(["b1"])
 338         return d
 339
 340     def tearDown(self):
 341         stopHTTPLog()
 342         return RunMixin.tearDown(self)
 343
 344     def doBuild(self, buildername):
 345         br = base.BuildRequest("forced", sourcestamp.SourceStamp())
 346         d = br.waitUntilFinished()
 347         self.control.getBuilder(buildername).requestBuild(br)
 348         return d
 349
 350     def assertNoURL(self, target):
 351         self.failUnlessIdentical(self.status.getURLForThing(target), None)
 352
 353     def assertURLEqual(self, target, expected):
 354         got = self.status.getURLForThing(target)
 355         full_expected = "http://dummy.example.org:8010/" + expected
 356         self.failUnlessEqual(got, full_expected)
 357
 358     def testMissingBase(self):
 359         noweb_config1 = geturl_config + "del c['buildbotURL']\n"
 360         d = self.master.loadConfig(noweb_config1)
 361         d.addCallback(self._testMissingBase_1)
 362         return d
 363     def _testMissingBase_1(self, res):
 364         s = self.status
 365         self.assertNoURL(s)
 366         builder_s = s.getBuilder("b1")
 367         self.assertNoURL(builder_s)
 368
 369     def testBase(self):
 370         s = self.status
 371         self.assertURLEqual(s, "")
 372         builder_s = s.getBuilder("b1")
 373         self.assertURLEqual(builder_s, "builders/b1")
 374
 375     def testChange(self):
 376         s = self.status
 377         c = Change("user", ["foo.c"], "comments")
 378         self.master.change_svc.addChange(c)
 379         # TODO: something more like s.getChanges(), requires IChange and
 380         # an accessor in IStatus. The HTML page exists already, though
 381         self.assertURLEqual(c, "changes/1")
 382
 383     def testBuild(self):
 384         # first we do some stuff so we'll have things to look at.
 385         s = self.status
 386         d = self.doBuild("b1")
 387         # maybe check IBuildSetStatus here?
 388         d.addCallback(self._testBuild_1)
 389         return d
 390
 391     def _testBuild_1(self, res):
 392         s = self.status
 393         builder_s = s.getBuilder("b1")
 394         build_s = builder_s.getLastFinishedBuild()
 395         self.assertURLEqual(build_s, "builders/b1/builds/0")
 396         # no page for builder.getEvent(-1)
 397         step = build_s.getSteps()[0]
 398         self.assertURLEqual(step, "builders/b1/builds/0/steps/remote%20dummy")
 399         # maybe page for build.getTestResults?
 400         self.assertURLEqual(step.getLogs()[0],
 401                             "builders/b1/builds/0/steps/remote%20dummy/logs/0")
 402
 403
 404
 405 class Logfile(BaseWeb, RunMixin, unittest.TestCase):
 406     def setUp(self):
 407         config = """
 408 from buildbot.status import html
 409 from buildbot.process.factory import BasicBuildFactory
 410 from buildbot.buildslave import BuildSlave
 411 f1 = BasicBuildFactory('cvsroot', 'cvsmodule')
 412 BuildmasterConfig = {
 413     'slaves': [BuildSlave('bot1', 'passwd1')],
 414     'schedulers': [],
 415     'builders': [{'name': 'builder1', 'slavename': 'bot1',
 416                   'builddir':'workdir', 'factory':f1}],
 417     'slavePortnum': 0,
 418     'status': [html.WebStatus(http_port=0)],
 419     }
 420 """
 421         if os.path.exists("test_logfile"):
 422             shutil.rmtree("test_logfile")
 423         os.mkdir("test_logfile")
 424         self.master = m = ConfiguredMaster("test_logfile", config)
 425         m.startService()
 426         # hack to find out what randomly-assigned port it is listening on
 427         port = self.find_webstatus(m).getPortnum()
 428         self.port = port
 429         # insert an event
 430
 431         req = base.BuildRequest("reason", sourcestamp.SourceStamp())
 432         build1 = base.Build([req])
 433         bs = m.status.getBuilder("builder1").newBuild()
 434         bs.setReason("reason")
 435         bs.buildStarted(build1)
 436
 437         step1 = BuildStep(name="setup")
 438         step1.setBuild(build1)
 439         bss = bs.addStepWithName("setup")
 440         step1.setStepStatus(bss)
 441         bss.stepStarted()
 442
 443         log1 = step1.addLog("output")
 444         log1.addStdout("some stdout\n")
 445         log1.finish()
 446
 447         log2 = step1.addHTMLLog("error", "<html>ouch</html>")
 448
 449         log3 = step1.addLog("big")
 450         log3.addStdout("big log\n")
 451         for i in range(1000):
 452             log3.addStdout("a" * 500)
 453             log3.addStderr("b" * 500)
 454         log3.finish()
 455
 456         log4 = step1.addCompleteLog("bigcomplete",
 457                                     "big2 log\n" + "a" * 1*1000*1000)
 458
 459         log5 = step1.addLog("mixed")
 460         log5.addHeader("header content")
 461         log5.addStdout("this is stdout content")
 462         log5.addStderr("errors go here")
 463         log5.addEntry(5, "non-standard content on channel 5")
 464         log5.addStderr(" and some trailing stderr")
 465
 466         d = defer.maybeDeferred(step1.step_status.stepFinished,
 467                                 builder.SUCCESS)
 468         bs.buildFinished()
 469         return d
 470
 471     def getLogPath(self, stepname, logname):
 472         return ("/builders/builder1/builds/0/steps/%s/logs/%s" %
 473                 (stepname, logname))
 474
 475     def getLogURL(self, stepname, logname):
 476         return ("http://localhost:%d" % self.port
 477                 + self.getLogPath(stepname, logname))
 478
 479     def test_logfile1(self):
 480         d = client.getPage("http://localhost:%d/" % self.port)
 481         def _check(page):
 482             self.failUnless(page)
 483         d.addCallback(_check)
 484         return d
 485
 486     def test_logfile2(self):
 487         logurl = self.getLogURL("setup", "output")
 488         d = client.getPage(logurl)
 489         def _check(logbody):
 490             self.failUnless(logbody)
 491         d.addCallback(_check)
 492         return d
 493
 494     def test_logfile3(self):
 495         logurl = self.getLogURL("setup", "output")
 496         d = client.getPage(logurl + "/text")
 497         def _check(logtext):
 498             self.failUnlessEqual(logtext, "some stdout\n")
 499         d.addCallback(_check)
 500         return d
 501
 502     def test_logfile4(self):
 503         logurl = self.getLogURL("setup", "error")
 504         d = client.getPage(logurl)
 505         def _check(logbody):
 506             self.failUnlessEqual(logbody, "<html>ouch</html>")
 507         d.addCallback(_check)
 508         return d
 509
 510     def test_logfile5(self):
 511         # this is log3, which is about 1MB in size, made up of alternating
 512         # stdout/stderr chunks. buildbot-0.6.6, when run against
 513         # twisted-1.3.0, fails to resume sending chunks after the client
 514         # stalls for a few seconds, because of a recursive doWrite() call
 515         # that was fixed in twisted-2.0.0
 516         p = SlowReader("GET %s HTTP/1.0\r\n\r\n"
 517                        % self.getLogPath("setup", "big"))
 518         cf = CFactory(p)
 519         c = reactor.connectTCP("localhost", self.port, cf)
 520         d = p.d
 521         def _check(res):
 522             self.failUnlessIn("big log", p.data)
 523             self.failUnlessIn("a"*100, p.data)
 524             self.failUnless(p.count > 1*1000*1000)
 525         d.addCallback(_check)
 526         return d
 527
 528     def test_logfile6(self):
 529         # this is log4, which is about 1MB in size, one big chunk.
 530         # buildbot-0.6.6 dies as the NetstringReceiver barfs on the
 531         # saved logfile, because it was using one big chunk and exceeding
 532         # NetstringReceiver.MAX_LENGTH
 533         p = SlowReader("GET %s HTTP/1.0\r\n\r\n"
 534                        % self.getLogPath("setup", "bigcomplete"))
 535         cf = CFactory(p)
 536         c = reactor.connectTCP("localhost", self.port, cf)
 537         d = p.d
 538         def _check(res):
 539             self.failUnlessIn("big2 log", p.data)
 540             self.failUnlessIn("a"*100, p.data)
 541             self.failUnless(p.count > 1*1000*1000)
 542         d.addCallback(_check)
 543         return d
 544
 545     def test_logfile7(self):
 546         # this is log5, with mixed content on the tree standard channels
 547         # as well as on channel 5
 548
 549         class SpanParser(HTMLParser):
 550             '''Parser subclass to gather all the log spans from the log page'''
 551             def __init__(self, test):
 552                 self.spans = []
 553                 self.test = test
 554                 self.inSpan = False
 555                 HTMLParser.__init__(self)
 556
 557             def handle_starttag(self, tag, attrs):
 558                 if tag == 'span':
 559                     self.inSpan = True
 560                     cls = attrs[0]
 561                     self.test.failUnless(cls[0] == 'class')
 562                     self.spans.append([cls[1],''])
 563
 564             def handle_data(self, data):
 565                 if self.inSpan:
 566                     self.spans[-1][1] += data
 567
 568             def handle_endtag(self, tag):
 569                 if tag == 'span':
 570                     self.inSpan = False
 571
 572         logurl = self.getLogURL("setup", "mixed")
 573         d = client.getPage(logurl, timeout=2)
 574         def _check(logbody):
 575             try:
 576                 p = SpanParser(self)
 577                 p.feed(logbody)
 578                 p.close
 579             except Exception, e:
 580                 print e
 581             self.failUnlessEqual(len(p.spans), 4)
 582             self.failUnlessEqual(p.spans[0][0], 'header')
 583             self.failUnlessEqual(p.spans[0][1], 'header content')
 584             self.failUnlessEqual(p.spans[1][0], 'stdout')
 585             self.failUnlessEqual(p.spans[1][1], 'this is stdout content')
 586             self.failUnlessEqual(p.spans[2][0], 'stderr')
 587             self.failUnlessEqual(p.spans[2][1], 'errors go here')
 588             self.failUnlessEqual(p.spans[3][0], 'stderr')
 589             self.failUnlessEqual(p.spans[3][1], ' and some trailing stderr')
 590         def _fail(err):
 591             pass
 592         d.addCallback(_check)
 593         d.addErrback(_fail)
 594         return d