buildbot/test/test_web.py

   1 # -*- test-case-name: buildbot.test.test_web -*-
   2
   3 import os, time, shutil
   4 from twisted.python import components
   5
   6 from twisted.trial import unittest
   7 from buildbot.test.runutils import RunMixin
   8
   9 from twisted.internet import reactor, defer, protocol
  10 from twisted.internet.interfaces import IReactorUNIX
  11 from twisted.web import client
  12
  13 from buildbot import master, interfaces, sourcestamp
  14 from buildbot.status import html, builder
  15 from buildbot.status.web import waterfall
  16 from buildbot.changes.changes import Change
  17 from buildbot.process import base
  18 from buildbot.process.buildstep import BuildStep
  19 from buildbot.test.runutils import setupBuildStepStatus
  20
  21 class ConfiguredMaster(master.BuildMaster):
  22     """This BuildMaster variant has a static config file, provided as a
  23     string when it is created."""
  24
  25     def __init__(self, basedir, config):
  26         self.config = config
  27         master.BuildMaster.__init__(self, basedir)
  28
  29     def loadTheConfigFile(self):
  30         self.loadConfig(self.config)
  31
  32 components.registerAdapter(master.Control, ConfiguredMaster,
  33                            interfaces.IControl)
  34
  35
  36 base_config = """
  37 from buildbot.changes.pb import PBChangeSource
  38 from buildbot.status import html
  39 from buildbot.buildslave import BuildSlave
  40 from buildbot.scheduler import Scheduler
  41 from buildbot.process.factory import BuildFactory
  42
  43 BuildmasterConfig = c = {
  44     'change_source': PBChangeSource(),
  45     'slaves': [BuildSlave('bot1name', 'bot1passwd')],
  46     'schedulers': [Scheduler('name', None, 60, ['builder1'])],
  47     'builders': [{'name': 'builder1', 'slavename': 'bot1name',
  48                   'builddir': 'builder1', 'factory': BuildFactory()}],
  49     'slavePortnum': 0,
  50     }
  51 """
  52
  53
  54
  55 class DistribUNIX:
  56     def __init__(self, unixpath):
  57         from twisted.web import server, resource, distrib
  58         root = resource.Resource()
  59         self.r = r = distrib.ResourceSubscription("unix", unixpath)
  60         root.putChild('remote', r)
  61         self.p = p = reactor.listenTCP(0, server.Site(root))
  62         self.portnum = p.getHost().port
  63     def shutdown(self):
  64         d = defer.maybeDeferred(self.p.stopListening)
  65         return d
  66
  67 class DistribTCP:
  68     def __init__(self, port):
  69         from twisted.web import server, resource, distrib
  70         root = resource.Resource()
  71         self.r = r = distrib.ResourceSubscription("localhost", port)
  72         root.putChild('remote', r)
  73         self.p = p = reactor.listenTCP(0, server.Site(root))
  74         self.portnum = p.getHost().port
  75     def shutdown(self):
  76         d = defer.maybeDeferred(self.p.stopListening)
  77         d.addCallback(self._shutdown_1)
  78         return d
  79     def _shutdown_1(self, res):
  80         return self.r.publisher.broker.transport.loseConnection()
  81
  82 class SlowReader(protocol.Protocol):
  83     didPause = False
  84     count = 0
  85     data = ""
  86     def __init__(self, req):
  87         self.req = req
  88         self.d = defer.Deferred()
  89     def connectionMade(self):
  90         self.transport.write(self.req)
  91     def dataReceived(self, data):
  92         self.data += data
  93         self.count += len(data)
  94         if not self.didPause and self.count > 10*1000:
  95             self.didPause = True
  96             self.transport.pauseProducing()
  97             reactor.callLater(2, self.resume)
  98     def resume(self):
  99         self.transport.resumeProducing()
 100     def connectionLost(self, why):
 101         self.d.callback(None)
 102
 103 class CFactory(protocol.ClientFactory):
 104     def __init__(self, p):
 105         self.p = p
 106     def buildProtocol(self, addr):
 107         self.p.factory = self
 108         return self.p
 109
 110 def stopHTTPLog():
 111     # grr.
 112     from twisted.web import http
 113     http._logDateTimeStop()
 114
 115 class BaseWeb:
 116     master = None
 117
 118     def failUnlessIn(self, substr, string, note=None):
 119         self.failUnless(string.find(substr) != -1, note)
 120
 121     def tearDown(self):
 122         stopHTTPLog()
 123         if self.master:
 124             d = self.master.stopService()
 125             return d
 126
 127     def find_webstatus(self, master):
 128         for child in list(master):
 129             if isinstance(child, html.WebStatus):
 130                 return child
 131
 132     def find_waterfall(self, master):
 133         for child in list(master):
 134             if isinstance(child, html.Waterfall):
 135                 return child
 136
 137 class Ports(BaseWeb, unittest.TestCase):
 138
 139     def test_webPortnum(self):
 140         # run a regular web server on a TCP socket
 141         config = base_config + "c['status'] = [html.WebStatus(http_port=0)]\n"
 142         os.mkdir("test_web1")
 143         self.master = m = ConfiguredMaster("test_web1", config)
 144         m.startService()
 145         # hack to find out what randomly-assigned port it is listening on
 146         port = self.find_webstatus(m).getPortnum()
 147
 148         d = client.getPage("http://localhost:%d/waterfall" % port)
 149         def _check(page):
 150             #print page
 151             self.failUnless(page)
 152         d.addCallback(_check)
 153         return d
 154     test_webPortnum.timeout = 10
 155
 156     def test_webPathname(self):
 157         # running a t.web.distrib server over a UNIX socket
 158         if not IReactorUNIX.providedBy(reactor):
 159             raise unittest.SkipTest("UNIX sockets not supported here")
 160         config = (base_config +
 161                   "c['status'] = [html.WebStatus(distrib_port='.web-pb')]\n")
 162         os.mkdir("test_web2")
 163         self.master = m = ConfiguredMaster("test_web2", config)
 164         m.startService()
 165
 166         p = DistribUNIX("test_web2/.web-pb")
 167
 168         d = client.getPage("http://localhost:%d/remote/waterfall" % p.portnum)
 169         def _check(page):
 170             self.failUnless(page)
 171         d.addCallback(_check)
 172         def _done(res):
 173             d1 = p.shutdown()
 174             d1.addCallback(lambda x: res)
 175             return d1
 176         d.addBoth(_done)
 177         return d
 178     test_webPathname.timeout = 10
 179
 180
 181     def test_webPathname_port(self):
 182         # running a t.web.distrib server over TCP
 183         config = (base_config +
 184                   "c['status'] = [html.WebStatus(distrib_port=0)]\n")
 185         os.mkdir("test_web3")
 186         self.master = m = ConfiguredMaster("test_web3", config)
 187         m.startService()
 188         dport = self.find_webstatus(m).getPortnum()
 189
 190         p = DistribTCP(dport)
 191
 192         d = client.getPage("http://localhost:%d/remote/waterfall" % p.portnum)
 193         def _check(page):
 194             self.failUnlessIn("BuildBot", page)
 195         d.addCallback(_check)
 196         def _done(res):
 197             d1 = p.shutdown()
 198             d1.addCallback(lambda x: res)
 199             return d1
 200         d.addBoth(_done)
 201         return d
 202     test_webPathname_port.timeout = 10
 203
 204
 205 class Waterfall(BaseWeb, unittest.TestCase):
 206     def test_waterfall(self):
 207         os.mkdir("test_web4")
 208         os.mkdir("my-maildir"); os.mkdir("my-maildir/new")
 209         self.robots_txt = os.path.abspath(os.path.join("test_web4",
 210                                                        "robots.txt"))
 211         self.robots_txt_contents = "User-agent: *\nDisallow: /\n"
 212         f = open(self.robots_txt, "w")
 213         f.write(self.robots_txt_contents)
 214         f.close()
 215         # this is the right way to configure the Waterfall status
 216         config1 = base_config + """
 217 from buildbot.changes import mail
 218 c['change_source'] = mail.SyncmailMaildirSource('my-maildir')
 219 c['status'] = [html.Waterfall(http_port=0, robots_txt=%s)]
 220 """ % repr(self.robots_txt)
 221
 222         self.master = m = ConfiguredMaster("test_web4", config1)
 223         m.startService()
 224         port = self.find_waterfall(m).getPortnum()
 225         self.port = port
 226         # insert an event
 227         m.change_svc.addChange(Change("user", ["foo.c"], "comments"))
 228
 229         d = client.getPage("http://localhost:%d/" % port)
 230
 231         def _check1(page):
 232             self.failUnless(page)
 233             self.failUnlessIn("current activity", page)
 234             self.failUnlessIn("<html", page)
 235             TZ = time.tzname[time.daylight]
 236             self.failUnlessIn("time (%s)" % TZ, page)
 237
 238             # phase=0 is really for debugging the waterfall layout
 239             return client.getPage("http://localhost:%d/?phase=0" % self.port)
 240         d.addCallback(_check1)
 241
 242         def _check2(page):
 243             self.failUnless(page)
 244             self.failUnlessIn("<html", page)
 245
 246             return client.getPage("http://localhost:%d/changes" % self.port)
 247         d.addCallback(_check2)
 248
 249         def _check3(changes):
 250             self.failUnlessIn("<li>Syncmail mailing list in maildir " +
 251                               "my-maildir</li>", changes)
 252
 253             return client.getPage("http://localhost:%d/robots.txt" % self.port)
 254         d.addCallback(_check3)
 255
 256         def _check4(robotstxt):
 257             self.failUnless(robotstxt == self.robots_txt_contents)
 258         d.addCallback(_check4)
 259
 260         return d
 261
 262     test_waterfall.timeout = 10
 263
 264 class WaterfallSteps(unittest.TestCase):
 265
 266     # failUnlessSubstring copied from twisted-2.1.0, because this helps us
 267     # maintain compatibility with python2.2.
 268     def failUnlessSubstring(self, substring, astring, msg=None):
 269         """a python2.2 friendly test to assert that substring is found in
 270         astring parameters follow the semantics of failUnlessIn
 271         """
 272         if astring.find(substring) == -1:
 273             raise self.failureException(msg or "%r not found in %r"
 274                                         % (substring, astring))
 275         return substring
 276     assertSubstring = failUnlessSubstring
 277
 278     def test_urls(self):
 279         s = setupBuildStepStatus("test_web.test_urls")
 280         s.addURL("coverage", "http://coverage.example.org/target")
 281         s.addURL("icon", "http://coverage.example.org/icon.png")
 282         class FakeRequest:
 283             prepath = []
 284             postpath = []
 285             def childLink(self, name):
 286                 return name
 287         req = FakeRequest()
 288         box = waterfall.IBox(s).getBox(req)
 289         td = box.td()
 290         e1 = '[<a href="http://coverage.example.org/target" class="BuildStep external">coverage</a>]'
 291         self.failUnlessSubstring(e1, td)
 292         e2 = '[<a href="http://coverage.example.org/icon.png" class="BuildStep external">icon</a>]'
 293         self.failUnlessSubstring(e2, td)
 294
 295
 296
 297 geturl_config = """
 298 from buildbot.status import html
 299 from buildbot.changes import mail
 300 from buildbot.process import factory
 301 from buildbot.steps import dummy
 302 from buildbot.scheduler import Scheduler
 303 from buildbot.changes.base import ChangeSource
 304 from buildbot.buildslave import BuildSlave
 305 s = factory.s
 306
 307 class DiscardScheduler(Scheduler):
 308     def addChange(self, change):
 309         pass
 310 class DummyChangeSource(ChangeSource):
 311     pass
 312
 313 BuildmasterConfig = c = {}
 314 c['slaves'] = [BuildSlave('bot1', 'sekrit'), BuildSlave('bot2', 'sekrit')]
 315 c['change_source'] = DummyChangeSource()
 316 c['schedulers'] = [DiscardScheduler('discard', None, 60, ['b1'])]
 317 c['slavePortnum'] = 0
 318 c['status'] = [html.Waterfall(http_port=0)]
 319
 320 f = factory.BuildFactory([s(dummy.RemoteDummy, timeout=1)])
 321
 322 c['builders'] = [
 323     {'name': 'b1', 'slavenames': ['bot1','bot2'],
 324      'builddir': 'b1', 'factory': f},
 325     ]
 326 c['buildbotURL'] = 'http://dummy.example.org:8010/'
 327
 328 """
 329
 330 class GetURL(RunMixin, unittest.TestCase):
 331
 332     def setUp(self):
 333         RunMixin.setUp(self)
 334         self.master.loadConfig(geturl_config)
 335         self.master.startService()
 336         d = self.connectSlave(["b1"])
 337         return d
 338
 339     def tearDown(self):
 340         stopHTTPLog()
 341         return RunMixin.tearDown(self)
 342
 343     def doBuild(self, buildername):
 344         br = base.BuildRequest("forced", sourcestamp.SourceStamp())
 345         d = br.waitUntilFinished()
 346         self.control.getBuilder(buildername).requestBuild(br)
 347         return d
 348
 349     def assertNoURL(self, target):
 350         self.failUnlessIdentical(self.status.getURLForThing(target), None)
 351
 352     def assertURLEqual(self, target, expected):
 353         got = self.status.getURLForThing(target)
 354         full_expected = "http://dummy.example.org:8010/" + expected
 355         self.failUnlessEqual(got, full_expected)
 356
 357     def testMissingBase(self):
 358         noweb_config1 = geturl_config + "del c['buildbotURL']\n"
 359         d = self.master.loadConfig(noweb_config1)
 360         d.addCallback(self._testMissingBase_1)
 361         return d
 362     def _testMissingBase_1(self, res):
 363         s = self.status
 364         self.assertNoURL(s)
 365         builder = s.getBuilder("b1")
 366         self.assertNoURL(builder)
 367
 368     def testBase(self):
 369         s = self.status
 370         self.assertURLEqual(s, "")
 371         builder = s.getBuilder("b1")
 372         self.assertURLEqual(builder, "b1")
 373
 374     def testChange(self):
 375         s = self.status
 376         c = Change("user", ["foo.c"], "comments")
 377         self.master.change_svc.addChange(c)
 378         # TODO: something more like s.getChanges(), requires IChange and
 379         # an accessor in IStatus. The HTML page exists already, though
 380         self.assertURLEqual(c, "changes/1")
 381
 382     def testBuild(self):
 383         # first we do some stuff so we'll have things to look at.
 384         s = self.status
 385         d = self.doBuild("b1")
 386         # maybe check IBuildSetStatus here?
 387         d.addCallback(self._testBuild_1)
 388         return d
 389
 390     def _testBuild_1(self, res):
 391         s = self.status
 392         builder = s.getBuilder("b1")
 393         build = builder.getLastFinishedBuild()
 394         self.assertURLEqual(build, "b1/builds/0")
 395         # no page for builder.getEvent(-1)
 396         step = build.getSteps()[0]
 397         self.assertURLEqual(step, "b1/builds/0/step-remote%20dummy")
 398         # maybe page for build.getTestResults?
 399         self.assertURLEqual(step.getLogs()[0],
 400                             "b1/builds/0/step-remote%20dummy/0")
 401
 402
 403
 404 class Logfile(BaseWeb, RunMixin, unittest.TestCase):
 405     def setUp(self):
 406         config = """
 407 from buildbot.status import html
 408 from buildbot.process.factory import BasicBuildFactory
 409 from buildbot.buildslave import BuildSlave
 410 f1 = BasicBuildFactory('cvsroot', 'cvsmodule')
 411 BuildmasterConfig = {
 412     'slaves': [BuildSlave('bot1', 'passwd1')],
 413     'schedulers': [],
 414     'builders': [{'name': 'builder1', 'slavename': 'bot1',
 415                   'builddir':'workdir', 'factory':f1}],
 416     'slavePortnum': 0,
 417     'status': [html.WebStatus(http_port=0)],
 418     }
 419 """
 420         if os.path.exists("test_logfile"):
 421             shutil.rmtree("test_logfile")
 422         os.mkdir("test_logfile")
 423         self.master = m = ConfiguredMaster("test_logfile", config)
 424         m.startService()
 425         # hack to find out what randomly-assigned port it is listening on
 426         port = self.find_webstatus(m).getPortnum()
 427         self.port = port
 428         # insert an event
 429
 430         req = base.BuildRequest("reason", sourcestamp.SourceStamp())
 431         build1 = base.Build([req])
 432         bs = m.status.getBuilder("builder1").newBuild()
 433         bs.setReason("reason")
 434         bs.buildStarted(build1)
 435
 436         step1 = BuildStep(name="setup")
 437         step1.setBuild(build1)
 438         bss = bs.addStepWithName("setup")
 439         step1.setStepStatus(bss)
 440         bss.stepStarted()
 441
 442         log1 = step1.addLog("output")
 443         log1.addStdout("some stdout\n")
 444         log1.finish()
 445
 446         log2 = step1.addHTMLLog("error", "<html>ouch</html>")
 447
 448         log3 = step1.addLog("big")
 449         log3.addStdout("big log\n")
 450         for i in range(1000):
 451             log3.addStdout("a" * 500)
 452             log3.addStderr("b" * 500)
 453         log3.finish()
 454
 455         log4 = step1.addCompleteLog("bigcomplete",
 456                                     "big2 log\n" + "a" * 1*1000*1000)
 457
 458         step1.step_status.stepFinished(builder.SUCCESS)
 459         bs.buildFinished()
 460
 461     def getLogPath(self, stepname, logname):
 462         return ("/builders/builder1/builds/0/steps/%s/logs/%s" %
 463                 (stepname, logname))
 464
 465     def getLogURL(self, stepname, logname):
 466         return ("http://localhost:%d" % self.port
 467                 + self.getLogPath(stepname, logname))
 468
 469     def test_logfile1(self):
 470         d = client.getPage("http://localhost:%d/" % self.port)
 471         def _check(page):
 472             self.failUnless(page)
 473         d.addCallback(_check)
 474         return d
 475
 476     def test_logfile2(self):
 477         logurl = self.getLogURL("setup", "output")
 478         d = client.getPage(logurl)
 479         def _check(logbody):
 480             self.failUnless(logbody)
 481         d.addCallback(_check)
 482         return d
 483
 484     def test_logfile3(self):
 485         logurl = self.getLogURL("setup", "output")
 486         d = client.getPage(logurl + "/text")
 487         def _check(logtext):
 488             self.failUnlessEqual(logtext, "some stdout\n")
 489         d.addCallback(_check)
 490         return d
 491
 492     def test_logfile4(self):
 493         logurl = self.getLogURL("setup", "error")
 494         d = client.getPage(logurl)
 495         def _check(logbody):
 496             self.failUnlessEqual(logbody, "<html>ouch</html>")
 497         d.addCallback(_check)
 498         return d
 499
 500     def test_logfile5(self):
 501         # this is log3, which is about 1MB in size, made up of alternating
 502         # stdout/stderr chunks. buildbot-0.6.6, when run against
 503         # twisted-1.3.0, fails to resume sending chunks after the client
 504         # stalls for a few seconds, because of a recursive doWrite() call
 505         # that was fixed in twisted-2.0.0
 506         p = SlowReader("GET %s HTTP/1.0\r\n\r\n"
 507                        % self.getLogPath("setup", "big"))
 508         cf = CFactory(p)
 509         c = reactor.connectTCP("localhost", self.port, cf)
 510         d = p.d
 511         def _check(res):
 512             self.failUnlessIn("big log", p.data)
 513             self.failUnlessIn("a"*100, p.data)
 514             self.failUnless(p.count > 1*1000*1000)
 515         d.addCallback(_check)
 516         return d
 517
 518     def test_logfile6(self):
 519         # this is log4, which is about 1MB in size, one big chunk.
 520         # buildbot-0.6.6 dies as the NetstringReceiver barfs on the
 521         # saved logfile, because it was using one big chunk and exceeding
 522         # NetstringReceiver.MAX_LENGTH
 523         p = SlowReader("GET %s HTTP/1.0\r\n\r\n"
 524                        % self.getLogPath("setup", "bigcomplete"))
 525         cf = CFactory(p)
 526         c = reactor.connectTCP("localhost", self.port, cf)
 527         d = p.d
 528         def _check(res):
 529             self.failUnlessIn("big2 log", p.data)
 530             self.failUnlessIn("a"*100, p.data)
 531             self.failUnless(p.count > 1*1000*1000)
 532         d.addCallback(_check)
 533         return d
 534
 535