buildbot/test/test_web.py

   1 # -*- test-case-name: buildbot.test.test_web -*-
   2
   3 import os, time, shutil
   4 from twisted.python import components
   5
   6 from twisted.trial import unittest
   7 from buildbot.test.runutils import RunMixin
   8
   9 from twisted.internet import reactor, defer, protocol
  10 from twisted.internet.interfaces import IReactorUNIX
  11 from twisted.web import client
  12
  13 from buildbot import master, interfaces, sourcestamp
  14 from buildbot.status import html, builder
  15 from buildbot.status.web import waterfall
  16 from buildbot.changes.changes import Change
  17 from buildbot.process import base
  18 from buildbot.process.buildstep import BuildStep
  19 from buildbot.test.runutils import setupBuildStepStatus
  20
  21 class ConfiguredMaster(master.BuildMaster):
  22     """This BuildMaster variant has a static config file, provided as a
  23     string when it is created."""
  24
  25     def __init__(self, basedir, config):
  26         self.config = config
  27         master.BuildMaster.__init__(self, basedir)
  28
  29     def loadTheConfigFile(self):
  30         self.loadConfig(self.config)
  31
  32 components.registerAdapter(master.Control, ConfiguredMaster,
  33                            interfaces.IControl)
  34
  35
  36 base_config = """
  37 from buildbot.status import html
  38 BuildmasterConfig = c = {
  39     'slaves': [],
  40     'schedulers': [],
  41     'builders': [],
  42     'slavePortnum': 0,
  43     }
  44 """
  45
  46
  47
  48 class DistribUNIX:
  49     def __init__(self, unixpath):
  50         from twisted.web import server, resource, distrib
  51         root = resource.Resource()
  52         self.r = r = distrib.ResourceSubscription("unix", unixpath)
  53         root.putChild('remote', r)
  54         self.p = p = reactor.listenTCP(0, server.Site(root))
  55         self.portnum = p.getHost().port
  56     def shutdown(self):
  57         d = defer.maybeDeferred(self.p.stopListening)
  58         return d
  59
  60 class DistribTCP:
  61     def __init__(self, port):
  62         from twisted.web import server, resource, distrib
  63         root = resource.Resource()
  64         self.r = r = distrib.ResourceSubscription("localhost", port)
  65         root.putChild('remote', r)
  66         self.p = p = reactor.listenTCP(0, server.Site(root))
  67         self.portnum = p.getHost().port
  68     def shutdown(self):
  69         d = defer.maybeDeferred(self.p.stopListening)
  70         d.addCallback(self._shutdown_1)
  71         return d
  72     def _shutdown_1(self, res):
  73         return self.r.publisher.broker.transport.loseConnection()
  74
  75 class SlowReader(protocol.Protocol):
  76     didPause = False
  77     count = 0
  78     data = ""
  79     def __init__(self, req):
  80         self.req = req
  81         self.d = defer.Deferred()
  82     def connectionMade(self):
  83         self.transport.write(self.req)
  84     def dataReceived(self, data):
  85         self.data += data
  86         self.count += len(data)
  87         if not self.didPause and self.count > 10*1000:
  88             self.didPause = True
  89             self.transport.pauseProducing()
  90             reactor.callLater(2, self.resume)
  91     def resume(self):
  92         self.transport.resumeProducing()
  93     def connectionLost(self, why):
  94         self.d.callback(None)
  95
  96 class CFactory(protocol.ClientFactory):
  97     def __init__(self, p):
  98         self.p = p
  99     def buildProtocol(self, addr):
 100         self.p.factory = self
 101         return self.p
 102
 103 def stopHTTPLog():
 104     # grr.
 105     from twisted.web import http
 106     http._logDateTimeStop()
 107
 108 class BaseWeb:
 109     master = None
 110
 111     def failUnlessIn(self, substr, string):
 112         self.failUnless(string.find(substr) != -1)
 113
 114     def tearDown(self):
 115         stopHTTPLog()
 116         if self.master:
 117             d = self.master.stopService()
 118             return d
 119
 120     def find_webstatus(self, master):
 121         for child in list(master):
 122             if isinstance(child, html.WebStatus):
 123                 return child
 124
 125     def find_waterfall(self, master):
 126         for child in list(master):
 127             if isinstance(child, html.Waterfall):
 128                 return child
 129
 130 class Ports(BaseWeb, unittest.TestCase):
 131
 132     def test_webPortnum(self):
 133         # run a regular web server on a TCP socket
 134         config = base_config + "c['status'] = [html.WebStatus(http_port=0)]\n"
 135         os.mkdir("test_web1")
 136         self.master = m = ConfiguredMaster("test_web1", config)
 137         m.startService()
 138         # hack to find out what randomly-assigned port it is listening on
 139         port = self.find_webstatus(m).getPortnum()
 140
 141         d = client.getPage("http://localhost:%d/waterfall" % port)
 142         def _check(page):
 143             #print page
 144             self.failUnless(page)
 145         d.addCallback(_check)
 146         return d
 147     test_webPortnum.timeout = 10
 148
 149     def test_webPathname(self):
 150         # running a t.web.distrib server over a UNIX socket
 151         if not IReactorUNIX.providedBy(reactor):
 152             raise unittest.SkipTest("UNIX sockets not supported here")
 153         config = (base_config +
 154                   "c['status'] = [html.WebStatus(distrib_port='.web-pb')]\n")
 155         os.mkdir("test_web2")
 156         self.master = m = ConfiguredMaster("test_web2", config)
 157         m.startService()
 158
 159         p = DistribUNIX("test_web2/.web-pb")
 160
 161         d = client.getPage("http://localhost:%d/remote/waterfall" % p.portnum)
 162         def _check(page):
 163             self.failUnless(page)
 164         d.addCallback(_check)
 165         def _done(res):
 166             d1 = p.shutdown()
 167             d1.addCallback(lambda x: res)
 168             return d1
 169         d.addBoth(_done)
 170         return d
 171     test_webPathname.timeout = 10
 172
 173
 174     def test_webPathname_port(self):
 175         # running a t.web.distrib server over TCP
 176         config = (base_config +
 177                   "c['status'] = [html.WebStatus(distrib_port=0)]\n")
 178         os.mkdir("test_web3")
 179         self.master = m = ConfiguredMaster("test_web3", config)
 180         m.startService()
 181         dport = self.find_webstatus(m).getPortnum()
 182
 183         p = DistribTCP(dport)
 184
 185         d = client.getPage("http://localhost:%d/remote/waterfall" % p.portnum)
 186         def _check(page):
 187             self.failUnlessIn("BuildBot", page)
 188         d.addCallback(_check)
 189         def _done(res):
 190             d1 = p.shutdown()
 191             d1.addCallback(lambda x: res)
 192             return d1
 193         d.addBoth(_done)
 194         return d
 195     test_webPathname_port.timeout = 10
 196
 197
 198 class Waterfall(BaseWeb, unittest.TestCase):
 199     def test_waterfall(self):
 200         os.mkdir("test_web4")
 201         os.mkdir("my-maildir"); os.mkdir("my-maildir/new")
 202         self.robots_txt = os.path.abspath(os.path.join("test_web4",
 203                                                        "robots.txt"))
 204         self.robots_txt_contents = "User-agent: *\nDisallow: /\n"
 205         f = open(self.robots_txt, "w")
 206         f.write(self.robots_txt_contents)
 207         f.close()
 208         # this is the right way to configure the Waterfall status
 209         config1 = base_config + """
 210 from buildbot.changes import mail
 211 c['change_source'] = mail.SyncmailMaildirSource('my-maildir')
 212 c['status'] = [html.Waterfall(http_port=0, robots_txt=%s)]
 213 """ % repr(self.robots_txt)
 214
 215         self.master = m = ConfiguredMaster("test_web4", config1)
 216         m.startService()
 217         port = self.find_waterfall(m).getPortnum()
 218         self.port = port
 219         # insert an event
 220         m.change_svc.addChange(Change("user", ["foo.c"], "comments"))
 221
 222         d = client.getPage("http://localhost:%d/" % port)
 223
 224         def _check1(page):
 225             self.failUnless(page)
 226             self.failUnlessIn("current activity", page)
 227             self.failUnlessIn("<html", page)
 228             TZ = time.tzname[time.daylight]
 229             self.failUnlessIn("time (%s)" % TZ, page)
 230
 231             # phase=0 is really for debugging the waterfall layout
 232             return client.getPage("http://localhost:%d/?phase=0" % self.port)
 233         d.addCallback(_check1)
 234
 235         def _check2(page):
 236             self.failUnless(page)
 237             self.failUnlessIn("<html", page)
 238
 239             return client.getPage("http://localhost:%d/changes" % self.port)
 240         d.addCallback(_check2)
 241
 242         def _check3(changes):
 243             self.failUnlessIn("<li>Syncmail mailing list in maildir " +
 244                               "my-maildir</li>", changes)
 245
 246             return client.getPage("http://localhost:%d/robots.txt" % self.port)
 247         d.addCallback(_check3)
 248
 249         def _check4(robotstxt):
 250             self.failUnless(robotstxt == self.robots_txt_contents)
 251         d.addCallback(_check4)
 252
 253         return d
 254
 255     test_waterfall.timeout = 10
 256
 257 class WaterfallSteps(unittest.TestCase):
 258
 259     # failUnlessSubstring copied from twisted-2.1.0, because this helps us
 260     # maintain compatibility with python2.2.
 261     def failUnlessSubstring(self, substring, astring, msg=None):
 262         """a python2.2 friendly test to assert that substring is found in
 263         astring parameters follow the semantics of failUnlessIn
 264         """
 265         if astring.find(substring) == -1:
 266             raise self.failureException(msg or "%r not found in %r"
 267                                         % (substring, astring))
 268         return substring
 269     assertSubstring = failUnlessSubstring
 270
 271     def test_urls(self):
 272         s = setupBuildStepStatus("test_web.test_urls")
 273         s.addURL("coverage", "http://coverage.example.org/target")
 274         s.addURL("icon", "http://coverage.example.org/icon.png")
 275         box = waterfall.IBox(s).getBox()
 276         td = box.td()
 277         e1 = '[<a href="http://coverage.example.org/target" class="BuildStep external">coverage</a>]'
 278         self.failUnlessSubstring(e1, td)
 279         e2 = '[<a href="http://coverage.example.org/icon.png" class="BuildStep external">icon</a>]'
 280         self.failUnlessSubstring(e2, td)
 281
 282
 283
 284 geturl_config = """
 285 from buildbot.status import html
 286 from buildbot.changes import mail
 287 from buildbot.process import factory
 288 from buildbot.steps import dummy
 289 from buildbot.scheduler import Scheduler
 290 from buildbot.changes.base import ChangeSource
 291 from buildbot.buildslave import BuildSlave
 292 s = factory.s
 293
 294 class DiscardScheduler(Scheduler):
 295     def addChange(self, change):
 296         pass
 297 class DummyChangeSource(ChangeSource):
 298     pass
 299
 300 BuildmasterConfig = c = {}
 301 c['slaves'] = [BuildSlave('bot1', 'sekrit'), BuildSlave('bot2', 'sekrit')]
 302 c['change_source'] = DummyChangeSource()
 303 c['schedulers'] = [DiscardScheduler('discard', None, 60, ['b1'])]
 304 c['slavePortnum'] = 0
 305 c['status'] = [html.Waterfall(http_port=0)]
 306
 307 f = factory.BuildFactory([s(dummy.RemoteDummy, timeout=1)])
 308
 309 c['builders'] = [
 310     {'name': 'b1', 'slavenames': ['bot1','bot2'],
 311      'builddir': 'b1', 'factory': f},
 312     ]
 313 c['buildbotURL'] = 'http://dummy.example.org:8010/'
 314
 315 """
 316
 317 class GetURL(RunMixin, unittest.TestCase):
 318
 319     def setUp(self):
 320         RunMixin.setUp(self)
 321         self.master.loadConfig(geturl_config)
 322         self.master.startService()
 323         d = self.connectSlave(["b1"])
 324         return d
 325
 326     def tearDown(self):
 327         stopHTTPLog()
 328         return RunMixin.tearDown(self)
 329
 330     def doBuild(self, buildername):
 331         br = base.BuildRequest("forced", sourcestamp.SourceStamp())
 332         d = br.waitUntilFinished()
 333         self.control.getBuilder(buildername).requestBuild(br)
 334         return d
 335
 336     def assertNoURL(self, target):
 337         self.failUnlessIdentical(self.status.getURLForThing(target), None)
 338
 339     def assertURLEqual(self, target, expected):
 340         got = self.status.getURLForThing(target)
 341         full_expected = "http://dummy.example.org:8010/" + expected
 342         self.failUnlessEqual(got, full_expected)
 343
 344     def testMissingBase(self):
 345         noweb_config1 = geturl_config + "del c['buildbotURL']\n"
 346         d = self.master.loadConfig(noweb_config1)
 347         d.addCallback(self._testMissingBase_1)
 348         return d
 349     def _testMissingBase_1(self, res):
 350         s = self.status
 351         self.assertNoURL(s)
 352         builder = s.getBuilder("b1")
 353         self.assertNoURL(builder)
 354
 355     def testBase(self):
 356         s = self.status
 357         self.assertURLEqual(s, "")
 358         builder = s.getBuilder("b1")
 359         self.assertURLEqual(builder, "b1")
 360
 361     def testChange(self):
 362         s = self.status
 363         c = Change("user", ["foo.c"], "comments")
 364         self.master.change_svc.addChange(c)
 365         # TODO: something more like s.getChanges(), requires IChange and
 366         # an accessor in IStatus. The HTML page exists already, though
 367         self.assertURLEqual(c, "changes/1")
 368
 369     def testBuild(self):
 370         # first we do some stuff so we'll have things to look at.
 371         s = self.status
 372         d = self.doBuild("b1")
 373         # maybe check IBuildSetStatus here?
 374         d.addCallback(self._testBuild_1)
 375         return d
 376
 377     def _testBuild_1(self, res):
 378         s = self.status
 379         builder = s.getBuilder("b1")
 380         build = builder.getLastFinishedBuild()
 381         self.assertURLEqual(build, "b1/builds/0")
 382         # no page for builder.getEvent(-1)
 383         step = build.getSteps()[0]
 384         self.assertURLEqual(step, "b1/builds/0/step-remote%20dummy")
 385         # maybe page for build.getTestResults?
 386         self.assertURLEqual(step.getLogs()[0],
 387                             "b1/builds/0/step-remote%20dummy/0")
 388
 389
 390
 391 class Logfile(BaseWeb, RunMixin, unittest.TestCase):
 392     def setUp(self):
 393         config = """
 394 from buildbot.status import html
 395 from buildbot.process.factory import BasicBuildFactory
 396 from buildbot.buildslave import BuildSlave
 397 f1 = BasicBuildFactory('cvsroot', 'cvsmodule')
 398 BuildmasterConfig = {
 399     'slaves': [BuildSlave('bot1', 'passwd1')],
 400     'schedulers': [],
 401     'builders': [{'name': 'builder1', 'slavename': 'bot1',
 402                   'builddir':'workdir', 'factory':f1}],
 403     'slavePortnum': 0,
 404     'status': [html.WebStatus(http_port=0)],
 405     }
 406 """
 407         if os.path.exists("test_logfile"):
 408             shutil.rmtree("test_logfile")
 409         os.mkdir("test_logfile")
 410         self.master = m = ConfiguredMaster("test_logfile", config)
 411         m.startService()
 412         # hack to find out what randomly-assigned port it is listening on
 413         port = self.find_webstatus(m).getPortnum()
 414         self.port = port
 415         # insert an event
 416
 417         req = base.BuildRequest("reason", sourcestamp.SourceStamp())
 418         build1 = base.Build([req])
 419         bs = m.status.getBuilder("builder1").newBuild()
 420         bs.setReason("reason")
 421         bs.buildStarted(build1)
 422
 423         step1 = BuildStep(name="setup")
 424         step1.setBuild(build1)
 425         bss = bs.addStepWithName("setup")
 426         step1.setStepStatus(bss)
 427         bss.stepStarted()
 428
 429         log1 = step1.addLog("output")
 430         log1.addStdout("some stdout\n")
 431         log1.finish()
 432
 433         log2 = step1.addHTMLLog("error", "<html>ouch</html>")
 434
 435         log3 = step1.addLog("big")
 436         log3.addStdout("big log\n")
 437         for i in range(1000):
 438             log3.addStdout("a" * 500)
 439             log3.addStderr("b" * 500)
 440         log3.finish()
 441
 442         log4 = step1.addCompleteLog("bigcomplete",
 443                                     "big2 log\n" + "a" * 1*1000*1000)
 444
 445         step1.step_status.stepFinished(builder.SUCCESS)
 446         bs.buildFinished()
 447
 448     def getLogPath(self, stepname, logname):
 449         return ("/builders/builder1/builds/0/steps/%s/logs/%s" %
 450                 (stepname, logname))
 451
 452     def getLogURL(self, stepname, logname):
 453         return ("http://localhost:%d" % self.port
 454                 + self.getLogPath(stepname, logname))
 455
 456     def test_logfile1(self):
 457         d = client.getPage("http://localhost:%d/" % self.port)
 458         def _check(page):
 459             self.failUnless(page)
 460         d.addCallback(_check)
 461         return d
 462
 463     def test_logfile2(self):
 464         logurl = self.getLogURL("setup", "output")
 465         d = client.getPage(logurl)
 466         def _check(logbody):
 467             self.failUnless(logbody)
 468         d.addCallback(_check)
 469         return d
 470
 471     def test_logfile3(self):
 472         logurl = self.getLogURL("setup", "output")
 473         d = client.getPage(logurl + "/text")
 474         def _check(logtext):
 475             self.failUnlessEqual(logtext, "some stdout\n")
 476         d.addCallback(_check)
 477         return d
 478
 479     def test_logfile4(self):
 480         logurl = self.getLogURL("setup", "error")
 481         d = client.getPage(logurl)
 482         def _check(logbody):
 483             self.failUnlessEqual(logbody, "<html>ouch</html>")
 484         d.addCallback(_check)
 485         return d
 486
 487     def test_logfile5(self):
 488         # this is log3, which is about 1MB in size, made up of alternating
 489         # stdout/stderr chunks. buildbot-0.6.6, when run against
 490         # twisted-1.3.0, fails to resume sending chunks after the client
 491         # stalls for a few seconds, because of a recursive doWrite() call
 492         # that was fixed in twisted-2.0.0
 493         p = SlowReader("GET %s HTTP/1.0\r\n\r\n"
 494                        % self.getLogPath("setup", "big"))
 495         cf = CFactory(p)
 496         c = reactor.connectTCP("localhost", self.port, cf)
 497         d = p.d
 498         def _check(res):
 499             self.failUnlessIn("big log", p.data)
 500             self.failUnlessIn("a"*100, p.data)
 501             self.failUnless(p.count > 1*1000*1000)
 502         d.addCallback(_check)
 503         return d
 504
 505     def test_logfile6(self):
 506         # this is log4, which is about 1MB in size, one big chunk.
 507         # buildbot-0.6.6 dies as the NetstringReceiver barfs on the
 508         # saved logfile, because it was using one big chunk and exceeding
 509         # NetstringReceiver.MAX_LENGTH
 510         p = SlowReader("GET %s HTTP/1.0\r\n\r\n"
 511                        % self.getLogPath("setup", "bigcomplete"))
 512         cf = CFactory(p)
 513         c = reactor.connectTCP("localhost", self.port, cf)
 514         d = p.d
 515         def _check(res):
 516             self.failUnlessIn("big2 log", p.data)
 517             self.failUnlessIn("a"*100, p.data)
 518             self.failUnless(p.count > 1*1000*1000)
 519         d.addCallback(_check)
 520         return d
 521
 522