buildbot/test/test_status.py

   1 # -*- test-case-name: buildbot.test.test_status -*-
   2
   3 import email, os
   4
   5 from zope.interface import implements
   6 from twisted.internet import defer, reactor
   7 from twisted.trial import unittest
   8
   9 from buildbot import interfaces
  10 from buildbot.sourcestamp import SourceStamp
  11 from buildbot.process.base import BuildRequest
  12 from buildbot.status import builder, base
  13
  14 mail = None
  15 try:
  16     from buildbot.status import mail
  17 except ImportError:
  18     pass
  19 from buildbot.status import progress, client # NEEDS COVERAGE
  20 from buildbot.test.runutils import RunMixin
  21
  22 class MyStep:
  23     build = None
  24     def getName(self):
  25         return "step"
  26
  27 class MyLogFileProducer(builder.LogFileProducer):
  28     # The reactor.callLater(0) in LogFileProducer.resumeProducing is a bit of
  29     # a nuisance from a testing point of view. This subclass adds a Deferred
  30     # to that call so we can find out when it is complete.
  31     def resumeProducing(self):
  32         d = defer.Deferred()
  33         reactor.callLater(0, self._resumeProducing, d)
  34         return d
  35     def _resumeProducing(self, d):
  36         builder.LogFileProducer._resumeProducing(self)
  37         reactor.callLater(0, d.callback, None)
  38
  39 class MyLog(builder.LogFile):
  40     def __init__(self, basedir, name, text=None, step=None):
  41         self.fakeBuilderBasedir = basedir
  42         if not step:
  43             step = MyStep()
  44         builder.LogFile.__init__(self, step, name, name)
  45         if text:
  46             self.addStdout(text)
  47             self.finish()
  48     def getFilename(self):
  49         return os.path.join(self.fakeBuilderBasedir, self.name)
  50
  51     def subscribeConsumer(self, consumer):
  52         p = MyLogFileProducer(self, consumer)
  53         d = p.resumeProducing()
  54         return d
  55
  56 class MyHTMLLog(builder.HTMLLogFile):
  57     def __init__(self, basedir, name, html):
  58         step = MyStep()
  59         builder.HTMLLogFile.__init__(self, step, name, name, html)
  60
  61 class MyLogSubscriber:
  62     def __init__(self):
  63         self.chunks = []
  64     def logChunk(self, build, step, log, channel, text):
  65         self.chunks.append((channel, text))
  66
  67 class MyLogConsumer:
  68     def __init__(self, limit=None):
  69         self.chunks = []
  70         self.finished = False
  71         self.limit = limit
  72     def registerProducer(self, producer, streaming):
  73         self.producer = producer
  74         self.streaming = streaming
  75     def unregisterProducer(self):
  76         self.producer = None
  77     def writeChunk(self, chunk):
  78         self.chunks.append(chunk)
  79         if self.limit:
  80             self.limit -= 1
  81             if self.limit == 0:
  82                 self.producer.pauseProducing()
  83     def finish(self):
  84         self.finished = True
  85
  86 if mail:
  87     class MyMailer(mail.MailNotifier):
  88         def sendMessage(self, m, recipients):
  89             self.parent.messages.append((m, recipients))
  90
  91 class MyStatus:
  92     def getBuildbotURL(self):
  93         return self.url
  94     def getURLForThing(self, thing):
  95         return None
  96
  97 class MyBuilder(builder.BuilderStatus):
  98     nextBuildNumber = 0
  99
 100 class MyBuild(builder.BuildStatus):
 101     testlogs = []
 102     def __init__(self, parent, number, results):
 103         builder.BuildStatus.__init__(self, parent, number)
 104         self.results = results
 105         self.source = SourceStamp(revision="1.14")
 106         self.reason = "build triggered by changes"
 107         self.finished = True
 108     def getLogs(self):
 109         return self.testlogs
 110
 111 class MyLookup:
 112     implements(interfaces.IEmailLookup)
 113
 114     def getAddress(self, user):
 115         d = defer.Deferred()
 116         # With me now is Mr Thomas Walters of West Hartlepool who is totally
 117         # invisible.
 118         if user == "Thomas_Walters":
 119             d.callback(None)
 120         else:
 121             d.callback(user + "@" + "dev.com")
 122         return d
 123
 124 class Mail(unittest.TestCase):
 125
 126     def setUp(self):
 127         self.builder = MyBuilder("builder1")
 128
 129     def stall(self, res, timeout):
 130         d = defer.Deferred()
 131         reactor.callLater(timeout, d.callback, res)
 132         return d
 133
 134     def makeBuild(self, number, results):
 135         return MyBuild(self.builder, number, results)
 136
 137     def failUnlessIn(self, substring, string):
 138         self.failUnless(string.find(substring) != -1,
 139                         "didn't see '%s' in '%s'" % (substring, string))
 140
 141     def getProjectName(self):
 142         return "PROJECT"
 143
 144     def getBuildbotURL(self):
 145         return "BUILDBOT_URL"
 146
 147     def getURLForThing(self, thing):
 148         return None
 149
 150     def testBuild1(self):
 151         mailer = MyMailer(fromaddr="buildbot@example.com",
 152                           extraRecipients=["recip@example.com",
 153                                            "recip2@example.com"],
 154                           lookup=mail.Domain("dev.com"))
 155         mailer.parent = self
 156         mailer.status = self
 157         self.messages = []
 158
 159         b1 = self.makeBuild(3, builder.SUCCESS)
 160         b1.blamelist = ["bob"]
 161
 162         mailer.buildFinished("builder1", b1, b1.results)
 163         self.failUnless(len(self.messages) == 1)
 164         m,r = self.messages.pop()
 165         t = m.as_string()
 166         self.failUnlessIn("To: bob@dev.com, recip2@example.com, "
 167                           "recip@example.com\n", t)
 168         self.failUnlessIn("From: buildbot@example.com\n", t)
 169         self.failUnlessIn("Subject: buildbot success in PROJECT on builder1\n", t)
 170         self.failUnlessIn("Date: ", t)
 171         self.failUnlessIn("Build succeeded!\n", t)
 172         self.failUnlessIn("Buildbot URL: BUILDBOT_URL\n", t)
 173
 174     def testBuild2(self):
 175         mailer = MyMailer(fromaddr="buildbot@example.com",
 176                           extraRecipients=["recip@example.com",
 177                                            "recip2@example.com"],
 178                           lookup="dev.com",
 179                           sendToInterestedUsers=False)
 180         mailer.parent = self
 181         mailer.status = self
 182         self.messages = []
 183
 184         b1 = self.makeBuild(3, builder.SUCCESS)
 185         b1.blamelist = ["bob"]
 186
 187         mailer.buildFinished("builder1", b1, b1.results)
 188         self.failUnless(len(self.messages) == 1)
 189         m,r = self.messages.pop()
 190         t = m.as_string()
 191         self.failUnlessIn("To: recip2@example.com, "
 192                           "recip@example.com\n", t)
 193         self.failUnlessIn("From: buildbot@example.com\n", t)
 194         self.failUnlessIn("Subject: buildbot success in PROJECT on builder1\n", t)
 195         self.failUnlessIn("Build succeeded!\n", t)
 196         self.failUnlessIn("Buildbot URL: BUILDBOT_URL\n", t)
 197
 198     def testBuildStatusCategory(self):
 199         # a status client only interested in a category should only receive
 200         # from that category
 201         mailer = MyMailer(fromaddr="buildbot@example.com",
 202                           extraRecipients=["recip@example.com",
 203                                            "recip2@example.com"],
 204                           lookup="dev.com",
 205                           sendToInterestedUsers=False,
 206                           categories=["debug"])
 207
 208         mailer.parent = self
 209         mailer.status = self
 210         self.messages = []
 211
 212         b1 = self.makeBuild(3, builder.SUCCESS)
 213         b1.blamelist = ["bob"]
 214
 215         mailer.buildFinished("builder1", b1, b1.results)
 216         self.failIf(self.messages)
 217
 218     def testBuilderCategory(self):
 219         # a builder in a certain category should notify status clients that
 220         # did not list categories, or categories including this one
 221         mailer1 = MyMailer(fromaddr="buildbot@example.com",
 222                            extraRecipients=["recip@example.com",
 223                                             "recip2@example.com"],
 224                            lookup="dev.com",
 225                            sendToInterestedUsers=False)
 226         mailer2 = MyMailer(fromaddr="buildbot@example.com",
 227                            extraRecipients=["recip@example.com",
 228                                             "recip2@example.com"],
 229                            lookup="dev.com",
 230                            sendToInterestedUsers=False,
 231                            categories=["active"])
 232         mailer3 = MyMailer(fromaddr="buildbot@example.com",
 233                            extraRecipients=["recip@example.com",
 234                                             "recip2@example.com"],
 235                            lookup="dev.com",
 236                            sendToInterestedUsers=False,
 237                            categories=["active", "debug"])
 238
 239         builderd = MyBuilder("builder2", "debug")
 240
 241         mailer1.parent = self
 242         mailer1.status = self
 243         mailer2.parent = self
 244         mailer2.status = self
 245         mailer3.parent = self
 246         mailer3.status = self
 247         self.messages = []
 248
 249         t = mailer1.builderAdded("builder2", builderd)
 250         self.assertEqual(len(mailer1.watched), 1)
 251         self.assertEqual(t, mailer1)
 252         t = mailer2.builderAdded("builder2", builderd)
 253         self.assertEqual(len(mailer2.watched), 0)
 254         self.assertEqual(t, None)
 255         t = mailer3.builderAdded("builder2", builderd)
 256         self.assertEqual(len(mailer3.watched), 1)
 257         self.assertEqual(t, mailer3)
 258
 259         b2 = MyBuild(builderd, 3, builder.SUCCESS)
 260         b2.blamelist = ["bob"]
 261
 262         mailer1.buildFinished("builder2", b2, b2.results)
 263         self.failUnlessEqual(len(self.messages), 1)
 264         self.messages = []
 265         mailer2.buildFinished("builder2", b2, b2.results)
 266         self.failUnlessEqual(len(self.messages), 0)
 267         self.messages = []
 268         mailer3.buildFinished("builder2", b2, b2.results)
 269         self.failUnlessEqual(len(self.messages), 1)
 270
 271     def testFailure(self):
 272         mailer = MyMailer(fromaddr="buildbot@example.com", mode="problem",
 273                           extraRecipients=["recip@example.com",
 274                                            "recip2@example.com"],
 275                           lookup=MyLookup())
 276         mailer.parent = self
 277         mailer.status = self
 278         self.messages = []
 279
 280         b1 = self.makeBuild(3, builder.SUCCESS)
 281         b1.blamelist = ["dev1", "dev2"]
 282         b2 = self.makeBuild(4, builder.FAILURE)
 283         b2.setText(["snarkleack", "polarization", "failed"])
 284         b2.blamelist = ["dev3", "dev3", "dev3", "dev4",
 285                         "Thomas_Walters"]
 286         mailer.buildFinished("builder1", b1, b1.results)
 287         self.failIf(self.messages)
 288         mailer.buildFinished("builder1", b2, b2.results)
 289         self.failUnless(len(self.messages) == 1)
 290         m,r = self.messages.pop()
 291         t = m.as_string()
 292         self.failUnlessIn("To: dev3@dev.com, dev4@dev.com, "
 293                           "recip2@example.com, recip@example.com\n", t)
 294         self.failUnlessIn("From: buildbot@example.com\n", t)
 295         self.failUnlessIn("Subject: buildbot failure in PROJECT on builder1\n", t)
 296         self.failUnlessIn("The Buildbot has detected a new failure", t)
 297         self.failUnlessIn("BUILD FAILED: snarkleack polarization failed\n", t)
 298         self.failUnlessEqual(r, ["dev3@dev.com", "dev4@dev.com",
 299                                  "recip2@example.com", "recip@example.com"])
 300
 301     def testLogs(self):
 302         basedir = "test_status_logs"
 303         os.mkdir(basedir)
 304         mailer = MyMailer(fromaddr="buildbot@example.com", addLogs=True,
 305                           extraRecipients=["recip@example.com",
 306                                            "recip2@example.com"])
 307         mailer.parent = self
 308         mailer.status = self
 309         self.messages = []
 310
 311         b1 = self.makeBuild(3, builder.WARNINGS)
 312         b1.testlogs = [MyLog(basedir, 'compile', "Compile log here\n"),
 313                        MyLog(basedir,
 314                              'test', "Test log here\nTest 4 failed\n"),
 315                    ]
 316         b1.text = ["unusual", "gnarzzler", "output"]
 317         mailer.buildFinished("builder1", b1, b1.results)
 318         self.failUnless(len(self.messages) == 1)
 319         m,r = self.messages.pop()
 320         t = m.as_string()
 321         self.failUnlessIn("Subject: buildbot warnings in PROJECT on builder1\n", t)
 322         m2 = email.message_from_string(t)
 323         p = m2.get_payload()
 324         self.failUnlessEqual(len(p), 3)
 325
 326         self.failUnlessIn("Build Had Warnings: unusual gnarzzler output\n",
 327                           p[0].get_payload())
 328
 329         self.failUnlessEqual(p[1].get_filename(), "step.compile")
 330         self.failUnlessEqual(p[1].get_payload(), "Compile log here\n")
 331
 332         self.failUnlessEqual(p[2].get_filename(), "step.test")
 333         self.failUnlessIn("Test log here\n", p[2].get_payload())
 334
 335     def testMail(self):
 336         basedir = "test_status_mail"
 337         os.mkdir(basedir)
 338         dest = os.environ.get("BUILDBOT_TEST_MAIL")
 339         if not dest:
 340             raise unittest.SkipTest("define BUILDBOT_TEST_MAIL=dest to run this")
 341         mailer = mail.MailNotifier(fromaddr="buildbot@example.com",
 342                                    addLogs=True,
 343                                    extraRecipients=[dest])
 344         s = MyStatus()
 345         s.url = "project URL"
 346         mailer.status = s
 347
 348         b1 = self.makeBuild(3, builder.SUCCESS)
 349         b1.testlogs = [MyLog(basedir, 'compile', "Compile log here\n"),
 350                        MyLog(basedir,
 351                              'test', "Test log here\nTest 4 failed\n"),
 352                    ]
 353
 354         print "sending mail to", dest
 355         d = mailer.buildFinished("builder1", b1, b1.results)
 356         # When this fires, the mail has been sent, but the SMTP connection is
 357         # still up (because smtp.sendmail relies upon the server to hang up).
 358         # Spin for a moment to avoid the "unclean reactor" warning that Trial
 359         # gives us if we finish before the socket is disconnected. Really,
 360         # sendmail() ought to hang up the connection once it is finished:
 361         # otherwise a malicious SMTP server could make us consume lots of
 362         # memory.
 363         d.addCallback(self.stall, 0.1)
 364         return d
 365
 366 if not mail:
 367     Mail.skip = "the Twisted Mail package is not installed"
 368
 369 class Progress(unittest.TestCase):
 370     def testWavg(self):
 371         bp = progress.BuildProgress([])
 372         e = progress.Expectations(bp)
 373         # wavg(old, current)
 374         self.failUnlessEqual(e.wavg(None, None), None)
 375         self.failUnlessEqual(e.wavg(None, 3), 3)
 376         self.failUnlessEqual(e.wavg(3, None), 3)
 377         self.failUnlessEqual(e.wavg(3, 4), 3.5)
 378         e.decay = 0.1
 379         self.failUnlessEqual(e.wavg(3, 4), 3.1)
 380
 381
 382 class Results(unittest.TestCase):
 383
 384     def testAddResults(self):
 385         b = builder.BuildStatus(builder.BuilderStatus("test"), 12)
 386         testname = ("buildbot", "test", "test_status", "Results",
 387                     "testAddResults")
 388         r1 = builder.TestResult(name=testname,
 389                                 results=builder.SUCCESS,
 390                                 text=["passed"],
 391                                 logs={'output': ""},
 392                                 )
 393         b.addTestResult(r1)
 394
 395         res = b.getTestResults()
 396         self.failUnlessEqual(res.keys(), [testname])
 397         t = res[testname]
 398         self.failUnless(interfaces.ITestResult.providedBy(t))
 399         self.failUnlessEqual(t.getName(), testname)
 400         self.failUnlessEqual(t.getResults(), builder.SUCCESS)
 401         self.failUnlessEqual(t.getText(), ["passed"])
 402         self.failUnlessEqual(t.getLogs(), {'output': ""})
 403
 404 class Log(unittest.TestCase):
 405     def setUpClass(self):
 406         self.basedir = "status_log_add"
 407         os.mkdir(self.basedir)
 408
 409     def testAdd(self):
 410         l = MyLog(self.basedir, "compile", step=13)
 411         self.failUnlessEqual(l.getName(), "compile")
 412         self.failUnlessEqual(l.getStep(), 13)
 413         l.addHeader("HEADER\n")
 414         l.addStdout("Some text\n")
 415         l.addStderr("Some error\n")
 416         l.addStdout("Some more text\n")
 417         self.failIf(l.isFinished())
 418         l.finish()
 419         self.failUnless(l.isFinished())
 420         self.failUnlessEqual(l.getText(),
 421                              "Some text\nSome error\nSome more text\n")
 422         self.failUnlessEqual(l.getTextWithHeaders(),
 423                              "HEADER\n" +
 424                              "Some text\nSome error\nSome more text\n")
 425         self.failUnlessEqual(len(list(l.getChunks())), 4)
 426
 427         self.failUnless(l.hasContents())
 428         os.unlink(l.getFilename())
 429         self.failIf(l.hasContents())
 430
 431     def TODO_testDuplicate(self):
 432         # create multiple logs for the same step with the same logname, make
 433         # sure their on-disk filenames are suitably uniquified. This
 434         # functionality actually lives in BuildStepStatus and BuildStatus, so
 435         # this test must involve more than just the MyLog class.
 436
 437         # naieve approach, doesn't work
 438         l1 = MyLog(self.basedir, "duplicate")
 439         l1.addStdout("Some text\n")
 440         l1.finish()
 441         l2 = MyLog(self.basedir, "duplicate")
 442         l2.addStdout("Some more text\n")
 443         l2.finish()
 444         self.failIfEqual(l1.getFilename(), l2.getFilename())
 445
 446     def testMerge1(self):
 447         l = MyLog(self.basedir, "merge1")
 448         l.addHeader("HEADER\n")
 449         l.addStdout("Some text\n")
 450         l.addStdout("Some more text\n")
 451         l.addStdout("more\n")
 452         l.finish()
 453         self.failUnlessEqual(l.getText(),
 454                              "Some text\nSome more text\nmore\n")
 455         self.failUnlessEqual(l.getTextWithHeaders(),
 456                              "HEADER\n" +
 457                              "Some text\nSome more text\nmore\n")
 458         self.failUnlessEqual(len(list(l.getChunks())), 2)
 459
 460     def testMerge2(self):
 461         l = MyLog(self.basedir, "merge2")
 462         l.addHeader("HEADER\n")
 463         for i in xrange(1000):
 464             l.addStdout("aaaa")
 465         for i in xrange(30):
 466             l.addStderr("bbbb")
 467         for i in xrange(10):
 468             l.addStdout("cc")
 469         target = 1000*"aaaa" + 30 * "bbbb" + 10 * "cc"
 470         self.failUnlessEqual(len(l.getText()), len(target))
 471         self.failUnlessEqual(l.getText(), target)
 472         l.finish()
 473         self.failUnlessEqual(len(l.getText()), len(target))
 474         self.failUnlessEqual(l.getText(), target)
 475         self.failUnlessEqual(len(list(l.getChunks())), 4)
 476
 477     def testMerge3(self):
 478         l = MyLog(self.basedir, "merge3")
 479         l.chunkSize = 100
 480         l.addHeader("HEADER\n")
 481         for i in xrange(8):
 482             l.addStdout(10*"a")
 483         for i in xrange(8):
 484             l.addStdout(10*"a")
 485         self.failUnlessEqual(list(l.getChunks()),
 486                              [(builder.HEADER, "HEADER\n"),
 487                               (builder.STDOUT, 100*"a"),
 488                               (builder.STDOUT, 60*"a")])
 489         l.finish()
 490         self.failUnlessEqual(l.getText(), 160*"a")
 491
 492     def testReadlines(self):
 493         l = MyLog(self.basedir, "chunks")
 494         l.addHeader("HEADER\n") # should be ignored
 495         l.addStdout("Some text\n")
 496         l.addStdout("Some More Text\nAnd Some More\n")
 497         l.addStderr("Some Stderr\n")
 498         l.addStdout("Last line\n")
 499         l.finish()
 500         alllines = list(l.readlines())
 501         self.failUnlessEqual(len(alllines), 4)
 502         self.failUnlessEqual(alllines[0], "Some text\n")
 503         self.failUnlessEqual(alllines[2], "And Some More\n")
 504         self.failUnlessEqual(alllines[3], "Last line\n")
 505         stderr = list(l.readlines(interfaces.LOG_CHANNEL_STDERR))
 506         self.failUnlessEqual(len(stderr), 1)
 507         self.failUnlessEqual(stderr[0], "Some Stderr\n")
 508         lines = l.readlines()
 509         if False: # TODO: l.readlines() is not yet an iterator
 510             # verify that it really is an iterator
 511             line0 = lines.next()
 512             self.failUnlessEqual(line0, "Some text\n")
 513             line1 = lines.next()
 514             line2 = lines.next()
 515             self.failUnlessEqual(line2, "And Some More\n")
 516
 517
 518     def testChunks(self):
 519         l = MyLog(self.basedir, "chunks")
 520         c1 = l.getChunks()
 521         l.addHeader("HEADER\n")
 522         l.addStdout("Some text\n")
 523         self.failUnlessEqual("".join(l.getChunks(onlyText=True)),
 524                              "HEADER\nSome text\n")
 525         c2 = l.getChunks()
 526
 527         l.addStdout("Some more text\n")
 528         self.failUnlessEqual("".join(l.getChunks(onlyText=True)),
 529                              "HEADER\nSome text\nSome more text\n")
 530         c3 = l.getChunks()
 531
 532         l.addStdout("more\n")
 533         l.finish()
 534
 535         self.failUnlessEqual(list(c1), [])
 536         self.failUnlessEqual(list(c2), [(builder.HEADER, "HEADER\n"),
 537                                         (builder.STDOUT, "Some text\n")])
 538         self.failUnlessEqual(list(c3), [(builder.HEADER, "HEADER\n"),
 539                                         (builder.STDOUT,
 540                                          "Some text\nSome more text\n")])
 541
 542         self.failUnlessEqual(l.getText(),
 543                              "Some text\nSome more text\nmore\n")
 544         self.failUnlessEqual(l.getTextWithHeaders(),
 545                              "HEADER\n" +
 546                              "Some text\nSome more text\nmore\n")
 547         self.failUnlessEqual(len(list(l.getChunks())), 2)
 548
 549     def testUpgrade(self):
 550         l = MyLog(self.basedir, "upgrade")
 551         l.addHeader("HEADER\n")
 552         l.addStdout("Some text\n")
 553         l.addStdout("Some more text\n")
 554         l.addStdout("more\n")
 555         l.finish()
 556         self.failUnless(l.hasContents())
 557         # now doctor it to look like a 0.6.4-era non-upgraded logfile
 558         l.entries = list(l.getChunks())
 559         del l.filename
 560         os.unlink(l.getFilename())
 561         # now make sure we can upgrade it
 562         l.upgrade("upgrade")
 563         self.failUnlessEqual(l.getText(),
 564                              "Some text\nSome more text\nmore\n")
 565         self.failUnlessEqual(len(list(l.getChunks())), 2)
 566         self.failIf(l.entries)
 567
 568         # now, do it again, but make it look like an upgraded 0.6.4 logfile
 569         # (i.e. l.filename is missing, but the contents are there on disk)
 570         l.entries = list(l.getChunks())
 571         del l.filename
 572         l.upgrade("upgrade")
 573         self.failUnlessEqual(l.getText(),
 574                              "Some text\nSome more text\nmore\n")
 575         self.failUnlessEqual(len(list(l.getChunks())), 2)
 576         self.failIf(l.entries)
 577         self.failUnless(l.hasContents())
 578
 579     def testHTMLUpgrade(self):
 580         l = MyHTMLLog(self.basedir, "upgrade", "log contents")
 581         l.upgrade("filename")
 582
 583     def testSubscribe(self):
 584         l1 = MyLog(self.basedir, "subscribe1")
 585         l1.finish()
 586         self.failUnless(l1.isFinished())
 587
 588         s = MyLogSubscriber()
 589         l1.subscribe(s, True)
 590         l1.unsubscribe(s)
 591         self.failIf(s.chunks)
 592
 593         s = MyLogSubscriber()
 594         l1.subscribe(s, False)
 595         l1.unsubscribe(s)
 596         self.failIf(s.chunks)
 597
 598         finished = []
 599         l2 = MyLog(self.basedir, "subscribe2")
 600         l2.waitUntilFinished().addCallback(finished.append)
 601         l2.addHeader("HEADER\n")
 602         s1 = MyLogSubscriber()
 603         l2.subscribe(s1, True)
 604         s2 = MyLogSubscriber()
 605         l2.subscribe(s2, False)
 606         self.failUnlessEqual(s1.chunks, [(builder.HEADER, "HEADER\n")])
 607         self.failUnlessEqual(s2.chunks, [])
 608
 609         l2.addStdout("Some text\n")
 610         self.failUnlessEqual(s1.chunks, [(builder.HEADER, "HEADER\n"),
 611                                          (builder.STDOUT, "Some text\n")])
 612         self.failUnlessEqual(s2.chunks, [(builder.STDOUT, "Some text\n")])
 613         l2.unsubscribe(s1)
 614
 615         l2.addStdout("Some more text\n")
 616         self.failUnlessEqual(s1.chunks, [(builder.HEADER, "HEADER\n"),
 617                                          (builder.STDOUT, "Some text\n")])
 618         self.failUnlessEqual(s2.chunks, [(builder.STDOUT, "Some text\n"),
 619                                          (builder.STDOUT, "Some more text\n"),
 620                                          ])
 621         self.failIf(finished)
 622         l2.finish()
 623         self.failUnlessEqual(finished, [l2])
 624
 625     def testConsumer(self):
 626         l1 = MyLog(self.basedir, "consumer1")
 627         l1.finish()
 628         self.failUnless(l1.isFinished())
 629
 630         s = MyLogConsumer()
 631         d = l1.subscribeConsumer(s)
 632         d.addCallback(self._testConsumer_1, s)
 633         return d
 634     testConsumer.timeout = 5
 635     def _testConsumer_1(self, res, s):
 636         self.failIf(s.chunks)
 637         self.failUnless(s.finished)
 638         self.failIf(s.producer) # producer should be registered and removed
 639
 640         l2 = MyLog(self.basedir, "consumer2")
 641         l2.addHeader("HEADER\n")
 642         l2.finish()
 643         self.failUnless(l2.isFinished())
 644
 645         s = MyLogConsumer()
 646         d = l2.subscribeConsumer(s)
 647         d.addCallback(self._testConsumer_2, s)
 648         return d
 649     def _testConsumer_2(self, res, s):
 650         self.failUnlessEqual(s.chunks, [(builder.HEADER, "HEADER\n")])
 651         self.failUnless(s.finished)
 652         self.failIf(s.producer) # producer should be registered and removed
 653
 654
 655         l2 = MyLog(self.basedir, "consumer3")
 656         l2.chunkSize = 1000
 657         l2.addHeader("HEADER\n")
 658         l2.addStdout(800*"a")
 659         l2.addStdout(800*"a") # should now have two chunks on disk, 1000+600
 660         l2.addStdout(800*"b") # HEADER,1000+600*a on disk, 800*a in memory
 661         l2.addStdout(800*"b") # HEADER,1000+600*a,1000+600*b on disk
 662         l2.addStdout(200*"c") # HEADER,1000+600*a,1000+600*b on disk,
 663                               # 200*c in memory
 664
 665         s = MyLogConsumer(limit=1)
 666         d = l2.subscribeConsumer(s)
 667         d.addCallback(self._testConsumer_3, l2, s)
 668         return d
 669     def _testConsumer_3(self, res, l2, s):
 670         self.failUnless(s.streaming)
 671         self.failUnlessEqual(s.chunks, [(builder.HEADER, "HEADER\n")])
 672         s.limit = 1
 673         d = s.producer.resumeProducing()
 674         d.addCallback(self._testConsumer_4, l2, s)
 675         return d
 676     def _testConsumer_4(self, res, l2, s):
 677         self.failUnlessEqual(s.chunks, [(builder.HEADER, "HEADER\n"),
 678                                         (builder.STDOUT, 1000*"a"),
 679                                         ])
 680         s.limit = None
 681         d = s.producer.resumeProducing()
 682         d.addCallback(self._testConsumer_5, l2, s)
 683         return d
 684     def _testConsumer_5(self, res, l2, s):
 685         self.failUnlessEqual(s.chunks, [(builder.HEADER, "HEADER\n"),
 686                                         (builder.STDOUT, 1000*"a"),
 687                                         (builder.STDOUT, 600*"a"),
 688                                         (builder.STDOUT, 1000*"b"),
 689                                         (builder.STDOUT, 600*"b"),
 690                                         (builder.STDOUT, 200*"c")])
 691         l2.addStdout(1000*"c") # HEADER,1600*a,1600*b,1200*c on disk
 692         self.failUnlessEqual(s.chunks, [(builder.HEADER, "HEADER\n"),
 693                                         (builder.STDOUT, 1000*"a"),
 694                                         (builder.STDOUT, 600*"a"),
 695                                         (builder.STDOUT, 1000*"b"),
 696                                         (builder.STDOUT, 600*"b"),
 697                                         (builder.STDOUT, 200*"c"),
 698                                         (builder.STDOUT, 1000*"c")])
 699         l2.finish()
 700         self.failUnlessEqual(s.chunks, [(builder.HEADER, "HEADER\n"),
 701                                         (builder.STDOUT, 1000*"a"),
 702                                         (builder.STDOUT, 600*"a"),
 703                                         (builder.STDOUT, 1000*"b"),
 704                                         (builder.STDOUT, 600*"b"),
 705                                         (builder.STDOUT, 200*"c"),
 706                                         (builder.STDOUT, 1000*"c")])
 707         self.failIf(s.producer)
 708         self.failUnless(s.finished)
 709
 710     def testLargeSummary(self):
 711         bigtext = "a" * 200000 # exceed the NetstringReceiver 100KB limit
 712         l = MyLog(self.basedir, "large", bigtext)
 713         s = MyLogConsumer()
 714         d = l.subscribeConsumer(s)
 715         def _check(res):
 716             for ctype,chunk in s.chunks:
 717                 self.failUnless(len(chunk) < 100000)
 718             merged = "".join([c[1] for c in s.chunks])
 719             self.failUnless(merged == bigtext)
 720         d.addCallback(_check)
 721         # when this fails, it fails with a timeout, and there is an exception
 722         # sent to log.err(). This AttributeError exception is in
 723         # NetstringReceiver.dataReceived where it does
 724         # self.transport.loseConnection() because of the NetstringParseError,
 725         # however self.transport is None
 726         return d
 727     testLargeSummary.timeout = 5
 728
 729 config_base = """
 730 from buildbot.process import factory
 731 from buildbot.steps import dummy
 732 from buildbot.slave import BuildSlave
 733 s = factory.s
 734
 735 f1 = factory.QuickBuildFactory('fakerep', 'cvsmodule', configure=None)
 736
 737 f2 = factory.BuildFactory([
 738     s(dummy.Dummy, timeout=1),
 739     s(dummy.RemoteDummy, timeout=2),
 740     ])
 741
 742 BuildmasterConfig = c = {}
 743 c['slaves'] = [BuildSlave('bot1', 'sekrit')]
 744 c['sources'] = []
 745 c['schedulers'] = []
 746 c['builders'] = []
 747 c['builders'].append({'name':'quick', 'slavename':'bot1',
 748                       'builddir': 'quickdir', 'factory': f1})
 749 c['slavePortnum'] = 0
 750 """
 751
 752 config_2 = config_base + """
 753 c['builders'] = [{'name': 'dummy', 'slavename': 'bot1',
 754                   'builddir': 'dummy1', 'factory': f2},
 755                  {'name': 'testdummy', 'slavename': 'bot1',
 756                   'builddir': 'dummy2', 'factory': f2, 'category': 'test'}]
 757 """
 758
 759 class STarget(base.StatusReceiver):
 760     debug = False
 761
 762     def __init__(self, mode):
 763         self.mode = mode
 764         self.events = []
 765     def announce(self):
 766         if self.debug:
 767             print self.events[-1]
 768
 769     def builderAdded(self, name, builder):
 770         self.events.append(("builderAdded", name, builder))
 771         self.announce()
 772         if "builder" in self.mode:
 773             return self
 774     def builderChangedState(self, name, state):
 775         self.events.append(("builderChangedState", name, state))
 776         self.announce()
 777     def buildStarted(self, name, build):
 778         self.events.append(("buildStarted", name, build))
 779         self.announce()
 780         if "eta" in self.mode:
 781             self.eta_build = build.getETA()
 782         if "build" in self.mode:
 783             return self
 784     def buildETAUpdate(self, build, ETA):
 785         self.events.append(("buildETAUpdate", build, ETA))
 786         self.announce()
 787     def stepStarted(self, build, step):
 788         self.events.append(("stepStarted", build, step))
 789         self.announce()
 790         if 0 and "eta" in self.mode:
 791             print "TIMES", step.getTimes()
 792             print "ETA", step.getETA()
 793             print "EXP", step.getExpectations()
 794         if "step" in self.mode:
 795             return self
 796     def stepETAUpdate(self, build, step, ETA, expectations):
 797         self.events.append(("stepETAUpdate", build, step, ETA, expectations))
 798         self.announce()
 799     def logStarted(self, build, step, log):
 800         self.events.append(("logStarted", build, step, log))
 801         self.announce()
 802     def logFinished(self, build, step, log):
 803         self.events.append(("logFinished", build, step, log))
 804         self.announce()
 805     def stepFinished(self, build, step, results):
 806         self.events.append(("stepFinished", build, step, results))
 807         if 0 and "eta" in self.mode:
 808             print "post-EXP", step.getExpectations()
 809         self.announce()
 810     def buildFinished(self, name, build, results):
 811         self.events.append(("buildFinished", name, build, results))
 812         self.announce()
 813     def builderRemoved(self, name):
 814         self.events.append(("builderRemoved", name))
 815         self.announce()
 816
 817 class Subscription(RunMixin, unittest.TestCase):
 818     # verify that StatusTargets can subscribe/unsubscribe properly
 819
 820     def testSlave(self):
 821         m = self.master
 822         s = m.getStatus()
 823         self.t1 = t1 = STarget(["builder"])
 824         #t1.debug = True; print
 825         s.subscribe(t1)
 826         self.failUnlessEqual(len(t1.events), 0)
 827
 828         self.t3 = t3 = STarget(["builder", "build", "step"])
 829         s.subscribe(t3)
 830
 831         m.loadConfig(config_2)
 832         m.readConfig = True
 833         m.startService()
 834
 835         self.failUnlessEqual(len(t1.events), 4)
 836         self.failUnlessEqual(t1.events[0][0:2], ("builderAdded", "dummy"))
 837         self.failUnlessEqual(t1.events[1],
 838                              ("builderChangedState", "dummy", "offline"))
 839         self.failUnlessEqual(t1.events[2][0:2], ("builderAdded", "testdummy"))
 840         self.failUnlessEqual(t1.events[3],
 841                              ("builderChangedState", "testdummy", "offline"))
 842         t1.events = []
 843
 844         self.failUnlessEqual(s.getBuilderNames(), ["dummy", "testdummy"])
 845         self.failUnlessEqual(s.getBuilderNames(categories=['test']),
 846                              ["testdummy"])
 847         self.s1 = s1 = s.getBuilder("dummy")
 848         self.failUnlessEqual(s1.getName(), "dummy")
 849         self.failUnlessEqual(s1.getState(), ("offline", []))
 850         self.failUnlessEqual(s1.getCurrentBuilds(), [])
 851         self.failUnlessEqual(s1.getLastFinishedBuild(), None)
 852         self.failUnlessEqual(s1.getBuild(-1), None)
 853         #self.failUnlessEqual(s1.getEvent(-1), foo("created"))
 854
 855         # status targets should, upon being subscribed, immediately get a
 856         # list of all current builders matching their category
 857         self.t2 = t2 = STarget([])
 858         s.subscribe(t2)
 859         self.failUnlessEqual(len(t2.events), 2)
 860         self.failUnlessEqual(t2.events[0][0:2], ("builderAdded", "dummy"))
 861         self.failUnlessEqual(t2.events[1][0:2], ("builderAdded", "testdummy"))
 862
 863         d = self.connectSlave(builders=["dummy", "testdummy"])
 864         d.addCallback(self._testSlave_1, t1)
 865         return d
 866
 867     def _testSlave_1(self, res, t1):
 868         self.failUnlessEqual(len(t1.events), 2)
 869         self.failUnlessEqual(t1.events[0],
 870                              ("builderChangedState", "dummy", "idle"))
 871         self.failUnlessEqual(t1.events[1],
 872                              ("builderChangedState", "testdummy", "idle"))
 873         t1.events = []
 874
 875         c = interfaces.IControl(self.master)
 876         req = BuildRequest("forced build for testing", SourceStamp())
 877         c.getBuilder("dummy").requestBuild(req)
 878         d = req.waitUntilFinished()
 879         d2 = self.master.botmaster.waitUntilBuilderIdle("dummy")
 880         dl = defer.DeferredList([d, d2])
 881         dl.addCallback(self._testSlave_2)
 882         return dl
 883
 884     def _testSlave_2(self, res):
 885         # t1 subscribes to builds, but not anything lower-level
 886         ev = self.t1.events
 887         self.failUnlessEqual(len(ev), 4)
 888         self.failUnlessEqual(ev[0][0:3],
 889                              ("builderChangedState", "dummy", "building"))
 890         self.failUnlessEqual(ev[1][0], "buildStarted")
 891         self.failUnlessEqual(ev[2][0:2]+ev[2][3:4],
 892                              ("buildFinished", "dummy", builder.SUCCESS))
 893         self.failUnlessEqual(ev[3][0:3],
 894                              ("builderChangedState", "dummy", "idle"))
 895
 896         self.failUnlessEqual([ev[0] for ev in self.t3.events],
 897                              ["builderAdded",
 898                               "builderChangedState", # offline
 899                               "builderAdded",
 900                               "builderChangedState", # idle
 901                               "builderChangedState", # offline
 902                               "builderChangedState", # idle
 903                               "builderChangedState", # building
 904                               "buildStarted",
 905                               "stepStarted", "stepETAUpdate", "stepFinished",
 906                               "stepStarted", "stepETAUpdate",
 907                               "logStarted", "logFinished", "stepFinished",
 908                               "buildFinished",
 909                               "builderChangedState", # idle
 910                               ])
 911
 912         b = self.s1.getLastFinishedBuild()
 913         self.failUnless(b)
 914         self.failUnlessEqual(b.getBuilder().getName(), "dummy")
 915         self.failUnlessEqual(b.getNumber(), 0)
 916         self.failUnlessEqual(b.getSourceStamp(), (None, None, None))
 917         self.failUnlessEqual(b.getReason(), "forced build for testing")
 918         self.failUnlessEqual(b.getChanges(), [])
 919         self.failUnlessEqual(b.getResponsibleUsers(), [])
 920         self.failUnless(b.isFinished())
 921         self.failUnlessEqual(b.getText(), ['build', 'successful'])
 922         self.failUnlessEqual(b.getColor(), "green")
 923         self.failUnlessEqual(b.getResults(), builder.SUCCESS)
 924
 925         steps = b.getSteps()
 926         self.failUnlessEqual(len(steps), 2)
 927
 928         eta = 0
 929         st1 = steps[0]
 930         self.failUnlessEqual(st1.getName(), "dummy")
 931         self.failUnless(st1.isFinished())
 932         self.failUnlessEqual(st1.getText(), ["delay", "1 secs"])
 933         start,finish = st1.getTimes()
 934         self.failUnless(0.5 < (finish-start) < 10)
 935         self.failUnlessEqual(st1.getExpectations(), [])
 936         self.failUnlessEqual(st1.getLogs(), [])
 937         eta += finish-start
 938
 939         st2 = steps[1]
 940         self.failUnlessEqual(st2.getName(), "remote dummy")
 941         self.failUnless(st2.isFinished())
 942         self.failUnlessEqual(st2.getText(),
 943                              ["remote", "delay", "2 secs"])
 944         start,finish = st2.getTimes()
 945         self.failUnless(1.5 < (finish-start) < 10)
 946         eta += finish-start
 947         self.failUnlessEqual(st2.getExpectations(), [('output', 38, None)])
 948         logs = st2.getLogs()
 949         self.failUnlessEqual(len(logs), 1)
 950         self.failUnlessEqual(logs[0].getName(), "stdio")
 951         self.failUnlessEqual(logs[0].getText(), "data")
 952
 953         self.eta = eta
 954         # now we run it a second time, and we should have an ETA
 955
 956         self.t4 = t4 = STarget(["builder", "build", "eta"])
 957         self.master.getStatus().subscribe(t4)
 958         c = interfaces.IControl(self.master)
 959         req = BuildRequest("forced build for testing", SourceStamp())
 960         c.getBuilder("dummy").requestBuild(req)
 961         d = req.waitUntilFinished()
 962         d2 = self.master.botmaster.waitUntilBuilderIdle("dummy")
 963         dl = defer.DeferredList([d, d2])
 964         dl.addCallback(self._testSlave_3)
 965         return dl
 966
 967     def _testSlave_3(self, res):
 968         t4 = self.t4
 969         eta = self.eta
 970         self.failUnless(eta-1 < t4.eta_build < eta+1, # should be 3 seconds
 971                         "t4.eta_build was %g, not in (%g,%g)"
 972                         % (t4.eta_build, eta-1, eta+1))
 973
 974
 975 class Client(unittest.TestCase):
 976     def testAdaptation(self):
 977         b = builder.BuilderStatus("bname")
 978         b2 = client.makeRemote(b)
 979         self.failUnless(isinstance(b2, client.RemoteBuilder))
 980         b3 = client.makeRemote(None)
 981         self.failUnless(b3 is None)