buildbot/test/test_status.py

   1 # -*- test-case-name: buildbot.test.test_status -*-
   2
   3 import email, os
   4
   5 from twisted.internet import defer, reactor
   6 from twisted.trial import unittest
   7
   8 from buildbot import interfaces
   9 from buildbot.sourcestamp import SourceStamp
  10 from buildbot.process.base import BuildRequest
  11 from buildbot.twcompat import implements, providedBy, maybeWait
  12 from buildbot.status import builder, base
  13
  14 mail = None
  15 try:
  16     from buildbot.status import mail
  17 except ImportError:
  18     pass
  19 from buildbot.status import progress, client # NEEDS COVERAGE
  20 from buildbot.test.runutils import RunMixin
  21
  22 class MyStep:
  23     build = None
  24     def getName(self):
  25         return "step"
  26
  27 class MyLogFileProducer(builder.LogFileProducer):
  28     # The reactor.callLater(0) in LogFileProducer.resumeProducing is a bit of
  29     # a nuisance from a testing point of view. This subclass adds a Deferred
  30     # to that call so we can find out when it is complete.
  31     def resumeProducing(self):
  32         d = defer.Deferred()
  33         reactor.callLater(0, self._resumeProducing, d)
  34         return d
  35     def _resumeProducing(self, d):
  36         builder.LogFileProducer._resumeProducing(self)
  37         reactor.callLater(0, d.callback, None)
  38
  39 class MyLog(builder.LogFile):
  40     def __init__(self, basedir, name, text=None, step=None):
  41         self.fakeBuilderBasedir = basedir
  42         if not step:
  43             step = MyStep()
  44         builder.LogFile.__init__(self, step, name, name)
  45         if text:
  46             self.addStdout(text)
  47             self.finish()
  48     def getFilename(self):
  49         return os.path.join(self.fakeBuilderBasedir, self.name)
  50
  51     def subscribeConsumer(self, consumer):
  52         p = MyLogFileProducer(self, consumer)
  53         d = p.resumeProducing()
  54         return d
  55
  56 class MyHTMLLog(builder.HTMLLogFile):
  57     def __init__(self, basedir, name, html):
  58         step = MyStep()
  59         builder.HTMLLogFile.__init__(self, step, name, name, html)
  60
  61 class MyLogSubscriber:
  62     def __init__(self):
  63         self.chunks = []
  64     def logChunk(self, build, step, log, channel, text):
  65         self.chunks.append((channel, text))
  66
  67 class MyLogConsumer:
  68     def __init__(self, limit=None):
  69         self.chunks = []
  70         self.finished = False
  71         self.limit = limit
  72     def registerProducer(self, producer, streaming):
  73         self.producer = producer
  74         self.streaming = streaming
  75     def unregisterProducer(self):
  76         self.producer = None
  77     def writeChunk(self, chunk):
  78         self.chunks.append(chunk)
  79         if self.limit:
  80             self.limit -= 1
  81             if self.limit == 0:
  82                 self.producer.pauseProducing()
  83     def finish(self):
  84         self.finished = True
  85
  86 if mail:
  87     class MyMailer(mail.MailNotifier):
  88         def sendMessage(self, m, recipients):
  89             self.parent.messages.append((m, recipients))
  90
  91 class MyStatus:
  92     def getBuildbotURL(self):
  93         return self.url
  94     def getURLForThing(self, thing):
  95         return None
  96
  97 class MyBuilder(builder.BuilderStatus):
  98     nextBuildNumber = 0
  99
 100 class MyBuild(builder.BuildStatus):
 101     testlogs = []
 102     def __init__(self, parent, number, results):
 103         builder.BuildStatus.__init__(self, parent, number)
 104         self.results = results
 105         self.source = SourceStamp(revision="1.14")
 106         self.reason = "build triggered by changes"
 107         self.finished = True
 108     def getLogs(self):
 109         return self.testlogs
 110
 111 class MyLookup:
 112     if implements:
 113         implements(interfaces.IEmailLookup)
 114     else:
 115         __implements__ = interfaces.IEmailLookup,
 116
 117     def getAddress(self, user):
 118         d = defer.Deferred()
 119         # With me now is Mr Thomas Walters of West Hartlepool who is totally
 120         # invisible.
 121         if user == "Thomas_Walters":
 122             d.callback(None)
 123         else:
 124             d.callback(user + "@" + "dev.com")
 125         return d
 126
 127 class Mail(unittest.TestCase):
 128
 129     def setUp(self):
 130         self.builder = MyBuilder("builder1")
 131
 132     def stall(self, res, timeout):
 133         d = defer.Deferred()
 134         reactor.callLater(timeout, d.callback, res)
 135         return d
 136
 137     def makeBuild(self, number, results):
 138         return MyBuild(self.builder, number, results)
 139
 140     def failUnlessIn(self, substring, string):
 141         self.failUnless(string.find(substring) != -1)
 142
 143     def getBuildbotURL(self):
 144         return "BUILDBOT_URL"
 145
 146     def getURLForThing(self, thing):
 147         return None
 148
 149     def testBuild1(self):
 150         mailer = MyMailer(fromaddr="buildbot@example.com",
 151                           extraRecipients=["recip@example.com",
 152                                            "recip2@example.com"],
 153                           lookup=mail.Domain("dev.com"))
 154         mailer.parent = self
 155         mailer.status = self
 156         self.messages = []
 157
 158         b1 = self.makeBuild(3, builder.SUCCESS)
 159         b1.blamelist = ["bob"]
 160
 161         mailer.buildFinished("builder1", b1, b1.results)
 162         self.failUnless(len(self.messages) == 1)
 163         m,r = self.messages.pop()
 164         t = m.as_string()
 165         self.failUnlessIn("To: bob@dev.com, recip2@example.com, "
 166                           "recip@example.com\n", t)
 167         self.failUnlessIn("From: buildbot@example.com\n", t)
 168         self.failUnlessIn("Subject: buildbot success in builder1\n", t)
 169         self.failUnlessIn("Date: ", t)
 170         self.failUnlessIn("Build succeeded!\n", t)
 171         self.failUnlessIn("Buildbot URL: BUILDBOT_URL\n", t)
 172
 173     def testBuild2(self):
 174         mailer = MyMailer(fromaddr="buildbot@example.com",
 175                           extraRecipients=["recip@example.com",
 176                                            "recip2@example.com"],
 177                           lookup="dev.com",
 178                           sendToInterestedUsers=False)
 179         mailer.parent = self
 180         mailer.status = self
 181         self.messages = []
 182
 183         b1 = self.makeBuild(3, builder.SUCCESS)
 184         b1.blamelist = ["bob"]
 185
 186         mailer.buildFinished("builder1", b1, b1.results)
 187         self.failUnless(len(self.messages) == 1)
 188         m,r = self.messages.pop()
 189         t = m.as_string()
 190         self.failUnlessIn("To: recip2@example.com, "
 191                           "recip@example.com\n", t)
 192         self.failUnlessIn("From: buildbot@example.com\n", t)
 193         self.failUnlessIn("Subject: buildbot success in builder1\n", t)
 194         self.failUnlessIn("Build succeeded!\n", t)
 195         self.failUnlessIn("Buildbot URL: BUILDBOT_URL\n", t)
 196
 197     def testBuildStatusCategory(self):
 198         # a status client only interested in a category should only receive
 199         # from that category
 200         mailer = MyMailer(fromaddr="buildbot@example.com",
 201                           extraRecipients=["recip@example.com",
 202                                            "recip2@example.com"],
 203                           lookup="dev.com",
 204                           sendToInterestedUsers=False,
 205                           categories=["debug"])
 206
 207         mailer.parent = self
 208         mailer.status = self
 209         self.messages = []
 210
 211         b1 = self.makeBuild(3, builder.SUCCESS)
 212         b1.blamelist = ["bob"]
 213
 214         mailer.buildFinished("builder1", b1, b1.results)
 215         self.failIf(self.messages)
 216
 217     def testBuilderCategory(self):
 218         # a builder in a certain category should notify status clients that
 219         # did not list categories, or categories including this one
 220         mailer1 = MyMailer(fromaddr="buildbot@example.com",
 221                            extraRecipients=["recip@example.com",
 222                                             "recip2@example.com"],
 223                            lookup="dev.com",
 224                            sendToInterestedUsers=False)
 225         mailer2 = MyMailer(fromaddr="buildbot@example.com",
 226                            extraRecipients=["recip@example.com",
 227                                             "recip2@example.com"],
 228                            lookup="dev.com",
 229                            sendToInterestedUsers=False,
 230                            categories=["active"])
 231         mailer3 = MyMailer(fromaddr="buildbot@example.com",
 232                            extraRecipients=["recip@example.com",
 233                                             "recip2@example.com"],
 234                            lookup="dev.com",
 235                            sendToInterestedUsers=False,
 236                            categories=["active", "debug"])
 237
 238         builderd = MyBuilder("builder2", "debug")
 239
 240         mailer1.parent = self
 241         mailer1.status = self
 242         mailer2.parent = self
 243         mailer2.status = self
 244         mailer3.parent = self
 245         mailer3.status = self
 246         self.messages = []
 247
 248         t = mailer1.builderAdded("builder2", builderd)
 249         self.assertEqual(len(mailer1.watched), 1)
 250         self.assertEqual(t, mailer1)
 251         t = mailer2.builderAdded("builder2", builderd)
 252         self.assertEqual(len(mailer2.watched), 0)
 253         self.assertEqual(t, None)
 254         t = mailer3.builderAdded("builder2", builderd)
 255         self.assertEqual(len(mailer3.watched), 1)
 256         self.assertEqual(t, mailer3)
 257
 258         b2 = MyBuild(builderd, 3, builder.SUCCESS)
 259         b2.blamelist = ["bob"]
 260
 261         mailer1.buildFinished("builder2", b2, b2.results)
 262         self.failUnlessEqual(len(self.messages), 1)
 263         self.messages = []
 264         mailer2.buildFinished("builder2", b2, b2.results)
 265         self.failUnlessEqual(len(self.messages), 0)
 266         self.messages = []
 267         mailer3.buildFinished("builder2", b2, b2.results)
 268         self.failUnlessEqual(len(self.messages), 1)
 269
 270     def testFailure(self):
 271         mailer = MyMailer(fromaddr="buildbot@example.com", mode="problem",
 272                           extraRecipients=["recip@example.com",
 273                                            "recip2@example.com"],
 274                           lookup=MyLookup())
 275         mailer.parent = self
 276         mailer.status = self
 277         self.messages = []
 278
 279         b1 = self.makeBuild(3, builder.SUCCESS)
 280         b1.blamelist = ["dev1", "dev2"]
 281         b2 = self.makeBuild(4, builder.FAILURE)
 282         b2.setText(["snarkleack", "polarization", "failed"])
 283         b2.blamelist = ["dev3", "dev3", "dev3", "dev4",
 284                         "Thomas_Walters"]
 285         mailer.buildFinished("builder1", b1, b1.results)
 286         self.failIf(self.messages)
 287         mailer.buildFinished("builder1", b2, b2.results)
 288         self.failUnless(len(self.messages) == 1)
 289         m,r = self.messages.pop()
 290         t = m.as_string()
 291         self.failUnlessIn("To: dev3@dev.com, dev4@dev.com, "
 292                           "recip2@example.com, recip@example.com\n", t)
 293         self.failUnlessIn("From: buildbot@example.com\n", t)
 294         self.failUnlessIn("Subject: buildbot failure in builder1\n", t)
 295         self.failUnlessIn("The Buildbot has detected a new failure", t)
 296         self.failUnlessIn("BUILD FAILED: snarkleack polarization failed\n", t)
 297         self.failUnlessEqual(r, ["dev3@dev.com", "dev4@dev.com",
 298                                  "recip2@example.com", "recip@example.com"])
 299
 300     def testLogs(self):
 301         basedir = "test_status_logs"
 302         os.mkdir(basedir)
 303         mailer = MyMailer(fromaddr="buildbot@example.com", addLogs=True,
 304                           extraRecipients=["recip@example.com",
 305                                            "recip2@example.com"])
 306         mailer.parent = self
 307         mailer.status = self
 308         self.messages = []
 309
 310         b1 = self.makeBuild(3, builder.WARNINGS)
 311         b1.testlogs = [MyLog(basedir, 'compile', "Compile log here\n"),
 312                        MyLog(basedir,
 313                              'test', "Test log here\nTest 4 failed\n"),
 314                    ]
 315         b1.text = ["unusual", "gnarzzler", "output"]
 316         mailer.buildFinished("builder1", b1, b1.results)
 317         self.failUnless(len(self.messages) == 1)
 318         m,r = self.messages.pop()
 319         t = m.as_string()
 320         self.failUnlessIn("Subject: buildbot warnings in builder1\n", t)
 321         m2 = email.message_from_string(t)
 322         p = m2.get_payload()
 323         self.failUnlessEqual(len(p), 3)
 324
 325         self.failUnlessIn("Build Had Warnings: unusual gnarzzler output\n",
 326                           p[0].get_payload())
 327
 328         self.failUnlessEqual(p[1].get_filename(), "step.compile")
 329         self.failUnlessEqual(p[1].get_payload(), "Compile log here\n")
 330
 331         self.failUnlessEqual(p[2].get_filename(), "step.test")
 332         self.failUnlessIn("Test log here\n", p[2].get_payload())
 333
 334     def testMail(self):
 335         basedir = "test_status_mail"
 336         os.mkdir(basedir)
 337         dest = os.environ.get("BUILDBOT_TEST_MAIL")
 338         if not dest:
 339             raise unittest.SkipTest("define BUILDBOT_TEST_MAIL=dest to run this")
 340         mailer = mail.MailNotifier(fromaddr="buildbot@example.com",
 341                                    addLogs=True,
 342                                    extraRecipients=[dest])
 343         s = MyStatus()
 344         s.url = "project URL"
 345         mailer.status = s
 346
 347         b1 = self.makeBuild(3, builder.SUCCESS)
 348         b1.testlogs = [MyLog(basedir, 'compile', "Compile log here\n"),
 349                        MyLog(basedir,
 350                              'test', "Test log here\nTest 4 failed\n"),
 351                    ]
 352
 353         print "sending mail to", dest
 354         d = mailer.buildFinished("builder1", b1, b1.results)
 355         # When this fires, the mail has been sent, but the SMTP connection is
 356         # still up (because smtp.sendmail relies upon the server to hang up).
 357         # Spin for a moment to avoid the "unclean reactor" warning that Trial
 358         # gives us if we finish before the socket is disconnected. Really,
 359         # sendmail() ought to hang up the connection once it is finished:
 360         # otherwise a malicious SMTP server could make us consume lots of
 361         # memory.
 362         d.addCallback(self.stall, 0.1)
 363         return maybeWait(d)
 364
 365 if not mail:
 366     Mail.skip = "the Twisted Mail package is not installed"
 367
 368 class Progress(unittest.TestCase):
 369     def testWavg(self):
 370         bp = progress.BuildProgress([])
 371         e = progress.Expectations(bp)
 372         # wavg(old, current)
 373         self.failUnlessEqual(e.wavg(None, None), None)
 374         self.failUnlessEqual(e.wavg(None, 3), 3)
 375         self.failUnlessEqual(e.wavg(3, None), 3)
 376         self.failUnlessEqual(e.wavg(3, 4), 3.5)
 377         e.decay = 0.1
 378         self.failUnlessEqual(e.wavg(3, 4), 3.1)
 379
 380
 381 class Results(unittest.TestCase):
 382
 383     def testAddResults(self):
 384         b = builder.BuildStatus(builder.BuilderStatus("test"), 12)
 385         testname = ("buildbot", "test", "test_status", "Results",
 386                     "testAddResults")
 387         r1 = builder.TestResult(name=testname,
 388                                 results=builder.SUCCESS,
 389                                 text=["passed"],
 390                                 logs={'output': ""},
 391                                 )
 392         b.addTestResult(r1)
 393
 394         res = b.getTestResults()
 395         self.failUnlessEqual(res.keys(), [testname])
 396         t = res[testname]
 397         self.failUnless(providedBy(t, interfaces.ITestResult))
 398         self.failUnlessEqual(t.getName(), testname)
 399         self.failUnlessEqual(t.getResults(), builder.SUCCESS)
 400         self.failUnlessEqual(t.getText(), ["passed"])
 401         self.failUnlessEqual(t.getLogs(), {'output': ""})
 402
 403 class Log(unittest.TestCase):
 404     def setUpClass(self):
 405         self.basedir = "status_log_add"
 406         os.mkdir(self.basedir)
 407
 408     def testAdd(self):
 409         l = MyLog(self.basedir, "compile", step=13)
 410         self.failUnlessEqual(l.getName(), "compile")
 411         self.failUnlessEqual(l.getStep(), 13)
 412         l.addHeader("HEADER\n")
 413         l.addStdout("Some text\n")
 414         l.addStderr("Some error\n")
 415         l.addStdout("Some more text\n")
 416         self.failIf(l.isFinished())
 417         l.finish()
 418         self.failUnless(l.isFinished())
 419         self.failUnlessEqual(l.getText(),
 420                              "Some text\nSome error\nSome more text\n")
 421         self.failUnlessEqual(l.getTextWithHeaders(),
 422                              "HEADER\n" +
 423                              "Some text\nSome error\nSome more text\n")
 424         self.failUnlessEqual(len(list(l.getChunks())), 4)
 425
 426         self.failUnless(l.hasContents())
 427         os.unlink(l.getFilename())
 428         self.failIf(l.hasContents())
 429
 430     def TODO_testDuplicate(self):
 431         # create multiple logs for the same step with the same logname, make
 432         # sure their on-disk filenames are suitably uniquified. This
 433         # functionality actually lives in BuildStepStatus and BuildStatus, so
 434         # this test must involve more than just the MyLog class.
 435
 436         # naieve approach, doesn't work
 437         l1 = MyLog(self.basedir, "duplicate")
 438         l1.addStdout("Some text\n")
 439         l1.finish()
 440         l2 = MyLog(self.basedir, "duplicate")
 441         l2.addStdout("Some more text\n")
 442         l2.finish()
 443         self.failIfEqual(l1.getFilename(), l2.getFilename())
 444
 445     def testMerge1(self):
 446         l = MyLog(self.basedir, "merge1")
 447         l.addHeader("HEADER\n")
 448         l.addStdout("Some text\n")
 449         l.addStdout("Some more text\n")
 450         l.addStdout("more\n")
 451         l.finish()
 452         self.failUnlessEqual(l.getText(),
 453                              "Some text\nSome more text\nmore\n")
 454         self.failUnlessEqual(l.getTextWithHeaders(),
 455                              "HEADER\n" +
 456                              "Some text\nSome more text\nmore\n")
 457         self.failUnlessEqual(len(list(l.getChunks())), 2)
 458
 459     def testMerge2(self):
 460         l = MyLog(self.basedir, "merge2")
 461         l.addHeader("HEADER\n")
 462         for i in xrange(1000):
 463             l.addStdout("aaaa")
 464         for i in xrange(30):
 465             l.addStderr("bbbb")
 466         for i in xrange(10):
 467             l.addStdout("cc")
 468         target = 1000*"aaaa" + 30 * "bbbb" + 10 * "cc"
 469         self.failUnlessEqual(len(l.getText()), len(target))
 470         self.failUnlessEqual(l.getText(), target)
 471         l.finish()
 472         self.failUnlessEqual(len(l.getText()), len(target))
 473         self.failUnlessEqual(l.getText(), target)
 474         self.failUnlessEqual(len(list(l.getChunks())), 4)
 475
 476     def testMerge3(self):
 477         l = MyLog(self.basedir, "merge3")
 478         l.chunkSize = 100
 479         l.addHeader("HEADER\n")
 480         for i in xrange(8):
 481             l.addStdout(10*"a")
 482         for i in xrange(8):
 483             l.addStdout(10*"a")
 484         self.failUnlessEqual(list(l.getChunks()),
 485                              [(builder.HEADER, "HEADER\n"),
 486                               (builder.STDOUT, 100*"a"),
 487                               (builder.STDOUT, 60*"a")])
 488         l.finish()
 489         self.failUnlessEqual(l.getText(), 160*"a")
 490
 491     def testReadlines(self):
 492         l = MyLog(self.basedir, "chunks")
 493         l.addHeader("HEADER\n") # should be ignored
 494         l.addStdout("Some text\n")
 495         l.addStdout("Some More Text\nAnd Some More\n")
 496         l.addStderr("Some Stderr\n")
 497         l.addStdout("Last line\n")
 498         l.finish()
 499         alllines = list(l.readlines())
 500         self.failUnlessEqual(len(alllines), 4)
 501         self.failUnlessEqual(alllines[0], "Some text\n")
 502         self.failUnlessEqual(alllines[2], "And Some More\n")
 503         self.failUnlessEqual(alllines[3], "Last line\n")
 504         stderr = list(l.readlines(interfaces.LOG_CHANNEL_STDERR))
 505         self.failUnlessEqual(len(stderr), 1)
 506         self.failUnlessEqual(stderr[0], "Some Stderr\n")
 507         lines = l.readlines()
 508         if False: # TODO: l.readlines() is not yet an iterator
 509             # verify that it really is an iterator
 510             line0 = lines.next()
 511             self.failUnlessEqual(line0, "Some text\n")
 512             line1 = lines.next()
 513             line2 = lines.next()
 514             self.failUnlessEqual(line2, "And Some More\n")
 515
 516
 517     def testChunks(self):
 518         l = MyLog(self.basedir, "chunks")
 519         c1 = l.getChunks()
 520         l.addHeader("HEADER\n")
 521         l.addStdout("Some text\n")
 522         self.failUnlessEqual("".join(l.getChunks(onlyText=True)),
 523                              "HEADER\nSome text\n")
 524         c2 = l.getChunks()
 525
 526         l.addStdout("Some more text\n")
 527         self.failUnlessEqual("".join(l.getChunks(onlyText=True)),
 528                              "HEADER\nSome text\nSome more text\n")
 529         c3 = l.getChunks()
 530
 531         l.addStdout("more\n")
 532         l.finish()
 533
 534         self.failUnlessEqual(list(c1), [])
 535         self.failUnlessEqual(list(c2), [(builder.HEADER, "HEADER\n"),
 536                                         (builder.STDOUT, "Some text\n")])
 537         self.failUnlessEqual(list(c3), [(builder.HEADER, "HEADER\n"),
 538                                         (builder.STDOUT,
 539                                          "Some text\nSome more text\n")])
 540
 541         self.failUnlessEqual(l.getText(),
 542                              "Some text\nSome more text\nmore\n")
 543         self.failUnlessEqual(l.getTextWithHeaders(),
 544                              "HEADER\n" +
 545                              "Some text\nSome more text\nmore\n")
 546         self.failUnlessEqual(len(list(l.getChunks())), 2)
 547
 548     def testUpgrade(self):
 549         l = MyLog(self.basedir, "upgrade")
 550         l.addHeader("HEADER\n")
 551         l.addStdout("Some text\n")
 552         l.addStdout("Some more text\n")
 553         l.addStdout("more\n")
 554         l.finish()
 555         self.failUnless(l.hasContents())
 556         # now doctor it to look like a 0.6.4-era non-upgraded logfile
 557         l.entries = list(l.getChunks())
 558         del l.filename
 559         os.unlink(l.getFilename())
 560         # now make sure we can upgrade it
 561         l.upgrade("upgrade")
 562         self.failUnlessEqual(l.getText(),
 563                              "Some text\nSome more text\nmore\n")
 564         self.failUnlessEqual(len(list(l.getChunks())), 2)
 565         self.failIf(l.entries)
 566
 567         # now, do it again, but make it look like an upgraded 0.6.4 logfile
 568         # (i.e. l.filename is missing, but the contents are there on disk)
 569         l.entries = list(l.getChunks())
 570         del l.filename
 571         l.upgrade("upgrade")
 572         self.failUnlessEqual(l.getText(),
 573                              "Some text\nSome more text\nmore\n")
 574         self.failUnlessEqual(len(list(l.getChunks())), 2)
 575         self.failIf(l.entries)
 576         self.failUnless(l.hasContents())
 577
 578     def testHTMLUpgrade(self):
 579         l = MyHTMLLog(self.basedir, "upgrade", "log contents")
 580         l.upgrade("filename")
 581
 582     def testSubscribe(self):
 583         l1 = MyLog(self.basedir, "subscribe1")
 584         l1.finish()
 585         self.failUnless(l1.isFinished())
 586
 587         s = MyLogSubscriber()
 588         l1.subscribe(s, True)
 589         l1.unsubscribe(s)
 590         self.failIf(s.chunks)
 591
 592         s = MyLogSubscriber()
 593         l1.subscribe(s, False)
 594         l1.unsubscribe(s)
 595         self.failIf(s.chunks)
 596
 597         finished = []
 598         l2 = MyLog(self.basedir, "subscribe2")
 599         l2.waitUntilFinished().addCallback(finished.append)
 600         l2.addHeader("HEADER\n")
 601         s1 = MyLogSubscriber()
 602         l2.subscribe(s1, True)
 603         s2 = MyLogSubscriber()
 604         l2.subscribe(s2, False)
 605         self.failUnlessEqual(s1.chunks, [(builder.HEADER, "HEADER\n")])
 606         self.failUnlessEqual(s2.chunks, [])
 607
 608         l2.addStdout("Some text\n")
 609         self.failUnlessEqual(s1.chunks, [(builder.HEADER, "HEADER\n"),
 610                                          (builder.STDOUT, "Some text\n")])
 611         self.failUnlessEqual(s2.chunks, [(builder.STDOUT, "Some text\n")])
 612         l2.unsubscribe(s1)
 613
 614         l2.addStdout("Some more text\n")
 615         self.failUnlessEqual(s1.chunks, [(builder.HEADER, "HEADER\n"),
 616                                          (builder.STDOUT, "Some text\n")])
 617         self.failUnlessEqual(s2.chunks, [(builder.STDOUT, "Some text\n"),
 618                                          (builder.STDOUT, "Some more text\n"),
 619                                          ])
 620         self.failIf(finished)
 621         l2.finish()
 622         self.failUnlessEqual(finished, [l2])
 623
 624     def testConsumer(self):
 625         l1 = MyLog(self.basedir, "consumer1")
 626         l1.finish()
 627         self.failUnless(l1.isFinished())
 628
 629         s = MyLogConsumer()
 630         d = l1.subscribeConsumer(s)
 631         d.addCallback(self._testConsumer_1, s)
 632         return maybeWait(d, 5)
 633     def _testConsumer_1(self, res, s):
 634         self.failIf(s.chunks)
 635         self.failUnless(s.finished)
 636         self.failIf(s.producer) # producer should be registered and removed
 637
 638         l2 = MyLog(self.basedir, "consumer2")
 639         l2.addHeader("HEADER\n")
 640         l2.finish()
 641         self.failUnless(l2.isFinished())
 642
 643         s = MyLogConsumer()
 644         d = l2.subscribeConsumer(s)
 645         d.addCallback(self._testConsumer_2, s)
 646         return d
 647     def _testConsumer_2(self, res, s):
 648         self.failUnlessEqual(s.chunks, [(builder.HEADER, "HEADER\n")])
 649         self.failUnless(s.finished)
 650         self.failIf(s.producer) # producer should be registered and removed
 651
 652
 653         l2 = MyLog(self.basedir, "consumer3")
 654         l2.chunkSize = 1000
 655         l2.addHeader("HEADER\n")
 656         l2.addStdout(800*"a")
 657         l2.addStdout(800*"a") # should now have two chunks on disk, 1000+600
 658         l2.addStdout(800*"b") # HEADER,1000+600*a on disk, 800*a in memory
 659         l2.addStdout(800*"b") # HEADER,1000+600*a,1000+600*b on disk
 660         l2.addStdout(200*"c") # HEADER,1000+600*a,1000+600*b on disk,
 661                               # 200*c in memory
 662
 663         s = MyLogConsumer(limit=1)
 664         d = l2.subscribeConsumer(s)
 665         d.addCallback(self._testConsumer_3, l2, s)
 666         return d
 667     def _testConsumer_3(self, res, l2, s):
 668         self.failUnless(s.streaming)
 669         self.failUnlessEqual(s.chunks, [(builder.HEADER, "HEADER\n")])
 670         s.limit = 1
 671         d = s.producer.resumeProducing()
 672         d.addCallback(self._testConsumer_4, l2, s)
 673         return d
 674     def _testConsumer_4(self, res, l2, s):
 675         self.failUnlessEqual(s.chunks, [(builder.HEADER, "HEADER\n"),
 676                                         (builder.STDOUT, 1000*"a"),
 677                                         ])
 678         s.limit = None
 679         d = s.producer.resumeProducing()
 680         d.addCallback(self._testConsumer_5, l2, s)
 681         return d
 682     def _testConsumer_5(self, res, l2, s):
 683         self.failUnlessEqual(s.chunks, [(builder.HEADER, "HEADER\n"),
 684                                         (builder.STDOUT, 1000*"a"),
 685                                         (builder.STDOUT, 600*"a"),
 686                                         (builder.STDOUT, 1000*"b"),
 687                                         (builder.STDOUT, 600*"b"),
 688                                         (builder.STDOUT, 200*"c")])
 689         l2.addStdout(1000*"c") # HEADER,1600*a,1600*b,1200*c on disk
 690         self.failUnlessEqual(s.chunks, [(builder.HEADER, "HEADER\n"),
 691                                         (builder.STDOUT, 1000*"a"),
 692                                         (builder.STDOUT, 600*"a"),
 693                                         (builder.STDOUT, 1000*"b"),
 694                                         (builder.STDOUT, 600*"b"),
 695                                         (builder.STDOUT, 200*"c"),
 696                                         (builder.STDOUT, 1000*"c")])
 697         l2.finish()
 698         self.failUnlessEqual(s.chunks, [(builder.HEADER, "HEADER\n"),
 699                                         (builder.STDOUT, 1000*"a"),
 700                                         (builder.STDOUT, 600*"a"),
 701                                         (builder.STDOUT, 1000*"b"),
 702                                         (builder.STDOUT, 600*"b"),
 703                                         (builder.STDOUT, 200*"c"),
 704                                         (builder.STDOUT, 1000*"c")])
 705         self.failIf(s.producer)
 706         self.failUnless(s.finished)
 707
 708     def testLargeSummary(self):
 709         bigtext = "a" * 200000 # exceed the NetstringReceiver 100KB limit
 710         l = MyLog(self.basedir, "large", bigtext)
 711         s = MyLogConsumer()
 712         d = l.subscribeConsumer(s)
 713         def _check(res):
 714             for ctype,chunk in s.chunks:
 715                 self.failUnless(len(chunk) < 100000)
 716             merged = "".join([c[1] for c in s.chunks])
 717             self.failUnless(merged == bigtext)
 718         d.addCallback(_check)
 719         # when this fails, it fails with a timeout, and there is an exception
 720         # sent to log.err(). This AttributeError exception is in
 721         # NetstringReceiver.dataReceived where it does
 722         # self.transport.loseConnection() because of the NetstringParseError,
 723         # however self.transport is None
 724         return maybeWait(d, 5)
 725     testLargeSummary.timeout = 5
 726
 727 config_base = """
 728 from buildbot.process import factory
 729 from buildbot.steps import dummy
 730 s = factory.s
 731
 732 f1 = factory.QuickBuildFactory('fakerep', 'cvsmodule', configure=None)
 733
 734 f2 = factory.BuildFactory([
 735     s(dummy.Dummy, timeout=1),
 736     s(dummy.RemoteDummy, timeout=2),
 737     ])
 738
 739 BuildmasterConfig = c = {}
 740 c['bots'] = [['bot1', 'sekrit']]
 741 c['sources'] = []
 742 c['schedulers'] = []
 743 c['builders'] = []
 744 c['builders'].append({'name':'quick', 'slavename':'bot1',
 745                       'builddir': 'quickdir', 'factory': f1})
 746 c['slavePortnum'] = 0
 747 """
 748
 749 config_2 = config_base + """
 750 c['builders'] = [{'name': 'dummy', 'slavename': 'bot1',
 751                   'builddir': 'dummy1', 'factory': f2},
 752                  {'name': 'testdummy', 'slavename': 'bot1',
 753                   'builddir': 'dummy2', 'factory': f2, 'category': 'test'}]
 754 """
 755
 756 class STarget(base.StatusReceiver):
 757     debug = False
 758
 759     def __init__(self, mode):
 760         self.mode = mode
 761         self.events = []
 762     def announce(self):
 763         if self.debug:
 764             print self.events[-1]
 765
 766     def builderAdded(self, name, builder):
 767         self.events.append(("builderAdded", name, builder))
 768         self.announce()
 769         if "builder" in self.mode:
 770             return self
 771     def builderChangedState(self, name, state):
 772         self.events.append(("builderChangedState", name, state))
 773         self.announce()
 774     def buildStarted(self, name, build):
 775         self.events.append(("buildStarted", name, build))
 776         self.announce()
 777         if "eta" in self.mode:
 778             self.eta_build = build.getETA()
 779         if "build" in self.mode:
 780             return self
 781     def buildETAUpdate(self, build, ETA):
 782         self.events.append(("buildETAUpdate", build, ETA))
 783         self.announce()
 784     def stepStarted(self, build, step):
 785         self.events.append(("stepStarted", build, step))
 786         self.announce()
 787         if 0 and "eta" in self.mode:
 788             print "TIMES", step.getTimes()
 789             print "ETA", step.getETA()
 790             print "EXP", step.getExpectations()
 791         if "step" in self.mode:
 792             return self
 793     def stepETAUpdate(self, build, step, ETA, expectations):
 794         self.events.append(("stepETAUpdate", build, step, ETA, expectations))
 795         self.announce()
 796     def logStarted(self, build, step, log):
 797         self.events.append(("logStarted", build, step, log))
 798         self.announce()
 799     def logFinished(self, build, step, log):
 800         self.events.append(("logFinished", build, step, log))
 801         self.announce()
 802     def stepFinished(self, build, step, results):
 803         self.events.append(("stepFinished", build, step, results))
 804         if 0 and "eta" in self.mode:
 805             print "post-EXP", step.getExpectations()
 806         self.announce()
 807     def buildFinished(self, name, build, results):
 808         self.events.append(("buildFinished", name, build, results))
 809         self.announce()
 810     def builderRemoved(self, name):
 811         self.events.append(("builderRemoved", name))
 812         self.announce()
 813
 814 class Subscription(RunMixin, unittest.TestCase):
 815     # verify that StatusTargets can subscribe/unsubscribe properly
 816
 817     def testSlave(self):
 818         m = self.master
 819         s = m.getStatus()
 820         self.t1 = t1 = STarget(["builder"])
 821         #t1.debug = True; print
 822         s.subscribe(t1)
 823         self.failUnlessEqual(len(t1.events), 0)
 824
 825         self.t3 = t3 = STarget(["builder", "build", "step"])
 826         s.subscribe(t3)
 827
 828         m.loadConfig(config_2)
 829         m.readConfig = True
 830         m.startService()
 831
 832         self.failUnlessEqual(len(t1.events), 4)
 833         self.failUnlessEqual(t1.events[0][0:2], ("builderAdded", "dummy"))
 834         self.failUnlessEqual(t1.events[1],
 835                              ("builderChangedState", "dummy", "offline"))
 836         self.failUnlessEqual(t1.events[2][0:2], ("builderAdded", "testdummy"))
 837         self.failUnlessEqual(t1.events[3],
 838                              ("builderChangedState", "testdummy", "offline"))
 839         t1.events = []
 840
 841         self.failUnlessEqual(s.getBuilderNames(), ["dummy", "testdummy"])
 842         self.failUnlessEqual(s.getBuilderNames(categories=['test']),
 843                              ["testdummy"])
 844         self.s1 = s1 = s.getBuilder("dummy")
 845         self.failUnlessEqual(s1.getName(), "dummy")
 846         self.failUnlessEqual(s1.getState(), ("offline", []))
 847         self.failUnlessEqual(s1.getCurrentBuilds(), [])
 848         self.failUnlessEqual(s1.getLastFinishedBuild(), None)
 849         self.failUnlessEqual(s1.getBuild(-1), None)
 850         #self.failUnlessEqual(s1.getEvent(-1), foo("created"))
 851
 852         # status targets should, upon being subscribed, immediately get a
 853         # list of all current builders matching their category
 854         self.t2 = t2 = STarget([])
 855         s.subscribe(t2)
 856         self.failUnlessEqual(len(t2.events), 2)
 857         self.failUnlessEqual(t2.events[0][0:2], ("builderAdded", "dummy"))
 858         self.failUnlessEqual(t2.events[1][0:2], ("builderAdded", "testdummy"))
 859
 860         d = self.connectSlave(builders=["dummy", "testdummy"])
 861         d.addCallback(self._testSlave_1, t1)
 862         return maybeWait(d)
 863
 864     def _testSlave_1(self, res, t1):
 865         self.failUnlessEqual(len(t1.events), 2)
 866         self.failUnlessEqual(t1.events[0],
 867                              ("builderChangedState", "dummy", "idle"))
 868         self.failUnlessEqual(t1.events[1],
 869                              ("builderChangedState", "testdummy", "idle"))
 870         t1.events = []
 871
 872         c = interfaces.IControl(self.master)
 873         req = BuildRequest("forced build for testing", SourceStamp())
 874         c.getBuilder("dummy").requestBuild(req)
 875         d = req.waitUntilFinished()
 876         d2 = self.master.botmaster.waitUntilBuilderIdle("dummy")
 877         dl = defer.DeferredList([d, d2])
 878         dl.addCallback(self._testSlave_2)
 879         return dl
 880
 881     def _testSlave_2(self, res):
 882         # t1 subscribes to builds, but not anything lower-level
 883         ev = self.t1.events
 884         self.failUnlessEqual(len(ev), 4)
 885         self.failUnlessEqual(ev[0][0:3],
 886                              ("builderChangedState", "dummy", "building"))
 887         self.failUnlessEqual(ev[1][0], "buildStarted")
 888         self.failUnlessEqual(ev[2][0:2]+ev[2][3:4],
 889                              ("buildFinished", "dummy", builder.SUCCESS))
 890         self.failUnlessEqual(ev[3][0:3],
 891                              ("builderChangedState", "dummy", "idle"))
 892
 893         self.failUnlessEqual([ev[0] for ev in self.t3.events],
 894                              ["builderAdded",
 895                               "builderChangedState", # offline
 896                               "builderAdded",
 897                               "builderChangedState", # idle
 898                               "builderChangedState", # offline
 899                               "builderChangedState", # idle
 900                               "builderChangedState", # building
 901                               "buildStarted",
 902                               "stepStarted", "stepETAUpdate", "stepFinished",
 903                               "stepStarted", "stepETAUpdate",
 904                               "logStarted", "logFinished", "stepFinished",
 905                               "buildFinished",
 906                               "builderChangedState", # idle
 907                               ])
 908
 909         b = self.s1.getLastFinishedBuild()
 910         self.failUnless(b)
 911         self.failUnlessEqual(b.getBuilder().getName(), "dummy")
 912         self.failUnlessEqual(b.getNumber(), 0)
 913         self.failUnlessEqual(b.getSourceStamp(), (None, None, None))
 914         self.failUnlessEqual(b.getReason(), "forced build for testing")
 915         self.failUnlessEqual(b.getChanges(), [])
 916         self.failUnlessEqual(b.getResponsibleUsers(), [])
 917         self.failUnless(b.isFinished())
 918         self.failUnlessEqual(b.getText(), ['build', 'successful'])
 919         self.failUnlessEqual(b.getColor(), "green")
 920         self.failUnlessEqual(b.getResults(), builder.SUCCESS)
 921
 922         steps = b.getSteps()
 923         self.failUnlessEqual(len(steps), 2)
 924
 925         eta = 0
 926         st1 = steps[0]
 927         self.failUnlessEqual(st1.getName(), "dummy")
 928         self.failUnless(st1.isFinished())
 929         self.failUnlessEqual(st1.getText(), ["delay", "1 secs"])
 930         start,finish = st1.getTimes()
 931         self.failUnless(0.5 < (finish-start) < 10)
 932         self.failUnlessEqual(st1.getExpectations(), [])
 933         self.failUnlessEqual(st1.getLogs(), [])
 934         eta += finish-start
 935
 936         st2 = steps[1]
 937         self.failUnlessEqual(st2.getName(), "remote dummy")
 938         self.failUnless(st2.isFinished())
 939         self.failUnlessEqual(st2.getText(),
 940                              ["remote", "delay", "2 secs"])
 941         start,finish = st2.getTimes()
 942         self.failUnless(1.5 < (finish-start) < 10)
 943         eta += finish-start
 944         self.failUnlessEqual(st2.getExpectations(), [('output', 38, None)])
 945         logs = st2.getLogs()
 946         self.failUnlessEqual(len(logs), 1)
 947         self.failUnlessEqual(logs[0].getName(), "stdio")
 948         self.failUnlessEqual(logs[0].getText(), "data")
 949
 950         self.eta = eta
 951         # now we run it a second time, and we should have an ETA
 952
 953         self.t4 = t4 = STarget(["builder", "build", "eta"])
 954         self.master.getStatus().subscribe(t4)
 955         c = interfaces.IControl(self.master)
 956         req = BuildRequest("forced build for testing", SourceStamp())
 957         c.getBuilder("dummy").requestBuild(req)
 958         d = req.waitUntilFinished()
 959         d2 = self.master.botmaster.waitUntilBuilderIdle("dummy")
 960         dl = defer.DeferredList([d, d2])
 961         dl.addCallback(self._testSlave_3)
 962         return dl
 963
 964     def _testSlave_3(self, res):
 965         t4 = self.t4
 966         eta = self.eta
 967         self.failUnless(eta-1 < t4.eta_build < eta+1, # should be 3 seconds
 968                         "t4.eta_build was %g, not in (%g,%g)"
 969                         % (t4.eta_build, eta-1, eta+1))
 970
 971
 972 class Client(unittest.TestCase):
 973     def testAdaptation(self):
 974         b = builder.BuilderStatus("bname")
 975         b2 = client.makeRemote(b)
 976         self.failUnless(isinstance(b2, client.RemoteBuilder))
 977         b3 = client.makeRemote(None)
 978         self.failUnless(b3 is None)