buildbot/test/test_run.py

   1 # -*- test-case-name: buildbot.test.test_run -*-
   2
   3 from twisted.trial import unittest
   4 from twisted.internet import reactor, defer
   5 import os
   6
   7 from buildbot import master, interfaces
   8 from buildbot.sourcestamp import SourceStamp
   9 from buildbot.changes import changes
  10 from buildbot.status import builder
  11 from buildbot.process.base import BuildRequest
  12
  13 from buildbot.test.runutils import RunMixin, rmtree
  14
  15 config_base = """
  16 from buildbot.process import factory
  17 from buildbot.steps import dummy
  18 from buildbot.buildslave import BuildSlave
  19 s = factory.s
  20
  21 f1 = factory.QuickBuildFactory('fakerep', 'cvsmodule', configure=None)
  22
  23 f2 = factory.BuildFactory([
  24     s(dummy.Dummy, timeout=1),
  25     s(dummy.RemoteDummy, timeout=2),
  26     ])
  27
  28 BuildmasterConfig = c = {}
  29 c['slaves'] = [BuildSlave('bot1', 'sekrit')]
  30 c['schedulers'] = []
  31 c['builders'] = []
  32 c['builders'].append({'name':'quick', 'slavename':'bot1',
  33                       'builddir': 'quickdir', 'factory': f1})
  34 c['slavePortnum'] = 0
  35 """
  36
  37 config_run = config_base + """
  38 from buildbot.scheduler import Scheduler
  39 c['schedulers'] = [Scheduler('quick', None, 120, ['quick'])]
  40 """
  41
  42 config_can_build = config_base + """
  43 from buildbot.buildslave import BuildSlave
  44 c['slaves'] = [ BuildSlave('bot1', 'sekrit') ]
  45
  46 from buildbot.scheduler import Scheduler
  47 c['schedulers'] = [Scheduler('dummy', None, 0.1, ['dummy'])]
  48
  49 c['builders'] = [{'name': 'dummy', 'slavename': 'bot1',
  50                   'builddir': 'dummy1', 'factory': f2}]
  51 """
  52
  53 config_cant_build = config_can_build + """
  54 class MyBuildSlave(BuildSlave):
  55     def canStartBuild(self): return False
  56 c['slaves'] = [ MyBuildSlave('bot1', 'sekrit') ]
  57 """
  58
  59 config_concurrency = config_base + """
  60 from buildbot.buildslave import BuildSlave
  61 c['slaves'] = [ BuildSlave('bot1', 'sekrit', max_builds=1) ]
  62
  63 from buildbot.scheduler import Scheduler
  64 c['schedulers'] = [Scheduler('dummy', None, 0.1, ['dummy', 'dummy2'])]
  65
  66 c['builders'].append({'name': 'dummy', 'slavename': 'bot1',
  67                       'builddir': 'dummy', 'factory': f2})
  68 c['builders'].append({'name': 'dummy2', 'slavename': 'bot1',
  69                       'builddir': 'dummy2', 'factory': f2})
  70 """
  71
  72 config_2 = config_base + """
  73 c['builders'] = [{'name': 'dummy', 'slavename': 'bot1',
  74                   'builddir': 'dummy1', 'factory': f2},
  75                  {'name': 'testdummy', 'slavename': 'bot1',
  76                   'builddir': 'dummy2', 'factory': f2, 'category': 'test'}]
  77 """
  78
  79 config_3 = config_2 + """
  80 c['builders'].append({'name': 'adummy', 'slavename': 'bot1',
  81                       'builddir': 'adummy3', 'factory': f2})
  82 c['builders'].append({'name': 'bdummy', 'slavename': 'bot1',
  83                       'builddir': 'adummy4', 'factory': f2,
  84                       'category': 'test'})
  85 """
  86
  87 config_4 = config_base + """
  88 c['builders'] = [{'name': 'dummy', 'slavename': 'bot1',
  89                   'builddir': 'dummy', 'factory': f2}]
  90 """
  91
  92 config_4_newbasedir = config_4 + """
  93 c['builders'] = [{'name': 'dummy', 'slavename': 'bot1',
  94                   'builddir': 'dummy2', 'factory': f2}]
  95 """
  96
  97 config_4_newbuilder = config_4_newbasedir + """
  98 c['builders'].append({'name': 'dummy2', 'slavename': 'bot1',
  99                       'builddir': 'dummy23', 'factory': f2})
 100 """
 101
 102 class Run(unittest.TestCase):
 103     def rmtree(self, d):
 104         rmtree(d)
 105
 106     def testMaster(self):
 107         self.rmtree("basedir")
 108         os.mkdir("basedir")
 109         m = master.BuildMaster("basedir")
 110         m.loadConfig(config_run)
 111         m.readConfig = True
 112         m.startService()
 113         cm = m.change_svc
 114         c = changes.Change("bob", ["Makefile", "foo/bar.c"], "changed stuff")
 115         cm.addChange(c)
 116         # verify that the Scheduler is now waiting
 117         s = m.allSchedulers()[0]
 118         self.failUnless(s.timer)
 119         # halting the service will also stop the timer
 120         d = defer.maybeDeferred(m.stopService)
 121         return d
 122
 123 class CanStartBuild(RunMixin, unittest.TestCase):
 124     def rmtree(self, d):
 125         rmtree(d)
 126
 127     def testCanStartBuild(self):
 128         return self.do_test(config_can_build, True)
 129
 130     def testCantStartBuild(self):
 131         return self.do_test(config_cant_build, False)
 132
 133     def do_test(self, config, builder_should_run):
 134         self.master.loadConfig(config)
 135         self.master.readConfig = True
 136         self.master.startService()
 137         d = self.connectSlave()
 138
 139         # send a change
 140         cm = self.master.change_svc
 141         c = changes.Change("bob", ["Makefile", "foo/bar.c"], "changed stuff")
 142         cm.addChange(c)
 143
 144         d.addCallback(self._do_test1, builder_should_run)
 145
 146         return d
 147
 148     def _do_test1(self, res, builder_should_run):
 149         # delay a little bit. Note that relying upon timers is a bit fragile,
 150         # in this case we're hoping that our 0.5 second timer will land us
 151         # somewhere in the middle of the [0.1s, 3.1s] window (after the 0.1
 152         # second Scheduler fires, then during the 3-second build), so that
 153         # when we sample BuildSlave.state, we'll see BUILDING (or IDLE if the
 154         # slave was told to be unavailable). On a heavily loaded system, our
 155         # 0.5 second timer might not actually fire until after the build has
 156         # completed. In the long run, it would be good to change this test to
 157         # pass under those circumstances too.
 158         d = defer.Deferred()
 159         reactor.callLater(.5, d.callback, builder_should_run)
 160         d.addCallback(self._do_test2)
 161         return d
 162
 163     def _do_test2(self, builder_should_run):
 164         b = self.master.botmaster.builders['dummy']
 165         self.failUnless(len(b.slaves) == 1)
 166
 167         bs = b.slaves[0]
 168         from buildbot.process.builder import IDLE, BUILDING
 169         if builder_should_run:
 170             self.failUnlessEqual(bs.state, BUILDING)
 171         else:
 172             self.failUnlessEqual(bs.state, IDLE)
 173
 174
 175 class ConcurrencyLimit(RunMixin, unittest.TestCase):
 176
 177     def testConcurrencyLimit(self):
 178         d = self.master.loadConfig(config_concurrency)
 179         d.addCallback(lambda res: self.master.startService())
 180         d.addCallback(lambda res: self.connectSlave())
 181
 182         def _send(res):
 183             # send a change. This will trigger both builders at the same
 184             # time, but since they share a slave, the max_builds=1 setting
 185             # will insure that only one of the two builds gets to run.
 186             cm = self.master.change_svc
 187             c = changes.Change("bob", ["Makefile", "foo/bar.c"],
 188                                "changed stuff")
 189             cm.addChange(c)
 190         d.addCallback(_send)
 191
 192         def _delay(res):
 193             d1 = defer.Deferred()
 194             reactor.callLater(1, d1.callback, None)
 195             # this test depends upon this 1s delay landing us in the middle
 196             # of one of the builds.
 197             return d1
 198         d.addCallback(_delay)
 199
 200         def _check(res):
 201             builders = [ self.master.botmaster.builders[bn]
 202                          for bn in ('dummy', 'dummy2') ]
 203             for builder in builders:
 204                 self.failUnless(len(builder.slaves) == 1)
 205
 206             from buildbot.process.builder import IDLE, BUILDING
 207             building_bs = [ builder
 208                             for builder in builders
 209                             if builder.slaves[0].state == BUILDING ]
 210             # assert that only one build is running right now. If the
 211             # max_builds= weren't in effect, this would be 2.
 212             self.failUnlessEqual(len(building_bs), 1)
 213         d.addCallback(_check)
 214
 215         return d
 216
 217
 218 class Ping(RunMixin, unittest.TestCase):
 219     def testPing(self):
 220         self.master.loadConfig(config_2)
 221         self.master.readConfig = True
 222         self.master.startService()
 223
 224         d = self.connectSlave()
 225         d.addCallback(self._testPing_1)
 226         return d
 227
 228     def _testPing_1(self, res):
 229         d = interfaces.IControl(self.master).getBuilder("dummy").ping(1)
 230         d.addCallback(self._testPing_2)
 231         return d
 232
 233     def _testPing_2(self, res):
 234         pass
 235
 236 class BuilderNames(unittest.TestCase):
 237
 238     def testGetBuilderNames(self):
 239         os.mkdir("bnames")
 240         m = master.BuildMaster("bnames")
 241         s = m.getStatus()
 242
 243         m.loadConfig(config_3)
 244         m.readConfig = True
 245
 246         self.failUnlessEqual(s.getBuilderNames(),
 247                              ["dummy", "testdummy", "adummy", "bdummy"])
 248         self.failUnlessEqual(s.getBuilderNames(categories=['test']),
 249                              ["testdummy", "bdummy"])
 250
 251 class Disconnect(RunMixin, unittest.TestCase):
 252
 253     def setUp(self):
 254         RunMixin.setUp(self)
 255
 256         # verify that disconnecting the slave during a build properly
 257         # terminates the build
 258         m = self.master
 259         s = self.status
 260         c = self.control
 261
 262         m.loadConfig(config_2)
 263         m.readConfig = True
 264         m.startService()
 265
 266         self.failUnlessEqual(s.getBuilderNames(), ["dummy", "testdummy"])
 267         self.s1 = s1 = s.getBuilder("dummy")
 268         self.failUnlessEqual(s1.getName(), "dummy")
 269         self.failUnlessEqual(s1.getState(), ("offline", []))
 270         self.failUnlessEqual(s1.getCurrentBuilds(), [])
 271         self.failUnlessEqual(s1.getLastFinishedBuild(), None)
 272         self.failUnlessEqual(s1.getBuild(-1), None)
 273
 274         d = self.connectSlave()
 275         d.addCallback(self._disconnectSetup_1)
 276         return d
 277
 278     def _disconnectSetup_1(self, res):
 279         self.failUnlessEqual(self.s1.getState(), ("idle", []))
 280
 281
 282     def verifyDisconnect(self, bs):
 283         self.failUnless(bs.isFinished())
 284
 285         step1 = bs.getSteps()[0]
 286         self.failUnlessEqual(step1.getText(), ["delay", "interrupted"])
 287         self.failUnlessEqual(step1.getResults()[0], builder.FAILURE)
 288
 289         self.failUnlessEqual(bs.getResults(), builder.FAILURE)
 290
 291     def verifyDisconnect2(self, bs):
 292         self.failUnless(bs.isFinished())
 293
 294         step1 = bs.getSteps()[1]
 295         self.failUnlessEqual(step1.getText(), ["remote", "delay", "2 secs",
 296                                                "failed", "slave", "lost"])
 297         self.failUnlessEqual(step1.getResults()[0], builder.FAILURE)
 298
 299         self.failUnlessEqual(bs.getResults(), builder.FAILURE)
 300
 301     def submitBuild(self):
 302         ss = SourceStamp()
 303         br = BuildRequest("forced build", ss, "dummy")
 304         self.control.getBuilder("dummy").requestBuild(br)
 305         d = defer.Deferred()
 306         def _started(bc):
 307             br.unsubscribe(_started)
 308             d.callback(bc)
 309         br.subscribe(_started)
 310         return d
 311
 312     def testIdle2(self):
 313         # now suppose the slave goes missing
 314         self.disappearSlave(allowReconnect=False)
 315
 316         # forcing a build will work: the build detect that the slave is no
 317         # longer available and will be re-queued. Wait 5 seconds, then check
 318         # to make sure the build is still in the 'waiting for a slave' queue.
 319         self.control.getBuilder("dummy").original.START_BUILD_TIMEOUT = 1
 320         req = BuildRequest("forced build", SourceStamp())
 321         self.failUnlessEqual(req.startCount, 0)
 322         self.control.getBuilder("dummy").requestBuild(req)
 323         # this should ping the slave, which doesn't respond, and then give up
 324         # after a second. The BuildRequest will be re-queued, and its
 325         # .startCount will be incremented.
 326         d = defer.Deferred()
 327         d.addCallback(self._testIdle2_1, req)
 328         reactor.callLater(3, d.callback, None)
 329         return d
 330     testIdle2.timeout = 5
 331
 332     def _testIdle2_1(self, res, req):
 333         self.failUnlessEqual(req.startCount, 1)
 334         cancelled = req.cancel()
 335         self.failUnless(cancelled)
 336
 337
 338     def testBuild1(self):
 339         # this next sequence is timing-dependent. The dummy build takes at
 340         # least 3 seconds to complete, and this batch of commands must
 341         # complete within that time.
 342         #
 343         d = self.submitBuild()
 344         d.addCallback(self._testBuild1_1)
 345         return d
 346
 347     def _testBuild1_1(self, bc):
 348         bs = bc.getStatus()
 349         # now kill the slave before it gets to start the first step
 350         d = self.shutdownAllSlaves() # dies before it gets started
 351         d.addCallback(self._testBuild1_2, bs)
 352         return d  # TODO: this used to have a 5-second timeout
 353
 354     def _testBuild1_2(self, res, bs):
 355         # now examine the just-stopped build and make sure it is really
 356         # stopped. This is checking for bugs in which the slave-detach gets
 357         # missed or causes an exception which prevents the build from being
 358         # marked as "finished due to an error".
 359         d = bs.waitUntilFinished()
 360         d2 = self.master.botmaster.waitUntilBuilderDetached("dummy")
 361         dl = defer.DeferredList([d, d2])
 362         dl.addCallback(self._testBuild1_3, bs)
 363         return dl # TODO: this had a 5-second timeout too
 364
 365     def _testBuild1_3(self, res, bs):
 366         self.failUnlessEqual(self.s1.getState()[0], "offline")
 367         self.verifyDisconnect(bs)
 368
 369
 370     def testBuild2(self):
 371         # this next sequence is timing-dependent
 372         d = self.submitBuild()
 373         d.addCallback(self._testBuild1_1)
 374         return d
 375     testBuild2.timeout = 30
 376
 377     def _testBuild1_1(self, bc):
 378         bs = bc.getStatus()
 379         # shutdown the slave while it's running the first step
 380         reactor.callLater(0.5, self.shutdownAllSlaves)
 381
 382         d = bs.waitUntilFinished()
 383         d.addCallback(self._testBuild2_2, bs)
 384         return d
 385
 386     def _testBuild2_2(self, res, bs):
 387         # we hit here when the build has finished. The builder is still being
 388         # torn down, however, so spin for another second to allow the
 389         # callLater(0) in Builder.detached to fire.
 390         d = defer.Deferred()
 391         reactor.callLater(1, d.callback, None)
 392         d.addCallback(self._testBuild2_3, bs)
 393         return d
 394
 395     def _testBuild2_3(self, res, bs):
 396         self.failUnlessEqual(self.s1.getState()[0], "offline")
 397         self.verifyDisconnect(bs)
 398
 399
 400     def testBuild3(self):
 401         # this next sequence is timing-dependent
 402         d = self.submitBuild()
 403         d.addCallback(self._testBuild3_1)
 404         return d
 405     testBuild3.timeout = 30
 406
 407     def _testBuild3_1(self, bc):
 408         bs = bc.getStatus()
 409         # kill the slave while it's running the first step
 410         reactor.callLater(0.5, self.killSlave)
 411         d = bs.waitUntilFinished()
 412         d.addCallback(self._testBuild3_2, bs)
 413         return d
 414
 415     def _testBuild3_2(self, res, bs):
 416         # the builder is still being torn down, so give it another second
 417         d = defer.Deferred()
 418         reactor.callLater(1, d.callback, None)
 419         d.addCallback(self._testBuild3_3, bs)
 420         return d
 421
 422     def _testBuild3_3(self, res, bs):
 423         self.failUnlessEqual(self.s1.getState()[0], "offline")
 424         self.verifyDisconnect(bs)
 425
 426
 427     def testBuild4(self):
 428         # this next sequence is timing-dependent
 429         d = self.submitBuild()
 430         d.addCallback(self._testBuild4_1)
 431         return d
 432     testBuild4.timeout = 30
 433
 434     def _testBuild4_1(self, bc):
 435         bs = bc.getStatus()
 436         # kill the slave while it's running the second (remote) step
 437         reactor.callLater(1.5, self.killSlave)
 438         d = bs.waitUntilFinished()
 439         d.addCallback(self._testBuild4_2, bs)
 440         return d
 441
 442     def _testBuild4_2(self, res, bs):
 443         # at this point, the slave is in the process of being removed, so it
 444         # could either be 'idle' or 'offline'. I think there is a
 445         # reactor.callLater(0) standing between here and the offline state.
 446         #reactor.iterate() # TODO: remove the need for this
 447
 448         self.failUnlessEqual(self.s1.getState()[0], "offline")
 449         self.verifyDisconnect2(bs)
 450
 451
 452     def testInterrupt(self):
 453         # this next sequence is timing-dependent
 454         d = self.submitBuild()
 455         d.addCallback(self._testInterrupt_1)
 456         return d
 457     testInterrupt.timeout = 30
 458
 459     def _testInterrupt_1(self, bc):
 460         bs = bc.getStatus()
 461         # halt the build while it's running the first step
 462         reactor.callLater(0.5, bc.stopBuild, "bang go splat")
 463         d = bs.waitUntilFinished()
 464         d.addCallback(self._testInterrupt_2, bs)
 465         return d
 466
 467     def _testInterrupt_2(self, res, bs):
 468         self.verifyDisconnect(bs)
 469
 470
 471     def testDisappear(self):
 472         bc = self.control.getBuilder("dummy")
 473
 474         # ping should succeed
 475         d = bc.ping(1)
 476         d.addCallback(self._testDisappear_1, bc)
 477         return d
 478
 479     def _testDisappear_1(self, res, bc):
 480         self.failUnlessEqual(res, True)
 481
 482         # now, before any build is run, make the slave disappear
 483         self.disappearSlave(allowReconnect=False)
 484
 485         # at this point, a ping to the slave should timeout
 486         d = bc.ping(1)
 487         d.addCallback(self. _testDisappear_2)
 488         return d
 489     def _testDisappear_2(self, res):
 490         self.failUnlessEqual(res, False)
 491
 492     def testDuplicate(self):
 493         bc = self.control.getBuilder("dummy")
 494         bs = self.status.getBuilder("dummy")
 495         ss = bs.getSlaves()[0]
 496
 497         self.failUnless(ss.isConnected())
 498         self.failUnlessEqual(ss.getAdmin(), "one")
 499
 500         # now, before any build is run, make the first slave disappear
 501         self.disappearSlave(allowReconnect=False)
 502
 503         d = self.master.botmaster.waitUntilBuilderDetached("dummy")
 504         # now let the new slave take over
 505         self.connectSlave2()
 506         d.addCallback(self._testDuplicate_1, ss)
 507         return d
 508     testDuplicate.timeout = 5
 509
 510     def _testDuplicate_1(self, res, ss):
 511         d = self.master.botmaster.waitUntilBuilderAttached("dummy")
 512         d.addCallback(self._testDuplicate_2, ss)
 513         return d
 514
 515     def _testDuplicate_2(self, res, ss):
 516         self.failUnless(ss.isConnected())
 517         self.failUnlessEqual(ss.getAdmin(), "two")
 518
 519
 520 class Disconnect2(RunMixin, unittest.TestCase):
 521
 522     def setUp(self):
 523         RunMixin.setUp(self)
 524         # verify that disconnecting the slave during a build properly
 525         # terminates the build
 526         m = self.master
 527         s = self.status
 528         c = self.control
 529
 530         m.loadConfig(config_2)
 531         m.readConfig = True
 532         m.startService()
 533
 534         self.failUnlessEqual(s.getBuilderNames(), ["dummy", "testdummy"])
 535         self.s1 = s1 = s.getBuilder("dummy")
 536         self.failUnlessEqual(s1.getName(), "dummy")
 537         self.failUnlessEqual(s1.getState(), ("offline", []))
 538         self.failUnlessEqual(s1.getCurrentBuilds(), [])
 539         self.failUnlessEqual(s1.getLastFinishedBuild(), None)
 540         self.failUnlessEqual(s1.getBuild(-1), None)
 541
 542         d = self.connectSlaveFastTimeout()
 543         d.addCallback(self._setup_disconnect2_1)
 544         return d
 545
 546     def _setup_disconnect2_1(self, res):
 547         self.failUnlessEqual(self.s1.getState(), ("idle", []))
 548
 549
 550     def testSlaveTimeout(self):
 551         # now suppose the slave goes missing. We want to find out when it
 552         # creates a new Broker, so we reach inside and mark it with the
 553         # well-known sigil of impending messy death.
 554         bd = self.slaves['bot1'].getServiceNamed("bot").builders["dummy"]
 555         broker = bd.remote.broker
 556         broker.redshirt = 1
 557
 558         # make sure the keepalives will keep the connection up
 559         d = defer.Deferred()
 560         reactor.callLater(5, d.callback, None)
 561         d.addCallback(self._testSlaveTimeout_1)
 562         return d
 563     testSlaveTimeout.timeout = 20
 564
 565     def _testSlaveTimeout_1(self, res):
 566         bd = self.slaves['bot1'].getServiceNamed("bot").builders["dummy"]
 567         if not bd.remote or not hasattr(bd.remote.broker, "redshirt"):
 568             self.fail("slave disconnected when it shouldn't have")
 569
 570         d = self.master.botmaster.waitUntilBuilderDetached("dummy")
 571         # whoops! how careless of me.
 572         self.disappearSlave(allowReconnect=True)
 573         # the slave will realize the connection is lost within 2 seconds, and
 574         # reconnect.
 575         d.addCallback(self._testSlaveTimeout_2)
 576         return d
 577
 578     def _testSlaveTimeout_2(self, res):
 579         # the ReconnectingPBClientFactory will attempt a reconnect in two
 580         # seconds.
 581         d = self.master.botmaster.waitUntilBuilderAttached("dummy")
 582         d.addCallback(self._testSlaveTimeout_3)
 583         return d
 584
 585     def _testSlaveTimeout_3(self, res):
 586         # make sure it is a new connection (i.e. a new Broker)
 587         bd = self.slaves['bot1'].getServiceNamed("bot").builders["dummy"]
 588         self.failUnless(bd.remote, "hey, slave isn't really connected")
 589         self.failIf(hasattr(bd.remote.broker, "redshirt"),
 590                     "hey, slave's Broker is still marked for death")
 591
 592
 593 class Basedir(RunMixin, unittest.TestCase):
 594     def testChangeBuilddir(self):
 595         m = self.master
 596         m.loadConfig(config_4)
 597         m.readConfig = True
 598         m.startService()
 599
 600         d = self.connectSlave()
 601         d.addCallback(self._testChangeBuilddir_1)
 602         return d
 603
 604     def _testChangeBuilddir_1(self, res):
 605         self.bot = bot = self.slaves['bot1'].bot
 606         self.builder = builder = bot.builders.get("dummy")
 607         self.failUnless(builder)
 608         self.failUnlessEqual(builder.builddir, "dummy")
 609         self.failUnlessEqual(builder.basedir,
 610                              os.path.join("slavebase-bot1", "dummy"))
 611
 612         d = self.master.loadConfig(config_4_newbasedir)
 613         d.addCallback(self._testChangeBuilddir_2)
 614         return d
 615
 616     def _testChangeBuilddir_2(self, res):
 617         bot = self.bot
 618         # this does NOT cause the builder to be replaced
 619         builder = bot.builders.get("dummy")
 620         self.failUnless(builder)
 621         self.failUnlessIdentical(self.builder, builder)
 622         # the basedir should be updated
 623         self.failUnlessEqual(builder.builddir, "dummy2")
 624         self.failUnlessEqual(builder.basedir,
 625                              os.path.join("slavebase-bot1", "dummy2"))
 626
 627         # add a new builder, which causes the basedir list to be reloaded
 628         d = self.master.loadConfig(config_4_newbuilder)
 629         return d
 630
 631 # TODO: test everything, from Change submission to Scheduler to Build to
 632 # Status. Use all the status types. Specifically I want to catch recurrences
 633 # of the bug where I forgot to make Waterfall inherit from StatusReceiver
 634 # such that buildSetSubmitted failed.
 635