buildbot/test/test_run.py

   1 # -*- test-case-name: buildbot.test.test_run -*-
   2
   3 from twisted.trial import unittest
   4 from twisted.internet import reactor, defer
   5 import os
   6
   7 from buildbot import master, interfaces
   8 from buildbot.sourcestamp import SourceStamp
   9 from buildbot.changes import changes
  10 from buildbot.status import builder
  11 from buildbot.process.base import BuildRequest
  12
  13 from buildbot.test.runutils import RunMixin, rmtree
  14
  15 config_base = """
  16 from buildbot.process import factory
  17 from buildbot.steps import dummy
  18 from buildbot.slave import BuildSlave
  19 s = factory.s
  20
  21 f1 = factory.QuickBuildFactory('fakerep', 'cvsmodule', configure=None)
  22
  23 f2 = factory.BuildFactory([
  24     s(dummy.Dummy, timeout=1),
  25     s(dummy.RemoteDummy, timeout=2),
  26     ])
  27
  28 BuildmasterConfig = c = {}
  29 c['slaves'] = [BuildSlave('bot1', 'sekrit')]
  30 c['sources'] = []
  31 c['schedulers'] = []
  32 c['builders'] = []
  33 c['builders'].append({'name':'quick', 'slavename':'bot1',
  34                       'builddir': 'quickdir', 'factory': f1})
  35 c['slavePortnum'] = 0
  36 """
  37
  38 config_run = config_base + """
  39 from buildbot.scheduler import Scheduler
  40 c['schedulers'] = [Scheduler('quick', None, 120, ['quick'])]
  41 """
  42
  43 config_2 = config_base + """
  44 c['builders'] = [{'name': 'dummy', 'slavename': 'bot1',
  45                   'builddir': 'dummy1', 'factory': f2},
  46                  {'name': 'testdummy', 'slavename': 'bot1',
  47                   'builddir': 'dummy2', 'factory': f2, 'category': 'test'}]
  48 """
  49
  50 config_3 = config_2 + """
  51 c['builders'].append({'name': 'adummy', 'slavename': 'bot1',
  52                       'builddir': 'adummy3', 'factory': f2})
  53 c['builders'].append({'name': 'bdummy', 'slavename': 'bot1',
  54                       'builddir': 'adummy4', 'factory': f2,
  55                       'category': 'test'})
  56 """
  57
  58 config_4 = config_base + """
  59 c['builders'] = [{'name': 'dummy', 'slavename': 'bot1',
  60                   'builddir': 'dummy', 'factory': f2}]
  61 """
  62
  63 config_4_newbasedir = config_4 + """
  64 c['builders'] = [{'name': 'dummy', 'slavename': 'bot1',
  65                   'builddir': 'dummy2', 'factory': f2}]
  66 """
  67
  68 config_4_newbuilder = config_4_newbasedir + """
  69 c['builders'].append({'name': 'dummy2', 'slavename': 'bot1',
  70                       'builddir': 'dummy23', 'factory': f2})
  71 """
  72
  73 class Run(unittest.TestCase):
  74     def rmtree(self, d):
  75         rmtree(d)
  76
  77     def testMaster(self):
  78         self.rmtree("basedir")
  79         os.mkdir("basedir")
  80         m = master.BuildMaster("basedir")
  81         m.loadConfig(config_run)
  82         m.readConfig = True
  83         m.startService()
  84         cm = m.change_svc
  85         c = changes.Change("bob", ["Makefile", "foo/bar.c"], "changed stuff")
  86         cm.addChange(c)
  87         # verify that the Scheduler is now waiting
  88         s = m.allSchedulers()[0]
  89         self.failUnless(s.timer)
  90         # halting the service will also stop the timer
  91         d = defer.maybeDeferred(m.stopService)
  92         return d
  93
  94 class Ping(RunMixin, unittest.TestCase):
  95     def testPing(self):
  96         self.master.loadConfig(config_2)
  97         self.master.readConfig = True
  98         self.master.startService()
  99
 100         d = self.connectSlave()
 101         d.addCallback(self._testPing_1)
 102         return d
 103
 104     def _testPing_1(self, res):
 105         d = interfaces.IControl(self.master).getBuilder("dummy").ping(1)
 106         d.addCallback(self._testPing_2)
 107         return d
 108
 109     def _testPing_2(self, res):
 110         pass
 111
 112 class BuilderNames(unittest.TestCase):
 113
 114     def testGetBuilderNames(self):
 115         os.mkdir("bnames")
 116         m = master.BuildMaster("bnames")
 117         s = m.getStatus()
 118
 119         m.loadConfig(config_3)
 120         m.readConfig = True
 121
 122         self.failUnlessEqual(s.getBuilderNames(),
 123                              ["dummy", "testdummy", "adummy", "bdummy"])
 124         self.failUnlessEqual(s.getBuilderNames(categories=['test']),
 125                              ["testdummy", "bdummy"])
 126
 127 class Disconnect(RunMixin, unittest.TestCase):
 128
 129     def setUp(self):
 130         RunMixin.setUp(self)
 131
 132         # verify that disconnecting the slave during a build properly
 133         # terminates the build
 134         m = self.master
 135         s = self.status
 136         c = self.control
 137
 138         m.loadConfig(config_2)
 139         m.readConfig = True
 140         m.startService()
 141
 142         self.failUnlessEqual(s.getBuilderNames(), ["dummy", "testdummy"])
 143         self.s1 = s1 = s.getBuilder("dummy")
 144         self.failUnlessEqual(s1.getName(), "dummy")
 145         self.failUnlessEqual(s1.getState(), ("offline", []))
 146         self.failUnlessEqual(s1.getCurrentBuilds(), [])
 147         self.failUnlessEqual(s1.getLastFinishedBuild(), None)
 148         self.failUnlessEqual(s1.getBuild(-1), None)
 149
 150         d = self.connectSlave()
 151         d.addCallback(self._disconnectSetup_1)
 152         return d
 153
 154     def _disconnectSetup_1(self, res):
 155         self.failUnlessEqual(self.s1.getState(), ("idle", []))
 156
 157
 158     def verifyDisconnect(self, bs):
 159         self.failUnless(bs.isFinished())
 160
 161         step1 = bs.getSteps()[0]
 162         self.failUnlessEqual(step1.getText(), ["delay", "interrupted"])
 163         self.failUnlessEqual(step1.getResults()[0], builder.FAILURE)
 164
 165         self.failUnlessEqual(bs.getResults(), builder.FAILURE)
 166
 167     def verifyDisconnect2(self, bs):
 168         self.failUnless(bs.isFinished())
 169
 170         step1 = bs.getSteps()[1]
 171         self.failUnlessEqual(step1.getText(), ["remote", "delay", "2 secs",
 172                                                "failed", "slave", "lost"])
 173         self.failUnlessEqual(step1.getResults()[0], builder.FAILURE)
 174
 175         self.failUnlessEqual(bs.getResults(), builder.FAILURE)
 176
 177     def submitBuild(self):
 178         ss = SourceStamp()
 179         br = BuildRequest("forced build", ss, "dummy")
 180         self.control.getBuilder("dummy").requestBuild(br)
 181         d = defer.Deferred()
 182         def _started(bc):
 183             br.unsubscribe(_started)
 184             d.callback(bc)
 185         br.subscribe(_started)
 186         return d
 187
 188     def testIdle2(self):
 189         # now suppose the slave goes missing
 190         self.disappearSlave(allowReconnect=False)
 191
 192         # forcing a build will work: the build detect that the slave is no
 193         # longer available and will be re-queued. Wait 5 seconds, then check
 194         # to make sure the build is still in the 'waiting for a slave' queue.
 195         self.control.getBuilder("dummy").original.START_BUILD_TIMEOUT = 1
 196         req = BuildRequest("forced build", SourceStamp())
 197         self.failUnlessEqual(req.startCount, 0)
 198         self.control.getBuilder("dummy").requestBuild(req)
 199         # this should ping the slave, which doesn't respond, and then give up
 200         # after a second. The BuildRequest will be re-queued, and its
 201         # .startCount will be incremented.
 202         d = defer.Deferred()
 203         d.addCallback(self._testIdle2_1, req)
 204         reactor.callLater(3, d.callback, None)
 205         return d
 206     testIdle2.timeout = 5
 207
 208     def _testIdle2_1(self, res, req):
 209         self.failUnlessEqual(req.startCount, 1)
 210         cancelled = req.cancel()
 211         self.failUnless(cancelled)
 212
 213
 214     def testBuild1(self):
 215         # this next sequence is timing-dependent. The dummy build takes at
 216         # least 3 seconds to complete, and this batch of commands must
 217         # complete within that time.
 218         #
 219         d = self.submitBuild()
 220         d.addCallback(self._testBuild1_1)
 221         return d
 222
 223     def _testBuild1_1(self, bc):
 224         bs = bc.getStatus()
 225         # now kill the slave before it gets to start the first step
 226         d = self.shutdownAllSlaves() # dies before it gets started
 227         d.addCallback(self._testBuild1_2, bs)
 228         return d  # TODO: this used to have a 5-second timeout
 229
 230     def _testBuild1_2(self, res, bs):
 231         # now examine the just-stopped build and make sure it is really
 232         # stopped. This is checking for bugs in which the slave-detach gets
 233         # missed or causes an exception which prevents the build from being
 234         # marked as "finished due to an error".
 235         d = bs.waitUntilFinished()
 236         d2 = self.master.botmaster.waitUntilBuilderDetached("dummy")
 237         dl = defer.DeferredList([d, d2])
 238         dl.addCallback(self._testBuild1_3, bs)
 239         return dl # TODO: this had a 5-second timeout too
 240
 241     def _testBuild1_3(self, res, bs):
 242         self.failUnlessEqual(self.s1.getState()[0], "offline")
 243         self.verifyDisconnect(bs)
 244
 245
 246     def testBuild2(self):
 247         # this next sequence is timing-dependent
 248         d = self.submitBuild()
 249         d.addCallback(self._testBuild1_1)
 250         return d
 251     testBuild2.timeout = 30
 252
 253     def _testBuild1_1(self, bc):
 254         bs = bc.getStatus()
 255         # shutdown the slave while it's running the first step
 256         reactor.callLater(0.5, self.shutdownAllSlaves)
 257
 258         d = bs.waitUntilFinished()
 259         d.addCallback(self._testBuild2_2, bs)
 260         return d
 261
 262     def _testBuild2_2(self, res, bs):
 263         # we hit here when the build has finished. The builder is still being
 264         # torn down, however, so spin for another second to allow the
 265         # callLater(0) in Builder.detached to fire.
 266         d = defer.Deferred()
 267         reactor.callLater(1, d.callback, None)
 268         d.addCallback(self._testBuild2_3, bs)
 269         return d
 270
 271     def _testBuild2_3(self, res, bs):
 272         self.failUnlessEqual(self.s1.getState()[0], "offline")
 273         self.verifyDisconnect(bs)
 274
 275
 276     def testBuild3(self):
 277         # this next sequence is timing-dependent
 278         d = self.submitBuild()
 279         d.addCallback(self._testBuild3_1)
 280         return d
 281     testBuild3.timeout = 30
 282
 283     def _testBuild3_1(self, bc):
 284         bs = bc.getStatus()
 285         # kill the slave while it's running the first step
 286         reactor.callLater(0.5, self.killSlave)
 287         d = bs.waitUntilFinished()
 288         d.addCallback(self._testBuild3_2, bs)
 289         return d
 290
 291     def _testBuild3_2(self, res, bs):
 292         # the builder is still being torn down, so give it another second
 293         d = defer.Deferred()
 294         reactor.callLater(1, d.callback, None)
 295         d.addCallback(self._testBuild3_3, bs)
 296         return d
 297
 298     def _testBuild3_3(self, res, bs):
 299         self.failUnlessEqual(self.s1.getState()[0], "offline")
 300         self.verifyDisconnect(bs)
 301
 302
 303     def testBuild4(self):
 304         # this next sequence is timing-dependent
 305         d = self.submitBuild()
 306         d.addCallback(self._testBuild4_1)
 307         return d
 308     testBuild4.timeout = 30
 309
 310     def _testBuild4_1(self, bc):
 311         bs = bc.getStatus()
 312         # kill the slave while it's running the second (remote) step
 313         reactor.callLater(1.5, self.killSlave)
 314         d = bs.waitUntilFinished()
 315         d.addCallback(self._testBuild4_2, bs)
 316         return d
 317
 318     def _testBuild4_2(self, res, bs):
 319         # at this point, the slave is in the process of being removed, so it
 320         # could either be 'idle' or 'offline'. I think there is a
 321         # reactor.callLater(0) standing between here and the offline state.
 322         #reactor.iterate() # TODO: remove the need for this
 323
 324         self.failUnlessEqual(self.s1.getState()[0], "offline")
 325         self.verifyDisconnect2(bs)
 326
 327
 328     def testInterrupt(self):
 329         # this next sequence is timing-dependent
 330         d = self.submitBuild()
 331         d.addCallback(self._testInterrupt_1)
 332         return d
 333     testInterrupt.timeout = 30
 334
 335     def _testInterrupt_1(self, bc):
 336         bs = bc.getStatus()
 337         # halt the build while it's running the first step
 338         reactor.callLater(0.5, bc.stopBuild, "bang go splat")
 339         d = bs.waitUntilFinished()
 340         d.addCallback(self._testInterrupt_2, bs)
 341         return d
 342
 343     def _testInterrupt_2(self, res, bs):
 344         self.verifyDisconnect(bs)
 345
 346
 347     def testDisappear(self):
 348         bc = self.control.getBuilder("dummy")
 349
 350         # ping should succeed
 351         d = bc.ping(1)
 352         d.addCallback(self._testDisappear_1, bc)
 353         return d
 354
 355     def _testDisappear_1(self, res, bc):
 356         self.failUnlessEqual(res, True)
 357
 358         # now, before any build is run, make the slave disappear
 359         self.disappearSlave(allowReconnect=False)
 360
 361         # at this point, a ping to the slave should timeout
 362         d = bc.ping(1)
 363         d.addCallback(self. _testDisappear_2)
 364         return d
 365     def _testDisappear_2(self, res):
 366         self.failUnlessEqual(res, False)
 367
 368     def testDuplicate(self):
 369         bc = self.control.getBuilder("dummy")
 370         bs = self.status.getBuilder("dummy")
 371         ss = bs.getSlaves()[0]
 372
 373         self.failUnless(ss.isConnected())
 374         self.failUnlessEqual(ss.getAdmin(), "one")
 375
 376         # now, before any build is run, make the first slave disappear
 377         self.disappearSlave(allowReconnect=False)
 378
 379         d = self.master.botmaster.waitUntilBuilderDetached("dummy")
 380         # now let the new slave take over
 381         self.connectSlave2()
 382         d.addCallback(self._testDuplicate_1, ss)
 383         return d
 384     testDuplicate.timeout = 5
 385
 386     def _testDuplicate_1(self, res, ss):
 387         d = self.master.botmaster.waitUntilBuilderAttached("dummy")
 388         d.addCallback(self._testDuplicate_2, ss)
 389         return d
 390
 391     def _testDuplicate_2(self, res, ss):
 392         self.failUnless(ss.isConnected())
 393         self.failUnlessEqual(ss.getAdmin(), "two")
 394
 395
 396 class Disconnect2(RunMixin, unittest.TestCase):
 397
 398     def setUp(self):
 399         RunMixin.setUp(self)
 400         # verify that disconnecting the slave during a build properly
 401         # terminates the build
 402         m = self.master
 403         s = self.status
 404         c = self.control
 405
 406         m.loadConfig(config_2)
 407         m.readConfig = True
 408         m.startService()
 409
 410         self.failUnlessEqual(s.getBuilderNames(), ["dummy", "testdummy"])
 411         self.s1 = s1 = s.getBuilder("dummy")
 412         self.failUnlessEqual(s1.getName(), "dummy")
 413         self.failUnlessEqual(s1.getState(), ("offline", []))
 414         self.failUnlessEqual(s1.getCurrentBuilds(), [])
 415         self.failUnlessEqual(s1.getLastFinishedBuild(), None)
 416         self.failUnlessEqual(s1.getBuild(-1), None)
 417
 418         d = self.connectSlaveFastTimeout()
 419         d.addCallback(self._setup_disconnect2_1)
 420         return d
 421
 422     def _setup_disconnect2_1(self, res):
 423         self.failUnlessEqual(self.s1.getState(), ("idle", []))
 424
 425
 426     def testSlaveTimeout(self):
 427         # now suppose the slave goes missing. We want to find out when it
 428         # creates a new Broker, so we reach inside and mark it with the
 429         # well-known sigil of impending messy death.
 430         bd = self.slaves['bot1'].getServiceNamed("bot").builders["dummy"]
 431         broker = bd.remote.broker
 432         broker.redshirt = 1
 433
 434         # make sure the keepalives will keep the connection up
 435         d = defer.Deferred()
 436         reactor.callLater(5, d.callback, None)
 437         d.addCallback(self._testSlaveTimeout_1)
 438         return d
 439     testSlaveTimeout.timeout = 20
 440
 441     def _testSlaveTimeout_1(self, res):
 442         bd = self.slaves['bot1'].getServiceNamed("bot").builders["dummy"]
 443         if not bd.remote or not hasattr(bd.remote.broker, "redshirt"):
 444             self.fail("slave disconnected when it shouldn't have")
 445
 446         d = self.master.botmaster.waitUntilBuilderDetached("dummy")
 447         # whoops! how careless of me.
 448         self.disappearSlave(allowReconnect=True)
 449         # the slave will realize the connection is lost within 2 seconds, and
 450         # reconnect.
 451         d.addCallback(self._testSlaveTimeout_2)
 452         return d
 453
 454     def _testSlaveTimeout_2(self, res):
 455         # the ReconnectingPBClientFactory will attempt a reconnect in two
 456         # seconds.
 457         d = self.master.botmaster.waitUntilBuilderAttached("dummy")
 458         d.addCallback(self._testSlaveTimeout_3)
 459         return d
 460
 461     def _testSlaveTimeout_3(self, res):
 462         # make sure it is a new connection (i.e. a new Broker)
 463         bd = self.slaves['bot1'].getServiceNamed("bot").builders["dummy"]
 464         self.failUnless(bd.remote, "hey, slave isn't really connected")
 465         self.failIf(hasattr(bd.remote.broker, "redshirt"),
 466                     "hey, slave's Broker is still marked for death")
 467
 468
 469 class Basedir(RunMixin, unittest.TestCase):
 470     def testChangeBuilddir(self):
 471         m = self.master
 472         m.loadConfig(config_4)
 473         m.readConfig = True
 474         m.startService()
 475
 476         d = self.connectSlave()
 477         d.addCallback(self._testChangeBuilddir_1)
 478         return d
 479
 480     def _testChangeBuilddir_1(self, res):
 481         self.bot = bot = self.slaves['bot1'].bot
 482         self.builder = builder = bot.builders.get("dummy")
 483         self.failUnless(builder)
 484         self.failUnlessEqual(builder.builddir, "dummy")
 485         self.failUnlessEqual(builder.basedir,
 486                              os.path.join("slavebase-bot1", "dummy"))
 487
 488         d = self.master.loadConfig(config_4_newbasedir)
 489         d.addCallback(self._testChangeBuilddir_2)
 490         return d
 491
 492     def _testChangeBuilddir_2(self, res):
 493         bot = self.bot
 494         # this does NOT cause the builder to be replaced
 495         builder = bot.builders.get("dummy")
 496         self.failUnless(builder)
 497         self.failUnlessIdentical(self.builder, builder)
 498         # the basedir should be updated
 499         self.failUnlessEqual(builder.builddir, "dummy2")
 500         self.failUnlessEqual(builder.basedir,
 501                              os.path.join("slavebase-bot1", "dummy2"))
 502
 503         # add a new builder, which causes the basedir list to be reloaded
 504         d = self.master.loadConfig(config_4_newbuilder)
 505         return d
 506
 507 # TODO: test everything, from Change submission to Scheduler to Build to
 508 # Status. Use all the status types. Specifically I want to catch recurrences
 509 # of the bug where I forgot to make Waterfall inherit from StatusReceiver
 510 # such that buildSetSubmitted failed.
 511