buildbot/test/test_run.py

   1 # -*- test-case-name: buildbot.test.test_run -*-
   2
   3 from twisted.trial import unittest
   4 from twisted.internet import reactor, defer
   5 from twisted.python import log
   6 import sys, os, os.path, shutil, time, errno
   7 #log.startLogging(sys.stderr)
   8
   9 from buildbot import master, interfaces
  10 from buildbot.sourcestamp import SourceStamp
  11 from buildbot.slave import bot
  12 from buildbot.changes import changes
  13 from buildbot.status import builder
  14 from buildbot.process.base import BuildRequest
  15 from buildbot.twcompat import maybeWait
  16
  17 from buildbot.test.runutils import RunMixin
  18
  19 config_base = """
  20 from buildbot.process import factory, step
  21 s = factory.s
  22
  23 f1 = factory.QuickBuildFactory('fakerep', 'cvsmodule', configure=None)
  24
  25 f2 = factory.BuildFactory([
  26     s(step.Dummy, timeout=1),
  27     s(step.RemoteDummy, timeout=2),
  28     ])
  29
  30 BuildmasterConfig = c = {}
  31 c['bots'] = [['bot1', 'sekrit']]
  32 c['sources'] = []
  33 c['schedulers'] = []
  34 c['builders'] = []
  35 c['builders'].append({'name':'quick', 'slavename':'bot1',
  36                       'builddir': 'quickdir', 'factory': f1})
  37 c['slavePortnum'] = 0
  38 """
  39
  40 config_run = config_base + """
  41 from buildbot.scheduler import Scheduler
  42 c['schedulers'] = [Scheduler('quick', None, 120, ['quick'])]
  43 """
  44
  45 config_2 = config_base + """
  46 c['builders'] = [{'name': 'dummy', 'slavename': 'bot1',
  47                   'builddir': 'dummy1', 'factory': f2},
  48                  {'name': 'testdummy', 'slavename': 'bot1',
  49                   'builddir': 'dummy2', 'factory': f2, 'category': 'test'}]
  50 """
  51
  52 config_3 = config_2 + """
  53 c['builders'].append({'name': 'adummy', 'slavename': 'bot1',
  54                       'builddir': 'adummy3', 'factory': f2})
  55 c['builders'].append({'name': 'bdummy', 'slavename': 'bot1',
  56                       'builddir': 'adummy4', 'factory': f2,
  57                       'category': 'test'})
  58 """
  59
  60 config_4 = config_base + """
  61 c['builders'] = [{'name': 'dummy', 'slavename': 'bot1',
  62                   'builddir': 'dummy', 'factory': f2}]
  63 """
  64
  65 config_4_newbasedir = config_4 + """
  66 c['builders'] = [{'name': 'dummy', 'slavename': 'bot1',
  67                   'builddir': 'dummy2', 'factory': f2}]
  68 """
  69
  70 config_4_newbuilder = config_4_newbasedir + """
  71 c['builders'].append({'name': 'dummy2', 'slavename': 'bot1',
  72                       'builddir': 'dummy23', 'factory': f2})
  73 """
  74
  75 class Run(unittest.TestCase):
  76     def rmtree(self, d):
  77         try:
  78             shutil.rmtree(d, ignore_errors=1)
  79         except OSError, e:
  80             # stupid 2.2 appears to ignore ignore_errors
  81             if e.errno != errno.ENOENT:
  82                 raise
  83
  84     def testMaster(self):
  85         self.rmtree("basedir")
  86         os.mkdir("basedir")
  87         m = master.BuildMaster("basedir")
  88         m.loadConfig(config_run)
  89         m.readConfig = True
  90         m.startService()
  91         cm = m.change_svc
  92         c = changes.Change("bob", ["Makefile", "foo/bar.c"], "changed stuff")
  93         cm.addChange(c)
  94         # verify that the Scheduler is now waiting
  95         s = m.allSchedulers()[0]
  96         self.failUnless(s.timer)
  97         # halting the service will also stop the timer
  98         d = defer.maybeDeferred(m.stopService)
  99         return maybeWait(d)
 100
 101 class Ping(RunMixin, unittest.TestCase):
 102     def testPing(self):
 103         self.master.loadConfig(config_2)
 104         self.master.readConfig = True
 105         self.master.startService()
 106
 107         d = self.connectSlave()
 108         d.addCallback(self._testPing_1)
 109         return maybeWait(d)
 110
 111     def _testPing_1(self, res):
 112         d = interfaces.IControl(self.master).getBuilder("dummy").ping(1)
 113         d.addCallback(self._testPing_2)
 114         return d
 115
 116     def _testPing_2(self, res):
 117         pass
 118
 119 class BuilderNames(unittest.TestCase):
 120
 121     def testGetBuilderNames(self):
 122         os.mkdir("bnames")
 123         m = master.BuildMaster("bnames")
 124         s = m.getStatus()
 125
 126         m.loadConfig(config_3)
 127         m.readConfig = True
 128
 129         self.failUnlessEqual(s.getBuilderNames(),
 130                              ["dummy", "testdummy", "adummy", "bdummy"])
 131         self.failUnlessEqual(s.getBuilderNames(categories=['test']),
 132                              ["testdummy", "bdummy"])
 133
 134 class Disconnect(RunMixin, unittest.TestCase):
 135
 136     def setUp(self):
 137         RunMixin.setUp(self)
 138
 139         # verify that disconnecting the slave during a build properly
 140         # terminates the build
 141         m = self.master
 142         s = self.status
 143         c = self.control
 144
 145         m.loadConfig(config_2)
 146         m.readConfig = True
 147         m.startService()
 148
 149         self.failUnlessEqual(s.getBuilderNames(), ["dummy", "testdummy"])
 150         self.s1 = s1 = s.getBuilder("dummy")
 151         self.failUnlessEqual(s1.getName(), "dummy")
 152         self.failUnlessEqual(s1.getState(), ("offline", []))
 153         self.failUnlessEqual(s1.getCurrentBuilds(), [])
 154         self.failUnlessEqual(s1.getLastFinishedBuild(), None)
 155         self.failUnlessEqual(s1.getBuild(-1), None)
 156
 157         d = self.connectSlave()
 158         d.addCallback(self._disconnectSetup_1)
 159         return maybeWait(d)
 160
 161     def _disconnectSetup_1(self, res):
 162         self.failUnlessEqual(self.s1.getState(), ("idle", []))
 163
 164
 165     def verifyDisconnect(self, bs):
 166         self.failUnless(bs.isFinished())
 167
 168         step1 = bs.getSteps()[0]
 169         self.failUnlessEqual(step1.getText(), ["delay", "interrupted"])
 170         self.failUnlessEqual(step1.getResults()[0], builder.FAILURE)
 171
 172         self.failUnlessEqual(bs.getResults(), builder.FAILURE)
 173
 174     def verifyDisconnect2(self, bs):
 175         self.failUnless(bs.isFinished())
 176
 177         step1 = bs.getSteps()[1]
 178         self.failUnlessEqual(step1.getText(), ["remote", "delay", "2 secs",
 179                                                "failed", "slave", "lost"])
 180         self.failUnlessEqual(step1.getResults()[0], builder.FAILURE)
 181
 182         self.failUnlessEqual(bs.getResults(), builder.FAILURE)
 183
 184
 185     def testIdle1(self):
 186         # disconnect the slave before the build starts
 187         d = self.shutdownAllSlaves() # dies before it gets started
 188         d.addCallback(self._testIdle1_1)
 189         return d
 190     def _testIdle1_1(self, res):
 191         # trying to force a build now will cause an error. Regular builds
 192         # just wait for the slave to re-appear, but forced builds that
 193         # cannot be run right away trigger NoSlaveErrors
 194         fb = self.control.getBuilder("dummy").forceBuild
 195         self.failUnlessRaises(interfaces.NoSlaveError,
 196                               fb, None, "forced build")
 197
 198     def testIdle2(self):
 199         # now suppose the slave goes missing
 200         self.slaves['bot1'].bf.continueTrying = 0
 201         self.disappearSlave()
 202
 203         # forcing a build will work: the build detect that the slave is no
 204         # longer available and will be re-queued. Wait 5 seconds, then check
 205         # to make sure the build is still in the 'waiting for a slave' queue.
 206         self.control.getBuilder("dummy").original.START_BUILD_TIMEOUT = 1
 207         req = BuildRequest("forced build", SourceStamp())
 208         self.failUnlessEqual(req.startCount, 0)
 209         self.control.getBuilder("dummy").requestBuild(req)
 210         # this should ping the slave, which doesn't respond, and then give up
 211         # after a second. The BuildRequest will be re-queued, and its
 212         # .startCount will be incremented.
 213         d = defer.Deferred()
 214         d.addCallback(self._testIdle2_1, req)
 215         reactor.callLater(3, d.callback, None)
 216         return maybeWait(d, 5)
 217     testIdle2.timeout = 5
 218
 219     def _testIdle2_1(self, res, req):
 220         self.failUnlessEqual(req.startCount, 1)
 221         cancelled = req.cancel()
 222         self.failUnless(cancelled)
 223
 224
 225     def testBuild1(self):
 226         # this next sequence is timing-dependent. The dummy build takes at
 227         # least 3 seconds to complete, and this batch of commands must
 228         # complete within that time.
 229         #
 230         d = self.control.getBuilder("dummy").forceBuild(None, "forced build")
 231         d.addCallback(self._testBuild1_1)
 232         return maybeWait(d)
 233
 234     def _testBuild1_1(self, bc):
 235         bs = bc.getStatus()
 236         # now kill the slave before it gets to start the first step
 237         d = self.shutdownAllSlaves() # dies before it gets started
 238         d.addCallback(self._testBuild1_2, bs)
 239         return d  # TODO: this used to have a 5-second timeout
 240
 241     def _testBuild1_2(self, res, bs):
 242         # now examine the just-stopped build and make sure it is really
 243         # stopped. This is checking for bugs in which the slave-detach gets
 244         # missed or causes an exception which prevents the build from being
 245         # marked as "finished due to an error".
 246         d = bs.waitUntilFinished()
 247         d2 = self.master.botmaster.waitUntilBuilderDetached("dummy")
 248         dl = defer.DeferredList([d, d2])
 249         dl.addCallback(self._testBuild1_3, bs)
 250         return dl # TODO: this had a 5-second timeout too
 251
 252     def _testBuild1_3(self, res, bs):
 253         self.failUnlessEqual(self.s1.getState()[0], "offline")
 254         self.verifyDisconnect(bs)
 255
 256
 257     def testBuild2(self):
 258         # this next sequence is timing-dependent
 259         d = self.control.getBuilder("dummy").forceBuild(None, "forced build")
 260         d.addCallback(self._testBuild1_1)
 261         return maybeWait(d, 30)
 262     testBuild2.timeout = 30
 263
 264     def _testBuild1_1(self, bc):
 265         bs = bc.getStatus()
 266         # shutdown the slave while it's running the first step
 267         reactor.callLater(0.5, self.shutdownAllSlaves)
 268
 269         d = bs.waitUntilFinished()
 270         d.addCallback(self._testBuild2_2, bs)
 271         return d
 272
 273     def _testBuild2_2(self, res, bs):
 274         # we hit here when the build has finished. The builder is still being
 275         # torn down, however, so spin for another second to allow the
 276         # callLater(0) in Builder.detached to fire.
 277         d = defer.Deferred()
 278         reactor.callLater(1, d.callback, None)
 279         d.addCallback(self._testBuild2_3, bs)
 280         return d
 281
 282     def _testBuild2_3(self, res, bs):
 283         self.failUnlessEqual(self.s1.getState()[0], "offline")
 284         self.verifyDisconnect(bs)
 285
 286
 287     def testBuild3(self):
 288         # this next sequence is timing-dependent
 289         d = self.control.getBuilder("dummy").forceBuild(None, "forced build")
 290         d.addCallback(self._testBuild3_1)
 291         return maybeWait(d, 30)
 292     testBuild3.timeout = 30
 293
 294     def _testBuild3_1(self, bc):
 295         bs = bc.getStatus()
 296         # kill the slave while it's running the first step
 297         reactor.callLater(0.5, self.killSlave)
 298         d = bs.waitUntilFinished()
 299         d.addCallback(self._testBuild3_2, bs)
 300         return d
 301
 302     def _testBuild3_2(self, res, bs):
 303         # the builder is still being torn down, so give it another second
 304         d = defer.Deferred()
 305         reactor.callLater(1, d.callback, None)
 306         d.addCallback(self._testBuild3_3, bs)
 307         return d
 308
 309     def _testBuild3_3(self, res, bs):
 310         self.failUnlessEqual(self.s1.getState()[0], "offline")
 311         self.verifyDisconnect(bs)
 312
 313
 314     def testBuild4(self):
 315         # this next sequence is timing-dependent
 316         d = self.control.getBuilder("dummy").forceBuild(None, "forced build")
 317         d.addCallback(self._testBuild4_1)
 318         return maybeWait(d, 30)
 319     testBuild4.timeout = 30
 320
 321     def _testBuild4_1(self, bc):
 322         bs = bc.getStatus()
 323         # kill the slave while it's running the second (remote) step
 324         reactor.callLater(1.5, self.killSlave)
 325         d = bs.waitUntilFinished()
 326         d.addCallback(self._testBuild4_2, bs)
 327         return d
 328
 329     def _testBuild4_2(self, res, bs):
 330         # at this point, the slave is in the process of being removed, so it
 331         # could either be 'idle' or 'offline'. I think there is a
 332         # reactor.callLater(0) standing between here and the offline state.
 333         #reactor.iterate() # TODO: remove the need for this
 334
 335         self.failUnlessEqual(self.s1.getState()[0], "offline")
 336         self.verifyDisconnect2(bs)
 337
 338
 339     def testInterrupt(self):
 340         # this next sequence is timing-dependent
 341         d = self.control.getBuilder("dummy").forceBuild(None, "forced build")
 342         d.addCallback(self._testInterrupt_1)
 343         return maybeWait(d, 30)
 344     testInterrupt.timeout = 30
 345
 346     def _testInterrupt_1(self, bc):
 347         bs = bc.getStatus()
 348         # halt the build while it's running the first step
 349         reactor.callLater(0.5, bc.stopBuild, "bang go splat")
 350         d = bs.waitUntilFinished()
 351         d.addCallback(self._testInterrupt_2, bs)
 352         return d
 353
 354     def _testInterrupt_2(self, res, bs):
 355         self.verifyDisconnect(bs)
 356
 357
 358     def testDisappear(self):
 359         bc = self.control.getBuilder("dummy")
 360
 361         # ping should succeed
 362         d = bc.ping(1)
 363         d.addCallback(self._testDisappear_1, bc)
 364         return maybeWait(d)
 365
 366     def _testDisappear_1(self, res, bc):
 367         self.failUnlessEqual(res, True)
 368
 369         # now, before any build is run, make the slave disappear
 370         self.slaves['bot1'].bf.continueTrying = 0
 371         self.disappearSlave()
 372
 373         # at this point, a ping to the slave should timeout
 374         d = bc.ping(1)
 375         d.addCallback(self. _testDisappear_2)
 376         return d
 377     def _testDisappear_2(self, res):
 378         self.failUnlessEqual(res, False)
 379
 380     def testDuplicate(self):
 381         bc = self.control.getBuilder("dummy")
 382         bs = self.status.getBuilder("dummy")
 383         ss = bs.getSlaves()[0]
 384
 385         self.failUnless(ss.isConnected())
 386         self.failUnlessEqual(ss.getAdmin(), "one")
 387
 388         # now, before any build is run, make the first slave disappear
 389         self.slaves['bot1'].bf.continueTrying = 0
 390         self.disappearSlave()
 391
 392         d = self.master.botmaster.waitUntilBuilderDetached("dummy")
 393         # now let the new slave take over
 394         self.connectSlave2()
 395         d.addCallback(self._testDuplicate_1, ss)
 396         return maybeWait(d, 2)
 397     testDuplicate.timeout = 5
 398
 399     def _testDuplicate_1(self, res, ss):
 400         d = self.master.botmaster.waitUntilBuilderAttached("dummy")
 401         d.addCallback(self._testDuplicate_2, ss)
 402         return d
 403
 404     def _testDuplicate_2(self, res, ss):
 405         self.failUnless(ss.isConnected())
 406         self.failUnlessEqual(ss.getAdmin(), "two")
 407
 408
 409 class Disconnect2(RunMixin, unittest.TestCase):
 410
 411     def setUp(self):
 412         RunMixin.setUp(self)
 413         # verify that disconnecting the slave during a build properly
 414         # terminates the build
 415         m = self.master
 416         s = self.status
 417         c = self.control
 418
 419         m.loadConfig(config_2)
 420         m.readConfig = True
 421         m.startService()
 422
 423         self.failUnlessEqual(s.getBuilderNames(), ["dummy", "testdummy"])
 424         self.s1 = s1 = s.getBuilder("dummy")
 425         self.failUnlessEqual(s1.getName(), "dummy")
 426         self.failUnlessEqual(s1.getState(), ("offline", []))
 427         self.failUnlessEqual(s1.getCurrentBuilds(), [])
 428         self.failUnlessEqual(s1.getLastFinishedBuild(), None)
 429         self.failUnlessEqual(s1.getBuild(-1), None)
 430
 431         d = self.connectSlaveFastTimeout()
 432         d.addCallback(self._setup_disconnect2_1)
 433         return maybeWait(d)
 434
 435     def _setup_disconnect2_1(self, res):
 436         self.failUnlessEqual(self.s1.getState(), ("idle", []))
 437
 438
 439     def testSlaveTimeout(self):
 440         # now suppose the slave goes missing. We want to find out when it
 441         # creates a new Broker, so we reach inside and mark it with the
 442         # well-known sigil of impending messy death.
 443         bd = self.slaves['bot1'].getServiceNamed("bot").builders["dummy"]
 444         broker = bd.remote.broker
 445         broker.redshirt = 1
 446
 447         # make sure the keepalives will keep the connection up
 448         d = defer.Deferred()
 449         reactor.callLater(5, d.callback, None)
 450         d.addCallback(self._testSlaveTimeout_1)
 451         return maybeWait(d, 20)
 452     testSlaveTimeout.timeout = 20
 453
 454     def _testSlaveTimeout_1(self, res):
 455         bd = self.slaves['bot1'].getServiceNamed("bot").builders["dummy"]
 456         if not bd.remote or not hasattr(bd.remote.broker, "redshirt"):
 457             self.fail("slave disconnected when it shouldn't have")
 458
 459         d = self.master.botmaster.waitUntilBuilderDetached("dummy")
 460         # whoops! how careless of me.
 461         self.disappearSlave()
 462         # the slave will realize the connection is lost within 2 seconds, and
 463         # reconnect.
 464         d.addCallback(self._testSlaveTimeout_2)
 465         return d
 466
 467     def _testSlaveTimeout_2(self, res):
 468         # the ReconnectingPBClientFactory will attempt a reconnect in two
 469         # seconds.
 470         d = self.master.botmaster.waitUntilBuilderAttached("dummy")
 471         d.addCallback(self._testSlaveTimeout_3)
 472         return d
 473
 474     def _testSlaveTimeout_3(self, res):
 475         # make sure it is a new connection (i.e. a new Broker)
 476         bd = self.slaves['bot1'].getServiceNamed("bot").builders["dummy"]
 477         self.failUnless(bd.remote, "hey, slave isn't really connected")
 478         self.failIf(hasattr(bd.remote.broker, "redshirt"),
 479                     "hey, slave's Broker is still marked for death")
 480
 481
 482 class Basedir(RunMixin, unittest.TestCase):
 483     def testChangeBuilddir(self):
 484         m = self.master
 485         m.loadConfig(config_4)
 486         m.readConfig = True
 487         m.startService()
 488
 489         d = self.connectSlave()
 490         d.addCallback(self._testChangeBuilddir_1)
 491         return maybeWait(d)
 492
 493     def _testChangeBuilddir_1(self, res):
 494         self.bot = bot = self.slaves['bot1'].bot
 495         self.builder = builder = bot.builders.get("dummy")
 496         self.failUnless(builder)
 497         self.failUnlessEqual(builder.builddir, "dummy")
 498         self.failUnlessEqual(builder.basedir,
 499                              os.path.join("slavebase-bot1", "dummy"))
 500
 501         d = self.master.loadConfig(config_4_newbasedir)
 502         d.addCallback(self._testChangeBuilddir_2)
 503         return d
 504
 505     def _testChangeBuilddir_2(self, res):
 506         bot = self.bot
 507         # this causes the builder to be replaced
 508         self.failIfIdentical(self.builder, bot.builders.get("dummy"))
 509         builder = bot.builders.get("dummy")
 510         self.failUnless(builder)
 511         # the basedir should be updated
 512         self.failUnlessEqual(builder.builddir, "dummy2")
 513         self.failUnlessEqual(builder.basedir,
 514                              os.path.join("slavebase-bot1", "dummy2"))
 515
 516         # add a new builder, which causes the basedir list to be reloaded
 517         d = self.master.loadConfig(config_4_newbuilder)
 518         return d
 519
 520 # TODO: test everything, from Change submission to Scheduler to Build to
 521 # Status. Use all the status types. Specifically I want to catch recurrences
 522 # of the bug where I forgot to make Waterfall inherit from StatusReceiver
 523 # such that buildSetSubmitted failed.
 524