buildbot/test/test_slaves.py

   1 # -*- test-case-name: buildbot.test.test_slaves -*-
   2
   3 from twisted.trial import unittest
   4 from buildbot.twcompat import maybeWait
   5 from twisted.internet import defer, reactor
   6 from twisted.python import log
   7
   8 from buildbot.test.runutils import RunMixin
   9 from buildbot.sourcestamp import SourceStamp
  10 from buildbot.process.base import BuildRequest
  11 from buildbot.status.builder import SUCCESS
  12 from buildbot.slave import bot
  13
  14 config_1 = """
  15 from buildbot.process import factory
  16 from buildbot.steps import dummy
  17 s = factory.s
  18
  19 BuildmasterConfig = c = {}
  20 c['bots'] = [('bot1', 'sekrit'), ('bot2', 'sekrit'), ('bot3', 'sekrit')]
  21 c['sources'] = []
  22 c['schedulers'] = []
  23 c['slavePortnum'] = 0
  24 c['schedulers'] = []
  25
  26 f1 = factory.BuildFactory([s(dummy.RemoteDummy, timeout=1)])
  27 f2 = factory.BuildFactory([s(dummy.RemoteDummy, timeout=2)])
  28
  29 c['builders'] = [
  30     {'name': 'b1', 'slavenames': ['bot1','bot2','bot3'],
  31      'builddir': 'b1', 'factory': f1},
  32     ]
  33 """
  34
  35 config_2 = config_1 + """
  36
  37 c['builders'] = [
  38     {'name': 'b1', 'slavenames': ['bot1','bot2','bot3'],
  39      'builddir': 'b1', 'factory': f2},
  40     ]
  41
  42 """
  43
  44 class Slave(RunMixin, unittest.TestCase):
  45
  46     def setUp(self):
  47         RunMixin.setUp(self)
  48         self.master.loadConfig(config_1)
  49         self.master.startService()
  50         d = self.connectSlave(["b1"])
  51         d.addCallback(lambda res: self.connectSlave(["b1"], "bot2"))
  52         return maybeWait(d)
  53
  54     def doBuild(self, buildername):
  55         br = BuildRequest("forced", SourceStamp())
  56         d = br.waitUntilFinished()
  57         self.control.getBuilder(buildername).requestBuild(br)
  58         return d
  59
  60     def testSequence(self):
  61         # make sure both slaves appear in the list.
  62         attached_slaves = [c for c in self.master.botmaster.slaves.values()
  63                            if c.slave]
  64         self.failUnlessEqual(len(attached_slaves), 2)
  65         b = self.master.botmaster.builders["b1"]
  66         self.failUnlessEqual(len(b.slaves), 2)
  67
  68         # since the current scheduling algorithm is simple and does not
  69         # rotate or attempt any sort of load-balancing, two builds in
  70         # sequence should both use the first slave. This may change later if
  71         # we move to a more sophisticated scheme.
  72
  73         d = self.doBuild("b1")
  74         d.addCallback(self._testSequence_1)
  75         return maybeWait(d)
  76     def _testSequence_1(self, res):
  77         self.failUnlessEqual(res.getResults(), SUCCESS)
  78         self.failUnlessEqual(res.getSlavename(), "bot1")
  79
  80         d = self.doBuild("b1")
  81         d.addCallback(self._testSequence_2)
  82         return d
  83     def _testSequence_2(self, res):
  84         self.failUnlessEqual(res.getSlavename(), "bot1")
  85
  86
  87     def testSimultaneous(self):
  88         # make sure we can actually run two builds at the same time
  89         d1 = self.doBuild("b1")
  90         d2 = self.doBuild("b1")
  91         d1.addCallback(self._testSimultaneous_1, d2)
  92         return maybeWait(d1)
  93     def _testSimultaneous_1(self, res, d2):
  94         self.failUnlessEqual(res.getResults(), SUCCESS)
  95         self.failUnlessEqual(res.getSlavename(), "bot1")
  96         d2.addCallback(self._testSimultaneous_2)
  97         return d2
  98     def _testSimultaneous_2(self, res):
  99         self.failUnlessEqual(res.getResults(), SUCCESS)
 100         self.failUnlessEqual(res.getSlavename(), "bot2")
 101
 102     def testFallback1(self):
 103         # detach the first slave, verify that a build is run using the second
 104         # slave instead
 105         d = self.shutdownSlave("bot1", "b1")
 106         d.addCallback(self._testFallback1_1)
 107         return maybeWait(d)
 108     def _testFallback1_1(self, res):
 109         attached_slaves = [c for c in self.master.botmaster.slaves.values()
 110                            if c.slave]
 111         self.failUnlessEqual(len(attached_slaves), 1)
 112         self.failUnlessEqual(len(self.master.botmaster.builders["b1"].slaves),
 113                              1)
 114         d = self.doBuild("b1")
 115         d.addCallback(self._testFallback1_2)
 116         return d
 117     def _testFallback1_2(self, res):
 118         self.failUnlessEqual(res.getResults(), SUCCESS)
 119         self.failUnlessEqual(res.getSlavename(), "bot2")
 120
 121     def testFallback2(self):
 122         # Disable the first slave, so that a slaveping will timeout. Then
 123         # start a build, and verify that the non-failing (second) one is
 124         # claimed for the build, and that the failing one is removed from the
 125         # list.
 126
 127         # reduce the ping time so we'll failover faster
 128         self.master.botmaster.builders["b1"].START_BUILD_TIMEOUT = 1
 129         self.disappearSlave("bot1", "b1")
 130         d = self.doBuild("b1")
 131         d.addCallback(self._testFallback2_1)
 132         return maybeWait(d)
 133     def _testFallback2_1(self, res):
 134         self.failUnlessEqual(res.getResults(), SUCCESS)
 135         self.failUnlessEqual(res.getSlavename(), "bot2")
 136         b1slaves = self.master.botmaster.builders["b1"].slaves
 137         self.failUnlessEqual(len(b1slaves), 1)
 138         self.failUnlessEqual(b1slaves[0].slave.slavename, "bot2")
 139
 140
 141     def notFinished(self, brs):
 142         # utility method
 143         builds = brs.getBuilds()
 144         self.failIf(len(builds) > 1)
 145         if builds:
 146             self.failIf(builds[0].isFinished())
 147
 148     def testDontClaimPingingSlave(self):
 149         # have two slaves connect for the same builder. Do something to the
 150         # first one so that slavepings are delayed (but do not fail
 151         # outright).
 152         timers = []
 153         self.slaves['bot1'].debugOpts["stallPings"] = (10, timers)
 154         br = BuildRequest("forced", SourceStamp())
 155         d1 = br.waitUntilFinished()
 156         self.control.getBuilder("b1").requestBuild(br)
 157         s1 = br.status # this is a BuildRequestStatus
 158         # give it a chance to start pinging
 159         d2 = defer.Deferred()
 160         d2.addCallback(self._testDontClaimPingingSlave_1, d1, s1, timers)
 161         reactor.callLater(1, d2.callback, None)
 162         return maybeWait(d2)
 163     def _testDontClaimPingingSlave_1(self, res, d1, s1, timers):
 164         # now the first build is running (waiting on the ping), so start the
 165         # second build. This should claim the second slave, not the first,
 166         # because the first is busy doing the ping.
 167         self.notFinished(s1)
 168         d3 = self.doBuild("b1")
 169         d3.addCallback(self._testDontClaimPingingSlave_2, d1, s1, timers)
 170         return d3
 171     def _testDontClaimPingingSlave_2(self, res, d1, s1, timers):
 172         self.failUnlessEqual(res.getSlavename(), "bot2")
 173         self.notFinished(s1)
 174         # now let the ping complete
 175         self.failUnlessEqual(len(timers), 1)
 176         timers[0].reset(0)
 177         d1.addCallback(self._testDontClaimPingingSlave_3)
 178         return d1
 179     def _testDontClaimPingingSlave_3(self, res):
 180         self.failUnlessEqual(res.getSlavename(), "bot1")
 181
 182 config_3 = """
 183 from buildbot.process import factory
 184 from buildbot.steps import dummy
 185 s = factory.s
 186
 187 BuildmasterConfig = c = {}
 188 c['bots'] = [('bot1', 'sekrit')]
 189 c['sources'] = []
 190 c['schedulers'] = []
 191 c['slavePortnum'] = 0
 192 c['schedulers'] = []
 193
 194 f1 = factory.BuildFactory([s(dummy.Wait, handle='one')])
 195 f2 = factory.BuildFactory([s(dummy.Wait, handle='two')])
 196 f3 = factory.BuildFactory([s(dummy.Wait, handle='three')])
 197
 198 c['builders'] = [
 199     {'name': 'b1', 'slavenames': ['bot1'],
 200      'builddir': 'b1', 'factory': f1},
 201     ]
 202 """
 203
 204 config_4 = config_3 + """
 205 c['builders'] = [
 206     {'name': 'b1', 'slavenames': ['bot1'],
 207      'builddir': 'b1', 'factory': f2},
 208     ]
 209 """
 210
 211 config_5 = config_3 + """
 212 c['builders'] = [
 213     {'name': 'b1', 'slavenames': ['bot1'],
 214      'builddir': 'b1', 'factory': f3},
 215     ]
 216 """
 217
 218 from buildbot.slave.commands import waitCommandRegistry
 219
 220 class Reconfig(RunMixin, unittest.TestCase):
 221
 222     def setUp(self):
 223         RunMixin.setUp(self)
 224         self.master.loadConfig(config_3)
 225         self.master.startService()
 226         d = self.connectSlave(["b1"])
 227         return maybeWait(d)
 228
 229     def _one_started(self):
 230         log.msg("testReconfig._one_started")
 231         self.build1_started = True
 232         self.d1.callback(None)
 233         return self.d2
 234
 235     def _two_started(self):
 236         log.msg("testReconfig._two_started")
 237         self.build2_started = True
 238         self.d3.callback(None)
 239         return self.d4
 240
 241     def _three_started(self):
 242         log.msg("testReconfig._three_started")
 243         self.build3_started = True
 244         self.d5.callback(None)
 245         return self.d6
 246
 247     def testReconfig(self):
 248         # reconfiguring a Builder should not interrupt any running Builds. No
 249         # queued BuildRequests should be lost. The next Build started should
 250         # use the new process.
 251         slave1 = self.slaves['bot1']
 252         bot1 = slave1.getServiceNamed('bot')
 253         sb1 = bot1.builders['b1']
 254         self.failUnless(isinstance(sb1, bot.SlaveBuilder))
 255         self.failUnless(sb1.running)
 256         b1 = self.master.botmaster.builders['b1']
 257         self.orig_b1 = b1
 258
 259         self.d1 = d1 = defer.Deferred()
 260         self.d2 = d2 = defer.Deferred()
 261         self.d3, self.d4 = defer.Deferred(), defer.Deferred()
 262         self.d5, self.d6 = defer.Deferred(), defer.Deferred()
 263         self.build1_started = False
 264         self.build2_started = False
 265         self.build3_started = False
 266         waitCommandRegistry[("one","build1")] = self._one_started
 267         waitCommandRegistry[("two","build2")] = self._two_started
 268         waitCommandRegistry[("three","build3")] = self._three_started
 269
 270         # use different branches to make sure these cannot be merged
 271         br1 = BuildRequest("build1", SourceStamp(branch="1"))
 272         b1.submitBuildRequest(br1)
 273         br2 = BuildRequest("build2", SourceStamp(branch="2"))
 274         b1.submitBuildRequest(br2)
 275         br3 = BuildRequest("build3", SourceStamp(branch="3"))
 276         b1.submitBuildRequest(br3)
 277         self.requests = (br1, br2, br3)
 278         # all three are now in the queue
 279
 280         # wait until the first one has started
 281         d1.addCallback(self._testReconfig_2)
 282         return d1
 283
 284     def _testReconfig_2(self, res):
 285         log.msg("_testReconfig_2")
 286         # confirm that it is building
 287         brs = self.requests[0].status.getBuilds()
 288         self.failUnlessEqual(len(brs), 1)
 289         self.build1 = brs[0]
 290         self.failUnlessEqual(self.build1.getCurrentStep().getName(), "wait")
 291         # br1 is building, br2 and br3 are in the queue (in that order). Now
 292         # we reconfigure the Builder.
 293         self.failUnless(self.build1_started)
 294         d = self.master.loadConfig(config_4)
 295         d.addCallback(self._testReconfig_3)
 296         return d
 297
 298     def _testReconfig_3(self, res):
 299         log.msg("_testReconfig_3")
 300         # now check to see that br1 is still building, and that br2 and br3
 301         # are in the queue of the new builder
 302         b1 = self.master.botmaster.builders['b1']
 303         self.failIfIdentical(b1, self.orig_b1)
 304         self.failIf(self.build1.isFinished())
 305         self.failUnlessEqual(self.build1.getCurrentStep().getName(), "wait")
 306         self.failUnlessEqual(len(b1.buildable), 2)
 307         self.failUnless(self.requests[1] in b1.buildable)
 308         self.failUnless(self.requests[2] in b1.buildable)
 309
 310         # allow br1 to finish, and make sure its status is delivered normally
 311         d = self.requests[0].waitUntilFinished()
 312         d.addCallback(self._testReconfig_4)
 313         self.d2.callback(None)
 314         return d
 315
 316     def _testReconfig_4(self, bs):
 317         log.msg("_testReconfig_4")
 318         self.failUnlessEqual(bs.getReason(), "build1")
 319         self.failUnless(bs.isFinished())
 320         self.failUnlessEqual(bs.getResults(), SUCCESS)
 321
 322         # at this point, the first build has finished, and there is a pending
 323         # call to start the second build. Once that pending call fires, there
 324         # is a network roundtrip before the 'wait' RemoteCommand is delivered
 325         # to the slave. We need to wait for both events to happen before we
 326         # can check to make sure it is using the correct process. Just wait a
 327         # full second.
 328         d = defer.Deferred()
 329         d.addCallback(self._testReconfig_5)
 330         reactor.callLater(1, d.callback, None)
 331         return d
 332
 333     def _testReconfig_5(self, res):
 334         log.msg("_testReconfig_5")
 335         # at this point the next build ought to be running
 336         b1 = self.master.botmaster.builders['b1']
 337         self.failUnlessEqual(len(b1.buildable), 1)
 338         self.failUnless(self.requests[2] in b1.buildable)
 339         self.failUnlessEqual(len(b1.building), 1)
 340         # and it ought to be using the new process
 341         self.failUnless(self.build2_started)
 342
 343         # now, while the second build is running, change the config multiple
 344         # times.
 345
 346         d = self.master.loadConfig(config_3)
 347         d.addCallback(lambda res: self.master.loadConfig(config_4))
 348         d.addCallback(lambda res: self.master.loadConfig(config_5))
 349         def _done(res):
 350             # then once that's done, allow the second build to finish and
 351             # wait for it to complete
 352             da = self.requests[1].waitUntilFinished()
 353             self.d4.callback(None)
 354             return da
 355         d.addCallback(_done)
 356         def _done2(res):
 357             # and once *that*'s done, wait another second to let the third
 358             # build start
 359             db = defer.Deferred()
 360             reactor.callLater(1, db.callback, None)
 361             return db
 362         d.addCallback(_done2)
 363         d.addCallback(self._testReconfig_6)
 364         return d
 365
 366     def _testReconfig_6(self, res):
 367         log.msg("_testReconfig_6")
 368         # now check to see that the third build is running
 369         self.failUnless(self.build3_started)
 370
 371         # we're done
 372
 373
 374
 375 class Slave2(RunMixin, unittest.TestCase):
 376
 377     revision = 0
 378
 379     def setUp(self):
 380         RunMixin.setUp(self)
 381         self.master.loadConfig(config_1)
 382         self.master.startService()
 383
 384     def doBuild(self, buildername, reason="forced"):
 385         # we need to prevent these builds from being merged, so we create
 386         # each of them with a different revision specifier. The revision is
 387         # ignored because our build process does not have a source checkout
 388         # step.
 389         self.revision += 1
 390         br = BuildRequest(reason, SourceStamp(revision=self.revision))
 391         d = br.waitUntilFinished()
 392         self.control.getBuilder(buildername).requestBuild(br)
 393         return d
 394
 395     def testFirstComeFirstServed(self):
 396         # submit three builds, then connect a slave which fails the
 397         # slaveping. The first build will claim the slave, do the slaveping,
 398         # give up, and re-queue the build. Verify that the build gets
 399         # re-queued in front of all other builds. This may be tricky, because
 400         # the other builds may attempt to claim the just-failed slave.
 401
 402         d1 = self.doBuild("b1", "first")
 403         d2 = self.doBuild("b1", "second")
 404         #buildable = self.master.botmaster.builders["b1"].buildable
 405         #print [b.reason for b in buildable]
 406
 407         # specifically, I want the poor build to get precedence over any
 408         # others that were waiting. To test this, we need more builds than
 409         # slaves.
 410
 411         # now connect a broken slave. The first build started as soon as it
 412         # connects, so by the time we get to our _1 method, the ill-fated
 413         # build has already started.
 414         d = self.connectSlave(["b1"], opts={"failPingOnce": True})
 415         d.addCallback(self._testFirstComeFirstServed_1, d1, d2)
 416         return maybeWait(d)
 417     def _testFirstComeFirstServed_1(self, res, d1, d2):
 418         # the master has send the slaveping. When this is received, it will
 419         # fail, causing the master to hang up on the slave. When it
 420         # reconnects, it should find the first build at the front of the
 421         # queue. If we simply wait for both builds to complete, then look at
 422         # the status logs, we should see that the builds ran in the correct
 423         # order.
 424
 425         d = defer.DeferredList([d1,d2])
 426         d.addCallback(self._testFirstComeFirstServed_2)
 427         return d
 428     def _testFirstComeFirstServed_2(self, res):
 429         b = self.status.getBuilder("b1")
 430         builds = b.getBuild(0), b.getBuild(1)
 431         reasons = [build.getReason() for build in builds]
 432         self.failUnlessEqual(reasons, ["first", "second"])
 433