buildbot/test/runutils.py

   1
   2 import signal
   3 import shutil, os, errno
   4 from twisted.internet import defer, reactor
   5 from twisted.python import log, util
   6
   7 from buildbot import master, interfaces
   8 from buildbot.twcompat import maybeWait
   9 from buildbot.slave import bot
  10 from buildbot.process.builder import Builder
  11 from buildbot.process.base import BuildRequest, Build
  12 from buildbot.process.buildstep import BuildStep
  13 from buildbot.sourcestamp import SourceStamp
  14 from buildbot.status import builder
  15
  16 class MyBot(bot.Bot):
  17     def remote_getSlaveInfo(self):
  18         return self.parent.info
  19
  20 class MyBuildSlave(bot.BuildSlave):
  21     botClass = MyBot
  22
  23 def rmtree(d):
  24     try:
  25         shutil.rmtree(d, ignore_errors=1)
  26     except OSError, e:
  27         # stupid 2.2 appears to ignore ignore_errors
  28         if e.errno != errno.ENOENT:
  29             raise
  30
  31 class RunMixin:
  32     master = None
  33
  34     def rmtree(self, d):
  35         rmtree(d)
  36
  37     def setUp(self):
  38         self.slaves = {}
  39         self.rmtree("basedir")
  40         os.mkdir("basedir")
  41         self.master = master.BuildMaster("basedir")
  42         self.status = self.master.getStatus()
  43         self.control = interfaces.IControl(self.master)
  44
  45     def connectOneSlave(self, slavename, opts={}):
  46         port = self.master.slavePort._port.getHost().port
  47         self.rmtree("slavebase-%s" % slavename)
  48         os.mkdir("slavebase-%s" % slavename)
  49         slave = MyBuildSlave("localhost", port, slavename, "sekrit",
  50                              "slavebase-%s" % slavename,
  51                              keepalive=0, usePTY=1, debugOpts=opts)
  52         slave.info = {"admin": "one"}
  53         self.slaves[slavename] = slave
  54         slave.startService()
  55
  56     def connectSlave(self, builders=["dummy"], slavename="bot1",
  57                      opts={}):
  58         # connect buildslave 'slavename' and wait for it to connect to all of
  59         # the given builders
  60         dl = []
  61         # initiate call for all of them, before waiting on result,
  62         # otherwise we might miss some
  63         for b in builders:
  64             dl.append(self.master.botmaster.waitUntilBuilderAttached(b))
  65         d = defer.DeferredList(dl)
  66         self.connectOneSlave(slavename, opts)
  67         return d
  68
  69     def connectSlaves(self, slavenames, builders):
  70         dl = []
  71         # initiate call for all of them, before waiting on result,
  72         # otherwise we might miss some
  73         for b in builders:
  74             dl.append(self.master.botmaster.waitUntilBuilderAttached(b))
  75         d = defer.DeferredList(dl)
  76         for name in slavenames:
  77             self.connectOneSlave(name)
  78         return d
  79
  80     def connectSlave2(self):
  81         # this takes over for bot1, so it has to share the slavename
  82         port = self.master.slavePort._port.getHost().port
  83         self.rmtree("slavebase-bot2")
  84         os.mkdir("slavebase-bot2")
  85         # this uses bot1, really
  86         slave = MyBuildSlave("localhost", port, "bot1", "sekrit",
  87                              "slavebase-bot2", keepalive=0, usePTY=1)
  88         slave.info = {"admin": "two"}
  89         self.slaves['bot2'] = slave
  90         slave.startService()
  91
  92     def connectSlaveFastTimeout(self):
  93         # this slave has a very fast keepalive timeout
  94         port = self.master.slavePort._port.getHost().port
  95         self.rmtree("slavebase-bot1")
  96         os.mkdir("slavebase-bot1")
  97         slave = MyBuildSlave("localhost", port, "bot1", "sekrit",
  98                              "slavebase-bot1", keepalive=2, usePTY=1,
  99                              keepaliveTimeout=1)
 100         slave.info = {"admin": "one"}
 101         self.slaves['bot1'] = slave
 102         slave.startService()
 103         d = self.master.botmaster.waitUntilBuilderAttached("dummy")
 104         return d
 105
 106     # things to start builds
 107     def requestBuild(self, builder):
 108         # returns a Deferred that fires with an IBuildStatus object when the
 109         # build is finished
 110         req = BuildRequest("forced build", SourceStamp())
 111         self.control.getBuilder(builder).requestBuild(req)
 112         return req.waitUntilFinished()
 113
 114     def failUnlessBuildSucceeded(self, bs):
 115         if bs.getResults() != builder.SUCCESS:
 116             log.msg("failUnlessBuildSucceeded noticed that the build failed")
 117             self.logBuildResults(bs)
 118         self.failUnless(bs.getResults() == builder.SUCCESS)
 119         return bs # useful for chaining
 120
 121     def logBuildResults(self, bs):
 122         # emit the build status and the contents of all logs to test.log
 123         log.msg("logBuildResults starting")
 124         log.msg(" bs.getResults() == %s" % builder.Results[bs.getResults()])
 125         log.msg(" bs.isFinished() == %s" % bs.isFinished())
 126         for s in bs.getSteps():
 127             for l in s.getLogs():
 128                 log.msg("--- START step %s / log %s ---" % (s.getName(),
 129                                                             l.getName()))
 130                 if not l.getName().endswith(".html"):
 131                     log.msg(l.getTextWithHeaders())
 132                 log.msg("--- STOP ---")
 133         log.msg("logBuildResults finished")
 134
 135     def tearDown(self):
 136         log.msg("doing tearDown")
 137         d = self.shutdownAllSlaves()
 138         d.addCallback(self._tearDown_1)
 139         d.addCallback(self._tearDown_2)
 140         return maybeWait(d)
 141     def _tearDown_1(self, res):
 142         if self.master:
 143             return defer.maybeDeferred(self.master.stopService)
 144     def _tearDown_2(self, res):
 145         self.master = None
 146         log.msg("tearDown done")
 147
 148
 149     # various forms of slave death
 150
 151     def shutdownAllSlaves(self):
 152         # the slave has disconnected normally: they SIGINT'ed it, or it shut
 153         # down willingly. This will kill child processes and give them a
 154         # chance to finish up. We return a Deferred that will fire when
 155         # everything is finished shutting down.
 156
 157         log.msg("doing shutdownAllSlaves")
 158         dl = []
 159         for slave in self.slaves.values():
 160             dl.append(slave.waitUntilDisconnected())
 161             dl.append(defer.maybeDeferred(slave.stopService))
 162         d = defer.DeferredList(dl)
 163         d.addCallback(self._shutdownAllSlavesDone)
 164         return d
 165     def _shutdownAllSlavesDone(self, res):
 166         for name in self.slaves.keys():
 167             del self.slaves[name]
 168         return self.master.botmaster.waitUntilBuilderFullyDetached("dummy")
 169
 170     def shutdownSlave(self, slavename, buildername):
 171         # this slave has disconnected normally: they SIGINT'ed it, or it shut
 172         # down willingly. This will kill child processes and give them a
 173         # chance to finish up. We return a Deferred that will fire when
 174         # everything is finished shutting down, and the given Builder knows
 175         # that the slave has gone away.
 176
 177         s = self.slaves[slavename]
 178         dl = [self.master.botmaster.waitUntilBuilderDetached(buildername),
 179               s.waitUntilDisconnected()]
 180         d = defer.DeferredList(dl)
 181         d.addCallback(self._shutdownSlave_done, slavename)
 182         s.stopService()
 183         return d
 184     def _shutdownSlave_done(self, res, slavename):
 185         del self.slaves[slavename]
 186
 187     def killSlave(self):
 188         # the slave has died, its host sent a FIN. The .notifyOnDisconnect
 189         # callbacks will terminate the current step, so the build should be
 190         # flunked (no further steps should be started).
 191         self.slaves['bot1'].bf.continueTrying = 0
 192         bot = self.slaves['bot1'].getServiceNamed("bot")
 193         broker = bot.builders["dummy"].remote.broker
 194         broker.transport.loseConnection()
 195         del self.slaves['bot1']
 196
 197     def disappearSlave(self, slavename="bot1", buildername="dummy"):
 198         # the slave's host has vanished off the net, leaving the connection
 199         # dangling. This will be detected quickly by app-level keepalives or
 200         # a ping, or slowly by TCP timeouts.
 201
 202         # simulate this by replacing the slave Broker's .dataReceived method
 203         # with one that just throws away all data.
 204         def discard(data):
 205             pass
 206         bot = self.slaves[slavename].getServiceNamed("bot")
 207         broker = bot.builders[buildername].remote.broker
 208         broker.dataReceived = discard # seal its ears
 209         broker.transport.write = discard # and take away its voice
 210
 211     def ghostSlave(self):
 212         # the slave thinks it has lost the connection, and initiated a
 213         # reconnect. The master doesn't yet realize it has lost the previous
 214         # connection, and sees two connections at once.
 215         raise NotImplementedError
 216
 217
 218 def setupBuildStepStatus(basedir):
 219     """Return a BuildStep with a suitable BuildStepStatus object, ready to
 220     use."""
 221     os.mkdir(basedir)
 222     botmaster = None
 223     s0 = builder.Status(botmaster, basedir)
 224     s1 = s0.builderAdded("buildername", "buildername")
 225     s2 = builder.BuildStatus(s1, 1)
 226     s3 = builder.BuildStepStatus(s2)
 227     s3.setName("foostep")
 228     s3.started = True
 229     s3.stepStarted()
 230     return s3
 231
 232 def makeBuildStep(basedir, step_class=BuildStep, **kwargs):
 233     bss = setupBuildStepStatus(basedir)
 234
 235     ss = SourceStamp()
 236     setup = {'name': "builder1", "slavename": "bot1",
 237              'builddir': "builddir", 'factory': None}
 238     b0 = Builder(setup, bss.getBuild().getBuilder())
 239     br = BuildRequest("reason", ss)
 240     b = Build([br])
 241     b.setBuilder(b0)
 242     s = step_class(build=b, **kwargs)
 243     s.setStepStatus(bss)
 244     b.setupStatus(bss.getBuild())
 245     return s
 246
 247
 248 def findDir():
 249     # the same directory that holds this script
 250     return util.sibpath(__file__, ".")
 251
 252 class SignalMixin:
 253     sigchldHandler = None
 254
 255     def setUpClass(self):
 256         # make sure SIGCHLD handler is installed, as it should be on
 257         # reactor.run(). problem is reactor may not have been run when this
 258         # test runs.
 259         if hasattr(reactor, "_handleSigchld") and hasattr(signal, "SIGCHLD"):
 260             self.sigchldHandler = signal.signal(signal.SIGCHLD,
 261                                                 reactor._handleSigchld)
 262
 263     def tearDownClass(self):
 264         if self.sigchldHandler:
 265             signal.signal(signal.SIGCHLD, self.sigchldHandler)
 266
 267 # these classes are used to test SlaveCommands in isolation
 268
 269 class FakeSlaveBuilder:
 270     debug = False
 271     def __init__(self, usePTY, basedir):
 272         self.updates = []
 273         self.basedir = basedir
 274         self.usePTY = usePTY
 275
 276     def sendUpdate(self, data):
 277         if self.debug:
 278             print "FakeSlaveBuilder.sendUpdate", data
 279         self.updates.append(data)
 280
 281
 282 class SlaveCommandTestBase(SignalMixin):
 283     usePTY = False
 284
 285     def setUpBuilder(self, basedir):
 286         if not os.path.exists(basedir):
 287             os.mkdir(basedir)
 288         self.builder = FakeSlaveBuilder(self.usePTY, basedir)
 289
 290     def startCommand(self, cmdclass, args):
 291         stepId = 0
 292         self.cmd = c = cmdclass(self.builder, stepId, args)
 293         c.running = True
 294         d = c.doStart()
 295         return d
 296
 297     def collectUpdates(self, res=None):
 298         logs = {}
 299         for u in self.builder.updates:
 300             for k in u.keys():
 301                 if k == "log":
 302                     logname,data = u[k]
 303                     oldlog = logs.get(("log",logname), "")
 304                     logs[("log",logname)] = oldlog + data
 305                 elif k == "rc":
 306                     pass
 307                 else:
 308                     logs[k] = logs.get(k, "") + u[k]
 309         return logs
 310
 311     def findRC(self):
 312         for u in self.builder.updates:
 313             if "rc" in u:
 314                 return u["rc"]
 315         return None
 316
 317     def printStderr(self):
 318         for u in self.builder.updates:
 319             if "stderr" in u:
 320                 print u["stderr"]
 321
 322 # ----------------------------------------
 323
 324 class LocalWrapper:
 325     # r = pb.Referenceable()
 326     # w = LocalWrapper(r)
 327     # now you can do things like w.callRemote()
 328     def __init__(self, target):
 329         self.target = target
 330
 331     def callRemote(self, name, *args, **kwargs):
 332         d = defer.maybeDeferred(self._callRemote, name, *args, **kwargs)
 333         return d
 334
 335     def _callRemote(self, name, *args, **kwargs):
 336         method = getattr(self.target, "remote_"+name)
 337         return method(*args, **kwargs)
 338
 339     def notifyOnDisconnect(self, observer):
 340         pass
 341     def dontNotifyOnDisconnect(self, observer):
 342         pass
 343
 344
 345 class LocalSlaveBuilder(bot.SlaveBuilder):
 346     """I am object that behaves like a pb.RemoteReference, but in fact I
 347     invoke methods locally."""
 348     _arg_filter = None
 349
 350     def setArgFilter(self, filter):
 351         self._arg_filter = filter
 352
 353     def remote_startCommand(self, stepref, stepId, command, args):
 354         if self._arg_filter:
 355             args = self._arg_filter(args)
 356         # stepref should be a RemoteReference to the RemoteCommand
 357         return bot.SlaveBuilder.remote_startCommand(self,
 358                                                     LocalWrapper(stepref),
 359                                                     stepId, command, args)
 360
 361 class StepTester:
 362     """Utility class to exercise BuildSteps and RemoteCommands, without
 363     really using a Build or a Bot. No networks are used.
 364
 365     Use this as follows::
 366
 367     class MyTest(StepTester, unittest.TestCase):
 368         def testOne(self):
 369             self.slavebase = 'testOne.slave'
 370             self.masterbase = 'testOne.master'
 371             sb = self.makeSlaveBuilder()
 372             step = self.makeStep(stepclass, **kwargs)
 373             d = self.runStep(step)
 374             d.addCallback(_checkResults)
 375             return d
 376     """
 377
 378     #slavebase = "slavebase"
 379     slavebuilderbase = "slavebuilderbase"
 380     #masterbase = "masterbase"
 381
 382     def makeSlaveBuilder(self):
 383         os.mkdir(self.slavebase)
 384         os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase))
 385         b = bot.Bot(self.slavebase, False)
 386         b.startService()
 387         sb = LocalSlaveBuilder("slavebuildername", False)
 388         sb.setArgFilter(self.filterArgs)
 389         sb.usePTY = False
 390         sb.setServiceParent(b)
 391         sb.setBuilddir(self.slavebuilderbase)
 392         self.remote = LocalWrapper(sb)
 393         return sb
 394
 395     workdir = "build"
 396     def makeStep(self, factory, **kwargs):
 397         if not kwargs.has_key("workdir"):
 398             kwargs['workdir'] = self.workdir
 399         step = makeBuildStep(self.masterbase, factory, **kwargs)
 400         return step
 401
 402     def runStep(self, step):
 403         d = defer.maybeDeferred(step.startStep, self.remote)
 404         return d
 405
 406     def wrap(self, target):
 407         return LocalWrapper(target)
 408
 409     def filterArgs(self, args):
 410         # this can be overridden
 411         return args