buildbot/locks.py

   1 # -*- test-case-name: buildbot.test.test_locks -*-
   2
   3 from twisted.python import log
   4 from twisted.internet import reactor, defer
   5 from buildbot import util
   6
   7 if False: # for debugging
   8     def debuglog(msg):
   9         log.msg(msg)
  10 else:
  11     def debuglog(msg):
  12         pass
  13
  14 class BaseLock:
  15     description = "<BaseLock>"
  16
  17     def __init__(self, name, maxCount=1):
  18         self.name = name
  19         self.waiting = []
  20         self.owners = []
  21         self.maxCount=maxCount
  22
  23     def __repr__(self):
  24         return self.description
  25
  26     def isAvailable(self):
  27         debuglog("%s isAvailable: self.owners=%r" % (self, self.owners))
  28         return len(self.owners) < self.maxCount
  29
  30     def claim(self, owner):
  31         debuglog("%s claim(%s)" % (self, owner))
  32         assert owner is not None
  33         assert len(self.owners) < self.maxCount, "ask for isAvailable() first"
  34         self.owners.append(owner)
  35         debuglog(" %s is claimed" % (self,))
  36
  37     def release(self, owner):
  38         debuglog("%s release(%s)" % (self, owner))
  39         assert owner in self.owners
  40         self.owners.remove(owner)
  41         # who can we wake up?
  42         if self.waiting:
  43             d = self.waiting.pop(0)
  44             reactor.callLater(0, d.callback, self)
  45
  46     def waitUntilMaybeAvailable(self, owner):
  47         """Fire when the lock *might* be available. The caller will need to
  48         check with isAvailable() when the deferred fires. This loose form is
  49         used to avoid deadlocks. If we were interested in a stronger form,
  50         this would be named 'waitUntilAvailable', and the deferred would fire
  51         after the lock had been claimed.
  52         """
  53         debuglog("%s waitUntilAvailable(%s)" % (self, owner))
  54         if self.isAvailable():
  55             return defer.succeed(self)
  56         d = defer.Deferred()
  57         self.waiting.append(d)
  58         return d
  59
  60
  61 class RealMasterLock(BaseLock):
  62     def __init__(self, lockid):
  63         BaseLock.__init__(self, lockid.name, lockid.maxCount)
  64         self.description = "<MasterLock(%s, %s)>" % (self.name, self.maxCount)
  65
  66     def getLock(self, slave):
  67         return self
  68
  69 class RealSlaveLock:
  70     def __init__(self, lockid):
  71         self.name = lockid.name
  72         self.maxCount = lockid.maxCount
  73         self.maxCountForSlave = lockid.maxCountForSlave
  74         self.description = "<SlaveLock(%s, %s, %s)>" % (self.name,
  75                                                         self.maxCount,
  76                                                         self.maxCountForSlave)
  77         self.locks = {}
  78
  79     def __repr__(self):
  80         return self.description
  81
  82     def getLock(self, slavebuilder):
  83         slavename = slavebuilder.slave.slavename
  84         if not self.locks.has_key(slavename):
  85             maxCount = self.maxCountForSlave.get(slavename,
  86                                                  self.maxCount)
  87             lock = self.locks[slavename] = BaseLock(self.name, maxCount)
  88             desc = "<SlaveLock(%s, %s)[%s] %d>" % (self.name, maxCount,
  89                                                    slavename, id(lock))
  90             lock.description = desc
  91             self.locks[slavename] = lock
  92         return self.locks[slavename]
  93
  94
  95 # master.cfg should only reference the following MasterLock and SlaveLock
  96 # classes. They are identifiers that will be turned into real Locks later,
  97 # via the BotMaster.getLockByID method.
  98
  99 class MasterLock(util.ComparableMixin):
 100     """I am a semaphore that limits the number of simultaneous actions.
 101
 102     Builds and BuildSteps can declare that they wish to claim me as they run.
 103     Only a limited number of such builds or steps will be able to run
 104     simultaneously. By default this number is one, but my maxCount parameter
 105     can be raised to allow two or three or more operations to happen at the
 106     same time.
 107
 108     Use this to protect a resource that is shared among all builders and all
 109     slaves, for example to limit the load on a common SVN repository.
 110     """
 111
 112     compare_attrs = ['name', 'maxCount']
 113     lockClass = RealMasterLock
 114     def __init__(self, name, maxCount=1):
 115         self.name = name
 116         self.maxCount = maxCount
 117
 118 class SlaveLock(util.ComparableMixin):
 119     """I am a semaphore that limits simultaneous actions on each buildslave.
 120
 121     Builds and BuildSteps can declare that they wish to claim me as they run.
 122     Only a limited number of such builds or steps will be able to run
 123     simultaneously on any given buildslave. By default this number is one,
 124     but my maxCount parameter can be raised to allow two or three or more
 125     operations to happen on a single buildslave at the same time.
 126
 127     Use this to protect a resource that is shared among all the builds taking
 128     place on each slave, for example to limit CPU or memory load on an
 129     underpowered machine.
 130
 131     Each buildslave will get an independent copy of this semaphore. By
 132     default each copy will use the same owner count (set with maxCount), but
 133     you can provide maxCountForSlave with a dictionary that maps slavename to
 134     owner count, to allow some slaves more parallelism than others.
 135
 136     """
 137
 138     compare_attrs = ['name', 'maxCount', '_maxCountForSlaveList']
 139     lockClass = RealSlaveLock
 140     def __init__(self, name, maxCount=1, maxCountForSlave={}):
 141         self.name = name
 142         self.maxCount = maxCount
 143         self.maxCountForSlave = maxCountForSlave
 144         # for comparison purposes, turn this dictionary into a stably-sorted
 145         # list of tuples
 146         self._maxCountForSlaveList = self.maxCountForSlave.items()
 147         self._maxCountForSlaveList.sort()
 148         self._maxCountForSlaveList = tuple(self._maxCountForSlaveList)