buildbot/locks.py

   1 # -*- test-case-name: buildbot.test.test_locks -*-
   2
   3 from twisted.python import log
   4 from twisted.internet import reactor, defer
   5 from buildbot import util
   6
   7 if False: # for debugging
   8     debuglog = log.msg
   9 else:
  10     debuglog = lambda m: None
  11
  12 class BaseLock:
  13     """
  14     Class handling claiming and releasing of L{self}, and keeping track of
  15     current and waiting owners.
  16
  17     @note: Ideally, we'd like to maintain FIFO order. The place to do that
  18            would be the L{isAvailable()} function. However, this function is
  19            called by builds/steps both for the first time, and after waking
  20            them up by L{self} from the L{self.waiting} queue. There is
  21            currently no way of distinguishing between them.
  22     """
  23     description = "<BaseLock>"
  24
  25     def __init__(self, name, maxCount=1):
  26         self.name = name        # Name of the lock
  27         self.waiting = []       # Current queue, tuples (LockAccess, deferred)
  28         self.owners = []        # Current owners, tuples (owner, LockAccess)
  29         self.maxCount=maxCount  # maximal number of counting owners
  30
  31     def __repr__(self):
  32         return self.description
  33
  34     def _getOwnersCount(self):
  35         """ Return the number of current exclusive and counting owners.
  36
  37             @return: Tuple (number exclusive owners, number counting owners)
  38         """
  39         num_excl, num_counting = 0, 0
  40         for owner in self.owners:
  41             if owner[1].mode == 'exclusive':
  42                 num_excl = num_excl + 1
  43             else: # mode == 'counting'
  44                 num_counting = num_counting + 1
  45
  46         assert (num_excl == 1 and num_counting == 0) \
  47                 or (num_excl == 0 and num_counting <= self.maxCount)
  48         return num_excl, num_counting
  49
  50
  51     def isAvailable(self, access):
  52         """ Return a boolean whether the lock is available for claiming """
  53         debuglog("%s isAvailable(%s): self.owners=%r"
  54                                             % (self, access, self.owners))
  55         num_excl, num_counting = self._getOwnersCount()
  56         if access.mode == 'counting':
  57             # Wants counting access
  58             return num_excl == 0 and num_counting < self.maxCount
  59         else:
  60             # Wants exclusive access
  61             return num_excl == 0 and num_counting == 0
  62
  63     def claim(self, owner, access):
  64         """ Claim the lock (lock must be available) """
  65         debuglog("%s claim(%s, %s)" % (self, owner, access.mode))
  66         assert owner is not None
  67         assert self.isAvailable(access), "ask for isAvailable() first"
  68
  69         assert isinstance(access, LockAccess)
  70         assert access.mode in ['counting', 'exclusive']
  71         self.owners.append((owner, access))
  72         debuglog(" %s is claimed '%s'" % (self, access.mode))
  73
  74     def release(self, owner, access):
  75         """ Release the lock """
  76         assert isinstance(access, LockAccess)
  77
  78         debuglog("%s release(%s, %s)" % (self, owner, access.mode))
  79         entry = (owner, access)
  80         assert entry in self.owners
  81         self.owners.remove(entry)
  82         # who can we wake up?
  83         # After an exclusive access, we may need to wake up several waiting.
  84         # Break out of the loop when the first waiting client should not be awakened.
  85         num_excl, num_counting = self._getOwnersCount()
  86         while len(self.waiting) > 0:
  87             access, d = self.waiting[0]
  88             if access.mode == 'counting':
  89                 if num_excl > 0 or num_counting == self.maxCount:
  90                     break
  91                 else:
  92                     num_counting = num_counting + 1
  93             else:
  94                 # access.mode == 'exclusive'
  95                 if num_excl > 0 or num_counting > 0:
  96                     break
  97                 else:
  98                     num_excl = num_excl + 1
  99
 100             del self.waiting[0]
 101             reactor.callLater(0, d.callback, self)
 102
 103     def waitUntilMaybeAvailable(self, owner, access):
 104         """Fire when the lock *might* be available. The caller will need to
 105         check with isAvailable() when the deferred fires. This loose form is
 106         used to avoid deadlocks. If we were interested in a stronger form,
 107         this would be named 'waitUntilAvailable', and the deferred would fire
 108         after the lock had been claimed.
 109         """
 110         debuglog("%s waitUntilAvailable(%s)" % (self, owner))
 111         assert isinstance(access, LockAccess)
 112         if self.isAvailable(access):
 113             return defer.succeed(self)
 114         d = defer.Deferred()
 115         self.waiting.append((access, d))
 116         return d
 117
 118
 119 class RealMasterLock(BaseLock):
 120     def __init__(self, lockid):
 121         BaseLock.__init__(self, lockid.name, lockid.maxCount)
 122         self.description = "<MasterLock(%s, %s)>" % (self.name, self.maxCount)
 123
 124     def getLock(self, slave):
 125         return self
 126
 127 class RealSlaveLock:
 128     def __init__(self, lockid):
 129         self.name = lockid.name
 130         self.maxCount = lockid.maxCount
 131         self.maxCountForSlave = lockid.maxCountForSlave
 132         self.description = "<SlaveLock(%s, %s, %s)>" % (self.name,
 133                                                         self.maxCount,
 134                                                         self.maxCountForSlave)
 135         self.locks = {}
 136
 137     def __repr__(self):
 138         return self.description
 139
 140     def getLock(self, slavebuilder):
 141         slavename = slavebuilder.slave.slavename
 142         if not self.locks.has_key(slavename):
 143             maxCount = self.maxCountForSlave.get(slavename,
 144                                                  self.maxCount)
 145             lock = self.locks[slavename] = BaseLock(self.name, maxCount)
 146             desc = "<SlaveLock(%s, %s)[%s] %d>" % (self.name, maxCount,
 147                                                    slavename, id(lock))
 148             lock.description = desc
 149             self.locks[slavename] = lock
 150         return self.locks[slavename]
 151
 152
 153 class LockAccess:
 154     """ I am an object representing a way to access a lock.
 155
 156     @param lockid: LockId instance that should be accessed.
 157     @type  lockid: A MasterLock or SlaveLock instance.
 158
 159     @param mode: Mode of accessing the lock.
 160     @type  mode: A string, either 'counting' or 'exclusive'.
 161     """
 162     def __init__(self, lockid, mode):
 163         self.lockid = lockid
 164         self.mode = mode
 165
 166         assert isinstance(lockid, (MasterLock, SlaveLock))
 167         assert mode in ['counting', 'exclusive']
 168
 169
 170 class BaseLockId(util.ComparableMixin):
 171     """ Abstract base class for LockId classes.
 172
 173     Sets up the 'access()' function for the LockId's available to the user
 174     (MasterLock and SlaveLock classes).
 175     Derived classes should add
 176     - Comparison with the L{util.ComparableMixin} via the L{compare_attrs}
 177       class variable.
 178     - Link to the actual lock class should be added with the L{lockClass}
 179       class variable.
 180     """
 181     def access(self, mode):
 182         """ Express how the lock should be accessed """
 183         assert mode in ['counting', 'exclusive']
 184         return LockAccess(self, mode)
 185
 186     def defaultAccess(self):
 187         """ For buildbot 0.7.7 compability: When user doesn't specify an access
 188             mode, this one is chosen.
 189         """
 190         return self.access('counting')
 191
 192
 193
 194 # master.cfg should only reference the following MasterLock and SlaveLock
 195 # classes. They are identifiers that will be turned into real Locks later,
 196 # via the BotMaster.getLockByID method.
 197
 198 class MasterLock(BaseLockId):
 199     """I am a semaphore that limits the number of simultaneous actions.
 200
 201     Builds and BuildSteps can declare that they wish to claim me as they run.
 202     Only a limited number of such builds or steps will be able to run
 203     simultaneously. By default this number is one, but my maxCount parameter
 204     can be raised to allow two or three or more operations to happen at the
 205     same time.
 206
 207     Use this to protect a resource that is shared among all builders and all
 208     slaves, for example to limit the load on a common SVN repository.
 209     """
 210
 211     compare_attrs = ['name', 'maxCount']
 212     lockClass = RealMasterLock
 213     def __init__(self, name, maxCount=1):
 214         self.name = name
 215         self.maxCount = maxCount
 216
 217 class SlaveLock(BaseLockId):
 218     """I am a semaphore that limits simultaneous actions on each buildslave.
 219
 220     Builds and BuildSteps can declare that they wish to claim me as they run.
 221     Only a limited number of such builds or steps will be able to run
 222     simultaneously on any given buildslave. By default this number is one,
 223     but my maxCount parameter can be raised to allow two or three or more
 224     operations to happen on a single buildslave at the same time.
 225
 226     Use this to protect a resource that is shared among all the builds taking
 227     place on each slave, for example to limit CPU or memory load on an
 228     underpowered machine.
 229
 230     Each buildslave will get an independent copy of this semaphore. By
 231     default each copy will use the same owner count (set with maxCount), but
 232     you can provide maxCountForSlave with a dictionary that maps slavename to
 233     owner count, to allow some slaves more parallelism than others.
 234
 235     """
 236
 237     compare_attrs = ['name', 'maxCount', '_maxCountForSlaveList']
 238     lockClass = RealSlaveLock
 239     def __init__(self, name, maxCount=1, maxCountForSlave={}):
 240         self.name = name
 241         self.maxCount = maxCount
 242         self.maxCountForSlave = maxCountForSlave
 243         # for comparison purposes, turn this dictionary into a stably-sorted
 244         # list of tuples
 245         self._maxCountForSlaveList = self.maxCountForSlave.items()
 246         self._maxCountForSlaveList.sort()
 247         self._maxCountForSlaveList = tuple(self._maxCountForSlaveList)