buildbot/locks.py

   1 # -*- test-case-name: buildbot.test.test_locks -*-
   2
   3 from twisted.python import log
   4 from twisted.internet import reactor, defer
   5 from buildbot import util
   6
   7 if False: # for debugging
   8     debuglog = log.msg
   9 else:
  10     debuglog = lambda m: None
  11
  12 class BaseLock:
  13     description = "<BaseLock>"
  14
  15     def __init__(self, name, maxCount=1):
  16         self.name = name
  17         self.waiting = []
  18         self.owners = []
  19         self.maxCount=maxCount
  20
  21     def __repr__(self):
  22         return self.description
  23
  24     def isAvailable(self):
  25         debuglog("%s isAvailable: self.owners=%r" % (self, self.owners))
  26         return len(self.owners) < self.maxCount
  27
  28     def claim(self, owner):
  29         debuglog("%s claim(%s)" % (self, owner))
  30         assert owner is not None
  31         assert len(self.owners) < self.maxCount, "ask for isAvailable() first"
  32         self.owners.append(owner)
  33         debuglog(" %s is claimed" % (self,))
  34
  35     def release(self, owner):
  36         debuglog("%s release(%s)" % (self, owner))
  37         assert owner in self.owners
  38         self.owners.remove(owner)
  39         # who can we wake up?
  40         if self.waiting:
  41             d = self.waiting.pop(0)
  42             reactor.callLater(0, d.callback, self)
  43
  44     def waitUntilMaybeAvailable(self, owner):
  45         """Fire when the lock *might* be available. The caller will need to
  46         check with isAvailable() when the deferred fires. This loose form is
  47         used to avoid deadlocks. If we were interested in a stronger form,
  48         this would be named 'waitUntilAvailable', and the deferred would fire
  49         after the lock had been claimed.
  50         """
  51         debuglog("%s waitUntilAvailable(%s)" % (self, owner))
  52         if self.isAvailable():
  53             return defer.succeed(self)
  54         d = defer.Deferred()
  55         self.waiting.append(d)
  56         return d
  57
  58
  59 class RealMasterLock(BaseLock):
  60     def __init__(self, lockid):
  61         BaseLock.__init__(self, lockid.name, lockid.maxCount)
  62         self.description = "<MasterLock(%s, %s)>" % (self.name, self.maxCount)
  63
  64     def getLock(self, slave):
  65         return self
  66
  67 class RealSlaveLock:
  68     def __init__(self, lockid):
  69         self.name = lockid.name
  70         self.maxCount = lockid.maxCount
  71         self.maxCountForSlave = lockid.maxCountForSlave
  72         self.description = "<SlaveLock(%s, %s, %s)>" % (self.name,
  73                                                         self.maxCount,
  74                                                         self.maxCountForSlave)
  75         self.locks = {}
  76
  77     def __repr__(self):
  78         return self.description
  79
  80     def getLock(self, slavebuilder):
  81         slavename = slavebuilder.slave.slavename
  82         if not self.locks.has_key(slavename):
  83             maxCount = self.maxCountForSlave.get(slavename,
  84                                                  self.maxCount)
  85             lock = self.locks[slavename] = BaseLock(self.name, maxCount)
  86             desc = "<SlaveLock(%s, %s)[%s] %d>" % (self.name, maxCount,
  87                                                    slavename, id(lock))
  88             lock.description = desc
  89             self.locks[slavename] = lock
  90         return self.locks[slavename]
  91
  92
  93 # master.cfg should only reference the following MasterLock and SlaveLock
  94 # classes. They are identifiers that will be turned into real Locks later,
  95 # via the BotMaster.getLockByID method.
  96
  97 class MasterLock(util.ComparableMixin):
  98     """I am a semaphore that limits the number of simultaneous actions.
  99
 100     Builds and BuildSteps can declare that they wish to claim me as they run.
 101     Only a limited number of such builds or steps will be able to run
 102     simultaneously. By default this number is one, but my maxCount parameter
 103     can be raised to allow two or three or more operations to happen at the
 104     same time.
 105
 106     Use this to protect a resource that is shared among all builders and all
 107     slaves, for example to limit the load on a common SVN repository.
 108     """
 109
 110     compare_attrs = ['name', 'maxCount']
 111     lockClass = RealMasterLock
 112     def __init__(self, name, maxCount=1):
 113         self.name = name
 114         self.maxCount = maxCount
 115
 116 class SlaveLock(util.ComparableMixin):
 117     """I am a semaphore that limits simultaneous actions on each buildslave.
 118
 119     Builds and BuildSteps can declare that they wish to claim me as they run.
 120     Only a limited number of such builds or steps will be able to run
 121     simultaneously on any given buildslave. By default this number is one,
 122     but my maxCount parameter can be raised to allow two or three or more
 123     operations to happen on a single buildslave at the same time.
 124
 125     Use this to protect a resource that is shared among all the builds taking
 126     place on each slave, for example to limit CPU or memory load on an
 127     underpowered machine.
 128
 129     Each buildslave will get an independent copy of this semaphore. By
 130     default each copy will use the same owner count (set with maxCount), but
 131     you can provide maxCountForSlave with a dictionary that maps slavename to
 132     owner count, to allow some slaves more parallelism than others.
 133
 134     """
 135
 136     compare_attrs = ['name', 'maxCount', '_maxCountForSlaveList']
 137     lockClass = RealSlaveLock
 138     def __init__(self, name, maxCount=1, maxCountForSlave={}):
 139         self.name = name
 140         self.maxCount = maxCount
 141         self.maxCountForSlave = maxCountForSlave
 142         # for comparison purposes, turn this dictionary into a stably-sorted
 143         # list of tuples
 144         self._maxCountForSlaveList = self.maxCountForSlave.items()
 145         self._maxCountForSlaveList.sort()
 146         self._maxCountForSlaveList = tuple(self._maxCountForSlaveList)