buildbot/status/builder.py

   1 # -*- test-case-name: buildbot.test.test_status -*-
   2
   3 from zope.interface import implements
   4 from twisted.python import log
   5 from twisted.persisted import styles
   6 from twisted.internet import reactor, defer, threads
   7 from twisted.protocols import basic
   8 from buildbot.process.properties import Properties
   9
  10 import weakref
  11 import os, shutil, sys, re, urllib, itertools
  12 import gc
  13 from cPickle import load, dump
  14 from cStringIO import StringIO
  15 from bz2 import BZ2File
  16
  17 # sibling imports
  18 from buildbot import interfaces, util, sourcestamp
  19
  20 SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION = range(5)
  21 Results = ["success", "warnings", "failure", "skipped", "exception"]
  22
  23
  24 # build processes call the following methods:
  25 #
  26 #  setDefaults
  27 #
  28 #  currentlyBuilding
  29 #  currentlyIdle
  30 #  currentlyInterlocked
  31 #  currentlyOffline
  32 #  currentlyWaiting
  33 #
  34 #  setCurrentActivity
  35 #  updateCurrentActivity
  36 #  addFileToCurrentActivity
  37 #  finishCurrentActivity
  38 #
  39 #  startBuild
  40 #  finishBuild
  41
  42 STDOUT = interfaces.LOG_CHANNEL_STDOUT
  43 STDERR = interfaces.LOG_CHANNEL_STDERR
  44 HEADER = interfaces.LOG_CHANNEL_HEADER
  45 ChunkTypes = ["stdout", "stderr", "header"]
  46
  47 class LogFileScanner(basic.NetstringReceiver):
  48     def __init__(self, chunk_cb, channels=[]):
  49         self.chunk_cb = chunk_cb
  50         self.channels = channels
  51
  52     def stringReceived(self, line):
  53         channel = int(line[0])
  54         if not self.channels or (channel in self.channels):
  55             self.chunk_cb((channel, line[1:]))
  56
  57 class LogFileProducer:
  58     """What's the plan?
  59
  60     the LogFile has just one FD, used for both reading and writing.
  61     Each time you add an entry, fd.seek to the end and then write.
  62
  63     Each reader (i.e. Producer) keeps track of their own offset. The reader
  64     starts by seeking to the start of the logfile, and reading forwards.
  65     Between each hunk of file they yield chunks, so they must remember their
  66     offset before yielding and re-seek back to that offset before reading
  67     more data. When their read() returns EOF, they're finished with the first
  68     phase of the reading (everything that's already been written to disk).
  69
  70     After EOF, the remaining data is entirely in the current entries list.
  71     These entries are all of the same channel, so we can do one "".join and
  72     obtain a single chunk to be sent to the listener. But since that involves
  73     a yield, and more data might arrive after we give up control, we have to
  74     subscribe them before yielding. We can't subscribe them any earlier,
  75     otherwise they'd get data out of order.
  76
  77     We're using a generator in the first place so that the listener can
  78     throttle us, which means they're pulling. But the subscription means
  79     we're pushing. Really we're a Producer. In the first phase we can be
  80     either a PullProducer or a PushProducer. In the second phase we're only a
  81     PushProducer.
  82
  83     So the client gives a LogFileConsumer to File.subscribeConsumer . This
  84     Consumer must have registerProducer(), unregisterProducer(), and
  85     writeChunk(), and is just like a regular twisted.interfaces.IConsumer,
  86     except that writeChunk() takes chunks (tuples of (channel,text)) instead
  87     of the normal write() which takes just text. The LogFileConsumer is
  88     allowed to call stopProducing, pauseProducing, and resumeProducing on the
  89     producer instance it is given. """
  90
  91     paused = False
  92     subscribed = False
  93     BUFFERSIZE = 2048
  94
  95     def __init__(self, logfile, consumer):
  96         self.logfile = logfile
  97         self.consumer = consumer
  98         self.chunkGenerator = self.getChunks()
  99         consumer.registerProducer(self, True)
 100
 101     def getChunks(self):
 102         f = self.logfile.getFile()
 103         offset = 0
 104         chunks = []
 105         p = LogFileScanner(chunks.append)
 106         f.seek(offset)
 107         data = f.read(self.BUFFERSIZE)
 108         offset = f.tell()
 109         while data:
 110             p.dataReceived(data)
 111             while chunks:
 112                 c = chunks.pop(0)
 113                 yield c
 114             f.seek(offset)
 115             data = f.read(self.BUFFERSIZE)
 116             offset = f.tell()
 117         del f
 118
 119         # now subscribe them to receive new entries
 120         self.subscribed = True
 121         self.logfile.watchers.append(self)
 122         d = self.logfile.waitUntilFinished()
 123
 124         # then give them the not-yet-merged data
 125         if self.logfile.runEntries:
 126             channel = self.logfile.runEntries[0][0]
 127             text = "".join([c[1] for c in self.logfile.runEntries])
 128             yield (channel, text)
 129
 130         # now we've caught up to the present. Anything further will come from
 131         # the logfile subscription. We add the callback *after* yielding the
 132         # data from runEntries, because the logfile might have finished
 133         # during the yield.
 134         d.addCallback(self.logfileFinished)
 135
 136     def stopProducing(self):
 137         # TODO: should we still call consumer.finish? probably not.
 138         self.paused = True
 139         self.consumer = None
 140         self.done()
 141
 142     def done(self):
 143         if self.chunkGenerator:
 144             self.chunkGenerator = None # stop making chunks
 145         if self.subscribed:
 146             self.logfile.watchers.remove(self)
 147             self.subscribed = False
 148
 149     def pauseProducing(self):
 150         self.paused = True
 151
 152     def resumeProducing(self):
 153         # Twisted-1.3.0 has a bug which causes hangs when resumeProducing
 154         # calls transport.write (there is a recursive loop, fixed in 2.0 in
 155         # t.i.abstract.FileDescriptor.doWrite by setting the producerPaused
 156         # flag *before* calling resumeProducing). To work around this, we
 157         # just put off the real resumeProducing for a moment. This probably
 158         # has a performance hit, but I'm going to assume that the log files
 159         # are not retrieved frequently enough for it to be an issue.
 160
 161         reactor.callLater(0, self._resumeProducing)
 162
 163     def _resumeProducing(self):
 164         self.paused = False
 165         if not self.chunkGenerator:
 166             return
 167         try:
 168             while not self.paused:
 169                 chunk = self.chunkGenerator.next()
 170                 self.consumer.writeChunk(chunk)
 171                 # we exit this when the consumer says to stop, or we run out
 172                 # of chunks
 173         except StopIteration:
 174             # if the generator finished, it will have done releaseFile
 175             self.chunkGenerator = None
 176         # now everything goes through the subscription, and they don't get to
 177         # pause anymore
 178
 179     def logChunk(self, build, step, logfile, channel, chunk):
 180         if self.consumer:
 181             self.consumer.writeChunk((channel, chunk))
 182
 183     def logfileFinished(self, logfile):
 184         self.done()
 185         if self.consumer:
 186             self.consumer.unregisterProducer()
 187             self.consumer.finish()
 188             self.consumer = None
 189
 190 def _tryremove(filename, timeout, retries):
 191     """Try to remove a file, and if failed, try again in timeout.
 192     Increases the timeout by a factor of 4, and only keeps trying for
 193     another retries-amount of times.
 194
 195     """
 196     try:
 197         os.unlink(filename)
 198     except OSError:
 199         if retries > 0:
 200             reactor.callLater(timeout, _tryremove, filename, timeout * 4,
 201                               retries - 1)
 202         else:
 203             log.msg("giving up on removing %s after over %d seconds" %
 204                     (filename, timeout))
 205
 206 class LogFile:
 207     """A LogFile keeps all of its contents on disk, in a non-pickle format to
 208     which new entries can easily be appended. The file on disk has a name
 209     like 12-log-compile-output, under the Builder's directory. The actual
 210     filename is generated (before the LogFile is created) by
 211     L{BuildStatus.generateLogfileName}.
 212
 213     Old LogFile pickles (which kept their contents in .entries) must be
 214     upgraded. The L{BuilderStatus} is responsible for doing this, when it
 215     loads the L{BuildStatus} into memory. The Build pickle is not modified,
 216     so users who go from 0.6.5 back to 0.6.4 don't have to lose their
 217     logs."""
 218
 219     implements(interfaces.IStatusLog, interfaces.ILogFile)
 220
 221     finished = False
 222     length = 0
 223     chunkSize = 10*1000
 224     runLength = 0
 225     runEntries = [] # provided so old pickled builds will getChunks() ok
 226     entries = None
 227     BUFFERSIZE = 2048
 228     filename = None # relative to the Builder's basedir
 229     openfile = None
 230
 231     def __init__(self, parent, name, logfilename):
 232         """
 233         @type  parent: L{BuildStepStatus}
 234         @param parent: the Step that this log is a part of
 235         @type  name: string
 236         @param name: the name of this log, typically 'output'
 237         @type  logfilename: string
 238         @param logfilename: the Builder-relative pathname for the saved entries
 239         """
 240         self.step = parent
 241         self.name = name
 242         self.filename = logfilename
 243         fn = self.getFilename()
 244         if os.path.exists(fn):
 245             # the buildmaster was probably stopped abruptly, before the
 246             # BuilderStatus could be saved, so BuilderStatus.nextBuildNumber
 247             # is out of date, and we're overlapping with earlier builds now.
 248             # Warn about it, but then overwrite the old pickle file
 249             log.msg("Warning: Overwriting old serialized Build at %s" % fn)
 250         self.openfile = open(fn, "w+")
 251         self.runEntries = []
 252         self.watchers = []
 253         self.finishedWatchers = []
 254
 255     def getFilename(self):
 256         return os.path.join(self.step.build.builder.basedir, self.filename)
 257
 258     def hasContents(self):
 259         return os.path.exists(self.getFilename() + '.bz2') or \
 260             os.path.exists(self.getFilename())
 261
 262     def getName(self):
 263         return self.name
 264
 265     def getStep(self):
 266         return self.step
 267
 268     def isFinished(self):
 269         return self.finished
 270     def waitUntilFinished(self):
 271         if self.finished:
 272             d = defer.succeed(self)
 273         else:
 274             d = defer.Deferred()
 275             self.finishedWatchers.append(d)
 276         return d
 277
 278     def logfileExists(self):
 279         if self.openfile: return True
 280         fn = self.getFilename()
 281         for f in (fn, fn + ".bz2"):
 282             if os.path.exists(f): return True
 283         return False
 284
 285     def getFile(self):
 286         if self.openfile:
 287             # this is the filehandle we're using to write to the log, so
 288             # don't close it!
 289             return self.openfile
 290         # otherwise they get their own read-only handle
 291         # try a compressed log first
 292         try:
 293             return BZ2File(self.getFilename() + ".bz2", "r")
 294         except IOError:
 295             pass
 296         return open(self.getFilename(), "r")
 297
 298     def getText(self):
 299         # this produces one ginormous string
 300         return "".join(self.getChunks([STDOUT, STDERR], onlyText=True))
 301
 302     def getTextWithHeaders(self):
 303         return "".join(self.getChunks(onlyText=True))
 304
 305     def getChunks(self, channels=[], onlyText=False):
 306         # generate chunks for everything that was logged at the time we were
 307         # first called, so remember how long the file was when we started.
 308         # Don't read beyond that point. The current contents of
 309         # self.runEntries will follow.
 310
 311         # this returns an iterator, which means arbitrary things could happen
 312         # while we're yielding. This will faithfully deliver the log as it
 313         # existed when it was started, and not return anything after that
 314         # point. To use this in subscribe(catchup=True) without missing any
 315         # data, you must insure that nothing will be added to the log during
 316         # yield() calls.
 317
 318         f = self.getFile()
 319         offset = 0
 320         f.seek(0, 2)
 321         remaining = f.tell()
 322
 323         leftover = None
 324         if self.runEntries and (not channels or
 325                                 (self.runEntries[0][0] in channels)):
 326             leftover = (self.runEntries[0][0],
 327                         "".join([c[1] for c in self.runEntries]))
 328
 329         # freeze the state of the LogFile by passing a lot of parameters into
 330         # a generator
 331         return self._generateChunks(f, offset, remaining, leftover,
 332                                     channels, onlyText)
 333
 334     def _generateChunks(self, f, offset, remaining, leftover,
 335                         channels, onlyText):
 336         chunks = []
 337         p = LogFileScanner(chunks.append, channels)
 338         f.seek(offset)
 339         data = f.read(min(remaining, self.BUFFERSIZE))
 340         remaining -= len(data)
 341         offset = f.tell()
 342         while data:
 343             p.dataReceived(data)
 344             while chunks:
 345                 channel, text = chunks.pop(0)
 346                 if onlyText:
 347                     yield text
 348                 else:
 349                     yield (channel, text)
 350             f.seek(offset)
 351             data = f.read(min(remaining, self.BUFFERSIZE))
 352             remaining -= len(data)
 353             offset = f.tell()
 354         del f
 355
 356         if leftover:
 357             if onlyText:
 358                 yield leftover[1]
 359             else:
 360                 yield leftover
 361
 362     def readlines(self, channel=STDOUT):
 363         """Return an iterator that produces newline-terminated lines,
 364         excluding header chunks."""
 365         # TODO: make this memory-efficient, by turning it into a generator
 366         # that retrieves chunks as necessary, like a pull-driven version of
 367         # twisted.protocols.basic.LineReceiver
 368         alltext = "".join(self.getChunks([channel], onlyText=True))
 369         io = StringIO(alltext)
 370         return io.readlines()
 371
 372     def subscribe(self, receiver, catchup):
 373         if self.finished:
 374             return
 375         self.watchers.append(receiver)
 376         if catchup:
 377             for channel, text in self.getChunks():
 378                 # TODO: add logChunks(), to send over everything at once?
 379                 receiver.logChunk(self.step.build, self.step, self,
 380                                   channel, text)
 381
 382     def unsubscribe(self, receiver):
 383         if receiver in self.watchers:
 384             self.watchers.remove(receiver)
 385
 386     def subscribeConsumer(self, consumer):
 387         p = LogFileProducer(self, consumer)
 388         p.resumeProducing()
 389
 390     # interface used by the build steps to add things to the log
 391
 392     def merge(self):
 393         # merge all .runEntries (which are all of the same type) into a
 394         # single chunk for .entries
 395         if not self.runEntries:
 396             return
 397         channel = self.runEntries[0][0]
 398         text = "".join([c[1] for c in self.runEntries])
 399         assert channel < 10
 400         f = self.openfile
 401         f.seek(0, 2)
 402         offset = 0
 403         while offset < len(text):
 404             size = min(len(text)-offset, self.chunkSize)
 405             f.write("%d:%d" % (1 + size, channel))
 406             f.write(text[offset:offset+size])
 407             f.write(",")
 408             offset += size
 409         self.runEntries = []
 410         self.runLength = 0
 411
 412     def addEntry(self, channel, text):
 413         assert not self.finished
 414         # we only add to .runEntries here. merge() is responsible for adding
 415         # merged chunks to .entries
 416         if self.runEntries and channel != self.runEntries[0][0]:
 417             self.merge()
 418         self.runEntries.append((channel, text))
 419         self.runLength += len(text)
 420         if self.runLength >= self.chunkSize:
 421             self.merge()
 422
 423         for w in self.watchers:
 424             w.logChunk(self.step.build, self.step, self, channel, text)
 425         self.length += len(text)
 426
 427     def addStdout(self, text):
 428         self.addEntry(STDOUT, text)
 429     def addStderr(self, text):
 430         self.addEntry(STDERR, text)
 431     def addHeader(self, text):
 432         self.addEntry(HEADER, text)
 433
 434     def finish(self):
 435         self.merge()
 436         if self.openfile:
 437             # we don't do an explicit close, because there might be readers
 438             # shareing the filehandle. As soon as they stop reading, the
 439             # filehandle will be released and automatically closed. We will
 440             # do a sync, however, to make sure the log gets saved in case of
 441             # a crash.
 442             self.openfile.flush()
 443             os.fsync(self.openfile.fileno())
 444             del self.openfile
 445         self.finished = True
 446         watchers = self.finishedWatchers
 447         self.finishedWatchers = []
 448         for w in watchers:
 449             w.callback(self)
 450         self.watchers = []
 451
 452
 453     def compressLog(self):
 454         compressed = self.getFilename() + ".bz2.tmp"
 455         d = threads.deferToThread(self._compressLog, compressed)
 456         d.addCallback(self._renameCompressedLog, compressed)
 457         d.addErrback(self._cleanupFailedCompress, compressed)
 458         return d
 459
 460     def _compressLog(self, compressed):
 461         infile = self.getFile()
 462         cf = BZ2File(compressed, 'w')
 463         bufsize = 1024*1024
 464         while True:
 465             buf = infile.read(bufsize)
 466             cf.write(buf)
 467             if len(buf) < bufsize:
 468                 break
 469         cf.close()
 470     def _renameCompressedLog(self, rv, compressed):
 471         filename = self.getFilename() + '.bz2'
 472         if sys.platform == 'win32':
 473             # windows cannot rename a file on top of an existing one, so
 474             # fall back to delete-first. There are ways this can fail and
 475             # lose the builder's history, so we avoid using it in the
 476             # general (non-windows) case
 477             if os.path.exists(filename):
 478                 os.unlink(filename)
 479         os.rename(compressed, filename)
 480         _tryremove(self.getFilename(), 1, 5)
 481     def _cleanupFailedCompress(self, failure, compressed):
 482         log.msg("failed to compress %s" % self.getFilename())
 483         if os.path.exists(compressed):
 484             _tryremove(compressed, 1, 5)
 485         failure.trap() # reraise the failure
 486
 487     # persistence stuff
 488     def __getstate__(self):
 489         d = self.__dict__.copy()
 490         del d['step'] # filled in upon unpickling
 491         del d['watchers']
 492         del d['finishedWatchers']
 493         d['entries'] = [] # let 0.6.4 tolerate the saved log. TODO: really?
 494         if d.has_key('finished'):
 495             del d['finished']
 496         if d.has_key('openfile'):
 497             del d['openfile']
 498         return d
 499
 500     def __setstate__(self, d):
 501         self.__dict__ = d
 502         self.watchers = [] # probably not necessary
 503         self.finishedWatchers = [] # same
 504         # self.step must be filled in by our parent
 505         self.finished = True
 506
 507     def upgrade(self, logfilename):
 508         """Save our .entries to a new-style offline log file (if necessary),
 509         and modify our in-memory representation to use it. The original
 510         pickled LogFile (inside the pickled Build) won't be modified."""
 511         self.filename = logfilename
 512         if not os.path.exists(self.getFilename()):
 513             self.openfile = open(self.getFilename(), "w")
 514             self.finished = False
 515             for channel,text in self.entries:
 516                 self.addEntry(channel, text)
 517             self.finish() # releases self.openfile, which will be closed
 518         del self.entries
 519
 520 class HTMLLogFile:
 521     implements(interfaces.IStatusLog)
 522
 523     filename = None
 524
 525     def __init__(self, parent, name, logfilename, html):
 526         self.step = parent
 527         self.name = name
 528         self.filename = logfilename
 529         self.html = html
 530
 531     def getName(self):
 532         return self.name # set in BuildStepStatus.addLog
 533     def getStep(self):
 534         return self.step
 535
 536     def isFinished(self):
 537         return True
 538     def waitUntilFinished(self):
 539         return defer.succeed(self)
 540
 541     def hasContents(self):
 542         return True
 543     def getText(self):
 544         return self.html # looks kinda like text
 545     def getTextWithHeaders(self):
 546         return self.html
 547     def getChunks(self):
 548         return [(STDERR, self.html)]
 549
 550     def subscribe(self, receiver, catchup):
 551         pass
 552     def unsubscribe(self, receiver):
 553         pass
 554
 555     def finish(self):
 556         pass
 557
 558     def __getstate__(self):
 559         d = self.__dict__.copy()
 560         del d['step']
 561         return d
 562
 563     def upgrade(self, logfilename):
 564         pass
 565
 566
 567 class Event:
 568     implements(interfaces.IStatusEvent)
 569
 570     started = None
 571     finished = None
 572     text = []
 573
 574     # IStatusEvent methods
 575     def getTimes(self):
 576         return (self.started, self.finished)
 577     def getText(self):
 578         return self.text
 579     def getLogs(self):
 580         return []
 581
 582     def finish(self):
 583         self.finished = util.now()
 584
 585 class TestResult:
 586     implements(interfaces.ITestResult)
 587
 588     def __init__(self, name, results, text, logs):
 589         assert isinstance(name, tuple)
 590         self.name = name
 591         self.results = results
 592         self.text = text
 593         self.logs = logs
 594
 595     def getName(self):
 596         return self.name
 597
 598     def getResults(self):
 599         return self.results
 600
 601     def getText(self):
 602         return self.text
 603
 604     def getLogs(self):
 605         return self.logs
 606
 607
 608 class BuildSetStatus:
 609     implements(interfaces.IBuildSetStatus)
 610
 611     def __init__(self, source, reason, builderNames, bsid=None):
 612         self.source = source
 613         self.reason = reason
 614         self.builderNames = builderNames
 615         self.id = bsid
 616         self.successWatchers = []
 617         self.finishedWatchers = []
 618         self.stillHopeful = True
 619         self.finished = False
 620
 621     def setBuildRequestStatuses(self, buildRequestStatuses):
 622         self.buildRequests = buildRequestStatuses
 623     def setResults(self, results):
 624         # the build set succeeds only if all its component builds succeed
 625         self.results = results
 626     def giveUpHope(self):
 627         self.stillHopeful = False
 628
 629
 630     def notifySuccessWatchers(self):
 631         for d in self.successWatchers:
 632             d.callback(self)
 633         self.successWatchers = []
 634
 635     def notifyFinishedWatchers(self):
 636         self.finished = True
 637         for d in self.finishedWatchers:
 638             d.callback(self)
 639         self.finishedWatchers = []
 640
 641     # methods for our clients
 642
 643     def getSourceStamp(self):
 644         return self.source
 645     def getReason(self):
 646         return self.reason
 647     def getResults(self):
 648         return self.results
 649     def getID(self):
 650         return self.id
 651
 652     def getBuilderNames(self):
 653         return self.builderNames
 654     def getBuildRequests(self):
 655         return self.buildRequests
 656     def isFinished(self):
 657         return self.finished
 658
 659     def waitUntilSuccess(self):
 660         if self.finished or not self.stillHopeful:
 661             # the deferreds have already fired
 662             return defer.succeed(self)
 663         d = defer.Deferred()
 664         self.successWatchers.append(d)
 665         return d
 666
 667     def waitUntilFinished(self):
 668         if self.finished:
 669             return defer.succeed(self)
 670         d = defer.Deferred()
 671         self.finishedWatchers.append(d)
 672         return d
 673
 674 class BuildRequestStatus:
 675     implements(interfaces.IBuildRequestStatus)
 676
 677     def __init__(self, source, builderName):
 678         self.source = source
 679         self.builderName = builderName
 680         self.builds = [] # list of BuildStatus objects
 681         self.observers = []
 682         self.submittedAt = None
 683
 684     def buildStarted(self, build):
 685         self.builds.append(build)
 686         for o in self.observers[:]:
 687             o(build)
 688
 689     # methods called by our clients
 690     def getSourceStamp(self):
 691         return self.source
 692     def getBuilderName(self):
 693         return self.builderName
 694     def getBuilds(self):
 695         return self.builds
 696
 697     def subscribe(self, observer):
 698         self.observers.append(observer)
 699         for b in self.builds:
 700             observer(b)
 701     def unsubscribe(self, observer):
 702         self.observers.remove(observer)
 703
 704     def getSubmitTime(self):
 705         return self.submittedAt
 706     def setSubmitTime(self, t):
 707         self.submittedAt = t
 708
 709
 710 class BuildStepStatus(styles.Versioned):
 711     """
 712     I represent a collection of output status for a
 713     L{buildbot.process.step.BuildStep}.
 714
 715     Statistics contain any information gleaned from a step that is
 716     not in the form of a logfile.  As an example, steps that run
 717     tests might gather statistics about the number of passed, failed,
 718     or skipped tests.
 719
 720     @type progress: L{buildbot.status.progress.StepProgress}
 721     @cvar progress: tracks ETA for the step
 722     @type text: list of strings
 723     @cvar text: list of short texts that describe the command and its status
 724     @type text2: list of strings
 725     @cvar text2: list of short texts added to the overall build description
 726     @type logs: dict of string -> L{buildbot.status.builder.LogFile}
 727     @ivar logs: logs of steps
 728     @type statistics: dict
 729     @ivar statistics: results from running this step
 730     """
 731     # note that these are created when the Build is set up, before each
 732     # corresponding BuildStep has started.
 733     implements(interfaces.IBuildStepStatus, interfaces.IStatusEvent)
 734     persistenceVersion = 2
 735
 736     started = None
 737     finished = None
 738     progress = None
 739     text = []
 740     results = (None, [])
 741     text2 = []
 742     watchers = []
 743     updates = {}
 744     finishedWatchers = []
 745     statistics = {}
 746
 747     def __init__(self, parent):
 748         assert interfaces.IBuildStatus(parent)
 749         self.build = parent
 750         self.logs = []
 751         self.urls = {}
 752         self.watchers = []
 753         self.updates = {}
 754         self.finishedWatchers = []
 755         self.statistics = {}
 756
 757     def getName(self):
 758         """Returns a short string with the name of this step. This string
 759         may have spaces in it."""
 760         return self.name
 761
 762     def getBuild(self):
 763         return self.build
 764
 765     def getTimes(self):
 766         return (self.started, self.finished)
 767
 768     def getExpectations(self):
 769         """Returns a list of tuples (name, current, target)."""
 770         if not self.progress:
 771             return []
 772         ret = []
 773         metrics = self.progress.progress.keys()
 774         metrics.sort()
 775         for m in metrics:
 776             t = (m, self.progress.progress[m], self.progress.expectations[m])
 777             ret.append(t)
 778         return ret
 779
 780     def getLogs(self):
 781         return self.logs
 782
 783     def getURLs(self):
 784         return self.urls.copy()
 785
 786     def isFinished(self):
 787         return (self.finished is not None)
 788
 789     def waitUntilFinished(self):
 790         if self.finished:
 791             d = defer.succeed(self)
 792         else:
 793             d = defer.Deferred()
 794             self.finishedWatchers.append(d)
 795         return d
 796
 797     # while the step is running, the following methods make sense.
 798     # Afterwards they return None
 799
 800     def getETA(self):
 801         if self.started is None:
 802             return None # not started yet
 803         if self.finished is not None:
 804             return None # already finished
 805         if not self.progress:
 806             return None # no way to predict
 807         return self.progress.remaining()
 808
 809     # Once you know the step has finished, the following methods are legal.
 810     # Before this step has finished, they all return None.
 811
 812     def getText(self):
 813         """Returns a list of strings which describe the step. These are
 814         intended to be displayed in a narrow column. If more space is
 815         available, the caller should join them together with spaces before
 816         presenting them to the user."""
 817         return self.text
 818
 819     def getResults(self):
 820         """Return a tuple describing the results of the step.
 821         'result' is one of the constants in L{buildbot.status.builder}:
 822         SUCCESS, WARNINGS, FAILURE, or SKIPPED.
 823         'strings' is an optional list of strings that the step wants to
 824         append to the overall build's results. These strings are usually
 825         more terse than the ones returned by getText(): in particular,
 826         successful Steps do not usually contribute any text to the
 827         overall build.
 828
 829         @rtype:   tuple of int, list of strings
 830         @returns: (result, strings)
 831         """
 832         return (self.results, self.text2)
 833
 834     def hasStatistic(self, name):
 835         """Return true if this step has a value for the given statistic.
 836         """
 837         return self.statistics.has_key(name)
 838
 839     def getStatistic(self, name, default=None):
 840         """Return the given statistic, if present
 841         """
 842         return self.statistics.get(name, default)
 843
 844     # subscription interface
 845
 846     def subscribe(self, receiver, updateInterval=10):
 847         # will get logStarted, logFinished, stepETAUpdate
 848         assert receiver not in self.watchers
 849         self.watchers.append(receiver)
 850         self.sendETAUpdate(receiver, updateInterval)
 851
 852     def sendETAUpdate(self, receiver, updateInterval):
 853         self.updates[receiver] = None
 854         # they might unsubscribe during stepETAUpdate
 855         receiver.stepETAUpdate(self.build, self,
 856                            self.getETA(), self.getExpectations())
 857         if receiver in self.watchers:
 858             self.updates[receiver] = reactor.callLater(updateInterval,
 859                                                        self.sendETAUpdate,
 860                                                        receiver,
 861                                                        updateInterval)
 862
 863     def unsubscribe(self, receiver):
 864         if receiver in self.watchers:
 865             self.watchers.remove(receiver)
 866         if receiver in self.updates:
 867             if self.updates[receiver] is not None:
 868                 self.updates[receiver].cancel()
 869             del self.updates[receiver]
 870
 871
 872     # methods to be invoked by the BuildStep
 873
 874     def setName(self, stepname):
 875         self.name = stepname
 876
 877     def setColor(self, color):
 878         log.msg("BuildStepStatus.setColor is no longer supported -- ignoring color %s" % (color,))
 879
 880     def setProgress(self, stepprogress):
 881         self.progress = stepprogress
 882
 883     def stepStarted(self):
 884         self.started = util.now()
 885         if self.build:
 886             self.build.stepStarted(self)
 887
 888     def addLog(self, name):
 889         assert self.started # addLog before stepStarted won't notify watchers
 890         logfilename = self.build.generateLogfileName(self.name, name)
 891         log = LogFile(self, name, logfilename)
 892         self.logs.append(log)
 893         for w in self.watchers:
 894             receiver = w.logStarted(self.build, self, log)
 895             if receiver:
 896                 log.subscribe(receiver, True)
 897                 d = log.waitUntilFinished()
 898                 d.addCallback(lambda log: log.unsubscribe(receiver))
 899         d = log.waitUntilFinished()
 900         d.addCallback(self.logFinished)
 901         return log
 902
 903     def addHTMLLog(self, name, html):
 904         assert self.started # addLog before stepStarted won't notify watchers
 905         logfilename = self.build.generateLogfileName(self.name, name)
 906         log = HTMLLogFile(self, name, logfilename, html)
 907         self.logs.append(log)
 908         for w in self.watchers:
 909             receiver = w.logStarted(self.build, self, log)
 910             # TODO: think about this: there isn't much point in letting
 911             # them subscribe
 912             #if receiver:
 913             #    log.subscribe(receiver, True)
 914             w.logFinished(self.build, self, log)
 915
 916     def logFinished(self, log):
 917         for w in self.watchers:
 918             w.logFinished(self.build, self, log)
 919
 920     def addURL(self, name, url):
 921         self.urls[name] = url
 922
 923     def setText(self, text):
 924         self.text = text
 925         for w in self.watchers:
 926             w.stepTextChanged(self.build, self, text)
 927     def setText2(self, text):
 928         self.text2 = text
 929         for w in self.watchers:
 930             w.stepText2Changed(self.build, self, text)
 931
 932     def setStatistic(self, name, value):
 933         """Set the given statistic.  Usually called by subclasses.
 934         """
 935         self.statistics[name] = value
 936
 937     def stepFinished(self, results):
 938         self.finished = util.now()
 939         self.results = results
 940         cld = [] # deferreds for log compression
 941         logCompressionLimit = self.build.builder.logCompressionLimit
 942         for loog in self.logs:
 943             if not loog.isFinished():
 944                 loog.finish()
 945             # if log compression is on, and it's a real LogFile,
 946             # HTMLLogFiles aren't files
 947             if logCompressionLimit is not False and \
 948                     isinstance(loog, LogFile):
 949                 if os.path.getsize(loog.getFilename()) > logCompressionLimit:
 950                     cld.append(loog.compressLog())
 951
 952         for r in self.updates.keys():
 953             if self.updates[r] is not None:
 954                 self.updates[r].cancel()
 955                 del self.updates[r]
 956
 957         watchers = self.finishedWatchers
 958         self.finishedWatchers = []
 959         for w in watchers:
 960             w.callback(self)
 961         if cld:
 962             return defer.DeferredList(cld)
 963
 964     def checkLogfiles(self):
 965         # filter out logs that have been deleted
 966         self.logs = [ l for l in self.logs if l.logfileExists() ]
 967
 968     # persistence
 969
 970     def __getstate__(self):
 971         d = styles.Versioned.__getstate__(self)
 972         del d['build'] # filled in when loading
 973         if d.has_key('progress'):
 974             del d['progress']
 975         del d['watchers']
 976         del d['finishedWatchers']
 977         del d['updates']
 978         return d
 979
 980     def __setstate__(self, d):
 981         styles.Versioned.__setstate__(self, d)
 982         # self.build must be filled in by our parent
 983
 984         # point the logs to this object
 985         for loog in self.logs:
 986             loog.step = self
 987
 988     def upgradeToVersion1(self):
 989         if not hasattr(self, "urls"):
 990             self.urls = {}
 991
 992     def upgradeToVersion2(self):
 993         if not hasattr(self, "statistics"):
 994             self.statistics = {}
 995
 996
 997 class BuildStatus(styles.Versioned):
 998     implements(interfaces.IBuildStatus, interfaces.IStatusEvent)
 999     persistenceVersion = 3
1000
1001     source = None
1002     reason = None
1003     changes = []
1004     blamelist = []
1005     requests = []
1006     progress = None
1007     started = None
1008     finished = None
1009     currentStep = None
1010     text = []
1011     results = None
1012     slavename = "???"
1013
1014     # these lists/dicts are defined here so that unserialized instances have
1015     # (empty) values. They are set in __init__ to new objects to make sure
1016     # each instance gets its own copy.
1017     watchers = []
1018     updates = {}
1019     finishedWatchers = []
1020     testResults = {}
1021
1022     def __init__(self, parent, number):
1023         """
1024         @type  parent: L{BuilderStatus}
1025         @type  number: int
1026         """
1027         assert interfaces.IBuilderStatus(parent)
1028         self.builder = parent
1029         self.number = number
1030         self.watchers = []
1031         self.updates = {}
1032         self.finishedWatchers = []
1033         self.steps = []
1034         self.testResults = {}
1035         self.properties = Properties()
1036         self.requests = []
1037
1038     def __repr__(self):
1039         return "<%s #%s>" % (self.__class__.__name__, self.number)
1040
1041     # IBuildStatus
1042
1043     def getBuilder(self):
1044         """
1045         @rtype: L{BuilderStatus}
1046         """
1047         return self.builder
1048
1049     def getProperty(self, propname):
1050         return self.properties[propname]
1051
1052     def getProperties(self):
1053         return self.properties
1054
1055     def getNumber(self):
1056         return self.number
1057
1058     def getPreviousBuild(self):
1059         if self.number == 0:
1060             return None
1061         return self.builder.getBuild(self.number-1)
1062
1063     def getSourceStamp(self, absolute=False):
1064         if not absolute or not self.properties.has_key('got_revision'):
1065             return self.source
1066         return self.source.getAbsoluteSourceStamp(self.properties['got_revision'])
1067
1068     def getReason(self):
1069         return self.reason
1070
1071     def getChanges(self):
1072         return self.changes
1073
1074     def getRequests(self):
1075         return self.requests
1076
1077     def getResponsibleUsers(self):
1078         return self.blamelist
1079
1080     def getInterestedUsers(self):
1081         # TODO: the Builder should add others: sheriffs, domain-owners
1082         return self.blamelist + self.properties.getProperty('owners', [])
1083
1084     def getSteps(self):
1085         """Return a list of IBuildStepStatus objects. For invariant builds
1086         (those which always use the same set of Steps), this should be the
1087         complete list, however some of the steps may not have started yet
1088         (step.getTimes()[0] will be None). For variant builds, this may not
1089         be complete (asking again later may give you more of them)."""
1090         return self.steps
1091
1092     def getTimes(self):
1093         return (self.started, self.finished)
1094
1095     _sentinel = [] # used as a sentinel to indicate unspecified initial_value
1096     def getSummaryStatistic(self, name, summary_fn, initial_value=_sentinel):
1097         """Summarize the named statistic over all steps in which it
1098         exists, using combination_fn and initial_value to combine multiple
1099         results into a single result.  This translates to a call to Python's
1100         X{reduce}::
1101             return reduce(summary_fn, step_stats_list, initial_value)
1102         """
1103         step_stats_list = [
1104                 st.getStatistic(name)
1105                 for st in self.steps
1106                 if st.hasStatistic(name) ]
1107         if initial_value is self._sentinel:
1108             return reduce(summary_fn, step_stats_list)
1109         else:
1110             return reduce(summary_fn, step_stats_list, initial_value)
1111
1112     def isFinished(self):
1113         return (self.finished is not None)
1114
1115     def waitUntilFinished(self):
1116         if self.finished:
1117             d = defer.succeed(self)
1118         else:
1119             d = defer.Deferred()
1120             self.finishedWatchers.append(d)
1121         return d
1122
1123     # while the build is running, the following methods make sense.
1124     # Afterwards they return None
1125
1126     def getETA(self):
1127         if self.finished is not None:
1128             return None
1129         if not self.progress:
1130             return None
1131         eta = self.progress.eta()
1132         if eta is None:
1133             return None
1134         return eta - util.now()
1135
1136     def getCurrentStep(self):
1137         return self.currentStep
1138
1139     # Once you know the build has finished, the following methods are legal.
1140     # Before ths build has finished, they all return None.
1141
1142     def getText(self):
1143         text = []
1144         text.extend(self.text)
1145         for s in self.steps:
1146             text.extend(s.text2)
1147         return text
1148
1149     def getResults(self):
1150         return self.results
1151
1152     def getSlavename(self):
1153         return self.slavename
1154
1155     def getTestResults(self):
1156         return self.testResults
1157
1158     def getLogs(self):
1159         # TODO: steps should contribute significant logs instead of this
1160         # hack, which returns every log from every step. The logs should get
1161         # names like "compile" and "test" instead of "compile.output"
1162         logs = []
1163         for s in self.steps:
1164             for log in s.getLogs():
1165                 logs.append(log)
1166         return logs
1167
1168     # subscription interface
1169
1170     def subscribe(self, receiver, updateInterval=None):
1171         # will receive stepStarted and stepFinished messages
1172         # and maybe buildETAUpdate
1173         self.watchers.append(receiver)
1174         if updateInterval is not None:
1175             self.sendETAUpdate(receiver, updateInterval)
1176
1177     def sendETAUpdate(self, receiver, updateInterval):
1178         self.updates[receiver] = None
1179         ETA = self.getETA()
1180         if ETA is not None:
1181             receiver.buildETAUpdate(self, self.getETA())
1182         # they might have unsubscribed during buildETAUpdate
1183         if receiver in self.watchers:
1184             self.updates[receiver] = reactor.callLater(updateInterval,
1185                                                        self.sendETAUpdate,
1186                                                        receiver,
1187                                                        updateInterval)
1188
1189     def unsubscribe(self, receiver):
1190         if receiver in self.watchers:
1191             self.watchers.remove(receiver)
1192         if receiver in self.updates:
1193             if self.updates[receiver] is not None:
1194                 self.updates[receiver].cancel()
1195             del self.updates[receiver]
1196
1197     # methods for the base.Build to invoke
1198
1199     def addStepWithName(self, name):
1200         """The Build is setting up, and has added a new BuildStep to its
1201         list. Create a BuildStepStatus object to which it can send status
1202         updates."""
1203
1204         s = BuildStepStatus(self)
1205         s.setName(name)
1206         self.steps.append(s)
1207         return s
1208
1209     def setProperty(self, propname, value, source):
1210         self.properties.setProperty(propname, value, source)
1211
1212     def addTestResult(self, result):
1213         self.testResults[result.getName()] = result
1214
1215     def setSourceStamp(self, sourceStamp):
1216         self.source = sourceStamp
1217         self.changes = self.source.changes
1218
1219     def setRequests(self, requests):
1220         self.requests = requests
1221
1222     def setReason(self, reason):
1223         self.reason = reason
1224     def setBlamelist(self, blamelist):
1225         self.blamelist = blamelist
1226     def setProgress(self, progress):
1227         self.progress = progress
1228
1229     def buildStarted(self, build):
1230         """The Build has been set up and is about to be started. It can now
1231         be safely queried, so it is time to announce the new build."""
1232
1233         self.started = util.now()
1234         # now that we're ready to report status, let the BuilderStatus tell
1235         # the world about us
1236         self.builder.buildStarted(self)
1237
1238     def setSlavename(self, slavename):
1239         self.slavename = slavename
1240
1241     def setText(self, text):
1242         assert isinstance(text, (list, tuple))
1243         self.text = text
1244     def setResults(self, results):
1245         self.results = results
1246
1247     def buildFinished(self):
1248         self.currentStep = None
1249         self.finished = util.now()
1250
1251         for r in self.updates.keys():
1252             if self.updates[r] is not None:
1253                 self.updates[r].cancel()
1254                 del self.updates[r]
1255
1256         watchers = self.finishedWatchers
1257         self.finishedWatchers = []
1258         for w in watchers:
1259             w.callback(self)
1260
1261     # methods called by our BuildStepStatus children
1262
1263     def stepStarted(self, step):
1264         self.currentStep = step
1265         name = self.getBuilder().getName()
1266         for w in self.watchers:
1267             receiver = w.stepStarted(self, step)
1268             if receiver:
1269                 if type(receiver) == type(()):
1270                     step.subscribe(receiver[0], receiver[1])
1271                 else:
1272                     step.subscribe(receiver)
1273                 d = step.waitUntilFinished()
1274                 d.addCallback(lambda step: step.unsubscribe(receiver))
1275
1276         step.waitUntilFinished().addCallback(self._stepFinished)
1277
1278     def _stepFinished(self, step):
1279         results = step.getResults()
1280         for w in self.watchers:
1281             w.stepFinished(self, step, results)
1282
1283     # methods called by our BuilderStatus parent
1284
1285     def pruneSteps(self):
1286         # this build is very old: remove the build steps too
1287         self.steps = []
1288
1289     # persistence stuff
1290
1291     def generateLogfileName(self, stepname, logname):
1292         """Return a filename (relative to the Builder's base directory) where
1293         the logfile's contents can be stored uniquely.
1294
1295         The base filename is made by combining our build number, the Step's
1296         name, and the log's name, then removing unsuitable characters. The
1297         filename is then made unique by appending _0, _1, etc, until it does
1298         not collide with any other logfile.
1299
1300         These files are kept in the Builder's basedir (rather than a
1301         per-Build subdirectory) because that makes cleanup easier: cron and
1302         find will help get rid of the old logs, but the empty directories are
1303         more of a hassle to remove."""
1304
1305         starting_filename = "%d-log-%s-%s" % (self.number, stepname, logname)
1306         starting_filename = re.sub(r'[^\w\.\-]', '_', starting_filename)
1307         # now make it unique
1308         unique_counter = 0
1309         filename = starting_filename
1310         while filename in [l.filename
1311                            for step in self.steps
1312                            for l in step.getLogs()
1313                            if l.filename]:
1314             filename = "%s_%d" % (starting_filename, unique_counter)
1315             unique_counter += 1
1316         return filename
1317
1318     def __getstate__(self):
1319         d = styles.Versioned.__getstate__(self)
1320         # for now, a serialized Build is always "finished". We will never
1321         # save unfinished builds.
1322         if not self.finished:
1323             d['finished'] = True
1324             # TODO: push an "interrupted" step so it is clear that the build
1325             # was interrupted. The builder will have a 'shutdown' event, but
1326             # someone looking at just this build will be confused as to why
1327             # the last log is truncated.
1328         for k in 'builder', 'watchers', 'updates', 'requests', 'finishedWatchers':
1329             if k in d: del d[k]
1330         return d
1331
1332     def __setstate__(self, d):
1333         styles.Versioned.__setstate__(self, d)
1334         # self.builder must be filled in by our parent when loading
1335         for step in self.steps:
1336             step.build = self
1337         self.watchers = []
1338         self.updates = {}
1339         self.finishedWatchers = []
1340
1341     def upgradeToVersion1(self):
1342         if hasattr(self, "sourceStamp"):
1343             # the old .sourceStamp attribute wasn't actually very useful
1344             maxChangeNumber, patch = self.sourceStamp
1345             changes = getattr(self, 'changes', [])
1346             source = sourcestamp.SourceStamp(branch=None,
1347                                              revision=None,
1348                                              patch=patch,
1349                                              changes=changes)
1350             self.source = source
1351             self.changes = source.changes
1352             del self.sourceStamp
1353
1354     def upgradeToVersion2(self):
1355         self.properties = {}
1356
1357     def upgradeToVersion3(self):
1358         # in version 3, self.properties became a Properties object
1359         propdict = self.properties
1360         self.properties = Properties()
1361         self.properties.update(propdict, "Upgrade from previous version")
1362
1363     def upgradeLogfiles(self):
1364         # upgrade any LogFiles that need it. This must occur after we've been
1365         # attached to our Builder, and after we know about all LogFiles of
1366         # all Steps (to get the filenames right).
1367         assert self.builder
1368         for s in self.steps:
1369             for l in s.getLogs():
1370                 if l.filename:
1371                     pass # new-style, log contents are on disk
1372                 else:
1373                     logfilename = self.generateLogfileName(s.name, l.name)
1374                     # let the logfile update its .filename pointer,
1375                     # transferring its contents onto disk if necessary
1376                     l.upgrade(logfilename)
1377
1378     def checkLogfiles(self):
1379         # check that all logfiles exist, and remove references to any that
1380         # have been deleted (e.g., by purge())
1381         for s in self.steps:
1382             s.checkLogfiles()
1383
1384     def saveYourself(self):
1385         filename = os.path.join(self.builder.basedir, "%d" % self.number)
1386         if os.path.isdir(filename):
1387             # leftover from 0.5.0, which stored builds in directories
1388             shutil.rmtree(filename, ignore_errors=True)
1389         tmpfilename = filename + ".tmp"
1390         try:
1391             dump(self, open(tmpfilename, "wb"), -1)
1392             if sys.platform == 'win32':
1393                 # windows cannot rename a file on top of an existing one, so
1394                 # fall back to delete-first. There are ways this can fail and
1395                 # lose the builder's history, so we avoid using it in the
1396                 # general (non-windows) case
1397                 if os.path.exists(filename):
1398                     os.unlink(filename)
1399             os.rename(tmpfilename, filename)
1400         except:
1401             log.msg("unable to save build %s-#%d" % (self.builder.name,
1402                                                      self.number))
1403             log.err()
1404
1405
1406
1407 class BuilderStatus(styles.Versioned):
1408     """I handle status information for a single process.base.Builder object.
1409     That object sends status changes to me (frequently as Events), and I
1410     provide them on demand to the various status recipients, like the HTML
1411     waterfall display and the live status clients. It also sends build
1412     summaries to me, which I log and provide to status clients who aren't
1413     interested in seeing details of the individual build steps.
1414
1415     I am responsible for maintaining the list of historic Events and Builds,
1416     pruning old ones, and loading them from / saving them to disk.
1417
1418     I live in the buildbot.process.base.Builder object, in the
1419     .builder_status attribute.
1420
1421     @type  category: string
1422     @ivar  category: user-defined category this builder belongs to; can be
1423                      used to filter on in status clients
1424     """
1425
1426     implements(interfaces.IBuilderStatus, interfaces.IEventSource)
1427     persistenceVersion = 1
1428
1429     # these limit the amount of memory we consume, as well as the size of the
1430     # main Builder pickle. The Build and LogFile pickles on disk must be
1431     # handled separately.
1432     buildCacheSize = 15
1433     eventHorizon = 50 # forget events beyond this
1434
1435     # these limit on-disk storage
1436     logHorizon = 40 # forget logs in steps in builds beyond this
1437     buildHorizon = 100 # forget builds beyond this
1438
1439     category = None
1440     currentBigState = "offline" # or idle/waiting/interlocked/building
1441     basedir = None # filled in by our parent
1442
1443     def __init__(self, buildername, category=None):
1444         self.name = buildername
1445         self.category = category
1446
1447         self.slavenames = []
1448         self.events = []
1449         # these three hold Events, and are used to retrieve the current
1450         # state of the boxes.
1451         self.lastBuildStatus = None
1452         #self.currentBig = None
1453         #self.currentSmall = None
1454         self.currentBuilds = []
1455         self.pendingBuilds = []
1456         self.nextBuild = None
1457         self.watchers = []
1458         self.buildCache = weakref.WeakValueDictionary()
1459         self.buildCache_LRU = []
1460         self.logCompressionLimit = False # default to no compression for tests
1461
1462     # persistence
1463
1464     def __getstate__(self):
1465         # when saving, don't record transient stuff like what builds are
1466         # currently running, because they won't be there when we start back
1467         # up. Nor do we save self.watchers, nor anything that gets set by our
1468         # parent like .basedir and .status
1469         d = styles.Versioned.__getstate__(self)
1470         d['watchers'] = []
1471         del d['buildCache']
1472         del d['buildCache_LRU']
1473         for b in self.currentBuilds:
1474             b.saveYourself()
1475             # TODO: push a 'hey, build was interrupted' event
1476         del d['currentBuilds']
1477         del d['pendingBuilds']
1478         del d['currentBigState']
1479         del d['basedir']
1480         del d['status']
1481         del d['nextBuildNumber']
1482         return d
1483
1484     def __setstate__(self, d):
1485         # when loading, re-initialize the transient stuff. Remember that
1486         # upgradeToVersion1 and such will be called after this finishes.
1487         styles.Versioned.__setstate__(self, d)
1488         self.buildCache = weakref.WeakValueDictionary()
1489         self.buildCache_LRU = []
1490         self.currentBuilds = []
1491         self.pendingBuilds = []
1492         self.watchers = []
1493         self.slavenames = []
1494         # self.basedir must be filled in by our parent
1495         # self.status must be filled in by our parent
1496
1497     def upgradeToVersion1(self):
1498         if hasattr(self, 'slavename'):
1499             self.slavenames = [self.slavename]
1500             del self.slavename
1501         if hasattr(self, 'nextBuildNumber'):
1502             del self.nextBuildNumber # determineNextBuildNumber chooses this
1503
1504     def determineNextBuildNumber(self):
1505         """Scan our directory of saved BuildStatus instances to determine
1506         what our self.nextBuildNumber should be. Set it one larger than the
1507         highest-numbered build we discover. This is called by the top-level
1508         Status object shortly after we are created or loaded from disk.
1509         """
1510         existing_builds = [int(f)
1511                            for f in os.listdir(self.basedir)
1512                            if re.match("^\d+$", f)]
1513         if existing_builds:
1514             self.nextBuildNumber = max(existing_builds) + 1
1515         else:
1516             self.nextBuildNumber = 0
1517
1518     def setLogCompressionLimit(self, lowerLimit):
1519         self.logCompressionLimit = lowerLimit
1520
1521     def saveYourself(self):
1522         for b in self.currentBuilds:
1523             if not b.isFinished:
1524                 # interrupted build, need to save it anyway.
1525                 # BuildStatus.saveYourself will mark it as interrupted.
1526                 b.saveYourself()
1527         filename = os.path.join(self.basedir, "builder")
1528         tmpfilename = filename + ".tmp"
1529         try:
1530             dump(self, open(tmpfilename, "wb"), -1)
1531             if sys.platform == 'win32':
1532                 # windows cannot rename a file on top of an existing one
1533                 if os.path.exists(filename):
1534                     os.unlink(filename)
1535             os.rename(tmpfilename, filename)
1536         except:
1537             log.msg("unable to save builder %s" % self.name)
1538             log.err()
1539
1540
1541     # build cache management
1542
1543     def makeBuildFilename(self, number):
1544         return os.path.join(self.basedir, "%d" % number)
1545
1546     def touchBuildCache(self, build):
1547         self.buildCache[build.number] = build
1548         if build in self.buildCache_LRU:
1549             self.buildCache_LRU.remove(build)
1550         self.buildCache_LRU = self.buildCache_LRU[-(self.buildCacheSize-1):] + [ build ]
1551         return build
1552
1553     def getBuildByNumber(self, number):
1554         # first look in currentBuilds
1555         for b in self.currentBuilds:
1556             if b.number == number:
1557                 return self.touchBuildCache(b)
1558
1559         # then in the buildCache
1560         if number in self.buildCache:
1561             return self.touchBuildCache(self.buildCache[number])
1562
1563         # then fall back to loading it from disk
1564         filename = self.makeBuildFilename(number)
1565         try:
1566             log.msg("Loading builder %s's build %d from on-disk pickle"
1567                 % (self.name, number))
1568             build = load(open(filename, "rb"))
1569             styles.doUpgrade()
1570             build.builder = self
1571             # handle LogFiles from after 0.5.0 and before 0.6.5
1572             build.upgradeLogfiles()
1573             # check that logfiles exist
1574             build.checkLogfiles()
1575             return self.touchBuildCache(build)
1576         except IOError:
1577             raise IndexError("no such build %d" % number)
1578         except EOFError:
1579             raise IndexError("corrupted build pickle %d" % number)
1580
1581     def prune(self):
1582         gc.collect()
1583
1584         # begin by pruning our own events
1585         self.events = self.events[-self.eventHorizon:]
1586
1587         # get the horizons straight
1588         if self.buildHorizon:
1589             earliest_build = self.nextBuildNumber - self.buildHorizon
1590         else:
1591             earliest_build = 0
1592
1593         if self.logHorizon:
1594             earliest_log = self.nextBuildNumber - self.logHorizon
1595         else:
1596             earliest_log = 0
1597
1598         if earliest_log < earliest_build:
1599             earliest_log = earliest_build
1600
1601         if earliest_build == 0:
1602             return
1603
1604         # skim the directory and delete anything that shouldn't be there anymore
1605         build_re = re.compile(r"^([0-9]+)$")
1606         build_log_re = re.compile(r"^([0-9]+)-.*$")
1607         for filename in os.listdir(self.basedir):
1608             num = None
1609             mo = build_re.match(filename)
1610             is_logfile = False
1611             if mo:
1612                 num = int(mo.group(1))
1613             else:
1614                 mo = build_log_re.match(filename)
1615                 if mo:
1616                     num = int(mo.group(1))
1617                     is_logfile = True
1618
1619             if not num: continue
1620             if num in self.buildCache: continue
1621
1622             if (is_logfile and num < earliest_log) or num < earliest_build:
1623                 pathname = os.path.join(self.basedir, filename)
1624                 log.msg("pruning '%s'" % pathname)
1625                 try: os.unlink(pathname)
1626                 except OSError: pass
1627
1628     # IBuilderStatus methods
1629     def getName(self):
1630         return self.name
1631
1632     def getState(self):
1633         return (self.currentBigState, self.currentBuilds)
1634
1635     def getSlaves(self):
1636         return [self.status.getSlave(name) for name in self.slavenames]
1637
1638     def getPendingBuilds(self):
1639         return self.pendingBuilds
1640
1641     def getCurrentBuilds(self):
1642         return self.currentBuilds
1643
1644     def getLastFinishedBuild(self):
1645         b = self.getBuild(-1)
1646         if not (b and b.isFinished()):
1647             b = self.getBuild(-2)
1648         return b
1649
1650     def getBuild(self, number):
1651         if number < 0:
1652             number = self.nextBuildNumber + number
1653         if number < 0 or number >= self.nextBuildNumber:
1654             return None
1655
1656         try:
1657             return self.getBuildByNumber(number)
1658         except IndexError:
1659             return None
1660
1661     def getEvent(self, number):
1662         try:
1663             return self.events[number]
1664         except IndexError:
1665             return None
1666
1667     def generateFinishedBuilds(self, branches=[],
1668                                num_builds=None,
1669                                max_buildnum=None,
1670                                finished_before=None,
1671                                max_search=200):
1672         got = 0
1673         for Nb in itertools.count(1):
1674             if Nb > self.nextBuildNumber:
1675                 break
1676             if Nb > max_search:
1677                 break
1678             build = self.getBuild(-Nb)
1679             if build is None:
1680                 continue
1681             if max_buildnum is not None:
1682                 if build.getNumber() > max_buildnum:
1683                     continue
1684             if not build.isFinished():
1685                 continue
1686             if finished_before is not None:
1687                 start, end = build.getTimes()
1688                 if end >= finished_before:
1689                     continue
1690             if branches:
1691                 if build.getSourceStamp().branch not in branches:
1692                     continue
1693             got += 1
1694             yield build
1695             if num_builds is not None:
1696                 if got >= num_builds:
1697                     return
1698
1699     def eventGenerator(self, branches=[]):
1700         """This function creates a generator which will provide all of this
1701         Builder's status events, starting with the most recent and
1702         progressing backwards in time. """
1703
1704         # remember the oldest-to-earliest flow here. "next" means earlier.
1705
1706         # TODO: interleave build steps and self.events by timestamp.
1707         # TODO: um, I think we're already doing that.
1708
1709         # TODO: there's probably something clever we could do here to
1710         # interleave two event streams (one from self.getBuild and the other
1711         # from self.getEvent), which would be simpler than this control flow
1712
1713         eventIndex = -1
1714         e = self.getEvent(eventIndex)
1715         for Nb in range(1, self.nextBuildNumber+1):
1716             b = self.getBuild(-Nb)
1717             if not b:
1718                 break
1719             if branches and not b.getSourceStamp().branch in branches:
1720                 continue
1721             steps = b.getSteps()
1722             for Ns in range(1, len(steps)+1):
1723                 if steps[-Ns].started:
1724                     step_start = steps[-Ns].getTimes()[0]
1725                     while e is not None and e.getTimes()[0] > step_start:
1726                         yield e
1727                         eventIndex -= 1
1728                         e = self.getEvent(eventIndex)
1729                     yield steps[-Ns]
1730             yield b
1731         while e is not None:
1732             yield e
1733             eventIndex -= 1
1734             e = self.getEvent(eventIndex)
1735
1736     def subscribe(self, receiver):
1737         # will get builderChangedState, buildStarted, and buildFinished
1738         self.watchers.append(receiver)
1739         self.publishState(receiver)
1740
1741     def unsubscribe(self, receiver):
1742         self.watchers.remove(receiver)
1743
1744     ## Builder interface (methods called by the Builder which feeds us)
1745
1746     def setSlavenames(self, names):
1747         self.slavenames = names
1748
1749     def addEvent(self, text=[]):
1750         # this adds a duration event. When it is done, the user should call
1751         # e.finish(). They can also mangle it by modifying .text
1752         e = Event()
1753         e.started = util.now()
1754         e.text = text
1755         self.events.append(e)
1756         return e # they are free to mangle it further
1757
1758     def addPointEvent(self, text=[]):
1759         # this adds a point event, one which occurs as a single atomic
1760         # instant of time.
1761         e = Event()
1762         e.started = util.now()
1763         e.finished = 0
1764         e.text = text
1765         self.events.append(e)
1766         return e # for consistency, but they really shouldn't touch it
1767
1768     def setBigState(self, state):
1769         needToUpdate = state != self.currentBigState
1770         self.currentBigState = state
1771         if needToUpdate:
1772             self.publishState()
1773
1774     def publishState(self, target=None):
1775         state = self.currentBigState
1776
1777         if target is not None:
1778             # unicast
1779             target.builderChangedState(self.name, state)
1780             return
1781         for w in self.watchers:
1782             try:
1783                 w.builderChangedState(self.name, state)
1784             except:
1785                 log.msg("Exception caught publishing state to %r" % w)
1786                 log.err()
1787
1788     def newBuild(self):
1789         """The Builder has decided to start a build, but the Build object is
1790         not yet ready to report status (it has not finished creating the
1791         Steps). Create a BuildStatus object that it can use."""
1792         number = self.nextBuildNumber
1793         self.nextBuildNumber += 1
1794         # TODO: self.saveYourself(), to make sure we don't forget about the
1795         # build number we've just allocated. This is not quite as important
1796         # as it was before we switch to determineNextBuildNumber, but I think
1797         # it may still be useful to have the new build save itself.
1798         s = BuildStatus(self, number)
1799         s.waitUntilFinished().addCallback(self._buildFinished)
1800         return s
1801
1802     def addBuildRequest(self, brstatus):
1803         self.pendingBuilds.append(brstatus)
1804         for w in self.watchers:
1805             w.requestSubmitted(brstatus)
1806
1807     def removeBuildRequest(self, brstatus):
1808         self.pendingBuilds.remove(brstatus)
1809
1810     # buildStarted is called by our child BuildStatus instances
1811     def buildStarted(self, s):
1812         """Now the BuildStatus object is ready to go (it knows all of its
1813         Steps, its ETA, etc), so it is safe to notify our watchers."""
1814
1815         assert s.builder is self # paranoia
1816         assert s.number == self.nextBuildNumber - 1
1817         assert s not in self.currentBuilds
1818         self.currentBuilds.append(s)
1819         self.touchBuildCache(s)
1820
1821         # now that the BuildStatus is prepared to answer queries, we can
1822         # announce the new build to all our watchers
1823
1824         for w in self.watchers: # TODO: maybe do this later? callLater(0)?
1825             try:
1826                 receiver = w.buildStarted(self.getName(), s)
1827                 if receiver:
1828                     if type(receiver) == type(()):
1829                         s.subscribe(receiver[0], receiver[1])
1830                     else:
1831                         s.subscribe(receiver)
1832                     d = s.waitUntilFinished()
1833                     d.addCallback(lambda s: s.unsubscribe(receiver))
1834             except:
1835                 log.msg("Exception caught notifying %r of buildStarted event" % w)
1836                 log.err()
1837
1838     def _buildFinished(self, s):
1839         assert s in self.currentBuilds
1840         s.saveYourself()
1841         self.currentBuilds.remove(s)
1842
1843         name = self.getName()
1844         results = s.getResults()
1845         for w in self.watchers:
1846             try:
1847                 w.buildFinished(name, s, results)
1848             except:
1849                 log.msg("Exception caught notifying %r of buildFinished event" % w)
1850                 log.err()
1851
1852         self.prune() # conserve disk
1853
1854
1855     # waterfall display (history)
1856
1857     # I want some kind of build event that holds everything about the build:
1858     # why, what changes went into it, the results of the build, itemized
1859     # test results, etc. But, I do kind of need something to be inserted in
1860     # the event log first, because intermixing step events and the larger
1861     # build event is fraught with peril. Maybe an Event-like-thing that
1862     # doesn't have a file in it but does have links. Hmm, that's exactly
1863     # what it does now. The only difference would be that this event isn't
1864     # pushed to the clients.
1865
1866     # publish to clients
1867     def sendLastBuildStatus(self, client):
1868         #client.newLastBuildStatus(self.lastBuildStatus)
1869         pass
1870     def sendCurrentActivityBigToEveryone(self):
1871         for s in self.subscribers:
1872             self.sendCurrentActivityBig(s)
1873     def sendCurrentActivityBig(self, client):
1874         state = self.currentBigState
1875         if state == "offline":
1876             client.currentlyOffline()
1877         elif state == "idle":
1878             client.currentlyIdle()
1879         elif state == "building":
1880             client.currentlyBuilding()
1881         else:
1882             log.msg("Hey, self.currentBigState is weird:", state)
1883
1884
1885     ## HTML display interface
1886
1887     def getEventNumbered(self, num):
1888         # deal with dropped events, pruned events
1889         first = self.events[0].number
1890         if first + len(self.events)-1 != self.events[-1].number:
1891             log.msg(self,
1892                     "lost an event somewhere: [0] is %d, [%d] is %d" % \
1893                     (self.events[0].number,
1894                      len(self.events) - 1,
1895                      self.events[-1].number))
1896             for e in self.events:
1897                 log.msg("e[%d]: " % e.number, e)
1898             return None
1899         offset = num - first
1900         log.msg(self, "offset", offset)
1901         try:
1902             return self.events[offset]
1903         except IndexError:
1904             return None
1905
1906     ## Persistence of Status
1907     def loadYourOldEvents(self):
1908         if hasattr(self, "allEvents"):
1909             # first time, nothing to get from file. Note that this is only if
1910             # the Application gets .run() . If it gets .save()'ed, then the
1911             # .allEvents attribute goes away in the initial __getstate__ and
1912             # we try to load a non-existent file.
1913             return
1914         self.allEvents = self.loadFile("events", [])
1915         if self.allEvents:
1916             self.nextEventNumber = self.allEvents[-1].number + 1
1917         else:
1918             self.nextEventNumber = 0
1919     def saveYourOldEvents(self):
1920         self.saveFile("events", self.allEvents)
1921
1922     ## clients
1923
1924     def addClient(self, client):
1925         if client not in self.subscribers:
1926             self.subscribers.append(client)
1927             self.sendLastBuildStatus(client)
1928             self.sendCurrentActivityBig(client)
1929             client.newEvent(self.currentSmall)
1930     def removeClient(self, client):
1931         if client in self.subscribers:
1932             self.subscribers.remove(client)
1933
1934 class SlaveStatus:
1935     implements(interfaces.ISlaveStatus)
1936
1937     admin = None
1938     host = None
1939     connected = False
1940     graceful_shutdown = False
1941
1942     def __init__(self, name):
1943         self.name = name
1944         self._lastMessageReceived = 0
1945         self.runningBuilds = []
1946         self.graceful_callbacks = []
1947
1948     def getName(self):
1949         return self.name
1950     def getAdmin(self):
1951         return self.admin
1952     def getHost(self):
1953         return self.host
1954     def isConnected(self):
1955         return self.connected
1956     def lastMessageReceived(self):
1957         return self._lastMessageReceived
1958     def getRunningBuilds(self):
1959         return self.runningBuilds
1960
1961     def setAdmin(self, admin):
1962         self.admin = admin
1963     def setHost(self, host):
1964         self.host = host
1965     def setConnected(self, isConnected):
1966         self.connected = isConnected
1967     def setLastMessageReceived(self, when):
1968         self._lastMessageReceived = when
1969
1970     def buildStarted(self, build):
1971         self.runningBuilds.append(build)
1972     def buildFinished(self, build):
1973         self.runningBuilds.remove(build)
1974
1975     def getGraceful(self):
1976         """Return the graceful shutdown flag"""
1977         return self.graceful_shutdown
1978     def setGraceful(self, graceful):
1979         """Set the graceful shutdown flag, and notify all the watchers"""
1980         self.graceful_shutdown = graceful
1981         for cb in self.graceful_callbacks:
1982             reactor.callLater(0, cb, graceful)
1983     def addGracefulWatcher(self, watcher):
1984         """Add watcher to the list of watchers to be notified when the
1985         graceful shutdown flag is changed."""
1986         if not watcher in self.graceful_callbacks:
1987             self.graceful_callbacks.append(watcher)
1988     def removeGracefulWatcher(self, watcher):
1989         """Remove watcher from the list of watchers to be notified when the
1990         graceful shutdown flag is changed."""
1991         if watcher in self.graceful_callbacks:
1992             self.graceful_callbacks.remove(watcher)
1993
1994 class Status:
1995     """
1996     I represent the status of the buildmaster.
1997     """
1998     implements(interfaces.IStatus)
1999
2000     def __init__(self, botmaster, basedir):
2001         """
2002         @type  botmaster: L{buildbot.master.BotMaster}
2003         @param botmaster: the Status object uses C{.botmaster} to get at
2004                           both the L{buildbot.master.BuildMaster} (for
2005                           various buildbot-wide parameters) and the
2006                           actual Builders (to get at their L{BuilderStatus}
2007                           objects). It is not allowed to change or influence
2008                           anything through this reference.
2009         @type  basedir: string
2010         @param basedir: this provides a base directory in which saved status
2011                         information (changes.pck, saved Build status
2012                         pickles) can be stored
2013         """
2014         self.botmaster = botmaster
2015         self.basedir = basedir
2016         self.watchers = []
2017         self.activeBuildSets = []
2018         assert os.path.isdir(basedir)
2019         # compress logs bigger than 4k, a good default on linux
2020         self.logCompressionLimit = 4*1024
2021
2022
2023     # methods called by our clients
2024
2025     def getProjectName(self):
2026         return self.botmaster.parent.projectName
2027     def getProjectURL(self):
2028         return self.botmaster.parent.projectURL
2029     def getBuildbotURL(self):
2030         return self.botmaster.parent.buildbotURL
2031
2032     def getURLForThing(self, thing):
2033         prefix = self.getBuildbotURL()
2034         if not prefix:
2035             return None
2036         if interfaces.IStatus.providedBy(thing):
2037             return prefix
2038         if interfaces.ISchedulerStatus.providedBy(thing):
2039             pass
2040         if interfaces.IBuilderStatus.providedBy(thing):
2041             builder = thing
2042             return prefix + "builders/%s" % (
2043                 urllib.quote(builder.getName(), safe=''),
2044                 )
2045         if interfaces.IBuildStatus.providedBy(thing):
2046             build = thing
2047             builder = build.getBuilder()
2048             return prefix + "builders/%s/builds/%d" % (
2049                 urllib.quote(builder.getName(), safe=''),
2050                 build.getNumber())
2051         if interfaces.IBuildStepStatus.providedBy(thing):
2052             step = thing
2053             build = step.getBuild()
2054             builder = build.getBuilder()
2055             return prefix + "builders/%s/builds/%d/steps/%s" % (
2056                 urllib.quote(builder.getName(), safe=''),
2057                 build.getNumber(),
2058                 urllib.quote(step.getName(), safe=''))
2059         # IBuildSetStatus
2060         # IBuildRequestStatus
2061         # ISlaveStatus
2062
2063         # IStatusEvent
2064         if interfaces.IStatusEvent.providedBy(thing):
2065             from buildbot.changes import changes
2066             # TODO: this is goofy, create IChange or something
2067             if isinstance(thing, changes.Change):
2068                 change = thing
2069                 return "%schanges/%d" % (prefix, change.number)
2070
2071         if interfaces.IStatusLog.providedBy(thing):
2072             log = thing
2073             step = log.getStep()
2074             build = step.getBuild()
2075             builder = build.getBuilder()
2076
2077             logs = step.getLogs()
2078             for i in range(len(logs)):
2079                 if log is logs[i]:
2080                     lognum = i
2081                     break
2082             else:
2083                 return None
2084             return prefix + "builders/%s/builds/%d/steps/%s/logs/%d" % (
2085                 urllib.quote(builder.getName(), safe=''),
2086                 build.getNumber(),
2087                 urllib.quote(step.getName(), safe=''),
2088                 lognum)
2089
2090     def getChangeSources(self):
2091         return list(self.botmaster.parent.change_svc)
2092
2093     def getChange(self, number):
2094         return self.botmaster.parent.change_svc.getChangeNumbered(number)
2095
2096     def getSchedulers(self):
2097         return self.botmaster.parent.allSchedulers()
2098
2099     def getBuilderNames(self, categories=None):
2100         if categories == None:
2101             return self.botmaster.builderNames[:] # don't let them break it
2102
2103         l = []
2104         # respect addition order
2105         for name in self.botmaster.builderNames:
2106             builder = self.botmaster.builders[name]
2107             if builder.builder_status.category in categories:
2108                 l.append(name)
2109         return l
2110
2111     def getBuilder(self, name):
2112         """
2113         @rtype: L{BuilderStatus}
2114         """
2115         return self.botmaster.builders[name].builder_status
2116
2117     def getSlaveNames(self):
2118         return self.botmaster.slaves.keys()
2119
2120     def getSlave(self, slavename):
2121         return self.botmaster.slaves[slavename].slave_status
2122
2123     def getBuildSets(self):
2124         return self.activeBuildSets[:]
2125
2126     def generateFinishedBuilds(self, builders=[], branches=[],
2127                                num_builds=None, finished_before=None,
2128                                max_search=200):
2129
2130         def want_builder(bn):
2131             if builders:
2132                 return bn in builders
2133             return True
2134         builder_names = [bn
2135                          for bn in self.getBuilderNames()
2136                          if want_builder(bn)]
2137
2138         # 'sources' is a list of generators, one for each Builder we're
2139         # using. When the generator is exhausted, it is replaced in this list
2140         # with None.
2141         sources = []
2142         for bn in builder_names:
2143             b = self.getBuilder(bn)
2144             g = b.generateFinishedBuilds(branches,
2145                                          finished_before=finished_before,
2146                                          max_search=max_search)
2147             sources.append(g)
2148
2149         # next_build the next build from each source
2150         next_build = [None] * len(sources)
2151
2152         def refill():
2153             for i,g in enumerate(sources):
2154                 if next_build[i]:
2155                     # already filled
2156                     continue
2157                 if not g:
2158                     # already exhausted
2159                     continue
2160                 try:
2161                     next_build[i] = g.next()
2162                 except StopIteration:
2163                     next_build[i] = None
2164                     sources[i] = None
2165
2166         got = 0
2167         while True:
2168             refill()
2169             # find the latest build among all the candidates
2170             candidates = [(i, b, b.getTimes()[1])
2171                           for i,b in enumerate(next_build)
2172                           if b is not None]
2173             candidates.sort(lambda x,y: cmp(x[2], y[2]))
2174             if not candidates:
2175                 return
2176
2177             # and remove it from the list
2178             i, build, finshed_time = candidates[-1]
2179             next_build[i] = None
2180             got += 1
2181             yield build
2182             if num_builds is not None:
2183                 if got >= num_builds:
2184                     return
2185
2186     def subscribe(self, target):
2187         self.watchers.append(target)
2188         for name in self.botmaster.builderNames:
2189             self.announceNewBuilder(target, name, self.getBuilder(name))
2190     def unsubscribe(self, target):
2191         self.watchers.remove(target)
2192
2193
2194     # methods called by upstream objects
2195
2196     def announceNewBuilder(self, target, name, builder_status):
2197         t = target.builderAdded(name, builder_status)
2198         if t:
2199             builder_status.subscribe(t)
2200
2201     def builderAdded(self, name, basedir, category=None):
2202         """
2203         @rtype: L{BuilderStatus}
2204         """
2205         filename = os.path.join(self.basedir, basedir, "builder")
2206         log.msg("trying to load status pickle from %s" % filename)
2207         builder_status = None
2208         try:
2209             builder_status = load(open(filename, "rb"))
2210             styles.doUpgrade()
2211         except IOError:
2212             log.msg("no saved status pickle, creating a new one")
2213         except:
2214             log.msg("error while loading status pickle, creating a new one")
2215             log.msg("error follows:")
2216             log.err()
2217         if not builder_status:
2218             builder_status = BuilderStatus(name, category)
2219             builder_status.addPointEvent(["builder", "created"])
2220         log.msg("added builder %s in category %s" % (name, category))
2221         # an unpickled object might not have category set from before,
2222         # so set it here to make sure
2223         builder_status.category = category
2224         builder_status.basedir = os.path.join(self.basedir, basedir)
2225         builder_status.name = name # it might have been updated
2226         builder_status.status = self
2227
2228         if not os.path.isdir(builder_status.basedir):
2229             os.makedirs(builder_status.basedir)
2230         builder_status.determineNextBuildNumber()
2231
2232         builder_status.setBigState("offline")
2233         builder_status.setLogCompressionLimit(self.logCompressionLimit)
2234
2235         for t in self.watchers:
2236             self.announceNewBuilder(t, name, builder_status)
2237
2238         return builder_status
2239
2240     def builderRemoved(self, name):
2241         for t in self.watchers:
2242             t.builderRemoved(name)
2243
2244     def buildsetSubmitted(self, bss):
2245         self.activeBuildSets.append(bss)
2246         bss.waitUntilFinished().addCallback(self.activeBuildSets.remove)
2247         for t in self.watchers:
2248             t.buildsetSubmitted(bss)