buildbot/status/builder.py

   1 # -*- test-case-name: buildbot.test.test_status -*-
   2
   3 from zope.interface import implements
   4 from twisted.python import log
   5 from twisted.persisted import styles
   6 from twisted.internet import reactor, defer, threads
   7 from twisted.protocols import basic
   8 from buildbot.process.properties import Properties
   9
  10 import weakref
  11 import os, shutil, sys, re, urllib, itertools
  12 from cPickle import load, dump
  13 from cStringIO import StringIO
  14 from bz2 import BZ2File
  15
  16 # sibling imports
  17 from buildbot import interfaces, util, sourcestamp
  18
  19 SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION = range(5)
  20 Results = ["success", "warnings", "failure", "skipped", "exception"]
  21
  22
  23 # build processes call the following methods:
  24 #
  25 #  setDefaults
  26 #
  27 #  currentlyBuilding
  28 #  currentlyIdle
  29 #  currentlyInterlocked
  30 #  currentlyOffline
  31 #  currentlyWaiting
  32 #
  33 #  setCurrentActivity
  34 #  updateCurrentActivity
  35 #  addFileToCurrentActivity
  36 #  finishCurrentActivity
  37 #
  38 #  startBuild
  39 #  finishBuild
  40
  41 STDOUT = interfaces.LOG_CHANNEL_STDOUT
  42 STDERR = interfaces.LOG_CHANNEL_STDERR
  43 HEADER = interfaces.LOG_CHANNEL_HEADER
  44 ChunkTypes = ["stdout", "stderr", "header"]
  45
  46 class LogFileScanner(basic.NetstringReceiver):
  47     def __init__(self, chunk_cb, channels=[]):
  48         self.chunk_cb = chunk_cb
  49         self.channels = channels
  50
  51     def stringReceived(self, line):
  52         channel = int(line[0])
  53         if not self.channels or (channel in self.channels):
  54             self.chunk_cb((channel, line[1:]))
  55
  56 class LogFileProducer:
  57     """What's the plan?
  58
  59     the LogFile has just one FD, used for both reading and writing.
  60     Each time you add an entry, fd.seek to the end and then write.
  61
  62     Each reader (i.e. Producer) keeps track of their own offset. The reader
  63     starts by seeking to the start of the logfile, and reading forwards.
  64     Between each hunk of file they yield chunks, so they must remember their
  65     offset before yielding and re-seek back to that offset before reading
  66     more data. When their read() returns EOF, they're finished with the first
  67     phase of the reading (everything that's already been written to disk).
  68
  69     After EOF, the remaining data is entirely in the current entries list.
  70     These entries are all of the same channel, so we can do one "".join and
  71     obtain a single chunk to be sent to the listener. But since that involves
  72     a yield, and more data might arrive after we give up control, we have to
  73     subscribe them before yielding. We can't subscribe them any earlier,
  74     otherwise they'd get data out of order.
  75
  76     We're using a generator in the first place so that the listener can
  77     throttle us, which means they're pulling. But the subscription means
  78     we're pushing. Really we're a Producer. In the first phase we can be
  79     either a PullProducer or a PushProducer. In the second phase we're only a
  80     PushProducer.
  81
  82     So the client gives a LogFileConsumer to File.subscribeConsumer . This
  83     Consumer must have registerProducer(), unregisterProducer(), and
  84     writeChunk(), and is just like a regular twisted.interfaces.IConsumer,
  85     except that writeChunk() takes chunks (tuples of (channel,text)) instead
  86     of the normal write() which takes just text. The LogFileConsumer is
  87     allowed to call stopProducing, pauseProducing, and resumeProducing on the
  88     producer instance it is given. """
  89
  90     paused = False
  91     subscribed = False
  92     BUFFERSIZE = 2048
  93
  94     def __init__(self, logfile, consumer):
  95         self.logfile = logfile
  96         self.consumer = consumer
  97         self.chunkGenerator = self.getChunks()
  98         consumer.registerProducer(self, True)
  99
 100     def getChunks(self):
 101         f = self.logfile.getFile()
 102         offset = 0
 103         chunks = []
 104         p = LogFileScanner(chunks.append)
 105         f.seek(offset)
 106         data = f.read(self.BUFFERSIZE)
 107         offset = f.tell()
 108         while data:
 109             p.dataReceived(data)
 110             while chunks:
 111                 c = chunks.pop(0)
 112                 yield c
 113             f.seek(offset)
 114             data = f.read(self.BUFFERSIZE)
 115             offset = f.tell()
 116         del f
 117
 118         # now subscribe them to receive new entries
 119         self.subscribed = True
 120         self.logfile.watchers.append(self)
 121         d = self.logfile.waitUntilFinished()
 122
 123         # then give them the not-yet-merged data
 124         if self.logfile.runEntries:
 125             channel = self.logfile.runEntries[0][0]
 126             text = "".join([c[1] for c in self.logfile.runEntries])
 127             yield (channel, text)
 128
 129         # now we've caught up to the present. Anything further will come from
 130         # the logfile subscription. We add the callback *after* yielding the
 131         # data from runEntries, because the logfile might have finished
 132         # during the yield.
 133         d.addCallback(self.logfileFinished)
 134
 135     def stopProducing(self):
 136         # TODO: should we still call consumer.finish? probably not.
 137         self.paused = True
 138         self.consumer = None
 139         self.done()
 140
 141     def done(self):
 142         if self.chunkGenerator:
 143             self.chunkGenerator = None # stop making chunks
 144         if self.subscribed:
 145             self.logfile.watchers.remove(self)
 146             self.subscribed = False
 147
 148     def pauseProducing(self):
 149         self.paused = True
 150
 151     def resumeProducing(self):
 152         # Twisted-1.3.0 has a bug which causes hangs when resumeProducing
 153         # calls transport.write (there is a recursive loop, fixed in 2.0 in
 154         # t.i.abstract.FileDescriptor.doWrite by setting the producerPaused
 155         # flag *before* calling resumeProducing). To work around this, we
 156         # just put off the real resumeProducing for a moment. This probably
 157         # has a performance hit, but I'm going to assume that the log files
 158         # are not retrieved frequently enough for it to be an issue.
 159
 160         reactor.callLater(0, self._resumeProducing)
 161
 162     def _resumeProducing(self):
 163         self.paused = False
 164         if not self.chunkGenerator:
 165             return
 166         try:
 167             while not self.paused:
 168                 chunk = self.chunkGenerator.next()
 169                 self.consumer.writeChunk(chunk)
 170                 # we exit this when the consumer says to stop, or we run out
 171                 # of chunks
 172         except StopIteration:
 173             # if the generator finished, it will have done releaseFile
 174             self.chunkGenerator = None
 175         # now everything goes through the subscription, and they don't get to
 176         # pause anymore
 177
 178     def logChunk(self, build, step, logfile, channel, chunk):
 179         if self.consumer:
 180             self.consumer.writeChunk((channel, chunk))
 181
 182     def logfileFinished(self, logfile):
 183         self.done()
 184         if self.consumer:
 185             self.consumer.unregisterProducer()
 186             self.consumer.finish()
 187             self.consumer = None
 188
 189 def _tryremove(filename, timeout, retries):
 190     """Try to remove a file, and if failed, try again in timeout.
 191     Increases the timeout by a factor of 4, and only keeps trying for
 192     another retries-amount of times.
 193
 194     """
 195     try:
 196         os.unlink(filename)
 197     except OSError:
 198         if retries > 0:
 199             reactor.callLater(timeout, _tryremove, filename, timeout * 4,
 200                               retries - 1)
 201         else:
 202             log.msg("giving up on removing %s after over %d seconds" %
 203                     (filename, timeout))
 204
 205 class LogFile:
 206     """A LogFile keeps all of its contents on disk, in a non-pickle format to
 207     which new entries can easily be appended. The file on disk has a name
 208     like 12-log-compile-output, under the Builder's directory. The actual
 209     filename is generated (before the LogFile is created) by
 210     L{BuildStatus.generateLogfileName}.
 211
 212     Old LogFile pickles (which kept their contents in .entries) must be
 213     upgraded. The L{BuilderStatus} is responsible for doing this, when it
 214     loads the L{BuildStatus} into memory. The Build pickle is not modified,
 215     so users who go from 0.6.5 back to 0.6.4 don't have to lose their
 216     logs."""
 217
 218     implements(interfaces.IStatusLog, interfaces.ILogFile)
 219
 220     finished = False
 221     length = 0
 222     chunkSize = 10*1000
 223     runLength = 0
 224     runEntries = [] # provided so old pickled builds will getChunks() ok
 225     entries = None
 226     BUFFERSIZE = 2048
 227     filename = None # relative to the Builder's basedir
 228     openfile = None
 229
 230     def __init__(self, parent, name, logfilename):
 231         """
 232         @type  parent: L{BuildStepStatus}
 233         @param parent: the Step that this log is a part of
 234         @type  name: string
 235         @param name: the name of this log, typically 'output'
 236         @type  logfilename: string
 237         @param logfilename: the Builder-relative pathname for the saved entries
 238         """
 239         self.step = parent
 240         self.name = name
 241         self.filename = logfilename
 242         fn = self.getFilename()
 243         if os.path.exists(fn):
 244             # the buildmaster was probably stopped abruptly, before the
 245             # BuilderStatus could be saved, so BuilderStatus.nextBuildNumber
 246             # is out of date, and we're overlapping with earlier builds now.
 247             # Warn about it, but then overwrite the old pickle file
 248             log.msg("Warning: Overwriting old serialized Build at %s" % fn)
 249         self.openfile = open(fn, "w+")
 250         self.runEntries = []
 251         self.watchers = []
 252         self.finishedWatchers = []
 253
 254     def getFilename(self):
 255         return os.path.join(self.step.build.builder.basedir, self.filename)
 256
 257     def hasContents(self):
 258         return os.path.exists(self.getFilename() + '.bz2') or \
 259             os.path.exists(self.getFilename())
 260
 261     def getName(self):
 262         return self.name
 263
 264     def getStep(self):
 265         return self.step
 266
 267     def isFinished(self):
 268         return self.finished
 269     def waitUntilFinished(self):
 270         if self.finished:
 271             d = defer.succeed(self)
 272         else:
 273             d = defer.Deferred()
 274             self.finishedWatchers.append(d)
 275         return d
 276
 277     def getFile(self):
 278         if self.openfile:
 279             # this is the filehandle we're using to write to the log, so
 280             # don't close it!
 281             return self.openfile
 282         # otherwise they get their own read-only handle
 283         # try a compressed log first
 284         try:
 285             return BZ2File(self.getFilename() + ".bz2", "r")
 286         except IOError:
 287             pass
 288         return open(self.getFilename(), "r")
 289
 290     def getText(self):
 291         # this produces one ginormous string
 292         return "".join(self.getChunks([STDOUT, STDERR], onlyText=True))
 293
 294     def getTextWithHeaders(self):
 295         return "".join(self.getChunks(onlyText=True))
 296
 297     def getChunks(self, channels=[], onlyText=False):
 298         # generate chunks for everything that was logged at the time we were
 299         # first called, so remember how long the file was when we started.
 300         # Don't read beyond that point. The current contents of
 301         # self.runEntries will follow.
 302
 303         # this returns an iterator, which means arbitrary things could happen
 304         # while we're yielding. This will faithfully deliver the log as it
 305         # existed when it was started, and not return anything after that
 306         # point. To use this in subscribe(catchup=True) without missing any
 307         # data, you must insure that nothing will be added to the log during
 308         # yield() calls.
 309
 310         f = self.getFile()
 311         offset = 0
 312         f.seek(0, 2)
 313         remaining = f.tell()
 314
 315         leftover = None
 316         if self.runEntries and (not channels or
 317                                 (self.runEntries[0][0] in channels)):
 318             leftover = (self.runEntries[0][0],
 319                         "".join([c[1] for c in self.runEntries]))
 320
 321         # freeze the state of the LogFile by passing a lot of parameters into
 322         # a generator
 323         return self._generateChunks(f, offset, remaining, leftover,
 324                                     channels, onlyText)
 325
 326     def _generateChunks(self, f, offset, remaining, leftover,
 327                         channels, onlyText):
 328         chunks = []
 329         p = LogFileScanner(chunks.append, channels)
 330         f.seek(offset)
 331         data = f.read(min(remaining, self.BUFFERSIZE))
 332         remaining -= len(data)
 333         offset = f.tell()
 334         while data:
 335             p.dataReceived(data)
 336             while chunks:
 337                 channel, text = chunks.pop(0)
 338                 if onlyText:
 339                     yield text
 340                 else:
 341                     yield (channel, text)
 342             f.seek(offset)
 343             data = f.read(min(remaining, self.BUFFERSIZE))
 344             remaining -= len(data)
 345             offset = f.tell()
 346         del f
 347
 348         if leftover:
 349             if onlyText:
 350                 yield leftover[1]
 351             else:
 352                 yield leftover
 353
 354     def readlines(self, channel=STDOUT):
 355         """Return an iterator that produces newline-terminated lines,
 356         excluding header chunks."""
 357         # TODO: make this memory-efficient, by turning it into a generator
 358         # that retrieves chunks as necessary, like a pull-driven version of
 359         # twisted.protocols.basic.LineReceiver
 360         alltext = "".join(self.getChunks([channel], onlyText=True))
 361         io = StringIO(alltext)
 362         return io.readlines()
 363
 364     def subscribe(self, receiver, catchup):
 365         if self.finished:
 366             return
 367         self.watchers.append(receiver)
 368         if catchup:
 369             for channel, text in self.getChunks():
 370                 # TODO: add logChunks(), to send over everything at once?
 371                 receiver.logChunk(self.step.build, self.step, self,
 372                                   channel, text)
 373
 374     def unsubscribe(self, receiver):
 375         if receiver in self.watchers:
 376             self.watchers.remove(receiver)
 377
 378     def subscribeConsumer(self, consumer):
 379         p = LogFileProducer(self, consumer)
 380         p.resumeProducing()
 381
 382     # interface used by the build steps to add things to the log
 383
 384     def merge(self):
 385         # merge all .runEntries (which are all of the same type) into a
 386         # single chunk for .entries
 387         if not self.runEntries:
 388             return
 389         channel = self.runEntries[0][0]
 390         text = "".join([c[1] for c in self.runEntries])
 391         assert channel < 10
 392         f = self.openfile
 393         f.seek(0, 2)
 394         offset = 0
 395         while offset < len(text):
 396             size = min(len(text)-offset, self.chunkSize)
 397             f.write("%d:%d" % (1 + size, channel))
 398             f.write(text[offset:offset+size])
 399             f.write(",")
 400             offset += size
 401         self.runEntries = []
 402         self.runLength = 0
 403
 404     def addEntry(self, channel, text):
 405         assert not self.finished
 406         # we only add to .runEntries here. merge() is responsible for adding
 407         # merged chunks to .entries
 408         if self.runEntries and channel != self.runEntries[0][0]:
 409             self.merge()
 410         self.runEntries.append((channel, text))
 411         self.runLength += len(text)
 412         if self.runLength >= self.chunkSize:
 413             self.merge()
 414
 415         for w in self.watchers:
 416             w.logChunk(self.step.build, self.step, self, channel, text)
 417         self.length += len(text)
 418
 419     def addStdout(self, text):
 420         self.addEntry(STDOUT, text)
 421     def addStderr(self, text):
 422         self.addEntry(STDERR, text)
 423     def addHeader(self, text):
 424         self.addEntry(HEADER, text)
 425
 426     def finish(self):
 427         self.merge()
 428         if self.openfile:
 429             # we don't do an explicit close, because there might be readers
 430             # shareing the filehandle. As soon as they stop reading, the
 431             # filehandle will be released and automatically closed. We will
 432             # do a sync, however, to make sure the log gets saved in case of
 433             # a crash.
 434             self.openfile.flush()
 435             os.fsync(self.openfile.fileno())
 436             del self.openfile
 437         self.finished = True
 438         watchers = self.finishedWatchers
 439         self.finishedWatchers = []
 440         for w in watchers:
 441             w.callback(self)
 442         self.watchers = []
 443
 444
 445     def compressLog(self):
 446         compressed = self.getFilename() + ".bz2.tmp"
 447         d = threads.deferToThread(self._compressLog, compressed)
 448         d.addCallback(self._renameCompressedLog, compressed)
 449         d.addErrback(self._cleanupFailedCompress, compressed)
 450         return d
 451
 452     def _compressLog(self, compressed):
 453         infile = self.getFile()
 454         cf = BZ2File(compressed, 'w')
 455         bufsize = 1024*1024
 456         while True:
 457             buf = infile.read(bufsize)
 458             cf.write(buf)
 459             if len(buf) < bufsize:
 460                 break
 461         cf.close()
 462     def _renameCompressedLog(self, rv, compressed):
 463         filename = self.getFilename() + '.bz2'
 464         if sys.platform == 'win32':
 465             # windows cannot rename a file on top of an existing one, so
 466             # fall back to delete-first. There are ways this can fail and
 467             # lose the builder's history, so we avoid using it in the
 468             # general (non-windows) case
 469             if os.path.exists(filename):
 470                 os.unlink(filename)
 471         os.rename(compressed, filename)
 472         _tryremove(self.getFilename(), 1, 5)
 473     def _cleanupFailedCompress(self, failure, compressed):
 474         log.msg("failed to compress %s" % self.getFilename())
 475         if os.path.exists(compressed):
 476             _tryremove(compressed, 1, 5)
 477         failure.trap() # reraise the failure
 478
 479     # persistence stuff
 480     def __getstate__(self):
 481         d = self.__dict__.copy()
 482         del d['step'] # filled in upon unpickling
 483         del d['watchers']
 484         del d['finishedWatchers']
 485         d['entries'] = [] # let 0.6.4 tolerate the saved log. TODO: really?
 486         if d.has_key('finished'):
 487             del d['finished']
 488         if d.has_key('openfile'):
 489             del d['openfile']
 490         return d
 491
 492     def __setstate__(self, d):
 493         self.__dict__ = d
 494         self.watchers = [] # probably not necessary
 495         self.finishedWatchers = [] # same
 496         # self.step must be filled in by our parent
 497         self.finished = True
 498
 499     def upgrade(self, logfilename):
 500         """Save our .entries to a new-style offline log file (if necessary),
 501         and modify our in-memory representation to use it. The original
 502         pickled LogFile (inside the pickled Build) won't be modified."""
 503         self.filename = logfilename
 504         if not os.path.exists(self.getFilename()):
 505             self.openfile = open(self.getFilename(), "w")
 506             self.finished = False
 507             for channel,text in self.entries:
 508                 self.addEntry(channel, text)
 509             self.finish() # releases self.openfile, which will be closed
 510         del self.entries
 511
 512 class HTMLLogFile:
 513     implements(interfaces.IStatusLog)
 514
 515     filename = None
 516
 517     def __init__(self, parent, name, logfilename, html):
 518         self.step = parent
 519         self.name = name
 520         self.filename = logfilename
 521         self.html = html
 522
 523     def getName(self):
 524         return self.name # set in BuildStepStatus.addLog
 525     def getStep(self):
 526         return self.step
 527
 528     def isFinished(self):
 529         return True
 530     def waitUntilFinished(self):
 531         return defer.succeed(self)
 532
 533     def hasContents(self):
 534         return True
 535     def getText(self):
 536         return self.html # looks kinda like text
 537     def getTextWithHeaders(self):
 538         return self.html
 539     def getChunks(self):
 540         return [(STDERR, self.html)]
 541
 542     def subscribe(self, receiver, catchup):
 543         pass
 544     def unsubscribe(self, receiver):
 545         pass
 546
 547     def finish(self):
 548         pass
 549
 550     def __getstate__(self):
 551         d = self.__dict__.copy()
 552         del d['step']
 553         return d
 554
 555     def upgrade(self, logfilename):
 556         pass
 557
 558
 559 class Event:
 560     implements(interfaces.IStatusEvent)
 561
 562     started = None
 563     finished = None
 564     text = []
 565
 566     # IStatusEvent methods
 567     def getTimes(self):
 568         return (self.started, self.finished)
 569     def getText(self):
 570         return self.text
 571     def getLogs(self):
 572         return []
 573
 574     def finish(self):
 575         self.finished = util.now()
 576
 577 class TestResult:
 578     implements(interfaces.ITestResult)
 579
 580     def __init__(self, name, results, text, logs):
 581         assert isinstance(name, tuple)
 582         self.name = name
 583         self.results = results
 584         self.text = text
 585         self.logs = logs
 586
 587     def getName(self):
 588         return self.name
 589
 590     def getResults(self):
 591         return self.results
 592
 593     def getText(self):
 594         return self.text
 595
 596     def getLogs(self):
 597         return self.logs
 598
 599
 600 class BuildSetStatus:
 601     implements(interfaces.IBuildSetStatus)
 602
 603     def __init__(self, source, reason, builderNames, bsid=None):
 604         self.source = source
 605         self.reason = reason
 606         self.builderNames = builderNames
 607         self.id = bsid
 608         self.successWatchers = []
 609         self.finishedWatchers = []
 610         self.stillHopeful = True
 611         self.finished = False
 612
 613     def setBuildRequestStatuses(self, buildRequestStatuses):
 614         self.buildRequests = buildRequestStatuses
 615     def setResults(self, results):
 616         # the build set succeeds only if all its component builds succeed
 617         self.results = results
 618     def giveUpHope(self):
 619         self.stillHopeful = False
 620
 621
 622     def notifySuccessWatchers(self):
 623         for d in self.successWatchers:
 624             d.callback(self)
 625         self.successWatchers = []
 626
 627     def notifyFinishedWatchers(self):
 628         self.finished = True
 629         for d in self.finishedWatchers:
 630             d.callback(self)
 631         self.finishedWatchers = []
 632
 633     # methods for our clients
 634
 635     def getSourceStamp(self):
 636         return self.source
 637     def getReason(self):
 638         return self.reason
 639     def getResults(self):
 640         return self.results
 641     def getID(self):
 642         return self.id
 643
 644     def getBuilderNames(self):
 645         return self.builderNames
 646     def getBuildRequests(self):
 647         return self.buildRequests
 648     def isFinished(self):
 649         return self.finished
 650
 651     def waitUntilSuccess(self):
 652         if self.finished or not self.stillHopeful:
 653             # the deferreds have already fired
 654             return defer.succeed(self)
 655         d = defer.Deferred()
 656         self.successWatchers.append(d)
 657         return d
 658
 659     def waitUntilFinished(self):
 660         if self.finished:
 661             return defer.succeed(self)
 662         d = defer.Deferred()
 663         self.finishedWatchers.append(d)
 664         return d
 665
 666 class BuildRequestStatus:
 667     implements(interfaces.IBuildRequestStatus)
 668
 669     def __init__(self, source, builderName):
 670         self.source = source
 671         self.builderName = builderName
 672         self.builds = [] # list of BuildStatus objects
 673         self.observers = []
 674         self.submittedAt = None
 675
 676     def buildStarted(self, build):
 677         self.builds.append(build)
 678         for o in self.observers[:]:
 679             o(build)
 680
 681     # methods called by our clients
 682     def getSourceStamp(self):
 683         return self.source
 684     def getBuilderName(self):
 685         return self.builderName
 686     def getBuilds(self):
 687         return self.builds
 688
 689     def subscribe(self, observer):
 690         self.observers.append(observer)
 691         for b in self.builds:
 692             observer(b)
 693     def unsubscribe(self, observer):
 694         self.observers.remove(observer)
 695
 696     def getSubmitTime(self):
 697         return self.submittedAt
 698     def setSubmitTime(self, t):
 699         self.submittedAt = t
 700
 701
 702 class BuildStepStatus(styles.Versioned):
 703     """
 704     I represent a collection of output status for a
 705     L{buildbot.process.step.BuildStep}.
 706
 707     Statistics contain any information gleaned from a step that is
 708     not in the form of a logfile.  As an example, steps that run
 709     tests might gather statistics about the number of passed, failed,
 710     or skipped tests.
 711
 712     @type progress: L{buildbot.status.progress.StepProgress}
 713     @cvar progress: tracks ETA for the step
 714     @type text: list of strings
 715     @cvar text: list of short texts that describe the command and its status
 716     @type text2: list of strings
 717     @cvar text2: list of short texts added to the overall build description
 718     @type logs: dict of string -> L{buildbot.status.builder.LogFile}
 719     @ivar logs: logs of steps
 720     @type statistics: dict
 721     @ivar statistics: results from running this step
 722     """
 723     # note that these are created when the Build is set up, before each
 724     # corresponding BuildStep has started.
 725     implements(interfaces.IBuildStepStatus, interfaces.IStatusEvent)
 726     persistenceVersion = 2
 727
 728     started = None
 729     finished = None
 730     progress = None
 731     text = []
 732     results = (None, [])
 733     text2 = []
 734     watchers = []
 735     updates = {}
 736     finishedWatchers = []
 737     statistics = {}
 738
 739     def __init__(self, parent):
 740         assert interfaces.IBuildStatus(parent)
 741         self.build = parent
 742         self.logs = []
 743         self.urls = {}
 744         self.watchers = []
 745         self.updates = {}
 746         self.finishedWatchers = []
 747         self.statistics = {}
 748
 749     def getName(self):
 750         """Returns a short string with the name of this step. This string
 751         may have spaces in it."""
 752         return self.name
 753
 754     def getBuild(self):
 755         return self.build
 756
 757     def getTimes(self):
 758         return (self.started, self.finished)
 759
 760     def getExpectations(self):
 761         """Returns a list of tuples (name, current, target)."""
 762         if not self.progress:
 763             return []
 764         ret = []
 765         metrics = self.progress.progress.keys()
 766         metrics.sort()
 767         for m in metrics:
 768             t = (m, self.progress.progress[m], self.progress.expectations[m])
 769             ret.append(t)
 770         return ret
 771
 772     def getLogs(self):
 773         return self.logs
 774
 775     def getURLs(self):
 776         return self.urls.copy()
 777
 778     def isFinished(self):
 779         return (self.finished is not None)
 780
 781     def waitUntilFinished(self):
 782         if self.finished:
 783             d = defer.succeed(self)
 784         else:
 785             d = defer.Deferred()
 786             self.finishedWatchers.append(d)
 787         return d
 788
 789     # while the step is running, the following methods make sense.
 790     # Afterwards they return None
 791
 792     def getETA(self):
 793         if self.started is None:
 794             return None # not started yet
 795         if self.finished is not None:
 796             return None # already finished
 797         if not self.progress:
 798             return None # no way to predict
 799         return self.progress.remaining()
 800
 801     # Once you know the step has finished, the following methods are legal.
 802     # Before this step has finished, they all return None.
 803
 804     def getText(self):
 805         """Returns a list of strings which describe the step. These are
 806         intended to be displayed in a narrow column. If more space is
 807         available, the caller should join them together with spaces before
 808         presenting them to the user."""
 809         return self.text
 810
 811     def getResults(self):
 812         """Return a tuple describing the results of the step.
 813         'result' is one of the constants in L{buildbot.status.builder}:
 814         SUCCESS, WARNINGS, FAILURE, or SKIPPED.
 815         'strings' is an optional list of strings that the step wants to
 816         append to the overall build's results. These strings are usually
 817         more terse than the ones returned by getText(): in particular,
 818         successful Steps do not usually contribute any text to the
 819         overall build.
 820
 821         @rtype:   tuple of int, list of strings
 822         @returns: (result, strings)
 823         """
 824         return (self.results, self.text2)
 825
 826     def hasStatistic(self, name):
 827         """Return true if this step has a value for the given statistic.
 828         """
 829         return self.statistics.has_key(name)
 830
 831     def getStatistic(self, name, default=None):
 832         """Return the given statistic, if present
 833         """
 834         return self.statistics.get(name, default)
 835
 836     # subscription interface
 837
 838     def subscribe(self, receiver, updateInterval=10):
 839         # will get logStarted, logFinished, stepETAUpdate
 840         assert receiver not in self.watchers
 841         self.watchers.append(receiver)
 842         self.sendETAUpdate(receiver, updateInterval)
 843
 844     def sendETAUpdate(self, receiver, updateInterval):
 845         self.updates[receiver] = None
 846         # they might unsubscribe during stepETAUpdate
 847         receiver.stepETAUpdate(self.build, self,
 848                            self.getETA(), self.getExpectations())
 849         if receiver in self.watchers:
 850             self.updates[receiver] = reactor.callLater(updateInterval,
 851                                                        self.sendETAUpdate,
 852                                                        receiver,
 853                                                        updateInterval)
 854
 855     def unsubscribe(self, receiver):
 856         if receiver in self.watchers:
 857             self.watchers.remove(receiver)
 858         if receiver in self.updates:
 859             if self.updates[receiver] is not None:
 860                 self.updates[receiver].cancel()
 861             del self.updates[receiver]
 862
 863
 864     # methods to be invoked by the BuildStep
 865
 866     def setName(self, stepname):
 867         self.name = stepname
 868
 869     def setColor(self, color):
 870         log.msg("BuildStepStatus.setColor is no longer supported -- ignoring color %s" % (color,))
 871
 872     def setProgress(self, stepprogress):
 873         self.progress = stepprogress
 874
 875     def stepStarted(self):
 876         self.started = util.now()
 877         if self.build:
 878             self.build.stepStarted(self)
 879
 880     def addLog(self, name):
 881         assert self.started # addLog before stepStarted won't notify watchers
 882         logfilename = self.build.generateLogfileName(self.name, name)
 883         log = LogFile(self, name, logfilename)
 884         self.logs.append(log)
 885         for w in self.watchers:
 886             receiver = w.logStarted(self.build, self, log)
 887             if receiver:
 888                 log.subscribe(receiver, True)
 889                 d = log.waitUntilFinished()
 890                 d.addCallback(lambda log: log.unsubscribe(receiver))
 891         d = log.waitUntilFinished()
 892         d.addCallback(self.logFinished)
 893         return log
 894
 895     def addHTMLLog(self, name, html):
 896         assert self.started # addLog before stepStarted won't notify watchers
 897         logfilename = self.build.generateLogfileName(self.name, name)
 898         log = HTMLLogFile(self, name, logfilename, html)
 899         self.logs.append(log)
 900         for w in self.watchers:
 901             receiver = w.logStarted(self.build, self, log)
 902             # TODO: think about this: there isn't much point in letting
 903             # them subscribe
 904             #if receiver:
 905             #    log.subscribe(receiver, True)
 906             w.logFinished(self.build, self, log)
 907
 908     def logFinished(self, log):
 909         for w in self.watchers:
 910             w.logFinished(self.build, self, log)
 911
 912     def addURL(self, name, url):
 913         self.urls[name] = url
 914
 915     def setText(self, text):
 916         self.text = text
 917         for w in self.watchers:
 918             w.stepTextChanged(self.build, self, text)
 919     def setText2(self, text):
 920         self.text2 = text
 921         for w in self.watchers:
 922             w.stepText2Changed(self.build, self, text)
 923
 924     def setStatistic(self, name, value):
 925         """Set the given statistic.  Usually called by subclasses.
 926         """
 927         self.statistics[name] = value
 928
 929     def stepFinished(self, results):
 930         self.finished = util.now()
 931         self.results = results
 932         cld = [] # deferreds for log compression
 933         logCompressionLimit = self.build.builder.logCompressionLimit
 934         for loog in self.logs:
 935             if not loog.isFinished():
 936                 loog.finish()
 937             # if log compression is on, and it's a real LogFile,
 938             # HTMLLogFiles aren't files
 939             if logCompressionLimit is not False and \
 940                     isinstance(loog, LogFile):
 941                 if os.path.getsize(loog.getFilename()) > logCompressionLimit:
 942                     cld.append(loog.compressLog())
 943
 944         for r in self.updates.keys():
 945             if self.updates[r] is not None:
 946                 self.updates[r].cancel()
 947                 del self.updates[r]
 948
 949         watchers = self.finishedWatchers
 950         self.finishedWatchers = []
 951         for w in watchers:
 952             w.callback(self)
 953         if cld:
 954             return defer.DeferredList(cld)
 955
 956     # persistence
 957
 958     def __getstate__(self):
 959         d = styles.Versioned.__getstate__(self)
 960         del d['build'] # filled in when loading
 961         if d.has_key('progress'):
 962             del d['progress']
 963         del d['watchers']
 964         del d['finishedWatchers']
 965         del d['updates']
 966         return d
 967
 968     def __setstate__(self, d):
 969         styles.Versioned.__setstate__(self, d)
 970         # self.build must be filled in by our parent
 971         for loog in self.logs:
 972             loog.step = self
 973
 974     def upgradeToVersion1(self):
 975         if not hasattr(self, "urls"):
 976             self.urls = {}
 977
 978     def upgradeToVersion2(self):
 979         if not hasattr(self, "statistics"):
 980             self.statistics = {}
 981
 982
 983 class BuildStatus(styles.Versioned):
 984     implements(interfaces.IBuildStatus, interfaces.IStatusEvent)
 985     persistenceVersion = 3
 986
 987     source = None
 988     reason = None
 989     changes = []
 990     blamelist = []
 991     requests = []
 992     progress = None
 993     started = None
 994     finished = None
 995     currentStep = None
 996     text = []
 997     results = None
 998     slavename = "???"
 999
1000     # these lists/dicts are defined here so that unserialized instances have
1001     # (empty) values. They are set in __init__ to new objects to make sure
1002     # each instance gets its own copy.
1003     watchers = []
1004     updates = {}
1005     finishedWatchers = []
1006     testResults = {}
1007
1008     def __init__(self, parent, number):
1009         """
1010         @type  parent: L{BuilderStatus}
1011         @type  number: int
1012         """
1013         assert interfaces.IBuilderStatus(parent)
1014         self.builder = parent
1015         self.number = number
1016         self.watchers = []
1017         self.updates = {}
1018         self.finishedWatchers = []
1019         self.steps = []
1020         self.testResults = {}
1021         self.properties = Properties()
1022         self.requests = []
1023
1024     def __repr__(self):
1025         return "<%s #%s>" % (self.__class__.__name__, self.number)
1026
1027     # IBuildStatus
1028
1029     def getBuilder(self):
1030         """
1031         @rtype: L{BuilderStatus}
1032         """
1033         return self.builder
1034
1035     def getProperty(self, propname):
1036         return self.properties[propname]
1037
1038     def getProperties(self):
1039         return self.properties
1040
1041     def getNumber(self):
1042         return self.number
1043
1044     def getPreviousBuild(self):
1045         if self.number == 0:
1046             return None
1047         return self.builder.getBuild(self.number-1)
1048
1049     def getSourceStamp(self, absolute=False):
1050         if not absolute or not self.properties.has_key('got_revision'):
1051             return self.source
1052         return self.source.getAbsoluteSourceStamp(self.properties['got_revision'])
1053
1054     def getReason(self):
1055         return self.reason
1056
1057     def getChanges(self):
1058         return self.changes
1059
1060     def getRequests(self):
1061         return self.requests
1062
1063     def getResponsibleUsers(self):
1064         return self.blamelist
1065
1066     def getInterestedUsers(self):
1067         # TODO: the Builder should add others: sheriffs, domain-owners
1068         return self.blamelist + self.properties.getProperty('owners', [])
1069
1070     def getSteps(self):
1071         """Return a list of IBuildStepStatus objects. For invariant builds
1072         (those which always use the same set of Steps), this should be the
1073         complete list, however some of the steps may not have started yet
1074         (step.getTimes()[0] will be None). For variant builds, this may not
1075         be complete (asking again later may give you more of them)."""
1076         return self.steps
1077
1078     def getTimes(self):
1079         return (self.started, self.finished)
1080
1081     _sentinel = [] # used as a sentinel to indicate unspecified initial_value
1082     def getSummaryStatistic(self, name, summary_fn, initial_value=_sentinel):
1083         """Summarize the named statistic over all steps in which it
1084         exists, using combination_fn and initial_value to combine multiple
1085         results into a single result.  This translates to a call to Python's
1086         X{reduce}::
1087             return reduce(summary_fn, step_stats_list, initial_value)
1088         """
1089         step_stats_list = [
1090                 st.getStatistic(name)
1091                 for st in self.steps
1092                 if st.hasStatistic(name) ]
1093         if initial_value is self._sentinel:
1094             return reduce(summary_fn, step_stats_list)
1095         else:
1096             return reduce(summary_fn, step_stats_list, initial_value)
1097
1098     def isFinished(self):
1099         return (self.finished is not None)
1100
1101     def waitUntilFinished(self):
1102         if self.finished:
1103             d = defer.succeed(self)
1104         else:
1105             d = defer.Deferred()
1106             self.finishedWatchers.append(d)
1107         return d
1108
1109     # while the build is running, the following methods make sense.
1110     # Afterwards they return None
1111
1112     def getETA(self):
1113         if self.finished is not None:
1114             return None
1115         if not self.progress:
1116             return None
1117         eta = self.progress.eta()
1118         if eta is None:
1119             return None
1120         return eta - util.now()
1121
1122     def getCurrentStep(self):
1123         return self.currentStep
1124
1125     # Once you know the build has finished, the following methods are legal.
1126     # Before ths build has finished, they all return None.
1127
1128     def getText(self):
1129         text = []
1130         text.extend(self.text)
1131         for s in self.steps:
1132             text.extend(s.text2)
1133         return text
1134
1135     def getResults(self):
1136         return self.results
1137
1138     def getSlavename(self):
1139         return self.slavename
1140
1141     def getTestResults(self):
1142         return self.testResults
1143
1144     def getLogs(self):
1145         # TODO: steps should contribute significant logs instead of this
1146         # hack, which returns every log from every step. The logs should get
1147         # names like "compile" and "test" instead of "compile.output"
1148         logs = []
1149         for s in self.steps:
1150             for log in s.getLogs():
1151                 logs.append(log)
1152         return logs
1153
1154     # subscription interface
1155
1156     def subscribe(self, receiver, updateInterval=None):
1157         # will receive stepStarted and stepFinished messages
1158         # and maybe buildETAUpdate
1159         self.watchers.append(receiver)
1160         if updateInterval is not None:
1161             self.sendETAUpdate(receiver, updateInterval)
1162
1163     def sendETAUpdate(self, receiver, updateInterval):
1164         self.updates[receiver] = None
1165         ETA = self.getETA()
1166         if ETA is not None:
1167             receiver.buildETAUpdate(self, self.getETA())
1168         # they might have unsubscribed during buildETAUpdate
1169         if receiver in self.watchers:
1170             self.updates[receiver] = reactor.callLater(updateInterval,
1171                                                        self.sendETAUpdate,
1172                                                        receiver,
1173                                                        updateInterval)
1174
1175     def unsubscribe(self, receiver):
1176         if receiver in self.watchers:
1177             self.watchers.remove(receiver)
1178         if receiver in self.updates:
1179             if self.updates[receiver] is not None:
1180                 self.updates[receiver].cancel()
1181             del self.updates[receiver]
1182
1183     # methods for the base.Build to invoke
1184
1185     def addStepWithName(self, name):
1186         """The Build is setting up, and has added a new BuildStep to its
1187         list. Create a BuildStepStatus object to which it can send status
1188         updates."""
1189
1190         s = BuildStepStatus(self)
1191         s.setName(name)
1192         self.steps.append(s)
1193         return s
1194
1195     def setProperty(self, propname, value, source):
1196         self.properties.setProperty(propname, value, source)
1197
1198     def addTestResult(self, result):
1199         self.testResults[result.getName()] = result
1200
1201     def setSourceStamp(self, sourceStamp):
1202         self.source = sourceStamp
1203         self.changes = self.source.changes
1204
1205     def setRequests(self, requests):
1206         self.requests = requests
1207
1208     def setReason(self, reason):
1209         self.reason = reason
1210     def setBlamelist(self, blamelist):
1211         self.blamelist = blamelist
1212     def setProgress(self, progress):
1213         self.progress = progress
1214
1215     def buildStarted(self, build):
1216         """The Build has been set up and is about to be started. It can now
1217         be safely queried, so it is time to announce the new build."""
1218
1219         self.started = util.now()
1220         # now that we're ready to report status, let the BuilderStatus tell
1221         # the world about us
1222         self.builder.buildStarted(self)
1223
1224     def setSlavename(self, slavename):
1225         self.slavename = slavename
1226
1227     def setText(self, text):
1228         assert isinstance(text, (list, tuple))
1229         self.text = text
1230     def setResults(self, results):
1231         self.results = results
1232
1233     def buildFinished(self):
1234         self.currentStep = None
1235         self.finished = util.now()
1236
1237         for r in self.updates.keys():
1238             if self.updates[r] is not None:
1239                 self.updates[r].cancel()
1240                 del self.updates[r]
1241
1242         watchers = self.finishedWatchers
1243         self.finishedWatchers = []
1244         for w in watchers:
1245             w.callback(self)
1246
1247     # methods called by our BuildStepStatus children
1248
1249     def stepStarted(self, step):
1250         self.currentStep = step
1251         name = self.getBuilder().getName()
1252         for w in self.watchers:
1253             receiver = w.stepStarted(self, step)
1254             if receiver:
1255                 if type(receiver) == type(()):
1256                     step.subscribe(receiver[0], receiver[1])
1257                 else:
1258                     step.subscribe(receiver)
1259                 d = step.waitUntilFinished()
1260                 d.addCallback(lambda step: step.unsubscribe(receiver))
1261
1262         step.waitUntilFinished().addCallback(self._stepFinished)
1263
1264     def _stepFinished(self, step):
1265         results = step.getResults()
1266         for w in self.watchers:
1267             w.stepFinished(self, step, results)
1268
1269     # methods called by our BuilderStatus parent
1270
1271     def pruneLogs(self):
1272         # this build is somewhat old: remove the build logs to save space
1273         # TODO: delete logs visible through IBuildStatus.getLogs
1274         for s in self.steps:
1275             s.pruneLogs()
1276
1277     def pruneSteps(self):
1278         # this build is very old: remove the build steps too
1279         self.steps = []
1280
1281     # persistence stuff
1282
1283     def generateLogfileName(self, stepname, logname):
1284         """Return a filename (relative to the Builder's base directory) where
1285         the logfile's contents can be stored uniquely.
1286
1287         The base filename is made by combining our build number, the Step's
1288         name, and the log's name, then removing unsuitable characters. The
1289         filename is then made unique by appending _0, _1, etc, until it does
1290         not collide with any other logfile.
1291
1292         These files are kept in the Builder's basedir (rather than a
1293         per-Build subdirectory) because that makes cleanup easier: cron and
1294         find will help get rid of the old logs, but the empty directories are
1295         more of a hassle to remove."""
1296
1297         starting_filename = "%d-log-%s-%s" % (self.number, stepname, logname)
1298         starting_filename = re.sub(r'[^\w\.\-]', '_', starting_filename)
1299         # now make it unique
1300         unique_counter = 0
1301         filename = starting_filename
1302         while filename in [l.filename
1303                            for step in self.steps
1304                            for l in step.getLogs()
1305                            if l.filename]:
1306             filename = "%s_%d" % (starting_filename, unique_counter)
1307             unique_counter += 1
1308         return filename
1309
1310     def __getstate__(self):
1311         d = styles.Versioned.__getstate__(self)
1312         # for now, a serialized Build is always "finished". We will never
1313         # save unfinished builds.
1314         if not self.finished:
1315             d['finished'] = True
1316             # TODO: push an "interrupted" step so it is clear that the build
1317             # was interrupted. The builder will have a 'shutdown' event, but
1318             # someone looking at just this build will be confused as to why
1319             # the last log is truncated.
1320         del d['builder'] # filled in by our parent when loading
1321         del d['watchers']
1322         del d['updates']
1323         del d['requests']
1324         del d['finishedWatchers']
1325         return d
1326
1327     def __setstate__(self, d):
1328         styles.Versioned.__setstate__(self, d)
1329         # self.builder must be filled in by our parent when loading
1330         for step in self.steps:
1331             step.build = self
1332         self.watchers = []
1333         self.updates = {}
1334         self.finishedWatchers = []
1335
1336     def upgradeToVersion1(self):
1337         if hasattr(self, "sourceStamp"):
1338             # the old .sourceStamp attribute wasn't actually very useful
1339             maxChangeNumber, patch = self.sourceStamp
1340             changes = getattr(self, 'changes', [])
1341             source = sourcestamp.SourceStamp(branch=None,
1342                                              revision=None,
1343                                              patch=patch,
1344                                              changes=changes)
1345             self.source = source
1346             self.changes = source.changes
1347             del self.sourceStamp
1348
1349     def upgradeToVersion2(self):
1350         self.properties = {}
1351
1352     def upgradeToVersion3(self):
1353         # in version 3, self.properties became a Properties object
1354         propdict = self.properties
1355         self.properties = Properties()
1356         self.properties.update(propdict, "Upgrade from previous version")
1357
1358     def upgradeLogfiles(self):
1359         # upgrade any LogFiles that need it. This must occur after we've been
1360         # attached to our Builder, and after we know about all LogFiles of
1361         # all Steps (to get the filenames right).
1362         assert self.builder
1363         for s in self.steps:
1364             for l in s.getLogs():
1365                 if l.filename:
1366                     pass # new-style, log contents are on disk
1367                 else:
1368                     logfilename = self.generateLogfileName(s.name, l.name)
1369                     # let the logfile update its .filename pointer,
1370                     # transferring its contents onto disk if necessary
1371                     l.upgrade(logfilename)
1372
1373     def saveYourself(self):
1374         filename = os.path.join(self.builder.basedir, "%d" % self.number)
1375         if os.path.isdir(filename):
1376             # leftover from 0.5.0, which stored builds in directories
1377             shutil.rmtree(filename, ignore_errors=True)
1378         tmpfilename = filename + ".tmp"
1379         try:
1380             dump(self, open(tmpfilename, "wb"), -1)
1381             if sys.platform == 'win32':
1382                 # windows cannot rename a file on top of an existing one, so
1383                 # fall back to delete-first. There are ways this can fail and
1384                 # lose the builder's history, so we avoid using it in the
1385                 # general (non-windows) case
1386                 if os.path.exists(filename):
1387                     os.unlink(filename)
1388             os.rename(tmpfilename, filename)
1389         except:
1390             log.msg("unable to save build %s-#%d" % (self.builder.name,
1391                                                      self.number))
1392             log.err()
1393
1394
1395
1396 class BuilderStatus(styles.Versioned):
1397     """I handle status information for a single process.base.Builder object.
1398     That object sends status changes to me (frequently as Events), and I
1399     provide them on demand to the various status recipients, like the HTML
1400     waterfall display and the live status clients. It also sends build
1401     summaries to me, which I log and provide to status clients who aren't
1402     interested in seeing details of the individual build steps.
1403
1404     I am responsible for maintaining the list of historic Events and Builds,
1405     pruning old ones, and loading them from / saving them to disk.
1406
1407     I live in the buildbot.process.base.Builder object, in the
1408     .builder_status attribute.
1409
1410     @type  category: string
1411     @ivar  category: user-defined category this builder belongs to; can be
1412                      used to filter on in status clients
1413     """
1414
1415     implements(interfaces.IBuilderStatus, interfaces.IEventSource)
1416     persistenceVersion = 1
1417
1418     # these limit the amount of memory we consume, as well as the size of the
1419     # main Builder pickle. The Build and LogFile pickles on disk must be
1420     # handled separately.
1421     buildCacheSize = 15
1422     buildHorizon = 100 # forget builds beyond this
1423     stepHorizon = 50 # forget steps in builds beyond this
1424     eventHorizon = 50 # forget events beyond this
1425
1426     category = None
1427     currentBigState = "offline" # or idle/waiting/interlocked/building
1428     basedir = None # filled in by our parent
1429
1430     def __init__(self, buildername, category=None):
1431         self.name = buildername
1432         self.category = category
1433
1434         self.slavenames = []
1435         self.events = []
1436         # these three hold Events, and are used to retrieve the current
1437         # state of the boxes.
1438         self.lastBuildStatus = None
1439         #self.currentBig = None
1440         #self.currentSmall = None
1441         self.currentBuilds = []
1442         self.pendingBuilds = []
1443         self.nextBuild = None
1444         self.watchers = []
1445         self.buildCache = weakref.WeakValueDictionary()
1446         self.buildCache_LRU = []
1447         self.logCompressionLimit = False # default to no compression for tests
1448
1449     # persistence
1450
1451     def __getstate__(self):
1452         # when saving, don't record transient stuff like what builds are
1453         # currently running, because they won't be there when we start back
1454         # up. Nor do we save self.watchers, nor anything that gets set by our
1455         # parent like .basedir and .status
1456         d = styles.Versioned.__getstate__(self)
1457         d['watchers'] = []
1458         del d['buildCache']
1459         del d['buildCache_LRU']
1460         for b in self.currentBuilds:
1461             b.saveYourself()
1462             # TODO: push a 'hey, build was interrupted' event
1463         del d['currentBuilds']
1464         del d['pendingBuilds']
1465         del d['currentBigState']
1466         del d['basedir']
1467         del d['status']
1468         del d['nextBuildNumber']
1469         return d
1470
1471     def __setstate__(self, d):
1472         # when loading, re-initialize the transient stuff. Remember that
1473         # upgradeToVersion1 and such will be called after this finishes.
1474         styles.Versioned.__setstate__(self, d)
1475         self.buildCache = weakref.WeakValueDictionary()
1476         self.buildCache_LRU = []
1477         self.currentBuilds = []
1478         self.pendingBuilds = []
1479         self.watchers = []
1480         self.slavenames = []
1481         # self.basedir must be filled in by our parent
1482         # self.status must be filled in by our parent
1483
1484     def upgradeToVersion1(self):
1485         if hasattr(self, 'slavename'):
1486             self.slavenames = [self.slavename]
1487             del self.slavename
1488         if hasattr(self, 'nextBuildNumber'):
1489             del self.nextBuildNumber # determineNextBuildNumber chooses this
1490
1491     def determineNextBuildNumber(self):
1492         """Scan our directory of saved BuildStatus instances to determine
1493         what our self.nextBuildNumber should be. Set it one larger than the
1494         highest-numbered build we discover. This is called by the top-level
1495         Status object shortly after we are created or loaded from disk.
1496         """
1497         existing_builds = [int(f)
1498                            for f in os.listdir(self.basedir)
1499                            if re.match("^\d+$", f)]
1500         if existing_builds:
1501             self.nextBuildNumber = max(existing_builds) + 1
1502         else:
1503             self.nextBuildNumber = 0
1504
1505     def setLogCompressionLimit(self, lowerLimit):
1506         self.logCompressionLimit = lowerLimit
1507
1508     def saveYourself(self):
1509         for b in self.currentBuilds:
1510             if not b.isFinished:
1511                 # interrupted build, need to save it anyway.
1512                 # BuildStatus.saveYourself will mark it as interrupted.
1513                 b.saveYourself()
1514         filename = os.path.join(self.basedir, "builder")
1515         tmpfilename = filename + ".tmp"
1516         try:
1517             dump(self, open(tmpfilename, "wb"), -1)
1518             if sys.platform == 'win32':
1519                 # windows cannot rename a file on top of an existing one
1520                 if os.path.exists(filename):
1521                     os.unlink(filename)
1522             os.rename(tmpfilename, filename)
1523         except:
1524             log.msg("unable to save builder %s" % self.name)
1525             log.err()
1526
1527
1528     # build cache management
1529
1530     def touchBuildCache(self, build):
1531         self.buildCache[build.number] = build
1532         if build in self.buildCache_LRU:
1533             self.buildCache_LRU.remove(build)
1534         self.buildCache_LRU = self.buildCache_LRU[-(self.buildCacheSize-1):] + [ build ]
1535         return build
1536
1537     def getBuildByNumber(self, number):
1538         # first look in currentBuilds
1539         for b in self.currentBuilds:
1540             if b.number == number:
1541                 return self.touchBuildCache(b)
1542
1543         # then in the buildCache
1544         if number in self.buildCache:
1545             return self.touchBuildCache(self.buildCache[number])
1546
1547         # then fall back to loading it from disk
1548         filename = os.path.join(self.basedir, "%d" % number)
1549         try:
1550             log.msg("Loading builder %s's build %d from on-disk pickle"
1551                 % (self.name, number))
1552             build = load(open(filename, "rb"))
1553             styles.doUpgrade()
1554             build.builder = self
1555             # handle LogFiles from after 0.5.0 and before 0.6.5
1556             build.upgradeLogfiles()
1557             return self.touchBuildCache(build)
1558         except IOError:
1559             raise IndexError("no such build %d" % number)
1560         except EOFError:
1561             raise IndexError("corrupted build pickle %d" % number)
1562
1563     def prune(self):
1564         return # TODO: change this to walk through the filesystem
1565         # first, blow away all builds beyond our build horizon
1566         self.builds = self.builds[-self.buildHorizon:]
1567         # then prune steps in builds past the step horizon
1568         for b in self.builds[0:-self.stepHorizon]:
1569             b.pruneSteps()
1570
1571     def pruneEvents(self):
1572         self.events = self.events[-self.eventHorizon:]
1573
1574     # IBuilderStatus methods
1575     def getName(self):
1576         return self.name
1577
1578     def getState(self):
1579         return (self.currentBigState, self.currentBuilds)
1580
1581     def getSlaves(self):
1582         return [self.status.getSlave(name) for name in self.slavenames]
1583
1584     def getPendingBuilds(self):
1585         return self.pendingBuilds
1586
1587     def getCurrentBuilds(self):
1588         return self.currentBuilds
1589
1590     def getLastFinishedBuild(self):
1591         b = self.getBuild(-1)
1592         if not (b and b.isFinished()):
1593             b = self.getBuild(-2)
1594         return b
1595
1596     def getBuild(self, number):
1597         if number < 0:
1598             number = self.nextBuildNumber + number
1599         if number < 0 or number >= self.nextBuildNumber:
1600             return None
1601
1602         try:
1603             return self.getBuildByNumber(number)
1604         except IndexError:
1605             return None
1606
1607     def getEvent(self, number):
1608         try:
1609             return self.events[number]
1610         except IndexError:
1611             return None
1612
1613     def generateFinishedBuilds(self, branches=[],
1614                                num_builds=None,
1615                                max_buildnum=None,
1616                                finished_before=None,
1617                                max_search=200):
1618         got = 0
1619         for Nb in itertools.count(1):
1620             if Nb > self.nextBuildNumber:
1621                 break
1622             if Nb > max_search:
1623                 break
1624             build = self.getBuild(-Nb)
1625             if build is None:
1626                 continue
1627             if max_buildnum is not None:
1628                 if build.getNumber() > max_buildnum:
1629                     continue
1630             if not build.isFinished():
1631                 continue
1632             if finished_before is not None:
1633                 start, end = build.getTimes()
1634                 if end >= finished_before:
1635                     continue
1636             if branches:
1637                 if build.getSourceStamp().branch not in branches:
1638                     continue
1639             got += 1
1640             yield build
1641             if num_builds is not None:
1642                 if got >= num_builds:
1643                     return
1644
1645     def eventGenerator(self, branches=[]):
1646         """This function creates a generator which will provide all of this
1647         Builder's status events, starting with the most recent and
1648         progressing backwards in time. """
1649
1650         # remember the oldest-to-earliest flow here. "next" means earlier.
1651
1652         # TODO: interleave build steps and self.events by timestamp.
1653         # TODO: um, I think we're already doing that.
1654
1655         # TODO: there's probably something clever we could do here to
1656         # interleave two event streams (one from self.getBuild and the other
1657         # from self.getEvent), which would be simpler than this control flow
1658
1659         eventIndex = -1
1660         e = self.getEvent(eventIndex)
1661         for Nb in range(1, self.nextBuildNumber+1):
1662             b = self.getBuild(-Nb)
1663             if not b:
1664                 break
1665             if branches and not b.getSourceStamp().branch in branches:
1666                 continue
1667             steps = b.getSteps()
1668             for Ns in range(1, len(steps)+1):
1669                 if steps[-Ns].started:
1670                     step_start = steps[-Ns].getTimes()[0]
1671                     while e is not None and e.getTimes()[0] > step_start:
1672                         yield e
1673                         eventIndex -= 1
1674                         e = self.getEvent(eventIndex)
1675                     yield steps[-Ns]
1676             yield b
1677         while e is not None:
1678             yield e
1679             eventIndex -= 1
1680             e = self.getEvent(eventIndex)
1681
1682     def subscribe(self, receiver):
1683         # will get builderChangedState, buildStarted, and buildFinished
1684         self.watchers.append(receiver)
1685         self.publishState(receiver)
1686
1687     def unsubscribe(self, receiver):
1688         self.watchers.remove(receiver)
1689
1690     ## Builder interface (methods called by the Builder which feeds us)
1691
1692     def setSlavenames(self, names):
1693         self.slavenames = names
1694
1695     def addEvent(self, text=[]):
1696         # this adds a duration event. When it is done, the user should call
1697         # e.finish(). They can also mangle it by modifying .text
1698         e = Event()
1699         e.started = util.now()
1700         e.text = text
1701         self.events.append(e)
1702         self.pruneEvents()
1703         return e # they are free to mangle it further
1704
1705     def addPointEvent(self, text=[]):
1706         # this adds a point event, one which occurs as a single atomic
1707         # instant of time.
1708         e = Event()
1709         e.started = util.now()
1710         e.finished = 0
1711         e.text = text
1712         self.events.append(e)
1713         self.pruneEvents()
1714         return e # for consistency, but they really shouldn't touch it
1715
1716     def setBigState(self, state):
1717         needToUpdate = state != self.currentBigState
1718         self.currentBigState = state
1719         if needToUpdate:
1720             self.publishState()
1721
1722     def publishState(self, target=None):
1723         state = self.currentBigState
1724
1725         if target is not None:
1726             # unicast
1727             target.builderChangedState(self.name, state)
1728             return
1729         for w in self.watchers:
1730             try:
1731                 w.builderChangedState(self.name, state)
1732             except:
1733                 log.msg("Exception caught publishing state to %r" % w)
1734                 log.err()
1735
1736     def newBuild(self):
1737         """The Builder has decided to start a build, but the Build object is
1738         not yet ready to report status (it has not finished creating the
1739         Steps). Create a BuildStatus object that it can use."""
1740         number = self.nextBuildNumber
1741         self.nextBuildNumber += 1
1742         # TODO: self.saveYourself(), to make sure we don't forget about the
1743         # build number we've just allocated. This is not quite as important
1744         # as it was before we switch to determineNextBuildNumber, but I think
1745         # it may still be useful to have the new build save itself.
1746         s = BuildStatus(self, number)
1747         s.waitUntilFinished().addCallback(self._buildFinished)
1748         return s
1749
1750     def addBuildRequest(self, brstatus):
1751         self.pendingBuilds.append(brstatus)
1752         for w in self.watchers:
1753             w.requestSubmitted(brstatus)
1754
1755     def removeBuildRequest(self, brstatus):
1756         self.pendingBuilds.remove(brstatus)
1757
1758     # buildStarted is called by our child BuildStatus instances
1759     def buildStarted(self, s):
1760         """Now the BuildStatus object is ready to go (it knows all of its
1761         Steps, its ETA, etc), so it is safe to notify our watchers."""
1762
1763         assert s.builder is self # paranoia
1764         assert s.number == self.nextBuildNumber - 1
1765         assert s not in self.currentBuilds
1766         self.currentBuilds.append(s)
1767         self.touchBuildCache(s)
1768
1769         # now that the BuildStatus is prepared to answer queries, we can
1770         # announce the new build to all our watchers
1771
1772         for w in self.watchers: # TODO: maybe do this later? callLater(0)?
1773             try:
1774                 receiver = w.buildStarted(self.getName(), s)
1775                 if receiver:
1776                     if type(receiver) == type(()):
1777                         s.subscribe(receiver[0], receiver[1])
1778                     else:
1779                         s.subscribe(receiver)
1780                     d = s.waitUntilFinished()
1781                     d.addCallback(lambda s: s.unsubscribe(receiver))
1782             except:
1783                 log.msg("Exception caught notifying %r of buildStarted event" % w)
1784                 log.err()
1785
1786     def _buildFinished(self, s):
1787         assert s in self.currentBuilds
1788         s.saveYourself()
1789         self.currentBuilds.remove(s)
1790
1791         name = self.getName()
1792         results = s.getResults()
1793         for w in self.watchers:
1794             try:
1795                 w.buildFinished(name, s, results)
1796             except:
1797                 log.msg("Exception caught notifying %r of buildFinished event" % w)
1798                 log.err()
1799
1800         self.prune() # conserve disk
1801
1802
1803     # waterfall display (history)
1804
1805     # I want some kind of build event that holds everything about the build:
1806     # why, what changes went into it, the results of the build, itemized
1807     # test results, etc. But, I do kind of need something to be inserted in
1808     # the event log first, because intermixing step events and the larger
1809     # build event is fraught with peril. Maybe an Event-like-thing that
1810     # doesn't have a file in it but does have links. Hmm, that's exactly
1811     # what it does now. The only difference would be that this event isn't
1812     # pushed to the clients.
1813
1814     # publish to clients
1815     def sendLastBuildStatus(self, client):
1816         #client.newLastBuildStatus(self.lastBuildStatus)
1817         pass
1818     def sendCurrentActivityBigToEveryone(self):
1819         for s in self.subscribers:
1820             self.sendCurrentActivityBig(s)
1821     def sendCurrentActivityBig(self, client):
1822         state = self.currentBigState
1823         if state == "offline":
1824             client.currentlyOffline()
1825         elif state == "idle":
1826             client.currentlyIdle()
1827         elif state == "building":
1828             client.currentlyBuilding()
1829         else:
1830             log.msg("Hey, self.currentBigState is weird:", state)
1831
1832
1833     ## HTML display interface
1834
1835     def getEventNumbered(self, num):
1836         # deal with dropped events, pruned events
1837         first = self.events[0].number
1838         if first + len(self.events)-1 != self.events[-1].number:
1839             log.msg(self,
1840                     "lost an event somewhere: [0] is %d, [%d] is %d" % \
1841                     (self.events[0].number,
1842                      len(self.events) - 1,
1843                      self.events[-1].number))
1844             for e in self.events:
1845                 log.msg("e[%d]: " % e.number, e)
1846             return None
1847         offset = num - first
1848         log.msg(self, "offset", offset)
1849         try:
1850             return self.events[offset]
1851         except IndexError:
1852             return None
1853
1854     ## Persistence of Status
1855     def loadYourOldEvents(self):
1856         if hasattr(self, "allEvents"):
1857             # first time, nothing to get from file. Note that this is only if
1858             # the Application gets .run() . If it gets .save()'ed, then the
1859             # .allEvents attribute goes away in the initial __getstate__ and
1860             # we try to load a non-existent file.
1861             return
1862         self.allEvents = self.loadFile("events", [])
1863         if self.allEvents:
1864             self.nextEventNumber = self.allEvents[-1].number + 1
1865         else:
1866             self.nextEventNumber = 0
1867     def saveYourOldEvents(self):
1868         self.saveFile("events", self.allEvents)
1869
1870     ## clients
1871
1872     def addClient(self, client):
1873         if client not in self.subscribers:
1874             self.subscribers.append(client)
1875             self.sendLastBuildStatus(client)
1876             self.sendCurrentActivityBig(client)
1877             client.newEvent(self.currentSmall)
1878     def removeClient(self, client):
1879         if client in self.subscribers:
1880             self.subscribers.remove(client)
1881
1882 class SlaveStatus:
1883     implements(interfaces.ISlaveStatus)
1884
1885     admin = None
1886     host = None
1887     connected = False
1888     graceful_shutdown = False
1889
1890     def __init__(self, name):
1891         self.name = name
1892         self._lastMessageReceived = 0
1893         self.runningBuilds = []
1894         self.graceful_callbacks = []
1895
1896     def getName(self):
1897         return self.name
1898     def getAdmin(self):
1899         return self.admin
1900     def getHost(self):
1901         return self.host
1902     def isConnected(self):
1903         return self.connected
1904     def lastMessageReceived(self):
1905         return self._lastMessageReceived
1906     def getRunningBuilds(self):
1907         return self.runningBuilds
1908
1909     def setAdmin(self, admin):
1910         self.admin = admin
1911     def setHost(self, host):
1912         self.host = host
1913     def setConnected(self, isConnected):
1914         self.connected = isConnected
1915     def setLastMessageReceived(self, when):
1916         self._lastMessageReceived = when
1917
1918     def buildStarted(self, build):
1919         self.runningBuilds.append(build)
1920     def buildFinished(self, build):
1921         self.runningBuilds.remove(build)
1922
1923     def getGraceful(self):
1924         """Return the graceful shutdown flag"""
1925         return self.graceful_shutdown
1926     def setGraceful(self, graceful):
1927         """Set the graceful shutdown flag, and notify all the watchers"""
1928         self.graceful_shutdown = graceful
1929         for cb in self.graceful_callbacks:
1930             reactor.callLater(0, cb, graceful)
1931     def addGracefulWatcher(self, watcher):
1932         """Add watcher to the list of watchers to be notified when the
1933         graceful shutdown flag is changed."""
1934         if not watcher in self.graceful_callbacks:
1935             self.graceful_callbacks.append(watcher)
1936     def removeGracefulWatcher(self, watcher):
1937         """Remove watcher from the list of watchers to be notified when the
1938         graceful shutdown flag is changed."""
1939         if watcher in self.graceful_callbacks:
1940             self.graceful_callbacks.remove(watcher)
1941
1942 class Status:
1943     """
1944     I represent the status of the buildmaster.
1945     """
1946     implements(interfaces.IStatus)
1947
1948     def __init__(self, botmaster, basedir):
1949         """
1950         @type  botmaster: L{buildbot.master.BotMaster}
1951         @param botmaster: the Status object uses C{.botmaster} to get at
1952                           both the L{buildbot.master.BuildMaster} (for
1953                           various buildbot-wide parameters) and the
1954                           actual Builders (to get at their L{BuilderStatus}
1955                           objects). It is not allowed to change or influence
1956                           anything through this reference.
1957         @type  basedir: string
1958         @param basedir: this provides a base directory in which saved status
1959                         information (changes.pck, saved Build status
1960                         pickles) can be stored
1961         """
1962         self.botmaster = botmaster
1963         self.basedir = basedir
1964         self.watchers = []
1965         self.activeBuildSets = []
1966         assert os.path.isdir(basedir)
1967         # compress logs bigger than 4k, a good default on linux
1968         self.logCompressionLimit = 4*1024
1969
1970
1971     # methods called by our clients
1972
1973     def getProjectName(self):
1974         return self.botmaster.parent.projectName
1975     def getProjectURL(self):
1976         return self.botmaster.parent.projectURL
1977     def getBuildbotURL(self):
1978         return self.botmaster.parent.buildbotURL
1979
1980     def getURLForThing(self, thing):
1981         prefix = self.getBuildbotURL()
1982         if not prefix:
1983             return None
1984         if interfaces.IStatus.providedBy(thing):
1985             return prefix
1986         if interfaces.ISchedulerStatus.providedBy(thing):
1987             pass
1988         if interfaces.IBuilderStatus.providedBy(thing):
1989             builder = thing
1990             return prefix + "builders/%s" % (
1991                 urllib.quote(builder.getName(), safe=''),
1992                 )
1993         if interfaces.IBuildStatus.providedBy(thing):
1994             build = thing
1995             builder = build.getBuilder()
1996             return prefix + "builders/%s/builds/%d" % (
1997                 urllib.quote(builder.getName(), safe=''),
1998                 build.getNumber())
1999         if interfaces.IBuildStepStatus.providedBy(thing):
2000             step = thing
2001             build = step.getBuild()
2002             builder = build.getBuilder()
2003             return prefix + "builders/%s/builds/%d/steps/%s" % (
2004                 urllib.quote(builder.getName(), safe=''),
2005                 build.getNumber(),
2006                 urllib.quote(step.getName(), safe=''))
2007         # IBuildSetStatus
2008         # IBuildRequestStatus
2009         # ISlaveStatus
2010
2011         # IStatusEvent
2012         if interfaces.IStatusEvent.providedBy(thing):
2013             from buildbot.changes import changes
2014             # TODO: this is goofy, create IChange or something
2015             if isinstance(thing, changes.Change):
2016                 change = thing
2017                 return "%schanges/%d" % (prefix, change.number)
2018
2019         if interfaces.IStatusLog.providedBy(thing):
2020             log = thing
2021             step = log.getStep()
2022             build = step.getBuild()
2023             builder = build.getBuilder()
2024
2025             logs = step.getLogs()
2026             for i in range(len(logs)):
2027                 if log is logs[i]:
2028                     lognum = i
2029                     break
2030             else:
2031                 return None
2032             return prefix + "builders/%s/builds/%d/steps/%s/logs/%d" % (
2033                 urllib.quote(builder.getName(), safe=''),
2034                 build.getNumber(),
2035                 urllib.quote(step.getName(), safe=''),
2036                 lognum)
2037
2038     def getChangeSources(self):
2039         return list(self.botmaster.parent.change_svc)
2040
2041     def getChange(self, number):
2042         return self.botmaster.parent.change_svc.getChangeNumbered(number)
2043
2044     def getSchedulers(self):
2045         return self.botmaster.parent.allSchedulers()
2046
2047     def getBuilderNames(self, categories=None):
2048         if categories == None:
2049             return self.botmaster.builderNames[:] # don't let them break it
2050
2051         l = []
2052         # respect addition order
2053         for name in self.botmaster.builderNames:
2054             builder = self.botmaster.builders[name]
2055             if builder.builder_status.category in categories:
2056                 l.append(name)
2057         return l
2058
2059     def getBuilder(self, name):
2060         """
2061         @rtype: L{BuilderStatus}
2062         """
2063         return self.botmaster.builders[name].builder_status
2064
2065     def getSlaveNames(self):
2066         return self.botmaster.slaves.keys()
2067
2068     def getSlave(self, slavename):
2069         return self.botmaster.slaves[slavename].slave_status
2070
2071     def getBuildSets(self):
2072         return self.activeBuildSets[:]
2073
2074     def generateFinishedBuilds(self, builders=[], branches=[],
2075                                num_builds=None, finished_before=None,
2076                                max_search=200):
2077
2078         def want_builder(bn):
2079             if builders:
2080                 return bn in builders
2081             return True
2082         builder_names = [bn
2083                          for bn in self.getBuilderNames()
2084                          if want_builder(bn)]
2085
2086         # 'sources' is a list of generators, one for each Builder we're
2087         # using. When the generator is exhausted, it is replaced in this list
2088         # with None.
2089         sources = []
2090         for bn in builder_names:
2091             b = self.getBuilder(bn)
2092             g = b.generateFinishedBuilds(branches,
2093                                          finished_before=finished_before,
2094                                          max_search=max_search)
2095             sources.append(g)
2096
2097         # next_build the next build from each source
2098         next_build = [None] * len(sources)
2099
2100         def refill():
2101             for i,g in enumerate(sources):
2102                 if next_build[i]:
2103                     # already filled
2104                     continue
2105                 if not g:
2106                     # already exhausted
2107                     continue
2108                 try:
2109                     next_build[i] = g.next()
2110                 except StopIteration:
2111                     next_build[i] = None
2112                     sources[i] = None
2113
2114         got = 0
2115         while True:
2116             refill()
2117             # find the latest build among all the candidates
2118             candidates = [(i, b, b.getTimes()[1])
2119                           for i,b in enumerate(next_build)
2120                           if b is not None]
2121             candidates.sort(lambda x,y: cmp(x[2], y[2]))
2122             if not candidates:
2123                 return
2124
2125             # and remove it from the list
2126             i, build, finshed_time = candidates[-1]
2127             next_build[i] = None
2128             got += 1
2129             yield build
2130             if num_builds is not None:
2131                 if got >= num_builds:
2132                     return
2133
2134     def subscribe(self, target):
2135         self.watchers.append(target)
2136         for name in self.botmaster.builderNames:
2137             self.announceNewBuilder(target, name, self.getBuilder(name))
2138     def unsubscribe(self, target):
2139         self.watchers.remove(target)
2140
2141
2142     # methods called by upstream objects
2143
2144     def announceNewBuilder(self, target, name, builder_status):
2145         t = target.builderAdded(name, builder_status)
2146         if t:
2147             builder_status.subscribe(t)
2148
2149     def builderAdded(self, name, basedir, category=None):
2150         """
2151         @rtype: L{BuilderStatus}
2152         """
2153         filename = os.path.join(self.basedir, basedir, "builder")
2154         log.msg("trying to load status pickle from %s" % filename)
2155         builder_status = None
2156         try:
2157             builder_status = load(open(filename, "rb"))
2158             styles.doUpgrade()
2159         except IOError:
2160             log.msg("no saved status pickle, creating a new one")
2161         except:
2162             log.msg("error while loading status pickle, creating a new one")
2163             log.msg("error follows:")
2164             log.err()
2165         if not builder_status:
2166             builder_status = BuilderStatus(name, category)
2167             builder_status.addPointEvent(["builder", "created"])
2168         log.msg("added builder %s in category %s" % (name, category))
2169         # an unpickled object might not have category set from before,
2170         # so set it here to make sure
2171         builder_status.category = category
2172         builder_status.basedir = os.path.join(self.basedir, basedir)
2173         builder_status.name = name # it might have been updated
2174         builder_status.status = self
2175
2176         if not os.path.isdir(builder_status.basedir):
2177             os.makedirs(builder_status.basedir)
2178         builder_status.determineNextBuildNumber()
2179
2180         builder_status.setBigState("offline")
2181         builder_status.setLogCompressionLimit(self.logCompressionLimit)
2182
2183         for t in self.watchers:
2184             self.announceNewBuilder(t, name, builder_status)
2185
2186         return builder_status
2187
2188     def builderRemoved(self, name):
2189         for t in self.watchers:
2190             t.builderRemoved(name)
2191
2192     def prune(self):
2193         for b in self.botmaster.builders.values():
2194             b.builder_status.prune()
2195
2196     def buildsetSubmitted(self, bss):
2197         self.activeBuildSets.append(bss)
2198         bss.waitUntilFinished().addCallback(self.activeBuildSets.remove)
2199         for t in self.watchers:
2200             t.buildsetSubmitted(bss)