From aea14d3f245badd92c5e197803cf56cc4e6d96be Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Thu, 19 Mar 2009 12:31:10 -0400 Subject: [PATCH] (refs #459) actually prune old logfiles and builds Builds older than buildbot.status.builder.BuilderStatus.buildHorizon are deleted entirely, while logfiles in builds older than logHorizon are deleted, leaving the builds and steps intact. The existing stepHorizion was removed, as it's too time-consuming to open each build pickle to determine whether the steps have been removed. --- buildbot/status/builder.py | 104 +++++++++++++++++++++++++++++++------------ buildbot/status/web/build.py | 39 ++++++++-------- buildbot/test/test_status.py | 46 ++++++++++++++++++- 3 files changed, 140 insertions(+), 49 deletions(-) diff --git a/buildbot/status/builder.py b/buildbot/status/builder.py index a54e73b..e880ac7 100644 --- a/buildbot/status/builder.py +++ b/buildbot/status/builder.py @@ -9,6 +9,7 @@ from buildbot.process.properties import Properties import weakref import os, shutil, sys, re, urllib, itertools +import gc from cPickle import load, dump from cStringIO import StringIO from bz2 import BZ2File @@ -274,6 +275,13 @@ class LogFile: self.finishedWatchers.append(d) return d + def logfileExists(self): + if self.openfile: return True + fn = self.getFilename() + for f in (fn, fn + ".bz2"): + if os.path.exists(f): return True + return False + def getFile(self): if self.openfile: # this is the filehandle we're using to write to the log, so @@ -953,6 +961,10 @@ class BuildStepStatus(styles.Versioned): if cld: return defer.DeferredList(cld) + def checkLogfiles(self): + # filter out logs that have been deleted + self.logs = [ l for l in self.logs if l.logfileExists() ] + # persistence def __getstate__(self): @@ -968,6 +980,8 @@ class BuildStepStatus(styles.Versioned): def __setstate__(self, d): styles.Versioned.__setstate__(self, d) # self.build must be filled in by our parent + + # point the logs to this object for loog in self.logs: loog.step = self @@ -1268,12 +1282,6 @@ class BuildStatus(styles.Versioned): # methods called by our BuilderStatus parent - def pruneLogs(self): - # this build is somewhat old: remove the build logs to save space - # TODO: delete logs visible through IBuildStatus.getLogs - for s in self.steps: - s.pruneLogs() - def pruneSteps(self): # this build is very old: remove the build steps too self.steps = [] @@ -1317,11 +1325,8 @@ class BuildStatus(styles.Versioned): # was interrupted. The builder will have a 'shutdown' event, but # someone looking at just this build will be confused as to why # the last log is truncated. - del d['builder'] # filled in by our parent when loading - del d['watchers'] - del d['updates'] - del d['requests'] - del d['finishedWatchers'] + for k in 'builder', 'watchers', 'updates', 'requests', 'finishedWatchers': + if k in d: del d[k] return d def __setstate__(self, d): @@ -1370,6 +1375,12 @@ class BuildStatus(styles.Versioned): # transferring its contents onto disk if necessary l.upgrade(logfilename) + def checkLogfiles(self): + # check that all logfiles exist, and remove references to any that + # have been deleted (e.g., by purge()) + for s in self.steps: + s.checkLogfiles() + def saveYourself(self): filename = os.path.join(self.builder.basedir, "%d" % self.number) if os.path.isdir(filename): @@ -1419,10 +1430,12 @@ class BuilderStatus(styles.Versioned): # main Builder pickle. The Build and LogFile pickles on disk must be # handled separately. buildCacheSize = 15 - buildHorizon = 100 # forget builds beyond this - stepHorizon = 50 # forget steps in builds beyond this eventHorizon = 50 # forget events beyond this + # these limit on-disk storage + logHorizon = 40 # forget logs in steps in builds beyond this + buildHorizon = 100 # forget builds beyond this + category = None currentBigState = "offline" # or idle/waiting/interlocked/building basedir = None # filled in by our parent @@ -1527,6 +1540,9 @@ class BuilderStatus(styles.Versioned): # build cache management + def makeBuildFilename(self, number): + return os.path.join(self.basedir, "%d" % number) + def touchBuildCache(self, build): self.buildCache[build.number] = build if build in self.buildCache_LRU: @@ -1545,7 +1561,7 @@ class BuilderStatus(styles.Versioned): return self.touchBuildCache(self.buildCache[number]) # then fall back to loading it from disk - filename = os.path.join(self.basedir, "%d" % number) + filename = self.makeBuildFilename(number) try: log.msg("Loading builder %s's build %d from on-disk pickle" % (self.name, number)) @@ -1554,6 +1570,8 @@ class BuilderStatus(styles.Versioned): build.builder = self # handle LogFiles from after 0.5.0 and before 0.6.5 build.upgradeLogfiles() + # check that logfiles exist + build.checkLogfiles() return self.touchBuildCache(build) except IOError: raise IndexError("no such build %d" % number) @@ -1561,16 +1579,52 @@ class BuilderStatus(styles.Versioned): raise IndexError("corrupted build pickle %d" % number) def prune(self): - return # TODO: change this to walk through the filesystem - # first, blow away all builds beyond our build horizon - self.builds = self.builds[-self.buildHorizon:] - # then prune steps in builds past the step horizon - for b in self.builds[0:-self.stepHorizon]: - b.pruneSteps() - - def pruneEvents(self): + gc.collect() + + # begin by pruning our own events self.events = self.events[-self.eventHorizon:] + # get the horizons straight + if self.buildHorizon: + earliest_build = self.nextBuildNumber - self.buildHorizon + else: + earliest_build = 0 + + if self.logHorizon: + earliest_log = self.nextBuildNumber - self.logHorizon + else: + earliest_log = 0 + + if earliest_log < earliest_build: + earliest_log = earliest_build + + if earliest_build == 0: + return + + # skim the directory and delete anything that shouldn't be there anymore + build_re = re.compile(r"^([0-9]+)$") + build_log_re = re.compile(r"^([0-9]+)-.*$") + for filename in os.listdir(self.basedir): + num = None + mo = build_re.match(filename) + is_logfile = False + if mo: + num = int(mo.group(1)) + else: + mo = build_log_re.match(filename) + if mo: + num = int(mo.group(1)) + is_logfile = True + + if not num: continue + if num in self.buildCache: continue + + if (is_logfile and num < earliest_log) or num < earliest_build: + pathname = os.path.join(self.basedir, filename) + log.msg("pruning '%s'" % pathname) + try: os.unlink(pathname) + except OSError: pass + # IBuilderStatus methods def getName(self): return self.name @@ -1699,7 +1753,6 @@ class BuilderStatus(styles.Versioned): e.started = util.now() e.text = text self.events.append(e) - self.pruneEvents() return e # they are free to mangle it further def addPointEvent(self, text=[]): @@ -1710,7 +1763,6 @@ class BuilderStatus(styles.Versioned): e.finished = 0 e.text = text self.events.append(e) - self.pruneEvents() return e # for consistency, but they really shouldn't touch it def setBigState(self, state): @@ -2189,10 +2241,6 @@ class Status: for t in self.watchers: t.builderRemoved(name) - def prune(self): - for b in self.botmaster.builders.values(): - b.builder_status.prune() - def buildsetSubmitted(self, bss): self.activeBuildSets.append(bss) bss.waitUntilFinished().addCallback(self.activeBuildSets.remove) diff --git a/buildbot/status/web/build.py b/buildbot/status/web/build.py index 5d01358..6adaf1f 100644 --- a/buildbot/status/web/build.py +++ b/buildbot/status/web/build.py @@ -103,26 +103,25 @@ class StatusResourceBuild(HtmlResource): # for name, target in urls.items(): # text.append('[%s]' % # (target, ex_url_class, html.escape(name))) - if b.getLogs(): - data += "
    \n" - for s in b.getSteps(): - name = s.getName() - data += ("
  1. %s [%s]\n" - % (req.childLink("steps/%s" % urllib.quote(name)), - name, - " ".join(s.getText()))) - if s.getLogs(): - data += "
      \n" - for logfile in s.getLogs(): - logname = logfile.getName() - logurl = req.childLink("steps/%s/logs/%s" % - (urllib.quote(name), - urllib.quote(logname))) - data += ("
    1. %s
      2. \n" % - (logurl, logfile.getName())) - data += "
    \n" - data += "
    2. \n" - data += "
\n" + data += "
    \n" + for s in b.getSteps(): + name = s.getName() + data += ("
  1. %s [%s]\n" + % (req.childLink("steps/%s" % urllib.quote(name)), + name, + " ".join(s.getText()))) + if s.getLogs(): + data += "
      \n" + for logfile in s.getLogs(): + logname = logfile.getName() + logurl = req.childLink("steps/%s/logs/%s" % + (urllib.quote(name), + urllib.quote(logname))) + data += ("
    1. %s
      2. \n" % + (logurl, logfile.getName())) + data += "
    \n" + data += "
    2. \n" + data += "
\n" data += "

Build Properties:

\n" data += "\n" diff --git a/buildbot/test/test_status.py b/buildbot/test/test_status.py index b3c162a..7676987 100644 --- a/buildbot/test/test_status.py +++ b/buildbot/test/test_status.py @@ -21,7 +21,7 @@ try: except ImportError: pass from buildbot.status import progress, client # NEEDS COVERAGE -from buildbot.test.runutils import RunMixin, setupBuildStepStatus +from buildbot.test.runutils import RunMixin, setupBuildStepStatus, rmtree class MyStep: build = None @@ -1629,3 +1629,47 @@ class BuildExpectation(unittest.TestCase): build.buildFinished(['sometext'], builder.FAILURE) self.failUnlessEqual(b.expectations, None, 'Zero expectation for a failed build') + +class Pruning(unittest.TestCase): + def runTest(self, files, buildHorizon, logHorizon): + bstat = builder.BuilderStatus("foo") + bstat.buildHorizon = buildHorizon + bstat.logHorizon = logHorizon + bstat.basedir = "prune-test" + + rmtree(bstat.basedir) + os.mkdir(bstat.basedir) + for filename in files: + open(os.path.join(bstat.basedir, filename), "w").write("TEST") + bstat.determineNextBuildNumber() + + bstat.prune() + + remaining = os.listdir(bstat.basedir) + remaining.sort() + return remaining + + files_base = [ + '10', + '11', + '12', '12-log-bar', '12-log-foo', + '13', '13-log-foo', + '14', '14-log-bar', '14-log-foo', + ] + + def test_rmlogs(self): + remaining = self.runTest(self.files_base, 5, 2) + self.failUnlessEqual(remaining, [ + '10', + '11', + '12', + '13', '13-log-foo', + '14', '14-log-bar', '14-log-foo', + ]) + + def test_rmbuilds(self): + remaining = self.runTest(self.files_base, 2, 0) + self.failUnlessEqual(remaining, [ + '13', '13-log-foo', + '14', '14-log-bar', '14-log-foo', + ]) -- 2.11.4.GIT
NameValueSource