xapian-maintainer-tools/buildbot/tarsource.py

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 # Based on SVNPoller
   5
   6 from twisted.python import log
   7 from twisted.internet import defer, reactor, utils
   8 from twisted.internet.task import LoopingCall
   9
  10 from buildbot import util
  11 from buildbot.changes import base
  12 from buildbot.changes.changes import Change
  13 from buildbot.process.buildstep import RemoteShellCommand
  14 from buildbot.steps.source import Source
  15
  16 import re
  17
  18 tarlink_re = re.compile('<a href="([a-zA-Z0-9_\.-]+).tar.xz">')
  19
  20 def parsehtml(html, archives):
  21     max_revision = 0
  22     links = []
  23     for link in tarlink_re.findall(html):
  24         for archive in archives:
  25             if link.startswith(archive):
  26                 revision = link[len(archive):]
  27                 svnpos = revision.find('svn')
  28                 if svnpos > 0:
  29                     revision = int(revision[svnpos + 3:])
  30                     if revision > max_revision:
  31                         links = []
  32                         max_revision = revision
  33                     if revision == max_revision:
  34                         links.append(link)
  35     if max_revision == 0:
  36         log.msg("No valid links found")
  37         return None
  38     elif len(links) == len(archives):
  39         log.msg("Parsing html index page: found all archives for revision: %d" % (max_revision, ))
  40         return (max_revision, links)
  41     else:
  42         log.msg("Latest revision (%d) is not complete: found the following links: %r" % (max_revision, links, ))
  43         return None
  44
  45 class TarPoller(base.ChangeSource, util.ComparableMixin):
  46
  47     compare_attrs = ["tarball_root", "branch", "archives", "pollinterval",]
  48     last_revision = None
  49     loop = None
  50     parent = None
  51     working = False
  52
  53     def __init__(self, tarball_root, branch=None, archives=None, pollinterval=60):
  54         self.tarball_root = tarball_root
  55         self.branch = branch
  56         self.archives = archives
  57         self.pollinterval = pollinterval
  58         self.loop = LoopingCall(self.checktar)
  59
  60     def startService(self):
  61         log.msg("TarPoller(%s) starting" % self.tarball_root)
  62         base.ChangeSource.startService(self)
  63         # Don't start the loop just yet because the reactor isn't running.
  64         # Give it a chance to go and install our SIGCHLD handler before
  65         # spawning processes.
  66         reactor.callLater(0, self.loop.start, self.pollinterval)
  67
  68     def stopService(self):
  69         log.msg("TarPoller(%s) shutting down" % self.tarball_root)
  70         self.loop.stop()
  71         return base.ChangeSource.stopService(self)
  72
  73     def describe(self):
  74         return "TarPoller watching %s" % self.tarball_root
  75
  76     def checktar(self):
  77         if self.working:
  78             log.msg("TarPoller(%s) overrun: timer fired but the previous "
  79                     "poll had not yet finished.")
  80             self.overrun_counter += 1
  81             return defer.succeed(None)
  82         self.working = True
  83
  84         log.msg("TarPoller polling")
  85         d = utils.getProcessOutput("curl", ['-s', self.tarball_root], {})
  86         d.addCallback(parsehtml, self.archives)
  87         d.addCallback(self.create_changes)
  88         d.addCallback(self.submit_changes)
  89
  90         d.addCallbacks(self.finished_ok, self.finished_failure)
  91         return d
  92
  93     def create_changes(self, info):
  94         if info is None:
  95             # Got no consistent revision
  96             log.msg('TarPoller: got no consistent revision')
  97             return None
  98         (revision, links) = info
  99         if self.last_revision is None:
 100             log.msg('TarPoller: start revision is %r' % (revision,))
 101             self.last_revision = revision
 102             return None
 103         if self.last_revision == revision:
 104             # No change from last revision
 105             log.msg('TarPoller: still at previous revision: %r' % (revision,))
 106             return None
 107         self.last_revision = revision
 108
 109         log.msg('TarPoller: new revision found: %r' % (revision,))
 110         c = Change(who='',
 111                    files=[],
 112                    comments='',
 113                    revision=revision,
 114                    branch=self.branch)
 115         c.links = links
 116         return c
 117
 118     def submit_changes(self, change):
 119         if change is not None:
 120             self.master.addChange(change)
 121
 122     def finished_ok(self, res):
 123         log.msg("TarPoller finished polling")
 124         log.msg('_finished : %s' % res)
 125         assert self.working
 126         self.working = False
 127         return res
 128
 129     def finished_failure(self, f):
 130         log.msg("TarPoller failed")
 131         log.msg('_finished : %s' % f)
 132         assert self.working
 133         self.working = False
 134         return None # eat the failure
 135
 136
 137 class Tar(Source):
 138     name = "tar"
 139     def __init__(self, rooturl, archives, **kwargs):
 140         self.branch = None
 141         Source.__init__(self, **kwargs)
 142         self.rooturl = rooturl
 143         self.archives = archives
 144         self.working = False
 145
 146     def computeSourceRevision(self, changes):
 147         if not changes: return None
 148         changelist = [c.revision for c in changes]
 149         if len(changelist) == 0: return None
 150         changelist.sort()
 151         return changelist[-1]
 152
 153     def startVC(self, branch, revision, patch):
 154         d = utils.getProcessOutput("curl", ['-s', self.rooturl], {})
 155         d.addCallback(self.doStartVC, branch, revision, patch)
 156         d.addErrback(self.failed)
 157
 158     def doStartVC(self, html, branch, revision, path):
 159         info = parsehtml(html, self.archives)
 160
 161         if info is None:
 162             # Got no consistent revision
 163             log.msg('Tar: got no consistent revision')
 164             self.failed('Couldn\'t get consistent revision')
 165             return
 166         (revision, links) = info
 167
 168         cmdargs = ['curl', '-s']
 169         for link in links:
 170             cmdargs.append('-O')
 171             cmdargs.append(self.rooturl + link + '.tar.xz')
 172
 173         cmd = RemoteShellCommand('build', command=(cmdargs))
 174         self.startCommand(cmd)
 175