master/contrib/svn_buildbot.py

   1 #!/usr/bin/python
   2
   3 # this requires python >=2.3 for the 'sets' module.
   4
   5 # The sets.py from python-2.3 appears to work fine under python2.2 . To
   6 # install this script on a host with only python2.2, copy
   7 # /usr/lib/python2.3/sets.py from a newer python into somewhere on your
   8 # PYTHONPATH, then edit the #! line above to invoke python2.2
   9
  10 # python2.1 is right out
  11
  12 # If you run this program as part of your SVN post-commit hooks, it will
  13 # deliver Change notices to a buildmaster that is running a PBChangeSource
  14 # instance.
  15
  16 # edit your svn-repository/hooks/post-commit file, and add lines that look
  17 # like this:
  18
  19 '''
  20 # set up PYTHONPATH to contain Twisted/buildbot perhaps, if not already
  21 # installed site-wide
  22 . ~/.environment
  23
  24 /path/to/svn_buildbot.py --repository "$REPOS" --revision "$REV" \
  25 --bbserver localhost --bbport 9989
  26 '''
  27
  28 import commands
  29 import sys
  30 import os
  31 import re
  32 import sets
  33
  34 # We have hackish "-d" handling here rather than in the Options
  35 # subclass below because a common error will be to not have twisted in
  36 # PYTHONPATH; we want to be able to print that error to the log if
  37 # debug mode is on, so we set it up before the imports.
  38
  39 DEBUG = None
  40
  41 if '-d' in sys.argv:
  42     i = sys.argv.index('-d')
  43     DEBUG = sys.argv[i+1]
  44     del sys.argv[i]
  45     del sys.argv[i]
  46
  47 if DEBUG:
  48     f = open(DEBUG, 'a')
  49     sys.stderr = f
  50     sys.stdout = f
  51
  52
  53 from twisted.internet import defer, reactor
  54 from twisted.python import usage
  55 from twisted.spread import pb
  56 from twisted.cred import credentials
  57
  58
  59 class Options(usage.Options):
  60     optParameters = [
  61         ['repository', 'r', None,
  62          "The repository that was changed."],
  63         ['revision', 'v', None,
  64          "The revision that we want to examine (default: latest)"],
  65         ['bbserver', 's', 'localhost',
  66          "The hostname of the server that buildbot is running on"],
  67         ['bbport', 'p', 8007,
  68          "The port that buildbot is listening on"],
  69         ['include', 'f', None,
  70          '''\
  71 Search the list of changed files for this regular expression, and if there is
  72 at least one match notify buildbot; otherwise buildbot will not do a build.
  73 You may provide more than one -f argument to try multiple
  74 patterns.  If no filter is given, buildbot will always be notified.'''],
  75         ['filter', 'f', None, "Same as --include.  (Deprecated)"],
  76         ['exclude', 'F', None,
  77          '''\
  78 The inverse of --filter.  Changed files matching this expression will never
  79 be considered for a build.
  80 You may provide more than one -F argument to try multiple
  81 patterns.  Excludes override includes, that is, patterns that match both an
  82 include and an exclude will be excluded.'''],
  83         ['encoding', 'e', "utf8",
  84          "The encoding of the strings from subversion (default: utf8)" ],
  85         ]
  86     optFlags = [
  87         ['dryrun', 'n', "Do not actually send changes"],
  88         ]
  89
  90     def __init__(self):
  91         usage.Options.__init__(self)
  92         self._includes = []
  93         self._excludes = []
  94         self['includes'] = None
  95         self['excludes'] = None
  96
  97     def opt_include(self, arg):
  98         self._includes.append('.*%s.*' % (arg, ))
  99
 100     opt_filter = opt_include
 101
 102     def opt_exclude(self, arg):
 103         self._excludes.append('.*%s.*' % (arg, ))
 104
 105     def postOptions(self):
 106         if self['repository'] is None:
 107             raise usage.error("You must pass --repository")
 108         if self._includes:
 109             self['includes'] = '(%s)' % ('|'.join(self._includes), )
 110         if self._excludes:
 111             self['excludes'] = '(%s)' % ('|'.join(self._excludes), )
 112
 113
 114 def split_file_dummy(changed_file):
 115     """Split the repository-relative filename into a tuple of (branchname,
 116     branch_relative_filename). If you have no branches, this should just
 117     return (None, changed_file).
 118     """
 119     return (None, changed_file)
 120
 121
 122 # this version handles repository layouts that look like:
 123 #  trunk/files..                  -> trunk
 124 #  branches/branch1/files..       -> branches/branch1
 125 #  branches/branch2/files..       -> branches/branch2
 126 #
 127
 128
 129 def split_file_branches(changed_file):
 130     pieces = changed_file.split(os.sep)
 131     if pieces[0] == 'branches':
 132         return (os.path.join(*pieces[:2]),
 133                 os.path.join(*pieces[2:]))
 134     if pieces[0] == 'trunk':
 135         return (pieces[0], os.path.join(*pieces[1:]))
 136     ## there are other sibilings of 'trunk' and 'branches'. Pretend they are
 137     ## all just funny-named branches, and let the Schedulers ignore them.
 138     #return (pieces[0], os.path.join(*pieces[1:]))
 139
 140     raise RuntimeError("cannot determine branch for '%s'" % changed_file)
 141
 142
 143 split_file = split_file_dummy
 144
 145
 146 class ChangeSender:
 147
 148     def getChanges(self, opts):
 149         """Generate and stash a list of Change dictionaries, ready to be sent
 150         to the buildmaster's PBChangeSource."""
 151
 152         # first we extract information about the files that were changed
 153         repo = opts['repository']
 154         print "Repo:", repo
 155         rev_arg = ''
 156         if opts['revision']:
 157             rev_arg = '-r %s' % (opts['revision'], )
 158         changed = commands.getoutput('svnlook changed %s "%s"' % (
 159             rev_arg, repo)).split('\n')
 160         # the first 4 columns can contain status information
 161         changed = [x[4:] for x in changed]
 162
 163         message = commands.getoutput('svnlook log %s "%s"' % (rev_arg, repo))
 164         who = commands.getoutput('svnlook author %s "%s"' % (rev_arg, repo))
 165         revision = opts.get('revision')
 166         if revision is not None:
 167             revision = str(int(revision))
 168
 169         # see if we even need to notify buildbot by looking at filters first
 170         changestring = '\n'.join(changed)
 171         fltpat = opts['includes']
 172         if fltpat:
 173             included = sets.Set(re.findall(fltpat, changestring))
 174         else:
 175             included = sets.Set(changed)
 176
 177         expat = opts['excludes']
 178         if expat:
 179             excluded = sets.Set(re.findall(expat, changestring))
 180         else:
 181             excluded = sets.Set([])
 182         if len(included.difference(excluded)) == 0:
 183             print changestring
 184             print """\
 185     Buildbot was not interested, no changes matched any of these filters:\n %s
 186     or all the changes matched these exclusions:\n %s\
 187     """ % (fltpat, expat)
 188             sys.exit(0)
 189
 190         # now see which branches are involved
 191         files_per_branch = {}
 192         for f in changed:
 193             branch, filename = split_file(f)
 194             if branch in files_per_branch.keys():
 195                 files_per_branch[branch].append(filename)
 196             else:
 197                 files_per_branch[branch] = [filename]
 198
 199         # now create the Change dictionaries
 200         changes = []
 201         encoding = opts['encoding']
 202         for branch in files_per_branch.keys():
 203             d = {'who': unicode(who, encoding=encoding),
 204                  'repository': unicode(repo, encoding=encoding),
 205                  'comments': unicode(message, encoding=encoding),
 206                  'revision': revision
 207                  }
 208             if branch:
 209                 d['branch'] = unicode(branch, encoding=encoding)
 210             else:
 211                 d['branch'] = branch
 212
 213             files = []
 214             for file in files_per_branch[branch]:
 215                 files.append(unicode(file, encoding=encoding))
 216             d['files'] = files
 217
 218             changes.append(d)
 219
 220         return changes
 221
 222     def sendChanges(self, opts, changes):
 223         pbcf = pb.PBClientFactory()
 224         reactor.connectTCP(opts['bbserver'], int(opts['bbport']), pbcf)
 225         d = pbcf.login(credentials.UsernamePassword('change', 'changepw'))
 226         d.addCallback(self.sendAllChanges, changes)
 227         return d
 228
 229     def sendAllChanges(self, remote, changes):
 230         dl = [remote.callRemote('addChange', change)
 231               for change in changes]
 232         return defer.DeferredList(dl)
 233
 234     def run(self):
 235         opts = Options()
 236         try:
 237             opts.parseOptions()
 238         except usage.error, ue:
 239             print opts
 240             print "%s: %s" % (sys.argv[0], ue)
 241             sys.exit()
 242
 243         changes = self.getChanges(opts)
 244         if opts['dryrun']:
 245             for i, c in enumerate(changes):
 246                 print "CHANGE #%d" % (i+1)
 247                 keys = c.keys()
 248                 keys.sort()
 249                 for k in keys:
 250                     print "[%10s]: %s" % (k, c[k])
 251             print "*NOT* sending any changes"
 252             return
 253
 254         d = self.sendChanges(opts, changes)
 255
 256         def quit(*why):
 257             print "quitting! because", why
 258             reactor.stop()
 259
 260         def failed(f):
 261             print "FAILURE"
 262             print f
 263             reactor.stop()
 264
 265         d.addCallback(quit, "SUCCESS")
 266         d.addErrback(failed)
 267         reactor.callLater(60, quit, "TIMEOUT")
 268         reactor.run()
 269
 270
 271 if __name__ == '__main__':
 272     s = ChangeSender()
 273     s.run()