contrib/svn_buildbot.py

   1 #!/usr/bin/python
   2
   3 # this requires python >=2.3 for the 'sets' module.
   4
   5 # The sets.py from python-2.3 appears to work fine under python2.2 . To
   6 # install this script on a host with only python2.2, copy
   7 # /usr/lib/python2.3/sets.py from a newer python into somewhere on your
   8 # PYTHONPATH, then edit the #! line above to invoke python2.2
   9
  10 # python2.1 is right out
  11
  12 # If you run this program as part of your SVN post-commit hooks, it will
  13 # deliver Change notices to a buildmaster that is running a PBChangeSource
  14 # instance.
  15
  16 # edit your svn-repository/hooks/post-commit file, and add lines that look
  17 # like this:
  18
  19 '''
  20 # set up PYTHONPATH to contain Twisted/buildbot perhaps, if not already
  21 # installed site-wide
  22 . ~/.environment
  23
  24 /path/to/svn_buildbot.py --repository "$REPOS" --revision "$REV" \
  25 --bbserver localhost --bbport 9989
  26 '''
  27
  28 import commands
  29 import sys
  30 import os
  31 import re
  32 import sets
  33
  34 # We have hackish "-d" handling here rather than in the Options
  35 # subclass below because a common error will be to not have twisted in
  36 # PYTHONPATH; we want to be able to print that error to the log if
  37 # debug mode is on, so we set it up before the imports.
  38
  39 DEBUG = None
  40
  41 if '-d' in sys.argv:
  42     i = sys.argv.index('-d')
  43     DEBUG = sys.argv[i+1]
  44     del sys.argv[i]
  45     del sys.argv[i]
  46
  47 if DEBUG:
  48     f = open(DEBUG, 'a')
  49     sys.stderr = f
  50     sys.stdout = f
  51
  52
  53 from twisted.internet import defer, reactor
  54 from twisted.python import usage
  55 from twisted.spread import pb
  56 from twisted.cred import credentials
  57
  58
  59 class Options(usage.Options):
  60     optParameters = [
  61         ['repository', 'r', None,
  62          "The repository that was changed."],
  63         ['revision', 'v', None,
  64          "The revision that we want to examine (default: latest)"],
  65         ['bbserver', 's', 'localhost',
  66          "The hostname of the server that buildbot is running on"],
  67         ['bbport', 'p', 8007,
  68          "The port that buildbot is listening on"],
  69         ['include', 'f', None,
  70          '''\
  71 Search the list of changed files for this regular expression, and if there is
  72 at least one match notify buildbot; otherwise buildbot will not do a build.
  73 You may provide more than one -f argument to try multiple
  74 patterns.  If no filter is given, buildbot will always be notified.'''],
  75         ['filter', 'f', None, "Same as --include.  (Deprecated)"],
  76         ['exclude', 'F', None,
  77          '''\
  78 The inverse of --filter.  Changed files matching this expression will never
  79 be considered for a build.
  80 You may provide more than one -F argument to try multiple
  81 patterns.  Excludes override includes, that is, patterns that match both an
  82 include and an exclude will be excluded.'''],
  83         ]
  84     optFlags = [
  85         ['dryrun', 'n', "Do not actually send changes"],
  86         ]
  87
  88     def __init__(self):
  89         usage.Options.__init__(self)
  90         self._includes = []
  91         self._excludes = []
  92         self['includes'] = None
  93         self['excludes'] = None
  94
  95     def opt_include(self, arg):
  96         self._includes.append('.*%s.*' % (arg, ))
  97
  98     opt_filter = opt_include
  99
 100     def opt_exclude(self, arg):
 101         self._excludes.append('.*%s.*' % (arg, ))
 102
 103     def postOptions(self):
 104         if self['repository'] is None:
 105             raise usage.error("You must pass --repository")
 106         if self._includes:
 107             self['includes'] = '(%s)' % ('|'.join(self._includes), )
 108         if self._excludes:
 109             self['excludes'] = '(%s)' % ('|'.join(self._excludes), )
 110
 111
 112 def split_file_dummy(changed_file):
 113     """Split the repository-relative filename into a tuple of (branchname,
 114     branch_relative_filename). If you have no branches, this should just
 115     return (None, changed_file).
 116     """
 117     return (None, changed_file)
 118
 119
 120 # this version handles repository layouts that look like:
 121 #  trunk/files..                  -> trunk
 122 #  branches/branch1/files..       -> branches/branch1
 123 #  branches/branch2/files..       -> branches/branch2
 124 #
 125
 126
 127 def split_file_branches(changed_file):
 128     pieces = changed_file.split(os.sep)
 129     if pieces[0] == 'branches':
 130         return (os.path.join(*pieces[:2]),
 131                 os.path.join(*pieces[2:]))
 132     if pieces[0] == 'trunk':
 133         return (pieces[0], os.path.join(*pieces[1:]))
 134     ## there are other sibilings of 'trunk' and 'branches'. Pretend they are
 135     ## all just funny-named branches, and let the Schedulers ignore them.
 136     #return (pieces[0], os.path.join(*pieces[1:]))
 137
 138     raise RuntimeError("cannot determine branch for '%s'" % changed_file)
 139
 140
 141 split_file = split_file_dummy
 142
 143
 144 class ChangeSender:
 145
 146     def getChanges(self, opts):
 147         """Generate and stash a list of Change dictionaries, ready to be sent
 148         to the buildmaster's PBChangeSource."""
 149
 150         # first we extract information about the files that were changed
 151         repo = opts['repository']
 152         print "Repo:", repo
 153         rev_arg = ''
 154         if opts['revision']:
 155             rev_arg = '-r %s' % (opts['revision'], )
 156         changed = commands.getoutput('svnlook changed %s "%s"' % (
 157             rev_arg, repo)).split('\n')
 158         # the first 4 columns can contain status information
 159         changed = [x[4:] for x in changed]
 160
 161         message = commands.getoutput('svnlook log %s "%s"' % (rev_arg, repo))
 162         who = commands.getoutput('svnlook author %s "%s"' % (rev_arg, repo))
 163         revision = opts.get('revision')
 164         if revision is not None:
 165             revision = int(revision)
 166
 167         # see if we even need to notify buildbot by looking at filters first
 168         changestring = '\n'.join(changed)
 169         fltpat = opts['includes']
 170         if fltpat:
 171             included = sets.Set(re.findall(fltpat, changestring))
 172         else:
 173             included = sets.Set(changed)
 174
 175         expat = opts['excludes']
 176         if expat:
 177             excluded = sets.Set(re.findall(expat, changestring))
 178         else:
 179             excluded = sets.Set([])
 180         if len(included.difference(excluded)) == 0:
 181             print changestring
 182             print """\
 183     Buildbot was not interested, no changes matched any of these filters:\n %s
 184     or all the changes matched these exclusions:\n %s\
 185     """ % (fltpat, expat)
 186             sys.exit(0)
 187
 188         # now see which branches are involved
 189         files_per_branch = {}
 190         for f in changed:
 191             branch, filename = split_file(f)
 192             if branch in files_per_branch.keys():
 193                 files_per_branch[branch].append(filename)
 194             else:
 195                 files_per_branch[branch] = [filename]
 196
 197         # now create the Change dictionaries
 198         changes = []
 199         for branch in files_per_branch.keys():
 200             d = {'who': who,
 201                  'branch': branch,
 202                  'files': files_per_branch[branch],
 203                  'comments': message,
 204                  'revision': revision}
 205             changes.append(d)
 206
 207         return changes
 208
 209     def sendChanges(self, opts, changes):
 210         pbcf = pb.PBClientFactory()
 211         reactor.connectTCP(opts['bbserver'], int(opts['bbport']), pbcf)
 212         d = pbcf.login(credentials.UsernamePassword('change', 'changepw'))
 213         d.addCallback(self.sendAllChanges, changes)
 214         return d
 215
 216     def sendAllChanges(self, remote, changes):
 217         dl = [remote.callRemote('addChange', change)
 218               for change in changes]
 219         return defer.DeferredList(dl)
 220
 221     def run(self):
 222         opts = Options()
 223         try:
 224             opts.parseOptions()
 225         except usage.error, ue:
 226             print opts
 227             print "%s: %s" % (sys.argv[0], ue)
 228             sys.exit()
 229
 230         changes = self.getChanges(opts)
 231         if opts['dryrun']:
 232             for i, c in enumerate(changes):
 233                 print "CHANGE #%d" % (i+1)
 234                 keys = c.keys()
 235                 keys.sort()
 236                 for k in keys:
 237                     print "[%10s]: %s" % (k, c[k])
 238             print "*NOT* sending any changes"
 239             return
 240
 241         d = self.sendChanges(opts, changes)
 242
 243         def quit(*why):
 244             print "quitting! because", why
 245             reactor.stop()
 246
 247         def failed(f):
 248             print "FAILURE"
 249             print f
 250             reactor.stop()
 251
 252         d.addCallback(quit, "SUCCESS")
 253         d.addErrback(failed)
 254         reactor.callLater(60, quit, "TIMEOUT")
 255         reactor.run()
 256
 257
 258 if __name__ == '__main__':
 259     s = ChangeSender()
 260     s.run()