contrib/svn_buildbot.py

   1 #!/usr/bin/python
   2
   3 # this requires python >=2.3 for the 'sets' module.
   4
   5 # The sets.py from python-2.3 appears to work fine under python2.2 . To
   6 # install this script on a host with only python2.2, copy
   7 # /usr/lib/python2.3/sets.py from a newer python into somewhere on your
   8 # PYTHONPATH, then edit the #! line above to invoke python2.2
   9
  10 # python2.1 is right out
  11
  12 # If you run this program as part of your SVN post-commit hooks, it will
  13 # deliver Change notices to a buildmaster that is running a PBChangeSource
  14 # instance.
  15
  16 # edit your svn-repository/hooks/post-commit file, and add lines that look
  17 # like this:
  18
  19 '''
  20 # set up PYTHONPATH to contain Twisted/buildbot perhaps, if not already
  21 # installed site-wide
  22 . ~/.environment
  23
  24 /path/to/svn_buildbot.py --repository "$REPOS" --revision "$REV" --bbserver localhost --bbport 9989
  25 '''
  26
  27 import commands, sys, os
  28 import re
  29 import sets
  30
  31 # We have hackish "-d" handling here rather than in the Options
  32 # subclass below because a common error will be to not have twisted in
  33 # PYTHONPATH; we want to be able to print that error to the log if
  34 # debug mode is on, so we set it up before the imports.
  35
  36 DEBUG = None
  37
  38 if '-d' in sys.argv:
  39     i = sys.argv.index('-d')
  40     DEBUG = sys.argv[i+1]
  41     del sys.argv[i]
  42     del sys.argv[i]
  43
  44 if DEBUG:
  45     f = open(DEBUG, 'a')
  46     sys.stderr = f
  47     sys.stdout = f
  48
  49 from twisted.internet import defer, reactor
  50 from twisted.python import usage
  51 from twisted.spread import pb
  52 from twisted.cred import credentials
  53
  54 class Options(usage.Options):
  55     optParameters = [
  56         ['repository', 'r', None,
  57          "The repository that was changed."],
  58         ['revision', 'v', None,
  59          "The revision that we want to examine (default: latest)"],
  60         ['bbserver', 's', 'localhost',
  61          "The hostname of the server that buildbot is running on"],
  62         ['bbport', 'p', 8007,
  63          "The port that buildbot is listening on"],
  64         ['include', 'f', None,
  65          '''\
  66 Search the list of changed files for this regular expression, and if there is
  67 at least one match notify buildbot; otherwise buildbot will not do a build.
  68 You may provide more than one -f argument to try multiple
  69 patterns.  If no filter is given, buildbot will always be notified.'''],
  70         ['filter', 'f', None, "Same as --include.  (Deprecated)"],
  71         ['exclude', 'F', None,
  72          '''\
  73 The inverse of --filter.  Changed files matching this expression will never
  74 be considered for a build.
  75 You may provide more than one -F argument to try multiple
  76 patterns.  Excludes override includes, that is, patterns that match both an
  77 include and an exclude will be excluded.'''],
  78         ]
  79     optFlags = [
  80         ['dryrun', 'n', "Do not actually send changes"],
  81         ]
  82
  83     def __init__(self):
  84         usage.Options.__init__(self)
  85         self._includes = []
  86         self._excludes = []
  87         self['includes'] = None
  88         self['excludes'] = None
  89
  90     def opt_include(self, arg):
  91         self._includes.append('.*%s.*' % (arg,))
  92     opt_filter = opt_include
  93
  94     def opt_exclude(self, arg):
  95         self._excludes.append('.*%s.*' % (arg,))
  96
  97     def postOptions(self):
  98         if self['repository'] is None:
  99             raise usage.error("You must pass --repository")
 100         if self._includes:
 101             self['includes'] = '(%s)' % ('|'.join(self._includes),)
 102         if self._excludes:
 103             self['excludes'] = '(%s)' % ('|'.join(self._excludes),)
 104
 105 def split_file_dummy(changed_file):
 106     """Split the repository-relative filename into a tuple of (branchname,
 107     branch_relative_filename). If you have no branches, this should just
 108     return (None, changed_file).
 109     """
 110     return (None, changed_file)
 111
 112 # this version handles repository layouts that look like:
 113 #  trunk/files..                  -> trunk
 114 #  branches/branch1/files..       -> branches/branch1
 115 #  branches/branch2/files..       -> branches/branch2
 116 #
 117 def split_file_branches(changed_file):
 118     pieces = changed_file.split(os.sep)
 119     if pieces[0] == 'branches':
 120         return (os.path.join(*pieces[:2]),
 121                 os.path.join(*pieces[2:]))
 122     if pieces[0] == 'trunk':
 123         return (pieces[0], os.path.join(*pieces[1:]))
 124     ## there are other sibilings of 'trunk' and 'branches'. Pretend they are
 125     ## all just funny-named branches, and let the Schedulers ignore them.
 126     #return (pieces[0], os.path.join(*pieces[1:]))
 127
 128     raise RuntimeError("cannot determine branch for '%s'" % changed_file)
 129
 130 split_file = split_file_dummy
 131
 132
 133 class ChangeSender:
 134
 135     def getChanges(self, opts):
 136         """Generate and stash a list of Change dictionaries, ready to be sent
 137         to the buildmaster's PBChangeSource."""
 138
 139         # first we extract information about the files that were changed
 140         repo = opts['repository']
 141         print "Repo:", repo
 142         rev_arg = ''
 143         if opts['revision']:
 144             rev_arg = '-r %s' % (opts['revision'],)
 145         changed = commands.getoutput('svnlook changed %s "%s"' % (rev_arg,
 146                                                                   repo)
 147                                      ).split('\n')
 148         # the first 4 columns can contain status information
 149         changed = [x[4:] for x in changed]
 150
 151         message = commands.getoutput('svnlook log %s "%s"' % (rev_arg, repo))
 152         who = commands.getoutput('svnlook author %s "%s"' % (rev_arg, repo))
 153         revision = opts.get('revision')
 154         if revision is not None:
 155             revision = int(revision)
 156
 157         # see if we even need to notify buildbot by looking at filters first
 158         changestring = '\n'.join(changed)
 159         fltpat = opts['includes']
 160         if fltpat:
 161             included = sets.Set(re.findall(fltpat, changestring))
 162         else:
 163             included = sets.Set(changed)
 164
 165         expat = opts['excludes']
 166         if expat:
 167             excluded = sets.Set(re.findall(expat, changestring))
 168         else:
 169             excluded = sets.Set([])
 170         if len(included.difference(excluded)) == 0:
 171             print changestring
 172             print """\
 173     Buildbot was not interested, no changes matched any of these filters:\n %s
 174     or all the changes matched these exclusions:\n %s\
 175     """ % (fltpat, expat)
 176             sys.exit(0)
 177
 178         # now see which branches are involved
 179         files_per_branch = {}
 180         for f in changed:
 181             branch, filename = split_file(f)
 182             if files_per_branch.has_key(branch):
 183                 files_per_branch[branch].append(filename)
 184             else:
 185                 files_per_branch[branch] = [filename]
 186
 187         # now create the Change dictionaries
 188         changes = []
 189         for branch in files_per_branch.keys():
 190             d = {'who': who,
 191                  'branch': branch,
 192                  'files': files_per_branch[branch],
 193                  'comments': message,
 194                  'revision': revision}
 195             changes.append(d)
 196
 197         return changes
 198
 199     def sendChanges(self, opts, changes):
 200         pbcf = pb.PBClientFactory()
 201         reactor.connectTCP(opts['bbserver'], int(opts['bbport']), pbcf)
 202         d = pbcf.login(credentials.UsernamePassword('change', 'changepw'))
 203         d.addCallback(self.sendAllChanges, changes)
 204         return d
 205
 206     def sendAllChanges(self, remote, changes):
 207         dl = [remote.callRemote('addChange', change)
 208               for change in changes]
 209         return defer.DeferredList(dl)
 210
 211     def run(self):
 212         opts = Options()
 213         try:
 214             opts.parseOptions()
 215         except usage.error, ue:
 216             print opts
 217             print "%s: %s" % (sys.argv[0], ue)
 218             sys.exit()
 219
 220         changes = self.getChanges(opts)
 221         if opts['dryrun']:
 222             for i,c in enumerate(changes):
 223                 print "CHANGE #%d" % (i+1)
 224                 keys = c.keys()
 225                 keys.sort()
 226                 for k in keys:
 227                     print "[%10s]: %s" % (k, c[k])
 228             print "*NOT* sending any changes"
 229             return
 230
 231         d = self.sendChanges(opts, changes)
 232
 233         def quit(*why):
 234             print "quitting! because", why
 235             reactor.stop()
 236
 237         def failed(f):
 238             print "FAILURE"
 239             print f
 240             reactor.stop()
 241
 242         d.addCallback(quit, "SUCCESS")
 243         d.addErrback(failed)
 244         reactor.callLater(60, quit, "TIMEOUT")
 245         reactor.run()
 246
 247 if __name__ == '__main__':
 248     s = ChangeSender()
 249     s.run()
 250
 251