contrib/svn_buildbot.py

   1 #!/usr/bin/env python2.3
   2
   3 # this requires python >=2.3 for the 'sets' module.
   4
   5 # The sets.py from python-2.3 appears to work fine under python2.2 . To
   6 # install this script on a host with only python2.2, copy
   7 # /usr/lib/python2.3/sets.py from a newer python into somewhere on your
   8 # PYTHONPATH, then edit the #! line above to invoke python2.2
   9
  10 # python2.1 is right out
  11
  12 # If you run this program as part of your SVN post-commit hooks, it will
  13 # deliver Change notices to a buildmaster that is running a PBChangeSource
  14 # instance.
  15
  16 # edit your svn-repository/hooks/post-commit file, and add lines that look
  17 # like this:
  18
  19 '''
  20 # set up PYTHONPATH to contain Twisted/buildbot perhaps, if not already
  21 # installed site-wide
  22 . ~/.environment
  23
  24 /path/to/svn_buildbot.py --repository "$REPOS" --revision "$REV" --bbserver localhost --bbport 9989
  25 '''
  26
  27 import commands, sys, os
  28 import re
  29 import sets
  30
  31 # We have hackish "-d" handling here rather than in the Options
  32 # subclass below because a common error will be to not have twisted in
  33 # PYTHONPATH; we want to be able to print that error to the log if
  34 # debug mode is on, so we set it up before the imports.
  35
  36 DEBUG = None
  37
  38 if '-d' in sys.argv:
  39     i = sys.argv.index('-d')
  40     DEBUG = sys.argv[i+1]
  41     del sys.argv[i]
  42     del sys.argv[i]
  43
  44 if DEBUG:
  45     f = open(DEBUG, 'a')
  46     sys.stderr = f
  47     sys.stdout = f
  48
  49 from twisted.internet import defer, reactor
  50 from twisted.python import usage
  51 from twisted.spread import pb
  52 from twisted.cred import credentials
  53
  54 class Options(usage.Options):
  55     optParameters = [
  56         ['repository', 'r', None,
  57          "The repository that was changed."],
  58         ['revision', 'v', None,
  59          "The revision that we want to examine (default: latest)"],
  60         ['bbserver', 's', 'localhost',
  61          "The hostname of the server that buildbot is running on"],
  62         ['bbport', 'p', 8007,
  63          "The port that buildbot is listening on"],
  64         ['include', 'f', None,
  65          '''\
  66 Search the list of changed files for this regular expression, and if there is
  67 at least one match notify buildbot; otherwise buildbot will not do a build.
  68 You may provide more than one -f argument to try multiple
  69 patterns.  If no filter is given, buildbot will always be notified.'''],
  70         ['filter', 'f', None, "Same as --include.  (Deprecated)"],
  71         ['exclude', 'F', None,
  72          '''\
  73 The inverse of --filter.  Changed files matching this expression will never
  74 be considered for a build.
  75 You may provide more than one -F argument to try multiple
  76 patterns.  Excludes override includes, that is, patterns that match both an
  77 include and an exclude will be excluded.'''],
  78         ]
  79     optFlags = [
  80         ['dryrun', 'n', "Do not actually send changes"],
  81         ]
  82
  83     def __init__(self):
  84         usage.Options.__init__(self)
  85         self._includes = []
  86         self._excludes = []
  87         self['includes'] = None
  88         self['excludes'] = None
  89
  90     def opt_include(self, arg):
  91         self._includes.append('.*%s.*' % (arg,))
  92     opt_filter = opt_include
  93
  94     def opt_exclude(self, arg):
  95         self._excludes.append('.*%s.*' % (arg,))
  96
  97     def postOptions(self):
  98         if self['repository'] is None:
  99             raise usage.error("You must pass --repository")
 100         if self._includes:
 101             self['includes'] = '(%s)' % ('|'.join(self._includes),)
 102         if self._excludes:
 103             self['excludes'] = '(%s)' % ('|'.join(self._excludes),)
 104
 105 def split_file_dummy(changed_file):
 106     """Split the repository-relative filename into a tuple of (branchname,
 107     branch_relative_filename). If you have no branches, this should just
 108     return (None, changed_file).
 109     """
 110     return (None, changed_file)
 111
 112 # this version handles repository layouts that look like:
 113 #  trunk/files..                  -> trunk
 114 #  branches/branch1/files..       -> branches/branch1
 115 #  branches/branch2/files..       -> branches/branch2
 116 #
 117 def split_file_branches(changed_file):
 118     pieces = changed_file.split(os.sep)
 119     if pieces[0] == 'branches':
 120         return (os.path.join(*pieces[:2]),
 121                 os.path.join(*pieces[2:]))
 122     if pieces[0] == 'trunk':
 123         return (pieces[0], os.path.join(*pieces[1:]))
 124     ## there are other sibilings of 'trunk' and 'branches'. Pretend they are
 125     ## all just funny-named branches, and let the Schedulers ignore them.
 126     #return (pieces[0], os.path.join(*pieces[1:]))
 127
 128     raise RuntimeError("cannot determine branch for '%s'" % changed_file)
 129
 130 split_file = split_file_dummy
 131
 132
 133 class ChangeSender:
 134
 135     def getChanges(self, opts):
 136         """Generate and stash a list of Change dictionaries, ready to be sent
 137         to the buildmaster's PBChangeSource."""
 138
 139         # first we extract information about the files that were changed
 140         repo = opts['repository']
 141         print "Repo:", repo
 142         rev_arg = ''
 143         if opts['revision']:
 144             rev_arg = '-r %s' % (opts['revision'],)
 145         changed = commands.getoutput('svnlook changed %s "%s"' % (rev_arg,
 146                                                                   repo)
 147                                      ).split('\n')
 148         changed = [x[1:].strip() for x in changed]
 149
 150         message = commands.getoutput('svnlook log %s "%s"' % (rev_arg, repo))
 151         who = commands.getoutput('svnlook author %s "%s"' % (rev_arg, repo))
 152         revision = opts.get('revision')
 153         if revision is not None:
 154             revision = int(revision)
 155
 156         # see if we even need to notify buildbot by looking at filters first
 157         changestring = '\n'.join(changed)
 158         fltpat = opts['includes']
 159         if fltpat:
 160             included = sets.Set(re.findall(fltpat, changestring))
 161         else:
 162             included = sets.Set(changed)
 163
 164         expat = opts['excludes']
 165         if expat:
 166             excluded = sets.Set(re.findall(expat, changestring))
 167         else:
 168             excluded = sets.Set([])
 169         if len(included.difference(excluded)) == 0:
 170             print changestring
 171             print """\
 172     Buildbot was not interested, no changes matched any of these filters:\n %s
 173     or all the changes matched these exclusions:\n %s\
 174     """ % (fltpat, expat)
 175             sys.exit(0)
 176
 177         # now see which branches are involved
 178         files_per_branch = {}
 179         for f in changed:
 180             branch, filename = split_file(f)
 181             if files_per_branch.has_key(branch):
 182                 files_per_branch[branch].append(filename)
 183             else:
 184                 files_per_branch[branch] = [filename]
 185
 186         # now create the Change dictionaries
 187         changes = []
 188         for branch in files_per_branch.keys():
 189             d = {'who': who,
 190                  'branch': branch,
 191                  'files': files_per_branch[branch],
 192                  'comments': message,
 193                  'revision': revision}
 194             changes.append(d)
 195
 196         return changes
 197
 198     def sendChanges(self, opts, changes):
 199         pbcf = pb.PBClientFactory()
 200         reactor.connectTCP(opts['bbserver'], int(opts['bbport']), pbcf)
 201         d = pbcf.login(credentials.UsernamePassword('change', 'changepw'))
 202         d.addCallback(self.sendAllChanges, changes)
 203         return d
 204
 205     def sendAllChanges(self, remote, changes):
 206         dl = [remote.callRemote('addChange', change)
 207               for change in changes]
 208         return defer.DeferredList(dl)
 209
 210     def run(self):
 211         opts = Options()
 212         try:
 213             opts.parseOptions()
 214         except usage.error, ue:
 215             print opts
 216             print "%s: %s" % (sys.argv[0], ue)
 217             sys.exit()
 218
 219         changes = self.getChanges(opts)
 220         if opts['dryrun']:
 221             for i,c in enumerate(changes):
 222                 print "CHANGE #%d" % (i+1)
 223                 keys = c.keys()
 224                 keys.sort()
 225                 for k in keys:
 226                     print "[%10s]: %s" % (k, c[k])
 227             print "*NOT* sending any changes"
 228             return
 229
 230         d = self.sendChanges(opts, changes)
 231
 232         def quit(*why):
 233             print "quitting! because", why
 234             reactor.stop()
 235
 236         def failed(f):
 237             print "FAILURE"
 238             print f
 239             reactor.stop()
 240
 241         d.addCallback(quit, "SUCCESS")
 242         d.addErrback(failed)
 243         reactor.callLater(60, quit, "TIMEOUT")
 244         reactor.run()
 245
 246 if __name__ == '__main__':
 247     s = ChangeSender()
 248     s.run()
 249
 250