contrib/git_buildbot.py

   1 #! /usr/bin/env python
   2
   3 # This script expects one line for each new revision on the form
   4 #   <oldrev> <newrev> <refname>
   5 #
   6 # For example:
   7 #   aa453216d1b3e49e7f6f98441fa56946ddcd6a20
   8 #   68f7abf4e6f922807889f52bc043ecd31b79f814 refs/heads/master
   9 #
  10 # Each of these changes will be passed to the buildbot server along
  11 # with any other change information we manage to extract from the
  12 # repository.
  13 #
  14 # This script is meant to be run from hooks/post-receive in the git
  15 # repository. It can also be run at client side with hooks/post-merge
  16 # after using this wrapper:
  17
  18 #!/bin/sh
  19 # PRE=$(git rev-parse 'HEAD@{1}')
  20 # POST=$(git rev-parse HEAD)
  21 # SYMNAME=$(git rev-parse --symbolic-full-name HEAD)
  22 # echo "$PRE $POST $SYMNAME" | git_buildbot.py
  23 #
  24 # Largely based on contrib/hooks/post-receive-email from git.
  25
  26 import commands
  27 import logging
  28 import os
  29 import re
  30 import sys
  31
  32 from twisted.spread import pb
  33 from twisted.cred import credentials
  34 from twisted.internet import reactor
  35
  36 from buildbot.scripts import runner
  37 from optparse import OptionParser
  38
  39 # Modify this to fit your setup, or pass in --master server:host on the
  40 # command line
  41
  42 master = "localhost:9989"
  43
  44 # The GIT_DIR environment variable must have been set up so that any
  45 # git commands that are executed will operate on the repository we're
  46 # installed in.
  47
  48 changes = []
  49
  50
  51 def connectFailed(error):
  52     logging.error("Could not connect to %s: %s"
  53             % (master, error.getErrorMessage()))
  54     return error
  55
  56
  57 def addChange(dummy, remote, changei):
  58     logging.debug("addChange %s, %s" % (repr(remote), repr(changei)))
  59     try:
  60         c = changei.next()
  61     except StopIteration:
  62         remote.broker.transport.loseConnection()
  63         return None
  64
  65     logging.info("New revision: %s" % c['revision'][:8])
  66     for key, value in c.iteritems():
  67         logging.debug("  %s: %s" % (key, value))
  68
  69     d = remote.callRemote('addChange', c)
  70     d.addCallback(addChange, remote, changei)
  71     return d
  72
  73
  74 def connected(remote):
  75     return addChange(None, remote, changes.__iter__())
  76
  77
  78 def grab_commit_info(c, rev):
  79     # Extract information about committer and files using git show
  80     f = os.popen("git show --raw --pretty=full %s" % rev, 'r')
  81
  82     files = []
  83
  84     while True:
  85         line = f.readline()
  86         if not line:
  87             break
  88
  89         m = re.match(r"^:.*[MAD]\s+(.+)$", line)
  90         if m:
  91             logging.debug("Got file: %s" % m.group(1))
  92             files.append(m.group(1))
  93             continue
  94
  95         m = re.match(r"^Author:\s+(.+)$", line)
  96         if m:
  97             logging.debug("Got author: %s" % m.group(1))
  98             c['who'] = m.group(1)
  99
 100         if re.match(r"^Merge: .*$", line):
 101             files.append('merge')
 102
 103     c['files'] = files
 104     status = f.close()
 105     if status:
 106         logging.warning("git show exited with status %d" % status)
 107
 108
 109 def gen_changes(input, branch):
 110     while True:
 111         line = input.readline()
 112         if not line:
 113             break
 114
 115         logging.debug("Change: %s" % line)
 116
 117         m = re.match(r"^([0-9a-f]+) (.*)$", line.strip())
 118         c = {'revision': m.group(1),
 119              'comments': m.group(2),
 120              'branch': branch,
 121         }
 122         grab_commit_info(c, m.group(1))
 123         changes.append(c)
 124
 125
 126 def gen_create_branch_changes(newrev, refname, branch):
 127     # A new branch has been created. Generate changes for everything
 128     # up to `newrev' which does not exist in any branch but `refname'.
 129     #
 130     # Note that this may be inaccurate if two new branches are created
 131     # at the same time, pointing to the same commit, or if there are
 132     # commits that only exists in a common subset of the new branches.
 133
 134     logging.info("Branch `%s' created" % branch)
 135
 136     f = os.popen("git rev-parse --not --branches"
 137             + "| grep -v $(git rev-parse %s)" % refname
 138             + "| git rev-list --reverse --pretty=oneline --stdin %s" % newrev,
 139             'r')
 140
 141     gen_changes(f, branch)
 142
 143     status = f.close()
 144     if status:
 145         logging.warning("git rev-list exited with status %d" % status)
 146
 147
 148 def gen_update_branch_changes(oldrev, newrev, refname, branch):
 149     # A branch has been updated. If it was a fast-forward update,
 150     # generate Change events for everything between oldrev and newrev.
 151     #
 152     # In case of a forced update, first generate a "fake" Change event
 153     # rewinding the branch to the common ancestor of oldrev and
 154     # newrev. Then, generate Change events for each commit between the
 155     # common ancestor and newrev.
 156
 157     logging.info("Branch `%s' updated %s .. %s"
 158             % (branch, oldrev[:8], newrev[:8]))
 159
 160     baserev = commands.getoutput("git merge-base %s %s" % (oldrev, newrev))
 161     logging.debug("oldrev=%s newrev=%s baserev=%s" % (oldrev, newrev, baserev))
 162     if baserev != oldrev:
 163         c = {'revision': baserev,
 164              'comments': "Rewind branch",
 165              'branch': branch,
 166              'who': "dummy",
 167         }
 168         logging.info("Branch %s was rewound to %s" % (branch, baserev[:8]))
 169         files = []
 170         f = os.popen("git diff --raw %s..%s" % (oldrev, baserev), 'r')
 171         while True:
 172             line = f.readline()
 173             if not line:
 174                 break
 175
 176             file = re.match(r"^:.*[MAD]\s*(.+)$", line).group(1)
 177             logging.debug("  Rewound file: %s" % file)
 178             files.append(file)
 179
 180         status = f.close()
 181         if status:
 182             logging.warning("git diff exited with status %d" % status)
 183
 184         if files:
 185             c['files'] = files
 186             changes.append(c)
 187
 188     if newrev != baserev:
 189         # Not a pure rewind
 190         f = os.popen("git rev-list --reverse --pretty=oneline %s..%s"
 191                 % (baserev, newrev), 'r')
 192         gen_changes(f, branch)
 193
 194         status = f.close()
 195         if status:
 196             logging.warning("git rev-list exited with status %d" % status)
 197
 198
 199 def cleanup(res):
 200     reactor.stop()
 201
 202
 203 def process_changes():
 204     # Read branch updates from stdin and generate Change events
 205     while True:
 206         line = sys.stdin.readline()
 207         if not line:
 208             break
 209
 210         [oldrev, newrev, refname] = line.split(None, 2)
 211
 212         # We only care about regular heads, i.e. branches
 213         m = re.match(r"^refs\/heads\/(.+)$", refname)
 214         if not m:
 215             logging.info("Ignoring refname `%s': Not a branch" % refname)
 216             continue
 217
 218         branch = m.group(1)
 219
 220         # Find out if the branch was created, deleted or updated. Branches
 221         # being deleted aren't really interesting.
 222         if re.match(r"^0*$", newrev):
 223             logging.info("Branch `%s' deleted, ignoring" % branch)
 224             continue
 225         elif re.match(r"^0*$", oldrev):
 226             gen_create_branch_changes(newrev, refname, branch)
 227         else:
 228             gen_update_branch_changes(oldrev, newrev, refname, branch)
 229
 230     # Submit the changes, if any
 231     if not changes:
 232         logging.warning("No changes found")
 233         return
 234
 235     host, port = master.split(':')
 236     port = int(port)
 237
 238     f = pb.PBClientFactory()
 239     d = f.login(credentials.UsernamePassword("change", "changepw"))
 240     reactor.connectTCP(host, port, f)
 241
 242     d.addErrback(connectFailed)
 243     d.addCallback(connected)
 244     d.addBoth(cleanup)
 245
 246     reactor.run()
 247
 248
 249 def parse_options():
 250     parser = OptionParser()
 251     parser.add_option("-l", "--logfile", action="store", type="string",
 252             help="Log to the specified file")
 253     parser.add_option("-v", "--verbose", action="count",
 254             help="Be more verbose. Ignored if -l is not specified.")
 255     master_help = ("Build master to push to. Default is %(master)s" %
 256                    { 'master' : master })
 257     parser.add_option("-m", "--master", action="store", type="string",
 258             help=master_help)
 259     options, args = parser.parse_args()
 260     return options
 261
 262
 263 # Log errors and critical messages to stderr. Optionally log
 264 # information to a file as well (we'll set that up later.)
 265 stderr = logging.StreamHandler(sys.stderr)
 266 fmt = logging.Formatter("git_buildbot: %(levelname)s: %(message)s")
 267 stderr.setLevel(logging.ERROR)
 268 stderr.setFormatter(fmt)
 269 logging.getLogger().addHandler(stderr)
 270 logging.getLogger().setLevel(logging.DEBUG)
 271
 272 try:
 273     options = parse_options()
 274     level = logging.WARNING
 275     if options.verbose:
 276         level -= 10 * options.verbose
 277         if level < 0:
 278             level = 0
 279
 280     if options.logfile:
 281         logfile = logging.FileHandler(options.logfile)
 282         logfile.setLevel(level)
 283         fmt = logging.Formatter("%(asctime)s %(levelname)s: %(message)s")
 284         logfile.setFormatter(fmt)
 285         logging.getLogger().addHandler(logfile)
 286
 287     if options.master:
 288         master=options.master
 289
 290     process_changes()
 291 except SystemExit:
 292     pass
 293 except:
 294     logging.exception("Unhandled exception")
 295     sys.exit(1)