contrib/bzr_buildbot.py

   1 # Copyright (C) 2008-2009 Canonical
   2 #
   3 # This program is free software: you can redistribute it and/or modify
   4 # it under the terms of the GNU General Public License as published by
   5 # the Free Software Foundation, either version 3 of the License, or
   6 # (at your option) any later version.
   7 #
   8 # This program is distributed in the hope that it will be useful,
   9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11 # GNU General Public License for more details.
  12 #
  13 # You should have received a copy of the GNU General Public License
  14 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  15
  16 """\
  17 bzr buildbot integration
  18 ========================
  19
  20 This file contains both bzr commit/change hooks and a bzr poller.
  21
  22 ------------
  23 Requirements
  24 ------------
  25
  26 This has been tested with buildbot 0.7.9, bzr 1.10, and Twisted 8.1.0.  It
  27 should work in subsequent releases.
  28
  29 For the hook to work, Twisted must be installed in the same Python that bzr
  30 uses.
  31
  32 -----
  33 Hooks
  34 -----
  35
  36 To install, put this file in a bzr plugins directory (e.g.,
  37 ~/.bazaar/plugins). Then, in one of your bazaar conf files (e.g.,
  38 ~/.bazaar/locations.conf), set the location you want to connect with buildbot
  39 with these keys:
  40
  41 - buildbot_on: one of 'commit', 'push, or 'change'.  Turns the plugin on to
  42   report changes via commit, changes via push, or any changes to the trunk.
  43   'change' is recommended.
  44
  45 - buildbot_server: (required to send to a buildbot master) the URL of the
  46   buildbot master to which you will connect (as of this writing, the same
  47   server and port to which slaves connect).
  48
  49 - buildbot_port: (optional, defaults to 9989) the port of the buildbot master
  50   to which you will connect (as of this writing, the same server and port to
  51   which slaves connect)
  52
  53 - buildbot_pqm: (optional, defaults to not pqm) Normally, the user that
  54   commits the revision is the user that is responsible for the change.  When
  55   run in a pqm (Patch Queue Manager, see https://launchpad.net/pqm)
  56   environment, the user that commits is the Patch Queue Manager, and the user
  57   that committed the *parent* revision is responsible for the change.  To turn
  58   on the pqm mode, set this value to any of (case-insensitive) "Yes", "Y",
  59   "True", or "T".
  60
  61 - buildbot_dry_run: (optional, defaults to not a dry run) Normally, the
  62   post-commit hook will attempt to communicate with the configured buildbot
  63   server and port. If this parameter is included and any of (case-insensitive)
  64   "Yes", "Y", "True", or "T", then the hook will simply print what it would
  65   have sent, but not attempt to contact the buildbot master.
  66
  67 - buildbot_send_branch_name: (optional, defaults to not sending the branch
  68   name) If your buildbot's bzr source build step uses a repourl, do
  69   *not* turn this on. If your buildbot's bzr build step uses a baseURL, then
  70   you may set this value to any of (case-insensitive) "Yes", "Y", "True", or
  71   "T" to have the buildbot master append the branch name to the baseURL.
  72
  73 When buildbot no longer has a hardcoded password, it will be a configuration
  74 option here as well.
  75
  76 ------
  77 Poller
  78 ------
  79
  80 Put this file somewhere that your buildbot configuration can import it.  Even
  81 in the same directory as the master.cfg should work.  Install the poller in
  82 the buildbot configuration as with any other change source.  Minimally,
  83 provide a URL that you want to poll (bzr://, bzr+ssh://, or lp:), though make
  84 sure the buildbot user has necessary privileges.  You may also want to specify
  85 these optional values.
  86
  87 poll_interval: the number of seconds to wait between polls.  Defaults to 10
  88                minutes.
  89
  90 branch_name: any value to be used as the branch name.  Defaults to None, or
  91              specify a string, or specify the constants from this file SHORT
  92              or FULL to get the short branch name or full branch address.
  93
  94 blame_merge_author: normally, the user that commits the revision is the user
  95                     that is responsible for the change. When run in a pqm
  96                     (Patch Queue Manager, see https://launchpad.net/pqm)
  97                     environment, the user that commits is the Patch Queue
  98                     Manager, and the user that committed the merged, *parent*
  99                     revision is responsible for the change. set this value to
 100                     True if this is pointed against a PQM-managed branch.
 101
 102 -------------------
 103 Contact Information
 104 -------------------
 105
 106 Maintainer/author: gary.poster@canonical.com
 107 """
 108
 109 try:
 110     import buildbot.util
 111     import buildbot.changes.base
 112     import buildbot.changes.changes
 113 except ImportError:
 114     DEFINE_POLLER = False
 115 else:
 116     DEFINE_POLLER = True
 117 import bzrlib.branch
 118 import bzrlib.errors
 119 import bzrlib.trace
 120 import twisted.cred.credentials
 121 import twisted.internet.base
 122 import twisted.internet.defer
 123 import twisted.internet.reactor
 124 import twisted.internet.selectreactor
 125 import twisted.internet.task
 126 import twisted.internet.threads
 127 import twisted.python.log
 128 import twisted.spread.pb
 129
 130
 131 #############################################################################
 132 # This is the code that the poller and the hooks share.
 133
 134 def generate_change(branch,
 135                     old_revno=None, old_revid=None,
 136                     new_revno=None, new_revid=None,
 137                     blame_merge_author=False):
 138     """Return a dict of information about a change to the branch.
 139
 140     Dict has keys of "files", "who", "comments", and "revision", as used by
 141     the buildbot Change (and the PBChangeSource).
 142
 143     If only the branch is given, the most recent change is returned.
 144
 145     If only the new_revno is given, the comparison is expected to be between
 146     it and the previous revno (new_revno -1) in the branch.
 147
 148     Passing old_revid and new_revid is only an optimization, included because
 149     bzr hooks usually provide this information.
 150
 151     blame_merge_author means that the author of the merged branch is
 152     identified as the "who", not the person who committed the branch itself.
 153     This is typically used for PQM.
 154     """
 155     change = {} # files, who, comments, revision; NOT branch (= branch.nick)
 156     if new_revno is None:
 157         new_revno = branch.revno()
 158     if new_revid is None:
 159         new_revid = branch.get_rev_id(new_revno)
 160     # TODO: This falls over if this is the very first revision
 161     if old_revno is None:
 162         old_revno = new_revno -1
 163     if old_revid is None:
 164         old_revid = branch.get_rev_id(old_revno)
 165     repository = branch.repository
 166     new_rev = repository.get_revision(new_revid)
 167     if blame_merge_author:
 168         # this is a pqm commit or something like it
 169         change['who'] = repository.get_revision(
 170             new_rev.parent_ids[-1]).get_apparent_author()
 171     else:
 172         change['who'] = new_rev.get_apparent_author()
 173     # maybe useful to know:
 174     # name, email = bzrtools.config.parse_username(change['who'])
 175     change['comments'] = new_rev.message
 176     change['revision'] = new_revno
 177     files = change['files'] = []
 178     changes = repository.revision_tree(new_revid).changes_from(
 179         repository.revision_tree(old_revid))
 180     for (collection, name) in ((changes.added, 'ADDED'),
 181                                (changes.removed, 'REMOVED'),
 182                                (changes.modified, 'MODIFIED')):
 183         for info in collection:
 184             path = info[0]
 185             kind = info[2]
 186             files.append(' '.join([path, kind, name]))
 187     for info in changes.renamed:
 188         oldpath, newpath, id, kind, text_modified, meta_modified = info
 189         elements = [oldpath, kind,'RENAMED', newpath]
 190         if text_modified or meta_modified:
 191             elements.append('MODIFIED')
 192         files.append(' '.join(elements))
 193     return change
 194
 195 #############################################################################
 196 # poller
 197
 198 # We don't want to make the hooks unnecessarily depend on buildbot being
 199 # installed locally, so we conditionally create the BzrPoller class.
 200 if DEFINE_POLLER:
 201
 202     FULL = object()
 203     SHORT = object()
 204
 205
 206     class BzrPoller(buildbot.changes.base.ChangeSource,
 207                     buildbot.util.ComparableMixin):
 208
 209         compare_attrs = ['url']
 210
 211         def __init__(self, url, poll_interval=10*60, blame_merge_author=False,
 212                      branch_name=None):
 213             # poll_interval is in seconds, so default poll_interval is 10
 214             # minutes.
 215             # bzr+ssh://bazaar.launchpad.net/~launchpad-pqm/launchpad/devel/
 216             # works, lp:~launchpad-pqm/launchpad/devel/ doesn't without help.
 217             if url.startswith('lp:'):
 218                 url = 'bzr+ssh://bazaar.launchpad.net/' + url[3:]
 219             self.url = url
 220             self.poll_interval = poll_interval
 221             self.loop = twisted.internet.task.LoopingCall(self.poll)
 222             self.blame_merge_author = blame_merge_author
 223             self.branch_name = branch_name
 224
 225         def startService(self):
 226             twisted.python.log.msg("BzrPoller(%s) starting" % self.url)
 227             buildbot.changes.base.ChangeSource.startService(self)
 228             twisted.internet.reactor.callWhenRunning(
 229                 self.loop.start, self.poll_interval)
 230             for change in reversed(self.parent.changes):
 231                 if change.branch == self.url:
 232                     self.last_revision = change.revision
 233                     break
 234             else:
 235                 self.last_revision = None
 236             self.polling = False
 237
 238         def stopService(self):
 239             twisted.python.log.msg("BzrPoller(%s) shutting down" % self.url)
 240             self.loop.stop()
 241             return buildbot.changes.base.ChangeSource.stopService(self)
 242
 243         def describe(self):
 244             return "BzrPoller watching %s" % self.url
 245
 246         @twisted.internet.defer.inlineCallbacks
 247         def poll(self):
 248             if self.polling: # this is called in a loop, and the loop might
 249                 # conceivably overlap.
 250                 return
 251             self.polling = True
 252             try:
 253                 # On a big tree, even individual elements of the bzr commands
 254                 # can take awhile. So we just push the bzr work off to a
 255                 # thread.
 256                 try:
 257                     changes = yield twisted.internet.threads.deferToThread(
 258                         self.getRawChanges)
 259                 except (SystemExit, KeyboardInterrupt):
 260                     raise
 261                 except:
 262                     # we'll try again next poll.  Meanwhile, let's report.
 263                     twisted.python.log.err()
 264                 else:
 265                     for change in changes:
 266                         yield self.addChange(
 267                             buildbot.changes.changes.Change(**change))
 268                         self.last_revision = change['revision']
 269             finally:
 270                 self.polling = False
 271
 272         def getRawChanges(self):
 273             branch = bzrlib.branch.Branch.open_containing(self.url)[0]
 274             if self.branch_name is FULL:
 275                 branch_name = self.url
 276             elif self.branch_name is SHORT:
 277                 branch_name = branch.nick
 278             else: # presumably a string or maybe None
 279                 branch_name = self.branch_name
 280             changes = []
 281             change = generate_change(
 282                 branch, blame_merge_author=self.blame_merge_author)
 283             if (self.last_revision is None or
 284                 change['revision'] > self.last_revision):
 285                 changes.append(change)
 286                 if self.last_revision is not None:
 287                     while self.last_revision + 1 < change['revision']:
 288                         change = generate_change(
 289                             branch, new_revno=change['revision']-1,
 290                             blame_merge_author=self.blame_merge_author)
 291                         change['branch'] = branch_name
 292                         changes.append(change)
 293             changes.reverse()
 294             return changes
 295
 296         def addChange(self, change):
 297             d = twisted.internet.defer.Deferred()
 298             def _add_change():
 299                 d.callback(
 300                     self.parent.addChange(change))
 301             twisted.internet.reactor.callLater(0, _add_change)
 302             return d
 303
 304 #############################################################################
 305 # hooks
 306
 307 HOOK_KEY = 'buildbot_on'
 308 SERVER_KEY = 'buildbot_server'
 309 PORT_KEY = 'buildbot_port'
 310 DRYRUN_KEY = 'buildbot_dry_run'
 311 PQM_KEY = 'buildbot_pqm'
 312 SEND_BRANCHNAME_KEY = 'buildbot_send_branch_name'
 313
 314 PUSH_VALUE = 'push'
 315 COMMIT_VALUE = 'commit'
 316 CHANGE_VALUE = 'change'
 317
 318 def _is_true(config, key):
 319     val = config.get_user_option(key)
 320     return val is not None and val.lower().strip() in (
 321         'y', 'yes', 't', 'true')
 322
 323 def _installed_hook(branch):
 324     value = branch.get_config().get_user_option(HOOK_KEY)
 325     if value is not None:
 326         value = value.strip().lower()
 327         if value not in (PUSH_VALUE, COMMIT_VALUE, CHANGE_VALUE):
 328             raise bzrlib.errors.BzrError(
 329                 '%s, if set, must be one of %s, %s, or %s' % (
 330                     HOOK_KEY, PUSH_VALUE, COMMIT_VALUE, CHANGE_VALUE))
 331     return value
 332
 333 ##########################
 334 # Work around Twisted bug.
 335 # See http://twistedmatrix.com/trac/ticket/3591
 336 import operator
 337 import socket
 338 from twisted.internet import defer
 339 from twisted.python import failure
 340
 341 # replaces twisted.internet.thread equivalent
 342 def _putResultInDeferred(reactor, deferred, f, args, kwargs):
 343     """
 344     Run a function and give results to a Deferred.
 345     """
 346     try:
 347         result = f(*args, **kwargs)
 348     except:
 349         f = failure.Failure()
 350         reactor.callFromThread(deferred.errback, f)
 351     else:
 352         reactor.callFromThread(deferred.callback, result)
 353
 354 # would be a proposed addition.  deferToThread could use it
 355 def deferToThreadInReactor(reactor, f, *args, **kwargs):
 356     """
 357     Run function in thread and return result as Deferred.
 358     """
 359     d = defer.Deferred()
 360     reactor.callInThread(_putResultInDeferred, reactor, d, f, args, kwargs)
 361     return d
 362
 363 # uses its own reactor for the threaded calls, unlike Twisted's
 364 class ThreadedResolver(twisted.internet.base.ThreadedResolver):
 365     def getHostByName(self, name, timeout = (1, 3, 11, 45)):
 366         if timeout:
 367             timeoutDelay = reduce(operator.add, timeout)
 368         else:
 369             timeoutDelay = 60
 370         userDeferred = defer.Deferred()
 371         lookupDeferred = deferToThreadInReactor(
 372             self.reactor, socket.gethostbyname, name)
 373         cancelCall = self.reactor.callLater(
 374             timeoutDelay, self._cleanup, name, lookupDeferred)
 375         self._runningQueries[lookupDeferred] = (userDeferred, cancelCall)
 376         lookupDeferred.addBoth(self._checkTimeout, name, lookupDeferred)
 377         return userDeferred
 378 ##########################
 379
 380 def send_change(branch, old_revno, old_revid, new_revno, new_revid, hook):
 381     config = branch.get_config()
 382     server = config.get_user_option(SERVER_KEY)
 383     if not server:
 384         bzrlib.trace.warning(
 385             'bzr_buildbot: ERROR.  If %s is set, %s must be set',
 386             HOOK_KEY, SERVER_KEY)
 387         return
 388     change = generate_change(
 389         branch, old_revno, old_revid, new_revno, new_revid,
 390         blame_merge_author=_is_true(config, PQM_KEY))
 391     if _is_true(config, SEND_BRANCHNAME_KEY):
 392         change['branch'] = branch.nick
 393     # as of this writing (in Buildbot 0.7.9), 9989 is the default port when
 394     # you make a buildbot master.
 395     port = int(config.get_user_option(PORT_KEY) or 9989)
 396     # if dry run, stop.
 397     if _is_true(config, DRYRUN_KEY):
 398         bzrlib.trace.note("bzr_buildbot DRY RUN "
 399                           "(*not* sending changes to %s:%d on %s)",
 400                           server, port, hook)
 401         keys = change.keys()
 402         keys.sort()
 403         for k in keys:
 404             bzrlib.trace.note("[%10s]: %s", k, change[k])
 405         return
 406     # We instantiate our own reactor so that this can run within a server.
 407     reactor = twisted.internet.selectreactor.SelectReactor()
 408     # See other reference to http://twistedmatrix.com/trac/ticket/3591
 409     # above.  This line can go away with a release of Twisted that addresses
 410     # this issue.
 411     reactor.resolver = ThreadedResolver(reactor)
 412     pbcf = twisted.spread.pb.PBClientFactory()
 413     reactor.connectTCP(server, port, pbcf)
 414     deferred = pbcf.login(
 415         twisted.cred.credentials.UsernamePassword('change', 'changepw'))
 416
 417     def sendChanges(remote):
 418         """Send changes to buildbot."""
 419         bzrlib.trace.mutter("bzrbuildout sending changes: %s", change)
 420         return remote.callRemote('addChange', change)
 421
 422     deferred.addCallback(sendChanges)
 423
 424     def quit(ignore, msg):
 425         bzrlib.trace.note("bzrbuildout: %s", msg)
 426         reactor.stop()
 427
 428     def failed(failure):
 429         bzrlib.trace.warning("bzrbuildout: FAILURE\n %s", failure)
 430         reactor.stop()
 431
 432     deferred.addCallback(quit, "SUCCESS")
 433     deferred.addErrback(failed)
 434     reactor.callLater(60, quit, None, "TIMEOUT")
 435     bzrlib.trace.note(
 436         "bzr_buildbot: SENDING CHANGES to buildbot master %s:%d on %s",
 437         server, port, hook)
 438     reactor.run(installSignalHandlers=False) # run in a thread when in server
 439
 440 def post_commit(local_branch, master_branch, # branch is the master_branch
 441                 old_revno, old_revid, new_revno, new_revid):
 442     if _installed_hook(master_branch) == COMMIT_VALUE:
 443         send_change(master_branch,
 444                      old_revid, old_revid, new_revno, new_revid, COMMIT_VALUE)
 445
 446 def post_push(result):
 447     if _installed_hook(result.target_branch) == PUSH_VALUE:
 448         send_change(result.target_branch,
 449                      result.old_revid, result.old_revid,
 450                      result.new_revno, result.new_revid, PUSH_VALUE)
 451
 452 def post_change_branch_tip(result):
 453     if _installed_hook(result.branch) == CHANGE_VALUE:
 454         send_change(result.branch,
 455                      result.old_revid, result.old_revid,
 456                      result.new_revno, result.new_revid, CHANGE_VALUE)
 457
 458 bzrlib.branch.Branch.hooks.install_named_hook(
 459     'post_commit', post_commit,
 460     'send change to buildbot master')
 461 bzrlib.branch.Branch.hooks.install_named_hook(
 462     'post_push', post_push,
 463     'send change to buildbot master')
 464 bzrlib.branch.Branch.hooks.install_named_hook(
 465     'post_change_branch_tip', post_change_branch_tip,
 466     'send change to buildbot master')