contrib/bzr_buildbot.py

   1 # Copyright (C) 2008-2009 Canonical
   2 #
   3 # This program is free software: you can redistribute it and/or modify
   4 # it under the terms of the GNU General Public License as published by
   5 # the Free Software Foundation, either version 3 of the License, or
   6 # (at your option) any later version.
   7 #
   8 # This program is distributed in the hope that it will be useful,
   9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11 # GNU General Public License for more details.
  12 #
  13 # You should have received a copy of the GNU General Public License
  14 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  15
  16 """\
  17 bzr buildbot integration
  18 ========================
  19
  20 This file contains both bzr commit/change hooks and a bzr poller.
  21
  22 ------------
  23 Requirements
  24 ------------
  25
  26 This has been tested with buildbot 0.7.9, bzr 1.10, and Twisted 8.1.0.  It
  27 should work in subsequent releases.
  28
  29 For the hook to work, Twisted must be installed in the same Python that bzr
  30 uses.
  31
  32 -----
  33 Hooks
  34 -----
  35
  36 To install, put this file in a bzr plugins directory (e.g.,
  37 ~/.bazaar/plugins). Then, in one of your bazaar conf files (e.g.,
  38 ~/.bazaar/locations.conf), set the location you want to connect with buildbot
  39 with these keys:
  40
  41 - buildbot_on: one of 'commit', 'push, or 'change'.  Turns the plugin on to
  42   report changes via commit, changes via push, or any changes to the trunk.
  43   'change' is recommended.
  44
  45 - buildbot_server: (required to send to a buildbot master) the URL of the
  46   buildbot master to which you will connect (as of this writing, the same
  47   server and port to which slaves connect).
  48
  49 - buildbot_port: (optional, defaults to 9989) the port of the buildbot master
  50   to which you will connect (as of this writing, the same server and port to
  51   which slaves connect)
  52
  53 - buildbot_pqm: (optional, defaults to not pqm) Normally, the user that
  54   commits the revision is the user that is responsible for the change.  When
  55   run in a pqm (Patch Queue Manager, see https://launchpad.net/pqm)
  56   environment, the user that commits is the Patch Queue Manager, and the user
  57   that committed the *parent* revision is responsible for the change.  To turn
  58   on the pqm mode, set this value to any of (case-insensitive) "Yes", "Y",
  59   "True", or "T".
  60
  61 - buildbot_dry_run: (optional, defaults to not a dry run) Normally, the
  62   post-commit hook will attempt to communicate with the configured buildbot
  63   server and port. If this parameter is included and any of (case-insensitive)
  64   "Yes", "Y", "True", or "T", then the hook will simply print what it would
  65   have sent, but not attempt to contact the buildbot master.
  66
  67 - buildbot_send_branch_name: (optional, defaults to not sending the branch
  68   name) If your buildbot's bzr source build step uses a repourl, do
  69   *not* turn this on. If your buildbot's bzr build step uses a baseURL, then
  70   you may set this value to any of (case-insensitive) "Yes", "Y", "True", or
  71   "T" to have the buildbot master append the branch name to the baseURL.
  72
  73 When buildbot no longer has a hardcoded password, it will be a configuration
  74 option here as well.
  75
  76 ------
  77 Poller
  78 ------
  79
  80 Put this file somewhere that your buildbot configuration can import it.  Even
  81 in the same directory as the master.cfg should work.  Install the poller in
  82 the buildbot configuration as with any other change source.  Minimally,
  83 provide a URL that you want to poll (bzr://, bzr+ssh://, or lp:), though make
  84 sure the buildbot user has necessary privileges.  You may also want to specify
  85 these optional values.
  86
  87 poll_interval: the number of seconds to wait between polls.  Defaults to 10
  88                minutes.
  89
  90 branch_name: any value to be used as the branch name.  Defaults to None, or
  91              specify a string, or specify the constants from this file SHORT
  92              or FULL to get the short branch name or full branch address.
  93
  94 blame_merge_author: normally, the user that commits the revision is the user
  95                     that is responsible for the change. When run in a pqm
  96                     (Patch Queue Manager, see https://launchpad.net/pqm)
  97                     environment, the user that commits is the Patch Queue
  98                     Manager, and the user that committed the merged, *parent*
  99                     revision is responsible for the change. set this value to
 100                     True if this is pointed against a PQM-managed branch.
 101
 102 -------------------
 103 Contact Information
 104 -------------------
 105
 106 Maintainer/author: gary.poster@canonical.com
 107 """
 108
 109 try:
 110     import buildbot.util
 111     import buildbot.changes.base
 112     import buildbot.changes.changes
 113 except ImportError:
 114     DEFINE_POLLER = False
 115 else:
 116     DEFINE_POLLER = True
 117 import bzrlib.branch
 118 import bzrlib.errors
 119 import bzrlib.trace
 120 import twisted.cred.credentials
 121 import twisted.internet.base
 122 import twisted.internet.defer
 123 import twisted.internet.reactor
 124 import twisted.internet.selectreactor
 125 import twisted.internet.task
 126 import twisted.internet.threads
 127 import twisted.python.log
 128 import twisted.spread.pb
 129
 130
 131 #############################################################################
 132 # This is the code that the poller and the hooks share.
 133
 134 def generate_change(branch,
 135                     old_revno=None, old_revid=None,
 136                     new_revno=None, new_revid=None,
 137                     blame_merge_author=False):
 138     """Return a dict of information about a change to the branch.
 139
 140     Dict has keys of "files", "who", "comments", and "revision", as used by
 141     the buildbot Change (and the PBChangeSource).
 142
 143     If only the branch is given, the most recent change is returned.
 144
 145     If only the new_revno is given, the comparison is expected to be between
 146     it and the previous revno (new_revno -1) in the branch.
 147
 148     Passing old_revid and new_revid is only an optimization, included because
 149     bzr hooks usually provide this information.
 150
 151     blame_merge_author means that the author of the merged branch is
 152     identified as the "who", not the person who committed the branch itself.
 153     This is typically used for PQM.
 154     """
 155     change = {} # files, who, comments, revision; NOT branch (= branch.nick)
 156     if new_revno is None:
 157         new_revno = branch.revno()
 158     if new_revid is None:
 159         new_revid = branch.get_rev_id(new_revno)
 160     # TODO: This falls over if this is the very first revision
 161     if old_revno is None:
 162         old_revno = new_revno -1
 163     if old_revid is None:
 164         old_revid = branch.get_rev_id(old_revno)
 165     repository = branch.repository
 166     new_rev = repository.get_revision(new_revid)
 167     if blame_merge_author:
 168         # this is a pqm commit or something like it
 169         change['who'] = repository.get_revision(
 170             new_rev.parent_ids[-1]).get_apparent_author()
 171     else:
 172         change['who'] = new_rev.get_apparent_author()
 173     # maybe useful to know:
 174     # name, email = bzrtools.config.parse_username(change['who'])
 175     change['comments'] = new_rev.message
 176     change['revision'] = new_revno
 177     files = change['files'] = []
 178     changes = repository.revision_tree(new_revid).changes_from(
 179         repository.revision_tree(old_revid))
 180     for (collection, name) in ((changes.added, 'ADDED'),
 181                                (changes.removed, 'REMOVED'),
 182                                (changes.modified, 'MODIFIED')):
 183         for info in collection:
 184             path = info[0]
 185             kind = info[2]
 186             files.append(' '.join([path, kind, name]))
 187     for info in changes.renamed:
 188         oldpath, newpath, id, kind, text_modified, meta_modified = info
 189         elements = [oldpath, kind,'RENAMED', newpath]
 190         if text_modified or meta_modified:
 191             elements.append('MODIFIED')
 192         files.append(' '.join(elements))
 193     return change
 194
 195 #############################################################################
 196 # poller
 197
 198 # We don't want to make the hooks unnecessarily depend on buildbot being
 199 # installed locally, so we conditionally create the BzrPoller class.
 200 if DEFINE_POLLER:
 201
 202     FULL = object()
 203     SHORT = object()
 204
 205
 206     class BzrPoller(buildbot.changes.base.ChangeSource,
 207                     buildbot.util.ComparableMixin):
 208
 209         compare_attrs = ['url']
 210
 211         def __init__(self, url, poll_interval=10*60, blame_merge_author=False,
 212                      branch_name=None):
 213             # poll_interval is in seconds, so default poll_interval is 10
 214             # minutes.
 215             # bzr+ssh://bazaar.launchpad.net/~launchpad-pqm/launchpad/devel/
 216             # works, lp:~launchpad-pqm/launchpad/devel/ doesn't without help.
 217             if url.startswith('lp:'):
 218                 url = 'bzr+ssh://bazaar.launchpad.net/' + url[3:]
 219             self.url = url
 220             self.poll_interval = poll_interval
 221             self.loop = twisted.internet.task.LoopingCall(self.poll)
 222             self.blame_merge_author = blame_merge_author
 223             self.branch_name = branch_name
 224
 225         def startService(self):
 226             twisted.python.log.msg("BzrPoller(%s) starting" % self.url)
 227             buildbot.changes.base.ChangeSource.startService(self)
 228             if self.branch_name is FULL:
 229                 ourbranch = self.url
 230             elif self.branch_name is SHORT:
 231                 # We are in a bit of trouble, as we cannot really know what our
 232                 # branch is until we have polled new changes.
 233                 # Seems we would have to wait until we polled the first time,
 234                 # and only then do the filtering, grabbing the branch name from
 235                 # whatever we polled.
 236                 # For now, leave it as it was previously (compare against
 237                 # self.url); at least now things work when specifying the
 238                 # branch name explicitly.
 239                 ourbranch = self.url
 240             else:
 241                 ourbranch = self.branch_name
 242             for change in reversed(self.parent.changes):
 243                 if change.branch == ourbranch:
 244                     self.last_revision = change.revision
 245                     break
 246             else:
 247                 self.last_revision = None
 248             self.polling = False
 249             twisted.internet.reactor.callWhenRunning(
 250                 self.loop.start, self.poll_interval)
 251
 252         def stopService(self):
 253             twisted.python.log.msg("BzrPoller(%s) shutting down" % self.url)
 254             self.loop.stop()
 255             return buildbot.changes.base.ChangeSource.stopService(self)
 256
 257         def describe(self):
 258             return "BzrPoller watching %s" % self.url
 259
 260         @twisted.internet.defer.inlineCallbacks
 261         def poll(self):
 262             if self.polling: # this is called in a loop, and the loop might
 263                 # conceivably overlap.
 264                 return
 265             self.polling = True
 266             try:
 267                 # On a big tree, even individual elements of the bzr commands
 268                 # can take awhile. So we just push the bzr work off to a
 269                 # thread.
 270                 try:
 271                     changes = yield twisted.internet.threads.deferToThread(
 272                         self.getRawChanges)
 273                 except (SystemExit, KeyboardInterrupt):
 274                     raise
 275                 except:
 276                     # we'll try again next poll.  Meanwhile, let's report.
 277                     twisted.python.log.err()
 278                 else:
 279                     for change in changes:
 280                         yield self.addChange(
 281                             buildbot.changes.changes.Change(**change))
 282                         self.last_revision = change['revision']
 283             finally:
 284                 self.polling = False
 285
 286         def getRawChanges(self):
 287             branch = bzrlib.branch.Branch.open_containing(self.url)[0]
 288             if self.branch_name is FULL:
 289                 branch_name = self.url
 290             elif self.branch_name is SHORT:
 291                 branch_name = branch.nick
 292             else: # presumably a string or maybe None
 293                 branch_name = self.branch_name
 294             changes = []
 295             change = generate_change(
 296                 branch, blame_merge_author=self.blame_merge_author)
 297             if (self.last_revision is None or
 298                 change['revision'] > self.last_revision):
 299                 change['branch'] = branch_name
 300                 changes.append(change)
 301                 if self.last_revision is not None:
 302                     while self.last_revision + 1 < change['revision']:
 303                         change = generate_change(
 304                             branch, new_revno=change['revision']-1,
 305                             blame_merge_author=self.blame_merge_author)
 306                         change['branch'] = branch_name
 307                         changes.append(change)
 308             changes.reverse()
 309             return changes
 310
 311         def addChange(self, change):
 312             d = twisted.internet.defer.Deferred()
 313             def _add_change():
 314                 d.callback(
 315                     self.parent.addChange(change))
 316             twisted.internet.reactor.callLater(0, _add_change)
 317             return d
 318
 319 #############################################################################
 320 # hooks
 321
 322 HOOK_KEY = 'buildbot_on'
 323 SERVER_KEY = 'buildbot_server'
 324 PORT_KEY = 'buildbot_port'
 325 DRYRUN_KEY = 'buildbot_dry_run'
 326 PQM_KEY = 'buildbot_pqm'
 327 SEND_BRANCHNAME_KEY = 'buildbot_send_branch_name'
 328
 329 PUSH_VALUE = 'push'
 330 COMMIT_VALUE = 'commit'
 331 CHANGE_VALUE = 'change'
 332
 333 def _is_true(config, key):
 334     val = config.get_user_option(key)
 335     return val is not None and val.lower().strip() in (
 336         'y', 'yes', 't', 'true')
 337
 338 def _installed_hook(branch):
 339     value = branch.get_config().get_user_option(HOOK_KEY)
 340     if value is not None:
 341         value = value.strip().lower()
 342         if value not in (PUSH_VALUE, COMMIT_VALUE, CHANGE_VALUE):
 343             raise bzrlib.errors.BzrError(
 344                 '%s, if set, must be one of %s, %s, or %s' % (
 345                     HOOK_KEY, PUSH_VALUE, COMMIT_VALUE, CHANGE_VALUE))
 346     return value
 347
 348 ##########################
 349 # Work around Twisted bug.
 350 # See http://twistedmatrix.com/trac/ticket/3591
 351 import operator
 352 import socket
 353 from twisted.internet import defer
 354 from twisted.python import failure
 355
 356 # replaces twisted.internet.thread equivalent
 357 def _putResultInDeferred(reactor, deferred, f, args, kwargs):
 358     """
 359     Run a function and give results to a Deferred.
 360     """
 361     try:
 362         result = f(*args, **kwargs)
 363     except:
 364         f = failure.Failure()
 365         reactor.callFromThread(deferred.errback, f)
 366     else:
 367         reactor.callFromThread(deferred.callback, result)
 368
 369 # would be a proposed addition.  deferToThread could use it
 370 def deferToThreadInReactor(reactor, f, *args, **kwargs):
 371     """
 372     Run function in thread and return result as Deferred.
 373     """
 374     d = defer.Deferred()
 375     reactor.callInThread(_putResultInDeferred, reactor, d, f, args, kwargs)
 376     return d
 377
 378 # uses its own reactor for the threaded calls, unlike Twisted's
 379 class ThreadedResolver(twisted.internet.base.ThreadedResolver):
 380     def getHostByName(self, name, timeout = (1, 3, 11, 45)):
 381         if timeout:
 382             timeoutDelay = reduce(operator.add, timeout)
 383         else:
 384             timeoutDelay = 60
 385         userDeferred = defer.Deferred()
 386         lookupDeferred = deferToThreadInReactor(
 387             self.reactor, socket.gethostbyname, name)
 388         cancelCall = self.reactor.callLater(
 389             timeoutDelay, self._cleanup, name, lookupDeferred)
 390         self._runningQueries[lookupDeferred] = (userDeferred, cancelCall)
 391         lookupDeferred.addBoth(self._checkTimeout, name, lookupDeferred)
 392         return userDeferred
 393 ##########################
 394
 395 def send_change(branch, old_revno, old_revid, new_revno, new_revid, hook):
 396     config = branch.get_config()
 397     server = config.get_user_option(SERVER_KEY)
 398     if not server:
 399         bzrlib.trace.warning(
 400             'bzr_buildbot: ERROR.  If %s is set, %s must be set',
 401             HOOK_KEY, SERVER_KEY)
 402         return
 403     change = generate_change(
 404         branch, old_revno, old_revid, new_revno, new_revid,
 405         blame_merge_author=_is_true(config, PQM_KEY))
 406     if _is_true(config, SEND_BRANCHNAME_KEY):
 407         change['branch'] = branch.nick
 408     # as of this writing (in Buildbot 0.7.9), 9989 is the default port when
 409     # you make a buildbot master.
 410     port = int(config.get_user_option(PORT_KEY) or 9989)
 411     # if dry run, stop.
 412     if _is_true(config, DRYRUN_KEY):
 413         bzrlib.trace.note("bzr_buildbot DRY RUN "
 414                           "(*not* sending changes to %s:%d on %s)",
 415                           server, port, hook)
 416         keys = change.keys()
 417         keys.sort()
 418         for k in keys:
 419             bzrlib.trace.note("[%10s]: %s", k, change[k])
 420         return
 421     # We instantiate our own reactor so that this can run within a server.
 422     reactor = twisted.internet.selectreactor.SelectReactor()
 423     # See other reference to http://twistedmatrix.com/trac/ticket/3591
 424     # above.  This line can go away with a release of Twisted that addresses
 425     # this issue.
 426     reactor.resolver = ThreadedResolver(reactor)
 427     pbcf = twisted.spread.pb.PBClientFactory()
 428     reactor.connectTCP(server, port, pbcf)
 429     deferred = pbcf.login(
 430         twisted.cred.credentials.UsernamePassword('change', 'changepw'))
 431
 432     def sendChanges(remote):
 433         """Send changes to buildbot."""
 434         bzrlib.trace.mutter("bzrbuildout sending changes: %s", change)
 435         return remote.callRemote('addChange', change)
 436
 437     deferred.addCallback(sendChanges)
 438
 439     def quit(ignore, msg):
 440         bzrlib.trace.note("bzrbuildout: %s", msg)
 441         reactor.stop()
 442
 443     def failed(failure):
 444         bzrlib.trace.warning("bzrbuildout: FAILURE\n %s", failure)
 445         reactor.stop()
 446
 447     deferred.addCallback(quit, "SUCCESS")
 448     deferred.addErrback(failed)
 449     reactor.callLater(60, quit, None, "TIMEOUT")
 450     bzrlib.trace.note(
 451         "bzr_buildbot: SENDING CHANGES to buildbot master %s:%d on %s",
 452         server, port, hook)
 453     reactor.run(installSignalHandlers=False) # run in a thread when in server
 454
 455 def post_commit(local_branch, master_branch, # branch is the master_branch
 456                 old_revno, old_revid, new_revno, new_revid):
 457     if _installed_hook(master_branch) == COMMIT_VALUE:
 458         send_change(master_branch,
 459                      old_revid, old_revid, new_revno, new_revid, COMMIT_VALUE)
 460
 461 def post_push(result):
 462     if _installed_hook(result.target_branch) == PUSH_VALUE:
 463         send_change(result.target_branch,
 464                      result.old_revid, result.old_revid,
 465                      result.new_revno, result.new_revid, PUSH_VALUE)
 466
 467 def post_change_branch_tip(result):
 468     if _installed_hook(result.branch) == CHANGE_VALUE:
 469         send_change(result.branch,
 470                      result.old_revid, result.old_revid,
 471                      result.new_revno, result.new_revid, CHANGE_VALUE)
 472
 473 bzrlib.branch.Branch.hooks.install_named_hook(
 474     'post_commit', post_commit,
 475     'send change to buildbot master')
 476 bzrlib.branch.Branch.hooks.install_named_hook(
 477     'post_push', post_push,
 478     'send change to buildbot master')
 479 bzrlib.branch.Branch.hooks.install_named_hook(
 480     'post_change_branch_tip', post_change_branch_tip,
 481     'send change to buildbot master')