testcase for #35, dependent schedulers lost on reconfig; reconfigure without changes...
[buildbot.git] / contrib / bzr_buildbot.py
bloba7424a4cdd1fcdd7492d145c6c9402ba345df446
1 # Copyright (C) 2008-2009 Canonical
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU General Public License as published by
5 # the Free Software Foundation, either version 3 of the License, or
6 # (at your option) any later version.
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details.
13 # You should have received a copy of the GNU General Public License
14 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 """\
17 bzr buildbot integration
18 ========================
20 This file contains both bzr commit/change hooks and a bzr poller.
22 ------------
23 Requirements
24 ------------
26 This has been tested with buildbot 0.7.9, bzr 1.10, and Twisted 8.1.0. It
27 should work in subsequent releases.
29 For the hook to work, Twisted must be installed in the same Python that bzr
30 uses.
32 -----
33 Hooks
34 -----
36 To install, put this file in a bzr plugins directory (e.g.,
37 ~/.bazaar/plugins). Then, in one of your bazaar conf files (e.g.,
38 ~/.bazaar/locations.conf), set the location you want to connect with buildbot
39 with these keys:
41 - buildbot_on: one of 'commit', 'push, or 'change'. Turns the plugin on to
42 report changes via commit, changes via push, or any changes to the trunk.
43 'change' is recommended.
45 - buildbot_server: (required to send to a buildbot master) the URL of the
46 buildbot master to which you will connect (as of this writing, the same
47 server and port to which slaves connect).
49 - buildbot_port: (optional, defaults to 9989) the port of the buildbot master
50 to which you will connect (as of this writing, the same server and port to
51 which slaves connect)
53 - buildbot_pqm: (optional, defaults to not pqm) Normally, the user that
54 commits the revision is the user that is responsible for the change. When
55 run in a pqm (Patch Queue Manager, see https://launchpad.net/pqm)
56 environment, the user that commits is the Patch Queue Manager, and the user
57 that committed the *parent* revision is responsible for the change. To turn
58 on the pqm mode, set this value to any of (case-insensitive) "Yes", "Y",
59 "True", or "T".
61 - buildbot_dry_run: (optional, defaults to not a dry run) Normally, the
62 post-commit hook will attempt to communicate with the configured buildbot
63 server and port. If this parameter is included and any of (case-insensitive)
64 "Yes", "Y", "True", or "T", then the hook will simply print what it would
65 have sent, but not attempt to contact the buildbot master.
67 - buildbot_send_branch_name: (optional, defaults to not sending the branch
68 name) If your buildbot's bzr source build step uses a repourl, do
69 *not* turn this on. If your buildbot's bzr build step uses a baseURL, then
70 you may set this value to any of (case-insensitive) "Yes", "Y", "True", or
71 "T" to have the buildbot master append the branch name to the baseURL.
73 When buildbot no longer has a hardcoded password, it will be a configuration
74 option here as well.
76 ------
77 Poller
78 ------
80 Put this file somewhere that your buildbot configuration can import it. Even
81 in the same directory as the master.cfg should work. Install the poller in
82 the buildbot configuration as with any other change source. Minimally,
83 provide a URL that you want to poll (bzr://, bzr+ssh://, or lp:), though make
84 sure the buildbot user has necessary privileges. You may also want to specify
85 these optional values.
87 poll_interval: the number of seconds to wait between polls. Defaults to 10
88 minutes.
90 branch_name: any value to be used as the branch name. Defaults to None, or
91 specify a string, or specify the constants from this file SHORT
92 or FULL to get the short branch name or full branch address.
94 blame_merge_author: normally, the user that commits the revision is the user
95 that is responsible for the change. When run in a pqm
96 (Patch Queue Manager, see https://launchpad.net/pqm)
97 environment, the user that commits is the Patch Queue
98 Manager, and the user that committed the merged, *parent*
99 revision is responsible for the change. set this value to
100 True if this is pointed against a PQM-managed branch.
102 -------------------
103 Contact Information
104 -------------------
106 Maintainer/author: gary.poster@canonical.com
109 try:
110 import buildbot.util
111 import buildbot.changes.base
112 import buildbot.changes.changes
113 except ImportError:
114 DEFINE_POLLER = False
115 else:
116 DEFINE_POLLER = True
117 import bzrlib.branch
118 import bzrlib.errors
119 import bzrlib.trace
120 import twisted.cred.credentials
121 import twisted.internet.base
122 import twisted.internet.defer
123 import twisted.internet.reactor
124 import twisted.internet.selectreactor
125 import twisted.internet.task
126 import twisted.internet.threads
127 import twisted.python.log
128 import twisted.spread.pb
131 #############################################################################
132 # This is the code that the poller and the hooks share.
134 def generate_change(branch,
135 old_revno=None, old_revid=None,
136 new_revno=None, new_revid=None,
137 blame_merge_author=False):
138 """Return a dict of information about a change to the branch.
140 Dict has keys of "files", "who", "comments", and "revision", as used by
141 the buildbot Change (and the PBChangeSource).
143 If only the branch is given, the most recent change is returned.
145 If only the new_revno is given, the comparison is expected to be between
146 it and the previous revno (new_revno -1) in the branch.
148 Passing old_revid and new_revid is only an optimization, included because
149 bzr hooks usually provide this information.
151 blame_merge_author means that the author of the merged branch is
152 identified as the "who", not the person who committed the branch itself.
153 This is typically used for PQM.
155 change = {} # files, who, comments, revision; NOT branch (= branch.nick)
156 if new_revno is None:
157 new_revno = branch.revno()
158 if new_revid is None:
159 new_revid = branch.get_rev_id(new_revno)
160 # TODO: This falls over if this is the very first revision
161 if old_revno is None:
162 old_revno = new_revno -1
163 if old_revid is None:
164 old_revid = branch.get_rev_id(old_revno)
165 repository = branch.repository
166 new_rev = repository.get_revision(new_revid)
167 if blame_merge_author:
168 # this is a pqm commit or something like it
169 change['who'] = repository.get_revision(
170 new_rev.parent_ids[-1]).get_apparent_author()
171 else:
172 change['who'] = new_rev.get_apparent_author()
173 # maybe useful to know:
174 # name, email = bzrtools.config.parse_username(change['who'])
175 change['comments'] = new_rev.message
176 change['revision'] = new_revno
177 files = change['files'] = []
178 changes = repository.revision_tree(new_revid).changes_from(
179 repository.revision_tree(old_revid))
180 for (collection, name) in ((changes.added, 'ADDED'),
181 (changes.removed, 'REMOVED'),
182 (changes.modified, 'MODIFIED')):
183 for info in collection:
184 path = info[0]
185 kind = info[2]
186 files.append(' '.join([path, kind, name]))
187 for info in changes.renamed:
188 oldpath, newpath, id, kind, text_modified, meta_modified = info
189 elements = [oldpath, kind,'RENAMED', newpath]
190 if text_modified or meta_modified:
191 elements.append('MODIFIED')
192 files.append(' '.join(elements))
193 return change
195 #############################################################################
196 # poller
198 # We don't want to make the hooks unnecessarily depend on buildbot being
199 # installed locally, so we conditionally create the BzrPoller class.
200 if DEFINE_POLLER:
202 FULL = object()
203 SHORT = object()
206 class BzrPoller(buildbot.changes.base.ChangeSource,
207 buildbot.util.ComparableMixin):
209 compare_attrs = ['url']
211 def __init__(self, url, poll_interval=10*60, blame_merge_author=False,
212 branch_name=None):
213 # poll_interval is in seconds, so default poll_interval is 10
214 # minutes.
215 # bzr+ssh://bazaar.launchpad.net/~launchpad-pqm/launchpad/devel/
216 # works, lp:~launchpad-pqm/launchpad/devel/ doesn't without help.
217 if url.startswith('lp:'):
218 url = 'bzr+ssh://bazaar.launchpad.net/' + url[3:]
219 self.url = url
220 self.poll_interval = poll_interval
221 self.loop = twisted.internet.task.LoopingCall(self.poll)
222 self.blame_merge_author = blame_merge_author
223 self.branch_name = branch_name
225 def startService(self):
226 twisted.python.log.msg("BzrPoller(%s) starting" % self.url)
227 buildbot.changes.base.ChangeSource.startService(self)
228 twisted.internet.reactor.callWhenRunning(
229 self.loop.start, self.poll_interval)
230 for change in reversed(self.parent.changes):
231 if change.branch == self.url:
232 self.last_revision = change.revision
233 break
234 else:
235 self.last_revision = None
236 self.polling = False
238 def stopService(self):
239 twisted.python.log.msg("BzrPoller(%s) shutting down" % self.url)
240 self.loop.stop()
241 return buildbot.changes.base.ChangeSource.stopService(self)
243 def describe(self):
244 return "BzrPoller watching %s" % self.url
246 @twisted.internet.defer.inlineCallbacks
247 def poll(self):
248 if self.polling: # this is called in a loop, and the loop might
249 # conceivably overlap.
250 return
251 self.polling = True
252 try:
253 # On a big tree, even individual elements of the bzr commands
254 # can take awhile. So we just push the bzr work off to a
255 # thread.
256 try:
257 changes = yield twisted.internet.threads.deferToThread(
258 self.getRawChanges)
259 except (SystemExit, KeyboardInterrupt):
260 raise
261 except:
262 # we'll try again next poll. Meanwhile, let's report.
263 twisted.python.log.err()
264 else:
265 for change in changes:
266 yield self.addChange(
267 buildbot.changes.changes.Change(**change))
268 self.last_revision = change['revision']
269 finally:
270 self.polling = False
272 def getRawChanges(self):
273 branch = bzrlib.branch.Branch.open_containing(self.url)[0]
274 if self.branch_name is FULL:
275 branch_name = self.url
276 elif self.branch_name is SHORT:
277 branch_name = branch.nick
278 else: # presumably a string or maybe None
279 branch_name = self.branch_name
280 changes = []
281 change = generate_change(
282 branch, blame_merge_author=self.blame_merge_author)
283 if (self.last_revision is None or
284 change['revision'] > self.last_revision):
285 changes.append(change)
286 if self.last_revision is not None:
287 while self.last_revision + 1 < change['revision']:
288 change = generate_change(
289 branch, new_revno=change['revision']-1,
290 blame_merge_author=self.blame_merge_author)
291 change['branch'] = branch_name
292 changes.append(change)
293 changes.reverse()
294 return changes
296 def addChange(self, change):
297 d = twisted.internet.defer.Deferred()
298 def _add_change():
299 d.callback(
300 self.parent.addChange(change))
301 twisted.internet.reactor.callLater(0, _add_change)
302 return d
304 #############################################################################
305 # hooks
307 HOOK_KEY = 'buildbot_on'
308 SERVER_KEY = 'buildbot_server'
309 PORT_KEY = 'buildbot_port'
310 DRYRUN_KEY = 'buildbot_dry_run'
311 PQM_KEY = 'buildbot_pqm'
312 SEND_BRANCHNAME_KEY = 'buildbot_send_branch_name'
314 PUSH_VALUE = 'push'
315 COMMIT_VALUE = 'commit'
316 CHANGE_VALUE = 'change'
318 def _is_true(config, key):
319 val = config.get_user_option(key)
320 return val is not None and val.lower().strip() in (
321 'y', 'yes', 't', 'true')
323 def _installed_hook(branch):
324 value = branch.get_config().get_user_option(HOOK_KEY)
325 if value is not None:
326 value = value.strip().lower()
327 if value not in (PUSH_VALUE, COMMIT_VALUE, CHANGE_VALUE):
328 raise bzrlib.errors.BzrError(
329 '%s, if set, must be one of %s, %s, or %s' % (
330 HOOK_KEY, PUSH_VALUE, COMMIT_VALUE, CHANGE_VALUE))
331 return value
333 ##########################
334 # Work around Twisted bug.
335 # See http://twistedmatrix.com/trac/ticket/3591
336 import operator
337 import socket
338 from twisted.internet import defer
339 from twisted.python import failure
341 # replaces twisted.internet.thread equivalent
342 def _putResultInDeferred(reactor, deferred, f, args, kwargs):
344 Run a function and give results to a Deferred.
346 try:
347 result = f(*args, **kwargs)
348 except:
349 f = failure.Failure()
350 reactor.callFromThread(deferred.errback, f)
351 else:
352 reactor.callFromThread(deferred.callback, result)
354 # would be a proposed addition. deferToThread could use it
355 def deferToThreadInReactor(reactor, f, *args, **kwargs):
357 Run function in thread and return result as Deferred.
359 d = defer.Deferred()
360 reactor.callInThread(_putResultInDeferred, reactor, d, f, args, kwargs)
361 return d
363 # uses its own reactor for the threaded calls, unlike Twisted's
364 class ThreadedResolver(twisted.internet.base.ThreadedResolver):
365 def getHostByName(self, name, timeout = (1, 3, 11, 45)):
366 if timeout:
367 timeoutDelay = reduce(operator.add, timeout)
368 else:
369 timeoutDelay = 60
370 userDeferred = defer.Deferred()
371 lookupDeferred = deferToThreadInReactor(
372 self.reactor, socket.gethostbyname, name)
373 cancelCall = self.reactor.callLater(
374 timeoutDelay, self._cleanup, name, lookupDeferred)
375 self._runningQueries[lookupDeferred] = (userDeferred, cancelCall)
376 lookupDeferred.addBoth(self._checkTimeout, name, lookupDeferred)
377 return userDeferred
378 ##########################
380 def send_change(branch, old_revno, old_revid, new_revno, new_revid, hook):
381 config = branch.get_config()
382 server = config.get_user_option(SERVER_KEY)
383 if not server:
384 bzrlib.trace.warning(
385 'bzr_buildbot: ERROR. If %s is set, %s must be set',
386 HOOK_KEY, SERVER_KEY)
387 return
388 change = generate_change(
389 branch, old_revno, old_revid, new_revno, new_revid,
390 blame_merge_author=_is_true(config, PQM_KEY))
391 if _is_true(config, SEND_BRANCHNAME_KEY):
392 change['branch'] = branch.nick
393 # as of this writing (in Buildbot 0.7.9), 9989 is the default port when
394 # you make a buildbot master.
395 port = int(config.get_user_option(PORT_KEY) or 9989)
396 # if dry run, stop.
397 if _is_true(config, DRYRUN_KEY):
398 bzrlib.trace.note("bzr_buildbot DRY RUN "
399 "(*not* sending changes to %s:%d on %s)",
400 server, port, hook)
401 keys = change.keys()
402 keys.sort()
403 for k in keys:
404 bzrlib.trace.note("[%10s]: %s", k, change[k])
405 return
406 # We instantiate our own reactor so that this can run within a server.
407 reactor = twisted.internet.selectreactor.SelectReactor()
408 # See other reference to http://twistedmatrix.com/trac/ticket/3591
409 # above. This line can go away with a release of Twisted that addresses
410 # this issue.
411 reactor.resolver = ThreadedResolver(reactor)
412 pbcf = twisted.spread.pb.PBClientFactory()
413 reactor.connectTCP(server, port, pbcf)
414 deferred = pbcf.login(
415 twisted.cred.credentials.UsernamePassword('change', 'changepw'))
417 def sendChanges(remote):
418 """Send changes to buildbot."""
419 bzrlib.trace.mutter("bzrbuildout sending changes: %s", change)
420 return remote.callRemote('addChange', change)
422 deferred.addCallback(sendChanges)
424 def quit(ignore, msg):
425 bzrlib.trace.note("bzrbuildout: %s", msg)
426 reactor.stop()
428 def failed(failure):
429 bzrlib.trace.warning("bzrbuildout: FAILURE\n %s", failure)
430 reactor.stop()
432 deferred.addCallback(quit, "SUCCESS")
433 deferred.addErrback(failed)
434 reactor.callLater(60, quit, None, "TIMEOUT")
435 bzrlib.trace.note(
436 "bzr_buildbot: SENDING CHANGES to buildbot master %s:%d on %s",
437 server, port, hook)
438 reactor.run(installSignalHandlers=False) # run in a thread when in server
440 def post_commit(local_branch, master_branch, # branch is the master_branch
441 old_revno, old_revid, new_revno, new_revid):
442 if _installed_hook(master_branch) == COMMIT_VALUE:
443 send_change(master_branch,
444 old_revid, old_revid, new_revno, new_revid, COMMIT_VALUE)
446 def post_push(result):
447 if _installed_hook(result.target_branch) == PUSH_VALUE:
448 send_change(result.target_branch,
449 result.old_revid, result.old_revid,
450 result.new_revno, result.new_revid, PUSH_VALUE)
452 def post_change_branch_tip(result):
453 if _installed_hook(result.branch) == CHANGE_VALUE:
454 send_change(result.branch,
455 result.old_revid, result.old_revid,
456 result.new_revno, result.new_revid, CHANGE_VALUE)
458 bzrlib.branch.Branch.hooks.install_named_hook(
459 'post_commit', post_commit,
460 'send change to buildbot master')
461 bzrlib.branch.Branch.hooks.install_named_hook(
462 'post_push', post_push,
463 'send change to buildbot master')
464 bzrlib.branch.Branch.hooks.install_named_hook(
465 'post_change_branch_tip', post_change_branch_tip,
466 'send change to buildbot master')