1 # Copyright (C) 2008-2009 Canonical
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU General Public License as published by
5 # the Free Software Foundation, either version 3 of the License, or
6 # (at your option) any later version.
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details.
13 # You should have received a copy of the GNU General Public License
14 # along with this program. If not, see <http://www.gnu.org/licenses/>.
17 bzr buildbot integration
18 ========================
20 This file contains both bzr commit/change hooks and a bzr poller.
26 This has been tested with buildbot 0.7.9, bzr 1.10, and Twisted 8.1.0. It
27 should work in subsequent releases.
29 For the hook to work, Twisted must be installed in the same Python that bzr
36 To install, put this file in a bzr plugins directory (e.g.,
37 ~/.bazaar/plugins). Then, in one of your bazaar conf files (e.g.,
38 ~/.bazaar/locations.conf), set the location you want to connect with buildbot
41 - buildbot_on: one of 'commit', 'push, or 'change'. Turns the plugin on to
42 report changes via commit, changes via push, or any changes to the trunk.
43 'change' is recommended.
45 - buildbot_server: (required to send to a buildbot master) the URL of the
46 buildbot master to which you will connect (as of this writing, the same
47 server and port to which slaves connect).
49 - buildbot_port: (optional, defaults to 9989) the port of the buildbot master
50 to which you will connect (as of this writing, the same server and port to
53 - buildbot_pqm: (optional, defaults to not pqm) Normally, the user that
54 commits the revision is the user that is responsible for the change. When
55 run in a pqm (Patch Queue Manager, see https://launchpad.net/pqm)
56 environment, the user that commits is the Patch Queue Manager, and the user
57 that committed the *parent* revision is responsible for the change. To turn
58 on the pqm mode, set this value to any of (case-insensitive) "Yes", "Y",
61 - buildbot_dry_run: (optional, defaults to not a dry run) Normally, the
62 post-commit hook will attempt to communicate with the configured buildbot
63 server and port. If this parameter is included and any of (case-insensitive)
64 "Yes", "Y", "True", or "T", then the hook will simply print what it would
65 have sent, but not attempt to contact the buildbot master.
67 - buildbot_send_branch_name: (optional, defaults to not sending the branch
68 name) If your buildbot's bzr source build step uses a repourl, do
69 *not* turn this on. If your buildbot's bzr build step uses a baseURL, then
70 you may set this value to any of (case-insensitive) "Yes", "Y", "True", or
71 "T" to have the buildbot master append the branch name to the baseURL.
73 When buildbot no longer has a hardcoded password, it will be a configuration
80 Put this file somewhere that your buildbot configuration can import it. Even
81 in the same directory as the master.cfg should work. Install the poller in
82 the buildbot configuration as with any other change source. Minimally,
83 provide a URL that you want to poll (bzr://, bzr+ssh://, or lp:), though make
84 sure the buildbot user has necessary privileges. You may also want to specify
85 these optional values.
87 poll_interval: the number of seconds to wait between polls. Defaults to 10
90 branch_name: any value to be used as the branch name. Defaults to None, or
91 specify a string, or specify the constants from this file SHORT
92 or FULL to get the short branch name or full branch address.
94 blame_merge_author: normally, the user that commits the revision is the user
95 that is responsible for the change. When run in a pqm
96 (Patch Queue Manager, see https://launchpad.net/pqm)
97 environment, the user that commits is the Patch Queue
98 Manager, and the user that committed the merged, *parent*
99 revision is responsible for the change. set this value to
100 True if this is pointed against a PQM-managed branch.
106 Maintainer/author: gary.poster@canonical.com
111 import buildbot
.changes
.base
112 import buildbot
.changes
.changes
114 DEFINE_POLLER
= False
120 import twisted
.cred
.credentials
121 import twisted
.internet
.base
122 import twisted
.internet
.defer
123 import twisted
.internet
.reactor
124 import twisted
.internet
.selectreactor
125 import twisted
.internet
.task
126 import twisted
.internet
.threads
127 import twisted
.python
.log
128 import twisted
.spread
.pb
131 #############################################################################
132 # This is the code that the poller and the hooks share.
134 def generate_change(branch
,
135 old_revno
=None, old_revid
=None,
136 new_revno
=None, new_revid
=None,
137 blame_merge_author
=False):
138 """Return a dict of information about a change to the branch.
140 Dict has keys of "files", "who", "comments", and "revision", as used by
141 the buildbot Change (and the PBChangeSource).
143 If only the branch is given, the most recent change is returned.
145 If only the new_revno is given, the comparison is expected to be between
146 it and the previous revno (new_revno -1) in the branch.
148 Passing old_revid and new_revid is only an optimization, included because
149 bzr hooks usually provide this information.
151 blame_merge_author means that the author of the merged branch is
152 identified as the "who", not the person who committed the branch itself.
153 This is typically used for PQM.
155 change
= {} # files, who, comments, revision; NOT branch (= branch.nick)
156 if new_revno
is None:
157 new_revno
= branch
.revno()
158 if new_revid
is None:
159 new_revid
= branch
.get_rev_id(new_revno
)
160 # TODO: This falls over if this is the very first revision
161 if old_revno
is None:
162 old_revno
= new_revno
-1
163 if old_revid
is None:
164 old_revid
= branch
.get_rev_id(old_revno
)
165 repository
= branch
.repository
166 new_rev
= repository
.get_revision(new_revid
)
167 if blame_merge_author
:
168 # this is a pqm commit or something like it
169 change
['who'] = repository
.get_revision(
170 new_rev
.parent_ids
[-1]).get_apparent_author()
172 change
['who'] = new_rev
.get_apparent_author()
173 # maybe useful to know:
174 # name, email = bzrtools.config.parse_username(change['who'])
175 change
['comments'] = new_rev
.message
176 change
['revision'] = new_revno
177 files
= change
['files'] = []
178 changes
= repository
.revision_tree(new_revid
).changes_from(
179 repository
.revision_tree(old_revid
))
180 for (collection
, name
) in ((changes
.added
, 'ADDED'),
181 (changes
.removed
, 'REMOVED'),
182 (changes
.modified
, 'MODIFIED')):
183 for info
in collection
:
186 files
.append(' '.join([path
, kind
, name
]))
187 for info
in changes
.renamed
:
188 oldpath
, newpath
, id, kind
, text_modified
, meta_modified
= info
189 elements
= [oldpath
, kind
,'RENAMED', newpath
]
190 if text_modified
or meta_modified
:
191 elements
.append('MODIFIED')
192 files
.append(' '.join(elements
))
195 #############################################################################
198 # We don't want to make the hooks unnecessarily depend on buildbot being
199 # installed locally, so we conditionally create the BzrPoller class.
206 class BzrPoller(buildbot
.changes
.base
.ChangeSource
,
207 buildbot
.util
.ComparableMixin
):
209 compare_attrs
= ['url']
211 def __init__(self
, url
, poll_interval
=10*60, blame_merge_author
=False,
213 # poll_interval is in seconds, so default poll_interval is 10
215 # bzr+ssh://bazaar.launchpad.net/~launchpad-pqm/launchpad/devel/
216 # works, lp:~launchpad-pqm/launchpad/devel/ doesn't without help.
217 if url
.startswith('lp:'):
218 url
= 'bzr+ssh://bazaar.launchpad.net/' + url
[3:]
220 self
.poll_interval
= poll_interval
221 self
.loop
= twisted
.internet
.task
.LoopingCall(self
.poll
)
222 self
.blame_merge_author
= blame_merge_author
223 self
.branch_name
= branch_name
225 def startService(self
):
226 twisted
.python
.log
.msg("BzrPoller(%s) starting" % self
.url
)
227 buildbot
.changes
.base
.ChangeSource
.startService(self
)
228 twisted
.internet
.reactor
.callWhenRunning(
229 self
.loop
.start
, self
.poll_interval
)
230 for change
in reversed(self
.parent
.changes
):
231 if change
.branch
== self
.url
:
232 self
.last_revision
= change
.revision
235 self
.last_revision
= None
238 def stopService(self
):
239 twisted
.python
.log
.msg("BzrPoller(%s) shutting down" % self
.url
)
241 return buildbot
.changes
.base
.ChangeSource
.stopService(self
)
244 return "BzrPoller watching %s" % self
.url
246 @twisted.internet
.defer
.inlineCallbacks
248 if self
.polling
: # this is called in a loop, and the loop might
249 # conceivably overlap.
253 # On a big tree, even individual elements of the bzr commands
254 # can take awhile. So we just push the bzr work off to a
257 changes
= yield twisted
.internet
.threads
.deferToThread(
259 except (SystemExit, KeyboardInterrupt):
262 # we'll try again next poll. Meanwhile, let's report.
263 twisted
.python
.log
.err()
265 for change
in changes
:
266 yield self
.addChange(
267 buildbot
.changes
.changes
.Change(**change
))
268 self
.last_revision
= change
['revision']
272 def getRawChanges(self
):
273 branch
= bzrlib
.branch
.Branch
.open_containing(self
.url
)[0]
274 if self
.branch_name
is FULL
:
275 branch_name
= self
.url
276 elif self
.branch_name
is SHORT
:
277 branch_name
= branch
.nick
278 else: # presumably a string or maybe None
279 branch_name
= self
.branch_name
281 change
= generate_change(
282 branch
, blame_merge_author
=self
.blame_merge_author
)
283 if (self
.last_revision
is None or
284 change
['revision'] > self
.last_revision
):
285 change
['branch'] = branch_name
286 changes
.append(change
)
287 if self
.last_revision
is not None:
288 while self
.last_revision
+ 1 < change
['revision']:
289 change
= generate_change(
290 branch
, new_revno
=change
['revision']-1,
291 blame_merge_author
=self
.blame_merge_author
)
292 change
['branch'] = branch_name
293 changes
.append(change
)
297 def addChange(self
, change
):
298 d
= twisted
.internet
.defer
.Deferred()
301 self
.parent
.addChange(change
))
302 twisted
.internet
.reactor
.callLater(0, _add_change
)
305 #############################################################################
308 HOOK_KEY
= 'buildbot_on'
309 SERVER_KEY
= 'buildbot_server'
310 PORT_KEY
= 'buildbot_port'
311 DRYRUN_KEY
= 'buildbot_dry_run'
312 PQM_KEY
= 'buildbot_pqm'
313 SEND_BRANCHNAME_KEY
= 'buildbot_send_branch_name'
316 COMMIT_VALUE
= 'commit'
317 CHANGE_VALUE
= 'change'
319 def _is_true(config
, key
):
320 val
= config
.get_user_option(key
)
321 return val
is not None and val
.lower().strip() in (
322 'y', 'yes', 't', 'true')
324 def _installed_hook(branch
):
325 value
= branch
.get_config().get_user_option(HOOK_KEY
)
326 if value
is not None:
327 value
= value
.strip().lower()
328 if value
not in (PUSH_VALUE
, COMMIT_VALUE
, CHANGE_VALUE
):
329 raise bzrlib
.errors
.BzrError(
330 '%s, if set, must be one of %s, %s, or %s' % (
331 HOOK_KEY
, PUSH_VALUE
, COMMIT_VALUE
, CHANGE_VALUE
))
334 ##########################
335 # Work around Twisted bug.
336 # See http://twistedmatrix.com/trac/ticket/3591
339 from twisted
.internet
import defer
340 from twisted
.python
import failure
342 # replaces twisted.internet.thread equivalent
343 def _putResultInDeferred(reactor
, deferred
, f
, args
, kwargs
):
345 Run a function and give results to a Deferred.
348 result
= f(*args
, **kwargs
)
350 f
= failure
.Failure()
351 reactor
.callFromThread(deferred
.errback
, f
)
353 reactor
.callFromThread(deferred
.callback
, result
)
355 # would be a proposed addition. deferToThread could use it
356 def deferToThreadInReactor(reactor
, f
, *args
, **kwargs
):
358 Run function in thread and return result as Deferred.
361 reactor
.callInThread(_putResultInDeferred
, reactor
, d
, f
, args
, kwargs
)
364 # uses its own reactor for the threaded calls, unlike Twisted's
365 class ThreadedResolver(twisted
.internet
.base
.ThreadedResolver
):
366 def getHostByName(self
, name
, timeout
= (1, 3, 11, 45)):
368 timeoutDelay
= reduce(operator
.add
, timeout
)
371 userDeferred
= defer
.Deferred()
372 lookupDeferred
= deferToThreadInReactor(
373 self
.reactor
, socket
.gethostbyname
, name
)
374 cancelCall
= self
.reactor
.callLater(
375 timeoutDelay
, self
._cleanup
, name
, lookupDeferred
)
376 self
._runningQueries
[lookupDeferred
] = (userDeferred
, cancelCall
)
377 lookupDeferred
.addBoth(self
._checkTimeout
, name
, lookupDeferred
)
379 ##########################
381 def send_change(branch
, old_revno
, old_revid
, new_revno
, new_revid
, hook
):
382 config
= branch
.get_config()
383 server
= config
.get_user_option(SERVER_KEY
)
385 bzrlib
.trace
.warning(
386 'bzr_buildbot: ERROR. If %s is set, %s must be set',
387 HOOK_KEY
, SERVER_KEY
)
389 change
= generate_change(
390 branch
, old_revno
, old_revid
, new_revno
, new_revid
,
391 blame_merge_author
=_is_true(config
, PQM_KEY
))
392 if _is_true(config
, SEND_BRANCHNAME_KEY
):
393 change
['branch'] = branch
.nick
394 # as of this writing (in Buildbot 0.7.9), 9989 is the default port when
395 # you make a buildbot master.
396 port
= int(config
.get_user_option(PORT_KEY
) or 9989)
398 if _is_true(config
, DRYRUN_KEY
):
399 bzrlib
.trace
.note("bzr_buildbot DRY RUN "
400 "(*not* sending changes to %s:%d on %s)",
405 bzrlib
.trace
.note("[%10s]: %s", k
, change
[k
])
407 # We instantiate our own reactor so that this can run within a server.
408 reactor
= twisted
.internet
.selectreactor
.SelectReactor()
409 # See other reference to http://twistedmatrix.com/trac/ticket/3591
410 # above. This line can go away with a release of Twisted that addresses
412 reactor
.resolver
= ThreadedResolver(reactor
)
413 pbcf
= twisted
.spread
.pb
.PBClientFactory()
414 reactor
.connectTCP(server
, port
, pbcf
)
415 deferred
= pbcf
.login(
416 twisted
.cred
.credentials
.UsernamePassword('change', 'changepw'))
418 def sendChanges(remote
):
419 """Send changes to buildbot."""
420 bzrlib
.trace
.mutter("bzrbuildout sending changes: %s", change
)
421 return remote
.callRemote('addChange', change
)
423 deferred
.addCallback(sendChanges
)
425 def quit(ignore
, msg
):
426 bzrlib
.trace
.note("bzrbuildout: %s", msg
)
430 bzrlib
.trace
.warning("bzrbuildout: FAILURE\n %s", failure
)
433 deferred
.addCallback(quit
, "SUCCESS")
434 deferred
.addErrback(failed
)
435 reactor
.callLater(60, quit
, None, "TIMEOUT")
437 "bzr_buildbot: SENDING CHANGES to buildbot master %s:%d on %s",
439 reactor
.run(installSignalHandlers
=False) # run in a thread when in server
441 def post_commit(local_branch
, master_branch
, # branch is the master_branch
442 old_revno
, old_revid
, new_revno
, new_revid
):
443 if _installed_hook(master_branch
) == COMMIT_VALUE
:
444 send_change(master_branch
,
445 old_revid
, old_revid
, new_revno
, new_revid
, COMMIT_VALUE
)
447 def post_push(result
):
448 if _installed_hook(result
.target_branch
) == PUSH_VALUE
:
449 send_change(result
.target_branch
,
450 result
.old_revid
, result
.old_revid
,
451 result
.new_revno
, result
.new_revid
, PUSH_VALUE
)
453 def post_change_branch_tip(result
):
454 if _installed_hook(result
.branch
) == CHANGE_VALUE
:
455 send_change(result
.branch
,
456 result
.old_revid
, result
.old_revid
,
457 result
.new_revno
, result
.new_revid
, CHANGE_VALUE
)
459 bzrlib
.branch
.Branch
.hooks
.install_named_hook(
460 'post_commit', post_commit
,
461 'send change to buildbot master')
462 bzrlib
.branch
.Branch
.hooks
.install_named_hook(
463 'post_push', post_push
,
464 'send change to buildbot master')
465 bzrlib
.branch
.Branch
.hooks
.install_named_hook(
466 'post_change_branch_tip', post_change_branch_tip
,
467 'send change to buildbot master')