From b401c4f3128a463628a52c77dcd3a359554734b9 Mon Sep 17 00:00:00 2001 From: warner Date: Mon, 12 Jun 2006 12:01:00 +0100 Subject: [PATCH] applied patch SF#1473939, initial Perforce support --- buildbot/changes/p4poller.py | 144 +++++++++++++++++------- buildbot/process/step.py | 69 ++++++++++-- buildbot/slave/commands.py | 131 +++++++++++++++++++++- buildbot/test/test_p4poller.py | 208 ++++++++++++++++++++++++++++++++++ buildbot/test/test_vc.py | 246 +++++++++++++++++++++++++++++++++++++++-- docs/buildbot.texinfo | 106 +++++++++++++++--- 6 files changed, 826 insertions(+), 78 deletions(-) create mode 100644 buildbot/test/test_p4poller.py diff --git a/buildbot/changes/p4poller.py b/buildbot/changes/p4poller.py index d14e57c..1c7a00d 100644 --- a/buildbot/changes/p4poller.py +++ b/buildbot/changes/p4poller.py @@ -1,28 +1,51 @@ -#! /usr/bin/python +# -*- test-case-name: buildbot.test.test_p4poller -*- # Many thanks to Dave Peticolas for contributing this module -from twisted.internet import defer +import re +import time + +from twisted.python import log, failure +from twisted.internet import defer, reactor from twisted.internet.utils import getProcessOutput from twisted.internet.task import LoopingCall from buildbot import util from buildbot.changes import base, changes +def get_simple_split(branchfile): + """Splits the branchfile argument and assuming branch is + the first path component in branchfile, will return + branch and file else None.""" + + index = branchfile.find('/') + if index == -1: return None, None + branch, file = branchfile.split('/', 1) + return branch, file + class P4Source(base.ChangeSource, util.ComparableMixin): """This source will poll a perforce repository for changes and submit them to the change master.""" - compare_attrs = ["p4port", "p4user", "p4passwd", "p4client", "p4base", + compare_attrs = ["p4port", "p4user", "p4passwd", "p4base", "p4bin", "pollinterval", "histmax"] + changes_line_re = re.compile( + r"Change (?P\d+) on \S+ by \S+@\S+ '.+'$") + describe_header_re = re.compile( + r"Change \d+ by (?P\S+)@\S+ on (?P.+)$") + file_re = re.compile(r"^\.\.\. (?P[^#]+)#\d+ \w+$") + datefmt = '%Y/%m/%d %H:%M:%S' + parent = None # filled in when we're added last_change = None loop = None volatile = ['loop'] + working = False - def __init__(self, p4port, p4user, p4passwd=None, p4client=None, - p4base='//...', p4bin='p4', + def __init__(self, p4port=None, p4user=None, p4passwd=None, + p4base='//', p4bin='p4', + split_file=lambda branchfile: (None, branchfile), pollinterval=60 * 10, histmax=100): """ @type p4port: string @@ -31,13 +54,13 @@ class P4Source(base.ChangeSource, util.ComparableMixin): @param p4user: p4 user @type p4passwd: string @param p4passwd: p4 passwd - @type p4client: string - @param p4client: name of p4 client to poll @type p4base: string @param p4base: p4 file specification to limit a poll to - (i.e., //...) + without the trailing '...' (i.e., //) @type p4bin: string @param p4bin: path to p4 binary, defaults to just 'p4' + @type split_file: func + $param split_file: splits a filename into branch and filename. @type pollinterval: int @param pollinterval: interval in seconds between polls @type histmax: int @@ -47,28 +70,49 @@ class P4Source(base.ChangeSource, util.ComparableMixin): self.p4port = p4port self.p4user = p4user self.p4passwd = p4passwd - self.p4client = p4client self.p4base = p4base self.p4bin = p4bin + self.split_file = split_file self.pollinterval = pollinterval self.histmax = histmax def startService(self): self.loop = LoopingCall(self.checkp4) - self.loop.start(self.pollinterval) base.ChangeSource.startService(self) + # Don't start the loop just yet because the reactor isn't running. + # Give it a chance to go and install our SIGCHLD handler before + # spawning processes. + reactor.callLater(0, self.loop.start, self.pollinterval) + def stopService(self): self.loop.stop() return base.ChangeSource.stopService(self) def describe(self): - return "p4source %s-%s %s" % (self.p4port, self.p4client, self.p4base) + return "p4source %s %s" % (self.p4port, self.p4base) def checkp4(self): - d = self._get_changes() - d.addCallback(self._process_changes) - d.addCallback(self._handle_changes) + # Our return value is only used for unit testing. + if self.working: + log.msg("Skipping checkp4 because last one has not finished") + return defer.succeed(None) + else: + self.working = True + d = self._get_changes() + d.addCallback(self._process_changes) + d.addBoth(self._finished) + return d + + def _finished(self, res): + assert self.working + self.working = False + + # Again, the return value is only for unit testing. + # If there's a failure, log it so it isn't lost. + if isinstance(res, failure.Failure): + log.msg('P4 poll failed: %s' % res) + return res def _get_changes(self): args = [] @@ -78,9 +122,7 @@ class P4Source(base.ChangeSource, util.ComparableMixin): args.extend(['-u', self.p4user]) if self.p4passwd: args.extend(['-P', self.p4passwd]) - if self.p4client: - args.extend(['-c', self.p4client]) - args.extend(['changes', '-m', str(self.histmax), self.p4base]) + args.extend(['changes', '-m', str(self.histmax), self.p4base + '...']) env = {} return getProcessOutput(self.p4bin, args, env) @@ -90,18 +132,26 @@ class P4Source(base.ChangeSource, util.ComparableMixin): for line in result.split('\n'): line = line.strip() if not line: continue - _, num, _, date, _, user, _ = line.split(' ', 6) + m = self.changes_line_re.match(line) + assert m, "Unexpected 'p4 changes' output: %r" % result + num = m.group('num') if last_change is None: + log.msg('P4Poller: starting at change %s' % num) self.last_change = num return [] - if last_change == num: break - change = {'num' : num, 'date' : date, 'user' : user.split('@')[0]} - changelists.append(change) + if last_change == num: + break + changelists.append(num) changelists.reverse() # oldest first - ds = [self._get_change(c) for c in changelists] - return defer.DeferredList(ds) - def _get_change(self, change): + # Retrieve each sequentially. + d = defer.succeed(None) + for c in changelists: + d.addCallback(self._get_describe, c) + d.addCallback(self._process_describe, c) + return d + + def _get_describe(self, dummy, num): args = [] if self.p4port: args.extend(['-p', self.p4port]) @@ -109,34 +159,44 @@ class P4Source(base.ChangeSource, util.ComparableMixin): args.extend(['-u', self.p4user]) if self.p4passwd: args.extend(['-P', self.p4passwd]) - if self.p4client: - args.extend(['-c', self.p4client]) - args.extend(['describe', '-s', change['num']]) + args.extend(['describe', '-s', num]) env = {} d = getProcessOutput(self.p4bin, args, env) - d.addCallback(self._process_change, change) return d - def _process_change(self, result, change): + def _process_describe(self, result, num): lines = result.split('\n') + m = self.describe_header_re.match(lines[0]) + assert m, "Unexpected 'p4 describe -s' result: %r" % result + who = m.group('who') + when = time.mktime(time.strptime(m.group('when'), self.datefmt)) comments = '' while not lines[0].startswith('Affected files'): comments += lines.pop(0) + '\n' - change['comments'] = comments lines.pop(0) # affected files - files = [] + + branch_files = {} # dict for branch mapped to file(s) while lines: line = lines.pop(0).strip() if not line: continue - files.append(line.split(' ')[1]) - change['files'] = files - return change - - def _handle_changes(self, result): - for success, change in result: - if not success: continue - c = changes.Change(change['user'], change['files'], - change['comments'], - revision=change['num']) + m = self.file_re.match(line) + assert m, "Invalid file line: %r" % line + path = m.group('path') + if path.startswith(self.p4base): + branch, file = self.split_file(path[len(self.p4base):]) + if (branch == None and file == None): continue + if branch_files.has_key(branch): + branch_files[branch].append(file) + else: + branch_files[branch] = [file] + + for branch in branch_files: + c = changes.Change(who=who, + files=branch_files[branch], + comments=comments, + revision=num, + when=when, + branch=branch) self.parent.addChange(c) - self.last_change = change['num'] + + self.last_change = num diff --git a/buildbot/process/step.py b/buildbot/process/step.py index 9704de7..0231ef8 100644 --- a/buildbot/process/step.py +++ b/buildbot/process/step.py @@ -1822,24 +1822,69 @@ class Mercurial(Source): self.startCommand(cmd) -class todo_P4(Source): +class P4(Source): + """ P4 is a class for accessing perforce revision control""" name = "p4" - # to create the working directory for the first time: - # need to create a View. The 'Root' parameter will have to be filled - # in by the buildslave with the abspath of the basedir. Then the - # setup process involves 'p4 client' to set up the view. After - # that, 'p4 sync' does all the necessary updating. - # P4PORT=P4PORT P4CLIENT=name p4 client + def __init__(self, p4base, defaultBranch=None, p4port=None, p4user=None, + p4passwd=None, p4extra_views=[], + p4client='buildbot_%(slave)s_%(builder)s', **kwargs): + """ + @type p4base: string + @param p4base: A view into a perforce depot, typically + "//depot/proj/" + + @type defaultBranch: string + @param defaultBranch: Identify a branch to build by default. Perforce + is a view based branching system. So, the branch + is normally the name after the base. For example, + branch=1.0 is view=//depot/proj/1.0/... + branch=1.1 is view=//depot/proj/1.1/... + + @type p4port: string + @param p4port: Specify the perforce server to connection in the format + :. Example "perforce.example.com:1666" + + @type p4user: string + @param p4user: The perforce user to run the command as. + + @type p4passwd: string + @param p4passwd: The password for the perforce user. - def __init__(self, p4port, view, **kwargs): + @type p4extra_views: list of tuples + @param p4extra_views: Extra views to be added to + the client that is being used. + + @type p4client: string + @param p4client: The perforce client to use for this buildslave. + """ + + self.branch = defaultBranch Source.__init__(self, **kwargs) - self.args.update({'p4port': p4port, - 'view': view, - }) + self.args['p4port'] = p4port + self.args['p4user'] = p4user + self.args['p4passwd'] = p4passwd + self.args['p4base'] = p4base + self.args['p4extra_views'] = p4extra_views + self.args['p4client'] = p4client % { + 'slave': self.build.slavename, + 'builder': self.build.builder.name, + } + + def computeSourceRevision(self, changes): + if not changes: + return None + lastChange = max([int(c.revision) for c in changes]) + return lastChange def startVC(self, branch, revision, patch): - cmd = LoggedRemoteCommand("p4", self.args) + slavever = self.slaveVersion("p4") + assert slavever, "slave is too old, does not know about p4" + args = dict(self.args) + args['branch'] = branch or self.branch + args['revision'] = revision + args['patch'] = patch + cmd = LoggedRemoteCommand("p4", args) self.startCommand(cmd) class P4Sync(Source): diff --git a/buildbot/slave/commands.py b/buildbot/slave/commands.py index a8df324..61fe90e 100644 --- a/buildbot/slave/commands.py +++ b/buildbot/slave/commands.py @@ -1439,6 +1439,128 @@ class Mercurial(SourceBase): registerSlaveCommand("hg", Mercurial, cvs_ver) +class P4(SourceBase): + """A P4 source-updater. + + ['p4port'] (required): host:port for server to access + ['p4user'] (optional): user to use for access + ['p4passwd'] (optional): passwd to try for the user + ['p4client'] (optional): client spec to use + ['p4views'] (optional): client views to use + """ + + header = "p4" + + def setup(self, args): + SourceBase.setup(self, args) + self.p4port = args['p4port'] + self.p4client = args['p4client'] + self.p4user = args['p4user'] + self.p4passwd = args['p4passwd'] + self.p4base = args['p4base'] + self.p4extra_views = args['p4extra_views'] + self.p4mode = args['mode'] + self.p4branch = args['branch'] + self.p4logname = os.environ['LOGNAME'] + + self.sourcedata = str([ + # Perforce server. + self.p4port, + + # Client spec. + self.p4client, + + # Depot side of view spec. + self.p4base, + self.p4branch, + self.p4extra_views, + + # Local side of view spec (srcdir is made from these). + self.builder.basedir, + self.mode, + self.workdir + ]) + + + def sourcedirIsUpdateable(self): + if os.path.exists(os.path.join(self.builder.basedir, + self.srcdir, ".buildbot-patched")): + return False + # We assume our client spec is still around. + # We just say we aren't updateable if the dir doesn't exist so we + # don't get ENOENT checking the sourcedata. + return os.path.isdir(os.path.join(self.builder.basedir, + self.srcdir)) + + def doVCUpdate(self): + return self._doP4Sync(force=False) + + def _doP4Sync(self, force): + command = ['p4'] + + if self.p4port: + command.extend(['-p', self.p4port]) + if self.p4user: + command.extend(['-u', self.p4user]) + if self.p4passwd: + command.extend(['-P', self.p4passwd]) + if self.p4client: + command.extend(['-c', self.p4client]) + command.extend(['sync']) + if force: + command.extend(['-f']) + if self.revision: + command.extend(['@' + str(self.revision)]) + env = {} + c = ShellCommand(self.builder, command, self.builder.basedir, + environ=env, sendRC=False, timeout=self.timeout, + keepStdout=True) + self.command = c + d = c.start() + d.addCallback(self._abandonOnFailure) + return d + + + def doVCFull(self): + env = {} + command = ['p4'] + client_spec = '' + client_spec += "Client: %s\n\n" % self.p4client + client_spec += "Owner: %s\n\n" % self.p4logname + client_spec += "Description:\n\tCreated by %s\n\n" % self.p4logname + client_spec += "Root:\t%s\n\n" % self.builder.basedir + client_spec += "Options:\tallwrite rmdir\n\n" + client_spec += "LineEnd:\tlocal\n\n" + + # Setup a view + client_spec += "View:\n\t%s" % (self.p4base) + if self.p4branch: + client_spec += "%s/" % (self.p4branch) + client_spec += "... //%s/%s/...\n" % (self.p4client, self.srcdir) + if self.p4extra_views: + for k, v in self.p4extra_views: + client_spec += "\t%s/... //%s/%s%s/...\n" % (k, self.p4client, + self.srcdir, v) + if self.p4port: + command.extend(['-p', self.p4port]) + if self.p4user: + command.extend(['-u', self.p4user]) + if self.p4passwd: + command.extend(['-P', self.p4passwd]) + command.extend(['client', '-i']) + log.msg(client_spec) + c = ShellCommand(self.builder, command, self.builder.basedir, + environ=env, sendRC=False, timeout=self.timeout, + stdin=client_spec) + self.command = c + d = c.start() + d.addCallback(self._abandonOnFailure) + d.addCallback(lambda _: self._doP4Sync(force=True)) + return d + +registerSlaveCommand("p4", P4, cvs_ver) + + class P4Sync(SourceBase): """A partial P4 source-updater. Requires manual setup of a per-slave P4 environment. The only thing which comes from the master is P4PORT. @@ -1463,7 +1585,7 @@ class P4Sync(SourceBase): def sourcedirIsUpdateable(self): return True - def doVCUpdate(self): + def _doVC(self, force): d = os.path.join(self.builder.basedir, self.srcdir) command = [self.vcexe] if self.p4port: @@ -1475,6 +1597,8 @@ class P4Sync(SourceBase): if self.p4client: command.extend(['-c', self.p4client]) command.extend(['sync']) + if force: + command.extend(['-f']) if self.revision: command.extend(['@' + self.revision]) env = {} @@ -1483,7 +1607,10 @@ class P4Sync(SourceBase): self.command = c return c.start() + def doVCUpdate(self): + return self._doVC(force=False) + def doVCFull(self): - return self.doVCUpdate() + return self._doVC(force=True) registerSlaveCommand("p4sync", P4Sync, cvs_ver) diff --git a/buildbot/test/test_p4poller.py b/buildbot/test/test_p4poller.py new file mode 100644 index 0000000..83b9582 --- /dev/null +++ b/buildbot/test/test_p4poller.py @@ -0,0 +1,208 @@ +import sys +import time + +from twisted.python import log, failure +from twisted.internet import defer +from twisted.trial import unittest + +from buildbot.twcompat import maybeWait +from buildbot.changes.changes import Change +from buildbot.changes.p4poller import P4Source, get_simple_split + +#log.startLogging(sys.stderr) + +first_p4changes = \ +"""Change 1 on 2006/04/13 by slamb@testclient 'first rev' +""" + +second_p4changes = \ +"""Change 3 on 2006/04/13 by bob@testclient 'short desc truncated' +Change 2 on 2006/04/13 by slamb@testclient 'bar' +""" + +third_p4changes = \ +"""Change 5 on 2006/04/13 by mpatel@testclient 'first rev' +""" + +change_4_log = \ +"""Change 4 by mpatel@testclient on 2006/04/13 21:55:39 + + short desc truncated because this is a long description. +""" +change_3_log = \ +"""Change 3 by bob@testclient on 2006/04/13 21:51:39 + + short desc truncated because this is a long description. +""" + +change_2_log = \ +"""Change 2 by slamb@testclient on 2006/04/13 21:46:23 + + creation +""" + +p4change = { + '3': change_3_log + +"""Affected files ... + +... //depot/myproject/branch_b/branch_b_file#1 add +... //depot/myproject/branch_b/whatbranch#1 branch +... //depot/myproject/branch_c/whatbranch#1 branch +""", + '2': change_2_log + +"""Affected files ... + +... //depot/myproject/trunk/whatbranch#1 add +... //depot/otherproject/trunk/something#1 add +""", + '5': change_4_log + +"""Affected files ... + +... //depot/myproject/branch_b/branch_b_file#1 add +... //depot/myproject/branch_b#75 edit +... //depot/myproject/branch_c/branch_c_file#1 add +""", +} + + +class MockP4Source(P4Source): + """Test P4Source which doesn't actually invoke p4.""" + invocation = 0 + + def __init__(self, p4changes, p4change, *args, **kwargs): + P4Source.__init__(self, *args, **kwargs) + self.p4changes = p4changes + self.p4change = p4change + + def _get_changes(self): + assert self.working + result = self.p4changes[self.invocation] + self.invocation += 1 + return defer.succeed(result) + + def _get_describe(self, dummy, num): + assert self.working + return defer.succeed(self.p4change[num]) + +class TestP4Poller(unittest.TestCase): + def setUp(self): + self.changes = [] + self.addChange = self.changes.append + + def testCheck(self): + """successful checks""" + self.t = MockP4Source(p4changes=[first_p4changes, second_p4changes], + p4change=p4change, + p4port=None, p4user=None, + p4base='//depot/myproject/', + split_file=lambda x: x.split('/', 1)) + self.t.parent = self + + # The first time, it just learns the change to start at. + self.assert_(self.t.last_change is None) + self.assert_(not self.t.working) + return maybeWait(self.t.checkp4().addCallback(self._testCheck2)) + + def _testCheck2(self, res): + self.assertEquals(self.changes, []) + self.assertEquals(self.t.last_change, '1') + + # Subsequent times, it returns Change objects for new changes. + return self.t.checkp4().addCallback(self._testCheck3) + + def _testCheck3(self, res): + self.assertEquals(len(self.changes), 3) + self.assertEquals(self.t.last_change, '3') + self.assert_(not self.t.working) + + # They're supposed to go oldest to newest, so this one must be first. + self.assertEquals(self.changes[0].asText(), + Change(who='slamb', + files=['whatbranch'], + comments=change_2_log, + revision='2', + when=time.mktime((2006, 4, 13, 21, 46, 23, 3, 103, -1)), + branch='trunk').asText()) + + # These two can happen in either order, since they're from the same + # Perforce change. + self.assertIn( + Change(who='bob', + files=['branch_b_file', + 'whatbranch'], + comments=change_3_log, + revision='3', + when=time.mktime((2006, 4, 13, 21, 51, 39, 3, 103, -1)), + branch='branch_b').asText(), + [c.asText() for c in self.changes]) + self.assertIn( + Change(who='bob', + files=['whatbranch'], + comments=change_3_log, + revision='3', + when=time.mktime((2006, 4, 13, 21, 51, 39, 3, 103, -1)), + branch='branch_c').asText(), + [c.asText() for c in self.changes]) + + def testFailedChanges(self): + """'p4 changes' failure is properly reported""" + self.t = MockP4Source(p4changes=['Perforce client error:\n...'], + p4change={}, + p4port=None, p4user=None) + self.t.parent = self + d = self.t.checkp4() + d.addBoth(self._testFailedChanges2) + return maybeWait(d) + + def _testFailedChanges2(self, f): + self.assert_(isinstance(f, failure.Failure)) + self.assertIn('Perforce client error', str(f)) + self.assert_(not self.t.working) + + def testFailedDescribe(self): + """'p4 describe' failure is properly reported""" + c = dict(p4change) + c['3'] = 'Perforce client error:\n...' + self.t = MockP4Source(p4changes=[first_p4changes, second_p4changes], + p4change=c, p4port=None, p4user=None) + self.t.parent = self + d = self.t.checkp4() + d.addCallback(self._testFailedDescribe2) + return maybeWait(d) + + def _testFailedDescribe2(self, res): + # first time finds nothing; check again. + return self.t.checkp4().addBoth(self._testFailedDescribe3) + + def _testFailedDescribe3(self, f): + self.assert_(isinstance(f, failure.Failure)) + self.assertIn('Perforce client error', str(f)) + self.assert_(not self.t.working) + self.assertEquals(self.t.last_change, '2') + + def testAlreadyWorking(self): + """don't launch a new poll while old is still going""" + self.t = P4Source() + self.t.working = True + self.assert_(self.t.last_change is None) + d = self.t.checkp4() + d.addCallback(self._testAlreadyWorking2) + + def _testAlreadyWorking2(self, res): + self.assert_(self.t.last_change is None) + + def testSplitFile(self): + """Make sure split file works on branch only changes""" + self.t = MockP4Source(p4changes=[third_p4changes], + p4change=p4change, + p4port=None, p4user=None, + p4base='//depot/myproject/', + split_file=get_simple_split) + self.t.parent = self + self.t.last_change = 50 + d = self.t.checkp4() + d.addCallback(self._testSplitFile) + + def _testSplitFile(self, res): + self.assertEquals(len(self.changes), 2) + self.assertEquals(self.t.last_change, '5') diff --git a/buildbot/test/test_vc.py b/buildbot/test/test_vc.py index f65e755..d4b9f02 100644 --- a/buildbot/test/test_vc.py +++ b/buildbot/test/test_vc.py @@ -6,7 +6,35 @@ import sys, os, signal, shutil, time, re from email.Utils import mktime_tz, parsedate_tz from twisted.trial import unittest -from twisted.internet import defer, reactor, utils +from twisted.internet import defer, reactor, utils, protocol, error +try: + from twisted.python.procutils import which +except ImportError: + # copied from Twisted circa 2.2.0 + def which(name, flags=os.X_OK): + """Search PATH for executable files with the given name. + + @type name: C{str} + @param name: The name for which to search. + + @type flags: C{int} + @param flags: Arguments to L{os.access}. + + @rtype: C{list} + @param: A list of the full paths to files found, in the + order in which they were found. + """ + result = [] + exts = filter(None, os.environ.get('PATHEXT', '').split(os.pathsep)) + for p in os.environ['PATH'].split(os.pathsep): + p = os.path.join(p, name) + if os.access(p, flags): + result.append(p) + for e in exts: + pext = p + e + if os.access(pext, flags): + result.append(pext) + return result #defer.Deferred.debug = True @@ -47,6 +75,49 @@ from twisted.internet.defer import waitForDeferred, deferredGenerator # small web server to provide access to the repository files while the test # is running). +# Perforce starts the daemon running on localhost. Unfortunately, it must +# use a predetermined Internet-domain port number, unless we want to go +# all-out: bind the listen socket ourselves and pretend to be inetd. + +try: + import cStringIO as StringIO +except ImportError: + import StringIO + +class _PutEverythingGetter(protocol.ProcessProtocol): + def __init__(self, deferred, stdin): + self.deferred = deferred + self.outBuf = StringIO.StringIO() + self.errBuf = StringIO.StringIO() + self.outReceived = self.outBuf.write + self.errReceived = self.errBuf.write + self.stdin = stdin + + def connectionMade(self): + if self.stdin is not None: + self.transport.write(self.stdin) + self.transport.closeStdin() + + def processEnded(self, reason): + out = self.outBuf.getvalue() + err = self.errBuf.getvalue() + e = reason.value + code = e.exitCode + if e.signal: + self.deferred.errback((out, err, e.signal)) + else: + self.deferred.callback((out, err, code)) + +def myGetProcessOutputAndValue(executable, args=(), env={}, path='.', + reactor=None, stdin=None): + """Like twisted.internet.utils.getProcessOutputAndValue but takes + stdin, too.""" + if reactor is None: + from twisted.internet import reactor + d = defer.Deferred() + p = _PutEverythingGetter(d, stdin) + reactor.spawnProcess(p, executable, (executable,)+tuple(args), env, path) + return d config_vc = """ from buildbot.process import factory, step @@ -313,7 +384,7 @@ class BaseHelper: self.branch.append(rev) self.allrevs.append(rev) - def runCommand(self, basedir, command, failureIsOk=False): + def runCommand(self, basedir, command, failureIsOk=False, stdin=None): # all commands passed to do() should be strings or lists. If they are # strings, none of the arguments may have spaces. This makes the # commands less verbose at the expense of restricting what they can @@ -323,8 +394,9 @@ class BaseHelper: #print "do %s" % command env = os.environ.copy() env['LC_ALL'] = "C" - d = utils.getProcessOutputAndValue(command[0], command[1:], - env=env, path=basedir) + d = myGetProcessOutputAndValue(command[0], command[1:], + env=env, path=basedir, + stdin=stdin) def check((out, err, code)): #print #print "command: %s" % command @@ -342,14 +414,15 @@ class BaseHelper: d.addCallback(check) return d - def do(self, basedir, command, failureIsOk=False): - d = self.runCommand(basedir, command, failureIsOk=failureIsOk) + def do(self, basedir, command, failureIsOk=False, stdin=None): + d = self.runCommand(basedir, command, failureIsOk=failureIsOk, + stdin=stdin) return waitForDeferred(d) - def dovc(self, basedir, command, failureIsOk=False): + def dovc(self, basedir, command, failureIsOk=False, stdin=None): """Like do(), but the VC binary will be prepended to COMMAND.""" command = self.vcexe + " " + command - return self.do(basedir, command, failureIsOk) + return self.do(basedir, command, failureIsOk, stdin) class VCBase(SignalMixin): metadir = None @@ -1315,6 +1388,163 @@ class SVN(VCBase, unittest.TestCase): VCS.registerVC(SVN.vc_name, SVNHelper()) + +class P4Support(VCBase): + metadir = None + branchname = "branch" + vctype = "step.P4" + p4port = 'localhost:1666' + pid = None + base_descr = 'Change: new\nDescription: asdf\nFiles:\n' + + class _P4DProtocol(protocol.ProcessProtocol): + def __init__(self): + self.started = defer.Deferred() + self.ended = defer.Deferred() + + def outReceived(self, data): + # When it says starting, it has bound to the socket. + if self.started: + if data.startswith('Perforce Server starting...'): + self.started.callback(None) + else: + try: + raise Exception('p4d said %r' % data) + except: + self.started.errback(failure.Failure()) + self.started = None + + def processEnded(self, status_object): + if status_object.check(error.ProcessDone): + self.ended.callback(None) + else: + self.ended.errback(status_object) + + def _start_p4d(self): + proto = self._P4DProtocol() + reactor.spawnProcess(proto, self.p4dexe, ['p4d', '-p', self.p4port], + env=os.environ, path=self.p4rep) + return proto.started, proto.ended + + def capable(self): + global VCS + if not VCS.has_key("p4"): + VCS["p4"] = False + p4paths = which('p4') + p4dpaths = which('p4d') + if p4paths and p4dpaths: + self.vcexe = p4paths[0] + self.p4dexe = p4dpaths[0] + VCS["p4"] = True + if not VCS["p4"]: + raise unittest.SkipTest("No usable Perforce was found") + + def vc_create(self): + tmp = os.path.join(self.repbase, "p4tmp") + self.p4rep = os.path.join(self.repbase, 'P4-Repository') + os.mkdir(self.p4rep) + + # Launch p4d. + started, self.p4d_shutdown = self._start_p4d() + w = waitForDeferred(started) + yield w; w.getResult() + + # Create client spec. + os.mkdir(tmp) + clispec = 'Client: creator\n' + clispec += 'Root: %s\n' % tmp + clispec += 'View:\n' + clispec += '\t//depot/... //creator/...\n' + w = self.dovc(tmp, '-p %s client -i' % self.p4port, stdin=clispec) + yield w; w.getResult() + + # Create first rev (trunk). + self.populate(os.path.join(tmp, 'trunk')) + files = ['main.c', 'version.c', 'subdir/subdir.c'] + w = self.do(tmp, ['sh', '-c', + "p4 -p %s -c creator add " % self.p4port + + " ".join(['trunk/%s' % f for f in files])]) + yield w; w.getResult() + descr = self.base_descr + for file in files: + descr += '\t//depot/trunk/%s\n' % file + w = self.dovc(tmp, "-p %s -c creator submit -i" % self.p4port, + stdin=descr) + yield w; out = w.getResult() + m = re.search(r'Change (\d+) submitted.', out) + assert m.group(1) == '1' + self.addTrunkRev(m.group(1)) + + # Create second rev (branch). + w = self.dovc(tmp, '-p %s -c creator integrate ' % self.p4port + + '//depot/trunk/... //depot/branch/...') + yield w; w.getResult() + w = self.do(tmp, ['sh', '-c', + "p4 -p %s -c creator edit branch/main.c" + % self.p4port]) + yield w; w.getResult() + self.populate_branch(os.path.join(tmp, 'branch')) + descr = self.base_descr + for file in files: + descr += '\t//depot/branch/%s\n' % file + w = self.dovc(tmp, "-p %s -c creator submit -i" % self.p4port, + stdin=descr) + yield w; out = w.getResult() + m = re.search(r'Change (\d+) submitted.', out) + self.addBranchRev(m.group(1)) + vc_create = deferredGenerator(vc_create) + + def vc_revise(self): + tmp = os.path.join(self.repbase, "p4tmp") + self.version += 1 + version_c = VERSION_C % self.version + w = self.do(tmp, ['sh', '-c', + 'p4 -p %s -c creator edit trunk/version.c' + % self.p4port]) + yield w; w.getResult() + open(os.path.join(tmp, "trunk/version.c"), "w").write(version_c) + descr = self.base_descr + '\t//depot/trunk/version.c\n' + w = self.dovc(tmp, "-p %s -c creator submit -i" % self.p4port, + stdin=descr) + yield w; out = w.getResult() + m = re.search(r'Change (\d+) submitted.', out) + self.addTrunkRev(m.group(1)) + vc_revise = deferredGenerator(vc_revise) + + def setUp2(self, res): + if self.p4d_shutdown is None: + started, self.p4d_shutdown = self._start_p4d() + return started + + def tearDown2(self): + self.p4d_shutdown = None + d = self.runCommand(self.repbase, '%s -p %s admin stop' + % (self.vcexe, self.p4port)) + return d.addCallback(lambda _: self.p4d_shutdown) + +class P4(P4Support, unittest.TestCase): + + def testCheckout(self): + self.vcargs = { 'p4port': self.p4port, 'p4base': '//depot/', + 'defaultBranch': 'trunk' } + d = self.do_vctest(testRetry=False) + # TODO: like arch and darcs, sync does nothing when server is not + # changed. + return maybeWait(d) + + def testPatch(self): + self.vcargs = { 'p4port': self.p4port, 'p4base': '//depot/', + 'defaultBranch': 'trunk' } + d = self.do_patch() + return maybeWait(d) + + def testBranch(self): + self.vcargs = { 'p4port': self.p4port, 'p4base': '//depot/', + 'defaultBranch': 'trunk' } + d = self.do_branch() + return maybeWait(d) + + class DarcsHelper(BaseHelper): branchname = "branch" try_branchname = "branch" diff --git a/docs/buildbot.texinfo b/docs/buildbot.texinfo index 7f74203..b7fe47b 100644 --- a/docs/buildbot.texinfo +++ b/docs/buildbot.texinfo @@ -128,6 +128,7 @@ Change Sources * CVSToys - mail notification:: * Other mail notification ChangeSources:: * PBChangeSource:: +* P4Source:: Build Process @@ -150,7 +151,7 @@ Source Checkout * Mercurial:: * Arch:: * Bazaar:: -* P4Sync:: +* P4:: Simple ShellCommand Subclasses @@ -1165,6 +1166,13 @@ statically-specified @code{defaultBranch}). The @code{baseURL} and @code{branch} are simply concatenated together to derive the @code{svnurl} to use for the checkout. +@uref{http://www.perforce.com/, Perforce} is similar. The server +is specified through a @code{P4PORT} parameter. Module and branch +are specified in a single depot path, and revisions are +depot-wide. When branches are used, the @code{p4base} and +@code{defaultBranch} are concatenated together to produce the depot +path. + @uref{http://wiki.gnuarch.org/, Arch} and @uref{http://bazaar.canonical.com/, Bazaar} specify a repository by URL, as well as a @code{version} which is kind of like a branch name. @@ -2302,6 +2310,7 @@ a suitable @code{MaildirSource}. * CVSToys - mail notification:: * Other mail notification ChangeSources:: * PBChangeSource:: +* P4Source:: @end menu @node Choosing ChangeSources, CVSToys - PBService, Change Sources, Change Sources @@ -2447,7 +2456,7 @@ used in mail sent by Syncmail. @code{BonsaiMaildirSource} parses messages sent out by Bonsai. -@node PBChangeSource, , Other mail notification ChangeSources, Change Sources +@node PBChangeSource, P4Source, Other mail notification ChangeSources, Change Sources @subsection PBChangeSource The last kind of ChangeSource actually listens on a TCP port for @@ -2500,6 +2509,55 @@ connection. @end table +@node P4Source, , PBChangeSource, Change Sources +@subsection P4Source + +The @code{P4Source} periodically polls a @uref{http://www.perforce.com/, +Perforce} depot for changes. It accepts the following arguments: + +@table @samp +@item @code{p4base} +The base depot path to watch, without the trailing '/...'. + +@item @code{p4port} +The Perforce server to connect to (as host:port). + +@item @code{p4user} +The Perforce user. + +@item @code{p4passwd} +The Perforce password. + +@item @code{split_file} +A function that maps a pathname, without the leading @code{p4base}, to a +(branch, filename) tuple. The default just returns (None, branchfile), +which effectively disables branch support. You should supply a function +which understands your repository structure. + +@item @code{pollinterval} +How often to poll, in seconds. Defaults to 600 (10 minutes). + +@item @code{histmax} +The maximum number of changes to inspect at a time. If more than this +number occur since the last poll, older changes will be silently +ignored. +@end table + +@heading Example + +This configuration uses the @code{P4PORT}, @code{P4USER}, and @code{P4PASSWD} +specified in the buildmaster's environment. It watches a project in which the +branch name is simply the next path component, and the file is all path +components after. + +@example +import buildbot.changes.p4poller +c['sources'].append(p4poller.P4Source( + p4base='//depot/project/', + split_file=lambda branchfile: branchfile.split('/',1) +)) +@end example + @node Build Process, Status Delivery, Getting Source Code Changes, Top @chapter Build Process @@ -2733,7 +2791,7 @@ arguments are described on the following pages. * Mercurial:: * Arch:: * Bazaar:: -* P4Sync:: +* P4:: @end menu @node CVS, SVN, Source Checkout, Source Checkout @@ -3005,7 +3063,7 @@ step, below. @end table -@node Bazaar, P4Sync, Arch, Source Checkout +@node Bazaar, P4, Arch, Source Checkout @subsubsection Bazaar @cindex Bazaar Checkout @@ -3021,22 +3079,42 @@ name when you do @code{baz register-archive}, so we must provide it ourselves). -@node P4Sync, , Bazaar, Source Checkout -@subsubsection P4Sync +@node P4, , Bazaar, Source Checkout +@subsubsection P4 @cindex Perforce Update -The @code{P4Sync} build step performs a -@uref{http://www.perforce.com/, Perforce} update. It is a temporary -facility: a more complete P4 checkout step (named @code{P4}) will -eventually replace it. This step requires significant manual setup on -each build slave. It takes the following arguments. +The @code{P4} build step creates a @uref{http://www.perforce.com/, +Perforce} client specification and performs an update. @table @code +@item p4base +A view into the Perforce depot without branch name or trailing "...". +Typically "//depot/proj/". +@item defaultBranch +A branch name to append on build requests if none is specified. +Typically "trunk". @item p4port -(required): the host:port string describing how to get to the P4 Depot -(repository), used as the P4PORT environment variable for all p4 -commands +(optional): the host:port string describing how to get to the P4 Depot +(repository), used as the -p argument for all p4 commands. +@item p4user +(optional): the Perforce user, used as the -u argument to all p4 +commands. +@item p4passwd +(optional): the Perforce password, used as the -p argument to all p4 +commands. +@item p4extra_views +(optional): a list of (depotpath, clientpath) tuples containing extra +views to be mapped into the client specification. Both will have +"/..." appended automatically. The client name and source directory +will be prepended to the client path. +@item p4client +(optional): The name of the client to use. In mode='copy' and +mode='update', it's particularly important that a unique name is used +for each checkout directory to avoid incorrect synchronization. For +this reason, Python percent substitution will be performed on this value +to replace %(slave)s with the slave name and %(builder)s with the +builder name. The default is "buildbot_%(slave)s_%(build)s". @end table @node ShellCommand, Simple ShellCommand Subclasses, Source Checkout, Build Steps -- 2.11.4.GIT