1 # -*- test-case-name: buildbot.test.test_slavecommand -*-
3 import os
, re
, signal
, shutil
, types
, time
4 from stat
import ST_CTIME
, ST_MTIME
, ST_SIZE
6 from zope
.interface
import implements
7 from twisted
.internet
.protocol
import ProcessProtocol
8 from twisted
.internet
import reactor
, defer
, task
9 from twisted
.python
import log
, failure
, runtime
10 from twisted
.python
.procutils
import which
12 from buildbot
.slave
.interfaces
import ISlaveCommand
13 from buildbot
.slave
.registry
import registerSlaveCommand
15 # this used to be a CVS $-style "Revision" auto-updated keyword, but since I
16 # moved to Darcs as the primary repository, this is updated manually each
17 # time this file is changed. The last cvs_ver that was here was 1.51 .
18 command_version
= "2.5"
21 # >=1.17: commands are interruptable
22 # >=1.28: Arch understands 'revision', added Bazaar
23 # >=1.33: Source classes understand 'retry'
24 # >=1.39: Source classes correctly handle changes in branch (except Git)
25 # Darcs accepts 'revision' (now all do but Git) (well, and P4Sync)
26 # Arch/Baz should accept 'build-config'
27 # >=1.51: (release 0.7.3)
28 # >= 2.1: SlaveShellCommand now accepts 'initial_stdin', 'keep_stdin_open',
29 # and 'logfiles'. It now sends 'log' messages in addition to
30 # stdout/stdin/header/rc. It acquired writeStdin/closeStdin methods,
31 # but these are not remotely callable yet.
32 # (not externally visible: ShellCommandPP has writeStdin/closeStdin.
33 # ShellCommand accepts new arguments (logfiles=, initialStdin=,
34 # keepStdinOpen=) and no longer accepts stdin=)
36 # >= 2.2: added monotone, uploadFile, and downloadFile (release 0.7.5)
37 # >= 2.3: added bzr (release 0.7.6)
38 # >= 2.4: Git understands 'revision' and branches
39 # >= 2.5: workaround added for remote 'hg clone --rev REV' when hg<0.9.2
41 class CommandInterrupted(Exception):
43 class TimeoutError(Exception):
46 class AbandonChain(Exception):
47 """A series of chained steps can raise this exception to indicate that
48 one of the intermediate ShellCommands has failed, such that there is no
49 point in running the remainder. 'rc' should be the non-zero exit code of
50 the failing ShellCommand."""
53 return "<AbandonChain rc=%s>" % self
.args
[0]
56 possibles
= which(name
)
58 raise RuntimeError("Couldn't find executable for '%s'" % name
)
61 def rmdirRecursive(dir):
62 """This is a replacement for shutil.rmtree that works better under
63 windows. Thanks to Bear at the OSAF for the code."""
64 if not os
.path
.exists(dir):
67 if os
.path
.islink(dir):
71 # Verify the directory is read/write/execute for the current user
74 for name
in os
.listdir(dir):
75 full_name
= os
.path
.join(dir, name
)
76 # on Windows, if we don't have write permission we can't remove
77 # the file/directory either, so turn that on
79 if not os
.access(full_name
, os
.W_OK
):
80 # I think this is now redundant, but I don't have an NT
81 # machine to test on, so I'm going to leave it in place
83 os
.chmod(full_name
, 0600)
85 if os
.path
.isdir(full_name
):
86 rmdirRecursive(full_name
)
88 os
.chmod(full_name
, 0700)
92 class ShellCommandPP(ProcessProtocol
):
95 def __init__(self
, command
):
96 self
.command
= command
97 self
.pending_stdin
= ""
98 self
.stdin_finished
= False
100 def writeStdin(self
, data
):
101 assert not self
.stdin_finished
103 self
.transport
.write(data
)
105 self
.pending_stdin
+= data
107 def closeStdin(self
):
109 if self
.debug
: log
.msg(" closing stdin")
110 self
.transport
.closeStdin()
111 self
.stdin_finished
= True
113 def connectionMade(self
):
115 log
.msg("ShellCommandPP.connectionMade")
116 if not self
.command
.process
:
118 log
.msg(" assigning self.command.process: %s" %
120 self
.command
.process
= self
.transport
122 # TODO: maybe we shouldn't close stdin when using a PTY. I can't test
123 # this yet, recent debian glibc has a bug which causes thread-using
124 # test cases to SIGHUP trial, and the workaround is to either run
125 # the whole test with /bin/sh -c " ".join(argv) (way gross) or to
126 # not use a PTY. Once the bug is fixed, I'll be able to test what
127 # happens when you close stdin on a pty. My concern is that it will
128 # SIGHUP the child (since we are, in a sense, hanging up on them).
129 # But it may well be that keeping stdout open prevents the SIGHUP
131 #if not self.command.usePTY:
133 if self
.pending_stdin
:
134 if self
.debug
: log
.msg(" writing to stdin")
135 self
.transport
.write(self
.pending_stdin
)
136 if self
.stdin_finished
:
137 if self
.debug
: log
.msg(" closing stdin")
138 self
.transport
.closeStdin()
140 def outReceived(self
, data
):
142 log
.msg("ShellCommandPP.outReceived")
143 self
.command
.addStdout(data
)
145 def errReceived(self
, data
):
147 log
.msg("ShellCommandPP.errReceived")
148 self
.command
.addStderr(data
)
150 def processEnded(self
, status_object
):
152 log
.msg("ShellCommandPP.processEnded", status_object
)
153 # status_object is a Failure wrapped around an
154 # error.ProcessTerminated or and error.ProcessDone.
155 # requires twisted >= 1.0.4 to overcome a bug in process.py
156 sig
= status_object
.value
.signal
157 rc
= status_object
.value
.exitCode
158 self
.command
.finished(sig
, rc
)
160 class LogFileWatcher
:
163 def __init__(self
, command
, name
, logfile
):
164 self
.command
= command
166 self
.logfile
= logfile
167 log
.msg("LogFileWatcher created to watch %s" % logfile
)
168 # we are created before the ShellCommand starts. If the logfile we're
169 # supposed to be watching already exists, record its size and
170 # ctime/mtime so we can tell when it starts to change.
171 self
.old_logfile_stats
= self
.statFile()
174 # every 2 seconds we check on the file again
175 self
.poller
= task
.LoopingCall(self
.poll
)
178 self
.poller
.start(self
.POLL_INTERVAL
).addErrback(self
._cleanupPoll
)
180 def _cleanupPoll(self
, err
):
181 log
.err(err
, msg
="Polling error")
186 if self
.poller
is not None:
192 if os
.path
.exists(self
.logfile
):
193 s
= os
.stat(self
.logfile
)
194 return (s
[ST_CTIME
], s
[ST_MTIME
], s
[ST_SIZE
])
200 if s
== self
.old_logfile_stats
:
201 return # not started yet
203 # the file was there, but now it's deleted. Forget about the
204 # initial state, clearly the process has deleted the logfile
205 # in preparation for creating a new one.
206 self
.old_logfile_stats
= None
207 return # no file to work with
208 self
.f
= open(self
.logfile
, "rb")
210 self
.f
.seek(self
.f
.tell(), 0)
212 data
= self
.f
.read(10000)
215 self
.command
.addLogfile(self
.name
, data
)
219 # This is a helper class, used by SlaveCommands to run programs in a
225 CHUNK_LIMIT
= 128*1024
227 # For sending elapsed time:
230 # I wish we had easy access to CLOCK_MONOTONIC in Python:
231 # http://www.opengroup.org/onlinepubs/000095399/functions/clock_getres.html
232 # Then changes to the system clock during a run wouldn't effect the "elapsed
235 def __init__(self
, builder
, command
,
236 workdir
, environ
=None,
237 sendStdout
=True, sendStderr
=True, sendRC
=True,
238 timeout
=None, initialStdin
=None, keepStdinOpen
=False,
239 keepStdout
=False, keepStderr
=False,
243 @param keepStdout: if True, we keep a copy of all the stdout text
244 that we've seen. This copy is available in
245 self.stdout, which can be read after the command
247 @param keepStderr: same, for stderr
251 self
.builder
= builder
252 self
.command
= command
253 self
.sendStdout
= sendStdout
254 self
.sendStderr
= sendStderr
256 self
.logfiles
= logfiles
257 self
.workdir
= workdir
258 self
.environ
= os
.environ
.copy()
260 if environ
.has_key('PYTHONPATH'):
261 ppath
= environ
['PYTHONPATH']
262 # Need to do os.pathsep translation. We could either do that
263 # by replacing all incoming ':'s with os.pathsep, or by
264 # accepting lists. I like lists better.
265 if not isinstance(ppath
, str):
266 # If it's not a string, treat it as a sequence to be
267 # turned in to a string.
268 ppath
= os
.pathsep
.join(ppath
)
270 if self
.environ
.has_key('PYTHONPATH'):
271 # special case, prepend the builder's items to the
272 # existing ones. This will break if you send over empty
273 # strings, so don't do that.
274 ppath
= ppath
+ os
.pathsep
+ self
.environ
['PYTHONPATH']
276 environ
['PYTHONPATH'] = ppath
278 self
.environ
.update(environ
)
279 self
.initialStdin
= initialStdin
280 self
.keepStdinOpen
= keepStdinOpen
281 self
.timeout
= timeout
283 self
.keepStdout
= keepStdout
284 self
.keepStderr
= keepStderr
286 # usePTY=True is a convenience for cleaning up all children and
287 # grandchildren of a hung command. Fall back to usePTY=False on
288 # systems where ptys cause problems.
290 self
.usePTY
= self
.builder
.usePTY
291 if runtime
.platformType
!= "posix":
292 self
.usePTY
= False # PTYs are posix-only
293 if initialStdin
is not None:
294 # for .closeStdin to matter, we must use a pipe, not a PTY
297 self
.logFileWatchers
= []
298 for name
,filename
in self
.logfiles
.items():
299 w
= LogFileWatcher(self
, name
,
300 os
.path
.join(self
.workdir
, filename
))
301 self
.logFileWatchers
.append(w
)
304 return "<slavecommand.ShellCommand '%s'>" % self
.command
306 def sendStatus(self
, status
):
307 self
.builder
.sendUpdate(status
)
310 # return a Deferred which fires (with the exit code) when the command
316 self
.deferred
= defer
.Deferred()
320 log
.msg("error in ShellCommand._startCommand")
322 # pretend it was a shell error
323 self
.deferred
.errback(AbandonChain(-1))
326 def _startCommand(self
):
327 # ensure workdir exists
328 if not os
.path
.isdir(self
.workdir
):
329 os
.makedirs(self
.workdir
)
330 log
.msg("ShellCommand._startCommand")
332 self
.sendStatus({'header': "command '%s' in dir %s" % \
333 (self
.command
, self
.workdir
)})
334 self
.sendStatus({'header': "(not really)\n"})
335 self
.finished(None, 0)
338 self
.pp
= ShellCommandPP(self
)
340 if type(self
.command
) in types
.StringTypes
:
341 if runtime
.platformType
== 'win32':
342 argv
= [os
.environ
['COMSPEC'], '/c', self
.command
]
344 # for posix, use /bin/sh. for other non-posix, well, doesn't
346 argv
= ['/bin/sh', '-c', self
.command
]
348 if runtime
.platformType
== 'win32':
349 argv
= [os
.environ
['COMSPEC'], '/c'] + list(self
.command
)
353 # $PWD usually indicates the current directory; spawnProcess may not
354 # update this value, though, so we set it explicitly here.
355 self
.environ
['PWD'] = os
.path
.abspath(self
.workdir
)
357 # self.stdin is handled in ShellCommandPP.connectionMade
359 # first header line is the command in plain text, argv joined with
360 # spaces. You should be able to cut-and-paste this into a shell to
361 # obtain the same results. If there are spaces in the arguments, too
365 self
.sendStatus({'header': msg
+"\n"})
367 # then comes the secondary information
368 msg
= " in dir %s" % (self
.workdir
,)
370 msg
+= " (timeout %d secs)" % (self
.timeout
,)
372 self
.sendStatus({'header': msg
+"\n"})
374 msg
= " watching logfiles %s" % (self
.logfiles
,)
376 self
.sendStatus({'header': msg
+"\n"})
378 # then the argv array for resolving unambiguity
379 msg
= " argv: %s" % (argv
,)
381 self
.sendStatus({'header': msg
+"\n"})
383 # then the environment, since it sometimes causes problems
384 msg
= " environment:\n"
385 env_names
= self
.environ
.keys()
387 for name
in env_names
:
388 msg
+= " %s=%s\n" % (name
, self
.environ
[name
])
389 log
.msg(" environment: %s" % (self
.environ
,))
390 self
.sendStatus({'header': msg
})
392 if self
.initialStdin
:
393 msg
= " writing %d bytes to stdin" % len(self
.initialStdin
)
395 self
.sendStatus({'header': msg
+"\n"})
397 if self
.keepStdinOpen
:
398 msg
= " leaving stdin open"
400 msg
= " closing stdin"
402 self
.sendStatus({'header': msg
+"\n"})
404 msg
= " using PTY: %s" % bool(self
.usePTY
)
406 self
.sendStatus({'header': msg
+"\n"})
408 # this will be buffered until connectionMade is called
409 if self
.initialStdin
:
410 self
.pp
.writeStdin(self
.initialStdin
)
411 if not self
.keepStdinOpen
:
414 # win32eventreactor's spawnProcess (under twisted <= 2.0.1) returns
415 # None, as opposed to all the posixbase-derived reactors (which
416 # return the new Process object). This is a nuisance. We can make up
417 # for it by having the ProcessProtocol give us their .transport
418 # attribute after they get one. I'd prefer to get it from
419 # spawnProcess because I'm concerned about returning from this method
420 # without having a valid self.process to work with. (if kill() were
421 # called right after we return, but somehow before connectionMade
422 # were called, then kill() would blow up).
424 self
.startTime
= time
.time()
425 p
= reactor
.spawnProcess(self
.pp
, argv
[0], argv
,
429 # connectionMade might have been called during spawnProcess
433 # connectionMade also closes stdin as long as we're not using a PTY.
434 # This is intended to kill off inappropriately interactive commands
435 # better than the (long) hung-command timeout. ProcessPTY should be
436 # enhanced to allow the same childFDs argument that Process takes,
437 # which would let us connect stdin to /dev/null .
440 self
.timer
= reactor
.callLater(self
.timeout
, self
.doTimeout
)
442 for w
in self
.logFileWatchers
:
446 def _chunkForSend(self
, data
):
447 # limit the chunks that we send over PB to 128k, since it has a
448 # hardwired string-size limit of 640k.
449 LIMIT
= self
.CHUNK_LIMIT
450 for i
in range(0, len(data
), LIMIT
):
451 yield data
[i
:i
+LIMIT
]
453 def addStdout(self
, data
):
455 for chunk
in self
._chunkForSend
(data
):
456 self
.sendStatus({'stdout': chunk
})
460 self
.timer
.reset(self
.timeout
)
462 def addStderr(self
, data
):
464 for chunk
in self
._chunkForSend
(data
):
465 self
.sendStatus({'stderr': chunk
})
469 self
.timer
.reset(self
.timeout
)
471 def addLogfile(self
, name
, data
):
472 for chunk
in self
._chunkForSend
(data
):
473 self
.sendStatus({'log': (name
, chunk
)})
475 self
.timer
.reset(self
.timeout
)
477 def finished(self
, sig
, rc
):
478 self
.elapsedTime
= time
.time() - self
.startTime
479 log
.msg("command finished with signal %s, exit code %s, elapsedTime: %0.6f" % (sig
,rc
,self
.elapsedTime
))
480 for w
in self
.logFileWatchers
:
481 # this will send the final updates
488 {'header': "process killed by signal %d\n" % sig
})
489 self
.sendStatus({'rc': rc
})
490 self
.sendStatus({'header': "elapsedTime=%0.6f\n" % self
.elapsedTime
})
499 log
.msg("Hey, command %s finished twice" % self
)
501 def failed(self
, why
):
502 log
.msg("ShellCommand.failed: command failed: %s" % (why
,))
511 log
.msg("Hey, command %s finished twice" % self
)
515 msg
= "command timed out: %d seconds without output" % self
.timeout
519 # This may be called by the timeout, or when the user has decided to
524 if hasattr(self
.process
, "pid"):
525 msg
+= ", killing pid %d" % self
.process
.pid
527 self
.sendStatus({'header': "\n" + msg
+ "\n"})
530 if runtime
.platformType
== "posix":
532 # really want to kill off all child processes too. Process
533 # Groups are ideal for this, but that requires
534 # spawnProcess(usePTY=1). Try both ways in case process was
535 # not started that way.
537 # the test suite sets self.KILL=None to tell us we should
538 # only pretend to kill the child. This lets us test the
542 if self
.KILL
is not None:
543 sig
= getattr(signal
, "SIG"+ self
.KILL
, None)
545 if self
.KILL
== None:
546 log
.msg("self.KILL==None, only pretending to kill child")
548 log
.msg("signal module is missing SIG%s" % self
.KILL
)
549 elif not hasattr(os
, "kill"):
550 log
.msg("os module is missing the 'kill' function")
552 log
.msg("trying os.kill(-pid, %d)" % (sig
,))
553 # TODO: maybe use os.killpg instead of a negative pid?
554 os
.kill(-self
.process
.pid
, sig
)
555 log
.msg(" signal %s sent successfully" % sig
)
558 # probably no-such-process, maybe because there is no process
563 if self
.KILL
is None:
564 log
.msg("self.KILL==None, only pretending to kill child")
566 log
.msg("trying process.signalProcess('KILL')")
567 self
.process
.signalProcess(self
.KILL
)
568 log
.msg(" signal %s sent successfully" % (self
.KILL
,))
571 # could be no-such-process, because they finished very recently
574 log
.msg("signalProcess/os.kill failed both times")
576 if runtime
.platformType
== "posix":
577 # we only do this under posix because the win32eventreactor
578 # blocks here until the process has terminated, while closing
579 # stderr. This is weird.
580 self
.pp
.transport
.loseConnection()
582 # finished ought to be called momentarily. Just in case it doesn't,
583 # set a timer which will abandon the command.
584 self
.timer
= reactor
.callLater(self
.BACKUP_TIMEOUT
,
585 self
.doBackupTimeout
)
587 def doBackupTimeout(self
):
588 log
.msg("we tried to kill the process, and it wouldn't die.."
591 self
.sendStatus({'header': "SIGKILL failed to kill process\n"})
593 self
.sendStatus({'header': "using fake rc=-1\n"})
594 self
.sendStatus({'rc': -1})
595 self
.failed(TimeoutError("SIGKILL failed to kill process"))
598 def writeStdin(self
, data
):
599 self
.pp
.writeStdin(data
)
601 def closeStdin(self
):
606 implements(ISlaveCommand
)
608 """This class defines one command that can be invoked by the build master.
609 The command is executed on the slave side, and always sends back a
610 completion message when it finishes. It may also send intermediate status
611 as it runs (by calling builder.sendStatus). Some commands can be
612 interrupted (either by the build master or a local timeout), in which
613 case the step is expected to complete normally with a status message that
614 indicates an error occurred.
616 These commands are used by BuildSteps on the master side. Each kind of
617 BuildStep uses a single Command. The slave must implement all the
618 Commands required by the set of BuildSteps used for any given build:
619 this is checked at startup time.
621 All Commands are constructed with the same signature:
622 c = CommandClass(builder, args)
623 where 'builder' is the parent SlaveBuilder object, and 'args' is a
624 dict that is interpreted per-command.
626 The setup(args) method is available for setup, and is run from __init__.
628 The Command is started with start(). This method must be implemented in a
629 subclass, and it should return a Deferred. When your step is done, you
630 should fire the Deferred (the results are not used). If the command is
631 interrupted, it should fire the Deferred anyway.
633 While the command runs. it may send status messages back to the
634 buildmaster by calling self.sendStatus(statusdict). The statusdict is
635 interpreted by the master-side BuildStep however it likes.
637 A separate completion message is sent when the deferred fires, which
638 indicates that the Command has finished, but does not carry any status
639 data. If the Command needs to return an exit code of some sort, that
640 should be sent as a regular status message before the deferred is fired .
641 Once builder.commandComplete has been run, no more status messages may be
644 If interrupt() is called, the Command should attempt to shut down as
645 quickly as possible. Child processes should be killed, new ones should
646 not be started. The Command should send some kind of error status update,
647 then complete as usual by firing the Deferred.
649 .interrupted should be set by interrupt(), and can be tested to avoid
650 sending multiple error status messages.
652 If .running is False, the bot is shutting down (or has otherwise lost the
653 connection to the master), and should not send any status messages. This
654 is checked in Command.sendStatus .
659 # sendStatus(dict) (zero or more)
660 # commandComplete() or commandInterrupted() (one, at end)
664 running
= False # set by Builder, cleared on shutdown or when the
667 def __init__(self
, builder
, stepId
, args
):
668 self
.builder
= builder
669 self
.stepId
= stepId
# just for logging
673 def setup(self
, args
):
674 """Override this in a subclass to extract items from the args dict."""
679 d
= defer
.maybeDeferred(self
.start
)
680 d
.addBoth(self
.commandComplete
)
684 """Start the command. This method should return a Deferred that will
685 fire when the command has completed. The Deferred's argument will be
688 This method should be overridden by subclasses."""
689 raise NotImplementedError, "You must implement this in a subclass"
691 def sendStatus(self
, status
):
692 """Send a status update to the master."""
694 log
.msg("sendStatus", status
)
696 log
.msg("would sendStatus but not .running")
698 self
.builder
.sendUpdate(status
)
700 def doInterrupt(self
):
705 """Override this in a subclass to allow commands to be interrupted.
706 May be called multiple times, test and set self.interrupted=True if
710 def commandComplete(self
, res
):
714 # utility methods, mostly used by SlaveShellCommand and the like
716 def _abandonOnFailure(self
, rc
):
717 if type(rc
) is not int:
718 log
.msg("weird, _abandonOnFailure was given rc=%s (%s)" % \
720 assert isinstance(rc
, int)
722 raise AbandonChain(rc
)
725 def _sendRC(self
, res
):
726 self
.sendStatus({'rc': 0})
728 def _checkAbandoned(self
, why
):
729 log
.msg("_checkAbandoned", why
)
730 why
.trap(AbandonChain
)
731 log
.msg(" abandoning chain", why
.value
)
732 self
.sendStatus({'rc': why
.value
.args
[0]})
737 class SlaveFileUploadCommand(Command
):
739 Upload a file from slave to build master
742 - ['workdir']: base directory to use
743 - ['slavesrc']: name of the slave-side file to read from
744 - ['writer']: RemoteReference to a transfer._FileWriter object
745 - ['maxsize']: max size (in bytes) of file to write
746 - ['blocksize']: max size for each data block
750 def setup(self
, args
):
751 self
.workdir
= args
['workdir']
752 self
.filename
= args
['slavesrc']
753 self
.writer
= args
['writer']
754 self
.remaining
= args
['maxsize']
755 self
.blocksize
= args
['blocksize']
761 log
.msg('SlaveFileUploadCommand started')
764 self
.path
= os
.path
.join(self
.builder
.basedir
,
766 os
.path
.expanduser(self
.filename
))
768 self
.fp
= open(self
.path
, 'rb')
770 log
.msg('Opened %r for upload' % self
.path
)
772 # TODO: this needs cleanup
774 self
.stderr
= 'Cannot open file %r for upload' % self
.path
777 log
.msg('Cannot open file %r for upload' % self
.path
)
779 self
.sendStatus({'header': "sending %s" % self
.path
})
782 reactor
.callLater(0, self
._loop
, d
)
784 # close the file, but pass through any errors from _loop
785 d1
= self
.writer
.callRemote("close")
786 d1
.addErrback(log
.err
)
787 d1
.addCallback(lambda ignored
: res
)
790 d
.addBoth(self
.finished
)
793 def _loop(self
, fire_when_done
):
794 d
= defer
.maybeDeferred(self
._writeBlock
)
797 fire_when_done
.callback(None)
799 self
._loop
(fire_when_done
)
801 fire_when_done
.errback(why
)
802 d
.addCallbacks(_done
, _err
)
805 def _writeBlock(self
):
806 """Write a block of data to the remote writer"""
808 if self
.interrupted
or self
.fp
is None:
810 log
.msg('SlaveFileUploadCommand._writeBlock(): end')
813 length
= self
.blocksize
814 if self
.remaining
is not None and length
> self
.remaining
:
815 length
= self
.remaining
818 if self
.stderr
is None:
819 self
.stderr
= 'Maximum filesize reached, truncating file %r' \
824 data
= self
.fp
.read(length
)
827 log
.msg('SlaveFileUploadCommand._writeBlock(): '+
828 'allowed=%d readlen=%d' % (length
, len(data
)))
830 log
.msg("EOF: callRemote(close)")
833 if self
.remaining
is not None:
834 self
.remaining
= self
.remaining
- len(data
)
835 assert self
.remaining
>= 0
836 d
= self
.writer
.callRemote('write', data
)
837 d
.addCallback(lambda res
: False)
842 log
.msg('interrupted')
845 if self
.stderr
is None:
846 self
.stderr
= 'Upload of %r interrupted' % self
.path
848 self
.interrupted
= True
849 # the next _writeBlock call will notice the .interrupted flag
851 def finished(self
, res
):
853 log
.msg('finished: stderr=%r, rc=%r' % (self
.stderr
, self
.rc
))
854 if self
.stderr
is None:
855 self
.sendStatus({'rc': self
.rc
})
857 self
.sendStatus({'stderr': self
.stderr
, 'rc': self
.rc
})
860 registerSlaveCommand("uploadFile", SlaveFileUploadCommand
, command_version
)
863 class SlaveFileDownloadCommand(Command
):
865 Download a file from master to slave
868 - ['workdir']: base directory to use
869 - ['slavedest']: name of the slave-side file to be created
870 - ['reader']: RemoteReference to a transfer._FileReader object
871 - ['maxsize']: max size (in bytes) of file to write
872 - ['blocksize']: max size for each data block
873 - ['mode']: access mode for the new file
877 def setup(self
, args
):
878 self
.workdir
= args
['workdir']
879 self
.filename
= args
['slavedest']
880 self
.reader
= args
['reader']
881 self
.bytes_remaining
= args
['maxsize']
882 self
.blocksize
= args
['blocksize']
883 self
.mode
= args
['mode']
889 log
.msg('SlaveFileDownloadCommand starting')
892 self
.path
= os
.path
.join(self
.builder
.basedir
,
894 os
.path
.expanduser(self
.filename
))
896 dirname
= os
.path
.dirname(self
.path
)
897 if not os
.path
.exists(dirname
):
901 self
.fp
= open(self
.path
, 'wb')
903 log
.msg('Opened %r for download' % self
.path
)
904 if self
.mode
is not None:
905 # note: there is a brief window during which the new file
906 # will have the buildslave's default (umask) mode before we
907 # set the new one. Don't use this mode= feature to keep files
908 # private: use the buildslave's umask for that instead. (it
909 # is possible to call os.umask() before and after the open()
910 # call, but cleaning up from exceptions properly is more of a
911 # nuisance that way).
912 os
.chmod(self
.path
, self
.mode
)
914 # TODO: this still needs cleanup
916 self
.stderr
= 'Cannot open file %r for download' % self
.path
919 log
.msg('Cannot open file %r for download' % self
.path
)
922 reactor
.callLater(0, self
._loop
, d
)
924 # close the file, but pass through any errors from _loop
925 d1
= self
.reader
.callRemote('close')
926 d1
.addErrback(log
.err
)
927 d1
.addCallback(lambda ignored
: res
)
930 d
.addBoth(self
.finished
)
933 def _loop(self
, fire_when_done
):
934 d
= defer
.maybeDeferred(self
._readBlock
)
937 fire_when_done
.callback(None)
939 self
._loop
(fire_when_done
)
941 fire_when_done
.errback(why
)
942 d
.addCallbacks(_done
, _err
)
945 def _readBlock(self
):
946 """Read a block of data from the remote reader."""
948 if self
.interrupted
or self
.fp
is None:
950 log
.msg('SlaveFileDownloadCommand._readBlock(): end')
953 length
= self
.blocksize
954 if self
.bytes_remaining
is not None and length
> self
.bytes_remaining
:
955 length
= self
.bytes_remaining
958 if self
.stderr
is None:
959 self
.stderr
= 'Maximum filesize reached, truncating file %r' \
964 d
= self
.reader
.callRemote('read', length
)
965 d
.addCallback(self
._writeData
)
968 def _writeData(self
, data
):
970 log
.msg('SlaveFileDownloadCommand._readBlock(): readlen=%d' %
975 if self
.bytes_remaining
is not None:
976 self
.bytes_remaining
= self
.bytes_remaining
- len(data
)
977 assert self
.bytes_remaining
>= 0
983 log
.msg('interrupted')
986 if self
.stderr
is None:
987 self
.stderr
= 'Download of %r interrupted' % self
.path
989 self
.interrupted
= True
990 # now we wait for the next read request to return. _readBlock will
991 # abandon the file when it sees self.interrupted set.
993 def finished(self
, res
):
994 if self
.fp
is not None:
998 log
.msg('finished: stderr=%r, rc=%r' % (self
.stderr
, self
.rc
))
999 if self
.stderr
is None:
1000 self
.sendStatus({'rc': self
.rc
})
1002 self
.sendStatus({'stderr': self
.stderr
, 'rc': self
.rc
})
1005 registerSlaveCommand("downloadFile", SlaveFileDownloadCommand
, command_version
)
1009 class SlaveShellCommand(Command
):
1010 """This is a Command which runs a shell command. The args dict contains
1013 - ['command'] (required): a shell command to run. If this is a string,
1014 it will be run with /bin/sh (['/bin/sh',
1015 '-c', command]). If it is a list
1016 (preferred), it will be used directly.
1017 - ['workdir'] (required): subdirectory in which the command will be
1018 run, relative to the builder dir
1019 - ['env']: a dict of environment variables to augment/replace
1020 os.environ . PYTHONPATH is treated specially, and
1021 should be a list of path components to be prepended to
1022 any existing PYTHONPATH environment variable.
1023 - ['initial_stdin']: a string which will be written to the command's
1024 stdin as soon as it starts
1025 - ['keep_stdin_open']: unless True, the command's stdin will be
1026 closed as soon as initial_stdin has been
1027 written. Set this to True if you plan to write
1028 to stdin after the command has been started.
1029 - ['want_stdout']: 0 if stdout should be thrown away
1030 - ['want_stderr']: 0 if stderr should be thrown away
1031 - ['not_really']: 1 to skip execution and return rc=0
1032 - ['timeout']: seconds of silence to tolerate before killing command
1033 - ['logfiles']: dict mapping LogFile name to the workdir-relative
1034 filename of a local log file. This local file will be
1035 watched just like 'tail -f', and all changes will be
1036 written to 'log' status updates.
1038 ShellCommand creates the following status messages:
1039 - {'stdout': data} : when stdout data is available
1040 - {'stderr': data} : when stderr data is available
1041 - {'header': data} : when headers (command start/stop) are available
1042 - {'log': (logfile_name, data)} : when log files have new contents
1043 - {'rc': rc} : when the process has terminated
1048 # args['workdir'] is relative to Builder directory, and is required.
1049 assert args
['workdir'] is not None
1050 workdir
= os
.path
.join(self
.builder
.basedir
, args
['workdir'])
1052 c
= ShellCommand(self
.builder
, args
['command'],
1053 workdir
, environ
=args
.get('env'),
1054 timeout
=args
.get('timeout', None),
1055 sendStdout
=args
.get('want_stdout', True),
1056 sendStderr
=args
.get('want_stderr', True),
1058 initialStdin
=args
.get('initial_stdin'),
1059 keepStdinOpen
=args
.get('keep_stdin_open'),
1060 logfiles
=args
.get('logfiles', {}),
1063 d
= self
.command
.start()
1066 def interrupt(self
):
1067 self
.interrupted
= True
1068 self
.command
.kill("command interrupted")
1070 def writeStdin(self
, data
):
1071 self
.command
.writeStdin(data
)
1073 def closeStdin(self
):
1074 self
.command
.closeStdin()
1076 registerSlaveCommand("shell", SlaveShellCommand
, command_version
)
1079 class DummyCommand(Command
):
1081 I am a dummy no-op command that by default takes 5 seconds to complete.
1082 See L{buildbot.steps.dummy.RemoteDummy}
1086 self
.d
= defer
.Deferred()
1087 log
.msg(" starting dummy command [%s]" % self
.stepId
)
1088 self
.timer
= reactor
.callLater(1, self
.doStatus
)
1091 def interrupt(self
):
1092 if self
.interrupted
:
1096 self
.interrupted
= True
1100 log
.msg(" sending intermediate status")
1101 self
.sendStatus({'stdout': 'data'})
1102 timeout
= self
.args
.get('timeout', 5) + 1
1103 self
.timer
= reactor
.callLater(timeout
- 1, self
.finished
)
1106 log
.msg(" dummy command finished [%s]" % self
.stepId
)
1107 if self
.interrupted
:
1108 self
.sendStatus({'rc': 1})
1110 self
.sendStatus({'rc': 0})
1113 registerSlaveCommand("dummy", DummyCommand
, command_version
)
1116 # this maps handle names to a callable. When the WaitCommand starts, this
1117 # callable is invoked with no arguments. It should return a Deferred. When
1118 # that Deferred fires, our WaitCommand will finish.
1119 waitCommandRegistry
= {}
1121 class WaitCommand(Command
):
1123 I am a dummy command used by the buildbot unit test suite. I want for the
1124 unit test to tell us to finish. See L{buildbot.steps.dummy.Wait}
1128 self
.d
= defer
.Deferred()
1129 log
.msg(" starting wait command [%s]" % self
.stepId
)
1130 handle
= self
.args
['handle']
1131 cb
= waitCommandRegistry
[handle
]
1132 del waitCommandRegistry
[handle
]
1134 log
.msg(" wait-%s starting" % (handle
,))
1137 log
.msg(" wait-%s finishing: %s" % (handle
, res
))
1140 d
.addCallbacks(self
.finished
, self
.failed
)
1141 reactor
.callLater(0, _called
)
1144 def interrupt(self
):
1145 log
.msg(" wait command interrupted")
1146 if self
.interrupted
:
1148 self
.interrupted
= True
1149 self
.finished("interrupted")
1151 def finished(self
, res
):
1152 log
.msg(" wait command finished [%s]" % self
.stepId
)
1153 if self
.interrupted
:
1154 self
.sendStatus({'rc': 2})
1156 self
.sendStatus({'rc': 0})
1158 def failed(self
, why
):
1159 log
.msg(" wait command failed [%s]" % self
.stepId
)
1160 self
.sendStatus({'rc': 1})
1163 registerSlaveCommand("dummy.wait", WaitCommand
, command_version
)
1166 class SourceBase(Command
):
1167 """Abstract base class for Version Control System operations (checkout
1168 and update). This class extracts the following arguments from the
1169 dictionary received from the master:
1171 - ['workdir']: (required) the subdirectory where the buildable sources
1174 - ['mode']: one of update/copy/clobber/export, defaults to 'update'
1176 - ['revision']: If not None, this is an int or string which indicates
1177 which sources (along a time-like axis) should be used.
1178 It is the thing you provide as the CVS -r or -D
1181 - ['patch']: If not None, this is a tuple of (striplevel, patch)
1182 which contains a patch that should be applied after the
1183 checkout has occurred. Once applied, the tree is no
1184 longer eligible for use with mode='update', and it only
1185 makes sense to use this in conjunction with a
1186 ['revision'] argument. striplevel is an int, and patch
1187 is a string in standard unified diff format. The patch
1188 will be applied with 'patch -p%d <PATCH', with
1189 STRIPLEVEL substituted as %d. The command will fail if
1190 the patch process fails (rejected hunks).
1192 - ['timeout']: seconds of silence tolerated before we kill off the
1195 - ['retry']: If not None, this is a tuple of (delay, repeats)
1196 which means that any failed VC updates should be
1197 reattempted, up to REPEATS times, after a delay of
1198 DELAY seconds. This is intended to deal with slaves
1199 that experience transient network failures.
1204 def setup(self
, args
):
1205 # if we need to parse the output, use this environment. Otherwise
1206 # command output will be in whatever the buildslave's native language
1208 self
.env
= os
.environ
.copy()
1209 self
.env
['LC_ALL'] = "C"
1211 self
.workdir
= args
['workdir']
1212 self
.mode
= args
.get('mode', "update")
1213 self
.revision
= args
.get('revision')
1214 self
.patch
= args
.get('patch')
1215 self
.timeout
= args
.get('timeout', 120)
1216 self
.retry
= args
.get('retry')
1217 # VC-specific subclasses should override this to extract more args.
1218 # Make sure to upcall!
1221 self
.sendStatus({'header': "starting " + self
.header
+ "\n"})
1224 # self.srcdir is where the VC system should put the sources
1225 if self
.mode
== "copy":
1226 self
.srcdir
= "source" # hardwired directory name, sorry
1228 self
.srcdir
= self
.workdir
1229 self
.sourcedatafile
= os
.path
.join(self
.builder
.basedir
,
1231 ".buildbot-sourcedata")
1233 d
= defer
.succeed(None)
1234 # do we need to clobber anything?
1235 if self
.mode
in ("copy", "clobber", "export"):
1236 d
.addCallback(self
.doClobber
, self
.workdir
)
1237 if not (self
.sourcedirIsUpdateable() and self
.sourcedataMatches()):
1238 # the directory cannot be updated, so we have to clobber it.
1239 # Perhaps the master just changed modes from 'export' to
1241 d
.addCallback(self
.doClobber
, self
.srcdir
)
1243 d
.addCallback(self
.doVC
)
1245 if self
.mode
== "copy":
1246 d
.addCallback(self
.doCopy
)
1248 d
.addCallback(self
.doPatch
)
1249 d
.addCallbacks(self
._sendRC
, self
._checkAbandoned
)
1252 def interrupt(self
):
1253 self
.interrupted
= True
1255 self
.command
.kill("command interrupted")
1257 def doVC(self
, res
):
1258 if self
.interrupted
:
1259 raise AbandonChain(1)
1260 if self
.sourcedirIsUpdateable() and self
.sourcedataMatches():
1261 d
= self
.doVCUpdate()
1262 d
.addCallback(self
.maybeDoVCFallback
)
1265 d
.addBoth(self
.maybeDoVCRetry
)
1266 d
.addCallback(self
._abandonOnFailure
)
1267 d
.addCallback(self
._handleGotRevision
)
1268 d
.addCallback(self
.writeSourcedata
)
1271 def sourcedataMatches(self
):
1273 olddata
= open(self
.sourcedatafile
, "r").read()
1274 if olddata
!= self
.sourcedata
:
1280 def _handleGotRevision(self
, res
):
1281 d
= defer
.maybeDeferred(self
.parseGotRevision
)
1282 d
.addCallback(lambda got_revision
:
1283 self
.sendStatus({'got_revision': got_revision
}))
1286 def parseGotRevision(self
):
1287 """Override this in a subclass. It should return a string that
1288 represents which revision was actually checked out, or a Deferred
1289 that will fire with such a string. If, in a future build, you were to
1290 pass this 'got_revision' string in as the 'revision' component of a
1291 SourceStamp, you should wind up with the same source code as this
1292 checkout just obtained.
1294 It is probably most useful to scan self.command.stdout for a string
1295 of some sort. Be sure to set keepStdout=True on the VC command that
1296 you run, so that you'll have something available to look at.
1298 If this information is unavailable, just return None."""
1302 def writeSourcedata(self
, res
):
1303 open(self
.sourcedatafile
, "w").write(self
.sourcedata
)
1306 def sourcedirIsUpdateable(self
):
1307 raise NotImplementedError("this must be implemented in a subclass")
1309 def doVCUpdate(self
):
1310 raise NotImplementedError("this must be implemented in a subclass")
1313 raise NotImplementedError("this must be implemented in a subclass")
1315 def maybeDoVCFallback(self
, rc
):
1316 if type(rc
) is int and rc
== 0:
1318 if self
.interrupted
:
1319 raise AbandonChain(1)
1320 msg
= "update failed, clobbering and trying again"
1321 self
.sendStatus({'header': msg
+ "\n"})
1323 d
= self
.doClobber(None, self
.srcdir
)
1324 d
.addCallback(self
.doVCFallback2
)
1327 def doVCFallback2(self
, res
):
1328 msg
= "now retrying VC operation"
1329 self
.sendStatus({'header': msg
+ "\n"})
1332 d
.addBoth(self
.maybeDoVCRetry
)
1333 d
.addCallback(self
._abandonOnFailure
)
1336 def maybeDoVCRetry(self
, res
):
1337 """We get here somewhere after a VC chain has finished. res could
1340 - 0: the operation was successful
1341 - nonzero: the operation failed. retry if possible
1342 - AbandonChain: the operation failed, someone else noticed. retry.
1343 - Failure: some other exception, re-raise
1346 if isinstance(res
, failure
.Failure
):
1347 if self
.interrupted
:
1348 return res
# don't re-try interrupted builds
1349 res
.trap(AbandonChain
)
1351 if type(res
) is int and res
== 0:
1353 if self
.interrupted
:
1354 raise AbandonChain(1)
1355 # if we get here, we should retry, if possible
1357 delay
, repeats
= self
.retry
1359 self
.retry
= (delay
, repeats
-1)
1360 msg
= ("update failed, trying %d more times after %d seconds"
1362 self
.sendStatus({'header': msg
+ "\n"})
1364 d
= defer
.Deferred()
1365 d
.addCallback(lambda res
: self
.doVCFull())
1366 d
.addBoth(self
.maybeDoVCRetry
)
1367 reactor
.callLater(delay
, d
.callback
, None)
1371 def doClobber(self
, dummy
, dirname
):
1372 # TODO: remove the old tree in the background
1373 ## workdir = os.path.join(self.builder.basedir, self.workdir)
1374 ## deaddir = self.workdir + ".deleting"
1375 ## if os.path.isdir(workdir):
1377 ## os.rename(workdir, deaddir)
1378 ## # might fail if deaddir already exists: previous deletion
1379 ## # hasn't finished yet
1380 ## # start the deletion in the background
1381 ## # TODO: there was a solaris/NetApp/NFS problem where a
1382 ## # process that was still running out of the directory we're
1383 ## # trying to delete could prevent the rm-rf from working. I
1384 ## # think it stalled the rm, but maybe it just died with
1385 ## # permission issues. Try to detect this.
1386 ## os.commands("rm -rf %s &" % deaddir)
1388 ## # fall back to sequential delete-then-checkout
1390 d
= os
.path
.join(self
.builder
.basedir
, dirname
)
1391 if runtime
.platformType
!= "posix":
1392 # if we're running on w32, use rmtree instead. It will block,
1393 # but hopefully it won't take too long.
1395 return defer
.succeed(0)
1396 command
= ["rm", "-rf", d
]
1397 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
1398 sendRC
=0, timeout
=self
.timeout
)
1400 # Work around for Bug #255
1404 # sendRC=0 means the rm command will send stdout/stderr to the
1405 # master, but not the rc=0 when it finishes. That job is left to
1408 d
.addCallback(self
._abandonOnFailure
)
1411 def doCopy(self
, res
):
1412 # now copy tree to workdir
1413 fromdir
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
1414 todir
= os
.path
.join(self
.builder
.basedir
, self
.workdir
)
1415 if runtime
.platformType
!= "posix":
1416 shutil
.copytree(fromdir
, todir
)
1417 return defer
.succeed(0)
1418 command
= ['cp', '-R', '-P', '-p', fromdir
, todir
]
1419 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
1420 sendRC
=False, timeout
=self
.timeout
)
1423 d
.addCallback(self
._abandonOnFailure
)
1426 def doPatch(self
, res
):
1427 patchlevel
, diff
= self
.patch
1428 command
= [getCommand("patch"), '-p%d' % patchlevel
]
1429 dir = os
.path
.join(self
.builder
.basedir
, self
.workdir
)
1430 # mark the directory so we don't try to update it later
1431 open(os
.path
.join(dir, ".buildbot-patched"), "w").write("patched\n")
1432 # now apply the patch
1433 c
= ShellCommand(self
.builder
, command
, dir,
1434 sendRC
=False, timeout
=self
.timeout
,
1438 d
.addCallback(self
._abandonOnFailure
)
1442 class CVS(SourceBase
):
1443 """CVS-specific VC operation. In addition to the arguments handled by
1444 SourceBase, this command reads the following keys:
1446 ['cvsroot'] (required): the CVSROOT repository string
1447 ['cvsmodule'] (required): the module to be retrieved
1448 ['branch']: a '-r' tag or branch name to use for the checkout/update
1449 ['login']: a string for use as a password to 'cvs login'
1450 ['global_options']: a list of strings to use before the CVS verb
1453 header
= "cvs operation"
1455 def setup(self
, args
):
1456 SourceBase
.setup(self
, args
)
1457 self
.vcexe
= getCommand("cvs")
1458 self
.cvsroot
= args
['cvsroot']
1459 self
.cvsmodule
= args
['cvsmodule']
1460 self
.global_options
= args
.get('global_options', [])
1461 self
.branch
= args
.get('branch')
1462 self
.login
= args
.get('login')
1463 self
.sourcedata
= "%s\n%s\n%s\n" % (self
.cvsroot
, self
.cvsmodule
,
1466 def sourcedirIsUpdateable(self
):
1467 if os
.path
.exists(os
.path
.join(self
.builder
.basedir
,
1468 self
.srcdir
, ".buildbot-patched")):
1470 return os
.path
.isdir(os
.path
.join(self
.builder
.basedir
,
1471 self
.srcdir
, "CVS"))
1474 if self
.login
is not None:
1475 # need to do a 'cvs login' command first
1476 d
= self
.builder
.basedir
1477 command
= ([self
.vcexe
, '-d', self
.cvsroot
] + self
.global_options
1479 c
= ShellCommand(self
.builder
, command
, d
,
1480 sendRC
=False, timeout
=self
.timeout
,
1481 initialStdin
=self
.login
+"\n")
1484 d
.addCallback(self
._abandonOnFailure
)
1485 d
.addCallback(self
._didLogin
)
1488 return self
._didLogin
(None)
1490 def _didLogin(self
, res
):
1491 # now we really start
1492 return SourceBase
.start(self
)
1494 def doVCUpdate(self
):
1495 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
1496 command
= [self
.vcexe
, '-z3'] + self
.global_options
+ ['update', '-dP']
1498 command
+= ['-r', self
.branch
]
1500 command
+= ['-D', self
.revision
]
1501 c
= ShellCommand(self
.builder
, command
, d
,
1502 sendRC
=False, timeout
=self
.timeout
)
1507 d
= self
.builder
.basedir
1508 if self
.mode
== "export":
1512 command
= ([self
.vcexe
, '-d', self
.cvsroot
, '-z3'] +
1513 self
.global_options
+
1514 [verb
, '-d', self
.srcdir
])
1516 command
+= ['-r', self
.branch
]
1518 command
+= ['-D', self
.revision
]
1519 command
+= [self
.cvsmodule
]
1520 c
= ShellCommand(self
.builder
, command
, d
,
1521 sendRC
=False, timeout
=self
.timeout
)
1525 def parseGotRevision(self
):
1526 # CVS does not have any kind of revision stamp to speak of. We return
1527 # the current timestamp as a best-effort guess, but this depends upon
1528 # the local system having a clock that is
1529 # reasonably-well-synchronized with the repository.
1530 return time
.strftime("%Y-%m-%d %H:%M:%S +0000", time
.gmtime())
1532 registerSlaveCommand("cvs", CVS
, command_version
)
1534 class SVN(SourceBase
):
1535 """Subversion-specific VC operation. In addition to the arguments
1536 handled by SourceBase, this command reads the following keys:
1538 ['svnurl'] (required): the SVN repository string
1541 header
= "svn operation"
1543 def setup(self
, args
):
1544 SourceBase
.setup(self
, args
)
1545 self
.vcexe
= getCommand("svn")
1546 self
.svnurl
= args
['svnurl']
1547 self
.sourcedata
= "%s\n" % self
.svnurl
1549 def sourcedirIsUpdateable(self
):
1550 if os
.path
.exists(os
.path
.join(self
.builder
.basedir
,
1551 self
.srcdir
, ".buildbot-patched")):
1553 return os
.path
.isdir(os
.path
.join(self
.builder
.basedir
,
1554 self
.srcdir
, ".svn"))
1556 def doVCUpdate(self
):
1557 revision
= self
.args
['revision'] or 'HEAD'
1558 # update: possible for mode in ('copy', 'update')
1559 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
1560 command
= [self
.vcexe
, 'update', '--revision', str(revision
),
1561 '--non-interactive', '--no-auth-cache']
1562 c
= ShellCommand(self
.builder
, command
, d
,
1563 sendRC
=False, timeout
=self
.timeout
,
1569 revision
= self
.args
['revision'] or 'HEAD'
1570 d
= self
.builder
.basedir
1571 if self
.mode
== "export":
1572 command
= [self
.vcexe
, 'export', '--revision', str(revision
),
1573 '--non-interactive', '--no-auth-cache',
1574 self
.svnurl
, self
.srcdir
]
1576 # mode=='clobber', or copy/update on a broken workspace
1577 command
= [self
.vcexe
, 'checkout', '--revision', str(revision
),
1578 '--non-interactive', '--no-auth-cache',
1579 self
.svnurl
, self
.srcdir
]
1580 c
= ShellCommand(self
.builder
, command
, d
,
1581 sendRC
=False, timeout
=self
.timeout
,
1586 def getSvnVersionCommand(self
):
1588 Get the (shell) command used to determine SVN revision number
1591 return: list of strings, passable as the command argument to ShellCommand
1593 # svn checkout operations finish with 'Checked out revision 16657.'
1594 # svn update operations finish the line 'At revision 16654.'
1595 # But we don't use those. Instead, run 'svnversion'.
1596 svnversion_command
= getCommand("svnversion")
1597 # older versions of 'svnversion' (1.1.4) require the WC_PATH
1598 # argument, newer ones (1.3.1) do not.
1599 return [svnversion_command
, "."]
1601 def parseGotRevision(self
):
1602 c
= ShellCommand(self
.builder
,
1603 self
.getSvnVersionCommand(),
1604 os
.path
.join(self
.builder
.basedir
, self
.srcdir
),
1606 sendStdout
=False, sendStderr
=False, sendRC
=False,
1611 r_raw
= c
.stdout
.strip()
1612 # Extract revision from the version "number" string
1613 r
= r_raw
.rstrip('MS')
1614 r
= r
.split(':')[-1]
1617 got_version
= int(r
)
1619 msg
=("SVN.parseGotRevision unable to parse output "
1620 "of svnversion: '%s'" % r_raw
)
1622 self
.sendStatus({'header': msg
+ "\n"})
1624 d
.addCallback(_parse
)
1628 registerSlaveCommand("svn", SVN
, command_version
)
1630 class Darcs(SourceBase
):
1631 """Darcs-specific VC operation. In addition to the arguments
1632 handled by SourceBase, this command reads the following keys:
1634 ['repourl'] (required): the Darcs repository string
1637 header
= "darcs operation"
1639 def setup(self
, args
):
1640 SourceBase
.setup(self
, args
)
1641 self
.vcexe
= getCommand("darcs")
1642 self
.repourl
= args
['repourl']
1643 self
.sourcedata
= "%s\n" % self
.repourl
1644 self
.revision
= self
.args
.get('revision')
1646 def sourcedirIsUpdateable(self
):
1647 if os
.path
.exists(os
.path
.join(self
.builder
.basedir
,
1648 self
.srcdir
, ".buildbot-patched")):
1651 # checking out a specific revision requires a full 'darcs get'
1653 return os
.path
.isdir(os
.path
.join(self
.builder
.basedir
,
1654 self
.srcdir
, "_darcs"))
1656 def doVCUpdate(self
):
1657 assert not self
.revision
1658 # update: possible for mode in ('copy', 'update')
1659 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
1660 command
= [self
.vcexe
, 'pull', '--all', '--verbose']
1661 c
= ShellCommand(self
.builder
, command
, d
,
1662 sendRC
=False, timeout
=self
.timeout
)
1667 # checkout or export
1668 d
= self
.builder
.basedir
1669 command
= [self
.vcexe
, 'get', '--verbose', '--partial',
1670 '--repo-name', self
.srcdir
]
1672 # write the context to a file
1673 n
= os
.path
.join(self
.builder
.basedir
, ".darcs-context")
1675 f
.write(self
.revision
)
1677 # tell Darcs to use that context
1678 command
.append('--context')
1680 command
.append(self
.repourl
)
1682 c
= ShellCommand(self
.builder
, command
, d
,
1683 sendRC
=False, timeout
=self
.timeout
)
1687 d
.addCallback(self
.removeContextFile
, n
)
1690 def removeContextFile(self
, res
, n
):
1694 def parseGotRevision(self
):
1695 # we use 'darcs context' to find out what we wound up with
1696 command
= [self
.vcexe
, "changes", "--context"]
1697 c
= ShellCommand(self
.builder
, command
,
1698 os
.path
.join(self
.builder
.basedir
, self
.srcdir
),
1700 sendStdout
=False, sendStderr
=False, sendRC
=False,
1704 d
.addCallback(lambda res
: c
.stdout
)
1707 registerSlaveCommand("darcs", Darcs
, command_version
)
1709 class Monotone(SourceBase
):
1710 """Monotone-specific VC operation. In addition to the arguments handled
1711 by SourceBase, this command reads the following keys:
1713 ['server_addr'] (required): the address of the server to pull from
1714 ['branch'] (required): the branch the revision is on
1715 ['db_path'] (required): the local database path to use
1716 ['revision'] (required): the revision to check out
1717 ['monotone']: (required): path to monotone executable
1720 header
= "monotone operation"
1722 def setup(self
, args
):
1723 SourceBase
.setup(self
, args
)
1724 self
.server_addr
= args
["server_addr"]
1725 self
.branch
= args
["branch"]
1726 self
.db_path
= args
["db_path"]
1727 self
.revision
= args
["revision"]
1728 self
.monotone
= args
["monotone"]
1729 self
._made
_fulls
= False
1730 self
._pull
_timeout
= args
["timeout"]
1732 def _makefulls(self
):
1733 if not self
._made
_fulls
:
1734 basedir
= self
.builder
.basedir
1735 self
.full_db_path
= os
.path
.join(basedir
, self
.db_path
)
1736 self
.full_srcdir
= os
.path
.join(basedir
, self
.srcdir
)
1737 self
._made
_fulls
= True
1739 def sourcedirIsUpdateable(self
):
1741 if os
.path
.exists(os
.path
.join(self
.full_srcdir
,
1742 ".buildbot_patched")):
1744 return (os
.path
.isfile(self
.full_db_path
)
1745 and os
.path
.isdir(os
.path
.join(self
.full_srcdir
, "MT")))
1747 def doVCUpdate(self
):
1748 return self
._withFreshDb
(self
._doUpdate
)
1750 def _doUpdate(self
):
1751 # update: possible for mode in ('copy', 'update')
1752 command
= [self
.monotone
, "update",
1753 "-r", self
.revision
,
1755 c
= ShellCommand(self
.builder
, command
, self
.full_srcdir
,
1756 sendRC
=False, timeout
=self
.timeout
)
1761 return self
._withFreshDb
(self
._doFull
)
1764 command
= [self
.monotone
, "--db=" + self
.full_db_path
,
1766 "-r", self
.revision
,
1769 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
1770 sendRC
=False, timeout
=self
.timeout
)
1774 def _withFreshDb(self
, callback
):
1776 # first ensure the db exists and is usable
1777 if os
.path
.isfile(self
.full_db_path
):
1778 # already exists, so run 'db migrate' in case monotone has been
1780 command
= [self
.monotone
, "db", "migrate",
1781 "--db=" + self
.full_db_path
]
1783 # We'll be doing an initial pull, so up the timeout to 3 hours to
1784 # make sure it will have time to complete.
1785 self
._pull
_timeout
= max(self
._pull
_timeout
, 3 * 60 * 60)
1786 self
.sendStatus({"header": "creating database %s\n"
1787 % (self
.full_db_path
,)})
1788 command
= [self
.monotone
, "db", "init",
1789 "--db=" + self
.full_db_path
]
1790 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
1791 sendRC
=False, timeout
=self
.timeout
)
1794 d
.addCallback(self
._abandonOnFailure
)
1795 d
.addCallback(self
._didDbInit
)
1796 d
.addCallback(self
._didPull
, callback
)
1799 def _didDbInit(self
, res
):
1800 command
= [self
.monotone
, "--db=" + self
.full_db_path
,
1801 "pull", "--ticker=dot", self
.server_addr
, self
.branch
]
1802 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
1803 sendRC
=False, timeout
=self
._pull
_timeout
)
1804 self
.sendStatus({"header": "pulling %s from %s\n"
1805 % (self
.branch
, self
.server_addr
)})
1809 def _didPull(self
, res
, callback
):
1812 registerSlaveCommand("monotone", Monotone
, command_version
)
1815 class Git(SourceBase
):
1816 """Git specific VC operation. In addition to the arguments
1817 handled by SourceBase, this command reads the following keys:
1819 ['repourl'] (required): the upstream GIT repository string
1820 ['branch'] (optional): which version (i.e. branch or tag) to
1821 retrieve. Default: "master".
1824 header
= "git operation"
1826 def setup(self
, args
):
1827 SourceBase
.setup(self
, args
)
1828 self
.repourl
= args
['repourl']
1829 self
.branch
= args
.get('branch')
1831 self
.branch
= "master"
1832 self
.sourcedata
= "%s %s\n" % (self
.repourl
, self
.branch
)
1834 def _fullSrcdir(self
):
1835 return os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
1837 def _commitSpec(self
):
1839 return self
.revision
1842 def sourcedirIsUpdateable(self
):
1843 if os
.path
.exists(os
.path
.join(self
._fullSrcdir
(),
1844 ".buildbot-patched")):
1846 return os
.path
.isdir(os
.path
.join(self
._fullSrcdir
(), ".git"))
1848 def _didFetch(self
, res
):
1850 head
= self
.revision
1854 command
= ['git', 'reset', '--hard', head
]
1855 c
= ShellCommand(self
.builder
, command
, self
._fullSrcdir
(),
1856 sendRC
=False, timeout
=self
.timeout
)
1860 def doVCUpdate(self
):
1861 command
= ['git', 'fetch', self
.repourl
, self
.branch
]
1862 self
.sendStatus({"header": "fetching branch %s from %s\n"
1863 % (self
.branch
, self
.repourl
)})
1864 c
= ShellCommand(self
.builder
, command
, self
._fullSrcdir
(),
1865 sendRC
=False, timeout
=self
.timeout
)
1868 d
.addCallback(self
._abandonOnFailure
)
1869 d
.addCallback(self
._didFetch
)
1872 def _didInit(self
, res
):
1873 return self
.doVCUpdate()
1876 os
.mkdir(self
._fullSrcdir
())
1877 c
= ShellCommand(self
.builder
, ['git', 'init'], self
._fullSrcdir
(),
1878 sendRC
=False, timeout
=self
.timeout
)
1881 d
.addCallback(self
._abandonOnFailure
)
1882 d
.addCallback(self
._didInit
)
1885 def parseGotRevision(self
):
1886 command
= ['git', 'rev-parse', 'HEAD']
1887 c
= ShellCommand(self
.builder
, command
, self
._fullSrcdir
(),
1888 sendRC
=False, keepStdout
=True)
1892 hash = c
.stdout
.strip()
1896 d
.addCallback(_parse
)
1899 registerSlaveCommand("git", Git
, command_version
)
1901 class Arch(SourceBase
):
1902 """Arch-specific (tla-specific) VC operation. In addition to the
1903 arguments handled by SourceBase, this command reads the following keys:
1905 ['url'] (required): the repository string
1906 ['version'] (required): which version (i.e. branch) to retrieve
1907 ['revision'] (optional): the 'patch-NN' argument to check out
1908 ['archive']: the archive name to use. If None, use the archive's default
1909 ['build-config']: if present, give to 'tla build-config' after checkout
1912 header
= "arch operation"
1915 def setup(self
, args
):
1916 SourceBase
.setup(self
, args
)
1917 self
.vcexe
= getCommand("tla")
1918 self
.archive
= args
.get('archive')
1919 self
.url
= args
['url']
1920 self
.version
= args
['version']
1921 self
.revision
= args
.get('revision')
1922 self
.buildconfig
= args
.get('build-config')
1923 self
.sourcedata
= "%s\n%s\n%s\n" % (self
.url
, self
.version
,
1926 def sourcedirIsUpdateable(self
):
1928 # Arch cannot roll a directory backwards, so if they ask for a
1929 # specific revision, clobber the directory. Technically this
1930 # could be limited to the cases where the requested revision is
1931 # later than our current one, but it's too hard to extract the
1932 # current revision from the tree.
1934 if os
.path
.exists(os
.path
.join(self
.builder
.basedir
,
1935 self
.srcdir
, ".buildbot-patched")):
1937 return os
.path
.isdir(os
.path
.join(self
.builder
.basedir
,
1938 self
.srcdir
, "{arch}"))
1940 def doVCUpdate(self
):
1941 # update: possible for mode in ('copy', 'update')
1942 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
1943 command
= [self
.vcexe
, 'replay']
1945 command
.append(self
.revision
)
1946 c
= ShellCommand(self
.builder
, command
, d
,
1947 sendRC
=False, timeout
=self
.timeout
)
1952 # to do a checkout, we must first "register" the archive by giving
1953 # the URL to tla, which will go to the repository at that URL and
1954 # figure out the archive name. tla will tell you the archive name
1955 # when it is done, and all further actions must refer to this name.
1957 command
= [self
.vcexe
, 'register-archive', '--force', self
.url
]
1958 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
1959 sendRC
=False, keepStdout
=True,
1960 timeout
=self
.timeout
)
1963 d
.addCallback(self
._abandonOnFailure
)
1964 d
.addCallback(self
._didRegister
, c
)
1967 def _didRegister(self
, res
, c
):
1968 # find out what tla thinks the archive name is. If the user told us
1969 # to use something specific, make sure it matches.
1970 r
= re
.search(r
'Registering archive: (\S+)\s*$', c
.stdout
)
1972 msg
= "tla reports archive name is '%s'" % r
.group(1)
1974 self
.builder
.sendUpdate({'header': msg
+"\n"})
1975 if self
.archive
and r
.group(1) != self
.archive
:
1976 msg
= (" mismatch, we wanted an archive named '%s'"
1979 self
.builder
.sendUpdate({'header': msg
+"\n"})
1980 raise AbandonChain(-1)
1981 self
.archive
= r
.group(1)
1982 assert self
.archive
, "need archive name to continue"
1983 return self
._doGet
()
1988 ver
+= "--%s" % self
.revision
1989 command
= [self
.vcexe
, 'get', '--archive', self
.archive
,
1992 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
1993 sendRC
=False, timeout
=self
.timeout
)
1996 d
.addCallback(self
._abandonOnFailure
)
1997 if self
.buildconfig
:
1998 d
.addCallback(self
._didGet
)
2001 def _didGet(self
, res
):
2002 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2003 command
= [self
.vcexe
, 'build-config', self
.buildconfig
]
2004 c
= ShellCommand(self
.builder
, command
, d
,
2005 sendRC
=False, timeout
=self
.timeout
)
2008 d
.addCallback(self
._abandonOnFailure
)
2011 def parseGotRevision(self
):
2012 # using code from tryclient.TlaExtractor
2013 # 'tla logs --full' gives us ARCHIVE/BRANCH--REVISION
2014 # 'tla logs' gives us REVISION
2015 command
= [self
.vcexe
, "logs", "--full", "--reverse"]
2016 c
= ShellCommand(self
.builder
, command
,
2017 os
.path
.join(self
.builder
.basedir
, self
.srcdir
),
2019 sendStdout
=False, sendStderr
=False, sendRC
=False,
2024 tid
= c
.stdout
.split("\n")[0].strip()
2025 slash
= tid
.index("/")
2026 dd
= tid
.rindex("--")
2027 #branch = tid[slash+1:dd]
2028 baserev
= tid
[dd
+2:]
2030 d
.addCallback(_parse
)
2033 registerSlaveCommand("arch", Arch
, command_version
)
2036 """Bazaar (/usr/bin/baz) is an alternative client for Arch repositories.
2037 It is mostly option-compatible, but archive registration is different
2038 enough to warrant a separate Command.
2040 ['archive'] (required): the name of the archive being used
2043 def setup(self
, args
):
2044 Arch
.setup(self
, args
)
2045 self
.vcexe
= getCommand("baz")
2046 # baz doesn't emit the repository name after registration (and
2047 # grepping through the output of 'baz archives' is too hard), so we
2048 # require that the buildmaster configuration to provide both the
2049 # archive name and the URL.
2050 self
.archive
= args
['archive'] # required for Baz
2051 self
.sourcedata
= "%s\n%s\n%s\n" % (self
.url
, self
.version
,
2054 # in _didRegister, the regexp won't match, so we'll stick with the name
2058 # baz prefers ARCHIVE/VERSION. This will work even if
2059 # my-default-archive is not set.
2060 ver
= self
.archive
+ "/" + self
.version
2062 ver
+= "--%s" % self
.revision
2063 command
= [self
.vcexe
, 'get', '--no-pristine',
2065 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
2066 sendRC
=False, timeout
=self
.timeout
)
2069 d
.addCallback(self
._abandonOnFailure
)
2070 if self
.buildconfig
:
2071 d
.addCallback(self
._didGet
)
2074 def parseGotRevision(self
):
2075 # using code from tryclient.BazExtractor
2076 command
= [self
.vcexe
, "tree-id"]
2077 c
= ShellCommand(self
.builder
, command
,
2078 os
.path
.join(self
.builder
.basedir
, self
.srcdir
),
2080 sendStdout
=False, sendStderr
=False, sendRC
=False,
2085 tid
= c
.stdout
.strip()
2086 slash
= tid
.index("/")
2087 dd
= tid
.rindex("--")
2088 #branch = tid[slash+1:dd]
2089 baserev
= tid
[dd
+2:]
2091 d
.addCallback(_parse
)
2094 registerSlaveCommand("bazaar", Bazaar
, command_version
)
2097 class Bzr(SourceBase
):
2098 """bzr-specific VC operation. In addition to the arguments
2099 handled by SourceBase, this command reads the following keys:
2101 ['repourl'] (required): the Bzr repository string
2104 header
= "bzr operation"
2106 def setup(self
, args
):
2107 SourceBase
.setup(self
, args
)
2108 self
.vcexe
= getCommand("bzr")
2109 self
.repourl
= args
['repourl']
2110 self
.sourcedata
= "%s\n" % self
.repourl
2111 self
.revision
= self
.args
.get('revision')
2113 def sourcedirIsUpdateable(self
):
2114 if os
.path
.exists(os
.path
.join(self
.builder
.basedir
,
2115 self
.srcdir
, ".buildbot-patched")):
2118 # checking out a specific revision requires a full 'bzr checkout'
2120 return os
.path
.isdir(os
.path
.join(self
.builder
.basedir
,
2121 self
.srcdir
, ".bzr"))
2123 def doVCUpdate(self
):
2124 assert not self
.revision
2125 # update: possible for mode in ('copy', 'update')
2126 srcdir
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2127 command
= [self
.vcexe
, 'update']
2128 c
= ShellCommand(self
.builder
, command
, srcdir
,
2129 sendRC
=False, timeout
=self
.timeout
)
2134 # checkout or export
2135 d
= self
.builder
.basedir
2136 if self
.mode
== "export":
2137 # exporting in bzr requires a separate directory
2138 return self
.doVCExport()
2139 # originally I added --lightweight here, but then 'bzr revno' is
2140 # wrong. The revno reported in 'bzr version-info' is correct,
2141 # however. Maybe this is a bzr bug?
2143 # In addition, you cannot perform a 'bzr update' on a repo pulled
2144 # from an HTTP repository that used 'bzr checkout --lightweight'. You
2145 # get a "ERROR: Cannot lock: transport is read only" when you try.
2147 # So I won't bother using --lightweight for now.
2149 command
= [self
.vcexe
, 'checkout']
2151 command
.append('--revision')
2152 command
.append(str(self
.revision
))
2153 command
.append(self
.repourl
)
2154 command
.append(self
.srcdir
)
2156 c
= ShellCommand(self
.builder
, command
, d
,
2157 sendRC
=False, timeout
=self
.timeout
)
2162 def doVCExport(self
):
2163 tmpdir
= os
.path
.join(self
.builder
.basedir
, "export-temp")
2164 srcdir
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2165 command
= [self
.vcexe
, 'checkout', '--lightweight']
2167 command
.append('--revision')
2168 command
.append(str(self
.revision
))
2169 command
.append(self
.repourl
)
2170 command
.append(tmpdir
)
2171 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
2172 sendRC
=False, timeout
=self
.timeout
)
2176 command
= [self
.vcexe
, 'export', srcdir
]
2177 c
= ShellCommand(self
.builder
, command
, tmpdir
,
2178 sendRC
=False, timeout
=self
.timeout
)
2181 d
.addCallback(_export
)
2184 def get_revision_number(self
, out
):
2185 # it feels like 'bzr revno' sometimes gives different results than
2186 # the 'revno:' line from 'bzr version-info', and the one from
2187 # version-info is more likely to be correct.
2188 for line
in out
.split("\n"):
2189 colon
= line
.find(":")
2191 key
, value
= line
[:colon
], line
[colon
+2:]
2194 raise ValueError("unable to find revno: in bzr output: '%s'" % out
)
2196 def parseGotRevision(self
):
2197 command
= [self
.vcexe
, "version-info"]
2198 c
= ShellCommand(self
.builder
, command
,
2199 os
.path
.join(self
.builder
.basedir
, self
.srcdir
),
2201 sendStdout
=False, sendStderr
=False, sendRC
=False,
2207 return self
.get_revision_number(c
.stdout
)
2209 msg
=("Bzr.parseGotRevision unable to parse output "
2210 "of bzr version-info: '%s'" % c
.stdout
.strip())
2212 self
.sendStatus({'header': msg
+ "\n"})
2214 d
.addCallback(_parse
)
2217 registerSlaveCommand("bzr", Bzr
, command_version
)
2219 class Mercurial(SourceBase
):
2220 """Mercurial specific VC operation. In addition to the arguments
2221 handled by SourceBase, this command reads the following keys:
2223 ['repourl'] (required): the Cogito repository string
2226 header
= "mercurial operation"
2228 def setup(self
, args
):
2229 SourceBase
.setup(self
, args
)
2230 self
.vcexe
= getCommand("hg")
2231 self
.repourl
= args
['repourl']
2232 self
.sourcedata
= "%s\n" % self
.repourl
2236 def sourcedirIsUpdateable(self
):
2237 if os
.path
.exists(os
.path
.join(self
.builder
.basedir
,
2238 self
.srcdir
, ".buildbot-patched")):
2240 # like Darcs, to check out a specific (old) revision, we have to do a
2241 # full checkout. TODO: I think 'hg pull' plus 'hg update' might work
2244 return os
.path
.isdir(os
.path
.join(self
.builder
.basedir
,
2245 self
.srcdir
, ".hg"))
2247 def doVCUpdate(self
):
2248 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2249 command
= [self
.vcexe
, 'pull', '--update', '--verbose']
2250 if self
.args
.get('revision'):
2251 command
.extend(['--rev', self
.args
['revision']])
2252 c
= ShellCommand(self
.builder
, command
, d
,
2253 sendRC
=False, timeout
=self
.timeout
,
2257 d
.addCallback(self
._handleEmptyUpdate
)
2260 def _handleEmptyUpdate(self
, res
):
2261 if type(res
) is int and res
== 1:
2262 if self
.command
.stdout
.find("no changes found") != -1:
2263 # 'hg pull', when it doesn't have anything to do, exits with
2264 # rc=1, and there appears to be no way to shut this off. It
2265 # emits a distinctive message to stdout, though. So catch
2266 # this and pretend that it completed successfully.
2271 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2272 command
= [self
.vcexe
, 'clone', '-U']
2273 command
.extend([self
.repourl
, d
])
2274 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
2275 sendRC
=False, timeout
=self
.timeout
)
2280 updatecmd
=[self
.vcexe
, 'update', '--repository', d
]
2281 if self
.args
.get('revision'):
2282 updatecmd
.extend(['--rev', self
.args
['revision']])
2284 updatecmd
.extend(['--rev', self
.args
.get('branch', 'default')])
2285 self
.command
= ShellCommand(self
.builder
, updatecmd
,
2286 self
.builder
.basedir
, sendRC
=False, timeout
=self
.timeout
)
2287 return self
.command
.start()
2289 cmd1
.addCallback(_update
)
2292 def _updateToDesiredRevision(self
, res
):
2293 assert self
.args
.get('revision')
2294 newdir
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2295 # hg-0.9.1 and earlier (which need this fallback) also want to see
2296 # 'hg update REV' instead of 'hg update --rev REV'. Note that this is
2297 # the only place we use 'hg update', since what most VC tools mean
2298 # by, say, 'cvs update' is expressed as 'hg pull --update' instead.
2299 command
= [self
.vcexe
, 'update', self
.args
['revision']]
2300 c
= ShellCommand(self
.builder
, command
, newdir
,
2301 sendRC
=False, timeout
=self
.timeout
)
2304 def parseGotRevision(self
):
2305 # we use 'hg identify' to find out what we wound up with
2306 command
= [self
.vcexe
, "identify"]
2307 c
= ShellCommand(self
.builder
, command
,
2308 os
.path
.join(self
.builder
.basedir
, self
.srcdir
),
2310 sendStdout
=False, sendStderr
=False, sendRC
=False,
2314 m
= re
.search(r
'^(\w+)', c
.stdout
)
2316 d
.addCallback(_parse
)
2319 registerSlaveCommand("hg", Mercurial
, command_version
)
2322 class P4Base(SourceBase
):
2323 """Base class for P4 source-updaters
2325 ['p4port'] (required): host:port for server to access
2326 ['p4user'] (optional): user to use for access
2327 ['p4passwd'] (optional): passwd to try for the user
2328 ['p4client'] (optional): client spec to use
2330 def setup(self
, args
):
2331 SourceBase
.setup(self
, args
)
2332 self
.p4port
= args
['p4port']
2333 self
.p4client
= args
['p4client']
2334 self
.p4user
= args
['p4user']
2335 self
.p4passwd
= args
['p4passwd']
2337 def parseGotRevision(self
):
2338 # Executes a p4 command that will give us the latest changelist number
2339 # of any file under the current (or default) client:
2342 command
.extend(['-p', self
.p4port
])
2344 command
.extend(['-u', self
.p4user
])
2346 command
.extend(['-P', self
.p4passwd
])
2348 command
.extend(['-c', self
.p4client
])
2349 command
.extend(['changes', '-m', '1', '#have'])
2350 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
2351 environ
=self
.env
, timeout
=self
.timeout
,
2352 sendStdout
=True, sendStderr
=False, sendRC
=False,
2358 # 'p4 -c clien-name change -m 1 "#have"' will produce an output like:
2359 # "Change 28147 on 2008/04/07 by p4user@hostname..."
2360 # The number after "Change" is the one we want.
2361 m
= re
.match('Change\s+(\d+)\s+', c
.stdout
)
2365 d
.addCallback(_parse
)
2370 """A P4 source-updater.
2372 ['p4port'] (required): host:port for server to access
2373 ['p4user'] (optional): user to use for access
2374 ['p4passwd'] (optional): passwd to try for the user
2375 ['p4client'] (optional): client spec to use
2376 ['p4extra_views'] (optional): additional client views to use
2381 def setup(self
, args
):
2382 P4Base
.setup(self
, args
)
2383 self
.p4base
= args
['p4base']
2384 self
.p4extra_views
= args
['p4extra_views']
2385 self
.p4mode
= args
['mode']
2386 self
.p4branch
= args
['branch']
2388 self
.sourcedata
= str([
2395 # Depot side of view spec.
2400 # Local side of view spec (srcdir is made from these).
2401 self
.builder
.basedir
,
2407 def sourcedirIsUpdateable(self
):
2408 if os
.path
.exists(os
.path
.join(self
.builder
.basedir
,
2409 self
.srcdir
, ".buildbot-patched")):
2411 # We assume our client spec is still around.
2412 # We just say we aren't updateable if the dir doesn't exist so we
2413 # don't get ENOENT checking the sourcedata.
2414 return os
.path
.isdir(os
.path
.join(self
.builder
.basedir
,
2417 def doVCUpdate(self
):
2418 return self
._doP
4Sync
(force
=False)
2420 def _doP4Sync(self
, force
):
2424 command
.extend(['-p', self
.p4port
])
2426 command
.extend(['-u', self
.p4user
])
2428 command
.extend(['-P', self
.p4passwd
])
2430 command
.extend(['-c', self
.p4client
])
2431 command
.extend(['sync'])
2433 command
.extend(['-f'])
2435 command
.extend(['@' + str(self
.revision
)])
2437 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
2438 environ
=env
, sendRC
=False, timeout
=self
.timeout
,
2442 d
.addCallback(self
._abandonOnFailure
)
2450 client_spec
+= "Client: %s\n\n" % self
.p4client
2451 client_spec
+= "Owner: %s\n\n" % self
.p4user
2452 client_spec
+= "Description:\n\tCreated by %s\n\n" % self
.p4user
2453 client_spec
+= "Root:\t%s\n\n" % self
.builder
.basedir
2454 client_spec
+= "Options:\tallwrite rmdir\n\n"
2455 client_spec
+= "LineEnd:\tlocal\n\n"
2458 client_spec
+= "View:\n\t%s" % (self
.p4base
)
2460 client_spec
+= "%s/" % (self
.p4branch
)
2461 client_spec
+= "... //%s/%s/...\n" % (self
.p4client
, self
.srcdir
)
2462 if self
.p4extra_views
:
2463 for k
, v
in self
.p4extra_views
:
2464 client_spec
+= "\t%s/... //%s/%s%s/...\n" % (k
, self
.p4client
,
2467 command
.extend(['-p', self
.p4port
])
2469 command
.extend(['-u', self
.p4user
])
2471 command
.extend(['-P', self
.p4passwd
])
2472 command
.extend(['client', '-i'])
2473 log
.msg(client_spec
)
2474 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
2475 environ
=env
, sendRC
=False, timeout
=self
.timeout
,
2476 initialStdin
=client_spec
)
2479 d
.addCallback(self
._abandonOnFailure
)
2480 d
.addCallback(lambda _
: self
._doP
4Sync
(force
=True))
2483 registerSlaveCommand("p4", P4
, command_version
)
2486 class P4Sync(P4Base
):
2487 """A partial P4 source-updater. Requires manual setup of a per-slave P4
2488 environment. The only thing which comes from the master is P4PORT.
2489 'mode' is required to be 'copy'.
2491 ['p4port'] (required): host:port for server to access
2492 ['p4user'] (optional): user to use for access
2493 ['p4passwd'] (optional): passwd to try for the user
2494 ['p4client'] (optional): client spec to use
2499 def setup(self
, args
):
2500 P4Base
.setup(self
, args
)
2501 self
.vcexe
= getCommand("p4")
2503 def sourcedirIsUpdateable(self
):
2506 def _doVC(self
, force
):
2507 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2508 command
= [self
.vcexe
]
2510 command
.extend(['-p', self
.p4port
])
2512 command
.extend(['-u', self
.p4user
])
2514 command
.extend(['-P', self
.p4passwd
])
2516 command
.extend(['-c', self
.p4client
])
2517 command
.extend(['sync'])
2519 command
.extend(['-f'])
2521 command
.extend(['@' + self
.revision
])
2523 c
= ShellCommand(self
.builder
, command
, d
, environ
=env
,
2524 sendRC
=False, timeout
=self
.timeout
)
2528 def doVCUpdate(self
):
2529 return self
._doVC
(force
=False)
2532 return self
._doVC
(force
=True)
2534 registerSlaveCommand("p4sync", P4Sync
, command_version
)