1 # -*- test-case-name: buildbot.test.test_slavecommand -*-
3 import os
, sys
, re
, signal
, shutil
, types
, time
, tarfile
, tempfile
4 from stat
import ST_CTIME
, ST_MTIME
, ST_SIZE
6 from zope
.interface
import implements
7 from twisted
.internet
.protocol
import ProcessProtocol
8 from twisted
.internet
import reactor
, defer
, task
9 from twisted
.python
import log
, failure
, runtime
10 from twisted
.python
.procutils
import which
12 from buildbot
.slave
.interfaces
import ISlaveCommand
13 from buildbot
.slave
.registry
import registerSlaveCommand
14 from buildbot
.util
import to_text
16 # this used to be a CVS $-style "Revision" auto-updated keyword, but since I
17 # moved to Darcs as the primary repository, this is updated manually each
18 # time this file is changed. The last cvs_ver that was here was 1.51 .
19 command_version
= "2.8"
22 # >=1.17: commands are interruptable
23 # >=1.28: Arch understands 'revision', added Bazaar
24 # >=1.33: Source classes understand 'retry'
25 # >=1.39: Source classes correctly handle changes in branch (except Git)
26 # Darcs accepts 'revision' (now all do but Git) (well, and P4Sync)
27 # Arch/Baz should accept 'build-config'
28 # >=1.51: (release 0.7.3)
29 # >= 2.1: SlaveShellCommand now accepts 'initial_stdin', 'keep_stdin_open',
30 # and 'logfiles'. It now sends 'log' messages in addition to
31 # stdout/stdin/header/rc. It acquired writeStdin/closeStdin methods,
32 # but these are not remotely callable yet.
33 # (not externally visible: ShellCommandPP has writeStdin/closeStdin.
34 # ShellCommand accepts new arguments (logfiles=, initialStdin=,
35 # keepStdinOpen=) and no longer accepts stdin=)
37 # >= 2.2: added monotone, uploadFile, and downloadFile (release 0.7.5)
38 # >= 2.3: added bzr (release 0.7.6)
39 # >= 2.4: Git understands 'revision' and branches
40 # >= 2.5: workaround added for remote 'hg clone --rev REV' when hg<0.9.2
41 # >= 2.6: added uploadDirectory
42 # >= 2.7: added usePTY option to SlaveShellCommand
43 # >= 2.8: added username and password args to SVN class
45 class CommandInterrupted(Exception):
47 class TimeoutError(Exception):
51 """An obfuscated string in a command"""
52 def __init__(self
, real
, fake
):
62 def get_real(command
):
64 if type(command
) == types
.ListType
:
67 if isinstance(elt
, Obfuscated
):
70 rv
.append(to_text(elt
))
72 get_real
= staticmethod(get_real
)
74 def get_fake(command
):
76 if type(command
) == types
.ListType
:
79 if isinstance(elt
, Obfuscated
):
82 rv
.append(to_text(elt
))
84 get_fake
= staticmethod(get_fake
)
86 class AbandonChain(Exception):
87 """A series of chained steps can raise this exception to indicate that
88 one of the intermediate ShellCommands has failed, such that there is no
89 point in running the remainder. 'rc' should be the non-zero exit code of
90 the failing ShellCommand."""
93 return "<AbandonChain rc=%s>" % self
.args
[0]
96 possibles
= which(name
)
98 raise RuntimeError("Couldn't find executable for '%s'" % name
)
101 def rmdirRecursive(dir):
102 """This is a replacement for shutil.rmtree that works better under
103 windows. Thanks to Bear at the OSAF for the code."""
104 if not os
.path
.exists(dir):
107 if os
.path
.islink(dir):
111 # Verify the directory is read/write/execute for the current user
114 for name
in os
.listdir(dir):
115 full_name
= os
.path
.join(dir, name
)
116 # on Windows, if we don't have write permission we can't remove
117 # the file/directory either, so turn that on
119 if not os
.access(full_name
, os
.W_OK
):
120 # I think this is now redundant, but I don't have an NT
121 # machine to test on, so I'm going to leave it in place
123 os
.chmod(full_name
, 0600)
125 if os
.path
.isdir(full_name
):
126 rmdirRecursive(full_name
)
128 os
.chmod(full_name
, 0700)
132 class ShellCommandPP(ProcessProtocol
):
135 def __init__(self
, command
):
136 self
.command
= command
137 self
.pending_stdin
= ""
138 self
.stdin_finished
= False
140 def writeStdin(self
, data
):
141 assert not self
.stdin_finished
143 self
.transport
.write(data
)
145 self
.pending_stdin
+= data
147 def closeStdin(self
):
149 if self
.debug
: log
.msg(" closing stdin")
150 self
.transport
.closeStdin()
151 self
.stdin_finished
= True
153 def connectionMade(self
):
155 log
.msg("ShellCommandPP.connectionMade")
156 if not self
.command
.process
:
158 log
.msg(" assigning self.command.process: %s" %
160 self
.command
.process
= self
.transport
162 # TODO: maybe we shouldn't close stdin when using a PTY. I can't test
163 # this yet, recent debian glibc has a bug which causes thread-using
164 # test cases to SIGHUP trial, and the workaround is to either run
165 # the whole test with /bin/sh -c " ".join(argv) (way gross) or to
166 # not use a PTY. Once the bug is fixed, I'll be able to test what
167 # happens when you close stdin on a pty. My concern is that it will
168 # SIGHUP the child (since we are, in a sense, hanging up on them).
169 # But it may well be that keeping stdout open prevents the SIGHUP
171 #if not self.command.usePTY:
173 if self
.pending_stdin
:
174 if self
.debug
: log
.msg(" writing to stdin")
175 self
.transport
.write(self
.pending_stdin
)
176 if self
.stdin_finished
:
177 if self
.debug
: log
.msg(" closing stdin")
178 self
.transport
.closeStdin()
180 def outReceived(self
, data
):
182 log
.msg("ShellCommandPP.outReceived")
183 self
.command
.addStdout(data
)
185 def errReceived(self
, data
):
187 log
.msg("ShellCommandPP.errReceived")
188 self
.command
.addStderr(data
)
190 def processEnded(self
, status_object
):
192 log
.msg("ShellCommandPP.processEnded", status_object
)
193 # status_object is a Failure wrapped around an
194 # error.ProcessTerminated or and error.ProcessDone.
195 # requires twisted >= 1.0.4 to overcome a bug in process.py
196 sig
= status_object
.value
.signal
197 rc
= status_object
.value
.exitCode
198 self
.command
.finished(sig
, rc
)
200 class LogFileWatcher
:
203 def __init__(self
, command
, name
, logfile
, follow
=False):
204 self
.command
= command
206 self
.logfile
= logfile
208 log
.msg("LogFileWatcher created to watch %s" % logfile
)
209 # we are created before the ShellCommand starts. If the logfile we're
210 # supposed to be watching already exists, record its size and
211 # ctime/mtime so we can tell when it starts to change.
212 self
.old_logfile_stats
= self
.statFile()
215 # follow the file, only sending back lines
216 # added since we started watching
219 # every 2 seconds we check on the file again
220 self
.poller
= task
.LoopingCall(self
.poll
)
223 self
.poller
.start(self
.POLL_INTERVAL
).addErrback(self
._cleanupPoll
)
225 def _cleanupPoll(self
, err
):
226 log
.err(err
, msg
="Polling error")
231 if self
.poller
is not None:
237 if os
.path
.exists(self
.logfile
):
238 s
= os
.stat(self
.logfile
)
239 return (s
[ST_CTIME
], s
[ST_MTIME
], s
[ST_SIZE
])
245 if s
== self
.old_logfile_stats
:
246 return # not started yet
248 # the file was there, but now it's deleted. Forget about the
249 # initial state, clearly the process has deleted the logfile
250 # in preparation for creating a new one.
251 self
.old_logfile_stats
= None
252 return # no file to work with
253 self
.f
= open(self
.logfile
, "rb")
254 # if we only want new lines, seek to
255 # where we stat'd so we only find new
260 self
.f
.seek(self
.f
.tell(), 0)
262 data
= self
.f
.read(10000)
265 self
.command
.addLogfile(self
.name
, data
)
269 # This is a helper class, used by SlaveCommands to run programs in a
275 CHUNK_LIMIT
= 128*1024
277 # For sending elapsed time:
280 # I wish we had easy access to CLOCK_MONOTONIC in Python:
281 # http://www.opengroup.org/onlinepubs/000095399/functions/clock_getres.html
282 # Then changes to the system clock during a run wouldn't effect the "elapsed
285 def __init__(self
, builder
, command
,
286 workdir
, environ
=None,
287 sendStdout
=True, sendStderr
=True, sendRC
=True,
288 timeout
=None, initialStdin
=None, keepStdinOpen
=False,
289 keepStdout
=False, keepStderr
=False, logEnviron
=True,
290 logfiles
={}, usePTY
="slave-config"):
293 @param keepStdout: if True, we keep a copy of all the stdout text
294 that we've seen. This copy is available in
295 self.stdout, which can be read after the command
297 @param keepStderr: same, for stderr
299 @param usePTY: "slave-config" -> use the SlaveBuilder's usePTY;
300 otherwise, true to use a PTY, false to not use a PTY.
303 self
.builder
= builder
304 self
.command
= Obfuscated
.get_real(command
)
305 self
.fake_command
= Obfuscated
.get_fake(command
)
306 self
.sendStdout
= sendStdout
307 self
.sendStderr
= sendStderr
309 self
.logfiles
= logfiles
310 self
.workdir
= workdir
311 if not os
.path
.exists(workdir
):
313 self
.environ
= os
.environ
.copy()
315 if environ
.has_key('PYTHONPATH'):
316 ppath
= environ
['PYTHONPATH']
317 # Need to do os.pathsep translation. We could either do that
318 # by replacing all incoming ':'s with os.pathsep, or by
319 # accepting lists. I like lists better.
320 if not isinstance(ppath
, str):
321 # If it's not a string, treat it as a sequence to be
322 # turned in to a string.
323 ppath
= os
.pathsep
.join(ppath
)
325 if self
.environ
.has_key('PYTHONPATH'):
326 # special case, prepend the builder's items to the
327 # existing ones. This will break if you send over empty
328 # strings, so don't do that.
329 ppath
= ppath
+ os
.pathsep
+ self
.environ
['PYTHONPATH']
331 environ
['PYTHONPATH'] = ppath
333 self
.environ
.update(environ
)
334 self
.initialStdin
= initialStdin
335 self
.keepStdinOpen
= keepStdinOpen
336 self
.logEnviron
= logEnviron
337 self
.timeout
= timeout
339 self
.keepStdout
= keepStdout
340 self
.keepStderr
= keepStderr
343 if usePTY
== "slave-config":
344 self
.usePTY
= self
.builder
.usePTY
348 # usePTY=True is a convenience for cleaning up all children and
349 # grandchildren of a hung command. Fall back to usePTY=False on systems
350 # and in situations where ptys cause problems. PTYs are posix-only,
351 # and for .closeStdin to matter, we must use a pipe, not a PTY
352 if runtime
.platformType
!= "posix" or initialStdin
is not None:
353 if self
.usePTY
and usePTY
!= "slave-config":
354 self
.sendStatus({'header': "WARNING: disabling usePTY for this command"})
357 self
.logFileWatchers
= []
358 for name
,filevalue
in self
.logfiles
.items():
362 # check for a dictionary of options
363 # filename is required, others are optional
364 if type(filevalue
) == dict:
365 filename
= filevalue
['filename']
366 follow
= filevalue
.get('follow', False)
368 w
= LogFileWatcher(self
, name
,
369 os
.path
.join(self
.workdir
, filename
),
371 self
.logFileWatchers
.append(w
)
374 return "<slavecommand.ShellCommand '%s'>" % self
.fake_command
376 def sendStatus(self
, status
):
377 self
.builder
.sendUpdate(status
)
380 # return a Deferred which fires (with the exit code) when the command
386 self
.deferred
= defer
.Deferred()
390 log
.msg("error in ShellCommand._startCommand")
392 # pretend it was a shell error
393 self
.deferred
.errback(AbandonChain(-1))
396 def _startCommand(self
):
397 # ensure workdir exists
398 if not os
.path
.isdir(self
.workdir
):
399 os
.makedirs(self
.workdir
)
400 log
.msg("ShellCommand._startCommand")
402 self
.sendStatus({'header': "command '%s' in dir %s" % \
403 (self
.fake_command
, self
.workdir
)})
404 self
.sendStatus({'header': "(not really)\n"})
405 self
.finished(None, 0)
408 self
.pp
= ShellCommandPP(self
)
410 if type(self
.command
) in types
.StringTypes
:
411 if runtime
.platformType
== 'win32':
412 argv
= os
.environ
['COMSPEC'].split() # allow %COMSPEC% to have args
413 if '/c' not in argv
: argv
+= ['/c']
414 argv
+= [self
.command
]
416 # for posix, use /bin/sh. for other non-posix, well, doesn't
418 argv
= ['/bin/sh', '-c', self
.command
]
419 display
= self
.fake_command
421 if runtime
.platformType
== 'win32' and not self
.command
[0].lower().endswith(".exe"):
422 argv
= os
.environ
['COMSPEC'].split() # allow %COMSPEC% to have args
423 if '/c' not in argv
: argv
+= ['/c']
424 argv
+= list(self
.command
)
427 display
= " ".join(self
.fake_command
)
429 # $PWD usually indicates the current directory; spawnProcess may not
430 # update this value, though, so we set it explicitly here. This causes
431 # weird problems (bug #456) on msys, though..
432 if not self
.environ
.get('MACHTYPE', None) == 'i686-pc-msys':
433 self
.environ
['PWD'] = os
.path
.abspath(self
.workdir
)
435 # self.stdin is handled in ShellCommandPP.connectionMade
437 # first header line is the command in plain text, argv joined with
438 # spaces. You should be able to cut-and-paste this into a shell to
439 # obtain the same results. If there are spaces in the arguments, too
441 log
.msg(" " + display
)
442 self
.sendStatus({'header': display
+"\n"})
444 # then comes the secondary information
445 msg
= " in dir %s" % (self
.workdir
,)
447 msg
+= " (timeout %d secs)" % (self
.timeout
,)
449 self
.sendStatus({'header': msg
+"\n"})
451 msg
= " watching logfiles %s" % (self
.logfiles
,)
453 self
.sendStatus({'header': msg
+"\n"})
455 # then the obfuscated command array for resolving unambiguity
456 msg
= " argv: %s" % (self
.fake_command
,)
458 self
.sendStatus({'header': msg
+"\n"})
460 # then the environment, since it sometimes causes problems
462 msg
= " environment:\n"
463 env_names
= self
.environ
.keys()
465 for name
in env_names
:
466 msg
+= " %s=%s\n" % (name
, self
.environ
[name
])
467 log
.msg(" environment: %s" % (self
.environ
,))
468 self
.sendStatus({'header': msg
})
470 if self
.initialStdin
:
471 msg
= " writing %d bytes to stdin" % len(self
.initialStdin
)
473 self
.sendStatus({'header': msg
+"\n"})
475 if self
.keepStdinOpen
:
476 msg
= " leaving stdin open"
478 msg
= " closing stdin"
480 self
.sendStatus({'header': msg
+"\n"})
482 msg
= " using PTY: %s" % bool(self
.usePTY
)
484 self
.sendStatus({'header': msg
+"\n"})
486 # this will be buffered until connectionMade is called
487 if self
.initialStdin
:
488 self
.pp
.writeStdin(self
.initialStdin
)
489 if not self
.keepStdinOpen
:
492 # win32eventreactor's spawnProcess (under twisted <= 2.0.1) returns
493 # None, as opposed to all the posixbase-derived reactors (which
494 # return the new Process object). This is a nuisance. We can make up
495 # for it by having the ProcessProtocol give us their .transport
496 # attribute after they get one. I'd prefer to get it from
497 # spawnProcess because I'm concerned about returning from this method
498 # without having a valid self.process to work with. (if kill() were
499 # called right after we return, but somehow before connectionMade
500 # were called, then kill() would blow up).
502 self
.startTime
= time
.time()
504 p
= reactor
.spawnProcess(self
.pp
, argv
[0], argv
,
508 # connectionMade might have been called during spawnProcess
512 # connectionMade also closes stdin as long as we're not using a PTY.
513 # This is intended to kill off inappropriately interactive commands
514 # better than the (long) hung-command timeout. ProcessPTY should be
515 # enhanced to allow the same childFDs argument that Process takes,
516 # which would let us connect stdin to /dev/null .
519 self
.timer
= reactor
.callLater(self
.timeout
, self
.doTimeout
)
521 for w
in self
.logFileWatchers
:
525 def _chunkForSend(self
, data
):
526 # limit the chunks that we send over PB to 128k, since it has a
527 # hardwired string-size limit of 640k.
528 LIMIT
= self
.CHUNK_LIMIT
529 for i
in range(0, len(data
), LIMIT
):
530 yield data
[i
:i
+LIMIT
]
532 def addStdout(self
, data
):
534 for chunk
in self
._chunkForSend
(data
):
535 self
.sendStatus({'stdout': chunk
})
539 self
.timer
.reset(self
.timeout
)
541 def addStderr(self
, data
):
543 for chunk
in self
._chunkForSend
(data
):
544 self
.sendStatus({'stderr': chunk
})
548 self
.timer
.reset(self
.timeout
)
550 def addLogfile(self
, name
, data
):
551 for chunk
in self
._chunkForSend
(data
):
552 self
.sendStatus({'log': (name
, chunk
)})
554 self
.timer
.reset(self
.timeout
)
556 def finished(self
, sig
, rc
):
557 self
.elapsedTime
= time
.time() - self
.startTime
558 log
.msg("command finished with signal %s, exit code %s, elapsedTime: %0.6f" % (sig
,rc
,self
.elapsedTime
))
559 for w
in self
.logFileWatchers
:
560 # this will send the final updates
567 {'header': "process killed by signal %d\n" % sig
})
568 self
.sendStatus({'rc': rc
})
569 self
.sendStatus({'header': "elapsedTime=%0.6f\n" % self
.elapsedTime
})
578 log
.msg("Hey, command %s finished twice" % self
)
580 def failed(self
, why
):
581 log
.msg("ShellCommand.failed: command failed: %s" % (why
,))
590 log
.msg("Hey, command %s finished twice" % self
)
594 msg
= "command timed out: %d seconds without output" % self
.timeout
598 # This may be called by the timeout, or when the user has decided to
603 if hasattr(self
.process
, "pid") and self
.process
.pid
is not None:
604 msg
+= ", killing pid %s" % self
.process
.pid
606 self
.sendStatus({'header': "\n" + msg
+ "\n"})
609 if runtime
.platformType
== "posix":
611 # really want to kill off all child processes too. Process
612 # Groups are ideal for this, but that requires
613 # spawnProcess(usePTY=1). Try both ways in case process was
614 # not started that way.
616 # the test suite sets self.KILL=None to tell us we should
617 # only pretend to kill the child. This lets us test the
621 if self
.KILL
is not None:
622 sig
= getattr(signal
, "SIG"+ self
.KILL
, None)
624 if self
.KILL
== None:
625 log
.msg("self.KILL==None, only pretending to kill child")
627 log
.msg("signal module is missing SIG%s" % self
.KILL
)
628 elif not hasattr(os
, "kill"):
629 log
.msg("os module is missing the 'kill' function")
630 elif not hasattr(self
.process
, "pid") or self
.process
.pid
is None:
631 log
.msg("self.process has no pid")
633 log
.msg("trying os.kill(-pid, %d)" % (sig
,))
634 # TODO: maybe use os.killpg instead of a negative pid?
635 os
.kill(-self
.process
.pid
, sig
)
636 log
.msg(" signal %s sent successfully" % sig
)
639 # probably no-such-process, maybe because there is no process
644 if self
.KILL
is None:
645 log
.msg("self.KILL==None, only pretending to kill child")
647 log
.msg("trying process.signalProcess('KILL')")
648 self
.process
.signalProcess(self
.KILL
)
649 log
.msg(" signal %s sent successfully" % (self
.KILL
,))
652 # could be no-such-process, because they finished very recently
655 log
.msg("signalProcess/os.kill failed both times")
657 if runtime
.platformType
== "posix":
658 # we only do this under posix because the win32eventreactor
659 # blocks here until the process has terminated, while closing
660 # stderr. This is weird.
661 self
.pp
.transport
.loseConnection()
663 # finished ought to be called momentarily. Just in case it doesn't,
664 # set a timer which will abandon the command.
665 self
.timer
= reactor
.callLater(self
.BACKUP_TIMEOUT
,
666 self
.doBackupTimeout
)
668 def doBackupTimeout(self
):
669 log
.msg("we tried to kill the process, and it wouldn't die.."
672 self
.sendStatus({'header': "SIGKILL failed to kill process\n"})
674 self
.sendStatus({'header': "using fake rc=-1\n"})
675 self
.sendStatus({'rc': -1})
676 self
.failed(TimeoutError("SIGKILL failed to kill process"))
679 def writeStdin(self
, data
):
680 self
.pp
.writeStdin(data
)
682 def closeStdin(self
):
687 implements(ISlaveCommand
)
689 """This class defines one command that can be invoked by the build master.
690 The command is executed on the slave side, and always sends back a
691 completion message when it finishes. It may also send intermediate status
692 as it runs (by calling builder.sendStatus). Some commands can be
693 interrupted (either by the build master or a local timeout), in which
694 case the step is expected to complete normally with a status message that
695 indicates an error occurred.
697 These commands are used by BuildSteps on the master side. Each kind of
698 BuildStep uses a single Command. The slave must implement all the
699 Commands required by the set of BuildSteps used for any given build:
700 this is checked at startup time.
702 All Commands are constructed with the same signature:
703 c = CommandClass(builder, args)
704 where 'builder' is the parent SlaveBuilder object, and 'args' is a
705 dict that is interpreted per-command.
707 The setup(args) method is available for setup, and is run from __init__.
709 The Command is started with start(). This method must be implemented in a
710 subclass, and it should return a Deferred. When your step is done, you
711 should fire the Deferred (the results are not used). If the command is
712 interrupted, it should fire the Deferred anyway.
714 While the command runs. it may send status messages back to the
715 buildmaster by calling self.sendStatus(statusdict). The statusdict is
716 interpreted by the master-side BuildStep however it likes.
718 A separate completion message is sent when the deferred fires, which
719 indicates that the Command has finished, but does not carry any status
720 data. If the Command needs to return an exit code of some sort, that
721 should be sent as a regular status message before the deferred is fired .
722 Once builder.commandComplete has been run, no more status messages may be
725 If interrupt() is called, the Command should attempt to shut down as
726 quickly as possible. Child processes should be killed, new ones should
727 not be started. The Command should send some kind of error status update,
728 then complete as usual by firing the Deferred.
730 .interrupted should be set by interrupt(), and can be tested to avoid
731 sending multiple error status messages.
733 If .running is False, the bot is shutting down (or has otherwise lost the
734 connection to the master), and should not send any status messages. This
735 is checked in Command.sendStatus .
740 # sendStatus(dict) (zero or more)
741 # commandComplete() or commandInterrupted() (one, at end)
745 running
= False # set by Builder, cleared on shutdown or when the
748 def __init__(self
, builder
, stepId
, args
):
749 self
.builder
= builder
750 self
.stepId
= stepId
# just for logging
754 def setup(self
, args
):
755 """Override this in a subclass to extract items from the args dict."""
760 d
= defer
.maybeDeferred(self
.start
)
761 d
.addBoth(self
.commandComplete
)
765 """Start the command. This method should return a Deferred that will
766 fire when the command has completed. The Deferred's argument will be
769 This method should be overridden by subclasses."""
770 raise NotImplementedError, "You must implement this in a subclass"
772 def sendStatus(self
, status
):
773 """Send a status update to the master."""
775 log
.msg("sendStatus", status
)
777 log
.msg("would sendStatus but not .running")
779 self
.builder
.sendUpdate(status
)
781 def doInterrupt(self
):
786 """Override this in a subclass to allow commands to be interrupted.
787 May be called multiple times, test and set self.interrupted=True if
791 def commandComplete(self
, res
):
795 # utility methods, mostly used by SlaveShellCommand and the like
797 def _abandonOnFailure(self
, rc
):
798 if type(rc
) is not int:
799 log
.msg("weird, _abandonOnFailure was given rc=%s (%s)" % \
801 assert isinstance(rc
, int)
803 raise AbandonChain(rc
)
806 def _sendRC(self
, res
):
807 self
.sendStatus({'rc': 0})
809 def _checkAbandoned(self
, why
):
810 log
.msg("_checkAbandoned", why
)
811 why
.trap(AbandonChain
)
812 log
.msg(" abandoning chain", why
.value
)
813 self
.sendStatus({'rc': why
.value
.args
[0]})
818 class SlaveFileUploadCommand(Command
):
820 Upload a file from slave to build master
823 - ['workdir']: base directory to use
824 - ['slavesrc']: name of the slave-side file to read from
825 - ['writer']: RemoteReference to a transfer._FileWriter object
826 - ['maxsize']: max size (in bytes) of file to write
827 - ['blocksize']: max size for each data block
831 def setup(self
, args
):
832 self
.workdir
= args
['workdir']
833 self
.filename
= args
['slavesrc']
834 self
.writer
= args
['writer']
835 self
.remaining
= args
['maxsize']
836 self
.blocksize
= args
['blocksize']
842 log
.msg('SlaveFileUploadCommand started')
845 self
.path
= os
.path
.join(self
.builder
.basedir
,
847 os
.path
.expanduser(self
.filename
))
849 self
.fp
= open(self
.path
, 'rb')
851 log
.msg('Opened %r for upload' % self
.path
)
853 # TODO: this needs cleanup
855 self
.stderr
= 'Cannot open file %r for upload' % self
.path
858 log
.msg('Cannot open file %r for upload' % self
.path
)
860 self
.sendStatus({'header': "sending %s" % self
.path
})
863 reactor
.callLater(0, self
._loop
, d
)
865 # close the file, but pass through any errors from _loop
866 d1
= self
.writer
.callRemote("close")
867 d1
.addErrback(log
.err
)
868 d1
.addCallback(lambda ignored
: res
)
871 d
.addBoth(self
.finished
)
874 def _loop(self
, fire_when_done
):
875 d
= defer
.maybeDeferred(self
._writeBlock
)
878 fire_when_done
.callback(None)
880 self
._loop
(fire_when_done
)
882 fire_when_done
.errback(why
)
883 d
.addCallbacks(_done
, _err
)
886 def _writeBlock(self
):
887 """Write a block of data to the remote writer"""
889 if self
.interrupted
or self
.fp
is None:
891 log
.msg('SlaveFileUploadCommand._writeBlock(): end')
894 length
= self
.blocksize
895 if self
.remaining
is not None and length
> self
.remaining
:
896 length
= self
.remaining
899 if self
.stderr
is None:
900 self
.stderr
= 'Maximum filesize reached, truncating file %r' \
905 data
= self
.fp
.read(length
)
908 log
.msg('SlaveFileUploadCommand._writeBlock(): '+
909 'allowed=%d readlen=%d' % (length
, len(data
)))
911 log
.msg("EOF: callRemote(close)")
914 if self
.remaining
is not None:
915 self
.remaining
= self
.remaining
- len(data
)
916 assert self
.remaining
>= 0
917 d
= self
.writer
.callRemote('write', data
)
918 d
.addCallback(lambda res
: False)
923 log
.msg('interrupted')
926 if self
.stderr
is None:
927 self
.stderr
= 'Upload of %r interrupted' % self
.path
929 self
.interrupted
= True
930 # the next _writeBlock call will notice the .interrupted flag
932 def finished(self
, res
):
934 log
.msg('finished: stderr=%r, rc=%r' % (self
.stderr
, self
.rc
))
935 if self
.stderr
is None:
936 self
.sendStatus({'rc': self
.rc
})
938 self
.sendStatus({'stderr': self
.stderr
, 'rc': self
.rc
})
941 registerSlaveCommand("uploadFile", SlaveFileUploadCommand
, command_version
)
944 class SlaveDirectoryUploadCommand(SlaveFileUploadCommand
):
946 Upload a directory from slave to build master
949 - ['workdir']: base directory to use
950 - ['slavesrc']: name of the slave-side directory to read from
951 - ['writer']: RemoteReference to a transfer._DirectoryWriter object
952 - ['maxsize']: max size (in bytes) of file to write
953 - ['blocksize']: max size for each data block
954 - ['compress']: one of [None, 'bz2', 'gz']
958 def setup(self
, args
):
959 self
.workdir
= args
['workdir']
960 self
.dirname
= args
['slavesrc']
961 self
.writer
= args
['writer']
962 self
.remaining
= args
['maxsize']
963 self
.blocksize
= args
['blocksize']
964 self
.compress
= args
['compress']
970 log
.msg('SlaveDirectoryUploadCommand started')
972 self
.path
= os
.path
.join(self
.builder
.basedir
,
974 os
.path
.expanduser(self
.dirname
))
976 log
.msg("path: %r" % self
.path
)
978 # Create temporary archive
979 fd
, self
.tarname
= tempfile
.mkstemp()
980 fileobj
= os
.fdopen(fd
, 'w')
981 if self
.compress
== 'bz2':
983 elif self
.compress
== 'gz':
987 archive
= tarfile
.open(name
=self
.tarname
, mode
=mode
, fileobj
=fileobj
)
988 archive
.add(self
.path
, '')
993 self
.fp
= open(self
.tarname
, 'rb')
995 self
.sendStatus({'header': "sending %s" % self
.path
})
998 reactor
.callLater(0, self
._loop
, d
)
1000 # unpack the archive, but pass through any errors from _loop
1001 d1
= self
.writer
.callRemote("unpack")
1002 d1
.addErrback(log
.err
)
1003 d1
.addCallback(lambda ignored
: res
)
1005 d
.addCallback(unpack
)
1006 d
.addBoth(self
.finished
)
1009 def finished(self
, res
):
1010 os
.remove(self
.tarname
)
1012 log
.msg('finished: stderr=%r, rc=%r' % (self
.stderr
, self
.rc
))
1013 if self
.stderr
is None:
1014 self
.sendStatus({'rc': self
.rc
})
1016 self
.sendStatus({'stderr': self
.stderr
, 'rc': self
.rc
})
1019 registerSlaveCommand("uploadDirectory", SlaveDirectoryUploadCommand
, command_version
)
1022 class SlaveFileDownloadCommand(Command
):
1024 Download a file from master to slave
1027 - ['workdir']: base directory to use
1028 - ['slavedest']: name of the slave-side file to be created
1029 - ['reader']: RemoteReference to a transfer._FileReader object
1030 - ['maxsize']: max size (in bytes) of file to write
1031 - ['blocksize']: max size for each data block
1032 - ['mode']: access mode for the new file
1036 def setup(self
, args
):
1037 self
.workdir
= args
['workdir']
1038 self
.filename
= args
['slavedest']
1039 self
.reader
= args
['reader']
1040 self
.bytes_remaining
= args
['maxsize']
1041 self
.blocksize
= args
['blocksize']
1042 self
.mode
= args
['mode']
1048 log
.msg('SlaveFileDownloadCommand starting')
1051 self
.path
= os
.path
.join(self
.builder
.basedir
,
1053 os
.path
.expanduser(self
.filename
))
1055 dirname
= os
.path
.dirname(self
.path
)
1056 if not os
.path
.exists(dirname
):
1057 os
.makedirs(dirname
)
1060 self
.fp
= open(self
.path
, 'wb')
1062 log
.msg('Opened %r for download' % self
.path
)
1063 if self
.mode
is not None:
1064 # note: there is a brief window during which the new file
1065 # will have the buildslave's default (umask) mode before we
1066 # set the new one. Don't use this mode= feature to keep files
1067 # private: use the buildslave's umask for that instead. (it
1068 # is possible to call os.umask() before and after the open()
1069 # call, but cleaning up from exceptions properly is more of a
1070 # nuisance that way).
1071 os
.chmod(self
.path
, self
.mode
)
1073 # TODO: this still needs cleanup
1075 self
.stderr
= 'Cannot open file %r for download' % self
.path
1078 log
.msg('Cannot open file %r for download' % self
.path
)
1080 d
= defer
.Deferred()
1081 reactor
.callLater(0, self
._loop
, d
)
1083 # close the file, but pass through any errors from _loop
1084 d1
= self
.reader
.callRemote('close')
1085 d1
.addErrback(log
.err
)
1086 d1
.addCallback(lambda ignored
: res
)
1089 d
.addBoth(self
.finished
)
1092 def _loop(self
, fire_when_done
):
1093 d
= defer
.maybeDeferred(self
._readBlock
)
1094 def _done(finished
):
1096 fire_when_done
.callback(None)
1098 self
._loop
(fire_when_done
)
1100 fire_when_done
.errback(why
)
1101 d
.addCallbacks(_done
, _err
)
1104 def _readBlock(self
):
1105 """Read a block of data from the remote reader."""
1107 if self
.interrupted
or self
.fp
is None:
1109 log
.msg('SlaveFileDownloadCommand._readBlock(): end')
1112 length
= self
.blocksize
1113 if self
.bytes_remaining
is not None and length
> self
.bytes_remaining
:
1114 length
= self
.bytes_remaining
1117 if self
.stderr
is None:
1118 self
.stderr
= 'Maximum filesize reached, truncating file %r' \
1123 d
= self
.reader
.callRemote('read', length
)
1124 d
.addCallback(self
._writeData
)
1127 def _writeData(self
, data
):
1129 log
.msg('SlaveFileDownloadCommand._readBlock(): readlen=%d' %
1134 if self
.bytes_remaining
is not None:
1135 self
.bytes_remaining
= self
.bytes_remaining
- len(data
)
1136 assert self
.bytes_remaining
>= 0
1140 def interrupt(self
):
1142 log
.msg('interrupted')
1143 if self
.interrupted
:
1145 if self
.stderr
is None:
1146 self
.stderr
= 'Download of %r interrupted' % self
.path
1148 self
.interrupted
= True
1149 # now we wait for the next read request to return. _readBlock will
1150 # abandon the file when it sees self.interrupted set.
1152 def finished(self
, res
):
1153 if self
.fp
is not None:
1157 log
.msg('finished: stderr=%r, rc=%r' % (self
.stderr
, self
.rc
))
1158 if self
.stderr
is None:
1159 self
.sendStatus({'rc': self
.rc
})
1161 self
.sendStatus({'stderr': self
.stderr
, 'rc': self
.rc
})
1164 registerSlaveCommand("downloadFile", SlaveFileDownloadCommand
, command_version
)
1168 class SlaveShellCommand(Command
):
1169 """This is a Command which runs a shell command. The args dict contains
1172 - ['command'] (required): a shell command to run. If this is a string,
1173 it will be run with /bin/sh (['/bin/sh',
1174 '-c', command]). If it is a list
1175 (preferred), it will be used directly.
1176 - ['workdir'] (required): subdirectory in which the command will be
1177 run, relative to the builder dir
1178 - ['env']: a dict of environment variables to augment/replace
1179 os.environ . PYTHONPATH is treated specially, and
1180 should be a list of path components to be prepended to
1181 any existing PYTHONPATH environment variable.
1182 - ['initial_stdin']: a string which will be written to the command's
1183 stdin as soon as it starts
1184 - ['keep_stdin_open']: unless True, the command's stdin will be
1185 closed as soon as initial_stdin has been
1186 written. Set this to True if you plan to write
1187 to stdin after the command has been started.
1188 - ['want_stdout']: 0 if stdout should be thrown away
1189 - ['want_stderr']: 0 if stderr should be thrown away
1190 - ['usePTY']: True or False if the command should use a PTY (defaults to
1191 configuration of the slave)
1192 - ['not_really']: 1 to skip execution and return rc=0
1193 - ['timeout']: seconds of silence to tolerate before killing command
1194 - ['logfiles']: dict mapping LogFile name to the workdir-relative
1195 filename of a local log file. This local file will be
1196 watched just like 'tail -f', and all changes will be
1197 written to 'log' status updates.
1199 ShellCommand creates the following status messages:
1200 - {'stdout': data} : when stdout data is available
1201 - {'stderr': data} : when stderr data is available
1202 - {'header': data} : when headers (command start/stop) are available
1203 - {'log': (logfile_name, data)} : when log files have new contents
1204 - {'rc': rc} : when the process has terminated
1209 # args['workdir'] is relative to Builder directory, and is required.
1210 assert args
['workdir'] is not None
1211 workdir
= os
.path
.join(self
.builder
.basedir
, args
['workdir'])
1213 c
= ShellCommand(self
.builder
, args
['command'],
1214 workdir
, environ
=args
.get('env'),
1215 timeout
=args
.get('timeout', None),
1216 sendStdout
=args
.get('want_stdout', True),
1217 sendStderr
=args
.get('want_stderr', True),
1219 initialStdin
=args
.get('initial_stdin'),
1220 keepStdinOpen
=args
.get('keep_stdin_open'),
1221 logfiles
=args
.get('logfiles', {}),
1222 usePTY
=args
.get('usePTY', "slave-config"),
1225 d
= self
.command
.start()
1228 def interrupt(self
):
1229 self
.interrupted
= True
1230 self
.command
.kill("command interrupted")
1232 def writeStdin(self
, data
):
1233 self
.command
.writeStdin(data
)
1235 def closeStdin(self
):
1236 self
.command
.closeStdin()
1238 registerSlaveCommand("shell", SlaveShellCommand
, command_version
)
1241 class DummyCommand(Command
):
1243 I am a dummy no-op command that by default takes 5 seconds to complete.
1244 See L{buildbot.steps.dummy.RemoteDummy}
1248 self
.d
= defer
.Deferred()
1249 log
.msg(" starting dummy command [%s]" % self
.stepId
)
1250 self
.timer
= reactor
.callLater(1, self
.doStatus
)
1253 def interrupt(self
):
1254 if self
.interrupted
:
1258 self
.interrupted
= True
1262 log
.msg(" sending intermediate status")
1263 self
.sendStatus({'stdout': 'data'})
1264 timeout
= self
.args
.get('timeout', 5) + 1
1265 self
.timer
= reactor
.callLater(timeout
- 1, self
.finished
)
1268 log
.msg(" dummy command finished [%s]" % self
.stepId
)
1269 if self
.interrupted
:
1270 self
.sendStatus({'rc': 1})
1272 self
.sendStatus({'rc': 0})
1275 registerSlaveCommand("dummy", DummyCommand
, command_version
)
1278 # this maps handle names to a callable. When the WaitCommand starts, this
1279 # callable is invoked with no arguments. It should return a Deferred. When
1280 # that Deferred fires, our WaitCommand will finish.
1281 waitCommandRegistry
= {}
1283 class WaitCommand(Command
):
1285 I am a dummy command used by the buildbot unit test suite. I want for the
1286 unit test to tell us to finish. See L{buildbot.steps.dummy.Wait}
1290 self
.d
= defer
.Deferred()
1291 log
.msg(" starting wait command [%s]" % self
.stepId
)
1292 handle
= self
.args
['handle']
1293 cb
= waitCommandRegistry
[handle
]
1294 del waitCommandRegistry
[handle
]
1296 log
.msg(" wait-%s starting" % (handle
,))
1299 log
.msg(" wait-%s finishing: %s" % (handle
, res
))
1302 d
.addCallbacks(self
.finished
, self
.failed
)
1303 reactor
.callLater(0, _called
)
1306 def interrupt(self
):
1307 log
.msg(" wait command interrupted")
1308 if self
.interrupted
:
1310 self
.interrupted
= True
1311 self
.finished("interrupted")
1313 def finished(self
, res
):
1314 log
.msg(" wait command finished [%s]" % self
.stepId
)
1315 if self
.interrupted
:
1316 self
.sendStatus({'rc': 2})
1318 self
.sendStatus({'rc': 0})
1320 def failed(self
, why
):
1321 log
.msg(" wait command failed [%s]" % self
.stepId
)
1322 self
.sendStatus({'rc': 1})
1325 registerSlaveCommand("dummy.wait", WaitCommand
, command_version
)
1328 class SourceBase(Command
):
1329 """Abstract base class for Version Control System operations (checkout
1330 and update). This class extracts the following arguments from the
1331 dictionary received from the master:
1333 - ['workdir']: (required) the subdirectory where the buildable sources
1336 - ['mode']: one of update/copy/clobber/export, defaults to 'update'
1338 - ['revision']: If not None, this is an int or string which indicates
1339 which sources (along a time-like axis) should be used.
1340 It is the thing you provide as the CVS -r or -D
1343 - ['patch']: If not None, this is a tuple of (striplevel, patch)
1344 which contains a patch that should be applied after the
1345 checkout has occurred. Once applied, the tree is no
1346 longer eligible for use with mode='update', and it only
1347 makes sense to use this in conjunction with a
1348 ['revision'] argument. striplevel is an int, and patch
1349 is a string in standard unified diff format. The patch
1350 will be applied with 'patch -p%d <PATCH', with
1351 STRIPLEVEL substituted as %d. The command will fail if
1352 the patch process fails (rejected hunks).
1354 - ['timeout']: seconds of silence tolerated before we kill off the
1357 - ['retry']: If not None, this is a tuple of (delay, repeats)
1358 which means that any failed VC updates should be
1359 reattempted, up to REPEATS times, after a delay of
1360 DELAY seconds. This is intended to deal with slaves
1361 that experience transient network failures.
1366 def setup(self
, args
):
1367 # if we need to parse the output, use this environment. Otherwise
1368 # command output will be in whatever the buildslave's native language
1370 self
.env
= os
.environ
.copy()
1371 self
.env
['LC_MESSAGES'] = "C"
1373 self
.workdir
= args
['workdir']
1374 self
.mode
= args
.get('mode', "update")
1375 self
.revision
= args
.get('revision')
1376 self
.patch
= args
.get('patch')
1377 self
.timeout
= args
.get('timeout', 120)
1378 self
.retry
= args
.get('retry')
1379 # VC-specific subclasses should override this to extract more args.
1380 # Make sure to upcall!
1383 self
.sendStatus({'header': "starting " + self
.header
+ "\n"})
1386 # self.srcdir is where the VC system should put the sources
1387 if self
.mode
== "copy":
1388 self
.srcdir
= "source" # hardwired directory name, sorry
1390 self
.srcdir
= self
.workdir
1391 self
.sourcedatafile
= os
.path
.join(self
.builder
.basedir
,
1393 ".buildbot-sourcedata")
1395 d
= defer
.succeed(None)
1396 self
.maybeClobber(d
)
1397 if not (self
.sourcedirIsUpdateable() and self
.sourcedataMatches()):
1398 # the directory cannot be updated, so we have to clobber it.
1399 # Perhaps the master just changed modes from 'export' to
1401 d
.addCallback(self
.doClobber
, self
.srcdir
)
1403 d
.addCallback(self
.doVC
)
1405 if self
.mode
== "copy":
1406 d
.addCallback(self
.doCopy
)
1408 d
.addCallback(self
.doPatch
)
1409 d
.addCallbacks(self
._sendRC
, self
._checkAbandoned
)
1412 def maybeClobber(self
, d
):
1413 # do we need to clobber anything?
1414 if self
.mode
in ("copy", "clobber", "export"):
1415 d
.addCallback(self
.doClobber
, self
.workdir
)
1417 def interrupt(self
):
1418 self
.interrupted
= True
1420 self
.command
.kill("command interrupted")
1422 def doVC(self
, res
):
1423 if self
.interrupted
:
1424 raise AbandonChain(1)
1425 if self
.sourcedirIsUpdateable() and self
.sourcedataMatches():
1426 d
= self
.doVCUpdate()
1427 d
.addCallback(self
.maybeDoVCFallback
)
1430 d
.addBoth(self
.maybeDoVCRetry
)
1431 d
.addCallback(self
._abandonOnFailure
)
1432 d
.addCallback(self
._handleGotRevision
)
1433 d
.addCallback(self
.writeSourcedata
)
1436 def sourcedataMatches(self
):
1438 olddata
= open(self
.sourcedatafile
, "r").read()
1439 if olddata
!= self
.sourcedata
:
1445 def _handleGotRevision(self
, res
):
1446 d
= defer
.maybeDeferred(self
.parseGotRevision
)
1447 d
.addCallback(lambda got_revision
:
1448 self
.sendStatus({'got_revision': got_revision
}))
1451 def parseGotRevision(self
):
1452 """Override this in a subclass. It should return a string that
1453 represents which revision was actually checked out, or a Deferred
1454 that will fire with such a string. If, in a future build, you were to
1455 pass this 'got_revision' string in as the 'revision' component of a
1456 SourceStamp, you should wind up with the same source code as this
1457 checkout just obtained.
1459 It is probably most useful to scan self.command.stdout for a string
1460 of some sort. Be sure to set keepStdout=True on the VC command that
1461 you run, so that you'll have something available to look at.
1463 If this information is unavailable, just return None."""
1467 def writeSourcedata(self
, res
):
1468 open(self
.sourcedatafile
, "w").write(self
.sourcedata
)
1471 def sourcedirIsUpdateable(self
):
1472 raise NotImplementedError("this must be implemented in a subclass")
1474 def doVCUpdate(self
):
1475 raise NotImplementedError("this must be implemented in a subclass")
1478 raise NotImplementedError("this must be implemented in a subclass")
1480 def maybeDoVCFallback(self
, rc
):
1481 if type(rc
) is int and rc
== 0:
1483 if self
.interrupted
:
1484 raise AbandonChain(1)
1485 msg
= "update failed, clobbering and trying again"
1486 self
.sendStatus({'header': msg
+ "\n"})
1488 d
= self
.doClobber(None, self
.srcdir
)
1489 d
.addCallback(self
.doVCFallback2
)
1492 def doVCFallback2(self
, res
):
1493 msg
= "now retrying VC operation"
1494 self
.sendStatus({'header': msg
+ "\n"})
1497 d
.addBoth(self
.maybeDoVCRetry
)
1498 d
.addCallback(self
._abandonOnFailure
)
1501 def maybeDoVCRetry(self
, res
):
1502 """We get here somewhere after a VC chain has finished. res could
1505 - 0: the operation was successful
1506 - nonzero: the operation failed. retry if possible
1507 - AbandonChain: the operation failed, someone else noticed. retry.
1508 - Failure: some other exception, re-raise
1511 if isinstance(res
, failure
.Failure
):
1512 if self
.interrupted
:
1513 return res
# don't re-try interrupted builds
1514 res
.trap(AbandonChain
)
1516 if type(res
) is int and res
== 0:
1518 if self
.interrupted
:
1519 raise AbandonChain(1)
1520 # if we get here, we should retry, if possible
1522 delay
, repeats
= self
.retry
1524 self
.retry
= (delay
, repeats
-1)
1525 msg
= ("update failed, trying %d more times after %d seconds"
1527 self
.sendStatus({'header': msg
+ "\n"})
1529 d
= defer
.Deferred()
1530 self
.maybeClobber(d
)
1531 d
.addCallback(lambda res
: self
.doVCFull())
1532 d
.addBoth(self
.maybeDoVCRetry
)
1533 reactor
.callLater(delay
, d
.callback
, None)
1537 def doClobber(self
, dummy
, dirname
):
1538 # TODO: remove the old tree in the background
1539 ## workdir = os.path.join(self.builder.basedir, self.workdir)
1540 ## deaddir = self.workdir + ".deleting"
1541 ## if os.path.isdir(workdir):
1543 ## os.rename(workdir, deaddir)
1544 ## # might fail if deaddir already exists: previous deletion
1545 ## # hasn't finished yet
1546 ## # start the deletion in the background
1547 ## # TODO: there was a solaris/NetApp/NFS problem where a
1548 ## # process that was still running out of the directory we're
1549 ## # trying to delete could prevent the rm-rf from working. I
1550 ## # think it stalled the rm, but maybe it just died with
1551 ## # permission issues. Try to detect this.
1552 ## os.commands("rm -rf %s &" % deaddir)
1554 ## # fall back to sequential delete-then-checkout
1556 d
= os
.path
.join(self
.builder
.basedir
, dirname
)
1557 if runtime
.platformType
!= "posix":
1558 # if we're running on w32, use rmtree instead. It will block,
1559 # but hopefully it won't take too long.
1561 return defer
.succeed(0)
1562 command
= ["rm", "-rf", d
]
1563 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
1564 sendRC
=0, timeout
=self
.timeout
, usePTY
=False)
1567 # sendRC=0 means the rm command will send stdout/stderr to the
1568 # master, but not the rc=0 when it finishes. That job is left to
1571 d
.addCallback(self
._abandonOnFailure
)
1574 def doCopy(self
, res
):
1575 # now copy tree to workdir
1576 fromdir
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
1577 todir
= os
.path
.join(self
.builder
.basedir
, self
.workdir
)
1578 if runtime
.platformType
!= "posix":
1579 self
.sendStatus({'header': "Since we're on a non-POSIX platform, "
1580 "we're not going to try to execute cp in a subprocess, but instead "
1581 "use shutil.copytree(), which will block until it is complete. "
1582 "fromdir: %s, todir: %s\n" % (fromdir
, todir
)})
1583 shutil
.copytree(fromdir
, todir
)
1584 return defer
.succeed(0)
1586 if not os
.path
.exists(os
.path
.dirname(todir
)):
1587 os
.makedirs(os
.path
.dirname(todir
))
1588 if os
.path
.exists(todir
):
1589 # I don't think this happens, but just in case..
1590 log
.msg("cp target '%s' already exists -- cp will not do what you think!" % todir
)
1592 command
= ['cp', '-R', '-P', '-p', fromdir
, todir
]
1593 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
1594 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
1597 d
.addCallback(self
._abandonOnFailure
)
1600 def doPatch(self
, res
):
1601 patchlevel
, diff
= self
.patch
1602 command
= [getCommand("patch"), '-p%d' % patchlevel
]
1603 dir = os
.path
.join(self
.builder
.basedir
, self
.workdir
)
1604 # mark the directory so we don't try to update it later
1605 open(os
.path
.join(dir, ".buildbot-patched"), "w").write("patched\n")
1606 # now apply the patch
1607 c
= ShellCommand(self
.builder
, command
, dir,
1608 sendRC
=False, timeout
=self
.timeout
,
1609 initialStdin
=diff
, usePTY
=False)
1612 d
.addCallback(self
._abandonOnFailure
)
1616 class CVS(SourceBase
):
1617 """CVS-specific VC operation. In addition to the arguments handled by
1618 SourceBase, this command reads the following keys:
1620 ['cvsroot'] (required): the CVSROOT repository string
1621 ['cvsmodule'] (required): the module to be retrieved
1622 ['branch']: a '-r' tag or branch name to use for the checkout/update
1623 ['login']: a string for use as a password to 'cvs login'
1624 ['global_options']: a list of strings to use before the CVS verb
1625 ['checkout_options']: a list of strings to use after checkout,
1626 but before revision and branch specifiers
1629 header
= "cvs operation"
1631 def setup(self
, args
):
1632 SourceBase
.setup(self
, args
)
1633 self
.vcexe
= getCommand("cvs")
1634 self
.cvsroot
= args
['cvsroot']
1635 self
.cvsmodule
= args
['cvsmodule']
1636 self
.global_options
= args
.get('global_options', [])
1637 self
.checkout_options
= args
.get('checkout_options', [])
1638 self
.branch
= args
.get('branch')
1639 self
.login
= args
.get('login')
1640 self
.sourcedata
= "%s\n%s\n%s\n" % (self
.cvsroot
, self
.cvsmodule
,
1643 def sourcedirIsUpdateable(self
):
1644 if os
.path
.exists(os
.path
.join(self
.builder
.basedir
,
1645 self
.srcdir
, ".buildbot-patched")):
1647 return os
.path
.isdir(os
.path
.join(self
.builder
.basedir
,
1648 self
.srcdir
, "CVS"))
1651 if self
.login
is not None:
1652 # need to do a 'cvs login' command first
1653 d
= self
.builder
.basedir
1654 command
= ([self
.vcexe
, '-d', self
.cvsroot
] + self
.global_options
1656 c
= ShellCommand(self
.builder
, command
, d
,
1657 sendRC
=False, timeout
=self
.timeout
,
1658 initialStdin
=self
.login
+"\n", usePTY
=False)
1661 d
.addCallback(self
._abandonOnFailure
)
1662 d
.addCallback(self
._didLogin
)
1665 return self
._didLogin
(None)
1667 def _didLogin(self
, res
):
1668 # now we really start
1669 return SourceBase
.start(self
)
1671 def doVCUpdate(self
):
1672 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
1673 command
= [self
.vcexe
, '-z3'] + self
.global_options
+ ['update', '-dP']
1675 command
+= ['-r', self
.branch
]
1677 command
+= ['-D', self
.revision
]
1678 c
= ShellCommand(self
.builder
, command
, d
,
1679 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
1684 d
= self
.builder
.basedir
1685 if self
.mode
== "export":
1689 command
= ([self
.vcexe
, '-d', self
.cvsroot
, '-z3'] +
1690 self
.global_options
+
1691 [verb
, '-d', self
.srcdir
])
1693 if verb
== "checkout":
1694 command
+= self
.checkout_options
1696 command
+= ['-r', self
.branch
]
1698 command
+= ['-D', self
.revision
]
1699 command
+= [self
.cvsmodule
]
1701 c
= ShellCommand(self
.builder
, command
, d
,
1702 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
1706 def parseGotRevision(self
):
1707 # CVS does not have any kind of revision stamp to speak of. We return
1708 # the current timestamp as a best-effort guess, but this depends upon
1709 # the local system having a clock that is
1710 # reasonably-well-synchronized with the repository.
1711 return time
.strftime("%Y-%m-%d %H:%M:%S +0000", time
.gmtime())
1713 registerSlaveCommand("cvs", CVS
, command_version
)
1715 class SVN(SourceBase
):
1716 """Subversion-specific VC operation. In addition to the arguments
1717 handled by SourceBase, this command reads the following keys:
1719 ['svnurl'] (required): the SVN repository string
1720 ['username'] Username passed to the svn command
1721 ['password'] Password passed to the svn command
1724 header
= "svn operation"
1726 def setup(self
, args
):
1727 SourceBase
.setup(self
, args
)
1728 self
.vcexe
= getCommand("svn")
1729 self
.svnurl
= args
['svnurl']
1730 self
.sourcedata
= "%s\n" % self
.svnurl
1733 if args
.has_key('username'):
1734 self
.svn_args
.extend(["--username", args
['username']])
1735 if args
.has_key('password'):
1736 self
.svn_args
.extend(["--password", Obfuscated(args
['password'], "XXXX")])
1737 if args
.get('extra_args', None) is not None:
1738 self
.svn_args
.extend(args
['extra_args'])
1740 def sourcedirIsUpdateable(self
):
1741 if os
.path
.exists(os
.path
.join(self
.builder
.basedir
,
1742 self
.srcdir
, ".buildbot-patched")):
1744 return os
.path
.isdir(os
.path
.join(self
.builder
.basedir
,
1745 self
.srcdir
, ".svn"))
1747 def doVCUpdate(self
):
1748 revision
= self
.args
['revision'] or 'HEAD'
1749 # update: possible for mode in ('copy', 'update')
1750 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
1751 command
= [self
.vcexe
, 'update'] + \
1753 ['--revision', str(revision
),
1754 '--non-interactive', '--no-auth-cache']
1755 c
= ShellCommand(self
.builder
, command
, d
,
1756 sendRC
=False, timeout
=self
.timeout
,
1757 keepStdout
=True, usePTY
=False)
1762 revision
= self
.args
['revision'] or 'HEAD'
1763 d
= self
.builder
.basedir
1764 if self
.mode
== "export":
1765 command
= [self
.vcexe
, 'export'] + \
1767 ['--revision', str(revision
),
1768 '--non-interactive', '--no-auth-cache',
1769 self
.svnurl
, self
.srcdir
]
1771 # mode=='clobber', or copy/update on a broken workspace
1772 command
= [self
.vcexe
, 'checkout'] + \
1774 ['--revision', str(revision
),
1775 '--non-interactive', '--no-auth-cache',
1776 self
.svnurl
, self
.srcdir
]
1777 c
= ShellCommand(self
.builder
, command
, d
,
1778 sendRC
=False, timeout
=self
.timeout
,
1779 keepStdout
=True, usePTY
=False)
1783 def getSvnVersionCommand(self
):
1785 Get the (shell) command used to determine SVN revision number
1788 return: list of strings, passable as the command argument to ShellCommand
1790 # svn checkout operations finish with 'Checked out revision 16657.'
1791 # svn update operations finish the line 'At revision 16654.'
1792 # But we don't use those. Instead, run 'svnversion'.
1793 svnversion_command
= getCommand("svnversion")
1794 # older versions of 'svnversion' (1.1.4) require the WC_PATH
1795 # argument, newer ones (1.3.1) do not.
1796 return [svnversion_command
, "."]
1798 def parseGotRevision(self
):
1799 c
= ShellCommand(self
.builder
,
1800 self
.getSvnVersionCommand(),
1801 os
.path
.join(self
.builder
.basedir
, self
.srcdir
),
1803 sendStdout
=False, sendStderr
=False, sendRC
=False,
1804 keepStdout
=True, usePTY
=False)
1807 r_raw
= c
.stdout
.strip()
1808 # Extract revision from the version "number" string
1809 r
= r_raw
.rstrip('MS')
1810 r
= r
.split(':')[-1]
1813 got_version
= int(r
)
1815 msg
=("SVN.parseGotRevision unable to parse output "
1816 "of svnversion: '%s'" % r_raw
)
1818 self
.sendStatus({'header': msg
+ "\n"})
1820 d
.addCallback(_parse
)
1824 registerSlaveCommand("svn", SVN
, command_version
)
1826 class Darcs(SourceBase
):
1827 """Darcs-specific VC operation. In addition to the arguments
1828 handled by SourceBase, this command reads the following keys:
1830 ['repourl'] (required): the Darcs repository string
1833 header
= "darcs operation"
1835 def setup(self
, args
):
1836 SourceBase
.setup(self
, args
)
1837 self
.vcexe
= getCommand("darcs")
1838 self
.repourl
= args
['repourl']
1839 self
.sourcedata
= "%s\n" % self
.repourl
1840 self
.revision
= self
.args
.get('revision')
1842 def sourcedirIsUpdateable(self
):
1843 if os
.path
.exists(os
.path
.join(self
.builder
.basedir
,
1844 self
.srcdir
, ".buildbot-patched")):
1847 # checking out a specific revision requires a full 'darcs get'
1849 return os
.path
.isdir(os
.path
.join(self
.builder
.basedir
,
1850 self
.srcdir
, "_darcs"))
1852 def doVCUpdate(self
):
1853 assert not self
.revision
1854 # update: possible for mode in ('copy', 'update')
1855 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
1856 command
= [self
.vcexe
, 'pull', '--all', '--verbose']
1857 c
= ShellCommand(self
.builder
, command
, d
,
1858 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
1863 # checkout or export
1864 d
= self
.builder
.basedir
1865 command
= [self
.vcexe
, 'get', '--verbose', '--partial',
1866 '--repo-name', self
.srcdir
]
1868 # write the context to a file
1869 n
= os
.path
.join(self
.builder
.basedir
, ".darcs-context")
1871 f
.write(self
.revision
)
1873 # tell Darcs to use that context
1874 command
.append('--context')
1876 command
.append(self
.repourl
)
1878 c
= ShellCommand(self
.builder
, command
, d
,
1879 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
1883 d
.addCallback(self
.removeContextFile
, n
)
1886 def removeContextFile(self
, res
, n
):
1890 def parseGotRevision(self
):
1891 # we use 'darcs context' to find out what we wound up with
1892 command
= [self
.vcexe
, "changes", "--context"]
1893 c
= ShellCommand(self
.builder
, command
,
1894 os
.path
.join(self
.builder
.basedir
, self
.srcdir
),
1896 sendStdout
=False, sendStderr
=False, sendRC
=False,
1897 keepStdout
=True, usePTY
=False)
1899 d
.addCallback(lambda res
: c
.stdout
)
1902 registerSlaveCommand("darcs", Darcs
, command_version
)
1904 class Monotone(SourceBase
):
1905 """Monotone-specific VC operation. In addition to the arguments handled
1906 by SourceBase, this command reads the following keys:
1908 ['server_addr'] (required): the address of the server to pull from
1909 ['branch'] (required): the branch the revision is on
1910 ['db_path'] (required): the local database path to use
1911 ['revision'] (required): the revision to check out
1912 ['monotone']: (required): path to monotone executable
1915 header
= "monotone operation"
1917 def setup(self
, args
):
1918 SourceBase
.setup(self
, args
)
1919 self
.server_addr
= args
["server_addr"]
1920 self
.branch
= args
["branch"]
1921 self
.db_path
= args
["db_path"]
1922 self
.revision
= args
["revision"]
1923 self
.monotone
= args
["monotone"]
1924 self
._made
_fulls
= False
1925 self
._pull
_timeout
= args
["timeout"]
1927 def _makefulls(self
):
1928 if not self
._made
_fulls
:
1929 basedir
= self
.builder
.basedir
1930 self
.full_db_path
= os
.path
.join(basedir
, self
.db_path
)
1931 self
.full_srcdir
= os
.path
.join(basedir
, self
.srcdir
)
1932 self
._made
_fulls
= True
1934 def sourcedirIsUpdateable(self
):
1936 if os
.path
.exists(os
.path
.join(self
.full_srcdir
,
1937 ".buildbot_patched")):
1939 return (os
.path
.isfile(self
.full_db_path
)
1940 and os
.path
.isdir(os
.path
.join(self
.full_srcdir
, "MT")))
1942 def doVCUpdate(self
):
1943 return self
._withFreshDb
(self
._doUpdate
)
1945 def _doUpdate(self
):
1946 # update: possible for mode in ('copy', 'update')
1947 command
= [self
.monotone
, "update",
1948 "-r", self
.revision
,
1950 c
= ShellCommand(self
.builder
, command
, self
.full_srcdir
,
1951 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
1956 return self
._withFreshDb
(self
._doFull
)
1959 command
= [self
.monotone
, "--db=" + self
.full_db_path
,
1961 "-r", self
.revision
,
1964 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
1965 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
1969 def _withFreshDb(self
, callback
):
1971 # first ensure the db exists and is usable
1972 if os
.path
.isfile(self
.full_db_path
):
1973 # already exists, so run 'db migrate' in case monotone has been
1975 command
= [self
.monotone
, "db", "migrate",
1976 "--db=" + self
.full_db_path
]
1978 # We'll be doing an initial pull, so up the timeout to 3 hours to
1979 # make sure it will have time to complete.
1980 self
._pull
_timeout
= max(self
._pull
_timeout
, 3 * 60 * 60)
1981 self
.sendStatus({"header": "creating database %s\n"
1982 % (self
.full_db_path
,)})
1983 command
= [self
.monotone
, "db", "init",
1984 "--db=" + self
.full_db_path
]
1985 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
1986 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
1989 d
.addCallback(self
._abandonOnFailure
)
1990 d
.addCallback(self
._didDbInit
)
1991 d
.addCallback(self
._didPull
, callback
)
1994 def _didDbInit(self
, res
):
1995 command
= [self
.monotone
, "--db=" + self
.full_db_path
,
1996 "pull", "--ticker=dot", self
.server_addr
, self
.branch
]
1997 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
1998 sendRC
=False, timeout
=self
._pull
_timeout
, usePTY
=False)
1999 self
.sendStatus({"header": "pulling %s from %s\n"
2000 % (self
.branch
, self
.server_addr
)})
2004 def _didPull(self
, res
, callback
):
2007 registerSlaveCommand("monotone", Monotone
, command_version
)
2010 class Git(SourceBase
):
2011 """Git specific VC operation. In addition to the arguments
2012 handled by SourceBase, this command reads the following keys:
2014 ['repourl'] (required): the upstream GIT repository string
2015 ['branch'] (optional): which version (i.e. branch or tag) to
2016 retrieve. Default: "master".
2017 ['submodules'] (optional): whether to initialize and update
2018 submodules. Default: False.
2021 header
= "git operation"
2023 def setup(self
, args
):
2024 SourceBase
.setup(self
, args
)
2025 self
.repourl
= args
['repourl']
2026 self
.branch
= args
.get('branch')
2028 self
.branch
= "master"
2029 self
.sourcedata
= "%s %s\n" % (self
.repourl
, self
.branch
)
2030 self
.submodules
= args
.get('submodules')
2032 def _fullSrcdir(self
):
2033 return os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2035 def _commitSpec(self
):
2037 return self
.revision
2040 def sourcedirIsUpdateable(self
):
2041 return os
.path
.isdir(os
.path
.join(self
._fullSrcdir
(), ".git"))
2043 def readSourcedata(self
):
2044 return open(self
.sourcedatafile
, "r").read()
2046 # If the repourl matches the sourcedata file, then
2047 # we can say that the sourcedata matches. We can
2048 # ignore branch changes, since Git can work with
2049 # many branches fetched, and we deal with it properly
2051 def sourcedataMatches(self
):
2053 olddata
= self
.readSourcedata()
2054 if not olddata
.startswith(self
.repourl
+' '):
2060 def _didSubmodules(self
, res
):
2061 command
= ['git', 'submodule', 'update', '--init']
2062 c
= ShellCommand(self
.builder
, command
, self
._fullSrcdir
(),
2063 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
2067 def _didFetch(self
, res
):
2069 head
= self
.revision
2073 command
= ['git', 'reset', '--hard', head
]
2074 c
= ShellCommand(self
.builder
, command
, self
._fullSrcdir
(),
2075 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
2079 d
.addCallback(self
._abandonOnFailure
)
2080 d
.addCallback(self
._didSubmodules
)
2083 # Update first runs "git clean", removing local changes, This,
2084 # combined with the later "git reset" equates clobbering the repo,
2085 # but it's much more efficient.
2086 def doVCUpdate(self
):
2087 command
= ['git', 'clean', '-f', '-d', '-x']
2088 c
= ShellCommand(self
.builder
, command
, self
._fullSrcdir
(),
2089 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
2092 d
.addCallback(self
._abandonOnFailure
)
2093 d
.addCallback(self
._didClean
)
2096 def _didClean(self
, dummy
):
2097 command
= ['git', 'fetch', '-t', self
.repourl
, self
.branch
]
2098 self
.sendStatus({"header": "fetching branch %s from %s\n"
2099 % (self
.branch
, self
.repourl
)})
2100 c
= ShellCommand(self
.builder
, command
, self
._fullSrcdir
(),
2101 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
2104 d
.addCallback(self
._abandonOnFailure
)
2105 d
.addCallback(self
._didFetch
)
2108 def _didInit(self
, res
):
2109 return self
.doVCUpdate()
2112 os
.mkdir(self
._fullSrcdir
())
2113 c
= ShellCommand(self
.builder
, ['git', 'init'], self
._fullSrcdir
(),
2114 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
2117 d
.addCallback(self
._abandonOnFailure
)
2118 d
.addCallback(self
._didInit
)
2121 def parseGotRevision(self
):
2122 command
= ['git', 'rev-parse', 'HEAD']
2123 c
= ShellCommand(self
.builder
, command
, self
._fullSrcdir
(),
2124 sendRC
=False, keepStdout
=True, usePTY
=False)
2127 hash = c
.stdout
.strip()
2131 d
.addCallback(_parse
)
2134 registerSlaveCommand("git", Git
, command_version
)
2136 class Arch(SourceBase
):
2137 """Arch-specific (tla-specific) VC operation. In addition to the
2138 arguments handled by SourceBase, this command reads the following keys:
2140 ['url'] (required): the repository string
2141 ['version'] (required): which version (i.e. branch) to retrieve
2142 ['revision'] (optional): the 'patch-NN' argument to check out
2143 ['archive']: the archive name to use. If None, use the archive's default
2144 ['build-config']: if present, give to 'tla build-config' after checkout
2147 header
= "arch operation"
2150 def setup(self
, args
):
2151 SourceBase
.setup(self
, args
)
2152 self
.vcexe
= getCommand("tla")
2153 self
.archive
= args
.get('archive')
2154 self
.url
= args
['url']
2155 self
.version
= args
['version']
2156 self
.revision
= args
.get('revision')
2157 self
.buildconfig
= args
.get('build-config')
2158 self
.sourcedata
= "%s\n%s\n%s\n" % (self
.url
, self
.version
,
2161 def sourcedirIsUpdateable(self
):
2163 # Arch cannot roll a directory backwards, so if they ask for a
2164 # specific revision, clobber the directory. Technically this
2165 # could be limited to the cases where the requested revision is
2166 # later than our current one, but it's too hard to extract the
2167 # current revision from the tree.
2169 if os
.path
.exists(os
.path
.join(self
.builder
.basedir
,
2170 self
.srcdir
, ".buildbot-patched")):
2172 return os
.path
.isdir(os
.path
.join(self
.builder
.basedir
,
2173 self
.srcdir
, "{arch}"))
2175 def doVCUpdate(self
):
2176 # update: possible for mode in ('copy', 'update')
2177 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2178 command
= [self
.vcexe
, 'replay']
2180 command
.append(self
.revision
)
2181 c
= ShellCommand(self
.builder
, command
, d
,
2182 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
2187 # to do a checkout, we must first "register" the archive by giving
2188 # the URL to tla, which will go to the repository at that URL and
2189 # figure out the archive name. tla will tell you the archive name
2190 # when it is done, and all further actions must refer to this name.
2192 command
= [self
.vcexe
, 'register-archive', '--force', self
.url
]
2193 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
2194 sendRC
=False, keepStdout
=True,
2195 timeout
=self
.timeout
, usePTY
=False)
2198 d
.addCallback(self
._abandonOnFailure
)
2199 d
.addCallback(self
._didRegister
, c
)
2202 def _didRegister(self
, res
, c
):
2203 # find out what tla thinks the archive name is. If the user told us
2204 # to use something specific, make sure it matches.
2205 r
= re
.search(r
'Registering archive: (\S+)\s*$', c
.stdout
)
2207 msg
= "tla reports archive name is '%s'" % r
.group(1)
2209 self
.builder
.sendUpdate({'header': msg
+"\n"})
2210 if self
.archive
and r
.group(1) != self
.archive
:
2211 msg
= (" mismatch, we wanted an archive named '%s'"
2214 self
.builder
.sendUpdate({'header': msg
+"\n"})
2215 raise AbandonChain(-1)
2216 self
.archive
= r
.group(1)
2217 assert self
.archive
, "need archive name to continue"
2218 return self
._doGet
()
2223 ver
+= "--%s" % self
.revision
2224 command
= [self
.vcexe
, 'get', '--archive', self
.archive
,
2227 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
2228 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
2231 d
.addCallback(self
._abandonOnFailure
)
2232 if self
.buildconfig
:
2233 d
.addCallback(self
._didGet
)
2236 def _didGet(self
, res
):
2237 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2238 command
= [self
.vcexe
, 'build-config', self
.buildconfig
]
2239 c
= ShellCommand(self
.builder
, command
, d
,
2240 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
2243 d
.addCallback(self
._abandonOnFailure
)
2246 def parseGotRevision(self
):
2247 # using code from tryclient.TlaExtractor
2248 # 'tla logs --full' gives us ARCHIVE/BRANCH--REVISION
2249 # 'tla logs' gives us REVISION
2250 command
= [self
.vcexe
, "logs", "--full", "--reverse"]
2251 c
= ShellCommand(self
.builder
, command
,
2252 os
.path
.join(self
.builder
.basedir
, self
.srcdir
),
2254 sendStdout
=False, sendStderr
=False, sendRC
=False,
2255 keepStdout
=True, usePTY
=False)
2258 tid
= c
.stdout
.split("\n")[0].strip()
2259 slash
= tid
.index("/")
2260 dd
= tid
.rindex("--")
2261 #branch = tid[slash+1:dd]
2262 baserev
= tid
[dd
+2:]
2264 d
.addCallback(_parse
)
2267 registerSlaveCommand("arch", Arch
, command_version
)
2270 """Bazaar (/usr/bin/baz) is an alternative client for Arch repositories.
2271 It is mostly option-compatible, but archive registration is different
2272 enough to warrant a separate Command.
2274 ['archive'] (required): the name of the archive being used
2277 def setup(self
, args
):
2278 Arch
.setup(self
, args
)
2279 self
.vcexe
= getCommand("baz")
2280 # baz doesn't emit the repository name after registration (and
2281 # grepping through the output of 'baz archives' is too hard), so we
2282 # require that the buildmaster configuration to provide both the
2283 # archive name and the URL.
2284 self
.archive
= args
['archive'] # required for Baz
2285 self
.sourcedata
= "%s\n%s\n%s\n" % (self
.url
, self
.version
,
2288 # in _didRegister, the regexp won't match, so we'll stick with the name
2292 # baz prefers ARCHIVE/VERSION. This will work even if
2293 # my-default-archive is not set.
2294 ver
= self
.archive
+ "/" + self
.version
2296 ver
+= "--%s" % self
.revision
2297 command
= [self
.vcexe
, 'get', '--no-pristine',
2299 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
2300 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
2303 d
.addCallback(self
._abandonOnFailure
)
2304 if self
.buildconfig
:
2305 d
.addCallback(self
._didGet
)
2308 def parseGotRevision(self
):
2309 # using code from tryclient.BazExtractor
2310 command
= [self
.vcexe
, "tree-id"]
2311 c
= ShellCommand(self
.builder
, command
,
2312 os
.path
.join(self
.builder
.basedir
, self
.srcdir
),
2314 sendStdout
=False, sendStderr
=False, sendRC
=False,
2315 keepStdout
=True, usePTY
=False)
2318 tid
= c
.stdout
.strip()
2319 slash
= tid
.index("/")
2320 dd
= tid
.rindex("--")
2321 #branch = tid[slash+1:dd]
2322 baserev
= tid
[dd
+2:]
2324 d
.addCallback(_parse
)
2327 registerSlaveCommand("bazaar", Bazaar
, command_version
)
2330 class Bzr(SourceBase
):
2331 """bzr-specific VC operation. In addition to the arguments
2332 handled by SourceBase, this command reads the following keys:
2334 ['repourl'] (required): the Bzr repository string
2337 header
= "bzr operation"
2339 def setup(self
, args
):
2340 SourceBase
.setup(self
, args
)
2341 self
.vcexe
= getCommand("bzr")
2342 self
.repourl
= args
['repourl']
2343 self
.sourcedata
= "%s\n" % self
.repourl
2344 self
.revision
= self
.args
.get('revision')
2345 self
.forceSharedRepo
= args
.get('forceSharedRepo')
2347 def sourcedirIsUpdateable(self
):
2348 if os
.path
.exists(os
.path
.join(self
.builder
.basedir
,
2349 self
.srcdir
, ".buildbot-patched")):
2352 # checking out a specific revision requires a full 'bzr checkout'
2354 return os
.path
.isdir(os
.path
.join(self
.builder
.basedir
,
2355 self
.srcdir
, ".bzr"))
2359 # Continue with start() method in superclass.
2360 return SourceBase
.start(self
)
2362 if self
.forceSharedRepo
:
2363 d
= self
.doForceSharedRepo();
2369 def doVCUpdate(self
):
2370 assert not self
.revision
2371 # update: possible for mode in ('copy', 'update')
2372 srcdir
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2373 command
= [self
.vcexe
, 'update']
2374 c
= ShellCommand(self
.builder
, command
, srcdir
,
2375 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
2380 # checkout or export
2381 d
= self
.builder
.basedir
2382 if self
.mode
== "export":
2383 # exporting in bzr requires a separate directory
2384 return self
.doVCExport()
2385 # originally I added --lightweight here, but then 'bzr revno' is
2386 # wrong. The revno reported in 'bzr version-info' is correct,
2387 # however. Maybe this is a bzr bug?
2389 # In addition, you cannot perform a 'bzr update' on a repo pulled
2390 # from an HTTP repository that used 'bzr checkout --lightweight'. You
2391 # get a "ERROR: Cannot lock: transport is read only" when you try.
2393 # So I won't bother using --lightweight for now.
2395 command
= [self
.vcexe
, 'checkout']
2397 command
.append('--revision')
2398 command
.append(str(self
.revision
))
2399 command
.append(self
.repourl
)
2400 command
.append(self
.srcdir
)
2402 c
= ShellCommand(self
.builder
, command
, d
,
2403 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
2408 def doVCExport(self
):
2409 tmpdir
= os
.path
.join(self
.builder
.basedir
, "export-temp")
2410 srcdir
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2411 command
= [self
.vcexe
, 'checkout', '--lightweight']
2413 command
.append('--revision')
2414 command
.append(str(self
.revision
))
2415 command
.append(self
.repourl
)
2416 command
.append(tmpdir
)
2417 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
2418 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
2422 command
= [self
.vcexe
, 'export', srcdir
]
2423 c
= ShellCommand(self
.builder
, command
, tmpdir
,
2424 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
2427 d
.addCallback(_export
)
2430 def doForceSharedRepo(self
):
2431 # Don't send stderr. When there is no shared repo, this might confuse
2432 # users, as they will see a bzr error message. But having no shared
2433 # repo is not an error, just an indication that we need to make one.
2434 c
= ShellCommand(self
.builder
, [self
.vcexe
, 'info', '.'],
2435 self
.builder
.basedir
,
2436 sendStderr
=False, sendRC
=False, usePTY
=False)
2438 def afterCheckSharedRepo(res
):
2439 if type(res
) is int and res
!= 0:
2440 log
.msg("No shared repo found, creating it")
2441 # bzr info fails, try to create shared repo.
2442 c
= ShellCommand(self
.builder
, [self
.vcexe
, 'init-repo', '.'],
2443 self
.builder
.basedir
,
2444 sendRC
=False, usePTY
=False)
2448 return defer
.succeed(res
)
2449 d
.addCallback(afterCheckSharedRepo
)
2452 def get_revision_number(self
, out
):
2453 # it feels like 'bzr revno' sometimes gives different results than
2454 # the 'revno:' line from 'bzr version-info', and the one from
2455 # version-info is more likely to be correct.
2456 for line
in out
.split("\n"):
2457 colon
= line
.find(":")
2459 key
, value
= line
[:colon
], line
[colon
+2:]
2462 raise ValueError("unable to find revno: in bzr output: '%s'" % out
)
2464 def parseGotRevision(self
):
2465 command
= [self
.vcexe
, "version-info"]
2466 c
= ShellCommand(self
.builder
, command
,
2467 os
.path
.join(self
.builder
.basedir
, self
.srcdir
),
2469 sendStdout
=False, sendStderr
=False, sendRC
=False,
2470 keepStdout
=True, usePTY
=False)
2474 return self
.get_revision_number(c
.stdout
)
2476 msg
=("Bzr.parseGotRevision unable to parse output "
2477 "of bzr version-info: '%s'" % c
.stdout
.strip())
2479 self
.sendStatus({'header': msg
+ "\n"})
2481 d
.addCallback(_parse
)
2484 registerSlaveCommand("bzr", Bzr
, command_version
)
2486 class Mercurial(SourceBase
):
2487 """Mercurial specific VC operation. In addition to the arguments
2488 handled by SourceBase, this command reads the following keys:
2490 ['repourl'] (required): the Mercurial repository string
2493 header
= "mercurial operation"
2495 def setup(self
, args
):
2496 SourceBase
.setup(self
, args
)
2497 self
.vcexe
= getCommand("hg")
2498 self
.repourl
= args
['repourl']
2499 self
.clobberOnBranchChange
= args
['clobberOnBranchChange']
2500 self
.sourcedata
= "%s\n" % self
.repourl
2501 self
.branchType
= args
['branchType']
2505 def sourcedirIsUpdateable(self
):
2506 return os
.path
.isdir(os
.path
.join(self
.builder
.basedir
,
2507 self
.srcdir
, ".hg"))
2509 def doVCUpdate(self
):
2510 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2511 command
= [self
.vcexe
, 'pull', '--verbose', self
.repourl
]
2512 c
= ShellCommand(self
.builder
, command
, d
,
2513 sendRC
=False, timeout
=self
.timeout
,
2514 keepStdout
=True, usePTY
=False)
2517 d
.addCallback(self
._handleEmptyUpdate
)
2518 d
.addCallback(self
._update
)
2521 def _handleEmptyUpdate(self
, res
):
2522 if type(res
) is int and res
== 1:
2523 if self
.command
.stdout
.find("no changes found") != -1:
2524 # 'hg pull', when it doesn't have anything to do, exits with
2525 # rc=1, and there appears to be no way to shut this off. It
2526 # emits a distinctive message to stdout, though. So catch
2527 # this and pretend that it completed successfully.
2532 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2533 command
= [self
.vcexe
, 'clone', '--verbose', '--noupdate']
2535 # if got revision, clobbering and in dirname, only clone to specific revision
2536 # (otherwise, do full clone to re-use .hg dir for subsequent byuilds)
2537 if self
.args
.get('revision') and self
.mode
== 'clobber' and self
.branchType
== 'dirname':
2538 command
.extend(['--rev', self
.args
.get('revision')])
2539 command
.extend([self
.repourl
, d
])
2541 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
2542 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
2545 cmd1
.addCallback(self
._update
)
2548 def _clobber(self
, dummy
, dirname
):
2550 return self
.doVCFull()
2552 c
= self
.doClobber(dummy
, dirname
)
2553 c
.addCallback(_vcfull
)
2557 def _purge(self
, dummy
, dirname
):
2558 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2559 purge
= [self
.vcexe
, 'purge', '--all']
2560 purgeCmd
= ShellCommand(self
.builder
, purge
, d
,
2561 sendStdout
=False, sendStderr
=False,
2562 keepStdout
=True, keepStderr
=True, usePTY
=False)
2566 # purge failed, we need to switch to a classic clobber
2567 msg
= "'hg purge' failed: %s\n%s. Clobbering." % (purgeCmd
.stdout
, purgeCmd
.stderr
)
2568 self
.sendStatus({'header': msg
+ "\n"})
2571 return self
._clobber
(dummy
, dirname
)
2573 # Purge was a success, then we need to update
2574 return self
._update
2(res
)
2576 p
= purgeCmd
.start()
2577 p
.addCallback(_clobber
)
2580 def _update(self
, res
):
2584 # compare current branch to update
2585 self
.update_branch
= self
.args
.get('branch', 'default')
2587 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2588 parentscmd
= [self
.vcexe
, 'identify', '--num', '--branch']
2589 cmd
= ShellCommand(self
.builder
, parentscmd
, d
,
2590 sendStdout
=False, sendStderr
=False,
2591 keepStdout
=True, keepStderr
=True, usePTY
=False)
2595 def _parseIdentify(res
):
2597 msg
= "'hg identify' failed: %s\n%s" % (cmd
.stdout
, cmd
.stderr
)
2598 self
.sendStatus({'header': msg
+ "\n"})
2602 log
.msg('Output: %s' % cmd
.stdout
)
2604 match
= re
.search(r
'^(.+) (.+)$', cmd
.stdout
)
2607 rev
= match
.group(1)
2608 current_branch
= match
.group(2)
2611 msg
= "Fresh hg repo, don't worry about in-repo branch name"
2614 elif os
.path
.exists(os
.path
.join(self
.builder
.basedir
,
2615 self
.srcdir
, ".buildbot-patched")):
2616 self
.clobber
= self
._purge
2618 elif self
.update_branch
!= current_branch
:
2619 msg
= "Working dir is on in-repo branch '%s' and build needs '%s'." % (current_branch
, self
.update_branch
)
2620 if self
.clobberOnBranchChange
:
2621 msg
+= ' Cloberring.'
2625 self
.sendStatus({'header': msg
+ "\n"})
2628 # Clobbers only if clobberOnBranchChange is set
2629 if self
.clobberOnBranchChange
:
2630 self
.clobber
= self
._purge
2633 msg
= "Working dir on same in-repo branch as build (%s)." % (current_branch
)
2638 def _checkRepoURL(res
):
2639 parentscmd
= [self
.vcexe
, 'paths', 'default']
2640 cmd2
= ShellCommand(self
.builder
, parentscmd
, d
,
2641 sendStdout
=False, sendStderr
=False,
2642 keepStdout
=True, keepStderr
=True, usePTY
=False)
2644 def _parseRepoURL(res
):
2646 if "not found!" == cmd2
.stderr
.strip():
2647 msg
= "hg default path not set. Not checking repo url for clobber test"
2651 msg
= "'hg paths default' failed: %s\n%s" % (cmd2
.stdout
, cmd2
.stderr
)
2655 oldurl
= cmd2
.stdout
.strip()
2657 log
.msg("Repo cloned from: '%s'" % oldurl
)
2659 if sys
.platform
== "win32":
2660 oldurl
= oldurl
.lower().replace('\\', '/')
2661 repourl
= self
.repourl
.lower().replace('\\', '/')
2662 if repourl
.startswith('file://'):
2663 repourl
= repourl
.split('file://')[1]
2665 repourl
= self
.repourl
2667 if oldurl
!= repourl
:
2668 self
.clobber
= self
._clobber
2669 msg
= "RepoURL changed from '%s' in wc to '%s' in update. Clobbering" % (oldurl
, repourl
)
2675 c
.addCallback(_parseRepoURL
)
2678 def _maybeClobber(res
):
2680 msg
= "Clobber flag set. Doing clobbering"
2684 return self
.doVCFull()
2686 return self
.clobber(None, self
.srcdir
)
2691 c
.addCallback(_parseIdentify
)
2692 c
.addCallback(_checkRepoURL
)
2693 c
.addCallback(_maybeClobber
)
2694 c
.addCallback(self
._update
2)
2697 def _update2(self
, res
):
2698 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2700 updatecmd
=[self
.vcexe
, 'update', '--clean', '--repository', d
]
2701 if self
.args
.get('revision'):
2702 updatecmd
.extend(['--rev', self
.args
['revision']])
2704 updatecmd
.extend(['--rev', self
.args
.get('branch', 'default')])
2705 self
.command
= ShellCommand(self
.builder
, updatecmd
,
2706 self
.builder
.basedir
, sendRC
=False,
2707 timeout
=self
.timeout
, usePTY
=False)
2708 return self
.command
.start()
2710 def parseGotRevision(self
):
2711 # we use 'hg identify' to find out what we wound up with
2712 command
= [self
.vcexe
, "identify"]
2713 c
= ShellCommand(self
.builder
, command
,
2714 os
.path
.join(self
.builder
.basedir
, self
.srcdir
),
2716 sendStdout
=False, sendStderr
=False, sendRC
=False,
2717 keepStdout
=True, usePTY
=False)
2720 m
= re
.search(r
'^(\w+)', c
.stdout
)
2722 d
.addCallback(_parse
)
2725 registerSlaveCommand("hg", Mercurial
, command_version
)
2728 class P4Base(SourceBase
):
2729 """Base class for P4 source-updaters
2731 ['p4port'] (required): host:port for server to access
2732 ['p4user'] (optional): user to use for access
2733 ['p4passwd'] (optional): passwd to try for the user
2734 ['p4client'] (optional): client spec to use
2736 def setup(self
, args
):
2737 SourceBase
.setup(self
, args
)
2738 self
.p4port
= args
['p4port']
2739 self
.p4client
= args
['p4client']
2740 self
.p4user
= args
['p4user']
2741 self
.p4passwd
= args
['p4passwd']
2743 def parseGotRevision(self
):
2744 # Executes a p4 command that will give us the latest changelist number
2745 # of any file under the current (or default) client:
2748 command
.extend(['-p', self
.p4port
])
2750 command
.extend(['-u', self
.p4user
])
2752 command
.extend(['-P', self
.p4passwd
])
2754 command
.extend(['-c', self
.p4client
])
2755 command
.extend(['changes', '-m', '1', '#have'])
2756 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
2757 environ
=self
.env
, timeout
=self
.timeout
,
2758 sendStdout
=True, sendStderr
=False, sendRC
=False,
2759 keepStdout
=True, usePTY
=False)
2764 # 'p4 -c clien-name change -m 1 "#have"' will produce an output like:
2765 # "Change 28147 on 2008/04/07 by p4user@hostname..."
2766 # The number after "Change" is the one we want.
2767 m
= re
.match('Change\s+(\d+)\s+', c
.stdout
)
2771 d
.addCallback(_parse
)
2776 """A P4 source-updater.
2778 ['p4port'] (required): host:port for server to access
2779 ['p4user'] (optional): user to use for access
2780 ['p4passwd'] (optional): passwd to try for the user
2781 ['p4client'] (optional): client spec to use
2782 ['p4extra_views'] (optional): additional client views to use
2787 def setup(self
, args
):
2788 P4Base
.setup(self
, args
)
2789 self
.p4base
= args
['p4base']
2790 self
.p4extra_views
= args
['p4extra_views']
2791 self
.p4mode
= args
['mode']
2792 self
.p4branch
= args
['branch']
2794 self
.sourcedata
= str([
2801 # Depot side of view spec.
2806 # Local side of view spec (srcdir is made from these).
2807 self
.builder
.basedir
,
2813 def sourcedirIsUpdateable(self
):
2814 if os
.path
.exists(os
.path
.join(self
.builder
.basedir
,
2815 self
.srcdir
, ".buildbot-patched")):
2817 # We assume our client spec is still around.
2818 # We just say we aren't updateable if the dir doesn't exist so we
2819 # don't get ENOENT checking the sourcedata.
2820 return os
.path
.isdir(os
.path
.join(self
.builder
.basedir
,
2823 def doVCUpdate(self
):
2824 return self
._doP
4Sync
(force
=False)
2826 def _doP4Sync(self
, force
):
2830 command
.extend(['-p', self
.p4port
])
2832 command
.extend(['-u', self
.p4user
])
2834 command
.extend(['-P', self
.p4passwd
])
2836 command
.extend(['-c', self
.p4client
])
2837 command
.extend(['sync'])
2839 command
.extend(['-f'])
2841 command
.extend(['@' + str(self
.revision
)])
2843 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
2844 environ
=env
, sendRC
=False, timeout
=self
.timeout
,
2845 keepStdout
=True, usePTY
=False)
2848 d
.addCallback(self
._abandonOnFailure
)
2856 client_spec
+= "Client: %s\n\n" % self
.p4client
2857 client_spec
+= "Owner: %s\n\n" % self
.p4user
2858 client_spec
+= "Description:\n\tCreated by %s\n\n" % self
.p4user
2859 client_spec
+= "Root:\t%s\n\n" % self
.builder
.basedir
2860 client_spec
+= "Options:\tallwrite rmdir\n\n"
2861 client_spec
+= "LineEnd:\tlocal\n\n"
2864 client_spec
+= "View:\n\t%s" % (self
.p4base
)
2866 client_spec
+= "%s/" % (self
.p4branch
)
2867 client_spec
+= "... //%s/%s/...\n" % (self
.p4client
, self
.srcdir
)
2868 if self
.p4extra_views
:
2869 for k
, v
in self
.p4extra_views
:
2870 client_spec
+= "\t%s/... //%s/%s%s/...\n" % (k
, self
.p4client
,
2873 command
.extend(['-p', self
.p4port
])
2875 command
.extend(['-u', self
.p4user
])
2877 command
.extend(['-P', self
.p4passwd
])
2878 command
.extend(['client', '-i'])
2879 log
.msg(client_spec
)
2880 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
2881 environ
=env
, sendRC
=False, timeout
=self
.timeout
,
2882 initialStdin
=client_spec
, usePTY
=False)
2885 d
.addCallback(self
._abandonOnFailure
)
2886 d
.addCallback(lambda _
: self
._doP
4Sync
(force
=True))
2889 def parseGotRevision(self
):
2892 rv
= str(self
.revision
)
2895 registerSlaveCommand("p4", P4
, command_version
)
2898 class P4Sync(P4Base
):
2899 """A partial P4 source-updater. Requires manual setup of a per-slave P4
2900 environment. The only thing which comes from the master is P4PORT.
2901 'mode' is required to be 'copy'.
2903 ['p4port'] (required): host:port for server to access
2904 ['p4user'] (optional): user to use for access
2905 ['p4passwd'] (optional): passwd to try for the user
2906 ['p4client'] (optional): client spec to use
2911 def setup(self
, args
):
2912 P4Base
.setup(self
, args
)
2913 self
.vcexe
= getCommand("p4")
2915 def sourcedirIsUpdateable(self
):
2918 def _doVC(self
, force
):
2919 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2920 command
= [self
.vcexe
]
2922 command
.extend(['-p', self
.p4port
])
2924 command
.extend(['-u', self
.p4user
])
2926 command
.extend(['-P', self
.p4passwd
])
2928 command
.extend(['-c', self
.p4client
])
2929 command
.extend(['sync'])
2931 command
.extend(['-f'])
2933 command
.extend(['@' + self
.revision
])
2935 c
= ShellCommand(self
.builder
, command
, d
, environ
=env
,
2936 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
2940 def doVCUpdate(self
):
2941 return self
._doVC
(force
=False)
2944 return self
._doVC
(force
=True)
2946 def parseGotRevision(self
):
2949 rv
= str(self
.revision
)
2952 registerSlaveCommand("p4sync", P4Sync
, command_version
)