1 # -*- test-case-name: buildbot.test.test_slavecommand -*-
3 import os
, re
, signal
, shutil
, types
, time
4 from stat
import ST_CTIME
, ST_MTIME
, ST_SIZE
6 from zope
.interface
import implements
7 from twisted
.internet
.protocol
import ProcessProtocol
8 from twisted
.internet
import reactor
, defer
, task
9 from twisted
.python
import log
, failure
, runtime
10 from twisted
.python
.procutils
import which
12 from buildbot
.slave
.interfaces
import ISlaveCommand
13 from buildbot
.slave
.registry
import registerSlaveCommand
15 # this used to be a CVS $-style "Revision" auto-updated keyword, but since I
16 # moved to Darcs as the primary repository, this is updated manually each
17 # time this file is changed. The last cvs_ver that was here was 1.51 .
18 command_version
= "2.7"
21 # >=1.17: commands are interruptable
22 # >=1.28: Arch understands 'revision', added Bazaar
23 # >=1.33: Source classes understand 'retry'
24 # >=1.39: Source classes correctly handle changes in branch (except Git)
25 # Darcs accepts 'revision' (now all do but Git) (well, and P4Sync)
26 # Arch/Baz should accept 'build-config'
27 # >=1.51: (release 0.7.3)
28 # >= 2.1: SlaveShellCommand now accepts 'initial_stdin', 'keep_stdin_open',
29 # and 'logfiles'. It now sends 'log' messages in addition to
30 # stdout/stdin/header/rc. It acquired writeStdin/closeStdin methods,
31 # but these are not remotely callable yet.
32 # (not externally visible: ShellCommandPP has writeStdin/closeStdin.
33 # ShellCommand accepts new arguments (logfiles=, initialStdin=,
34 # keepStdinOpen=) and no longer accepts stdin=)
36 # >= 2.2: added monotone, uploadFile, and downloadFile (release 0.7.5)
37 # >= 2.3: added bzr (release 0.7.6)
38 # >= 2.4: Git understands 'revision' and branches
39 # >= 2.5: workaround added for remote 'hg clone --rev REV' when hg<0.9.2
40 # >= 2.6: added uploadDirectory
41 # >= 2.7: added usePTY option to SlaveShellCommand
43 class CommandInterrupted(Exception):
45 class TimeoutError(Exception):
48 class AbandonChain(Exception):
49 """A series of chained steps can raise this exception to indicate that
50 one of the intermediate ShellCommands has failed, such that there is no
51 point in running the remainder. 'rc' should be the non-zero exit code of
52 the failing ShellCommand."""
55 return "<AbandonChain rc=%s>" % self
.args
[0]
58 possibles
= which(name
)
60 raise RuntimeError("Couldn't find executable for '%s'" % name
)
63 def rmdirRecursive(dir):
64 """This is a replacement for shutil.rmtree that works better under
65 windows. Thanks to Bear at the OSAF for the code."""
66 if not os
.path
.exists(dir):
69 if os
.path
.islink(dir):
73 # Verify the directory is read/write/execute for the current user
76 for name
in os
.listdir(dir):
77 full_name
= os
.path
.join(dir, name
)
78 # on Windows, if we don't have write permission we can't remove
79 # the file/directory either, so turn that on
81 if not os
.access(full_name
, os
.W_OK
):
82 # I think this is now redundant, but I don't have an NT
83 # machine to test on, so I'm going to leave it in place
85 os
.chmod(full_name
, 0600)
87 if os
.path
.isdir(full_name
):
88 rmdirRecursive(full_name
)
90 os
.chmod(full_name
, 0700)
94 class ShellCommandPP(ProcessProtocol
):
97 def __init__(self
, command
):
98 self
.command
= command
99 self
.pending_stdin
= ""
100 self
.stdin_finished
= False
102 def writeStdin(self
, data
):
103 assert not self
.stdin_finished
105 self
.transport
.write(data
)
107 self
.pending_stdin
+= data
109 def closeStdin(self
):
111 if self
.debug
: log
.msg(" closing stdin")
112 self
.transport
.closeStdin()
113 self
.stdin_finished
= True
115 def connectionMade(self
):
117 log
.msg("ShellCommandPP.connectionMade")
118 if not self
.command
.process
:
120 log
.msg(" assigning self.command.process: %s" %
122 self
.command
.process
= self
.transport
124 # TODO: maybe we shouldn't close stdin when using a PTY. I can't test
125 # this yet, recent debian glibc has a bug which causes thread-using
126 # test cases to SIGHUP trial, and the workaround is to either run
127 # the whole test with /bin/sh -c " ".join(argv) (way gross) or to
128 # not use a PTY. Once the bug is fixed, I'll be able to test what
129 # happens when you close stdin on a pty. My concern is that it will
130 # SIGHUP the child (since we are, in a sense, hanging up on them).
131 # But it may well be that keeping stdout open prevents the SIGHUP
133 #if not self.command.usePTY:
135 if self
.pending_stdin
:
136 if self
.debug
: log
.msg(" writing to stdin")
137 self
.transport
.write(self
.pending_stdin
)
138 if self
.stdin_finished
:
139 if self
.debug
: log
.msg(" closing stdin")
140 self
.transport
.closeStdin()
142 def outReceived(self
, data
):
144 log
.msg("ShellCommandPP.outReceived")
145 self
.command
.addStdout(data
)
147 def errReceived(self
, data
):
149 log
.msg("ShellCommandPP.errReceived")
150 self
.command
.addStderr(data
)
152 def processEnded(self
, status_object
):
154 log
.msg("ShellCommandPP.processEnded", status_object
)
155 # status_object is a Failure wrapped around an
156 # error.ProcessTerminated or and error.ProcessDone.
157 # requires twisted >= 1.0.4 to overcome a bug in process.py
158 sig
= status_object
.value
.signal
159 rc
= status_object
.value
.exitCode
160 self
.command
.finished(sig
, rc
)
162 class LogFileWatcher
:
165 def __init__(self
, command
, name
, logfile
):
166 self
.command
= command
168 self
.logfile
= logfile
169 log
.msg("LogFileWatcher created to watch %s" % logfile
)
170 # we are created before the ShellCommand starts. If the logfile we're
171 # supposed to be watching already exists, record its size and
172 # ctime/mtime so we can tell when it starts to change.
173 self
.old_logfile_stats
= self
.statFile()
176 # every 2 seconds we check on the file again
177 self
.poller
= task
.LoopingCall(self
.poll
)
180 self
.poller
.start(self
.POLL_INTERVAL
).addErrback(self
._cleanupPoll
)
182 def _cleanupPoll(self
, err
):
183 log
.err(err
, msg
="Polling error")
188 if self
.poller
is not None:
194 if os
.path
.exists(self
.logfile
):
195 s
= os
.stat(self
.logfile
)
196 return (s
[ST_CTIME
], s
[ST_MTIME
], s
[ST_SIZE
])
202 if s
== self
.old_logfile_stats
:
203 return # not started yet
205 # the file was there, but now it's deleted. Forget about the
206 # initial state, clearly the process has deleted the logfile
207 # in preparation for creating a new one.
208 self
.old_logfile_stats
= None
209 return # no file to work with
210 self
.f
= open(self
.logfile
, "rb")
212 self
.f
.seek(self
.f
.tell(), 0)
214 data
= self
.f
.read(10000)
217 self
.command
.addLogfile(self
.name
, data
)
221 # This is a helper class, used by SlaveCommands to run programs in a
227 CHUNK_LIMIT
= 128*1024
229 # For sending elapsed time:
232 # I wish we had easy access to CLOCK_MONOTONIC in Python:
233 # http://www.opengroup.org/onlinepubs/000095399/functions/clock_getres.html
234 # Then changes to the system clock during a run wouldn't effect the "elapsed
237 def __init__(self
, builder
, command
,
238 workdir
, environ
=None,
239 sendStdout
=True, sendStderr
=True, sendRC
=True,
240 timeout
=None, initialStdin
=None, keepStdinOpen
=False,
241 keepStdout
=False, keepStderr
=False, logEnviron
=True,
242 logfiles
={}, usePTY
="slave-config"):
245 @param keepStdout: if True, we keep a copy of all the stdout text
246 that we've seen. This copy is available in
247 self.stdout, which can be read after the command
249 @param keepStderr: same, for stderr
251 @param usePTY: "slave-config" -> use the SlaveBuilder's usePTY;
252 otherwise, true to use a PTY, false to not use a PTY.
255 self
.builder
= builder
256 self
.command
= command
257 self
.sendStdout
= sendStdout
258 self
.sendStderr
= sendStderr
260 self
.logfiles
= logfiles
261 self
.workdir
= workdir
262 self
.environ
= os
.environ
.copy()
264 if environ
.has_key('PYTHONPATH'):
265 ppath
= environ
['PYTHONPATH']
266 # Need to do os.pathsep translation. We could either do that
267 # by replacing all incoming ':'s with os.pathsep, or by
268 # accepting lists. I like lists better.
269 if not isinstance(ppath
, str):
270 # If it's not a string, treat it as a sequence to be
271 # turned in to a string.
272 ppath
= os
.pathsep
.join(ppath
)
274 if self
.environ
.has_key('PYTHONPATH'):
275 # special case, prepend the builder's items to the
276 # existing ones. This will break if you send over empty
277 # strings, so don't do that.
278 ppath
= ppath
+ os
.pathsep
+ self
.environ
['PYTHONPATH']
280 environ
['PYTHONPATH'] = ppath
282 self
.environ
.update(environ
)
283 self
.initialStdin
= initialStdin
284 self
.keepStdinOpen
= keepStdinOpen
285 self
.logEnviron
= logEnviron
286 self
.timeout
= timeout
288 self
.keepStdout
= keepStdout
289 self
.keepStderr
= keepStderr
292 if usePTY
== "slave-config":
293 self
.usePTY
= self
.builder
.usePTY
297 # usePTY=True is a convenience for cleaning up all children and
298 # grandchildren of a hung command. Fall back to usePTY=False on systems
299 # and in situations where ptys cause problems. PTYs are posix-only,
300 # and for .closeStdin to matter, we must use a pipe, not a PTY
301 if runtime
.platformType
!= "posix" or initialStdin
is not None:
302 if self
.usePTY
and usePTY
!= "slave-config":
303 self
.sendStatus({'header': "WARNING: disabling usePTY for this command"})
306 self
.logFileWatchers
= []
307 for name
,filename
in self
.logfiles
.items():
308 w
= LogFileWatcher(self
, name
,
309 os
.path
.join(self
.workdir
, filename
))
310 self
.logFileWatchers
.append(w
)
313 return "<slavecommand.ShellCommand '%s'>" % self
.command
315 def sendStatus(self
, status
):
316 self
.builder
.sendUpdate(status
)
319 # return a Deferred which fires (with the exit code) when the command
325 self
.deferred
= defer
.Deferred()
329 log
.msg("error in ShellCommand._startCommand")
331 # pretend it was a shell error
332 self
.deferred
.errback(AbandonChain(-1))
335 def _startCommand(self
):
336 # ensure workdir exists
337 if not os
.path
.isdir(self
.workdir
):
338 os
.makedirs(self
.workdir
)
339 log
.msg("ShellCommand._startCommand")
341 self
.sendStatus({'header': "command '%s' in dir %s" % \
342 (self
.command
, self
.workdir
)})
343 self
.sendStatus({'header': "(not really)\n"})
344 self
.finished(None, 0)
347 self
.pp
= ShellCommandPP(self
)
349 if type(self
.command
) in types
.StringTypes
:
350 if runtime
.platformType
== 'win32':
351 argv
= os
.environ
['COMSPEC'].split() # allow %COMSPEC% to have args
352 if '/c' not in argv
: argv
+= ['/c']
353 argv
+= [self
.command
]
355 # for posix, use /bin/sh. for other non-posix, well, doesn't
357 argv
= ['/bin/sh', '-c', self
.command
]
359 if runtime
.platformType
== 'win32':
360 argv
= os
.environ
['COMSPEC'].split() # allow %COMSPEC% to have args
361 if '/c' not in argv
: argv
+= ['/c']
362 argv
+= list(self
.command
)
366 # $PWD usually indicates the current directory; spawnProcess may not
367 # update this value, though, so we set it explicitly here.
368 self
.environ
['PWD'] = os
.path
.abspath(self
.workdir
)
370 # self.stdin is handled in ShellCommandPP.connectionMade
372 # first header line is the command in plain text, argv joined with
373 # spaces. You should be able to cut-and-paste this into a shell to
374 # obtain the same results. If there are spaces in the arguments, too
378 self
.sendStatus({'header': msg
+"\n"})
380 # then comes the secondary information
381 msg
= " in dir %s" % (self
.workdir
,)
383 msg
+= " (timeout %d secs)" % (self
.timeout
,)
385 self
.sendStatus({'header': msg
+"\n"})
387 msg
= " watching logfiles %s" % (self
.logfiles
,)
389 self
.sendStatus({'header': msg
+"\n"})
391 # then the argv array for resolving unambiguity
392 msg
= " argv: %s" % (argv
,)
394 self
.sendStatus({'header': msg
+"\n"})
396 # then the environment, since it sometimes causes problems
398 msg
= " environment:\n"
399 env_names
= self
.environ
.keys()
401 for name
in env_names
:
402 msg
+= " %s=%s\n" % (name
, self
.environ
[name
])
403 log
.msg(" environment: %s" % (self
.environ
,))
404 self
.sendStatus({'header': msg
})
406 if self
.initialStdin
:
407 msg
= " writing %d bytes to stdin" % len(self
.initialStdin
)
409 self
.sendStatus({'header': msg
+"\n"})
411 if self
.keepStdinOpen
:
412 msg
= " leaving stdin open"
414 msg
= " closing stdin"
416 self
.sendStatus({'header': msg
+"\n"})
418 msg
= " using PTY: %s" % bool(self
.usePTY
)
420 self
.sendStatus({'header': msg
+"\n"})
422 # this will be buffered until connectionMade is called
423 if self
.initialStdin
:
424 self
.pp
.writeStdin(self
.initialStdin
)
425 if not self
.keepStdinOpen
:
428 # win32eventreactor's spawnProcess (under twisted <= 2.0.1) returns
429 # None, as opposed to all the posixbase-derived reactors (which
430 # return the new Process object). This is a nuisance. We can make up
431 # for it by having the ProcessProtocol give us their .transport
432 # attribute after they get one. I'd prefer to get it from
433 # spawnProcess because I'm concerned about returning from this method
434 # without having a valid self.process to work with. (if kill() were
435 # called right after we return, but somehow before connectionMade
436 # were called, then kill() would blow up).
438 self
.startTime
= time
.time()
439 p
= reactor
.spawnProcess(self
.pp
, argv
[0], argv
,
443 # connectionMade might have been called during spawnProcess
447 # connectionMade also closes stdin as long as we're not using a PTY.
448 # This is intended to kill off inappropriately interactive commands
449 # better than the (long) hung-command timeout. ProcessPTY should be
450 # enhanced to allow the same childFDs argument that Process takes,
451 # which would let us connect stdin to /dev/null .
454 self
.timer
= reactor
.callLater(self
.timeout
, self
.doTimeout
)
456 for w
in self
.logFileWatchers
:
460 def _chunkForSend(self
, data
):
461 # limit the chunks that we send over PB to 128k, since it has a
462 # hardwired string-size limit of 640k.
463 LIMIT
= self
.CHUNK_LIMIT
464 for i
in range(0, len(data
), LIMIT
):
465 yield data
[i
:i
+LIMIT
]
467 def addStdout(self
, data
):
469 for chunk
in self
._chunkForSend
(data
):
470 self
.sendStatus({'stdout': chunk
})
474 self
.timer
.reset(self
.timeout
)
476 def addStderr(self
, data
):
478 for chunk
in self
._chunkForSend
(data
):
479 self
.sendStatus({'stderr': chunk
})
483 self
.timer
.reset(self
.timeout
)
485 def addLogfile(self
, name
, data
):
486 for chunk
in self
._chunkForSend
(data
):
487 self
.sendStatus({'log': (name
, chunk
)})
489 self
.timer
.reset(self
.timeout
)
491 def finished(self
, sig
, rc
):
492 self
.elapsedTime
= time
.time() - self
.startTime
493 log
.msg("command finished with signal %s, exit code %s, elapsedTime: %0.6f" % (sig
,rc
,self
.elapsedTime
))
494 for w
in self
.logFileWatchers
:
495 # this will send the final updates
502 {'header': "process killed by signal %d\n" % sig
})
503 self
.sendStatus({'rc': rc
})
504 self
.sendStatus({'header': "elapsedTime=%0.6f\n" % self
.elapsedTime
})
513 log
.msg("Hey, command %s finished twice" % self
)
515 def failed(self
, why
):
516 log
.msg("ShellCommand.failed: command failed: %s" % (why
,))
525 log
.msg("Hey, command %s finished twice" % self
)
529 msg
= "command timed out: %d seconds without output" % self
.timeout
533 # This may be called by the timeout, or when the user has decided to
538 if hasattr(self
.process
, "pid"):
539 msg
+= ", killing pid %d" % self
.process
.pid
541 self
.sendStatus({'header': "\n" + msg
+ "\n"})
544 if runtime
.platformType
== "posix":
546 # really want to kill off all child processes too. Process
547 # Groups are ideal for this, but that requires
548 # spawnProcess(usePTY=1). Try both ways in case process was
549 # not started that way.
551 # the test suite sets self.KILL=None to tell us we should
552 # only pretend to kill the child. This lets us test the
556 if self
.KILL
is not None:
557 sig
= getattr(signal
, "SIG"+ self
.KILL
, None)
559 if self
.KILL
== None:
560 log
.msg("self.KILL==None, only pretending to kill child")
562 log
.msg("signal module is missing SIG%s" % self
.KILL
)
563 elif not hasattr(os
, "kill"):
564 log
.msg("os module is missing the 'kill' function")
566 log
.msg("trying os.kill(-pid, %d)" % (sig
,))
567 # TODO: maybe use os.killpg instead of a negative pid?
568 os
.kill(-self
.process
.pid
, sig
)
569 log
.msg(" signal %s sent successfully" % sig
)
572 # probably no-such-process, maybe because there is no process
577 if self
.KILL
is None:
578 log
.msg("self.KILL==None, only pretending to kill child")
580 log
.msg("trying process.signalProcess('KILL')")
581 self
.process
.signalProcess(self
.KILL
)
582 log
.msg(" signal %s sent successfully" % (self
.KILL
,))
585 # could be no-such-process, because they finished very recently
588 log
.msg("signalProcess/os.kill failed both times")
590 if runtime
.platformType
== "posix":
591 # we only do this under posix because the win32eventreactor
592 # blocks here until the process has terminated, while closing
593 # stderr. This is weird.
594 self
.pp
.transport
.loseConnection()
596 # finished ought to be called momentarily. Just in case it doesn't,
597 # set a timer which will abandon the command.
598 self
.timer
= reactor
.callLater(self
.BACKUP_TIMEOUT
,
599 self
.doBackupTimeout
)
601 def doBackupTimeout(self
):
602 log
.msg("we tried to kill the process, and it wouldn't die.."
605 self
.sendStatus({'header': "SIGKILL failed to kill process\n"})
607 self
.sendStatus({'header': "using fake rc=-1\n"})
608 self
.sendStatus({'rc': -1})
609 self
.failed(TimeoutError("SIGKILL failed to kill process"))
612 def writeStdin(self
, data
):
613 self
.pp
.writeStdin(data
)
615 def closeStdin(self
):
620 implements(ISlaveCommand
)
622 """This class defines one command that can be invoked by the build master.
623 The command is executed on the slave side, and always sends back a
624 completion message when it finishes. It may also send intermediate status
625 as it runs (by calling builder.sendStatus). Some commands can be
626 interrupted (either by the build master or a local timeout), in which
627 case the step is expected to complete normally with a status message that
628 indicates an error occurred.
630 These commands are used by BuildSteps on the master side. Each kind of
631 BuildStep uses a single Command. The slave must implement all the
632 Commands required by the set of BuildSteps used for any given build:
633 this is checked at startup time.
635 All Commands are constructed with the same signature:
636 c = CommandClass(builder, args)
637 where 'builder' is the parent SlaveBuilder object, and 'args' is a
638 dict that is interpreted per-command.
640 The setup(args) method is available for setup, and is run from __init__.
642 The Command is started with start(). This method must be implemented in a
643 subclass, and it should return a Deferred. When your step is done, you
644 should fire the Deferred (the results are not used). If the command is
645 interrupted, it should fire the Deferred anyway.
647 While the command runs. it may send status messages back to the
648 buildmaster by calling self.sendStatus(statusdict). The statusdict is
649 interpreted by the master-side BuildStep however it likes.
651 A separate completion message is sent when the deferred fires, which
652 indicates that the Command has finished, but does not carry any status
653 data. If the Command needs to return an exit code of some sort, that
654 should be sent as a regular status message before the deferred is fired .
655 Once builder.commandComplete has been run, no more status messages may be
658 If interrupt() is called, the Command should attempt to shut down as
659 quickly as possible. Child processes should be killed, new ones should
660 not be started. The Command should send some kind of error status update,
661 then complete as usual by firing the Deferred.
663 .interrupted should be set by interrupt(), and can be tested to avoid
664 sending multiple error status messages.
666 If .running is False, the bot is shutting down (or has otherwise lost the
667 connection to the master), and should not send any status messages. This
668 is checked in Command.sendStatus .
673 # sendStatus(dict) (zero or more)
674 # commandComplete() or commandInterrupted() (one, at end)
678 running
= False # set by Builder, cleared on shutdown or when the
681 def __init__(self
, builder
, stepId
, args
):
682 self
.builder
= builder
683 self
.stepId
= stepId
# just for logging
687 def setup(self
, args
):
688 """Override this in a subclass to extract items from the args dict."""
693 d
= defer
.maybeDeferred(self
.start
)
694 d
.addBoth(self
.commandComplete
)
698 """Start the command. This method should return a Deferred that will
699 fire when the command has completed. The Deferred's argument will be
702 This method should be overridden by subclasses."""
703 raise NotImplementedError, "You must implement this in a subclass"
705 def sendStatus(self
, status
):
706 """Send a status update to the master."""
708 log
.msg("sendStatus", status
)
710 log
.msg("would sendStatus but not .running")
712 self
.builder
.sendUpdate(status
)
714 def doInterrupt(self
):
719 """Override this in a subclass to allow commands to be interrupted.
720 May be called multiple times, test and set self.interrupted=True if
724 def commandComplete(self
, res
):
728 # utility methods, mostly used by SlaveShellCommand and the like
730 def _abandonOnFailure(self
, rc
):
731 if type(rc
) is not int:
732 log
.msg("weird, _abandonOnFailure was given rc=%s (%s)" % \
734 assert isinstance(rc
, int)
736 raise AbandonChain(rc
)
739 def _sendRC(self
, res
):
740 self
.sendStatus({'rc': 0})
742 def _checkAbandoned(self
, why
):
743 log
.msg("_checkAbandoned", why
)
744 why
.trap(AbandonChain
)
745 log
.msg(" abandoning chain", why
.value
)
746 self
.sendStatus({'rc': why
.value
.args
[0]})
751 class SlaveFileUploadCommand(Command
):
753 Upload a file from slave to build master
756 - ['workdir']: base directory to use
757 - ['slavesrc']: name of the slave-side file to read from
758 - ['writer']: RemoteReference to a transfer._FileWriter object
759 - ['maxsize']: max size (in bytes) of file to write
760 - ['blocksize']: max size for each data block
764 def setup(self
, args
):
765 self
.workdir
= args
['workdir']
766 self
.filename
= args
['slavesrc']
767 self
.writer
= args
['writer']
768 self
.remaining
= args
['maxsize']
769 self
.blocksize
= args
['blocksize']
775 log
.msg('SlaveFileUploadCommand started')
778 self
.path
= os
.path
.join(self
.builder
.basedir
,
780 os
.path
.expanduser(self
.filename
))
782 self
.fp
= open(self
.path
, 'rb')
784 log
.msg('Opened %r for upload' % self
.path
)
786 # TODO: this needs cleanup
788 self
.stderr
= 'Cannot open file %r for upload' % self
.path
791 log
.msg('Cannot open file %r for upload' % self
.path
)
793 self
.sendStatus({'header': "sending %s" % self
.path
})
796 reactor
.callLater(0, self
._loop
, d
)
798 # close the file, but pass through any errors from _loop
799 d1
= self
.writer
.callRemote("close")
800 d1
.addErrback(log
.err
)
801 d1
.addCallback(lambda ignored
: res
)
804 d
.addBoth(self
.finished
)
807 def _loop(self
, fire_when_done
):
808 d
= defer
.maybeDeferred(self
._writeBlock
)
811 fire_when_done
.callback(None)
813 self
._loop
(fire_when_done
)
815 fire_when_done
.errback(why
)
816 d
.addCallbacks(_done
, _err
)
819 def _writeBlock(self
):
820 """Write a block of data to the remote writer"""
822 if self
.interrupted
or self
.fp
is None:
824 log
.msg('SlaveFileUploadCommand._writeBlock(): end')
827 length
= self
.blocksize
828 if self
.remaining
is not None and length
> self
.remaining
:
829 length
= self
.remaining
832 if self
.stderr
is None:
833 self
.stderr
= 'Maximum filesize reached, truncating file %r' \
838 data
= self
.fp
.read(length
)
841 log
.msg('SlaveFileUploadCommand._writeBlock(): '+
842 'allowed=%d readlen=%d' % (length
, len(data
)))
844 log
.msg("EOF: callRemote(close)")
847 if self
.remaining
is not None:
848 self
.remaining
= self
.remaining
- len(data
)
849 assert self
.remaining
>= 0
850 d
= self
.writer
.callRemote('write', data
)
851 d
.addCallback(lambda res
: False)
856 log
.msg('interrupted')
859 if self
.stderr
is None:
860 self
.stderr
= 'Upload of %r interrupted' % self
.path
862 self
.interrupted
= True
863 # the next _writeBlock call will notice the .interrupted flag
865 def finished(self
, res
):
867 log
.msg('finished: stderr=%r, rc=%r' % (self
.stderr
, self
.rc
))
868 if self
.stderr
is None:
869 self
.sendStatus({'rc': self
.rc
})
871 self
.sendStatus({'stderr': self
.stderr
, 'rc': self
.rc
})
874 registerSlaveCommand("uploadFile", SlaveFileUploadCommand
, command_version
)
877 class SlaveDirectoryUploadCommand(Command
):
879 Upload a directory from slave to build master
882 - ['workdir']: base directory to use
883 - ['slavesrc']: name of the slave-side directory to read from
884 - ['writer']: RemoteReference to a transfer._DirectoryWriter object
885 - ['maxsize']: max size (in bytes) of file to write
886 - ['blocksize']: max size for each data block
890 def setup(self
, args
):
891 self
.workdir
= args
['workdir']
892 self
.dirname
= args
['slavesrc']
893 self
.writer
= args
['writer']
894 self
.remaining
= args
['maxsize']
895 self
.blocksize
= args
['blocksize']
901 log
.msg('SlaveDirectoryUploadCommand started')
903 # create some lists with all files and directories
907 self
.baseRoot
= os
.path
.join(self
.builder
.basedir
,
909 os
.path
.expanduser(self
.dirname
))
911 log
.msg("baseRoot: %r" % self
.baseRoot
)
913 for root
, dirs
, files
in os
.walk(self
.baseRoot
):
916 while (tempRoot
!= self
.baseRoot
):
917 tempRoot
, tempRelRoot
= os
.path
.split(tempRoot
)
918 relRoot
= os
.path
.join(tempRelRoot
, relRoot
)
920 foundFiles
.append(os
.path
.join(relRoot
, name
))
921 for directory
in dirs
:
922 foundDirs
.append(os
.path
.join(relRoot
, directory
))
925 log
.msg("foundDirs: %s" % (str(foundDirs
)))
926 log
.msg("foundFiles: %s" % (str(foundFiles
)))
928 # create all directories on the master, to catch also empty ones
929 for dirname
in foundDirs
:
930 self
.writer
.callRemote("createdir", dirname
)
932 for filename
in foundFiles
:
933 self
._writeFile
(filename
)
937 def _writeFile(self
, filename
):
938 """Write a file to the remote writer"""
940 log
.msg("_writeFile: %r" % (filename
))
941 self
.writer
.callRemote('open', filename
)
942 data
= open(os
.path
.join(self
.baseRoot
, filename
), "r").read()
943 self
.writer
.callRemote('write', data
)
944 self
.writer
.callRemote('close')
949 log
.msg('interrupted')
952 if self
.stderr
is None:
953 self
.stderr
= 'Upload of %r interrupted' % self
.path
955 self
.interrupted
= True
956 # the next _writeBlock call will notice the .interrupted flag
958 def finished(self
, res
):
960 log
.msg('finished: stderr=%r, rc=%r' % (self
.stderr
, self
.rc
))
961 if self
.stderr
is None:
962 self
.sendStatus({'rc': self
.rc
})
964 self
.sendStatus({'stderr': self
.stderr
, 'rc': self
.rc
})
967 registerSlaveCommand("uploadDirectory", SlaveDirectoryUploadCommand
, command_version
)
970 class SlaveFileDownloadCommand(Command
):
972 Download a file from master to slave
975 - ['workdir']: base directory to use
976 - ['slavedest']: name of the slave-side file to be created
977 - ['reader']: RemoteReference to a transfer._FileReader object
978 - ['maxsize']: max size (in bytes) of file to write
979 - ['blocksize']: max size for each data block
980 - ['mode']: access mode for the new file
984 def setup(self
, args
):
985 self
.workdir
= args
['workdir']
986 self
.filename
= args
['slavedest']
987 self
.reader
= args
['reader']
988 self
.bytes_remaining
= args
['maxsize']
989 self
.blocksize
= args
['blocksize']
990 self
.mode
= args
['mode']
996 log
.msg('SlaveFileDownloadCommand starting')
999 self
.path
= os
.path
.join(self
.builder
.basedir
,
1001 os
.path
.expanduser(self
.filename
))
1003 dirname
= os
.path
.dirname(self
.path
)
1004 if not os
.path
.exists(dirname
):
1005 os
.makedirs(dirname
)
1008 self
.fp
= open(self
.path
, 'wb')
1010 log
.msg('Opened %r for download' % self
.path
)
1011 if self
.mode
is not None:
1012 # note: there is a brief window during which the new file
1013 # will have the buildslave's default (umask) mode before we
1014 # set the new one. Don't use this mode= feature to keep files
1015 # private: use the buildslave's umask for that instead. (it
1016 # is possible to call os.umask() before and after the open()
1017 # call, but cleaning up from exceptions properly is more of a
1018 # nuisance that way).
1019 os
.chmod(self
.path
, self
.mode
)
1021 # TODO: this still needs cleanup
1023 self
.stderr
= 'Cannot open file %r for download' % self
.path
1026 log
.msg('Cannot open file %r for download' % self
.path
)
1028 d
= defer
.Deferred()
1029 reactor
.callLater(0, self
._loop
, d
)
1031 # close the file, but pass through any errors from _loop
1032 d1
= self
.reader
.callRemote('close')
1033 d1
.addErrback(log
.err
)
1034 d1
.addCallback(lambda ignored
: res
)
1037 d
.addBoth(self
.finished
)
1040 def _loop(self
, fire_when_done
):
1041 d
= defer
.maybeDeferred(self
._readBlock
)
1042 def _done(finished
):
1044 fire_when_done
.callback(None)
1046 self
._loop
(fire_when_done
)
1048 fire_when_done
.errback(why
)
1049 d
.addCallbacks(_done
, _err
)
1052 def _readBlock(self
):
1053 """Read a block of data from the remote reader."""
1055 if self
.interrupted
or self
.fp
is None:
1057 log
.msg('SlaveFileDownloadCommand._readBlock(): end')
1060 length
= self
.blocksize
1061 if self
.bytes_remaining
is not None and length
> self
.bytes_remaining
:
1062 length
= self
.bytes_remaining
1065 if self
.stderr
is None:
1066 self
.stderr
= 'Maximum filesize reached, truncating file %r' \
1071 d
= self
.reader
.callRemote('read', length
)
1072 d
.addCallback(self
._writeData
)
1075 def _writeData(self
, data
):
1077 log
.msg('SlaveFileDownloadCommand._readBlock(): readlen=%d' %
1082 if self
.bytes_remaining
is not None:
1083 self
.bytes_remaining
= self
.bytes_remaining
- len(data
)
1084 assert self
.bytes_remaining
>= 0
1088 def interrupt(self
):
1090 log
.msg('interrupted')
1091 if self
.interrupted
:
1093 if self
.stderr
is None:
1094 self
.stderr
= 'Download of %r interrupted' % self
.path
1096 self
.interrupted
= True
1097 # now we wait for the next read request to return. _readBlock will
1098 # abandon the file when it sees self.interrupted set.
1100 def finished(self
, res
):
1101 if self
.fp
is not None:
1105 log
.msg('finished: stderr=%r, rc=%r' % (self
.stderr
, self
.rc
))
1106 if self
.stderr
is None:
1107 self
.sendStatus({'rc': self
.rc
})
1109 self
.sendStatus({'stderr': self
.stderr
, 'rc': self
.rc
})
1112 registerSlaveCommand("downloadFile", SlaveFileDownloadCommand
, command_version
)
1116 class SlaveShellCommand(Command
):
1117 """This is a Command which runs a shell command. The args dict contains
1120 - ['command'] (required): a shell command to run. If this is a string,
1121 it will be run with /bin/sh (['/bin/sh',
1122 '-c', command]). If it is a list
1123 (preferred), it will be used directly.
1124 - ['workdir'] (required): subdirectory in which the command will be
1125 run, relative to the builder dir
1126 - ['env']: a dict of environment variables to augment/replace
1127 os.environ . PYTHONPATH is treated specially, and
1128 should be a list of path components to be prepended to
1129 any existing PYTHONPATH environment variable.
1130 - ['initial_stdin']: a string which will be written to the command's
1131 stdin as soon as it starts
1132 - ['keep_stdin_open']: unless True, the command's stdin will be
1133 closed as soon as initial_stdin has been
1134 written. Set this to True if you plan to write
1135 to stdin after the command has been started.
1136 - ['want_stdout']: 0 if stdout should be thrown away
1137 - ['want_stderr']: 0 if stderr should be thrown away
1138 - ['usePTY']: True or False if the command should use a PTY (defaults to
1139 configuration of the slave)
1140 - ['not_really']: 1 to skip execution and return rc=0
1141 - ['timeout']: seconds of silence to tolerate before killing command
1142 - ['logfiles']: dict mapping LogFile name to the workdir-relative
1143 filename of a local log file. This local file will be
1144 watched just like 'tail -f', and all changes will be
1145 written to 'log' status updates.
1147 ShellCommand creates the following status messages:
1148 - {'stdout': data} : when stdout data is available
1149 - {'stderr': data} : when stderr data is available
1150 - {'header': data} : when headers (command start/stop) are available
1151 - {'log': (logfile_name, data)} : when log files have new contents
1152 - {'rc': rc} : when the process has terminated
1157 # args['workdir'] is relative to Builder directory, and is required.
1158 assert args
['workdir'] is not None
1159 workdir
= os
.path
.join(self
.builder
.basedir
, args
['workdir'])
1161 c
= ShellCommand(self
.builder
, args
['command'],
1162 workdir
, environ
=args
.get('env'),
1163 timeout
=args
.get('timeout', None),
1164 sendStdout
=args
.get('want_stdout', True),
1165 sendStderr
=args
.get('want_stderr', True),
1167 initialStdin
=args
.get('initial_stdin'),
1168 keepStdinOpen
=args
.get('keep_stdin_open'),
1169 logfiles
=args
.get('logfiles', {}),
1170 usePTY
=args
.get('usePTY', "slave-config"),
1173 d
= self
.command
.start()
1176 def interrupt(self
):
1177 self
.interrupted
= True
1178 self
.command
.kill("command interrupted")
1180 def writeStdin(self
, data
):
1181 self
.command
.writeStdin(data
)
1183 def closeStdin(self
):
1184 self
.command
.closeStdin()
1186 registerSlaveCommand("shell", SlaveShellCommand
, command_version
)
1189 class DummyCommand(Command
):
1191 I am a dummy no-op command that by default takes 5 seconds to complete.
1192 See L{buildbot.steps.dummy.RemoteDummy}
1196 self
.d
= defer
.Deferred()
1197 log
.msg(" starting dummy command [%s]" % self
.stepId
)
1198 self
.timer
= reactor
.callLater(1, self
.doStatus
)
1201 def interrupt(self
):
1202 if self
.interrupted
:
1206 self
.interrupted
= True
1210 log
.msg(" sending intermediate status")
1211 self
.sendStatus({'stdout': 'data'})
1212 timeout
= self
.args
.get('timeout', 5) + 1
1213 self
.timer
= reactor
.callLater(timeout
- 1, self
.finished
)
1216 log
.msg(" dummy command finished [%s]" % self
.stepId
)
1217 if self
.interrupted
:
1218 self
.sendStatus({'rc': 1})
1220 self
.sendStatus({'rc': 0})
1223 registerSlaveCommand("dummy", DummyCommand
, command_version
)
1226 # this maps handle names to a callable. When the WaitCommand starts, this
1227 # callable is invoked with no arguments. It should return a Deferred. When
1228 # that Deferred fires, our WaitCommand will finish.
1229 waitCommandRegistry
= {}
1231 class WaitCommand(Command
):
1233 I am a dummy command used by the buildbot unit test suite. I want for the
1234 unit test to tell us to finish. See L{buildbot.steps.dummy.Wait}
1238 self
.d
= defer
.Deferred()
1239 log
.msg(" starting wait command [%s]" % self
.stepId
)
1240 handle
= self
.args
['handle']
1241 cb
= waitCommandRegistry
[handle
]
1242 del waitCommandRegistry
[handle
]
1244 log
.msg(" wait-%s starting" % (handle
,))
1247 log
.msg(" wait-%s finishing: %s" % (handle
, res
))
1250 d
.addCallbacks(self
.finished
, self
.failed
)
1251 reactor
.callLater(0, _called
)
1254 def interrupt(self
):
1255 log
.msg(" wait command interrupted")
1256 if self
.interrupted
:
1258 self
.interrupted
= True
1259 self
.finished("interrupted")
1261 def finished(self
, res
):
1262 log
.msg(" wait command finished [%s]" % self
.stepId
)
1263 if self
.interrupted
:
1264 self
.sendStatus({'rc': 2})
1266 self
.sendStatus({'rc': 0})
1268 def failed(self
, why
):
1269 log
.msg(" wait command failed [%s]" % self
.stepId
)
1270 self
.sendStatus({'rc': 1})
1273 registerSlaveCommand("dummy.wait", WaitCommand
, command_version
)
1276 class SourceBase(Command
):
1277 """Abstract base class for Version Control System operations (checkout
1278 and update). This class extracts the following arguments from the
1279 dictionary received from the master:
1281 - ['workdir']: (required) the subdirectory where the buildable sources
1284 - ['mode']: one of update/copy/clobber/export, defaults to 'update'
1286 - ['revision']: If not None, this is an int or string which indicates
1287 which sources (along a time-like axis) should be used.
1288 It is the thing you provide as the CVS -r or -D
1291 - ['patch']: If not None, this is a tuple of (striplevel, patch)
1292 which contains a patch that should be applied after the
1293 checkout has occurred. Once applied, the tree is no
1294 longer eligible for use with mode='update', and it only
1295 makes sense to use this in conjunction with a
1296 ['revision'] argument. striplevel is an int, and patch
1297 is a string in standard unified diff format. The patch
1298 will be applied with 'patch -p%d <PATCH', with
1299 STRIPLEVEL substituted as %d. The command will fail if
1300 the patch process fails (rejected hunks).
1302 - ['timeout']: seconds of silence tolerated before we kill off the
1305 - ['retry']: If not None, this is a tuple of (delay, repeats)
1306 which means that any failed VC updates should be
1307 reattempted, up to REPEATS times, after a delay of
1308 DELAY seconds. This is intended to deal with slaves
1309 that experience transient network failures.
1314 def setup(self
, args
):
1315 # if we need to parse the output, use this environment. Otherwise
1316 # command output will be in whatever the buildslave's native language
1318 self
.env
= os
.environ
.copy()
1319 self
.env
['LC_MESSAGES'] = "C"
1321 self
.workdir
= args
['workdir']
1322 self
.mode
= args
.get('mode', "update")
1323 self
.revision
= args
.get('revision')
1324 self
.patch
= args
.get('patch')
1325 self
.timeout
= args
.get('timeout', 120)
1326 self
.retry
= args
.get('retry')
1327 # VC-specific subclasses should override this to extract more args.
1328 # Make sure to upcall!
1331 self
.sendStatus({'header': "starting " + self
.header
+ "\n"})
1334 # self.srcdir is where the VC system should put the sources
1335 if self
.mode
== "copy":
1336 self
.srcdir
= "source" # hardwired directory name, sorry
1338 self
.srcdir
= self
.workdir
1339 self
.sourcedatafile
= os
.path
.join(self
.builder
.basedir
,
1341 ".buildbot-sourcedata")
1343 d
= defer
.succeed(None)
1344 self
.maybeClobber(d
)
1345 if not (self
.sourcedirIsUpdateable() and self
.sourcedataMatches()):
1346 # the directory cannot be updated, so we have to clobber it.
1347 # Perhaps the master just changed modes from 'export' to
1349 d
.addCallback(self
.doClobber
, self
.srcdir
)
1351 d
.addCallback(self
.doVC
)
1353 if self
.mode
== "copy":
1354 d
.addCallback(self
.doCopy
)
1356 d
.addCallback(self
.doPatch
)
1357 d
.addCallbacks(self
._sendRC
, self
._checkAbandoned
)
1360 def maybeClobber(self
, d
):
1361 # do we need to clobber anything?
1362 if self
.mode
in ("copy", "clobber", "export"):
1363 d
.addCallback(self
.doClobber
, self
.workdir
)
1365 def interrupt(self
):
1366 self
.interrupted
= True
1368 self
.command
.kill("command interrupted")
1370 def doVC(self
, res
):
1371 if self
.interrupted
:
1372 raise AbandonChain(1)
1373 if self
.sourcedirIsUpdateable() and self
.sourcedataMatches():
1374 d
= self
.doVCUpdate()
1375 d
.addCallback(self
.maybeDoVCFallback
)
1378 d
.addBoth(self
.maybeDoVCRetry
)
1379 d
.addCallback(self
._abandonOnFailure
)
1380 d
.addCallback(self
._handleGotRevision
)
1381 d
.addCallback(self
.writeSourcedata
)
1384 def sourcedataMatches(self
):
1386 olddata
= open(self
.sourcedatafile
, "r").read()
1387 if olddata
!= self
.sourcedata
:
1393 def _handleGotRevision(self
, res
):
1394 d
= defer
.maybeDeferred(self
.parseGotRevision
)
1395 d
.addCallback(lambda got_revision
:
1396 self
.sendStatus({'got_revision': got_revision
}))
1399 def parseGotRevision(self
):
1400 """Override this in a subclass. It should return a string that
1401 represents which revision was actually checked out, or a Deferred
1402 that will fire with such a string. If, in a future build, you were to
1403 pass this 'got_revision' string in as the 'revision' component of a
1404 SourceStamp, you should wind up with the same source code as this
1405 checkout just obtained.
1407 It is probably most useful to scan self.command.stdout for a string
1408 of some sort. Be sure to set keepStdout=True on the VC command that
1409 you run, so that you'll have something available to look at.
1411 If this information is unavailable, just return None."""
1415 def writeSourcedata(self
, res
):
1416 open(self
.sourcedatafile
, "w").write(self
.sourcedata
)
1419 def sourcedirIsUpdateable(self
):
1420 raise NotImplementedError("this must be implemented in a subclass")
1422 def doVCUpdate(self
):
1423 raise NotImplementedError("this must be implemented in a subclass")
1426 raise NotImplementedError("this must be implemented in a subclass")
1428 def maybeDoVCFallback(self
, rc
):
1429 if type(rc
) is int and rc
== 0:
1431 if self
.interrupted
:
1432 raise AbandonChain(1)
1433 msg
= "update failed, clobbering and trying again"
1434 self
.sendStatus({'header': msg
+ "\n"})
1436 d
= self
.doClobber(None, self
.srcdir
)
1437 d
.addCallback(self
.doVCFallback2
)
1440 def doVCFallback2(self
, res
):
1441 msg
= "now retrying VC operation"
1442 self
.sendStatus({'header': msg
+ "\n"})
1445 d
.addBoth(self
.maybeDoVCRetry
)
1446 d
.addCallback(self
._abandonOnFailure
)
1449 def maybeDoVCRetry(self
, res
):
1450 """We get here somewhere after a VC chain has finished. res could
1453 - 0: the operation was successful
1454 - nonzero: the operation failed. retry if possible
1455 - AbandonChain: the operation failed, someone else noticed. retry.
1456 - Failure: some other exception, re-raise
1459 if isinstance(res
, failure
.Failure
):
1460 if self
.interrupted
:
1461 return res
# don't re-try interrupted builds
1462 res
.trap(AbandonChain
)
1464 if type(res
) is int and res
== 0:
1466 if self
.interrupted
:
1467 raise AbandonChain(1)
1468 # if we get here, we should retry, if possible
1470 delay
, repeats
= self
.retry
1472 self
.retry
= (delay
, repeats
-1)
1473 msg
= ("update failed, trying %d more times after %d seconds"
1475 self
.sendStatus({'header': msg
+ "\n"})
1477 d
= defer
.Deferred()
1478 self
.maybeClobber(d
)
1479 d
.addCallback(lambda res
: self
.doVCFull())
1480 d
.addBoth(self
.maybeDoVCRetry
)
1481 reactor
.callLater(delay
, d
.callback
, None)
1485 def doClobber(self
, dummy
, dirname
):
1486 # TODO: remove the old tree in the background
1487 ## workdir = os.path.join(self.builder.basedir, self.workdir)
1488 ## deaddir = self.workdir + ".deleting"
1489 ## if os.path.isdir(workdir):
1491 ## os.rename(workdir, deaddir)
1492 ## # might fail if deaddir already exists: previous deletion
1493 ## # hasn't finished yet
1494 ## # start the deletion in the background
1495 ## # TODO: there was a solaris/NetApp/NFS problem where a
1496 ## # process that was still running out of the directory we're
1497 ## # trying to delete could prevent the rm-rf from working. I
1498 ## # think it stalled the rm, but maybe it just died with
1499 ## # permission issues. Try to detect this.
1500 ## os.commands("rm -rf %s &" % deaddir)
1502 ## # fall back to sequential delete-then-checkout
1504 d
= os
.path
.join(self
.builder
.basedir
, dirname
)
1505 if runtime
.platformType
!= "posix":
1506 # if we're running on w32, use rmtree instead. It will block,
1507 # but hopefully it won't take too long.
1509 return defer
.succeed(0)
1510 command
= ["rm", "-rf", d
]
1511 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
1512 sendRC
=0, timeout
=self
.timeout
, usePTY
=False)
1515 # sendRC=0 means the rm command will send stdout/stderr to the
1516 # master, but not the rc=0 when it finishes. That job is left to
1519 d
.addCallback(self
._abandonOnFailure
)
1522 def doCopy(self
, res
):
1523 # now copy tree to workdir
1524 fromdir
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
1525 todir
= os
.path
.join(self
.builder
.basedir
, self
.workdir
)
1526 if runtime
.platformType
!= "posix":
1527 self
.sendStatus({'header': "Since we're on a non-POSIX platform, "
1528 "we're not going to try to execute cp in a subprocess, but instead "
1529 "use shutil.copytree(), which will block until it is complete. "
1530 "fromdir: %s, todir: %s\n" % (fromdir
, todir
)})
1531 shutil
.copytree(fromdir
, todir
)
1532 return defer
.succeed(0)
1534 if not os
.path
.exists(os
.path
.dirname(todir
)):
1535 os
.makedirs(os
.path
.dirname(todir
))
1536 if os
.path
.exists(todir
):
1537 # I don't think this happens, but just in case..
1538 log
.msg("cp target '%s' already exists -- cp will not do what you think!" % todir
)
1540 command
= ['cp', '-R', '-P', '-p', fromdir
, todir
]
1541 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
1542 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
1545 d
.addCallback(self
._abandonOnFailure
)
1548 def doPatch(self
, res
):
1549 patchlevel
, diff
= self
.patch
1550 command
= [getCommand("patch"), '-p%d' % patchlevel
]
1551 dir = os
.path
.join(self
.builder
.basedir
, self
.workdir
)
1552 # mark the directory so we don't try to update it later
1553 open(os
.path
.join(dir, ".buildbot-patched"), "w").write("patched\n")
1554 # now apply the patch
1555 c
= ShellCommand(self
.builder
, command
, dir,
1556 sendRC
=False, timeout
=self
.timeout
,
1557 initialStdin
=diff
, usePTY
=False)
1560 d
.addCallback(self
._abandonOnFailure
)
1564 class CVS(SourceBase
):
1565 """CVS-specific VC operation. In addition to the arguments handled by
1566 SourceBase, this command reads the following keys:
1568 ['cvsroot'] (required): the CVSROOT repository string
1569 ['cvsmodule'] (required): the module to be retrieved
1570 ['branch']: a '-r' tag or branch name to use for the checkout/update
1571 ['login']: a string for use as a password to 'cvs login'
1572 ['global_options']: a list of strings to use before the CVS verb
1575 header
= "cvs operation"
1577 def setup(self
, args
):
1578 SourceBase
.setup(self
, args
)
1579 self
.vcexe
= getCommand("cvs")
1580 self
.cvsroot
= args
['cvsroot']
1581 self
.cvsmodule
= args
['cvsmodule']
1582 self
.global_options
= args
.get('global_options', [])
1583 self
.branch
= args
.get('branch')
1584 self
.login
= args
.get('login')
1585 self
.sourcedata
= "%s\n%s\n%s\n" % (self
.cvsroot
, self
.cvsmodule
,
1588 def sourcedirIsUpdateable(self
):
1589 if os
.path
.exists(os
.path
.join(self
.builder
.basedir
,
1590 self
.srcdir
, ".buildbot-patched")):
1592 return os
.path
.isdir(os
.path
.join(self
.builder
.basedir
,
1593 self
.srcdir
, "CVS"))
1596 if self
.login
is not None:
1597 # need to do a 'cvs login' command first
1598 d
= self
.builder
.basedir
1599 command
= ([self
.vcexe
, '-d', self
.cvsroot
] + self
.global_options
1601 c
= ShellCommand(self
.builder
, command
, d
,
1602 sendRC
=False, timeout
=self
.timeout
,
1603 initialStdin
=self
.login
+"\n", usePTY
=False)
1606 d
.addCallback(self
._abandonOnFailure
)
1607 d
.addCallback(self
._didLogin
)
1610 return self
._didLogin
(None)
1612 def _didLogin(self
, res
):
1613 # now we really start
1614 return SourceBase
.start(self
)
1616 def doVCUpdate(self
):
1617 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
1618 command
= [self
.vcexe
, '-z3'] + self
.global_options
+ ['update', '-dP']
1620 command
+= ['-r', self
.branch
]
1622 command
+= ['-D', self
.revision
]
1623 c
= ShellCommand(self
.builder
, command
, d
,
1624 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
1629 d
= self
.builder
.basedir
1630 if self
.mode
== "export":
1634 command
= ([self
.vcexe
, '-d', self
.cvsroot
, '-z3'] +
1635 self
.global_options
+
1636 [verb
, '-d', self
.srcdir
])
1638 command
+= ['-r', self
.branch
]
1640 command
+= ['-D', self
.revision
]
1641 command
+= [self
.cvsmodule
]
1642 c
= ShellCommand(self
.builder
, command
, d
,
1643 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
1647 def parseGotRevision(self
):
1648 # CVS does not have any kind of revision stamp to speak of. We return
1649 # the current timestamp as a best-effort guess, but this depends upon
1650 # the local system having a clock that is
1651 # reasonably-well-synchronized with the repository.
1652 return time
.strftime("%Y-%m-%d %H:%M:%S +0000", time
.gmtime())
1654 registerSlaveCommand("cvs", CVS
, command_version
)
1656 class SVN(SourceBase
):
1657 """Subversion-specific VC operation. In addition to the arguments
1658 handled by SourceBase, this command reads the following keys:
1660 ['svnurl'] (required): the SVN repository string
1663 header
= "svn operation"
1665 def setup(self
, args
):
1666 SourceBase
.setup(self
, args
)
1667 self
.vcexe
= getCommand("svn")
1668 self
.svnurl
= args
['svnurl']
1669 self
.sourcedata
= "%s\n" % self
.svnurl
1671 def sourcedirIsUpdateable(self
):
1672 if os
.path
.exists(os
.path
.join(self
.builder
.basedir
,
1673 self
.srcdir
, ".buildbot-patched")):
1675 return os
.path
.isdir(os
.path
.join(self
.builder
.basedir
,
1676 self
.srcdir
, ".svn"))
1678 def doVCUpdate(self
):
1679 revision
= self
.args
['revision'] or 'HEAD'
1680 # update: possible for mode in ('copy', 'update')
1681 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
1682 command
= [self
.vcexe
, 'update', '--revision', str(revision
),
1683 '--non-interactive', '--no-auth-cache']
1684 c
= ShellCommand(self
.builder
, command
, d
,
1685 sendRC
=False, timeout
=self
.timeout
,
1686 keepStdout
=True, usePTY
=False)
1691 revision
= self
.args
['revision'] or 'HEAD'
1692 d
= self
.builder
.basedir
1693 if self
.mode
== "export":
1694 command
= [self
.vcexe
, 'export', '--revision', str(revision
),
1695 '--non-interactive', '--no-auth-cache',
1696 self
.svnurl
, self
.srcdir
]
1698 # mode=='clobber', or copy/update on a broken workspace
1699 command
= [self
.vcexe
, 'checkout', '--revision', str(revision
),
1700 '--non-interactive', '--no-auth-cache',
1701 self
.svnurl
, self
.srcdir
]
1702 c
= ShellCommand(self
.builder
, command
, d
,
1703 sendRC
=False, timeout
=self
.timeout
,
1704 keepStdout
=True, usePTY
=False)
1708 def getSvnVersionCommand(self
):
1710 Get the (shell) command used to determine SVN revision number
1713 return: list of strings, passable as the command argument to ShellCommand
1715 # svn checkout operations finish with 'Checked out revision 16657.'
1716 # svn update operations finish the line 'At revision 16654.'
1717 # But we don't use those. Instead, run 'svnversion'.
1718 svnversion_command
= getCommand("svnversion")
1719 # older versions of 'svnversion' (1.1.4) require the WC_PATH
1720 # argument, newer ones (1.3.1) do not.
1721 return [svnversion_command
, "."]
1723 def parseGotRevision(self
):
1724 c
= ShellCommand(self
.builder
,
1725 self
.getSvnVersionCommand(),
1726 os
.path
.join(self
.builder
.basedir
, self
.srcdir
),
1728 sendStdout
=False, sendStderr
=False, sendRC
=False,
1729 keepStdout
=True, usePTY
=False)
1732 r_raw
= c
.stdout
.strip()
1733 # Extract revision from the version "number" string
1734 r
= r_raw
.rstrip('MS')
1735 r
= r
.split(':')[-1]
1738 got_version
= int(r
)
1740 msg
=("SVN.parseGotRevision unable to parse output "
1741 "of svnversion: '%s'" % r_raw
)
1743 self
.sendStatus({'header': msg
+ "\n"})
1745 d
.addCallback(_parse
)
1749 registerSlaveCommand("svn", SVN
, command_version
)
1751 class Darcs(SourceBase
):
1752 """Darcs-specific VC operation. In addition to the arguments
1753 handled by SourceBase, this command reads the following keys:
1755 ['repourl'] (required): the Darcs repository string
1758 header
= "darcs operation"
1760 def setup(self
, args
):
1761 SourceBase
.setup(self
, args
)
1762 self
.vcexe
= getCommand("darcs")
1763 self
.repourl
= args
['repourl']
1764 self
.sourcedata
= "%s\n" % self
.repourl
1765 self
.revision
= self
.args
.get('revision')
1767 def sourcedirIsUpdateable(self
):
1768 if os
.path
.exists(os
.path
.join(self
.builder
.basedir
,
1769 self
.srcdir
, ".buildbot-patched")):
1772 # checking out a specific revision requires a full 'darcs get'
1774 return os
.path
.isdir(os
.path
.join(self
.builder
.basedir
,
1775 self
.srcdir
, "_darcs"))
1777 def doVCUpdate(self
):
1778 assert not self
.revision
1779 # update: possible for mode in ('copy', 'update')
1780 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
1781 command
= [self
.vcexe
, 'pull', '--all', '--verbose']
1782 c
= ShellCommand(self
.builder
, command
, d
,
1783 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
1788 # checkout or export
1789 d
= self
.builder
.basedir
1790 command
= [self
.vcexe
, 'get', '--verbose', '--partial',
1791 '--repo-name', self
.srcdir
]
1793 # write the context to a file
1794 n
= os
.path
.join(self
.builder
.basedir
, ".darcs-context")
1796 f
.write(self
.revision
)
1798 # tell Darcs to use that context
1799 command
.append('--context')
1801 command
.append(self
.repourl
)
1803 c
= ShellCommand(self
.builder
, command
, d
,
1804 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
1808 d
.addCallback(self
.removeContextFile
, n
)
1811 def removeContextFile(self
, res
, n
):
1815 def parseGotRevision(self
):
1816 # we use 'darcs context' to find out what we wound up with
1817 command
= [self
.vcexe
, "changes", "--context"]
1818 c
= ShellCommand(self
.builder
, command
,
1819 os
.path
.join(self
.builder
.basedir
, self
.srcdir
),
1821 sendStdout
=False, sendStderr
=False, sendRC
=False,
1822 keepStdout
=True, usePTY
=False)
1824 d
.addCallback(lambda res
: c
.stdout
)
1827 registerSlaveCommand("darcs", Darcs
, command_version
)
1829 class Monotone(SourceBase
):
1830 """Monotone-specific VC operation. In addition to the arguments handled
1831 by SourceBase, this command reads the following keys:
1833 ['server_addr'] (required): the address of the server to pull from
1834 ['branch'] (required): the branch the revision is on
1835 ['db_path'] (required): the local database path to use
1836 ['revision'] (required): the revision to check out
1837 ['monotone']: (required): path to monotone executable
1840 header
= "monotone operation"
1842 def setup(self
, args
):
1843 SourceBase
.setup(self
, args
)
1844 self
.server_addr
= args
["server_addr"]
1845 self
.branch
= args
["branch"]
1846 self
.db_path
= args
["db_path"]
1847 self
.revision
= args
["revision"]
1848 self
.monotone
= args
["monotone"]
1849 self
._made
_fulls
= False
1850 self
._pull
_timeout
= args
["timeout"]
1852 def _makefulls(self
):
1853 if not self
._made
_fulls
:
1854 basedir
= self
.builder
.basedir
1855 self
.full_db_path
= os
.path
.join(basedir
, self
.db_path
)
1856 self
.full_srcdir
= os
.path
.join(basedir
, self
.srcdir
)
1857 self
._made
_fulls
= True
1859 def sourcedirIsUpdateable(self
):
1861 if os
.path
.exists(os
.path
.join(self
.full_srcdir
,
1862 ".buildbot_patched")):
1864 return (os
.path
.isfile(self
.full_db_path
)
1865 and os
.path
.isdir(os
.path
.join(self
.full_srcdir
, "MT")))
1867 def doVCUpdate(self
):
1868 return self
._withFreshDb
(self
._doUpdate
)
1870 def _doUpdate(self
):
1871 # update: possible for mode in ('copy', 'update')
1872 command
= [self
.monotone
, "update",
1873 "-r", self
.revision
,
1875 c
= ShellCommand(self
.builder
, command
, self
.full_srcdir
,
1876 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
1881 return self
._withFreshDb
(self
._doFull
)
1884 command
= [self
.monotone
, "--db=" + self
.full_db_path
,
1886 "-r", self
.revision
,
1889 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
1890 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
1894 def _withFreshDb(self
, callback
):
1896 # first ensure the db exists and is usable
1897 if os
.path
.isfile(self
.full_db_path
):
1898 # already exists, so run 'db migrate' in case monotone has been
1900 command
= [self
.monotone
, "db", "migrate",
1901 "--db=" + self
.full_db_path
]
1903 # We'll be doing an initial pull, so up the timeout to 3 hours to
1904 # make sure it will have time to complete.
1905 self
._pull
_timeout
= max(self
._pull
_timeout
, 3 * 60 * 60)
1906 self
.sendStatus({"header": "creating database %s\n"
1907 % (self
.full_db_path
,)})
1908 command
= [self
.monotone
, "db", "init",
1909 "--db=" + self
.full_db_path
]
1910 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
1911 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
1914 d
.addCallback(self
._abandonOnFailure
)
1915 d
.addCallback(self
._didDbInit
)
1916 d
.addCallback(self
._didPull
, callback
)
1919 def _didDbInit(self
, res
):
1920 command
= [self
.monotone
, "--db=" + self
.full_db_path
,
1921 "pull", "--ticker=dot", self
.server_addr
, self
.branch
]
1922 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
1923 sendRC
=False, timeout
=self
._pull
_timeout
, usePTY
=False)
1924 self
.sendStatus({"header": "pulling %s from %s\n"
1925 % (self
.branch
, self
.server_addr
)})
1929 def _didPull(self
, res
, callback
):
1932 registerSlaveCommand("monotone", Monotone
, command_version
)
1935 class Git(SourceBase
):
1936 """Git specific VC operation. In addition to the arguments
1937 handled by SourceBase, this command reads the following keys:
1939 ['repourl'] (required): the upstream GIT repository string
1940 ['branch'] (optional): which version (i.e. branch or tag) to
1941 retrieve. Default: "master".
1944 header
= "git operation"
1946 def setup(self
, args
):
1947 SourceBase
.setup(self
, args
)
1948 self
.repourl
= args
['repourl']
1949 self
.branch
= args
.get('branch')
1951 self
.branch
= "master"
1952 self
.sourcedata
= "%s %s\n" % (self
.repourl
, self
.branch
)
1954 def _fullSrcdir(self
):
1955 return os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
1957 def _commitSpec(self
):
1959 return self
.revision
1962 def sourcedirIsUpdateable(self
):
1963 if os
.path
.exists(os
.path
.join(self
._fullSrcdir
(),
1964 ".buildbot-patched")):
1966 return os
.path
.isdir(os
.path
.join(self
._fullSrcdir
(), ".git"))
1968 def readSourcedata(self
):
1969 return open(self
.sourcedatafile
, "r").read()
1971 # If the repourl matches the sourcedata file, then
1972 # we can say that the sourcedata matches. We can
1973 # ignore branch changes, since Git can work with
1974 # many branches fetched, and we deal with it properly
1976 def sourcedataMatches(self
):
1978 olddata
= self
.readSourcedata()
1979 if not olddata
.startswith(self
.repourl
+' '):
1985 def _didFetch(self
, res
):
1987 head
= self
.revision
1991 command
= ['git', 'reset', '--hard', head
]
1992 c
= ShellCommand(self
.builder
, command
, self
._fullSrcdir
(),
1993 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
1997 # Update first runs "git clean", removing local changes,
1998 # if the branch to be checked out has changed. This, combined
1999 # with the later "git reset" equates clobbering the repo,
2000 # but it's much more efficient.
2001 def doVCUpdate(self
):
2003 # Check to see if our branch has changed
2004 diffbranch
= self
.sourcedata
!= self
.readSourcedata()
2008 command
= ['git', 'clean', '-f', '-d']
2009 c
= ShellCommand(self
.builder
, command
, self
._fullSrcdir
(),
2010 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
2013 d
.addCallback(self
._abandonOnFailure
)
2014 d
.addCallback(self
._didClean
)
2016 return self
._didClean
(None)
2018 def _didClean(self
, dummy
):
2019 command
= ['git', 'fetch', '-t', self
.repourl
, self
.branch
]
2020 self
.sendStatus({"header": "fetching branch %s from %s\n"
2021 % (self
.branch
, self
.repourl
)})
2022 c
= ShellCommand(self
.builder
, command
, self
._fullSrcdir
(),
2023 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
2026 d
.addCallback(self
._abandonOnFailure
)
2027 d
.addCallback(self
._didFetch
)
2030 def _didInit(self
, res
):
2031 return self
.doVCUpdate()
2034 os
.mkdir(self
._fullSrcdir
())
2035 c
= ShellCommand(self
.builder
, ['git', 'init'], self
._fullSrcdir
(),
2036 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
2039 d
.addCallback(self
._abandonOnFailure
)
2040 d
.addCallback(self
._didInit
)
2043 def parseGotRevision(self
):
2044 command
= ['git', 'rev-parse', 'HEAD']
2045 c
= ShellCommand(self
.builder
, command
, self
._fullSrcdir
(),
2046 sendRC
=False, keepStdout
=True, usePTY
=False)
2049 hash = c
.stdout
.strip()
2053 d
.addCallback(_parse
)
2056 registerSlaveCommand("git", Git
, command_version
)
2058 class Arch(SourceBase
):
2059 """Arch-specific (tla-specific) VC operation. In addition to the
2060 arguments handled by SourceBase, this command reads the following keys:
2062 ['url'] (required): the repository string
2063 ['version'] (required): which version (i.e. branch) to retrieve
2064 ['revision'] (optional): the 'patch-NN' argument to check out
2065 ['archive']: the archive name to use. If None, use the archive's default
2066 ['build-config']: if present, give to 'tla build-config' after checkout
2069 header
= "arch operation"
2072 def setup(self
, args
):
2073 SourceBase
.setup(self
, args
)
2074 self
.vcexe
= getCommand("tla")
2075 self
.archive
= args
.get('archive')
2076 self
.url
= args
['url']
2077 self
.version
= args
['version']
2078 self
.revision
= args
.get('revision')
2079 self
.buildconfig
= args
.get('build-config')
2080 self
.sourcedata
= "%s\n%s\n%s\n" % (self
.url
, self
.version
,
2083 def sourcedirIsUpdateable(self
):
2085 # Arch cannot roll a directory backwards, so if they ask for a
2086 # specific revision, clobber the directory. Technically this
2087 # could be limited to the cases where the requested revision is
2088 # later than our current one, but it's too hard to extract the
2089 # current revision from the tree.
2091 if os
.path
.exists(os
.path
.join(self
.builder
.basedir
,
2092 self
.srcdir
, ".buildbot-patched")):
2094 return os
.path
.isdir(os
.path
.join(self
.builder
.basedir
,
2095 self
.srcdir
, "{arch}"))
2097 def doVCUpdate(self
):
2098 # update: possible for mode in ('copy', 'update')
2099 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2100 command
= [self
.vcexe
, 'replay']
2102 command
.append(self
.revision
)
2103 c
= ShellCommand(self
.builder
, command
, d
,
2104 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
2109 # to do a checkout, we must first "register" the archive by giving
2110 # the URL to tla, which will go to the repository at that URL and
2111 # figure out the archive name. tla will tell you the archive name
2112 # when it is done, and all further actions must refer to this name.
2114 command
= [self
.vcexe
, 'register-archive', '--force', self
.url
]
2115 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
2116 sendRC
=False, keepStdout
=True,
2117 timeout
=self
.timeout
, usePTY
=False)
2120 d
.addCallback(self
._abandonOnFailure
)
2121 d
.addCallback(self
._didRegister
, c
)
2124 def _didRegister(self
, res
, c
):
2125 # find out what tla thinks the archive name is. If the user told us
2126 # to use something specific, make sure it matches.
2127 r
= re
.search(r
'Registering archive: (\S+)\s*$', c
.stdout
)
2129 msg
= "tla reports archive name is '%s'" % r
.group(1)
2131 self
.builder
.sendUpdate({'header': msg
+"\n"})
2132 if self
.archive
and r
.group(1) != self
.archive
:
2133 msg
= (" mismatch, we wanted an archive named '%s'"
2136 self
.builder
.sendUpdate({'header': msg
+"\n"})
2137 raise AbandonChain(-1)
2138 self
.archive
= r
.group(1)
2139 assert self
.archive
, "need archive name to continue"
2140 return self
._doGet
()
2145 ver
+= "--%s" % self
.revision
2146 command
= [self
.vcexe
, 'get', '--archive', self
.archive
,
2149 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
2150 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
2153 d
.addCallback(self
._abandonOnFailure
)
2154 if self
.buildconfig
:
2155 d
.addCallback(self
._didGet
)
2158 def _didGet(self
, res
):
2159 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2160 command
= [self
.vcexe
, 'build-config', self
.buildconfig
]
2161 c
= ShellCommand(self
.builder
, command
, d
,
2162 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
2165 d
.addCallback(self
._abandonOnFailure
)
2168 def parseGotRevision(self
):
2169 # using code from tryclient.TlaExtractor
2170 # 'tla logs --full' gives us ARCHIVE/BRANCH--REVISION
2171 # 'tla logs' gives us REVISION
2172 command
= [self
.vcexe
, "logs", "--full", "--reverse"]
2173 c
= ShellCommand(self
.builder
, command
,
2174 os
.path
.join(self
.builder
.basedir
, self
.srcdir
),
2176 sendStdout
=False, sendStderr
=False, sendRC
=False,
2177 keepStdout
=True, usePTY
=False)
2180 tid
= c
.stdout
.split("\n")[0].strip()
2181 slash
= tid
.index("/")
2182 dd
= tid
.rindex("--")
2183 #branch = tid[slash+1:dd]
2184 baserev
= tid
[dd
+2:]
2186 d
.addCallback(_parse
)
2189 registerSlaveCommand("arch", Arch
, command_version
)
2192 """Bazaar (/usr/bin/baz) is an alternative client for Arch repositories.
2193 It is mostly option-compatible, but archive registration is different
2194 enough to warrant a separate Command.
2196 ['archive'] (required): the name of the archive being used
2199 def setup(self
, args
):
2200 Arch
.setup(self
, args
)
2201 self
.vcexe
= getCommand("baz")
2202 # baz doesn't emit the repository name after registration (and
2203 # grepping through the output of 'baz archives' is too hard), so we
2204 # require that the buildmaster configuration to provide both the
2205 # archive name and the URL.
2206 self
.archive
= args
['archive'] # required for Baz
2207 self
.sourcedata
= "%s\n%s\n%s\n" % (self
.url
, self
.version
,
2210 # in _didRegister, the regexp won't match, so we'll stick with the name
2214 # baz prefers ARCHIVE/VERSION. This will work even if
2215 # my-default-archive is not set.
2216 ver
= self
.archive
+ "/" + self
.version
2218 ver
+= "--%s" % self
.revision
2219 command
= [self
.vcexe
, 'get', '--no-pristine',
2221 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
2222 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
2225 d
.addCallback(self
._abandonOnFailure
)
2226 if self
.buildconfig
:
2227 d
.addCallback(self
._didGet
)
2230 def parseGotRevision(self
):
2231 # using code from tryclient.BazExtractor
2232 command
= [self
.vcexe
, "tree-id"]
2233 c
= ShellCommand(self
.builder
, command
,
2234 os
.path
.join(self
.builder
.basedir
, self
.srcdir
),
2236 sendStdout
=False, sendStderr
=False, sendRC
=False,
2237 keepStdout
=True, usePTY
=False)
2240 tid
= c
.stdout
.strip()
2241 slash
= tid
.index("/")
2242 dd
= tid
.rindex("--")
2243 #branch = tid[slash+1:dd]
2244 baserev
= tid
[dd
+2:]
2246 d
.addCallback(_parse
)
2249 registerSlaveCommand("bazaar", Bazaar
, command_version
)
2252 class Bzr(SourceBase
):
2253 """bzr-specific VC operation. In addition to the arguments
2254 handled by SourceBase, this command reads the following keys:
2256 ['repourl'] (required): the Bzr repository string
2259 header
= "bzr operation"
2261 def setup(self
, args
):
2262 SourceBase
.setup(self
, args
)
2263 self
.vcexe
= getCommand("bzr")
2264 self
.repourl
= args
['repourl']
2265 self
.sourcedata
= "%s\n" % self
.repourl
2266 self
.revision
= self
.args
.get('revision')
2268 def sourcedirIsUpdateable(self
):
2269 if os
.path
.exists(os
.path
.join(self
.builder
.basedir
,
2270 self
.srcdir
, ".buildbot-patched")):
2273 # checking out a specific revision requires a full 'bzr checkout'
2275 return os
.path
.isdir(os
.path
.join(self
.builder
.basedir
,
2276 self
.srcdir
, ".bzr"))
2278 def doVCUpdate(self
):
2279 assert not self
.revision
2280 # update: possible for mode in ('copy', 'update')
2281 srcdir
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2282 command
= [self
.vcexe
, 'update']
2283 c
= ShellCommand(self
.builder
, command
, srcdir
,
2284 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
2289 # checkout or export
2290 d
= self
.builder
.basedir
2291 if self
.mode
== "export":
2292 # exporting in bzr requires a separate directory
2293 return self
.doVCExport()
2294 # originally I added --lightweight here, but then 'bzr revno' is
2295 # wrong. The revno reported in 'bzr version-info' is correct,
2296 # however. Maybe this is a bzr bug?
2298 # In addition, you cannot perform a 'bzr update' on a repo pulled
2299 # from an HTTP repository that used 'bzr checkout --lightweight'. You
2300 # get a "ERROR: Cannot lock: transport is read only" when you try.
2302 # So I won't bother using --lightweight for now.
2304 command
= [self
.vcexe
, 'checkout']
2306 command
.append('--revision')
2307 command
.append(str(self
.revision
))
2308 command
.append(self
.repourl
)
2309 command
.append(self
.srcdir
)
2311 c
= ShellCommand(self
.builder
, command
, d
,
2312 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
2317 def doVCExport(self
):
2318 tmpdir
= os
.path
.join(self
.builder
.basedir
, "export-temp")
2319 srcdir
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2320 command
= [self
.vcexe
, 'checkout', '--lightweight']
2322 command
.append('--revision')
2323 command
.append(str(self
.revision
))
2324 command
.append(self
.repourl
)
2325 command
.append(tmpdir
)
2326 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
2327 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
2331 command
= [self
.vcexe
, 'export', srcdir
]
2332 c
= ShellCommand(self
.builder
, command
, tmpdir
,
2333 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
2336 d
.addCallback(_export
)
2339 def get_revision_number(self
, out
):
2340 # it feels like 'bzr revno' sometimes gives different results than
2341 # the 'revno:' line from 'bzr version-info', and the one from
2342 # version-info is more likely to be correct.
2343 for line
in out
.split("\n"):
2344 colon
= line
.find(":")
2346 key
, value
= line
[:colon
], line
[colon
+2:]
2349 raise ValueError("unable to find revno: in bzr output: '%s'" % out
)
2351 def parseGotRevision(self
):
2352 command
= [self
.vcexe
, "version-info"]
2353 c
= ShellCommand(self
.builder
, command
,
2354 os
.path
.join(self
.builder
.basedir
, self
.srcdir
),
2356 sendStdout
=False, sendStderr
=False, sendRC
=False,
2357 keepStdout
=True, usePTY
=False)
2361 return self
.get_revision_number(c
.stdout
)
2363 msg
=("Bzr.parseGotRevision unable to parse output "
2364 "of bzr version-info: '%s'" % c
.stdout
.strip())
2366 self
.sendStatus({'header': msg
+ "\n"})
2368 d
.addCallback(_parse
)
2371 registerSlaveCommand("bzr", Bzr
, command_version
)
2373 class Mercurial(SourceBase
):
2374 """Mercurial specific VC operation. In addition to the arguments
2375 handled by SourceBase, this command reads the following keys:
2377 ['repourl'] (required): the Cogito repository string
2380 header
= "mercurial operation"
2382 def setup(self
, args
):
2383 SourceBase
.setup(self
, args
)
2384 self
.vcexe
= getCommand("hg")
2385 self
.repourl
= args
['repourl']
2386 self
.sourcedata
= "%s\n" % self
.repourl
2390 def sourcedirIsUpdateable(self
):
2391 if os
.path
.exists(os
.path
.join(self
.builder
.basedir
,
2392 self
.srcdir
, ".buildbot-patched")):
2394 # like Darcs, to check out a specific (old) revision, we have to do a
2395 # full checkout. TODO: I think 'hg pull' plus 'hg update' might work
2398 return os
.path
.isdir(os
.path
.join(self
.builder
.basedir
,
2399 self
.srcdir
, ".hg"))
2401 def doVCUpdate(self
):
2402 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2403 command
= [self
.vcexe
, 'pull', '--verbose', self
.repourl
]
2404 c
= ShellCommand(self
.builder
, command
, d
,
2405 sendRC
=False, timeout
=self
.timeout
,
2406 keepStdout
=True, usePTY
=False)
2409 d
.addCallback(self
._handleEmptyUpdate
)
2410 d
.addCallback(self
._update
)
2413 def _handleEmptyUpdate(self
, res
):
2414 if type(res
) is int and res
== 1:
2415 if self
.command
.stdout
.find("no changes found") != -1:
2416 # 'hg pull', when it doesn't have anything to do, exits with
2417 # rc=1, and there appears to be no way to shut this off. It
2418 # emits a distinctive message to stdout, though. So catch
2419 # this and pretend that it completed successfully.
2424 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2425 command
= [self
.vcexe
, 'init', d
]
2426 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
2427 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
2432 return self
.doVCUpdate()
2434 cmd1
.addCallback(_vcupdate
)
2437 def _update(self
, res
):
2441 # compare current branch to update
2442 self
.update_branch
= self
.args
.get('branch', 'default')
2444 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2445 parentscmd
= [self
.vcexe
, 'identify', '--num', '--branch']
2446 cmd
= ShellCommand(self
.builder
, parentscmd
, d
,
2447 sendStdout
=False, sendStderr
=False,
2448 keepStdout
=True, keepStderr
=True, usePTY
=False)
2452 msg
= "'hg identify' failed: %s\n%s" % (cmd
.stdout
, cmd
.stderr
)
2453 self
.sendStatus({'header': msg
+ "\n"})
2457 log
.msg('Output: %s' % cmd
.stdout
)
2459 match
= re
.search(r
'^(.+) (.+)$', cmd
.stdout
)
2462 rev
= match
.group(1)
2463 current_branch
= match
.group(2)
2466 msg
= "Fresh hg repo, don't worry about branch"
2469 elif self
.update_branch
!= current_branch
:
2470 msg
= "Working dir is on branch '%s' and build needs '%s'. Clobbering." % (current_branch
, self
.update_branch
)
2471 self
.sendStatus({'header': msg
+ "\n"})
2475 return self
.doVCFull()
2477 d
= self
.doClobber(None, self
.srcdir
)
2478 d
.addCallback(_vcfull
)
2482 msg
= "Working dir on same branch as build (%s)." % (current_branch
)
2488 c
.addCallback(_parse
)
2489 c
.addCallback(self
._update
2)
2492 def _update2(self
, res
):
2493 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2495 updatecmd
=[self
.vcexe
, 'update', '--clean', '--repository', d
]
2496 if self
.args
.get('revision'):
2497 updatecmd
.extend(['--rev', self
.args
['revision']])
2499 updatecmd
.extend(['--rev', self
.args
.get('branch', 'default')])
2500 self
.command
= ShellCommand(self
.builder
, updatecmd
,
2501 self
.builder
.basedir
, sendRC
=False,
2502 timeout
=self
.timeout
, usePTY
=False)
2503 return self
.command
.start()
2505 def parseGotRevision(self
):
2506 # we use 'hg identify' to find out what we wound up with
2507 command
= [self
.vcexe
, "identify"]
2508 c
= ShellCommand(self
.builder
, command
,
2509 os
.path
.join(self
.builder
.basedir
, self
.srcdir
),
2511 sendStdout
=False, sendStderr
=False, sendRC
=False,
2512 keepStdout
=True, usePTY
=False)
2515 m
= re
.search(r
'^(\w+)', c
.stdout
)
2517 d
.addCallback(_parse
)
2520 registerSlaveCommand("hg", Mercurial
, command_version
)
2523 class P4Base(SourceBase
):
2524 """Base class for P4 source-updaters
2526 ['p4port'] (required): host:port for server to access
2527 ['p4user'] (optional): user to use for access
2528 ['p4passwd'] (optional): passwd to try for the user
2529 ['p4client'] (optional): client spec to use
2531 def setup(self
, args
):
2532 SourceBase
.setup(self
, args
)
2533 self
.p4port
= args
['p4port']
2534 self
.p4client
= args
['p4client']
2535 self
.p4user
= args
['p4user']
2536 self
.p4passwd
= args
['p4passwd']
2538 def parseGotRevision(self
):
2539 # Executes a p4 command that will give us the latest changelist number
2540 # of any file under the current (or default) client:
2543 command
.extend(['-p', self
.p4port
])
2545 command
.extend(['-u', self
.p4user
])
2547 command
.extend(['-P', self
.p4passwd
])
2549 command
.extend(['-c', self
.p4client
])
2550 command
.extend(['changes', '-m', '1', '#have'])
2551 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
2552 environ
=self
.env
, timeout
=self
.timeout
,
2553 sendStdout
=True, sendStderr
=False, sendRC
=False,
2554 keepStdout
=True, usePTY
=False)
2559 # 'p4 -c clien-name change -m 1 "#have"' will produce an output like:
2560 # "Change 28147 on 2008/04/07 by p4user@hostname..."
2561 # The number after "Change" is the one we want.
2562 m
= re
.match('Change\s+(\d+)\s+', c
.stdout
)
2566 d
.addCallback(_parse
)
2571 """A P4 source-updater.
2573 ['p4port'] (required): host:port for server to access
2574 ['p4user'] (optional): user to use for access
2575 ['p4passwd'] (optional): passwd to try for the user
2576 ['p4client'] (optional): client spec to use
2577 ['p4extra_views'] (optional): additional client views to use
2582 def setup(self
, args
):
2583 P4Base
.setup(self
, args
)
2584 self
.p4base
= args
['p4base']
2585 self
.p4extra_views
= args
['p4extra_views']
2586 self
.p4mode
= args
['mode']
2587 self
.p4branch
= args
['branch']
2589 self
.sourcedata
= str([
2596 # Depot side of view spec.
2601 # Local side of view spec (srcdir is made from these).
2602 self
.builder
.basedir
,
2608 def sourcedirIsUpdateable(self
):
2609 if os
.path
.exists(os
.path
.join(self
.builder
.basedir
,
2610 self
.srcdir
, ".buildbot-patched")):
2612 # We assume our client spec is still around.
2613 # We just say we aren't updateable if the dir doesn't exist so we
2614 # don't get ENOENT checking the sourcedata.
2615 return os
.path
.isdir(os
.path
.join(self
.builder
.basedir
,
2618 def doVCUpdate(self
):
2619 return self
._doP
4Sync
(force
=False)
2621 def _doP4Sync(self
, force
):
2625 command
.extend(['-p', self
.p4port
])
2627 command
.extend(['-u', self
.p4user
])
2629 command
.extend(['-P', self
.p4passwd
])
2631 command
.extend(['-c', self
.p4client
])
2632 command
.extend(['sync'])
2634 command
.extend(['-f'])
2636 command
.extend(['@' + str(self
.revision
)])
2638 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
2639 environ
=env
, sendRC
=False, timeout
=self
.timeout
,
2640 keepStdout
=True, usePTY
=False)
2643 d
.addCallback(self
._abandonOnFailure
)
2651 client_spec
+= "Client: %s\n\n" % self
.p4client
2652 client_spec
+= "Owner: %s\n\n" % self
.p4user
2653 client_spec
+= "Description:\n\tCreated by %s\n\n" % self
.p4user
2654 client_spec
+= "Root:\t%s\n\n" % self
.builder
.basedir
2655 client_spec
+= "Options:\tallwrite rmdir\n\n"
2656 client_spec
+= "LineEnd:\tlocal\n\n"
2659 client_spec
+= "View:\n\t%s" % (self
.p4base
)
2661 client_spec
+= "%s/" % (self
.p4branch
)
2662 client_spec
+= "... //%s/%s/...\n" % (self
.p4client
, self
.srcdir
)
2663 if self
.p4extra_views
:
2664 for k
, v
in self
.p4extra_views
:
2665 client_spec
+= "\t%s/... //%s/%s%s/...\n" % (k
, self
.p4client
,
2668 command
.extend(['-p', self
.p4port
])
2670 command
.extend(['-u', self
.p4user
])
2672 command
.extend(['-P', self
.p4passwd
])
2673 command
.extend(['client', '-i'])
2674 log
.msg(client_spec
)
2675 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
2676 environ
=env
, sendRC
=False, timeout
=self
.timeout
,
2677 initialStdin
=client_spec
, usePTY
=False)
2680 d
.addCallback(self
._abandonOnFailure
)
2681 d
.addCallback(lambda _
: self
._doP
4Sync
(force
=True))
2684 registerSlaveCommand("p4", P4
, command_version
)
2687 class P4Sync(P4Base
):
2688 """A partial P4 source-updater. Requires manual setup of a per-slave P4
2689 environment. The only thing which comes from the master is P4PORT.
2690 'mode' is required to be 'copy'.
2692 ['p4port'] (required): host:port for server to access
2693 ['p4user'] (optional): user to use for access
2694 ['p4passwd'] (optional): passwd to try for the user
2695 ['p4client'] (optional): client spec to use
2700 def setup(self
, args
):
2701 P4Base
.setup(self
, args
)
2702 self
.vcexe
= getCommand("p4")
2704 def sourcedirIsUpdateable(self
):
2707 def _doVC(self
, force
):
2708 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2709 command
= [self
.vcexe
]
2711 command
.extend(['-p', self
.p4port
])
2713 command
.extend(['-u', self
.p4user
])
2715 command
.extend(['-P', self
.p4passwd
])
2717 command
.extend(['-c', self
.p4client
])
2718 command
.extend(['sync'])
2720 command
.extend(['-f'])
2722 command
.extend(['@' + self
.revision
])
2724 c
= ShellCommand(self
.builder
, command
, d
, environ
=env
,
2725 sendRC
=False, timeout
=self
.timeout
, usePTY
=False)
2729 def doVCUpdate(self
):
2730 return self
._doVC
(force
=False)
2733 return self
._doVC
(force
=True)
2735 registerSlaveCommand("p4sync", P4Sync
, command_version
)