1 # -*- test-case-name: buildbot.test.test_slavecommand -*-
3 import os
, re
, signal
, shutil
, types
, time
4 from stat
import ST_CTIME
, ST_MTIME
, ST_SIZE
6 from zope
.interface
import implements
7 from twisted
.internet
.protocol
import ProcessProtocol
8 from twisted
.internet
import reactor
, defer
, task
9 from twisted
.python
import log
, failure
, runtime
10 from twisted
.python
.procutils
import which
12 from buildbot
.slave
.interfaces
import ISlaveCommand
13 from buildbot
.slave
.registry
import registerSlaveCommand
15 # this used to be a CVS $-style "Revision" auto-updated keyword, but since I
16 # moved to Darcs as the primary repository, this is updated manually each
17 # time this file is changed. The last cvs_ver that was here was 1.51 .
18 command_version
= "2.5"
21 # >=1.17: commands are interruptable
22 # >=1.28: Arch understands 'revision', added Bazaar
23 # >=1.33: Source classes understand 'retry'
24 # >=1.39: Source classes correctly handle changes in branch (except Git)
25 # Darcs accepts 'revision' (now all do but Git) (well, and P4Sync)
26 # Arch/Baz should accept 'build-config'
27 # >=1.51: (release 0.7.3)
28 # >= 2.1: SlaveShellCommand now accepts 'initial_stdin', 'keep_stdin_open',
29 # and 'logfiles'. It now sends 'log' messages in addition to
30 # stdout/stdin/header/rc. It acquired writeStdin/closeStdin methods,
31 # but these are not remotely callable yet.
32 # (not externally visible: ShellCommandPP has writeStdin/closeStdin.
33 # ShellCommand accepts new arguments (logfiles=, initialStdin=,
34 # keepStdinOpen=) and no longer accepts stdin=)
36 # >= 2.2: added monotone, uploadFile, and downloadFile (release 0.7.5)
37 # >= 2.3: added bzr (release 0.7.6)
38 # >= 2.4: Git understands 'revision' and branches
39 # >= 2.5: workaround added for remote 'hg clone --rev REV' when hg<0.9.2
41 class CommandInterrupted(Exception):
43 class TimeoutError(Exception):
46 class AbandonChain(Exception):
47 """A series of chained steps can raise this exception to indicate that
48 one of the intermediate ShellCommands has failed, such that there is no
49 point in running the remainder. 'rc' should be the non-zero exit code of
50 the failing ShellCommand."""
53 return "<AbandonChain rc=%s>" % self
.args
[0]
56 possibles
= which(name
)
58 raise RuntimeError("Couldn't find executable for '%s'" % name
)
61 def rmdirRecursive(dir):
62 """This is a replacement for shutil.rmtree that works better under
63 windows. Thanks to Bear at the OSAF for the code."""
64 if not os
.path
.exists(dir):
67 if os
.path
.islink(dir):
71 # Verify the directory is read/write/execute for the current user
74 for name
in os
.listdir(dir):
75 full_name
= os
.path
.join(dir, name
)
76 # on Windows, if we don't have write permission we can't remove
77 # the file/directory either, so turn that on
79 if not os
.access(full_name
, os
.W_OK
):
80 # I think this is now redundant, but I don't have an NT
81 # machine to test on, so I'm going to leave it in place
83 os
.chmod(full_name
, 0600)
85 if os
.path
.isdir(full_name
):
86 rmdirRecursive(full_name
)
88 os
.chmod(full_name
, 0700)
92 class ShellCommandPP(ProcessProtocol
):
95 def __init__(self
, command
):
96 self
.command
= command
97 self
.pending_stdin
= ""
98 self
.stdin_finished
= False
100 def writeStdin(self
, data
):
101 assert not self
.stdin_finished
103 self
.transport
.write(data
)
105 self
.pending_stdin
+= data
107 def closeStdin(self
):
109 if self
.debug
: log
.msg(" closing stdin")
110 self
.transport
.closeStdin()
111 self
.stdin_finished
= True
113 def connectionMade(self
):
115 log
.msg("ShellCommandPP.connectionMade")
116 if not self
.command
.process
:
118 log
.msg(" assigning self.command.process: %s" %
120 self
.command
.process
= self
.transport
122 # TODO: maybe we shouldn't close stdin when using a PTY. I can't test
123 # this yet, recent debian glibc has a bug which causes thread-using
124 # test cases to SIGHUP trial, and the workaround is to either run
125 # the whole test with /bin/sh -c " ".join(argv) (way gross) or to
126 # not use a PTY. Once the bug is fixed, I'll be able to test what
127 # happens when you close stdin on a pty. My concern is that it will
128 # SIGHUP the child (since we are, in a sense, hanging up on them).
129 # But it may well be that keeping stdout open prevents the SIGHUP
131 #if not self.command.usePTY:
133 if self
.pending_stdin
:
134 if self
.debug
: log
.msg(" writing to stdin")
135 self
.transport
.write(self
.pending_stdin
)
136 if self
.stdin_finished
:
137 if self
.debug
: log
.msg(" closing stdin")
138 self
.transport
.closeStdin()
140 def outReceived(self
, data
):
142 log
.msg("ShellCommandPP.outReceived")
143 self
.command
.addStdout(data
)
145 def errReceived(self
, data
):
147 log
.msg("ShellCommandPP.errReceived")
148 self
.command
.addStderr(data
)
150 def processEnded(self
, status_object
):
152 log
.msg("ShellCommandPP.processEnded", status_object
)
153 # status_object is a Failure wrapped around an
154 # error.ProcessTerminated or and error.ProcessDone.
155 # requires twisted >= 1.0.4 to overcome a bug in process.py
156 sig
= status_object
.value
.signal
157 rc
= status_object
.value
.exitCode
158 self
.command
.finished(sig
, rc
)
160 class LogFileWatcher
:
163 def __init__(self
, command
, name
, logfile
):
164 self
.command
= command
166 self
.logfile
= logfile
167 log
.msg("LogFileWatcher created to watch %s" % logfile
)
168 # we are created before the ShellCommand starts. If the logfile we're
169 # supposed to be watching already exists, record its size and
170 # ctime/mtime so we can tell when it starts to change.
171 self
.old_logfile_stats
= self
.statFile()
174 # every 2 seconds we check on the file again
175 self
.poller
= task
.LoopingCall(self
.poll
)
178 self
.poller
.start(self
.POLL_INTERVAL
).addErrback(self
._cleanupPoll
)
180 def _cleanupPoll(self
, err
):
181 log
.err(err
, msg
="Polling error")
186 if self
.poller
is not None:
192 if os
.path
.exists(self
.logfile
):
193 s
= os
.stat(self
.logfile
)
194 return (s
[ST_CTIME
], s
[ST_MTIME
], s
[ST_SIZE
])
200 if s
== self
.old_logfile_stats
:
201 return # not started yet
203 # the file was there, but now it's deleted. Forget about the
204 # initial state, clearly the process has deleted the logfile
205 # in preparation for creating a new one.
206 self
.old_logfile_stats
= None
207 return # no file to work with
208 self
.f
= open(self
.logfile
, "rb")
210 self
.f
.seek(self
.f
.tell(), 0)
212 data
= self
.f
.read(10000)
215 self
.command
.addLogfile(self
.name
, data
)
219 # This is a helper class, used by SlaveCommands to run programs in a
225 CHUNK_LIMIT
= 128*1024
227 # For sending elapsed time:
230 # I wish we had easy access to CLOCK_MONOTONIC in Python:
231 # http://www.opengroup.org/onlinepubs/000095399/functions/clock_getres.html
232 # Then changes to the system clock during a run wouldn't effect the "elapsed
235 def __init__(self
, builder
, command
,
236 workdir
, environ
=None,
237 sendStdout
=True, sendStderr
=True, sendRC
=True,
238 timeout
=None, initialStdin
=None, keepStdinOpen
=False,
239 keepStdout
=False, keepStderr
=False,
243 @param keepStdout: if True, we keep a copy of all the stdout text
244 that we've seen. This copy is available in
245 self.stdout, which can be read after the command
247 @param keepStderr: same, for stderr
251 self
.builder
= builder
252 self
.command
= command
253 self
.sendStdout
= sendStdout
254 self
.sendStderr
= sendStderr
256 self
.logfiles
= logfiles
257 self
.workdir
= workdir
258 self
.environ
= os
.environ
.copy()
260 if environ
.has_key('PYTHONPATH'):
261 ppath
= environ
['PYTHONPATH']
262 # Need to do os.pathsep translation. We could either do that
263 # by replacing all incoming ':'s with os.pathsep, or by
264 # accepting lists. I like lists better.
265 if not isinstance(ppath
, str):
266 # If it's not a string, treat it as a sequence to be
267 # turned in to a string.
268 ppath
= os
.pathsep
.join(ppath
)
270 if self
.environ
.has_key('PYTHONPATH'):
271 # special case, prepend the builder's items to the
272 # existing ones. This will break if you send over empty
273 # strings, so don't do that.
274 ppath
= ppath
+ os
.pathsep
+ self
.environ
['PYTHONPATH']
276 environ
['PYTHONPATH'] = ppath
278 self
.environ
.update(environ
)
279 self
.initialStdin
= initialStdin
280 self
.keepStdinOpen
= keepStdinOpen
281 self
.timeout
= timeout
283 self
.keepStdout
= keepStdout
284 self
.keepStderr
= keepStderr
286 # usePTY=True is a convenience for cleaning up all children and
287 # grandchildren of a hung command. Fall back to usePTY=False on
288 # systems where ptys cause problems.
290 self
.usePTY
= self
.builder
.usePTY
291 if runtime
.platformType
!= "posix":
292 self
.usePTY
= False # PTYs are posix-only
293 if initialStdin
is not None:
294 # for .closeStdin to matter, we must use a pipe, not a PTY
297 self
.logFileWatchers
= []
298 for name
,filename
in self
.logfiles
.items():
299 w
= LogFileWatcher(self
, name
,
300 os
.path
.join(self
.workdir
, filename
))
301 self
.logFileWatchers
.append(w
)
304 return "<slavecommand.ShellCommand '%s'>" % self
.command
306 def sendStatus(self
, status
):
307 self
.builder
.sendUpdate(status
)
310 # return a Deferred which fires (with the exit code) when the command
316 self
.deferred
= defer
.Deferred()
320 log
.msg("error in ShellCommand._startCommand")
322 # pretend it was a shell error
323 self
.deferred
.errback(AbandonChain(-1))
326 def _startCommand(self
):
327 # ensure workdir exists
328 if not os
.path
.isdir(self
.workdir
):
329 os
.makedirs(self
.workdir
)
330 log
.msg("ShellCommand._startCommand")
332 self
.sendStatus({'header': "command '%s' in dir %s" % \
333 (self
.command
, self
.workdir
)})
334 self
.sendStatus({'header': "(not really)\n"})
335 self
.finished(None, 0)
338 self
.pp
= ShellCommandPP(self
)
340 if type(self
.command
) in types
.StringTypes
:
341 if runtime
.platformType
== 'win32':
342 argv
= [os
.environ
['COMSPEC'], '/c', self
.command
]
344 # for posix, use /bin/sh. for other non-posix, well, doesn't
346 argv
= ['/bin/sh', '-c', self
.command
]
348 if runtime
.platformType
== 'win32':
349 argv
= [os
.environ
['COMSPEC'], '/c'] + list(self
.command
)
353 # self.stdin is handled in ShellCommandPP.connectionMade
355 # first header line is the command in plain text, argv joined with
356 # spaces. You should be able to cut-and-paste this into a shell to
357 # obtain the same results. If there are spaces in the arguments, too
361 self
.sendStatus({'header': msg
+"\n"})
363 # then comes the secondary information
364 msg
= " in dir %s" % (self
.workdir
,)
366 msg
+= " (timeout %d secs)" % (self
.timeout
,)
368 self
.sendStatus({'header': msg
+"\n"})
370 msg
= " watching logfiles %s" % (self
.logfiles
,)
372 self
.sendStatus({'header': msg
+"\n"})
374 # then the argv array for resolving unambiguity
375 msg
= " argv: %s" % (argv
,)
377 self
.sendStatus({'header': msg
+"\n"})
379 # then the environment, since it sometimes causes problems
380 msg
= " environment:\n"
381 env_names
= self
.environ
.keys()
383 for name
in env_names
:
384 msg
+= " %s=%s\n" % (name
, self
.environ
[name
])
385 log
.msg(" environment: %s" % (self
.environ
,))
386 self
.sendStatus({'header': msg
})
388 if self
.initialStdin
:
389 msg
= " writing %d bytes to stdin" % len(self
.initialStdin
)
391 self
.sendStatus({'header': msg
+"\n"})
393 if self
.keepStdinOpen
:
394 msg
= " leaving stdin open"
396 msg
= " closing stdin"
398 self
.sendStatus({'header': msg
+"\n"})
400 msg
= " using PTY: %s" % bool(self
.usePTY
)
402 self
.sendStatus({'header': msg
+"\n"})
404 # this will be buffered until connectionMade is called
405 if self
.initialStdin
:
406 self
.pp
.writeStdin(self
.initialStdin
)
407 if not self
.keepStdinOpen
:
410 # win32eventreactor's spawnProcess (under twisted <= 2.0.1) returns
411 # None, as opposed to all the posixbase-derived reactors (which
412 # return the new Process object). This is a nuisance. We can make up
413 # for it by having the ProcessProtocol give us their .transport
414 # attribute after they get one. I'd prefer to get it from
415 # spawnProcess because I'm concerned about returning from this method
416 # without having a valid self.process to work with. (if kill() were
417 # called right after we return, but somehow before connectionMade
418 # were called, then kill() would blow up).
420 self
.startTime
= time
.time()
421 p
= reactor
.spawnProcess(self
.pp
, argv
[0], argv
,
425 # connectionMade might have been called during spawnProcess
429 # connectionMade also closes stdin as long as we're not using a PTY.
430 # This is intended to kill off inappropriately interactive commands
431 # better than the (long) hung-command timeout. ProcessPTY should be
432 # enhanced to allow the same childFDs argument that Process takes,
433 # which would let us connect stdin to /dev/null .
436 self
.timer
= reactor
.callLater(self
.timeout
, self
.doTimeout
)
438 for w
in self
.logFileWatchers
:
442 def _chunkForSend(self
, data
):
443 # limit the chunks that we send over PB to 128k, since it has a
444 # hardwired string-size limit of 640k.
445 LIMIT
= self
.CHUNK_LIMIT
446 for i
in range(0, len(data
), LIMIT
):
447 yield data
[i
:i
+LIMIT
]
449 def addStdout(self
, data
):
451 for chunk
in self
._chunkForSend
(data
):
452 self
.sendStatus({'stdout': chunk
})
456 self
.timer
.reset(self
.timeout
)
458 def addStderr(self
, data
):
460 for chunk
in self
._chunkForSend
(data
):
461 self
.sendStatus({'stderr': chunk
})
465 self
.timer
.reset(self
.timeout
)
467 def addLogfile(self
, name
, data
):
468 for chunk
in self
._chunkForSend
(data
):
469 self
.sendStatus({'log': (name
, chunk
)})
471 self
.timer
.reset(self
.timeout
)
473 def finished(self
, sig
, rc
):
474 self
.elapsedTime
= time
.time() - self
.startTime
475 log
.msg("command finished with signal %s, exit code %s, elapsedTime: %0.6f" % (sig
,rc
,self
.elapsedTime
))
476 for w
in self
.logFileWatchers
:
477 # this will send the final updates
484 {'header': "process killed by signal %d\n" % sig
})
485 self
.sendStatus({'rc': rc
})
486 self
.sendStatus({'header': "elapsedTime=%0.6f\n" % self
.elapsedTime
})
495 log
.msg("Hey, command %s finished twice" % self
)
497 def failed(self
, why
):
498 log
.msg("ShellCommand.failed: command failed: %s" % (why
,))
507 log
.msg("Hey, command %s finished twice" % self
)
511 msg
= "command timed out: %d seconds without output" % self
.timeout
515 # This may be called by the timeout, or when the user has decided to
520 if hasattr(self
.process
, "pid"):
521 msg
+= ", killing pid %d" % self
.process
.pid
523 self
.sendStatus({'header': "\n" + msg
+ "\n"})
526 if runtime
.platformType
== "posix":
528 # really want to kill off all child processes too. Process
529 # Groups are ideal for this, but that requires
530 # spawnProcess(usePTY=1). Try both ways in case process was
531 # not started that way.
533 # the test suite sets self.KILL=None to tell us we should
534 # only pretend to kill the child. This lets us test the
538 if self
.KILL
is not None:
539 sig
= getattr(signal
, "SIG"+ self
.KILL
, None)
541 if self
.KILL
== None:
542 log
.msg("self.KILL==None, only pretending to kill child")
544 log
.msg("signal module is missing SIG%s" % self
.KILL
)
545 elif not hasattr(os
, "kill"):
546 log
.msg("os module is missing the 'kill' function")
548 log
.msg("trying os.kill(-pid, %d)" % (sig
,))
549 # TODO: maybe use os.killpg instead of a negative pid?
550 os
.kill(-self
.process
.pid
, sig
)
551 log
.msg(" signal %s sent successfully" % sig
)
554 # probably no-such-process, maybe because there is no process
559 if self
.KILL
is None:
560 log
.msg("self.KILL==None, only pretending to kill child")
562 log
.msg("trying process.signalProcess('KILL')")
563 self
.process
.signalProcess(self
.KILL
)
564 log
.msg(" signal %s sent successfully" % (self
.KILL
,))
567 # could be no-such-process, because they finished very recently
570 log
.msg("signalProcess/os.kill failed both times")
572 if runtime
.platformType
== "posix":
573 # we only do this under posix because the win32eventreactor
574 # blocks here until the process has terminated, while closing
575 # stderr. This is weird.
576 self
.pp
.transport
.loseConnection()
578 # finished ought to be called momentarily. Just in case it doesn't,
579 # set a timer which will abandon the command.
580 self
.timer
= reactor
.callLater(self
.BACKUP_TIMEOUT
,
581 self
.doBackupTimeout
)
583 def doBackupTimeout(self
):
584 log
.msg("we tried to kill the process, and it wouldn't die.."
587 self
.sendStatus({'header': "SIGKILL failed to kill process\n"})
589 self
.sendStatus({'header': "using fake rc=-1\n"})
590 self
.sendStatus({'rc': -1})
591 self
.failed(TimeoutError("SIGKILL failed to kill process"))
594 def writeStdin(self
, data
):
595 self
.pp
.writeStdin(data
)
597 def closeStdin(self
):
602 implements(ISlaveCommand
)
604 """This class defines one command that can be invoked by the build master.
605 The command is executed on the slave side, and always sends back a
606 completion message when it finishes. It may also send intermediate status
607 as it runs (by calling builder.sendStatus). Some commands can be
608 interrupted (either by the build master or a local timeout), in which
609 case the step is expected to complete normally with a status message that
610 indicates an error occurred.
612 These commands are used by BuildSteps on the master side. Each kind of
613 BuildStep uses a single Command. The slave must implement all the
614 Commands required by the set of BuildSteps used for any given build:
615 this is checked at startup time.
617 All Commands are constructed with the same signature:
618 c = CommandClass(builder, args)
619 where 'builder' is the parent SlaveBuilder object, and 'args' is a
620 dict that is interpreted per-command.
622 The setup(args) method is available for setup, and is run from __init__.
624 The Command is started with start(). This method must be implemented in a
625 subclass, and it should return a Deferred. When your step is done, you
626 should fire the Deferred (the results are not used). If the command is
627 interrupted, it should fire the Deferred anyway.
629 While the command runs. it may send status messages back to the
630 buildmaster by calling self.sendStatus(statusdict). The statusdict is
631 interpreted by the master-side BuildStep however it likes.
633 A separate completion message is sent when the deferred fires, which
634 indicates that the Command has finished, but does not carry any status
635 data. If the Command needs to return an exit code of some sort, that
636 should be sent as a regular status message before the deferred is fired .
637 Once builder.commandComplete has been run, no more status messages may be
640 If interrupt() is called, the Command should attempt to shut down as
641 quickly as possible. Child processes should be killed, new ones should
642 not be started. The Command should send some kind of error status update,
643 then complete as usual by firing the Deferred.
645 .interrupted should be set by interrupt(), and can be tested to avoid
646 sending multiple error status messages.
648 If .running is False, the bot is shutting down (or has otherwise lost the
649 connection to the master), and should not send any status messages. This
650 is checked in Command.sendStatus .
655 # sendStatus(dict) (zero or more)
656 # commandComplete() or commandInterrupted() (one, at end)
660 running
= False # set by Builder, cleared on shutdown or when the
663 def __init__(self
, builder
, stepId
, args
):
664 self
.builder
= builder
665 self
.stepId
= stepId
# just for logging
669 def setup(self
, args
):
670 """Override this in a subclass to extract items from the args dict."""
675 d
= defer
.maybeDeferred(self
.start
)
676 d
.addBoth(self
.commandComplete
)
680 """Start the command. This method should return a Deferred that will
681 fire when the command has completed. The Deferred's argument will be
684 This method should be overridden by subclasses."""
685 raise NotImplementedError, "You must implement this in a subclass"
687 def sendStatus(self
, status
):
688 """Send a status update to the master."""
690 log
.msg("sendStatus", status
)
692 log
.msg("would sendStatus but not .running")
694 self
.builder
.sendUpdate(status
)
696 def doInterrupt(self
):
701 """Override this in a subclass to allow commands to be interrupted.
702 May be called multiple times, test and set self.interrupted=True if
706 def commandComplete(self
, res
):
710 # utility methods, mostly used by SlaveShellCommand and the like
712 def _abandonOnFailure(self
, rc
):
713 if type(rc
) is not int:
714 log
.msg("weird, _abandonOnFailure was given rc=%s (%s)" % \
716 assert isinstance(rc
, int)
718 raise AbandonChain(rc
)
721 def _sendRC(self
, res
):
722 self
.sendStatus({'rc': 0})
724 def _checkAbandoned(self
, why
):
725 log
.msg("_checkAbandoned", why
)
726 why
.trap(AbandonChain
)
727 log
.msg(" abandoning chain", why
.value
)
728 self
.sendStatus({'rc': why
.value
.args
[0]})
733 class SlaveFileUploadCommand(Command
):
735 Upload a file from slave to build master
738 - ['workdir']: base directory to use
739 - ['slavesrc']: name of the slave-side file to read from
740 - ['writer']: RemoteReference to a transfer._FileWriter object
741 - ['maxsize']: max size (in bytes) of file to write
742 - ['blocksize']: max size for each data block
746 def setup(self
, args
):
747 self
.workdir
= args
['workdir']
748 self
.filename
= args
['slavesrc']
749 self
.writer
= args
['writer']
750 self
.remaining
= args
['maxsize']
751 self
.blocksize
= args
['blocksize']
757 log
.msg('SlaveFileUploadCommand started')
760 self
.path
= os
.path
.join(self
.builder
.basedir
,
762 os
.path
.expanduser(self
.filename
))
764 self
.fp
= open(self
.path
, 'rb')
766 log
.msg('Opened %r for upload' % self
.path
)
768 # TODO: this needs cleanup
770 self
.stderr
= 'Cannot open file %r for upload' % self
.path
773 log
.msg('Cannot open file %r for upload' % self
.path
)
775 self
.sendStatus({'header': "sending %s" % self
.path
})
778 reactor
.callLater(0, self
._loop
, d
)
780 # close the file, but pass through any errors from _loop
781 d1
= self
.writer
.callRemote("close")
782 d1
.addErrback(log
.err
)
783 d1
.addCallback(lambda ignored
: res
)
786 d
.addBoth(self
.finished
)
789 def _loop(self
, fire_when_done
):
790 d
= defer
.maybeDeferred(self
._writeBlock
)
793 fire_when_done
.callback(None)
795 self
._loop
(fire_when_done
)
797 fire_when_done
.errback(why
)
798 d
.addCallbacks(_done
, _err
)
801 def _writeBlock(self
):
802 """Write a block of data to the remote writer"""
804 if self
.interrupted
or self
.fp
is None:
806 log
.msg('SlaveFileUploadCommand._writeBlock(): end')
809 length
= self
.blocksize
810 if self
.remaining
is not None and length
> self
.remaining
:
811 length
= self
.remaining
814 if self
.stderr
is None:
815 self
.stderr
= 'Maximum filesize reached, truncating file %r' \
820 data
= self
.fp
.read(length
)
823 log
.msg('SlaveFileUploadCommand._writeBlock(): '+
824 'allowed=%d readlen=%d' % (length
, len(data
)))
826 log
.msg("EOF: callRemote(close)")
829 if self
.remaining
is not None:
830 self
.remaining
= self
.remaining
- len(data
)
831 assert self
.remaining
>= 0
832 d
= self
.writer
.callRemote('write', data
)
833 d
.addCallback(lambda res
: False)
838 log
.msg('interrupted')
841 if self
.stderr
is None:
842 self
.stderr
= 'Upload of %r interrupted' % self
.path
844 self
.interrupted
= True
845 # the next _writeBlock call will notice the .interrupted flag
847 def finished(self
, res
):
849 log
.msg('finished: stderr=%r, rc=%r' % (self
.stderr
, self
.rc
))
850 if self
.stderr
is None:
851 self
.sendStatus({'rc': self
.rc
})
853 self
.sendStatus({'stderr': self
.stderr
, 'rc': self
.rc
})
856 registerSlaveCommand("uploadFile", SlaveFileUploadCommand
, command_version
)
859 class SlaveFileDownloadCommand(Command
):
861 Download a file from master to slave
864 - ['workdir']: base directory to use
865 - ['slavedest']: name of the slave-side file to be created
866 - ['reader']: RemoteReference to a transfer._FileReader object
867 - ['maxsize']: max size (in bytes) of file to write
868 - ['blocksize']: max size for each data block
869 - ['mode']: access mode for the new file
873 def setup(self
, args
):
874 self
.workdir
= args
['workdir']
875 self
.filename
= args
['slavedest']
876 self
.reader
= args
['reader']
877 self
.bytes_remaining
= args
['maxsize']
878 self
.blocksize
= args
['blocksize']
879 self
.mode
= args
['mode']
885 log
.msg('SlaveFileDownloadCommand starting')
888 self
.path
= os
.path
.join(self
.builder
.basedir
,
890 os
.path
.expanduser(self
.filename
))
892 dirname
= os
.path
.dirname(self
.path
)
893 if not os
.path
.exists(dirname
):
897 self
.fp
= open(self
.path
, 'wb')
899 log
.msg('Opened %r for download' % self
.path
)
900 if self
.mode
is not None:
901 # note: there is a brief window during which the new file
902 # will have the buildslave's default (umask) mode before we
903 # set the new one. Don't use this mode= feature to keep files
904 # private: use the buildslave's umask for that instead. (it
905 # is possible to call os.umask() before and after the open()
906 # call, but cleaning up from exceptions properly is more of a
907 # nuisance that way).
908 os
.chmod(self
.path
, self
.mode
)
910 # TODO: this still needs cleanup
912 self
.stderr
= 'Cannot open file %r for download' % self
.path
915 log
.msg('Cannot open file %r for download' % self
.path
)
918 reactor
.callLater(0, self
._loop
, d
)
920 # close the file, but pass through any errors from _loop
921 d1
= self
.reader
.callRemote('close')
922 d1
.addErrback(log
.err
)
923 d1
.addCallback(lambda ignored
: res
)
926 d
.addBoth(self
.finished
)
929 def _loop(self
, fire_when_done
):
930 d
= defer
.maybeDeferred(self
._readBlock
)
933 fire_when_done
.callback(None)
935 self
._loop
(fire_when_done
)
937 fire_when_done
.errback(why
)
938 d
.addCallbacks(_done
, _err
)
941 def _readBlock(self
):
942 """Read a block of data from the remote reader."""
944 if self
.interrupted
or self
.fp
is None:
946 log
.msg('SlaveFileDownloadCommand._readBlock(): end')
949 length
= self
.blocksize
950 if self
.bytes_remaining
is not None and length
> self
.bytes_remaining
:
951 length
= self
.bytes_remaining
954 if self
.stderr
is None:
955 self
.stderr
= 'Maximum filesize reached, truncating file %r' \
960 d
= self
.reader
.callRemote('read', length
)
961 d
.addCallback(self
._writeData
)
964 def _writeData(self
, data
):
966 log
.msg('SlaveFileDownloadCommand._readBlock(): readlen=%d' %
971 if self
.bytes_remaining
is not None:
972 self
.bytes_remaining
= self
.bytes_remaining
- len(data
)
973 assert self
.bytes_remaining
>= 0
979 log
.msg('interrupted')
982 if self
.stderr
is None:
983 self
.stderr
= 'Download of %r interrupted' % self
.path
985 self
.interrupted
= True
986 # now we wait for the next read request to return. _readBlock will
987 # abandon the file when it sees self.interrupted set.
989 def finished(self
, res
):
990 if self
.fp
is not None:
994 log
.msg('finished: stderr=%r, rc=%r' % (self
.stderr
, self
.rc
))
995 if self
.stderr
is None:
996 self
.sendStatus({'rc': self
.rc
})
998 self
.sendStatus({'stderr': self
.stderr
, 'rc': self
.rc
})
1001 registerSlaveCommand("downloadFile", SlaveFileDownloadCommand
, command_version
)
1005 class SlaveShellCommand(Command
):
1006 """This is a Command which runs a shell command. The args dict contains
1009 - ['command'] (required): a shell command to run. If this is a string,
1010 it will be run with /bin/sh (['/bin/sh',
1011 '-c', command]). If it is a list
1012 (preferred), it will be used directly.
1013 - ['workdir'] (required): subdirectory in which the command will be
1014 run, relative to the builder dir
1015 - ['env']: a dict of environment variables to augment/replace
1016 os.environ . PYTHONPATH is treated specially, and
1017 should be a list of path components to be prepended to
1018 any existing PYTHONPATH environment variable.
1019 - ['initial_stdin']: a string which will be written to the command's
1020 stdin as soon as it starts
1021 - ['keep_stdin_open']: unless True, the command's stdin will be
1022 closed as soon as initial_stdin has been
1023 written. Set this to True if you plan to write
1024 to stdin after the command has been started.
1025 - ['want_stdout']: 0 if stdout should be thrown away
1026 - ['want_stderr']: 0 if stderr should be thrown away
1027 - ['not_really']: 1 to skip execution and return rc=0
1028 - ['timeout']: seconds of silence to tolerate before killing command
1029 - ['logfiles']: dict mapping LogFile name to the workdir-relative
1030 filename of a local log file. This local file will be
1031 watched just like 'tail -f', and all changes will be
1032 written to 'log' status updates.
1034 ShellCommand creates the following status messages:
1035 - {'stdout': data} : when stdout data is available
1036 - {'stderr': data} : when stderr data is available
1037 - {'header': data} : when headers (command start/stop) are available
1038 - {'log': (logfile_name, data)} : when log files have new contents
1039 - {'rc': rc} : when the process has terminated
1044 # args['workdir'] is relative to Builder directory, and is required.
1045 assert args
['workdir'] is not None
1046 workdir
= os
.path
.join(self
.builder
.basedir
, args
['workdir'])
1048 c
= ShellCommand(self
.builder
, args
['command'],
1049 workdir
, environ
=args
.get('env'),
1050 timeout
=args
.get('timeout', None),
1051 sendStdout
=args
.get('want_stdout', True),
1052 sendStderr
=args
.get('want_stderr', True),
1054 initialStdin
=args
.get('initial_stdin'),
1055 keepStdinOpen
=args
.get('keep_stdin_open'),
1056 logfiles
=args
.get('logfiles', {}),
1059 d
= self
.command
.start()
1062 def interrupt(self
):
1063 self
.interrupted
= True
1064 self
.command
.kill("command interrupted")
1066 def writeStdin(self
, data
):
1067 self
.command
.writeStdin(data
)
1069 def closeStdin(self
):
1070 self
.command
.closeStdin()
1072 registerSlaveCommand("shell", SlaveShellCommand
, command_version
)
1075 class DummyCommand(Command
):
1077 I am a dummy no-op command that by default takes 5 seconds to complete.
1078 See L{buildbot.steps.dummy.RemoteDummy}
1082 self
.d
= defer
.Deferred()
1083 log
.msg(" starting dummy command [%s]" % self
.stepId
)
1084 self
.timer
= reactor
.callLater(1, self
.doStatus
)
1087 def interrupt(self
):
1088 if self
.interrupted
:
1092 self
.interrupted
= True
1096 log
.msg(" sending intermediate status")
1097 self
.sendStatus({'stdout': 'data'})
1098 timeout
= self
.args
.get('timeout', 5) + 1
1099 self
.timer
= reactor
.callLater(timeout
- 1, self
.finished
)
1102 log
.msg(" dummy command finished [%s]" % self
.stepId
)
1103 if self
.interrupted
:
1104 self
.sendStatus({'rc': 1})
1106 self
.sendStatus({'rc': 0})
1109 registerSlaveCommand("dummy", DummyCommand
, command_version
)
1112 # this maps handle names to a callable. When the WaitCommand starts, this
1113 # callable is invoked with no arguments. It should return a Deferred. When
1114 # that Deferred fires, our WaitCommand will finish.
1115 waitCommandRegistry
= {}
1117 class WaitCommand(Command
):
1119 I am a dummy command used by the buildbot unit test suite. I want for the
1120 unit test to tell us to finish. See L{buildbot.steps.dummy.Wait}
1124 self
.d
= defer
.Deferred()
1125 log
.msg(" starting wait command [%s]" % self
.stepId
)
1126 handle
= self
.args
['handle']
1127 cb
= waitCommandRegistry
[handle
]
1128 del waitCommandRegistry
[handle
]
1130 log
.msg(" wait-%s starting" % (handle
,))
1133 log
.msg(" wait-%s finishing: %s" % (handle
, res
))
1136 d
.addCallbacks(self
.finished
, self
.failed
)
1137 reactor
.callLater(0, _called
)
1140 def interrupt(self
):
1141 log
.msg(" wait command interrupted")
1142 if self
.interrupted
:
1144 self
.interrupted
= True
1145 self
.finished("interrupted")
1147 def finished(self
, res
):
1148 log
.msg(" wait command finished [%s]" % self
.stepId
)
1149 if self
.interrupted
:
1150 self
.sendStatus({'rc': 2})
1152 self
.sendStatus({'rc': 0})
1154 def failed(self
, why
):
1155 log
.msg(" wait command failed [%s]" % self
.stepId
)
1156 self
.sendStatus({'rc': 1})
1159 registerSlaveCommand("dummy.wait", WaitCommand
, command_version
)
1162 class SourceBase(Command
):
1163 """Abstract base class for Version Control System operations (checkout
1164 and update). This class extracts the following arguments from the
1165 dictionary received from the master:
1167 - ['workdir']: (required) the subdirectory where the buildable sources
1170 - ['mode']: one of update/copy/clobber/export, defaults to 'update'
1172 - ['revision']: If not None, this is an int or string which indicates
1173 which sources (along a time-like axis) should be used.
1174 It is the thing you provide as the CVS -r or -D
1177 - ['patch']: If not None, this is a tuple of (striplevel, patch)
1178 which contains a patch that should be applied after the
1179 checkout has occurred. Once applied, the tree is no
1180 longer eligible for use with mode='update', and it only
1181 makes sense to use this in conjunction with a
1182 ['revision'] argument. striplevel is an int, and patch
1183 is a string in standard unified diff format. The patch
1184 will be applied with 'patch -p%d <PATCH', with
1185 STRIPLEVEL substituted as %d. The command will fail if
1186 the patch process fails (rejected hunks).
1188 - ['timeout']: seconds of silence tolerated before we kill off the
1191 - ['retry']: If not None, this is a tuple of (delay, repeats)
1192 which means that any failed VC updates should be
1193 reattempted, up to REPEATS times, after a delay of
1194 DELAY seconds. This is intended to deal with slaves
1195 that experience transient network failures.
1200 def setup(self
, args
):
1201 # if we need to parse the output, use this environment. Otherwise
1202 # command output will be in whatever the buildslave's native language
1204 self
.env
= os
.environ
.copy()
1205 self
.env
['LC_ALL'] = "C"
1207 self
.workdir
= args
['workdir']
1208 self
.mode
= args
.get('mode', "update")
1209 self
.revision
= args
.get('revision')
1210 self
.patch
= args
.get('patch')
1211 self
.timeout
= args
.get('timeout', 120)
1212 self
.retry
= args
.get('retry')
1213 # VC-specific subclasses should override this to extract more args.
1214 # Make sure to upcall!
1217 self
.sendStatus({'header': "starting " + self
.header
+ "\n"})
1220 # self.srcdir is where the VC system should put the sources
1221 if self
.mode
== "copy":
1222 self
.srcdir
= "source" # hardwired directory name, sorry
1224 self
.srcdir
= self
.workdir
1225 self
.sourcedatafile
= os
.path
.join(self
.builder
.basedir
,
1227 ".buildbot-sourcedata")
1229 d
= defer
.succeed(None)
1230 # do we need to clobber anything?
1231 if self
.mode
in ("copy", "clobber", "export"):
1232 d
.addCallback(self
.doClobber
, self
.workdir
)
1233 if not (self
.sourcedirIsUpdateable() and self
.sourcedataMatches()):
1234 # the directory cannot be updated, so we have to clobber it.
1235 # Perhaps the master just changed modes from 'export' to
1237 d
.addCallback(self
.doClobber
, self
.srcdir
)
1239 d
.addCallback(self
.doVC
)
1241 if self
.mode
== "copy":
1242 d
.addCallback(self
.doCopy
)
1244 d
.addCallback(self
.doPatch
)
1245 d
.addCallbacks(self
._sendRC
, self
._checkAbandoned
)
1248 def interrupt(self
):
1249 self
.interrupted
= True
1251 self
.command
.kill("command interrupted")
1253 def doVC(self
, res
):
1254 if self
.interrupted
:
1255 raise AbandonChain(1)
1256 if self
.sourcedirIsUpdateable() and self
.sourcedataMatches():
1257 d
= self
.doVCUpdate()
1258 d
.addCallback(self
.maybeDoVCFallback
)
1261 d
.addBoth(self
.maybeDoVCRetry
)
1262 d
.addCallback(self
._abandonOnFailure
)
1263 d
.addCallback(self
._handleGotRevision
)
1264 d
.addCallback(self
.writeSourcedata
)
1267 def sourcedataMatches(self
):
1269 olddata
= open(self
.sourcedatafile
, "r").read()
1270 if olddata
!= self
.sourcedata
:
1276 def _handleGotRevision(self
, res
):
1277 d
= defer
.maybeDeferred(self
.parseGotRevision
)
1278 d
.addCallback(lambda got_revision
:
1279 self
.sendStatus({'got_revision': got_revision
}))
1282 def parseGotRevision(self
):
1283 """Override this in a subclass. It should return a string that
1284 represents which revision was actually checked out, or a Deferred
1285 that will fire with such a string. If, in a future build, you were to
1286 pass this 'got_revision' string in as the 'revision' component of a
1287 SourceStamp, you should wind up with the same source code as this
1288 checkout just obtained.
1290 It is probably most useful to scan self.command.stdout for a string
1291 of some sort. Be sure to set keepStdout=True on the VC command that
1292 you run, so that you'll have something available to look at.
1294 If this information is unavailable, just return None."""
1298 def writeSourcedata(self
, res
):
1299 open(self
.sourcedatafile
, "w").write(self
.sourcedata
)
1302 def sourcedirIsUpdateable(self
):
1303 raise NotImplementedError("this must be implemented in a subclass")
1305 def doVCUpdate(self
):
1306 raise NotImplementedError("this must be implemented in a subclass")
1309 raise NotImplementedError("this must be implemented in a subclass")
1311 def maybeDoVCFallback(self
, rc
):
1312 if type(rc
) is int and rc
== 0:
1314 if self
.interrupted
:
1315 raise AbandonChain(1)
1316 msg
= "update failed, clobbering and trying again"
1317 self
.sendStatus({'header': msg
+ "\n"})
1319 d
= self
.doClobber(None, self
.srcdir
)
1320 d
.addCallback(self
.doVCFallback2
)
1323 def doVCFallback2(self
, res
):
1324 msg
= "now retrying VC operation"
1325 self
.sendStatus({'header': msg
+ "\n"})
1328 d
.addBoth(self
.maybeDoVCRetry
)
1329 d
.addCallback(self
._abandonOnFailure
)
1332 def maybeDoVCRetry(self
, res
):
1333 """We get here somewhere after a VC chain has finished. res could
1336 - 0: the operation was successful
1337 - nonzero: the operation failed. retry if possible
1338 - AbandonChain: the operation failed, someone else noticed. retry.
1339 - Failure: some other exception, re-raise
1342 if isinstance(res
, failure
.Failure
):
1343 if self
.interrupted
:
1344 return res
# don't re-try interrupted builds
1345 res
.trap(AbandonChain
)
1347 if type(res
) is int and res
== 0:
1349 if self
.interrupted
:
1350 raise AbandonChain(1)
1351 # if we get here, we should retry, if possible
1353 delay
, repeats
= self
.retry
1355 self
.retry
= (delay
, repeats
-1)
1356 msg
= ("update failed, trying %d more times after %d seconds"
1358 self
.sendStatus({'header': msg
+ "\n"})
1360 d
= defer
.Deferred()
1361 d
.addCallback(lambda res
: self
.doVCFull())
1362 d
.addBoth(self
.maybeDoVCRetry
)
1363 reactor
.callLater(delay
, d
.callback
, None)
1367 def doClobber(self
, dummy
, dirname
):
1368 # TODO: remove the old tree in the background
1369 ## workdir = os.path.join(self.builder.basedir, self.workdir)
1370 ## deaddir = self.workdir + ".deleting"
1371 ## if os.path.isdir(workdir):
1373 ## os.rename(workdir, deaddir)
1374 ## # might fail if deaddir already exists: previous deletion
1375 ## # hasn't finished yet
1376 ## # start the deletion in the background
1377 ## # TODO: there was a solaris/NetApp/NFS problem where a
1378 ## # process that was still running out of the directory we're
1379 ## # trying to delete could prevent the rm-rf from working. I
1380 ## # think it stalled the rm, but maybe it just died with
1381 ## # permission issues. Try to detect this.
1382 ## os.commands("rm -rf %s &" % deaddir)
1384 ## # fall back to sequential delete-then-checkout
1386 d
= os
.path
.join(self
.builder
.basedir
, dirname
)
1387 if runtime
.platformType
!= "posix":
1388 # if we're running on w32, use rmtree instead. It will block,
1389 # but hopefully it won't take too long.
1391 return defer
.succeed(0)
1392 command
= ["rm", "-rf", d
]
1393 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
1394 sendRC
=0, timeout
=self
.timeout
)
1396 # sendRC=0 means the rm command will send stdout/stderr to the
1397 # master, but not the rc=0 when it finishes. That job is left to
1400 d
.addCallback(self
._abandonOnFailure
)
1403 def doCopy(self
, res
):
1404 # now copy tree to workdir
1405 fromdir
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
1406 todir
= os
.path
.join(self
.builder
.basedir
, self
.workdir
)
1407 if runtime
.platformType
!= "posix":
1408 shutil
.copytree(fromdir
, todir
)
1409 return defer
.succeed(0)
1410 command
= ['cp', '-R', '-P', '-p', fromdir
, todir
]
1411 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
1412 sendRC
=False, timeout
=self
.timeout
)
1415 d
.addCallback(self
._abandonOnFailure
)
1418 def doPatch(self
, res
):
1419 patchlevel
, diff
= self
.patch
1420 command
= [getCommand("patch"), '-p%d' % patchlevel
]
1421 dir = os
.path
.join(self
.builder
.basedir
, self
.workdir
)
1422 # mark the directory so we don't try to update it later
1423 open(os
.path
.join(dir, ".buildbot-patched"), "w").write("patched\n")
1424 # now apply the patch
1425 c
= ShellCommand(self
.builder
, command
, dir,
1426 sendRC
=False, timeout
=self
.timeout
,
1430 d
.addCallback(self
._abandonOnFailure
)
1434 class CVS(SourceBase
):
1435 """CVS-specific VC operation. In addition to the arguments handled by
1436 SourceBase, this command reads the following keys:
1438 ['cvsroot'] (required): the CVSROOT repository string
1439 ['cvsmodule'] (required): the module to be retrieved
1440 ['branch']: a '-r' tag or branch name to use for the checkout/update
1441 ['login']: a string for use as a password to 'cvs login'
1442 ['global_options']: a list of strings to use before the CVS verb
1445 header
= "cvs operation"
1447 def setup(self
, args
):
1448 SourceBase
.setup(self
, args
)
1449 self
.vcexe
= getCommand("cvs")
1450 self
.cvsroot
= args
['cvsroot']
1451 self
.cvsmodule
= args
['cvsmodule']
1452 self
.global_options
= args
.get('global_options', [])
1453 self
.branch
= args
.get('branch')
1454 self
.login
= args
.get('login')
1455 self
.sourcedata
= "%s\n%s\n%s\n" % (self
.cvsroot
, self
.cvsmodule
,
1458 def sourcedirIsUpdateable(self
):
1459 if os
.path
.exists(os
.path
.join(self
.builder
.basedir
,
1460 self
.srcdir
, ".buildbot-patched")):
1462 return os
.path
.isdir(os
.path
.join(self
.builder
.basedir
,
1463 self
.srcdir
, "CVS"))
1466 if self
.login
is not None:
1467 # need to do a 'cvs login' command first
1468 d
= self
.builder
.basedir
1469 command
= ([self
.vcexe
, '-d', self
.cvsroot
] + self
.global_options
1471 c
= ShellCommand(self
.builder
, command
, d
,
1472 sendRC
=False, timeout
=self
.timeout
,
1473 initialStdin
=self
.login
+"\n")
1476 d
.addCallback(self
._abandonOnFailure
)
1477 d
.addCallback(self
._didLogin
)
1480 return self
._didLogin
(None)
1482 def _didLogin(self
, res
):
1483 # now we really start
1484 return SourceBase
.start(self
)
1486 def doVCUpdate(self
):
1487 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
1488 command
= [self
.vcexe
, '-z3'] + self
.global_options
+ ['update', '-dP']
1490 command
+= ['-r', self
.branch
]
1492 command
+= ['-D', self
.revision
]
1493 c
= ShellCommand(self
.builder
, command
, d
,
1494 sendRC
=False, timeout
=self
.timeout
)
1499 d
= self
.builder
.basedir
1500 if self
.mode
== "export":
1504 command
= ([self
.vcexe
, '-d', self
.cvsroot
, '-z3'] +
1505 self
.global_options
+
1506 [verb
, '-d', self
.srcdir
])
1508 command
+= ['-r', self
.branch
]
1510 command
+= ['-D', self
.revision
]
1511 command
+= [self
.cvsmodule
]
1512 c
= ShellCommand(self
.builder
, command
, d
,
1513 sendRC
=False, timeout
=self
.timeout
)
1517 def parseGotRevision(self
):
1518 # CVS does not have any kind of revision stamp to speak of. We return
1519 # the current timestamp as a best-effort guess, but this depends upon
1520 # the local system having a clock that is
1521 # reasonably-well-synchronized with the repository.
1522 return time
.strftime("%Y-%m-%d %H:%M:%S +0000", time
.gmtime())
1524 registerSlaveCommand("cvs", CVS
, command_version
)
1526 class SVN(SourceBase
):
1527 """Subversion-specific VC operation. In addition to the arguments
1528 handled by SourceBase, this command reads the following keys:
1530 ['svnurl'] (required): the SVN repository string
1533 header
= "svn operation"
1535 def setup(self
, args
):
1536 SourceBase
.setup(self
, args
)
1537 self
.vcexe
= getCommand("svn")
1538 self
.svnurl
= args
['svnurl']
1539 self
.sourcedata
= "%s\n" % self
.svnurl
1541 def sourcedirIsUpdateable(self
):
1542 if os
.path
.exists(os
.path
.join(self
.builder
.basedir
,
1543 self
.srcdir
, ".buildbot-patched")):
1545 return os
.path
.isdir(os
.path
.join(self
.builder
.basedir
,
1546 self
.srcdir
, ".svn"))
1548 def doVCUpdate(self
):
1549 revision
= self
.args
['revision'] or 'HEAD'
1550 # update: possible for mode in ('copy', 'update')
1551 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
1552 command
= [self
.vcexe
, 'update', '--revision', str(revision
),
1553 '--non-interactive', '--no-auth-cache']
1554 c
= ShellCommand(self
.builder
, command
, d
,
1555 sendRC
=False, timeout
=self
.timeout
,
1561 revision
= self
.args
['revision'] or 'HEAD'
1562 d
= self
.builder
.basedir
1563 if self
.mode
== "export":
1564 command
= [self
.vcexe
, 'export', '--revision', str(revision
),
1565 '--non-interactive', '--no-auth-cache',
1566 self
.svnurl
, self
.srcdir
]
1568 # mode=='clobber', or copy/update on a broken workspace
1569 command
= [self
.vcexe
, 'checkout', '--revision', str(revision
),
1570 '--non-interactive', '--no-auth-cache',
1571 self
.svnurl
, self
.srcdir
]
1572 c
= ShellCommand(self
.builder
, command
, d
,
1573 sendRC
=False, timeout
=self
.timeout
,
1578 def getSvnVersionCommand(self
):
1580 Get the (shell) command used to determine SVN revision number
1583 return: list of strings, passable as the command argument to ShellCommand
1585 # svn checkout operations finish with 'Checked out revision 16657.'
1586 # svn update operations finish the line 'At revision 16654.'
1587 # But we don't use those. Instead, run 'svnversion'.
1588 svnversion_command
= getCommand("svnversion")
1589 # older versions of 'svnversion' (1.1.4) require the WC_PATH
1590 # argument, newer ones (1.3.1) do not.
1591 return [svnversion_command
, "."]
1593 def parseGotRevision(self
):
1594 c
= ShellCommand(self
.builder
,
1595 self
.getSvnVersionCommand(),
1596 os
.path
.join(self
.builder
.basedir
, self
.srcdir
),
1598 sendStdout
=False, sendStderr
=False, sendRC
=False,
1603 r_raw
= c
.stdout
.strip()
1604 # Extract revision from the version "number" string
1605 r
= r_raw
.rstrip('MS')
1606 r
= r
.split(':')[-1]
1609 got_version
= int(r
)
1611 msg
=("SVN.parseGotRevision unable to parse output "
1612 "of svnversion: '%s'" % r_raw
)
1614 self
.sendStatus({'header': msg
+ "\n"})
1616 d
.addCallback(_parse
)
1620 registerSlaveCommand("svn", SVN
, command_version
)
1622 class Darcs(SourceBase
):
1623 """Darcs-specific VC operation. In addition to the arguments
1624 handled by SourceBase, this command reads the following keys:
1626 ['repourl'] (required): the Darcs repository string
1629 header
= "darcs operation"
1631 def setup(self
, args
):
1632 SourceBase
.setup(self
, args
)
1633 self
.vcexe
= getCommand("darcs")
1634 self
.repourl
= args
['repourl']
1635 self
.sourcedata
= "%s\n" % self
.repourl
1636 self
.revision
= self
.args
.get('revision')
1638 def sourcedirIsUpdateable(self
):
1639 if os
.path
.exists(os
.path
.join(self
.builder
.basedir
,
1640 self
.srcdir
, ".buildbot-patched")):
1643 # checking out a specific revision requires a full 'darcs get'
1645 return os
.path
.isdir(os
.path
.join(self
.builder
.basedir
,
1646 self
.srcdir
, "_darcs"))
1648 def doVCUpdate(self
):
1649 assert not self
.revision
1650 # update: possible for mode in ('copy', 'update')
1651 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
1652 command
= [self
.vcexe
, 'pull', '--all', '--verbose']
1653 c
= ShellCommand(self
.builder
, command
, d
,
1654 sendRC
=False, timeout
=self
.timeout
)
1659 # checkout or export
1660 d
= self
.builder
.basedir
1661 command
= [self
.vcexe
, 'get', '--verbose', '--partial',
1662 '--repo-name', self
.srcdir
]
1664 # write the context to a file
1665 n
= os
.path
.join(self
.builder
.basedir
, ".darcs-context")
1667 f
.write(self
.revision
)
1669 # tell Darcs to use that context
1670 command
.append('--context')
1672 command
.append(self
.repourl
)
1674 c
= ShellCommand(self
.builder
, command
, d
,
1675 sendRC
=False, timeout
=self
.timeout
)
1679 d
.addCallback(self
.removeContextFile
, n
)
1682 def removeContextFile(self
, res
, n
):
1686 def parseGotRevision(self
):
1687 # we use 'darcs context' to find out what we wound up with
1688 command
= [self
.vcexe
, "changes", "--context"]
1689 c
= ShellCommand(self
.builder
, command
,
1690 os
.path
.join(self
.builder
.basedir
, self
.srcdir
),
1692 sendStdout
=False, sendStderr
=False, sendRC
=False,
1696 d
.addCallback(lambda res
: c
.stdout
)
1699 registerSlaveCommand("darcs", Darcs
, command_version
)
1701 class Monotone(SourceBase
):
1702 """Monotone-specific VC operation. In addition to the arguments handled
1703 by SourceBase, this command reads the following keys:
1705 ['server_addr'] (required): the address of the server to pull from
1706 ['branch'] (required): the branch the revision is on
1707 ['db_path'] (required): the local database path to use
1708 ['revision'] (required): the revision to check out
1709 ['monotone']: (required): path to monotone executable
1712 header
= "monotone operation"
1714 def setup(self
, args
):
1715 SourceBase
.setup(self
, args
)
1716 self
.server_addr
= args
["server_addr"]
1717 self
.branch
= args
["branch"]
1718 self
.db_path
= args
["db_path"]
1719 self
.revision
= args
["revision"]
1720 self
.monotone
= args
["monotone"]
1721 self
._made
_fulls
= False
1722 self
._pull
_timeout
= args
["timeout"]
1724 def _makefulls(self
):
1725 if not self
._made
_fulls
:
1726 basedir
= self
.builder
.basedir
1727 self
.full_db_path
= os
.path
.join(basedir
, self
.db_path
)
1728 self
.full_srcdir
= os
.path
.join(basedir
, self
.srcdir
)
1729 self
._made
_fulls
= True
1731 def sourcedirIsUpdateable(self
):
1733 if os
.path
.exists(os
.path
.join(self
.full_srcdir
,
1734 ".buildbot_patched")):
1736 return (os
.path
.isfile(self
.full_db_path
)
1737 and os
.path
.isdir(os
.path
.join(self
.full_srcdir
, "MT")))
1739 def doVCUpdate(self
):
1740 return self
._withFreshDb
(self
._doUpdate
)
1742 def _doUpdate(self
):
1743 # update: possible for mode in ('copy', 'update')
1744 command
= [self
.monotone
, "update",
1745 "-r", self
.revision
,
1747 c
= ShellCommand(self
.builder
, command
, self
.full_srcdir
,
1748 sendRC
=False, timeout
=self
.timeout
)
1753 return self
._withFreshDb
(self
._doFull
)
1756 command
= [self
.monotone
, "--db=" + self
.full_db_path
,
1758 "-r", self
.revision
,
1761 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
1762 sendRC
=False, timeout
=self
.timeout
)
1766 def _withFreshDb(self
, callback
):
1768 # first ensure the db exists and is usable
1769 if os
.path
.isfile(self
.full_db_path
):
1770 # already exists, so run 'db migrate' in case monotone has been
1772 command
= [self
.monotone
, "db", "migrate",
1773 "--db=" + self
.full_db_path
]
1775 # We'll be doing an initial pull, so up the timeout to 3 hours to
1776 # make sure it will have time to complete.
1777 self
._pull
_timeout
= max(self
._pull
_timeout
, 3 * 60 * 60)
1778 self
.sendStatus({"header": "creating database %s\n"
1779 % (self
.full_db_path
,)})
1780 command
= [self
.monotone
, "db", "init",
1781 "--db=" + self
.full_db_path
]
1782 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
1783 sendRC
=False, timeout
=self
.timeout
)
1786 d
.addCallback(self
._abandonOnFailure
)
1787 d
.addCallback(self
._didDbInit
)
1788 d
.addCallback(self
._didPull
, callback
)
1791 def _didDbInit(self
, res
):
1792 command
= [self
.monotone
, "--db=" + self
.full_db_path
,
1793 "pull", "--ticker=dot", self
.server_addr
, self
.branch
]
1794 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
1795 sendRC
=False, timeout
=self
._pull
_timeout
)
1796 self
.sendStatus({"header": "pulling %s from %s\n"
1797 % (self
.branch
, self
.server_addr
)})
1801 def _didPull(self
, res
, callback
):
1804 registerSlaveCommand("monotone", Monotone
, command_version
)
1807 class Git(SourceBase
):
1808 """Git specific VC operation. In addition to the arguments
1809 handled by SourceBase, this command reads the following keys:
1811 ['repourl'] (required): the upstream GIT repository string
1812 ['branch'] (optional): which version (i.e. branch or tag) to
1813 retrieve. Default: "master".
1816 header
= "git operation"
1818 def setup(self
, args
):
1819 SourceBase
.setup(self
, args
)
1820 self
.repourl
= args
['repourl']
1821 self
.branch
= args
.get('branch')
1823 self
.branch
= "master"
1824 self
.sourcedata
= "%s %s\n" % (self
.repourl
, self
.branch
)
1826 def _fullSrcdir(self
):
1827 return os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
1829 def _commitSpec(self
):
1831 return self
.revision
1834 def sourcedirIsUpdateable(self
):
1835 if os
.path
.exists(os
.path
.join(self
._fullSrcdir
(),
1836 ".buildbot-patched")):
1838 return os
.path
.isdir(os
.path
.join(self
._fullSrcdir
(), ".git"))
1840 def _didFetch(self
, res
):
1842 head
= self
.revision
1846 command
= ['git', 'reset', '--hard', head
]
1847 c
= ShellCommand(self
.builder
, command
, self
._fullSrcdir
(),
1848 sendRC
=False, timeout
=self
.timeout
)
1852 def doVCUpdate(self
):
1853 command
= ['git', 'fetch', self
.repourl
, self
.branch
]
1854 self
.sendStatus({"header": "fetching branch %s from %s\n"
1855 % (self
.branch
, self
.repourl
)})
1856 c
= ShellCommand(self
.builder
, command
, self
._fullSrcdir
(),
1857 sendRC
=False, timeout
=self
.timeout
)
1860 d
.addCallback(self
._abandonOnFailure
)
1861 d
.addCallback(self
._didFetch
)
1864 def _didInit(self
, res
):
1865 return self
.doVCUpdate()
1868 os
.mkdir(self
._fullSrcdir
())
1869 c
= ShellCommand(self
.builder
, ['git', 'init'], self
._fullSrcdir
(),
1870 sendRC
=False, timeout
=self
.timeout
)
1873 d
.addCallback(self
._abandonOnFailure
)
1874 d
.addCallback(self
._didInit
)
1877 def parseGotRevision(self
):
1878 command
= ['git', 'rev-parse', 'HEAD']
1879 c
= ShellCommand(self
.builder
, command
, self
._fullSrcdir
(),
1880 sendRC
=False, keepStdout
=True)
1884 hash = c
.stdout
.strip()
1888 d
.addCallback(_parse
)
1891 registerSlaveCommand("git", Git
, command_version
)
1893 class Arch(SourceBase
):
1894 """Arch-specific (tla-specific) VC operation. In addition to the
1895 arguments handled by SourceBase, this command reads the following keys:
1897 ['url'] (required): the repository string
1898 ['version'] (required): which version (i.e. branch) to retrieve
1899 ['revision'] (optional): the 'patch-NN' argument to check out
1900 ['archive']: the archive name to use. If None, use the archive's default
1901 ['build-config']: if present, give to 'tla build-config' after checkout
1904 header
= "arch operation"
1907 def setup(self
, args
):
1908 SourceBase
.setup(self
, args
)
1909 self
.vcexe
= getCommand("tla")
1910 self
.archive
= args
.get('archive')
1911 self
.url
= args
['url']
1912 self
.version
= args
['version']
1913 self
.revision
= args
.get('revision')
1914 self
.buildconfig
= args
.get('build-config')
1915 self
.sourcedata
= "%s\n%s\n%s\n" % (self
.url
, self
.version
,
1918 def sourcedirIsUpdateable(self
):
1920 # Arch cannot roll a directory backwards, so if they ask for a
1921 # specific revision, clobber the directory. Technically this
1922 # could be limited to the cases where the requested revision is
1923 # later than our current one, but it's too hard to extract the
1924 # current revision from the tree.
1926 if os
.path
.exists(os
.path
.join(self
.builder
.basedir
,
1927 self
.srcdir
, ".buildbot-patched")):
1929 return os
.path
.isdir(os
.path
.join(self
.builder
.basedir
,
1930 self
.srcdir
, "{arch}"))
1932 def doVCUpdate(self
):
1933 # update: possible for mode in ('copy', 'update')
1934 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
1935 command
= [self
.vcexe
, 'replay']
1937 command
.append(self
.revision
)
1938 c
= ShellCommand(self
.builder
, command
, d
,
1939 sendRC
=False, timeout
=self
.timeout
)
1944 # to do a checkout, we must first "register" the archive by giving
1945 # the URL to tla, which will go to the repository at that URL and
1946 # figure out the archive name. tla will tell you the archive name
1947 # when it is done, and all further actions must refer to this name.
1949 command
= [self
.vcexe
, 'register-archive', '--force', self
.url
]
1950 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
1951 sendRC
=False, keepStdout
=True,
1952 timeout
=self
.timeout
)
1955 d
.addCallback(self
._abandonOnFailure
)
1956 d
.addCallback(self
._didRegister
, c
)
1959 def _didRegister(self
, res
, c
):
1960 # find out what tla thinks the archive name is. If the user told us
1961 # to use something specific, make sure it matches.
1962 r
= re
.search(r
'Registering archive: (\S+)\s*$', c
.stdout
)
1964 msg
= "tla reports archive name is '%s'" % r
.group(1)
1966 self
.builder
.sendUpdate({'header': msg
+"\n"})
1967 if self
.archive
and r
.group(1) != self
.archive
:
1968 msg
= (" mismatch, we wanted an archive named '%s'"
1971 self
.builder
.sendUpdate({'header': msg
+"\n"})
1972 raise AbandonChain(-1)
1973 self
.archive
= r
.group(1)
1974 assert self
.archive
, "need archive name to continue"
1975 return self
._doGet
()
1980 ver
+= "--%s" % self
.revision
1981 command
= [self
.vcexe
, 'get', '--archive', self
.archive
,
1984 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
1985 sendRC
=False, timeout
=self
.timeout
)
1988 d
.addCallback(self
._abandonOnFailure
)
1989 if self
.buildconfig
:
1990 d
.addCallback(self
._didGet
)
1993 def _didGet(self
, res
):
1994 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
1995 command
= [self
.vcexe
, 'build-config', self
.buildconfig
]
1996 c
= ShellCommand(self
.builder
, command
, d
,
1997 sendRC
=False, timeout
=self
.timeout
)
2000 d
.addCallback(self
._abandonOnFailure
)
2003 def parseGotRevision(self
):
2004 # using code from tryclient.TlaExtractor
2005 # 'tla logs --full' gives us ARCHIVE/BRANCH--REVISION
2006 # 'tla logs' gives us REVISION
2007 command
= [self
.vcexe
, "logs", "--full", "--reverse"]
2008 c
= ShellCommand(self
.builder
, command
,
2009 os
.path
.join(self
.builder
.basedir
, self
.srcdir
),
2011 sendStdout
=False, sendStderr
=False, sendRC
=False,
2016 tid
= c
.stdout
.split("\n")[0].strip()
2017 slash
= tid
.index("/")
2018 dd
= tid
.rindex("--")
2019 #branch = tid[slash+1:dd]
2020 baserev
= tid
[dd
+2:]
2022 d
.addCallback(_parse
)
2025 registerSlaveCommand("arch", Arch
, command_version
)
2028 """Bazaar (/usr/bin/baz) is an alternative client for Arch repositories.
2029 It is mostly option-compatible, but archive registration is different
2030 enough to warrant a separate Command.
2032 ['archive'] (required): the name of the archive being used
2035 def setup(self
, args
):
2036 Arch
.setup(self
, args
)
2037 self
.vcexe
= getCommand("baz")
2038 # baz doesn't emit the repository name after registration (and
2039 # grepping through the output of 'baz archives' is too hard), so we
2040 # require that the buildmaster configuration to provide both the
2041 # archive name and the URL.
2042 self
.archive
= args
['archive'] # required for Baz
2043 self
.sourcedata
= "%s\n%s\n%s\n" % (self
.url
, self
.version
,
2046 # in _didRegister, the regexp won't match, so we'll stick with the name
2050 # baz prefers ARCHIVE/VERSION. This will work even if
2051 # my-default-archive is not set.
2052 ver
= self
.archive
+ "/" + self
.version
2054 ver
+= "--%s" % self
.revision
2055 command
= [self
.vcexe
, 'get', '--no-pristine',
2057 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
2058 sendRC
=False, timeout
=self
.timeout
)
2061 d
.addCallback(self
._abandonOnFailure
)
2062 if self
.buildconfig
:
2063 d
.addCallback(self
._didGet
)
2066 def parseGotRevision(self
):
2067 # using code from tryclient.BazExtractor
2068 command
= [self
.vcexe
, "tree-id"]
2069 c
= ShellCommand(self
.builder
, command
,
2070 os
.path
.join(self
.builder
.basedir
, self
.srcdir
),
2072 sendStdout
=False, sendStderr
=False, sendRC
=False,
2077 tid
= c
.stdout
.strip()
2078 slash
= tid
.index("/")
2079 dd
= tid
.rindex("--")
2080 #branch = tid[slash+1:dd]
2081 baserev
= tid
[dd
+2:]
2083 d
.addCallback(_parse
)
2086 registerSlaveCommand("bazaar", Bazaar
, command_version
)
2089 class Bzr(SourceBase
):
2090 """bzr-specific VC operation. In addition to the arguments
2091 handled by SourceBase, this command reads the following keys:
2093 ['repourl'] (required): the Bzr repository string
2096 header
= "bzr operation"
2098 def setup(self
, args
):
2099 SourceBase
.setup(self
, args
)
2100 self
.vcexe
= getCommand("bzr")
2101 self
.repourl
= args
['repourl']
2102 self
.sourcedata
= "%s\n" % self
.repourl
2103 self
.revision
= self
.args
.get('revision')
2105 def sourcedirIsUpdateable(self
):
2106 if os
.path
.exists(os
.path
.join(self
.builder
.basedir
,
2107 self
.srcdir
, ".buildbot-patched")):
2110 # checking out a specific revision requires a full 'bzr checkout'
2112 return os
.path
.isdir(os
.path
.join(self
.builder
.basedir
,
2113 self
.srcdir
, ".bzr"))
2115 def doVCUpdate(self
):
2116 assert not self
.revision
2117 # update: possible for mode in ('copy', 'update')
2118 srcdir
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2119 command
= [self
.vcexe
, 'update']
2120 c
= ShellCommand(self
.builder
, command
, srcdir
,
2121 sendRC
=False, timeout
=self
.timeout
)
2126 # checkout or export
2127 d
= self
.builder
.basedir
2128 if self
.mode
== "export":
2129 # exporting in bzr requires a separate directory
2130 return self
.doVCExport()
2131 # originally I added --lightweight here, but then 'bzr revno' is
2132 # wrong. The revno reported in 'bzr version-info' is correct,
2133 # however. Maybe this is a bzr bug?
2135 # In addition, you cannot perform a 'bzr update' on a repo pulled
2136 # from an HTTP repository that used 'bzr checkout --lightweight'. You
2137 # get a "ERROR: Cannot lock: transport is read only" when you try.
2139 # So I won't bother using --lightweight for now.
2141 command
= [self
.vcexe
, 'checkout']
2143 command
.append('--revision')
2144 command
.append(str(self
.revision
))
2145 command
.append(self
.repourl
)
2146 command
.append(self
.srcdir
)
2148 c
= ShellCommand(self
.builder
, command
, d
,
2149 sendRC
=False, timeout
=self
.timeout
)
2154 def doVCExport(self
):
2155 tmpdir
= os
.path
.join(self
.builder
.basedir
, "export-temp")
2156 srcdir
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2157 command
= [self
.vcexe
, 'checkout', '--lightweight']
2159 command
.append('--revision')
2160 command
.append(str(self
.revision
))
2161 command
.append(self
.repourl
)
2162 command
.append(tmpdir
)
2163 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
2164 sendRC
=False, timeout
=self
.timeout
)
2168 command
= [self
.vcexe
, 'export', srcdir
]
2169 c
= ShellCommand(self
.builder
, command
, tmpdir
,
2170 sendRC
=False, timeout
=self
.timeout
)
2173 d
.addCallback(_export
)
2176 def get_revision_number(self
, out
):
2177 # it feels like 'bzr revno' sometimes gives different results than
2178 # the 'revno:' line from 'bzr version-info', and the one from
2179 # version-info is more likely to be correct.
2180 for line
in out
.split("\n"):
2181 colon
= line
.find(":")
2183 key
, value
= line
[:colon
], line
[colon
+2:]
2186 raise ValueError("unable to find revno: in bzr output: '%s'" % out
)
2188 def parseGotRevision(self
):
2189 command
= [self
.vcexe
, "version-info"]
2190 c
= ShellCommand(self
.builder
, command
,
2191 os
.path
.join(self
.builder
.basedir
, self
.srcdir
),
2193 sendStdout
=False, sendStderr
=False, sendRC
=False,
2199 return self
.get_revision_number(c
.stdout
)
2201 msg
=("Bzr.parseGotRevision unable to parse output "
2202 "of bzr version-info: '%s'" % c
.stdout
.strip())
2204 self
.sendStatus({'header': msg
+ "\n"})
2206 d
.addCallback(_parse
)
2209 registerSlaveCommand("bzr", Bzr
, command_version
)
2211 class Mercurial(SourceBase
):
2212 """Mercurial specific VC operation. In addition to the arguments
2213 handled by SourceBase, this command reads the following keys:
2215 ['repourl'] (required): the Cogito repository string
2218 header
= "mercurial operation"
2220 def setup(self
, args
):
2221 SourceBase
.setup(self
, args
)
2222 self
.vcexe
= getCommand("hg")
2223 self
.repourl
= args
['repourl']
2224 self
.sourcedata
= "%s\n" % self
.repourl
2228 def sourcedirIsUpdateable(self
):
2229 if os
.path
.exists(os
.path
.join(self
.builder
.basedir
,
2230 self
.srcdir
, ".buildbot-patched")):
2232 # like Darcs, to check out a specific (old) revision, we have to do a
2233 # full checkout. TODO: I think 'hg pull' plus 'hg update' might work
2236 return os
.path
.isdir(os
.path
.join(self
.builder
.basedir
,
2237 self
.srcdir
, ".hg"))
2239 def doVCUpdate(self
):
2240 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2241 command
= [self
.vcexe
, 'pull', '--update', '--verbose']
2242 if self
.args
['revision']:
2243 command
.extend(['--rev', self
.args
['revision']])
2244 c
= ShellCommand(self
.builder
, command
, d
,
2245 sendRC
=False, timeout
=self
.timeout
,
2249 d
.addCallback(self
._handleEmptyUpdate
)
2252 def _handleEmptyUpdate(self
, res
):
2253 if type(res
) is int and res
== 1:
2254 if self
.command
.stdout
.find("no changes found") != -1:
2255 # 'hg pull', when it doesn't have anything to do, exits with
2256 # rc=1, and there appears to be no way to shut this off. It
2257 # emits a distinctive message to stdout, though. So catch
2258 # this and pretend that it completed successfully.
2263 newdir
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2264 command
= [self
.vcexe
, 'clone']
2265 if self
.args
['revision']:
2266 command
.extend(['--rev', self
.args
['revision']])
2267 command
.extend([self
.repourl
, newdir
])
2268 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
2269 sendRC
=False, keepStdout
=True, keepStderr
=True,
2270 timeout
=self
.timeout
)
2273 d
.addCallback(self
._maybeFallback
, c
)
2276 def _maybeFallback(self
, res
, c
):
2277 # to do 'hg clone -r REV' (i.e. to check out a specific revision)
2278 # from a remote (HTTP) repository, both the client and the server
2279 # need to be hg-0.9.2 or newer. If this caused a checkout failure, we
2280 # fall back to doing a checkout of HEAD (spelled 'tip' in hg
2281 # parlance) and then 'hg update' *backwards* to the desired revision.
2286 # hg-0.6 didn't even have the 'clone' command
2288 "hg clone: option --rev not recognized",
2289 # hg-0.8, 0.8.1, 0.9
2290 "abort: clone -r not supported yet for remote repositories.",
2292 ("abort: clone by revision not supported yet for "
2293 "remote repositories"),
2294 # hg-0.9.2 and later say this when the other end is too old
2295 ("abort: src repository does not support revision lookup "
2296 "and so doesn't support clone by revision"),
2299 fallback_is_useful
= False
2300 for errmsg
in errmsgs
:
2301 # the error message might be in stdout if we're using PTYs, which
2302 # merge stdout and stderr.
2303 if errmsg
in c
.stdout
or errmsg
in c
.stderr
:
2304 fallback_is_useful
= True
2306 if not fallback_is_useful
:
2307 return res
# must be some other error
2309 # ok, do the fallback
2310 newdir
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2311 command
= [self
.vcexe
, 'clone']
2312 command
.extend([self
.repourl
, newdir
])
2313 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
2314 sendRC
=False, timeout
=self
.timeout
)
2317 d
.addCallback(self
._abandonOnFailure
)
2318 d
.addCallback(self
._updateToDesiredRevision
)
2321 def _updateToDesiredRevision(self
, res
):
2322 assert self
.args
['revision']
2323 newdir
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2324 # hg-0.9.1 and earlier (which need this fallback) also want to see
2325 # 'hg update REV' instead of 'hg update --rev REV'. Note that this is
2326 # the only place we use 'hg update', since what most VC tools mean
2327 # by, say, 'cvs update' is expressed as 'hg pull --update' instead.
2328 command
= [self
.vcexe
, 'update', self
.args
['revision']]
2329 c
= ShellCommand(self
.builder
, command
, newdir
,
2330 sendRC
=False, timeout
=self
.timeout
)
2333 def parseGotRevision(self
):
2334 # we use 'hg identify' to find out what we wound up with
2335 command
= [self
.vcexe
, "identify"]
2336 c
= ShellCommand(self
.builder
, command
,
2337 os
.path
.join(self
.builder
.basedir
, self
.srcdir
),
2339 sendStdout
=False, sendStderr
=False, sendRC
=False,
2343 m
= re
.search(r
'^(\w+)', c
.stdout
)
2345 d
.addCallback(_parse
)
2348 registerSlaveCommand("hg", Mercurial
, command_version
)
2351 class P4(SourceBase
):
2352 """A P4 source-updater.
2354 ['p4port'] (required): host:port for server to access
2355 ['p4user'] (optional): user to use for access
2356 ['p4passwd'] (optional): passwd to try for the user
2357 ['p4client'] (optional): client spec to use
2358 ['p4extra_views'] (optional): additional client views to use
2363 def setup(self
, args
):
2364 SourceBase
.setup(self
, args
)
2365 self
.p4port
= args
['p4port']
2366 self
.p4client
= args
['p4client']
2367 self
.p4user
= args
['p4user']
2368 self
.p4passwd
= args
['p4passwd']
2369 self
.p4base
= args
['p4base']
2370 self
.p4extra_views
= args
['p4extra_views']
2371 self
.p4mode
= args
['mode']
2372 self
.p4branch
= args
['branch']
2374 self
.sourcedata
= str([
2381 # Depot side of view spec.
2386 # Local side of view spec (srcdir is made from these).
2387 self
.builder
.basedir
,
2393 def sourcedirIsUpdateable(self
):
2394 if os
.path
.exists(os
.path
.join(self
.builder
.basedir
,
2395 self
.srcdir
, ".buildbot-patched")):
2397 # We assume our client spec is still around.
2398 # We just say we aren't updateable if the dir doesn't exist so we
2399 # don't get ENOENT checking the sourcedata.
2400 return os
.path
.isdir(os
.path
.join(self
.builder
.basedir
,
2403 def doVCUpdate(self
):
2404 return self
._doP
4Sync
(force
=False)
2406 def _doP4Sync(self
, force
):
2410 command
.extend(['-p', self
.p4port
])
2412 command
.extend(['-u', self
.p4user
])
2414 command
.extend(['-P', self
.p4passwd
])
2416 command
.extend(['-c', self
.p4client
])
2417 command
.extend(['sync'])
2419 command
.extend(['-f'])
2421 command
.extend(['@' + str(self
.revision
)])
2423 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
2424 environ
=env
, sendRC
=False, timeout
=self
.timeout
,
2428 d
.addCallback(self
._abandonOnFailure
)
2436 client_spec
+= "Client: %s\n\n" % self
.p4client
2437 client_spec
+= "Owner: %s\n\n" % self
.p4user
2438 client_spec
+= "Description:\n\tCreated by %s\n\n" % self
.p4user
2439 client_spec
+= "Root:\t%s\n\n" % self
.builder
.basedir
2440 client_spec
+= "Options:\tallwrite rmdir\n\n"
2441 client_spec
+= "LineEnd:\tlocal\n\n"
2444 client_spec
+= "View:\n\t%s" % (self
.p4base
)
2446 client_spec
+= "%s/" % (self
.p4branch
)
2447 client_spec
+= "... //%s/%s/...\n" % (self
.p4client
, self
.srcdir
)
2448 if self
.p4extra_views
:
2449 for k
, v
in self
.p4extra_views
:
2450 client_spec
+= "\t%s/... //%s/%s%s/...\n" % (k
, self
.p4client
,
2453 command
.extend(['-p', self
.p4port
])
2455 command
.extend(['-u', self
.p4user
])
2457 command
.extend(['-P', self
.p4passwd
])
2458 command
.extend(['client', '-i'])
2459 log
.msg(client_spec
)
2460 c
= ShellCommand(self
.builder
, command
, self
.builder
.basedir
,
2461 environ
=env
, sendRC
=False, timeout
=self
.timeout
,
2462 initialStdin
=client_spec
)
2465 d
.addCallback(self
._abandonOnFailure
)
2466 d
.addCallback(lambda _
: self
._doP
4Sync
(force
=True))
2469 registerSlaveCommand("p4", P4
, command_version
)
2472 class P4Sync(SourceBase
):
2473 """A partial P4 source-updater. Requires manual setup of a per-slave P4
2474 environment. The only thing which comes from the master is P4PORT.
2475 'mode' is required to be 'copy'.
2477 ['p4port'] (required): host:port for server to access
2478 ['p4user'] (optional): user to use for access
2479 ['p4passwd'] (optional): passwd to try for the user
2480 ['p4client'] (optional): client spec to use
2485 def setup(self
, args
):
2486 SourceBase
.setup(self
, args
)
2487 self
.vcexe
= getCommand("p4")
2488 self
.p4port
= args
['p4port']
2489 self
.p4user
= args
['p4user']
2490 self
.p4passwd
= args
['p4passwd']
2491 self
.p4client
= args
['p4client']
2493 def sourcedirIsUpdateable(self
):
2496 def _doVC(self
, force
):
2497 d
= os
.path
.join(self
.builder
.basedir
, self
.srcdir
)
2498 command
= [self
.vcexe
]
2500 command
.extend(['-p', self
.p4port
])
2502 command
.extend(['-u', self
.p4user
])
2504 command
.extend(['-P', self
.p4passwd
])
2506 command
.extend(['-c', self
.p4client
])
2507 command
.extend(['sync'])
2509 command
.extend(['-f'])
2511 command
.extend(['@' + self
.revision
])
2513 c
= ShellCommand(self
.builder
, command
, d
, environ
=env
,
2514 sendRC
=False, timeout
=self
.timeout
)
2518 def doVCUpdate(self
):
2519 return self
._doVC
(force
=False)
2522 return self
._doVC
(force
=True)
2524 registerSlaveCommand("p4sync", P4Sync
, command_version
)