1 # -*- test-case-name: buildbot.test.test_steps -*-
3 from zope
.interface
import implements
4 from twisted
.internet
import reactor
, defer
, error
5 from twisted
.protocols
import basic
6 from twisted
.spread
import pb
7 from twisted
.python
import log
8 from twisted
.python
.failure
import Failure
9 from twisted
.web
.util
import formatFailure
11 from buildbot
import interfaces
, locks
12 from buildbot
.status
import progress
13 from buildbot
.status
.builder
import SUCCESS
, WARNINGS
, FAILURE
, SKIPPED
, \
17 BuildStep and RemoteCommand classes for master-side representation of the
21 class RemoteCommand(pb
.Referenceable
):
23 I represent a single command to be run on the slave. I handle the details
24 of reliably gathering status updates from the slave (acknowledging each),
25 and (eventually, in a future release) recovering from interrupted builds.
26 This is the master-side object that is known to the slave-side
27 L{buildbot.slave.bot.SlaveBuilder}, to which status updates are sent.
29 My command should be started by calling .run(), which returns a
30 Deferred that will fire when the command has finished, or will
31 errback if an exception is raised.
33 Typically __init__ or run() will set up self.remote_command to be a
34 string which corresponds to one of the SlaveCommands registered in
35 the buildslave, and self.args to a dictionary of arguments that will
36 be passed to the SlaveCommand instance.
38 start, remoteUpdate, and remoteComplete are available to be overridden
40 @type commandCounter: list of one int
41 @cvar commandCounter: provides a unique value for each
42 RemoteCommand executed across all slaves
44 @ivar active: whether the command is currently running
46 commandCounter
= [0] # we use a list as a poor man's singleton
49 def __init__(self
, remote_command
, args
):
51 @type remote_command: string
52 @param remote_command: remote command to start. This will be
54 L{buildbot.slave.bot.SlaveBuilder.remote_startCommand}
55 and needs to have been registered
57 L{buildbot.slave.registry.registerSlaveCommand}
59 @param args: arguments to send to the remote command
62 self
.remote_command
= remote_command
65 def __getstate__(self
):
66 dict = self
.__dict
__.copy()
67 # Remove the remote ref: if necessary (only for resumed builds), it
68 # will be reattached at resume time
69 if dict.has_key("remote"):
73 def run(self
, step
, remote
):
77 c
= self
.commandCounter
[0]
78 self
.commandCounter
[0] += 1
79 #self.commandID = "%d %d" % (c, random.randint(0, 1000000))
80 self
.commandID
= "%d" % c
81 log
.msg("%s: RemoteCommand.run [%s]" % (self
, self
.commandID
))
82 self
.deferred
= defer
.Deferred()
84 d
= defer
.maybeDeferred(self
.start
)
86 # _finished is called with an error for unknown commands, errors
87 # that occur while the command is starting (including OSErrors in
88 # exec()), StaleBroker (when the connection was lost before we
89 # started), and pb.PBConnectionLost (when the slave isn't responding
90 # over this connection, perhaps it had a power failure, or NAT
91 # weirdness). If this happens, self.deferred is fired right away.
92 d
.addErrback(self
._finished
)
94 # Connections which are lost while the command is running are caught
95 # when our parent Step calls our .lostRemote() method.
100 Tell the slave to start executing the remote command.
102 @rtype: L{twisted.internet.defer.Deferred}
103 @returns: a deferred that will fire when the remote command is
104 done (with None as the result)
106 # This method only initiates the remote command.
107 # We will receive remote_update messages as the command runs.
108 # We will get a single remote_complete when it finishes.
109 # We should fire self.deferred when the command is done.
110 d
= self
.remote
.callRemote("startCommand", self
, self
.commandID
,
111 self
.remote_command
, self
.args
)
114 def interrupt(self
, why
):
115 # TODO: consider separating this into interrupt() and stop(), where
116 # stop() unconditionally calls _finished, but interrupt() merely
117 # asks politely for the command to stop soon.
119 log
.msg("RemoteCommand.interrupt", self
, why
)
121 log
.msg(" but this RemoteCommand is already inactive")
124 log
.msg(" but our .remote went away")
126 if isinstance(why
, Failure
) and why
.check(error
.ConnectionLost
):
127 log
.msg("RemoteCommand.disconnect: lost slave")
132 # tell the remote command to halt. Returns a Deferred that will fire
133 # when the interrupt command has been delivered.
135 d
= defer
.maybeDeferred(self
.remote
.callRemote
, "interruptCommand",
136 self
.commandID
, str(why
))
137 # the slave may not have remote_interruptCommand
138 d
.addErrback(self
._interruptFailed
)
141 def _interruptFailed(self
, why
):
142 log
.msg("RemoteCommand._interruptFailed", self
)
143 # TODO: forcibly stop the Command now, since we can't stop it
147 def remote_update(self
, updates
):
149 I am called by the slave's L{buildbot.slave.bot.SlaveBuilder} so
150 I can receive updates from the running remote command.
152 @type updates: list of [object, int]
153 @param updates: list of updates from the remote command
155 self
.buildslave
.messageReceivedFromSlave()
157 for (update
, num
) in updates
:
158 #log.msg("update[%d]:" % num)
160 if self
.active
: # ignore late updates
161 self
.remoteUpdate(update
)
163 # log failure, terminate build, let slave retire the update
164 self
._finished
(Failure())
165 # TODO: what if multiple updates arrive? should
166 # skip the rest but ack them all
167 if num
> max_updatenum
:
171 def remoteUpdate(self
, update
):
172 raise NotImplementedError("You must implement this in a subclass")
174 def remote_complete(self
, failure
=None):
176 Called by the slave's L{buildbot.slave.bot.SlaveBuilder} to
177 notify me the remote command has finished.
179 @type failure: L{twisted.python.failure.Failure} or None
183 self
.buildslave
.messageReceivedFromSlave()
184 # call the real remoteComplete a moment later, but first return an
185 # acknowledgement so the slave can retire the completion message.
187 reactor
.callLater(0, self
._finished
, failure
)
190 def _finished(self
, failure
=None):
192 # call .remoteComplete. If it raises an exception, or returns the
193 # Failure that we gave it, our self.deferred will be errbacked. If
194 # it does not (either it ate the Failure or there the step finished
195 # normally and it didn't raise a new exception), self.deferred will
197 d
= defer
.maybeDeferred(self
.remoteComplete
, failure
)
198 # arrange for the callback to get this RemoteCommand instance
199 # instead of just None
200 d
.addCallback(lambda r
: self
)
201 # this fires the original deferred we returned from .run(),
202 # with self as the result, or a failure
203 d
.addBoth(self
.deferred
.callback
)
205 def remoteComplete(self
, maybeFailure
):
206 """Subclasses can override this.
208 This is called when the RemoteCommand has finished. 'maybeFailure'
209 will be None if the command completed normally, or a Failure
210 instance in one of the following situations:
212 - the slave was lost before the command was started
213 - the slave didn't respond to the startCommand message
214 - the slave raised an exception while starting the command
215 (bad command name, bad args, OSError from missing executable)
216 - the slave raised an exception while finishing the command
217 (they send back a remote_complete message with a Failure payload)
220 - slave disconnected while the command was running
222 This method should do cleanup, like closing log files. It should
223 normally return the 'failure' argument, so that any exceptions will
224 be propagated to the Step. If it wants to consume them, return None
229 class LoggedRemoteCommand(RemoteCommand
):
232 I am a L{RemoteCommand} which gathers output from the remote command into
233 one or more local log files. My C{self.logs} dictionary contains
234 references to these L{buildbot.status.builder.LogFile} instances. Any
235 stdout/stderr/header updates from the slave will be put into
236 C{self.logs['stdio']}, if it exists. If the remote command uses other log
237 files, they will go into other entries in C{self.logs}.
239 If you want to use stdout or stderr, you should create a LogFile named
240 'stdio' and pass it to my useLog() message. Otherwise stdout/stderr will
241 be ignored, which is probably not what you want.
243 Unless you tell me otherwise, when my command completes I will close all
244 the LogFiles that I know about.
246 @ivar logs: maps logname to a LogFile instance
247 @ivar _closeWhenFinished: maps logname to a boolean. If true, this
248 LogFile will be closed when the RemoteCommand
249 finishes. LogFiles which are shared between
250 multiple RemoteCommands should use False here.
257 def __init__(self
, *args
, **kwargs
):
259 self
._closeWhenFinished
= {}
260 RemoteCommand
.__init
__(self
, *args
, **kwargs
)
263 return "<RemoteCommand '%s' at %d>" % (self
.remote_command
, id(self
))
265 def useLog(self
, loog
, closeWhenFinished
=False, logfileName
=None):
266 """Start routing messages from a remote logfile to a local LogFile
268 I take a local ILogFile instance in 'loog', and arrange to route
269 remote log messages for the logfile named 'logfileName' into it. By
270 default this logfileName comes from the ILogFile itself (using the
271 name by which the ILogFile will be displayed), but the 'logfileName'
272 argument can be used to override this. For example, if
273 logfileName='stdio', this logfile will collect text from the stdout
274 and stderr of the command.
276 @param loog: an instance which implements ILogFile
277 @param closeWhenFinished: a boolean, set to False if the logfile
278 will be shared between multiple
279 RemoteCommands. If True, the logfile will
280 be closed when this ShellCommand is done
282 @param logfileName: a string, which indicates which remote log file
283 should be routed into this ILogFile. This should
284 match one of the keys of the logfiles= argument
289 assert interfaces
.ILogFile
.providedBy(loog
)
291 logfileName
= loog
.getName()
292 assert logfileName
not in self
.logs
293 self
.logs
[logfileName
] = loog
294 self
._closeWhenFinished
[logfileName
] = closeWhenFinished
297 log
.msg("LoggedRemoteCommand.start")
298 if 'stdio' not in self
.logs
:
299 log
.msg("LoggedRemoteCommand (%s) is running a command, but "
300 "it isn't being logged to anything. This seems unusual."
303 return RemoteCommand
.start(self
)
305 def addStdout(self
, data
):
306 if 'stdio' in self
.logs
:
307 self
.logs
['stdio'].addStdout(data
)
308 def addStderr(self
, data
):
309 if 'stdio' in self
.logs
:
310 self
.logs
['stdio'].addStderr(data
)
311 def addHeader(self
, data
):
312 if 'stdio' in self
.logs
:
313 self
.logs
['stdio'].addHeader(data
)
315 def addToLog(self
, logname
, data
):
316 if logname
in self
.logs
:
317 self
.logs
[logname
].addStdout(data
)
319 log
.msg("%s.addToLog: no such log %s" % (self
, logname
))
321 def remoteUpdate(self
, update
):
323 for k
,v
in update
.items():
324 log
.msg("Update[%s]: %s" % (k
,v
))
325 if update
.has_key('stdout'):
327 self
.addStdout(update
['stdout'])
328 if update
.has_key('stderr'):
330 self
.addStderr(update
['stderr'])
331 if update
.has_key('header'):
333 self
.addHeader(update
['header'])
334 if update
.has_key('log'):
335 # 'log': (logname, data)
336 logname
, data
= update
['log']
337 self
.addToLog(logname
, data
)
338 if update
.has_key('rc'):
339 rc
= self
.rc
= update
['rc']
340 log
.msg("%s rc=%s" % (self
, rc
))
341 self
.addHeader("program finished with exit code %d\n" % rc
)
344 if k
not in ('stdout', 'stderr', 'header', 'rc'):
345 if k
not in self
.updates
:
347 self
.updates
[k
].append(update
[k
])
349 def remoteComplete(self
, maybeFailure
):
350 for name
,loog
in self
.logs
.items():
351 if self
._closeWhenFinished
[name
]:
353 loog
.addHeader("\nremoteFailed: %s" % maybeFailure
)
355 log
.msg("closing log %s" % loog
)
361 implements(interfaces
.ILogObserver
)
363 def setStep(self
, step
):
366 def setLog(self
, loog
):
367 assert interfaces
.IStatusLog
.providedBy(loog
)
368 loog
.subscribe(self
, True)
370 def logChunk(self
, build
, step
, log
, channel
, text
):
371 if channel
== interfaces
.LOG_CHANNEL_STDOUT
:
372 self
.outReceived(text
)
373 elif channel
== interfaces
.LOG_CHANNEL_STDERR
:
374 self
.errReceived(text
)
376 # TODO: add a logEnded method? er, stepFinished?
378 def outReceived(self
, data
):
379 """This will be called with chunks of stdout data. Override this in
383 def errReceived(self
, data
):
384 """This will be called with chunks of stderr data. Override this in
389 class LogLineObserver(LogObserver
):
391 self
.stdoutParser
= basic
.LineOnlyReceiver()
392 self
.stdoutParser
.delimiter
= "\n"
393 self
.stdoutParser
.lineReceived
= self
.outLineReceived
394 self
.stdoutParser
.transport
= self
# for the .disconnecting attribute
395 self
.disconnecting
= False
397 self
.stderrParser
= basic
.LineOnlyReceiver()
398 self
.stderrParser
.delimiter
= "\n"
399 self
.stderrParser
.lineReceived
= self
.errLineReceived
400 self
.stderrParser
.transport
= self
402 def setMaxLineLength(self
, max_length
):
404 Set the maximum line length: lines longer than max_length are
405 dropped. Default is 16384 bytes. Use sys.maxint for effective
408 self
.stdoutParser
.MAX_LENGTH
= max_length
409 self
.stderrParser
.MAX_LENGTH
= max_length
411 def outReceived(self
, data
):
412 self
.stdoutParser
.dataReceived(data
)
414 def errReceived(self
, data
):
415 self
.stderrParser
.dataReceived(data
)
417 def outLineReceived(self
, line
):
418 """This will be called with complete stdout lines (not including the
419 delimiter). Override this in your observer."""
422 def errLineReceived(self
, line
):
423 """This will be called with complete lines of stderr (not including
424 the delimiter). Override this in your observer."""
428 class RemoteShellCommand(LoggedRemoteCommand
):
429 """This class helps you run a shell command on the build slave. It will
430 accumulate all the command's output into a Log named 'stdio'. When the
431 command is finished, it will fire a Deferred. You can then check the
432 results of the command and parse the output however you like."""
434 def __init__(self
, workdir
, command
, env
=None,
435 want_stdout
=1, want_stderr
=1,
436 timeout
=20*60, logfiles
={}, usePTY
="slave-config"):
438 @type workdir: string
439 @param workdir: directory where the command ought to run,
440 relative to the Builder's home directory. Defaults to
441 '.': the same as the Builder's homedir. This should
442 probably be '.' for the initial 'cvs checkout'
443 command (which creates a workdir), and the Build-wide
444 workdir for all subsequent commands (including
445 compiles and 'cvs update').
447 @type command: list of strings (or string)
448 @param command: the shell command to run, like 'make all' or
449 'cvs update'. This should be a list or tuple
450 which can be used directly as the argv array.
451 For backwards compatibility, if this is a
452 string, the text will be given to '/bin/sh -c
455 @type env: dict of string->string
456 @param env: environment variables to add or change for the
457 slave. Each command gets a separate
458 environment; all inherit the slave's initial
459 one. TODO: make it possible to delete some or
460 all of the slave's environment.
462 @type want_stdout: bool
463 @param want_stdout: defaults to True. Set to False if stdout should
464 be thrown away. Do this to avoid storing or
465 sending large amounts of useless data.
467 @type want_stderr: bool
468 @param want_stderr: False if stderr should be thrown away
471 @param timeout: tell the remote that if the command fails to
472 produce any output for this number of seconds,
473 the command is hung and should be killed. Use
474 None to disable the timeout.
477 self
.command
= command
# stash .command, set it later
479 # avoid mutating the original master.cfg dictionary. Each
480 # ShellCommand gets its own copy, any start() methods won't be
481 # able to modify the original.
483 args
= {'workdir': workdir
,
485 'want_stdout': want_stdout
,
486 'want_stderr': want_stderr
,
487 'logfiles': logfiles
,
491 LoggedRemoteCommand
.__init
__(self
, "shell", args
)
494 self
.args
['command'] = self
.command
495 if self
.remote_command
== "shell":
496 # non-ShellCommand slavecommands are responsible for doing this
498 if self
.step
.slaveVersion("shell", "old") == "old":
499 self
.args
['dir'] = self
.args
['workdir']
500 what
= "command '%s' in dir '%s'" % (self
.args
['command'],
501 self
.args
['workdir'])
503 return LoggedRemoteCommand
.start(self
)
506 return "<RemoteShellCommand '%s'>" % repr(self
.command
)
510 I represent a single step of the build process. This step may involve
511 zero or more commands to be run in the build slave, as well as arbitrary
512 processing on the master side. Regardless of how many slave commands are
513 run, the BuildStep will result in a single status value.
515 The step is started by calling startStep(), which returns a Deferred that
516 fires when the step finishes. See C{startStep} for a description of the
517 results provided by that Deferred.
519 __init__ and start are good methods to override. Don't forget to upcall
520 BuildStep.__init__ or bad things will happen.
522 To launch a RemoteCommand, pass it to .runCommand and wait on the
525 Each BuildStep generates status as it runs. This status data is fed to
526 the L{buildbot.status.builder.BuildStepStatus} listener that sits in
527 C{self.step_status}. It can also feed progress data (like how much text
528 is output by a shell command) to the
529 L{buildbot.status.progress.StepProgress} object that lives in
530 C{self.progress}, by calling C{self.setProgress(metric, value)} as it
533 @type build: L{buildbot.process.base.Build}
534 @ivar build: the parent Build which is executing this step
536 @type progress: L{buildbot.status.progress.StepProgress}
537 @ivar progress: tracks ETA for the step
539 @type step_status: L{buildbot.status.builder.BuildStepStatus}
540 @ivar step_status: collects output status
543 # these parameters are used by the parent Build object to decide how to
544 # interpret our results. haltOnFailure will affect the build process
545 # immediately, the others will be taken into consideration when
546 # determining the overall build status.
548 # steps that are makred as alwaysRun will be run regardless of the outcome
549 # of previous steps (especially steps with haltOnFailure=True)
550 haltOnFailure
= False
551 flunkOnWarnings
= False
552 flunkOnFailure
= False
553 warnOnWarnings
= False
554 warnOnFailure
= False
557 # 'parms' holds a list of all the parameters we care about, to allow
558 # users to instantiate a subclass of BuildStep with a mixture of
559 # arguments, some of which are for us, some of which are for the subclass
560 # (or a delegate of the subclass, like how ShellCommand delivers many
561 # arguments to the RemoteShellCommand that it creates). Such delegating
562 # subclasses will use this list to figure out which arguments are meant
563 # for us and which should be given to someone else.
564 parms
= ['name', 'locks',
577 progressMetrics
= () # 'time' is implicit
578 useProgress
= True # set to False if step is really unpredictable
582 # doStepIf can be False, True, or a function that returns False or True
585 def __init__(self
, **kwargs
):
586 self
.factory
= (self
.__class
__, dict(kwargs
))
587 for p
in self
.__class
__.parms
:
588 if kwargs
.has_key(p
):
589 setattr(self
, p
, kwargs
[p
])
592 why
= "%s.__init__ got unexpected keyword argument(s) %s" \
593 % (self
, kwargs
.keys())
595 self
._pendingLogObservers
= []
597 def setBuild(self
, build
):
598 # subclasses which wish to base their behavior upon qualities of the
599 # Build (e.g. use the list of changed files to run unit tests only on
600 # code which has been modified) should do so here. The Build is not
601 # available during __init__, but setBuild() will be called just
605 def setBuildSlave(self
, buildslave
):
606 self
.buildslave
= buildslave
608 def setDefaultWorkdir(self
, workdir
):
609 # The Build calls this just after __init__(). ShellCommand
610 # and variants use a slave-side workdir, but some other steps
611 # do not. Subclasses which use a workdir should use the value
612 # set by this method unless they were constructed with
613 # something more specific.
616 def addFactoryArguments(self
, **kwargs
):
617 self
.factory
[1].update(kwargs
)
619 def getStepFactory(self
):
622 def setStepStatus(self
, step_status
):
623 self
.step_status
= step_status
625 def setupProgress(self
):
627 sp
= progress
.StepProgress(self
.name
, self
.progressMetrics
)
629 self
.step_status
.setProgress(sp
)
633 def setProgress(self
, metric
, value
):
634 """BuildSteps can call self.setProgress() to announce progress along
637 self
.progress
.setProgress(metric
, value
)
639 def getProperty(self
, propname
):
640 return self
.build
.getProperty(propname
)
642 def setProperty(self
, propname
, value
, source
="Step"):
643 self
.build
.setProperty(propname
, value
, source
)
645 def startStep(self
, remote
):
646 """Begin the step. This returns a Deferred that will fire when the
649 This deferred fires with a tuple of (result, [extra text]), although
650 older steps used to return just the 'result' value, so the receiving
651 L{base.Build} needs to be prepared to handle that too. C{result} is
652 one of the SUCCESS/WARNINGS/FAILURE/SKIPPED constants from
653 L{buildbot.status.builder}, and the extra text is a list of short
654 strings which should be appended to the Build's text results. This
655 text allows a test-case step which fails to append B{17 tests} to the
656 Build's status, in addition to marking the build as failing.
658 The deferred will errback if the step encounters an exception,
659 including an exception on the slave side (or if the slave goes away
660 altogether). Failures in shell commands (rc!=0) will B{not} cause an
661 errback, in general the BuildStep will evaluate the results and
662 decide whether to treat it as a WARNING or FAILURE.
664 @type remote: L{twisted.spread.pb.RemoteReference}
665 @param remote: a reference to the slave's
666 L{buildbot.slave.bot.SlaveBuilder} instance where any
667 RemoteCommands may be run
671 self
.deferred
= defer
.Deferred()
672 # convert all locks into their real form
674 for access
in self
.locks
:
675 if not isinstance(access
, locks
.LockAccess
):
676 # Buildbot 0.7.7 compability: user did not specify access
677 access
= access
.defaultAccess()
678 lock
= self
.build
.builder
.botmaster
.getLockByID(access
.lockid
)
679 lock_list
.append((lock
, access
))
680 self
.locks
= lock_list
681 # then narrow SlaveLocks down to the slave that this build is being
683 self
.locks
= [(l
.getLock(self
.build
.slavebuilder
), la
) for l
, la
in self
.locks
]
684 for l
, la
in self
.locks
:
685 if l
in self
.build
.locks
:
686 log
.msg("Hey, lock %s is claimed by both a Step (%s) and the"
687 " parent Build (%s)" % (l
, self
, self
.build
))
688 raise RuntimeError("lock claimed by both Step and Build")
689 d
= self
.acquireLocks()
690 d
.addCallback(self
._startStep
_2)
693 def acquireLocks(self
, res
=None):
694 log
.msg("acquireLocks(step %s, locks %s)" % (self
, self
.locks
))
696 return defer
.succeed(None)
697 for lock
, access
in self
.locks
:
698 if not lock
.isAvailable(access
):
699 log
.msg("step %s waiting for lock %s" % (self
, lock
))
700 d
= lock
.waitUntilMaybeAvailable(self
, access
)
701 d
.addCallback(self
.acquireLocks
)
703 # all locks are available, claim them all
704 for lock
, access
in self
.locks
:
705 lock
.claim(self
, access
)
706 return defer
.succeed(None)
708 def _startStep_2(self
, res
):
710 self
.progress
.start()
711 self
.step_status
.stepStarted()
714 if isinstance(self
.doStepIf
, bool):
715 if not self
.doStepIf
:
717 elif not self
.doStepIf(self
):
724 # this return value from self.start is a shortcut
725 # to finishing the step immediately
726 reactor
.callLater(0, self
.finished
, SKIPPED
)
728 log
.msg("BuildStep.startStep exception in .start")
729 self
.failed(Failure())
732 """Begin the step. Override this method and add code to do local
733 processing, fire off remote commands, etc.
735 To spawn a command in the buildslave, create a RemoteCommand instance
736 and run it with self.runCommand::
738 c = RemoteCommandFoo(args)
739 d = self.runCommand(c)
740 d.addCallback(self.fooDone).addErrback(self.failed)
742 As the step runs, it should send status information to the
745 self.step_status.setText(['compile', 'failed'])
746 self.step_status.setText2(['4', 'warnings'])
748 To have some code parse stdio (or other log stream) in realtime, add
749 a LogObserver subclass. This observer can use self.step.setProgress()
750 to provide better progress notification to the step.::
752 self.addLogObserver('stdio', MyLogObserver())
754 To add a LogFile, use self.addLog. Make sure it gets closed when it
755 finishes. When giving a Logfile to a RemoteShellCommand, just ask it
756 to close the log when the command completes::
758 log = self.addLog('output')
759 cmd = RemoteShellCommand(args)
760 cmd.useLog(log, closeWhenFinished=True)
762 You can also create complete Logfiles with generated text in a single
765 self.addCompleteLog('warnings', text)
767 When the step is done, it should call self.finished(result). 'result'
768 will be provided to the L{buildbot.process.base.Build}, and should be
769 one of the constants defined above: SUCCESS, WARNINGS, FAILURE, or
772 If the step encounters an exception, it should call self.failed(why).
773 'why' should be a Failure object. This automatically fails the whole
774 build with an exception. It is a good idea to add self.failed as an
775 errback to any Deferreds you might obtain.
777 If the step decides it does not need to be run, start() can return
778 the constant SKIPPED. This fires the callback immediately: it is not
779 necessary to call .finished yourself. This can also indicate to the
780 status-reporting mechanism that this step should not be displayed.
782 A step can be configured to only run under certain conditions. To
783 do this, set the step's doStepIf to a boolean value, or to a function
784 that returns a boolean value. If the value or function result is
785 False, then the step will return SKIPPED without doing anything,
786 otherwise the step will be executed normally. If you set doStepIf
787 to a function, that function should accept one parameter, which will
788 be the Step object itself."""
790 raise NotImplementedError("your subclass must implement this method")
792 def interrupt(self
, reason
):
793 """Halt the command, either because the user has decided to cancel
794 the build ('reason' is a string), or because the slave has
795 disconnected ('reason' is a ConnectionLost Failure). Any further
796 local processing should be skipped, and the Step completed with an
797 error status. The results text should say something useful like
798 ['step', 'interrupted'] or ['remote', 'lost']"""
801 def releaseLocks(self
):
802 log
.msg("releaseLocks(%s): %s" % (self
, self
.locks
))
803 for lock
, access
in self
.locks
:
804 lock
.release(self
, access
)
806 def finished(self
, results
):
808 self
.progress
.finish()
809 self
.step_status
.stepFinished(results
)
811 self
.deferred
.callback(results
)
813 def failed(self
, why
):
814 # if isinstance(why, pb.CopiedFailure): # a remote exception might
815 # only have short traceback, so formatFailure is not as useful as
816 # you'd like (no .frames, so no traceback is displayed)
817 log
.msg("BuildStep.failed, traceback follows")
821 self
.progress
.finish()
822 self
.addHTMLLog("err.html", formatFailure(why
))
823 self
.addCompleteLog("err.text", why
.getTraceback())
824 # could use why.getDetailedTraceback() for more information
825 self
.step_status
.setText([self
.name
, "exception"])
826 self
.step_status
.setText2([self
.name
])
827 self
.step_status
.stepFinished(EXCEPTION
)
829 log
.msg("exception during failure processing")
831 # the progress stuff may still be whacked (the StepStatus may
832 # think that it is still running), but the build overall will now
837 log
.msg("exception while releasing locks")
840 log
.msg("BuildStep.failed now firing callback")
841 self
.deferred
.callback(EXCEPTION
)
843 # utility methods that BuildSteps may find useful
845 def slaveVersion(self
, command
, oldversion
=None):
846 """Return the version number of the given slave command. For the
847 commands defined in buildbot.slave.commands, this is the value of
848 'cvs_ver' at the top of that file. Non-existent commands will return
849 a value of None. Buildslaves running buildbot-0.5.0 or earlier did
850 not respond to the version query: commands on those slaves will
851 return a value of OLDVERSION, so you can distinguish between old
852 buildslaves and missing commands.
854 If you know that <=0.5.0 buildslaves have the command you want (CVS
855 and SVN existed back then, but none of the other VC systems), then it
856 makes sense to call this with oldversion='old'. If the command you
857 want is newer than that, just leave oldversion= unspecified, and the
858 command will return None for a buildslave that does not implement the
861 return self
.build
.getSlaveCommandVersion(command
, oldversion
)
863 def slaveVersionIsOlderThan(self
, command
, minversion
):
864 sv
= self
.build
.getSlaveCommandVersion(command
, None)
867 # the version we get back is a string form of the CVS version number
868 # of the slave's buildbot/slave/commands.py, something like 1.39 .
869 # This might change in the future (I might move away from CVS), but
870 # if so I'll keep updating that string with suitably-comparable
872 if sv
.split(".") < minversion
.split("."):
876 def getSlaveName(self
):
877 return self
.build
.getSlaveName()
879 def addLog(self
, name
):
880 loog
= self
.step_status
.addLog(name
)
881 self
._connectPendingLogObservers
()
884 def getLog(self
, name
):
885 for l
in self
.step_status
.getLogs():
886 if l
.getName() == name
:
888 raise KeyError("no log named '%s'" % (name
,))
890 def addCompleteLog(self
, name
, text
):
891 log
.msg("addCompleteLog(%s)" % name
)
892 loog
= self
.step_status
.addLog(name
)
893 size
= loog
.chunkSize
894 for start
in range(0, len(text
), size
):
895 loog
.addStdout(text
[start
:start
+size
])
897 self
._connectPendingLogObservers
()
899 def addHTMLLog(self
, name
, html
):
900 log
.msg("addHTMLLog(%s)" % name
)
901 self
.step_status
.addHTMLLog(name
, html
)
902 self
._connectPendingLogObservers
()
904 def addLogObserver(self
, logname
, observer
):
905 assert interfaces
.ILogObserver
.providedBy(observer
)
906 observer
.setStep(self
)
907 self
._pendingLogObservers
.append((logname
, observer
))
908 self
._connectPendingLogObservers
()
910 def _connectPendingLogObservers(self
):
911 if not self
._pendingLogObservers
:
913 if not self
.step_status
:
916 for loog
in self
.step_status
.getLogs():
917 current_logs
[loog
.getName()] = loog
918 for logname
, observer
in self
._pendingLogObservers
[:]:
919 if logname
in current_logs
:
920 observer
.setLog(current_logs
[logname
])
921 self
._pendingLogObservers
.remove((logname
, observer
))
923 def addURL(self
, name
, url
):
924 """Add a BuildStep URL to this step.
926 An HREF to this URL will be added to any HTML representations of this
927 step. This allows a step to provide links to external web pages,
928 perhaps to provide detailed HTML code coverage results or other forms
931 self
.step_status
.addURL(name
, url
)
933 def runCommand(self
, c
):
934 c
.buildslave
= self
.buildslave
935 d
= c
.run(self
, self
.remote
)
939 class OutputProgressObserver(LogObserver
):
942 def __init__(self
, name
):
945 def logChunk(self
, build
, step
, log
, channel
, text
):
946 self
.length
+= len(text
)
947 self
.step
.setProgress(self
.name
, self
.length
)
949 class LoggingBuildStep(BuildStep
):
950 """This is an abstract base class, suitable for inheritance by all
951 BuildSteps that invoke RemoteCommands which emit stdout/stderr messages.
954 progressMetrics
= ('output',)
957 parms
= BuildStep
.parms
+ ['logfiles']
959 def __init__(self
, logfiles
={}, *args
, **kwargs
):
960 BuildStep
.__init
__(self
, *args
, **kwargs
)
961 self
.addFactoryArguments(logfiles
=logfiles
)
962 # merge a class-level 'logfiles' attribute with one passed in as an
964 self
.logfiles
= self
.logfiles
.copy()
965 self
.logfiles
.update(logfiles
)
966 self
.addLogObserver('stdio', OutputProgressObserver("output"))
968 def describe(self
, done
=False):
969 raise NotImplementedError("implement this in a subclass")
971 def startCommand(self
, cmd
, errorMessages
=[]):
973 @param cmd: a suitable RemoteCommand which will be launched, with
974 all output being put into our self.stdio_log LogFile
976 log
.msg("ShellCommand.startCommand(cmd=%s)" % (cmd
,))
977 log
.msg(" cmd.args = %r" % (cmd
.args
))
978 self
.cmd
= cmd
# so we can interrupt it
979 self
.step_status
.setText(self
.describe(False))
981 # stdio is the first log
982 self
.stdio_log
= stdio_log
= self
.addLog("stdio")
983 cmd
.useLog(stdio_log
, True)
984 for em
in errorMessages
:
985 stdio_log
.addHeader(em
)
986 # TODO: consider setting up self.stdio_log earlier, and have the
987 # code that passes in errorMessages instead call
988 # self.stdio_log.addHeader() directly.
990 # there might be other logs
991 self
.setupLogfiles(cmd
, self
.logfiles
)
993 d
= self
.runCommand(cmd
) # might raise ConnectionLost
994 d
.addCallback(lambda res
: self
.commandComplete(cmd
))
995 d
.addCallback(lambda res
: self
.createSummary(cmd
.logs
['stdio']))
996 d
.addCallback(lambda res
: self
.evaluateCommand(cmd
)) # returns results
997 def _gotResults(results
):
998 self
.setStatus(cmd
, results
)
1000 d
.addCallback(_gotResults
) # returns results
1001 d
.addCallbacks(self
.finished
, self
.checkDisconnect
)
1002 d
.addErrback(self
.failed
)
1004 def setupLogfiles(self
, cmd
, logfiles
):
1005 """Set up any additional logfiles= logs.
1007 for logname
,remotefilename
in logfiles
.items():
1008 # tell the BuildStepStatus to add a LogFile
1009 newlog
= self
.addLog(logname
)
1010 # and tell the LoggedRemoteCommand to feed it
1011 cmd
.useLog(newlog
, True)
1013 def interrupt(self
, reason
):
1014 # TODO: consider adding an INTERRUPTED or STOPPED status to use
1015 # instead of FAILURE, might make the text a bit more clear.
1016 # 'reason' can be a Failure, or text
1017 self
.addCompleteLog('interrupt', str(reason
))
1018 d
= self
.cmd
.interrupt(reason
)
1021 def checkDisconnect(self
, f
):
1022 f
.trap(error
.ConnectionLost
)
1023 self
.step_status
.setText(self
.describe(True) +
1024 ["failed", "slave", "lost"])
1025 self
.step_status
.setText2(["failed", "slave", "lost"])
1026 return self
.finished(FAILURE
)
1028 # to refine the status output, override one or more of the following
1029 # methods. Change as little as possible: start with the first ones on
1030 # this list and only proceed further if you have to
1032 # createSummary: add additional Logfiles with summarized results
1033 # evaluateCommand: decides whether the step was successful or not
1035 # getText: create the final per-step text strings
1036 # describeText2: create the strings added to the overall build status
1038 # getText2: only adds describeText2() when the step affects build status
1040 # setStatus: handles all status updating
1042 # commandComplete is available for general-purpose post-completion work.
1043 # It is a good place to do one-time parsing of logfiles, counting
1044 # warnings and errors. It should probably stash such counts in places
1045 # like self.warnings so they can be picked up later by your getText
1048 # TODO: most of this stuff should really be on BuildStep rather than
1049 # ShellCommand. That involves putting the status-setup stuff in
1050 # .finished, which would make it hard to turn off.
1052 def commandComplete(self
, cmd
):
1053 """This is a general-purpose hook method for subclasses. It will be
1054 called after the remote command has finished, but before any of the
1055 other hook functions are called."""
1058 def createSummary(self
, log
):
1059 """To create summary logs, do something like this:
1060 warnings = grep('^Warning:', log.getText())
1061 self.addCompleteLog('warnings', warnings)
1065 def evaluateCommand(self
, cmd
):
1066 """Decide whether the command was SUCCESS, WARNINGS, or FAILURE.
1067 Override this to, say, declare WARNINGS if there is any stderr
1068 activity, or to say that rc!=0 is not actually an error."""
1072 # if cmd.log.getStderr(): return WARNINGS
1075 def getText(self
, cmd
, results
):
1076 if results
== SUCCESS
:
1077 return self
.describe(True)
1078 elif results
== WARNINGS
:
1079 return self
.describe(True) + ["warnings"]
1081 return self
.describe(True) + ["failed"]
1083 def getText2(self
, cmd
, results
):
1084 """We have decided to add a short note about ourselves to the overall
1085 build description, probably because something went wrong. Return a
1086 short list of short strings. If your subclass counts test failures or
1087 warnings of some sort, this is a good place to announce the count."""
1088 # return ["%d warnings" % warningcount]
1089 # return ["%d tests" % len(failedTests)]
1092 def maybeGetText2(self
, cmd
, results
):
1093 if results
== SUCCESS
:
1094 # successful steps do not add anything to the build's text
1096 elif results
== WARNINGS
:
1097 if (self
.flunkOnWarnings
or self
.warnOnWarnings
):
1098 # we're affecting the overall build, so tell them why
1099 return self
.getText2(cmd
, results
)
1101 if (self
.haltOnFailure
or self
.flunkOnFailure
1102 or self
.warnOnFailure
):
1103 # we're affecting the overall build, so tell them why
1104 return self
.getText2(cmd
, results
)
1107 def setStatus(self
, cmd
, results
):
1108 # this is good enough for most steps, but it can be overridden to
1109 # get more control over the displayed text
1110 self
.step_status
.setText(self
.getText(cmd
, results
))
1111 self
.step_status
.setText2(self
.maybeGetText2(cmd
, results
))
1113 # (WithProeprties used to be available in this module)
1114 from buildbot
.process
.properties
import WithProperties
1115 _hush_pyflakes
= [WithProperties
]