buildbot/steps/shell.py

   1 # -*- test-case-name: buildbot.test.test_steps,buildbot.test.test_properties -*-
   2
   3 import types, re
   4 from twisted.python import log
   5 from buildbot import util
   6 from buildbot.process.buildstep import LoggingBuildStep, RemoteShellCommand
   7 from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE
   8
   9 class _BuildPropertyDictionary:
  10     def __init__(self, build):
  11         self.build = build
  12     def __getitem__(self, name):
  13         p = self.build.getProperty(name)
  14         if p is None:
  15             p = ""
  16         return p
  17
  18 class WithProperties(util.ComparableMixin):
  19     """This is a marker class, used in ShellCommand's command= argument to
  20     indicate that we want to interpolate a build property.
  21     """
  22
  23     compare_attrs = ('fmtstring', 'args')
  24
  25     def __init__(self, fmtstring, *args):
  26         self.fmtstring = fmtstring
  27         self.args = args
  28
  29     def render(self, build):
  30         if self.args:
  31             strings = []
  32             for name in self.args:
  33                 p = build.getProperty(name)
  34                 if p is None:
  35                     p = ""
  36                 strings.append(p)
  37             s = self.fmtstring % tuple(strings)
  38         else:
  39             s = self.fmtstring % _BuildPropertyDictionary(build)
  40         return s
  41
  42 class ShellCommand(LoggingBuildStep):
  43     """I run a single shell command on the buildslave. I return FAILURE if
  44     the exit code of that command is non-zero, SUCCESS otherwise. To change
  45     this behavior, override my .evaluateCommand method.
  46
  47     By default, a failure of this step will mark the whole build as FAILURE.
  48     To override this, give me an argument of flunkOnFailure=False .
  49
  50     I create a single Log named 'log' which contains the output of the
  51     command. To create additional summary Logs, override my .createSummary
  52     method.
  53
  54     The shell command I run (a list of argv strings) can be provided in
  55     several ways:
  56       - a class-level .command attribute
  57       - a command= parameter to my constructor (overrides .command)
  58       - set explicitly with my .setCommand() method (overrides both)
  59
  60     @ivar command: a list of argv strings (or WithProperties instances).
  61                    This will be used by start() to create a
  62                    RemoteShellCommand instance.
  63
  64     @ivar logfiles: a dict mapping log NAMEs to workdir-relative FILENAMEs
  65                     of their corresponding logfiles. The contents of the file
  66                     named FILENAME will be put into a LogFile named NAME, ina
  67                     something approximating real-time. (note that logfiles=
  68                     is actually handled by our parent class LoggingBuildStep)
  69
  70     """
  71
  72     name = "shell"
  73     description = None # set this to a list of short strings to override
  74     descriptionDone = None # alternate description when the step is complete
  75     command = None # set this to a command, or set in kwargs
  76     # logfiles={} # you can also set 'logfiles' to a dictionary, and it
  77     #               will be merged with any logfiles= argument passed in
  78     #               to __init__
  79
  80     # override this on a specific ShellCommand if you want to let it fail
  81     # without dooming the entire build to a status of FAILURE
  82     flunkOnFailure = True
  83
  84     def __init__(self, workdir,
  85                  description=None, descriptionDone=None,
  86                  command=None,
  87                  **kwargs):
  88         # most of our arguments get passed through to the RemoteShellCommand
  89         # that we create, but first strip out the ones that we pass to
  90         # BuildStep (like haltOnFailure and friends), and a couple that we
  91         # consume ourselves.
  92         self.workdir = workdir # required by RemoteShellCommand
  93         if description:
  94             self.description = description
  95         if isinstance(self.description, str):
  96             self.description = [self.description]
  97         if descriptionDone:
  98             self.descriptionDone = descriptionDone
  99         if isinstance(self.descriptionDone, str):
 100             self.descriptionDone = [self.descriptionDone]
 101         if command:
 102             self.command = command
 103
 104         # pull out the ones that LoggingBuildStep wants, then upcall
 105         buildstep_kwargs = {}
 106         for k in kwargs.keys()[:]:
 107             if k in self.__class__.parms:
 108                 buildstep_kwargs[k] = kwargs[k]
 109                 del kwargs[k]
 110         LoggingBuildStep.__init__(self, **buildstep_kwargs)
 111
 112         # everything left over goes to the RemoteShellCommand
 113         kwargs['workdir'] = workdir # including a copy of 'workdir'
 114         self.remote_kwargs = kwargs
 115
 116
 117     def setCommand(self, command):
 118         self.command = command
 119
 120     def describe(self, done=False):
 121         """Return a list of short strings to describe this step, for the
 122         status display. This uses the first few words of the shell command.
 123         You can replace this by setting .description in your subclass, or by
 124         overriding this method to describe the step better.
 125
 126         @type  done: boolean
 127         @param done: whether the command is complete or not, to improve the
 128                      way the command is described. C{done=False} is used
 129                      while the command is still running, so a single
 130                      imperfect-tense verb is appropriate ('compiling',
 131                      'testing', ...) C{done=True} is used when the command
 132                      has finished, and the default getText() method adds some
 133                      text, so a simple noun is appropriate ('compile',
 134                      'tests' ...)
 135         """
 136
 137         if done and self.descriptionDone is not None:
 138             return self.descriptionDone
 139         if self.description is not None:
 140             return self.description
 141
 142         words = self.command
 143         # TODO: handle WithProperties here
 144         if isinstance(words, types.StringTypes):
 145             words = words.split()
 146         if len(words) < 1:
 147             return ["???"]
 148         if len(words) == 1:
 149             return ["'%s'" % words[0]]
 150         if len(words) == 2:
 151             return ["'%s" % words[0], "%s'" % words[1]]
 152         return ["'%s" % words[0], "%s" % words[1], "...'"]
 153
 154     def _interpolateProperties(self, command):
 155         # interpolate any build properties into our command
 156         if not isinstance(command, (list, tuple)):
 157             return command
 158         command_argv = []
 159         for argv in command:
 160             if isinstance(argv, WithProperties):
 161                 command_argv.append(argv.render(self.build))
 162             else:
 163                 command_argv.append(argv)
 164         return command_argv
 165
 166     def setupEnvironment(self, cmd):
 167         # merge in anything from Build.slaveEnvironment . Earlier steps
 168         # (perhaps ones which compile libraries or sub-projects that need to
 169         # be referenced by later steps) can add keys to
 170         # self.build.slaveEnvironment to affect later steps.
 171         slaveEnv = self.build.slaveEnvironment
 172         if slaveEnv:
 173             if cmd.args['env'] is None:
 174                 cmd.args['env'] = {}
 175             cmd.args['env'].update(slaveEnv)
 176             # note that each RemoteShellCommand gets its own copy of the
 177             # dictionary, so we shouldn't be affecting anyone but ourselves.
 178
 179     def checkForOldSlaveAndLogfiles(self):
 180         if not self.logfiles:
 181             return # doesn't matter
 182         if not self.slaveVersionIsOlderThan("shell", "2.1"):
 183             return # slave is new enough
 184         # this buildslave is too old and will ignore the 'logfiles'
 185         # argument. You'll either have to pull the logfiles manually
 186         # (say, by using 'cat' in a separate RemoteShellCommand) or
 187         # upgrade the buildslave.
 188         msg1 = ("Warning: buildslave %s is too old "
 189                 "to understand logfiles=, ignoring it."
 190                % self.getSlaveName())
 191         msg2 = "You will have to pull this logfile (%s) manually."
 192         log.msg(msg1)
 193         for logname,remotefilename in self.logfiles.items():
 194             newlog = self.addLog(logname)
 195             newlog.addHeader(msg1 + "\n")
 196             newlog.addHeader(msg2 % remotefilename + "\n")
 197             newlog.finish()
 198         # now prevent setupLogfiles() from adding them
 199         self.logfiles = {}
 200
 201     def start(self):
 202         # this block is specific to ShellCommands. subclasses that don't need
 203         # to set up an argv array, an environment, or extra logfiles= (like
 204         # the Source subclasses) can just skip straight to startCommand()
 205         command = self._interpolateProperties(self.command)
 206         assert isinstance(command, (list, tuple, str))
 207         # create the actual RemoteShellCommand instance now
 208         kwargs = self.remote_kwargs
 209         kwargs['command'] = command
 210         kwargs['logfiles'] = self.logfiles
 211         cmd = RemoteShellCommand(**kwargs)
 212         self.setupEnvironment(cmd)
 213         self.checkForOldSlaveAndLogfiles()
 214
 215         self.startCommand(cmd)
 216
 217
 218
 219 class TreeSize(ShellCommand):
 220     name = "treesize"
 221     command = ["du", "-s", "."]
 222     kb = None
 223
 224     def commandComplete(self, cmd):
 225         out = cmd.log.getText()
 226         m = re.search(r'^(\d+)', out)
 227         if m:
 228             self.kb = int(m.group(1))
 229
 230     def evaluateCommand(self, cmd):
 231         if cmd.rc != 0:
 232             return FAILURE
 233         if self.kb is None:
 234             return WARNINGS # not sure how 'du' could fail, but whatever
 235         return SUCCESS
 236
 237     def getText(self, cmd, results):
 238         if self.kb is not None:
 239             return ["treesize", "%d kb" % self.kb]
 240         return ["treesize", "unknown"]
 241
 242 class Configure(ShellCommand):
 243
 244     name = "configure"
 245     haltOnFailure = 1
 246     description = ["configuring"]
 247     descriptionDone = ["configure"]
 248     command = ["./configure"]
 249
 250 class Compile(ShellCommand):
 251
 252     name = "compile"
 253     haltOnFailure = 1
 254     description = ["compiling"]
 255     descriptionDone = ["compile"]
 256     command = ["make", "all"]
 257
 258     OFFprogressMetrics = ('output',)
 259     # things to track: number of files compiled, number of directories
 260     # traversed (assuming 'make' is being used)
 261
 262     def createSummary(self, cmd):
 263         # TODO: grep for the characteristic GCC warning/error lines and
 264         # assemble them into a pair of buffers
 265         pass
 266
 267 class Test(ShellCommand):
 268
 269     name = "test"
 270     warnOnFailure = 1
 271     description = ["testing"]
 272     descriptionDone = ["test"]
 273     command = ["make", "test"]