buildbot/steps/transfer.py

   1 # -*- test-case-name: buildbot.test.test_transfer -*-
   2
   3 import os.path
   4 from twisted.internet import reactor
   5 from twisted.spread import pb
   6 from twisted.python import log
   7 from buildbot.process.buildstep import RemoteCommand, BuildStep
   8 from buildbot.process.buildstep import SUCCESS, FAILURE
   9 from buildbot.interfaces import BuildSlaveTooOldError
  10
  11
  12 class _FileWriter(pb.Referenceable):
  13     """
  14     Helper class that acts as a file-object with write access
  15     """
  16
  17     def __init__(self, destfile, maxsize, mode):
  18         # Create missing directories.
  19         destfile = os.path.abspath(destfile)
  20         dirname = os.path.dirname(destfile)
  21         if not os.path.exists(dirname):
  22             os.makedirs(dirname)
  23
  24         self.destfile = destfile
  25         self.fp = open(destfile, "wb")
  26         if mode is not None:
  27             os.chmod(destfile, mode)
  28         self.remaining = maxsize
  29
  30     def remote_write(self, data):
  31         """
  32         Called from remote slave to write L{data} to L{fp} within boundaries
  33         of L{maxsize}
  34
  35         @type  data: C{string}
  36         @param data: String of data to write
  37         """
  38         if self.remaining is not None:
  39             if len(data) > self.remaining:
  40                 data = data[:self.remaining]
  41             self.fp.write(data)
  42             self.remaining = self.remaining - len(data)
  43         else:
  44             self.fp.write(data)
  45
  46     def remote_close(self):
  47         """
  48         Called by remote slave to state that no more data will be transfered
  49         """
  50         self.fp.close()
  51         self.fp = None
  52
  53     def __del__(self):
  54         # unclean shutdown, the file is probably truncated, so delete it
  55         # altogether rather than deliver a corrupted file
  56         fp = getattr(self, "fp", None)
  57         if fp:
  58             fp.close()
  59             os.unlink(self.destfile)
  60
  61
  62 class StatusRemoteCommand(RemoteCommand):
  63     def __init__(self, remote_command, args):
  64         RemoteCommand.__init__(self, remote_command, args)
  65
  66         self.rc = None
  67         self.stderr = ''
  68
  69     def remoteUpdate(self, update):
  70         #log.msg('StatusRemoteCommand: update=%r' % update)
  71         if 'rc' in update:
  72             self.rc = update['rc']
  73         if 'stderr' in update:
  74             self.stderr = self.stderr + update['stderr'] + '\n'
  75
  76
  77 class FileUpload(BuildStep):
  78     """
  79     Build step to transfer a file from the slave to the master.
  80
  81     arguments:
  82
  83     - ['slavesrc']   filename of source file at slave, relative to workdir
  84     - ['masterdest'] filename of destination file at master
  85     - ['workdir']    string with slave working directory relative to builder
  86                      base dir, default 'build'
  87     - ['maxsize']    maximum size of the file, default None (=unlimited)
  88     - ['blocksize']  maximum size of each block being transfered
  89     - ['mode']       file access mode for the resulting master-side file.
  90                      The default (=None) is to leave it up to the umask of
  91                      the buildmaster process.
  92
  93     """
  94
  95     DEFAULT_WORKDIR = "build"           # is this redundant?
  96
  97     name = 'upload'
  98
  99     def __init__(self, slavesrc, masterdest,
 100                  workdir=None, maxsize=None, blocksize=16*1024, mode=None,
 101                  **buildstep_kwargs):
 102         BuildStep.__init__(self, **buildstep_kwargs)
 103         self.addFactoryArguments(slavesrc=slavesrc,
 104                                  masterdest=masterdest,
 105                                  workdir=workdir,
 106                                  maxsize=maxsize,
 107                                  blocksize=blocksize,
 108                                  mode=mode,
 109                                  )
 110
 111         self.slavesrc = slavesrc
 112         self.masterdest = masterdest
 113         self.workdir = workdir
 114         self.maxsize = maxsize
 115         self.blocksize = blocksize
 116         assert isinstance(mode, (int, type(None)))
 117         self.mode = mode
 118
 119     def setDefaultWorkdir(self, workdir):
 120         if self.workdir is None:
 121             self.workdir = workdir
 122
 123     def start(self):
 124         version = self.slaveVersion("uploadFile")
 125         properties = self.build.getProperties()
 126
 127         if not version:
 128             m = "slave is too old, does not know about uploadFile"
 129             raise BuildSlaveTooOldError(m)
 130
 131         source = properties.render(self.slavesrc)
 132         masterdest = properties.render(self.masterdest)
 133         # we rely upon the fact that the buildmaster runs chdir'ed into its
 134         # basedir to make sure that relative paths in masterdest are expanded
 135         # properly. TODO: maybe pass the master's basedir all the way down
 136         # into the BuildStep so we can do this better.
 137         masterdest = os.path.expanduser(masterdest)
 138         log.msg("FileUpload started, from slave %r to master %r"
 139                 % (source, masterdest))
 140
 141         self.step_status.setColor('yellow')
 142         self.step_status.setText(['uploading', os.path.basename(source)])
 143
 144         # we use maxsize to limit the amount of data on both sides
 145         fileWriter = _FileWriter(masterdest, self.maxsize, self.mode)
 146
 147         # default arguments
 148         args = {
 149             'slavesrc': source,
 150             'workdir': self.workdir or self.DEFAULT_WORKDIR,
 151             'writer': fileWriter,
 152             'maxsize': self.maxsize,
 153             'blocksize': self.blocksize,
 154             }
 155
 156         self.cmd = StatusRemoteCommand('uploadFile', args)
 157         d = self.runCommand(self.cmd)
 158         d.addCallback(self.finished).addErrback(self.failed)
 159
 160     def finished(self, result):
 161         if self.cmd.stderr != '':
 162             self.addCompleteLog('stderr', self.cmd.stderr)
 163
 164         if self.cmd.rc is None or self.cmd.rc == 0:
 165             self.step_status.setColor('green')
 166             return BuildStep.finished(self, SUCCESS)
 167         self.step_status.setColor('red')
 168         return BuildStep.finished(self, FAILURE)
 169
 170
 171
 172
 173
 174 class _FileReader(pb.Referenceable):
 175     """
 176     Helper class that acts as a file-object with read access
 177     """
 178
 179     def __init__(self, fp):
 180         self.fp = fp
 181
 182     def remote_read(self, maxlength):
 183         """
 184         Called from remote slave to read at most L{maxlength} bytes of data
 185
 186         @type  maxlength: C{integer}
 187         @param maxlength: Maximum number of data bytes that can be returned
 188
 189         @return: Data read from L{fp}
 190         @rtype: C{string} of bytes read from file
 191         """
 192         if self.fp is None:
 193             return ''
 194
 195         data = self.fp.read(maxlength)
 196         return data
 197
 198     def remote_close(self):
 199         """
 200         Called by remote slave to state that no more data will be transfered
 201         """
 202         if self.fp is not None:
 203             self.fp.close()
 204             self.fp = None
 205
 206
 207 class FileDownload(BuildStep):
 208     """
 209     Download the first 'maxsize' bytes of a file, from the buildmaster to the
 210     buildslave. Set the mode of the file
 211
 212     Arguments::
 213
 214      ['mastersrc'] filename of source file at master
 215      ['slavedest'] filename of destination file at slave
 216      ['workdir']   string with slave working directory relative to builder
 217                    base dir, default 'build'
 218      ['maxsize']   maximum size of the file, default None (=unlimited)
 219      ['blocksize'] maximum size of each block being transfered
 220      ['mode']      use this to set the access permissions of the resulting
 221                    buildslave-side file. This is traditionally an octal
 222                    integer, like 0644 to be world-readable (but not
 223                    world-writable), or 0600 to only be readable by
 224                    the buildslave account, or 0755 to be world-executable.
 225                    The default (=None) is to leave it up to the umask of
 226                    the buildslave process.
 227
 228     """
 229
 230     DEFAULT_WORKDIR = "build"           # is this redundant?
 231
 232     name = 'download'
 233
 234     def __init__(self, mastersrc, slavedest,
 235                  workdir=None, maxsize=None, blocksize=16*1024, mode=None,
 236                  **buildstep_kwargs):
 237         BuildStep.__init__(self, **buildstep_kwargs)
 238         self.addFactoryArguments(mastersrc=mastersrc,
 239                                  slavedest=slavedest,
 240                                  workdir=workdir,
 241                                  maxsize=maxsize,
 242                                  blocksize=blocksize,
 243                                  mode=mode,
 244                                  )
 245
 246         self.mastersrc = mastersrc
 247         self.slavedest = slavedest
 248         self.workdir = workdir
 249         self.maxsize = maxsize
 250         self.blocksize = blocksize
 251         assert isinstance(mode, (int, type(None)))
 252         self.mode = mode
 253
 254     def setDefaultWorkdir(self, workdir):
 255         if self.workdir is None:
 256             self.workdir = workdir
 257
 258     def start(self):
 259         properties = self.build.getProperties()
 260
 261         version = self.slaveVersion("downloadFile")
 262         if not version:
 263             m = "slave is too old, does not know about downloadFile"
 264             raise BuildSlaveTooOldError(m)
 265
 266         # we are currently in the buildmaster's basedir, so any non-absolute
 267         # paths will be interpreted relative to that
 268         source = os.path.expanduser(properties.render(self.mastersrc))
 269         slavedest = properties.render(self.slavedest)
 270         log.msg("FileDownload started, from master %r to slave %r" %
 271                 (source, slavedest))
 272
 273         self.step_status.setColor('yellow')
 274         self.step_status.setText(['downloading', "to",
 275                                   os.path.basename(slavedest)])
 276
 277         # setup structures for reading the file
 278         try:
 279             fp = open(source, 'rb')
 280         except IOError:
 281             # if file does not exist, bail out with an error
 282             self.addCompleteLog('stderr',
 283                                 'File %r not available at master' % source)
 284             # TODO: once BuildStep.start() gets rewritten to use
 285             # maybeDeferred, just re-raise the exception here.
 286             reactor.callLater(0, BuildStep.finished, self, FAILURE)
 287             return
 288         fileReader = _FileReader(fp)
 289
 290         # default arguments
 291         args = {
 292             'slavedest': slavedest,
 293             'maxsize': self.maxsize,
 294             'reader': fileReader,
 295             'blocksize': self.blocksize,
 296             'workdir': self.workdir or self.DEFAULT_WORKDIR,
 297             'mode': self.mode,
 298             }
 299
 300         self.cmd = StatusRemoteCommand('downloadFile', args)
 301         d = self.runCommand(self.cmd)
 302         d.addCallback(self.finished).addErrback(self.failed)
 303
 304     def finished(self, result):
 305         if self.cmd.stderr != '':
 306             self.addCompleteLog('stderr', self.cmd.stderr)
 307
 308         if self.cmd.rc is None or self.cmd.rc == 0:
 309             self.step_status.setColor('green')
 310             return BuildStep.finished(self, SUCCESS)
 311         self.step_status.setColor('red')
 312         return BuildStep.finished(self, FAILURE)
 313