buildbot/steps/transfer.py

   1 # -*- test-case-name: buildbot.test.test_transfer -*-
   2
   3 import os.path, tarfile, tempfile
   4 from twisted.internet import reactor
   5 from twisted.spread import pb
   6 from twisted.python import log
   7 from buildbot.process.buildstep import RemoteCommand, BuildStep
   8 from buildbot.process.buildstep import SUCCESS, FAILURE, SKIPPED
   9 from buildbot.interfaces import BuildSlaveTooOldError
  10
  11
  12 class _FileWriter(pb.Referenceable):
  13     """
  14     Helper class that acts as a file-object with write access
  15     """
  16
  17     def __init__(self, destfile, maxsize, mode):
  18         # Create missing directories.
  19         destfile = os.path.abspath(destfile)
  20         dirname = os.path.dirname(destfile)
  21         if not os.path.exists(dirname):
  22             os.makedirs(dirname)
  23
  24         self.destfile = destfile
  25         self.fp = open(destfile, "wb")
  26         if mode is not None:
  27             os.chmod(destfile, mode)
  28         self.remaining = maxsize
  29
  30     def remote_write(self, data):
  31         """
  32         Called from remote slave to write L{data} to L{fp} within boundaries
  33         of L{maxsize}
  34
  35         @type  data: C{string}
  36         @param data: String of data to write
  37         """
  38         if self.remaining is not None:
  39             if len(data) > self.remaining:
  40                 data = data[:self.remaining]
  41             self.fp.write(data)
  42             self.remaining = self.remaining - len(data)
  43         else:
  44             self.fp.write(data)
  45
  46     def remote_close(self):
  47         """
  48         Called by remote slave to state that no more data will be transfered
  49         """
  50         self.fp.close()
  51         self.fp = None
  52
  53     def __del__(self):
  54         # unclean shutdown, the file is probably truncated, so delete it
  55         # altogether rather than deliver a corrupted file
  56         fp = getattr(self, "fp", None)
  57         if fp:
  58             fp.close()
  59             os.unlink(self.destfile)
  60
  61
  62 class _DirectoryWriter(_FileWriter):
  63     """
  64     A DirectoryWriter is implemented as a FileWriter, with an added post-processing
  65     step to unpack the archive, once the transfer has completed.
  66     """
  67
  68     def __init__(self, destroot, maxsize, mode):
  69         self.destroot = destroot
  70
  71         self.fd, self.tarname = tempfile.mkstemp()
  72
  73         _FileWriter.__init__(self, self.tarname, maxsize, mode)
  74
  75     def remote_unpack(self):
  76         """
  77         Called by remote slave to state that no more data will be transfered
  78         """
  79         if self.fp:
  80             self.fp.close()
  81                 self.fp = None
  82         fileobj = os.fdopen(self.fd, 'r')
  83         archive = tarfile.open(name=self.tarname, mode="r|bz2", fileobj=fileobj)
  84         archive.extractall(path=self.destroot)
  85         os.remove(self.tarname)
  86
  87
  88 class StatusRemoteCommand(RemoteCommand):
  89     def __init__(self, remote_command, args):
  90         RemoteCommand.__init__(self, remote_command, args)
  91
  92         self.rc = None
  93         self.stderr = ''
  94
  95     def remoteUpdate(self, update):
  96         #log.msg('StatusRemoteCommand: update=%r' % update)
  97         if 'rc' in update:
  98             self.rc = update['rc']
  99         if 'stderr' in update:
 100             self.stderr = self.stderr + update['stderr'] + '\n'
 101
 102 class _TransferBuildStep(BuildStep):
 103     """
 104     Base class for FileUpload and FileDownload to factor out common
 105     functionality.
 106     """
 107     DEFAULT_WORKDIR = "build"           # is this redundant?
 108
 109     def setDefaultWorkdir(self, workdir):
 110         if self.workdir is None:
 111             self.workdir = workdir
 112
 113     def _getWorkdir(self):
 114         properties = self.build.getProperties()
 115         if self.workdir is None:
 116             workdir = self.DEFAULT_WORKDIR
 117         else:
 118             workdir = self.workdir
 119         return properties.render(workdir)
 120
 121     def finished(self, result):
 122         # Subclasses may choose to skip a transfer. In those cases, self.cmd
 123         # will be None, and we should just let BuildStep.finished() handle
 124         # the rest
 125         if result == SKIPPED:
 126             return BuildStep.finished(self, SKIPPED)
 127         if self.cmd.stderr != '':
 128             self.addCompleteLog('stderr', self.cmd.stderr)
 129
 130         if self.cmd.rc is None or self.cmd.rc == 0:
 131             return BuildStep.finished(self, SUCCESS)
 132         return BuildStep.finished(self, FAILURE)
 133
 134
 135 class FileUpload(_TransferBuildStep):
 136     """
 137     Build step to transfer a file from the slave to the master.
 138
 139     arguments:
 140
 141     - ['slavesrc']   filename of source file at slave, relative to workdir
 142     - ['masterdest'] filename of destination file at master
 143     - ['workdir']    string with slave working directory relative to builder
 144                      base dir, default 'build'
 145     - ['maxsize']    maximum size of the file, default None (=unlimited)
 146     - ['blocksize']  maximum size of each block being transfered
 147     - ['mode']       file access mode for the resulting master-side file.
 148                      The default (=None) is to leave it up to the umask of
 149                      the buildmaster process.
 150
 151     """
 152
 153     name = 'upload'
 154
 155     def __init__(self, slavesrc, masterdest,
 156                  workdir=None, maxsize=None, blocksize=16*1024, mode=None,
 157                  **buildstep_kwargs):
 158         BuildStep.__init__(self, **buildstep_kwargs)
 159         self.addFactoryArguments(slavesrc=slavesrc,
 160                                  masterdest=masterdest,
 161                                  workdir=workdir,
 162                                  maxsize=maxsize,
 163                                  blocksize=blocksize,
 164                                  mode=mode,
 165                                  )
 166
 167         self.slavesrc = slavesrc
 168         self.masterdest = masterdest
 169         self.workdir = workdir
 170         self.maxsize = maxsize
 171         self.blocksize = blocksize
 172         assert isinstance(mode, (int, type(None)))
 173         self.mode = mode
 174
 175     def start(self):
 176         version = self.slaveVersion("uploadFile")
 177         properties = self.build.getProperties()
 178
 179         if not version:
 180             m = "slave is too old, does not know about uploadFile"
 181             raise BuildSlaveTooOldError(m)
 182
 183         source = properties.render(self.slavesrc)
 184         masterdest = properties.render(self.masterdest)
 185         # we rely upon the fact that the buildmaster runs chdir'ed into its
 186         # basedir to make sure that relative paths in masterdest are expanded
 187         # properly. TODO: maybe pass the master's basedir all the way down
 188         # into the BuildStep so we can do this better.
 189         masterdest = os.path.expanduser(masterdest)
 190         log.msg("FileUpload started, from slave %r to master %r"
 191                 % (source, masterdest))
 192
 193         self.step_status.setText(['uploading', os.path.basename(source)])
 194
 195         # we use maxsize to limit the amount of data on both sides
 196         fileWriter = _FileWriter(masterdest, self.maxsize, self.mode)
 197
 198         # default arguments
 199         args = {
 200             'slavesrc': source,
 201             'workdir': self._getWorkdir(),
 202             'writer': fileWriter,
 203             'maxsize': self.maxsize,
 204             'blocksize': self.blocksize,
 205             }
 206
 207         self.cmd = StatusRemoteCommand('uploadFile', args)
 208         d = self.runCommand(self.cmd)
 209         d.addCallback(self.finished).addErrback(self.failed)
 210
 211
 212 class DirectoryUpload(BuildStep):
 213     """
 214     Build step to transfer a directory from the slave to the master.
 215
 216     arguments:
 217
 218     - ['slavesrc']   name of source directory at slave, relative to workdir
 219     - ['masterdest'] name of destination directory at master
 220     - ['workdir']    string with slave working directory relative to builder
 221                      base dir, default 'build'
 222     - ['maxsize']    maximum size of each file, default None (=unlimited)
 223     - ['blocksize']  maximum size of each block being transfered
 224     - ['mode']       file access mode for the resulting master-side file.
 225                      The default (=None) is to leave it up to the umask of
 226                      the buildmaster process.
 227
 228     """
 229
 230     name = 'upload'
 231
 232     def __init__(self, slavesrc, masterdest,
 233                  workdir="build", maxsize=None, blocksize=16*1024, mode=None,
 234                  **buildstep_kwargs):
 235         BuildStep.__init__(self, **buildstep_kwargs)
 236         self.addFactoryArguments(slavesrc=slavesrc,
 237                                  masterdest=masterdest,
 238                                  workdir=workdir,
 239                                  maxsize=maxsize,
 240                                  blocksize=blocksize,
 241                                  mode=mode,
 242                                  )
 243
 244         self.slavesrc = slavesrc
 245         self.masterdest = masterdest
 246         self.workdir = workdir
 247         self.maxsize = maxsize
 248         self.blocksize = blocksize
 249         assert isinstance(mode, (int, type(None)))
 250         self.mode = mode
 251
 252     def start(self):
 253         version = self.slaveVersion("uploadDirectory")
 254         properties = self.build.getProperties()
 255
 256         if not version:
 257             m = "slave is too old, does not know about uploadDirectory"
 258             raise BuildSlaveTooOldError(m)
 259
 260         source = properties.render(self.slavesrc)
 261         masterdest = properties.render(self.masterdest)
 262         # we rely upon the fact that the buildmaster runs chdir'ed into its
 263         # basedir to make sure that relative paths in masterdest are expanded
 264         # properly. TODO: maybe pass the master's basedir all the way down
 265         # into the BuildStep so we can do this better.
 266         masterdest = os.path.expanduser(masterdest)
 267         log.msg("DirectoryUpload started, from slave %r to master %r"
 268                 % (source, masterdest))
 269
 270         self.step_status.setText(['uploading', os.path.basename(source)])
 271
 272         # we use maxsize to limit the amount of data on both sides
 273         dirWriter = _DirectoryWriter(masterdest, self.maxsize, self.mode)
 274
 275         # default arguments
 276         args = {
 277             'slavesrc': source,
 278             'workdir': self.workdir,
 279             'writer': dirWriter,
 280             'maxsize': self.maxsize,
 281             'blocksize': self.blocksize,
 282             }
 283
 284         self.cmd = StatusRemoteCommand('uploadDirectory', args)
 285         d = self.runCommand(self.cmd)
 286         d.addCallback(self.finished).addErrback(self.failed)
 287
 288     def finished(self, result):
 289         # Subclasses may choose to skip a transfer. In those cases, self.cmd
 290         # will be None, and we should just let BuildStep.finished() handle
 291         # the rest
 292         if result == SKIPPED:
 293             return BuildStep.finished(self, SKIPPED)
 294         if self.cmd.stderr != '':
 295             self.addCompleteLog('stderr', self.cmd.stderr)
 296
 297         if self.cmd.rc is None or self.cmd.rc == 0:
 298             return BuildStep.finished(self, SUCCESS)
 299         return BuildStep.finished(self, FAILURE)
 300
 301
 302
 303
 304 class _FileReader(pb.Referenceable):
 305     """
 306     Helper class that acts as a file-object with read access
 307     """
 308
 309     def __init__(self, fp):
 310         self.fp = fp
 311
 312     def remote_read(self, maxlength):
 313         """
 314         Called from remote slave to read at most L{maxlength} bytes of data
 315
 316         @type  maxlength: C{integer}
 317         @param maxlength: Maximum number of data bytes that can be returned
 318
 319         @return: Data read from L{fp}
 320         @rtype: C{string} of bytes read from file
 321         """
 322         if self.fp is None:
 323             return ''
 324
 325         data = self.fp.read(maxlength)
 326         return data
 327
 328     def remote_close(self):
 329         """
 330         Called by remote slave to state that no more data will be transfered
 331         """
 332         if self.fp is not None:
 333             self.fp.close()
 334             self.fp = None
 335
 336
 337 class FileDownload(_TransferBuildStep):
 338     """
 339     Download the first 'maxsize' bytes of a file, from the buildmaster to the
 340     buildslave. Set the mode of the file
 341
 342     Arguments::
 343
 344      ['mastersrc'] filename of source file at master
 345      ['slavedest'] filename of destination file at slave
 346      ['workdir']   string with slave working directory relative to builder
 347                    base dir, default 'build'
 348      ['maxsize']   maximum size of the file, default None (=unlimited)
 349      ['blocksize'] maximum size of each block being transfered
 350      ['mode']      use this to set the access permissions of the resulting
 351                    buildslave-side file. This is traditionally an octal
 352                    integer, like 0644 to be world-readable (but not
 353                    world-writable), or 0600 to only be readable by
 354                    the buildslave account, or 0755 to be world-executable.
 355                    The default (=None) is to leave it up to the umask of
 356                    the buildslave process.
 357
 358     """
 359     name = 'download'
 360
 361     def __init__(self, mastersrc, slavedest,
 362                  workdir=None, maxsize=None, blocksize=16*1024, mode=None,
 363                  **buildstep_kwargs):
 364         BuildStep.__init__(self, **buildstep_kwargs)
 365         self.addFactoryArguments(mastersrc=mastersrc,
 366                                  slavedest=slavedest,
 367                                  workdir=workdir,
 368                                  maxsize=maxsize,
 369                                  blocksize=blocksize,
 370                                  mode=mode,
 371                                  )
 372
 373         self.mastersrc = mastersrc
 374         self.slavedest = slavedest
 375         self.workdir = workdir
 376         self.maxsize = maxsize
 377         self.blocksize = blocksize
 378         assert isinstance(mode, (int, type(None)))
 379         self.mode = mode
 380
 381     def start(self):
 382         properties = self.build.getProperties()
 383
 384         version = self.slaveVersion("downloadFile")
 385         if not version:
 386             m = "slave is too old, does not know about downloadFile"
 387             raise BuildSlaveTooOldError(m)
 388
 389         # we are currently in the buildmaster's basedir, so any non-absolute
 390         # paths will be interpreted relative to that
 391         source = os.path.expanduser(properties.render(self.mastersrc))
 392         slavedest = properties.render(self.slavedest)
 393         log.msg("FileDownload started, from master %r to slave %r" %
 394                 (source, slavedest))
 395
 396         self.step_status.setText(['downloading', "to",
 397                                   os.path.basename(slavedest)])
 398
 399         # setup structures for reading the file
 400         try:
 401             fp = open(source, 'rb')
 402         except IOError:
 403             # if file does not exist, bail out with an error
 404             self.addCompleteLog('stderr',
 405                                 'File %r not available at master' % source)
 406             # TODO: once BuildStep.start() gets rewritten to use
 407             # maybeDeferred, just re-raise the exception here.
 408             reactor.callLater(0, BuildStep.finished, self, FAILURE)
 409             return
 410         fileReader = _FileReader(fp)
 411
 412         # default arguments
 413         args = {
 414             'slavedest': slavedest,
 415             'maxsize': self.maxsize,
 416             'reader': fileReader,
 417             'blocksize': self.blocksize,
 418             'workdir': self._getWorkdir(),
 419             'mode': self.mode,
 420             }
 421
 422         self.cmd = StatusRemoteCommand('downloadFile', args)
 423         d = self.runCommand(self.cmd)
 424         d.addCallback(self.finished).addErrback(self.failed)
 425