From c1f3bca74408500dfdd71d40dd144fb3e2e3ff9c Mon Sep 17 00:00:00 2001 From: warner Date: Sat, 9 Sep 2006 02:44:26 +0100 Subject: [PATCH] filetransfer: some stylistic changes --- buildbot/process/step.py | 224 ++++++++++++++++++++++++++++++++++++++++++++- buildbot/slave/commands.py | 213 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 436 insertions(+), 1 deletion(-) diff --git a/buildbot/process/step.py b/buildbot/process/step.py index e8ba469..18c0633 100644 --- a/buildbot/process/step.py +++ b/buildbot/process/step.py @@ -1,6 +1,6 @@ # -*- test-case-name: buildbot.test.test_steps -*- -import time, random, types, re, warnings +import os, time, random, types, re, warnings from email.Utils import formatdate from twisted.internet import reactor, defer, error @@ -1066,6 +1066,228 @@ class LoggingBuildStep(BuildStep): self.step_status.setText(self.getText(cmd, results)) self.step_status.setText2(self.maybeGetText2(cmd, results)) +# +# ================================================================= +# + +class FileIO(pb.Referenceable): + """ + Helper base class that acts as remote-accessible file-object + """ + + def __init__(self,fp): + self.fp = fp + + def remote_close(self): + """ + Called by remote slave to state that no more data will be transfered + """ + if self.fp is not None: + self.fp.close() + self.fp = None + +class FileWriter(FileIO): + """ + Helper class that acts as a file-object with write access + """ + + def __init__(self,fp, maxsize=None): + FileIO.__init__(self,fp) + self.maxsize = maxsize + + def remote_write(self,data): + """ + Called from remote slave to write L{data} to L{fp} within boundaries + of L{maxsize} + + @type data: C{string} + @param data: String of data to write + """ + if self.fp is not None: + if self.maxsize is not None: + if len(data) > self.maxsize: + data = data[:self.maxsize] + self.fp.write(data) + self.maxsize = self.maxsize - len(data) + else: + self.fp.write(data) + +class FileReader(FileIO): + """ + Helper class that acts as a file-object with read access + """ + + def remote_read(self,maxlength): + """ + Called from remote slave to read at most L{maxlength} bytes of data + + @type maxlength: C{integer} + @param maxlength: Maximum number of data bytes that can be returned + + @return: Data read from L{fp} + @rtype: C{string} of bytes read from file + """ + if self.fp is None: + return '' + + data = self.fp.read(maxlength) + return data + + +class StatusRemoteCommand(RemoteCommand): + def __init__(self, remote_command, args): + RemoteCommand.__init__(self, remote_command, args) + + self.rc = None + self.stderr = '' + + def remoteUpdate(self, update): + #log.msg('StatusRemoteCommand: update=%r' % update) + if 'rc' in update: + self.rc = update['rc'] + if 'stderr' in update: + self.stderr = self.stderr + update['stderr'] + '\n' + + +class FileUpload(BuildStep): + """ + Build step to transfer a file from the slave to the master. + + arguments: + + - ['slavesrc'] filename of source file at slave, relative to workdir + - ['masterdest'] filename of destination file at master + - ['workdir'] string with slave working directory relative to builder + base dir, default 'build' + - ['maxsize'] maximum size of the file, default None (=unlimited) + - ['blocksize'] maximum size of each block being transfered + + """ + + name = 'upload' + + def __init__(self, build, **kwargs): + buildstep_kwargs = {} + for k in kwargs.keys()[:]: + if k in BuildStep.parms: + buildstep_kwargs[k] = kwargs[k] + del kwargs[k] + BuildStep.__init__(self,build,**buildstep_kwargs) + + self.args = kwargs + self.fileWriter = None + + def start(self): + log.msg("FileUpload started, from slave %r to master %r" + % (self.args['slavesrc'],self.args['masterdest'])) + + self.step_status.setColor('yellow') + self.step_status.setText(['uploading', self.args['slavesrc']]) + + fp = open(self.args['masterdest'],'w') + self.fileWriter = FileWriter(fp) + + # default arguments + args = { + 'maxsize': None, + 'blocksize': 16*1024, + 'workdir': 'build', + } + args.update(self.args) + args['writer'] = self.fileWriter + + self.cmd = StatusRemoteCommand('uploadFile', args) + d = self.runCommand(self.cmd) + d.addCallback(self.finished).addErrback(self.failed) + + def finished(self,result): + if self.cmd.stderr != '': + self.addCompleteLog('stderr', self.cmd.stderr) + + self.fileWriter = None + + if self.cmd.rc is None or self.cmd.rc == 0: + self.step_status.setColor('green') + return BuildStep.finished(self,SUCCESS) + self.step_status.setColor('red') + return BuildStep.finished(self,FAILURE) + +class FileDownload(BuildStep): + """ + Build step to download a file + arguments: + + ['mastersrc'] filename of source file at master + ['slavedest'] filename of destination file at slave + ['workdir'] string with slave working directory relative to builder + base dir, default 'build' + ['maxsize'] maximum size of the file, default None (=unlimited) + ['blocksize'] maximum size of each block being transfered + + """ + + name = 'download' + + def __init__(self,build, **kwargs): + buildstep_kwargs = {} + for k in kwargs.keys()[:]: + if k in BuildStep.parms: + buildstep_kwargs[k] = kwargs[k] + del kwargs[k] + BuildStep.__init__(self,build,**buildstep_kwargs) + + self.args = kwargs + self.fileReader = None + + def start(self): + log.msg("FileDownload started, from master %r to slave %r" + % (self.args['mastersrc'],self.args['slavedest'])) + + self.step_status.setColor('yellow') + self.step_status.setText(['downloading', self.args['slavedest']]) + + # If file does not exist, bail out with an error + if not os.path.isfile(self.args['mastersrc']): + self.addCompleteLog('stderr', + 'File %r not available at master' % self.args['mastersrc']) + reactor.callLater(0, self.reportFail) + return + + # setup structures for reading the file + fp = open(self.args['mastersrc'],'r') + self.fileReader = FileReader(fp) + + a = self.args.copy() + a['reader'] = self.fileReader + + # add defaults for optional settings + for k,dv in [('maxsize',None),('blocksize',16*1024),('workdir','build')]: + if k not in a: + a[k] = dv + + self.cmd = StatusRemoteCommand('downloadFile', a) + d = self.runCommand(self.cmd) + d.addCallback(self.finished).addErrback(self.failed) + + def finished(self,result): + if self.cmd.stderr != '': + self.addCompleteLog('stderr', self.cmd.stderr) + + self.fileReader = None + + if self.cmd.rc is None or self.cmd.rc == 0: + self.step_status.setColor('green') + return BuildStep.finished(self,SUCCESS) + return self.reportFail() + + def reportFail(self): + self.step_status.setColor('red') + return BuildStep.finished(self,FAILURE) + +# +# ================================================================= +# + # -*- test-case-name: buildbot.test.test_properties -*- diff --git a/buildbot/slave/commands.py b/buildbot/slave/commands.py index 64a0f9e..62e7c45 100644 --- a/buildbot/slave/commands.py +++ b/buildbot/slave/commands.py @@ -666,6 +666,219 @@ class Command: return None + +class SlaveFileUploadCommand(Command): + """ + Upload a file from slave to build master + Arguments: + + - ['workdir']: directory to use + - ['slavesrc']: name of the file to upload to the buildmaster + - ['writer']: object for remote writing + - ['maxsize']: max size (in bytes) of file to write + - ['blocksize']: max size for one data block + + """ + + def setup(self,args): + self.workdir = args['workdir'] + self.filename = os.path.basename(args['slavesrc']) + self.writer = args['writer'] + self.maxsize = args['maxsize'] + self.blocksize = args['blocksize'] + self.stderr = None + self.rc = 0 + + self.debug = 0 + if self.debug: log.msg('SlaveFileUploadCommand started') + + # Open file + self.path = os.path.join(self.builder.basedir,self.workdir,self.filename) + try: + self.fp = open(self.path, 'r') + if self.debug: log.msg('Opened %r for upload' % self.path) + except: + self.fp = None + self.stderr = 'Cannot open file %r for upload' % self.path + self.rc = 1 + if self.debug: log.msg('Cannot open file %r for upload' % self.path) + + + def start(self): + self.cmd = defer.Deferred() + reactor.callLater(0, self._writeBlock) + + return self.cmd + + def _writeBlock(self): + """ + Write a block of data to the remote writer + """ + if self.interrupted or self.fp is None: + if self.debug: log.msg('SlaveFileUploadCommand._writeBlock(): end') + d = self.writer.callRemote('close') + d.addCallback(lambda _: self.finished()) + return + + length = self.blocksize + if self.maxsize is not None and length > self.maxsize: + length = self.maxsize + + if length <= 0: + if self.stderr is None: + self.stderr = 'Maximum filesize reached, truncating file %r' \ + % self.path + self.rc = 1 + data = '' + else: + data = self.fp.read(length) + + if self.debug: log.msg('SlaveFileUploadCommand._writeBlock(): '+ + 'allowed=%d readlen=%d' % (length,len(data))) + if len(data) == 0: + d = self.writer.callRemote('close') + d.addCallback(lambda _: self.finished()) + else: + if self.maxsize is not None: + self.maxsize = self.maxsize - len(data) + assert self.maxsize >= 0 + d = self.writer.callRemote('write',data) + d.addCallback(lambda _: self._writeBlock()) + + + def interrupt(self): + if self.debug: log.msg('interrupted') + if self.interrupted: + return + if self.stderr is None: + self.stderr = 'Upload of %r interrupted' % self.path + self.rc = 1 + self.interrupted = True + self.finished() + + + def finished(self): + if self.debug: log.msg('finished: stderr=%r, rc=%r' % (self.stderr,self.rc)) + if self.stderr is None: + self.sendStatus({'rc':self.rc}) + else: + self.sendStatus({'stderr':self.stderr, 'rc':self.rc}) + self.cmd.callback(0) + +registerSlaveCommand("uploadFile", SlaveFileUploadCommand, command_version) + + +class SlaveFileDownloadCommand(Command): + """ + Download a file from master to slave + Arguments: + + - ['workdir']: directory to use + - ['slavedest']: name of the file to upload to the buildmaster + - ['reader']: object for remote writing + - ['maxsize']: max size (in bytes) of file to write + - ['blocksize']: max size for one data block + + """ + + def setup(self,args): + self.workdir = args['workdir'] + self.filename = os.path.basename(args['slavedest']) + self.reader = args['reader'] + self.maxsize = args['maxsize'] + self.blocksize = args['blocksize'] + self.stderr = None + self.rc = 0 + + self.debug = 0 + if self.debug: log.msg('SlaveFileDownloadCommand started') + + # Open file + self.path = os.path.join(self.builder.basedir,self.workdir,self.filename) + try: + self.fp = open(self.path, 'w') + if self.debug: log.msg('Opened %r for download' % self.path) + except: + self.fp = None + self.stderr = 'Cannot open file %r for download' % self.path + self.rc = 1 + if self.debug: + log.msg('Cannot open file %r for download' % self.path) + + + def start(self): + self.cmd = defer.Deferred() + reactor.callLater(0, self._readBlock) + + return self.cmd + + def _readBlock(self): + """ + Read a block of data from the remote reader + """ + if self.interrupted or self.fp is None: + if self.debug: log.msg('SlaveFileDownloadCommand._readBlock(): end') + d = self.reader.callRemote('close') + d.addCallback(lambda _: self.finished()) + return + + length = self.blocksize + if self.maxsize is not None and length > self.maxsize: + length = self.maxsize + + if length <= 0: + if self.stderr is None: + self.stderr = 'Maximum filesize reached, truncating file %r' \ + % self.path + self.rc = 1 + d = self.reader.callRemote('close') + d.addCallback(lambda _: self.finished()) + else: + d = self.reader.callRemote('read', length) + d.addCallback(self._writeData) + + def _writeData(self,data): + if self.debug: log.msg('SlaveFileDownloadCommand._readBlock(): '+ + 'readlen=%d' % len(data)) + if len(data) == 0: + d = self.reader.callRemote('close') + d.addCallback(lambda _: self.finished()) + else: + if self.maxsize is not None: + self.maxsize = self.maxsize - len(data) + assert self.maxsize >= 0 + self.fp.write(data) + self._readBlock() # setup call back for next block (or finish) + + + def interrupt(self): + if self.debug: log.msg('interrupted') + if self.interrupted: + return + if self.stderr is None: + self.stderr = 'Download of %r interrupted' % self.path + self.rc = 1 + self.interrupted = True + self.finished() + + + def finished(self): + if self.fp is not None: + self.fp.close() + + if self.debug: log.msg('finished: stderr=%r, rc=%r' + % (self.stderr,self.rc)) + if self.stderr is None: + self.sendStatus({'rc':self.rc}) + else: + self.sendStatus({'stderr':self.stderr, 'rc':self.rc}) + self.cmd.callback(0) + + +registerSlaveCommand("downloadFile", SlaveFileDownloadCommand, command_version) + + + class SlaveShellCommand(Command): """This is a Command which runs a shell command. The args dict contains the following keys: -- 2.11.4.GIT