From 4573cc224e7f8fc9df144e5c5ca33f86cc935cdf Mon Sep 17 00:00:00 2001 From: =?utf8?q?Ren=C3=A9=20M=C3=BCller?= Date: Sun, 15 Feb 2009 18:14:45 -0500 Subject: [PATCH] (refs #393) add DirectoryUpload --- buildbot/slave/commands.py | 96 +++++++++++- buildbot/steps/transfer.py | 155 +++++++++++++++++++ buildbot/test/test_transfer.py | 327 ++++++++++++++++++++++++++++++++++++++--- docs/buildbot.texinfo | 24 +++ 4 files changed, 578 insertions(+), 24 deletions(-) diff --git a/buildbot/slave/commands.py b/buildbot/slave/commands.py index 1199e55..940169b 100644 --- a/buildbot/slave/commands.py +++ b/buildbot/slave/commands.py @@ -15,7 +15,7 @@ from buildbot.slave.registry import registerSlaveCommand # this used to be a CVS $-style "Revision" auto-updated keyword, but since I # moved to Darcs as the primary repository, this is updated manually each # time this file is changed. The last cvs_ver that was here was 1.51 . -command_version = "2.5" +command_version = "2.6" # version history: # >=1.17: commands are interruptable @@ -37,6 +37,7 @@ command_version = "2.5" # >= 2.3: added bzr (release 0.7.6) # >= 2.4: Git understands 'revision' and branches # >= 2.5: workaround added for remote 'hg clone --rev REV' when hg<0.9.2 +# >= 2.6: added uploadDirectory class CommandInterrupted(Exception): pass @@ -864,6 +865,99 @@ class SlaveFileUploadCommand(Command): registerSlaveCommand("uploadFile", SlaveFileUploadCommand, command_version) +class SlaveDirectoryUploadCommand(Command): + """ + Upload a directory from slave to build master + Arguments: + + - ['workdir']: base directory to use + - ['slavesrc']: name of the slave-side directory to read from + - ['writer']: RemoteReference to a transfer._DirectoryWriter object + - ['maxsize']: max size (in bytes) of file to write + - ['blocksize']: max size for each data block + """ + debug = True + + def setup(self, args): + self.workdir = args['workdir'] + self.dirname = args['slavesrc'] + self.writer = args['writer'] + self.remaining = args['maxsize'] + self.blocksize = args['blocksize'] + self.stderr = None + self.rc = 0 + + def start(self): + if self.debug: + log.msg('SlaveDirectoryUploadCommand started') + + # create some lists with all files and directories + foundFiles = [] + foundDirs = [] + + self.baseRoot = os.path.join(self.builder.basedir, + self.workdir, + os.path.expanduser(self.dirname)) + if self.debug: + log.msg("baseRoot: %r" % self.baseRoot) + + for root, dirs, files in os.walk(self.baseRoot): + tempRoot = root + relRoot = '' + while (tempRoot != self.baseRoot): + tempRoot, tempRelRoot = os.path.split(tempRoot) + relRoot = os.path.join(tempRelRoot, relRoot) + for name in files: + foundFiles.append(os.path.join(relRoot, name)) + for directory in dirs: + foundDirs.append(os.path.join(relRoot, directory)) + + if self.debug: + log.msg("foundDirs: %s" % (str(foundDirs))) + log.msg("foundFiles: %s" % (str(foundFiles))) + + # create all directories on the master, to catch also empty ones + for dirname in foundDirs: + self.writer.callRemote("createdir", dirname) + + for filename in foundFiles: + self._writeFile(filename) + + return None + + def _writeFile(self, filename): + """Write a file to the remote writer""" + + log.msg("_writeFile: %r" % (filename)) + self.writer.callRemote('open', filename) + data = open(os.path.join(self.baseRoot, filename), "r").read() + self.writer.callRemote('write', data) + self.writer.callRemote('close') + return None + + def interrupt(self): + if self.debug: + log.msg('interrupted') + if self.interrupted: + return + if self.stderr is None: + self.stderr = 'Upload of %r interrupted' % self.path + self.rc = 1 + self.interrupted = True + # the next _writeBlock call will notice the .interrupted flag + + def finished(self, res): + if self.debug: + log.msg('finished: stderr=%r, rc=%r' % (self.stderr, self.rc)) + if self.stderr is None: + self.sendStatus({'rc': self.rc}) + else: + self.sendStatus({'stderr': self.stderr, 'rc': self.rc}) + return res + +registerSlaveCommand("uploadDirectory", SlaveDirectoryUploadCommand, command_version) + + class SlaveFileDownloadCommand(Command): """ Download a file from master to slave diff --git a/buildbot/steps/transfer.py b/buildbot/steps/transfer.py index 435c281..b53e409 100644 --- a/buildbot/steps/transfer.py +++ b/buildbot/steps/transfer.py @@ -59,6 +59,74 @@ class _FileWriter(pb.Referenceable): os.unlink(self.destfile) +class _DirectoryWriter(pb.Referenceable): + """ + Helper class that acts as a directory-object with write access + """ + + def __init__(self, destroot, maxsize, mode): + self.destroot = destroot + # Create missing directories. + self.destroot = os.path.abspath(self.destroot) + if not os.path.exists(self.destroot): + os.makedirs(self.destroot) + + self.fp = None + self.mode = mode + self.maxsize = maxsize + + def remote_createdir(self, dirname): + # This function is needed to transfer empty directories. + dirname = os.path.join(self.destroot, dirname) + dirname = os.path.abspath(dirname) + if not os.path.exists(dirname): + os.makedirs(dirname) + + def remote_open(self, destfile): + # Create missing directories. + destfile = os.path.join(self.destroot, destfile) + destfile = os.path.abspath(destfile) + dirname = os.path.dirname(destfile) + if not os.path.exists(dirname): + os.makedirs(dirname) + + self.fp = open(destfile, "wb") + if self.mode is not None: + os.chmod(destfile, self.mode) + self.remaining = self.maxsize + + def remote_write(self, data): + """ + Called from remote slave to write L{data} to L{fp} within boundaries + of L{maxsize} + + @type data: C{string} + @param data: String of data to write + """ + if self.remaining is not None: + if len(data) > self.remaining: + data = data[:self.remaining] + self.fp.write(data) + self.remaining = self.remaining - len(data) + else: + self.fp.write(data) + + def remote_close(self): + """ + Called by remote slave to state that no more data will be transfered + """ + if self.fp: + self.fp.close() + self.fp = None + + def __del__(self): + # unclean shutdown, the file is probably truncated, so delete it + # altogether rather than deliver a corrupted file + fp = getattr(self, "fp", None) + if fp: + fp.close() + + class StatusRemoteCommand(RemoteCommand): def __init__(self, remote_command, args): RemoteCommand.__init__(self, remote_command, args) @@ -181,6 +249,93 @@ class FileUpload(_TransferBuildStep): return BuildStep.finished(self, FAILURE) +class DirectoryUpload(BuildStep): + """ + Build step to transfer a directory from the slave to the master. + + arguments: + + - ['slavesrc'] name of source directory at slave, relative to workdir + - ['masterdest'] name of destination directory at master + - ['workdir'] string with slave working directory relative to builder + base dir, default 'build' + - ['maxsize'] maximum size of each file, default None (=unlimited) + - ['blocksize'] maximum size of each block being transfered + - ['mode'] file access mode for the resulting master-side file. + The default (=None) is to leave it up to the umask of + the buildmaster process. + + """ + + name = 'upload' + + def __init__(self, slavesrc, masterdest, + workdir="build", maxsize=None, blocksize=16*1024, mode=None, + **buildstep_kwargs): + BuildStep.__init__(self, **buildstep_kwargs) + self.addFactoryArguments(slavesrc=slavesrc, + masterdest=masterdest, + workdir=workdir, + maxsize=maxsize, + blocksize=blocksize, + mode=mode, + ) + + self.slavesrc = slavesrc + self.masterdest = masterdest + self.workdir = workdir + self.maxsize = maxsize + self.blocksize = blocksize + assert isinstance(mode, (int, type(None))) + self.mode = mode + + def start(self): + version = self.slaveVersion("uploadDirectory") + properties = self.build.getProperties() + + if not version: + m = "slave is too old, does not know about uploadDirectory" + raise BuildSlaveTooOldError(m) + + source = properties.render(self.slavesrc) + masterdest = properties.render(self.masterdest) + # we rely upon the fact that the buildmaster runs chdir'ed into its + # basedir to make sure that relative paths in masterdest are expanded + # properly. TODO: maybe pass the master's basedir all the way down + # into the BuildStep so we can do this better. + masterdest = os.path.expanduser(masterdest) + log.msg("DirectoryUpload started, from slave %r to master %r" + % (source, masterdest)) + + self.step_status.setColor('yellow') + self.step_status.setText(['uploading', os.path.basename(source)]) + + # we use maxsize to limit the amount of data on both sides + dirWriter = _DirectoryWriter(masterdest, self.maxsize, self.mode) + + # default arguments + args = { + 'slavesrc': source, + 'workdir': self.workdir, + 'writer': dirWriter, + 'maxsize': self.maxsize, + 'blocksize': self.blocksize, + } + + self.cmd = StatusRemoteCommand('uploadDirectory', args) + d = self.runCommand(self.cmd) + d.addCallback(self.finished).addErrback(self.failed) + + def finished(self, result): + if self.cmd.stderr != '': + self.addCompleteLog('stderr', self.cmd.stderr) + + if self.cmd.rc is None or self.cmd.rc == 0: + self.step_status.setColor('green') + return BuildStep.finished(self, SUCCESS) + self.step_status.setColor('red') + return BuildStep.finished(self, FAILURE) + diff --git a/buildbot/test/test_transfer.py b/buildbot/test/test_transfer.py index d090f24..c85c630 100644 --- a/buildbot/test/test_transfer.py +++ b/buildbot/test/test_transfer.py @@ -4,7 +4,7 @@ import os from stat import ST_MODE from twisted.trial import unittest from buildbot.process.buildstep import WithProperties -from buildbot.steps.transfer import FileUpload, FileDownload +from buildbot.steps.transfer import FileUpload, FileDownload, DirectoryUpload from buildbot.test.runutils import StepTester from buildbot.status.builder import SUCCESS, FAILURE @@ -12,7 +12,7 @@ from buildbot.status.builder import SUCCESS, FAILURE # catch and wrap them. If the LocalAsRemote wrapper were a proper membrane, # we wouldn't have to do this. -class Upload(StepTester, unittest.TestCase): +class UploadFile(StepTester, unittest.TestCase): def filterArgs(self, args): if "writer" in args: @@ -20,8 +20,8 @@ class Upload(StepTester, unittest.TestCase): return args def testSuccess(self): - self.slavebase = "Upload.testSuccess.slave" - self.masterbase = "Upload.testSuccess.master" + self.slavebase = "UploadFile.testSuccess.slave" + self.masterbase = "UploadFile.testSuccess.master" sb = self.makeSlaveBuilder() os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase, "build")) @@ -58,8 +58,8 @@ class Upload(StepTester, unittest.TestCase): return d def testMaxsize(self): - self.slavebase = "Upload.testMaxsize.slave" - self.masterbase = "Upload.testMaxsize.master" + self.slavebase = "UploadFile.testMaxsize.slave" + self.masterbase = "UploadFile.testMaxsize.master" sb = self.makeSlaveBuilder() os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase, "build")) @@ -94,8 +94,8 @@ class Upload(StepTester, unittest.TestCase): return d def testMode(self): - self.slavebase = "Upload.testMode.slave" - self.masterbase = "Upload.testMode.master" + self.slavebase = "UploadFile.testMode.slave" + self.masterbase = "UploadFile.testMode.master" sb = self.makeSlaveBuilder() os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase, "build")) @@ -134,8 +134,8 @@ class Upload(StepTester, unittest.TestCase): return d def testMissingFile(self): - self.slavebase = "Upload.testMissingFile.slave" - self.masterbase = "Upload.testMissingFile.master" + self.slavebase = "UploadFile.testMissingFile.slave" + self.masterbase = "UploadFile.testMissingFile.master" sb = self.makeSlaveBuilder() step = self.makeStep(FileUpload, slavesrc="MISSING.txt", @@ -155,8 +155,8 @@ class Upload(StepTester, unittest.TestCase): return d def testLotsOfBlocks(self): - self.slavebase = "Upload.testLotsOfBlocks.slave" - self.masterbase = "Upload.testLotsOfBlocks.master" + self.slavebase = "UploadFile.testLotsOfBlocks.slave" + self.masterbase = "UploadFile.testLotsOfBlocks.master" sb = self.makeSlaveBuilder() os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase, "build")) @@ -239,7 +239,7 @@ class Upload(StepTester, unittest.TestCase): self.failUnlessEqual(step._getWorkdir(), "build.1") -class Download(StepTester, unittest.TestCase): +class DownloadFile(StepTester, unittest.TestCase): def filterArgs(self, args): if "reader" in args: @@ -247,8 +247,8 @@ class Download(StepTester, unittest.TestCase): return args def testSuccess(self): - self.slavebase = "Download.testSuccess.slave" - self.masterbase = "Download.testSuccess.master" + self.slavebase = "DownloadFile.testSuccess.slave" + self.masterbase = "DownloadFile.testSuccess.master" sb = self.makeSlaveBuilder() os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase, "build")) @@ -277,8 +277,8 @@ class Download(StepTester, unittest.TestCase): return d def testMaxsize(self): - self.slavebase = "Download.testMaxsize.slave" - self.masterbase = "Download.testMaxsize.master" + self.slavebase = "DownloadFile.testMaxsize.slave" + self.masterbase = "DownloadFile.testMaxsize.master" sb = self.makeSlaveBuilder() os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase, "build")) @@ -310,8 +310,8 @@ class Download(StepTester, unittest.TestCase): return d def testMode(self): - self.slavebase = "Download.testMode.slave" - self.masterbase = "Download.testMode.master" + self.slavebase = "DownloadFile.testMode.slave" + self.masterbase = "DownloadFile.testMode.master" sb = self.makeSlaveBuilder() os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase, "build")) @@ -346,8 +346,8 @@ class Download(StepTester, unittest.TestCase): return d def testMissingFile(self): - self.slavebase = "Download.testMissingFile.slave" - self.masterbase = "Download.testMissingFile.master" + self.slavebase = "DownloadFile.testMissingFile.slave" + self.masterbase = "DownloadFile.testMissingFile.master" sb = self.makeSlaveBuilder() os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase, "build")) @@ -373,8 +373,8 @@ class Download(StepTester, unittest.TestCase): return d def testLotsOfBlocks(self): - self.slavebase = "Download.testLotsOfBlocks.slave" - self.masterbase = "Download.testLotsOfBlocks.master" + self.slavebase = "DownloadFile.testLotsOfBlocks.slave" + self.masterbase = "DownloadFile.testLotsOfBlocks.master" sb = self.makeSlaveBuilder() os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase, "build")) @@ -432,6 +432,287 @@ class Download(StepTester, unittest.TestCase): +class UploadDirectory(StepTester, unittest.TestCase): + + def filterArgs(self, args): + if "writer" in args: + args["writer"] = self.wrap(args["writer"]) + return args + + def testSuccess(self): + self.slavebase = "UploadDirectory.testSuccess.slave" + self.masterbase = "UploadDirectory.testSuccess.master" + sb = self.makeSlaveBuilder() + os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase, + "build")) + # the buildmaster normally runs chdir'ed into masterbase, so uploaded + # files will appear there. Under trial, we're chdir'ed into + # _trial_temp instead, so use a different masterdest= to keep the + # uploaded file in a test-local directory + masterdest = os.path.join(self.masterbase, "dest_dir") + step = self.makeStep(DirectoryUpload, + slavesrc="source_dir", + masterdest=masterdest) + slavesrc = os.path.join(self.slavebase, + self.slavebuilderbase, + "build", + "source_dir") + dircount = 5 + content = [] + content.append("this is one source file\n" * 1000) + content.append("this is a second source file\n" * 978) + content.append("this is a third source file\n" * 473) + os.mkdir(slavesrc) + for i in range(dircount): + os.mkdir(os.path.join(slavesrc, "d%i" % (i))) + for j in range(dircount): + curdir = os.path.join("d%i" % (i), "e%i" % (j)) + os.mkdir(os.path.join(slavesrc, curdir)) + for h in range(3): + open(os.path.join(slavesrc, curdir, "file%i" % (h)), "w").write(content[h]) + for j in range(dircount): + #empty dirs, must be uploaded too + curdir = os.path.join("d%i" % (i), "f%i" % (j)) + os.mkdir(os.path.join(slavesrc, curdir)) + + d = self.runStep(step) + def _checkUpload(results): + step_status = step.step_status + #l = step_status.getLogs() + #if l: + # logtext = l[0].getText() + # print logtext + self.failUnlessEqual(results, SUCCESS) + self.failUnless(os.path.exists(masterdest)) + for i in range(dircount): + for j in range(dircount): + curdir = os.path.join("d%i" % (i), "e%i" % (j)) + self.failUnless(os.path.exists(os.path.join(masterdest, curdir))) + for h in range(3): + masterdest_contents = open(os.path.join(masterdest, curdir, "file%i" % (h)), "r").read() + self.failUnlessEqual(masterdest_contents, content[h]) + for j in range(dircount): + curdir = os.path.join("d%i" % (i), "f%i" % (j)) + self.failUnless(os.path.exists(os.path.join(masterdest, curdir))) + d.addCallback(_checkUpload) + return d + + def testOneEmptyDir(self): + self.slavebase = "UploadDirectory.testOneEmptyDir.slave" + self.masterbase = "UploadDirectory.testOneEmptyDir.master" + sb = self.makeSlaveBuilder() + os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase, + "build")) + # the buildmaster normally runs chdir'ed into masterbase, so uploaded + # files will appear there. Under trial, we're chdir'ed into + # _trial_temp instead, so use a different masterdest= to keep the + # uploaded file in a test-local directory + masterdest = os.path.join(self.masterbase, "dest_dir") + step = self.makeStep(DirectoryUpload, + slavesrc="source_dir", + masterdest=masterdest) + slavesrc = os.path.join(self.slavebase, + self.slavebuilderbase, + "build", + "source_dir") + os.mkdir(slavesrc) + + d = self.runStep(step) + def _checkUpload(results): + step_status = step.step_status + #l = step_status.getLogs() + #if l: + # logtext = l[0].getText() + # print logtext + self.failUnlessEqual(results, SUCCESS) + self.failUnless(os.path.exists(masterdest)) + d.addCallback(_checkUpload) + return d + + def testManyEmptyDirs(self): + self.slavebase = "UploadDirectory.testManyEmptyDirs.slave" + self.masterbase = "UploadDirectory.testManyEmptyDirs.master" + sb = self.makeSlaveBuilder() + os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase, + "build")) + # the buildmaster normally runs chdir'ed into masterbase, so uploaded + # files will appear there. Under trial, we're chdir'ed into + # _trial_temp instead, so use a different masterdest= to keep the + # uploaded file in a test-local directory + masterdest = os.path.join(self.masterbase, "dest_dir") + step = self.makeStep(DirectoryUpload, + slavesrc="source_dir", + masterdest=masterdest) + slavesrc = os.path.join(self.slavebase, + self.slavebuilderbase, + "build", + "source_dir") + dircount = 25 + os.mkdir(slavesrc) + for i in range(dircount): + os.mkdir(os.path.join(slavesrc, "d%i" % (i))) + for j in range(dircount): + curdir = os.path.join("d%i" % (i), "e%i" % (j)) + os.mkdir(os.path.join(slavesrc, curdir)) + curdir = os.path.join("d%i" % (i), "f%i" % (j)) + os.mkdir(os.path.join(slavesrc, curdir)) + + d = self.runStep(step) + def _checkUpload(results): + step_status = step.step_status + #l = step_status.getLogs() + #if l: + # logtext = l[0].getText() + # print logtext + self.failUnlessEqual(results, SUCCESS) + self.failUnless(os.path.exists(masterdest)) + for i in range(dircount): + for j in range(dircount): + curdir = os.path.join("d%i" % (i), "e%i" % (j)) + self.failUnless(os.path.exists(os.path.join(masterdest, curdir))) + curdir = os.path.join("d%i" % (i), "f%i" % (j)) + self.failUnless(os.path.exists(os.path.join(masterdest, curdir))) + d.addCallback(_checkUpload) + return d + + def testOneDirOneFile(self): + self.slavebase = "UploadDirectory.testOneDirOneFile.slave" + self.masterbase = "UploadDirectory.testOneDirOneFile.master" + sb = self.makeSlaveBuilder() + os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase, + "build")) + # the buildmaster normally runs chdir'ed into masterbase, so uploaded + # files will appear there. Under trial, we're chdir'ed into + # _trial_temp instead, so use a different masterdest= to keep the + # uploaded file in a test-local directory + masterdest = os.path.join(self.masterbase, "dest_dir") + step = self.makeStep(DirectoryUpload, + slavesrc="source_dir", + masterdest=masterdest) + slavesrc = os.path.join(self.slavebase, + self.slavebuilderbase, + "build", + "source_dir") + os.mkdir(slavesrc) + content = "this is one source file\n" * 1000 + open(os.path.join(slavesrc, "srcfile"), "w").write(content) + + d = self.runStep(step) + def _checkUpload(results): + step_status = step.step_status + #l = step_status.getLogs() + #if l: + # logtext = l[0].getText() + # print logtext + self.failUnlessEqual(results, SUCCESS) + self.failUnless(os.path.exists(masterdest)) + masterdest_contents = open(os.path.join(masterdest, "srcfile"), "r").read() + self.failUnlessEqual(masterdest_contents, content) + d.addCallback(_checkUpload) + return d + + def testOneDirManyFiles(self): + self.slavebase = "UploadDirectory.testOneDirManyFile.slave" + self.masterbase = "UploadDirectory.testOneDirManyFile.master" + sb = self.makeSlaveBuilder() + os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase, + "build")) + # the buildmaster normally runs chdir'ed into masterbase, so uploaded + # files will appear there. Under trial, we're chdir'ed into + # _trial_temp instead, so use a different masterdest= to keep the + # uploaded file in a test-local directory + masterdest = os.path.join(self.masterbase, "dest_dir") + step = self.makeStep(DirectoryUpload, + slavesrc="source_dir", + masterdest=masterdest) + slavesrc = os.path.join(self.slavebase, + self.slavebuilderbase, + "build", + "source_dir") + filecount = 20 + os.mkdir(slavesrc) + content = [] + content.append("this is one source file\n" * 1000) + content.append("this is a second source file\n" * 978) + content.append("this is a third source file\n" * 473) + for i in range(3): + for j in range(filecount): + open(os.path.join(slavesrc, "srcfile%i_%i" % (i, j)), "w").write(content[i]) + + d = self.runStep(step) + def _checkUpload(results): + step_status = step.step_status + #l = step_status.getLogs() + #if l: + # logtext = l[0].getText() + # print logtext + self.failUnlessEqual(results, SUCCESS) + self.failUnless(os.path.exists(masterdest)) + for i in range(3): + for j in range(filecount): + masterdest_contents = open(os.path.join(masterdest, "srcfile%i_%i" % (i, j)), "r").read() + self.failUnlessEqual(masterdest_contents, content[i]) + d.addCallback(_checkUpload) + return d + + def testManyDirsManyFiles(self): + self.slavebase = "UploadDirectory.testManyDirsManyFile.slave" + self.masterbase = "UploadDirectory.testManyDirsManyFile.master" + sb = self.makeSlaveBuilder() + os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase, + "build")) + # the buildmaster normally runs chdir'ed into masterbase, so uploaded + # files will appear there. Under trial, we're chdir'ed into + # _trial_temp instead, so use a different masterdest= to keep the + # uploaded file in a test-local directory + masterdest = os.path.join(self.masterbase, "dest_dir") + step = self.makeStep(DirectoryUpload, + slavesrc="source_dir", + masterdest=masterdest) + slavesrc = os.path.join(self.slavebase, + self.slavebuilderbase, + "build", + "source_dir") + dircount = 10 + os.mkdir(slavesrc) + for i in range(dircount): + os.mkdir(os.path.join(slavesrc, "d%i" % (i))) + for j in range(dircount): + curdir = os.path.join("d%i" % (i), "e%i" % (j)) + os.mkdir(os.path.join(slavesrc, curdir)) + curdir = os.path.join("d%i" % (i), "f%i" % (j)) + os.mkdir(os.path.join(slavesrc, curdir)) + + filecount = 5 + content = [] + content.append("this is one source file\n" * 1000) + content.append("this is a second source file\n" * 978) + content.append("this is a third source file\n" * 473) + for i in range(dircount): + for j in range(dircount): + for k in range(3): + for l in range(filecount): + open(os.path.join(slavesrc, "d%i" % (i), "e%i" % (j), "srcfile%i_%i" % (k, l)), "w").write(content[k]) + + d = self.runStep(step) + def _checkUpload(results): + step_status = step.step_status + #l = step_status.getLogs() + #if l: + # logtext = l[0].getText() + # print logtext + self.failUnlessEqual(results, SUCCESS) + self.failUnless(os.path.exists(masterdest)) + for i in range(dircount): + for j in range(dircount): + for k in range(3): + for l in range(filecount): + masterdest_contents = open(os.path.join(masterdest, "d%i" % (i), "e%i" % (j), "srcfile%i_%i" % (k, l)), "r").read() + self.failUnlessEqual(masterdest_contents, content[k]) + d.addCallback(_checkUpload) + return d + + # TODO: # test relative paths, ~/paths # need to implement expanduser() for slave-side diff --git a/docs/buildbot.texinfo b/docs/buildbot.texinfo index 73f2fa4..29ddbaf 100644 --- a/docs/buildbot.texinfo +++ b/docs/buildbot.texinfo @@ -5680,6 +5680,7 @@ f.addStep(PyFlakes(command=["pyflakes", "src"])) @cindex File Transfer @bsindex buildbot.steps.transfer.FileUpload @bsindex buildbot.steps.transfer.FileDownload +@bsindex buildbot.steps.transfer.DirectoryUpload Most of the work involved in a build will take place on the buildslave. But occasionally it is useful to do some work on the @@ -5759,6 +5760,29 @@ umask tends to be fairly restrictive, but at least on the buildslave you can make it less restrictive with a --umask command-line option at creation time (@pxref{Buildslave Options}). +@subheading Transfering Directories + +To transfer complete directories from the buildslave to the master, there +is a BuildStep named @code{DirectoryUpload}. It works like @code{FileUpload}, +just for directories. However it does not support the @code{maxsize}, +@code{blocksize} and @code{mode} arguments. As an example, let's assume an +generated project documentation, which consists of many files (like the output +of doxygen or epydoc). We want to move the entire documentation to the +buildmaster, into a @code{~/public_html/docs} directory. On the slave-side +the directory can be found under @code{docs}: + +@example +from buildbot.steps.shell import ShellCommand +from buildbot.steps.transfer import DirectoryUpload + +f.addStep(ShellCommand(command=["make", "docs"])) +f.addStep(DirectoryUpload(slavesrc="docs", + masterdest="~/public_html/docs")) +@end example + +The DirectoryUpload step will create all necessary directories and +transfers empty directories, too. + @node Triggering Schedulers, Writing New BuildSteps, Transferring Files, Build Steps @subsection Triggering Schedulers -- 2.11.4.GIT