buildbot/test/test_transfer.py

   1 # -*- test-case-name: buildbot.test.test_transfer -*-
   2
   3 import os
   4 from stat import ST_MODE
   5 from twisted.trial import unittest
   6 from buildbot.steps.transfer import FileUpload, FileDownload
   7 from buildbot.test.runutils import StepTester
   8 from buildbot.status.builder import SUCCESS, FAILURE
   9
  10
  11 # these steps pass a pb.Referenceable inside their arguments, so we have to
  12 # catch and wrap them. If the LocalAsRemote wrapper were a proper membrane,
  13 # we wouldn't have to do this.
  14
  15 class Upload(StepTester, unittest.TestCase):
  16
  17     def filterArgs(self, args):
  18         if "writer" in args:
  19             args["writer"] = self.wrap(args["writer"])
  20         return args
  21
  22     def testSuccess(self):
  23         self.slavebase = "Upload.testSuccess.slave"
  24         self.masterbase = "Upload.testSuccess.master"
  25         sb = self.makeSlaveBuilder()
  26         os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase,
  27                               "build"))
  28         # the buildmaster normally runs chdir'ed into masterbase, so uploaded
  29         # files will appear there. Under trial, we're chdir'ed into
  30         # _trial_temp instead, so use a different masterdest= to keep the
  31         # uploaded file in a test-local directory
  32         masterdest = os.path.join(self.masterbase, "dest.text")
  33         step = self.makeStep(FileUpload,
  34                              slavesrc="source.txt",
  35                              masterdest=masterdest)
  36         slavesrc = os.path.join(self.slavebase,
  37                                 self.slavebuilderbase,
  38                                 "build",
  39                                 "source.txt")
  40         contents = "this is the source file\n" * 1000
  41         open(slavesrc, "w").write(contents)
  42         f = open(masterdest, "w")
  43         f.write("overwrite me\n")
  44         f.close()
  45
  46         d = self.runStep(step)
  47         def _checkUpload(results):
  48             step_status = step.step_status
  49             #l = step_status.getLogs()
  50             #if l:
  51             #    logtext = l[0].getText()
  52             #    print logtext
  53             self.failUnlessEqual(results, SUCCESS)
  54             self.failUnless(os.path.exists(masterdest))
  55             masterdest_contents = open(masterdest, "r").read()
  56             self.failUnlessEqual(masterdest_contents, contents)
  57         d.addCallback(_checkUpload)
  58         return d
  59
  60     def testMaxsize(self):
  61         self.slavebase = "Upload.testMaxsize.slave"
  62         self.masterbase = "Upload.testMaxsize.master"
  63         sb = self.makeSlaveBuilder()
  64         os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase,
  65                               "build"))
  66         masterdest = os.path.join(self.masterbase, "dest2.text")
  67         step = self.makeStep(FileUpload,
  68                              slavesrc="source.txt",
  69                              masterdest=masterdest,
  70                              maxsize=12345)
  71         slavesrc = os.path.join(self.slavebase,
  72                                 self.slavebuilderbase,
  73                                 "build",
  74                                 "source.txt")
  75         contents = "this is the source file\n" * 1000
  76         open(slavesrc, "w").write(contents)
  77         f = open(masterdest, "w")
  78         f.write("overwrite me\n")
  79         f.close()
  80
  81         d = self.runStep(step)
  82         def _checkUpload(results):
  83             step_status = step.step_status
  84             #l = step_status.getLogs()
  85             #if l:
  86             #    logtext = l[0].getText()
  87             #    print logtext
  88             self.failUnlessEqual(results, FAILURE)
  89             self.failUnless(os.path.exists(masterdest))
  90             masterdest_contents = open(masterdest, "r").read()
  91             self.failUnlessEqual(len(masterdest_contents), 12345)
  92             self.failUnlessEqual(masterdest_contents, contents[:12345])
  93         d.addCallback(_checkUpload)
  94         return d
  95
  96     def testMode(self):
  97         self.slavebase = "Upload.testMode.slave"
  98         self.masterbase = "Upload.testMode.master"
  99         sb = self.makeSlaveBuilder()
 100         os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase,
 101                               "build"))
 102         masterdest = os.path.join(self.masterbase, "dest3.text")
 103         step = self.makeStep(FileUpload,
 104                              slavesrc="source.txt",
 105                              masterdest=masterdest,
 106                              mode=0755)
 107         slavesrc = os.path.join(self.slavebase,
 108                                 self.slavebuilderbase,
 109                                 "build",
 110                                 "source.txt")
 111         contents = "this is the source file\n"
 112         open(slavesrc, "w").write(contents)
 113         f = open(masterdest, "w")
 114         f.write("overwrite me\n")
 115         f.close()
 116
 117         d = self.runStep(step)
 118         def _checkUpload(results):
 119             step_status = step.step_status
 120             #l = step_status.getLogs()
 121             #if l:
 122             #    logtext = l[0].getText()
 123             #    print logtext
 124             self.failUnlessEqual(results, SUCCESS)
 125             self.failUnless(os.path.exists(masterdest))
 126             masterdest_contents = open(masterdest, "r").read()
 127             self.failUnlessEqual(masterdest_contents, contents)
 128             # and with 0777 to ignore sticky bits
 129             dest_mode = os.stat(masterdest)[ST_MODE] & 0777
 130             self.failUnlessEqual(dest_mode, 0755,
 131                                  "target mode was %o, we wanted %o" %
 132                                  (dest_mode, 0755))
 133         d.addCallback(_checkUpload)
 134         return d
 135
 136     def testMissingFile(self):
 137         self.slavebase = "Upload.testMissingFile.slave"
 138         self.masterbase = "Upload.testMissingFile.master"
 139         sb = self.makeSlaveBuilder()
 140         step = self.makeStep(FileUpload,
 141                              slavesrc="MISSING.txt",
 142                              masterdest="dest.txt")
 143         masterdest = os.path.join(self.masterbase, "dest4.txt")
 144
 145         d = self.runStep(step)
 146         def _checkUpload(results):
 147             step_status = step.step_status
 148             self.failUnlessEqual(results, FAILURE)
 149             self.failIf(os.path.exists(masterdest))
 150             l = step_status.getLogs()
 151             logtext = l[0].getText().strip()
 152             self.failUnless(logtext.startswith("Cannot open file"))
 153             self.failUnless(logtext.endswith("for upload"))
 154         d.addCallback(_checkUpload)
 155         return d
 156
 157
 158
 159 class Download(StepTester, unittest.TestCase):
 160
 161     def filterArgs(self, args):
 162         if "reader" in args:
 163             args["reader"] = self.wrap(args["reader"])
 164         return args
 165
 166     def testSuccess(self):
 167         self.slavebase = "Download.testSuccess.slave"
 168         self.masterbase = "Download.testSuccess.master"
 169         sb = self.makeSlaveBuilder()
 170         os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase,
 171                               "build"))
 172         mastersrc = os.path.join(self.masterbase, "source.text")
 173         slavedest = os.path.join(self.slavebase,
 174                                  self.slavebuilderbase,
 175                                  "build",
 176                                  "dest.txt")
 177         step = self.makeStep(FileDownload,
 178                              mastersrc=mastersrc,
 179                              slavedest="dest.txt")
 180         contents = "this is the source file\n" * 1000  # 24kb, so two blocks
 181         open(mastersrc, "w").write(contents)
 182         f = open(slavedest, "w")
 183         f.write("overwrite me\n")
 184         f.close()
 185
 186         d = self.runStep(step)
 187         def _checkDownload(results):
 188             step_status = step.step_status
 189             self.failUnlessEqual(results, SUCCESS)
 190             self.failUnless(os.path.exists(slavedest))
 191             slavedest_contents = open(slavedest, "r").read()
 192             self.failUnlessEqual(slavedest_contents, contents)
 193         d.addCallback(_checkDownload)
 194         return d
 195
 196     def testMaxsize(self):
 197         self.slavebase = "Download.testMaxsize.slave"
 198         self.masterbase = "Download.testMaxsize.master"
 199         sb = self.makeSlaveBuilder()
 200         os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase,
 201                               "build"))
 202         mastersrc = os.path.join(self.masterbase, "source.text")
 203         slavedest = os.path.join(self.slavebase,
 204                                  self.slavebuilderbase,
 205                                  "build",
 206                                  "dest.txt")
 207         step = self.makeStep(FileDownload,
 208                              mastersrc=mastersrc,
 209                              slavedest="dest.txt",
 210                              maxsize=12345)
 211         contents = "this is the source file\n" * 1000  # 24kb, so two blocks
 212         open(mastersrc, "w").write(contents)
 213         f = open(slavedest, "w")
 214         f.write("overwrite me\n")
 215         f.close()
 216
 217         d = self.runStep(step)
 218         def _checkDownload(results):
 219             step_status = step.step_status
 220             # the file should be truncated, and the step a FAILURE
 221             self.failUnlessEqual(results, FAILURE)
 222             self.failUnless(os.path.exists(slavedest))
 223             slavedest_contents = open(slavedest, "r").read()
 224             self.failUnlessEqual(len(slavedest_contents), 12345)
 225             self.failUnlessEqual(slavedest_contents, contents[:12345])
 226         d.addCallback(_checkDownload)
 227         return d
 228
 229     def testMode(self):
 230         self.slavebase = "Download.testMode.slave"
 231         self.masterbase = "Download.testMode.master"
 232         sb = self.makeSlaveBuilder()
 233         os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase,
 234                               "build"))
 235         mastersrc = os.path.join(self.masterbase, "source.text")
 236         slavedest = os.path.join(self.slavebase,
 237                                  self.slavebuilderbase,
 238                                  "build",
 239                                  "dest.txt")
 240         step = self.makeStep(FileDownload,
 241                              mastersrc=mastersrc,
 242                              slavedest="dest.txt",
 243                              mode=0755)
 244         contents = "this is the source file\n"
 245         open(mastersrc, "w").write(contents)
 246         f = open(slavedest, "w")
 247         f.write("overwrite me\n")
 248         f.close()
 249
 250         d = self.runStep(step)
 251         def _checkDownload(results):
 252             step_status = step.step_status
 253             self.failUnlessEqual(results, SUCCESS)
 254             self.failUnless(os.path.exists(slavedest))
 255             slavedest_contents = open(slavedest, "r").read()
 256             self.failUnlessEqual(slavedest_contents, contents)
 257             # and with 0777 to ignore sticky bits
 258             dest_mode = os.stat(slavedest)[ST_MODE] & 0777
 259             self.failUnlessEqual(dest_mode, 0755,
 260                                  "target mode was %o, we wanted %o" %
 261                                  (dest_mode, 0755))
 262         d.addCallback(_checkDownload)
 263         return d
 264
 265     def testMissingFile(self):
 266         self.slavebase = "Download.testMissingFile.slave"
 267         self.masterbase = "Download.testMissingFile.master"
 268         sb = self.makeSlaveBuilder()
 269         os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase,
 270                               "build"))
 271         mastersrc = os.path.join(self.masterbase, "MISSING.text")
 272         slavedest = os.path.join(self.slavebase,
 273                                  self.slavebuilderbase,
 274                                  "build",
 275                                  "dest.txt")
 276         step = self.makeStep(FileDownload,
 277                              mastersrc=mastersrc,
 278                              slavedest="dest.txt")
 279
 280         d = self.runStep(step)
 281         def _checkDownload(results):
 282             step_status = step.step_status
 283             self.failUnlessEqual(results, FAILURE)
 284             self.failIf(os.path.exists(slavedest))
 285             l = step_status.getLogs()
 286             logtext = l[0].getText().strip()
 287             self.failUnless(logtext.endswith(" not available at master"))
 288         d.addCallbacks(_checkDownload)
 289
 290         return d
 291
 292
 293 # TODO:
 294 #  test relative paths, ~/paths
 295 #   need to implement expanduser() for slave-side
 296 #  test error message when master-side file is in a missing directory
 297 #  remove workdir= default?
 298