buildbot/test/test_transfer.py

   1 # -*- test-case-name: buildbot.test.test_transfer -*-
   2
   3 import os
   4 from stat import ST_MODE
   5 from twisted.trial import unittest
   6 from buildbot.process.buildstep import WithProperties
   7 from buildbot.steps.transfer import FileUpload, FileDownload, DirectoryUpload
   8 from buildbot.test.runutils import StepTester
   9 from buildbot.status.builder import SUCCESS, FAILURE
  10
  11 # these steps pass a pb.Referenceable inside their arguments, so we have to
  12 # catch and wrap them. If the LocalAsRemote wrapper were a proper membrane,
  13 # we wouldn't have to do this.
  14
  15 class UploadFile(StepTester, unittest.TestCase):
  16
  17     def filterArgs(self, args):
  18         if "writer" in args:
  19             args["writer"] = self.wrap(args["writer"])
  20         return args
  21
  22     def testSuccess(self):
  23         self.slavebase = "UploadFile.testSuccess.slave"
  24         self.masterbase = "UploadFile.testSuccess.master"
  25         sb = self.makeSlaveBuilder()
  26         os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase,
  27                               "build"))
  28         # the buildmaster normally runs chdir'ed into masterbase, so uploaded
  29         # files will appear there. Under trial, we're chdir'ed into
  30         # _trial_temp instead, so use a different masterdest= to keep the
  31         # uploaded file in a test-local directory
  32         masterdest = os.path.join(self.masterbase, "dest.text")
  33         step = self.makeStep(FileUpload,
  34                              slavesrc="source.txt",
  35                              masterdest=masterdest)
  36         slavesrc = os.path.join(self.slavebase,
  37                                 self.slavebuilderbase,
  38                                 "build",
  39                                 "source.txt")
  40         contents = "this is the source file\n" * 1000
  41         open(slavesrc, "w").write(contents)
  42         f = open(masterdest, "w")
  43         f.write("overwrite me\n")
  44         f.close()
  45
  46         d = self.runStep(step)
  47         def _checkUpload(results):
  48             step_status = step.step_status
  49             #l = step_status.getLogs()
  50             #if l:
  51             #    logtext = l[0].getText()
  52             #    print logtext
  53             self.failUnlessEqual(results, SUCCESS)
  54             self.failUnless(os.path.exists(masterdest))
  55             masterdest_contents = open(masterdest, "r").read()
  56             self.failUnlessEqual(masterdest_contents, contents)
  57         d.addCallback(_checkUpload)
  58         return d
  59
  60     def testMaxsize(self):
  61         self.slavebase = "UploadFile.testMaxsize.slave"
  62         self.masterbase = "UploadFile.testMaxsize.master"
  63         sb = self.makeSlaveBuilder()
  64         os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase,
  65                               "build"))
  66         masterdest = os.path.join(self.masterbase, "dest2.text")
  67         step = self.makeStep(FileUpload,
  68                              slavesrc="source.txt",
  69                              masterdest=masterdest,
  70                              maxsize=12345)
  71         slavesrc = os.path.join(self.slavebase,
  72                                 self.slavebuilderbase,
  73                                 "build",
  74                                 "source.txt")
  75         contents = "this is the source file\n" * 1000
  76         open(slavesrc, "w").write(contents)
  77         f = open(masterdest, "w")
  78         f.write("overwrite me\n")
  79         f.close()
  80
  81         d = self.runStep(step)
  82         def _checkUpload(results):
  83             step_status = step.step_status
  84             #l = step_status.getLogs()
  85             #if l:
  86             #    logtext = l[0].getText()
  87             #    print logtext
  88             self.failUnlessEqual(results, FAILURE)
  89             self.failUnless(os.path.exists(masterdest))
  90             masterdest_contents = open(masterdest, "r").read()
  91             self.failUnlessEqual(len(masterdest_contents), 12345)
  92             self.failUnlessEqual(masterdest_contents, contents[:12345])
  93         d.addCallback(_checkUpload)
  94         return d
  95
  96     def testMode(self):
  97         self.slavebase = "UploadFile.testMode.slave"
  98         self.masterbase = "UploadFile.testMode.master"
  99         sb = self.makeSlaveBuilder()
 100         os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase,
 101                               "build"))
 102         masterdest = os.path.join(self.masterbase, "dest3.text")
 103         step = self.makeStep(FileUpload,
 104                              slavesrc="source.txt",
 105                              masterdest=masterdest,
 106                              mode=0755)
 107         slavesrc = os.path.join(self.slavebase,
 108                                 self.slavebuilderbase,
 109                                 "build",
 110                                 "source.txt")
 111         contents = "this is the source file\n"
 112         open(slavesrc, "w").write(contents)
 113         f = open(masterdest, "w")
 114         f.write("overwrite me\n")
 115         f.close()
 116
 117         d = self.runStep(step)
 118         def _checkUpload(results):
 119             step_status = step.step_status
 120             #l = step_status.getLogs()
 121             #if l:
 122             #    logtext = l[0].getText()
 123             #    print logtext
 124             self.failUnlessEqual(results, SUCCESS)
 125             self.failUnless(os.path.exists(masterdest))
 126             masterdest_contents = open(masterdest, "r").read()
 127             self.failUnlessEqual(masterdest_contents, contents)
 128             # and with 0777 to ignore sticky bits
 129             dest_mode = os.stat(masterdest)[ST_MODE] & 0777
 130             self.failUnlessEqual(dest_mode, 0755,
 131                                  "target mode was %o, we wanted %o" %
 132                                  (dest_mode, 0755))
 133         d.addCallback(_checkUpload)
 134         return d
 135
 136     def testMissingFile(self):
 137         self.slavebase = "UploadFile.testMissingFile.slave"
 138         self.masterbase = "UploadFile.testMissingFile.master"
 139         sb = self.makeSlaveBuilder()
 140         step = self.makeStep(FileUpload,
 141                              slavesrc="MISSING.txt",
 142                              masterdest="dest.txt")
 143         masterdest = os.path.join(self.masterbase, "dest4.txt")
 144
 145         d = self.runStep(step)
 146         def _checkUpload(results):
 147             step_status = step.step_status
 148             self.failUnlessEqual(results, FAILURE)
 149             self.failIf(os.path.exists(masterdest))
 150             l = step_status.getLogs()
 151             logtext = l[0].getText().strip()
 152             self.failUnless(logtext.startswith("Cannot open file"))
 153             self.failUnless(logtext.endswith("for upload"))
 154         d.addCallback(_checkUpload)
 155         return d
 156
 157     def testLotsOfBlocks(self):
 158         self.slavebase = "UploadFile.testLotsOfBlocks.slave"
 159         self.masterbase = "UploadFile.testLotsOfBlocks.master"
 160         sb = self.makeSlaveBuilder()
 161         os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase,
 162                               "build"))
 163         # the buildmaster normally runs chdir'ed into masterbase, so uploaded
 164         # files will appear there. Under trial, we're chdir'ed into
 165         # _trial_temp instead, so use a different masterdest= to keep the
 166         # uploaded file in a test-local directory
 167         masterdest = os.path.join(self.masterbase, "dest.text")
 168         step = self.makeStep(FileUpload,
 169                              slavesrc="source.txt",
 170                              masterdest=masterdest,
 171                              blocksize=15)
 172         slavesrc = os.path.join(self.slavebase,
 173                                 self.slavebuilderbase,
 174                                 "build",
 175                                 "source.txt")
 176         contents = "".join(["this is the source file #%d\n" % i
 177                             for i in range(1000)])
 178         open(slavesrc, "w").write(contents)
 179         f = open(masterdest, "w")
 180         f.write("overwrite me\n")
 181         f.close()
 182
 183         d = self.runStep(step)
 184         def _checkUpload(results):
 185             step_status = step.step_status
 186             #l = step_status.getLogs()
 187             #if l:
 188             #    logtext = l[0].getText()
 189             #    print logtext
 190             self.failUnlessEqual(results, SUCCESS)
 191             self.failUnless(os.path.exists(masterdest))
 192             masterdest_contents = open(masterdest, "r").read()
 193             self.failUnlessEqual(masterdest_contents, contents)
 194         d.addCallback(_checkUpload)
 195         return d
 196
 197     def testWorkdir(self):
 198         self.slavebase = "Upload.testWorkdir.slave"
 199         self.masterbase = "Upload.testWorkdir.master"
 200         sb = self.makeSlaveBuilder()
 201
 202         self.workdir = "mybuild"        # override default in StepTest
 203         full_workdir = os.path.join(
 204             self.slavebase, self.slavebuilderbase, self.workdir)
 205         os.mkdir(full_workdir)
 206
 207         masterdest = os.path.join(self.masterbase, "dest.txt")
 208
 209         step = self.makeStep(FileUpload,
 210                              slavesrc="source.txt",
 211                              masterdest=masterdest)
 212
 213         # Testing that the FileUpload's workdir is set when makeStep()
 214         # calls setDefaultWorkdir() is actually enough; carrying on and
 215         # making sure the upload actually succeeds is pure gravy.
 216         self.failUnlessEqual(self.workdir, step.workdir)
 217
 218         slavesrc = os.path.join(full_workdir, "source.txt")
 219         open(slavesrc, "w").write("upload me\n")
 220
 221         def _checkUpload(results):
 222             self.failUnlessEqual(results, SUCCESS)
 223             self.failUnless(os.path.isfile(masterdest))
 224
 225         d = self.runStep(step)
 226         d.addCallback(_checkUpload)
 227         return d
 228
 229     def testWithProperties(self):
 230         # test that workdir can be a WithProperties object
 231         self.slavebase = "Upload.testWithProperties.slave"
 232         self.masterbase = "Upload.testWithProperties.master"
 233         sb = self.makeSlaveBuilder()
 234
 235         step = self.makeStep(FileUpload,
 236                              slavesrc="src.txt",
 237                              masterdest="dest.txt")
 238         step.workdir = WithProperties("build.%s", "buildnumber")
 239
 240         self.failUnlessEqual(step._getWorkdir(), "build.1")
 241
 242 class DownloadFile(StepTester, unittest.TestCase):
 243
 244     def filterArgs(self, args):
 245         if "reader" in args:
 246             args["reader"] = self.wrap(args["reader"])
 247         return args
 248
 249     def testSuccess(self):
 250         self.slavebase = "DownloadFile.testSuccess.slave"
 251         self.masterbase = "DownloadFile.testSuccess.master"
 252         sb = self.makeSlaveBuilder()
 253         os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase,
 254                               "build"))
 255         mastersrc = os.path.join(self.masterbase, "source.text")
 256         slavedest = os.path.join(self.slavebase,
 257                                  self.slavebuilderbase,
 258                                  "build",
 259                                  "dest.txt")
 260         step = self.makeStep(FileDownload,
 261                              mastersrc=mastersrc,
 262                              slavedest="dest.txt")
 263         contents = "this is the source file\n" * 1000  # 24kb, so two blocks
 264         open(mastersrc, "w").write(contents)
 265         f = open(slavedest, "w")
 266         f.write("overwrite me\n")
 267         f.close()
 268
 269         d = self.runStep(step)
 270         def _checkDownload(results):
 271             step_status = step.step_status
 272             self.failUnlessEqual(results, SUCCESS)
 273             self.failUnless(os.path.exists(slavedest))
 274             slavedest_contents = open(slavedest, "r").read()
 275             self.failUnlessEqual(slavedest_contents, contents)
 276         d.addCallback(_checkDownload)
 277         return d
 278
 279     def testMaxsize(self):
 280         self.slavebase = "DownloadFile.testMaxsize.slave"
 281         self.masterbase = "DownloadFile.testMaxsize.master"
 282         sb = self.makeSlaveBuilder()
 283         os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase,
 284                               "build"))
 285         mastersrc = os.path.join(self.masterbase, "source.text")
 286         slavedest = os.path.join(self.slavebase,
 287                                  self.slavebuilderbase,
 288                                  "build",
 289                                  "dest.txt")
 290         step = self.makeStep(FileDownload,
 291                              mastersrc=mastersrc,
 292                              slavedest="dest.txt",
 293                              maxsize=12345)
 294         contents = "this is the source file\n" * 1000  # 24kb, so two blocks
 295         open(mastersrc, "w").write(contents)
 296         f = open(slavedest, "w")
 297         f.write("overwrite me\n")
 298         f.close()
 299
 300         d = self.runStep(step)
 301         def _checkDownload(results):
 302             step_status = step.step_status
 303             # the file should be truncated, and the step a FAILURE
 304             self.failUnlessEqual(results, FAILURE)
 305             self.failUnless(os.path.exists(slavedest))
 306             slavedest_contents = open(slavedest, "r").read()
 307             self.failUnlessEqual(len(slavedest_contents), 12345)
 308             self.failUnlessEqual(slavedest_contents, contents[:12345])
 309         d.addCallback(_checkDownload)
 310         return d
 311
 312     def testMode(self):
 313         self.slavebase = "DownloadFile.testMode.slave"
 314         self.masterbase = "DownloadFile.testMode.master"
 315         sb = self.makeSlaveBuilder()
 316         os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase,
 317                               "build"))
 318         mastersrc = os.path.join(self.masterbase, "source.text")
 319         slavedest = os.path.join(self.slavebase,
 320                                  self.slavebuilderbase,
 321                                  "build",
 322                                  "dest.txt")
 323         step = self.makeStep(FileDownload,
 324                              mastersrc=mastersrc,
 325                              slavedest="dest.txt",
 326                              mode=0755)
 327         contents = "this is the source file\n"
 328         open(mastersrc, "w").write(contents)
 329         f = open(slavedest, "w")
 330         f.write("overwrite me\n")
 331         f.close()
 332
 333         d = self.runStep(step)
 334         def _checkDownload(results):
 335             step_status = step.step_status
 336             self.failUnlessEqual(results, SUCCESS)
 337             self.failUnless(os.path.exists(slavedest))
 338             slavedest_contents = open(slavedest, "r").read()
 339             self.failUnlessEqual(slavedest_contents, contents)
 340             # and with 0777 to ignore sticky bits
 341             dest_mode = os.stat(slavedest)[ST_MODE] & 0777
 342             self.failUnlessEqual(dest_mode, 0755,
 343                                  "target mode was %o, we wanted %o" %
 344                                  (dest_mode, 0755))
 345         d.addCallback(_checkDownload)
 346         return d
 347
 348     def testMissingFile(self):
 349         self.slavebase = "DownloadFile.testMissingFile.slave"
 350         self.masterbase = "DownloadFile.testMissingFile.master"
 351         sb = self.makeSlaveBuilder()
 352         os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase,
 353                               "build"))
 354         mastersrc = os.path.join(self.masterbase, "MISSING.text")
 355         slavedest = os.path.join(self.slavebase,
 356                                  self.slavebuilderbase,
 357                                  "build",
 358                                  "dest.txt")
 359         step = self.makeStep(FileDownload,
 360                              mastersrc=mastersrc,
 361                              slavedest="dest.txt")
 362
 363         d = self.runStep(step)
 364         def _checkDownload(results):
 365             step_status = step.step_status
 366             self.failUnlessEqual(results, FAILURE)
 367             self.failIf(os.path.exists(slavedest))
 368             l = step_status.getLogs()
 369             logtext = l[0].getText().strip()
 370             self.failUnless(logtext.endswith(" not available at master"))
 371         d.addCallbacks(_checkDownload)
 372
 373         return d
 374
 375     def testLotsOfBlocks(self):
 376         self.slavebase = "DownloadFile.testLotsOfBlocks.slave"
 377         self.masterbase = "DownloadFile.testLotsOfBlocks.master"
 378         sb = self.makeSlaveBuilder()
 379         os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase,
 380                               "build"))
 381         mastersrc = os.path.join(self.masterbase, "source.text")
 382         slavedest = os.path.join(self.slavebase,
 383                                  self.slavebuilderbase,
 384                                  "build",
 385                                  "dest.txt")
 386         step = self.makeStep(FileDownload,
 387                              mastersrc=mastersrc,
 388                              slavedest="dest.txt",
 389                              blocksize=15)
 390         contents = "".join(["this is the source file #%d\n" % i
 391                             for i in range(1000)])
 392         open(mastersrc, "w").write(contents)
 393         f = open(slavedest, "w")
 394         f.write("overwrite me\n")
 395         f.close()
 396
 397         d = self.runStep(step)
 398         def _checkDownload(results):
 399             step_status = step.step_status
 400             self.failUnlessEqual(results, SUCCESS)
 401             self.failUnless(os.path.exists(slavedest))
 402             slavedest_contents = open(slavedest, "r").read()
 403             self.failUnlessEqual(slavedest_contents, contents)
 404         d.addCallback(_checkDownload)
 405         return d
 406
 407     def testWorkdir(self):
 408         self.slavebase = "Download.testWorkdir.slave"
 409         self.masterbase = "Download.testWorkdir.master"
 410         sb = self.makeSlaveBuilder()
 411
 412         # As in Upload.testWorkdir(), it's enough to test that makeStep()'s
 413         # call of setDefaultWorkdir() actually sets step.workdir.
 414         self.workdir = "mybuild"
 415         step = self.makeStep(FileDownload,
 416                              mastersrc="foo",
 417                              slavedest="foo")
 418         self.failUnlessEqual(step.workdir, self.workdir)
 419
 420     def testWithProperties(self):
 421         # test that workdir can be a WithProperties object
 422         self.slavebase = "Download.testWithProperties.slave"
 423         self.masterbase = "Download.testWithProperties.master"
 424         sb = self.makeSlaveBuilder()
 425
 426         step = self.makeStep(FileDownload,
 427                              mastersrc="src.txt",
 428                              slavedest="dest.txt")
 429         step.workdir = WithProperties("build.%s", "buildnumber")
 430
 431         self.failUnlessEqual(step._getWorkdir(), "build.1")
 432
 433
 434
 435 class UploadDirectory(StepTester, unittest.TestCase):
 436
 437     def filterArgs(self, args):
 438         if "writer" in args:
 439             args["writer"] = self.wrap(args["writer"])
 440         return args
 441
 442     def testSuccess(self):
 443         self.slavebase = "UploadDirectory.testSuccess.slave"
 444         self.masterbase = "UploadDirectory.testSuccess.master"
 445         sb = self.makeSlaveBuilder()
 446         os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase,
 447                               "build"))
 448         # the buildmaster normally runs chdir'ed into masterbase, so uploaded
 449         # files will appear there. Under trial, we're chdir'ed into
 450         # _trial_temp instead, so use a different masterdest= to keep the
 451         # uploaded file in a test-local directory
 452         masterdest = os.path.join(self.masterbase, "dest_dir")
 453         step = self.makeStep(DirectoryUpload,
 454                              slavesrc="source_dir",
 455                              masterdest=masterdest)
 456         slavesrc = os.path.join(self.slavebase,
 457                                 self.slavebuilderbase,
 458                                 "build",
 459                                 "source_dir")
 460         dircount = 5
 461         content = []
 462         content.append("this is one source file\n" * 1000)
 463         content.append("this is a second source file\n" * 978)
 464         content.append("this is a third source file\n" * 473)
 465         os.mkdir(slavesrc)
 466         for i in range(dircount):
 467             os.mkdir(os.path.join(slavesrc, "d%i" % (i)))
 468             for j in range(dircount):
 469                 curdir = os.path.join("d%i" % (i), "e%i" % (j))
 470                 os.mkdir(os.path.join(slavesrc, curdir))
 471                 for h in range(3):
 472                     open(os.path.join(slavesrc, curdir, "file%i" % (h)), "w").write(content[h])
 473             for j in range(dircount):
 474                 #empty dirs, must be uploaded too
 475                 curdir = os.path.join("d%i" % (i), "f%i" % (j))
 476                 os.mkdir(os.path.join(slavesrc, curdir))
 477
 478         d = self.runStep(step)
 479         def _checkUpload(results):
 480             step_status = step.step_status
 481             #l = step_status.getLogs()
 482             #if l:
 483             #    logtext = l[0].getText()
 484             #    print logtext
 485             self.failUnlessEqual(results, SUCCESS)
 486             self.failUnless(os.path.exists(masterdest))
 487             for i in range(dircount):
 488                 for j in range(dircount):
 489                     curdir = os.path.join("d%i" % (i), "e%i" % (j))
 490                     self.failUnless(os.path.exists(os.path.join(masterdest, curdir)))
 491                     for h in range(3):
 492                         masterdest_contents = open(os.path.join(masterdest, curdir, "file%i" % (h)), "r").read()
 493                         self.failUnlessEqual(masterdest_contents, content[h])
 494                 for j in range(dircount):
 495                     curdir = os.path.join("d%i" % (i), "f%i" % (j))
 496                     self.failUnless(os.path.exists(os.path.join(masterdest, curdir)))
 497         d.addCallback(_checkUpload)
 498         return d
 499
 500     def testOneEmptyDir(self):
 501         self.slavebase = "UploadDirectory.testOneEmptyDir.slave"
 502         self.masterbase = "UploadDirectory.testOneEmptyDir.master"
 503         sb = self.makeSlaveBuilder()
 504         os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase,
 505                               "build"))
 506         # the buildmaster normally runs chdir'ed into masterbase, so uploaded
 507         # files will appear there. Under trial, we're chdir'ed into
 508         # _trial_temp instead, so use a different masterdest= to keep the
 509         # uploaded file in a test-local directory
 510         masterdest = os.path.join(self.masterbase, "dest_dir")
 511         step = self.makeStep(DirectoryUpload,
 512                              slavesrc="source_dir",
 513                              masterdest=masterdest)
 514         slavesrc = os.path.join(self.slavebase,
 515                                 self.slavebuilderbase,
 516                                 "build",
 517                                 "source_dir")
 518         os.mkdir(slavesrc)
 519
 520         d = self.runStep(step)
 521         def _checkUpload(results):
 522             step_status = step.step_status
 523             #l = step_status.getLogs()
 524             #if l:
 525             #    logtext = l[0].getText()
 526             #    print logtext
 527             self.failUnlessEqual(results, SUCCESS)
 528             self.failUnless(os.path.exists(masterdest))
 529         d.addCallback(_checkUpload)
 530         return d
 531
 532     def testManyEmptyDirs(self):
 533         self.slavebase = "UploadDirectory.testManyEmptyDirs.slave"
 534         self.masterbase = "UploadDirectory.testManyEmptyDirs.master"
 535         sb = self.makeSlaveBuilder()
 536         os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase,
 537                               "build"))
 538         # the buildmaster normally runs chdir'ed into masterbase, so uploaded
 539         # files will appear there. Under trial, we're chdir'ed into
 540         # _trial_temp instead, so use a different masterdest= to keep the
 541         # uploaded file in a test-local directory
 542         masterdest = os.path.join(self.masterbase, "dest_dir")
 543         step = self.makeStep(DirectoryUpload,
 544                              slavesrc="source_dir",
 545                              masterdest=masterdest)
 546         slavesrc = os.path.join(self.slavebase,
 547                                 self.slavebuilderbase,
 548                                 "build",
 549                                 "source_dir")
 550         dircount = 25
 551         os.mkdir(slavesrc)
 552         for i in range(dircount):
 553             os.mkdir(os.path.join(slavesrc, "d%i" % (i)))
 554             for j in range(dircount):
 555                 curdir = os.path.join("d%i" % (i), "e%i" % (j))
 556                 os.mkdir(os.path.join(slavesrc, curdir))
 557                 curdir = os.path.join("d%i" % (i), "f%i" % (j))
 558                 os.mkdir(os.path.join(slavesrc, curdir))
 559
 560         d = self.runStep(step)
 561         def _checkUpload(results):
 562             step_status = step.step_status
 563             #l = step_status.getLogs()
 564             #if l:
 565             #    logtext = l[0].getText()
 566             #    print logtext
 567             self.failUnlessEqual(results, SUCCESS)
 568             self.failUnless(os.path.exists(masterdest))
 569             for i in range(dircount):
 570                 for j in range(dircount):
 571                     curdir = os.path.join("d%i" % (i), "e%i" % (j))
 572                     self.failUnless(os.path.exists(os.path.join(masterdest, curdir)))
 573                     curdir = os.path.join("d%i" % (i), "f%i" % (j))
 574                     self.failUnless(os.path.exists(os.path.join(masterdest, curdir)))
 575         d.addCallback(_checkUpload)
 576         return d
 577
 578     def testOneDirOneFile(self):
 579         self.slavebase = "UploadDirectory.testOneDirOneFile.slave"
 580         self.masterbase = "UploadDirectory.testOneDirOneFile.master"
 581         sb = self.makeSlaveBuilder()
 582         os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase,
 583                               "build"))
 584         # the buildmaster normally runs chdir'ed into masterbase, so uploaded
 585         # files will appear there. Under trial, we're chdir'ed into
 586         # _trial_temp instead, so use a different masterdest= to keep the
 587         # uploaded file in a test-local directory
 588         masterdest = os.path.join(self.masterbase, "dest_dir")
 589         step = self.makeStep(DirectoryUpload,
 590                              slavesrc="source_dir",
 591                              masterdest=masterdest)
 592         slavesrc = os.path.join(self.slavebase,
 593                                 self.slavebuilderbase,
 594                                 "build",
 595                                 "source_dir")
 596         os.mkdir(slavesrc)
 597         content = "this is one source file\n" * 1000
 598         open(os.path.join(slavesrc, "srcfile"), "w").write(content)
 599
 600         d = self.runStep(step)
 601         def _checkUpload(results):
 602             step_status = step.step_status
 603             #l = step_status.getLogs()
 604             #if l:
 605             #    logtext = l[0].getText()
 606             #    print logtext
 607             self.failUnlessEqual(results, SUCCESS)
 608             self.failUnless(os.path.exists(masterdest))
 609             masterdest_contents = open(os.path.join(masterdest, "srcfile"), "r").read()
 610             self.failUnlessEqual(masterdest_contents, content)
 611         d.addCallback(_checkUpload)
 612         return d
 613
 614     def testOneDirManyFiles(self):
 615         self.slavebase = "UploadDirectory.testOneDirManyFile.slave"
 616         self.masterbase = "UploadDirectory.testOneDirManyFile.master"
 617         sb = self.makeSlaveBuilder()
 618         os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase,
 619                               "build"))
 620         # the buildmaster normally runs chdir'ed into masterbase, so uploaded
 621         # files will appear there. Under trial, we're chdir'ed into
 622         # _trial_temp instead, so use a different masterdest= to keep the
 623         # uploaded file in a test-local directory
 624         masterdest = os.path.join(self.masterbase, "dest_dir")
 625         step = self.makeStep(DirectoryUpload,
 626                              slavesrc="source_dir",
 627                              masterdest=masterdest)
 628         slavesrc = os.path.join(self.slavebase,
 629                                 self.slavebuilderbase,
 630                                 "build",
 631                                 "source_dir")
 632         filecount = 20
 633         os.mkdir(slavesrc)
 634         content = []
 635         content.append("this is one source file\n" * 1000)
 636         content.append("this is a second source file\n" * 978)
 637         content.append("this is a third source file\n" * 473)
 638         for i in range(3):
 639             for j in range(filecount):
 640                 open(os.path.join(slavesrc, "srcfile%i_%i" % (i, j)), "w").write(content[i])
 641
 642         d = self.runStep(step)
 643         def _checkUpload(results):
 644             step_status = step.step_status
 645             #l = step_status.getLogs()
 646             #if l:
 647             #    logtext = l[0].getText()
 648             #    print logtext
 649             self.failUnlessEqual(results, SUCCESS)
 650             self.failUnless(os.path.exists(masterdest))
 651             for i in range(3):
 652                 for j in range(filecount):
 653                     masterdest_contents = open(os.path.join(masterdest, "srcfile%i_%i" % (i, j)), "r").read()
 654                     self.failUnlessEqual(masterdest_contents, content[i])
 655         d.addCallback(_checkUpload)
 656         return d
 657
 658     def testManyDirsManyFiles(self):
 659         self.slavebase = "UploadDirectory.testManyDirsManyFile.slave"
 660         self.masterbase = "UploadDirectory.testManyDirsManyFile.master"
 661         sb = self.makeSlaveBuilder()
 662         os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase,
 663                               "build"))
 664         # the buildmaster normally runs chdir'ed into masterbase, so uploaded
 665         # files will appear there. Under trial, we're chdir'ed into
 666         # _trial_temp instead, so use a different masterdest= to keep the
 667         # uploaded file in a test-local directory
 668         masterdest = os.path.join(self.masterbase, "dest_dir")
 669         step = self.makeStep(DirectoryUpload,
 670                              slavesrc="source_dir",
 671                              masterdest=masterdest)
 672         slavesrc = os.path.join(self.slavebase,
 673                                 self.slavebuilderbase,
 674                                 "build",
 675                                 "source_dir")
 676         dircount = 10
 677         os.mkdir(slavesrc)
 678         for i in range(dircount):
 679             os.mkdir(os.path.join(slavesrc, "d%i" % (i)))
 680             for j in range(dircount):
 681                 curdir = os.path.join("d%i" % (i), "e%i" % (j))
 682                 os.mkdir(os.path.join(slavesrc, curdir))
 683                 curdir = os.path.join("d%i" % (i), "f%i" % (j))
 684                 os.mkdir(os.path.join(slavesrc, curdir))
 685
 686         filecount = 5
 687         content = []
 688         content.append("this is one source file\n" * 1000)
 689         content.append("this is a second source file\n" * 978)
 690         content.append("this is a third source file\n" * 473)
 691         for i in range(dircount):
 692             for j in range(dircount):
 693                 for k in range(3):
 694                     for l in range(filecount):
 695                         open(os.path.join(slavesrc, "d%i" % (i), "e%i" % (j), "srcfile%i_%i" % (k, l)), "w").write(content[k])
 696
 697         d = self.runStep(step)
 698         def _checkUpload(results):
 699             step_status = step.step_status
 700             #l = step_status.getLogs()
 701             #if l:
 702             #    logtext = l[0].getText()
 703             #    print logtext
 704             self.failUnlessEqual(results, SUCCESS)
 705             self.failUnless(os.path.exists(masterdest))
 706             for i in range(dircount):
 707                 for j in range(dircount):
 708                     for k in range(3):
 709                         for l in range(filecount):
 710                             masterdest_contents = open(os.path.join(masterdest, "d%i" % (i), "e%i" % (j), "srcfile%i_%i" % (k, l)), "r").read()
 711                             self.failUnlessEqual(masterdest_contents, content[k])
 712         d.addCallback(_checkUpload)
 713         return d
 714
 715
 716     def testBigFile(self):
 717         self.slavebase = "UploadDirectory.testBigFile.slave"
 718         self.masterbase = "UploadDirectory.testBigFile.master"
 719         sb = self.makeSlaveBuilder()
 720         os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase,
 721                               "build"))
 722         # the buildmaster normally runs chdir'ed into masterbase, so uploaded
 723         # files will appear there. Under trial, we're chdir'ed into
 724         # _trial_temp instead, so use a different masterdest= to keep the
 725         # uploaded file in a test-local directory
 726         masterdest = os.path.join(self.masterbase, "dest_dir")
 727         step = self.makeStep(DirectoryUpload,
 728                              slavesrc="source_dir",
 729                              masterdest=masterdest)
 730         slavesrc = os.path.join(self.slavebase,
 731                                 self.slavebuilderbase,
 732                                 "build",
 733                                 "source_dir")
 734         content = 'x' * 1024*1024*8
 735         os.mkdir(slavesrc)
 736         open(os.path.join(slavesrc, "file"), "w").write(content)
 737
 738         d = self.runStep(step)
 739         def _checkUpload(results):
 740             step_status = step.step_status
 741             self.failUnlessEqual(results, SUCCESS)
 742             self.failUnless(os.path.exists(masterdest))
 743             masterdest_contents = open(os.path.join(masterdest, "file"), "r").read()
 744             self.failUnlessEqual(masterdest_contents, content)
 745         d.addCallback(_checkUpload)
 746         return d
 747
 748
 749 # TODO:
 750 #  test relative paths, ~/paths
 751 #   need to implement expanduser() for slave-side
 752 #  test error message when master-side file is in a missing directory
 753 #  remove workdir= default?
 754