buildbot/test/test_scheduler.py

   1 # -*- test-case-name: buildbot.test.test_scheduler -*-
   2
   3 import os, time
   4
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, reactor
   7 from twisted.application import service
   8 from twisted.spread import pb
   9
  10 from buildbot import scheduler, sourcestamp, buildset, status
  11 from buildbot.changes.changes import Change
  12 from buildbot.scripts import tryclient
  13
  14
  15 class FakeMaster(service.MultiService):
  16     d = None
  17     def submitBuildSet(self, bs):
  18         self.sets.append(bs)
  19         if self.d:
  20             reactor.callLater(0, self.d.callback, bs)
  21             self.d = None
  22         return pb.Referenceable() # makes the cleanup work correctly
  23
  24 class Scheduling(unittest.TestCase):
  25     def setUp(self):
  26         self.master = master = FakeMaster()
  27         master.sets = []
  28         master.startService()
  29
  30     def tearDown(self):
  31         d = self.master.stopService()
  32         return d
  33
  34     def addScheduler(self, s):
  35         s.setServiceParent(self.master)
  36
  37     def testPeriodic1(self):
  38         self.addScheduler(scheduler.Periodic("quickly", ["a","b"], 2))
  39         d = defer.Deferred()
  40         reactor.callLater(5, d.callback, None)
  41         d.addCallback(self._testPeriodic1_1)
  42         return d
  43     def _testPeriodic1_1(self, res):
  44         self.failUnless(len(self.master.sets) > 1)
  45         s1 = self.master.sets[0]
  46         self.failUnlessEqual(s1.builderNames, ["a","b"])
  47         self.failUnlessEqual(s1.reason, "The Periodic scheduler named 'quickly' triggered this build")
  48
  49     def testNightly(self):
  50         # now == 15-Nov-2005, 00:05:36 AM . By using mktime, this is
  51         # converted into the local timezone, which happens to match what
  52         # Nightly is going to do anyway.
  53         MIN=60; HOUR=60*MIN; DAY=24*3600
  54         now = time.mktime((2005, 11, 15, 0, 5, 36, 1, 319, 0))
  55
  56         s = scheduler.Nightly('nightly', ["a"], hour=3)
  57         t = s.calculateNextRunTimeFrom(now)
  58         self.failUnlessEqual(int(t-now), 2*HOUR+54*MIN+24)
  59
  60         s = scheduler.Nightly('nightly', ["a"], minute=[3,8,54])
  61         t = s.calculateNextRunTimeFrom(now)
  62         self.failUnlessEqual(int(t-now), 2*MIN+24)
  63
  64         s = scheduler.Nightly('nightly', ["a"],
  65                               dayOfMonth=16, hour=1, minute=6)
  66         t = s.calculateNextRunTimeFrom(now)
  67         self.failUnlessEqual(int(t-now), DAY+HOUR+24)
  68
  69         s = scheduler.Nightly('nightly', ["a"],
  70                               dayOfMonth=16, hour=1, minute=3)
  71         t = s.calculateNextRunTimeFrom(now)
  72         self.failUnlessEqual(int(t-now), DAY+57*MIN+24)
  73
  74         s = scheduler.Nightly('nightly', ["a"],
  75                               dayOfMonth=15, hour=1, minute=3)
  76         t = s.calculateNextRunTimeFrom(now)
  77         self.failUnlessEqual(int(t-now), 57*MIN+24)
  78
  79         s = scheduler.Nightly('nightly', ["a"],
  80                               dayOfMonth=15, hour=0, minute=3)
  81         t = s.calculateNextRunTimeFrom(now)
  82         self.failUnlessEqual(int(t-now), 30*DAY-3*MIN+24)
  83
  84
  85     def isImportant(self, change):
  86         if "important" in change.files:
  87             return True
  88         return False
  89
  90     def testBranch(self):
  91         s = scheduler.Scheduler("b1", "branch1", 2, ["a","b"],
  92                                 fileIsImportant=self.isImportant)
  93         self.addScheduler(s)
  94
  95         c0 = Change("carol", ["important"], "other branch", branch="other")
  96         s.addChange(c0)
  97         self.failIf(s.timer)
  98         self.failIf(s.importantChanges)
  99
 100         c1 = Change("alice", ["important", "not important"], "some changes",
 101                     branch="branch1")
 102         s.addChange(c1)
 103         c2 = Change("bob", ["not important", "boring"], "some more changes",
 104                     branch="branch1")
 105         s.addChange(c2)
 106         c3 = Change("carol", ["important", "dull"], "even more changes",
 107                     branch="branch1")
 108         s.addChange(c3)
 109
 110         self.failUnlessEqual(s.importantChanges, [c1,c3])
 111         self.failUnlessEqual(s.unimportantChanges, [c2])
 112         self.failUnless(s.timer)
 113
 114         d = defer.Deferred()
 115         reactor.callLater(4, d.callback, None)
 116         d.addCallback(self._testBranch_1)
 117         return d
 118     def _testBranch_1(self, res):
 119         self.failUnlessEqual(len(self.master.sets), 1)
 120         s = self.master.sets[0].source
 121         self.failUnlessEqual(s.branch, "branch1")
 122         self.failUnlessEqual(s.revision, None)
 123         self.failUnlessEqual(len(s.changes), 3)
 124         self.failUnlessEqual(s.patch, None)
 125
 126
 127     def testAnyBranch(self):
 128         s = scheduler.AnyBranchScheduler("b1", None, 1, ["a","b"],
 129                                          fileIsImportant=self.isImportant)
 130         self.addScheduler(s)
 131
 132         c1 = Change("alice", ["important", "not important"], "some changes",
 133                     branch="branch1")
 134         s.addChange(c1)
 135         c2 = Change("bob", ["not important", "boring"], "some more changes",
 136                     branch="branch1")
 137         s.addChange(c2)
 138         c3 = Change("carol", ["important", "dull"], "even more changes",
 139                     branch="branch1")
 140         s.addChange(c3)
 141
 142         c4 = Change("carol", ["important"], "other branch", branch="branch2")
 143         s.addChange(c4)
 144
 145         c5 = Change("carol", ["important"], "default branch", branch=None)
 146         s.addChange(c5)
 147
 148         d = defer.Deferred()
 149         reactor.callLater(2, d.callback, None)
 150         d.addCallback(self._testAnyBranch_1)
 151         return d
 152     def _testAnyBranch_1(self, res):
 153         self.failUnlessEqual(len(self.master.sets), 3)
 154         self.master.sets.sort(lambda a,b: cmp(a.source.branch,
 155                                               b.source.branch))
 156
 157         s1 = self.master.sets[0].source
 158         self.failUnlessEqual(s1.branch, None)
 159         self.failUnlessEqual(s1.revision, None)
 160         self.failUnlessEqual(len(s1.changes), 1)
 161         self.failUnlessEqual(s1.patch, None)
 162
 163         s2 = self.master.sets[1].source
 164         self.failUnlessEqual(s2.branch, "branch1")
 165         self.failUnlessEqual(s2.revision, None)
 166         self.failUnlessEqual(len(s2.changes), 3)
 167         self.failUnlessEqual(s2.patch, None)
 168
 169         s3 = self.master.sets[2].source
 170         self.failUnlessEqual(s3.branch, "branch2")
 171         self.failUnlessEqual(s3.revision, None)
 172         self.failUnlessEqual(len(s3.changes), 1)
 173         self.failUnlessEqual(s3.patch, None)
 174
 175     def testAnyBranch2(self):
 176         # like testAnyBranch but without fileIsImportant
 177         s = scheduler.AnyBranchScheduler("b1", None, 2, ["a","b"])
 178         self.addScheduler(s)
 179         c1 = Change("alice", ["important", "not important"], "some changes",
 180                     branch="branch1")
 181         s.addChange(c1)
 182         c2 = Change("bob", ["not important", "boring"], "some more changes",
 183                     branch="branch1")
 184         s.addChange(c2)
 185         c3 = Change("carol", ["important", "dull"], "even more changes",
 186                     branch="branch1")
 187         s.addChange(c3)
 188
 189         c4 = Change("carol", ["important"], "other branch", branch="branch2")
 190         s.addChange(c4)
 191
 192         d = defer.Deferred()
 193         reactor.callLater(2, d.callback, None)
 194         d.addCallback(self._testAnyBranch2_1)
 195         return d
 196     def _testAnyBranch2_1(self, res):
 197         self.failUnlessEqual(len(self.master.sets), 2)
 198         self.master.sets.sort(lambda a,b: cmp(a.source.branch,
 199                                               b.source.branch))
 200         s1 = self.master.sets[0].source
 201         self.failUnlessEqual(s1.branch, "branch1")
 202         self.failUnlessEqual(s1.revision, None)
 203         self.failUnlessEqual(len(s1.changes), 3)
 204         self.failUnlessEqual(s1.patch, None)
 205
 206         s2 = self.master.sets[1].source
 207         self.failUnlessEqual(s2.branch, "branch2")
 208         self.failUnlessEqual(s2.revision, None)
 209         self.failUnlessEqual(len(s2.changes), 1)
 210         self.failUnlessEqual(s2.patch, None)
 211
 212
 213     def createMaildir(self, jobdir):
 214         os.mkdir(jobdir)
 215         os.mkdir(os.path.join(jobdir, "new"))
 216         os.mkdir(os.path.join(jobdir, "cur"))
 217         os.mkdir(os.path.join(jobdir, "tmp"))
 218
 219     jobcounter = 1
 220     def pushJob(self, jobdir, job):
 221         while 1:
 222             filename = "job_%d" % self.jobcounter
 223             self.jobcounter += 1
 224             if os.path.exists(os.path.join(jobdir, "new", filename)):
 225                 continue
 226             if os.path.exists(os.path.join(jobdir, "tmp", filename)):
 227                 continue
 228             if os.path.exists(os.path.join(jobdir, "cur", filename)):
 229                 continue
 230             break
 231         f = open(os.path.join(jobdir, "tmp", filename), "w")
 232         f.write(job)
 233         f.close()
 234         os.rename(os.path.join(jobdir, "tmp", filename),
 235                   os.path.join(jobdir, "new", filename))
 236
 237     def testTryJobdir(self):
 238         self.master.basedir = "try_jobdir"
 239         os.mkdir(self.master.basedir)
 240         jobdir = "jobdir1"
 241         jobdir_abs = os.path.join(self.master.basedir, jobdir)
 242         self.createMaildir(jobdir_abs)
 243         s = scheduler.Try_Jobdir("try1", ["a", "b"], jobdir)
 244         self.addScheduler(s)
 245         self.failIf(self.master.sets)
 246         job1 = tryclient.createJobfile("buildsetID",
 247                                        "branch1", "123", 1, "diff",
 248                                        ["a", "b"])
 249         self.master.d = d = defer.Deferred()
 250         self.pushJob(jobdir_abs, job1)
 251         d.addCallback(self._testTryJobdir_1)
 252         # N.B.: if we don't have DNotify, we poll every 10 seconds, so don't
 253         # set a .timeout here shorter than that. TODO: make it possible to
 254         # set the polling interval, so we can make it shorter.
 255         return d
 256
 257     def _testTryJobdir_1(self, bs):
 258         self.failUnlessEqual(bs.builderNames, ["a", "b"])
 259         self.failUnlessEqual(bs.source.branch, "branch1")
 260         self.failUnlessEqual(bs.source.revision, "123")
 261         self.failUnlessEqual(bs.source.patch, (1, "diff"))
 262
 263
 264     def testTryUserpass(self):
 265         up = [("alice","pw1"), ("bob","pw2")]
 266         s = scheduler.Try_Userpass("try2", ["a", "b"], 0, userpass=up)
 267         self.addScheduler(s)
 268         port = s.getPort()
 269         config = {'connect': 'pb',
 270                   'username': 'alice',
 271                   'passwd': 'pw1',
 272                   'master': "localhost:%d" % port,
 273                   'builders': ["a", "b"],
 274                   }
 275         t = tryclient.Try(config)
 276         ss = sourcestamp.SourceStamp("branch1", "123", (1, "diff"))
 277         t.sourcestamp = ss
 278         d2 = self.master.d = defer.Deferred()
 279         d = t.deliverJob()
 280         d.addCallback(self._testTryUserpass_1, t, d2)
 281         return d
 282     testTryUserpass.timeout = 5
 283     def _testTryUserpass_1(self, res, t, d2):
 284         # at this point, the Try object should have a RemoteReference to the
 285         # status object. The FakeMaster returns a stub.
 286         self.failUnless(t.buildsetStatus)
 287         d2.addCallback(self._testTryUserpass_2, t)
 288         return d2
 289     def _testTryUserpass_2(self, bs, t):
 290         # this should be the BuildSet submitted by the TryScheduler
 291         self.failUnlessEqual(bs.builderNames, ["a", "b"])
 292         self.failUnlessEqual(bs.source.branch, "branch1")
 293         self.failUnlessEqual(bs.source.revision, "123")
 294         self.failUnlessEqual(bs.source.patch, (1, "diff"))
 295
 296         t.cleanup()
 297
 298         # twisted-2.0.1 (but not later versions) seems to require a reactor
 299         # iteration before stopListening actually works. TODO: investigate
 300         # this.
 301         d = defer.Deferred()
 302         reactor.callLater(0, d.callback, None)
 303         return d
 304
 305     def testGetBuildSets(self):
 306         # validate IStatus.getBuildSets
 307         s = status.builder.Status(None, ".")
 308         bs1 = buildset.BuildSet(["a","b"], sourcestamp.SourceStamp(),
 309                                 reason="one", bsid="1")
 310         s.buildsetSubmitted(bs1.status)
 311         self.failUnlessEqual(s.getBuildSets(), [bs1.status])
 312         bs1.status.notifyFinishedWatchers()
 313         self.failUnlessEqual(s.getBuildSets(), [])
 314
 315     def testCategory(self):
 316         s1 = scheduler.Scheduler("b1", "branch1", 2, ["a","b"], categories=["categoryA", "both"])
 317         self.addScheduler(s1)
 318         s2 = scheduler.Scheduler("b2", "branch1", 2, ["a","b"], categories=["categoryB", "both"])
 319         self.addScheduler(s2)
 320
 321         c0 = Change("carol", ["important"], "branch1", branch="branch1", category="categoryA")
 322         s1.addChange(c0)
 323         s2.addChange(c0)
 324
 325         c1 = Change("carol", ["important"], "branch1", branch="branch1", category="categoryB")
 326         s1.addChange(c1)
 327         s2.addChange(c1)
 328
 329         c2 = Change("carol", ["important"], "branch1", branch="branch1")
 330         s1.addChange(c2)
 331         s2.addChange(c2)
 332
 333         c3 = Change("carol", ["important"], "branch1", branch="branch1", category="both")
 334         s1.addChange(c3)
 335         s2.addChange(c3)
 336
 337         self.failUnlessEqual(s1.importantChanges, [c0, c3])
 338         self.failUnlessEqual(s2.importantChanges, [c1, c3])
 339
 340         s = scheduler.Scheduler("b3", "branch1", 2, ["a","b"])
 341         self.addScheduler(s)
 342
 343         c0 = Change("carol", ["important"], "branch1", branch="branch1", category="categoryA")
 344         s.addChange(c0)
 345         c1 = Change("carol", ["important"], "branch1", branch="branch1", category="categoryB")
 346         s.addChange(c1)
 347
 348         self.failUnlessEqual(s.importantChanges, [c0, c1])