waterfall: fix DST calculation. Closes #137.
[buildbot.git] / buildbot / scheduler.py
blob2b290fd1ee38492f706602e846a599bb33247323
1 # -*- test-case-name: buildbot.test.test_dependencies -*-
3 import time, os.path
5 from zope.interface import implements
6 from twisted.internet import reactor
7 from twisted.application import service, internet, strports
8 from twisted.python import log, runtime
9 from twisted.protocols import basic
10 from twisted.cred import portal, checkers
11 from twisted.spread import pb
13 from buildbot import interfaces, buildset, util, pbutil
14 from buildbot.status import builder
15 from buildbot.sourcestamp import SourceStamp
16 from buildbot.changes.maildir import MaildirService
19 class BaseScheduler(service.MultiService, util.ComparableMixin):
20 implements(interfaces.IScheduler)
22 def __init__(self, name):
23 service.MultiService.__init__(self)
24 self.name = name
26 def __repr__(self):
27 # TODO: why can't id() return a positive number? %d is ugly.
28 return "<Scheduler '%s' at %d>" % (self.name, id(self))
30 def submit(self, bs):
31 self.parent.submitBuildSet(bs)
33 def addChange(self, change):
34 pass
36 class BaseUpstreamScheduler(BaseScheduler):
37 implements(interfaces.IUpstreamScheduler)
39 def __init__(self, name):
40 BaseScheduler.__init__(self, name)
41 self.successWatchers = []
43 def subscribeToSuccessfulBuilds(self, watcher):
44 self.successWatchers.append(watcher)
45 def unsubscribeToSuccessfulBuilds(self, watcher):
46 self.successWatchers.remove(watcher)
48 def submit(self, bs):
49 d = bs.waitUntilFinished()
50 d.addCallback(self.buildSetFinished)
51 self.parent.submitBuildSet(bs)
53 def buildSetFinished(self, bss):
54 if not self.running:
55 return
56 if bss.getResults() == builder.SUCCESS:
57 ss = bss.getSourceStamp()
58 for w in self.successWatchers:
59 w(ss)
62 class Scheduler(BaseUpstreamScheduler):
63 """The default Scheduler class will run a build after some period of time
64 called the C{treeStableTimer}, on a given set of Builders. It only pays
65 attention to a single branch. You you can provide a C{fileIsImportant}
66 function which will evaluate each Change to decide whether or not it
67 should trigger a new build.
68 """
70 fileIsImportant = None
71 compare_attrs = ('name', 'treeStableTimer', 'builderNames', 'branch',
72 'fileIsImportant')
74 def __init__(self, name, branch, treeStableTimer, builderNames,
75 fileIsImportant=None):
76 """
77 @param name: the name of this Scheduler
78 @param branch: The branch name that the Scheduler should pay
79 attention to. Any Change that is not on this branch
80 will be ignored. It can be set to None to only pay
81 attention to the default branch.
82 @param treeStableTimer: the duration, in seconds, for which the tree
83 must remain unchanged before a build will be
84 triggered. This is intended to avoid builds
85 of partially-committed fixes.
86 @param builderNames: a list of Builder names. When this Scheduler
87 decides to start a set of builds, they will be
88 run on the Builders named by this list.
90 @param fileIsImportant: A callable which takes one argument (a Change
91 instance) and returns True if the change is
92 worth building, and False if it is not.
93 Unimportant Changes are accumulated until the
94 build is triggered by an important change.
95 The default value of None means that all
96 Changes are important.
97 """
99 BaseUpstreamScheduler.__init__(self, name)
100 self.treeStableTimer = treeStableTimer
101 errmsg = ("The builderNames= argument to Scheduler must be a list "
102 "of Builder description names (i.e. the 'name' key of the "
103 "Builder specification dictionary)")
104 assert isinstance(builderNames, (list, tuple)), errmsg
105 for b in builderNames:
106 assert isinstance(b, str), errmsg
107 self.builderNames = builderNames
108 self.branch = branch
109 if fileIsImportant:
110 assert callable(fileIsImportant)
111 self.fileIsImportant = fileIsImportant
113 self.importantChanges = []
114 self.unimportantChanges = []
115 self.nextBuildTime = None
116 self.timer = None
118 def listBuilderNames(self):
119 return self.builderNames
121 def getPendingBuildTimes(self):
122 if self.nextBuildTime is not None:
123 return [self.nextBuildTime]
124 return []
126 def addChange(self, change):
127 if change.branch != self.branch:
128 log.msg("%s ignoring off-branch %s" % (self, change))
129 return
130 if not self.fileIsImportant:
131 self.addImportantChange(change)
132 elif self.fileIsImportant(change):
133 self.addImportantChange(change)
134 else:
135 self.addUnimportantChange(change)
137 def addImportantChange(self, change):
138 log.msg("%s: change is important, adding %s" % (self, change))
139 self.importantChanges.append(change)
140 self.nextBuildTime = max(self.nextBuildTime,
141 change.when + self.treeStableTimer)
142 self.setTimer(self.nextBuildTime)
144 def addUnimportantChange(self, change):
145 log.msg("%s: change is not important, adding %s" % (self, change))
146 self.unimportantChanges.append(change)
148 def setTimer(self, when):
149 log.msg("%s: setting timer to %s" %
150 (self, time.strftime("%H:%M:%S", time.localtime(when))))
151 now = util.now()
152 if when < now:
153 when = now + 1
154 if self.timer:
155 self.timer.cancel()
156 self.timer = reactor.callLater(when - now, self.fireTimer)
158 def stopTimer(self):
159 if self.timer:
160 self.timer.cancel()
161 self.timer = None
163 def fireTimer(self):
164 # clear out our state
165 self.timer = None
166 self.nextBuildTime = None
167 changes = self.importantChanges + self.unimportantChanges
168 self.importantChanges = []
169 self.unimportantChanges = []
171 # create a BuildSet, submit it to the BuildMaster
172 bs = buildset.BuildSet(self.builderNames,
173 SourceStamp(changes=changes))
174 self.submit(bs)
176 def stopService(self):
177 self.stopTimer()
178 return service.MultiService.stopService(self)
181 class AnyBranchScheduler(BaseUpstreamScheduler):
182 """This Scheduler will handle changes on a variety of branches. It will
183 accumulate Changes for each branch separately. It works by creating a
184 separate Scheduler for each new branch it sees."""
186 schedulerFactory = Scheduler
187 fileIsImportant = None
189 compare_attrs = ('name', 'branches', 'treeStableTimer', 'builderNames',
190 'fileIsImportant')
192 def __init__(self, name, branches, treeStableTimer, builderNames,
193 fileIsImportant=None):
195 @param name: the name of this Scheduler
196 @param branches: The branch names that the Scheduler should pay
197 attention to. Any Change that is not on one of these
198 branches will be ignored. It can be set to None to
199 accept changes from any branch. Don't use [] (an
200 empty list), because that means we don't pay
201 attention to *any* branches, so we'll never build
202 anything.
203 @param treeStableTimer: the duration, in seconds, for which the tree
204 must remain unchanged before a build will be
205 triggered. This is intended to avoid builds
206 of partially-committed fixes.
207 @param builderNames: a list of Builder names. When this Scheduler
208 decides to start a set of builds, they will be
209 run on the Builders named by this list.
211 @param fileIsImportant: A callable which takes one argument (a Change
212 instance) and returns True if the change is
213 worth building, and False if it is not.
214 Unimportant Changes are accumulated until the
215 build is triggered by an important change.
216 The default value of None means that all
217 Changes are important.
220 BaseUpstreamScheduler.__init__(self, name)
221 self.treeStableTimer = treeStableTimer
222 for b in builderNames:
223 assert isinstance(b, str)
224 self.builderNames = builderNames
225 self.branches = branches
226 if self.branches == []:
227 log.msg("AnyBranchScheduler %s: branches=[], so we will ignore "
228 "all branches, and never trigger any builds. Please set "
229 "branches=None to mean 'all branches'" % self)
230 # consider raising an exception here, to make this warning more
231 # prominent, but I can vaguely imagine situations where you might
232 # want to comment out branches temporarily and wouldn't
233 # appreciate it being treated as an error.
234 if fileIsImportant:
235 assert callable(fileIsImportant)
236 self.fileIsImportant = fileIsImportant
237 self.schedulers = {} # one per branch
239 def __repr__(self):
240 return "<AnyBranchScheduler '%s'>" % self.name
242 def listBuilderNames(self):
243 return self.builderNames
245 def getPendingBuildTimes(self):
246 bts = []
247 for s in self.schedulers.values():
248 if s.nextBuildTime is not None:
249 bts.append(s.nextBuildTime)
250 return bts
252 def addChange(self, change):
253 branch = change.branch
254 if self.branches is not None and branch not in self.branches:
255 log.msg("%s ignoring off-branch %s" % (self, change))
256 return
257 s = self.schedulers.get(branch)
258 if not s:
259 if branch:
260 name = self.name + "." + branch
261 else:
262 name = self.name + ".<default>"
263 s = self.schedulerFactory(name, branch,
264 self.treeStableTimer,
265 self.builderNames,
266 self.fileIsImportant)
267 s.successWatchers = self.successWatchers
268 s.setServiceParent(self)
269 # TODO: does this result in schedulers that stack up forever?
270 # When I make the persistify-pass, think about this some more.
271 self.schedulers[branch] = s
272 s.addChange(change)
274 def submitBuildSet(self, bs):
275 self.parent.submitBuildSet(bs)
278 class Dependent(BaseUpstreamScheduler):
279 """This scheduler runs some set of 'downstream' builds when the
280 'upstream' scheduler has completed successfully."""
282 compare_attrs = ('name', 'upstream', 'builders')
284 def __init__(self, name, upstream, builderNames):
285 assert interfaces.IUpstreamScheduler.providedBy(upstream)
286 BaseUpstreamScheduler.__init__(self, name)
287 self.upstream = upstream
288 self.builderNames = builderNames
290 def listBuilderNames(self):
291 return self.builderNames
293 def getPendingBuildTimes(self):
294 # report the upstream's value
295 return self.upstream.getPendingBuildTimes()
297 def startService(self):
298 service.MultiService.startService(self)
299 self.upstream.subscribeToSuccessfulBuilds(self.upstreamBuilt)
301 def stopService(self):
302 d = service.MultiService.stopService(self)
303 self.upstream.unsubscribeToSuccessfulBuilds(self.upstreamBuilt)
304 return d
306 def upstreamBuilt(self, ss):
307 bs = buildset.BuildSet(self.builderNames, ss)
308 self.submit(bs)
312 class Periodic(BaseUpstreamScheduler):
313 """Instead of watching for Changes, this Scheduler can just start a build
314 at fixed intervals. The C{periodicBuildTimer} parameter sets the number
315 of seconds to wait between such periodic builds. The first build will be
316 run immediately."""
318 # TODO: consider having this watch another (changed-based) scheduler and
319 # merely enforce a minimum time between builds.
321 compare_attrs = ('name', 'builderNames', 'periodicBuildTimer', 'branch')
323 def __init__(self, name, builderNames, periodicBuildTimer,
324 branch=None):
325 BaseUpstreamScheduler.__init__(self, name)
326 self.builderNames = builderNames
327 self.periodicBuildTimer = periodicBuildTimer
328 self.branch = branch
329 self.reason = ("The Periodic scheduler named '%s' triggered this build"
330 % name)
331 self.timer = internet.TimerService(self.periodicBuildTimer,
332 self.doPeriodicBuild)
333 self.timer.setServiceParent(self)
335 def listBuilderNames(self):
336 return self.builderNames
338 def getPendingBuildTimes(self):
339 # TODO: figure out when self.timer is going to fire next and report
340 # that
341 return []
343 def doPeriodicBuild(self):
344 bs = buildset.BuildSet(self.builderNames,
345 SourceStamp(branch=self.branch),
346 self.reason)
347 self.submit(bs)
351 class Nightly(BaseUpstreamScheduler):
352 """Imitate 'cron' scheduling. This can be used to schedule a nightly
353 build, or one which runs are certain times of the day, week, or month.
355 Pass some subset of minute, hour, dayOfMonth, month, and dayOfWeek; each
356 may be a single number or a list of valid values. The builds will be
357 triggered whenever the current time matches these values. Wildcards are
358 represented by a '*' string. All fields default to a wildcard except
359 'minute', so with no fields this defaults to a build every hour, on the
360 hour.
362 For example, the following master.cfg clause will cause a build to be
363 started every night at 3:00am::
365 s = Nightly('nightly', ['builder1', 'builder2'], hour=3, minute=0)
366 c['schedules'].append(s)
368 This scheduler will perform a build each monday morning at 6:23am and
369 again at 8:23am::
371 s = Nightly('BeforeWork', ['builder1'],
372 dayOfWeek=0, hour=[6,8], minute=23)
374 The following runs a build every two hours::
376 s = Nightly('every2hours', ['builder1'], hour=range(0, 24, 2))
378 And this one will run only on December 24th::
380 s = Nightly('SleighPreflightCheck', ['flying_circuits', 'radar'],
381 month=12, dayOfMonth=24, hour=12, minute=0)
383 For dayOfWeek and dayOfMonth, builds are triggered if the date matches
384 either of them. All time values are compared against the tuple returned
385 by time.localtime(), so month and dayOfMonth numbers start at 1, not
386 zero. dayOfWeek=0 is Monday, dayOfWeek=6 is Sunday.
389 compare_attrs = ('name', 'builderNames',
390 'minute', 'hour', 'dayOfMonth', 'month',
391 'dayOfWeek', 'branch')
393 def __init__(self, name, builderNames, minute=0, hour='*',
394 dayOfMonth='*', month='*', dayOfWeek='*',
395 branch=None):
396 # Setting minute=0 really makes this an 'Hourly' scheduler. This
397 # seemed like a better default than minute='*', which would result in
398 # a build every 60 seconds.
399 BaseUpstreamScheduler.__init__(self, name)
400 self.builderNames = builderNames
401 self.minute = minute
402 self.hour = hour
403 self.dayOfMonth = dayOfMonth
404 self.month = month
405 self.dayOfWeek = dayOfWeek
406 self.branch = branch
407 self.delayedRun = None
408 self.nextRunTime = None
409 self.reason = ("The Nightly scheduler named '%s' triggered this build"
410 % name)
412 def addTime(self, timetuple, secs):
413 return time.localtime(time.mktime(timetuple)+secs)
414 def findFirstValueAtLeast(self, values, value, default=None):
415 for v in values:
416 if v >= value: return v
417 return default
419 def setTimer(self):
420 self.nextRunTime = self.calculateNextRunTime()
421 self.delayedRun = reactor.callLater(self.nextRunTime - time.time(),
422 self.doPeriodicBuild)
424 def startService(self):
425 BaseUpstreamScheduler.startService(self)
426 self.setTimer()
428 def stopService(self):
429 BaseUpstreamScheduler.stopService(self)
430 self.delayedRun.cancel()
432 def isRunTime(self, timetuple):
433 def check(ourvalue, value):
434 if ourvalue == '*': return True
435 if isinstance(ourvalue, int): return value == ourvalue
436 return (value in ourvalue)
438 if not check(self.minute, timetuple[4]):
439 #print 'bad minute', timetuple[4], self.minute
440 return False
442 if not check(self.hour, timetuple[3]):
443 #print 'bad hour', timetuple[3], self.hour
444 return False
446 if not check(self.month, timetuple[1]):
447 #print 'bad month', timetuple[1], self.month
448 return False
450 if self.dayOfMonth != '*' and self.dayOfWeek != '*':
451 # They specified both day(s) of month AND day(s) of week.
452 # This means that we only have to match one of the two. If
453 # neither one matches, this time is not the right time.
454 if not (check(self.dayOfMonth, timetuple[2]) or
455 check(self.dayOfWeek, timetuple[6])):
456 #print 'bad day'
457 return False
458 else:
459 if not check(self.dayOfMonth, timetuple[2]):
460 #print 'bad day of month'
461 return False
463 if not check(self.dayOfWeek, timetuple[6]):
464 #print 'bad day of week'
465 return False
467 return True
469 def calculateNextRunTime(self):
470 return self.calculateNextRunTimeFrom(time.time())
472 def calculateNextRunTimeFrom(self, now):
473 dateTime = time.localtime(now)
475 # Remove seconds by advancing to at least the next minue
476 dateTime = self.addTime(dateTime, 60-dateTime[5])
478 # Now we just keep adding minutes until we find something that matches
480 # It not an efficient algorithm, but it'll *work* for now
481 yearLimit = dateTime[0]+2
482 while not self.isRunTime(dateTime):
483 dateTime = self.addTime(dateTime, 60)
484 #print 'Trying', time.asctime(dateTime)
485 assert dateTime[0] < yearLimit, 'Something is wrong with this code'
486 return time.mktime(dateTime)
488 def listBuilderNames(self):
489 return self.builderNames
491 def getPendingBuildTimes(self):
492 # TODO: figure out when self.timer is going to fire next and report
493 # that
494 if self.nextRunTime is None: return []
495 return [self.nextRunTime]
497 def doPeriodicBuild(self):
498 # Schedule the next run
499 self.setTimer()
501 # And trigger a build
502 bs = buildset.BuildSet(self.builderNames,
503 SourceStamp(branch=self.branch),
504 self.reason)
505 self.submit(bs)
507 def addChange(self, change):
508 pass
512 class TryBase(service.MultiService, util.ComparableMixin):
513 implements(interfaces.IScheduler)
515 def __init__(self, name, builderNames):
516 service.MultiService.__init__(self)
517 self.name = name
518 self.builderNames = builderNames
520 def listBuilderNames(self):
521 return self.builderNames
523 def getPendingBuildTimes(self):
524 # we can't predict what the developers are going to do in the future
525 return []
527 def addChange(self, change):
528 # Try schedulers ignore Changes
529 pass
532 class BadJobfile(Exception):
533 pass
535 class JobFileScanner(basic.NetstringReceiver):
536 def __init__(self):
537 self.strings = []
538 self.transport = self # so transport.loseConnection works
539 self.error = False
541 def stringReceived(self, s):
542 self.strings.append(s)
544 def loseConnection(self):
545 self.error = True
547 class Try_Jobdir(TryBase):
548 compare_attrs = ["name", "builderNames", "jobdir"]
550 def __init__(self, name, builderNames, jobdir):
551 TryBase.__init__(self, name, builderNames)
552 self.jobdir = jobdir
553 self.watcher = MaildirService()
554 self.watcher.setServiceParent(self)
556 def setServiceParent(self, parent):
557 self.watcher.setBasedir(os.path.join(parent.basedir, self.jobdir))
558 TryBase.setServiceParent(self, parent)
560 def parseJob(self, f):
561 # jobfiles are serialized build requests. Each is a list of
562 # serialized netstrings, in the following order:
563 # "1", the version number of this format
564 # buildsetID, arbitrary string, used to find the buildSet later
565 # branch name, "" for default-branch
566 # base revision, "" for HEAD
567 # patchlevel, usually "1"
568 # patch
569 # builderNames...
570 p = JobFileScanner()
571 p.dataReceived(f.read())
572 if p.error:
573 raise BadJobfile("unable to parse netstrings")
574 s = p.strings
575 ver = s.pop(0)
576 if ver != "1":
577 raise BadJobfile("unknown version '%s'" % ver)
578 buildsetID, branch, baserev, patchlevel, diff = s[:5]
579 builderNames = s[5:]
580 if branch == "":
581 branch = None
582 if baserev == "":
583 baserev = None
584 patchlevel = int(patchlevel)
585 patch = (patchlevel, diff)
586 ss = SourceStamp(branch, baserev, patch)
587 return builderNames, ss, buildsetID
589 def messageReceived(self, filename):
590 md = os.path.join(self.parent.basedir, self.jobdir)
591 if runtime.platformType == "posix":
592 # open the file before moving it, because I'm afraid that once
593 # it's in cur/, someone might delete it at any moment
594 path = os.path.join(md, "new", filename)
595 f = open(path, "r")
596 os.rename(os.path.join(md, "new", filename),
597 os.path.join(md, "cur", filename))
598 else:
599 # do this backwards under windows, because you can't move a file
600 # that somebody is holding open. This was causing a Permission
601 # Denied error on bear's win32-twisted1.3 buildslave.
602 os.rename(os.path.join(md, "new", filename),
603 os.path.join(md, "cur", filename))
604 path = os.path.join(md, "cur", filename)
605 f = open(path, "r")
607 try:
608 builderNames, ss, bsid = self.parseJob(f)
609 except BadJobfile:
610 log.msg("%s reports a bad jobfile in %s" % (self, filename))
611 log.err()
612 return
613 # compare builderNames against self.builderNames
614 # TODO: think about this some more.. why bother restricting it?
615 # perhaps self.builderNames should be used as the default list
616 # instead of being used as a restriction?
617 for b in builderNames:
618 if not b in self.builderNames:
619 log.msg("%s got jobfile %s with builder %s" % (self,
620 filename, b))
621 log.msg(" but that wasn't in our list: %s"
622 % (self.builderNames,))
623 return
625 reason = "'try' job"
626 bs = buildset.BuildSet(builderNames, ss, reason=reason, bsid=bsid)
627 self.parent.submitBuildSet(bs)
629 class Try_Userpass(TryBase):
630 compare_attrs = ["name", "builderNames", "port", "userpass"]
631 implements(portal.IRealm)
633 def __init__(self, name, builderNames, port, userpass):
634 TryBase.__init__(self, name, builderNames)
635 if type(port) is int:
636 port = "tcp:%d" % port
637 self.port = port
638 self.userpass = userpass
639 c = checkers.InMemoryUsernamePasswordDatabaseDontUse()
640 for user,passwd in self.userpass:
641 c.addUser(user, passwd)
643 p = portal.Portal(self)
644 p.registerChecker(c)
645 f = pb.PBServerFactory(p)
646 s = strports.service(port, f)
647 s.setServiceParent(self)
649 def getPort(self):
650 # utility method for tests: figure out which TCP port we just opened.
651 return self.services[0]._port.getHost().port
653 def requestAvatar(self, avatarID, mind, interface):
654 log.msg("%s got connection from user %s" % (self, avatarID))
655 assert interface == pb.IPerspective
656 p = Try_Userpass_Perspective(self, avatarID)
657 return (pb.IPerspective, p, lambda: None)
659 def submitBuildSet(self, bs):
660 return self.parent.submitBuildSet(bs)
662 class Try_Userpass_Perspective(pbutil.NewCredPerspective):
663 def __init__(self, parent, username):
664 self.parent = parent
665 self.username = username
667 def perspective_try(self, branch, revision, patch, builderNames):
668 log.msg("user %s requesting build on builders %s" % (self.username,
669 builderNames))
670 for b in builderNames:
671 if not b in self.parent.builderNames:
672 log.msg("%s got job with builder %s" % (self, b))
673 log.msg(" but that wasn't in our list: %s"
674 % (self.parent.builderNames,))
675 return
676 ss = SourceStamp(branch, revision, patch)
677 reason = "'try' job from user %s" % self.username
678 bs = buildset.BuildSet(builderNames, ss, reason=reason)
679 self.parent.submitBuildSet(bs)
681 # return a remotely-usable BuildSetStatus object
682 from buildbot.status.client import makeRemote
683 return makeRemote(bs.status)