1 # -*- test-case-name: buildbot.test.test_dependencies -*-
5 from zope
.interface
import implements
6 from twisted
.internet
import reactor
7 from twisted
.application
import service
, internet
, strports
8 from twisted
.python
import log
, runtime
9 from twisted
.protocols
import basic
10 from twisted
.cred
import portal
, checkers
11 from twisted
.spread
import pb
13 from buildbot
import interfaces
, buildset
, util
, pbutil
14 from buildbot
.status
import builder
15 from buildbot
.sourcestamp
import SourceStamp
16 from buildbot
.changes
.maildir
import MaildirService
19 class BaseScheduler(service
.MultiService
, util
.ComparableMixin
):
20 implements(interfaces
.IScheduler
)
22 def __init__(self
, name
):
23 service
.MultiService
.__init
__(self
)
27 # TODO: why can't id() return a positive number? %d is ugly.
28 return "<Scheduler '%s' at %d>" % (self
.name
, id(self
))
31 self
.parent
.submitBuildSet(bs
)
33 def addChange(self
, change
):
36 class BaseUpstreamScheduler(BaseScheduler
):
37 implements(interfaces
.IUpstreamScheduler
)
39 def __init__(self
, name
):
40 BaseScheduler
.__init
__(self
, name
)
41 self
.successWatchers
= []
43 def subscribeToSuccessfulBuilds(self
, watcher
):
44 self
.successWatchers
.append(watcher
)
45 def unsubscribeToSuccessfulBuilds(self
, watcher
):
46 self
.successWatchers
.remove(watcher
)
49 d
= bs
.waitUntilFinished()
50 d
.addCallback(self
.buildSetFinished
)
51 self
.parent
.submitBuildSet(bs
)
53 def buildSetFinished(self
, bss
):
56 if bss
.getResults() == builder
.SUCCESS
:
57 ss
= bss
.getSourceStamp()
58 for w
in self
.successWatchers
:
62 class Scheduler(BaseUpstreamScheduler
):
63 """The default Scheduler class will run a build after some period of time
64 called the C{treeStableTimer}, on a given set of Builders. It only pays
65 attention to a single branch. You you can provide a C{fileIsImportant}
66 function which will evaluate each Change to decide whether or not it
67 should trigger a new build.
70 fileIsImportant
= None
71 compare_attrs
= ('name', 'treeStableTimer', 'builderNames', 'branch',
74 def __init__(self
, name
, branch
, treeStableTimer
, builderNames
,
75 fileIsImportant
=None):
77 @param name: the name of this Scheduler
78 @param branch: The branch name that the Scheduler should pay
79 attention to. Any Change that is not on this branch
80 will be ignored. It can be set to None to only pay
81 attention to the default branch.
82 @param treeStableTimer: the duration, in seconds, for which the tree
83 must remain unchanged before a build will be
84 triggered. This is intended to avoid builds
85 of partially-committed fixes.
86 @param builderNames: a list of Builder names. When this Scheduler
87 decides to start a set of builds, they will be
88 run on the Builders named by this list.
90 @param fileIsImportant: A callable which takes one argument (a Change
91 instance) and returns True if the change is
92 worth building, and False if it is not.
93 Unimportant Changes are accumulated until the
94 build is triggered by an important change.
95 The default value of None means that all
96 Changes are important.
99 BaseUpstreamScheduler
.__init
__(self
, name
)
100 self
.treeStableTimer
= treeStableTimer
101 errmsg
= ("The builderNames= argument to Scheduler must be a list "
102 "of Builder description names (i.e. the 'name' key of the "
103 "Builder specification dictionary)")
104 assert isinstance(builderNames
, (list, tuple)), errmsg
105 for b
in builderNames
:
106 assert isinstance(b
, str), errmsg
107 self
.builderNames
= builderNames
110 assert callable(fileIsImportant
)
111 self
.fileIsImportant
= fileIsImportant
113 self
.importantChanges
= []
114 self
.unimportantChanges
= []
115 self
.nextBuildTime
= None
118 def listBuilderNames(self
):
119 return self
.builderNames
121 def getPendingBuildTimes(self
):
122 if self
.nextBuildTime
is not None:
123 return [self
.nextBuildTime
]
126 def addChange(self
, change
):
127 if change
.branch
!= self
.branch
:
128 log
.msg("%s ignoring off-branch %s" % (self
, change
))
130 if not self
.fileIsImportant
:
131 self
.addImportantChange(change
)
132 elif self
.fileIsImportant(change
):
133 self
.addImportantChange(change
)
135 self
.addUnimportantChange(change
)
137 def addImportantChange(self
, change
):
138 log
.msg("%s: change is important, adding %s" % (self
, change
))
139 self
.importantChanges
.append(change
)
140 self
.nextBuildTime
= max(self
.nextBuildTime
,
141 change
.when
+ self
.treeStableTimer
)
142 self
.setTimer(self
.nextBuildTime
)
144 def addUnimportantChange(self
, change
):
145 log
.msg("%s: change is not important, adding %s" % (self
, change
))
146 self
.unimportantChanges
.append(change
)
148 def setTimer(self
, when
):
149 log
.msg("%s: setting timer to %s" %
150 (self
, time
.strftime("%H:%M:%S", time
.localtime(when
))))
156 self
.timer
= reactor
.callLater(when
- now
, self
.fireTimer
)
164 # clear out our state
166 self
.nextBuildTime
= None
167 changes
= self
.importantChanges
+ self
.unimportantChanges
168 self
.importantChanges
= []
169 self
.unimportantChanges
= []
171 # create a BuildSet, submit it to the BuildMaster
172 bs
= buildset
.BuildSet(self
.builderNames
,
173 SourceStamp(changes
=changes
))
176 def stopService(self
):
178 return service
.MultiService
.stopService(self
)
181 class AnyBranchScheduler(BaseUpstreamScheduler
):
182 """This Scheduler will handle changes on a variety of branches. It will
183 accumulate Changes for each branch separately. It works by creating a
184 separate Scheduler for each new branch it sees."""
186 schedulerFactory
= Scheduler
187 fileIsImportant
= None
189 compare_attrs
= ('name', 'branches', 'treeStableTimer', 'builderNames',
192 def __init__(self
, name
, branches
, treeStableTimer
, builderNames
,
193 fileIsImportant
=None):
195 @param name: the name of this Scheduler
196 @param branches: The branch names that the Scheduler should pay
197 attention to. Any Change that is not on one of these
198 branches will be ignored. It can be set to None to
199 accept changes from any branch. Don't use [] (an
200 empty list), because that means we don't pay
201 attention to *any* branches, so we'll never build
203 @param treeStableTimer: the duration, in seconds, for which the tree
204 must remain unchanged before a build will be
205 triggered. This is intended to avoid builds
206 of partially-committed fixes.
207 @param builderNames: a list of Builder names. When this Scheduler
208 decides to start a set of builds, they will be
209 run on the Builders named by this list.
211 @param fileIsImportant: A callable which takes one argument (a Change
212 instance) and returns True if the change is
213 worth building, and False if it is not.
214 Unimportant Changes are accumulated until the
215 build is triggered by an important change.
216 The default value of None means that all
217 Changes are important.
220 BaseUpstreamScheduler
.__init
__(self
, name
)
221 self
.treeStableTimer
= treeStableTimer
222 for b
in builderNames
:
223 assert isinstance(b
, str)
224 self
.builderNames
= builderNames
225 self
.branches
= branches
226 if self
.branches
== []:
227 log
.msg("AnyBranchScheduler %s: branches=[], so we will ignore "
228 "all branches, and never trigger any builds. Please set "
229 "branches=None to mean 'all branches'" % self
)
230 # consider raising an exception here, to make this warning more
231 # prominent, but I can vaguely imagine situations where you might
232 # want to comment out branches temporarily and wouldn't
233 # appreciate it being treated as an error.
235 assert callable(fileIsImportant
)
236 self
.fileIsImportant
= fileIsImportant
237 self
.schedulers
= {} # one per branch
240 return "<AnyBranchScheduler '%s'>" % self
.name
242 def listBuilderNames(self
):
243 return self
.builderNames
245 def getPendingBuildTimes(self
):
247 for s
in self
.schedulers
.values():
248 if s
.nextBuildTime
is not None:
249 bts
.append(s
.nextBuildTime
)
252 def addChange(self
, change
):
253 branch
= change
.branch
254 if self
.branches
is not None and branch
not in self
.branches
:
255 log
.msg("%s ignoring off-branch %s" % (self
, change
))
257 s
= self
.schedulers
.get(branch
)
260 name
= self
.name
+ "." + branch
262 name
= self
.name
+ ".<default>"
263 s
= self
.schedulerFactory(name
, branch
,
264 self
.treeStableTimer
,
266 self
.fileIsImportant
)
267 s
.successWatchers
= self
.successWatchers
268 s
.setServiceParent(self
)
269 # TODO: does this result in schedulers that stack up forever?
270 # When I make the persistify-pass, think about this some more.
271 self
.schedulers
[branch
] = s
274 def submitBuildSet(self
, bs
):
275 self
.parent
.submitBuildSet(bs
)
278 class Dependent(BaseUpstreamScheduler
):
279 """This scheduler runs some set of 'downstream' builds when the
280 'upstream' scheduler has completed successfully."""
282 compare_attrs
= ('name', 'upstream', 'builders')
284 def __init__(self
, name
, upstream
, builderNames
):
285 assert interfaces
.IUpstreamScheduler
.providedBy(upstream
)
286 BaseUpstreamScheduler
.__init
__(self
, name
)
287 self
.upstream
= upstream
288 self
.builderNames
= builderNames
290 def listBuilderNames(self
):
291 return self
.builderNames
293 def getPendingBuildTimes(self
):
294 # report the upstream's value
295 return self
.upstream
.getPendingBuildTimes()
297 def startService(self
):
298 service
.MultiService
.startService(self
)
299 self
.upstream
.subscribeToSuccessfulBuilds(self
.upstreamBuilt
)
301 def stopService(self
):
302 d
= service
.MultiService
.stopService(self
)
303 self
.upstream
.unsubscribeToSuccessfulBuilds(self
.upstreamBuilt
)
306 def upstreamBuilt(self
, ss
):
307 bs
= buildset
.BuildSet(self
.builderNames
, ss
)
312 class Periodic(BaseUpstreamScheduler
):
313 """Instead of watching for Changes, this Scheduler can just start a build
314 at fixed intervals. The C{periodicBuildTimer} parameter sets the number
315 of seconds to wait between such periodic builds. The first build will be
318 # TODO: consider having this watch another (changed-based) scheduler and
319 # merely enforce a minimum time between builds.
321 compare_attrs
= ('name', 'builderNames', 'periodicBuildTimer', 'branch')
323 def __init__(self
, name
, builderNames
, periodicBuildTimer
,
325 BaseUpstreamScheduler
.__init
__(self
, name
)
326 self
.builderNames
= builderNames
327 self
.periodicBuildTimer
= periodicBuildTimer
329 self
.reason
= ("The Periodic scheduler named '%s' triggered this build"
331 self
.timer
= internet
.TimerService(self
.periodicBuildTimer
,
332 self
.doPeriodicBuild
)
333 self
.timer
.setServiceParent(self
)
335 def listBuilderNames(self
):
336 return self
.builderNames
338 def getPendingBuildTimes(self
):
339 # TODO: figure out when self.timer is going to fire next and report
343 def doPeriodicBuild(self
):
344 bs
= buildset
.BuildSet(self
.builderNames
,
345 SourceStamp(branch
=self
.branch
),
351 class Nightly(BaseUpstreamScheduler
):
352 """Imitate 'cron' scheduling. This can be used to schedule a nightly
353 build, or one which runs are certain times of the day, week, or month.
355 Pass some subset of minute, hour, dayOfMonth, month, and dayOfWeek; each
356 may be a single number or a list of valid values. The builds will be
357 triggered whenever the current time matches these values. Wildcards are
358 represented by a '*' string. All fields default to a wildcard except
359 'minute', so with no fields this defaults to a build every hour, on the
362 For example, the following master.cfg clause will cause a build to be
363 started every night at 3:00am::
365 s = Nightly('nightly', ['builder1', 'builder2'], hour=3, minute=0)
366 c['schedules'].append(s)
368 This scheduler will perform a build each monday morning at 6:23am and
371 s = Nightly('BeforeWork', ['builder1'],
372 dayOfWeek=0, hour=[6,8], minute=23)
374 The following runs a build every two hours::
376 s = Nightly('every2hours', ['builder1'], hour=range(0, 24, 2))
378 And this one will run only on December 24th::
380 s = Nightly('SleighPreflightCheck', ['flying_circuits', 'radar'],
381 month=12, dayOfMonth=24, hour=12, minute=0)
383 For dayOfWeek and dayOfMonth, builds are triggered if the date matches
384 either of them. All time values are compared against the tuple returned
385 by time.localtime(), so month and dayOfMonth numbers start at 1, not
386 zero. dayOfWeek=0 is Monday, dayOfWeek=6 is Sunday.
389 compare_attrs
= ('name', 'builderNames',
390 'minute', 'hour', 'dayOfMonth', 'month',
391 'dayOfWeek', 'branch')
393 def __init__(self
, name
, builderNames
, minute
=0, hour
='*',
394 dayOfMonth
='*', month
='*', dayOfWeek
='*',
396 # Setting minute=0 really makes this an 'Hourly' scheduler. This
397 # seemed like a better default than minute='*', which would result in
398 # a build every 60 seconds.
399 BaseUpstreamScheduler
.__init
__(self
, name
)
400 self
.builderNames
= builderNames
403 self
.dayOfMonth
= dayOfMonth
405 self
.dayOfWeek
= dayOfWeek
407 self
.delayedRun
= None
408 self
.nextRunTime
= None
409 self
.reason
= ("The Nightly scheduler named '%s' triggered this build"
412 def addTime(self
, timetuple
, secs
):
413 return time
.localtime(time
.mktime(timetuple
)+secs
)
414 def findFirstValueAtLeast(self
, values
, value
, default
=None):
416 if v
>= value
: return v
420 self
.nextRunTime
= self
.calculateNextRunTime()
421 self
.delayedRun
= reactor
.callLater(self
.nextRunTime
- time
.time(),
422 self
.doPeriodicBuild
)
424 def startService(self
):
425 BaseUpstreamScheduler
.startService(self
)
428 def stopService(self
):
429 BaseUpstreamScheduler
.stopService(self
)
430 self
.delayedRun
.cancel()
432 def isRunTime(self
, timetuple
):
433 def check(ourvalue
, value
):
434 if ourvalue
== '*': return True
435 if isinstance(ourvalue
, int): return value
== ourvalue
436 return (value
in ourvalue
)
438 if not check(self
.minute
, timetuple
[4]):
439 #print 'bad minute', timetuple[4], self.minute
442 if not check(self
.hour
, timetuple
[3]):
443 #print 'bad hour', timetuple[3], self.hour
446 if not check(self
.month
, timetuple
[1]):
447 #print 'bad month', timetuple[1], self.month
450 if self
.dayOfMonth
!= '*' and self
.dayOfWeek
!= '*':
451 # They specified both day(s) of month AND day(s) of week.
452 # This means that we only have to match one of the two. If
453 # neither one matches, this time is not the right time.
454 if not (check(self
.dayOfMonth
, timetuple
[2]) or
455 check(self
.dayOfWeek
, timetuple
[6])):
459 if not check(self
.dayOfMonth
, timetuple
[2]):
460 #print 'bad day of month'
463 if not check(self
.dayOfWeek
, timetuple
[6]):
464 #print 'bad day of week'
469 def calculateNextRunTime(self
):
470 return self
.calculateNextRunTimeFrom(time
.time())
472 def calculateNextRunTimeFrom(self
, now
):
473 dateTime
= time
.localtime(now
)
475 # Remove seconds by advancing to at least the next minue
476 dateTime
= self
.addTime(dateTime
, 60-dateTime
[5])
478 # Now we just keep adding minutes until we find something that matches
480 # It not an efficient algorithm, but it'll *work* for now
481 yearLimit
= dateTime
[0]+2
482 while not self
.isRunTime(dateTime
):
483 dateTime
= self
.addTime(dateTime
, 60)
484 #print 'Trying', time.asctime(dateTime)
485 assert dateTime
[0] < yearLimit
, 'Something is wrong with this code'
486 return time
.mktime(dateTime
)
488 def listBuilderNames(self
):
489 return self
.builderNames
491 def getPendingBuildTimes(self
):
492 # TODO: figure out when self.timer is going to fire next and report
494 if self
.nextRunTime
is None: return []
495 return [self
.nextRunTime
]
497 def doPeriodicBuild(self
):
498 # Schedule the next run
501 # And trigger a build
502 bs
= buildset
.BuildSet(self
.builderNames
,
503 SourceStamp(branch
=self
.branch
),
507 def addChange(self
, change
):
512 class TryBase(service
.MultiService
, util
.ComparableMixin
):
513 implements(interfaces
.IScheduler
)
515 def __init__(self
, name
, builderNames
):
516 service
.MultiService
.__init
__(self
)
518 self
.builderNames
= builderNames
520 def listBuilderNames(self
):
521 return self
.builderNames
523 def getPendingBuildTimes(self
):
524 # we can't predict what the developers are going to do in the future
527 def addChange(self
, change
):
528 # Try schedulers ignore Changes
532 class BadJobfile(Exception):
535 class JobFileScanner(basic
.NetstringReceiver
):
538 self
.transport
= self
# so transport.loseConnection works
541 def stringReceived(self
, s
):
542 self
.strings
.append(s
)
544 def loseConnection(self
):
547 class Try_Jobdir(TryBase
):
548 compare_attrs
= ["name", "builderNames", "jobdir"]
550 def __init__(self
, name
, builderNames
, jobdir
):
551 TryBase
.__init
__(self
, name
, builderNames
)
553 self
.watcher
= MaildirService()
554 self
.watcher
.setServiceParent(self
)
556 def setServiceParent(self
, parent
):
557 self
.watcher
.setBasedir(os
.path
.join(parent
.basedir
, self
.jobdir
))
558 TryBase
.setServiceParent(self
, parent
)
560 def parseJob(self
, f
):
561 # jobfiles are serialized build requests. Each is a list of
562 # serialized netstrings, in the following order:
563 # "1", the version number of this format
564 # buildsetID, arbitrary string, used to find the buildSet later
565 # branch name, "" for default-branch
566 # base revision, "" for HEAD
567 # patchlevel, usually "1"
571 p
.dataReceived(f
.read())
573 raise BadJobfile("unable to parse netstrings")
577 raise BadJobfile("unknown version '%s'" % ver
)
578 buildsetID
, branch
, baserev
, patchlevel
, diff
= s
[:5]
584 patchlevel
= int(patchlevel
)
585 patch
= (patchlevel
, diff
)
586 ss
= SourceStamp(branch
, baserev
, patch
)
587 return builderNames
, ss
, buildsetID
589 def messageReceived(self
, filename
):
590 md
= os
.path
.join(self
.parent
.basedir
, self
.jobdir
)
591 if runtime
.platformType
== "posix":
592 # open the file before moving it, because I'm afraid that once
593 # it's in cur/, someone might delete it at any moment
594 path
= os
.path
.join(md
, "new", filename
)
596 os
.rename(os
.path
.join(md
, "new", filename
),
597 os
.path
.join(md
, "cur", filename
))
599 # do this backwards under windows, because you can't move a file
600 # that somebody is holding open. This was causing a Permission
601 # Denied error on bear's win32-twisted1.3 buildslave.
602 os
.rename(os
.path
.join(md
, "new", filename
),
603 os
.path
.join(md
, "cur", filename
))
604 path
= os
.path
.join(md
, "cur", filename
)
608 builderNames
, ss
, bsid
= self
.parseJob(f
)
610 log
.msg("%s reports a bad jobfile in %s" % (self
, filename
))
613 # compare builderNames against self.builderNames
614 # TODO: think about this some more.. why bother restricting it?
615 # perhaps self.builderNames should be used as the default list
616 # instead of being used as a restriction?
617 for b
in builderNames
:
618 if not b
in self
.builderNames
:
619 log
.msg("%s got jobfile %s with builder %s" % (self
,
621 log
.msg(" but that wasn't in our list: %s"
622 % (self
.builderNames
,))
626 bs
= buildset
.BuildSet(builderNames
, ss
, reason
=reason
, bsid
=bsid
)
627 self
.parent
.submitBuildSet(bs
)
629 class Try_Userpass(TryBase
):
630 compare_attrs
= ["name", "builderNames", "port", "userpass"]
631 implements(portal
.IRealm
)
633 def __init__(self
, name
, builderNames
, port
, userpass
):
634 TryBase
.__init
__(self
, name
, builderNames
)
635 if type(port
) is int:
636 port
= "tcp:%d" % port
638 self
.userpass
= userpass
639 c
= checkers
.InMemoryUsernamePasswordDatabaseDontUse()
640 for user
,passwd
in self
.userpass
:
641 c
.addUser(user
, passwd
)
643 p
= portal
.Portal(self
)
645 f
= pb
.PBServerFactory(p
)
646 s
= strports
.service(port
, f
)
647 s
.setServiceParent(self
)
650 # utility method for tests: figure out which TCP port we just opened.
651 return self
.services
[0]._port
.getHost().port
653 def requestAvatar(self
, avatarID
, mind
, interface
):
654 log
.msg("%s got connection from user %s" % (self
, avatarID
))
655 assert interface
== pb
.IPerspective
656 p
= Try_Userpass_Perspective(self
, avatarID
)
657 return (pb
.IPerspective
, p
, lambda: None)
659 def submitBuildSet(self
, bs
):
660 return self
.parent
.submitBuildSet(bs
)
662 class Try_Userpass_Perspective(pbutil
.NewCredPerspective
):
663 def __init__(self
, parent
, username
):
665 self
.username
= username
667 def perspective_try(self
, branch
, revision
, patch
, builderNames
):
668 log
.msg("user %s requesting build on builders %s" % (self
.username
,
670 for b
in builderNames
:
671 if not b
in self
.parent
.builderNames
:
672 log
.msg("%s got job with builder %s" % (self
, b
))
673 log
.msg(" but that wasn't in our list: %s"
674 % (self
.parent
.builderNames
,))
676 ss
= SourceStamp(branch
, revision
, patch
)
677 reason
= "'try' job from user %s" % self
.username
678 bs
= buildset
.BuildSet(builderNames
, ss
, reason
=reason
)
679 self
.parent
.submitBuildSet(bs
)
681 # return a remotely-usable BuildSetStatus object
682 from buildbot
.status
.client
import makeRemote
683 return makeRemote(bs
.status
)