buildbot/scheduler.py

   1 # -*- test-case-name: buildbot.test.test_dependencies -*-
   2
   3 import time, os.path
   4
   5 from zope.interface import implements
   6 from twisted.internet import reactor
   7 from twisted.application import service, internet, strports
   8 from twisted.python import log, runtime
   9 from twisted.protocols import basic
  10 from twisted.cred import portal, checkers
  11 from twisted.spread import pb
  12
  13 from buildbot import interfaces, buildset, util, pbutil
  14 from buildbot.status import builder
  15 from buildbot.sourcestamp import SourceStamp
  16 from buildbot.changes.maildir import MaildirService
  17 from buildbot.process.properties import Properties
  18
  19
  20 class BaseScheduler(service.MultiService, util.ComparableMixin):
  21     """
  22     A Schduler creates BuildSets and submits them to the BuildMaster.
  23
  24     @ivar name: name of the scheduler
  25
  26     @ivar properties: additional properties specified in this
  27         scheduler's configuration
  28     @type properties: Properties object
  29     """
  30     implements(interfaces.IScheduler)
  31
  32     def __init__(self, name, properties={}):
  33         """
  34         @param name: name for this scheduler
  35
  36         @param properties: properties to be propagated from this scheduler
  37         @type properties: dict
  38         """
  39         service.MultiService.__init__(self)
  40         self.name = name
  41         self.properties = Properties()
  42         self.properties.update(properties, "Scheduler")
  43         self.properties.setProperty("scheduler", name, "Scheduler")
  44
  45     def __repr__(self):
  46         # TODO: why can't id() return a positive number? %d is ugly.
  47         return "<Scheduler '%s' at %d>" % (self.name, id(self))
  48
  49     def submitBuildSet(self, bs):
  50         self.parent.submitBuildSet(bs)
  51
  52     def addChange(self, change):
  53         pass
  54
  55 class BaseUpstreamScheduler(BaseScheduler):
  56     implements(interfaces.IUpstreamScheduler)
  57
  58     def __init__(self, name, properties={}):
  59         BaseScheduler.__init__(self, name, properties)
  60         self.successWatchers = []
  61
  62     def subscribeToSuccessfulBuilds(self, watcher):
  63         self.successWatchers.append(watcher)
  64     def unsubscribeToSuccessfulBuilds(self, watcher):
  65         self.successWatchers.remove(watcher)
  66
  67     def submitBuildSet(self, bs):
  68         d = bs.waitUntilFinished()
  69         d.addCallback(self.buildSetFinished)
  70         BaseScheduler.submitBuildSet(self, bs)
  71
  72     def buildSetFinished(self, bss):
  73         if not self.running:
  74             return
  75         if bss.getResults() == builder.SUCCESS:
  76             ss = bss.getSourceStamp()
  77             for w in self.successWatchers:
  78                 w(ss)
  79
  80
  81 class Scheduler(BaseUpstreamScheduler):
  82     """The default Scheduler class will run a build after some period of time
  83     called the C{treeStableTimer}, on a given set of Builders. It only pays
  84     attention to a single branch. You you can provide a C{fileIsImportant}
  85     function which will evaluate each Change to decide whether or not it
  86     should trigger a new build.
  87     """
  88
  89     fileIsImportant = None
  90     compare_attrs = ('name', 'treeStableTimer', 'builderNames', 'branch',
  91                      'fileIsImportant', 'properties', 'categories')
  92
  93     def __init__(self, name, branch, treeStableTimer, builderNames,
  94                  fileIsImportant=None, properties={}, categories=None):
  95         """
  96         @param name: the name of this Scheduler
  97         @param branch: The branch name that the Scheduler should pay
  98                        attention to. Any Change that is not on this branch
  99                        will be ignored. It can be set to None to only pay
 100                        attention to the default branch.
 101         @param treeStableTimer: the duration, in seconds, for which the tree
 102                                 must remain unchanged before a build will be
 103                                 triggered. This is intended to avoid builds
 104                                 of partially-committed fixes.
 105         @param builderNames: a list of Builder names. When this Scheduler
 106                              decides to start a set of builds, they will be
 107                              run on the Builders named by this list.
 108
 109         @param fileIsImportant: A callable which takes one argument (a Change
 110                                 instance) and returns True if the change is
 111                                 worth building, and False if it is not.
 112                                 Unimportant Changes are accumulated until the
 113                                 build is triggered by an important change.
 114                                 The default value of None means that all
 115                                 Changes are important.
 116
 117         @param properties: properties to apply to all builds started from this
 118                            scheduler
 119         @param categories: A list of categories of changes to accept
 120         """
 121
 122         BaseUpstreamScheduler.__init__(self, name, properties)
 123         self.treeStableTimer = treeStableTimer
 124         errmsg = ("The builderNames= argument to Scheduler must be a list "
 125                   "of Builder description names (i.e. the 'name' key of the "
 126                   "Builder specification dictionary)")
 127         assert isinstance(builderNames, (list, tuple)), errmsg
 128         for b in builderNames:
 129             assert isinstance(b, str), errmsg
 130         self.builderNames = builderNames
 131         self.branch = branch
 132         if fileIsImportant:
 133             assert callable(fileIsImportant)
 134             self.fileIsImportant = fileIsImportant
 135
 136         self.importantChanges = []
 137         self.unimportantChanges = []
 138         self.nextBuildTime = None
 139         self.timer = None
 140         self.categories = categories
 141
 142     def listBuilderNames(self):
 143         return self.builderNames
 144
 145     def getPendingBuildTimes(self):
 146         if self.nextBuildTime is not None:
 147             return [self.nextBuildTime]
 148         return []
 149
 150     def addChange(self, change):
 151         if change.branch != self.branch:
 152             log.msg("%s ignoring off-branch %s" % (self, change))
 153             return
 154         if self.categories is not None and change.category not in self.categories:
 155             log.msg("%s ignoring non-matching categories %s" % (self, change))
 156             return
 157         if not self.fileIsImportant:
 158             self.addImportantChange(change)
 159         elif self.fileIsImportant(change):
 160             self.addImportantChange(change)
 161         else:
 162             self.addUnimportantChange(change)
 163
 164     def addImportantChange(self, change):
 165         log.msg("%s: change is important, adding %s" % (self, change))
 166         self.importantChanges.append(change)
 167         self.nextBuildTime = max(self.nextBuildTime,
 168                                  change.when + self.treeStableTimer)
 169         self.setTimer(self.nextBuildTime)
 170
 171     def addUnimportantChange(self, change):
 172         log.msg("%s: change is not important, adding %s" % (self, change))
 173         self.unimportantChanges.append(change)
 174
 175     def setTimer(self, when):
 176         log.msg("%s: setting timer to %s" %
 177                 (self, time.strftime("%H:%M:%S", time.localtime(when))))
 178         now = util.now()
 179         if when < now:
 180             when = now
 181         if self.timer:
 182             self.timer.cancel()
 183         self.timer = reactor.callLater(when - now, self.fireTimer)
 184
 185     def stopTimer(self):
 186         if self.timer:
 187             self.timer.cancel()
 188             self.timer = None
 189
 190     def fireTimer(self):
 191         # clear out our state
 192         self.timer = None
 193         self.nextBuildTime = None
 194         changes = self.importantChanges + self.unimportantChanges
 195         self.importantChanges = []
 196         self.unimportantChanges = []
 197
 198         # create a BuildSet, submit it to the BuildMaster
 199         bs = buildset.BuildSet(self.builderNames,
 200                                SourceStamp(changes=changes),
 201                                properties=self.properties)
 202         self.submitBuildSet(bs)
 203
 204     def stopService(self):
 205         self.stopTimer()
 206         return service.MultiService.stopService(self)
 207
 208
 209 class AnyBranchScheduler(BaseUpstreamScheduler):
 210     """This Scheduler will handle changes on a variety of branches. It will
 211     accumulate Changes for each branch separately. It works by creating a
 212     separate Scheduler for each new branch it sees."""
 213
 214     schedulerFactory = Scheduler
 215     fileIsImportant = None
 216
 217     compare_attrs = ('name', 'branches', 'treeStableTimer', 'builderNames',
 218                      'fileIsImportant', 'properties')
 219
 220     def __init__(self, name, branches, treeStableTimer, builderNames,
 221                  fileIsImportant=None, properties={}):
 222         """
 223         @param name: the name of this Scheduler
 224         @param branches: The branch names that the Scheduler should pay
 225                          attention to. Any Change that is not on one of these
 226                          branches will be ignored. It can be set to None to
 227                          accept changes from any branch. Don't use [] (an
 228                          empty list), because that means we don't pay
 229                          attention to *any* branches, so we'll never build
 230                          anything.
 231         @param treeStableTimer: the duration, in seconds, for which the tree
 232                                 must remain unchanged before a build will be
 233                                 triggered. This is intended to avoid builds
 234                                 of partially-committed fixes.
 235         @param builderNames: a list of Builder names. When this Scheduler
 236                              decides to start a set of builds, they will be
 237                              run on the Builders named by this list.
 238
 239         @param fileIsImportant: A callable which takes one argument (a Change
 240                                 instance) and returns True if the change is
 241                                 worth building, and False if it is not.
 242                                 Unimportant Changes are accumulated until the
 243                                 build is triggered by an important change.
 244                                 The default value of None means that all
 245                                 Changes are important.
 246
 247         @param properties: properties to apply to all builds started from this
 248                            scheduler
 249         """
 250
 251         BaseUpstreamScheduler.__init__(self, name, properties)
 252         self.treeStableTimer = treeStableTimer
 253         for b in builderNames:
 254             assert isinstance(b, str)
 255         self.builderNames = builderNames
 256         self.branches = branches
 257         if self.branches == []:
 258             log.msg("AnyBranchScheduler %s: branches=[], so we will ignore "
 259                     "all branches, and never trigger any builds. Please set "
 260                     "branches=None to mean 'all branches'" % self)
 261             # consider raising an exception here, to make this warning more
 262             # prominent, but I can vaguely imagine situations where you might
 263             # want to comment out branches temporarily and wouldn't
 264             # appreciate it being treated as an error.
 265         if fileIsImportant:
 266             assert callable(fileIsImportant)
 267             self.fileIsImportant = fileIsImportant
 268         self.schedulers = {} # one per branch
 269
 270     def __repr__(self):
 271         return "<AnyBranchScheduler '%s'>" % self.name
 272
 273     def listBuilderNames(self):
 274         return self.builderNames
 275
 276     def getPendingBuildTimes(self):
 277         bts = []
 278         for s in self.schedulers.values():
 279             if s.nextBuildTime is not None:
 280                 bts.append(s.nextBuildTime)
 281         return bts
 282
 283     def buildSetFinished(self, bss):
 284         # we don't care if a build has finished; one of the per-branch builders
 285         # will take care of it, instead.
 286         pass
 287
 288     def addChange(self, change):
 289         branch = change.branch
 290         if self.branches is not None and branch not in self.branches:
 291             log.msg("%s ignoring off-branch %s" % (self, change))
 292             return
 293         s = self.schedulers.get(branch)
 294         if not s:
 295             if branch:
 296                 name = self.name + "." + branch
 297             else:
 298                 name = self.name + ".<default>"
 299             s = self.schedulerFactory(name, branch,
 300                                       self.treeStableTimer,
 301                                       self.builderNames,
 302                                       self.fileIsImportant)
 303             s.successWatchers = self.successWatchers
 304             s.setServiceParent(self)
 305             s.properties = self.properties
 306             # TODO: does this result in schedulers that stack up forever?
 307             # When I make the persistify-pass, think about this some more.
 308             self.schedulers[branch] = s
 309         s.addChange(change)
 310
 311
 312 class Dependent(BaseUpstreamScheduler):
 313     """This scheduler runs some set of 'downstream' builds when the
 314     'upstream' scheduler has completed successfully."""
 315     implements(interfaces.IDownstreamScheduler)
 316
 317     compare_attrs = ('name', 'upstream', 'builderNames', 'properties')
 318
 319     def __init__(self, name, upstream, builderNames, properties={}):
 320         assert interfaces.IUpstreamScheduler.providedBy(upstream)
 321         BaseUpstreamScheduler.__init__(self, name, properties)
 322         self.upstream_name = upstream.name
 323         self.upstream = None
 324         self.builderNames = builderNames
 325
 326     def listBuilderNames(self):
 327         return self.builderNames
 328
 329     def getPendingBuildTimes(self):
 330         # report the upstream's value
 331         return self.getUpstreamScheduler().getPendingBuildTimes()
 332
 333     def startService(self):
 334         service.MultiService.startService(self)
 335         self.upstream = self.findUpstreamScheduler()
 336         self.upstream.subscribeToSuccessfulBuilds(self.upstreamBuilt)
 337
 338     def stopService(self):
 339         d = service.MultiService.stopService(self)
 340         self.upstream.unsubscribeToSuccessfulBuilds(self.upstreamBuilt)
 341         self.upstream = None
 342         return d
 343
 344     def upstreamBuilt(self, ss):
 345         bs = buildset.BuildSet(self.builderNames, ss,
 346                     properties=self.properties)
 347         self.submitBuildSet(bs)
 348
 349     def findUpstreamScheduler(self):
 350         # find our *active* upstream scheduler (which may not be self.upstream!) by name
 351         upstream = None
 352         for s in self.parent.allSchedulers():
 353             if s.name == self.upstream_name and interfaces.IUpstreamScheduler.providedBy(s):
 354                 upstream = s
 355         if not upstream:
 356             log.msg("ERROR: Couldn't find upstream scheduler of name <%s>" %
 357                 self.upstream_name)
 358         return upstream
 359
 360     def checkUpstreamScheduler(self):
 361         # if we don't already have an upstream, then there's nothing to worry about
 362         if not self.upstream:
 363             return
 364
 365         upstream = self.findUpstreamScheduler()
 366
 367         # if it's already correct, we're good to go
 368         if upstream is self.upstream:
 369             return
 370
 371         # otherwise, associate with the new upstream.  We also keep listening
 372         # to the old upstream, in case it's in the middle of a build
 373         upstream.subscribeToSuccessfulBuilds(self.upstreamBuilt)
 374         self.upstream = upstream
 375         log.msg("Dependent <%s> connected to new Upstream <%s>" %
 376                 (self.name, up_name))
 377
 378 class Periodic(BaseUpstreamScheduler):
 379     """Instead of watching for Changes, this Scheduler can just start a build
 380     at fixed intervals. The C{periodicBuildTimer} parameter sets the number
 381     of seconds to wait between such periodic builds. The first build will be
 382     run immediately."""
 383
 384     # TODO: consider having this watch another (changed-based) scheduler and
 385     # merely enforce a minimum time between builds.
 386
 387     compare_attrs = ('name', 'builderNames', 'periodicBuildTimer', 'branch', 'properties')
 388
 389     def __init__(self, name, builderNames, periodicBuildTimer,
 390                  branch=None, properties={}):
 391         BaseUpstreamScheduler.__init__(self, name, properties)
 392         self.builderNames = builderNames
 393         self.periodicBuildTimer = periodicBuildTimer
 394         self.branch = branch
 395         self.reason = ("The Periodic scheduler named '%s' triggered this build"
 396                        % name)
 397         self.timer = internet.TimerService(self.periodicBuildTimer,
 398                                            self.doPeriodicBuild)
 399         self.timer.setServiceParent(self)
 400
 401     def listBuilderNames(self):
 402         return self.builderNames
 403
 404     def getPendingBuildTimes(self):
 405         # TODO: figure out when self.timer is going to fire next and report
 406         # that
 407         return []
 408
 409     def doPeriodicBuild(self):
 410         bs = buildset.BuildSet(self.builderNames,
 411                                SourceStamp(branch=self.branch),
 412                                self.reason,
 413                                properties=self.properties)
 414         self.submitBuildSet(bs)
 415
 416
 417
 418 class Nightly(BaseUpstreamScheduler):
 419     """Imitate 'cron' scheduling. This can be used to schedule a nightly
 420     build, or one which runs are certain times of the day, week, or month.
 421
 422     Pass some subset of minute, hour, dayOfMonth, month, and dayOfWeek; each
 423     may be a single number or a list of valid values. The builds will be
 424     triggered whenever the current time matches these values. Wildcards are
 425     represented by a '*' string. All fields default to a wildcard except
 426     'minute', so with no fields this defaults to a build every hour, on the
 427     hour.
 428
 429     For example, the following master.cfg clause will cause a build to be
 430     started every night at 3:00am::
 431
 432      s = Nightly('nightly', ['builder1', 'builder2'], hour=3, minute=0)
 433      c['schedules'].append(s)
 434
 435     This scheduler will perform a build each monday morning at 6:23am and
 436     again at 8:23am::
 437
 438      s = Nightly('BeforeWork', ['builder1'],
 439                  dayOfWeek=0, hour=[6,8], minute=23)
 440
 441     The following runs a build every two hours::
 442
 443      s = Nightly('every2hours', ['builder1'], hour=range(0, 24, 2))
 444
 445     And this one will run only on December 24th::
 446
 447      s = Nightly('SleighPreflightCheck', ['flying_circuits', 'radar'],
 448                  month=12, dayOfMonth=24, hour=12, minute=0)
 449
 450     For dayOfWeek and dayOfMonth, builds are triggered if the date matches
 451     either of them. All time values are compared against the tuple returned
 452     by time.localtime(), so month and dayOfMonth numbers start at 1, not
 453     zero. dayOfWeek=0 is Monday, dayOfWeek=6 is Sunday.
 454
 455     onlyIfChanged functionality
 456     s = Nightly('nightly', ['builder1', 'builder2'],
 457                 hour=3, minute=0, onlyIfChanged=True)
 458     When the flag is True (False by default), the build is trigged if
 459     the date matches and if the branch has changed
 460
 461     fileIsImportant parameter is implemented as defined in class Scheduler
 462     """
 463
 464     compare_attrs = ('name', 'builderNames',
 465                      'minute', 'hour', 'dayOfMonth', 'month',
 466                      'dayOfWeek', 'branch', 'onlyIfChanged',
 467                      'fileIsImportant', 'properties')
 468
 469     def __init__(self, name, builderNames, minute=0, hour='*',
 470                  dayOfMonth='*', month='*', dayOfWeek='*',
 471                  branch=None, fileIsImportant=None, onlyIfChanged=False, properties={}):
 472         # Setting minute=0 really makes this an 'Hourly' scheduler. This
 473         # seemed like a better default than minute='*', which would result in
 474         # a build every 60 seconds.
 475         BaseUpstreamScheduler.__init__(self, name, properties)
 476         self.builderNames = builderNames
 477         self.minute = minute
 478         self.hour = hour
 479         self.dayOfMonth = dayOfMonth
 480         self.month = month
 481         self.dayOfWeek = dayOfWeek
 482         self.branch = branch
 483         self.onlyIfChanged = onlyIfChanged
 484         self.delayedRun = None
 485         self.nextRunTime = None
 486         self.reason = ("The Nightly scheduler named '%s' triggered this build"
 487                        % name)
 488
 489         self.importantChanges   = []
 490         self.unimportantChanges = []
 491         self.fileIsImportant    = None
 492         if fileIsImportant:
 493             assert callable(fileIsImportant)
 494             self.fileIsImportant = fileIsImportant
 495
 496     def addTime(self, timetuple, secs):
 497         return time.localtime(time.mktime(timetuple)+secs)
 498     def findFirstValueAtLeast(self, values, value, default=None):
 499         for v in values:
 500             if v >= value: return v
 501         return default
 502
 503     def setTimer(self):
 504         self.nextRunTime = self.calculateNextRunTime()
 505         self.delayedRun = reactor.callLater(self.nextRunTime - time.time(),
 506                                             self.doPeriodicBuild)
 507
 508     def startService(self):
 509         BaseUpstreamScheduler.startService(self)
 510         self.setTimer()
 511
 512     def stopService(self):
 513         BaseUpstreamScheduler.stopService(self)
 514         self.delayedRun.cancel()
 515
 516     def isRunTime(self, timetuple):
 517         def check(ourvalue, value):
 518             if ourvalue == '*': return True
 519             if isinstance(ourvalue, int): return value == ourvalue
 520             return (value in ourvalue)
 521
 522         if not check(self.minute, timetuple[4]):
 523             #print 'bad minute', timetuple[4], self.minute
 524             return False
 525
 526         if not check(self.hour, timetuple[3]):
 527             #print 'bad hour', timetuple[3], self.hour
 528             return False
 529
 530         if not check(self.month, timetuple[1]):
 531             #print 'bad month', timetuple[1], self.month
 532             return False
 533
 534         if self.dayOfMonth != '*' and self.dayOfWeek != '*':
 535             # They specified both day(s) of month AND day(s) of week.
 536             # This means that we only have to match one of the two. If
 537             # neither one matches, this time is not the right time.
 538             if not (check(self.dayOfMonth, timetuple[2]) or
 539                     check(self.dayOfWeek, timetuple[6])):
 540                 #print 'bad day'
 541                 return False
 542         else:
 543             if not check(self.dayOfMonth, timetuple[2]):
 544                 #print 'bad day of month'
 545                 return False
 546
 547             if not check(self.dayOfWeek, timetuple[6]):
 548                 #print 'bad day of week'
 549                 return False
 550
 551         return True
 552
 553     def calculateNextRunTime(self):
 554         return self.calculateNextRunTimeFrom(time.time())
 555
 556     def calculateNextRunTimeFrom(self, now):
 557         dateTime = time.localtime(now)
 558
 559         # Remove seconds by advancing to at least the next minue
 560         dateTime = self.addTime(dateTime, 60-dateTime[5])
 561
 562         # Now we just keep adding minutes until we find something that matches
 563
 564         # It not an efficient algorithm, but it'll *work* for now
 565         yearLimit = dateTime[0]+2
 566         while not self.isRunTime(dateTime):
 567             dateTime = self.addTime(dateTime, 60)
 568             #print 'Trying', time.asctime(dateTime)
 569             assert dateTime[0] < yearLimit, 'Something is wrong with this code'
 570         return time.mktime(dateTime)
 571
 572     def listBuilderNames(self):
 573         return self.builderNames
 574
 575     def getPendingBuildTimes(self):
 576         # TODO: figure out when self.timer is going to fire next and report
 577         # that
 578         if self.nextRunTime is None: return []
 579         return [self.nextRunTime]
 580
 581     def doPeriodicBuild(self):
 582         # Schedule the next run
 583         self.setTimer()
 584
 585         if  self.onlyIfChanged:
 586             if len(self.importantChanges) > 0:
 587                 changes = self.importantChanges + self.unimportantChanges
 588                 # And trigger a build
 589                 log.msg("Nightly Scheduler <%s>: triggering build" % self.name)
 590                 bs = buildset.BuildSet(self.builderNames,
 591                                        SourceStamp(changes=changes),
 592                                        self.reason,
 593                                        properties=self.properties)
 594                 self.submitBuildSet(bs)
 595                 # Reset the change lists
 596                 self.importantChanges = []
 597                 self.unimportantChanges = []
 598             else:
 599                 log.msg("Nightly Scheduler <%s>: skipping build - No important change" % self.name)
 600         else:
 601             # And trigger a build
 602             bs = buildset.BuildSet(self.builderNames,
 603                                    SourceStamp(branch=self.branch),
 604                                    self.reason,
 605                                    properties=self.properties)
 606             self.submitBuildSet(bs)
 607
 608     def addChange(self, change):
 609         if  self.onlyIfChanged:
 610             if change.branch != self.branch:
 611                 log.msg("Nightly Scheduler <%s>: ignoring change %s on off-branch %s" % (self.name, change.revision, change.branch))
 612                 return
 613             if not self.fileIsImportant:
 614                 self.addImportantChange(change)
 615             elif self.fileIsImportant(change):
 616                 self.addImportantChange(change)
 617             else:
 618                 self.addUnimportantChange(change)
 619         else:
 620             log.msg("Nightly Scheduler <%s>: no add change" % self.name)
 621             pass
 622
 623     def addImportantChange(self, change):
 624         log.msg("Nightly Scheduler <%s>: change %s from %s is important, adding it" % (self.name, change.revision, change.who))
 625         self.importantChanges.append(change)
 626
 627     def addUnimportantChange(self, change):
 628         log.msg("Nightly Scheduler <%s>: change %s from %s is not important, adding it" % (self.name, change.revision, change.who))
 629         self.unimportantChanges.append(change)
 630
 631
 632 class TryBase(BaseScheduler):
 633     def __init__(self, name, builderNames, properties={}):
 634         BaseScheduler.__init__(self, name, properties)
 635         self.builderNames = builderNames
 636
 637     def listBuilderNames(self):
 638         return self.builderNames
 639
 640     def getPendingBuildTimes(self):
 641         # we can't predict what the developers are going to do in the future
 642         return []
 643
 644     def addChange(self, change):
 645         # Try schedulers ignore Changes
 646         pass
 647
 648     def processBuilderList(self, builderNames):
 649         # self.builderNames is the configured list of builders
 650         # available for try.  If the user supplies a list of builders,
 651         # it must be restricted to the configured list.  If not, build
 652         # on all of the configured builders.
 653         if builderNames:
 654             for b in builderNames:
 655                 if not b in self.builderNames:
 656                     log.msg("%s got with builder %s" % (self, b))
 657                     log.msg(" but that wasn't in our list: %s"
 658                             % (self.builderNames,))
 659                     return []
 660         else:
 661             builderNames = self.builderNames
 662         return builderNames
 663
 664 class BadJobfile(Exception):
 665     pass
 666
 667 class JobFileScanner(basic.NetstringReceiver):
 668     def __init__(self):
 669         self.strings = []
 670         self.transport = self # so transport.loseConnection works
 671         self.error = False
 672
 673     def stringReceived(self, s):
 674         self.strings.append(s)
 675
 676     def loseConnection(self):
 677         self.error = True
 678
 679 class Try_Jobdir(TryBase):
 680     compare_attrs = ( 'name', 'builderNames', 'jobdir', 'properties' )
 681
 682     def __init__(self, name, builderNames, jobdir, properties={}):
 683         TryBase.__init__(self, name, builderNames, properties)
 684         self.jobdir = jobdir
 685         self.watcher = MaildirService()
 686         self.watcher.setServiceParent(self)
 687
 688     def setServiceParent(self, parent):
 689         self.watcher.setBasedir(os.path.join(parent.basedir, self.jobdir))
 690         TryBase.setServiceParent(self, parent)
 691
 692     def parseJob(self, f):
 693         # jobfiles are serialized build requests. Each is a list of
 694         # serialized netstrings, in the following order:
 695         #  "1", the version number of this format
 696         #  buildsetID, arbitrary string, used to find the buildSet later
 697         #  branch name, "" for default-branch
 698         #  base revision, "" for HEAD
 699         #  patchlevel, usually "1"
 700         #  patch
 701         #  builderNames...
 702         p = JobFileScanner()
 703         p.dataReceived(f.read())
 704         if p.error:
 705             raise BadJobfile("unable to parse netstrings")
 706         s = p.strings
 707         ver = s.pop(0)
 708         if ver != "1":
 709             raise BadJobfile("unknown version '%s'" % ver)
 710         buildsetID, branch, baserev, patchlevel, diff = s[:5]
 711         builderNames = s[5:]
 712         if branch == "":
 713             branch = None
 714         if baserev == "":
 715             baserev = None
 716         patchlevel = int(patchlevel)
 717         patch = (patchlevel, diff)
 718         ss = SourceStamp(branch, baserev, patch)
 719         return builderNames, ss, buildsetID
 720
 721     def messageReceived(self, filename):
 722         md = os.path.join(self.parent.basedir, self.jobdir)
 723         if runtime.platformType == "posix":
 724             # open the file before moving it, because I'm afraid that once
 725             # it's in cur/, someone might delete it at any moment
 726             path = os.path.join(md, "new", filename)
 727             f = open(path, "r")
 728             os.rename(os.path.join(md, "new", filename),
 729                       os.path.join(md, "cur", filename))
 730         else:
 731             # do this backwards under windows, because you can't move a file
 732             # that somebody is holding open. This was causing a Permission
 733             # Denied error on bear's win32-twisted1.3 buildslave.
 734             os.rename(os.path.join(md, "new", filename),
 735                       os.path.join(md, "cur", filename))
 736             path = os.path.join(md, "cur", filename)
 737             f = open(path, "r")
 738
 739         try:
 740             builderNames, ss, bsid = self.parseJob(f)
 741         except BadJobfile:
 742             log.msg("%s reports a bad jobfile in %s" % (self, filename))
 743             log.err()
 744             return
 745         # Validate/fixup the builder names.
 746         builderNames = self.processBuilderList(builderNames)
 747         if not builderNames:
 748             return
 749         reason = "'try' job"
 750         bs = buildset.BuildSet(builderNames, ss, reason=reason,
 751                     bsid=bsid, properties=self.properties)
 752         self.submitBuildSet(bs)
 753
 754 class Try_Userpass(TryBase):
 755     compare_attrs = ( 'name', 'builderNames', 'port', 'userpass', 'properties' )
 756     implements(portal.IRealm)
 757
 758     def __init__(self, name, builderNames, port, userpass, properties={}):
 759         TryBase.__init__(self, name, builderNames, properties)
 760         if type(port) is int:
 761             port = "tcp:%d" % port
 762         self.port = port
 763         self.userpass = userpass
 764         c = checkers.InMemoryUsernamePasswordDatabaseDontUse()
 765         for user,passwd in self.userpass:
 766             c.addUser(user, passwd)
 767
 768         p = portal.Portal(self)
 769         p.registerChecker(c)
 770         f = pb.PBServerFactory(p)
 771         s = strports.service(port, f)
 772         s.setServiceParent(self)
 773
 774     def getPort(self):
 775         # utility method for tests: figure out which TCP port we just opened.
 776         return self.services[0]._port.getHost().port
 777
 778     def requestAvatar(self, avatarID, mind, interface):
 779         log.msg("%s got connection from user %s" % (self, avatarID))
 780         assert interface == pb.IPerspective
 781         p = Try_Userpass_Perspective(self, avatarID)
 782         return (pb.IPerspective, p, lambda: None)
 783
 784 class Try_Userpass_Perspective(pbutil.NewCredPerspective):
 785     def __init__(self, parent, username):
 786         self.parent = parent
 787         self.username = username
 788
 789     def perspective_try(self, branch, revision, patch, builderNames, properties={}):
 790         log.msg("user %s requesting build on builders %s" % (self.username,
 791                                                              builderNames))
 792         # Validate/fixup the builder names.
 793         builderNames = self.parent.processBuilderList(builderNames)
 794         if not builderNames:
 795             return
 796         ss = SourceStamp(branch, revision, patch)
 797         reason = "'try' job from user %s" % self.username
 798
 799         # roll the specified props in with our inherited props
 800         combined_props = Properties()
 801         combined_props.updateFromProperties(self.parent.properties)
 802         combined_props.update(properties, "try build")
 803
 804         bs = buildset.BuildSet(builderNames,
 805                                ss,
 806                                reason=reason,
 807                                properties=combined_props)
 808
 809         self.parent.submitBuildSet(bs)
 810
 811         # return a remotely-usable BuildSetStatus object
 812         from buildbot.status.client import makeRemote
 813         return makeRemote(bs.status)
 814
 815 class Triggerable(BaseUpstreamScheduler):
 816     """This scheduler doesn't do anything until it is triggered by a Trigger
 817     step in a factory. In general, that step will not complete until all of
 818     the builds that I fire have finished.
 819     """
 820
 821     compare_attrs = ('name', 'builderNames', 'properties')
 822
 823     def __init__(self, name, builderNames, properties={}):
 824         BaseUpstreamScheduler.__init__(self, name, properties)
 825         self.builderNames = builderNames
 826
 827     def listBuilderNames(self):
 828         return self.builderNames
 829
 830     def getPendingBuildTimes(self):
 831         return []
 832
 833     def trigger(self, ss, set_props=None):
 834         """Trigger this scheduler. Returns a deferred that will fire when the
 835         buildset is finished.
 836         """
 837
 838         # properties for this buildset are composed of our own properties,
 839         # potentially overridden by anything from the triggering build
 840         props = Properties()
 841         props.updateFromProperties(self.properties)
 842         if set_props: props.updateFromProperties(set_props)
 843
 844         bs = buildset.BuildSet(self.builderNames, ss, properties=props)
 845         d = bs.waitUntilFinished()
 846         self.submitBuildSet(bs)
 847         return d