buildbot/scheduler.py

   1 # -*- test-case-name: buildbot.test.test_dependencies -*-
   2
   3 import time, os.path
   4
   5 from zope.interface import implements
   6 from twisted.internet import reactor
   7 from twisted.application import service, internet, strports
   8 from twisted.python import log, runtime
   9 from twisted.protocols import basic
  10 from twisted.cred import portal, checkers
  11 from twisted.spread import pb
  12
  13 from buildbot import interfaces, buildset, util, pbutil
  14 from buildbot.status import builder
  15 from buildbot.sourcestamp import SourceStamp
  16 from buildbot.changes.maildir import MaildirService
  17 from buildbot.process.properties import Properties
  18
  19
  20 class BaseScheduler(service.MultiService, util.ComparableMixin):
  21     """
  22     A Schduler creates BuildSets and submits them to the BuildMaster.
  23
  24     @ivar name: name of the scheduler
  25
  26     @ivar properties: additional properties specified in this
  27         scheduler's configuration
  28     @type properties: Properties object
  29     """
  30     implements(interfaces.IScheduler)
  31
  32     def __init__(self, name, properties={}):
  33         """
  34         @param name: name for this scheduler
  35
  36         @param properties: properties to be propagated from this scheduler
  37         @type properties: dict
  38         """
  39         service.MultiService.__init__(self)
  40         self.name = name
  41         self.properties = Properties()
  42         self.properties.update(properties, "Scheduler")
  43         self.properties.setProperty("scheduler", name, "Scheduler")
  44
  45     def __repr__(self):
  46         # TODO: why can't id() return a positive number? %d is ugly.
  47         return "<Scheduler '%s' at %d>" % (self.name, id(self))
  48
  49     def submitBuildSet(self, bs):
  50         self.parent.submitBuildSet(bs)
  51
  52     def addChange(self, change):
  53         pass
  54
  55 class BaseUpstreamScheduler(BaseScheduler):
  56     implements(interfaces.IUpstreamScheduler)
  57
  58     def __init__(self, name, properties={}):
  59         BaseScheduler.__init__(self, name, properties)
  60         self.successWatchers = []
  61
  62     def subscribeToSuccessfulBuilds(self, watcher):
  63         self.successWatchers.append(watcher)
  64     def unsubscribeToSuccessfulBuilds(self, watcher):
  65         self.successWatchers.remove(watcher)
  66
  67     def submitBuildSet(self, bs):
  68         d = bs.waitUntilFinished()
  69         d.addCallback(self.buildSetFinished)
  70         BaseScheduler.submitBuildSet(self, bs)
  71
  72     def buildSetFinished(self, bss):
  73         if not self.running:
  74             return
  75         if bss.getResults() == builder.SUCCESS:
  76             ss = bss.getSourceStamp()
  77             for w in self.successWatchers:
  78                 w(ss)
  79
  80
  81 class Scheduler(BaseUpstreamScheduler):
  82     """The default Scheduler class will run a build after some period of time
  83     called the C{treeStableTimer}, on a given set of Builders. It only pays
  84     attention to a single branch. You you can provide a C{fileIsImportant}
  85     function which will evaluate each Change to decide whether or not it
  86     should trigger a new build.
  87     """
  88
  89     fileIsImportant = None
  90     compare_attrs = ('name', 'treeStableTimer', 'builderNames', 'branch',
  91                      'fileIsImportant', 'properties', 'categories')
  92
  93     def __init__(self, name, branch, treeStableTimer, builderNames,
  94                  fileIsImportant=None, properties={}, categories=None):
  95         """
  96         @param name: the name of this Scheduler
  97         @param branch: The branch name that the Scheduler should pay
  98                        attention to. Any Change that is not on this branch
  99                        will be ignored. It can be set to None to only pay
 100                        attention to the default branch.
 101         @param treeStableTimer: the duration, in seconds, for which the tree
 102                                 must remain unchanged before a build will be
 103                                 triggered. This is intended to avoid builds
 104                                 of partially-committed fixes.
 105         @param builderNames: a list of Builder names. When this Scheduler
 106                              decides to start a set of builds, they will be
 107                              run on the Builders named by this list.
 108
 109         @param fileIsImportant: A callable which takes one argument (a Change
 110                                 instance) and returns True if the change is
 111                                 worth building, and False if it is not.
 112                                 Unimportant Changes are accumulated until the
 113                                 build is triggered by an important change.
 114                                 The default value of None means that all
 115                                 Changes are important.
 116
 117         @param properties: properties to apply to all builds started from this
 118                            scheduler
 119         @param categories: A list of categories of changes to accept
 120         """
 121
 122         BaseUpstreamScheduler.__init__(self, name, properties)
 123         self.treeStableTimer = treeStableTimer
 124         errmsg = ("The builderNames= argument to Scheduler must be a list "
 125                   "of Builder description names (i.e. the 'name' key of the "
 126                   "Builder specification dictionary)")
 127         assert isinstance(builderNames, (list, tuple)), errmsg
 128         for b in builderNames:
 129             assert isinstance(b, str), errmsg
 130         self.builderNames = builderNames
 131         self.branch = branch
 132         if fileIsImportant:
 133             assert callable(fileIsImportant)
 134             self.fileIsImportant = fileIsImportant
 135
 136         self.importantChanges = []
 137         self.unimportantChanges = []
 138         self.nextBuildTime = None
 139         self.timer = None
 140         self.categories = categories
 141
 142     def listBuilderNames(self):
 143         return self.builderNames
 144
 145     def getPendingBuildTimes(self):
 146         if self.nextBuildTime is not None:
 147             return [self.nextBuildTime]
 148         return []
 149
 150     def addChange(self, change):
 151         if change.branch != self.branch:
 152             log.msg("%s ignoring off-branch %s" % (self, change))
 153             return
 154         if self.categories is not None and change.category not in self.categories:
 155             log.msg("%s ignoring non-matching categories %s" % (self, change))
 156             return
 157         if not self.fileIsImportant:
 158             self.addImportantChange(change)
 159         elif self.fileIsImportant(change):
 160             self.addImportantChange(change)
 161         else:
 162             self.addUnimportantChange(change)
 163
 164     def addImportantChange(self, change):
 165         log.msg("%s: change is important, adding %s" % (self, change))
 166         self.importantChanges.append(change)
 167         self.nextBuildTime = max(self.nextBuildTime,
 168                                  change.when + self.treeStableTimer)
 169         self.setTimer(self.nextBuildTime)
 170
 171     def addUnimportantChange(self, change):
 172         log.msg("%s: change is not important, adding %s" % (self, change))
 173         self.unimportantChanges.append(change)
 174
 175     def setTimer(self, when):
 176         log.msg("%s: setting timer to %s" %
 177                 (self, time.strftime("%H:%M:%S", time.localtime(when))))
 178         now = util.now()
 179         if when < now:
 180             when = now
 181         if self.timer:
 182             self.timer.cancel()
 183         self.timer = reactor.callLater(when - now, self.fireTimer)
 184
 185     def stopTimer(self):
 186         if self.timer:
 187             self.timer.cancel()
 188             self.timer = None
 189
 190     def fireTimer(self):
 191         # clear out our state
 192         self.timer = None
 193         self.nextBuildTime = None
 194         changes = self.importantChanges + self.unimportantChanges
 195         self.importantChanges = []
 196         self.unimportantChanges = []
 197
 198         # create a BuildSet, submit it to the BuildMaster
 199         bs = buildset.BuildSet(self.builderNames,
 200                                SourceStamp(changes=changes),
 201                                properties=self.properties)
 202         self.submitBuildSet(bs)
 203
 204     def stopService(self):
 205         self.stopTimer()
 206         return service.MultiService.stopService(self)
 207
 208
 209 class AnyBranchScheduler(BaseUpstreamScheduler):
 210     """This Scheduler will handle changes on a variety of branches. It will
 211     accumulate Changes for each branch separately. It works by creating a
 212     separate Scheduler for each new branch it sees."""
 213
 214     schedulerFactory = Scheduler
 215     fileIsImportant = None
 216
 217     compare_attrs = ('name', 'branches', 'treeStableTimer', 'builderNames',
 218                      'fileIsImportant', 'properties')
 219
 220     def __init__(self, name, branches, treeStableTimer, builderNames,
 221                  fileIsImportant=None, properties={}):
 222         """
 223         @param name: the name of this Scheduler
 224         @param branches: The branch names that the Scheduler should pay
 225                          attention to. Any Change that is not on one of these
 226                          branches will be ignored. It can be set to None to
 227                          accept changes from any branch. Don't use [] (an
 228                          empty list), because that means we don't pay
 229                          attention to *any* branches, so we'll never build
 230                          anything.
 231         @param treeStableTimer: the duration, in seconds, for which the tree
 232                                 must remain unchanged before a build will be
 233                                 triggered. This is intended to avoid builds
 234                                 of partially-committed fixes.
 235         @param builderNames: a list of Builder names. When this Scheduler
 236                              decides to start a set of builds, they will be
 237                              run on the Builders named by this list.
 238
 239         @param fileIsImportant: A callable which takes one argument (a Change
 240                                 instance) and returns True if the change is
 241                                 worth building, and False if it is not.
 242                                 Unimportant Changes are accumulated until the
 243                                 build is triggered by an important change.
 244                                 The default value of None means that all
 245                                 Changes are important.
 246
 247         @param properties: properties to apply to all builds started from this
 248                            scheduler
 249         """
 250
 251         BaseUpstreamScheduler.__init__(self, name, properties)
 252         self.treeStableTimer = treeStableTimer
 253         for b in builderNames:
 254             assert isinstance(b, str)
 255         self.builderNames = builderNames
 256         self.branches = branches
 257         if self.branches == []:
 258             log.msg("AnyBranchScheduler %s: branches=[], so we will ignore "
 259                     "all branches, and never trigger any builds. Please set "
 260                     "branches=None to mean 'all branches'" % self)
 261             # consider raising an exception here, to make this warning more
 262             # prominent, but I can vaguely imagine situations where you might
 263             # want to comment out branches temporarily and wouldn't
 264             # appreciate it being treated as an error.
 265         if fileIsImportant:
 266             assert callable(fileIsImportant)
 267             self.fileIsImportant = fileIsImportant
 268         self.schedulers = {} # one per branch
 269
 270     def __repr__(self):
 271         return "<AnyBranchScheduler '%s'>" % self.name
 272
 273     def listBuilderNames(self):
 274         return self.builderNames
 275
 276     def getPendingBuildTimes(self):
 277         bts = []
 278         for s in self.schedulers.values():
 279             if s.nextBuildTime is not None:
 280                 bts.append(s.nextBuildTime)
 281         return bts
 282
 283     def buildSetFinished(self, bss):
 284         # we don't care if a build has finished; one of the per-branch builders
 285         # will take care of it, instead.
 286         pass
 287
 288     def addChange(self, change):
 289         branch = change.branch
 290         if self.branches is not None and branch not in self.branches:
 291             log.msg("%s ignoring off-branch %s" % (self, change))
 292             return
 293         s = self.schedulers.get(branch)
 294         if not s:
 295             if branch:
 296                 name = self.name + "." + branch
 297             else:
 298                 name = self.name + ".<default>"
 299             s = self.schedulerFactory(name, branch,
 300                                       self.treeStableTimer,
 301                                       self.builderNames,
 302                                       self.fileIsImportant)
 303             s.successWatchers = self.successWatchers
 304             s.setServiceParent(self)
 305             s.properties = self.properties
 306             # TODO: does this result in schedulers that stack up forever?
 307             # When I make the persistify-pass, think about this some more.
 308             self.schedulers[branch] = s
 309         s.addChange(change)
 310
 311
 312 class Dependent(BaseUpstreamScheduler):
 313     """This scheduler runs some set of 'downstream' builds when the
 314     'upstream' scheduler has completed successfully."""
 315     implements(interfaces.IDownstreamScheduler)
 316
 317     compare_attrs = ('name', 'upstream', 'builderNames', 'properties')
 318
 319     def __init__(self, name, upstream, builderNames, properties={}):
 320         assert interfaces.IUpstreamScheduler.providedBy(upstream)
 321         BaseUpstreamScheduler.__init__(self, name, properties)
 322         self.upstream = upstream
 323         self.builderNames = builderNames
 324
 325     def listBuilderNames(self):
 326         return self.builderNames
 327
 328     def getPendingBuildTimes(self):
 329         # report the upstream's value
 330         return self.upstream.getPendingBuildTimes()
 331
 332     def startService(self):
 333         service.MultiService.startService(self)
 334         self.upstream.subscribeToSuccessfulBuilds(self.upstreamBuilt)
 335
 336     def stopService(self):
 337         d = service.MultiService.stopService(self)
 338         self.upstream.unsubscribeToSuccessfulBuilds(self.upstreamBuilt)
 339         return d
 340
 341     def upstreamBuilt(self, ss):
 342         bs = buildset.BuildSet(self.builderNames, ss,
 343                     properties=self.properties)
 344         self.submitBuildSet(bs)
 345
 346     def checkUpstreamScheduler(self):
 347         # find our *active* upstream scheduler (which may not be self.upstream!) by name
 348         up_name = self.upstream.name
 349         upstream = None
 350         for s in self.parent.allSchedulers():
 351             if s.name == up_name and interfaces.IUpstreamScheduler.providedBy(s):
 352                 upstream = s
 353         if not upstream:
 354             log.msg("ERROR: Couldn't find upstream scheduler of name <%s>" %
 355                 up_name)
 356
 357         # if it's already correct, we're good to go
 358         if upstream is self.upstream:
 359             return
 360
 361         # otherwise, associate with the new upstream.  We also keep listening
 362         # to the old upstream, in case it's in the middle of a build
 363         upstream.subscribeToSuccessfulBuilds(self.upstreamBuilt)
 364         self.upstream = upstream
 365         log.msg("Dependent <%s> connected to new Upstream <%s>" %
 366                 (self.name, up_name))
 367
 368 class Periodic(BaseUpstreamScheduler):
 369     """Instead of watching for Changes, this Scheduler can just start a build
 370     at fixed intervals. The C{periodicBuildTimer} parameter sets the number
 371     of seconds to wait between such periodic builds. The first build will be
 372     run immediately."""
 373
 374     # TODO: consider having this watch another (changed-based) scheduler and
 375     # merely enforce a minimum time between builds.
 376
 377     compare_attrs = ('name', 'builderNames', 'periodicBuildTimer', 'branch', 'properties')
 378
 379     def __init__(self, name, builderNames, periodicBuildTimer,
 380                  branch=None, properties={}):
 381         BaseUpstreamScheduler.__init__(self, name, properties)
 382         self.builderNames = builderNames
 383         self.periodicBuildTimer = periodicBuildTimer
 384         self.branch = branch
 385         self.reason = ("The Periodic scheduler named '%s' triggered this build"
 386                        % name)
 387         self.timer = internet.TimerService(self.periodicBuildTimer,
 388                                            self.doPeriodicBuild)
 389         self.timer.setServiceParent(self)
 390
 391     def listBuilderNames(self):
 392         return self.builderNames
 393
 394     def getPendingBuildTimes(self):
 395         # TODO: figure out when self.timer is going to fire next and report
 396         # that
 397         return []
 398
 399     def doPeriodicBuild(self):
 400         bs = buildset.BuildSet(self.builderNames,
 401                                SourceStamp(branch=self.branch),
 402                                self.reason,
 403                                properties=self.properties)
 404         self.submitBuildSet(bs)
 405
 406
 407
 408 class Nightly(BaseUpstreamScheduler):
 409     """Imitate 'cron' scheduling. This can be used to schedule a nightly
 410     build, or one which runs are certain times of the day, week, or month.
 411
 412     Pass some subset of minute, hour, dayOfMonth, month, and dayOfWeek; each
 413     may be a single number or a list of valid values. The builds will be
 414     triggered whenever the current time matches these values. Wildcards are
 415     represented by a '*' string. All fields default to a wildcard except
 416     'minute', so with no fields this defaults to a build every hour, on the
 417     hour.
 418
 419     For example, the following master.cfg clause will cause a build to be
 420     started every night at 3:00am::
 421
 422      s = Nightly('nightly', ['builder1', 'builder2'], hour=3, minute=0)
 423      c['schedules'].append(s)
 424
 425     This scheduler will perform a build each monday morning at 6:23am and
 426     again at 8:23am::
 427
 428      s = Nightly('BeforeWork', ['builder1'],
 429                  dayOfWeek=0, hour=[6,8], minute=23)
 430
 431     The following runs a build every two hours::
 432
 433      s = Nightly('every2hours', ['builder1'], hour=range(0, 24, 2))
 434
 435     And this one will run only on December 24th::
 436
 437      s = Nightly('SleighPreflightCheck', ['flying_circuits', 'radar'],
 438                  month=12, dayOfMonth=24, hour=12, minute=0)
 439
 440     For dayOfWeek and dayOfMonth, builds are triggered if the date matches
 441     either of them. All time values are compared against the tuple returned
 442     by time.localtime(), so month and dayOfMonth numbers start at 1, not
 443     zero. dayOfWeek=0 is Monday, dayOfWeek=6 is Sunday.
 444
 445     onlyIfChanged functionality
 446     s = Nightly('nightly', ['builder1', 'builder2'],
 447                 hour=3, minute=0, onlyIfChanged=True)
 448     When the flag is True (False by default), the build is trigged if
 449     the date matches and if the branch has changed
 450
 451     fileIsImportant parameter is implemented as defined in class Scheduler
 452     """
 453
 454     compare_attrs = ('name', 'builderNames',
 455                      'minute', 'hour', 'dayOfMonth', 'month',
 456                      'dayOfWeek', 'branch', 'onlyIfChanged',
 457                      'fileIsImportant', 'properties')
 458
 459     def __init__(self, name, builderNames, minute=0, hour='*',
 460                  dayOfMonth='*', month='*', dayOfWeek='*',
 461                  branch=None, fileIsImportant=None, onlyIfChanged=False, properties={}):
 462         # Setting minute=0 really makes this an 'Hourly' scheduler. This
 463         # seemed like a better default than minute='*', which would result in
 464         # a build every 60 seconds.
 465         BaseUpstreamScheduler.__init__(self, name, properties)
 466         self.builderNames = builderNames
 467         self.minute = minute
 468         self.hour = hour
 469         self.dayOfMonth = dayOfMonth
 470         self.month = month
 471         self.dayOfWeek = dayOfWeek
 472         self.branch = branch
 473         self.onlyIfChanged = onlyIfChanged
 474         self.delayedRun = None
 475         self.nextRunTime = None
 476         self.reason = ("The Nightly scheduler named '%s' triggered this build"
 477                        % name)
 478
 479         self.importantChanges   = []
 480         self.unimportantChanges = []
 481         self.fileIsImportant    = None
 482         if fileIsImportant:
 483             assert callable(fileIsImportant)
 484             self.fileIsImportant = fileIsImportant
 485
 486     def addTime(self, timetuple, secs):
 487         return time.localtime(time.mktime(timetuple)+secs)
 488     def findFirstValueAtLeast(self, values, value, default=None):
 489         for v in values:
 490             if v >= value: return v
 491         return default
 492
 493     def setTimer(self):
 494         self.nextRunTime = self.calculateNextRunTime()
 495         self.delayedRun = reactor.callLater(self.nextRunTime - time.time(),
 496                                             self.doPeriodicBuild)
 497
 498     def startService(self):
 499         BaseUpstreamScheduler.startService(self)
 500         self.setTimer()
 501
 502     def stopService(self):
 503         BaseUpstreamScheduler.stopService(self)
 504         self.delayedRun.cancel()
 505
 506     def isRunTime(self, timetuple):
 507         def check(ourvalue, value):
 508             if ourvalue == '*': return True
 509             if isinstance(ourvalue, int): return value == ourvalue
 510             return (value in ourvalue)
 511
 512         if not check(self.minute, timetuple[4]):
 513             #print 'bad minute', timetuple[4], self.minute
 514             return False
 515
 516         if not check(self.hour, timetuple[3]):
 517             #print 'bad hour', timetuple[3], self.hour
 518             return False
 519
 520         if not check(self.month, timetuple[1]):
 521             #print 'bad month', timetuple[1], self.month
 522             return False
 523
 524         if self.dayOfMonth != '*' and self.dayOfWeek != '*':
 525             # They specified both day(s) of month AND day(s) of week.
 526             # This means that we only have to match one of the two. If
 527             # neither one matches, this time is not the right time.
 528             if not (check(self.dayOfMonth, timetuple[2]) or
 529                     check(self.dayOfWeek, timetuple[6])):
 530                 #print 'bad day'
 531                 return False
 532         else:
 533             if not check(self.dayOfMonth, timetuple[2]):
 534                 #print 'bad day of month'
 535                 return False
 536
 537             if not check(self.dayOfWeek, timetuple[6]):
 538                 #print 'bad day of week'
 539                 return False
 540
 541         return True
 542
 543     def calculateNextRunTime(self):
 544         return self.calculateNextRunTimeFrom(time.time())
 545
 546     def calculateNextRunTimeFrom(self, now):
 547         dateTime = time.localtime(now)
 548
 549         # Remove seconds by advancing to at least the next minue
 550         dateTime = self.addTime(dateTime, 60-dateTime[5])
 551
 552         # Now we just keep adding minutes until we find something that matches
 553
 554         # It not an efficient algorithm, but it'll *work* for now
 555         yearLimit = dateTime[0]+2
 556         while not self.isRunTime(dateTime):
 557             dateTime = self.addTime(dateTime, 60)
 558             #print 'Trying', time.asctime(dateTime)
 559             assert dateTime[0] < yearLimit, 'Something is wrong with this code'
 560         return time.mktime(dateTime)
 561
 562     def listBuilderNames(self):
 563         return self.builderNames
 564
 565     def getPendingBuildTimes(self):
 566         # TODO: figure out when self.timer is going to fire next and report
 567         # that
 568         if self.nextRunTime is None: return []
 569         return [self.nextRunTime]
 570
 571     def doPeriodicBuild(self):
 572         # Schedule the next run
 573         self.setTimer()
 574
 575         if  self.onlyIfChanged:
 576             if len(self.importantChanges) > 0:
 577                 changes = self.importantChanges + self.unimportantChanges
 578                 # And trigger a build
 579                 log.msg("Nightly Scheduler <%s>: triggering build" % self.name)
 580                 bs = buildset.BuildSet(self.builderNames,
 581                                        SourceStamp(changes=changes),
 582                                        self.reason,
 583                                        properties=self.properties)
 584                 self.submitBuildSet(bs)
 585                 # Reset the change lists
 586                 self.importantChanges = []
 587                 self.unimportantChanges = []
 588             else:
 589                 log.msg("Nightly Scheduler <%s>: skipping build - No important change" % self.name)
 590         else:
 591             # And trigger a build
 592             bs = buildset.BuildSet(self.builderNames,
 593                                    SourceStamp(branch=self.branch),
 594                                    self.reason,
 595                                    properties=self.properties)
 596             self.submitBuildSet(bs)
 597
 598     def addChange(self, change):
 599         if  self.onlyIfChanged:
 600             if change.branch != self.branch:
 601                 log.msg("Nightly Scheduler <%s>: ignoring change %s on off-branch %s" % (self.name, change.revision, change.branch))
 602                 return
 603             if not self.fileIsImportant:
 604                 self.addImportantChange(change)
 605             elif self.fileIsImportant(change):
 606                 self.addImportantChange(change)
 607             else:
 608                 self.addUnimportantChange(change)
 609         else:
 610             log.msg("Nightly Scheduler <%s>: no add change" % self.name)
 611             pass
 612
 613     def addImportantChange(self, change):
 614         log.msg("Nightly Scheduler <%s>: change %s from %s is important, adding it" % (self.name, change.revision, change.who))
 615         self.importantChanges.append(change)
 616
 617     def addUnimportantChange(self, change):
 618         log.msg("Nightly Scheduler <%s>: change %s from %s is not important, adding it" % (self.name, change.revision, change.who))
 619         self.unimportantChanges.append(change)
 620
 621
 622 class TryBase(BaseScheduler):
 623     def __init__(self, name, builderNames, properties={}):
 624         BaseScheduler.__init__(self, name, properties)
 625         self.builderNames = builderNames
 626
 627     def listBuilderNames(self):
 628         return self.builderNames
 629
 630     def getPendingBuildTimes(self):
 631         # we can't predict what the developers are going to do in the future
 632         return []
 633
 634     def addChange(self, change):
 635         # Try schedulers ignore Changes
 636         pass
 637
 638     def processBuilderList(self, builderNames):
 639         # self.builderNames is the configured list of builders
 640         # available for try.  If the user supplies a list of builders,
 641         # it must be restricted to the configured list.  If not, build
 642         # on all of the configured builders.
 643         if builderNames:
 644             for b in builderNames:
 645                 if not b in self.builderNames:
 646                     log.msg("%s got with builder %s" % (self, b))
 647                     log.msg(" but that wasn't in our list: %s"
 648                             % (self.builderNames,))
 649                     return []
 650         else:
 651             builderNames = self.builderNames
 652         return builderNames
 653
 654 class BadJobfile(Exception):
 655     pass
 656
 657 class JobFileScanner(basic.NetstringReceiver):
 658     def __init__(self):
 659         self.strings = []
 660         self.transport = self # so transport.loseConnection works
 661         self.error = False
 662
 663     def stringReceived(self, s):
 664         self.strings.append(s)
 665
 666     def loseConnection(self):
 667         self.error = True
 668
 669 class Try_Jobdir(TryBase):
 670     compare_attrs = ( 'name', 'builderNames', 'jobdir', 'properties' )
 671
 672     def __init__(self, name, builderNames, jobdir, properties={}):
 673         TryBase.__init__(self, name, builderNames, properties)
 674         self.jobdir = jobdir
 675         self.watcher = MaildirService()
 676         self.watcher.setServiceParent(self)
 677
 678     def setServiceParent(self, parent):
 679         self.watcher.setBasedir(os.path.join(parent.basedir, self.jobdir))
 680         TryBase.setServiceParent(self, parent)
 681
 682     def parseJob(self, f):
 683         # jobfiles are serialized build requests. Each is a list of
 684         # serialized netstrings, in the following order:
 685         #  "1", the version number of this format
 686         #  buildsetID, arbitrary string, used to find the buildSet later
 687         #  branch name, "" for default-branch
 688         #  base revision, "" for HEAD
 689         #  patchlevel, usually "1"
 690         #  patch
 691         #  builderNames...
 692         p = JobFileScanner()
 693         p.dataReceived(f.read())
 694         if p.error:
 695             raise BadJobfile("unable to parse netstrings")
 696         s = p.strings
 697         ver = s.pop(0)
 698         if ver != "1":
 699             raise BadJobfile("unknown version '%s'" % ver)
 700         buildsetID, branch, baserev, patchlevel, diff = s[:5]
 701         builderNames = s[5:]
 702         if branch == "":
 703             branch = None
 704         if baserev == "":
 705             baserev = None
 706         patchlevel = int(patchlevel)
 707         patch = (patchlevel, diff)
 708         ss = SourceStamp(branch, baserev, patch)
 709         return builderNames, ss, buildsetID
 710
 711     def messageReceived(self, filename):
 712         md = os.path.join(self.parent.basedir, self.jobdir)
 713         if runtime.platformType == "posix":
 714             # open the file before moving it, because I'm afraid that once
 715             # it's in cur/, someone might delete it at any moment
 716             path = os.path.join(md, "new", filename)
 717             f = open(path, "r")
 718             os.rename(os.path.join(md, "new", filename),
 719                       os.path.join(md, "cur", filename))
 720         else:
 721             # do this backwards under windows, because you can't move a file
 722             # that somebody is holding open. This was causing a Permission
 723             # Denied error on bear's win32-twisted1.3 buildslave.
 724             os.rename(os.path.join(md, "new", filename),
 725                       os.path.join(md, "cur", filename))
 726             path = os.path.join(md, "cur", filename)
 727             f = open(path, "r")
 728
 729         try:
 730             builderNames, ss, bsid = self.parseJob(f)
 731         except BadJobfile:
 732             log.msg("%s reports a bad jobfile in %s" % (self, filename))
 733             log.err()
 734             return
 735         # Validate/fixup the builder names.
 736         builderNames = self.processBuilderList(builderNames)
 737         if not builderNames:
 738             return
 739         reason = "'try' job"
 740         bs = buildset.BuildSet(builderNames, ss, reason=reason,
 741                     bsid=bsid, properties=self.properties)
 742         self.submitBuildSet(bs)
 743
 744 class Try_Userpass(TryBase):
 745     compare_attrs = ( 'name', 'builderNames', 'port', 'userpass', 'properties' )
 746     implements(portal.IRealm)
 747
 748     def __init__(self, name, builderNames, port, userpass, properties={}):
 749         TryBase.__init__(self, name, builderNames, properties)
 750         if type(port) is int:
 751             port = "tcp:%d" % port
 752         self.port = port
 753         self.userpass = userpass
 754         c = checkers.InMemoryUsernamePasswordDatabaseDontUse()
 755         for user,passwd in self.userpass:
 756             c.addUser(user, passwd)
 757
 758         p = portal.Portal(self)
 759         p.registerChecker(c)
 760         f = pb.PBServerFactory(p)
 761         s = strports.service(port, f)
 762         s.setServiceParent(self)
 763
 764     def getPort(self):
 765         # utility method for tests: figure out which TCP port we just opened.
 766         return self.services[0]._port.getHost().port
 767
 768     def requestAvatar(self, avatarID, mind, interface):
 769         log.msg("%s got connection from user %s" % (self, avatarID))
 770         assert interface == pb.IPerspective
 771         p = Try_Userpass_Perspective(self, avatarID)
 772         return (pb.IPerspective, p, lambda: None)
 773
 774 class Try_Userpass_Perspective(pbutil.NewCredPerspective):
 775     def __init__(self, parent, username):
 776         self.parent = parent
 777         self.username = username
 778
 779     def perspective_try(self, branch, revision, patch, builderNames, properties={}):
 780         log.msg("user %s requesting build on builders %s" % (self.username,
 781                                                              builderNames))
 782         # Validate/fixup the builder names.
 783         builderNames = self.parent.processBuilderList(builderNames)
 784         if not builderNames:
 785             return
 786         ss = SourceStamp(branch, revision, patch)
 787         reason = "'try' job from user %s" % self.username
 788
 789         # roll the specified props in with our inherited props
 790         combined_props = Properties()
 791         combined_props.updateFromProperties(self.parent.properties)
 792         combined_props.update(properties, "try build")
 793
 794         bs = buildset.BuildSet(builderNames,
 795                                ss,
 796                                reason=reason,
 797                                properties=combined_props)
 798
 799         self.parent.submitBuildSet(bs)
 800
 801         # return a remotely-usable BuildSetStatus object
 802         from buildbot.status.client import makeRemote
 803         return makeRemote(bs.status)
 804
 805 class Triggerable(BaseUpstreamScheduler):
 806     """This scheduler doesn't do anything until it is triggered by a Trigger
 807     step in a factory. In general, that step will not complete until all of
 808     the builds that I fire have finished.
 809     """
 810
 811     compare_attrs = ('name', 'builderNames', 'properties')
 812
 813     def __init__(self, name, builderNames, properties={}):
 814         BaseUpstreamScheduler.__init__(self, name, properties)
 815         self.builderNames = builderNames
 816
 817     def listBuilderNames(self):
 818         return self.builderNames
 819
 820     def getPendingBuildTimes(self):
 821         return []
 822
 823     def trigger(self, ss, set_props=None):
 824         """Trigger this scheduler. Returns a deferred that will fire when the
 825         buildset is finished.
 826         """
 827
 828         # properties for this buildset are composed of our own properties,
 829         # potentially overridden by anything from the triggering build
 830         props = Properties()
 831         props.updateFromProperties(self.properties)
 832         if set_props: props.updateFromProperties(set_props)
 833
 834         bs = buildset.BuildSet(self.builderNames, ss, properties=props)
 835         d = bs.waitUntilFinished()
 836         self.submitBuildSet(bs)
 837         return d