buildbot/status/web/baseweb.py

   1
   2 import os, sys, urllib, weakref
   3 from itertools import count
   4
   5 from zope.interface import implements
   6 from twisted.python import log
   7 from twisted.application import strports, service
   8 from twisted.web import server, distrib, static, html
   9 from twisted.spread import pb
  10
  11 from buildbot.interfaces import IControl, IStatusReceiver
  12
  13 from buildbot.status.web.base import HtmlResource, Box, \
  14      build_get_class, ICurrentBox, OneLineMixin
  15 from buildbot.status.web.waterfall import WaterfallStatusResource
  16 from buildbot.status.web.changes import ChangesResource
  17 from buildbot.status.web.builder import BuildersResource
  18 from buildbot.status.web.slaves import BuildSlavesResource
  19 from buildbot.status.web.xmlrpc import XMLRPCServer
  20 from buildbot.status.web.about import AboutBuildbot
  21
  22 # this class contains the status services (WebStatus and the older Waterfall)
  23 # which can be put in c['status']. It also contains some of the resources
  24 # that are attached to the WebStatus at various well-known URLs, which the
  25 # admin might wish to attach (using WebStatus.putChild) at other URLs.
  26
  27
  28 class LastBuild(HtmlResource):
  29     def body(self, request):
  30         return "missing\n"
  31
  32 def getLastNBuilds(status, numbuilds, builders=[], branches=[]):
  33     """Return a list with the last few Builds, sorted by start time.
  34     builder_names=None means all builders
  35     """
  36
  37     # TODO: this unsorts the list of builder names, ick
  38     builder_names = set(status.getBuilderNames())
  39     if builders:
  40         builder_names = builder_names.intersection(set(builders))
  41
  42     # to make sure that we get everything, we must get 'numbuilds' builds
  43     # from *each* source, then sort by ending time, then trim to the last
  44     # 20. We could be more efficient, but it would require the same
  45     # gnarly code that the Waterfall uses to generate one event at a
  46     # time. TODO: factor that code out into some useful class.
  47     events = []
  48     for builder_name in builder_names:
  49         builder = status.getBuilder(builder_name)
  50         for build_number in count(1):
  51             if build_number > numbuilds:
  52                 break # enough from this builder, move on to another
  53             build = builder.getBuild(-build_number)
  54             if not build:
  55                 break # no more builds here, move on to the next builder
  56             #if not build.isFinished():
  57             #    continue
  58             (build_start, build_end) = build.getTimes()
  59             event = (build_start, builder_name, build)
  60             events.append(event)
  61     def _sorter(a, b):
  62         return cmp( a[:2], b[:2] )
  63     events.sort(_sorter)
  64     # now only return the actual build, and only return some of them
  65     return [e[2] for e in events[-numbuilds:]]
  66
  67
  68 # /one_line_per_build
  69 #  accepts builder=, branch=, numbuilds=
  70 class OneLinePerBuild(HtmlResource, OneLineMixin):
  71     """This shows one line per build, combining all builders together. Useful
  72     query arguments:
  73
  74     numbuilds=: how many lines to display
  75     builder=: show only builds for this builder. Multiple builder= arguments
  76               can be used to see builds from any builder in the set.
  77     """
  78
  79     title = "Recent Builds"
  80
  81     def __init__(self, numbuilds=20):
  82         HtmlResource.__init__(self)
  83         self.numbuilds = numbuilds
  84
  85     def getChild(self, path, req):
  86         status = self.getStatus(req)
  87         builder = status.getBuilder(path)
  88         return OneLinePerBuildOneBuilder(builder)
  89
  90     def body(self, req):
  91         status = self.getStatus(req)
  92         numbuilds = int(req.args.get("numbuilds", [self.numbuilds])[0])
  93         builders = req.args.get("builder", [])
  94         branches = [b for b in req.args.get("branch", []) if b]
  95
  96         g = status.generateFinishedBuilds(builders, branches, numbuilds)
  97
  98         data = ""
  99
 100         # really this is "up to %d builds"
 101         data += "<h1>Last %d finished builds: %s</h1>\n" % \
 102                 (numbuilds, " ".join(branches))
 103         if builders:
 104             data += ("<p>of builders: %s</p>\n" % (", ".join(builders)))
 105         data += "<ul>\n"
 106         got = 0
 107         for build in g:
 108             got += 1
 109             data += " <li>" + self.make_line(req, build) + "</li>\n"
 110         if not got:
 111             data += " <li>No matching builds found</li>\n"
 112         data += "</ul>\n"
 113         return data
 114
 115
 116
 117 # /one_line_per_build/$BUILDERNAME
 118 #  accepts branch=, numbuilds=
 119
 120 class OneLinePerBuildOneBuilder(HtmlResource, OneLineMixin):
 121     def __init__(self, builder, numbuilds=20):
 122         HtmlResource.__init__(self)
 123         self.builder = builder
 124         self.builder_name = builder.getName()
 125         self.numbuilds = numbuilds
 126         self.title = "Recent Builds of %s" % self.builder_name
 127
 128     def body(self, req):
 129         status = self.getStatus(req)
 130         numbuilds = int(req.args.get("numbuilds", [self.numbuilds])[0])
 131         branches = [b for b in req.args.get("branch", []) if b]
 132
 133         # walk backwards through all builds of a single builder
 134         g = self.builder.generateFinishedBuilds(branches, numbuilds)
 135
 136         data = ""
 137         data += ("<h1>Last %d builds of builder: %s</h1>\n" %
 138                  (numbuilds, self.builder_name))
 139         data += "<ul>\n"
 140         got = 0
 141         for build in g:
 142             got += 1
 143             data += " <li>" + self.make_line(req, build) + "</li>\n"
 144         if not got:
 145             data += " <li>No matching builds found</li>\n"
 146         data += "</ul>\n"
 147
 148         return data
 149
 150 # /one_box_per_builder
 151 #  accepts builder=, branch=
 152 class OneBoxPerBuilder(HtmlResource):
 153     """This shows a narrow table with one row per build. The leftmost column
 154     contains the builder name. The next column contains the results of the
 155     most recent build. The right-hand column shows the builder's current
 156     activity.
 157
 158     builder=: show only builds for this builder. Multiple builder= arguments
 159               can be used to see builds from any builder in the set.
 160     """
 161
 162     title = "Latest Build"
 163
 164     def body(self, req):
 165         status = self.getStatus(req)
 166
 167         builders = req.args.get("builder", status.getBuilderNames())
 168         branches = [b for b in req.args.get("branch", []) if b]
 169
 170         data = ""
 171
 172         data += "<h2>Latest builds: %s</h2>\n" % " ".join(branches)
 173         data += "<table>\n"
 174         for bn in builders:
 175             builder = status.getBuilder(bn)
 176             data += "<tr>\n"
 177             data += '<td class="box">%s</td>\n' % html.escape(bn)
 178             builds = list(builder.generateFinishedBuilds(branches,
 179                                                          num_builds=1))
 180             if builds:
 181                 b = builds[0]
 182                 url = (self.path_to_root(req) +
 183                        "builders/" +
 184                        urllib.quote(bn, safe='') +
 185                        "/builds/%d" % b.getNumber())
 186                 try:
 187                     label = b.getProperty("got_revision")
 188                 except KeyError:
 189                     label = None
 190                 if not label or len(str(label)) > 20:
 191                     label = "#%d" % b.getNumber()
 192                 text = ['<a href="%s">%s</a>' % (url, label)]
 193                 text.extend(b.getText())
 194                 box = Box(text, b.getColor(),
 195                           class_="LastBuild box %s" % build_get_class(b))
 196                 data += box.td(align="center")
 197             else:
 198                 data += '<td class="LastBuild box" >no build</td>\n'
 199             current_box = ICurrentBox(builder).getBox(status)
 200             data += current_box.td(align="center")
 201         data += "</table>\n"
 202         return data
 203
 204
 205
 206 HEADER = '''
 207 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
 208  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
 209
 210 <html
 211  xmlns="http://www.w3.org/1999/xhtml"
 212  lang="en"
 213  xml:lang="en">
 214 '''
 215
 216 HEAD_ELEMENTS = [
 217     '<title>%(title)s</title>',
 218     '<link href="%(root)sbuildbot.css" rel="stylesheet" type="text/css" />',
 219     ]
 220 BODY_ATTRS = {
 221     'vlink': "#800080",
 222     }
 223
 224 FOOTER = '''
 225 </html>
 226 '''
 227
 228
 229 class WebStatus(service.MultiService):
 230     implements(IStatusReceiver)
 231     # TODO: IStatusReceiver is really about things which subscribe to hear
 232     # about buildbot events. We need a different interface (perhaps a parent
 233     # of IStatusReceiver) for status targets that don't subscribe, like the
 234     # WebStatus class. buildbot.master.BuildMaster.loadConfig:737 asserts
 235     # that everything in c['status'] provides IStatusReceiver, but really it
 236     # should check that they provide IStatusTarget instead.
 237
 238     """
 239     The webserver provided by this class has the following resources:
 240
 241      /waterfall : the big time-oriented 'waterfall' display, with links
 242                   to individual changes, builders, builds, steps, and logs.
 243                   A number of query-arguments can be added to influence
 244                   the display.
 245      /builders/BUILDERNAME: a page summarizing the builder. This includes
 246                             references to the Schedulers that feed it,
 247                             any builds currently in the queue, which
 248                             buildslaves are designated or attached, and a
 249                             summary of the build process it uses.
 250      /builders/BUILDERNAME/builds/NUM: a page describing a single Build
 251      /builders/BUILDERNAME/builds/NUM/steps/STEPNAME: describes a single step
 252      /builders/BUILDERNAME/builds/NUM/steps/STEPNAME/logs/LOGNAME: a StatusLog
 253      /builders/BUILDERNAME/builds/NUM/tests : summarize test results
 254      /builders/BUILDERNAME/builds/NUM/tests/TEST.NAME: results of one test
 255      /changes : summarize all ChangeSources
 256      /changes/CHANGENUM: a page describing a single Change
 257      /schedulers/SCHEDULERNAME: a page describing a Scheduler, including
 258                                 a description of its behavior, a list of the
 259                                 Builders it triggers, and list of the Changes
 260                                 that are queued awaiting the tree-stable
 261                                 timer, and controls to accelerate the timer.
 262      /buildslaves : list all BuildSlaves
 263      /buildslaves/SLAVENAME : describe a single BuildSlave
 264      /one_line_per_build : summarize the last few builds, one line each
 265      /one_line_per_build/BUILDERNAME : same, but only for a single builder
 266      /one_box_per_builder : show the latest build and current activity
 267      /about : describe this buildmaster (Buildbot and support library versions)
 268      /xmlrpc : (not yet implemented) an XMLRPC server with build status
 269
 270
 271     All URLs for pages which are not defined here are used to look for files
 272     in BASEDIR/public_html/ , which means that /robots.txt or /buildbot.css
 273     or /favicon.ico can be placed in that directory.
 274
 275     If an index file (index.html, index.htm, or index, in that order) is
 276     present in public_html/, it will be used for the root resource. If not,
 277     the default behavior is to put a redirection to the /waterfall page.
 278
 279     All of the resources provided by this service use relative URLs to reach
 280     each other. The only absolute links are the c['projectURL'] links at the
 281     top and bottom of the page, and the buildbot home-page link at the
 282     bottom.
 283
 284     This webserver defines class attributes on elements so they can be styled
 285     with CSS stylesheets. All pages pull in public_html/buildbot.css, and you
 286     can cause additional stylesheets to be loaded by adding a suitable <link>
 287     to the WebStatus instance's .head_elements attribute.
 288
 289     Buildbot uses some generic classes to identify the type of object, and
 290     some more specific classes for the various kinds of those types. It does
 291     this by specifying both in the class attributes where applicable,
 292     separated by a space. It is important that in your CSS you declare the
 293     more generic class styles above the more specific ones. For example,
 294     first define a style for .Event, and below that for .SUCCESS
 295
 296     The following CSS class names are used:
 297         - Activity, Event, BuildStep, LastBuild: general classes
 298         - waiting, interlocked, building, offline, idle: Activity states
 299         - start, running, success, failure, warnings, skipped, exception:
 300           LastBuild and BuildStep states
 301         - Change: box with change
 302         - Builder: box for builder name (at top)
 303         - Project
 304         - Time
 305
 306     """
 307
 308     # we are not a ComparableMixin, and therefore the webserver will be
 309     # rebuilt every time we reconfig. This is because WebStatus.putChild()
 310     # makes it too difficult to tell whether two instances are the same or
 311     # not (we'd have to do a recursive traversal of all children to discover
 312     # all the changes).
 313
 314     def __init__(self, http_port=None, distrib_port=None, allowForce=False):
 315         """Run a web server that provides Buildbot status.
 316
 317         @type  http_port: int or L{twisted.application.strports} string
 318         @param http_port: a strports specification describing which port the
 319                           buildbot should use for its web server, with the
 320                           Waterfall display as the root page. For backwards
 321                           compatibility this can also be an int. Use
 322                           'tcp:8000' to listen on that port, or
 323                           'tcp:12345:interface=127.0.0.1' if you only want
 324                           local processes to connect to it (perhaps because
 325                           you are using an HTTP reverse proxy to make the
 326                           buildbot available to the outside world, and do not
 327                           want to make the raw port visible).
 328
 329         @type  distrib_port: int or L{twisted.application.strports} string
 330         @param distrib_port: Use this if you want to publish the Waterfall
 331                              page using web.distrib instead. The most common
 332                              case is to provide a string that is an absolute
 333                              pathname to the unix socket on which the
 334                              publisher should listen
 335                              (C{os.path.expanduser(~/.twistd-web-pb)} will
 336                              match the default settings of a standard
 337                              twisted.web 'personal web server'). Another
 338                              possibility is to pass an integer, which means
 339                              the publisher should listen on a TCP socket,
 340                              allowing the web server to be on a different
 341                              machine entirely. Both forms are provided for
 342                              backwards compatibility; the preferred form is a
 343                              strports specification like
 344                              'unix:/home/buildbot/.twistd-web-pb'. Providing
 345                              a non-absolute pathname will probably confuse
 346                              the strports parser.
 347         @param allowForce: boolean, if True then the webserver will allow
 348                            visitors to trigger and cancel builds
 349         """
 350
 351         service.MultiService.__init__(self)
 352         if type(http_port) is int:
 353             http_port = "tcp:%d" % http_port
 354         self.http_port = http_port
 355         if distrib_port is not None:
 356             if type(distrib_port) is int:
 357                 distrib_port = "tcp:%d" % distrib_port
 358             if distrib_port[0] in "/~.": # pathnames
 359                 distrib_port = "unix:%s" % distrib_port
 360         self.distrib_port = distrib_port
 361         self.allowForce = allowForce
 362
 363         # this will be replaced once we've been attached to a parent (and
 364         # thus have a basedir and can reference BASEDIR/public_html/)
 365         root = static.Data("placeholder", "text/plain")
 366         self.site = server.Site(root)
 367         self.childrenToBeAdded = {}
 368
 369         self.setupUsualPages()
 370
 371         # the following items are accessed by HtmlResource when it renders
 372         # each page.
 373         self.site.buildbot_service = self
 374         self.header = HEADER
 375         self.head_elements = HEAD_ELEMENTS[:]
 376         self.body_attrs = BODY_ATTRS.copy()
 377         self.footer = FOOTER
 378         self.template_values = {}
 379
 380         # keep track of cached connections so we can break them when we shut
 381         # down. See ticket #102 for more details.
 382         self.channels = weakref.WeakKeyDictionary()
 383
 384         if self.http_port is not None:
 385             s = strports.service(self.http_port, self.site)
 386             s.setServiceParent(self)
 387         if self.distrib_port is not None:
 388             f = pb.PBServerFactory(distrib.ResourcePublisher(self.site))
 389             s = strports.service(self.distrib_port, f)
 390             s.setServiceParent(self)
 391
 392     def setupUsualPages(self):
 393         #self.putChild("", IndexOrWaterfallRedirection())
 394         self.putChild("waterfall", WaterfallStatusResource())
 395         self.putChild("builders", BuildersResource()) # has builds/steps/logs
 396         self.putChild("changes", ChangesResource())
 397         self.putChild("buildslaves", BuildSlavesResource())
 398         #self.putChild("schedulers", SchedulersResource())
 399         self.putChild("one_line_per_build", OneLinePerBuild())
 400         self.putChild("one_box_per_builder", OneBoxPerBuilder())
 401         self.putChild("xmlrpc", XMLRPCServer())
 402         self.putChild("about", AboutBuildbot())
 403
 404     def __repr__(self):
 405         if self.http_port is None:
 406             return "<WebStatus on path %s at %s>" % (self.distrib_port,
 407                                                      hex(id(self)))
 408         if self.distrib_port is None:
 409             return "<WebStatus on port %s at %s>" % (self.http_port,
 410                                                      hex(id(self)))
 411         return ("<WebStatus on port %s and path %s at %s>" %
 412                 (self.http_port, self.distrib_port, hex(id(self))))
 413
 414     def setServiceParent(self, parent):
 415         service.MultiService.setServiceParent(self, parent)
 416         self.setupSite()
 417
 418     def setupSite(self):
 419         # this is responsible for creating the root resource. It isn't done
 420         # at __init__ time because we need to reference the parent's basedir.
 421         htmldir = os.path.join(self.parent.basedir, "public_html")
 422         if not os.path.isdir(htmldir):
 423             os.mkdir(htmldir)
 424         root = static.File(htmldir)
 425         log.msg("WebStatus using (%s)" % htmldir)
 426
 427         for name, child_resource in self.childrenToBeAdded.iteritems():
 428             root.putChild(name, child_resource)
 429
 430         self.site.resource = root
 431
 432     def putChild(self, name, child_resource):
 433         """This behaves a lot like root.putChild() . """
 434         self.childrenToBeAdded[name] = child_resource
 435
 436     def registerChannel(self, channel):
 437         self.channels[channel] = 1 # weakrefs
 438
 439     def stopService(self):
 440         for channel in self.channels:
 441             try:
 442                 channel.transport.loseConnection()
 443             except:
 444                 log.msg("WebStatus.stopService: error while disconnecting"
 445                         " leftover clients")
 446                 log.err()
 447         return service.MultiService.stopService(self)
 448
 449     def getStatus(self):
 450         return self.parent.getStatus()
 451     def getControl(self):
 452         if self.allowForce:
 453             return IControl(self.parent)
 454         return None
 455
 456     def getPortnum(self):
 457         # this is for the benefit of unit tests
 458         s = list(self)[0]
 459         return s._port.getHost().port
 460
 461 # resources can get access to the IStatus by calling
 462 # request.site.buildbot_service.getStatus()
 463
 464 # this is the compatibility class for the old waterfall. It is exactly like a
 465 # regular WebStatus except that the root resource (e.g. http://buildbot.net/)
 466 # always redirects to a WaterfallStatusResource, and the old arguments are
 467 # mapped into the new resource-tree approach. In the normal WebStatus, the
 468 # root resource either redirects the browser to /waterfall or serves
 469 # BASEDIR/public_html/index.html, and favicon/robots.txt are provided by
 470 # having the admin write actual files into BASEDIR/public_html/ .
 471
 472 # note: we don't use a util.Redirect here because HTTP requires that the
 473 # Location: header provide an absolute URI, and it's non-trivial to figure
 474 # out our absolute URI from here.
 475
 476 class Waterfall(WebStatus):
 477
 478     if hasattr(sys, "frozen"):
 479         # all 'data' files are in the directory of our executable
 480         here = os.path.dirname(sys.executable)
 481         buildbot_icon = os.path.abspath(os.path.join(here, "buildbot.png"))
 482         buildbot_css = os.path.abspath(os.path.join(here, "classic.css"))
 483     else:
 484         # running from source
 485         # the icon is sibpath(__file__, "../buildbot.png") . This is for
 486         # portability.
 487         up = os.path.dirname
 488         buildbot_icon = os.path.abspath(os.path.join(up(up(up(__file__))),
 489                                                      "buildbot.png"))
 490         buildbot_css = os.path.abspath(os.path.join(up(__file__),
 491                                                     "classic.css"))
 492
 493     compare_attrs = ["http_port", "distrib_port", "allowForce",
 494                      "categories", "css", "favicon", "robots_txt"]
 495
 496     def __init__(self, http_port=None, distrib_port=None, allowForce=True,
 497                  categories=None, css=buildbot_css, favicon=buildbot_icon,
 498                  robots_txt=None):
 499         WebStatus.__init__(self, http_port, distrib_port, allowForce)
 500         self.css = css
 501         if css:
 502             data = open(css, "rb").read()
 503             self.putChild("buildbot.css", static.Data(data, "text/plain"))
 504         self.favicon = favicon
 505         self.robots_txt = robots_txt
 506         if favicon:
 507             data = open(favicon, "rb").read()
 508             self.putChild("favicon.ico", static.Data(data, "image/x-icon"))
 509         if robots_txt:
 510             data = open(robots_txt, "rb").read()
 511             self.putChild("robots.txt", static.Data(data, "text/plain"))
 512         self.putChild("", WaterfallStatusResource(categories))