buildbot/status/web/baseweb.py

   1
   2 import os, sys, time, urllib
   3 from itertools import count
   4
   5 from zope.interface import implements
   6 from twisted.python import log
   7 from twisted.application import strports, service
   8 from twisted.web import server, distrib, static, html
   9 from twisted.spread import pb
  10
  11 from buildbot.interfaces import IControl, IStatusReceiver
  12
  13 from buildbot.status.web.base import HtmlResource, css_classes, Box, build_get_class, ICurrentBox
  14 from buildbot.status.web.waterfall import WaterfallStatusResource
  15 from buildbot.status.web.changes import ChangesResource
  16 from buildbot.status.web.builder import BuildersResource
  17 from buildbot.status.web.slaves import BuildSlavesResource
  18 from buildbot.status.web.xmlrpc import XMLRPCServer
  19 from buildbot.status.web.about import AboutBuildbot
  20
  21 # this class contains the status services (WebStatus and the older Waterfall)
  22 # which can be put in c['status']. It also contains some of the resources
  23 # that are attached to the WebStatus at various well-known URLs, which the
  24 # admin might wish to attach (using WebStatus.putChild) at other URLs.
  25
  26
  27 class LastBuild(HtmlResource):
  28     def body(self, request):
  29         return "missing\n"
  30
  31 def getLastNBuilds(status, numbuilds, builders=[], branches=[]):
  32     """Return a list with the last few Builds, sorted by start time.
  33     builder_names=None means all builders
  34     """
  35
  36     # TODO: this unsorts the list of builder names, ick
  37     builder_names = set(status.getBuilderNames())
  38     if builders:
  39         builder_names = builder_names.intersection(set(builders))
  40
  41     # to make sure that we get everything, we must get 'numbuilds' builds
  42     # from *each* source, then sort by ending time, then trim to the last
  43     # 20. We could be more efficient, but it would require the same
  44     # gnarly code that the Waterfall uses to generate one event at a
  45     # time. TODO: factor that code out into some useful class.
  46     events = []
  47     for builder_name in builder_names:
  48         builder = status.getBuilder(builder_name)
  49         for build_number in count(1):
  50             if build_number > numbuilds:
  51                 break # enough from this builder, move on to another
  52             build = builder.getBuild(-build_number)
  53             if not build:
  54                 break # no more builds here, move on to the next builder
  55             #if not build.isFinished():
  56             #    continue
  57             (build_start, build_end) = build.getTimes()
  58             event = (build_start, builder_name, build)
  59             events.append(event)
  60     def _sorter(a, b):
  61         return cmp( a[:2], b[:2] )
  62     events.sort(_sorter)
  63     # now only return the actual build, and only return some of them
  64     return [e[2] for e in events[-numbuilds:]]
  65
  66
  67 class OneLineMixin:
  68     LINE_TIME_FORMAT = "%b %d %H:%M"
  69
  70     def make_line(self, req, build):
  71         builder_name = build.getBuilder().getName()
  72         results = build.getResults()
  73         try:
  74             rev = build.getProperty("got_revision")
  75             if rev is None:
  76                 rev = "??"
  77         except KeyError:
  78             rev = "??"
  79         if len(rev) > 20:
  80             rev = "version is too-long"
  81         root = self.path_to_root(req)
  82         values = {'class': css_classes[results],
  83                   'builder_name': builder_name,
  84                   'buildnum': build.getNumber(),
  85                   'results': css_classes[results],
  86                   'buildurl': (root +
  87                                "builders/%s/builds/%d" % (builder_name,
  88                                                           build.getNumber())),
  89                   'builderurl': (root + "builders/%s" % builder_name),
  90                   'rev': rev,
  91                   'time': time.strftime(self.LINE_TIME_FORMAT,
  92                                         time.localtime(build.getTimes()[0])),
  93                   }
  94
  95         fmt = ('<font size="-1">(%(time)s)</font> '
  96                '<a href="%(builderurl)s">%(builder_name)s</a> '
  97                'rev=[%(rev)s] '
  98                '<a href="%(buildurl)s">#%(buildnum)d</a>: '
  99                '<span class="%(class)s">%(results)s</span> '
 100                )
 101         data = fmt % values
 102         return data
 103
 104 # /one_line_per_build
 105 #  accepts builder=, branch=, numbuilds=
 106 class OneLinePerBuild(HtmlResource, OneLineMixin):
 107     """This shows one line per build, combining all builders together. Useful
 108     query arguments:
 109
 110     numbuilds=: how many lines to display
 111     builder=: show only builds for this builder. Multiple builder= arguments
 112               can be used to see builds from any builder in the set.
 113     """
 114
 115     title = "Recent Builds"
 116
 117     def __init__(self, numbuilds=20):
 118         HtmlResource.__init__(self)
 119         self.numbuilds = numbuilds
 120
 121     def getChild(self, path, req):
 122         status = self.getStatus(req)
 123         builder = status.getBuilder(path)
 124         return OneLinePerBuildOneBuilder(builder)
 125
 126     def body(self, req):
 127         status = self.getStatus(req)
 128         numbuilds = int(req.args.get("numbuilds", [self.numbuilds])[0])
 129         builders = req.args.get("builder", [])
 130         branches = [b for b in req.args.get("branch", []) if b]
 131
 132         g = status.generateFinishedBuilds(builders, branches, numbuilds)
 133
 134         data = ""
 135
 136         # really this is "up to %d builds"
 137         data += "<h1>Last %d finished builds</h1>\n" % numbuilds
 138         if builders:
 139             data += ("<p>of builders: %s</p>\n" % (", ".join(builders)))
 140         data += "<ul>\n"
 141         got = 0
 142         for build in g:
 143             got += 1
 144             data += " <li>" + self.make_line(req, build) + "</li>\n"
 145         if not got:
 146             data += " <li>No matching builds found</li>\n"
 147         data += "</ul>\n"
 148         return data
 149
 150
 151
 152 # /one_line_per_build/$BUILDERNAME
 153 #  accepts branch=, numbuilds=
 154
 155 class OneLinePerBuildOneBuilder(HtmlResource, OneLineMixin):
 156     def __init__(self, builder, numbuilds=20):
 157         HtmlResource.__init__(self)
 158         self.builder = builder
 159         self.builder_name = builder.getName()
 160         self.numbuilds = numbuilds
 161         self.title = "Recent Builds of %s" % self.builder_name
 162
 163     def body(self, req):
 164         status = self.getStatus(req)
 165         numbuilds = int(req.args.get("numbuilds", [self.numbuilds])[0])
 166         branches = [b for b in req.args.get("branch", []) if b]
 167
 168         # walk backwards through all builds of a single builder
 169         g = self.builder.generateFinishedBuilds(branches, numbuilds)
 170
 171         data = ""
 172         data += ("<h1>Last %d builds of builder: %s</h1>\n" %
 173                  (numbuilds, self.builder_name))
 174         data += "<ul>\n"
 175         got = 0
 176         for build in g:
 177             got += 1
 178             data += " <li>" + self.make_line(req, build) + "</li>\n"
 179         if not got:
 180             data += " <li>No matching builds found</li>\n"
 181         data += "</ul>\n"
 182
 183         return data
 184
 185 # /one_box_per_builder
 186 #  accepts builder=, branch=
 187 class OneBoxPerBuilder(HtmlResource):
 188     """This shows a narrow table with one row per build. The leftmost column
 189     contains the builder name. The next column contains the results of the
 190     most recent build. The right-hand column shows the builder's current
 191     activity.
 192
 193     builder=: show only builds for this builder. Multiple builder= arguments
 194               can be used to see builds from any builder in the set.
 195     """
 196
 197     title = "Latest Build"
 198
 199     def body(self, req):
 200         status = self.getStatus(req)
 201
 202         builders = req.args.get("builder", status.getBuilderNames())
 203         branches = [b for b in req.args.get("branch", []) if b]
 204
 205         data = ""
 206
 207         data += "<h2>Latest builds</h2>\n"
 208         data += "<table>\n"
 209         for bn in builders:
 210             builder = status.getBuilder(bn)
 211             data += "<tr>\n"
 212             data += "<td>%s</td>\n" % html.escape(bn)
 213             b = builder.getLastFinishedBuild()
 214             if b:
 215                 url = (self.path_to_root(req) +
 216                        "builders/" +
 217                        urllib.quote(bn, safe='') +
 218                        "/builds/%d" % b.getNumber())
 219                 try:
 220                     label = b.getProperty("got_revision")
 221                 except KeyError:
 222                     label = None
 223                 if not label or len(str(label)) > 20:
 224                     label = "#%d" % b.getNumber()
 225                 text = ['<a href="%s">%s</a>' % (url, label)]
 226                 text.extend(b.getText())
 227                 box = Box(text, b.getColor(),
 228                           class_="LastBuild %s" % build_get_class(b))
 229                 data += box.td(align="center")
 230             else:
 231                 data += "<td></td>\n"
 232             current_box = ICurrentBox(builder).getBox(status)
 233             data += current_box.td(align="center")
 234         data += "</table>\n"
 235         return data
 236
 237
 238
 239 HEADER = '''
 240 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
 241  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
 242
 243 <html
 244  xmlns="http://www.w3.org/1999/xhtml"
 245  lang="en"
 246  xml:lang="en">
 247 '''
 248
 249 HEAD_ELEMENTS = [
 250     '<title>%(title)s</title>',
 251     '<link href="%(root)sbuildbot.css" rel="stylesheet" type="text/css" />',
 252     ]
 253 BODY_ATTRS = {
 254     'vlink': "#800080",
 255     }
 256
 257 FOOTER = '''
 258 </html>
 259 '''
 260
 261
 262 class WebStatus(service.MultiService):
 263     implements(IStatusReceiver)
 264     # TODO: IStatusReceiver is really about things which subscribe to hear
 265     # about buildbot events. We need a different interface (perhaps a parent
 266     # of IStatusReceiver) for status targets that don't subscribe, like the
 267     # WebStatus class. buildbot.master.BuildMaster.loadConfig:737 asserts
 268     # that everything in c['status'] provides IStatusReceiver, but really it
 269     # should check that they provide IStatusTarget instead.
 270
 271     """
 272     The webserver provided by this class has the following resources:
 273
 274      /waterfall : the big time-oriented 'waterfall' display, with links
 275                   to individual changes, builders, builds, steps, and logs.
 276                   A number of query-arguments can be added to influence
 277                   the display.
 278      /builders/BUILDERNAME: a page summarizing the builder. This includes
 279                             references to the Schedulers that feed it,
 280                             any builds currently in the queue, which
 281                             buildslaves are designated or attached, and a
 282                             summary of the build process it uses.
 283      /builders/BUILDERNAME/builds/NUM: a page describing a single Build
 284      /builders/BUILDERNAME/builds/NUM/steps/STEPNAME: describes a single step
 285      /builders/BUILDERNAME/builds/NUM/steps/STEPNAME/logs/LOGNAME: a StatusLog
 286      /builders/BUILDERNAME/builds/NUM/tests : summarize test results
 287      /builders/BUILDERNAME/builds/NUM/tests/TEST.NAME: results of one test
 288      /changes : summarize all ChangeSources
 289      /changes/CHANGENUM: a page describing a single Change
 290      /schedulers/SCHEDULERNAME: a page describing a Scheduler, including
 291                                 a description of its behavior, a list of the
 292                                 Builders it triggers, and list of the Changes
 293                                 that are queued awaiting the tree-stable
 294                                 timer, and controls to accelerate the timer.
 295      /others...
 296
 297     All URLs for pages which are not defined here are used to look for files
 298     in BASEDIR/public_html/ , which means that /robots.txt or /buildbot.css
 299     or /favicon.ico can be placed in that directory.
 300
 301     If an index file (index.html, index.htm, or index, in that order) is
 302     present in public_html/, it will be used for the root resource. If not,
 303     the default behavior is to put a redirection to the /waterfall page.
 304
 305     All of the resources provided by this service use relative URLs to reach
 306     each other. The only absolute links are the c['projectURL'] links at the
 307     top and bottom of the page, and the buildbot home-page link at the
 308     bottom.
 309
 310     This webserver defines class attributes on elements so they can be styled
 311     with CSS stylesheets. All pages pull in public_html/buildbot.css, and you
 312     can cause additional stylesheets to be loaded by adding a suitable <link>
 313     to the WebStatus instance's .head_elements attribute.
 314
 315     Buildbot uses some generic classes to identify the type of object, and
 316     some more specific classes for the various kinds of those types. It does
 317     this by specifying both in the class attributes where applicable,
 318     separated by a space. It is important that in your CSS you declare the
 319     more generic class styles above the more specific ones. For example,
 320     first define a style for .Event, and below that for .SUCCESS
 321
 322     The following CSS class names are used:
 323         - Activity, Event, BuildStep, LastBuild: general classes
 324         - waiting, interlocked, building, offline, idle: Activity states
 325         - start, running, success, failure, warnings, skipped, exception:
 326           LastBuild and BuildStep states
 327         - Change: box with change
 328         - Builder: box for builder name (at top)
 329         - Project
 330         - Time
 331
 332     """
 333
 334     # we are not a ComparableMixin, and therefore the webserver will be
 335     # rebuilt every time we reconfig.
 336
 337     def __init__(self, http_port=None, distrib_port=None, allowForce=False):
 338         """Run a web server that provides Buildbot status.
 339
 340         @type  http_port: int or L{twisted.application.strports} string
 341         @param http_port: a strports specification describing which port the
 342                           buildbot should use for its web server, with the
 343                           Waterfall display as the root page. For backwards
 344                           compatibility this can also be an int. Use
 345                           'tcp:8000' to listen on that port, or
 346                           'tcp:12345:interface=127.0.0.1' if you only want
 347                           local processes to connect to it (perhaps because
 348                           you are using an HTTP reverse proxy to make the
 349                           buildbot available to the outside world, and do not
 350                           want to make the raw port visible).
 351
 352         @type  distrib_port: int or L{twisted.application.strports} string
 353         @param distrib_port: Use this if you want to publish the Waterfall
 354                              page using web.distrib instead. The most common
 355                              case is to provide a string that is an absolute
 356                              pathname to the unix socket on which the
 357                              publisher should listen
 358                              (C{os.path.expanduser(~/.twistd-web-pb)} will
 359                              match the default settings of a standard
 360                              twisted.web 'personal web server'). Another
 361                              possibility is to pass an integer, which means
 362                              the publisher should listen on a TCP socket,
 363                              allowing the web server to be on a different
 364                              machine entirely. Both forms are provided for
 365                              backwards compatibility; the preferred form is a
 366                              strports specification like
 367                              'unix:/home/buildbot/.twistd-web-pb'. Providing
 368                              a non-absolute pathname will probably confuse
 369                              the strports parser.
 370         @param allowForce: boolean, if True then the webserver will allow
 371                            visitors to trigger and cancel builds
 372         """
 373
 374         service.MultiService.__init__(self)
 375         if type(http_port) is int:
 376             http_port = "tcp:%d" % http_port
 377         self.http_port = http_port
 378         if distrib_port is not None:
 379             if type(distrib_port) is int:
 380                 distrib_port = "tcp:%d" % distrib_port
 381             if distrib_port[0] in "/~.": # pathnames
 382                 distrib_port = "unix:%s" % distrib_port
 383         self.distrib_port = distrib_port
 384         self.allowForce = allowForce
 385
 386         # this will be replaced once we've been attached to a parent (and
 387         # thus have a basedir and can reference BASEDIR/public_html/)
 388         root = static.Data("placeholder", "text/plain")
 389         self.site = server.Site(root)
 390         self.childrenToBeAdded = {}
 391
 392         self.setupUsualPages()
 393
 394         # the following items are accessed by HtmlResource when it renders
 395         # each page.
 396         self.site.buildbot_service = self
 397         self.header = HEADER
 398         self.head_elements = HEAD_ELEMENTS[:]
 399         self.body_attrs = BODY_ATTRS.copy()
 400         self.footer = FOOTER
 401         self.template_values = {}
 402
 403         if self.http_port is not None:
 404             s = strports.service(self.http_port, self.site)
 405             s.setServiceParent(self)
 406         if self.distrib_port is not None:
 407             f = pb.PBServerFactory(distrib.ResourcePublisher(self.site))
 408             s = strports.service(self.distrib_port, f)
 409             s.setServiceParent(self)
 410
 411     def setupUsualPages(self):
 412         #self.putChild("", IndexOrWaterfallRedirection())
 413         self.putChild("waterfall", WaterfallStatusResource())
 414         self.putChild("builders", BuildersResource())
 415         self.putChild("changes", ChangesResource())
 416         self.putChild("buildslaves", BuildSlavesResource())
 417         #self.putChild("schedulers", SchedulersResource())
 418         self.putChild("one_line_per_build", OneLinePerBuild())
 419         self.putChild("one_box_per_builder", OneBoxPerBuilder())
 420         self.putChild("xmlrpc", XMLRPCServer())
 421         self.putChild("about", AboutBuildbot())
 422
 423     def __repr__(self):
 424         if self.http_port is None:
 425             return "<WebStatus on path %s>" % self.distrib_port
 426         if self.distrib_port is None:
 427             return "<WebStatus on port %s>" % self.http_port
 428         return "<WebStatus on port %s and path %s>" % (self.http_port,
 429                                                        self.distrib_port)
 430
 431     def setServiceParent(self, parent):
 432         service.MultiService.setServiceParent(self, parent)
 433         self.setupSite()
 434
 435     def setupSite(self):
 436         # this is responsible for creating the root resource. It isn't done
 437         # at __init__ time because we need to reference the parent's basedir.
 438         htmldir = os.path.join(self.parent.basedir, "public_html")
 439         if not os.path.isdir(htmldir):
 440             os.mkdir(htmldir)
 441         root = static.File(htmldir)
 442         log.msg("WebStatus using (%s)" % htmldir)
 443
 444         for name, child_resource in self.childrenToBeAdded.iteritems():
 445             root.putChild(name, child_resource)
 446
 447         self.site.resource = root
 448
 449     def putChild(self, name, child_resource):
 450         """This behaves a lot like root.putChild() . """
 451         self.childrenToBeAdded[name] = child_resource
 452
 453     def getStatus(self):
 454         return self.parent.getStatus()
 455     def getControl(self):
 456         if self.allowForce:
 457             return IControl(self.parent)
 458         return None
 459
 460     def getPortnum(self):
 461         # this is for the benefit of unit tests
 462         s = list(self)[0]
 463         return s._port.getHost().port
 464
 465 # resources can get access to the IStatus by calling
 466 # request.site.buildbot_service.getStatus()
 467
 468 # this is the compatibility class for the old waterfall. It is exactly like a
 469 # regular WebStatus except that the root resource (e.g. http://buildbot.net/)
 470 # always redirects to a WaterfallStatusResource, and the old arguments are
 471 # mapped into the new resource-tree approach. In the normal WebStatus, the
 472 # root resource either redirects the browser to /waterfall or serves
 473 # BASEDIR/public_html/index.html, and favicon/robots.txt are provided by
 474 # having the admin write actual files into BASEDIR/public_html/ .
 475
 476 # note: we don't use a util.Redirect here because HTTP requires that the
 477 # Location: header provide an absolute URI, and it's non-trivial to figure
 478 # out our absolute URI from here.
 479
 480 class Waterfall(WebStatus):
 481
 482     if hasattr(sys, "frozen"):
 483         # all 'data' files are in the directory of our executable
 484         here = os.path.dirname(sys.executable)
 485         buildbot_icon = os.path.abspath(os.path.join(here, "buildbot.png"))
 486         buildbot_css = os.path.abspath(os.path.join(here, "classic.css"))
 487     else:
 488         # running from source
 489         # the icon is sibpath(__file__, "../buildbot.png") . This is for
 490         # portability.
 491         up = os.path.dirname
 492         buildbot_icon = os.path.abspath(os.path.join(up(up(up(__file__))),
 493                                                      "buildbot.png"))
 494         buildbot_css = os.path.abspath(os.path.join(up(__file__),
 495                                                     "classic.css"))
 496
 497     compare_attrs = ["http_port", "distrib_port", "allowForce",
 498                      "categories", "css", "favicon", "robots_txt"]
 499
 500     def __init__(self, http_port=None, distrib_port=None, allowForce=True,
 501                  categories=None, css=buildbot_css, favicon=buildbot_icon,
 502                  robots_txt=None):
 503         WebStatus.__init__(self, http_port, distrib_port, allowForce)
 504         self.css = css
 505         if css:
 506             data = open(css, "rb").read()
 507             self.putChild("buildbot.css", static.Data(data, "text/plain"))
 508         self.favicon = favicon
 509         self.robots_txt = robots_txt
 510         if favicon:
 511             data = open(favicon, "rb").read()
 512             self.putChild("favicon.ico", static.Data(data, "image/x-icon"))
 513         if robots_txt:
 514             data = open(robots_txt, "rb").read()
 515             self.putChild("robots.txt", static.Data(data, "text/plain"))
 516         self.putChild("", WaterfallStatusResource(categories))