build_log.py

   1
   2 # Copyright (C) 2007 Mark Seaborn
   3 #
   4 # chroot_build is free software; you can redistribute it and/or modify
   5 # it under the terms of the GNU Lesser General Public License as
   6 # published by the Free Software Foundation; either version 2.1 of the
   7 # License, or (at your option) any later version.
   8 #
   9 # chroot_build is distributed in the hope that it will be useful, but
  10 # WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12 # Lesser General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Lesser General Public
  15 # License along with chroot_build; if not, write to the Free Software
  16 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
  17 # 02110-1301, USA.
  18
  19 import os
  20 import subprocess
  21 import time
  22
  23 from buildutils import remove_prefix
  24 import lxml.etree as etree
  25
  26
  27 class NodeStream(object):
  28
  29     def __init__(self, output):
  30         self.output = output
  31         self._names = set()
  32
  33     def alloc_name(self, name):
  34         if name in self._names:
  35             suffix = 1
  36             while True:
  37                 new_name = "%s_%i" % (name, suffix)
  38                 if new_name not in self._names:
  39                     break
  40                 suffix += 1
  41         else:
  42             new_name = name
  43         self._names.add(new_name)
  44         return new_name
  45
  46
  47 class NodeWriter(object):
  48
  49     def __init__(self, stream, id):
  50         self._stream = stream
  51         self._id = id
  52
  53     def new_child(self, tag_name, attrs=[], id_name=None):
  54         if id_name is None:
  55             id_name = tag_name
  56         new_id = self._stream.alloc_name(id_name)
  57         self._stream.output.write("%s add %s %s\n" %
  58                                   (self._id, tag_name, new_id))
  59         child = NodeWriter(self._stream, new_id)
  60         for key, value in attrs:
  61             child.add_attr(key, value)
  62         return child
  63
  64     def add_attr(self, key, value):
  65         assert isinstance(key, str)
  66         assert isinstance(value, str)
  67         assert " " not in key
  68         assert "\n" not in key
  69         assert "\n" not in value
  70         self._stream.output.write("%s %s %s\n" % (self._id, key, value))
  71
  72
  73 class StreamReader(object):
  74
  75     def __init__(self):
  76         self._root_node = etree.Element("root")
  77         self._map = {"root": self._root_node}
  78
  79     def process_line(self, line):
  80         node_id, attr, arg = line.split(" ", 2)
  81         node = self._map[node_id]
  82         if attr == "add":
  83             tag_name, new_node_id = arg.split(" ", 1)
  84             new_node = etree.Element(tag_name)
  85             assert arg not in self._map
  86             self._map[new_node_id] = new_node
  87             node.append(new_node)
  88         else:
  89             node.attrib[attr] = arg
  90
  91     def process_file(self, fh):
  92         for line in fh:
  93             self.process_line(line.rstrip("\n"))
  94
  95     def get_root(self):
  96         return self._root_node
  97
  98
  99 def get_xml_from_log(input_file):
 100     reader = StreamReader()
 101     reader.process_file(input_file)
 102     return reader.get_root()
 103
 104
 105 class LogDir(object):
 106
 107     def __init__(self, dir_path, get_time=time.time):
 108         self._dir_path = dir_path
 109         self._get_time = get_time
 110         self._counter = 0
 111         self._log_file = os.path.join(self._dir_path, "0000-log")
 112
 113     def make_filename(self, name):
 114         self._counter += 1
 115         basename = "%04i-%s" % (self._counter, name)
 116         return basename, os.path.join(self._dir_path, basename)
 117
 118     def make_logger(self):
 119         assert not os.path.exists(self._log_file)
 120         stream = NodeStream(open(self._log_file, "w", buffering=0))
 121         log = LogWriter(NodeWriter(stream, "root"),
 122                         self, "root", self._get_time)
 123         log.start()
 124         return log
 125
 126     def get_xml(self):
 127         log = get_xml_from_log(open(self._log_file, "r"))
 128         for file_node in log.xpath(".//file"):
 129             file_node.attrib["pathname"] = \
 130                 os.path.join(self._dir_path, file_node.attrib["filename"])
 131         return log
 132
 133     def get_timestamp(self):
 134         return os.stat(self._log_file).st_mtime
 135
 136
 137 class LogWriter(object):
 138
 139     def __init__(self, node, log_dir, name, get_time):
 140         self._node = node
 141         self._log_dir = log_dir
 142         self._name = name
 143         self._get_time = get_time
 144
 145     def start(self):
 146         self._node.add_attr("start_time", str(self._get_time()))
 147
 148     def message(self, message):
 149         self._node.new_child("message", [("text", message)])
 150
 151     def child_log(self, name, do_start=True):
 152         sublog = LogWriter(self._node.new_child("log", [("name", name)],
 153                                                 id_name=name),
 154                            self._log_dir, name, self._get_time)
 155         if do_start:
 156             sublog.start()
 157         return sublog
 158
 159     def make_file(self):
 160         relative_name, filename = self._log_dir.make_filename(self._name)
 161         self._node.new_child("file", [("filename", relative_name)])
 162         assert not os.path.exists(filename)
 163         return open(filename, "w")
 164
 165     def finish(self, result):
 166         self._node.add_attr("end_time", str(self._get_time()))
 167         self._node.add_attr("result", str(result))
 168
 169
 170 class DummyLogWriter(object):
 171
 172     def start(self):
 173         pass
 174
 175     def message(self, message):
 176         pass
 177
 178     def child_log(self, name, do_start=True):
 179         return DummyLogWriter()
 180
 181     def make_file(self):
 182         return open("/dev/null", "w")
 183
 184     def finish(self, result):
 185         pass
 186
 187
 188 class LogSetDir(object):
 189
 190     def __init__(self, dir_path, get_time=time.time):
 191         self._dir = dir_path
 192         self._get_time = get_time
 193
 194     def make_logger(self):
 195         time_now = time.gmtime(self._get_time())
 196         subdir_base = time.strftime("%Y/%m/%d", time_now)
 197         i = 0
 198         while True:
 199             log_dir = os.path.join(self._dir, "%s/%04i" % (subdir_base, i))
 200             if not os.path.exists(log_dir):
 201                 break
 202             i += 1
 203         os.makedirs(log_dir)
 204         return LogDir(log_dir, self._get_time).make_logger()
 205
 206     def _sorted_leafnames(self, dir_path):
 207         # For compatibility with existing log dirs, sort by number not
 208         # by string.
 209         leafnames = []
 210         for leafname in os.listdir(dir_path):
 211             try:
 212                 num = int(leafname)
 213             except ValueError:
 214                 pass
 215             else:
 216                 leafnames.append((num, leafname))
 217         leafnames.sort(reverse=True)
 218         return [name for num, name in leafnames]
 219
 220     def _get_logs(self, dir_path):
 221         if os.path.exists(os.path.join(dir_path, "0000-log")):
 222             yield LogDir(dir_path, self._get_time)
 223         else:
 224             if os.path.exists(dir_path):
 225                 for leafname in self._sorted_leafnames(dir_path):
 226                     for log in self._get_logs(os.path.join(dir_path, leafname)):
 227                         yield log
 228
 229     def get_logs(self):
 230         return self._get_logs(self._dir)
 231
 232
 233 class NullPathnameMapper(object):
 234
 235     def map_pathname(self, pathname):
 236         return pathname
 237
 238
 239 class PathnameMapper(object):
 240
 241     def __init__(self, dir_path):
 242         self._dir_path = dir_path
 243
 244     def map_pathname(self, pathname):
 245         return remove_prefix(self._dir_path + "/", pathname)
 246
 247
 248 def flatten(val):
 249     got = []
 250
 251     def recurse(x):
 252         if isinstance(x, (list, tuple)):
 253             for y in x:
 254                 recurse(y)
 255         else:
 256             got.append(x)
 257
 258     recurse(val)
 259     return got
 260
 261
 262 def tagp(tag_name, attrs, *children):
 263     element = etree.Element(tag_name)
 264     for key, value in attrs:
 265         element.attrib[key] = value
 266     last_child = None
 267     for child in flatten(children):
 268         if isinstance(child, basestring):
 269             assert last_child is None
 270             element.text = child
 271         else:
 272             element.append(child)
 273             last_child = child
 274     return element
 275
 276
 277 def tag(tag_name, *children):
 278     return tagp(tag_name, [], *children)
 279
 280
 281 def log_class(log):
 282     if "result" not in log.attrib:
 283         return "result_unknown"
 284     elif int(log.attrib["result"]) == 0:
 285         return "result_success"
 286     else:
 287         return "result_failure"
 288
 289
 290 def format_log(log, path_mapper, filter=lambda log: True):
 291     sub_logs = [format_log(sublog, path_mapper, filter)
 292                 for sublog in reversed(log.xpath("log"))]
 293     if not filter(log):
 294         return sub_logs
 295     classes = ["log", log_class(log)]
 296     html = tagp("div", [("class", " ".join(classes))])
 297     html.append(tag("span", log_duration(log) + log.attrib.get("name", "")))
 298     for file_node in log.xpath("file"):
 299         pathname = file_node.attrib["pathname"]
 300         relative_name = path_mapper.map_pathname(pathname)
 301         if os.stat(pathname).st_size > 0:
 302             html.append(tagp("a", [("href", relative_name)], "[log]"))
 303     html.extend(flatten(sub_logs))
 304     return html
 305
 306
 307 def format_time(log, attr):
 308     # TODO: use local timezone without breaking golden tests
 309     if attr in log.attrib:
 310         timeval = float(log.attrib[attr])
 311         string = time.strftime("%a, %d %b %Y %H:%M",
 312                                time.gmtime(timeval))
 313     else:
 314         string = "time not known"
 315     return tagp("div", [("class", "time")], string)
 316
 317
 318 def format_duration(seconds):
 319     if int(seconds) == 0:
 320         return "0s"
 321     got = []
 322     minutes, val = divmod(seconds, 60)
 323     got.append("%02is" % val)
 324     if minutes != 0:
 325         hours, val = divmod(minutes, 60)
 326         got.append("%02im" % val)
 327         if hours != 0:
 328             days, val = divmod(hours, 24)
 329             got.append("%02ih" % val)
 330             if days != 0:
 331                 got.append("%id" % days)
 332     return "".join(reversed(got)).lstrip("0")
 333
 334
 335 def log_duration(log):
 336     if "start_time" in log.attrib and "end_time" in log.attrib:
 337         return "[%s] " % format_duration(float(log.attrib["end_time"]) -
 338                                          float(log.attrib["start_time"]))
 339     else:
 340         return ""
 341
 342
 343 def format_top_log(log, path_mapper):
 344     return tag("div",
 345                format_time(log, "end_time"),
 346                format_log(log, path_mapper),
 347                format_time(log, "start_time"))
 348
 349
 350 def write_xml(fh, xml):
 351     fh.write(etree.tostring(xml, pretty_print=True))
 352
 353
 354 def write_xml_file(filename, xml):
 355     fh = open(filename, "w")
 356     try:
 357         write_xml(fh, xml)
 358     finally:
 359         fh.close()
 360
 361
 362 def format_logs(targets, path_mapper, dest_filename):
 363     headings = tag("tr", *[tag("th", target.get_name())
 364                            for target in targets])
 365     columns = tag("tr")
 366     for target in targets:
 367         column = tag("td")
 368         for log in target.get_logs():
 369             column.append(format_top_log(log.get_xml(), path_mapper))
 370         columns.append(column)
 371     html = wrap_body(tagp("table", [("class", "summary")],
 372                           headings, columns))
 373     write_xml_file(dest_filename, html)
 374
 375
 376 def format_short_summary(log, path_mapper):
 377     table = tagp("table", [("class", "short_results")])
 378     for child in reversed(log.xpath("log")):
 379         row = tag("tr")
 380         row.append(tag("td", child.attrib["name"]))
 381         row.append(tag("td", format_time(child, "end_time")))
 382         row.append(tag("td", log_duration(child)))
 383
 384         def filter(log):
 385             return int(log.attrib.get("result", "0")) != 0
 386
 387         info = flatten(format_log(child, path_mapper, filter))
 388         if len(info) == 0:
 389             info = tagp("div", [("class", "log %s" % log_class(child))], "ok")
 390         row.append(tag("td", info))
 391         table.append(row)
 392     return table
 393
 394
 395 def for_some(iterator):
 396     for x in iterator:
 397         if x:
 398             return True
 399     return False
 400
 401
 402 def log_has_failed(log):
 403     return (int(log.attrib.get("result", 0)) != 0 or
 404             for_some(log_has_failed(sublog) for sublog in log.xpath("log")))
 405
 406
 407 def warn_failures(logs, stamp_time):
 408     for log in logs:
 409         if log_has_failed(log.get_xml()):
 410             if log.get_timestamp() > stamp_time:
 411                 print "failed"
 412                 subprocess.call("beep -l 10 -f 1000", shell=True)
 413         return
 414     print "no completed logs"
 415
 416
 417 def wrap_body(body):
 418     return tagp("html", [],
 419                 tagp("link", [("rel", "stylesheet"), ("href", "log.css")]),
 420                 body)