test/functional/combine_logs.py

   1 #!/usr/bin/env python3
   2 """Combine logs from multiple bitcoin nodes as well as the test_framework log.
   3
   4 This streams the combined log output to stdout. Use combine_logs.py > outputfile
   5 to write to an outputfile."""
   6
   7 import argparse
   8 from collections import defaultdict, namedtuple
   9 import heapq
  10 import itertools
  11 import os
  12 import re
  13 import sys
  14
  15 # Matches on the date format at the start of the log event
  16 TIMESTAMP_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6}")
  17
  18 LogEvent = namedtuple('LogEvent', ['timestamp', 'source', 'event'])
  19
  20 def main():
  21     """Main function. Parses args, reads the log files and renders them as text or html."""
  22
  23     parser = argparse.ArgumentParser(usage='%(prog)s [options] <test temporary directory>', description=__doc__)
  24     parser.add_argument('-c', '--color', dest='color', action='store_true', help='outputs the combined log with events colored by source (requires posix terminal colors. Use less -r for viewing)')
  25     parser.add_argument('--html', dest='html', action='store_true', help='outputs the combined log as html. Requires jinja2. pip install jinja2')
  26     args, unknown_args = parser.parse_known_args()
  27
  28     if args.color and os.name != 'posix':
  29         print("Color output requires posix terminal colors.")
  30         sys.exit(1)
  31
  32     if args.html and args.color:
  33         print("Only one out of --color or --html should be specified")
  34         sys.exit(1)
  35
  36     # There should only be one unknown argument - the path of the temporary test directory
  37     if len(unknown_args) != 1:
  38         print("Unexpected arguments" + str(unknown_args))
  39         sys.exit(1)
  40
  41     log_events = read_logs(unknown_args[0])
  42
  43     print_logs(log_events, color=args.color, html=args.html)
  44
  45 def read_logs(tmp_dir):
  46     """Reads log files.
  47
  48     Delegates to generator function get_log_events() to provide individual log events
  49     for each of the input log files."""
  50
  51     files = [("test", "%s/test_framework.log" % tmp_dir)]
  52     for i in itertools.count():
  53         logfile = "{}/node{}/regtest/debug.log".format(tmp_dir, i)
  54         if not os.path.isfile(logfile):
  55             break
  56         files.append(("node%d" % i, logfile))
  57
  58     return heapq.merge(*[get_log_events(source, f) for source, f in files])
  59
  60 def get_log_events(source, logfile):
  61     """Generator function that returns individual log events.
  62
  63     Log events may be split over multiple lines. We use the timestamp
  64     regex match as the marker for a new log event."""
  65     try:
  66         with open(logfile, 'r') as infile:
  67             event = ''
  68             timestamp = ''
  69             for line in infile:
  70                 # skip blank lines
  71                 if line == '\n':
  72                     continue
  73                 # if this line has a timestamp, it's the start of a new log event.
  74                 time_match = TIMESTAMP_PATTERN.match(line)
  75                 if time_match:
  76                     if event:
  77                         yield LogEvent(timestamp=timestamp, source=source, event=event.rstrip())
  78                     event = line
  79                     timestamp = time_match.group()
  80                 # if it doesn't have a timestamp, it's a continuation line of the previous log.
  81                 else:
  82                     event += "\n" + line
  83             # Flush the final event
  84             yield LogEvent(timestamp=timestamp, source=source, event=event.rstrip())
  85     except FileNotFoundError:
  86         print("File %s could not be opened. Continuing without it." % logfile, file=sys.stderr)
  87
  88 def print_logs(log_events, color=False, html=False):
  89     """Renders the iterator of log events into text or html."""
  90     if not html:
  91         colors = defaultdict(lambda: '')
  92         if color:
  93             colors["test"] = "\033[0;36m"   # CYAN
  94             colors["node0"] = "\033[0;34m"  # BLUE
  95             colors["node1"] = "\033[0;32m"  # GREEN
  96             colors["node2"] = "\033[0;31m"  # RED
  97             colors["node3"] = "\033[0;33m"  # YELLOW
  98             colors["reset"] = "\033[0m"     # Reset font color
  99
 100         for event in log_events:
 101             print("{0} {1: <5} {2} {3}".format(colors[event.source.rstrip()], event.source, event.event, colors["reset"]))
 102
 103     else:
 104         try:
 105             import jinja2
 106         except ImportError:
 107             print("jinja2 not found. Try `pip install jinja2`")
 108             sys.exit(1)
 109         print(jinja2.Environment(loader=jinja2.FileSystemLoader('./'))
 110                     .get_template('combined_log_template.html')
 111                     .render(title="Combined Logs from testcase", log_events=[event._asdict() for event in log_events]))
 112
 113 if __name__ == '__main__':
 114     main()