From edd3522ff1926eeda6431fdcfebf9072d0b8ebc1 Mon Sep 17 00:00:00 2001 From: Rodrigo Lazo Date: Fri, 7 Oct 2011 02:13:43 -0300 Subject: [PATCH] Misc changes. --- main.py | 1 - monitor/collector.py | 39 +++++++++++++-------- monitor/extra/plotting.py | 8 +++-- monitor/http_display.py | 89 +++++++++++++++++++++-------------------------- 4 files changed, 70 insertions(+), 67 deletions(-) diff --git a/main.py b/main.py index 61b028c..6c43e71 100755 --- a/main.py +++ b/main.py @@ -66,7 +66,6 @@ def timed_collector(sources, interval): while (True): collected_data = collector.collect() store.insert_dict(collected_data) - print collected_data time.sleep(interval) diff --git a/monitor/collector.py b/monitor/collector.py index 7131fde..ea72525 100755 --- a/monitor/collector.py +++ b/monitor/collector.py @@ -36,7 +36,7 @@ import contextlib import threading import time -from urllib2 import urlopen +from urllib2 import urlopen, URLError REQUEST_TIMEOUT = 5 # Seconds to timeout remote requests @@ -73,21 +73,29 @@ def _retriever(source, timestamp, output_queue): the form """ - host_address = "%s:%s" % source + host_address = "%s:%s" % (source[0], source[1]) url = "http://%s/?format=text" % host_address vars_dict = {} - with contextlib.closing(urlopen(url, timeout=REQUEST_TIMEOUT)) as fd: - for line in fd: - elements = line.split() - if (len(elements) == 2): - fullvarname, value = elements - varname = fullvarname.split('@')[-1] - if '.' in value: - value = float(value) - else: - value = int(value) - vars_dict[varname] = (timestamp, value) - output_queue.put((host_address, vars_dict)) + try: + with contextlib.closing(urlopen(url, timeout=REQUEST_TIMEOUT)) as fd: + for line in fd: + elements = line.split() + if (len(elements) == 2): + fullvarname, value = elements + varname = fullvarname.split('@')[-1] + if value == 'true': + value = 1 + elif value == 'false': + value = 0 + elif '.' in value: + value = float(value) + else: + value = int(value) + vars_dict[varname] = (timestamp, value) + output_queue.put((host_address, vars_dict)) + except URLError: + print "ERROR RETRIEVING %s" % host_address + output_queue.put((None, None)) class HttpCollector(object): @@ -130,7 +138,8 @@ class HttpCollector(object): threads[-1].start() for _ in range(len(self._sources)): host_address, data = queue.get() - result[host_address] = data + if host_address is not None: + result[host_address] = data self._last_collected = result self._timestamp = timestamp return result diff --git a/monitor/extra/plotting.py b/monitor/extra/plotting.py index 990dcc7..cdbec9a 100755 --- a/monitor/extra/plotting.py +++ b/monitor/extra/plotting.py @@ -34,6 +34,7 @@ matplotlib.use('cairo.png') from datetime import datetime import matplotlib.pyplot as plot +from pylab import epoch2num import os.path @@ -62,9 +63,11 @@ def new_multigraph(data_dict, title, target_dir=""): data = values.get_all() xs = [x.timestamp for x in data] # pylint: disable=C0103 ys = [x.value for x in data] # pylint: disable=C0103 - plot.plot(xs, ys, label=name) + plot.plot_date(epoch2num(xs), + ys, '-', label=name) plot.title(title) plot.xlabel("timestamp") + plot.margins(0.1) plot.legend(loc="upper right") plot.grid(True) filename = "%s-%s.png" % (title.replace(' ', '_'), @@ -91,9 +94,10 @@ def new_graph(data_lst, title, target_dir): """ xs = [x[0] for x in data_lst] # pylint: disable=C0103 ys = [x[1] for x in data_lst] # pylint: disable=C0103 - plot.plot(xs, ys) + plot.plot_date(epoch2num(xs), ys, '-') plot.title(title) plot.xlabel("timestamp") + plot.margins(0.1) plot.legend(loc="upper right") plot.grid(True) now_str = datetime.now().strftime("%Y%m%d_%H%M%S") diff --git a/monitor/http_display.py b/monitor/http_display.py index b350852..30cb89c 100755 --- a/monitor/http_display.py +++ b/monitor/http_display.py @@ -39,12 +39,14 @@ __version__ = 0.1 import cherrypy -import os +from jinja2 import Environment, PackageLoader from datastore import DataStore, DataEntry, DataPoint # pylint: disable=W0611 from datetime import datetime -from extra import html_helpers, plotting -from string import Template # pylint: disable=W0402 +from extra import plotting + +def timestampformat(value, format_str='%H:%M / %d-%m-%Y'): + return datetime.fromtimestamp(value).strftime(format_str) # TODO: (09/19) add support for custom made plots based on GET request # params, providing an on-line plotting tool. @@ -56,6 +58,8 @@ class MonitorHttpHandler(object): self._store = store self._templates_path = templates_path self._static_path = static_path + self.env = Environment(loader=PackageLoader('monitor', 'www_templates')) + self.env.filters["timestampformat"] = timestampformat @cherrypy.expose def index(self): @@ -64,12 +68,10 @@ class MonitorHttpHandler(object): Template: index.html Vars: title, timestamp, body. """ - references = {"title": "Monitoring", - "timestamp": str(datetime.now()), - "body": "

seee

"} - with open(os.path.join(self._templates_path, "index.html")) as fd: - content = Template(fd.read()).safe_substitute(references) - return content + template = self.env.get_template('index.html') + return template.render({"title": "Monitoring", + "timestamp": str(datetime.now()), + "body": "

seee

"}) @cherrypy.expose def vars(self): @@ -81,14 +83,10 @@ class MonitorHttpHandler(object): Template: simple.html Vars: title, timestamp, body. """ - variables = {v: "/var?name=%s" % v for v in self._store.list_vars()} - body = html_helpers.map_as_ali(variables) - references = {"title": "List of monitored vars", - "timestamp": str(datetime.now()), - "body": body} - with open(os.path.join(self._templates_path, "simple.html")) as fd: - content = Template(fd.read()).safe_substitute(references) - return content + template = self.env.get_template('vars.html') + return template.render({"title": "List of monitored vars", + "timestamp": str(datetime.now()), + "vars" : self._store.list_vars()}) @cherrypy.expose def hosts(self): @@ -100,14 +98,10 @@ class MonitorHttpHandler(object): Template: simple.html Vars: title, timestamp, body. """ - hosts = {v: "/host?name=%s" % v for v in self._store.list_groups()} - body = html_helpers.map_as_ali(hosts) - references = {"title": "List of monitored hosts", - "timestamp": str(datetime.now()), - "body": body} - with open(os.path.join(self._templates_path, "simple.html")) as fd: - content = Template(fd.read()).safe_substitute(references) - return content + template = self.env.get_template('hosts.html') + return template.render({"title": "List of monitored vars", + "timestamp": str(datetime.now()), + "vars" : self._store.list_groups()}) @cherrypy.expose def var(self, name): @@ -121,17 +115,15 @@ class MonitorHttpHandler(object): Template: simple.html Vars: title, timestamp, body. """ - data = self._store.get_var(name) - if not data: - return "

Empty dataset. Check varname

" - filename, _ = plotting.new_multigraph(data, name, self._static_path) - body = "" % filename references = {"title": "Historic Values of var: %s" % name, - "timestamp": str(datetime.now()), - "body": body} - with open(os.path.join(self._templates_path, "simple.html")) as fd: - content = Template(fd.read()).safe_substitute(references) - return content + "timestamp": str(datetime.now())} + data = self._store.get_var(name) + if data: + filename, _ = plotting.new_multigraph(data, name, self._static_path) + references["filename"] = filename + references["items"] = [{'hostname' : k, 'value': v.get_latest()} for k, v in data.iteritems()] + template = self.env.get_template('var.html') + return template.render(references) @cherrypy.expose def host(self, name): @@ -146,21 +138,20 @@ class MonitorHttpHandler(object): Template: simple.html Vars: title, timestamp, body. """ - data = self._store.get_group(name) - if not data: - return "

Empty dataset. Check groupname

" - body = [] - for k, entry in data.iteritems(): - values = [x.as_tuple() for x in entry.get_all()] - filename, _ = plotting.new_graph(values, k, self._static_path) - body.append("

" % filename) references = {"title": "Historic Values of host: %s" % name, - "timestamp": str(datetime.now()), - "body": "\n".join(body)} - with open(os.path.join(self._templates_path, "simple.html")) as fd: - content = Template(fd.read()).safe_substitute(references) - return content - + "timestamp": str(datetime.now())} + data = self._store.get_group(name) + if data: + items = [] + for k, entry in data.iteritems(): + values = entry.get_all() + tuples = [x.as_tuple() for x in values] + filename, _ = plotting.new_graph(tuples, k, self._static_path) + items.append({'file': filename, 'value': values[-1]}) + references["items"] = items + print data + template = self.env.get_template('host.html') + return template.render(references) def main(store, port, templates_path, static_path): """HTTP server starter. -- 2.11.4.GIT