checks/job

   1 #!/usr/bin/python
   2 # -*- encoding: utf-8; py-indent-offset: 4 -*-
   3 # +------------------------------------------------------------------+
   4 # |             ____ _               _        __  __ _  __           |
   5 # |            / ___| |__   ___  ___| | __   |  \/  | |/ /           |
   6 # |           | |   | '_ \ / _ \/ __| |/ /   | |\/| | ' /            |
   7 # |           | |___| | | |  __/ (__|   <    | |  | | . \            |
   8 # |            \____|_| |_|\___|\___|_|\_\___|_|  |_|_|\_\           |
   9 # |                                                                  |
  10 # | Copyright Mathias Kettner 2014             mk@mathias-kettner.de |
  11 # +------------------------------------------------------------------+
  12 #
  13 # This file is part of Check_MK.
  14 # The official homepage is at http://mathias-kettner.de/check_mk.
  15 #
  16 # check_mk is free software;  you can redistribute it and/or modify it
  17 # under the  terms of the  GNU General Public License  as published by
  18 # the Free Software Foundation in version 2.  check_mk is  distributed
  19 # in the hope that it will be useful, but WITHOUT ANY WARRANTY;  with-
  20 # out even the implied warranty of  MERCHANTABILITY  or  FITNESS FOR A
  21 # PARTICULAR PURPOSE. See the  GNU General Public License for more de-
  22 # tails. You should have  received  a copy of the  GNU  General Public
  23 # License along with GNU Make; see the file  COPYING.  If  not,  write
  24 # to the Free Software Foundation, Inc., 51 Franklin St,  Fifth Floor,
  25 # Boston, MA 02110-1301 USA.
  26
  27 # <<<job>>>
  28 # ==> asd ASD <==
  29 # start_time 1389355839
  30 # exit_code 0
  31 # real_time 0:00.00
  32 # user_time 0.00
  33 # system_time 0.00
  34 # reads 0
  35 # writes 0
  36 # max_res_kbytes 1968
  37 # avg_mem_kbytes 0
  38 #
  39 #
  40 # ==> test <==
  41 # start_time 1389352839
  42 # exit_code 0
  43 # real_time 0:00.00
  44 # user_time 0.00
  45 # system_time 0.00
  46 # reads 0
  47 # writes 0
  48 # max_res_kbytes 1984
  49 # avg_mem_kbytes 0
  50
  51 factory_settings["job_default_levels"] = {
  52     "age": (0, 0)  # disabled as default
  53 }
  54
  55
  56 def parse_job(info):
  57     def job_parse_real_time(s):
  58         parts = s.split(':')
  59         min_sec, hour_sec = 0, 0
  60         if len(parts) == 3:
  61             hour_sec = int(parts[0]) * 60 * 60
  62         if len(parts) >= 2:
  63             min_sec = int(parts[-2]) * 60
  64         return float(parts[-1]) + min_sec + hour_sec
  65
  66     parsed = {}
  67     job = None
  68     prefix = None
  69     for line in info:
  70         node_info = line[0]
  71         line = line[1:]
  72         if line[0] == "==>" and line[-1] == "<==":
  73             jobname = " ".join(line[1:-1])
  74             if jobname.endswith(".running"):
  75                 jobname, jobstate = jobname.rsplit(".", 1)
  76                 prefix = "running_"
  77             else:
  78                 jobstate = None
  79                 prefix = ""
  80             job = parsed.setdefault(jobname, {}).setdefault(node_info, {
  81                 "state": jobstate,
  82             })
  83
  84         elif job is not None and prefix is not None and len(line) == 2:
  85             key, val = line
  86             # Convert several keys/values
  87             if key == 'real_time':
  88                 val = job_parse_real_time(val)
  89             elif key in ['user_time', 'system_time']:
  90                 val = float(val)
  91             elif key in [
  92                     'exit_code', 'invol_context_switches', 'vol_context_switches', 'start_time'
  93             ]:
  94                 val = int(val)
  95             elif key in ['max_res_kbytes', 'avg_mem_kbytes']:
  96                 key = key.replace('kbytes', 'bytes')
  97                 val = int(val) * 1000
  98             # TODO: Fix the code and remove the pragma below!
  99             job[prefix + key] = val  # pylint: disable=unsupported-assignment-operation
 100
 101     return parsed
 102
 103
 104 def inventory_job(parsed):
 105     for jobname, nodes in parsed.iteritems():
 106         for jobattrs in nodes.values():
 107             if jobattrs["state"] != "running":
 108                 yield jobname, {}
 109
 110
 111 @get_parsed_item_data
 112 def check_job(item, params, nodes):
 113     def process_start_time(value, state, warn, crit):
 114         display_value = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(value))
 115         job_age = time.time() - value
 116         if crit > 0 and job_age >= crit:
 117             state = max(state, 2)
 118             display_value += "(!!) (more than %s ago)" % get_age_human_readable(crit)
 119         elif warn > 0 and job_age >= warn:
 120             state = max(state, 1)
 121             display_value += "(!!) (more than %s ago)" % get_age_human_readable(warn)
 122         return state, display_value
 123
 124     states = []
 125     warn, crit = params.get('age')
 126     results = []
 127     for node_info, job in nodes.iteritems():
 128         if node_info:
 129             node_infotext = "[%s] " % node_info
 130         else:
 131             node_infotext = ""
 132
 133         if job.get("exit_code") is None:
 134             results.append((3, '%sGot incomplete information for this job' % node_infotext))
 135             states.append(3)
 136             continue
 137
 138         state = 0
 139         output = []
 140         perfdata = []
 141
 142         if 'running_start_time' in job:
 143             output.append('%sCurrently running' % node_infotext)
 144             state, display_value = process_start_time(job['running_start_time'], state, warn, crit)
 145             output.append('(Started: %s)' % display_value)
 146             results.append((state, ' '.join(output)))
 147             states.append(state)
 148             continue
 149
 150         txt = '%sExit-Code: %d' % (node_infotext, job['exit_code'])
 151         if job['exit_code'] != 0:
 152             state = max(state, 2)
 153             txt += ' (!!)'
 154         output.append(txt)
 155
 156         for key, title in [
 157             ('start_time', 'Started'),
 158             ('real_time', 'Real-Time'),
 159             ('user_time', 'User-Time'),
 160             ('system_time', 'System-Time'),
 161             ('reads', 'Filesystem Reads'),
 162             ('writes', 'Filesystem Writes'),
 163             ('max_res_bytes', 'Max. Memory'),
 164             ('avg_mem_bytes', 'Avg. Memory'),
 165             ('vol_context_switches', 'Vol. Context Switches'),
 166             ('invol_context_switches', 'Invol. Context Switches'),
 167         ]:
 168             value_str = job.get(key)
 169             if value_str is None:
 170                 continue
 171
 172             if key in ['max_res_bytes', 'avg_mem_bytes']:
 173                 value = int(value_str)
 174                 display_value = get_bytes_human_readable(value, 1000)
 175             elif key in ['real_time', 'user_time', 'system_time']:
 176                 value = float(value_str)
 177                 display_value = get_age_human_readable(value)
 178             elif key == 'start_time':
 179                 value = float(value_str)
 180                 state, display_value = process_start_time(value, state, warn, crit)
 181             else:
 182                 display_value = value_str
 183                 value = float(value_str)
 184
 185             output.append('%s: %s' % (title, display_value))
 186             perfdata.append((key, value))
 187         results.append((state, ', '.join(output), perfdata))
 188         states.append(state)
 189
 190     if params.get("outcome_on_cluster", "worst") == "best":
 191         infotexts = []
 192         perfdata = []
 193         for result in results:
 194             infotexts.append(result[1])
 195             if len(result) > 2:
 196                 perfdata += result[2]
 197         yield min(states), ", ".join(infotexts), perfdata
 198     else:
 199         for result in results:
 200             yield result
 201
 202
 203 check_info["job"] = {
 204     'parse_function': parse_job,
 205     'check_function': check_job,
 206     'inventory_function': inventory_job,
 207     'service_description': 'Job %s',
 208     'default_levels_variable': 'job_default_levels',
 209     'group': 'job',
 210     'has_perfdata': True,
 211     'node_info': True,
 212 }