checks/jolokia_metrics

   1 #!/usr/bin/python
   2 # -*- encoding: utf-8; py-indent-offset: 4 -*-
   3 # +------------------------------------------------------------------+
   4 # |             ____ _               _        __  __ _  __           |
   5 # |            / ___| |__   ___  ___| | __   |  \/  | |/ /           |
   6 # |           | |   | '_ \ / _ \/ __| |/ /   | |\/| | ' /            |
   7 # |           | |___| | | |  __/ (__|   <    | |  | | . \            |
   8 # |            \____|_| |_|\___|\___|_|\_\___|_|  |_|_|\_\           |
   9 # |                                                                  |
  10 # | Copyright Mathias Kettner 2014             mk@mathias-kettner.de |
  11 # +------------------------------------------------------------------+
  12 #
  13 # This file is part of Check_MK.
  14 # The official homepage is at http://mathias-kettner.de/check_mk.
  15 #
  16 # check_mk is free software;  you can redistribute it and/or modify it
  17 # under the  terms of the  GNU General Public License  as published by
  18 # the Free Software Foundation in version 2.  check_mk is  distributed
  19 # in the hope that it will be useful, but WITHOUT ANY WARRANTY;  with-
  20 # out even the implied warranty of  MERCHANTABILITY  or  FITNESS FOR A
  21 # PARTICULAR PURPOSE. See the  GNU General Public License for more de-
  22 # tails. You should have  received  a copy of the  GNU  General Public
  23 # License along with GNU Make; see the file  COPYING.  If  not,  write
  24 # to the Free Software Foundation, Inc., 51 Franklin St,  Fifth Floor,
  25 # Boston, MA 02110-1301 USA.
  26
  27 # Example output from agent:
  28 # <<<jolokia_metrics>>>
  29 # 8080 NonHeapMemoryUsage 101078952
  30 # 8080 NonHeapMemoryMax 184549376
  31 # 8080 HeapMemoryUsage 2362781664
  32 # 8080 HeapMemoryMax 9544663040
  33 # 8080 ThreadCount 78
  34 # 8080 DeamonThreadCount 72
  35 # 8080 PeakThreadCount 191
  36 # 8080 TotalStartedThreadCount 941
  37 # 8080 Uptime 572011375
  38 # 8080,java.lang:name=PS_MarkSweep,type=GarbageCollector CollectionCount 0
  39
  40 # MB warn, crit
  41 # jolokia_metrics_mem_default_levels = (2000, 3000)
  42
  43 # Number of threads warn, crit
  44 jolokia_metrics_threads_default_levels = (80, 100)
  45
  46 # Number of sessions low crit, low warn, high warn, high crit
  47 jolokia_metrics_app_sess_default_levels = (-1, -1, 800, 1000)
  48
  49 # Number of requests low crit, low warn, high warn, high crit
  50 jolokia_metrics_serv_req_default_levels = (-1, -1, 5000, 6000)
  51
  52 jolokia_metrics_queue_default_levels = (20, 50)
  53
  54 # Tomcat ThreadPools Count/Busy in relation to max value
  55 factory_settings["jolokia_metrics_tp_default_levels"] = {
  56     'currentThreadsBusy': (80, 90),
  57 }
  58
  59 #   .--Parse function------------------------------------------------------.
  60 #   |  ____                        __                  _   _               |
  61 #   | |  _ \ __ _ _ __ ___  ___   / _|_   _ _ __   ___| |_(_) ___  _ __    |
  62 #   | | |_) / _` | '__/ __|/ _ \ | |_| | | | '_ \ / __| __| |/ _ \| '_ \   |
  63 #   | |  __/ (_| | |  \__ \  __/ |  _| |_| | | | | (__| |_| | (_) | | | |  |
  64 #   | |_|   \__,_|_|  |___/\___| |_|  \__,_|_| |_|\___|\__|_|\___/|_| |_|  |
  65 #   |                                                                      |
  66 #   '----------------------------------------------------------------------'
  67
  68
  69 def jolokia_metrics_parse(info):
  70     parsed = {}
  71     for line in info:
  72         if len(line) > 1 and line[1] == "ERROR":
  73             continue
  74
  75         try:
  76             inst_raw, var, value = jolokia_basic_split(line, 3)
  77         except ValueError:
  78             continue
  79
  80         inst, attributes, positional = jolokoia_extract_opt(inst_raw)
  81
  82         parsed.setdefault(inst, {})
  83
  84         if 'type' in attributes:
  85             bean_name = attributes.pop('name')
  86             bean_type = attributes.pop('type')
  87             # backwards compatibility
  88             bean_type = {"GarbageCollector": "gc", "ThreadPool": "tp"}.get(bean_type, bean_type)
  89             # maybe do this for all types?
  90             if bean_type == "tp":
  91                 bean_name = bean_name.replace('"', '')
  92
  93             bean = parsed[inst].setdefault(bean_type, {}).setdefault(bean_name, {})
  94             bean[var] = value
  95             bean.update(attributes)
  96         else:
  97             if positional:
  98                 app = positional[0]
  99                 app_dict = parsed[inst].setdefault('apps', {}).setdefault(app, {})
 100                 if len(positional) > 1:
 101                     servlet = positional[1]
 102                     app_dict.setdefault('servlets', {}).setdefault(servlet, {})
 103                     app_dict['servlets'][servlet][var] = value
 104                 else:
 105                     app_dict[var] = value
 106             else:
 107                 parsed[inst][var] = value
 108     return parsed
 109
 110
 111 #.
 112 #   .--Generic inventory functions-----------------------------------------.
 113 #   |                   ____                      _                        |
 114 #   |                  / ___| ___ _ __   ___ _ __(_) ___                   |
 115 #   |                 | |  _ / _ \ '_ \ / _ \ '__| |/ __|                  |
 116 #   |                 | |_| |  __/ | | |  __/ |  | | (__                   |
 117 #   |                  \____|\___|_| |_|\___|_|  |_|\___|                  |
 118 #   |                                                                      |
 119 #   |             _                      _                                 |
 120 #   |            (_)_ ____   _____ _ __ | |_ ___  _ __ _   _               |
 121 #   |            | | '_ \ \ / / _ \ '_ \| __/ _ \| '__| | | |              |
 122 #   |            | | | | \ V /  __/ | | | || (_) | |  | |_| |              |
 123 #   |            |_|_| |_|\_/ \___|_| |_|\__\___/|_|   \__, |              |
 124 #   |                                                  |___/               |
 125 #   |              __                  _   _                               |
 126 #   |             / _|_   _ _ __   ___| |_(_) ___  _ __  ___               |
 127 #   |            | |_| | | | '_ \ / __| __| |/ _ \| '_ \/ __|              |
 128 #   |            |  _| |_| | | | | (__| |_| | (_) | | | \__ \              |
 129 #   |            |_|  \__,_|_| |_|\___|\__|_|\___/|_| |_|___/              |
 130 #   |                                                                      |
 131 #   '----------------------------------------------------------------------'
 132
 133
 134 def inventory_jolokia_metrics(info, what):
 135     parsed = jolokia_metrics_parse(info)
 136
 137     levels = None
 138
 139     if what == 'mem':
 140         levels = {}
 141     elif what == 'threads':
 142         levels = 'jolokia_metrics_threads_default_levels'
 143
 144     for instance, data in parsed.items():
 145         if data is None:
 146             continue  # No connection to agent currently
 147
 148         if what == 'uptime' and "Uptime" not in data:
 149             continue
 150         if what == 'mem' and ("HeapMemoryUsage" not in data or "NonHeapMemoryUsage" not in data or
 151                               "HeapMemoryMax" not in data or "NonHeapMemoryMax" not in data):
 152             # don't add memory check if we don't have the necessary data
 153             continue
 154         yield instance, levels
 155
 156
 157 def inventory_jolokia_metrics_apps(info, what):
 158     inv = []
 159     parsed = jolokia_metrics_parse(info)
 160
 161     if what == 'app_sess':
 162         levels = 'jolokia_metrics_app_sess_default_levels'
 163         needed_key = ["Sessions", "activeSessions"]
 164     elif what == 'bea_app_sess':
 165         levels = 'jolokia_metrics_app_sess_default_levels'
 166         needed_key = ["OpenSessionsCurrentCount"]
 167     elif what == 'queue':
 168         needed_key = ["QueueLength"]
 169         levels = "jolokia_metrics_queue_default_levels"
 170     # Only works on BEA
 171     elif what == 'bea_requests':
 172         needed_key = ["CompletedRequestCount"]
 173         levels = None
 174     elif what == 'requests':
 175         needed_key = ["requestCount"]
 176         levels = None
 177     elif what == 'threads':
 178         needed_key = ["StandbyThreadCount"]
 179         levels = None
 180     else:
 181         needed_key = ["Running", "stateName"]
 182         levels = None
 183
 184     # this handles information from BEA, they stack one level
 185     # higher than the rest.
 186     if what == 'bea_app_sess':
 187         for inst, vals in parsed.iteritems():
 188             if vals is None:
 189                 continue  # no data from agent
 190
 191             for app, appstate in vals.get('apps', {}).items():
 192                 if 'servlets' in appstate:
 193                     for nk in needed_key:
 194                         for servlet in appstate['servlets']:
 195                             if nk in appstate['servlets'][servlet]:
 196                                 inv.append(('%s %s %s' % (inst, app, servlet), levels))
 197                                 continue
 198     # This does the same for tomcat
 199     for inst, vals in parsed.iteritems():
 200         if vals is None:
 201             continue  # no data from agent
 202
 203         for app, appstate in vals.get('apps', {}).items():
 204             for nk in needed_key:
 205                 if nk in appstate:
 206                     inv.append(('%s %s' % (inst, app), levels))
 207                     continue
 208     return inv
 209
 210
 211 #.
 212 #   .--Arcane helpers------------------------------------------------------.
 213 #   |                     _                                                |
 214 #   |                    / \   _ __ ___ __ _ _ __   ___                    |
 215 #   |                   / _ \ | '__/ __/ _` | '_ \ / _ \                   |
 216 #   |                  / ___ \| | | (_| (_| | | | |  __/                   |
 217 #   |                 /_/   \_\_|  \___\__,_|_| |_|\___|                   |
 218 #   |                                                                      |
 219 #   |                  _          _                                        |
 220 #   |                 | |__   ___| |_ __   ___ _ __ ___                    |
 221 #   |                 | '_ \ / _ \ | '_ \ / _ \ '__/ __|                   |
 222 #   |                 | | | |  __/ | |_) |  __/ |  \__ \                   |
 223 #   |                 |_| |_|\___|_| .__/ \___|_|  |___/                   |
 224 #   |                              |_|                                     |
 225 #   +----------------------------------------------------------------------+
 226 #   | TODO: See if these can be removed altogether                         |
 227 #   '----------------------------------------------------------------------'
 228
 229
 230 # This bisects the app server and its values
 231 def jolokia_metrics_app(info, split_item):
 232     inst, app = split_item
 233     parsed = jolokia_metrics_parse(info)
 234     if parsed.get(inst, "") is None:
 235         raise MKCounterWrapped("No information from Jolokia agent")
 236     if not inst in parsed \
 237        or not app in parsed[inst].get('apps', {}):
 238         return None
 239     return parsed[inst]['apps'][app]
 240
 241
 242 # This bisects info from BEA and passes on to jolokia_metrics_app
 243 def jolokia_metrics_serv(info, split_item):
 244     inst, app, serv = split_item
 245     app = jolokia_metrics_app(info, (inst, app))
 246     if not app or not serv in app.get('servlets', {}):
 247         return None
 248     return app['servlets'][serv]
 249
 250
 251 def jolokia_metrics_gc(info, split_item):
 252     inst, _typ, gc = split_item
 253     parsed = jolokia_metrics_parse(info)
 254     if parsed.get(inst, "") is None:
 255         raise MKCounterWrapped("No information from Jolokia agent")
 256
 257     if not inst in parsed \
 258        or not gc in parsed[inst].get('gc', {}):
 259         return None
 260     return parsed[inst]['gc'][gc]
 261
 262
 263 #.
 264 #   .--Number of Requests--------------------------------------------------.
 265 #   |               ____                            _                      |
 266 #   |              |  _ \ ___  __ _ _   _  ___  ___| |_ ___                |
 267 #   |              | |_) / _ \/ _` | | | |/ _ \/ __| __/ __|               |
 268 #   |              |  _ <  __/ (_| | |_| |  __/\__ \ |_\__ \               |
 269 #   |              |_| \_\___|\__, |\__,_|\___||___/\__|___/               |
 270 #   |                            |_|                                       |
 271 #   '----------------------------------------------------------------------'
 272
 273
 274 def inventory_jolokia_metrics_serv(info):
 275     inv = []
 276     parsed = jolokia_metrics_parse(info)
 277     levels = None
 278     levels = 'jolokia_metrics_serv_req_default_levels'
 279     needed_key = "Requests"
 280     for inst, vals in parsed.iteritems():
 281         if vals is None:
 282             continue  # no data from agent
 283         for app, val in vals.get('apps', {}).iteritems():
 284             for serv, servinfo in val.get('servlets', {}).items():
 285                 if needed_key in servinfo:
 286                     inv.append(('%s %s %s' % (inst, app, serv), levels))
 287     return inv
 288
 289
 290 def check_jolokia_metrics_serv_req(item, params, info):
 291     lo_crit, lo_warn, hi_warn, hi_crit = params
 292     serv = jolokia_metrics_serv(info, item.split())
 293     if not serv or not 'Requests' in serv:
 294         return (3, "data not found in agent output")
 295     req = saveint(serv['Requests'])
 296
 297     status = 0
 298     status_txt = ''
 299     if lo_crit is not None and req <= lo_crit:
 300         status = 2
 301         status_txt = ' (Below or equal %d)' % lo_crit
 302     elif lo_warn is not None and req <= lo_warn:
 303         status = 1
 304         status_txt = ' (Below or equal %d)' % lo_warn
 305     elif hi_crit is not None and req >= hi_crit:
 306         status = 2
 307         status_txt = ' (Above or equal %d)' % hi_crit
 308     elif hi_warn is not None and req >= hi_warn:
 309         status = 1
 310         status_txt = ' (Above or equal %d)' % hi_warn
 311
 312     output = ['Requests: %d%s' % (req, status_txt)]
 313     perfdata = [('Requests', req, hi_warn, hi_crit)]
 314     wrapped = False
 315     this_time = time.time()
 316     try:
 317         rate = get_rate("jolokia_metrics.serv_req.%s" % item, this_time, req)
 318         output.append('RequestRate: %0.2f' % rate)
 319         perfdata.append(('RequestRate', rate))
 320     except MKCounterWrapped:
 321         wrapped = True
 322
 323     if wrapped:
 324         return (status, ', '.join(output))
 325     return (status, ', '.join(output), perfdata)
 326
 327
 328 check_info["jolokia_metrics.serv_req"] = {
 329     "includes": ["jolokia.include"],
 330     "service_description": "JVM %s Requests",
 331     "check_function": check_jolokia_metrics_serv_req,
 332     "inventory_function": inventory_jolokia_metrics_serv,
 333     "group": "jvm_requests",
 334     "has_perfdata": True,
 335 }
 336
 337 #.
 338 #   .--Garbage collection--------------------------------------------------.
 339 #   |                ____            _                                     |
 340 #   |               / ___| __ _ _ __| |__   __ _  __ _  ___                |
 341 #   |              | |  _ / _` | '__| '_ \ / _` |/ _` |/ _ \               |
 342 #   |              | |_| | (_| | |  | |_) | (_| | (_| |  __/               |
 343 #   |               \____|\__,_|_|  |_.__/ \__,_|\__, |\___|               |
 344 #   |                                            |___/                     |
 345 #   |                        _ _           _   _                           |
 346 #   |               ___ ___ | | | ___  ___| |_(_) ___  _ __                |
 347 #   |              / __/ _ \| | |/ _ \/ __| __| |/ _ \| '_ \               |
 348 #   |             | (_| (_) | | |  __/ (__| |_| | (_) | | | |              |
 349 #   |              \___\___/|_|_|\___|\___|\__|_|\___/|_| |_|              |
 350 #   |                                                                      |
 351 #   '----------------------------------------------------------------------'
 352
 353
 354 def inventory_jolokia_metrics_gc(info):
 355     inv = []
 356     parsed = jolokia_metrics_parse(info)
 357     for inst, vals in parsed.iteritems():
 358         if vals is None:
 359             continue  # no data from agent
 360         for gc in vals.get('gc', {}).iterkeys():
 361             inv.append(("%s GC %s" % (inst, gc), {}))
 362     return inv
 363
 364
 365 def check_jolokia_metrics_gc(item, params, info):
 366     gc = jolokia_metrics_gc(info, item.split())
 367     if gc is None:
 368         return
 369
 370     if params is None:
 371         params = {}
 372
 373
 374     crate = get_rate("jvm.gc.count.%s" % (item), \
 375                      time.time(), int(gc['CollectionCount']))
 376     crate = crate * 60.0
 377
 378     ctext = ''
 379     status = 0
 380     cwarn, ccrit = params.get('CollectionCount', (None, None))
 381     if cwarn is not None and ccrit is not None:
 382         if crate >= int(ccrit):
 383             status = 2
 384             ctext = " (Level %s) " % ccrit
 385         elif crate >= int(cwarn):
 386             status = 1
 387             ctext = " (Level %s) " % cwarn
 388
 389     yield status, "%.2f GC Count/minute%s" % (crate, ctext), \
 390                     [('CollectionCount', crate, cwarn, ccrit)]
 391
 392     if 'CollectionTime' in gc:
 393         twarn, tcrit = params.get('CollectionTime', (None, None))
 394         trate = get_rate("jvm.gc.time.%s" % (item), \
 395                          time.time(), int(gc['CollectionTime']))
 396         trate = trate * 60.0
 397
 398         ttext = ''
 399         status = 0
 400         if twarn is not None and tcrit is not None:
 401             if trate >= int(tcrit):
 402                 status = 2
 403                 ttext = "(Level %s) " % tcrit
 404             elif trate >= int(twarn):
 405                 status = 1
 406                 ttext = "(Level %s) " % twarn
 407
 408         yield status, "%.2f GC ms/minute%s" % (trate, ttext), \
 409                             [('CollectionTime', trate, twarn, tcrit)]
 410
 411
 412 check_info["jolokia_metrics.gc"] = {
 413     "includes": ["jolokia.include"],
 414     "service_description": "JVM %s",
 415     "check_function": check_jolokia_metrics_gc,
 416     "inventory_function": inventory_jolokia_metrics_gc,
 417     "group": "jvm_gc",
 418     "has_perfdata": True,
 419 }
 420
 421 #.
 422 #   .--Tomcat thread pool--------------------------------------------------.
 423 #   | _____                         _     _   _                        _   |
 424 #   ||_   _|__  _ __ ___   ___ __ _| |_  | |_| |__  _ __ ___  __ _  __| |  |
 425 #   |  | |/ _ \| '_ ` _ \ / __/ _` | __| | __| '_ \| '__/ _ \/ _` |/ _` |  |
 426 #   |  | | (_) | | | | | | (_| (_| | |_  | |_| | | | | |  __/ (_| | (_| |  |
 427 #   |  |_|\___/|_| |_| |_|\___\__,_|\__|  \__|_| |_|_|  \___|\__,_|\__,_|  |
 428 #   |                                                                      |
 429 #   |                                           _                          |
 430 #   |                         _ __   ___   ___ | |                         |
 431 #   |                        | '_ \ / _ \ / _ \| |                         |
 432 #   |                        | |_) | (_) | (_) | |                         |
 433 #   |                        | .__/ \___/ \___/|_|                         |
 434 #   |                        |_|                                           |
 435 #   '----------------------------------------------------------------------'
 436
 437
 438 def jolokia_metrics_tp(info, split_item):
 439     inst, _typ, threadpool_name = split_item
 440     parsed = jolokia_metrics_parse(info)
 441     if parsed.get(inst, "") is None:
 442         raise MKCounterWrapped("No information from Jolokia agent")
 443
 444     if not inst in parsed \
 445        or not threadpool_name in parsed[inst].get('tp', {}):
 446         return None
 447     return parsed[inst]['tp'][threadpool_name]
 448
 449
 450 def inventory_jolokia_metrics_tp(info):
 451     inv = []
 452     parsed = jolokia_metrics_parse(info)
 453     for inst, vals in parsed.iteritems():
 454         if vals is None:
 455             continue  # no data from agent
 456         for threadpool_name, threadpool_info in vals.get('tp', {}).iteritems():
 457             if "maxThreads" in threadpool_info:
 458                 inv.append(("%s ThreadPool %s" % (inst, threadpool_name), {}))
 459     return inv
 460
 461
 462 def check_jolokia_metrics_tp(item, params, info):
 463
 464     if params is None:
 465         params = {}
 466
 467     threadpool_info = jolokia_metrics_tp(info, item.split())
 468
 469     if threadpool_info is None or "maxThreads" not in threadpool_info:
 470         return
 471
 472     max_threads = float(threadpool_info["maxThreads"])
 473
 474     def check_thread_levels(what):
 475         threads_abs = int(threadpool_info[what])
 476         threads_perc = threads_abs * 100. / max_threads
 477         infotext = "%s: %d (%s)" % (what, threads_abs, get_percent_human_readable(threads_perc))
 478         status = 0
 479
 480         _params = params.get(what)
 481         if _params and _params[0] is not None and _params[1] is not None:
 482             warn, crit = params[what]
 483             levelstext = " (warn/crit at %s/%s)" % (get_percent_human_readable(warn),
 484                                                     get_percent_human_readable(crit))
 485             if threads_perc >= crit:
 486                 status = 2
 487                 infotext += levelstext
 488             elif threads_perc >= warn:
 489                 status = 1
 490                 infotext += levelstext
 491         else:
 492             warn, crit = ("", "")
 493         perfdata = [(what, threads_abs, warn, crit, 0, max_threads)]
 494         return status, infotext, perfdata
 495
 496     if 'currentThreadsBusy' in threadpool_info:
 497         yield check_thread_levels('currentThreadsBusy')
 498     if 'currentThreadCount' in threadpool_info:
 499         yield check_thread_levels('currentThreadCount')
 500     yield 0, "Maximum threads: %d" % max_threads
 501
 502
 503 check_info["jolokia_metrics.tp"] = {
 504     "includes": ["jolokia.include"],
 505     "default_levels_variable": "jolokia_metrics_tp_default_levels",
 506     "service_description": "JVM %s",
 507     "check_function": check_jolokia_metrics_tp,
 508     "inventory_function": inventory_jolokia_metrics_tp,
 509     "group": "jvm_tp",
 510     "has_perfdata": True,
 511 }
 512
 513 #.
 514 #   .--Memory--------------------------------------------------------------.
 515 #   |               __  __                                                 |
 516 #   |              |  \/  | ___ _ __ ___   ___  _ __ _   _                 |
 517 #   |              | |\/| |/ _ \ '_ ` _ \ / _ \| '__| | | |                |
 518 #   |              | |  | |  __/ | | | | | (_) | |  | |_| |                |
 519 #   |              |_|  |_|\___|_| |_| |_|\___/|_|   \__, |                |
 520 #   |                                                |___/                 |
 521 #   '----------------------------------------------------------------------'
 522
 523
 524 def check_jolokia_metrics_mem(item, params, info):
 525     parsed = jolokia_metrics_parse(info)
 526
 527     if parsed.get(item, "") is None:
 528         raise MKCounterWrapped("No information from Jolokia agent")
 529
 530     if item not in parsed:
 531         return
 532
 533     # convert old parameter version ( warn, crit )
 534     # represented levels of total heap
 535     if isinstance(params, tuple):
 536         params = {"total": params}
 537
 538     # rename totalheap to total
 539     # this block can be removed in the future (today 22.02.13)
 540     if "totalheap" in params:
 541         params = params.copy()
 542         params["total"] = params["totalheap"]
 543         del params["totalheap"]
 544
 545     d = parsed[item]
 546     mb = 1024 * 1024.0
 547
 548     if "HeapMemoryUsage" not in d or "NonHeapMemoryUsage" not in d\
 549         or "HeapMemoryMax" not in d or "NonHeapMemoryMax" not in d:
 550         return 3, "data in agent output incomplete"
 551
 552     heap = saveint(d["HeapMemoryUsage"]) / mb
 553     heapmax = saveint(d.get("HeapMemoryMax", -1)) / mb
 554     nonheap = saveint(d["NonHeapMemoryUsage"]) / mb
 555     nonheapmax = saveint(d.get("NonHeapMemoryMax", -1)) / mb
 556     total = heap + nonheap
 557     if heapmax > 0 and nonheapmax > 0:
 558         totalmax = heapmax + nonheapmax
 559     else:
 560         totalmax = 0
 561         heapmax = max(0, heapmax)
 562         nonheapmax = max(0, nonheapmax)
 563
 564     worst_state = 0
 565     perfdata = []
 566     info_list = []
 567
 568     for what, value, value_max in [
 569         ("heap", heap, heapmax),
 570         ("nonheap", nonheap, nonheapmax),
 571         ("total", total, totalmax),
 572     ]:
 573         param_state = 0
 574         level_info = ""
 575         used_info = ""
 576         if params.get(what):
 577             warn_level = 0
 578             crit_level = 0
 579             if isinstance(params[what][0], int):
 580                 if value_max:
 581                     warn_level = value_max - params[what][0]
 582                     crit_level = value_max - params[what][1]
 583
 584                 if what != "total":
 585                     perfdata.append((what, value, warn_level, crit_level, "", value_max))
 586
 587                 if not value_max:
 588                     param_state = 0
 589                 elif value >= crit_level:
 590                     param_state = 2
 591                     level_info = "%s(crit at %sMB free)" % (state_markers[2], params[what][1])
 592                 elif value >= warn_level:
 593                     param_state = 1
 594                     level_info = "%s(warn at %sMB free)" % (state_markers[1], params[what][0])
 595             else:
 596                 if value_max:
 597                     warn_level = value_max * params[what][0] / 100.0
 598                     crit_level = value_max * params[what][1] / 100.0
 599
 600                 if what != "total":
 601                     perfdata.append((what, value, warn_level, crit_level, "", value_max))
 602
 603                 if not value_max:
 604                     param_state = 0
 605                 elif value >= crit_level:
 606                     param_state = 2
 607                     level_info = "%s(crit at %s%%)" % (state_markers[2], params[what][1])
 608                 elif value >= warn_level:
 609                     param_state = 1
 610                     level_info = "%s(warn at %s%%)" % (state_markers[1], params[what][0])
 611         else:
 612             if what != "total":
 613                 perfdata.append((what, value, "", "", "", value_max))
 614
 615         if value_max:
 616             used_info = "/%.1f%% used" % (value / value_max * 100)
 617         info_list.append("%s: %0.fMB%s%s" % (what.title(), value, used_info, level_info))
 618         worst_state = max(param_state, worst_state)
 619
 620     return (worst_state, ', '.join(info_list), perfdata)
 621
 622
 623 check_info["jolokia_metrics.mem"] = {
 624     "includes": ["jolokia.include"],
 625     "service_description": "JVM %s Memory",
 626     "check_function": check_jolokia_metrics_mem,
 627     "inventory_function": lambda i: inventory_jolokia_metrics(i, "mem"),
 628     "has_perfdata": True,
 629     "group": "jvm_memory",
 630     "default_levels_variable": "jolokia_metrics_mem_default_levels"
 631 }
 632
 633 #.
 634 #   .--Threads-------------------------------------------------------------.
 635 #   |                _____ _                        _                      |
 636 #   |               |_   _| |__  _ __ ___  __ _  __| |___                  |
 637 #   |                 | | | '_ \| '__/ _ \/ _` |/ _` / __|                 |
 638 #   |                 | | | | | | | |  __/ (_| | (_| \__ \                 |
 639 #   |                 |_| |_| |_|_|  \___|\__,_|\__,_|___/                 |
 640 #   |                                                                      |
 641 #   '----------------------------------------------------------------------'
 642
 643
 644 def check_jolokia_metrics_threads(item, params, info):
 645     warn, crit = params
 646     parsed = jolokia_metrics_parse(info)
 647     if parsed.get(item, "") is None:
 648         raise MKCounterWrapped("No information from Jolokia agent")
 649     if item not in parsed:
 650         return (3, "data not found in agent output")
 651     d = parsed[item]
 652
 653     this_time = time.time()
 654     perfdata = []
 655     output = []
 656     status = 0
 657     for key in ['ThreadCount', 'DeamonThreadCount', 'PeakThreadCount', 'TotalStartedThreadCount']:
 658         if key not in d:
 659             continue  # The keys might be optional (saw jboss only sending ThreadCount)
 660
 661         val = int(d[key])
 662         status_info = ""
 663         if key == 'ThreadCount':
 664             # Thread count might lead to a warn/crit state
 665             if val >= crit:
 666                 status = 2
 667                 status_info = "(!!) (Levels at %d/%d)" % (warn, crit)
 668             elif val >= warn:
 669                 status = 1
 670                 status_info = "(!) (Levels at %d/%d)" % (warn, crit)
 671
 672             # Calculate the thread increase rate
 673             rate = get_rate(
 674                 "jolokia_metrics.threads.%s" % item, this_time, val, allow_negative=True)
 675             output.append('ThreadRate: %0.2f' % rate)
 676             perfdata.append(('ThreadRate', rate))
 677
 678         perfdata.append((key, val))
 679         output.append('%s: %d%s' % (key, val, status_info))
 680     return (status, ', '.join(output), perfdata)
 681
 682
 683 check_info["jolokia_metrics.threads"] = {
 684     "includes": ["jolokia.include"],
 685     "service_description": "JVM %s Threads",
 686     "check_function": check_jolokia_metrics_threads,
 687     "inventory_function": lambda i: inventory_jolokia_metrics(i, "threads"),
 688     "group": "jvm_threads",
 689     "has_perfdata": True,
 690 }
 691
 692 #.
 693 #   .--Uptime--------------------------------------------------------------.
 694 #   |                  _   _       _   _                                   |
 695 #   |                 | | | |_ __ | |_(_)_ __ ___   ___                    |
 696 #   |                 | | | | '_ \| __| | '_ ` _ \ / _ \                   |
 697 #   |                 | |_| | |_) | |_| | | | | | |  __/                   |
 698 #   |                  \___/| .__/ \__|_|_| |_| |_|\___|                   |
 699 #   |                       |_|                                            |
 700 #   '----------------------------------------------------------------------'
 701
 702
 703 def check_jolokia_metrics_uptime(item, params, info):
 704     parsed = jolokia_metrics_parse(info)
 705     if parsed.get(item, "") is None:
 706         raise MKCounterWrapped("No information from Jolokia agent")
 707     if item in parsed:
 708         if "Uptime" in parsed[item]:
 709             uptime = int(parsed[item]['Uptime']) / 1000
 710             return check_uptime_seconds(params, uptime)
 711
 712
 713 check_info["jolokia_metrics.uptime"] = {
 714     "includes": ["jolokia.include", 'uptime.include'],
 715     "service_description": "JVM %s Uptime",
 716     "check_function": check_jolokia_metrics_uptime,
 717     "inventory_function": lambda i: inventory_jolokia_metrics(i, "uptime"),
 718     "group": "jvm_uptime",
 719     "has_perfdata": True,
 720 }
 721
 722 #.
 723 #   .--App state-----------------------------------------------------------.
 724 #   |                _                      _        _                     |
 725 #   |               / \   _ __  _ __    ___| |_ __ _| |_ ___               |
 726 #   |              / _ \ | '_ \| '_ \  / __| __/ _` | __/ _ \              |
 727 #   |             / ___ \| |_) | |_) | \__ \ || (_| | ||  __/              |
 728 #   |            /_/   \_\ .__/| .__/  |___/\__\__,_|\__\___|              |
 729 #   |                    |_|   |_|                                         |
 730 #   '----------------------------------------------------------------------'
 731
 732
 733 def check_jolokia_metrics_app_state(item, _unused, info):
 734     app_state = 3
 735     app = jolokia_metrics_app(info, item.split())
 736
 737     # FIXME: this could be nicer.
 738     if app and "Running" in app:
 739         if app['Running'] == '1':
 740             app_state = 0
 741         else:
 742             app_state = 2
 743     # wenn in app statename steht
 744     elif app and "stateName" in app:
 745         if app['stateName'] == 'STARTED':
 746             app_state = 0
 747         else:
 748             app_state = 2
 749     if app_state == 3:
 750         return (3, "data not found in agent output")
 751     elif app_state == 0:
 752         return (0, 'application is running')
 753     elif app_state == 2:
 754         return (2, 'application is not running (Running: %s)')
 755
 756     return (3, 'error in agent output')
 757
 758
 759 check_info["jolokia_metrics.app_state"] = {
 760     "includes": ["jolokia.include"],
 761     "service_description": "JVM %s State",
 762     "check_function": check_jolokia_metrics_app_state,
 763     "inventory_function": lambda i: inventory_jolokia_metrics_apps(i, "app_state"),
 764     "has_perfdata": False,
 765 }
 766
 767 #.
 768 #   .--Unsorted------------------------------------------------------------.
 769 #   |              _   _                      _           _                |
 770 #   |             | | | |_ __  ___  ___  _ __| |_ ___  __| |               |
 771 #   |             | | | | '_ \/ __|/ _ \| '__| __/ _ \/ _` |               |
 772 #   |             | |_| | | | \__ \ (_) | |  | ||  __/ (_| |               |
 773 #   |              \___/|_| |_|___/\___/|_|   \__\___|\__,_|               |
 774 #   |                                                                      |
 775 #   '----------------------------------------------------------------------'
 776
 777
 778 def check_jolokia_metrics_app_sess(item, params, info):
 779     lo_crit, lo_warn, hi_warn, hi_crit = params
 780     if len(item.split()) == 3:
 781         app = jolokia_metrics_serv(info, item.split())
 782     elif len(item.split()) == 2:
 783         app = jolokia_metrics_app(info, item.split())
 784     if not app:
 785         return (3, "application not found")
 786     sessions = app.get('Sessions', app.get('activeSessions', app.get('OpenSessionsCurrentCount')))
 787     if sessions is None:
 788         return (3, "data not found in agent output")
 789     sess = saveint(sessions)
 790     maxActive = saveint(
 791         app.get('Sessions', app.get('maxActiveSessions', app.get('OpenSessionsCurrentCount'))))
 792
 793     status = 0
 794     status_txt = ''
 795     if lo_crit is not None and sess <= lo_crit:
 796         status = 2
 797         status_txt = ' (Below or equal %d)' % lo_crit
 798     elif lo_warn is not None and sess <= lo_warn:
 799         status = 1
 800         status_txt = ' (Below or equal %d)' % lo_warn
 801     elif hi_crit is not None and sess >= hi_crit:
 802         status = 2
 803         status_txt = ' (Above or equal %d)' % hi_crit
 804     elif hi_warn is not None and sess >= hi_warn:
 805         status = 1
 806         status_txt = ' (Above or equal %d)' % hi_warn
 807
 808     if maxActive and maxActive > 0:
 809         status_txt += " (max active sessions: %d)" % maxActive
 810
 811     return (status, '%d Sessions%s' % (sess, status_txt), [('sessions', sess, hi_warn, hi_crit)])
 812
 813
 814 def check_jolokia_metrics_bea_queue(item, params, info):
 815     app = jolokia_metrics_app(info, item.split())
 816     if not app:
 817         return (3, "application not found")
 818     if "QueueLength" not in app:
 819         return (3, "data not found in agent output")
 820     queuelength = int(app['QueueLength'])
 821
 822     status = 0
 823     warn, crit = params
 824     if queuelength >= crit:
 825         status = 2
 826     elif queuelength >= warn:
 827         status = 1
 828     return (status, 'queue length is %d' % queuelength, [("length", queuelength, warn, crit)])
 829
 830
 831 # FIXME: This check could work with any JVM
 832 # It has no levels
 833 # A candidate for 1.2.1 overhaul
 834 def check_jolokia_metrics_bea_requests(item, _no_params, info):
 835     app = jolokia_metrics_app(info, item.split())
 836     if not app:
 837         return (3, "application not found")
 838
 839     for nk in ["CompletedRequestCount", "requestCount"]:
 840         if nk in app:
 841             requests = int(app[nk])
 842             rate = get_rate("j4p.bea.requests.%s" % item, time.time(), requests)
 843             return (0, "%.2f requests/sec" % rate, [("rate", rate)])
 844
 845     return (3, "data not found in agent output")
 846
 847
 848 def check_jolokia_metrics_bea_threads(item, _no_params, info):
 849     app = jolokia_metrics_app(info, item.split())
 850     if not app:
 851         return (3, "data not found in agent output")
 852
 853     perfdata = []
 854     infos = []
 855     for varname, title in [("ExecuteThreadTotalCount", "total"), ("ExecuteThreadIdleCount", "idle"),
 856                            ("StandbyThreadCount", "standby"), ("HoggingThreadCount", "hogging")]:
 857         value = int(app[varname])
 858         perfdata.append((varname, value))
 859         infos.append("%s: %d" % (title, value))
 860
 861     return (0, ", ".join(infos), perfdata)
 862
 863
 864 check_info["jolokia_metrics.app_sess"] = {
 865     "includes": ["jolokia.include"],
 866     "service_description": "JVM %s Sessions",
 867     "check_function": check_jolokia_metrics_app_sess,
 868     "inventory_function": lambda i: inventory_jolokia_metrics_apps(i, "app_sess"),
 869     "group": "jvm_sessions",
 870     "has_perfdata": True,
 871 }
 872
 873 check_info["jolokia_metrics.requests"] = {
 874     "includes": ["jolokia.include"],
 875     "service_description": "JVM %s Requests",
 876     "check_function": check_jolokia_metrics_bea_requests,
 877     "inventory_function": lambda i: inventory_jolokia_metrics_apps(i, "requests"),
 878     "group": "jvm_requests",
 879     "has_perfdata": True,
 880 }
 881
 882 # Stuff found on BEA Weblogic
 883 check_info["jolokia_metrics.bea_queue"] = {
 884     "includes": ["jolokia.include"],
 885     "service_description": "JVM %s Queue",
 886     "check_function": check_jolokia_metrics_bea_queue,
 887     "inventory_function": lambda i: inventory_jolokia_metrics_apps(i, "queue"),
 888     "group": "jvm_queue",
 889     "has_perfdata": True,
 890 }
 891
 892 check_info["jolokia_metrics.bea_requests"] = {
 893     "includes": ["jolokia.include"],
 894     "service_description": "JVM %s Requests",
 895     "check_function": check_jolokia_metrics_bea_requests,
 896     "inventory_function": lambda i: inventory_jolokia_metrics_apps(i, "bea_requests"),
 897     "group": "jvm_requests",
 898     "has_perfdata": True,
 899 }
 900
 901 check_info["jolokia_metrics.bea_threads"] = {
 902     "includes": ["jolokia.include"],
 903     "service_description": "JVM %s Threads",
 904     "check_function": check_jolokia_metrics_bea_threads,
 905     "inventory_function": lambda i: inventory_jolokia_metrics_apps(i, "threads"),
 906     "group": "jvm_threads",
 907     "has_perfdata": True,
 908 }
 909
 910 check_info["jolokia_metrics.bea_sess"] = {
 911     "includes": ["jolokia.include"],
 912     "service_description": "JVM %s Sessions",
 913     "check_function": check_jolokia_metrics_app_sess,
 914     "inventory_function": lambda i: inventory_jolokia_metrics_apps(i, "bea_app_sess"),
 915     "group": "jvm_sessions",
 916     "has_perfdata": True,
 917 }
 918
 919 jolokia_metrics_perm_gen_default_levels = {"perm": (80.0, 100.0)}
 920
 921
 922 def inventory_jolokia_metrics_perm_gen(info):
 923     parsed = jolokia_metrics_parse(info)
 924     for instance, values in parsed.items():
 925         if values and "PermGenUsage" in values:
 926             yield instance, "jolokia_metrics_perm_gen_default_levels"
 927
 928
 929 def check_jolokia_metrics_perm_gen(item, params, info):
 930     parsed = jolokia_metrics_parse(info)
 931     if parsed.get(item, "") is None:
 932         raise MKCounterWrapped("No information from Jolokia agent")
 933     values = parsed.get(item)
 934     if values:
 935         warn, crit = params['perm']
 936         usage_bytes = int(values['PermGenUsage'])
 937         size_bytes = int(values['PermGenMax'])
 938         usage_perc = 100.0 / size_bytes * usage_bytes
 939         warn_bytes = size_bytes / 100.0 * warn
 940         crit_bytes = size_bytes / 100.0 * crit
 941         perfdata = [("mem_perm_used", usage_bytes, warn_bytes, crit_bytes, 0, size_bytes)]
 942         state = 0
 943         if usage_perc >= crit:
 944             state = 2
 945         elif usage_perc >= warn:
 946             state = 1
 947         if state != 0:
 948             levels = "(warn/crit at %2.f/%.2f %%)" % (warn, crit)
 949         else:
 950             levels = ""
 951         return state, "Usage: %.2f %% (%s of %s used) %s" % \
 952           ( usage_perc, get_bytes_human_readable(usage_bytes), get_bytes_human_readable(size_bytes), levels ), \
 953           perfdata
 954
 955
 956 check_info["jolokia_metrics.perm_gen"] = {
 957     "includes": ["jolokia.include"],
 958     "service_description": "JVM %s PermGen Usage",
 959     "check_function": check_jolokia_metrics_perm_gen,
 960     "inventory_function": inventory_jolokia_metrics_perm_gen,
 961     "group": "jvm_memory",
 962     "has_perfdata": True,
 963 }
 964
 965 jolokia_metrics_cache_hits_default_levels = {}
 966
 967
 968 def inventory_jolokia_metrics_cache(key, metrics, info):
 969     parsed = jolokia_metrics_parse(info)
 970     metrics_set = set(metrics)
 971     for inst, vals in [x for x in parsed.iteritems() if x[1] is not None]:
 972         for cache, cache_vars in vals.get("CacheStatistics", {}).iteritems():
 973             if metrics_set.intersection(cache_vars) == metrics_set:
 974                 if key is None:
 975                     yield "%s %s" % (inst, cache), None
 976                 else:
 977                     yield "%s %s" % (inst, cache), "jolokia_metrics_%s_default_levels" % key
 978
 979
 980 def check_jolokia_metrics_cache(metrics, totals, item, params, info):
 981     type_map = {
 982         "CacheHitPercentage": (float, 100.0, "%.1f%%"),
 983         "InMemoryHitPercentage": (float, 100.0, "%.1f%%"),
 984         "OnDiskHitPercentage": (float, 100.0, "%.1f%%"),
 985         "OffHeapHitPercentage": (float, 100.0, "%.1f%%"),
 986     }
 987
 988     parsed = jolokia_metrics_parse(info)
 989     try:
 990         inst, cache = item.split(" ")
 991
 992         # we display the "metrics" first, totals after, but to "fix" metrics based on zero-totals
 993         # we need to go over the totals once
 994         for total in totals:
 995             val = int(parsed[inst]["CacheStatistics"][cache][total])
 996             if val != 0:
 997                 break
 998
 999         for metric in metrics:
1000             type_, scale, format_str = type_map.get(metric, (int, 1, "%d"))
1001
1002             val = type_(parsed[inst]["CacheStatistics"][cache][metric]) * scale
1003             if isinstance(val, float) and val == 0.0:
1004                 # what a hack! we assume the float is based on the totals (all of them) and if they
1005                 # were all 0, so this float is 0/0, we want to display it as 1 as to not cause
1006                 # an alert
1007                 val = 1.0 * scale
1008             yield 0, ("%s: " + format_str) % (metric, val), [(metric, val)]
1009
1010         for total in totals:
1011             type_, scale, format_str = type_map.get(total, (int, 1, "%d"))
1012             val = type_(parsed[inst]["CacheStatistics"][cache][total]) * scale
1013             yield 0, ("%s: " + format_str) % (total, val), []
1014     except KeyError:
1015         # some element of the item was missing
1016         pass
1017
1018
1019 check_info["jolokia_metrics.cache_hits"] = {
1020     "includes": ["jolokia.include"],
1021     "service_description" : "JVM %s Cache Usage",
1022     "check_function"      : lambda item, params, parsed: check_jolokia_metrics_cache(["CacheHitPercentage", "ObjectCount"], ["CacheHits", "CacheMisses"], item, params, parsed),
1023     "inventory_function"  : lambda info: inventory_jolokia_metrics_cache("cache_hits", ["CacheHitPercentage", "ObjectCount", "CacheHits", "CacheMisses"], info),
1024     "has_perfdata"        : True,
1025 }
1026
1027 check_info["jolokia_metrics.in_memory"] = {
1028     "includes": ["jolokia.include"],
1029     "service_description" : "JVM %s In Memory",
1030     "check_function"      : lambda item, params, parsed: check_jolokia_metrics_cache(["InMemoryHitPercentage", "MemoryStoreObjectCount"], ["InMemoryHits", "InMemoryMisses"], item, params, parsed),
1031     "inventory_function"  : lambda info: inventory_jolokia_metrics_cache("cache_hits", ["InMemoryHitPercentage", "MemoryStoreObjectCount", "InMemoryHits", "InMemoryMisses"], info),
1032     "has_perfdata"        : True,
1033 }
1034
1035 check_info["jolokia_metrics.on_disk"] = {
1036     "includes": ["jolokia.include"],
1037     "service_description" : "JVM %s On Disk",
1038     "check_function"      : lambda item, params, parsed: check_jolokia_metrics_cache(["OnDiskHitPercentage", "DiskStoreObjectCount"], ["OnDiskHits", "OnDiskMisses"], item, params, parsed),
1039     "inventory_function"  : lambda info: inventory_jolokia_metrics_cache("cache_hits", ["OnDiskHitPercentage", "DiskStoreObjectCount", "OnDiskHits", "OnDiskMisses"], info),
1040     "has_perfdata"        : True,
1041 }
1042
1043 check_info["jolokia_metrics.off_heap"] = {
1044     "includes": ["jolokia.include"],
1045     "service_description" : "JVM %s Off Heap",
1046     "check_function"      : lambda item, params, parsed: check_jolokia_metrics_cache(["OffHeapHitPercentage", "OffHeapStoreObjectCount"], ["OffHeapHits", "OffHeapMisses"], item, params, parsed),
1047     "inventory_function"  : lambda info: inventory_jolokia_metrics_cache("cache_hits", ["OffHeapHitPercentage", "OffHeapStoreObjectCount", "OffHeapHits", "OffHeapMisses"], info),
1048     "has_perfdata"        : True,
1049 }
1050
1051 check_info["jolokia_metrics.writer"] = {
1052     "includes": ["jolokia.include"],
1053     "service_description" : "JVM %s Cache Writer",
1054     "check_function"      : lambda item, params, parsed: check_jolokia_metrics_cache(["WriterQueueLength", "WriterMaxQueueSize"], [], item, params, parsed),
1055     "inventory_function"  : lambda info: inventory_jolokia_metrics_cache("cache_hits", ["WriterQueueLength", "WriterMaxQueueSize"], info),
1056     "has_perfdata"        : True,
1057 }