bin/mkbench

   1 #!/usr/bin/env python
   2 # -*- encoding: utf-8; py-indent-offset: 4 -*-
   3 # +------------------------------------------------------------------+
   4 # |             ____ _               _        __  __ _  __           |
   5 # |            / ___| |__   ___  ___| | __   |  \/  | |/ /           |
   6 # |           | |   | '_ \ / _ \/ __| |/ /   | |\/| | ' /            |
   7 # |           | |___| | | |  __/ (__|   <    | |  | | . \            |
   8 # |            \____|_| |_|\___|\___|_|\_\___|_|  |_|_|\_\           |
   9 # |                                                                  |
  10 # | Copyright Mathias Kettner 2014             mk@mathias-kettner.de |
  11 # +------------------------------------------------------------------+
  12 #
  13 # This file is part of Check_MK.
  14 # The official homepage is at http://mathias-kettner.de/check_mk.
  15 #
  16 # check_mk is free software;  you can redistribute it and/or modify it
  17 # under the  terms of the  GNU General Public License  as published by
  18 # the Free Software Foundation in version 2.  check_mk is  distributed
  19 # in the hope that it will be useful, but WITHOUT ANY WARRANTY;  with-
  20 # out even the implied warranty of  MERCHANTABILITY  or  FITNESS FOR A
  21 # PARTICULAR PURPOSE. See the  GNU General Public License for more de-
  22 # tails. You should have  received  a copy of the  GNU  General Public
  23 # License along with GNU Make; see the file  COPYING.  If  not,  write
  24 # to the Free Software Foundation, Inc., 51 Franklin St,  Fifth Floor,
  25 # Boston, MA 02110-1301 USA.
  26
  27 __version__ = "1.6.0i1"
  28
  29 # TODO:
  30 # - Create a more detailed report out of the test results. For example
  31 #   graph the different values together with the timeline of the different
  32 #   test cases.
  33
  34 # Problems with large installations:
  35 # - Helper startup phases
  36 # - Memory usage of helpers
  37 # - Huge history files during restarts / rotations
  38 # - Config compilation is serialized
  39 # - Service check spread with many new services
  40 # Config tweaks needed:
  41 # - Number of helpers
  42 # - Immediate history log file rotation (too large files) -> cmc_log_limit
  43
  44 #.
  45 #   .--Imports-------------------------------------------------------------.
  46 #   |               ___                            _                       |
  47 #   |              |_ _|_ __ ___  _ __   ___  _ __| |_ ___                 |
  48 #   |               | || '_ ` _ \| '_ \ / _ \| '__| __/ __|                |
  49 #   |               | || | | | | | |_) | (_) | |  | |_\__ \                |
  50 #   |              |___|_| |_| |_| .__/ \___/|_|   \__|___/                |
  51 #   |                            |_|                                       |
  52 #   '----------------------------------------------------------------------'
  53
  54 import os
  55 import re
  56 import ast
  57 import sys
  58 import glob
  59 import time
  60 import getopt
  61 import pprint
  62 import select
  63 import shutil
  64 import signal
  65 import socket
  66 import logging
  67 import tempfile
  68 import threading
  69 import traceback
  70 import subprocess
  71 import collections
  72 import multiprocessing
  73
  74 import psutil
  75 import npyscreen
  76
  77 import livestatus
  78
  79 from cmk.utils.render import fmt_bytes
  80
  81 #.
  82 #   .--Helpers-------------------------------------------------------------.
  83 #   |                  _   _      _                                        |
  84 #   |                 | | | | ___| |_ __   ___ _ __ ___                    |
  85 #   |                 | |_| |/ _ \ | '_ \ / _ \ '__/ __|                   |
  86 #   |                 |  _  |  __/ | |_) |  __/ |  \__ \                   |
  87 #   |                 |_| |_|\___|_| .__/ \___|_|  |___/                   |
  88 #   |                              |_|                                     |
  89 #   +----------------------------------------------------------------------+
  90 #   | Different helper methods and classes                                 |
  91 #   '----------------------------------------------------------------------'
  92
  93
  94 # Changes the process name and process command line on of the running process
  95 # This works at least with Python 2.x on Linux
  96 def set_cmdline(cmdline):
  97     import ctypes
  98     import ctypes.util
  99     libc = ctypes.cdll.LoadLibrary(ctypes.util.find_library('c'))
 100
 101     argv = ctypes.POINTER(ctypes.c_char_p)()
 102     argc = ctypes.c_int()
 103     ctypes.pythonapi.Py_GetArgcArgv(ctypes.byref(argc), ctypes.byref(argv))
 104     cmdlen = sum([len(argv[i]) for i in range(argc.value)]) + argc.value
 105     new_cmdline = ctypes.c_char_p(cmdline.ljust(cmdlen, '\0'))
 106
 107     # replace the command line, which is available via /proc/<pid>/cmdline.
 108     # This is .e.g used by ps
 109     libc.memcpy(argv.contents, new_cmdline, cmdlen)
 110
 111     # replace the prctl name, which is available via /proc/<pid>/status.
 112     # This is for example used by top and killall
 113     libc.prctl(15, new_cmdline, 0, 0, 0)
 114
 115
 116 def omd_root():
 117     return os.environ["OMD_ROOT"]
 118
 119
 120 def site_id():
 121     return os.environ.get("OMD_SITE")
 122
 123
 124 def fmt_timespan(seconds):
 125     hours, secs = divmod(seconds, 3600)
 126     mins, secs = divmod(secs, 60)
 127     return "%02d:%02d:%02d" % (hours, mins, secs)
 128
 129
 130 def fmt_datetime(seconds):
 131     return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(seconds))
 132
 133
 134 class MKGeneralException(Exception):
 135     pass
 136
 137
 138 class MKTerminate(MKGeneralException):
 139     pass
 140
 141
 142 class MKSkipTestCase(MKGeneralException):
 143     pass
 144
 145
 146 class MKFailTestCase(MKGeneralException):
 147     pass
 148
 149
 150 class MKFailTestCaseHighLatency(MKFailTestCase):
 151     pass
 152
 153
 154 class LocalConnection(livestatus.SingleSiteConnection):
 155     def __init__(self, *args, **kwargs):
 156         livestatus.SingleSiteConnection.__init__(self, "unix:%s/tmp/run/live" % omd_root(), *args,
 157                                                  **kwargs)
 158         self.connect()
 159
 160
 161 def interrupt_handler(signum, frame):
 162     raise MKTerminate("Caught signal: %d" % signum)
 163
 164
 165 def register_signal_handlers():
 166     signal.signal(signal.SIGTERM, interrupt_handler)
 167
 168
 169 def usage(error=None):
 170     if error:
 171         sys.stderr.write("ERROR: %s\n" % error)
 172     sys.stdout.write("Usage: mkbench [OPTIONS] [MODE] [MODE_OPTIONS...]\n")
 173     sys.stdout.write("\n")
 174     sys.stdout.write("The tool is meant to execute different tasks in a Check_MK site\n")
 175     sys.stdout.write("on a system and measure the performance of the task on that\n")
 176     sys.stdout.write("system. The test is saving different static information of\n")
 177     sys.stdout.write("the system (Installed CPUs, memory, disks, ...) and captures\n")
 178     sys.stdout.write("the most important peformance relevant values during testing\n")
 179     sys.stdout.write("for later analysis and comparison.\n")
 180     sys.stdout.write("\n")
 181     sys.stdout.write("The results of mkbench are stored in 'var/mkbench'.\n")
 182     sys.stdout.write("\n")
 183     sys.stdout.write("The tool also monitors the performance of different values of\n")
 184     sys.stdout.write("the system and aborts tests in case thresholds are reached.\n")
 185     sys.stdout.write("\n")
 186     sys.stdout.write("BE AWARE: mkbench modifies the configuration of your site and will\n")
 187     sys.stdout.write("          delete existing files. Only execute this in test sites.\n")
 188     sys.stdout.write("\n")
 189     sys.stdout.write("MODES:\n")
 190     sys.stdout.write("\n")
 191     sys.stdout.write("    NONE             Execute a test in the current site\n")
 192     sys.stdout.write("    --report         List test results available for reporting\n")
 193     sys.stdout.write("    --report ID1     Create reports out of previous results\n")
 194     sys.stdout.write("\n")
 195     sys.stdout.write("TEST OPTIONS:\n")
 196     sys.stdout.write("    --num-hosts H    Set the initial number of hosts (Defaults to 100)\n")
 197     sys.stdout.write("    --num-helpers N  Set the initial number of Check_MK helpers\n")
 198     sys.stdout.write("\n")
 199     sys.stdout.write("OPTIONS:\n")
 200     sys.stdout.write("\n")
 201     sys.stdout.write("    --debug          Let Python exceptions come through\n")
 202     sys.stdout.write("    --version        Print the version of the program\n")
 203     sys.stdout.write("\n")
 204
 205     sys.exit(3)
 206
 207
 208 def get_host_ip(host_id):
 209     start_ip_int = ip_int_from_string("127.0.0.1")
 210     return string_from_ip_int(start_ip_int + host_id)
 211
 212
 213 def ip_int_from_string(ip_str):
 214     packed_ip = 0
 215     octets = ip_str.split('.')
 216     for oc in octets:
 217         packed_ip = (packed_ip << 8) | int(oc)
 218     return packed_ip
 219
 220
 221 def string_from_ip_int(ip_int):
 222     octets = []
 223     for _ in range(4):
 224         octets.insert(0, str(ip_int & 0xFF))
 225         ip_int >>= 8
 226     return '.'.join(octets)
 227
 228
 229 #.
 230 #   .--Testing-------------------------------------------------------------.
 231 #   |                   _____         _   _                                |
 232 #   |                  |_   _|__  ___| |_(_)_ __   __ _                    |
 233 #   |                    | |/ _ \/ __| __| | '_ \ / _` |                   |
 234 #   |                    | |  __/\__ \ |_| | | | | (_| |                   |
 235 #   |                    |_|\___||___/\__|_|_| |_|\__, |                   |
 236 #   |                                             |___/                    |
 237 #   +----------------------------------------------------------------------+
 238 #   | The major part of this tool. Execution of tests and measurement      |
 239 #   '----------------------------------------------------------------------'
 240
 241
 242 def run_performance_test(num_hosts, num_helpers):
 243     verify_site_is_running()
 244     verify_site_is_empty()
 245
 246     # Create a new test, running in a dedicated thread
 247     runner = TestRunner(num_hosts, num_helpers)
 248
 249     # Start measuring in a dedicated thread
 250     analyzer = Analyzer(runner)
 251
 252     # Show state on console
 253     renderer = ConsoleRenderer(runner)
 254
 255     try:
 256         # Save current state before starting
 257         runner.save()
 258
 259         # And now start the work
 260         analyzer.start()
 261         runner.start()
 262
 263         try:
 264             renderer.run(fork=False)
 265         except npyscreen.wgwidget.NotEnoughSpaceForWidget, e:
 266             raise MKGeneralException("Too small terminal (%s)" % e)
 267
 268         finally:
 269             sys.stdout.write("Terminating...\n")
 270             runner.test.log_to_stdout()
 271             runner.stop()
 272             runner.join()
 273     finally:
 274         analyzer.stop()
 275
 276
 277 class Test(object):
 278     STATE_INITIALIZING = "initializing"
 279     STATE_RUNNING = "running"
 280     STATE_STOPPED = "stopped"
 281
 282     RESULT_COMPLETED = "completed"
 283     RESULT_ABORTED = "aborted"  # TODO: Use this when aborted
 284     RESULT_FAILED = "failed"
 285
 286     @classmethod
 287     def data_dir(cls):
 288         return os.path.join(omd_root(), "var/mkbench")
 289
 290     @classmethod
 291     def test_dir(cls, id_):
 292         return os.path.join(cls.data_dir(), id_)
 293
 294     @classmethod
 295     def exists(cls, id_):
 296         return os.path.exists(cls.test_dir(id_))
 297
 298     def __init__(self, id_=None):
 299         self._log = []
 300         self._log_to_stdout = False
 301         self.version = __version__
 302         self.state = Test.STATE_INITIALIZING
 303         self._measurements = []
 304         self.result = None
 305         self.result_msg = None
 306         self.system_info = SystemInfo()
 307
 308         if id_ is None:
 309             self.system_info.collect()
 310             self.id_ = self._get_test_id()
 311             self.test_cases = []
 312         else:
 313             self.id_ = id_
 314             self.load()
 315
 316     def load(self):
 317         raw_state = ast.literal_eval(open(os.path.join(self.dir(), "state")).read())
 318         self.deserialize(raw_state)
 319         self.load_measurements()
 320
 321     # Persists the current state of the test. The measurements are persisting
 322     # their results on their own.
 323     def save(self):
 324         test_dir = self.dir()
 325         try:
 326             os.makedirs(test_dir)
 327         except OSError, e:
 328             if e.errno == 17:  # file exists
 329                 pass
 330             else:
 331                 raise
 332
 333         path = os.path.join(test_dir, "state")
 334         with tempfile.NamedTemporaryFile(
 335                 "w", dir=os.path.dirname(path), prefix=".%s.new" % os.path.basename(path),
 336                 delete=False) as tmp:
 337             tmp_path = tmp.name
 338             tmp.write(pprint.pformat(self.serialize()) + "\n")
 339         os.rename(tmp_path, path)
 340
 341     def deserialize(self, raw_state):
 342         self._log = raw_state["log"]
 343         self.version = raw_state["version"]
 344         self.state = raw_state["state"]
 345         self.result = raw_state["result"]
 346         self.result_msg = raw_state["result_msg"]
 347         self.system_info.deserialize(raw_state["system_info"])
 348         self.deserialize_test_cases(raw_state["test_cases"])
 349
 350     def serialize(self):
 351         return {
 352             "log": self._log,
 353             "version": self.version,
 354             "state": self.state,
 355             "result": self.result,
 356             "result_msg": self.result_msg,
 357             "system_info": self.system_info.serialize(),
 358             "test_cases": self.serialize_test_cases(),
 359         }
 360
 361     def serialize_test_cases(self):
 362         return [test_case.serialize() for test_case in self.test_cases]
 363
 364     def deserialize_test_cases(self, raw_cases):
 365         self.test_cases = []
 366         for raw_case in raw_cases:
 367             cls = globals()[raw_case.pop("__class_name__")]
 368             test_case = cls()
 369             test_case.deserialize(raw_case)
 370             self.test_cases.append(test_case)
 371
 372     def load_measurements(self):
 373         self._measurements = []
 374         for entry in sorted(os.listdir(self.dir())):
 375             if entry[0] == "." or not entry.endswith(".data"):
 376                 continue
 377
 378             m = Measurement()
 379             m.load(os.path.join(self.dir(), entry))
 380             self.add_measurement(m)
 381
 382     def _get_test_id(self):
 383         parts = [
 384             self.system_info.omd_version,
 385             self.system_info.core,
 386             "%d" % time.time(),
 387         ]
 388         return "_".join(parts)
 389
 390     def set_state(self, state, result=None, result_msg=None):
 391         self.state = state
 392         self.result = result
 393         self.result_msg = result_msg
 394
 395     def state_text(self):
 396         return self.state
 397
 398     def first_measurement(self):
 399         return self._measurements[0]
 400
 401     def current_measurement(self):
 402         return self._measurements[-1]
 403
 404     def add_measurement(self, m):
 405         self._measurements.append(m)
 406
 407     def has_measurements(self):
 408         return bool(self._measurements)
 409
 410     def get_measurements_by_test_case(self):
 411         by_test_case = collections.OrderedDict()
 412         for m in self._measurements:
 413             if m.test_case:
 414                 l = by_test_case.setdefault(m.test_case, [])
 415                 l.append(m)
 416         return by_test_case
 417
 418     def is_stopped(self):
 419         return self.state == Test.STATE_STOPPED
 420
 421     def get_log(self):
 422         return self._log
 423
 424     def log(self, level, msg):
 425         if self._log_to_stdout:
 426             sys.stdout.write("%d %s\n" % (time.time(), msg))
 427         self._log.append((time.time(), level, msg))
 428
 429     def info(self, msg):
 430         self.log(logging.INFO, msg)
 431
 432     def error(self, msg):
 433         self.log(logging.ERROR, msg)
 434
 435     def log_to_stdout(self):
 436         self._log_to_stdout = True
 437
 438     def dir(self):
 439         return Test.test_dir(self.id_)
 440
 441     def duration(self):
 442         return self.current_measurement().time - self.first_measurement().time
 443
 444
 445 class TestRunner(threading.Thread):
 446     host_steps = [
 447         100,
 448         200,
 449         500,
 450         1000,
 451         2000,
 452         5000,
 453         8000,
 454         10000,
 455         20000,
 456     ]
 457
 458     def __init__(self, num_hosts=100, num_helpers=20):
 459         super(TestRunner, self).__init__()
 460         self.name = "test-runner"
 461         self._stop = threading.Event()
 462         self.test = Test()
 463         self.test_case = None
 464
 465         # Helper vars needed for test case generator
 466         self._num_hosts = num_hosts
 467         self._num_helpers = num_helpers
 468
 469     def run(self):
 470         try:
 471             result, result_msg = None, None
 472
 473             self.wait_for_load_cooldown(max_wait=900, expected_load=5)
 474             self._prepare_site()
 475             self._set_test_state(Test.STATE_RUNNING)
 476             self._run_test()
 477
 478             result = Test.RESULT_COMPLETED
 479
 480         except Exception, e:
 481             if opt_debug:
 482                 raise
 483             self.test.error("Test failed: %s" % traceback.format_exc())
 484             result = Test.RESULT_FAILED
 485             result_msg = "Test failed: %s" % e
 486
 487         finally:
 488             self._set_test_state(Test.STATE_STOPPED, result, result_msg)
 489             self._cleanup_host_files()
 490
 491     def wait_for_load_cooldown(self, max_wait, expected_load):
 492         # After all helpers are ready, check for reaching previous load level
 493         reached = False
 494         while max_wait > 0:
 495             if self.shall_stop():
 496                 break
 497
 498             loadavg_5 = os.getloadavg()[1]
 499             if loadavg_5 > expected_load:
 500                 self.test.info("Waiting %d sec for cooldown (5min load: %0.2f > %0.2f)" %
 501                                (max_wait, loadavg_5, expected_load))
 502             else:
 503                 reached = True
 504                 break
 505             time.sleep(1)
 506             max_wait -= 1
 507
 508         if not reached:
 509             raise MKGeneralException("System has too high load")
 510
 511     def _set_test_state(self, state, result=None, result_msg=None):
 512         self.test.set_state(state, result, result_msg)
 513         self.save()
 514
 515     def _set_test_case_state(self, state, result=None, result_msg=None):
 516         self.test_case.set_state(state, result, result_msg)
 517         self.save()
 518
 519     def _prepare_site(self):
 520         self.test.info("Preparing site for testing")
 521         self._prepare_rrdcached()
 522
 523     def _prepare_rrdcached(self):
 524         with open("%s/etc/rrdcached.conf" % omd_root(), "w") as f:
 525             f.write("TIMEOUT=300\n" "RANDOM_DELAY=10\n" "FLUSH_TIMEOUT=7200\n" "WRITE_THREADS=4\n")
 526
 527         p = subprocess.Popen(["omd", "restart", "rrdcached"],
 528                              close_fds=True,
 529                              stdout=subprocess.PIPE,
 530                              stderr=subprocess.STDOUT)
 531         stdout = p.communicate()[0]
 532         if p.returncode != 0:
 533             raise MKGeneralException("Failed to restart rrdcached: %s" % stdout)
 534
 535     def _run_test(self):
 536         for test_case in self._generate_test_cases():
 537             if self._stop.isSet():
 538                 break
 539
 540             self.test.test_cases.append(test_case)
 541
 542             result, result_msg = None, None
 543             try:
 544                 self.test.info("Initializing %s" % test_case.name())
 545
 546                 for step in test_case.run():
 547                     if isinstance(step, Start):
 548                         # The measurements are only assigned to the test case when
 549                         # it tells us to do so by providing a Start() object
 550                         self.test.info("Start %s" % test_case.name())
 551                         self.test_case = test_case
 552                         self._set_test_case_state(TestCase.STATE_RUNNING)
 553
 554                     elif isinstance(step, Wait):
 555                         # Do many small sleeps while the test is running so that
 556                         # the test thread can persist the current state of the test
 557                         result = TestCase.RESULT_COMPLETED
 558                         while time.time() <= step.until:
 559                             try:
 560                                 self._while_waiting()
 561                             except MKSkipTestCase, e:
 562                                 self.test.info(
 563                                     "Skipping %s (%s): %s" %
 564                                     (test_case.name(), fmt_timespan(test_case.duration()), e))
 565                                 result = TestCase.RESULT_SKIPPED
 566                                 result_msg = "Skipped: %s" % e
 567                                 break  # Will iterate to next step (till test case end)
 568
 569                             if self._stop.isSet():
 570                                 result = TestCase.RESULT_ABORTED
 571                                 result_msg = "Test runner was stopped"
 572                                 return
 573
 574                             time.sleep(1)
 575
 576                 self.test.info(
 577                     "Finished %s (%s)" % (test_case.name(), fmt_timespan(test_case.duration())))
 578                 self._increase_hosts()
 579
 580             except MKFailTestCase, e:
 581                 result = TestCase.RESULT_FAILED
 582                 result_msg = "Test case aborted: %s" % e
 583                 self.test.error(result_msg)
 584
 585                 if isinstance(e, MKFailTestCaseHighLatency):
 586                     self._increase_helpers()
 587                 else:
 588                     break
 589
 590             except Exception, e:
 591                 result = TestCase.RESULT_FAILED
 592                 result_msg = "Test case failed: %s" % e
 593                 self.test.error(result_msg)
 594                 raise
 595
 596             finally:
 597                 if self.test_case:
 598                     self._set_test_case_state(TestCase.STATE_STOPPED, result, result_msg)
 599                     self.test_case = None
 600
 601         self.test.info("Finished testing")
 602
 603     def _increase_hosts(self):
 604         try:
 605             self._num_hosts = TestRunner.host_steps[TestRunner.host_steps.index(self._num_hosts) +
 606                                                     1]
 607         except (ValueError, IndexError):
 608             # Reached end!
 609             self.stop()
 610
 611     def _increase_helpers(self):
 612         self._num_helpers += 40
 613
 614     def _generate_test_cases(self):
 615         while not self._stop.isSet():
 616             yield TestAgentBasedStandardHosts(
 617                 self,
 618                 num_hosts=self._num_hosts,
 619                 planned_duration=15 * 60,
 620                 num_cmk_helpers=self._num_helpers)
 621         #return [
 622         #    TestAgentBasedStandardHosts(self, num_hosts=100,   planned_duration=15*60, num_cmk_helpers=20),
 623         #    TestAgentBasedStandardHosts(self, num_hosts=200,   planned_duration=15*60, num_cmk_helpers=20),
 624         #    TestAgentBasedStandardHosts(self, num_hosts=500,   planned_duration=15*60, num_cmk_helpers=20),
 625         #    TestAgentBasedStandardHosts(self, num_hosts=1000,  planned_duration=15*60, num_cmk_helpers=20),
 626         #    TestAgentBasedStandardHosts(self, num_hosts=2000,  planned_duration=15*60, num_cmk_helpers=20),
 627         #    TestAgentBasedStandardHosts(self, num_hosts=5000,  planned_duration=15*60, num_cmk_helpers=20),
 628         #    TestAgentBasedStandardHosts(self, num_hosts=8000,  planned_duration=15*60, num_cmk_helpers=20),
 629         #    TestAgentBasedStandardHosts(self, num_hosts=10000, planned_duration=15*60, num_cmk_helpers=20),
 630
 631         #    TestAgentBasedStandardHosts(self, num_hosts=2000,  planned_duration=15*60, num_cmk_helpers=40),
 632         #    TestAgentBasedStandardHosts(self, num_hosts=5000,  planned_duration=15*60, num_cmk_helpers=40),
 633         #    TestAgentBasedStandardHosts(self, num_hosts=8000,  planned_duration=15*60, num_cmk_helpers=40),
 634         #    TestAgentBasedStandardHosts(self, num_hosts=10000, planned_duration=15*60, num_cmk_helpers=40),
 635
 636         #    TestAgentBasedStandardHosts(self, num_hosts=5000,  planned_duration=15*60, num_cmk_helpers=60),
 637         #    TestAgentBasedStandardHosts(self, num_hosts=8000,  planned_duration=15*60, num_cmk_helpers=60),
 638         #    TestAgentBasedStandardHosts(self, num_hosts=10000, planned_duration=15*60, num_cmk_helpers=60),
 639
 640         #    TestAgentBasedStandardHosts(self, num_hosts=8000,  planned_duration=15*60, num_cmk_helpers=100),
 641         #    TestAgentBasedStandardHosts(self, num_hosts=10000, planned_duration=15*60, num_cmk_helpers=100),
 642         #    TestAgentBasedStandardHosts(self, num_hosts=20000, planned_duration=15*60, num_cmk_helpers=100),
 643         #]
 644
 645     def _while_waiting(self):
 646         self._check_thresholds()
 647         self.save()
 648
 649     def _check_thresholds(self):
 650         try:
 651             measurement = self.test.current_measurement()
 652         except IndexError:
 653             return
 654
 655         self._check_upper_thresholds(measurement)
 656         self._check_lower_thresholds(measurement)
 657
 658     def _check_upper_thresholds(self, m):
 659         # Abort when less than 100 MB are free
 660         if m.disk_usage.free < 100 * 1024 * 1024:
 661             raise MKFailTestCase("The disk %s has only %s left." %
 662                                  (self.test.system_info.block_device, fmt_bytes(m.disk_usage.free)))
 663
 664         if m.cpu_load[1] > self.test.system_info.cpu_count * 1.2:
 665             raise MKFailTestCase(
 666                 "The 5 min average CPU load %0.2f is higher than "
 667                 "number of CPUs (%d)." % (m.cpu_load[1], self.test.system_info.cpu_count))
 668
 669         # Check memory threshold. There are different values provided by psutil.
 670         # this value sounds good to me for testing.
 671         if m.memory.available < 100 * 1024 * 1024:
 672             raise MKFailTestCase(
 673                 "There is only %s memory available." % (fmt_bytes(m.memory.available)))
 674
 675         # Initial scheduling of the CMC spreads the service checks for a maximum of
 676         # 5 * check_interval. So we should terminate the test only after some initialization
 677         # time.
 678         if self.test_case.duration() < 450:
 679             return
 680
 681         if m.site_stats and m.site_stats["average_latency_cmk"] > 30:
 682             raise MKFailTestCaseHighLatency("The average check latency is too high (%0.2f sec)" %
 683                                             m.site_stats["average_latency_cmk"])
 684
 685     def _check_lower_thresholds(self, m):
 686         if self.test_case.duration() < 30:
 687             return  # Check for lower thresholds after X seconds
 688
 689         if m.disk_usage.percent > 70:
 690             return
 691
 692         if m.cpu_load[0] > self.test.system_info.cpu_count * 0.2:
 693             return
 694
 695         if m.memory.percent > 10:
 696             return
 697
 698         if m.site_stats and m.site_stats["helper_usage_cmk"] * 100 > 10:
 699             return
 700
 701         if m.site_stats and m.site_stats["average_latency_cmk"] > 2:
 702             return
 703
 704         raise MKSkipTestCase("System is bored.")
 705
 706     def save(self):
 707         set_symlink = not os.path.exists(self.test.dir())
 708
 709         self.test.save()
 710
 711         if set_symlink:
 712             self._update_last_symlink()
 713
 714     def _update_last_symlink(self):
 715         path = os.path.join(Test.data_dir(), "last")
 716         try:
 717             os.unlink(path)
 718         except OSError:
 719             pass
 720         os.symlink(self.test.id_, path)
 721
 722     def stop(self):
 723         self._stop.set()
 724
 725     def shall_stop(self):
 726         return self._stop.isSet()
 727
 728     def _cleanup_host_files(self, cleanup_rrds=False):
 729         for path in glob.glob("%s/var/check_mk/autochecks/*" % omd_root()):
 730             os.unlink(path)
 731
 732         for path in glob.glob("%s/tmp/check_mk/counters/*" % omd_root()):
 733             os.unlink(path)
 734
 735         if cleanup_rrds:
 736             for path in glob.glob("%s/var/check_mk/rrd/*" % omd_root()):
 737                 shutil.rmtree(path)
 738
 739             for path in glob.glob("%s/var/pnp4nagios/perfdata/*" % omd_root()):
 740                 shutil.rmtree(path)
 741
 742         for path in glob.glob("%s/var/check_mk/core/archive/history-*" % omd_root()):
 743             os.unlink(path)
 744
 745         for path in glob.glob("%s/var/check_mk/core/state*" % omd_root()):
 746             os.unlink(path)
 747
 748
 749 class Analyzer(threading.Thread):
 750     def __init__(self, runner):
 751         self._runner = runner
 752         super(Analyzer, self).__init__()
 753
 754         self.name = "analyzer"
 755         self.daemon = True
 756         self._stop = threading.Event()
 757
 758     def run(self):
 759         #self._runner.test.info("Analyzer started")
 760         try:
 761             while not self._stop.isSet() and not self._runner.test.is_stopped():
 762                 self._measure()
 763                 time.sleep(1)
 764         except Exception:
 765             if opt_debug:
 766                 raise
 767             self._runner.test.error("Analyzer failed: %s" % traceback.format_exc())
 768         self._runner.test.info("Analyzer stopped")
 769
 770     def _measure(self):
 771         m = Measurement(self._runner)
 772         if self._runner.test.has_measurements():
 773             last = self._runner.test.current_measurement()
 774         else:
 775             last = None
 776
 777         m.measure(last=last)
 778         self._runner.test.add_measurement(m)
 779
 780         m.save()
 781
 782     def is_stopped(self):
 783         return not self.is_alive()
 784
 785     def stop(self):
 786         self._stop.set()
 787
 788
 789 class Start(object):
 790     pass
 791
 792
 793 class Wait(object):
 794     def __init__(self, duration):
 795         self.until = time.time() + duration
 796
 797
 798 class SystemInfo(object):
 799     def __init__(self):
 800         super(SystemInfo, self).__init__()
 801         self.cpu_count = None
 802         self.cpu_info = None
 803         self.memory = None
 804         self.block_device = None
 805         self.linux_distro = None
 806         self.omd_version = None
 807         self.omd_site = None
 808         self.core = None
 809
 810     def collect(self):
 811         # Number of logical CPUs
 812         self.cpu_count = psutil.cpu_count()
 813         self.cpu_info = self._get_cpu_info()
 814         self.memory = psutil.virtual_memory().total
 815         self.block_device = self._find_block_device("/omd/sites")
 816         self.linux_distro = self._get_linux_distro()
 817         self.omd_version = self._get_omd_version()
 818         self.omd_site = site_id()
 819         self.core = self._get_monitoring_core()
 820
 821     def serialize(self):
 822         return dict([(k, v) for k, v in self.__dict__.items() if k[0] != "_"])
 823
 824     def deserialize(self, raw):
 825         self.__dict__.update(raw)
 826
 827     def _find_block_device(self, mount_path):
 828         mount_path = os.path.realpath(mount_path)
 829         while not os.path.ismount(mount_path):
 830             mount_path = os.path.dirname(mount_path)
 831
 832         dev = os.stat(mount_path).st_dev
 833         major = os.major(dev)
 834         minor = os.minor(dev)
 835
 836         return os.path.realpath("/sys/dev/block/%d:%d" % (major, minor))
 837
 838     def _get_linux_distro(self):
 839         return open("%s/share/omd/distro.info" % omd_root()).readline().split("=", 1)[1].strip()
 840
 841     def _get_omd_version(self):
 842         return os.path.basename(os.path.realpath("%s/version" % omd_root()))
 843
 844     def _get_monitoring_core(self):
 845         for l in open("%s/etc/omd/site.conf" % omd_root()):
 846             if l.startswith("CONFIG_CORE="):
 847                 return l.strip().split("=", 1)[1].strip("'")
 848         return None
 849
 850     def _get_cpu_info(self):
 851         node = {}
 852         num_threads_total = 0
 853         sockets = set([])
 854         for l in open("/proc/cpuinfo"):
 855             parts = l.split(": ", 1)
 856             if len(parts) != 2:
 857                 continue
 858             varname, value = parts[0].strip(), parts[1].strip()
 859
 860             if varname == "cpu cores":
 861                 node["cores_per_cpu"] = int(value)
 862             elif varname == "siblings":
 863                 node["threads_per_cpu"] = int(value)
 864             elif varname == "vendor_id":
 865                 node["vendor"] = {
 866                     "GenuineIntel": "intel",
 867                     "AuthenticAMD": "amd",
 868                 }.get(value, value)
 869             elif varname == "cache size":
 870                 node["cache_size"] = int(
 871                     value.split()[0]) * 1024  # everything is normalized to bytes!
 872             elif varname == "model name":
 873                 node["model"] = value
 874             # For the following two entries we assume that all
 875             # entries are numbered in increasing order in /proc/cpuinfo.
 876             elif varname == "processor":
 877                 num_threads_total = int(value) + 1
 878             elif varname == "physical id":
 879                 sockets.add(int(value))
 880             elif varname == "flags":
 881                 if re.search(" lm ", value):
 882                     node["arch"] = "x86_64"
 883                 else:
 884                     node["arch"] = "i386"
 885
 886         num_sockets = len(sockets)
 887
 888         if num_threads_total:
 889             node.setdefault("cores_per_cpu", 1)
 890             node.setdefault("threads_per_cpu", 1)
 891             node["cores"] = num_sockets * node["cores_per_cpu"]
 892             node["threads"] = num_sockets * node["threads_per_cpu"]
 893             node["cpus"] = num_sockets
 894
 895         return node
 896
 897
 898 DiskIO = collections.namedtuple("DiskIO", ["read_bytes", "write_bytes"])
 899
 900
 901 class Measurement(object):
 902     @classmethod
 903     def summary(cls, measurements):
 904         average_keys = [
 905             "cpu_load",
 906             "disk_io",
 907             "disk_usage",
 908             "memory",
 909             "site_stats",
 910         ]
 911
 912         average = Measurement()
 913
 914         for k in average_keys:
 915             values = []
 916             for m in measurements:
 917                 values.append(getattr(m, k))
 918
 919             if values:
 920                 if isinstance(values[0], tuple):
 921                     # averages of equal length tuples
 922                     averaged = tuple([sum(y) / len(y) for y in zip(*values)])
 923                 else:
 924                     # averages of dict items
 925                     averaged = {}
 926
 927                     real_values = [v for v in values if v is not None]
 928                     num_values = len(real_values)
 929
 930                     if real_values:
 931                         for key, value in real_values[0].items():
 932                             if isinstance(value, unicode):
 933                                 averaged[key] = value
 934                             else:
 935                                 averaged[key] = float(sum([v[key] for v in real_values
 936                                                           ])) / num_values
 937             else:
 938                 averaged = None
 939
 940             average.set_value(k, averaged)
 941
 942         return average
 943
 944     def __init__(self, runner=None):
 945         super(Measurement, self).__init__()
 946         self._runner = runner
 947
 948         self.test_case = None
 949         self.disk_counters = None
 950         self.site_state = None
 951
 952         self.time = None
 953         self.cpu_load = None
 954         self.memory = None
 955         self.disk_usage = None
 956         self.disk_io = None
 957         self.site_stats = None
 958
 959     def measure(self, last=None):
 960         self.test_case  = self._runner.test_case.name() \
 961                             if self._runner.test_case else None
 962         self.time = time.time()
 963         self.cpu_load = os.getloadavg()
 964         self.memory = psutil.virtual_memory()
 965         self.disk_usage = psutil.disk_usage(omd_root())
 966
 967         self._get_disk_io(last)
 968         self._get_site_state()
 969         self._get_site_stats()
 970
 971     def load(self, path):
 972         try:
 973             raw_data = ast.literal_eval(open(path).read())
 974             self.deserialize(raw_data)
 975         except Exception, e:
 976             if opt_debug:
 977                 raise
 978             raise MKGeneralException("Failed to load '%s': %s" % (path, e))
 979
 980     def save(self):
 981         test_dir = self._runner.test.dir()
 982
 983         path = os.path.join(test_dir, "%d.data" % self.time)
 984         with tempfile.NamedTemporaryFile(
 985                 "w", dir=os.path.dirname(path), prefix=".%s.new" % os.path.basename(path),
 986                 delete=False) as tmp:
 987             tmp_path = tmp.name
 988             tmp.write(pprint.pformat(self.serialize()) + "\n")
 989         os.rename(tmp_path, path)
 990
 991     def serialize(self):
 992         raw = {}
 993
 994         for k, v in self.__dict__.items():
 995             if k[0] == "_":
 996                 continue
 997
 998             # change named tuples to real tuples (for ast.literal_eval)
 999             if isinstance(v, tuple) and hasattr(v, "_fields"):
1000                 raw[k] = tuple(v)
1001             else:
1002                 raw[k] = v
1003
1004         return raw
1005
1006     def deserialize(self, raw):
1007         for k in self.__dict__:
1008             if k[0] != "_":
1009                 self.set_value(k, raw[k])
1010
1011     def set_value(self, k, val):
1012         """Set an attribute of the measurement. Cares about converting tuples
1013         to the correct namedtuples."""
1014         from psutil._common import sdiskusage
1015         from psutil._pslinux import svmem, sdiskio
1016
1017         if k == "memory":
1018             val = svmem(*val)
1019         elif k == "disk_usage":
1020             val = sdiskusage(*val)
1021         elif k == "disk_counters":
1022             val = sdiskio(*val)
1023         elif k == "disk_io":
1024             val = DiskIO(*val)
1025
1026         setattr(self, k, val)
1027
1028     def _get_disk_io(self, last):
1029         self.disk_counters = self._get_disk_counters()
1030
1031         if not last:
1032             self.disk_io = DiskIO(0.0, 0.0)
1033             return
1034
1035         read_bytes_per_sec = (self.disk_counters.read_bytes - last.disk_counters.read_bytes) \
1036                                 / (self.time - last.time)
1037
1038         write_bytes_per_sec = (self.disk_counters.write_bytes - last.disk_counters.write_bytes) \
1039                                 / (self.time - last.time)
1040
1041         self.disk_io = DiskIO(read_bytes_per_sec, write_bytes_per_sec)
1042
1043     def _get_disk_counters(self):
1044         # Use total disk IO of the system because e.g. software raid IO is not
1045         # accounted to the md0p2 partition in case of CMK is on that volume.
1046         #device_name = os.path.basename(self._runner.test.system_info.block_device)
1047         #disk_io     = psutil.disk_io_counters(perdisk=True)
1048         #return disk_io[device_name]
1049
1050         #part_name   = os.path.basename(self._runner.test.system_info.block_device)
1051         #device_name = part_name.rsplit("p", 1)[0]
1052         #disk_io     = psutil.disk_io_counters(perdisk=True)
1053         #return disk_io[device_name]
1054         return psutil.disk_io_counters()
1055
1056     def _get_site_state(self):
1057         self.site_state = subprocess.call(["omd", "status", "--bare"],
1058                                           stdout=file(os.devnull, "w"),
1059                                           close_fds=True)
1060
1061     def _get_site_stats(self):
1062         if self.site_state not in [ 0, 2 ] \
1063            or not os.path.exists("%s/tmp/run/live" % omd_root()):
1064             self.site_stats = None
1065             return
1066
1067         try:
1068             live = LocalConnection()
1069             live.set_timeout(5)
1070             self.site_stats = live.query_row_assoc("GET status")
1071             live.disconnect()
1072         except livestatus.MKLivestatusException, e:
1073             self._runner.test.error("Livestatus: %s" % e)
1074             self.site_stats = None
1075
1076
1077 #.
1078 #   .--AgentListener-------------------------------------------------------.
1079 #   |      _                    _   _     _     _                          |
1080 #   |     / \   __ _  ___ _ __ | |_| |   (_)___| |_ ___ _ __   ___ _ __    |
1081 #   |    / _ \ / _` |/ _ \ '_ \| __| |   | / __| __/ _ \ '_ \ / _ \ '__|   |
1082 #   |   / ___ \ (_| |  __/ | | | |_| |___| \__ \ ||  __/ | | |  __/ |      |
1083 #   |  /_/   \_\__, |\___|_| |_|\__|_____|_|___/\__\___|_| |_|\___|_|      |
1084 #   |          |___/                                                       |
1085 #   +----------------------------------------------------------------------+
1086 #   | Is executed in a dedicated process and serves the dynamic agent data |
1087 #   '----------------------------------------------------------------------'
1088
1089
1090 class AgentListener(multiprocessing.Process):
1091     def __init__(self, host_ids, *args, **kwargs):
1092         self._hosts = host_ids
1093         self._sockets = []
1094         self._socket_map = {}
1095         super(AgentListener, self).__init__(*args, **kwargs)
1096         self.name = "mkbench_agent-listener"
1097
1098         # Holds the counters per host that are used to track the dynamic agent data
1099         self._counters = {}
1100
1101     def run(self):
1102         try:
1103             self._set_process_name()
1104             self._open_sockets()
1105             self._serve()
1106         except (KeyboardInterrupt, MKTerminate):
1107             pass
1108         finally:
1109             self._close_sockets()
1110
1111     def _set_process_name(self):
1112         set_cmdline(self.name)
1113
1114     def _open_sockets(self):
1115         for host_id in self._hosts:
1116             address = (get_host_ip(host_id), 6559)
1117             try:
1118                 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1119                 s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
1120                 s.bind(address)
1121                 s.listen(1)
1122                 self._sockets.append(s)
1123                 self._socket_map[s.getsockname()] = host_id
1124             except Exception, e:
1125                 if opt_debug:
1126                     raise
1127                 sys.stderr.write("Failed to open %s:%d: %s\n" % (address[0], address[1], e))
1128                 raise
1129
1130     def _serve(self):
1131         while True:
1132             try:
1133                 try:
1134                     readable = select.select(self._sockets, [], [], 0.2)[0]
1135                 except select.error, e:
1136                     if e[0] == 4:
1137                         continue
1138                     else:
1139                         raise
1140
1141                 for s in readable:
1142                     time.sleep(0.5)
1143                     client_socket, _addr_info = s.accept()
1144                     host_id = self._socket_map[s.getsockname()]
1145                     client_socket.sendall(self._get_answer(host_id))
1146                     client_socket.close()
1147             except (KeyboardInterrupt, MKTerminate):
1148                 break
1149             except Exception, e:
1150                 sys.stderr.write("Error in listener: %s - %s\n" % (type(e), e))
1151
1152     def _close_sockets(self):
1153         for s in self._sockets:
1154             s.close()
1155         del self._sockets[:]
1156
1157     def _get_answer(self, host_id):
1158         agent_data = static_agent_data
1159         agent_data += self._section_kernel(host_id)
1160         return agent_data
1161
1162     def _update_counter(self, host_id, ident, per_sec):
1163         counters = self._counters.setdefault(host_id, {})
1164
1165         now = time.time()
1166         if ident not in counters:
1167             counters[ident] = now, 0
1168             return  # just initialied. Don't increase
1169
1170         else:
1171             last_update, last_value = counters[ident]
1172             duration = now - last_update
1173
1174             counters[ident] = now, last_value + (duration * per_sec)
1175
1176     def _section_kernel(self, host_id):
1177         self._update_counter(host_id, "kernel_ctxt", per_sec=5000)
1178         self._update_counter(host_id, "kernel_processes", per_sec=1)
1179         self._update_counter(host_id, "kernel_pgmajfault", per_sec=50)
1180
1181         # hundrets of a second
1182         num_cpus = 4
1183         for i in [""] + range(num_cpus):
1184             self._update_counter(host_id, "kernel_cpu%s_user" % i, per_sec=0.10)
1185             self._update_counter(host_id, "kernel_cpu%s_nice" % i, per_sec=0.05)
1186             self._update_counter(host_id, "kernel_cpu%s_system" % i, per_sec=0.10)
1187             self._update_counter(host_id, "kernel_cpu%s_idle" % i, per_sec=0.30)
1188             self._update_counter(host_id, "kernel_cpu%s_iowait" % i, per_sec=0.05)
1189
1190         counters = {k: v for k, (_t, v) in self._counters[host_id].iteritems()}
1191         counters["now"] = time.time()
1192
1193         cpu_output = ""
1194         for i in [""] + range(num_cpus):
1195             cpu_output += "cpu%s %d %d %d %d %d 0 0 0 0 0\n" % \
1196                 (i,
1197                  counters["kernel_cpu%s_user" % i],
1198                  counters["kernel_cpu%s_nice" % i],
1199                  counters["kernel_cpu%s_system" % i],
1200                  counters["kernel_cpu%s_idle" % i],
1201                  counters["kernel_cpu%s_iowait" % i])
1202
1203         section = """<<<kernel>>>
1204 %(now)d
1205 nr_free_pages 2544893
1206 nr_alloc_batch 3169
1207 nr_inactive_anon 52641
1208 nr_active_anon 980946
1209 nr_inactive_file 87489
1210 nr_active_file 314428
1211 nr_unevictable 0
1212 nr_mlock 0
1213 nr_anon_pages 979995
1214 nr_mapped 82615
1215 nr_file_pages 455554
1216 nr_dirty 10237
1217 nr_writeback 0
1218 nr_slab_reclaimable 34034
1219 nr_slab_unreclaimable 15947
1220 nr_page_table_pages 17569
1221 nr_kernel_stack 927
1222 nr_unstable 0
1223 nr_bounce 0
1224 nr_vmscan_write 0
1225 nr_vmscan_immediate_reclaim 0
1226 nr_writeback_temp 0
1227 nr_isolated_anon 0
1228 nr_isolated_file 0
1229 nr_shmem 53638
1230 nr_dirtied 3163803
1231 nr_written 1642042
1232 nr_pages_scanned 0
1233 numa_hit 43145578
1234 numa_miss 0
1235 numa_foreign 0
1236 numa_interleave 32848
1237 numa_local 43145578
1238 numa_other 0
1239 workingset_refault 0
1240 workingset_activate 0
1241 workingset_nodereclaim 0
1242 nr_anon_transparent_hugepages 495
1243 nr_free_cma 0
1244 nr_dirty_threshold 580548
1245 nr_dirty_background_threshold 289919
1246 pgpgin 886510
1247 pgpgout 6647136
1248 pswpin 0
1249 pswpout 0
1250 pgalloc_dma 0
1251 pgalloc_dma32 5351681
1252 pgalloc_normal 40330396
1253 pgalloc_movable 0
1254 pgfree 48229229
1255 pgactivate 194701
1256 pgdeactivate 0
1257 pgfault 45121049
1258 pgmajfault %(kernel_pgmajfault)d
1259 pgrefill_dma 0
1260 pgrefill_dma32 0
1261 pgrefill_normal 0
1262 pgrefill_movable 0
1263 pgsteal_kswapd_dma 0
1264 pgsteal_kswapd_dma32 0
1265 pgsteal_kswapd_normal 0
1266 pgsteal_kswapd_movable 0
1267 pgsteal_direct_dma 0
1268 pgsteal_direct_dma32 0
1269 pgsteal_direct_normal 0
1270 pgsteal_direct_movable 0
1271 pgscan_kswapd_dma 0
1272 pgscan_kswapd_dma32 0
1273 pgscan_kswapd_normal 0
1274 pgscan_kswapd_movable 0
1275 pgscan_direct_dma 0
1276 pgscan_direct_dma32 0
1277 pgscan_direct_normal 0
1278 pgscan_direct_movable 0
1279 pgscan_direct_throttle 0
1280 zone_reclaim_failed 0
1281 pginodesteal 0
1282 slabs_scanned 0
1283 kswapd_inodesteal 0
1284 kswapd_low_wmark_hit_quickly 0
1285 kswapd_high_wmark_hit_quickly 0
1286 pageoutrun 1
1287 allocstall 0
1288 pgrotated 0
1289 drop_pagecache 0
1290 drop_slab 0
1291 numa_pte_updates 0
1292 numa_huge_pte_updates 0
1293 numa_hint_faults 0
1294 numa_hint_faults_local 0
1295 numa_pages_migrated 0
1296 pgmigrate_success 0
1297 pgmigrate_fail 0
1298 compact_migrate_scanned 0
1299 compact_free_scanned 0
1300 compact_isolated 0
1301 compact_stall 0
1302 compact_fail 0
1303 compact_success 0
1304 htlb_buddy_alloc_success 0
1305 htlb_buddy_alloc_fail 0
1306 unevictable_pgs_culled 2964
1307 unevictable_pgs_scanned 0
1308 unevictable_pgs_rescued 1580
1309 unevictable_pgs_mlocked 4145
1310 unevictable_pgs_munlocked 4145
1311 unevictable_pgs_cleared 0
1312 unevictable_pgs_stranded 0
1313 thp_fault_alloc 2907
1314 thp_fault_fallback 0
1315 thp_collapse_alloc 500
1316 thp_collapse_alloc_failed 0
1317 thp_split 428
1318 thp_zero_page_alloc 1
1319 thp_zero_page_alloc_failed 0
1320 balloon_inflate 0
1321 balloon_deflate 0
1322 balloon_migrate 0
1323 intr 1664579 22 0 0 0 0 0 0 0 1 33196 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 20295 70340 0 0 0 0 0 0 0 2419 352 0 11 571 752 1616 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
1324 ctxt %(kernel_ctxt)d
1325 btime 1484812069
1326 processes %(kernel_processes)d
1327 procs_running 1
1328 procs_blocked 0
1329 softirq 3526225 0 845354 2 177933 55525 0 6372 577174 0 1863865
1330 """ % counters
1331
1332         return section + cpu_output
1333
1334
1335 static_agent_data = """<<<check_mk>>>
1336 Version: 2016.12.23
1337 AgentOS: linux
1338 Hostname: Klappspaten
1339 AgentDirectory: /etc/check_mk
1340 DataDirectory: /var/lib/check_mk_agent
1341 SpoolDirectory: /var/lib/check_mk_agent/spool
1342 PluginsDirectory: /usr/lib/check_mk_agent/plugins
1343 LocalDirectory: /usr/lib/check_mk_agent/local
1344 <<<df>>>
1345 /dev/mapper/rootvg-lv_root         ext4     1032088     509248    470412      52% /
1346 /dev/sda1                          ext4      516040      84856    404972      18% /boot
1347 /dev/mapper/rootvg-lv_home         ext4     3096336    1603828   1335248      55% /home
1348 /dev/mapper/rootvg-lv_opt          ext4     1548144     409064   1060452      28% /opt
1349 /dev/mapper/rootvg-lv_tmp          ext4     2064208      64980   1894392       4% /tmp
1350 /dev/mapper/rootvg-lv_usr          ext4     4128448    1738152   2180584      45% /usr
1351 /dev/mapper/rootvg-lv_local        ext4      516040      44136    445692      10% /usr/local
1352 /dev/mapper/rootvg-lv_var          ext4     2064208    1524036    435328      78% /var
1353 /dev/mapper/rootvg-lv_install      ext4    34060032   10740480  21589632      34% /install
1354 /dev/mapper/rootvg-lv_agent        ext4     4128448    1975432   1943316      51% /agent
1355 /dev/mapper/dwhvg-lv_dingdata   ext4   258030980  103616116 151793424      41% /dingdata
1356 /dev/mapper/dwhvg-lv_conndir       ext4     1032088      61064    918596       7% /conndir
1357 /dev/mapper/dwhvg-lv_backup        ext4    61927420   11445540  47336152      20% /backup
1358 <<<df>>>
1359 [df_inodes_start]
1360 /dev/mapper/rootvg-lv_root         ext4     65536  13287     52249   21% /
1361 /dev/sda1                          ext4     32768     50     32718    1% /boot
1362 /dev/mapper/rootvg-lv_home         ext4    196608   3193    193415    2% /home
1363 /dev/mapper/rootvg-lv_opt          ext4     98304    637     97667    1% /opt
1364 /dev/mapper/rootvg-lv_tmp          ext4    131072    161    130911    1% /tmp
1365 /dev/mapper/rootvg-lv_usr          ext4    262144  62008    200136   24% /usr
1366 /dev/mapper/rootvg-lv_local        ext4     32768    124     32644    1% /usr/local
1367 /dev/mapper/rootvg-lv_var          ext4    131072  61541     69531   47% /var
1368 /dev/mapper/rootvg-lv_install      ext4   2162688   8503   2154185    1% /install
1369 /dev/mapper/rootvg-lv_agent        ext4    262144  12469    249675    5% /agent
1370 /dev/mapper/dwhvg-lv_dingdata   ext4  16384000   2429  16381571    1% /dingdata
1371 /dev/mapper/dwhvg-lv_conndir       ext4     65536    205     65331    1% /conndir
1372 /dev/mapper/dwhvg-lv_backup        ext4   3932160     82   3932078    1% /backup
1373 [df_inodes_end]
1374 <<<nfsmounts>>>
1375 <<<mounts>>>
1376 /dev/mapper/rootvg-lv_root / ext4 rw,relatime,barrier=1,data=ordered 0 0
1377 /dev/sda1 /boot ext4 rw,relatime,barrier=1,data=ordered 0 0
1378 /dev/mapper/rootvg-lv_home /home ext4 rw,relatime,barrier=0,data=ordered 0 0
1379 /dev/mapper/rootvg-lv_opt /opt ext4 rw,relatime,barrier=0,data=ordered 0 0
1380 /dev/mapper/rootvg-lv_tmp /tmp ext4 rw,relatime,barrier=0,data=ordered 0 0
1381 /dev/mapper/rootvg-lv_usr /usr ext4 rw,relatime,barrier=0,data=ordered 0 0
1382 /dev/mapper/rootvg-lv_local /usr/local ext4 rw,relatime,barrier=0,data=ordered 0 0
1383 /dev/mapper/rootvg-lv_var /var ext4 rw,relatime,barrier=0,data=ordered 0 0
1384 /dev/mapper/rootvg-lv_install /install ext4 rw,relatime,barrier=0,data=ordered 0 0
1385 /dev/mapper/rootvg-lv_agent /agent ext4 rw,relatime,barrier=0,data=ordered 0 0
1386 /dev/mapper/dwhvg-lv_dingdata /dingdata ext4 rw,relatime,barrier=0,data=ordered 0 0
1387 /dev/mapper/dwhvg-lv_conndir /conndir ext4 rw,relatime,barrier=0,data=ordered 0 0
1388 /dev/mapper/dwhvg-lv_backup /backup ext4 rw,relatime,barrier=0,data=ordered 0 0
1389 <<<ps>>>
1390 (root,120480,6764,00:00:02/24:28,1) /sbin/init splash
1391 (root,0,0,00:00:00/24:28,2) [kthreadd]
1392 (root,0,0,00:00:00/24:28,3) [ksoftirqd/0]
1393 (root,0,0,00:00:00/24:28,5) [kworker/0:0H]
1394 (root,0,0,00:00:02/24:28,7) [rcu_sched]
1395 (root,0,0,00:00:00/24:28,8) [rcu_bh]
1396 (root,0,0,00:00:00/24:28,9) [migration/0]
1397 (root,0,0,00:00:00/24:28,10) [watchdog/0]
1398 (root,0,0,00:00:00/24:28,11) [watchdog/1]
1399 (root,0,0,00:00:00/24:28,12) [migration/1]
1400 (root,0,0,00:00:00/24:28,13) [ksoftirqd/1]
1401 (root,0,0,00:00:00/24:28,15) [kworker/1:0H]
1402 (root,0,0,00:00:00/24:28,16) [watchdog/2]
1403 (root,0,0,00:00:00/24:28,17) [migration/2]
1404 (root,0,0,00:00:00/24:28,18) [ksoftirqd/2]
1405 (root,0,0,00:00:00/24:28,20) [kworker/2:0H]
1406 (root,0,0,00:00:00/24:28,21) [watchdog/3]
1407 (root,0,0,00:00:00/24:28,22) [migration/3]
1408 (root,0,0,00:00:00/24:28,23) [ksoftirqd/3]
1409 (root,0,0,00:00:00/24:28,25) [kworker/3:0H]
1410 (root,0,0,00:00:00/24:28,26) [watchdog/4]
1411 (root,0,0,00:00:00/24:28,27) [migration/4]
1412 (root,0,0,00:00:00/24:28,28) [ksoftirqd/4]
1413 (root,0,0,00:00:00/24:28,29) [kworker/4:0]
1414 (root,0,0,00:00:00/24:28,30) [kworker/4:0H]
1415 (root,0,0,00:00:00/24:28,31) [watchdog/5]
1416 (root,0,0,00:00:00/24:28,32) [migration/5]
1417 (root,0,0,00:00:00/24:28,33) [ksoftirqd/5]
1418 (root,0,0,00:00:00/24:28,34) [kworker/5:0]
1419 (root,0,0,00:00:00/24:28,35) [kworker/5:0H]
1420 (root,0,0,00:00:00/24:28,36) [watchdog/6]
1421 (root,0,0,00:00:00/24:28,37) [migration/6]
1422 (root,0,0,00:00:00/24:28,38) [ksoftirqd/6]
1423 (root,0,0,00:00:00/24:28,40) [kworker/6:0H]
1424 (root,0,0,00:00:00/24:28,41) [watchdog/7]
1425 (root,0,0,00:00:00/24:28,42) [migration/7]
1426 (root,0,0,00:00:00/24:28,43) [ksoftirqd/7]
1427 (root,0,0,00:00:00/24:28,45) [kworker/7:0H]
1428 (root,0,0,00:00:00/24:28,46) [kdevtmpfs]
1429 (root,0,0,00:00:00/24:28,47) [netns]
1430 (root,0,0,00:00:00/24:28,48) [perf]
1431 (root,0,0,00:00:00/24:28,49) [khungtaskd]
1432 (root,0,0,00:00:00/24:28,50) [writeback]
1433 (root,0,0,00:00:00/24:28,51) [ksmd]
1434 (root,0,0,00:00:00/24:28,52) [khugepaged]
1435 (root,0,0,00:00:00/24:28,53) [crypto]
1436 (root,0,0,00:00:00/24:28,54) [kintegrityd]
1437 (root,0,0,00:00:00/24:28,55) [bioset]
1438 (root,0,0,00:00:00/24:28,56) [kblockd]
1439 (root,0,0,00:00:01/24:28,58) [kworker/u16:1]
1440 (root,0,0,00:00:00/24:28,59) [ata_sff]
1441 (root,0,0,00:00:00/24:28,60) [md]
1442 (root,0,0,00:00:00/24:28,61) [devfreq_wq]
1443 (root,0,0,00:00:00/24:28,62) [kworker/2:1]
1444 (root,0,0,00:00:00/24:28,65) [kswapd0]
1445 (root,0,0,00:00:00/24:28,66) [vmstat]
1446 (root,0,0,00:00:00/24:28,67) [fsnotify_mark]
1447 (root,0,0,00:00:00/24:28,68) [ecryptfs-kthrea]
1448 (root,0,0,00:00:00/24:28,84) [kthrotld]
1449 (root,0,0,00:00:00/24:28,85) [kworker/1:1]
1450 (root,0,0,00:00:00/24:28,88) [kworker/6:1]
1451 (root,0,0,00:00:00/24:28,89) [kworker/7:1]
1452 (root,0,0,00:00:00/24:28,90) [acpi_thermal_pm]
1453 (root,0,0,00:00:00/24:28,91) [bioset]
1454 (root,0,0,00:00:00/24:28,92) [bioset]
1455 (root,0,0,00:00:00/24:28,93) [bioset]
1456 (root,0,0,00:00:00/24:28,94) [bioset]
1457 (root,0,0,00:00:00/24:28,95) [bioset]
1458 (root,0,0,00:00:00/24:28,96) [bioset]
1459 (root,0,0,00:00:00/24:28,97) [bioset]
1460 (root,0,0,00:00:00/24:28,98) [bioset]
1461 (root,0,0,00:00:00/24:28,99) [bioset]
1462 (root,0,0,00:00:00/24:28,100) [bioset]
1463 (root,0,0,00:00:00/24:28,101) [bioset]
1464 (root,0,0,00:00:00/24:28,102) [bioset]
1465 (root,0,0,00:00:00/24:28,103) [bioset]
1466 (root,0,0,00:00:00/24:28,104) [bioset]
1467 (root,0,0,00:00:00/24:28,105) [bioset]
1468 (root,0,0,00:00:00/24:28,106) [bioset]
1469 (root,0,0,00:00:00/24:28,107) [bioset]
1470 (root,0,0,00:00:00/24:28,108) [bioset]
1471 (root,0,0,00:00:00/24:28,109) [bioset]
1472 (root,0,0,00:00:00/24:28,110) [bioset]
1473 (root,0,0,00:00:00/24:28,111) [bioset]
1474 (root,0,0,00:00:00/24:28,112) [bioset]
1475 (root,0,0,00:00:00/24:28,113) [bioset]
1476 (root,0,0,00:00:00/24:28,114) [bioset]
1477 (root,0,0,00:00:01/24:27,115) [kworker/u16:2]
1478 (root,0,0,00:00:00/24:27,118) [kworker/1:2]
1479 (root,0,0,00:00:00/24:27,122) [ipv6_addrconf]
1480 (root,0,0,00:00:00/24:27,135) [deferwq]
1481 (root,0,0,00:00:00/24:27,136) [charger_manager]
1482 (root,0,0,00:00:00/24:27,177) [scsi_eh_0]
1483 (root,0,0,00:00:00/24:27,178) [scsi_tmf_0]
1484 (root,0,0,00:00:00/24:27,179) [usb-storage]
1485 (root,0,0,00:00:01/24:26,211) [irq/36-(null)]
1486 (root,0,0,00:00:00/24:26,212) [scsi_eh_1]
1487 (root,0,0,00:00:00/24:26,213) [scsi_tmf_1]
1488 (root,0,0,00:00:00/24:26,218) [bioset]
1489 (root,0,0,00:00:00/24:26,289) [bioset]
1490 (root,0,0,00:00:01/24:26,291) [kworker/u16:4]
1491 (root,0,0,00:00:00/24:21,300) [kworker/7:1H]
1492 (root,0,0,00:00:00/24:20,301) [kworker/3:1H]
1493 (root,0,0,00:00:00/24:19,311) [kdmflush]
1494 (root,0,0,00:00:00/24:19,348) [bioset]
1495 (root,0,0,00:00:00/24:19,349) [kcryptd_io]
1496 (root,0,0,00:00:00/24:19,350) [kcryptd]
1497 (root,0,0,00:00:00/24:19,351) [dmcrypt_write]
1498 (root,0,0,00:00:00/24:19,352) [bioset]
1499 (root,0,0,00:00:00/24:19,362) [kdmflush]
1500 (root,0,0,00:00:00/24:19,364) [bioset]
1501 (root,0,0,00:00:00/24:19,367) [kdmflush]
1502 (root,0,0,00:00:00/24:19,368) [bioset]
1503 (root,0,0,00:00:00/02:55,393) [kworker/u16:8]
1504 (root,0,0,00:00:00/24:19,401) [jbd2/dm-1-8]
1505 (root,0,0,00:00:00/24:19,402) [ext4-rsv-conver]
1506 (root,0,0,00:00:01/24:19,404) [kworker/u16:5]
1507 (root,0,0,00:00:01/24:19,418) [kworker/u16:6]
1508 (root,0,0,00:00:01/24:19,419) [kworker/u16:7]
1509 (root,0,0,00:00:00/24:18,440) [kauditd]
1510 (root,35496,7576,00:00:00/24:18,461) /lib/systemd/systemd-journald
1511 (root,0,0,00:00:00/24:18,477) [rpciod]
1512 (root,168504,3752,00:00:00/24:18,490) /sbin/lvmetad -f
1513 (root,45468,4628,00:00:00/24:18,499) /lib/systemd/systemd-udevd
1514 (root,0,0,00:00:00/24:18,584) [kworker/4:2]
1515 (root,0,0,00:00:00/24:18,612) [kworker/6:2]
1516 (root,0,0,00:00:00/24:18,616) [irq/47-mei_me]
1517 (root,0,0,00:00:00/24:18,632) [cfg80211]
1518 (root,0,0,00:00:00/24:18,657) [kvm-irqfd-clean]
1519 (root,0,0,00:00:00/24:18,658) [kworker/4:1H]
1520 (root,0,0,00:00:00/24:18,662) [kworker/0:1H]
1521 (root,0,0,00:00:00/24:18,691) [kworker/2:3]
1522 (root,0,0,00:00:00/24:18,723) [kworker/6:1H]
1523 (root,0,0,00:00:00/24:18,724) [kworker/2:1H]
1524 (root,0,0,00:00:00/24:18,832) [kworker/1:1H]
1525 (root,0,0,00:00:00/24:18,874) [applesmc-led]
1526 (root,0,0,00:00:00/24:18,917) [ext4-rsv-conver]
1527 (root,23536,216,00:00:00/24:18,1015) /usr/sbin/rpc.idmapd
1528 (root,47628,3556,00:00:00/24:18,1095) /sbin/rpcbind -f -w
1529 (root,25216,4108,00:00:00/24:18,1112) /usr/sbin/smartd -n
1530 (mongodb,4484316,68116,00:00:03/24:18,1114) /usr/bin/mongod --config /etc/mongodb.conf
1531 (root,30536,3288,00:00:00/24:18,1115) /usr/sbin/cron -f
1532 (daemon,26044,2144,00:00:00/24:18,1119) /usr/sbin/atd -f
1533 (root,4400,1320,00:00:00/24:18,1121) /usr/sbin/acpid
1534 (root,0,0,00:00:00/24:18,1131) [kworker/5:3]
1535 (root,7492,948,00:00:00/24:18,1135) /usr/sbin/mbpfan -f
1536 (avahi,44932,3092,00:00:00/24:18,1137) avahi-daemon: running [Klappspaten.local]
1537 (root,278044,7180,00:00:00/24:18,1145) /usr/lib/accountsservice/accounts-daemon
1538 (root,10708,856,00:00:00/24:18,1147) /usr/sbin/avahi-dnsconfd -s
1539 (syslog,256396,3140,00:00:00/24:18,1169) /usr/sbin/rsyslogd -n
1540 (root,337316,8356,00:00:00/24:18,1172) /usr/sbin/ModemManager
1541 (messagebus,44276,5052,00:00:01/24:18,1178) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation
1542 (avahi,44788,344,00:00:00/24:18,1195) avahi-daemon: chroot helper
1543 (root,0,0,00:00:00/24:18,1238) [kworker/5:1H]
1544 (root,215828,18372,00:00:00/24:18,1241) /usr/lib/snapd/snapd
1545 (whoopsie,269224,9168,00:00:00/24:18,1243) /usr/bin/whoopsie -f
1546 (root,28680,3124,00:00:00/24:18,1245) /lib/systemd/systemd-logind
1547 (root,274960,9424,00:00:00/24:18,1247) /usr/sbin/cups-browsed
1548 (root,524564,16932,00:00:00/24:18,1257) /usr/sbin/NetworkManager --no-daemon
1549 (root,0,0,00:00:00/24:18,1292) [iprt]
1550 (root,0,0,00:00:00/24:17,1315) [irq/50-brcmf_pc]
1551 (root,0,0,00:00:00/24:17,1316) [msgbuf_txflow]
1552 (colord,302680,11180,00:00:00/24:17,1333) /usr/lib/colord/colord
1553 (root,283576,10984,00:00:00/24:17,1334) /usr/lib/policykit-1/polkitd --no-debug
1554 (root,0,0,00:00:00/24:17,1401) [kworker/3:2]
1555 (root,44128,6368,00:00:00/24:17,1402) /sbin/wpa_supplicant -u -s -O /run/wpa_supplicant
1556 (root,0,0,00:00:00/24:17,1421) [kworker/u17:0]
1557 (root,0,0,00:00:00/24:17,1422) [hci0]
1558 (root,0,0,00:00:00/24:17,1423) [hci0]
1559 (root,0,0,00:00:00/24:17,1424) [kworker/u17:1]
1560 (root,32084,4600,00:00:00/24:17,1433) /usr/lib/bluetooth/bluetoothd
1561 (root,65520,5688,00:00:00/24:17,1629) /usr/sbin/sshd -D
1562 (root,37976,900,00:00:00/24:17,1636) /usr/sbin/rpc.mountd --manage-gids
1563 (root,0,0,00:00:00/24:17,1642) [nfsd4_callbacks]
1564 (root,0,0,00:00:00/24:17,1643) [lockd]
1565 (root,0,0,00:00:00/24:17,1645) [nfsd]
1566 (root,0,0,00:00:00/24:17,1646) [nfsd]
1567 (root,0,0,00:00:00/24:17,1647) [nfsd]
1568 (root,0,0,00:00:00/24:17,1648) [nfsd]
1569 (root,0,0,00:00:00/24:17,1649) [nfsd]
1570 (root,0,0,00:00:00/24:17,1650) [nfsd]
1571 (root,0,0,00:00:00/24:17,1651) [nfsd]
1572 (root,0,0,00:00:00/24:17,1652) [nfsd]
1573 (root,209964,13996,00:00:04/24:17,1718) /opt/teamviewer/tv_bin/teamviewerd -d
1574 (root,19568,2216,00:00:00/24:17,1770) /usr/sbin/irqbalance --pid=/var/run/irqbalance.pid
1575 (ntp,110036,5144,00:00:00/24:17,1772) /usr/sbin/ntpd -p /var/run/ntpd.pid -g -u 108:115
1576 (root,15056,2096,00:00:00/24:17,1818) /usr/sbin/xinetd -pidfile /run/xinetd.pid -stayalive -inetd_compat -inetd_ipv6
1577 (root,276648,6344,00:00:00/24:17,1840) /usr/sbin/lightdm
1578 (snmp,61284,8000,00:00:00/24:17,1842) /usr/sbin/snmpd -Lsd -Lf /dev/null -u snmp -g snmp -I -smux mteTrigger mteTriggerConf -p /run/snmpd.pid
1579 (uml-net,4376,792,00:00:00/24:17,1868) /usr/bin/uml_switch -tap tap0 -unix /var/run/uml-utilities/uml_switch.ctl
1580 (root,83636,6164,00:00:00/24:17,1906) /usr/sbin/apache2 -k start
1581 (www-data,83552,3692,00:00:00/24:17,1911) /usr/sbin/apache2 -k start
1582 (www-data,83948,4356,00:00:00/24:17,1912) /usr/sbin/apache2 -k start
1583 (www-data,83948,4356,00:00:00/24:17,1913) /usr/sbin/apache2 -k start
1584 (www-data,83980,4356,00:00:00/24:17,1915) /usr/sbin/apache2 -k start
1585 (www-data,83980,4356,00:00:00/24:17,1916) /usr/sbin/apache2 -k start
1586 (www-data,83948,4356,00:00:00/24:17,1917) /usr/sbin/apache2 -k start
1587 (root,537576,93684,00:00:10/24:17,1991) /usr/lib/xorg/Xorg -core :0 -seat seat0 -auth /var/run/lightdm/root/:0 -nolisten tcp vt7 -novtswitch
1588 (root,17468,1704,00:00:00/24:17,1992) /sbin/agetty --noclear tty1 linux
1589 (root,65408,4548,00:00:00/24:16,2027) /usr/lib/postfix/sbin/master
1590 (postfix,67476,4416,00:00:00/24:16,2029) pickup -l -t unix -u -c
1591 (postfix,67648,4536,00:00:00/24:16,2030) qmgr -l -t unix -u
1592 (heute,53952,7716,00:00:00/24:16,2046) python /omd/sites/heute/bin/liveproxyd
1593 (root,241532,5356,00:00:00/24:16,2052) /usr/sbin/nmbd -D
1594 (heute,78568,9460,00:00:00/24:16,2073) python /omd/sites/heute/bin/mknotifyd
1595 (root,339396,15748,00:00:00/24:16,2168) /usr/sbin/smbd -D
1596 (root,331424,5816,00:00:00/24:16,2169) /usr/sbin/smbd -D
1597 (root,339404,8928,00:00:00/24:16,2171) /usr/sbin/smbd -D
1598 (heute,101268,10584,00:00:00/24:16,2173) /usr/sbin/apache2 -f /omd/sites/heute/etc/apache/apache.conf
1599 (heute,101228,4196,00:00:00/24:16,2176) /usr/sbin/apache2 -f /omd/sites/heute/etc/apache/apache.conf
1600 (heute,311868,68852,00:00:01/24:16,2177) /usr/sbin/apache2 -f /omd/sites/heute/etc/apache/apache.conf
1601 (root,0,0,00:00:00/10:32,2189) [kworker/u16:10]
1602 (root,228240,6648,00:00:00/24:16,2190) lightdm --session-child 12 15
1603 (heute2,221652,12080,00:00:00/24:16,2248) python /omd/sites/heute2/bin/mkeventd
1604 (heute2,53952,7448,00:00:00/24:16,2279) python /omd/sites/heute2/bin/liveproxyd
1605 (heute2,78852,9724,00:00:00/24:16,2289) python /omd/sites/heute2/bin/mknotifyd
1606 (heute2,251656,3172,00:00:00/24:15,2305) /omd/sites/heute2/bin/rrdcached -w 3600 -z 1800 -f 7200 -s heute2 -m 660 -l unix:/omd/sites/heute2/tmp/run/rrdcached.sock -p /omd/sites/heute2/tmp/rrdcached.pid -j /omd/sites/heute2/var/rrdcached
1607 (heute2,1530808,8520,00:00:00/24:15,2341) /omd/sites/heute2/bin/cmc /omd/sites/heute2/var/check_mk/core/config
1608 (heute2,218676,38616,00:00:01/24:15,2379) python /omd/sites/heute2/bin/cmk --create-rrd --keepalive
1609 (heute2,100348,27400,00:00:00/24:15,2383) python /omd/sites/heute2/bin/cmk --handle-alerts --keepalive
1610 (heute2,100644,28100,00:00:00/24:15,2387) python /omd/sites/heute2/bin/cmk --notify --keepalive
1611 (heute2,10492,1068,00:00:00/24:15,2391) checkhelper
1612 (heute2,10492,1004,00:00:00/24:15,2395) checkhelper
1613 (heute2,10492,968,00:00:00/24:15,2397) checkhelper
1614 (heute2,10492,1004,00:00:00/24:15,2398) checkhelper
1615 (heute2,10492,1016,00:00:00/24:15,2399) checkhelper
1616 (heute2,104244,31840,00:00:01/24:15,2401) python /omd/sites/heute2/bin/cmk --keepalive
1617 (heute2,104760,32396,00:00:00/24:15,2404) python /omd/sites/heute2/bin/cmk --keepalive
1618 (heute2,100344,27364,00:00:00/24:15,2405) python /omd/sites/heute2/bin/cmk --keepalive
1619 (heute2,100348,27540,00:00:00/24:15,2406) python /omd/sites/heute2/bin/cmk --keepalive
1620 (heute2,100348,27516,00:00:00/24:15,2407) python /omd/sites/heute2/bin/cmk --keepalive
1621 (heute2,100360,27488,00:00:00/24:15,2408) python /omd/sites/heute2/bin/cmk --keepalive
1622 (heute2,100348,27480,00:00:00/24:15,2409) python /omd/sites/heute2/bin/cmk --keepalive
1623 (heute2,100348,27564,00:00:00/24:15,2411) python /omd/sites/heute2/bin/cmk --keepalive
1624 (heute2,100348,27456,00:00:00/24:15,2413) python /omd/sites/heute2/bin/cmk --keepalive
1625 (heute2,100352,27316,00:00:00/24:15,2414) python /omd/sites/heute2/bin/cmk --keepalive
1626 (heute2,100352,27388,00:00:00/24:15,2415) python /omd/sites/heute2/bin/cmk --keepalive
1627 (heute2,100352,27468,00:00:00/24:15,2416) python /omd/sites/heute2/bin/cmk --keepalive
1628 (heute2,100356,27472,00:00:00/24:15,2417) python /omd/sites/heute2/bin/cmk --keepalive
1629 (heute2,100360,27376,00:00:00/24:15,2418) python /omd/sites/heute2/bin/cmk --keepalive
1630 (heute2,100356,27540,00:00:00/24:15,2419) python /omd/sites/heute2/bin/cmk --keepalive
1631 (heute2,100356,27512,00:00:00/24:15,2420) python /omd/sites/heute2/bin/cmk --keepalive
1632 (heute2,100348,27400,00:00:00/24:15,2421) python /omd/sites/heute2/bin/cmk --keepalive
1633 (heute2,100348,27444,00:00:00/24:15,2422) python /omd/sites/heute2/bin/cmk --keepalive
1634 (heute2,100348,27600,00:00:00/24:15,2441) python /omd/sites/heute2/bin/cmk --keepalive
1635 (heute2,100352,27300,00:00:00/24:15,2442) python /omd/sites/heute2/bin/cmk --keepalive
1636 (heute2,102436,27812,00:00:00/24:15,2443) python /omd/sites/heute2/bin/cmk --keepalive --real-time-checks
1637 (root,12596,1060,00:00:00/24:15,2448) icmpsender 8 0 1000
1638 (root,14848,1024,00:00:00/24:15,2449) icmpreceiver
1639 (heute2,101268,10624,00:00:00/24:15,2450) /usr/sbin/apache2 -f /omd/sites/heute2/etc/apache/apache.conf
1640 (heute2,101228,4216,00:00:00/24:15,2457) /usr/sbin/apache2 -f /omd/sites/heute2/etc/apache/apache.conf
1641 (heute2,313420,70412,00:00:01/24:15,2458) /usr/sbin/apache2 -f /omd/sites/heute2/etc/apache/apache.conf
1642 (test1,221484,11476,00:00:00/24:15,2511) python /omd/sites/test1/bin/mkeventd
1643 (rtkit,183544,3124,00:00:00/24:15,2524) /usr/lib/rtkit/rtkit-daemon
1644 (test1,54204,7964,00:00:00/24:15,2526) python /omd/sites/test1/bin/liveproxyd
1645 (test1,78848,9756,00:00:00/24:15,2568) python /omd/sites/test1/bin/mknotifyd
1646 (root,367748,8604,00:00:00/24:14,2580) /usr/lib/udisks2/udisksd --no-debug
1647 (test1,177660,3136,00:00:00/24:14,2582) /omd/sites/test1/bin/rrdcached -w 3600 -z 1800 -f 7200 -s test1 -m 660 -l unix:/omd/sites/test1/tmp/run/rrdcached.sock -p /omd/sites/test1/tmp/rrdcached.pid -j /omd/sites/test1/var/rrdcached
1648 (test1,1529284,6224,00:00:00/24:14,2642) /omd/sites/test1/bin/cmc /omd/sites/test1/var/check_mk/core/config
1649 (test1,218580,38660,00:00:01/24:14,2698) python /omd/sites/test1/share/check_mk/modules/check_mk.py --create-rrd --keepalive
1650 (test1,100332,27696,00:00:00/24:14,2700) python /omd/sites/test1/share/check_mk/modules/check_mk.py --handle-alerts --keepalive
1651 (test1,100336,27624,00:00:00/24:14,2702) python /omd/sites/test1/share/check_mk/modules/check_mk.py --notify --keepalive
1652 (root,0,0,00:00:00/24:14,2704) [krfcommd]
1653 (test1,4364,632,00:00:00/24:14,2711) checkhelper
1654 (test1,4364,636,00:00:00/24:14,2712) checkhelper
1655 (test1,4364,784,00:00:00/24:14,2713) checkhelper
1656 (test1,4364,652,00:00:00/24:14,2717) checkhelper
1657 (test1,4364,740,00:00:00/24:14,2737) checkhelper
1658 (test1,100332,27452,00:00:00/24:14,2738) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1659 (test1,100332,27748,00:00:00/24:14,2739) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1660 (test1,100332,27612,00:00:00/24:14,2741) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1661 (test1,100336,27660,00:00:00/24:14,2742) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1662 (root,349032,10440,00:00:00/24:14,2744) /usr/lib/upower/upowerd
1663 (test1,100324,27752,00:00:00/24:14,2747) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1664 (test1,100340,27676,00:00:00/24:14,2748) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1665 (test1,100332,27608,00:00:00/24:14,2751) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1666 (test1,100332,27612,00:00:00/24:14,2752) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1667 (test1,100332,27712,00:00:00/24:14,2753) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1668 (test1,100328,27660,00:00:00/24:14,2754) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1669 (test1,100328,27692,00:00:00/24:14,2755) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1670 (test1,100332,27572,00:00:00/24:14,2756) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1671 (test1,100328,27416,00:00:00/24:14,2757) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1672 (test1,100332,27668,00:00:00/24:14,2758) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1673 (test1,100388,27732,00:00:00/24:14,2759) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1674 (test1,100328,27916,00:00:00/24:14,2760) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1675 (test1,100332,27444,00:00:00/24:14,2761) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1676 (test1,100328,27524,00:00:00/24:14,2765) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1677 (test1,100328,27800,00:00:00/24:14,2768) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1678 (test1,100328,27412,00:00:00/24:14,2774) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1679 (test1,102668,27744,00:00:00/24:14,2775) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive --real-time-checks
1680 (root,12520,1160,00:00:00/24:14,2776) icmpsender 8 0 1000
1681 (root,8720,752,00:00:00/24:14,2777) icmpreceiver
1682 (heute,311868,68908,00:00:01/24:06,3812) /usr/sbin/apache2 -f /omd/sites/heute/etc/apache/apache.conf
1683 (www-data,83948,4364,00:00:00/24:06,4132) /usr/sbin/apache2 -k start
1684 (test1,101264,10604,00:00:00/24:05,4165) /usr/sbin/apache2 -f /omd/sites/test1/etc/apache/apache.conf
1685 (test1,15056,2052,00:00:00/24:05,4178) /omd/sites/test1/var/tmp/xinetd -pidfile /omd/sites/test1/tmp/run/xinetd.pid -filelog /omd/sites/test1/var/log/xinetd.log -f /omd/sites/test1/etc/xinetd.conf
1686 (test1,101224,4192,00:00:00/24:05,4186) /usr/sbin/apache2 -f /omd/sites/test1/etc/apache/apache.conf
1687 (test1,316088,70952,00:00:01/24:05,4187) /usr/sbin/apache2 -f /omd/sites/test1/etc/apache/apache.conf
1688 (test2,221480,11520,00:00:00/24:05,4200) python /omd/sites/test2/bin/mkeventd
1689 (test2,54204,7768,00:00:00/24:05,4209) python /omd/sites/test2/bin/liveproxyd
1690 (test2,78848,9656,00:00:00/24:05,4219) python /omd/sites/test2/bin/mknotifyd
1691 (test2,177660,3080,00:00:00/24:05,4224) /omd/sites/test2/bin/rrdcached -w 3600 -z 1800 -f 7200 -s test2 -m 660 -l unix:/omd/sites/test2/tmp/run/rrdcached.sock -p /omd/sites/test2/tmp/rrdcached.pid -j /omd/sites/test2/var/rrdcached
1692 (test2,1529284,6060,00:00:00/24:05,4244) /omd/sites/test2/bin/cmc /omd/sites/test2/var/check_mk/core/config
1693 (test2,218592,38636,00:00:01/24:05,4276) python /omd/sites/test2/share/check_mk/modules/check_mk.py --create-rrd --keepalive
1694 (test2,100332,27728,00:00:00/24:05,4277) python /omd/sites/test2/share/check_mk/modules/check_mk.py --handle-alerts --keepalive
1695 (test2,100332,27588,00:00:00/24:05,4279) python /omd/sites/test2/share/check_mk/modules/check_mk.py --notify --keepalive
1696 (test2,4364,784,00:00:00/24:05,4280) checkhelper
1697 (test2,4364,640,00:00:00/24:05,4282) checkhelper
1698 (test2,4364,652,00:00:00/24:05,4283) checkhelper
1699 (test2,4364,788,00:00:00/24:05,4284) checkhelper
1700 (test2,4364,732,00:00:00/24:05,4285) checkhelper
1701 (test2,100336,27776,00:00:00/24:05,4286) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1702 (test2,100332,27708,00:00:00/24:05,4287) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1703 (test2,100328,27540,00:00:00/24:05,4288) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1704 (test2,100356,27716,00:00:00/24:05,4289) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1705 (test2,100360,27400,00:00:00/24:05,4290) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1706 (test2,100336,27848,00:00:00/24:05,4291) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1707 (test2,100332,27704,00:00:00/24:05,4292) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1708 (test2,100328,27660,00:00:00/24:05,4293) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1709 (test2,100336,27696,00:00:00/24:05,4297) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1710 (test2,100332,27612,00:00:00/24:05,4298) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1711 (test2,100320,27428,00:00:00/24:05,4299) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1712 (test2,100328,27712,00:00:00/24:05,4300) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1713 (test2,100336,27620,00:00:00/24:05,4302) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1714 (test2,100328,27612,00:00:00/24:05,4303) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1715 (test2,100332,27584,00:00:00/24:05,4304) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1716 (test2,100336,27572,00:00:00/24:05,4305) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1717 (test2,100332,27844,00:00:00/24:05,4306) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1718 (test2,100332,27764,00:00:00/24:05,4307) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1719 (test2,100332,27680,00:00:00/24:05,4308) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1720 (test2,100328,27468,00:00:00/24:05,4317) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1721 (test2,102792,27924,00:00:00/24:05,4318) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive --real-time-checks
1722 (root,12520,1080,00:00:00/24:05,4319) icmpsender 8 0 1000
1723 (root,8720,732,00:00:00/24:05,4320) icmpreceiver
1724 (test2,101264,10592,00:00:00/23:56,4874) /usr/sbin/apache2 -f /omd/sites/test2/etc/apache/apache.conf
1725 (test2,15056,2056,00:00:00/23:56,4887) /omd/sites/test2/var/tmp/xinetd -pidfile /omd/sites/test2/tmp/run/xinetd.pid -filelog /omd/sites/test2/var/log/xinetd.log -f /omd/sites/test2/etc/xinetd.conf
1726 (test2,101224,4188,00:00:00/23:56,4888) /usr/sbin/apache2 -f /omd/sites/test2/etc/apache/apache.conf
1727 (test2,316032,70940,00:00:01/23:56,4889) /usr/sbin/apache2 -f /omd/sites/test2/etc/apache/apache.conf
1728 (test_modes,221652,11624,00:00:00/23:56,4907) python /omd/sites/test_modes/bin/mkeventd
1729 (test_modes,53952,7652,00:00:00/23:56,4915) python /omd/sites/test_modes/bin/liveproxyd
1730 (test_modes,78856,9744,00:00:00/23:56,4923) python /omd/sites/test_modes/bin/mknotifyd
1731 (test_modes,251392,3184,00:00:00/23:56,4927) /omd/sites/test_modes/bin/rrdcached -w 3600 -z 1800 -f 7200 -s test_modes -m 660 -l unix:/omd/sites/test_modes/tmp/run/rrdcached.sock -p /omd/sites/test_modes/tmp/rrdcached.pid -j /omd/sites/test_modes/var/rrdcached
1732 (test_modes,1529456,6236,00:00:00/23:56,4944) /omd/sites/test_modes/bin/cmc /omd/sites/test_modes/var/check_mk/core/config
1733 (test_modes,219156,43348,00:00:01/23:56,4974) python /omd/sites/test_modes/bin/cmk --create-rrd --keepalive
1734 (test_modes,97708,31144,00:00:01/23:56,4976) python /omd/sites/test_modes/bin/cmk --handle-alerts --keepalive
1735 (test_modes,97684,31224,00:00:01/23:56,4977) python /omd/sites/test_modes/bin/cmk --notify --keepalive
1736 (test_modes,7396,2260,00:00:00/23:56,4978) checkhelper
1737 (test_modes,7396,948,00:00:00/23:56,4979) checkhelper
1738 (test_modes,7396,792,00:00:00/23:56,4980) checkhelper
1739 (test_modes,7396,840,00:00:00/23:56,4981) checkhelper
1740 (test_modes,7396,808,00:00:00/23:56,4984) checkhelper
1741 (test_modes,108680,34108,00:00:01/23:56,4986) python /omd/sites/test_modes/bin/cmk --keepalive
1742 (test_modes,97440,30660,00:00:01/23:56,4987) python /omd/sites/test_modes/bin/cmk --keepalive
1743 (test_modes,97400,30628,00:00:01/23:56,4988) python /omd/sites/test_modes/bin/cmk --keepalive
1744 (test_modes,97444,30652,00:00:01/23:56,4989) python /omd/sites/test_modes/bin/cmk --keepalive
1745 (test_modes,97408,30580,00:00:01/23:56,4990) python /omd/sites/test_modes/bin/cmk --keepalive
1746 (test_modes,97420,30624,00:00:01/23:56,4991) python /omd/sites/test_modes/bin/cmk --keepalive
1747 (test_modes,97384,30736,00:00:01/23:56,4992) python /omd/sites/test_modes/bin/cmk --keepalive
1748 (test_modes,97416,30640,00:00:01/23:56,4993) python /omd/sites/test_modes/bin/cmk --keepalive
1749 (test_modes,97440,30620,00:00:01/23:56,4994) python /omd/sites/test_modes/bin/cmk --keepalive
1750 (test_modes,97440,30576,00:00:01/23:56,4996) python /omd/sites/test_modes/bin/cmk --keepalive
1751 (test_modes,97404,30560,00:00:01/23:56,4997) python /omd/sites/test_modes/bin/cmk --keepalive
1752 (test_modes,97408,30744,00:00:01/23:56,4998) python /omd/sites/test_modes/bin/cmk --keepalive
1753 (test_modes,97392,30572,00:00:01/23:56,4999) python /omd/sites/test_modes/bin/cmk --keepalive
1754 (test_modes,97420,30568,00:00:01/23:56,5000) python /omd/sites/test_modes/bin/cmk --keepalive
1755 (test_modes,97448,30556,00:00:01/23:56,5004) python /omd/sites/test_modes/bin/cmk --keepalive
1756 (test_modes,97436,30704,00:00:01/23:56,5005) python /omd/sites/test_modes/bin/cmk --keepalive
1757 (test_modes,97448,30660,00:00:01/23:56,5006) python /omd/sites/test_modes/bin/cmk --keepalive
1758 (test_modes,97412,30568,00:00:01/23:56,5007) python /omd/sites/test_modes/bin/cmk --keepalive
1759 (test_modes,97308,30680,00:00:01/23:56,5008) python /omd/sites/test_modes/bin/cmk --keepalive
1760 (test_modes,97444,30708,00:00:01/23:55,5014) python /omd/sites/test_modes/bin/cmk --keepalive
1761 (test_modes,99388,30532,00:00:01/23:55,5015) python /omd/sites/test_modes/bin/cmk --keepalive --real-time-checks
1762 (root,12612,1048,00:00:00/23:55,5016) icmpsender 8 0 1000
1763 (root,11756,872,00:00:00/23:55,5017) icmpreceiver
1764 (test_modes,101524,10648,00:00:00/23:46,5763) /usr/sbin/apache2 -f /omd/sites/test_modes/etc/apache/apache.conf
1765 (test_modes,101484,4220,00:00:00/23:46,5774) /usr/sbin/apache2 -f /omd/sites/test_modes/etc/apache/apache.conf
1766 (test_modes,313392,70500,00:00:01/23:46,5775) /usr/sbin/apache2 -f /omd/sites/test_modes/etc/apache/apache.conf
1767 (test_modes,15056,2172,00:00:00/23:46,5776) /omd/sites/test_modes/var/tmp/xinetd -pidfile /omd/sites/test_modes/tmp/run/xinetd.pid -filelog /omd/sites/test_modes/var/log/xinetd.log -f /omd/sites/test_modes/etc/xinetd.conf
1768 (test_modes,313392,70520,00:00:01/23:16,6881) /usr/sbin/apache2 -f /omd/sites/test_modes/etc/apache/apache.conf
1769 (test1,316088,70808,00:00:01/23:16,7013) /usr/sbin/apache2 -f /omd/sites/test1/etc/apache/apache.conf
1770 (heute2,313420,70432,00:00:01/23:16,7014) /usr/sbin/apache2 -f /omd/sites/heute2/etc/apache/apache.conf
1771 (test2,316032,71012,00:00:01/23:16,7023) /usr/sbin/apache2 -f /omd/sites/test2/etc/apache/apache.conf
1772 (root,0,0,00:00:00/09:45,7836) [kworker/3:0]
1773 (root,0,0,00:00:00/09:22,8288) [kworker/0:0]
1774 (heute,221660,11908,00:00:00/02:07,9604) python /omd/sites/heute/bin/mkeventd
1775 (heute,251392,3376,00:00:01/02:04,9785) /omd/sites/heute/bin/rrdcached -w 300 -z 10 -f 7200 -s heute -m 660 -l unix:/omd/sites/heute/tmp/run/rrdcached.sock -p /omd/sites/heute/tmp/rrdcached.pid -j /omd/sites/heute/var/rrdcached
1776 (root,22668,2276,00:00:00/00:29,10284) /bin/bash /usr/bin/check_mk_agent
1777 (heute,1529448,24964,00:00:00/00:27,10434) /omd/sites/heute/bin/cmc /omd/sites/heute/var/check_mk/core/config
1778 (heute,218612,38580,00:00:01/00:27,10456) python /omd/sites/heute/share/check_mk/modules/check_mk.py --create-rrd --keepalive
1779 (heute,100444,27424,00:00:00/00:27,10457) python /omd/sites/heute/share/check_mk/modules/check_mk.py --handle-alerts --keepalive
1780 (heute,100388,27436,00:00:00/00:27,10458) python /omd/sites/heute/share/check_mk/modules/check_mk.py --notify --keepalive
1781 (heute,7396,888,00:00:00/00:27,10459) checkhelper
1782 (heute,7396,788,00:00:00/00:27,10460) checkhelper
1783 (heute,7396,832,00:00:00/00:27,10461) checkhelper
1784 (heute,7396,780,00:00:00/00:27,10462) checkhelper
1785 (heute,7396,832,00:00:00/00:27,10463) checkhelper
1786 (heute,100412,27500,00:00:00/00:27,10464) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1787 (heute,100420,27440,00:00:00/00:27,10465) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1788 (heute,100420,27532,00:00:00/00:27,10466) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1789 (heute,100416,27468,00:00:00/00:27,10467) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1790 (heute,100420,27360,00:00:00/00:27,10468) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1791 (heute,100456,27572,00:00:00/00:27,10469) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1792 (heute,100452,27428,00:00:00/00:27,10470) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1793 (heute,100452,27488,00:00:00/00:27,10471) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1794 (heute,100428,27608,00:00:00/00:27,10472) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1795 (heute,100420,27320,00:00:00/00:27,10473) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1796 (heute,100456,27272,00:00:00/00:27,10474) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1797 (heute,100420,27492,00:00:00/00:27,10475) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1798 (heute,100420,27364,00:00:00/00:27,10476) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1799 (heute,100452,27512,00:00:00/00:27,10477) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1800 (heute,100452,27344,00:00:00/00:27,10478) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1801 (heute,100416,27540,00:00:00/00:27,10479) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1802 (heute,100456,27584,00:00:00/00:27,10480) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1803 (heute,100440,27344,00:00:00/00:27,10481) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1804 (heute,100416,27628,00:00:00/00:27,10482) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1805 (heute,100412,27448,00:00:00/00:27,10483) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1806 (heute,102500,27776,00:00:00/00:27,10484) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive --real-time-checks
1807 (root,12612,1092,00:00:00/00:27,10485) icmpsender 8 0 1000
1808 (root,11756,864,00:00:00/00:27,10486) icmpreceiver
1809 (root,0,0,00:00:00/18:03,15867) [kworker/0:2]
1810 (root,0,0,00:00:00/05:28,17253) [kworker/u16:3]
1811 (root,0,0,00:00:01/14:39,22323) [kworker/u16:0]
1812 (root,0,0,00:00:00/14:39,22324) [kworker/u16:9]
1813 (root,0,0,00:00:00/13:46,26453) [kworker/7:0]
1814 <<<mem>>>
1815 MemTotal:       16305836 kB
1816 MemFree:        10181860 kB
1817 MemAvailable:   11599688 kB
1818 Buffers:          176928 kB
1819 Cached:          1645288 kB
1820 SwapCached:            0 kB
1821 Active:          5181520 kB
1822 Inactive:         560520 kB
1823 Active(anon):    3923808 kB
1824 Inactive(anon):   210564 kB
1825 Active(file):    1257712 kB
1826 Inactive(file):   349956 kB
1827 Unevictable:           0 kB
1828 Mlocked:               0 kB
1829 SwapTotal:      16650236 kB
1830 SwapFree:       16650236 kB
1831 Dirty:             40948 kB
1832 Writeback:             0 kB
1833 AnonPages:       3919412 kB
1834 Mapped:           330372 kB
1835 Shmem:            214552 kB
1836 Slab:             199928 kB
1837 SReclaimable:     136136 kB
1838 SUnreclaim:        63792 kB
1839 KernelStack:       14832 kB
1840 PageTables:        70708 kB
1841 NFS_Unstable:          0 kB
1842 Bounce:                0 kB
1843 WritebackTmp:          0 kB
1844 CommitLimit:    24803152 kB
1845 Committed_AS:    7579556 kB
1846 VmallocTotal:   34359738367 kB
1847 VmallocUsed:           0 kB
1848 VmallocChunk:          0 kB
1849 HardwareCorrupted:     0 kB
1850 AnonHugePages:   1013760 kB
1851 CmaTotal:              0 kB
1852 CmaFree:               0 kB
1853 HugePages_Total:       0
1854 HugePages_Free:        0
1855 HugePages_Rsvd:        0
1856 HugePages_Surp:        0
1857 Hugepagesize:       2048 kB
1858 DirectMap4k:      195228 kB
1859 DirectMap2M:     8067072 kB
1860 DirectMap1G:     8388608 kB
1861 <<<cpu>>>
1862 2.05 2.31 1.62 1/920 11110 8
1863 <<<uptime>>>
1864 1468.80 10506.46
1865 <<<lnx_if:sep(58)>>>
1866     lo:462522167348 423391761    0    0    0     0          0         0 462522167348 423391761    0    0    0     0       0          0
1867   eth0:9663614968657 4226434136    0    0    0     0          0         0 41120440285551 1482439946    0    0    0     0       0          0
1868   eth1:9706218845482 3826133052    0    0    0     0          0         0 1746479361684 3786292851    0    0    0     0       0          0
1869   eth2:       0       0    0    0    0     0          0         0        0       0    0    0    0     0       0          0
1870   eth3:       0       0    0    0    0     0          0         0        0       0    0    0    0     0       0          0
1871   eth4:       0       0    0    0    0     0          0         0        0       0    0    0    0     0       0          0
1872   eth5:349900483789 5212016032    0    0    0     0          0         0 185966695259985 5288220641    0    0    0     0       0          0
1873  bond0:19369833814139 8052567188    0    0    0     0          0         0 42866919647235 5268732797    0    0    0     0       0          0
1874 [lo]
1875         Link detected: yes
1876         Address: 00:00:00:00:00:00
1877 [eth0]
1878         Speed: 1000Mb/s
1879         Duplex: Full
1880         Auto-negotiation: on
1881         Link detected: yes
1882         Address: e8:39:35:be:5d:90
1883 [eth1]
1884         Speed: 1000Mb/s
1885         Duplex: Full
1886         Auto-negotiation: on
1887         Link detected: yes
1888         Address: e8:39:35:be:5d:90
1889 [eth2]
1890         Speed: Unknown!
1891         Duplex: Unknown! (255)
1892         Auto-negotiation: off
1893         Link detected: no
1894         Address: e8:39:35:be:5d:92
1895 [eth3]
1896         Speed: Unknown!
1897         Duplex: Unknown! (255)
1898         Auto-negotiation: off
1899         Link detected: no
1900         Address: e8:39:35:be:5d:93
1901 [eth4]
1902         Speed: Unknown!
1903         Duplex: Unknown! (255)
1904         Auto-negotiation: off
1905         Link detected: no
1906         Address: 80:c1:6e:a9:22:38
1907 [eth5]
1908         Speed: 10000Mb/s
1909         Duplex: Full
1910         Auto-negotiation: off
1911         Link detected: yes
1912         Address: 80:c1:6e:a9:22:3c
1913 [bond0]
1914         Speed: 2000Mb/s
1915         Duplex: Full
1916         Auto-negotiation: off
1917         Link detected: yes
1918         Address: e8:39:35:be:5d:90
1919 <<<lnx_bonding:sep(58)>>>
1920 ==> bond0 <==
1921 Ethernet Channel Bonding Driver: v3.6.0 (September 26, 2009)
1922
1923 Bonding Mode: IEEE 802.3ad Dynamic link aggregation
1924 Transmit Hash Policy: layer2 (0)
1925 MII Status: up
1926 MII Polling Interval (ms): 100
1927 Up Delay (ms): 0
1928 Down Delay (ms): 0
1929
1930 802.3ad info
1931 LACP rate: slow
1932 Aggregator selection policy (ad_select): stable
1933 Active Aggregator Info:
1934         Aggregator ID: 1
1935         Number of ports: 2
1936         Actor Key: 17
1937         Partner Key: 203
1938         Partner Mac Address: 00:23:04:ee:be:0b
1939
1940 Slave Interface: eth0
1941 MII Status: up
1942 Speed: 1000 Mbps
1943 Duplex: full
1944 Link Failure Count: 0
1945 Permanent HW addr: e8:39:35:be:5d:90
1946 Aggregator ID: 1
1947 Slave queue ID: 0
1948
1949 Slave Interface: eth1
1950 MII Status: up
1951 Speed: 1000 Mbps
1952 Duplex: full
1953 Link Failure Count: 0
1954 Permanent HW addr: e8:39:35:be:5d:91
1955 Aggregator ID: 1
1956 Slave queue ID: 0
1957 ~
1958 <<<tcp_conn_stats>>>
1959 08 3
1960 01 4
1961 0A 34
1962 05 1
1963 06 194
1964 <<<multipath>>>
1965 mpathr (36006016067702d001acda7c3695ae211) dm-18 DGC,RAID 5
1966 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
1967 |-+- policy='round-robin 0' prio=0 status=active
1968 | |- 1:0:1:14 sdaq 66:160 active undef running
1969 | `- 3:0:1:14 sdbq 68:64  active undef running
1970 `-+- policy='round-robin 0' prio=0 status=enabled
1971   |- 2:0:0:14 sdq  65:0   active undef running
1972   `- 4:0:1:14 sdcq 69:224 active undef running
1973 mpathe (36006016067702d00fa66b0e3675ae211) dm-27 DGC,RAID 5
1974 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
1975 |-+- policy='round-robin 0' prio=0 status=active
1976 | |- 2:0:0:1  sdd  8:48   active undef running
1977 | `- 4:0:1:1  sdcd 69:16  active undef running
1978 `-+- policy='round-robin 0' prio=0 status=enabled
1979   |- 1:0:1:1  sdad 65:208 active undef running
1980   `- 3:0:1:1  sdbd 67:112 active undef running
1981 mpathq (36006016067702d00a46b8eaa695ae211) dm-13 DGC,RAID 5
1982 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
1983 |-+- policy='round-robin 0' prio=0 status=active
1984 | |- 2:0:0:13 sdp  8:240  active undef running
1985 | `- 4:0:1:13 sdcp 69:208 active undef running
1986 `-+- policy='round-robin 0' prio=0 status=enabled
1987   |- 1:0:1:13 sdap 66:144 active undef running
1988   `- 3:0:1:13 sdbp 68:48  active undef running
1989 mpathd (36006016067702d00c0eb668d675ae211) dm-2 DGC,RAID 5
1990 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
1991 |-+- policy='round-robin 0' prio=0 status=active
1992 | |- 1:0:1:0  sdac 65:192 active undef running
1993 | `- 3:0:1:0  sdbc 67:96  active undef running
1994 `-+- policy='round-robin 0' prio=0 status=enabled
1995   |- 2:0:0:0  sdc  8:32   active undef running
1996   `- 4:0:1:0  sdcc 69:0   active undef running
1997 mpathp (36006016067702d0088203c96695ae211) dm-26 DGC,RAID 5
1998 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
1999 |-+- policy='round-robin 0' prio=0 status=active
2000 | |- 1:0:1:12 sdao 66:128 active undef running
2001 | `- 3:0:1:12 sdbo 68:32  active undef running
2002 `-+- policy='round-robin 0' prio=0 status=enabled
2003   |- 2:0:0:12 sdo  8:224  active undef running
2004   `- 4:0:1:12 sdco 69:192 active undef running
2005 mpathc (36006016006b032002ec7d90f537be211) dm-9 DGC,RAID 5
2006 size=2.3T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2007 |-+- policy='round-robin 0' prio=0 status=active
2008 | |- 2:0:1:0  sdab 65:176 active undef running
2009 | `- 4:0:0:0  sdcb 68:240 active undef running
2010 `-+- policy='round-robin 0' prio=0 status=enabled
2011   |- 1:0:0:0  sdb  8:16   active undef running
2012   `- 3:0:0:0  sdbb 67:80  active undef running
2013 mpatho (36006016067702d00b85c207d695ae211) dm-21 DGC,RAID 5
2014 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2015 |-+- policy='round-robin 0' prio=0 status=active
2016 | |- 2:0:0:11 sdn  8:208  active undef running
2017 | `- 4:0:1:11 sdcn 69:176 active undef running
2018 `-+- policy='round-robin 0' prio=0 status=enabled
2019   |- 1:0:1:11 sdan 66:112 active undef running
2020   `- 3:0:1:11 sdbn 68:16  active undef running
2021 mpathb (36006016067702d00203a0733275be211) dm-17 DGC,RAID 5
2022 size=2.3T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2023 |-+- policy='round-robin 0' prio=0 status=active
2024 | |- 1:0:1:17 sdat 66:208 active undef running
2025 | `- 3:0:1:17 sdbt 68:112 active undef running
2026 `-+- policy='round-robin 0' prio=0 status=enabled
2027   |- 2:0:0:17 sdt  65:48  active undef running
2028   `- 4:0:1:17 sdct 70:16  active undef running
2029 mpathn (36006016067702d0006a5fe67695ae211) dm-8 DGC,RAID 5
2030 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2031 |-+- policy='round-robin 0' prio=0 status=active
2032 | |- 1:0:1:10 sdam 66:96  active undef running
2033 | `- 3:0:1:10 sdbm 68:0   active undef running
2034 `-+- policy='round-robin 0' prio=0 status=enabled
2035   |- 2:0:0:10 sdm  8:192  active undef running
2036   `- 4:0:1:10 sdcm 69:160 active undef running
2037 mpathz (3600601600cd031009e4a05bc2022e411) dm-20 DGC,RAID 5
2038 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2039 |-+- policy='round-robin 0' prio=0 status=active
2040 | |- 1:0:1:24 sdba 67:64  active undef running
2041 | `- 3:0:1:24 sdca 68:224 active undef running
2042 `-+- policy='round-robin 0' prio=0 status=enabled
2043   |- 2:0:0:24 sdaa 65:160 active undef running
2044   `- 4:0:1:24 sdda 70:128 active undef running
2045 mpathm (36006016067702d00fc1e5141695ae211) dm-10 DGC,RAID 5
2046 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2047 |-+- policy='round-robin 0' prio=0 status=active
2048 | |- 2:0:0:9  sdl  8:176  active undef running
2049 | `- 4:0:1:9  sdcl 69:144 active undef running
2050 `-+- policy='round-robin 0' prio=0 status=enabled
2051   |- 1:0:1:9  sdal 66:80  active undef running
2052   `- 3:0:1:9  sdbl 67:240 active undef running
2053 mpathy (3600601600cd03100d88010822022e411) dm-25 DGC,RAID 5
2054 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2055 |-+- policy='round-robin 0' prio=0 status=active
2056 | |- 2:0:0:23 sdz  65:144 active undef running
2057 | `- 4:0:1:23 sdcz 70:112 active undef running
2058 `-+- policy='round-robin 0' prio=0 status=enabled
2059   |- 1:0:1:23 sdaz 67:48  active undef running
2060   `- 3:0:1:23 sdbz 68:208 active undef running
2061 mpathl (36006016067702d00c646981c695ae211) dm-11 DGC,RAID 5
2062 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2063 |-+- policy='round-robin 0' prio=0 status=active
2064 | |- 1:0:1:8  sdak 66:64  active undef running
2065 | `- 3:0:1:8  sdbk 67:224 active undef running
2066 `-+- policy='round-robin 0' prio=0 status=enabled
2067   |- 2:0:0:8  sdk  8:160  active undef running
2068   `- 4:0:1:8  sdck 69:128 active undef running
2069 mpathx (3600601600cd03100ae96394fd111e411) dm-19 DGC,RAID 5
2070 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2071 |-+- policy='round-robin 0' prio=0 status=active
2072 | |- 1:0:1:22 sday 67:32  active undef running
2073 | `- 3:0:1:22 sdby 68:192 active undef running
2074 `-+- policy='round-robin 0' prio=0 status=enabled
2075   |- 2:0:0:22 sdy  65:128 active undef running
2076   `- 4:0:1:22 sdcy 70:96  active undef running
2077 mpathk (36006016067702d00d416f905695ae211) dm-5 DGC,RAID 5
2078 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2079 |-+- policy='round-robin 0' prio=0 status=active
2080 | |- 2:0:0:7  sdj  8:144  active undef running
2081 | `- 4:0:1:7  sdcj 69:112 active undef running
2082 `-+- policy='round-robin 0' prio=0 status=enabled
2083   |- 1:0:1:7  sdaj 66:48  active undef running
2084   `- 3:0:1:7  sdbj 67:208 active undef running
2085 mpathw (3600601600cd0310046b0242476fbe311) dm-23 DGC,RAID 5
2086 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2087 |-+- policy='round-robin 0' prio=0 status=active
2088 | |- 2:0:0:21 sdx  65:112 active undef running
2089 | `- 4:0:1:21 sdcx 70:80  active undef running
2090 `-+- policy='round-robin 0' prio=0 status=enabled
2091   |- 1:0:1:21 sdax 67:16  active undef running
2092   `- 3:0:1:21 sdbx 68:176 active undef running
2093 mpathj (36006016067702d001a8376ef685ae211) dm-6 DGC,RAID 5
2094 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2095 |-+- policy='round-robin 0' prio=0 status=active
2096 | |- 1:0:1:6  sdai 66:32  active undef running
2097 | `- 3:0:1:6  sdbi 67:192 active undef running
2098 `-+- policy='round-robin 0' prio=0 status=enabled
2099   |- 2:0:0:6  sdi  8:128  active undef running
2100   `- 4:0:1:6  sdci 69:96  active undef running
2101 mpathv (3600601600cd0310058dc725dd9c3e311) dm-22 DGC,RAID 5
2102 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2103 |-+- policy='round-robin 0' prio=0 status=active
2104 | |- 1:0:1:20 sdaw 67:0   active undef running
2105 | `- 3:0:1:20 sdbw 68:160 active undef running
2106 `-+- policy='round-robin 0' prio=0 status=enabled
2107   |- 2:0:0:20 sdw  65:96  active undef running
2108   `- 4:0:1:20 sdcw 70:64  active undef running
2109 mpathi (36006016067702d00a0b3acd5685ae211) dm-12 DGC,RAID 5
2110 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2111 |-+- policy='round-robin 0' prio=0 status=active
2112 | |- 2:0:0:5  sdh  8:112  active undef running
2113 | `- 4:0:1:5  sdch 69:80  active undef running
2114 `-+- policy='round-robin 0' prio=0 status=enabled
2115   |- 1:0:1:5  sdah 66:16  active undef running
2116   `- 3:0:1:5  sdbh 67:176 active undef running
2117 mpathu (3600601600cd0310042f4ce68f26ee311) dm-24 DGC,RAID 5
2118 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2119 |-+- policy='round-robin 0' prio=0 status=active
2120 | |- 2:0:0:19 sdv  65:80  active undef running
2121 | `- 4:0:1:19 sdcv 70:48  active undef running
2122 `-+- policy='round-robin 0' prio=0 status=enabled
2123   |- 1:0:1:19 sdav 66:240 active undef running
2124   `- 3:0:1:19 sdbv 68:144 active undef running
2125 mpathaa (3600601600cd03100f87503f8fb63e411) dm-14 DGC,RAID 5
2126 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2127 |-+- policy='round-robin 0' prio=0 status=active
2128 | |- 2:0:0:16 sdas 66:192 active undef running
2129 | `- 4:0:1:16 sdcs 70:0   active undef running
2130 `-+- policy='round-robin 0' prio=0 status=enabled
2131   |- 1:0:1:16 sds  65:32  active undef running
2132   `- 3:0:1:16 sdbs 68:96  active undef running
2133 mpathh (36006016067702d0068238084685ae211) dm-4 DGC,RAID 5
2134 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2135 |-+- policy='round-robin 0' prio=0 status=active
2136 | |- 1:0:1:4  sdag 66:0   active undef running
2137 | `- 3:0:1:4  sdbg 67:160 active undef running
2138 `-+- policy='round-robin 0' prio=0 status=enabled
2139   |- 2:0:0:4  sdg  8:96   active undef running
2140   `- 4:0:1:4  sdcg 69:64  active undef running
2141 mpatht (3600601600cd03100d6ef6cdfb034e311) dm-16 DGC,RAID 5
2142 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2143 |-+- policy='round-robin 0' prio=0 status=active
2144 | |- 2:0:0:18 sdu  65:64  active undef running
2145 | `- 4:0:1:18 sdcu 70:32  active undef running
2146 `-+- policy='round-robin 0' prio=0 status=enabled
2147   |- 1:0:1:18 sdau 66:224 active undef running
2148   `- 3:0:1:18 sdbu 68:128 active undef running
2149 mpathg (36006016067702d00607ac85b685ae211) dm-7 DGC,RAID 5
2150 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2151 |-+- policy='round-robin 0' prio=0 status=active
2152 | |- 2:0:0:3  sdf  8:80   active undef running
2153 | `- 4:0:1:3  sdcf 69:48  active undef running
2154 `-+- policy='round-robin 0' prio=0 status=enabled
2155   |- 1:0:1:3  sdaf 65:240 active undef running
2156   `- 3:0:1:3  sdbf 67:144 active undef running
2157 mpaths (36006016067702d007222c7d8695ae211) dm-15 DGC,RAID 5
2158 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2159 |-+- policy='round-robin 0' prio=0 status=active
2160 | |- 2:0:0:15 sdr  65:16  active undef running
2161 | `- 4:0:1:15 sdcr 69:240 active undef running
2162 `-+- policy='round-robin 0' prio=0 status=enabled
2163   |- 1:0:1:15 sdar 66:176 active undef running
2164   `- 3:0:1:15 sdbr 68:80  active undef running
2165 mpathf (36006016067702d00fc530441685ae211) dm-3 DGC,RAID 5
2166 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2167 |-+- policy='round-robin 0' prio=0 status=active
2168 | |- 1:0:1:2  sdae 65:224 active undef running
2169 | `- 3:0:1:2  sdbe 67:128 active undef running
2170 `-+- policy='round-robin 0' prio=0 status=enabled
2171   |- 2:0:0:2  sde  8:64   active undef running
2172   `- 4:0:1:2  sdce 69:32  active undef running
2173 <<<md>>>
2174 Personalities : [raid6] [raid5] [raid4] [raid1] [raid10]
2175 unused devices: <none>
2176 <<<diskstat>>>
2177 1416329024
2178    8      32 sdc 180321 12 1442664 1251715 0 0 0 0 0 1251539 1251539
2179    8      64 sde 180756 12 1446144 1041714 0 0 0 0 0 1041539 1041539
2180    8      96 sdg 180510 12 1444176 1107684 0 0 0 0 0 1107504 1107504
2181    8     144 sdj 169783423 12 43843540411 632255890 33607604 0 6267004625 272659047 0 419839234 904894095
2182    8     128 sdi 180646 12 1445264 1256029 0 0 0 0 0 1255842 1255842
2183    8      80 sdf 174080499 12 44161773254 876550093 33367613 0 6226195383 273863266 0 515985152 1150393431
2184    8     192 sdm 181055 12 1448536 476885 0 0 0 0 0 476709 476710
2185    8      16 sdb 181462 0 1451744 165494 0 0 0 0 0 165300 165300
2186    8     176 sdl 174430992 12 44043387762 474107972 33729801 0 6238082381 269983859 0 371355442 744068918
2187    8     160 sdk 180924 12 1447488 496736 0 0 0 0 0 496561 496561
2188    8     112 sdh 168369724 12 43780869363 739540124 34057534 0 6274171212 271958724 0 485493467 1011496108
2189    8     240 sdp 179081916 12 44237537945 479923725 34337443 0 6262719427 259427653 0 375627878 739328253
2190   65      16 sdr 172927164 12 43985653571 478338619 33530107 0 6260343800 265087257 0 373386477 743402893
2191    8       0 sda 763833 257293 20424501 1287709 93635159 72414348 1329396911 54406558 0 19533773 55678513
2192   65      64 sdu 172005076 12 43879459000 910813851 33484118 0 6271441545 265358933 0 529413152 1176144435
2193   65      48 sdt 182792 0 1462384 85775 0 0 0 0 0 85585 85585
2194   65       0 sdq 180988 12 1448000 489333 0 0 0 0 0 489149 489149
2195   65     128 sdy 181078 12 1448720 733531 0 0 0 0 0 733360 733360
2196   65     160 sdaa 182901 12 1463304 186498 0 0 0 0 0 186301 186301
2197    8     208 sdn 165811096 12 43489751870 463351380 33253088 0 6240545446 264381349 0 364249884 727730789
2198   65      96 sdw 180766 12 1446224 809284 0 0 0 0 0 809122 809122
2199   65     112 sdx 176288987 12 44186946417 835657359 33809994 0 6305896725 274596304 0 490712014 1110230675
2200   65      80 sdv 170318962 12 43977833942 944226323 33124746 0 6267302867 271187425 1 540600702 1215395619
2201   65     144 sdz 29327803 12 19834748713 94500294 70967487 0 37480100425 204162893 0 151405638 298652879
2202    8     224 sdo 181023 12 1448280 490418 0 0 0 0 0 490256 490256
2203   65     224 sdae 186309648 12 44878658934 885812511 33130641 0 6216997054 209754505 1 516642113 1095557121
2204   65     192 sdac 177995876 12 44049002280 883203293 33725709 0 6272541739 207590675 0 521909280 1090774650
2205   65     240 sdaf 180295 12 1442456 1045278 0 0 0 0 0 1045113 1045113
2206   65     208 sdad 179668 12 1437440 1338595 0 0 0 0 0 1338405 1338405
2207   66      16 sdah 180383 12 1443160 1102987 0 0 0 0 0 1102829 1102829
2208   66       0 sdag 170872794 12 43784837168 894290613 33545944 0 6246324421 210908020 0 521557389 1105174735
2209   66      32 sdai 181504782 12 44363628577 702399830 34081997 0 6305912018 204797792 0 474274130 907174012
2210   65     176 sdab 247172 0 37664418 350507 79274383 0 4091088125 183615953 0 18743294 183962880
2211   66      48 sdaj 180787 12 1446392 661241 0 0 0 0 0 661074 661075
2212   66      64 sdak 174245864 12 44036523847 447450546 33295568 0 6232289036 199808108 0 358725196 647235404
2213   66      80 sdal 180914 12 1447408 442235 0 0 0 0 0 442088 442088
2214   66      96 sdam 175041996 12 44170381939 446980535 33904064 0 6300198883 202889881 0 359058644 649849681
2215   66     112 sdan 180801 12 1446504 446759 0 0 0 0 0 446585 446585
2216   66     128 sdao 175192675 12 44022223946 452434848 33429281 0 6238432664 196552202 0 360931362 648967634
2217   66     160 sdaq 170071547 12 44008413266 443712383 33245032 0 6243941408 196678855 0 357874991 640365722
2218   66     144 sdap 180925 12 1447496 448385 0 0 0 0 0 448202 448202
2219   66     176 sdar 180962 12 1447792 447692 0 0 0 0 0 447521 447521
2220   66     208 sdat 11178914 0 1640585660 6508172 82750574 0 4075968015 183318258 0 20202802 189821198
2221   66     224 sdau 180608 12 1444960 828058 0 0 0 0 0 827863 827863
2222   66     240 sdav 180630 12 1445136 759952 0 0 0 0 0 759789 759789
2223   67      16 sdax 180698 12 1445680 754536 0 0 0 0 0 754381 754381
2224   67      32 sday 194834036 12 44230490997 1012668689 46147716 0 6579113969 219466311 0 503675456 1232105499
2225   67      48 sdaz 182986 12 1463984 171884 0 0 0 0 0 171680 171680
2226   67      64 sdba 29430660 12 19837422627 75988180 71865148 0 37489089073 205210368 0 146192791 281186694
2227   67      96 sdbc 177985736 12 44048994151 870445301 33735830 0 6277865722 202870095 0 516959883 1073296447
2228   67     160 sdbg 170861314 12 43783159047 882462595 33557397 0 6250557844 205498363 0 517056458 1087934877
2229   67     128 sdbe 186299662 12 44879578681 873038598 33140610 0 6222094799 204080767 0 512156350 1077111943
2230   67     112 sdbd 179674 12 1437488 1199756 0 0 0 0 0 1199607 1199607
2231   67     176 sdbh 180443 12 1443640 1165030 0 0 0 0 0 1164887 1164887
2232   67     144 sdbf 180334 12 1442768 1054992 0 0 0 0 0 1054844 1054844
2233   67       0 sdaw 172237941 12 44028533511 856018770 34063767 0 6277722377 209140501 0 502708439 1065138181
2234   67     192 sdbi 181487264 12 44359849697 692539799 34099488 0 6310822859 200192713 0 470207303 892710232
2235   67     208 sdbj 180820 12 1446656 658080 0 0 0 0 0 657912 657912
2236   68       0 sdbm 175030539 12 44165002768 440432812 33915514 0 6305875984 198048850 0 355051155 638462305
2237   68      16 sdbn 180842 12 1446832 445901 0 0 0 0 0 445737 445737
2238   67     240 sdbl 180959 12 1447768 440393 0 0 0 0 0 440250 440250
2239   68      48 sdbp 180971 12 1447864 427733 0 0 0 0 0 427571 427572
2240   68      32 sdbo 175188646 12 44022836016 444765923 33433320 0 6241705141 191938838 0 356473225 636685224
2241   68      80 sdbr 180987 12 1447992 435237 0 0 0 0 0 435084 435084
2242   67     224 sdbk 174242489 12 44035838106 440562259 33298959 0 6237545717 195136228 0 354761607 635676055
2243   67      80 sdbb 181695 0 1453608 151168 0 0 0 0 0 151011 151011
2244    8      48 sdd 170164487 12 43977422611 960984288 33095410 0 6270416349 274433282 0 547728889 1235392096
2245   68     128 sdbu 180635 12 1445176 816342 0 0 0 0 0 816157 816157
2246   68     112 sdbt 11180535 0 1641029565 6213475 82748952 0 4085426928 170733161 0 19624286 176941541
2247   68     160 sdbw 172226000 12 44026020943 843800787 34075671 0 6282424561 204058656 0 498054068 1047841683
2248   68     176 sdbx 180712 12 1445792 739235 0 0 0 0 0 739064 739064
2249   68     144 sdbv 180652 12 1445312 762070 0 0 0 0 0 761917 761917
2250   68     224 sdca 29434530 12 19847199942 74974270 71861288 0 37458821939 198452110 0 143345319 273418621
2251   68     208 sdbz 183017 12 1464232 167450 0 0 0 0 0 167281 167281
2252   68     192 sdby 194831142 12 44231144674 998352237 46150568 0 6580853462 214406449 0 498958973 1212736143
2253   68     240 sdcb 246959 0 38282366 346591 79274503 0 4070306818 197825372 0 18800649 198167887
2254   68      64 sdbq 170052822 12 44003342545 437435356 33263765 0 6252193096 192273318 0 353842472 629689046
2255   69       0 sdcc 180393 12 1443240 1191260 0 0 0 0 0 1191085 1191085
2256   69      16 sdcd 170155375 12 43977233214 954615083 33104521 0 6272932550 269379914 0 544438833 1223974580
2257   69      48 sdcf 174074108 12 44162468086 871606773 33374045 0 6228157343 268761461 0 512858811 1140349762
2258   69      64 sdcg 180582 12 1444752 1039199 0 0 0 0 0 1039018 1039018
2259   69      80 sdch 168364891 12 43781635102 733940875 34062438 0 6276678783 267117653 0 482424460 1001040414
2260   69      96 sdci 180742 12 1446032 1461326 0 0 0 0 0 1461153 1461153
2261   69     112 sdcj 169773690 12 43844470437 627177003 33617434 0 6272108509 267741754 0 416887613 894898535
2262   69     128 sdck 181022 12 1448272 397470 0 0 0 0 0 397318 397318
2263   69     144 sdcl 174425550 12 44043877357 470651620 33735361 0 6244608112 265091242 0 368338790 735731711
2264   69     160 sdcm 181179 12 1449528 399820 0 0 0 0 0 399670 399670
2265   69     176 sdcn 165805336 12 43489139065 459852314 33258939 0 6243539101 259542207 0 361118394 719391696
2266   69     208 sdcp 179075893 12 44235613446 476218679 34343594 0 6266874925 255091995 0 372410513 731286674
2267   69     224 sdcq 181077 12 1448712 388206 0 0 0 0 0 388040 388040
2268   69     240 sdcr 172925679 12 43986766379 474740104 33531707 0 6264954943 260378041 0 370276347 735093414
2269   70      16 sdct 182949 0 1463616 75280 0 0 0 0 0 75092 75092
2270   70      32 sdcu 172000515 12 43881551438 905360852 33488708 0 6274106221 260708386 0 525817139 1166049486
2271   70      48 sdcv 170317533 12 43980327184 937140947 33126218 0 6270338043 266645186 1 536720565 1203765605
2272   69     192 sdco 181121 12 1449064 420705 0 0 0 0 0 420569 420569
2273   70      64 sdcw 180810 12 1446576 737315 0 0 0 0 0 737144 737144
2274   70      80 sdcx 176285484 12 44187992047 830577294 33813549 0 6306223350 269392964 0 487508723 1099948790
2275   70      96 sdcy 181139 12 1449208 661518 0 0 0 0 0 661371 661371
2276   70     112 sdcz 29333492 12 19847258821 93679318 70961810 0 37452505355 199080400 0 148342124 292747890
2277   70     128 sdda 183031 12 1464344 172563 0 0 0 0 0 172380 172380
2278   69      32 sdce 180899 12 1447288 1018988 0 0 0 0 0 1018827 1018827
2279  253       0 dm-0 6993 0 184474 9111 1983314 0 15866512 917081 0 398390 926192
2280  253       1 dm-1 463 0 3704 663 0 0 0 0 0 504 663
2281  253       2 dm-2 355620743 181965 88095109287 1759320629 67461539 3586607 12550407461 530974294 0 685031846 2290275532
2282  253       3 dm-3 372247534 436932 89755343215 1766692941 66271251 3544279 12439091853 527944431 1 675870267 2294673597
2283  253       4 dm-4 341372887 289582 87565106255 1781914829 67103341 3541184 12496882265 546852230 0 682399766 2328802843
2284  253       5 dm-5 339195464 232151 87685117464 1266056490 67225038 3783221 12539113134 727059471 0 557041987 1993461010
2285  253       6 dm-6 362630521 188683 88720585882 1399068217 68181485 3622520 12616734877 516187962 0 629261336 1915204532
2286  253       7 dm-7 347793899 175304 88321355484 1755141077 66741658 3757966 12454352726 722314944 0 673325478 2477635984
2287  253       8 dm-8 349710138 189053 88332485339 893322661 67819578 3597465 12606074867 511580436 0 486480633 1404828342
2288  253       9 dm-9 129834 78285 73032240 423928 158548886 741875294 8161394943 408946368 0 23830227 409308176
2289  253      10 dm-10 348494628 170780 88084369615 951086974 67465162 3725637 12482690493 722886661 0 500387744 1674151941
2290  253      11 dm-11 348126225 197713 88069464737 894205421 66594527 3531631 12469834753 494562916 0 485887657 1388688948
2291  253      12 dm-12 336373719 258381 87559617105 1479076747 68119972 3785392 12550849995 711168313 0 640129098 2190434308
2292  253      13 dm-13 357795868 318210 88470255671 962221766 68681037 3708504 12529594352 658825438 0 507403974 1621115021
2293  253      15 dm-15 345490863 351487 87969523918 958962221 67061814 3784906 12525298743 700214690 0 503183769 1659323009
2294  253      16 dm-16 343644279 385533 87758119750 1825583470 66972826 3789906 12545547766 686023443 0 692735006 2511833548
2295  253      17 dm-17 21993029 4900505 3278683697 12928618 165499526 734924775 8161394943 384910692 0 27793966 397757231
2296  253      18 dm-18 339762144 337181 88008857819 886732799 66508797 3612637 12496134504 489654118 0 484152024 1376430941
2297  253      19 dm-19 389302830 222475 88458736695 2071186275 92298284 3552796 13159967431 561316323 0 668125162 2632451255
2298  253      20 dm-20 58498764 29179 39681690969 152895143 143726436 72262 74947911012 413516717 0 216996879 566145937
2299  253      21 dm-21 331254727 170920 86975997103 928940402 66512027 3743173 12484084547 692613143 0 491301422 1621766524
2300  253      22 dm-22 344102290 184997 88051661054 1706014307 68139438 3614308 12560146938 539611589 0 660738746 2245602928
2301  253      23 dm-23 352213003 214942 88372046528 1673950329 67623543 3853219 12612120075 737318454 0 645355899 2411542268
2302  253      24 dm-24 340275155 191741 87955270438 1890414127 66250964 3757863 12537640910 715552595 0 705992255 2606116574
2303  253      25 dm-25 58295032 29854 39679077238 189811590 141929297 73756 74932605780 416742838 0 220730140 606290020
2304  253      26 dm-26 350019080 169919 88042161842 902756087 66862601 3589118 12480137805 484464019 0 488945127 1387255496
2305  253      27 dm-27 339960452 402252 87951780353 1924302229 66199931 3786607 12543348899 730221212 0 713953512 2654743738
2306  253      58 dm-58 720 0 8426 1999 25091973 0 200735784 17886301 0 6577787 17915588
2307  253      59 dm-59 4585 0 189858 5496 6154916 0 49239328 69346841 0 1272814 69368241
2308  253      60 dm-60 29663 0 789962 38937 145652 0 1165216 1182859 0 31882 1221797
2309  253      61 dm-61 2483 0 41106 3499 1956 0 15648 1147 0 1740 4646
2310  253      62 dm-62 21884 0 615442 256846 93364571 0 746916568 44572422 0 3956584 44963459
2311  253      63 dm-63 2209 0 105266 3958 1433474 0 11467792 1691185 0 1659598 1695145
2312  253      64 dm-64 718300 0 10951069 1036294 13592348 0 108736055 10488518 0 1655581 11530820
2313  253      65 dm-65 180487 0 4235138 253796 9522619 0 76180952 5364362 0 1668153 5620801
2314  253      66 dm-66 18539 0 148306 22529 12763 0 102104 17661 0 2445 40189
2315  253      67 dm-67 20260 0 2865066 120881 4234727 0 33877816 202788643 0 251795 202954806
2316  253      68 dm-68 14328 0 275370 16886 10636477 0 85091816 12990462 0 2123205 13012289
2317  253      33 dm-33 2 0 2 1 2967567 0 19943214 3147629 0 2894996 3148970
2318  253      34 dm-34 260316 0 129544458 915203 383884750 0 3084082800 1982485129 0 5735608 1986345045
2319  253      35 dm-35 185 0 1440 113 2967567 0 19943214 4422825 0 3979685 4424014
2320  253      36 dm-36 17520 0 4984056 235445 383884750 0 3084082800 1610406883 0 5364824 1614013947
2321  253      37 dm-37 277836 0 134528514 1151916 383884750 0 3084082800 1397670470 0 8330660 1408574433
2322  253      38 dm-38 1 0 1 0 3157984 0 20801070 2919595 0 2842159 2919793
2323  253      39 dm-39 104873 0 50389434 271337 453554156 0 3628433248 1014232621 0 9291791 1016214459
2324  253      40 dm-40 18 0 128 20 3157984 0 20801070 4085395 0 3975849 4085581
2325  253      41 dm-41 28217 0 14144840 106309 453554156 0 3628433248 1488862180 0 11578686 1490928857
2326  253      42 dm-42 133090 0 64534274 378094 453554156 0 3628433248 3425594752 0 14281221 3432140713
2327  253      43 dm-43 1 0 1 0 2606 0 2606 1010 0 996 1010
2328  253      44 dm-44 364 0 10226 671 1167 0 9336 595 0 897 1266
2329  253      45 dm-45 3 0 3 2 2606 0 2606 3198 0 3161 3200
2330  253      46 dm-46 9 0 72 53 1167 0 9336 1191 0 658 1244
2331  253      47 dm-47 373 0 10298 724 1167 0 9336 4128 0 3343 4857
2332  253      48 dm-48 1 0 1 0 739701 0 5357469 406983 0 400044 406995
2333  253      49 dm-49 26485153 0 3093436614 19432378 20956882 0 1121648960 29607242 0 9955477 49070920
2334  253      50 dm-50 6 0 33 2 739701 0 5357469 827032 0 810277 827051
2335  253      51 dm-51 152139 0 52525212 305231 20956882 0 1121648960 45509459 0 2504061 45842266
2336  253      52 dm-52 24437304 0 3145961826 16458127 20193935 0 1121648960 70742718 0 11637871 87245526
2337  253      53 dm-53 1 0 1 0 157723 0 1096362 210158 0 204798 210189
2338  253      54 dm-54 41912 0 5292578 107227 35002449 0 280019592 106698529 0 467307 107031318
2339  253      55 dm-55 6 0 28 9 157723 0 1096362 298267 0 287825 298318
2340  253      56 dm-56 9385 0 1369448 55267 35002449 0 280019592 131355158 0 465576 131636552
2341  253      57 dm-57 51297 0 6662026 162742 35002449 0 280019592 404252302 0 638924 405270113
2342   65      32 sds 60166 0 481376 360919 0 0 0 0 0 360739 360841
2343  253      14 dm-14 129396173 83059 27104014468 680283380 34011478 1140542 5943777444 750978405 0 210345105 1431327505
2344   66     192 sdas 64759279 0 13552125660 325631056 17004589 0 2972152776 245107384 0 159785149 570730478
2345   68      96 sdbs 59991 0 479928 319839 0 0 0 0 0 319792 319793
2346   70       0 sdcs 64756720 0 13552847368 323976097 17006889 0 2971624668 240668581 0 158802441 564637339
2347 [dmsetup_info]
2348 mpathr 253:18
2349 dwhvg-lv_dingdata_rimage_0 253:39 dwhvg lv_dingdata_rimage_0
2350 mpathe 253:27
2351 dwhvg-lv_backup_rimage_1 253:51 dwhvg lv_backup_rimage_1
2352 rootvg-lv_swap 253:1 rootvg lv_swap
2353 rootvg-lv_usr 253:60 rootvg lv_usr
2354 rootvg-lv_root 253:0 rootvg lv_root
2355 rootvg-lv_var 253:62 rootvg lv_var
2356 mpathq 253:13
2357 mpathd 253:2
2358 dwhvg-lv_backup_rimage_0 253:49 dwhvg lv_backup_rimage_0
2359 dwhvg-lv_conndir 253:47 dwhvg lv_conndir
2360 mpathp 253:26
2361 mpathc 253:9
2362 mpatho 253:21
2363 mpathb 253:17
2364 rootvg-lv_install 253:66 rootvg lv_install
2365 rootvg-lv_local 253:61 rootvg lv_local
2366 mpathn 253:8
2367 dwhvg-lv_backup 253:52 dwhvg lv_backup
2368 mpathz 253:20
2369 mpathm 253:10
2370 dwhvg-lv_opt_uniserv 253:57 dwhvg lv_opt_uniserv
2371 rootvg-lv_opt 253:63 rootvg lv_opt
2372 mpathy 253:25
2373 mpathl 253:11
2374 rootvg-lv_oracle11 253:64 rootvg lv_oracle11
2375 dwhvg-lv_backup_rmeta_1 253:50 dwhvg lv_backup_rmeta_1
2376 mpathx 253:19
2377 mpathk 253:5
2378 dwhvg-lv_opt_uniserv_rmeta_1 253:55 dwhvg lv_opt_uniserv_rmeta_1
2379 dwhvg-lv_dingdata 253:42 dwhvg lv_dingdata
2380 dwhvg-lv_backup_rmeta_0 253:48 dwhvg lv_backup_rmeta_0
2381 mpathw 253:23
2382 mpathj 253:6
2383 dwhvg-lv_opt_uniserv_rmeta_0 253:53 dwhvg lv_opt_uniserv_rmeta_0
2384 mpathv 253:22
2385 dwhvg-lv_conndir_rimage_1 253:46 dwhvg lv_conndir_rimage_1
2386 mpathi 253:12
2387 dwhvg-lv_opt_uniserv_rimage_1 253:56 dwhvg lv_opt_uniserv_rimage_1
2388 dwhvg-lv_dingdata_rmeta_1 253:40 dwhvg lv_dingdata_rmeta_1
2389 mpathu 253:24
2390 mpathaa 253:14
2391 dwhvg-lv_conndir_rimage_0 253:44 dwhvg lv_conndir_rimage_0
2392 rootvg-lv_home 253:59 rootvg lv_home
2393 mpathh 253:4
2394 dwhvg-lv_opt_uniserv_rimage_0 253:54 dwhvg lv_opt_uniserv_rimage_0
2395 dwhvg-lv_conndir_rmeta_1 253:45 dwhvg lv_conndir_rmeta_1
2396 rootvg-lv_tmp 253:58 rootvg lv_tmp
2397 dwhvg-lv_dingdata_rmeta_0 253:38 dwhvg lv_dingdata_rmeta_0
2398 rootvg-lv_agent 253:68 rootvg lv_agent
2399 mpatht 253:16
2400 rootvg-lv_oragrid 253:65 rootvg lv_oragrid
2401 mpathg 253:7
2402 dwhvg-lv_conndir_rmeta_0 253:43 dwhvg lv_conndir_rmeta_0
2403 mpaths 253:15
2404 dwhvg-lv_dingdata_rimage_1 253:41 dwhvg lv_dingdata_rimage_1
2405 mpathf 253:3
2406 <<<md>>>
2407 Personalities :
2408 unused devices: <none>
2409 <<<ntp>>>
2410 * 172.33.44.49    29.244.104.45    2 u    2   64  377    0.302   -0.353   0.483
2411 + 172.44.33.19     29.244.104.33    2 u   18   64  377    0.408    0.051   0.319
2412 <<<postfix_mailq>>>
2413 QUEUE_deferred 4 0
2414 QUEUE_active 4 0
2415 <<<postfix_mailq>>>
2416 /var/spool/mqueue is empty
2417                 Total requests: 0
2418 <<<lnx_thermal>>>
2419 thermal_zone0 - x86_pkg_temp 59000 0 passive 0 passive
2420 thermal_zone1 - BAT0 35000
2421 <<<logwatch>>>
2422 [[[/var/log/messages:missing]]]
2423 [[[/var/log/kern.log]]]
2424 [[[/var/log/auth.log]]]
2425 [[[/var/log/syslog]]]
2426 """
2427
2428 #.
2429 #   .--Test Cases----------------------------------------------------------.
2430 #   |            _____         _      ____                                 |
2431 #   |           |_   _|__  ___| |_   / ___|__ _ ___  ___  ___              |
2432 #   |             | |/ _ \/ __| __| | |   / _` / __|/ _ \/ __|             |
2433 #   |             | |  __/\__ \ |_  | |__| (_| \__ \  __/\__ \             |
2434 #   |             |_|\___||___/\__|  \____\__,_|___/\___||___/             |
2435 #   |                                                                      |
2436 #   +----------------------------------------------------------------------+
2437 #   | The test cases implement different kinds of tests                    |
2438 #   '----------------------------------------------------------------------'
2439
2440
2441 class TestCase(object):
2442     STATE_INITIALIZING = "initializing"
2443     STATE_RUNNING = "running"
2444     STATE_STOPPED = "stopped"
2445
2446     RESULT_COMPLETED = "completed"
2447     RESULT_SKIPPED = "skipped"
2448     RESULT_ABORTED = "aborted"
2449     RESULT_FAILED = "failed"
2450
2451     def __init__(self):
2452         self.started_at = None
2453         self.finished_at = None
2454         self.state = None
2455         self.result = None
2456         self.result_msg = None
2457
2458     # Return a meaningful name to identify this test case
2459     def name(self):
2460         return "%s()" % self.__class__.__name__
2461
2462     # The run() method can be implemented as generator and yield several intermediate steps
2463     # to the Test() object. In case a Wait(duration) object is passed, the Test() object will
2464     # wait for the given duration and then call the run() method again.
2465     def run(self):
2466         raise NotImplementedError()
2467
2468     def duration(self):
2469         if self.started_at is None:
2470             return 0
2471         elif self.finished_at is None:
2472             return time.time() - self.started_at
2473         return self.finished_at - self.started_at
2474
2475     def serialize(self):
2476         d = dict([(k, v) for k, v in self.__dict__.items() if k[0] != "_"])
2477         d["__class_name__"] = self.__class__.__name__
2478         return d
2479
2480     def deserialize(self, raw):
2481         self.__dict__.update(raw)
2482
2483     def set_state(self, state, result=None, result_msg=None):
2484         self.state = state
2485         self.result = result
2486         self.result_msg = result_msg
2487
2488         if self.state == TestCase.STATE_RUNNING:
2489             self.started_at = time.time()
2490         elif self.state == TestCase.STATE_STOPPED:
2491             self.finished_at = time.time()
2492
2493     def state_text(self):
2494         return self.state
2495
2496
2497 class TestAgentBasedStandardHosts(TestCase):
2498     """A test that creates the specified number of "standard" hosts
2499
2500     It adds the hosts to Check_MK and starts the monitoring of these
2501     hosts. The hosts will be configured with a standard set of services
2502     which is assumed to be a good configuration for the average Check_MK
2503     agent based host.
2504
2505     The test will simulate an average network delay and process dynamic
2506     agent data that will lead to increasing counters in the checks.
2507     """
2508
2509     host_name_template = "std-agent-host-%05d"
2510
2511     def __init__(self, runner=None, num_hosts=None, planned_duration=None, num_cmk_helpers=20):
2512         super(TestAgentBasedStandardHosts, self).__init__()
2513         self._runner = runner
2514         self._test = runner.test if runner else None
2515
2516         self.num_hosts = num_hosts
2517         self.planned_duration = planned_duration
2518         self.num_cmk_helpers = num_cmk_helpers
2519
2520         self._listeners = []
2521         self._hosts_per_listener = 100
2522
2523     def name(self):
2524         return "%s(hosts=%d, duration=%d, helpers=%d)" % \
2525                    (self.__class__.__name__,
2526                     self.num_hosts,
2527                     self.planned_duration,
2528                     self.num_cmk_helpers)
2529
2530     def run(self):
2531         try:
2532             self._spawn_agent_listeners()
2533             self._write_global_settings()
2534             self._create_hosts()
2535             self._discover_services()
2536             self._activate_changes()
2537             self._create_rrds()
2538             self._wait_for_cmk_helpers_ready()
2539             yield Start()
2540             yield Wait(self.planned_duration)
2541         finally:
2542             self._cleanup_hosts()
2543             self._activate_changes()
2544             self._runner._cleanup_host_files()
2545             self._end_agent_listeners()
2546
2547     def _spawn_agent_listeners(self):
2548         self._test.info("  Starting agent listeners")
2549         for first_host_id in range(0, self.num_hosts, self._hosts_per_listener):
2550             host_ids = range(first_host_id, first_host_id + self._hosts_per_listener)
2551             listener = AgentListener(host_ids)
2552             listener.start()
2553             self._listeners.append(listener)
2554
2555     def _end_agent_listeners(self):
2556         self._test.info("  Stopping agent listeners")
2557         for listener in self._listeners:
2558             listener.terminate()
2559
2560         for listener in self._listeners:
2561             listener.join()
2562         self._test.info("  Stopped listeners")
2563
2564     def _write_global_settings(self):
2565         self._test.info("  Configuring site")
2566         config_dir = self._cmk_config_dir()
2567         try:
2568             os.makedirs(config_dir)
2569         except OSError, e:
2570             if e.errno == 17:  # file exists
2571                 pass
2572             else:
2573                 raise
2574
2575         with open("%s/global_settings.mk" % config_dir, "w") as f:
2576             f.write("# Created by mkbench\n"
2577                     "cmc_cmk_helpers = %d\n"
2578                     "cmc_log_limit = 209715200\n"
2579                     "cmc_initial_scheduling = {\n"
2580                     "    'burst': 100,\n"
2581                     "    'spread_cmk': 10,\n"
2582                     "    'spread_generic': 150\n"
2583                     "}\n"
2584                     "\n" % self.num_cmk_helpers)
2585
2586         with open("%s/rules.mk" % config_dir, "w") as f:
2587             f.write("agent_ports = [\n" "    ( 6559, [], ALL_HOSTS, {} ),\n" "] + agent_ports\n")
2588
2589             f.write("tcp_connect_timeouts = [\n"
2590                     "    ( 5.0, [], ALL_HOSTS, {} ),\n"
2591                     "] + tcp_connect_timeouts\n")
2592
2593             # Ensure RRDs are created with Check_MK default config
2594             f.write(
2595                 "cmc_host_rrd_config = [\n"
2596                 "      ({'rras': [\n"
2597                 "          (50.0, 1, 2880), (50.0, 5, 2880),\n"
2598                 "          (50.0, 30, 4320), (50.0, 360, 5840)\n"
2599                 "         ],\n"
2600                 "         'step': 60,\n"
2601                 "         'cfs': ['MIN', 'MAX', 'AVERAGE'],\n"
2602                 "         'format': 'cmc_single'\n"
2603                 "      },\n"
2604                 "      [], ALL_HOSTS, {\n"
2605                 "       'description': u'Default RRD configuration, using new single RRD format'\n"
2606                 "      }),\n"
2607                 "] + cmc_host_rrd_config\n"
2608                 "\n"
2609                 "\n"
2610                 "cmc_service_rrd_config = [\n"
2611                 "   ( {'rras': [\n"
2612                 "       (50.0, 1, 2880), (50.0, 5, 2880),\n"
2613                 "       (50.0, 30, 4320), (50.0, 360, 5840)\n"
2614                 "      ],\n"
2615                 "      'step': 60,\n"
2616                 "      'cfs': ['MIN', 'MAX', 'AVERAGE'],\n"
2617                 "      'format': 'cmc_single'\n"
2618                 "   },\n"
2619                 "   [], ALL_HOSTS, ALL_SERVICES ),\n"
2620                 "] + cmc_service_rrd_config\n")
2621
2622     def _create_hosts(self):
2623         self._test.info("  Creating hosts")
2624         config_dir = self._cmk_config_dir()
2625         try:
2626             os.makedirs(config_dir)
2627         except OSError, e:
2628             if e.errno == 17:  # file exists
2629                 pass
2630             else:
2631                 raise
2632
2633         all_hosts, ipaddresses, host_attributes = self._create_host_data()
2634
2635         with open("%s/hosts.mk" % config_dir, "w") as f:
2636             f.write("# Created by mkbench\n"
2637                     "\n"
2638                     "all_hosts += %s\n"
2639                     "\n"
2640                     "ipaddresses.update(%s)\n"
2641                     "\n"
2642                     "host_attributes.update(%s)\n" % (pprint.pformat(all_hosts),
2643                                                       pprint.pformat(ipaddresses),
2644                                                       pprint.pformat(host_attributes)))
2645
2646         with open("%s/rules.mk" % config_dir, "w") as f:
2647             f.write("agent_ports = [\n" "    ( 6559, [], ALL_HOSTS, {} ),\n" "] + agent_ports\n")
2648
2649             f.write("tcp_connect_timeouts = [\n"
2650                     "    ( 5.0, [], ALL_HOSTS, {} ),\n"
2651                     "] + tcp_connect_timeouts\n")
2652
2653     def _create_host_data(self):
2654         all_hosts, ipaddresses, host_attributes = [], {}, {}
2655
2656         for num in range(self.num_hosts):
2657             host_name = self.host_name_template % num
2658             ipaddress = get_host_ip(num)
2659
2660             all_hosts.append(
2661                 "%s|lan|ip-v4|cmk-agent|tcp|site:%s|ip-v4-only|prod|bench" % (host_name, site_id()))
2662
2663             ipaddresses[host_name] = ipaddress
2664
2665             host_attributes[host_name] = {
2666                 "ipaddress": ipaddress,
2667             }
2668
2669         return all_hosts, ipaddresses, host_attributes
2670
2671     def _discover_services(self):
2672         self._test.info("  Discovering services")
2673
2674         # Discver first host and copy the rest (all hosts are equal)
2675         first_host_name = self.host_name_template % 0
2676         self._discover_services_of_host(first_host_name)
2677
2678         autochecks_dir = "%s/var/check_mk/autochecks" % omd_root()
2679
2680         src_file = "%s/%s.mk" % (autochecks_dir, first_host_name)
2681         for num in range(1, self.num_hosts):
2682             host_name = self.host_name_template % num
2683             shutil.copy(src_file, "%s/%s.mk" % (autochecks_dir, host_name))
2684
2685     def _discover_services_of_host(self, host_name):
2686         p = subprocess.Popen(["cmk", "-II", host_name],
2687                              stdin=open(os.devnull),
2688                              close_fds=True,
2689                              stdout=subprocess.PIPE,
2690                              stderr=subprocess.STDOUT)
2691
2692         try:
2693             stdout = p.communicate()[0]
2694         except KeyboardInterrupt:
2695             try:
2696                 p.terminate()
2697             except OSError:
2698                 pass
2699             raise
2700
2701         if p.returncode != 0:
2702             raise MKGeneralException("'cmk -II %s' failed: %s" % (host_name, stdout))
2703
2704     #def _discover_services(self):
2705     #    self._test.info("  Discovering services")
2706     #    p = subprocess.Popen(["cmk", "-II"], stdin=open(os.devnull),
2707     #            close_fds=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
2708
2709     #    try:
2710     #        stdout = p.communicate()[0]
2711     #    except KeyboardInterrupt:
2712     #        try:
2713     #            p.terminate()
2714     #        except OSError:
2715     #            pass
2716     #        raise
2717
2718     #    if p.returncode != 0:
2719     #        raise MKGeneralException("'cmk -II' failed: %s" % stdout)
2720
2721     def _create_rrds(self):
2722         try:
2723             # The execution of service checks needs to be stopped to prevent
2724             # races with the core.
2725             LocalConnection().command("[%d] STOP_EXECUTING_SVC_CHECKS" % time.time())
2726
2727             self._test.info("  Creating RRDs (stopping core during this step)")
2728             first_host_name = self.host_name_template % 0
2729             self._create_rrds_of_host(first_host_name)
2730             self._copy_rrds_of(first_host_name)
2731             self._test.info("  Finished creating RRDs")
2732         finally:
2733             LocalConnection().command("[%d] START_EXECUTING_SVC_CHECKS" % time.time())
2734
2735     def _copy_rrds_of(self, first_host_name):
2736         rrd_dir = "%s/var/check_mk/rrd" % omd_root()
2737
2738         try:
2739             # We have to stop and start the CMC during manual RRD copy.
2740             # a) No races with the core
2741             # b) Cached RRD info held by the core during runtime
2742             subprocess.call(["omd", "stop", "cmc"], stdout=open(os.devnull, "w"), close_fds=True)
2743
2744             src_dir = "%s/%s" % (rrd_dir, first_host_name)
2745             for num in range(1, self.num_hosts):
2746                 if self._runner.shall_stop():
2747                     break
2748
2749                 if num % 100 == 0:
2750                     self._test.info("  Created RRDs for %d hosts" % num)
2751
2752                 host_name = self.host_name_template % num
2753                 target_dir = "%s/%s" % (rrd_dir, host_name)
2754
2755                 try:
2756                     shutil.copytree(src_dir, target_dir)
2757                 except OSError, e:
2758                     if e.errno == 17:  # file exists
2759                         pass
2760                     else:
2761                         raise
2762
2763                 # Replace the host name in the just copied info files
2764                 for f in glob.glob("%s/%s/*.info" % (rrd_dir, host_name)):
2765                     new_content = open(f).readlines()
2766                     for index, line in enumerate(new_content):
2767                         if line.startswith("HOST "):
2768                             new_content[index] = "HOST %s\n" % host_name
2769
2770                     open(f, "w").write("".join(new_content))
2771         finally:
2772             self._activate_changes()
2773
2774     # To create the RRDs, simply perform one check and wait for the RRDs to be created
2775     def _create_rrds_of_host(self, host_name):
2776         # Do 2 iterations to initialize the rrds that base on counter values
2777         for _i in xrange(2):
2778             p = subprocess.Popen(["cmk", "-v", host_name],
2779                                  stdin=open(os.devnull),
2780                                  close_fds=True,
2781                                  stdout=subprocess.PIPE,
2782                                  stderr=subprocess.STDOUT)
2783
2784             try:
2785                 stdout = p.communicate()[0]
2786             except KeyboardInterrupt:
2787                 try:
2788                     p.terminate()
2789                 except OSError:
2790                     pass
2791                 raise
2792
2793             if p.returncode != 0:
2794                 raise MKGeneralException("'cmk -v %s' failed: %s" % (host_name, stdout))
2795
2796             time.sleep(2)
2797
2798         num_rrds, wait_sec = None, 10
2799         while num_rrds is None or num_rrds < 30:
2800             if wait_sec == 0:
2801                 raise MKGeneralException("Did not create RRDs for '%s' "
2802                                          "after 10 seconds" % host_name)
2803             wait_sec -= 1
2804
2805             if self._runner.shall_stop():
2806                 break
2807             num_rrds = len(glob.glob("%s/var/check_mk/rrd/%s/*.rrd" % (omd_root(), host_name)))
2808             time.sleep(1)
2809         self._test.info("  Found %d RRDs per host" % num_rrds)
2810
2811     def _cmk_config_dir(self):
2812         return "%s/etc/check_mk/conf.d/wato/mkbench" % omd_root()
2813
2814     def _activate_changes(self):
2815         if self._test:
2816             self._test.info("  Activating changes")
2817         p = subprocess.Popen(["cmk", "-R"],
2818                              close_fds=True,
2819                              stdout=subprocess.PIPE,
2820                              stderr=subprocess.STDOUT)
2821
2822         try:
2823             stdout = p.communicate()[0]
2824         except KeyboardInterrupt:
2825             try:
2826                 p.terminate()
2827             except OSError:
2828                 pass
2829             raise
2830
2831         if p.returncode != 0:
2832             raise MKGeneralException("Failed to execute 'cmk -R': %s" % stdout)
2833
2834         # Wait for livestatus socket to be available
2835         while (self._runner and not self._runner.shall_stop()) \
2836               and not os.path.exists("%s/tmp/run/live" % omd_root()):
2837             time.sleep(1)
2838
2839     # To deal with the load peak while initializing the Check_MK helpers we
2840     # need to wait till all helpers are initialized. Since we have no one
2841     # to tell us the state of the helpers, we need to detect this somehow.
2842     # -> Find all helper processes and check the open file descriptors.
2843     def _wait_for_cmk_helpers_ready(self):
2844         num_not_ready, _loadavg_5_before = None, os.getloadavg()[1]
2845         while num_not_ready is None or num_not_ready > 0:
2846             if self._runner.shall_stop():
2847                 break
2848
2849             if num_not_ready is None:
2850                 self._test.info("  Waiting for helper initialization")
2851             else:
2852                 self._test.info(
2853                     "  Waiting for helper initialization (%d not ready)" % num_not_ready)
2854
2855             try:
2856                 matched_pids = subprocess.check_output(
2857                     ["pgrep", "-u",
2858                      site_id(), "-f", "python.*check_mk.py --keepalive$"]).split("\n")
2859             except subprocess.CalledProcessError, e:
2860                 self._test.error("  Found no helper yet (%s)" % e.returncode)
2861                 continue
2862
2863             num_not_ready = 0
2864             for p in matched_pids:
2865                 if p and not os.path.exists("/proc/%s/fd/3" % p):
2866                     num_not_ready += 1
2867             time.sleep(1)
2868
2869         self._test.info("  Finished helper initialization")
2870
2871     def _cleanup_hosts(self):
2872         if self._test:
2873             self._test.info("  Cleaning up test configs")
2874         for path in glob.glob("%s/*" % self._cmk_config_dir()):
2875             os.unlink(path)
2876
2877
2878 def run_cleanup_data():
2879     runner = TestRunner()
2880     test_case = TestAgentBasedStandardHosts()
2881     test_case._cleanup_hosts()
2882     test_case._activate_changes()
2883     runner._cleanup_host_files(cleanup_rrds=True)
2884     test_case._activate_changes()
2885
2886
2887 #.
2888 #   .--Reports-------------------------------------------------------------.
2889 #   |                 ____                       _                         |
2890 #   |                |  _ \ ___ _ __   ___  _ __| |_ ___                   |
2891 #   |                | |_) / _ \ '_ \ / _ \| '__| __/ __|                  |
2892 #   |                |  _ <  __/ |_) | (_) | |  | |_\__ \                  |
2893 #   |                |_| \_\___| .__/ \___/|_|   \__|___/                  |
2894 #   |                          |_|                                         |
2895 #   +----------------------------------------------------------------------+
2896 #   | Create some reports out of the test results                          |
2897 #   '----------------------------------------------------------------------'
2898
2899
2900 def run_report_list():
2901     for entry in sorted(os.listdir(Test.data_dir())):
2902         sys.stdout.write(entry + "\n")
2903
2904
2905 def run_report(test_id):
2906     if not Test.exists(test_id):
2907         raise MKGeneralException("The test '%s' does not exist." % test_id)
2908
2909     test = Test(test_id)
2910
2911     if test.state != Test.STATE_STOPPED:
2912         sys.stdout.write("WARNING: This test is not complete yet.\n")
2913         sys.stdout.write("\n")
2914
2915     sys.stdout.write("= SUMMARY ==================================\n")
2916     sys.stdout.write("\n")
2917     sys.stdout.write(" Test version : %s\n" % test.version)
2918     sys.stdout.write(" Date         : %s\n" % fmt_datetime(test.first_measurement().time))
2919     sys.stdout.write(" State        : %s\n" % test.state_text())
2920
2921     result_txt = test.result
2922     if test.result == "failed":
2923         result_txt += " (%s)" % test.result_msg
2924
2925     sys.stdout.write(" Result       : %s\n" % result_txt)
2926     sys.stdout.write(" Duration     : %s\n" % fmt_timespan(test.duration()))
2927     sys.stdout.write("\n")
2928     sys.stdout.write(" Version      : %s\n" % test.system_info.omd_version)
2929     sys.stdout.write(" Site         : %s\n" % test.system_info.omd_site)
2930     sys.stdout.write(" Core         : %s\n" % test.system_info.core)
2931     sys.stdout.write("\n")
2932
2933     test_cases_by_name = {}
2934     for test_case in test.test_cases:
2935         test_cases_by_name[test_case.name()] = test_case
2936
2937     for test_case_name, measurements in test.get_measurements_by_test_case().items():
2938         test_case = test_cases_by_name[test_case_name]
2939
2940         sys.stdout.write("= %s =================\n" % test_case_name)
2941         sys.stdout.write("\n")
2942
2943         values = Measurement.summary(measurements)
2944
2945         result_txt = test_case.result
2946         if test_case.result == "failed":
2947             result_txt += " (%s)" % test_case.result_msg
2948
2949         data = [
2950             "State       : %s" % test_case.state_text(),
2951             "Result      : %s" % result_txt,
2952             "Duration    : %s" % fmt_timespan(test_case.duration()),
2953             "",
2954             "CPU load    : %0.2f %0.2f %0.2f" % values.cpu_load,
2955             "Memory      : %0.2f %%" % values.memory.percent,
2956             "Disk %-6s : %s free" % (os.path.basename(test.system_info.block_device),
2957                                      fmt_bytes(values.disk_usage.free)),
2958             "Disk IO     : read %s/s, write %s/s" % (fmt_bytes(values.disk_io.read_bytes),
2959                                                      fmt_bytes(values.disk_io.write_bytes)),
2960         ]
2961
2962         if values.site_stats:
2963             data += [
2964                 "",
2965                 "Objects     : %d Hosts, %d Services" % (values.site_stats["num_hosts"],
2966                                                          values.site_stats["num_services"]),
2967                 "Check lat.  : %0.2f sec" % values.site_stats["average_latency_cmk"],
2968                 "Checks      : %0.2f Hosts/s %0.2f Services/s" %
2969                 (values.site_stats["host_checks_rate"], values.site_stats["service_checks_rate"]),
2970                 "CMK helpers : %0.2f %%" % (values.site_stats["helper_usage_cmk"] * 100),
2971             ]
2972         else:
2973             data += ["ERROR: Got no site statistics"]
2974
2975         for value in data:
2976             sys.stdout.write("  %s\n" % value)
2977
2978         sys.stdout.write("\n")
2979
2980
2981 #.
2982 #   .--GUI-----------------------------------------------------------------.
2983 #   |                            ____ _   _ ___                            |
2984 #   |                           / ___| | | |_ _|                           |
2985 #   |                          | |  _| | | || |                            |
2986 #   |                          | |_| | |_| || |                            |
2987 #   |                           \____|\___/|___|                           |
2988 #   |                                                                      |
2989 #   +----------------------------------------------------------------------+
2990 #   | The GUI rendering code                                               |
2991 #   '----------------------------------------------------------------------'
2992
2993
2994 # The class hierarchy is out of our control. Use it as intended by npyscreen.
2995 class Window(npyscreen.FormBaseNew):  # pylint: disable=too-many-ancestors
2996     pass
2997
2998
2999 class GeneralInformation(npyscreen.BoxTitle):
3000     def __init__(self, *args, **kwargs):
3001         super(GeneralInformation, self).__init__(*args, **kwargs)
3002         self.name = "General Information"
3003         self.editable = False
3004         self._runner = kwargs["runner"]
3005
3006     def update(self, clear=True):
3007         self.values = [
3008             "State      : %s" % self._runner.test.state_text(),
3009         ]
3010
3011         if self._runner.test.has_measurements():
3012             values = self._runner.test.current_measurement()
3013
3014             self.values += [
3015                 "Duration   : %s" % fmt_timespan(self._runner.test.duration()),
3016                 "",
3017                 "CPU load   : %0.2f %0.2f %0.2f" % values.cpu_load,
3018                 "Memory     : %0.2f %%" % values.memory.percent,
3019                 "Disk %-6s: %s free" % (os.path.basename(
3020                     self._runner.test.system_info.block_device), fmt_bytes(values.disk_usage.free)),
3021                 "Disk IO    : r %s/s, w %s/s" % (fmt_bytes(values.disk_io.read_bytes),
3022                                                  fmt_bytes(values.disk_io.write_bytes)),
3023             ]
3024         super(GeneralInformation, self).update(clear)
3025
3026
3027 class SiteInformation(npyscreen.BoxTitle):
3028     def __init__(self, *args, **kwargs):
3029         super(SiteInformation, self).__init__(*args, **kwargs)
3030         self.name = "Site"
3031         self.editable = False
3032         self._runner = kwargs["runner"]
3033
3034     def update(self, clear=True):
3035         self._update_info()
3036         super(SiteInformation, self).update(clear)
3037
3038     def _update_info(self):
3039         system_info = self._runner.test.system_info
3040
3041         self.values = [
3042             "Version     : %s" % system_info.omd_version,
3043             "Core        : %s" % system_info.core,
3044         ]
3045
3046         if not self._runner.test.has_measurements():
3047             return
3048
3049         values = self._runner.test.current_measurement()
3050         self.values += [
3051             "State       : %s" % self._site_state_text(values.site_state),
3052         ]
3053
3054         if not values.site_stats:
3055             return
3056
3057         self.values += [
3058             "Objects     : %d H, %d S" % (values.site_stats["num_hosts"],
3059                                           values.site_stats["num_services"]),
3060             "Check lat.  : %0.2f s" % values.site_stats["average_latency_cmk"],
3061             "Checks      : %0.2f H/s %0.2f S/s" % (values.site_stats["host_checks_rate"],
3062                                                    values.site_stats["service_checks_rate"]),
3063             "CMK helpers : %0.2f %%" % (values.site_stats["helper_usage_cmk"] * 100),
3064         ]
3065
3066     def _site_state_text(self, state):
3067         return {
3068             0: "running",
3069             1: "stopped",
3070             2: "partially running",
3071         }.get(state, "UNKNOWN (%d)" % state)
3072
3073
3074 class Log(npyscreen.BoxTitle):
3075     _contained_widget = npyscreen.BufferPager
3076
3077     def __init__(self, *args, **kwargs):
3078         kwargs["contained_widget_arguments"] = {
3079             "autowrap": True,
3080         }
3081         super(Log, self).__init__(*args, **kwargs)
3082         self.name = "Log"
3083         self.editable = False
3084         self._runner = kwargs["runner"]
3085
3086     def update(self, clear=True):
3087         entries = []
3088         for timestamp, _level, entry in self._runner.test.get_log():
3089             if "\n" not in entry:
3090                 lines = [entry]
3091             else:
3092                 lines = entry.split("\n")
3093
3094             for line in lines:
3095                 entries.append(
3096                     "%s %s" % (time.strftime("%H:%M:%S", time.localtime(timestamp)), line))
3097
3098         self.entry_widget.clearBuffer()
3099         self.entry_widget.buffer(entries, scroll_end=True)
3100         super(Log, self).update(clear)
3101
3102
3103 class HelpText(npyscreen.FixedText):
3104     def __init__(self, *args, **kwargs):
3105         super(HelpText, self).__init__(*args, **kwargs)
3106         self.value = "^C: quit"
3107
3108
3109 class ConsoleRenderer(npyscreen.NPSApp):
3110     def __init__(self, runner):
3111         super(ConsoleRenderer, self).__init__()
3112
3113         self._runner = runner
3114
3115         # GUI widgets
3116         self._window = None
3117         self._help = None
3118
3119     def main(self):
3120         npyscreen.setTheme(npyscreen.Themes.ElegantTheme)
3121         self.keypress_timeout_default = 10
3122
3123         self._window = Window(parentApp=self, name="Check_MK System Benchmark %s" % __version__)
3124
3125         max_y, max_x = self._window.curses_pad.getmaxyx()
3126         form_border = 1 * 2
3127         info_box_width = (max_x - form_border) / 2
3128         info_box_height = 10
3129         help_height = 2
3130         log_height = max_y - info_box_height - form_border - help_height
3131
3132         self._general = self._window.add(
3133             GeneralInformation,
3134             relx=1,
3135             rely=1,
3136             max_height=info_box_height,
3137             width=info_box_width,
3138             runner=self._runner)
3139
3140         self._site = self._window.add(
3141             SiteInformation,
3142             relx=info_box_width + 1,
3143             rely=1,
3144             max_height=info_box_height,
3145             width=info_box_width,
3146             runner=self._runner)
3147
3148         self._log = self._window.add(
3149             Log,
3150             relx=1,
3151             rely=info_box_height + 1,
3152             height=log_height,
3153             width=self._window.curses_pad.getmaxyx()[1] - 2,
3154             runner=self._runner,
3155         )
3156
3157         self._help = self._window.add(
3158             HelpText,
3159             height=1,
3160             relx=1,
3161             rely=-3,
3162         )
3163
3164         self._window.edit()
3165
3166     def while_waiting(self):
3167         self._general.update()
3168         self._window.display(clear=True)
3169
3170
3171 #.
3172 #   .--Main----------------------------------------------------------------.
3173 #   |                        __  __       _                                |
3174 #   |                       |  \/  | __ _(_)_ __                           |
3175 #   |                       | |\/| |/ _` | | '_ \                          |
3176 #   |                       | |  | | (_| | | | | |                         |
3177 #   |                       |_|  |_|\__,_|_|_| |_|                         |
3178 #   |                                                                      |
3179 #   +----------------------------------------------------------------------+
3180 #   |                                                                      |
3181 #   '----------------------------------------------------------------------'
3182
3183 opt_debug = False
3184
3185
3186 def main():
3187     global opt_debug
3188     register_signal_handlers()
3189
3190     set_cmdline(" ".join(["mkbench"] + sys.argv[1:]))
3191
3192     verify_is_site()
3193
3194     num_hosts, num_helpers = 100, 20
3195
3196     short_options = "h"
3197     long_options = [
3198         "help", "version", "debug", "report", "cleanup-data", "num-hosts=", "num-helpers="
3199     ]
3200
3201     try:
3202         opts, args = getopt.getopt(sys.argv[1:], short_options, long_options)
3203     except getopt.GetoptError, e:
3204         usage("%s" % e)
3205
3206     mode = "run"
3207
3208     for o, a in opts:
3209         if o in ["-h", "--help"]:
3210             usage()
3211         elif o == "--version":
3212             sys.stdout.write("mkbench %s\n" % __version__)
3213             sys.exit(0)
3214         elif o == "--debug":
3215             opt_debug = True
3216
3217         elif o == "--report":
3218             mode = "report"
3219
3220         elif o == "--cleanup-data":
3221             mode = "cleanup-data"
3222
3223         elif o == "--num-hosts":
3224             try:
3225                 num_hosts = int(a)
3226             except ValueError:
3227                 raise MKGeneralException("--num-hosts must be given as number")
3228
3229             if num_hosts not in TestRunner.host_steps:
3230                 raise MKGeneralException(
3231                     "--num-hosts must be one of: %s" % ", ".join(map(str, TestRunner.host_steps)))
3232
3233         elif o == "--num-helpers":
3234             try:
3235                 num_helpers = int(a)
3236             except ValueError:
3237                 raise MKGeneralException("--num-helpers must be given as number")
3238
3239     if mode == "run":
3240         run_performance_test(num_hosts, num_helpers)
3241     elif mode == "report":
3242         if args:
3243             run_report(args[0])
3244         else:
3245             run_report_list()
3246     elif mode == "cleanup-data":
3247         run_cleanup_data()
3248     else:
3249         raise NotImplementedError()
3250
3251
3252 def verify_is_site():
3253     if "OMD_ROOT" not in os.environ:
3254         raise MKGeneralException("mkbench can only be run as Check_MK site user")
3255
3256
3257 def verify_site_is_running():
3258     site_state = subprocess.call(["omd", "status", "--bare"],
3259                                  stdout=file(os.devnull, "w"),
3260                                  close_fds=True)
3261     if site_state != 0:
3262         raise MKGeneralException("Site needs to be running")
3263
3264
3265 def verify_site_is_empty():
3266     hosts = LocalConnection().query_column("GET hosts\nColumns: host_name")
3267     # Allow sites:
3268     # - empty
3269     # - created with "omd-vonheute"
3270     # - previously terminated mkbench runs
3271     if not hosts or hosts == ["heute"] or all([h.startswith("std-agent-host-") for h in hosts]):
3272         return
3273
3274     raise MKGeneralException("Site needs to be empty")
3275
3276
3277 if __name__ == "__main__":
3278     try:
3279         main()
3280     except KeyboardInterrupt:
3281         sys.stderr.write("Terminated.\n")
3282         sys.exit(0)
3283
3284     except MKGeneralException, e:
3285         sys.stderr.write("%s\n" % e)
3286         if opt_debug:
3287             raise
3288         sys.exit(3)