2 # -*- encoding: utf-8; py-indent-offset: 4 -*-
3 # +------------------------------------------------------------------+
4 # | ____ _ _ __ __ _ __ |
5 # | / ___| |__ ___ ___| | __ | \/ | |/ / |
6 # | | | | '_ \ / _ \/ __| |/ / | |\/| | ' / |
7 # | | |___| | | | __/ (__| < | | | | . \ |
8 # | \____|_| |_|\___|\___|_|\_\___|_| |_|_|\_\ |
10 # | Copyright Mathias Kettner 2014 mk@mathias-kettner.de |
11 # +------------------------------------------------------------------+
13 # This file is part of Check_MK.
14 # The official homepage is at http://mathias-kettner.de/check_mk.
16 # check_mk is free software; you can redistribute it and/or modify it
17 # under the terms of the GNU General Public License as published by
18 # the Free Software Foundation in version 2. check_mk is distributed
19 # in the hope that it will be useful, but WITHOUT ANY WARRANTY; with-
20 # out even the implied warranty of MERCHANTABILITY or FITNESS FOR A
21 # PARTICULAR PURPOSE. See the GNU General Public License for more de-
22 # tails. You should have received a copy of the GNU General Public
23 # License along with GNU Make; see the file COPYING. If not, write
24 # to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
25 # Boston, MA 02110-1301 USA.
27 __version__
= "1.6.0i1"
30 # - Create a more detailed report out of the test results. For example
31 # graph the different values together with the timeline of the different
34 # Problems with large installations:
35 # - Helper startup phases
36 # - Memory usage of helpers
37 # - Huge history files during restarts / rotations
38 # - Config compilation is serialized
39 # - Service check spread with many new services
40 # Config tweaks needed:
42 # - Immediate history log file rotation (too large files) -> cmc_log_limit
45 # .--Imports-------------------------------------------------------------.
47 # | |_ _|_ __ ___ _ __ ___ _ __| |_ ___ |
48 # | | || '_ ` _ \| '_ \ / _ \| '__| __/ __| |
49 # | | || | | | | | |_) | (_) | | | |_\__ \ |
50 # | |___|_| |_| |_| .__/ \___/|_| \__|___/ |
52 # '----------------------------------------------------------------------'
72 import multiprocessing
79 from cmk
.utils
.render
import fmt_bytes
82 # .--Helpers-------------------------------------------------------------.
84 # | | | | | ___| |_ __ ___ _ __ ___ |
85 # | | |_| |/ _ \ | '_ \ / _ \ '__/ __| |
86 # | | _ | __/ | |_) | __/ | \__ \ |
87 # | |_| |_|\___|_| .__/ \___|_| |___/ |
89 # +----------------------------------------------------------------------+
90 # | Different helper methods and classes |
91 # '----------------------------------------------------------------------'
94 # Changes the process name and process command line on of the running process
95 # This works at least with Python 2.x on Linux
96 def set_cmdline(cmdline
):
99 libc
= ctypes
.cdll
.LoadLibrary(ctypes
.util
.find_library('c'))
101 argv
= ctypes
.POINTER(ctypes
.c_char_p
)()
102 argc
= ctypes
.c_int()
103 ctypes
.pythonapi
.Py_GetArgcArgv(ctypes
.byref(argc
), ctypes
.byref(argv
))
104 cmdlen
= sum([len(argv
[i
]) for i
in range(argc
.value
)]) + argc
.value
105 new_cmdline
= ctypes
.c_char_p(cmdline
.ljust(cmdlen
, '\0'))
107 # replace the command line, which is available via /proc/<pid>/cmdline.
108 # This is .e.g used by ps
109 libc
.memcpy(argv
.contents
, new_cmdline
, cmdlen
)
111 # replace the prctl name, which is available via /proc/<pid>/status.
112 # This is for example used by top and killall
113 libc
.prctl(15, new_cmdline
, 0, 0, 0)
117 return os
.environ
["OMD_ROOT"]
121 return os
.environ
.get("OMD_SITE")
124 def fmt_timespan(seconds
):
125 hours
, secs
= divmod(seconds
, 3600)
126 mins
, secs
= divmod(secs
, 60)
127 return "%02d:%02d:%02d" % (hours
, mins
, secs
)
130 def fmt_datetime(seconds
):
131 return time
.strftime("%Y-%m-%d %H:%M:%S", time
.localtime(seconds
))
134 class MKGeneralException(Exception):
138 class MKTerminate(MKGeneralException
):
142 class MKSkipTestCase(MKGeneralException
):
146 class MKFailTestCase(MKGeneralException
):
150 class MKFailTestCaseHighLatency(MKFailTestCase
):
154 class LocalConnection(livestatus
.SingleSiteConnection
):
155 def __init__(self
, *args
, **kwargs
):
156 livestatus
.SingleSiteConnection
.__init
__(self
, "unix:%s/tmp/run/live" % omd_root(), *args
,
161 def interrupt_handler(signum
, frame
):
162 raise MKTerminate("Caught signal: %d" % signum
)
165 def register_signal_handlers():
166 signal
.signal(signal
.SIGTERM
, interrupt_handler
)
169 def usage(error
=None):
171 sys
.stderr
.write("ERROR: %s\n" % error
)
172 sys
.stdout
.write("Usage: mkbench [OPTIONS] [MODE] [MODE_OPTIONS...]\n")
173 sys
.stdout
.write("\n")
174 sys
.stdout
.write("The tool is meant to execute different tasks in a Check_MK site\n")
175 sys
.stdout
.write("on a system and measure the performance of the task on that\n")
176 sys
.stdout
.write("system. The test is saving different static information of\n")
177 sys
.stdout
.write("the system (Installed CPUs, memory, disks, ...) and captures\n")
178 sys
.stdout
.write("the most important peformance relevant values during testing\n")
179 sys
.stdout
.write("for later analysis and comparison.\n")
180 sys
.stdout
.write("\n")
181 sys
.stdout
.write("The results of mkbench are stored in 'var/mkbench'.\n")
182 sys
.stdout
.write("\n")
183 sys
.stdout
.write("The tool also monitors the performance of different values of\n")
184 sys
.stdout
.write("the system and aborts tests in case thresholds are reached.\n")
185 sys
.stdout
.write("\n")
186 sys
.stdout
.write("BE AWARE: mkbench modifies the configuration of your site and will\n")
187 sys
.stdout
.write(" delete existing files. Only execute this in test sites.\n")
188 sys
.stdout
.write("\n")
189 sys
.stdout
.write("MODES:\n")
190 sys
.stdout
.write("\n")
191 sys
.stdout
.write(" NONE Execute a test in the current site\n")
192 sys
.stdout
.write(" --report List test results available for reporting\n")
193 sys
.stdout
.write(" --report ID1 Create reports out of previous results\n")
194 sys
.stdout
.write("\n")
195 sys
.stdout
.write("TEST OPTIONS:\n")
196 sys
.stdout
.write(" --num-hosts H Set the initial number of hosts (Defaults to 100)\n")
197 sys
.stdout
.write(" --num-helpers N Set the initial number of Check_MK helpers\n")
198 sys
.stdout
.write("\n")
199 sys
.stdout
.write("OPTIONS:\n")
200 sys
.stdout
.write("\n")
201 sys
.stdout
.write(" --debug Let Python exceptions come through\n")
202 sys
.stdout
.write(" --version Print the version of the program\n")
203 sys
.stdout
.write("\n")
208 def get_host_ip(host_id
):
209 start_ip_int
= ip_int_from_string("127.0.0.1")
210 return string_from_ip_int(start_ip_int
+ host_id
)
213 def ip_int_from_string(ip_str
):
215 octets
= ip_str
.split('.')
217 packed_ip
= (packed_ip
<< 8) |
int(oc
)
221 def string_from_ip_int(ip_int
):
224 octets
.insert(0, str(ip_int
& 0xFF))
226 return '.'.join(octets
)
230 # .--Testing-------------------------------------------------------------.
232 # | |_ _|__ ___| |_(_)_ __ __ _ |
233 # | | |/ _ \/ __| __| | '_ \ / _` | |
234 # | | | __/\__ \ |_| | | | | (_| | |
235 # | |_|\___||___/\__|_|_| |_|\__, | |
237 # +----------------------------------------------------------------------+
238 # | The major part of this tool. Execution of tests and measurement |
239 # '----------------------------------------------------------------------'
242 def run_performance_test(num_hosts
, num_helpers
):
243 verify_site_is_running()
244 verify_site_is_empty()
246 # Create a new test, running in a dedicated thread
247 runner
= TestRunner(num_hosts
, num_helpers
)
249 # Start measuring in a dedicated thread
250 analyzer
= Analyzer(runner
)
252 # Show state on console
253 renderer
= ConsoleRenderer(runner
)
256 # Save current state before starting
259 # And now start the work
264 renderer
.run(fork
=False)
265 except npyscreen
.wgwidget
.NotEnoughSpaceForWidget
, e
:
266 raise MKGeneralException("Too small terminal (%s)" % e
)
269 sys
.stdout
.write("Terminating...\n")
270 runner
.test
.log_to_stdout()
278 STATE_INITIALIZING
= "initializing"
279 STATE_RUNNING
= "running"
280 STATE_STOPPED
= "stopped"
282 RESULT_COMPLETED
= "completed"
283 RESULT_ABORTED
= "aborted" # TODO: Use this when aborted
284 RESULT_FAILED
= "failed"
288 return os
.path
.join(omd_root(), "var/mkbench")
291 def test_dir(cls
, id_
):
292 return os
.path
.join(cls
.data_dir(), id_
)
295 def exists(cls
, id_
):
296 return os
.path
.exists(cls
.test_dir(id_
))
298 def __init__(self
, id_
=None):
300 self
._log
_to
_stdout
= False
301 self
.version
= __version__
302 self
.state
= Test
.STATE_INITIALIZING
303 self
._measurements
= []
305 self
.result_msg
= None
306 self
.system_info
= SystemInfo()
309 self
.system_info
.collect()
310 self
.id_
= self
._get
_test
_id
()
317 raw_state
= ast
.literal_eval(open(os
.path
.join(self
.dir(), "state")).read())
318 self
.deserialize(raw_state
)
319 self
.load_measurements()
321 # Persists the current state of the test. The measurements are persisting
322 # their results on their own.
324 test_dir
= self
.dir()
326 os
.makedirs(test_dir
)
328 if e
.errno
== 17: # file exists
333 path
= os
.path
.join(test_dir
, "state")
334 with tempfile
.NamedTemporaryFile(
335 "w", dir=os
.path
.dirname(path
), prefix
=".%s.new" % os
.path
.basename(path
),
336 delete
=False) as tmp
:
338 tmp
.write(pprint
.pformat(self
.serialize()) + "\n")
339 os
.rename(tmp_path
, path
)
341 def deserialize(self
, raw_state
):
342 self
._log
= raw_state
["log"]
343 self
.version
= raw_state
["version"]
344 self
.state
= raw_state
["state"]
345 self
.result
= raw_state
["result"]
346 self
.result_msg
= raw_state
["result_msg"]
347 self
.system_info
.deserialize(raw_state
["system_info"])
348 self
.deserialize_test_cases(raw_state
["test_cases"])
353 "version": self
.version
,
355 "result": self
.result
,
356 "result_msg": self
.result_msg
,
357 "system_info": self
.system_info
.serialize(),
358 "test_cases": self
.serialize_test_cases(),
361 def serialize_test_cases(self
):
362 return [test_case
.serialize() for test_case
in self
.test_cases
]
364 def deserialize_test_cases(self
, raw_cases
):
366 for raw_case
in raw_cases
:
367 cls
= globals()[raw_case
.pop("__class_name__")]
369 test_case
.deserialize(raw_case
)
370 self
.test_cases
.append(test_case
)
372 def load_measurements(self
):
373 self
._measurements
= []
374 for entry
in sorted(os
.listdir(self
.dir())):
375 if entry
[0] == "." or not entry
.endswith(".data"):
379 m
.load(os
.path
.join(self
.dir(), entry
))
380 self
.add_measurement(m
)
382 def _get_test_id(self
):
384 self
.system_info
.omd_version
,
385 self
.system_info
.core
,
388 return "_".join(parts
)
390 def set_state(self
, state
, result
=None, result_msg
=None):
393 self
.result_msg
= result_msg
395 def state_text(self
):
398 def first_measurement(self
):
399 return self
._measurements
[0]
401 def current_measurement(self
):
402 return self
._measurements
[-1]
404 def add_measurement(self
, m
):
405 self
._measurements
.append(m
)
407 def has_measurements(self
):
408 return bool(self
._measurements
)
410 def get_measurements_by_test_case(self
):
411 by_test_case
= collections
.OrderedDict()
412 for m
in self
._measurements
:
414 l
= by_test_case
.setdefault(m
.test_case
, [])
418 def is_stopped(self
):
419 return self
.state
== Test
.STATE_STOPPED
424 def log(self
, level
, msg
):
425 if self
._log
_to
_stdout
:
426 sys
.stdout
.write("%d %s\n" % (time
.time(), msg
))
427 self
._log
.append((time
.time(), level
, msg
))
430 self
.log(logging
.INFO
, msg
)
432 def error(self
, msg
):
433 self
.log(logging
.ERROR
, msg
)
435 def log_to_stdout(self
):
436 self
._log
_to
_stdout
= True
439 return Test
.test_dir(self
.id_
)
442 return self
.current_measurement().time
- self
.first_measurement().time
445 class TestRunner(threading
.Thread
):
458 def __init__(self
, num_hosts
=100, num_helpers
=20):
459 super(TestRunner
, self
).__init
__()
460 self
.name
= "test-runner"
461 self
._stop
= threading
.Event()
463 self
.test_case
= None
465 # Helper vars needed for test case generator
466 self
._num
_hosts
= num_hosts
467 self
._num
_helpers
= num_helpers
471 result
, result_msg
= None, None
473 self
.wait_for_load_cooldown(max_wait
=900, expected_load
=5)
475 self
._set
_test
_state
(Test
.STATE_RUNNING
)
478 result
= Test
.RESULT_COMPLETED
483 self
.test
.error("Test failed: %s" % traceback
.format_exc())
484 result
= Test
.RESULT_FAILED
485 result_msg
= "Test failed: %s" % e
488 self
._set
_test
_state
(Test
.STATE_STOPPED
, result
, result_msg
)
489 self
._cleanup
_host
_files
()
491 def wait_for_load_cooldown(self
, max_wait
, expected_load
):
492 # After all helpers are ready, check for reaching previous load level
495 if self
.shall_stop():
498 loadavg_5
= os
.getloadavg()[1]
499 if loadavg_5
> expected_load
:
500 self
.test
.info("Waiting %d sec for cooldown (5min load: %0.2f > %0.2f)" %
501 (max_wait
, loadavg_5
, expected_load
))
509 raise MKGeneralException("System has too high load")
511 def _set_test_state(self
, state
, result
=None, result_msg
=None):
512 self
.test
.set_state(state
, result
, result_msg
)
515 def _set_test_case_state(self
, state
, result
=None, result_msg
=None):
516 self
.test_case
.set_state(state
, result
, result_msg
)
519 def _prepare_site(self
):
520 self
.test
.info("Preparing site for testing")
521 self
._prepare
_rrdcached
()
523 def _prepare_rrdcached(self
):
524 with
open("%s/etc/rrdcached.conf" % omd_root(), "w") as f
:
525 f
.write("TIMEOUT=300\n" "RANDOM_DELAY=10\n" "FLUSH_TIMEOUT=7200\n" "WRITE_THREADS=4\n")
527 p
= subprocess
.Popen(["omd", "restart", "rrdcached"],
529 stdout
=subprocess
.PIPE
,
530 stderr
=subprocess
.STDOUT
)
531 stdout
= p
.communicate()[0]
532 if p
.returncode
!= 0:
533 raise MKGeneralException("Failed to restart rrdcached: %s" % stdout
)
536 for test_case
in self
._generate
_test
_cases
():
537 if self
._stop
.isSet():
540 self
.test
.test_cases
.append(test_case
)
542 result
, result_msg
= None, None
544 self
.test
.info("Initializing %s" % test_case
.name())
546 for step
in test_case
.run():
547 if isinstance(step
, Start
):
548 # The measurements are only assigned to the test case when
549 # it tells us to do so by providing a Start() object
550 self
.test
.info("Start %s" % test_case
.name())
551 self
.test_case
= test_case
552 self
._set
_test
_case
_state
(TestCase
.STATE_RUNNING
)
554 elif isinstance(step
, Wait
):
555 # Do many small sleeps while the test is running so that
556 # the test thread can persist the current state of the test
557 result
= TestCase
.RESULT_COMPLETED
558 while time
.time() <= step
.until
:
560 self
._while
_waiting
()
561 except MKSkipTestCase
, e
:
563 "Skipping %s (%s): %s" %
564 (test_case
.name(), fmt_timespan(test_case
.duration()), e
))
565 result
= TestCase
.RESULT_SKIPPED
566 result_msg
= "Skipped: %s" % e
567 break # Will iterate to next step (till test case end)
569 if self
._stop
.isSet():
570 result
= TestCase
.RESULT_ABORTED
571 result_msg
= "Test runner was stopped"
577 "Finished %s (%s)" % (test_case
.name(), fmt_timespan(test_case
.duration())))
578 self
._increase
_hosts
()
580 except MKFailTestCase
, e
:
581 result
= TestCase
.RESULT_FAILED
582 result_msg
= "Test case aborted: %s" % e
583 self
.test
.error(result_msg
)
585 if isinstance(e
, MKFailTestCaseHighLatency
):
586 self
._increase
_helpers
()
591 result
= TestCase
.RESULT_FAILED
592 result_msg
= "Test case failed: %s" % e
593 self
.test
.error(result_msg
)
598 self
._set
_test
_case
_state
(TestCase
.STATE_STOPPED
, result
, result_msg
)
599 self
.test_case
= None
601 self
.test
.info("Finished testing")
603 def _increase_hosts(self
):
605 self
._num
_hosts
= TestRunner
.host_steps
[TestRunner
.host_steps
.index(self
._num
_hosts
) +
607 except (ValueError, IndexError):
611 def _increase_helpers(self
):
612 self
._num
_helpers
+= 40
614 def _generate_test_cases(self
):
615 while not self
._stop
.isSet():
616 yield TestAgentBasedStandardHosts(
618 num_hosts
=self
._num
_hosts
,
619 planned_duration
=15 * 60,
620 num_cmk_helpers
=self
._num
_helpers
)
622 # TestAgentBasedStandardHosts(self, num_hosts=100, planned_duration=15*60, num_cmk_helpers=20),
623 # TestAgentBasedStandardHosts(self, num_hosts=200, planned_duration=15*60, num_cmk_helpers=20),
624 # TestAgentBasedStandardHosts(self, num_hosts=500, planned_duration=15*60, num_cmk_helpers=20),
625 # TestAgentBasedStandardHosts(self, num_hosts=1000, planned_duration=15*60, num_cmk_helpers=20),
626 # TestAgentBasedStandardHosts(self, num_hosts=2000, planned_duration=15*60, num_cmk_helpers=20),
627 # TestAgentBasedStandardHosts(self, num_hosts=5000, planned_duration=15*60, num_cmk_helpers=20),
628 # TestAgentBasedStandardHosts(self, num_hosts=8000, planned_duration=15*60, num_cmk_helpers=20),
629 # TestAgentBasedStandardHosts(self, num_hosts=10000, planned_duration=15*60, num_cmk_helpers=20),
631 # TestAgentBasedStandardHosts(self, num_hosts=2000, planned_duration=15*60, num_cmk_helpers=40),
632 # TestAgentBasedStandardHosts(self, num_hosts=5000, planned_duration=15*60, num_cmk_helpers=40),
633 # TestAgentBasedStandardHosts(self, num_hosts=8000, planned_duration=15*60, num_cmk_helpers=40),
634 # TestAgentBasedStandardHosts(self, num_hosts=10000, planned_duration=15*60, num_cmk_helpers=40),
636 # TestAgentBasedStandardHosts(self, num_hosts=5000, planned_duration=15*60, num_cmk_helpers=60),
637 # TestAgentBasedStandardHosts(self, num_hosts=8000, planned_duration=15*60, num_cmk_helpers=60),
638 # TestAgentBasedStandardHosts(self, num_hosts=10000, planned_duration=15*60, num_cmk_helpers=60),
640 # TestAgentBasedStandardHosts(self, num_hosts=8000, planned_duration=15*60, num_cmk_helpers=100),
641 # TestAgentBasedStandardHosts(self, num_hosts=10000, planned_duration=15*60, num_cmk_helpers=100),
642 # TestAgentBasedStandardHosts(self, num_hosts=20000, planned_duration=15*60, num_cmk_helpers=100),
645 def _while_waiting(self
):
646 self
._check
_thresholds
()
649 def _check_thresholds(self
):
651 measurement
= self
.test
.current_measurement()
655 self
._check
_upper
_thresholds
(measurement
)
656 self
._check
_lower
_thresholds
(measurement
)
658 def _check_upper_thresholds(self
, m
):
659 # Abort when less than 100 MB are free
660 if m
.disk_usage
.free
< 100 * 1024 * 1024:
661 raise MKFailTestCase("The disk %s has only %s left." %
662 (self
.test
.system_info
.block_device
, fmt_bytes(m
.disk_usage
.free
)))
664 if m
.cpu_load
[1] > self
.test
.system_info
.cpu_count
* 1.2:
665 raise MKFailTestCase(
666 "The 5 min average CPU load %0.2f is higher than "
667 "number of CPUs (%d)." % (m
.cpu_load
[1], self
.test
.system_info
.cpu_count
))
669 # Check memory threshold. There are different values provided by psutil.
670 # this value sounds good to me for testing.
671 if m
.memory
.available
< 100 * 1024 * 1024:
672 raise MKFailTestCase(
673 "There is only %s memory available." % (fmt_bytes(m
.memory
.available
)))
675 # Initial scheduling of the CMC spreads the service checks for a maximum of
676 # 5 * check_interval. So we should terminate the test only after some initialization
678 if self
.test_case
.duration() < 450:
681 if m
.site_stats
and m
.site_stats
["average_latency_cmk"] > 30:
682 raise MKFailTestCaseHighLatency("The average check latency is too high (%0.2f sec)" %
683 m
.site_stats
["average_latency_cmk"])
685 def _check_lower_thresholds(self
, m
):
686 if self
.test_case
.duration() < 30:
687 return # Check for lower thresholds after X seconds
689 if m
.disk_usage
.percent
> 70:
692 if m
.cpu_load
[0] > self
.test
.system_info
.cpu_count
* 0.2:
695 if m
.memory
.percent
> 10:
698 if m
.site_stats
and m
.site_stats
["helper_usage_cmk"] * 100 > 10:
701 if m
.site_stats
and m
.site_stats
["average_latency_cmk"] > 2:
704 raise MKSkipTestCase("System is bored.")
707 set_symlink
= not os
.path
.exists(self
.test
.dir())
712 self
._update
_last
_symlink
()
714 def _update_last_symlink(self
):
715 path
= os
.path
.join(Test
.data_dir(), "last")
720 os
.symlink(self
.test
.id_
, path
)
725 def shall_stop(self
):
726 return self
._stop
.isSet()
728 def _cleanup_host_files(self
, cleanup_rrds
=False):
729 for path
in glob
.glob("%s/var/check_mk/autochecks/*" % omd_root()):
732 for path
in glob
.glob("%s/tmp/check_mk/counters/*" % omd_root()):
736 for path
in glob
.glob("%s/var/check_mk/rrd/*" % omd_root()):
739 for path
in glob
.glob("%s/var/pnp4nagios/perfdata/*" % omd_root()):
742 for path
in glob
.glob("%s/var/check_mk/core/archive/history-*" % omd_root()):
745 for path
in glob
.glob("%s/var/check_mk/core/state*" % omd_root()):
749 class Analyzer(threading
.Thread
):
750 def __init__(self
, runner
):
751 self
._runner
= runner
752 super(Analyzer
, self
).__init
__()
754 self
.name
= "analyzer"
756 self
._stop
= threading
.Event()
759 #self._runner.test.info("Analyzer started")
761 while not self
._stop
.isSet() and not self
._runner
.test
.is_stopped():
767 self
._runner
.test
.error("Analyzer failed: %s" % traceback
.format_exc())
768 self
._runner
.test
.info("Analyzer stopped")
771 m
= Measurement(self
._runner
)
772 if self
._runner
.test
.has_measurements():
773 last
= self
._runner
.test
.current_measurement()
778 self
._runner
.test
.add_measurement(m
)
782 def is_stopped(self
):
783 return not self
.is_alive()
794 def __init__(self
, duration
):
795 self
.until
= time
.time() + duration
798 class SystemInfo(object):
800 super(SystemInfo
, self
).__init
__()
801 self
.cpu_count
= None
804 self
.block_device
= None
805 self
.linux_distro
= None
806 self
.omd_version
= None
811 # Number of logical CPUs
812 self
.cpu_count
= psutil
.cpu_count()
813 self
.cpu_info
= self
._get
_cpu
_info
()
814 self
.memory
= psutil
.virtual_memory().total
815 self
.block_device
= self
._find
_block
_device
("/omd/sites")
816 self
.linux_distro
= self
._get
_linux
_distro
()
817 self
.omd_version
= self
._get
_omd
_version
()
818 self
.omd_site
= site_id()
819 self
.core
= self
._get
_monitoring
_core
()
822 return dict([(k
, v
) for k
, v
in self
.__dict
__.items() if k
[0] != "_"])
824 def deserialize(self
, raw
):
825 self
.__dict
__.update(raw
)
827 def _find_block_device(self
, mount_path
):
828 mount_path
= os
.path
.realpath(mount_path
)
829 while not os
.path
.ismount(mount_path
):
830 mount_path
= os
.path
.dirname(mount_path
)
832 dev
= os
.stat(mount_path
).st_dev
833 major
= os
.major(dev
)
834 minor
= os
.minor(dev
)
836 return os
.path
.realpath("/sys/dev/block/%d:%d" % (major
, minor
))
838 def _get_linux_distro(self
):
839 return open("%s/share/omd/distro.info" % omd_root()).readline().split("=", 1)[1].strip()
841 def _get_omd_version(self
):
842 return os
.path
.basename(os
.path
.realpath("%s/version" % omd_root()))
844 def _get_monitoring_core(self
):
845 for l
in open("%s/etc/omd/site.conf" % omd_root()):
846 if l
.startswith("CONFIG_CORE="):
847 return l
.strip().split("=", 1)[1].strip("'")
850 def _get_cpu_info(self
):
852 num_threads_total
= 0
854 for l
in open("/proc/cpuinfo"):
855 parts
= l
.split(": ", 1)
858 varname
, value
= parts
[0].strip(), parts
[1].strip()
860 if varname
== "cpu cores":
861 node
["cores_per_cpu"] = int(value
)
862 elif varname
== "siblings":
863 node
["threads_per_cpu"] = int(value
)
864 elif varname
== "vendor_id":
866 "GenuineIntel": "intel",
867 "AuthenticAMD": "amd",
869 elif varname
== "cache size":
870 node
["cache_size"] = int(
871 value
.split()[0]) * 1024 # everything is normalized to bytes!
872 elif varname
== "model name":
873 node
["model"] = value
874 # For the following two entries we assume that all
875 # entries are numbered in increasing order in /proc/cpuinfo.
876 elif varname
== "processor":
877 num_threads_total
= int(value
) + 1
878 elif varname
== "physical id":
879 sockets
.add(int(value
))
880 elif varname
== "flags":
881 if re
.search(" lm ", value
):
882 node
["arch"] = "x86_64"
884 node
["arch"] = "i386"
886 num_sockets
= len(sockets
)
888 if num_threads_total
:
889 node
.setdefault("cores_per_cpu", 1)
890 node
.setdefault("threads_per_cpu", 1)
891 node
["cores"] = num_sockets
* node
["cores_per_cpu"]
892 node
["threads"] = num_sockets
* node
["threads_per_cpu"]
893 node
["cpus"] = num_sockets
898 DiskIO
= collections
.namedtuple("DiskIO", ["read_bytes", "write_bytes"])
901 class Measurement(object):
903 def summary(cls
, measurements
):
912 average
= Measurement()
914 for k
in average_keys
:
916 for m
in measurements
:
917 values
.append(getattr(m
, k
))
920 if isinstance(values
[0], tuple):
921 # averages of equal length tuples
922 averaged
= tuple([sum(y
) / len(y
) for y
in zip(*values
)])
924 # averages of dict items
927 real_values
= [v
for v
in values
if v
is not None]
928 num_values
= len(real_values
)
931 for key
, value
in real_values
[0].items():
932 if isinstance(value
, unicode):
933 averaged
[key
] = value
935 averaged
[key
] = float(sum([v
[key
] for v
in real_values
940 average
.set_value(k
, averaged
)
944 def __init__(self
, runner
=None):
945 super(Measurement
, self
).__init
__()
946 self
._runner
= runner
948 self
.test_case
= None
949 self
.disk_counters
= None
950 self
.site_state
= None
955 self
.disk_usage
= None
957 self
.site_stats
= None
959 def measure(self
, last
=None):
960 self
.test_case
= self
._runner
.test_case
.name() \
961 if self
._runner
.test_case
else None
962 self
.time
= time
.time()
963 self
.cpu_load
= os
.getloadavg()
964 self
.memory
= psutil
.virtual_memory()
965 self
.disk_usage
= psutil
.disk_usage(omd_root())
967 self
._get
_disk
_io
(last
)
968 self
._get
_site
_state
()
969 self
._get
_site
_stats
()
971 def load(self
, path
):
973 raw_data
= ast
.literal_eval(open(path
).read())
974 self
.deserialize(raw_data
)
978 raise MKGeneralException("Failed to load '%s': %s" % (path
, e
))
981 test_dir
= self
._runner
.test
.dir()
983 path
= os
.path
.join(test_dir
, "%d.data" % self
.time
)
984 with tempfile
.NamedTemporaryFile(
985 "w", dir=os
.path
.dirname(path
), prefix
=".%s.new" % os
.path
.basename(path
),
986 delete
=False) as tmp
:
988 tmp
.write(pprint
.pformat(self
.serialize()) + "\n")
989 os
.rename(tmp_path
, path
)
994 for k
, v
in self
.__dict
__.items():
998 # change named tuples to real tuples (for ast.literal_eval)
999 if isinstance(v
, tuple) and hasattr(v
, "_fields"):
1006 def deserialize(self
, raw
):
1007 for k
in self
.__dict
__:
1009 self
.set_value(k
, raw
[k
])
1011 def set_value(self
, k
, val
):
1012 """Set an attribute of the measurement. Cares about converting tuples
1013 to the correct namedtuples."""
1014 from psutil
._common
import sdiskusage
1015 from psutil
._pslinux
import svmem
, sdiskio
1019 elif k
== "disk_usage":
1020 val
= sdiskusage(*val
)
1021 elif k
== "disk_counters":
1023 elif k
== "disk_io":
1026 setattr(self
, k
, val
)
1028 def _get_disk_io(self
, last
):
1029 self
.disk_counters
= self
._get
_disk
_counters
()
1032 self
.disk_io
= DiskIO(0.0, 0.0)
1035 read_bytes_per_sec
= (self
.disk_counters
.read_bytes
- last
.disk_counters
.read_bytes
) \
1036 / (self
.time
- last
.time
)
1038 write_bytes_per_sec
= (self
.disk_counters
.write_bytes
- last
.disk_counters
.write_bytes
) \
1039 / (self
.time
- last
.time
)
1041 self
.disk_io
= DiskIO(read_bytes_per_sec
, write_bytes_per_sec
)
1043 def _get_disk_counters(self
):
1044 # Use total disk IO of the system because e.g. software raid IO is not
1045 # accounted to the md0p2 partition in case of CMK is on that volume.
1046 #device_name = os.path.basename(self._runner.test.system_info.block_device)
1047 #disk_io = psutil.disk_io_counters(perdisk=True)
1048 #return disk_io[device_name]
1050 #part_name = os.path.basename(self._runner.test.system_info.block_device)
1051 #device_name = part_name.rsplit("p", 1)[0]
1052 #disk_io = psutil.disk_io_counters(perdisk=True)
1053 #return disk_io[device_name]
1054 return psutil
.disk_io_counters()
1056 def _get_site_state(self
):
1057 self
.site_state
= subprocess
.call(["omd", "status", "--bare"],
1058 stdout
=file(os
.devnull
, "w"),
1061 def _get_site_stats(self
):
1062 if self
.site_state
not in [ 0, 2 ] \
1063 or not os
.path
.exists("%s/tmp/run/live" % omd_root()):
1064 self
.site_stats
= None
1068 live
= LocalConnection()
1070 self
.site_stats
= live
.query_row_assoc("GET status")
1072 except livestatus
.MKLivestatusException
, e
:
1073 self
._runner
.test
.error("Livestatus: %s" % e
)
1074 self
.site_stats
= None
1078 # .--AgentListener-------------------------------------------------------.
1080 # | / \ __ _ ___ _ __ | |_| | (_)___| |_ ___ _ __ ___ _ __ |
1081 # | / _ \ / _` |/ _ \ '_ \| __| | | / __| __/ _ \ '_ \ / _ \ '__| |
1082 # | / ___ \ (_| | __/ | | | |_| |___| \__ \ || __/ | | | __/ | |
1083 # | /_/ \_\__, |\___|_| |_|\__|_____|_|___/\__\___|_| |_|\___|_| |
1085 # +----------------------------------------------------------------------+
1086 # | Is executed in a dedicated process and serves the dynamic agent data |
1087 # '----------------------------------------------------------------------'
1090 class AgentListener(multiprocessing
.Process
):
1091 def __init__(self
, host_ids
, *args
, **kwargs
):
1092 self
._hosts
= host_ids
1094 self
._socket
_map
= {}
1095 super(AgentListener
, self
).__init
__(*args
, **kwargs
)
1096 self
.name
= "mkbench_agent-listener"
1098 # Holds the counters per host that are used to track the dynamic agent data
1103 self
._set
_process
_name
()
1104 self
._open
_sockets
()
1106 except (KeyboardInterrupt, MKTerminate
):
1109 self
._close
_sockets
()
1111 def _set_process_name(self
):
1112 set_cmdline(self
.name
)
1114 def _open_sockets(self
):
1115 for host_id
in self
._hosts
:
1116 address
= (get_host_ip(host_id
), 6559)
1118 s
= socket
.socket(socket
.AF_INET
, socket
.SOCK_STREAM
)
1119 s
.setsockopt(socket
.SOL_SOCKET
, socket
.SO_REUSEADDR
, 1)
1122 self
._sockets
.append(s
)
1123 self
._socket
_map
[s
.getsockname()] = host_id
1124 except Exception, e
:
1127 sys
.stderr
.write("Failed to open %s:%d: %s\n" % (address
[0], address
[1], e
))
1134 readable
= select
.select(self
._sockets
, [], [], 0.2)[0]
1135 except select
.error
, e
:
1143 client_socket
, _addr_info
= s
.accept()
1144 host_id
= self
._socket
_map
[s
.getsockname()]
1145 client_socket
.sendall(self
._get
_answer
(host_id
))
1146 client_socket
.close()
1147 except (KeyboardInterrupt, MKTerminate
):
1149 except Exception, e
:
1150 sys
.stderr
.write("Error in listener: %s - %s\n" % (type(e
), e
))
1152 def _close_sockets(self
):
1153 for s
in self
._sockets
:
1155 del self
._sockets
[:]
1157 def _get_answer(self
, host_id
):
1158 agent_data
= static_agent_data
1159 agent_data
+= self
._section
_kernel
(host_id
)
1162 def _update_counter(self
, host_id
, ident
, per_sec
):
1163 counters
= self
._counters
.setdefault(host_id
, {})
1166 if ident
not in counters
:
1167 counters
[ident
] = now
, 0
1168 return # just initialied. Don't increase
1171 last_update
, last_value
= counters
[ident
]
1172 duration
= now
- last_update
1174 counters
[ident
] = now
, last_value
+ (duration
* per_sec
)
1176 def _section_kernel(self
, host_id
):
1177 self
._update
_counter
(host_id
, "kernel_ctxt", per_sec
=5000)
1178 self
._update
_counter
(host_id
, "kernel_processes", per_sec
=1)
1179 self
._update
_counter
(host_id
, "kernel_pgmajfault", per_sec
=50)
1181 # hundrets of a second
1183 for i
in [""] + range(num_cpus
):
1184 self
._update
_counter
(host_id
, "kernel_cpu%s_user" % i
, per_sec
=0.10)
1185 self
._update
_counter
(host_id
, "kernel_cpu%s_nice" % i
, per_sec
=0.05)
1186 self
._update
_counter
(host_id
, "kernel_cpu%s_system" % i
, per_sec
=0.10)
1187 self
._update
_counter
(host_id
, "kernel_cpu%s_idle" % i
, per_sec
=0.30)
1188 self
._update
_counter
(host_id
, "kernel_cpu%s_iowait" % i
, per_sec
=0.05)
1190 counters
= {k
: v
for k
, (_t
, v
) in self
._counters
[host_id
].iteritems()}
1191 counters
["now"] = time
.time()
1194 for i
in [""] + range(num_cpus
):
1195 cpu_output
+= "cpu%s %d %d %d %d %d 0 0 0 0 0\n" % \
1197 counters
["kernel_cpu%s_user" % i
],
1198 counters
["kernel_cpu%s_nice" % i
],
1199 counters
["kernel_cpu%s_system" % i
],
1200 counters
["kernel_cpu%s_idle" % i
],
1201 counters
["kernel_cpu%s_iowait" % i
])
1203 section
= """<<<kernel>>>
1205 nr_free_pages 2544893
1207 nr_inactive_anon 52641
1208 nr_active_anon 980946
1209 nr_inactive_file 87489
1210 nr_active_file 314428
1213 nr_anon_pages 979995
1215 nr_file_pages 455554
1218 nr_slab_reclaimable 34034
1219 nr_slab_unreclaimable 15947
1220 nr_page_table_pages 17569
1225 nr_vmscan_immediate_reclaim 0
1236 numa_interleave 32848
1239 workingset_refault 0
1240 workingset_activate 0
1241 workingset_nodereclaim 0
1242 nr_anon_transparent_hugepages 495
1244 nr_dirty_threshold 580548
1245 nr_dirty_background_threshold 289919
1251 pgalloc_dma32 5351681
1252 pgalloc_normal 40330396
1258 pgmajfault %(kernel_pgmajfault)d
1263 pgsteal_kswapd_dma 0
1264 pgsteal_kswapd_dma32 0
1265 pgsteal_kswapd_normal 0
1266 pgsteal_kswapd_movable 0
1267 pgsteal_direct_dma 0
1268 pgsteal_direct_dma32 0
1269 pgsteal_direct_normal 0
1270 pgsteal_direct_movable 0
1272 pgscan_kswapd_dma32 0
1273 pgscan_kswapd_normal 0
1274 pgscan_kswapd_movable 0
1276 pgscan_direct_dma32 0
1277 pgscan_direct_normal 0
1278 pgscan_direct_movable 0
1279 pgscan_direct_throttle 0
1280 zone_reclaim_failed 0
1284 kswapd_low_wmark_hit_quickly 0
1285 kswapd_high_wmark_hit_quickly 0
1292 numa_huge_pte_updates 0
1294 numa_hint_faults_local 0
1295 numa_pages_migrated 0
1298 compact_migrate_scanned 0
1299 compact_free_scanned 0
1304 htlb_buddy_alloc_success 0
1305 htlb_buddy_alloc_fail 0
1306 unevictable_pgs_culled 2964
1307 unevictable_pgs_scanned 0
1308 unevictable_pgs_rescued 1580
1309 unevictable_pgs_mlocked 4145
1310 unevictable_pgs_munlocked 4145
1311 unevictable_pgs_cleared 0
1312 unevictable_pgs_stranded 0
1313 thp_fault_alloc 2907
1314 thp_fault_fallback 0
1315 thp_collapse_alloc 500
1316 thp_collapse_alloc_failed 0
1318 thp_zero_page_alloc 1
1319 thp_zero_page_alloc_failed 0
1323 intr 1664579 22 0 0 0 0 0 0 0 1 33196 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 20295 70340 0 0 0 0 0 0 0 2419 352 0 11 571 752 1616 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
1324 ctxt %(kernel_ctxt)d
1326 processes %(kernel_processes)d
1329 softirq 3526225 0 845354 2 177933 55525 0 6372 577174 0 1863865
1332 return section
+ cpu_output
1335 static_agent_data
= """<<<check_mk>>>
1338 Hostname: Klappspaten
1339 AgentDirectory: /etc/check_mk
1340 DataDirectory: /var/lib/check_mk_agent
1341 SpoolDirectory: /var/lib/check_mk_agent/spool
1342 PluginsDirectory: /usr/lib/check_mk_agent/plugins
1343 LocalDirectory: /usr/lib/check_mk_agent/local
1345 /dev/mapper/rootvg-lv_root ext4 1032088 509248 470412 52% /
1346 /dev/sda1 ext4 516040 84856 404972 18% /boot
1347 /dev/mapper/rootvg-lv_home ext4 3096336 1603828 1335248 55% /home
1348 /dev/mapper/rootvg-lv_opt ext4 1548144 409064 1060452 28% /opt
1349 /dev/mapper/rootvg-lv_tmp ext4 2064208 64980 1894392 4% /tmp
1350 /dev/mapper/rootvg-lv_usr ext4 4128448 1738152 2180584 45% /usr
1351 /dev/mapper/rootvg-lv_local ext4 516040 44136 445692 10% /usr/local
1352 /dev/mapper/rootvg-lv_var ext4 2064208 1524036 435328 78% /var
1353 /dev/mapper/rootvg-lv_install ext4 34060032 10740480 21589632 34% /install
1354 /dev/mapper/rootvg-lv_agent ext4 4128448 1975432 1943316 51% /agent
1355 /dev/mapper/dwhvg-lv_dingdata ext4 258030980 103616116 151793424 41% /dingdata
1356 /dev/mapper/dwhvg-lv_conndir ext4 1032088 61064 918596 7% /conndir
1357 /dev/mapper/dwhvg-lv_backup ext4 61927420 11445540 47336152 20% /backup
1360 /dev/mapper/rootvg-lv_root ext4 65536 13287 52249 21% /
1361 /dev/sda1 ext4 32768 50 32718 1% /boot
1362 /dev/mapper/rootvg-lv_home ext4 196608 3193 193415 2% /home
1363 /dev/mapper/rootvg-lv_opt ext4 98304 637 97667 1% /opt
1364 /dev/mapper/rootvg-lv_tmp ext4 131072 161 130911 1% /tmp
1365 /dev/mapper/rootvg-lv_usr ext4 262144 62008 200136 24% /usr
1366 /dev/mapper/rootvg-lv_local ext4 32768 124 32644 1% /usr/local
1367 /dev/mapper/rootvg-lv_var ext4 131072 61541 69531 47% /var
1368 /dev/mapper/rootvg-lv_install ext4 2162688 8503 2154185 1% /install
1369 /dev/mapper/rootvg-lv_agent ext4 262144 12469 249675 5% /agent
1370 /dev/mapper/dwhvg-lv_dingdata ext4 16384000 2429 16381571 1% /dingdata
1371 /dev/mapper/dwhvg-lv_conndir ext4 65536 205 65331 1% /conndir
1372 /dev/mapper/dwhvg-lv_backup ext4 3932160 82 3932078 1% /backup
1376 /dev/mapper/rootvg-lv_root / ext4 rw,relatime,barrier=1,data=ordered 0 0
1377 /dev/sda1 /boot ext4 rw,relatime,barrier=1,data=ordered 0 0
1378 /dev/mapper/rootvg-lv_home /home ext4 rw,relatime,barrier=0,data=ordered 0 0
1379 /dev/mapper/rootvg-lv_opt /opt ext4 rw,relatime,barrier=0,data=ordered 0 0
1380 /dev/mapper/rootvg-lv_tmp /tmp ext4 rw,relatime,barrier=0,data=ordered 0 0
1381 /dev/mapper/rootvg-lv_usr /usr ext4 rw,relatime,barrier=0,data=ordered 0 0
1382 /dev/mapper/rootvg-lv_local /usr/local ext4 rw,relatime,barrier=0,data=ordered 0 0
1383 /dev/mapper/rootvg-lv_var /var ext4 rw,relatime,barrier=0,data=ordered 0 0
1384 /dev/mapper/rootvg-lv_install /install ext4 rw,relatime,barrier=0,data=ordered 0 0
1385 /dev/mapper/rootvg-lv_agent /agent ext4 rw,relatime,barrier=0,data=ordered 0 0
1386 /dev/mapper/dwhvg-lv_dingdata /dingdata ext4 rw,relatime,barrier=0,data=ordered 0 0
1387 /dev/mapper/dwhvg-lv_conndir /conndir ext4 rw,relatime,barrier=0,data=ordered 0 0
1388 /dev/mapper/dwhvg-lv_backup /backup ext4 rw,relatime,barrier=0,data=ordered 0 0
1390 (root,120480,6764,00:00:02/24:28,1) /sbin/init splash
1391 (root,0,0,00:00:00/24:28,2) [kthreadd]
1392 (root,0,0,00:00:00/24:28,3) [ksoftirqd/0]
1393 (root,0,0,00:00:00/24:28,5) [kworker/0:0H]
1394 (root,0,0,00:00:02/24:28,7) [rcu_sched]
1395 (root,0,0,00:00:00/24:28,8) [rcu_bh]
1396 (root,0,0,00:00:00/24:28,9) [migration/0]
1397 (root,0,0,00:00:00/24:28,10) [watchdog/0]
1398 (root,0,0,00:00:00/24:28,11) [watchdog/1]
1399 (root,0,0,00:00:00/24:28,12) [migration/1]
1400 (root,0,0,00:00:00/24:28,13) [ksoftirqd/1]
1401 (root,0,0,00:00:00/24:28,15) [kworker/1:0H]
1402 (root,0,0,00:00:00/24:28,16) [watchdog/2]
1403 (root,0,0,00:00:00/24:28,17) [migration/2]
1404 (root,0,0,00:00:00/24:28,18) [ksoftirqd/2]
1405 (root,0,0,00:00:00/24:28,20) [kworker/2:0H]
1406 (root,0,0,00:00:00/24:28,21) [watchdog/3]
1407 (root,0,0,00:00:00/24:28,22) [migration/3]
1408 (root,0,0,00:00:00/24:28,23) [ksoftirqd/3]
1409 (root,0,0,00:00:00/24:28,25) [kworker/3:0H]
1410 (root,0,0,00:00:00/24:28,26) [watchdog/4]
1411 (root,0,0,00:00:00/24:28,27) [migration/4]
1412 (root,0,0,00:00:00/24:28,28) [ksoftirqd/4]
1413 (root,0,0,00:00:00/24:28,29) [kworker/4:0]
1414 (root,0,0,00:00:00/24:28,30) [kworker/4:0H]
1415 (root,0,0,00:00:00/24:28,31) [watchdog/5]
1416 (root,0,0,00:00:00/24:28,32) [migration/5]
1417 (root,0,0,00:00:00/24:28,33) [ksoftirqd/5]
1418 (root,0,0,00:00:00/24:28,34) [kworker/5:0]
1419 (root,0,0,00:00:00/24:28,35) [kworker/5:0H]
1420 (root,0,0,00:00:00/24:28,36) [watchdog/6]
1421 (root,0,0,00:00:00/24:28,37) [migration/6]
1422 (root,0,0,00:00:00/24:28,38) [ksoftirqd/6]
1423 (root,0,0,00:00:00/24:28,40) [kworker/6:0H]
1424 (root,0,0,00:00:00/24:28,41) [watchdog/7]
1425 (root,0,0,00:00:00/24:28,42) [migration/7]
1426 (root,0,0,00:00:00/24:28,43) [ksoftirqd/7]
1427 (root,0,0,00:00:00/24:28,45) [kworker/7:0H]
1428 (root,0,0,00:00:00/24:28,46) [kdevtmpfs]
1429 (root,0,0,00:00:00/24:28,47) [netns]
1430 (root,0,0,00:00:00/24:28,48) [perf]
1431 (root,0,0,00:00:00/24:28,49) [khungtaskd]
1432 (root,0,0,00:00:00/24:28,50) [writeback]
1433 (root,0,0,00:00:00/24:28,51) [ksmd]
1434 (root,0,0,00:00:00/24:28,52) [khugepaged]
1435 (root,0,0,00:00:00/24:28,53) [crypto]
1436 (root,0,0,00:00:00/24:28,54) [kintegrityd]
1437 (root,0,0,00:00:00/24:28,55) [bioset]
1438 (root,0,0,00:00:00/24:28,56) [kblockd]
1439 (root,0,0,00:00:01/24:28,58) [kworker/u16:1]
1440 (root,0,0,00:00:00/24:28,59) [ata_sff]
1441 (root,0,0,00:00:00/24:28,60) [md]
1442 (root,0,0,00:00:00/24:28,61) [devfreq_wq]
1443 (root,0,0,00:00:00/24:28,62) [kworker/2:1]
1444 (root,0,0,00:00:00/24:28,65) [kswapd0]
1445 (root,0,0,00:00:00/24:28,66) [vmstat]
1446 (root,0,0,00:00:00/24:28,67) [fsnotify_mark]
1447 (root,0,0,00:00:00/24:28,68) [ecryptfs-kthrea]
1448 (root,0,0,00:00:00/24:28,84) [kthrotld]
1449 (root,0,0,00:00:00/24:28,85) [kworker/1:1]
1450 (root,0,0,00:00:00/24:28,88) [kworker/6:1]
1451 (root,0,0,00:00:00/24:28,89) [kworker/7:1]
1452 (root,0,0,00:00:00/24:28,90) [acpi_thermal_pm]
1453 (root,0,0,00:00:00/24:28,91) [bioset]
1454 (root,0,0,00:00:00/24:28,92) [bioset]
1455 (root,0,0,00:00:00/24:28,93) [bioset]
1456 (root,0,0,00:00:00/24:28,94) [bioset]
1457 (root,0,0,00:00:00/24:28,95) [bioset]
1458 (root,0,0,00:00:00/24:28,96) [bioset]
1459 (root,0,0,00:00:00/24:28,97) [bioset]
1460 (root,0,0,00:00:00/24:28,98) [bioset]
1461 (root,0,0,00:00:00/24:28,99) [bioset]
1462 (root,0,0,00:00:00/24:28,100) [bioset]
1463 (root,0,0,00:00:00/24:28,101) [bioset]
1464 (root,0,0,00:00:00/24:28,102) [bioset]
1465 (root,0,0,00:00:00/24:28,103) [bioset]
1466 (root,0,0,00:00:00/24:28,104) [bioset]
1467 (root,0,0,00:00:00/24:28,105) [bioset]
1468 (root,0,0,00:00:00/24:28,106) [bioset]
1469 (root,0,0,00:00:00/24:28,107) [bioset]
1470 (root,0,0,00:00:00/24:28,108) [bioset]
1471 (root,0,0,00:00:00/24:28,109) [bioset]
1472 (root,0,0,00:00:00/24:28,110) [bioset]
1473 (root,0,0,00:00:00/24:28,111) [bioset]
1474 (root,0,0,00:00:00/24:28,112) [bioset]
1475 (root,0,0,00:00:00/24:28,113) [bioset]
1476 (root,0,0,00:00:00/24:28,114) [bioset]
1477 (root,0,0,00:00:01/24:27,115) [kworker/u16:2]
1478 (root,0,0,00:00:00/24:27,118) [kworker/1:2]
1479 (root,0,0,00:00:00/24:27,122) [ipv6_addrconf]
1480 (root,0,0,00:00:00/24:27,135) [deferwq]
1481 (root,0,0,00:00:00/24:27,136) [charger_manager]
1482 (root,0,0,00:00:00/24:27,177) [scsi_eh_0]
1483 (root,0,0,00:00:00/24:27,178) [scsi_tmf_0]
1484 (root,0,0,00:00:00/24:27,179) [usb-storage]
1485 (root,0,0,00:00:01/24:26,211) [irq/36-(null)]
1486 (root,0,0,00:00:00/24:26,212) [scsi_eh_1]
1487 (root,0,0,00:00:00/24:26,213) [scsi_tmf_1]
1488 (root,0,0,00:00:00/24:26,218) [bioset]
1489 (root,0,0,00:00:00/24:26,289) [bioset]
1490 (root,0,0,00:00:01/24:26,291) [kworker/u16:4]
1491 (root,0,0,00:00:00/24:21,300) [kworker/7:1H]
1492 (root,0,0,00:00:00/24:20,301) [kworker/3:1H]
1493 (root,0,0,00:00:00/24:19,311) [kdmflush]
1494 (root,0,0,00:00:00/24:19,348) [bioset]
1495 (root,0,0,00:00:00/24:19,349) [kcryptd_io]
1496 (root,0,0,00:00:00/24:19,350) [kcryptd]
1497 (root,0,0,00:00:00/24:19,351) [dmcrypt_write]
1498 (root,0,0,00:00:00/24:19,352) [bioset]
1499 (root,0,0,00:00:00/24:19,362) [kdmflush]
1500 (root,0,0,00:00:00/24:19,364) [bioset]
1501 (root,0,0,00:00:00/24:19,367) [kdmflush]
1502 (root,0,0,00:00:00/24:19,368) [bioset]
1503 (root,0,0,00:00:00/02:55,393) [kworker/u16:8]
1504 (root,0,0,00:00:00/24:19,401) [jbd2/dm-1-8]
1505 (root,0,0,00:00:00/24:19,402) [ext4-rsv-conver]
1506 (root,0,0,00:00:01/24:19,404) [kworker/u16:5]
1507 (root,0,0,00:00:01/24:19,418) [kworker/u16:6]
1508 (root,0,0,00:00:01/24:19,419) [kworker/u16:7]
1509 (root,0,0,00:00:00/24:18,440) [kauditd]
1510 (root,35496,7576,00:00:00/24:18,461) /lib/systemd/systemd-journald
1511 (root,0,0,00:00:00/24:18,477) [rpciod]
1512 (root,168504,3752,00:00:00/24:18,490) /sbin/lvmetad -f
1513 (root,45468,4628,00:00:00/24:18,499) /lib/systemd/systemd-udevd
1514 (root,0,0,00:00:00/24:18,584) [kworker/4:2]
1515 (root,0,0,00:00:00/24:18,612) [kworker/6:2]
1516 (root,0,0,00:00:00/24:18,616) [irq/47-mei_me]
1517 (root,0,0,00:00:00/24:18,632) [cfg80211]
1518 (root,0,0,00:00:00/24:18,657) [kvm-irqfd-clean]
1519 (root,0,0,00:00:00/24:18,658) [kworker/4:1H]
1520 (root,0,0,00:00:00/24:18,662) [kworker/0:1H]
1521 (root,0,0,00:00:00/24:18,691) [kworker/2:3]
1522 (root,0,0,00:00:00/24:18,723) [kworker/6:1H]
1523 (root,0,0,00:00:00/24:18,724) [kworker/2:1H]
1524 (root,0,0,00:00:00/24:18,832) [kworker/1:1H]
1525 (root,0,0,00:00:00/24:18,874) [applesmc-led]
1526 (root,0,0,00:00:00/24:18,917) [ext4-rsv-conver]
1527 (root,23536,216,00:00:00/24:18,1015) /usr/sbin/rpc.idmapd
1528 (root,47628,3556,00:00:00/24:18,1095) /sbin/rpcbind -f -w
1529 (root,25216,4108,00:00:00/24:18,1112) /usr/sbin/smartd -n
1530 (mongodb,4484316,68116,00:00:03/24:18,1114) /usr/bin/mongod --config /etc/mongodb.conf
1531 (root,30536,3288,00:00:00/24:18,1115) /usr/sbin/cron -f
1532 (daemon,26044,2144,00:00:00/24:18,1119) /usr/sbin/atd -f
1533 (root,4400,1320,00:00:00/24:18,1121) /usr/sbin/acpid
1534 (root,0,0,00:00:00/24:18,1131) [kworker/5:3]
1535 (root,7492,948,00:00:00/24:18,1135) /usr/sbin/mbpfan -f
1536 (avahi,44932,3092,00:00:00/24:18,1137) avahi-daemon: running [Klappspaten.local]
1537 (root,278044,7180,00:00:00/24:18,1145) /usr/lib/accountsservice/accounts-daemon
1538 (root,10708,856,00:00:00/24:18,1147) /usr/sbin/avahi-dnsconfd -s
1539 (syslog,256396,3140,00:00:00/24:18,1169) /usr/sbin/rsyslogd -n
1540 (root,337316,8356,00:00:00/24:18,1172) /usr/sbin/ModemManager
1541 (messagebus,44276,5052,00:00:01/24:18,1178) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation
1542 (avahi,44788,344,00:00:00/24:18,1195) avahi-daemon: chroot helper
1543 (root,0,0,00:00:00/24:18,1238) [kworker/5:1H]
1544 (root,215828,18372,00:00:00/24:18,1241) /usr/lib/snapd/snapd
1545 (whoopsie,269224,9168,00:00:00/24:18,1243) /usr/bin/whoopsie -f
1546 (root,28680,3124,00:00:00/24:18,1245) /lib/systemd/systemd-logind
1547 (root,274960,9424,00:00:00/24:18,1247) /usr/sbin/cups-browsed
1548 (root,524564,16932,00:00:00/24:18,1257) /usr/sbin/NetworkManager --no-daemon
1549 (root,0,0,00:00:00/24:18,1292) [iprt]
1550 (root,0,0,00:00:00/24:17,1315) [irq/50-brcmf_pc]
1551 (root,0,0,00:00:00/24:17,1316) [msgbuf_txflow]
1552 (colord,302680,11180,00:00:00/24:17,1333) /usr/lib/colord/colord
1553 (root,283576,10984,00:00:00/24:17,1334) /usr/lib/policykit-1/polkitd --no-debug
1554 (root,0,0,00:00:00/24:17,1401) [kworker/3:2]
1555 (root,44128,6368,00:00:00/24:17,1402) /sbin/wpa_supplicant -u -s -O /run/wpa_supplicant
1556 (root,0,0,00:00:00/24:17,1421) [kworker/u17:0]
1557 (root,0,0,00:00:00/24:17,1422) [hci0]
1558 (root,0,0,00:00:00/24:17,1423) [hci0]
1559 (root,0,0,00:00:00/24:17,1424) [kworker/u17:1]
1560 (root,32084,4600,00:00:00/24:17,1433) /usr/lib/bluetooth/bluetoothd
1561 (root,65520,5688,00:00:00/24:17,1629) /usr/sbin/sshd -D
1562 (root,37976,900,00:00:00/24:17,1636) /usr/sbin/rpc.mountd --manage-gids
1563 (root,0,0,00:00:00/24:17,1642) [nfsd4_callbacks]
1564 (root,0,0,00:00:00/24:17,1643) [lockd]
1565 (root,0,0,00:00:00/24:17,1645) [nfsd]
1566 (root,0,0,00:00:00/24:17,1646) [nfsd]
1567 (root,0,0,00:00:00/24:17,1647) [nfsd]
1568 (root,0,0,00:00:00/24:17,1648) [nfsd]
1569 (root,0,0,00:00:00/24:17,1649) [nfsd]
1570 (root,0,0,00:00:00/24:17,1650) [nfsd]
1571 (root,0,0,00:00:00/24:17,1651) [nfsd]
1572 (root,0,0,00:00:00/24:17,1652) [nfsd]
1573 (root,209964,13996,00:00:04/24:17,1718) /opt/teamviewer/tv_bin/teamviewerd -d
1574 (root,19568,2216,00:00:00/24:17,1770) /usr/sbin/irqbalance --pid=/var/run/irqbalance.pid
1575 (ntp,110036,5144,00:00:00/24:17,1772) /usr/sbin/ntpd -p /var/run/ntpd.pid -g -u 108:115
1576 (root,15056,2096,00:00:00/24:17,1818) /usr/sbin/xinetd -pidfile /run/xinetd.pid -stayalive -inetd_compat -inetd_ipv6
1577 (root,276648,6344,00:00:00/24:17,1840) /usr/sbin/lightdm
1578 (snmp,61284,8000,00:00:00/24:17,1842) /usr/sbin/snmpd -Lsd -Lf /dev/null -u snmp -g snmp -I -smux mteTrigger mteTriggerConf -p /run/snmpd.pid
1579 (uml-net,4376,792,00:00:00/24:17,1868) /usr/bin/uml_switch -tap tap0 -unix /var/run/uml-utilities/uml_switch.ctl
1580 (root,83636,6164,00:00:00/24:17,1906) /usr/sbin/apache2 -k start
1581 (www-data,83552,3692,00:00:00/24:17,1911) /usr/sbin/apache2 -k start
1582 (www-data,83948,4356,00:00:00/24:17,1912) /usr/sbin/apache2 -k start
1583 (www-data,83948,4356,00:00:00/24:17,1913) /usr/sbin/apache2 -k start
1584 (www-data,83980,4356,00:00:00/24:17,1915) /usr/sbin/apache2 -k start
1585 (www-data,83980,4356,00:00:00/24:17,1916) /usr/sbin/apache2 -k start
1586 (www-data,83948,4356,00:00:00/24:17,1917) /usr/sbin/apache2 -k start
1587 (root,537576,93684,00:00:10/24:17,1991) /usr/lib/xorg/Xorg -core :0 -seat seat0 -auth /var/run/lightdm/root/:0 -nolisten tcp vt7 -novtswitch
1588 (root,17468,1704,00:00:00/24:17,1992) /sbin/agetty --noclear tty1 linux
1589 (root,65408,4548,00:00:00/24:16,2027) /usr/lib/postfix/sbin/master
1590 (postfix,67476,4416,00:00:00/24:16,2029) pickup -l -t unix -u -c
1591 (postfix,67648,4536,00:00:00/24:16,2030) qmgr -l -t unix -u
1592 (heute,53952,7716,00:00:00/24:16,2046) python /omd/sites/heute/bin/liveproxyd
1593 (root,241532,5356,00:00:00/24:16,2052) /usr/sbin/nmbd -D
1594 (heute,78568,9460,00:00:00/24:16,2073) python /omd/sites/heute/bin/mknotifyd
1595 (root,339396,15748,00:00:00/24:16,2168) /usr/sbin/smbd -D
1596 (root,331424,5816,00:00:00/24:16,2169) /usr/sbin/smbd -D
1597 (root,339404,8928,00:00:00/24:16,2171) /usr/sbin/smbd -D
1598 (heute,101268,10584,00:00:00/24:16,2173) /usr/sbin/apache2 -f /omd/sites/heute/etc/apache/apache.conf
1599 (heute,101228,4196,00:00:00/24:16,2176) /usr/sbin/apache2 -f /omd/sites/heute/etc/apache/apache.conf
1600 (heute,311868,68852,00:00:01/24:16,2177) /usr/sbin/apache2 -f /omd/sites/heute/etc/apache/apache.conf
1601 (root,0,0,00:00:00/10:32,2189) [kworker/u16:10]
1602 (root,228240,6648,00:00:00/24:16,2190) lightdm --session-child 12 15
1603 (heute2,221652,12080,00:00:00/24:16,2248) python /omd/sites/heute2/bin/mkeventd
1604 (heute2,53952,7448,00:00:00/24:16,2279) python /omd/sites/heute2/bin/liveproxyd
1605 (heute2,78852,9724,00:00:00/24:16,2289) python /omd/sites/heute2/bin/mknotifyd
1606 (heute2,251656,3172,00:00:00/24:15,2305) /omd/sites/heute2/bin/rrdcached -w 3600 -z 1800 -f 7200 -s heute2 -m 660 -l unix:/omd/sites/heute2/tmp/run/rrdcached.sock -p /omd/sites/heute2/tmp/rrdcached.pid -j /omd/sites/heute2/var/rrdcached
1607 (heute2,1530808,8520,00:00:00/24:15,2341) /omd/sites/heute2/bin/cmc /omd/sites/heute2/var/check_mk/core/config
1608 (heute2,218676,38616,00:00:01/24:15,2379) python /omd/sites/heute2/bin/cmk --create-rrd --keepalive
1609 (heute2,100348,27400,00:00:00/24:15,2383) python /omd/sites/heute2/bin/cmk --handle-alerts --keepalive
1610 (heute2,100644,28100,00:00:00/24:15,2387) python /omd/sites/heute2/bin/cmk --notify --keepalive
1611 (heute2,10492,1068,00:00:00/24:15,2391) checkhelper
1612 (heute2,10492,1004,00:00:00/24:15,2395) checkhelper
1613 (heute2,10492,968,00:00:00/24:15,2397) checkhelper
1614 (heute2,10492,1004,00:00:00/24:15,2398) checkhelper
1615 (heute2,10492,1016,00:00:00/24:15,2399) checkhelper
1616 (heute2,104244,31840,00:00:01/24:15,2401) python /omd/sites/heute2/bin/cmk --keepalive
1617 (heute2,104760,32396,00:00:00/24:15,2404) python /omd/sites/heute2/bin/cmk --keepalive
1618 (heute2,100344,27364,00:00:00/24:15,2405) python /omd/sites/heute2/bin/cmk --keepalive
1619 (heute2,100348,27540,00:00:00/24:15,2406) python /omd/sites/heute2/bin/cmk --keepalive
1620 (heute2,100348,27516,00:00:00/24:15,2407) python /omd/sites/heute2/bin/cmk --keepalive
1621 (heute2,100360,27488,00:00:00/24:15,2408) python /omd/sites/heute2/bin/cmk --keepalive
1622 (heute2,100348,27480,00:00:00/24:15,2409) python /omd/sites/heute2/bin/cmk --keepalive
1623 (heute2,100348,27564,00:00:00/24:15,2411) python /omd/sites/heute2/bin/cmk --keepalive
1624 (heute2,100348,27456,00:00:00/24:15,2413) python /omd/sites/heute2/bin/cmk --keepalive
1625 (heute2,100352,27316,00:00:00/24:15,2414) python /omd/sites/heute2/bin/cmk --keepalive
1626 (heute2,100352,27388,00:00:00/24:15,2415) python /omd/sites/heute2/bin/cmk --keepalive
1627 (heute2,100352,27468,00:00:00/24:15,2416) python /omd/sites/heute2/bin/cmk --keepalive
1628 (heute2,100356,27472,00:00:00/24:15,2417) python /omd/sites/heute2/bin/cmk --keepalive
1629 (heute2,100360,27376,00:00:00/24:15,2418) python /omd/sites/heute2/bin/cmk --keepalive
1630 (heute2,100356,27540,00:00:00/24:15,2419) python /omd/sites/heute2/bin/cmk --keepalive
1631 (heute2,100356,27512,00:00:00/24:15,2420) python /omd/sites/heute2/bin/cmk --keepalive
1632 (heute2,100348,27400,00:00:00/24:15,2421) python /omd/sites/heute2/bin/cmk --keepalive
1633 (heute2,100348,27444,00:00:00/24:15,2422) python /omd/sites/heute2/bin/cmk --keepalive
1634 (heute2,100348,27600,00:00:00/24:15,2441) python /omd/sites/heute2/bin/cmk --keepalive
1635 (heute2,100352,27300,00:00:00/24:15,2442) python /omd/sites/heute2/bin/cmk --keepalive
1636 (heute2,102436,27812,00:00:00/24:15,2443) python /omd/sites/heute2/bin/cmk --keepalive --real-time-checks
1637 (root,12596,1060,00:00:00/24:15,2448) icmpsender 8 0 1000
1638 (root,14848,1024,00:00:00/24:15,2449) icmpreceiver
1639 (heute2,101268,10624,00:00:00/24:15,2450) /usr/sbin/apache2 -f /omd/sites/heute2/etc/apache/apache.conf
1640 (heute2,101228,4216,00:00:00/24:15,2457) /usr/sbin/apache2 -f /omd/sites/heute2/etc/apache/apache.conf
1641 (heute2,313420,70412,00:00:01/24:15,2458) /usr/sbin/apache2 -f /omd/sites/heute2/etc/apache/apache.conf
1642 (test1,221484,11476,00:00:00/24:15,2511) python /omd/sites/test1/bin/mkeventd
1643 (rtkit,183544,3124,00:00:00/24:15,2524) /usr/lib/rtkit/rtkit-daemon
1644 (test1,54204,7964,00:00:00/24:15,2526) python /omd/sites/test1/bin/liveproxyd
1645 (test1,78848,9756,00:00:00/24:15,2568) python /omd/sites/test1/bin/mknotifyd
1646 (root,367748,8604,00:00:00/24:14,2580) /usr/lib/udisks2/udisksd --no-debug
1647 (test1,177660,3136,00:00:00/24:14,2582) /omd/sites/test1/bin/rrdcached -w 3600 -z 1800 -f 7200 -s test1 -m 660 -l unix:/omd/sites/test1/tmp/run/rrdcached.sock -p /omd/sites/test1/tmp/rrdcached.pid -j /omd/sites/test1/var/rrdcached
1648 (test1,1529284,6224,00:00:00/24:14,2642) /omd/sites/test1/bin/cmc /omd/sites/test1/var/check_mk/core/config
1649 (test1,218580,38660,00:00:01/24:14,2698) python /omd/sites/test1/share/check_mk/modules/check_mk.py --create-rrd --keepalive
1650 (test1,100332,27696,00:00:00/24:14,2700) python /omd/sites/test1/share/check_mk/modules/check_mk.py --handle-alerts --keepalive
1651 (test1,100336,27624,00:00:00/24:14,2702) python /omd/sites/test1/share/check_mk/modules/check_mk.py --notify --keepalive
1652 (root,0,0,00:00:00/24:14,2704) [krfcommd]
1653 (test1,4364,632,00:00:00/24:14,2711) checkhelper
1654 (test1,4364,636,00:00:00/24:14,2712) checkhelper
1655 (test1,4364,784,00:00:00/24:14,2713) checkhelper
1656 (test1,4364,652,00:00:00/24:14,2717) checkhelper
1657 (test1,4364,740,00:00:00/24:14,2737) checkhelper
1658 (test1,100332,27452,00:00:00/24:14,2738) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1659 (test1,100332,27748,00:00:00/24:14,2739) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1660 (test1,100332,27612,00:00:00/24:14,2741) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1661 (test1,100336,27660,00:00:00/24:14,2742) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1662 (root,349032,10440,00:00:00/24:14,2744) /usr/lib/upower/upowerd
1663 (test1,100324,27752,00:00:00/24:14,2747) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1664 (test1,100340,27676,00:00:00/24:14,2748) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1665 (test1,100332,27608,00:00:00/24:14,2751) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1666 (test1,100332,27612,00:00:00/24:14,2752) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1667 (test1,100332,27712,00:00:00/24:14,2753) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1668 (test1,100328,27660,00:00:00/24:14,2754) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1669 (test1,100328,27692,00:00:00/24:14,2755) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1670 (test1,100332,27572,00:00:00/24:14,2756) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1671 (test1,100328,27416,00:00:00/24:14,2757) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1672 (test1,100332,27668,00:00:00/24:14,2758) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1673 (test1,100388,27732,00:00:00/24:14,2759) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1674 (test1,100328,27916,00:00:00/24:14,2760) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1675 (test1,100332,27444,00:00:00/24:14,2761) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1676 (test1,100328,27524,00:00:00/24:14,2765) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1677 (test1,100328,27800,00:00:00/24:14,2768) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1678 (test1,100328,27412,00:00:00/24:14,2774) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
1679 (test1,102668,27744,00:00:00/24:14,2775) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive --real-time-checks
1680 (root,12520,1160,00:00:00/24:14,2776) icmpsender 8 0 1000
1681 (root,8720,752,00:00:00/24:14,2777) icmpreceiver
1682 (heute,311868,68908,00:00:01/24:06,3812) /usr/sbin/apache2 -f /omd/sites/heute/etc/apache/apache.conf
1683 (www-data,83948,4364,00:00:00/24:06,4132) /usr/sbin/apache2 -k start
1684 (test1,101264,10604,00:00:00/24:05,4165) /usr/sbin/apache2 -f /omd/sites/test1/etc/apache/apache.conf
1685 (test1,15056,2052,00:00:00/24:05,4178) /omd/sites/test1/var/tmp/xinetd -pidfile /omd/sites/test1/tmp/run/xinetd.pid -filelog /omd/sites/test1/var/log/xinetd.log -f /omd/sites/test1/etc/xinetd.conf
1686 (test1,101224,4192,00:00:00/24:05,4186) /usr/sbin/apache2 -f /omd/sites/test1/etc/apache/apache.conf
1687 (test1,316088,70952,00:00:01/24:05,4187) /usr/sbin/apache2 -f /omd/sites/test1/etc/apache/apache.conf
1688 (test2,221480,11520,00:00:00/24:05,4200) python /omd/sites/test2/bin/mkeventd
1689 (test2,54204,7768,00:00:00/24:05,4209) python /omd/sites/test2/bin/liveproxyd
1690 (test2,78848,9656,00:00:00/24:05,4219) python /omd/sites/test2/bin/mknotifyd
1691 (test2,177660,3080,00:00:00/24:05,4224) /omd/sites/test2/bin/rrdcached -w 3600 -z 1800 -f 7200 -s test2 -m 660 -l unix:/omd/sites/test2/tmp/run/rrdcached.sock -p /omd/sites/test2/tmp/rrdcached.pid -j /omd/sites/test2/var/rrdcached
1692 (test2,1529284,6060,00:00:00/24:05,4244) /omd/sites/test2/bin/cmc /omd/sites/test2/var/check_mk/core/config
1693 (test2,218592,38636,00:00:01/24:05,4276) python /omd/sites/test2/share/check_mk/modules/check_mk.py --create-rrd --keepalive
1694 (test2,100332,27728,00:00:00/24:05,4277) python /omd/sites/test2/share/check_mk/modules/check_mk.py --handle-alerts --keepalive
1695 (test2,100332,27588,00:00:00/24:05,4279) python /omd/sites/test2/share/check_mk/modules/check_mk.py --notify --keepalive
1696 (test2,4364,784,00:00:00/24:05,4280) checkhelper
1697 (test2,4364,640,00:00:00/24:05,4282) checkhelper
1698 (test2,4364,652,00:00:00/24:05,4283) checkhelper
1699 (test2,4364,788,00:00:00/24:05,4284) checkhelper
1700 (test2,4364,732,00:00:00/24:05,4285) checkhelper
1701 (test2,100336,27776,00:00:00/24:05,4286) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1702 (test2,100332,27708,00:00:00/24:05,4287) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1703 (test2,100328,27540,00:00:00/24:05,4288) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1704 (test2,100356,27716,00:00:00/24:05,4289) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1705 (test2,100360,27400,00:00:00/24:05,4290) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1706 (test2,100336,27848,00:00:00/24:05,4291) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1707 (test2,100332,27704,00:00:00/24:05,4292) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1708 (test2,100328,27660,00:00:00/24:05,4293) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1709 (test2,100336,27696,00:00:00/24:05,4297) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1710 (test2,100332,27612,00:00:00/24:05,4298) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1711 (test2,100320,27428,00:00:00/24:05,4299) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1712 (test2,100328,27712,00:00:00/24:05,4300) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1713 (test2,100336,27620,00:00:00/24:05,4302) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1714 (test2,100328,27612,00:00:00/24:05,4303) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1715 (test2,100332,27584,00:00:00/24:05,4304) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1716 (test2,100336,27572,00:00:00/24:05,4305) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1717 (test2,100332,27844,00:00:00/24:05,4306) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1718 (test2,100332,27764,00:00:00/24:05,4307) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1719 (test2,100332,27680,00:00:00/24:05,4308) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1720 (test2,100328,27468,00:00:00/24:05,4317) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
1721 (test2,102792,27924,00:00:00/24:05,4318) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive --real-time-checks
1722 (root,12520,1080,00:00:00/24:05,4319) icmpsender 8 0 1000
1723 (root,8720,732,00:00:00/24:05,4320) icmpreceiver
1724 (test2,101264,10592,00:00:00/23:56,4874) /usr/sbin/apache2 -f /omd/sites/test2/etc/apache/apache.conf
1725 (test2,15056,2056,00:00:00/23:56,4887) /omd/sites/test2/var/tmp/xinetd -pidfile /omd/sites/test2/tmp/run/xinetd.pid -filelog /omd/sites/test2/var/log/xinetd.log -f /omd/sites/test2/etc/xinetd.conf
1726 (test2,101224,4188,00:00:00/23:56,4888) /usr/sbin/apache2 -f /omd/sites/test2/etc/apache/apache.conf
1727 (test2,316032,70940,00:00:01/23:56,4889) /usr/sbin/apache2 -f /omd/sites/test2/etc/apache/apache.conf
1728 (test_modes,221652,11624,00:00:00/23:56,4907) python /omd/sites/test_modes/bin/mkeventd
1729 (test_modes,53952,7652,00:00:00/23:56,4915) python /omd/sites/test_modes/bin/liveproxyd
1730 (test_modes,78856,9744,00:00:00/23:56,4923) python /omd/sites/test_modes/bin/mknotifyd
1731 (test_modes,251392,3184,00:00:00/23:56,4927) /omd/sites/test_modes/bin/rrdcached -w 3600 -z 1800 -f 7200 -s test_modes -m 660 -l unix:/omd/sites/test_modes/tmp/run/rrdcached.sock -p /omd/sites/test_modes/tmp/rrdcached.pid -j /omd/sites/test_modes/var/rrdcached
1732 (test_modes,1529456,6236,00:00:00/23:56,4944) /omd/sites/test_modes/bin/cmc /omd/sites/test_modes/var/check_mk/core/config
1733 (test_modes,219156,43348,00:00:01/23:56,4974) python /omd/sites/test_modes/bin/cmk --create-rrd --keepalive
1734 (test_modes,97708,31144,00:00:01/23:56,4976) python /omd/sites/test_modes/bin/cmk --handle-alerts --keepalive
1735 (test_modes,97684,31224,00:00:01/23:56,4977) python /omd/sites/test_modes/bin/cmk --notify --keepalive
1736 (test_modes,7396,2260,00:00:00/23:56,4978) checkhelper
1737 (test_modes,7396,948,00:00:00/23:56,4979) checkhelper
1738 (test_modes,7396,792,00:00:00/23:56,4980) checkhelper
1739 (test_modes,7396,840,00:00:00/23:56,4981) checkhelper
1740 (test_modes,7396,808,00:00:00/23:56,4984) checkhelper
1741 (test_modes,108680,34108,00:00:01/23:56,4986) python /omd/sites/test_modes/bin/cmk --keepalive
1742 (test_modes,97440,30660,00:00:01/23:56,4987) python /omd/sites/test_modes/bin/cmk --keepalive
1743 (test_modes,97400,30628,00:00:01/23:56,4988) python /omd/sites/test_modes/bin/cmk --keepalive
1744 (test_modes,97444,30652,00:00:01/23:56,4989) python /omd/sites/test_modes/bin/cmk --keepalive
1745 (test_modes,97408,30580,00:00:01/23:56,4990) python /omd/sites/test_modes/bin/cmk --keepalive
1746 (test_modes,97420,30624,00:00:01/23:56,4991) python /omd/sites/test_modes/bin/cmk --keepalive
1747 (test_modes,97384,30736,00:00:01/23:56,4992) python /omd/sites/test_modes/bin/cmk --keepalive
1748 (test_modes,97416,30640,00:00:01/23:56,4993) python /omd/sites/test_modes/bin/cmk --keepalive
1749 (test_modes,97440,30620,00:00:01/23:56,4994) python /omd/sites/test_modes/bin/cmk --keepalive
1750 (test_modes,97440,30576,00:00:01/23:56,4996) python /omd/sites/test_modes/bin/cmk --keepalive
1751 (test_modes,97404,30560,00:00:01/23:56,4997) python /omd/sites/test_modes/bin/cmk --keepalive
1752 (test_modes,97408,30744,00:00:01/23:56,4998) python /omd/sites/test_modes/bin/cmk --keepalive
1753 (test_modes,97392,30572,00:00:01/23:56,4999) python /omd/sites/test_modes/bin/cmk --keepalive
1754 (test_modes,97420,30568,00:00:01/23:56,5000) python /omd/sites/test_modes/bin/cmk --keepalive
1755 (test_modes,97448,30556,00:00:01/23:56,5004) python /omd/sites/test_modes/bin/cmk --keepalive
1756 (test_modes,97436,30704,00:00:01/23:56,5005) python /omd/sites/test_modes/bin/cmk --keepalive
1757 (test_modes,97448,30660,00:00:01/23:56,5006) python /omd/sites/test_modes/bin/cmk --keepalive
1758 (test_modes,97412,30568,00:00:01/23:56,5007) python /omd/sites/test_modes/bin/cmk --keepalive
1759 (test_modes,97308,30680,00:00:01/23:56,5008) python /omd/sites/test_modes/bin/cmk --keepalive
1760 (test_modes,97444,30708,00:00:01/23:55,5014) python /omd/sites/test_modes/bin/cmk --keepalive
1761 (test_modes,99388,30532,00:00:01/23:55,5015) python /omd/sites/test_modes/bin/cmk --keepalive --real-time-checks
1762 (root,12612,1048,00:00:00/23:55,5016) icmpsender 8 0 1000
1763 (root,11756,872,00:00:00/23:55,5017) icmpreceiver
1764 (test_modes,101524,10648,00:00:00/23:46,5763) /usr/sbin/apache2 -f /omd/sites/test_modes/etc/apache/apache.conf
1765 (test_modes,101484,4220,00:00:00/23:46,5774) /usr/sbin/apache2 -f /omd/sites/test_modes/etc/apache/apache.conf
1766 (test_modes,313392,70500,00:00:01/23:46,5775) /usr/sbin/apache2 -f /omd/sites/test_modes/etc/apache/apache.conf
1767 (test_modes,15056,2172,00:00:00/23:46,5776) /omd/sites/test_modes/var/tmp/xinetd -pidfile /omd/sites/test_modes/tmp/run/xinetd.pid -filelog /omd/sites/test_modes/var/log/xinetd.log -f /omd/sites/test_modes/etc/xinetd.conf
1768 (test_modes,313392,70520,00:00:01/23:16,6881) /usr/sbin/apache2 -f /omd/sites/test_modes/etc/apache/apache.conf
1769 (test1,316088,70808,00:00:01/23:16,7013) /usr/sbin/apache2 -f /omd/sites/test1/etc/apache/apache.conf
1770 (heute2,313420,70432,00:00:01/23:16,7014) /usr/sbin/apache2 -f /omd/sites/heute2/etc/apache/apache.conf
1771 (test2,316032,71012,00:00:01/23:16,7023) /usr/sbin/apache2 -f /omd/sites/test2/etc/apache/apache.conf
1772 (root,0,0,00:00:00/09:45,7836) [kworker/3:0]
1773 (root,0,0,00:00:00/09:22,8288) [kworker/0:0]
1774 (heute,221660,11908,00:00:00/02:07,9604) python /omd/sites/heute/bin/mkeventd
1775 (heute,251392,3376,00:00:01/02:04,9785) /omd/sites/heute/bin/rrdcached -w 300 -z 10 -f 7200 -s heute -m 660 -l unix:/omd/sites/heute/tmp/run/rrdcached.sock -p /omd/sites/heute/tmp/rrdcached.pid -j /omd/sites/heute/var/rrdcached
1776 (root,22668,2276,00:00:00/00:29,10284) /bin/bash /usr/bin/check_mk_agent
1777 (heute,1529448,24964,00:00:00/00:27,10434) /omd/sites/heute/bin/cmc /omd/sites/heute/var/check_mk/core/config
1778 (heute,218612,38580,00:00:01/00:27,10456) python /omd/sites/heute/share/check_mk/modules/check_mk.py --create-rrd --keepalive
1779 (heute,100444,27424,00:00:00/00:27,10457) python /omd/sites/heute/share/check_mk/modules/check_mk.py --handle-alerts --keepalive
1780 (heute,100388,27436,00:00:00/00:27,10458) python /omd/sites/heute/share/check_mk/modules/check_mk.py --notify --keepalive
1781 (heute,7396,888,00:00:00/00:27,10459) checkhelper
1782 (heute,7396,788,00:00:00/00:27,10460) checkhelper
1783 (heute,7396,832,00:00:00/00:27,10461) checkhelper
1784 (heute,7396,780,00:00:00/00:27,10462) checkhelper
1785 (heute,7396,832,00:00:00/00:27,10463) checkhelper
1786 (heute,100412,27500,00:00:00/00:27,10464) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1787 (heute,100420,27440,00:00:00/00:27,10465) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1788 (heute,100420,27532,00:00:00/00:27,10466) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1789 (heute,100416,27468,00:00:00/00:27,10467) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1790 (heute,100420,27360,00:00:00/00:27,10468) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1791 (heute,100456,27572,00:00:00/00:27,10469) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1792 (heute,100452,27428,00:00:00/00:27,10470) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1793 (heute,100452,27488,00:00:00/00:27,10471) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1794 (heute,100428,27608,00:00:00/00:27,10472) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1795 (heute,100420,27320,00:00:00/00:27,10473) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1796 (heute,100456,27272,00:00:00/00:27,10474) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1797 (heute,100420,27492,00:00:00/00:27,10475) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1798 (heute,100420,27364,00:00:00/00:27,10476) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1799 (heute,100452,27512,00:00:00/00:27,10477) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1800 (heute,100452,27344,00:00:00/00:27,10478) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1801 (heute,100416,27540,00:00:00/00:27,10479) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1802 (heute,100456,27584,00:00:00/00:27,10480) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1803 (heute,100440,27344,00:00:00/00:27,10481) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1804 (heute,100416,27628,00:00:00/00:27,10482) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1805 (heute,100412,27448,00:00:00/00:27,10483) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
1806 (heute,102500,27776,00:00:00/00:27,10484) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive --real-time-checks
1807 (root,12612,1092,00:00:00/00:27,10485) icmpsender 8 0 1000
1808 (root,11756,864,00:00:00/00:27,10486) icmpreceiver
1809 (root,0,0,00:00:00/18:03,15867) [kworker/0:2]
1810 (root,0,0,00:00:00/05:28,17253) [kworker/u16:3]
1811 (root,0,0,00:00:01/14:39,22323) [kworker/u16:0]
1812 (root,0,0,00:00:00/14:39,22324) [kworker/u16:9]
1813 (root,0,0,00:00:00/13:46,26453) [kworker/7:0]
1815 MemTotal: 16305836 kB
1816 MemFree: 10181860 kB
1817 MemAvailable: 11599688 kB
1823 Active(anon): 3923808 kB
1824 Inactive(anon): 210564 kB
1825 Active(file): 1257712 kB
1826 Inactive(file): 349956 kB
1829 SwapTotal: 16650236 kB
1830 SwapFree: 16650236 kB
1833 AnonPages: 3919412 kB
1837 SReclaimable: 136136 kB
1838 SUnreclaim: 63792 kB
1839 KernelStack: 14832 kB
1840 PageTables: 70708 kB
1844 CommitLimit: 24803152 kB
1845 Committed_AS: 7579556 kB
1846 VmallocTotal: 34359738367 kB
1849 HardwareCorrupted: 0 kB
1850 AnonHugePages: 1013760 kB
1857 Hugepagesize: 2048 kB
1858 DirectMap4k: 195228 kB
1859 DirectMap2M: 8067072 kB
1860 DirectMap1G: 8388608 kB
1862 2.05 2.31 1.62 1/920 11110 8
1865 <<<lnx_if:sep(58)>>>
1866 lo:462522167348 423391761 0 0 0 0 0 0 462522167348 423391761 0 0 0 0 0 0
1867 eth0:9663614968657 4226434136 0 0 0 0 0 0 41120440285551 1482439946 0 0 0 0 0 0
1868 eth1:9706218845482 3826133052 0 0 0 0 0 0 1746479361684 3786292851 0 0 0 0 0 0
1869 eth2: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
1870 eth3: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
1871 eth4: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
1872 eth5:349900483789 5212016032 0 0 0 0 0 0 185966695259985 5288220641 0 0 0 0 0 0
1873 bond0:19369833814139 8052567188 0 0 0 0 0 0 42866919647235 5268732797 0 0 0 0 0 0
1876 Address: 00:00:00:00:00:00
1880 Auto-negotiation: on
1882 Address: e8:39:35:be:5d:90
1886 Auto-negotiation: on
1888 Address: e8:39:35:be:5d:90
1891 Duplex: Unknown! (255)
1892 Auto-negotiation: off
1894 Address: e8:39:35:be:5d:92
1897 Duplex: Unknown! (255)
1898 Auto-negotiation: off
1900 Address: e8:39:35:be:5d:93
1903 Duplex: Unknown! (255)
1904 Auto-negotiation: off
1906 Address: 80:c1:6e:a9:22:38
1910 Auto-negotiation: off
1912 Address: 80:c1:6e:a9:22:3c
1916 Auto-negotiation: off
1918 Address: e8:39:35:be:5d:90
1919 <<<lnx_bonding:sep(58)>>>
1921 Ethernet Channel Bonding Driver: v3.6.0 (September 26, 2009)
1923 Bonding Mode: IEEE 802.3ad Dynamic link aggregation
1924 Transmit Hash Policy: layer2 (0)
1926 MII Polling Interval (ms): 100
1932 Aggregator selection policy (ad_select): stable
1933 Active Aggregator Info:
1938 Partner Mac Address: 00:23:04:ee:be:0b
1940 Slave Interface: eth0
1944 Link Failure Count: 0
1945 Permanent HW addr: e8:39:35:be:5d:90
1949 Slave Interface: eth1
1953 Link Failure Count: 0
1954 Permanent HW addr: e8:39:35:be:5d:91
1958 <<<tcp_conn_stats>>>
1965 mpathr (36006016067702d001acda7c3695ae211) dm-18 DGC,RAID 5
1966 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
1967 |-+- policy='round-robin 0' prio=0 status=active
1968 | |- 1:0:1:14 sdaq 66:160 active undef running
1969 | `- 3:0:1:14 sdbq 68:64 active undef running
1970 `-+- policy='round-robin 0' prio=0 status=enabled
1971 |- 2:0:0:14 sdq 65:0 active undef running
1972 `- 4:0:1:14 sdcq 69:224 active undef running
1973 mpathe (36006016067702d00fa66b0e3675ae211) dm-27 DGC,RAID 5
1974 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
1975 |-+- policy='round-robin 0' prio=0 status=active
1976 | |- 2:0:0:1 sdd 8:48 active undef running
1977 | `- 4:0:1:1 sdcd 69:16 active undef running
1978 `-+- policy='round-robin 0' prio=0 status=enabled
1979 |- 1:0:1:1 sdad 65:208 active undef running
1980 `- 3:0:1:1 sdbd 67:112 active undef running
1981 mpathq (36006016067702d00a46b8eaa695ae211) dm-13 DGC,RAID 5
1982 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
1983 |-+- policy='round-robin 0' prio=0 status=active
1984 | |- 2:0:0:13 sdp 8:240 active undef running
1985 | `- 4:0:1:13 sdcp 69:208 active undef running
1986 `-+- policy='round-robin 0' prio=0 status=enabled
1987 |- 1:0:1:13 sdap 66:144 active undef running
1988 `- 3:0:1:13 sdbp 68:48 active undef running
1989 mpathd (36006016067702d00c0eb668d675ae211) dm-2 DGC,RAID 5
1990 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
1991 |-+- policy='round-robin 0' prio=0 status=active
1992 | |- 1:0:1:0 sdac 65:192 active undef running
1993 | `- 3:0:1:0 sdbc 67:96 active undef running
1994 `-+- policy='round-robin 0' prio=0 status=enabled
1995 |- 2:0:0:0 sdc 8:32 active undef running
1996 `- 4:0:1:0 sdcc 69:0 active undef running
1997 mpathp (36006016067702d0088203c96695ae211) dm-26 DGC,RAID 5
1998 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
1999 |-+- policy='round-robin 0' prio=0 status=active
2000 | |- 1:0:1:12 sdao 66:128 active undef running
2001 | `- 3:0:1:12 sdbo 68:32 active undef running
2002 `-+- policy='round-robin 0' prio=0 status=enabled
2003 |- 2:0:0:12 sdo 8:224 active undef running
2004 `- 4:0:1:12 sdco 69:192 active undef running
2005 mpathc (36006016006b032002ec7d90f537be211) dm-9 DGC,RAID 5
2006 size=2.3T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2007 |-+- policy='round-robin 0' prio=0 status=active
2008 | |- 2:0:1:0 sdab 65:176 active undef running
2009 | `- 4:0:0:0 sdcb 68:240 active undef running
2010 `-+- policy='round-robin 0' prio=0 status=enabled
2011 |- 1:0:0:0 sdb 8:16 active undef running
2012 `- 3:0:0:0 sdbb 67:80 active undef running
2013 mpatho (36006016067702d00b85c207d695ae211) dm-21 DGC,RAID 5
2014 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2015 |-+- policy='round-robin 0' prio=0 status=active
2016 | |- 2:0:0:11 sdn 8:208 active undef running
2017 | `- 4:0:1:11 sdcn 69:176 active undef running
2018 `-+- policy='round-robin 0' prio=0 status=enabled
2019 |- 1:0:1:11 sdan 66:112 active undef running
2020 `- 3:0:1:11 sdbn 68:16 active undef running
2021 mpathb (36006016067702d00203a0733275be211) dm-17 DGC,RAID 5
2022 size=2.3T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2023 |-+- policy='round-robin 0' prio=0 status=active
2024 | |- 1:0:1:17 sdat 66:208 active undef running
2025 | `- 3:0:1:17 sdbt 68:112 active undef running
2026 `-+- policy='round-robin 0' prio=0 status=enabled
2027 |- 2:0:0:17 sdt 65:48 active undef running
2028 `- 4:0:1:17 sdct 70:16 active undef running
2029 mpathn (36006016067702d0006a5fe67695ae211) dm-8 DGC,RAID 5
2030 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2031 |-+- policy='round-robin 0' prio=0 status=active
2032 | |- 1:0:1:10 sdam 66:96 active undef running
2033 | `- 3:0:1:10 sdbm 68:0 active undef running
2034 `-+- policy='round-robin 0' prio=0 status=enabled
2035 |- 2:0:0:10 sdm 8:192 active undef running
2036 `- 4:0:1:10 sdcm 69:160 active undef running
2037 mpathz (3600601600cd031009e4a05bc2022e411) dm-20 DGC,RAID 5
2038 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2039 |-+- policy='round-robin 0' prio=0 status=active
2040 | |- 1:0:1:24 sdba 67:64 active undef running
2041 | `- 3:0:1:24 sdca 68:224 active undef running
2042 `-+- policy='round-robin 0' prio=0 status=enabled
2043 |- 2:0:0:24 sdaa 65:160 active undef running
2044 `- 4:0:1:24 sdda 70:128 active undef running
2045 mpathm (36006016067702d00fc1e5141695ae211) dm-10 DGC,RAID 5
2046 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2047 |-+- policy='round-robin 0' prio=0 status=active
2048 | |- 2:0:0:9 sdl 8:176 active undef running
2049 | `- 4:0:1:9 sdcl 69:144 active undef running
2050 `-+- policy='round-robin 0' prio=0 status=enabled
2051 |- 1:0:1:9 sdal 66:80 active undef running
2052 `- 3:0:1:9 sdbl 67:240 active undef running
2053 mpathy (3600601600cd03100d88010822022e411) dm-25 DGC,RAID 5
2054 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2055 |-+- policy='round-robin 0' prio=0 status=active
2056 | |- 2:0:0:23 sdz 65:144 active undef running
2057 | `- 4:0:1:23 sdcz 70:112 active undef running
2058 `-+- policy='round-robin 0' prio=0 status=enabled
2059 |- 1:0:1:23 sdaz 67:48 active undef running
2060 `- 3:0:1:23 sdbz 68:208 active undef running
2061 mpathl (36006016067702d00c646981c695ae211) dm-11 DGC,RAID 5
2062 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2063 |-+- policy='round-robin 0' prio=0 status=active
2064 | |- 1:0:1:8 sdak 66:64 active undef running
2065 | `- 3:0:1:8 sdbk 67:224 active undef running
2066 `-+- policy='round-robin 0' prio=0 status=enabled
2067 |- 2:0:0:8 sdk 8:160 active undef running
2068 `- 4:0:1:8 sdck 69:128 active undef running
2069 mpathx (3600601600cd03100ae96394fd111e411) dm-19 DGC,RAID 5
2070 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2071 |-+- policy='round-robin 0' prio=0 status=active
2072 | |- 1:0:1:22 sday 67:32 active undef running
2073 | `- 3:0:1:22 sdby 68:192 active undef running
2074 `-+- policy='round-robin 0' prio=0 status=enabled
2075 |- 2:0:0:22 sdy 65:128 active undef running
2076 `- 4:0:1:22 sdcy 70:96 active undef running
2077 mpathk (36006016067702d00d416f905695ae211) dm-5 DGC,RAID 5
2078 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2079 |-+- policy='round-robin 0' prio=0 status=active
2080 | |- 2:0:0:7 sdj 8:144 active undef running
2081 | `- 4:0:1:7 sdcj 69:112 active undef running
2082 `-+- policy='round-robin 0' prio=0 status=enabled
2083 |- 1:0:1:7 sdaj 66:48 active undef running
2084 `- 3:0:1:7 sdbj 67:208 active undef running
2085 mpathw (3600601600cd0310046b0242476fbe311) dm-23 DGC,RAID 5
2086 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2087 |-+- policy='round-robin 0' prio=0 status=active
2088 | |- 2:0:0:21 sdx 65:112 active undef running
2089 | `- 4:0:1:21 sdcx 70:80 active undef running
2090 `-+- policy='round-robin 0' prio=0 status=enabled
2091 |- 1:0:1:21 sdax 67:16 active undef running
2092 `- 3:0:1:21 sdbx 68:176 active undef running
2093 mpathj (36006016067702d001a8376ef685ae211) dm-6 DGC,RAID 5
2094 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2095 |-+- policy='round-robin 0' prio=0 status=active
2096 | |- 1:0:1:6 sdai 66:32 active undef running
2097 | `- 3:0:1:6 sdbi 67:192 active undef running
2098 `-+- policy='round-robin 0' prio=0 status=enabled
2099 |- 2:0:0:6 sdi 8:128 active undef running
2100 `- 4:0:1:6 sdci 69:96 active undef running
2101 mpathv (3600601600cd0310058dc725dd9c3e311) dm-22 DGC,RAID 5
2102 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2103 |-+- policy='round-robin 0' prio=0 status=active
2104 | |- 1:0:1:20 sdaw 67:0 active undef running
2105 | `- 3:0:1:20 sdbw 68:160 active undef running
2106 `-+- policy='round-robin 0' prio=0 status=enabled
2107 |- 2:0:0:20 sdw 65:96 active undef running
2108 `- 4:0:1:20 sdcw 70:64 active undef running
2109 mpathi (36006016067702d00a0b3acd5685ae211) dm-12 DGC,RAID 5
2110 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2111 |-+- policy='round-robin 0' prio=0 status=active
2112 | |- 2:0:0:5 sdh 8:112 active undef running
2113 | `- 4:0:1:5 sdch 69:80 active undef running
2114 `-+- policy='round-robin 0' prio=0 status=enabled
2115 |- 1:0:1:5 sdah 66:16 active undef running
2116 `- 3:0:1:5 sdbh 67:176 active undef running
2117 mpathu (3600601600cd0310042f4ce68f26ee311) dm-24 DGC,RAID 5
2118 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2119 |-+- policy='round-robin 0' prio=0 status=active
2120 | |- 2:0:0:19 sdv 65:80 active undef running
2121 | `- 4:0:1:19 sdcv 70:48 active undef running
2122 `-+- policy='round-robin 0' prio=0 status=enabled
2123 |- 1:0:1:19 sdav 66:240 active undef running
2124 `- 3:0:1:19 sdbv 68:144 active undef running
2125 mpathaa (3600601600cd03100f87503f8fb63e411) dm-14 DGC,RAID 5
2126 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2127 |-+- policy='round-robin 0' prio=0 status=active
2128 | |- 2:0:0:16 sdas 66:192 active undef running
2129 | `- 4:0:1:16 sdcs 70:0 active undef running
2130 `-+- policy='round-robin 0' prio=0 status=enabled
2131 |- 1:0:1:16 sds 65:32 active undef running
2132 `- 3:0:1:16 sdbs 68:96 active undef running
2133 mpathh (36006016067702d0068238084685ae211) dm-4 DGC,RAID 5
2134 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2135 |-+- policy='round-robin 0' prio=0 status=active
2136 | |- 1:0:1:4 sdag 66:0 active undef running
2137 | `- 3:0:1:4 sdbg 67:160 active undef running
2138 `-+- policy='round-robin 0' prio=0 status=enabled
2139 |- 2:0:0:4 sdg 8:96 active undef running
2140 `- 4:0:1:4 sdcg 69:64 active undef running
2141 mpatht (3600601600cd03100d6ef6cdfb034e311) dm-16 DGC,RAID 5
2142 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2143 |-+- policy='round-robin 0' prio=0 status=active
2144 | |- 2:0:0:18 sdu 65:64 active undef running
2145 | `- 4:0:1:18 sdcu 70:32 active undef running
2146 `-+- policy='round-robin 0' prio=0 status=enabled
2147 |- 1:0:1:18 sdau 66:224 active undef running
2148 `- 3:0:1:18 sdbu 68:128 active undef running
2149 mpathg (36006016067702d00607ac85b685ae211) dm-7 DGC,RAID 5
2150 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2151 |-+- policy='round-robin 0' prio=0 status=active
2152 | |- 2:0:0:3 sdf 8:80 active undef running
2153 | `- 4:0:1:3 sdcf 69:48 active undef running
2154 `-+- policy='round-robin 0' prio=0 status=enabled
2155 |- 1:0:1:3 sdaf 65:240 active undef running
2156 `- 3:0:1:3 sdbf 67:144 active undef running
2157 mpaths (36006016067702d007222c7d8695ae211) dm-15 DGC,RAID 5
2158 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2159 |-+- policy='round-robin 0' prio=0 status=active
2160 | |- 2:0:0:15 sdr 65:16 active undef running
2161 | `- 4:0:1:15 sdcr 69:240 active undef running
2162 `-+- policy='round-robin 0' prio=0 status=enabled
2163 |- 1:0:1:15 sdar 66:176 active undef running
2164 `- 3:0:1:15 sdbr 68:80 active undef running
2165 mpathf (36006016067702d00fc530441685ae211) dm-3 DGC,RAID 5
2166 size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
2167 |-+- policy='round-robin 0' prio=0 status=active
2168 | |- 1:0:1:2 sdae 65:224 active undef running
2169 | `- 3:0:1:2 sdbe 67:128 active undef running
2170 `-+- policy='round-robin 0' prio=0 status=enabled
2171 |- 2:0:0:2 sde 8:64 active undef running
2172 `- 4:0:1:2 sdce 69:32 active undef running
2174 Personalities : [raid6] [raid5] [raid4] [raid1] [raid10]
2175 unused devices: <none>
2178 8 32 sdc 180321 12 1442664 1251715 0 0 0 0 0 1251539 1251539
2179 8 64 sde 180756 12 1446144 1041714 0 0 0 0 0 1041539 1041539
2180 8 96 sdg 180510 12 1444176 1107684 0 0 0 0 0 1107504 1107504
2181 8 144 sdj 169783423 12 43843540411 632255890 33607604 0 6267004625 272659047 0 419839234 904894095
2182 8 128 sdi 180646 12 1445264 1256029 0 0 0 0 0 1255842 1255842
2183 8 80 sdf 174080499 12 44161773254 876550093 33367613 0 6226195383 273863266 0 515985152 1150393431
2184 8 192 sdm 181055 12 1448536 476885 0 0 0 0 0 476709 476710
2185 8 16 sdb 181462 0 1451744 165494 0 0 0 0 0 165300 165300
2186 8 176 sdl 174430992 12 44043387762 474107972 33729801 0 6238082381 269983859 0 371355442 744068918
2187 8 160 sdk 180924 12 1447488 496736 0 0 0 0 0 496561 496561
2188 8 112 sdh 168369724 12 43780869363 739540124 34057534 0 6274171212 271958724 0 485493467 1011496108
2189 8 240 sdp 179081916 12 44237537945 479923725 34337443 0 6262719427 259427653 0 375627878 739328253
2190 65 16 sdr 172927164 12 43985653571 478338619 33530107 0 6260343800 265087257 0 373386477 743402893
2191 8 0 sda 763833 257293 20424501 1287709 93635159 72414348 1329396911 54406558 0 19533773 55678513
2192 65 64 sdu 172005076 12 43879459000 910813851 33484118 0 6271441545 265358933 0 529413152 1176144435
2193 65 48 sdt 182792 0 1462384 85775 0 0 0 0 0 85585 85585
2194 65 0 sdq 180988 12 1448000 489333 0 0 0 0 0 489149 489149
2195 65 128 sdy 181078 12 1448720 733531 0 0 0 0 0 733360 733360
2196 65 160 sdaa 182901 12 1463304 186498 0 0 0 0 0 186301 186301
2197 8 208 sdn 165811096 12 43489751870 463351380 33253088 0 6240545446 264381349 0 364249884 727730789
2198 65 96 sdw 180766 12 1446224 809284 0 0 0 0 0 809122 809122
2199 65 112 sdx 176288987 12 44186946417 835657359 33809994 0 6305896725 274596304 0 490712014 1110230675
2200 65 80 sdv 170318962 12 43977833942 944226323 33124746 0 6267302867 271187425 1 540600702 1215395619
2201 65 144 sdz 29327803 12 19834748713 94500294 70967487 0 37480100425 204162893 0 151405638 298652879
2202 8 224 sdo 181023 12 1448280 490418 0 0 0 0 0 490256 490256
2203 65 224 sdae 186309648 12 44878658934 885812511 33130641 0 6216997054 209754505 1 516642113 1095557121
2204 65 192 sdac 177995876 12 44049002280 883203293 33725709 0 6272541739 207590675 0 521909280 1090774650
2205 65 240 sdaf 180295 12 1442456 1045278 0 0 0 0 0 1045113 1045113
2206 65 208 sdad 179668 12 1437440 1338595 0 0 0 0 0 1338405 1338405
2207 66 16 sdah 180383 12 1443160 1102987 0 0 0 0 0 1102829 1102829
2208 66 0 sdag 170872794 12 43784837168 894290613 33545944 0 6246324421 210908020 0 521557389 1105174735
2209 66 32 sdai 181504782 12 44363628577 702399830 34081997 0 6305912018 204797792 0 474274130 907174012
2210 65 176 sdab 247172 0 37664418 350507 79274383 0 4091088125 183615953 0 18743294 183962880
2211 66 48 sdaj 180787 12 1446392 661241 0 0 0 0 0 661074 661075
2212 66 64 sdak 174245864 12 44036523847 447450546 33295568 0 6232289036 199808108 0 358725196 647235404
2213 66 80 sdal 180914 12 1447408 442235 0 0 0 0 0 442088 442088
2214 66 96 sdam 175041996 12 44170381939 446980535 33904064 0 6300198883 202889881 0 359058644 649849681
2215 66 112 sdan 180801 12 1446504 446759 0 0 0 0 0 446585 446585
2216 66 128 sdao 175192675 12 44022223946 452434848 33429281 0 6238432664 196552202 0 360931362 648967634
2217 66 160 sdaq 170071547 12 44008413266 443712383 33245032 0 6243941408 196678855 0 357874991 640365722
2218 66 144 sdap 180925 12 1447496 448385 0 0 0 0 0 448202 448202
2219 66 176 sdar 180962 12 1447792 447692 0 0 0 0 0 447521 447521
2220 66 208 sdat 11178914 0 1640585660 6508172 82750574 0 4075968015 183318258 0 20202802 189821198
2221 66 224 sdau 180608 12 1444960 828058 0 0 0 0 0 827863 827863
2222 66 240 sdav 180630 12 1445136 759952 0 0 0 0 0 759789 759789
2223 67 16 sdax 180698 12 1445680 754536 0 0 0 0 0 754381 754381
2224 67 32 sday 194834036 12 44230490997 1012668689 46147716 0 6579113969 219466311 0 503675456 1232105499
2225 67 48 sdaz 182986 12 1463984 171884 0 0 0 0 0 171680 171680
2226 67 64 sdba 29430660 12 19837422627 75988180 71865148 0 37489089073 205210368 0 146192791 281186694
2227 67 96 sdbc 177985736 12 44048994151 870445301 33735830 0 6277865722 202870095 0 516959883 1073296447
2228 67 160 sdbg 170861314 12 43783159047 882462595 33557397 0 6250557844 205498363 0 517056458 1087934877
2229 67 128 sdbe 186299662 12 44879578681 873038598 33140610 0 6222094799 204080767 0 512156350 1077111943
2230 67 112 sdbd 179674 12 1437488 1199756 0 0 0 0 0 1199607 1199607
2231 67 176 sdbh 180443 12 1443640 1165030 0 0 0 0 0 1164887 1164887
2232 67 144 sdbf 180334 12 1442768 1054992 0 0 0 0 0 1054844 1054844
2233 67 0 sdaw 172237941 12 44028533511 856018770 34063767 0 6277722377 209140501 0 502708439 1065138181
2234 67 192 sdbi 181487264 12 44359849697 692539799 34099488 0 6310822859 200192713 0 470207303 892710232
2235 67 208 sdbj 180820 12 1446656 658080 0 0 0 0 0 657912 657912
2236 68 0 sdbm 175030539 12 44165002768 440432812 33915514 0 6305875984 198048850 0 355051155 638462305
2237 68 16 sdbn 180842 12 1446832 445901 0 0 0 0 0 445737 445737
2238 67 240 sdbl 180959 12 1447768 440393 0 0 0 0 0 440250 440250
2239 68 48 sdbp 180971 12 1447864 427733 0 0 0 0 0 427571 427572
2240 68 32 sdbo 175188646 12 44022836016 444765923 33433320 0 6241705141 191938838 0 356473225 636685224
2241 68 80 sdbr 180987 12 1447992 435237 0 0 0 0 0 435084 435084
2242 67 224 sdbk 174242489 12 44035838106 440562259 33298959 0 6237545717 195136228 0 354761607 635676055
2243 67 80 sdbb 181695 0 1453608 151168 0 0 0 0 0 151011 151011
2244 8 48 sdd 170164487 12 43977422611 960984288 33095410 0 6270416349 274433282 0 547728889 1235392096
2245 68 128 sdbu 180635 12 1445176 816342 0 0 0 0 0 816157 816157
2246 68 112 sdbt 11180535 0 1641029565 6213475 82748952 0 4085426928 170733161 0 19624286 176941541
2247 68 160 sdbw 172226000 12 44026020943 843800787 34075671 0 6282424561 204058656 0 498054068 1047841683
2248 68 176 sdbx 180712 12 1445792 739235 0 0 0 0 0 739064 739064
2249 68 144 sdbv 180652 12 1445312 762070 0 0 0 0 0 761917 761917
2250 68 224 sdca 29434530 12 19847199942 74974270 71861288 0 37458821939 198452110 0 143345319 273418621
2251 68 208 sdbz 183017 12 1464232 167450 0 0 0 0 0 167281 167281
2252 68 192 sdby 194831142 12 44231144674 998352237 46150568 0 6580853462 214406449 0 498958973 1212736143
2253 68 240 sdcb 246959 0 38282366 346591 79274503 0 4070306818 197825372 0 18800649 198167887
2254 68 64 sdbq 170052822 12 44003342545 437435356 33263765 0 6252193096 192273318 0 353842472 629689046
2255 69 0 sdcc 180393 12 1443240 1191260 0 0 0 0 0 1191085 1191085
2256 69 16 sdcd 170155375 12 43977233214 954615083 33104521 0 6272932550 269379914 0 544438833 1223974580
2257 69 48 sdcf 174074108 12 44162468086 871606773 33374045 0 6228157343 268761461 0 512858811 1140349762
2258 69 64 sdcg 180582 12 1444752 1039199 0 0 0 0 0 1039018 1039018
2259 69 80 sdch 168364891 12 43781635102 733940875 34062438 0 6276678783 267117653 0 482424460 1001040414
2260 69 96 sdci 180742 12 1446032 1461326 0 0 0 0 0 1461153 1461153
2261 69 112 sdcj 169773690 12 43844470437 627177003 33617434 0 6272108509 267741754 0 416887613 894898535
2262 69 128 sdck 181022 12 1448272 397470 0 0 0 0 0 397318 397318
2263 69 144 sdcl 174425550 12 44043877357 470651620 33735361 0 6244608112 265091242 0 368338790 735731711
2264 69 160 sdcm 181179 12 1449528 399820 0 0 0 0 0 399670 399670
2265 69 176 sdcn 165805336 12 43489139065 459852314 33258939 0 6243539101 259542207 0 361118394 719391696
2266 69 208 sdcp 179075893 12 44235613446 476218679 34343594 0 6266874925 255091995 0 372410513 731286674
2267 69 224 sdcq 181077 12 1448712 388206 0 0 0 0 0 388040 388040
2268 69 240 sdcr 172925679 12 43986766379 474740104 33531707 0 6264954943 260378041 0 370276347 735093414
2269 70 16 sdct 182949 0 1463616 75280 0 0 0 0 0 75092 75092
2270 70 32 sdcu 172000515 12 43881551438 905360852 33488708 0 6274106221 260708386 0 525817139 1166049486
2271 70 48 sdcv 170317533 12 43980327184 937140947 33126218 0 6270338043 266645186 1 536720565 1203765605
2272 69 192 sdco 181121 12 1449064 420705 0 0 0 0 0 420569 420569
2273 70 64 sdcw 180810 12 1446576 737315 0 0 0 0 0 737144 737144
2274 70 80 sdcx 176285484 12 44187992047 830577294 33813549 0 6306223350 269392964 0 487508723 1099948790
2275 70 96 sdcy 181139 12 1449208 661518 0 0 0 0 0 661371 661371
2276 70 112 sdcz 29333492 12 19847258821 93679318 70961810 0 37452505355 199080400 0 148342124 292747890
2277 70 128 sdda 183031 12 1464344 172563 0 0 0 0 0 172380 172380
2278 69 32 sdce 180899 12 1447288 1018988 0 0 0 0 0 1018827 1018827
2279 253 0 dm-0 6993 0 184474 9111 1983314 0 15866512 917081 0 398390 926192
2280 253 1 dm-1 463 0 3704 663 0 0 0 0 0 504 663
2281 253 2 dm-2 355620743 181965 88095109287 1759320629 67461539 3586607 12550407461 530974294 0 685031846 2290275532
2282 253 3 dm-3 372247534 436932 89755343215 1766692941 66271251 3544279 12439091853 527944431 1 675870267 2294673597
2283 253 4 dm-4 341372887 289582 87565106255 1781914829 67103341 3541184 12496882265 546852230 0 682399766 2328802843
2284 253 5 dm-5 339195464 232151 87685117464 1266056490 67225038 3783221 12539113134 727059471 0 557041987 1993461010
2285 253 6 dm-6 362630521 188683 88720585882 1399068217 68181485 3622520 12616734877 516187962 0 629261336 1915204532
2286 253 7 dm-7 347793899 175304 88321355484 1755141077 66741658 3757966 12454352726 722314944 0 673325478 2477635984
2287 253 8 dm-8 349710138 189053 88332485339 893322661 67819578 3597465 12606074867 511580436 0 486480633 1404828342
2288 253 9 dm-9 129834 78285 73032240 423928 158548886 741875294 8161394943 408946368 0 23830227 409308176
2289 253 10 dm-10 348494628 170780 88084369615 951086974 67465162 3725637 12482690493 722886661 0 500387744 1674151941
2290 253 11 dm-11 348126225 197713 88069464737 894205421 66594527 3531631 12469834753 494562916 0 485887657 1388688948
2291 253 12 dm-12 336373719 258381 87559617105 1479076747 68119972 3785392 12550849995 711168313 0 640129098 2190434308
2292 253 13 dm-13 357795868 318210 88470255671 962221766 68681037 3708504 12529594352 658825438 0 507403974 1621115021
2293 253 15 dm-15 345490863 351487 87969523918 958962221 67061814 3784906 12525298743 700214690 0 503183769 1659323009
2294 253 16 dm-16 343644279 385533 87758119750 1825583470 66972826 3789906 12545547766 686023443 0 692735006 2511833548
2295 253 17 dm-17 21993029 4900505 3278683697 12928618 165499526 734924775 8161394943 384910692 0 27793966 397757231
2296 253 18 dm-18 339762144 337181 88008857819 886732799 66508797 3612637 12496134504 489654118 0 484152024 1376430941
2297 253 19 dm-19 389302830 222475 88458736695 2071186275 92298284 3552796 13159967431 561316323 0 668125162 2632451255
2298 253 20 dm-20 58498764 29179 39681690969 152895143 143726436 72262 74947911012 413516717 0 216996879 566145937
2299 253 21 dm-21 331254727 170920 86975997103 928940402 66512027 3743173 12484084547 692613143 0 491301422 1621766524
2300 253 22 dm-22 344102290 184997 88051661054 1706014307 68139438 3614308 12560146938 539611589 0 660738746 2245602928
2301 253 23 dm-23 352213003 214942 88372046528 1673950329 67623543 3853219 12612120075 737318454 0 645355899 2411542268
2302 253 24 dm-24 340275155 191741 87955270438 1890414127 66250964 3757863 12537640910 715552595 0 705992255 2606116574
2303 253 25 dm-25 58295032 29854 39679077238 189811590 141929297 73756 74932605780 416742838 0 220730140 606290020
2304 253 26 dm-26 350019080 169919 88042161842 902756087 66862601 3589118 12480137805 484464019 0 488945127 1387255496
2305 253 27 dm-27 339960452 402252 87951780353 1924302229 66199931 3786607 12543348899 730221212 0 713953512 2654743738
2306 253 58 dm-58 720 0 8426 1999 25091973 0 200735784 17886301 0 6577787 17915588
2307 253 59 dm-59 4585 0 189858 5496 6154916 0 49239328 69346841 0 1272814 69368241
2308 253 60 dm-60 29663 0 789962 38937 145652 0 1165216 1182859 0 31882 1221797
2309 253 61 dm-61 2483 0 41106 3499 1956 0 15648 1147 0 1740 4646
2310 253 62 dm-62 21884 0 615442 256846 93364571 0 746916568 44572422 0 3956584 44963459
2311 253 63 dm-63 2209 0 105266 3958 1433474 0 11467792 1691185 0 1659598 1695145
2312 253 64 dm-64 718300 0 10951069 1036294 13592348 0 108736055 10488518 0 1655581 11530820
2313 253 65 dm-65 180487 0 4235138 253796 9522619 0 76180952 5364362 0 1668153 5620801
2314 253 66 dm-66 18539 0 148306 22529 12763 0 102104 17661 0 2445 40189
2315 253 67 dm-67 20260 0 2865066 120881 4234727 0 33877816 202788643 0 251795 202954806
2316 253 68 dm-68 14328 0 275370 16886 10636477 0 85091816 12990462 0 2123205 13012289
2317 253 33 dm-33 2 0 2 1 2967567 0 19943214 3147629 0 2894996 3148970
2318 253 34 dm-34 260316 0 129544458 915203 383884750 0 3084082800 1982485129 0 5735608 1986345045
2319 253 35 dm-35 185 0 1440 113 2967567 0 19943214 4422825 0 3979685 4424014
2320 253 36 dm-36 17520 0 4984056 235445 383884750 0 3084082800 1610406883 0 5364824 1614013947
2321 253 37 dm-37 277836 0 134528514 1151916 383884750 0 3084082800 1397670470 0 8330660 1408574433
2322 253 38 dm-38 1 0 1 0 3157984 0 20801070 2919595 0 2842159 2919793
2323 253 39 dm-39 104873 0 50389434 271337 453554156 0 3628433248 1014232621 0 9291791 1016214459
2324 253 40 dm-40 18 0 128 20 3157984 0 20801070 4085395 0 3975849 4085581
2325 253 41 dm-41 28217 0 14144840 106309 453554156 0 3628433248 1488862180 0 11578686 1490928857
2326 253 42 dm-42 133090 0 64534274 378094 453554156 0 3628433248 3425594752 0 14281221 3432140713
2327 253 43 dm-43 1 0 1 0 2606 0 2606 1010 0 996 1010
2328 253 44 dm-44 364 0 10226 671 1167 0 9336 595 0 897 1266
2329 253 45 dm-45 3 0 3 2 2606 0 2606 3198 0 3161 3200
2330 253 46 dm-46 9 0 72 53 1167 0 9336 1191 0 658 1244
2331 253 47 dm-47 373 0 10298 724 1167 0 9336 4128 0 3343 4857
2332 253 48 dm-48 1 0 1 0 739701 0 5357469 406983 0 400044 406995
2333 253 49 dm-49 26485153 0 3093436614 19432378 20956882 0 1121648960 29607242 0 9955477 49070920
2334 253 50 dm-50 6 0 33 2 739701 0 5357469 827032 0 810277 827051
2335 253 51 dm-51 152139 0 52525212 305231 20956882 0 1121648960 45509459 0 2504061 45842266
2336 253 52 dm-52 24437304 0 3145961826 16458127 20193935 0 1121648960 70742718 0 11637871 87245526
2337 253 53 dm-53 1 0 1 0 157723 0 1096362 210158 0 204798 210189
2338 253 54 dm-54 41912 0 5292578 107227 35002449 0 280019592 106698529 0 467307 107031318
2339 253 55 dm-55 6 0 28 9 157723 0 1096362 298267 0 287825 298318
2340 253 56 dm-56 9385 0 1369448 55267 35002449 0 280019592 131355158 0 465576 131636552
2341 253 57 dm-57 51297 0 6662026 162742 35002449 0 280019592 404252302 0 638924 405270113
2342 65 32 sds 60166 0 481376 360919 0 0 0 0 0 360739 360841
2343 253 14 dm-14 129396173 83059 27104014468 680283380 34011478 1140542 5943777444 750978405 0 210345105 1431327505
2344 66 192 sdas 64759279 0 13552125660 325631056 17004589 0 2972152776 245107384 0 159785149 570730478
2345 68 96 sdbs 59991 0 479928 319839 0 0 0 0 0 319792 319793
2346 70 0 sdcs 64756720 0 13552847368 323976097 17006889 0 2971624668 240668581 0 158802441 564637339
2349 dwhvg-lv_dingdata_rimage_0 253:39 dwhvg lv_dingdata_rimage_0
2351 dwhvg-lv_backup_rimage_1 253:51 dwhvg lv_backup_rimage_1
2352 rootvg-lv_swap 253:1 rootvg lv_swap
2353 rootvg-lv_usr 253:60 rootvg lv_usr
2354 rootvg-lv_root 253:0 rootvg lv_root
2355 rootvg-lv_var 253:62 rootvg lv_var
2358 dwhvg-lv_backup_rimage_0 253:49 dwhvg lv_backup_rimage_0
2359 dwhvg-lv_conndir 253:47 dwhvg lv_conndir
2364 rootvg-lv_install 253:66 rootvg lv_install
2365 rootvg-lv_local 253:61 rootvg lv_local
2367 dwhvg-lv_backup 253:52 dwhvg lv_backup
2370 dwhvg-lv_opt_uniserv 253:57 dwhvg lv_opt_uniserv
2371 rootvg-lv_opt 253:63 rootvg lv_opt
2374 rootvg-lv_oracle11 253:64 rootvg lv_oracle11
2375 dwhvg-lv_backup_rmeta_1 253:50 dwhvg lv_backup_rmeta_1
2378 dwhvg-lv_opt_uniserv_rmeta_1 253:55 dwhvg lv_opt_uniserv_rmeta_1
2379 dwhvg-lv_dingdata 253:42 dwhvg lv_dingdata
2380 dwhvg-lv_backup_rmeta_0 253:48 dwhvg lv_backup_rmeta_0
2383 dwhvg-lv_opt_uniserv_rmeta_0 253:53 dwhvg lv_opt_uniserv_rmeta_0
2385 dwhvg-lv_conndir_rimage_1 253:46 dwhvg lv_conndir_rimage_1
2387 dwhvg-lv_opt_uniserv_rimage_1 253:56 dwhvg lv_opt_uniserv_rimage_1
2388 dwhvg-lv_dingdata_rmeta_1 253:40 dwhvg lv_dingdata_rmeta_1
2391 dwhvg-lv_conndir_rimage_0 253:44 dwhvg lv_conndir_rimage_0
2392 rootvg-lv_home 253:59 rootvg lv_home
2394 dwhvg-lv_opt_uniserv_rimage_0 253:54 dwhvg lv_opt_uniserv_rimage_0
2395 dwhvg-lv_conndir_rmeta_1 253:45 dwhvg lv_conndir_rmeta_1
2396 rootvg-lv_tmp 253:58 rootvg lv_tmp
2397 dwhvg-lv_dingdata_rmeta_0 253:38 dwhvg lv_dingdata_rmeta_0
2398 rootvg-lv_agent 253:68 rootvg lv_agent
2400 rootvg-lv_oragrid 253:65 rootvg lv_oragrid
2402 dwhvg-lv_conndir_rmeta_0 253:43 dwhvg lv_conndir_rmeta_0
2404 dwhvg-lv_dingdata_rimage_1 253:41 dwhvg lv_dingdata_rimage_1
2408 unused devices: <none>
2410 * 172.33.44.49 29.244.104.45 2 u 2 64 377 0.302 -0.353 0.483
2411 + 172.44.33.19 29.244.104.33 2 u 18 64 377 0.408 0.051 0.319
2416 /var/spool/mqueue is empty
2419 thermal_zone0 - x86_pkg_temp 59000 0 passive 0 passive
2420 thermal_zone1 - BAT0 35000
2422 [[[/var/log/messages:missing]]]
2423 [[[/var/log/kern.log]]]
2424 [[[/var/log/auth.log]]]
2425 [[[/var/log/syslog]]]
2429 # .--Test Cases----------------------------------------------------------.
2431 # | |_ _|__ ___| |_ / ___|__ _ ___ ___ ___ |
2432 # | | |/ _ \/ __| __| | | / _` / __|/ _ \/ __| |
2433 # | | | __/\__ \ |_ | |__| (_| \__ \ __/\__ \ |
2434 # | |_|\___||___/\__| \____\__,_|___/\___||___/ |
2436 # +----------------------------------------------------------------------+
2437 # | The test cases implement different kinds of tests |
2438 # '----------------------------------------------------------------------'
2441 class TestCase(object):
2442 STATE_INITIALIZING
= "initializing"
2443 STATE_RUNNING
= "running"
2444 STATE_STOPPED
= "stopped"
2446 RESULT_COMPLETED
= "completed"
2447 RESULT_SKIPPED
= "skipped"
2448 RESULT_ABORTED
= "aborted"
2449 RESULT_FAILED
= "failed"
2452 self
.started_at
= None
2453 self
.finished_at
= None
2456 self
.result_msg
= None
2458 # Return a meaningful name to identify this test case
2460 return "%s()" % self
.__class
__.__name
__
2462 # The run() method can be implemented as generator and yield several intermediate steps
2463 # to the Test() object. In case a Wait(duration) object is passed, the Test() object will
2464 # wait for the given duration and then call the run() method again.
2466 raise NotImplementedError()
2469 if self
.started_at
is None:
2471 elif self
.finished_at
is None:
2472 return time
.time() - self
.started_at
2473 return self
.finished_at
- self
.started_at
2475 def serialize(self
):
2476 d
= dict([(k
, v
) for k
, v
in self
.__dict
__.items() if k
[0] != "_"])
2477 d
["__class_name__"] = self
.__class
__.__name
__
2480 def deserialize(self
, raw
):
2481 self
.__dict
__.update(raw
)
2483 def set_state(self
, state
, result
=None, result_msg
=None):
2485 self
.result
= result
2486 self
.result_msg
= result_msg
2488 if self
.state
== TestCase
.STATE_RUNNING
:
2489 self
.started_at
= time
.time()
2490 elif self
.state
== TestCase
.STATE_STOPPED
:
2491 self
.finished_at
= time
.time()
2493 def state_text(self
):
2497 class TestAgentBasedStandardHosts(TestCase
):
2498 """A test that creates the specified number of "standard" hosts
2500 It adds the hosts to Check_MK and starts the monitoring of these
2501 hosts. The hosts will be configured with a standard set of services
2502 which is assumed to be a good configuration for the average Check_MK
2505 The test will simulate an average network delay and process dynamic
2506 agent data that will lead to increasing counters in the checks.
2509 host_name_template
= "std-agent-host-%05d"
2511 def __init__(self
, runner
=None, num_hosts
=None, planned_duration
=None, num_cmk_helpers
=20):
2512 super(TestAgentBasedStandardHosts
, self
).__init
__()
2513 self
._runner
= runner
2514 self
._test
= runner
.test
if runner
else None
2516 self
.num_hosts
= num_hosts
2517 self
.planned_duration
= planned_duration
2518 self
.num_cmk_helpers
= num_cmk_helpers
2520 self
._listeners
= []
2521 self
._hosts
_per
_listener
= 100
2524 return "%s(hosts=%d, duration=%d, helpers=%d)" % \
2525 (self
.__class
__.__name
__,
2527 self
.planned_duration
,
2528 self
.num_cmk_helpers
)
2532 self
._spawn
_agent
_listeners
()
2533 self
._write
_global
_settings
()
2534 self
._create
_hosts
()
2535 self
._discover
_services
()
2536 self
._activate
_changes
()
2538 self
._wait
_for
_cmk
_helpers
_ready
()
2540 yield Wait(self
.planned_duration
)
2542 self
._cleanup
_hosts
()
2543 self
._activate
_changes
()
2544 self
._runner
._cleanup
_host
_files
()
2545 self
._end
_agent
_listeners
()
2547 def _spawn_agent_listeners(self
):
2548 self
._test
.info(" Starting agent listeners")
2549 for first_host_id
in range(0, self
.num_hosts
, self
._hosts
_per
_listener
):
2550 host_ids
= range(first_host_id
, first_host_id
+ self
._hosts
_per
_listener
)
2551 listener
= AgentListener(host_ids
)
2553 self
._listeners
.append(listener
)
2555 def _end_agent_listeners(self
):
2556 self
._test
.info(" Stopping agent listeners")
2557 for listener
in self
._listeners
:
2558 listener
.terminate()
2560 for listener
in self
._listeners
:
2562 self
._test
.info(" Stopped listeners")
2564 def _write_global_settings(self
):
2565 self
._test
.info(" Configuring site")
2566 config_dir
= self
._cmk
_config
_dir
()
2568 os
.makedirs(config_dir
)
2570 if e
.errno
== 17: # file exists
2575 with
open("%s/global_settings.mk" % config_dir
, "w") as f
:
2576 f
.write("# Created by mkbench\n"
2577 "cmc_cmk_helpers = %d\n"
2578 "cmc_log_limit = 209715200\n"
2579 "cmc_initial_scheduling = {\n"
2581 " 'spread_cmk': 10,\n"
2582 " 'spread_generic': 150\n"
2584 "\n" % self
.num_cmk_helpers
)
2586 with
open("%s/rules.mk" % config_dir
, "w") as f
:
2587 f
.write("agent_ports = [\n" " ( 6559, [], ALL_HOSTS, {} ),\n" "] + agent_ports\n")
2589 f
.write("tcp_connect_timeouts = [\n"
2590 " ( 5.0, [], ALL_HOSTS, {} ),\n"
2591 "] + tcp_connect_timeouts\n")
2593 # Ensure RRDs are created with Check_MK default config
2595 "cmc_host_rrd_config = [\n"
2597 " (50.0, 1, 2880), (50.0, 5, 2880),\n"
2598 " (50.0, 30, 4320), (50.0, 360, 5840)\n"
2601 " 'cfs': ['MIN', 'MAX', 'AVERAGE'],\n"
2602 " 'format': 'cmc_single'\n"
2604 " [], ALL_HOSTS, {\n"
2605 " 'description': u'Default RRD configuration, using new single RRD format'\n"
2607 "] + cmc_host_rrd_config\n"
2610 "cmc_service_rrd_config = [\n"
2612 " (50.0, 1, 2880), (50.0, 5, 2880),\n"
2613 " (50.0, 30, 4320), (50.0, 360, 5840)\n"
2616 " 'cfs': ['MIN', 'MAX', 'AVERAGE'],\n"
2617 " 'format': 'cmc_single'\n"
2619 " [], ALL_HOSTS, ALL_SERVICES ),\n"
2620 "] + cmc_service_rrd_config\n")
2622 def _create_hosts(self
):
2623 self
._test
.info(" Creating hosts")
2624 config_dir
= self
._cmk
_config
_dir
()
2626 os
.makedirs(config_dir
)
2628 if e
.errno
== 17: # file exists
2633 all_hosts
, ipaddresses
, host_attributes
= self
._create
_host
_data
()
2635 with
open("%s/hosts.mk" % config_dir
, "w") as f
:
2636 f
.write("# Created by mkbench\n"
2640 "ipaddresses.update(%s)\n"
2642 "host_attributes.update(%s)\n" % (pprint
.pformat(all_hosts
),
2643 pprint
.pformat(ipaddresses
),
2644 pprint
.pformat(host_attributes
)))
2646 with
open("%s/rules.mk" % config_dir
, "w") as f
:
2647 f
.write("agent_ports = [\n" " ( 6559, [], ALL_HOSTS, {} ),\n" "] + agent_ports\n")
2649 f
.write("tcp_connect_timeouts = [\n"
2650 " ( 5.0, [], ALL_HOSTS, {} ),\n"
2651 "] + tcp_connect_timeouts\n")
2653 def _create_host_data(self
):
2654 all_hosts
, ipaddresses
, host_attributes
= [], {}, {}
2656 for num
in range(self
.num_hosts
):
2657 host_name
= self
.host_name_template
% num
2658 ipaddress
= get_host_ip(num
)
2661 "%s|lan|ip-v4|cmk-agent|tcp|site:%s|ip-v4-only|prod|bench" % (host_name
, site_id()))
2663 ipaddresses
[host_name
] = ipaddress
2665 host_attributes
[host_name
] = {
2666 "ipaddress": ipaddress
,
2669 return all_hosts
, ipaddresses
, host_attributes
2671 def _discover_services(self
):
2672 self
._test
.info(" Discovering services")
2674 # Discver first host and copy the rest (all hosts are equal)
2675 first_host_name
= self
.host_name_template
% 0
2676 self
._discover
_services
_of
_host
(first_host_name
)
2678 autochecks_dir
= "%s/var/check_mk/autochecks" % omd_root()
2680 src_file
= "%s/%s.mk" % (autochecks_dir
, first_host_name
)
2681 for num
in range(1, self
.num_hosts
):
2682 host_name
= self
.host_name_template
% num
2683 shutil
.copy(src_file
, "%s/%s.mk" % (autochecks_dir
, host_name
))
2685 def _discover_services_of_host(self
, host_name
):
2686 p
= subprocess
.Popen(["cmk", "-II", host_name
],
2687 stdin
=open(os
.devnull
),
2689 stdout
=subprocess
.PIPE
,
2690 stderr
=subprocess
.STDOUT
)
2693 stdout
= p
.communicate()[0]
2694 except KeyboardInterrupt:
2701 if p
.returncode
!= 0:
2702 raise MKGeneralException("'cmk -II %s' failed: %s" % (host_name
, stdout
))
2704 #def _discover_services(self):
2705 # self._test.info(" Discovering services")
2706 # p = subprocess.Popen(["cmk", "-II"], stdin=open(os.devnull),
2707 # close_fds=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
2710 # stdout = p.communicate()[0]
2711 # except KeyboardInterrupt:
2718 # if p.returncode != 0:
2719 # raise MKGeneralException("'cmk -II' failed: %s" % stdout)
2721 def _create_rrds(self
):
2723 # The execution of service checks needs to be stopped to prevent
2724 # races with the core.
2725 LocalConnection().command("[%d] STOP_EXECUTING_SVC_CHECKS" % time
.time())
2727 self
._test
.info(" Creating RRDs (stopping core during this step)")
2728 first_host_name
= self
.host_name_template
% 0
2729 self
._create
_rrds
_of
_host
(first_host_name
)
2730 self
._copy
_rrds
_of
(first_host_name
)
2731 self
._test
.info(" Finished creating RRDs")
2733 LocalConnection().command("[%d] START_EXECUTING_SVC_CHECKS" % time
.time())
2735 def _copy_rrds_of(self
, first_host_name
):
2736 rrd_dir
= "%s/var/check_mk/rrd" % omd_root()
2739 # We have to stop and start the CMC during manual RRD copy.
2740 # a) No races with the core
2741 # b) Cached RRD info held by the core during runtime
2742 subprocess
.call(["omd", "stop", "cmc"], stdout
=open(os
.devnull
, "w"), close_fds
=True)
2744 src_dir
= "%s/%s" % (rrd_dir
, first_host_name
)
2745 for num
in range(1, self
.num_hosts
):
2746 if self
._runner
.shall_stop():
2750 self
._test
.info(" Created RRDs for %d hosts" % num
)
2752 host_name
= self
.host_name_template
% num
2753 target_dir
= "%s/%s" % (rrd_dir
, host_name
)
2756 shutil
.copytree(src_dir
, target_dir
)
2758 if e
.errno
== 17: # file exists
2763 # Replace the host name in the just copied info files
2764 for f
in glob
.glob("%s/%s/*.info" % (rrd_dir
, host_name
)):
2765 new_content
= open(f
).readlines()
2766 for index
, line
in enumerate(new_content
):
2767 if line
.startswith("HOST "):
2768 new_content
[index
] = "HOST %s\n" % host_name
2770 open(f
, "w").write("".join(new_content
))
2772 self
._activate
_changes
()
2774 # To create the RRDs, simply perform one check and wait for the RRDs to be created
2775 def _create_rrds_of_host(self
, host_name
):
2776 # Do 2 iterations to initialize the rrds that base on counter values
2777 for _i
in xrange(2):
2778 p
= subprocess
.Popen(["cmk", "-v", host_name
],
2779 stdin
=open(os
.devnull
),
2781 stdout
=subprocess
.PIPE
,
2782 stderr
=subprocess
.STDOUT
)
2785 stdout
= p
.communicate()[0]
2786 except KeyboardInterrupt:
2793 if p
.returncode
!= 0:
2794 raise MKGeneralException("'cmk -v %s' failed: %s" % (host_name
, stdout
))
2798 num_rrds
, wait_sec
= None, 10
2799 while num_rrds
is None or num_rrds
< 30:
2801 raise MKGeneralException("Did not create RRDs for '%s' "
2802 "after 10 seconds" % host_name
)
2805 if self
._runner
.shall_stop():
2807 num_rrds
= len(glob
.glob("%s/var/check_mk/rrd/%s/*.rrd" % (omd_root(), host_name
)))
2809 self
._test
.info(" Found %d RRDs per host" % num_rrds
)
2811 def _cmk_config_dir(self
):
2812 return "%s/etc/check_mk/conf.d/wato/mkbench" % omd_root()
2814 def _activate_changes(self
):
2816 self
._test
.info(" Activating changes")
2817 p
= subprocess
.Popen(["cmk", "-R"],
2819 stdout
=subprocess
.PIPE
,
2820 stderr
=subprocess
.STDOUT
)
2823 stdout
= p
.communicate()[0]
2824 except KeyboardInterrupt:
2831 if p
.returncode
!= 0:
2832 raise MKGeneralException("Failed to execute 'cmk -R': %s" % stdout
)
2834 # Wait for livestatus socket to be available
2835 while (self
._runner
and not self
._runner
.shall_stop()) \
2836 and not os
.path
.exists("%s/tmp/run/live" % omd_root()):
2839 # To deal with the load peak while initializing the Check_MK helpers we
2840 # need to wait till all helpers are initialized. Since we have no one
2841 # to tell us the state of the helpers, we need to detect this somehow.
2842 # -> Find all helper processes and check the open file descriptors.
2843 def _wait_for_cmk_helpers_ready(self
):
2844 num_not_ready
, _loadavg_5_before
= None, os
.getloadavg()[1]
2845 while num_not_ready
is None or num_not_ready
> 0:
2846 if self
._runner
.shall_stop():
2849 if num_not_ready
is None:
2850 self
._test
.info(" Waiting for helper initialization")
2853 " Waiting for helper initialization (%d not ready)" % num_not_ready
)
2856 matched_pids
= subprocess
.check_output(
2858 site_id(), "-f", "python.*check_mk.py --keepalive$"]).split("\n")
2859 except subprocess
.CalledProcessError
, e
:
2860 self
._test
.error(" Found no helper yet (%s)" % e
.returncode
)
2864 for p
in matched_pids
:
2865 if p
and not os
.path
.exists("/proc/%s/fd/3" % p
):
2869 self
._test
.info(" Finished helper initialization")
2871 def _cleanup_hosts(self
):
2873 self
._test
.info(" Cleaning up test configs")
2874 for path
in glob
.glob("%s/*" % self
._cmk
_config
_dir
()):
2878 def run_cleanup_data():
2879 runner
= TestRunner()
2880 test_case
= TestAgentBasedStandardHosts()
2881 test_case
._cleanup
_hosts
()
2882 test_case
._activate
_changes
()
2883 runner
._cleanup
_host
_files
(cleanup_rrds
=True)
2884 test_case
._activate
_changes
()
2888 # .--Reports-------------------------------------------------------------.
2890 # | | _ \ ___ _ __ ___ _ __| |_ ___ |
2891 # | | |_) / _ \ '_ \ / _ \| '__| __/ __| |
2892 # | | _ < __/ |_) | (_) | | | |_\__ \ |
2893 # | |_| \_\___| .__/ \___/|_| \__|___/ |
2895 # +----------------------------------------------------------------------+
2896 # | Create some reports out of the test results |
2897 # '----------------------------------------------------------------------'
2900 def run_report_list():
2901 for entry
in sorted(os
.listdir(Test
.data_dir())):
2902 sys
.stdout
.write(entry
+ "\n")
2905 def run_report(test_id
):
2906 if not Test
.exists(test_id
):
2907 raise MKGeneralException("The test '%s' does not exist." % test_id
)
2909 test
= Test(test_id
)
2911 if test
.state
!= Test
.STATE_STOPPED
:
2912 sys
.stdout
.write("WARNING: This test is not complete yet.\n")
2913 sys
.stdout
.write("\n")
2915 sys
.stdout
.write("= SUMMARY ==================================\n")
2916 sys
.stdout
.write("\n")
2917 sys
.stdout
.write(" Test version : %s\n" % test
.version
)
2918 sys
.stdout
.write(" Date : %s\n" % fmt_datetime(test
.first_measurement().time
))
2919 sys
.stdout
.write(" State : %s\n" % test
.state_text())
2921 result_txt
= test
.result
2922 if test
.result
== "failed":
2923 result_txt
+= " (%s)" % test
.result_msg
2925 sys
.stdout
.write(" Result : %s\n" % result_txt
)
2926 sys
.stdout
.write(" Duration : %s\n" % fmt_timespan(test
.duration()))
2927 sys
.stdout
.write("\n")
2928 sys
.stdout
.write(" Version : %s\n" % test
.system_info
.omd_version
)
2929 sys
.stdout
.write(" Site : %s\n" % test
.system_info
.omd_site
)
2930 sys
.stdout
.write(" Core : %s\n" % test
.system_info
.core
)
2931 sys
.stdout
.write("\n")
2933 test_cases_by_name
= {}
2934 for test_case
in test
.test_cases
:
2935 test_cases_by_name
[test_case
.name()] = test_case
2937 for test_case_name
, measurements
in test
.get_measurements_by_test_case().items():
2938 test_case
= test_cases_by_name
[test_case_name
]
2940 sys
.stdout
.write("= %s =================\n" % test_case_name
)
2941 sys
.stdout
.write("\n")
2943 values
= Measurement
.summary(measurements
)
2945 result_txt
= test_case
.result
2946 if test_case
.result
== "failed":
2947 result_txt
+= " (%s)" % test_case
.result_msg
2950 "State : %s" % test_case
.state_text(),
2951 "Result : %s" % result_txt
,
2952 "Duration : %s" % fmt_timespan(test_case
.duration()),
2954 "CPU load : %0.2f %0.2f %0.2f" % values
.cpu_load
,
2955 "Memory : %0.2f %%" % values
.memory
.percent
,
2956 "Disk %-6s : %s free" % (os
.path
.basename(test
.system_info
.block_device
),
2957 fmt_bytes(values
.disk_usage
.free
)),
2958 "Disk IO : read %s/s, write %s/s" % (fmt_bytes(values
.disk_io
.read_bytes
),
2959 fmt_bytes(values
.disk_io
.write_bytes
)),
2962 if values
.site_stats
:
2965 "Objects : %d Hosts, %d Services" % (values
.site_stats
["num_hosts"],
2966 values
.site_stats
["num_services"]),
2967 "Check lat. : %0.2f sec" % values
.site_stats
["average_latency_cmk"],
2968 "Checks : %0.2f Hosts/s %0.2f Services/s" %
2969 (values
.site_stats
["host_checks_rate"], values
.site_stats
["service_checks_rate"]),
2970 "CMK helpers : %0.2f %%" % (values
.site_stats
["helper_usage_cmk"] * 100),
2973 data
+= ["ERROR: Got no site statistics"]
2976 sys
.stdout
.write(" %s\n" % value
)
2978 sys
.stdout
.write("\n")
2982 # .--GUI-----------------------------------------------------------------.
2984 # | / ___| | | |_ _| |
2985 # | | | _| | | || | |
2986 # | | |_| | |_| || | |
2987 # | \____|\___/|___| |
2989 # +----------------------------------------------------------------------+
2990 # | The GUI rendering code |
2991 # '----------------------------------------------------------------------'
2994 # The class hierarchy is out of our control. Use it as intended by npyscreen.
2995 class Window(npyscreen
.FormBaseNew
): # pylint: disable=too-many-ancestors
2999 class GeneralInformation(npyscreen
.BoxTitle
):
3000 def __init__(self
, *args
, **kwargs
):
3001 super(GeneralInformation
, self
).__init
__(*args
, **kwargs
)
3002 self
.name
= "General Information"
3003 self
.editable
= False
3004 self
._runner
= kwargs
["runner"]
3006 def update(self
, clear
=True):
3008 "State : %s" % self
._runner
.test
.state_text(),
3011 if self
._runner
.test
.has_measurements():
3012 values
= self
._runner
.test
.current_measurement()
3015 "Duration : %s" % fmt_timespan(self
._runner
.test
.duration()),
3017 "CPU load : %0.2f %0.2f %0.2f" % values
.cpu_load
,
3018 "Memory : %0.2f %%" % values
.memory
.percent
,
3019 "Disk %-6s: %s free" % (os
.path
.basename(
3020 self
._runner
.test
.system_info
.block_device
), fmt_bytes(values
.disk_usage
.free
)),
3021 "Disk IO : r %s/s, w %s/s" % (fmt_bytes(values
.disk_io
.read_bytes
),
3022 fmt_bytes(values
.disk_io
.write_bytes
)),
3024 super(GeneralInformation
, self
).update(clear
)
3027 class SiteInformation(npyscreen
.BoxTitle
):
3028 def __init__(self
, *args
, **kwargs
):
3029 super(SiteInformation
, self
).__init
__(*args
, **kwargs
)
3031 self
.editable
= False
3032 self
._runner
= kwargs
["runner"]
3034 def update(self
, clear
=True):
3036 super(SiteInformation
, self
).update(clear
)
3038 def _update_info(self
):
3039 system_info
= self
._runner
.test
.system_info
3042 "Version : %s" % system_info
.omd_version
,
3043 "Core : %s" % system_info
.core
,
3046 if not self
._runner
.test
.has_measurements():
3049 values
= self
._runner
.test
.current_measurement()
3051 "State : %s" % self
._site
_state
_text
(values
.site_state
),
3054 if not values
.site_stats
:
3058 "Objects : %d H, %d S" % (values
.site_stats
["num_hosts"],
3059 values
.site_stats
["num_services"]),
3060 "Check lat. : %0.2f s" % values
.site_stats
["average_latency_cmk"],
3061 "Checks : %0.2f H/s %0.2f S/s" % (values
.site_stats
["host_checks_rate"],
3062 values
.site_stats
["service_checks_rate"]),
3063 "CMK helpers : %0.2f %%" % (values
.site_stats
["helper_usage_cmk"] * 100),
3066 def _site_state_text(self
, state
):
3070 2: "partially running",
3071 }.get(state
, "UNKNOWN (%d)" % state
)
3074 class Log(npyscreen
.BoxTitle
):
3075 _contained_widget
= npyscreen
.BufferPager
3077 def __init__(self
, *args
, **kwargs
):
3078 kwargs
["contained_widget_arguments"] = {
3081 super(Log
, self
).__init
__(*args
, **kwargs
)
3083 self
.editable
= False
3084 self
._runner
= kwargs
["runner"]
3086 def update(self
, clear
=True):
3088 for timestamp
, _level
, entry
in self
._runner
.test
.get_log():
3089 if "\n" not in entry
:
3092 lines
= entry
.split("\n")
3096 "%s %s" % (time
.strftime("%H:%M:%S", time
.localtime(timestamp
)), line
))
3098 self
.entry_widget
.clearBuffer()
3099 self
.entry_widget
.buffer(entries
, scroll_end
=True)
3100 super(Log
, self
).update(clear
)
3103 class HelpText(npyscreen
.FixedText
):
3104 def __init__(self
, *args
, **kwargs
):
3105 super(HelpText
, self
).__init
__(*args
, **kwargs
)
3106 self
.value
= "^C: quit"
3109 class ConsoleRenderer(npyscreen
.NPSApp
):
3110 def __init__(self
, runner
):
3111 super(ConsoleRenderer
, self
).__init
__()
3113 self
._runner
= runner
3120 npyscreen
.setTheme(npyscreen
.Themes
.ElegantTheme
)
3121 self
.keypress_timeout_default
= 10
3123 self
._window
= Window(parentApp
=self
, name
="Check_MK System Benchmark %s" % __version__
)
3125 max_y
, max_x
= self
._window
.curses_pad
.getmaxyx()
3127 info_box_width
= (max_x
- form_border
) / 2
3128 info_box_height
= 10
3130 log_height
= max_y
- info_box_height
- form_border
- help_height
3132 self
._general
= self
._window
.add(
3136 max_height
=info_box_height
,
3137 width
=info_box_width
,
3138 runner
=self
._runner
)
3140 self
._site
= self
._window
.add(
3142 relx
=info_box_width
+ 1,
3144 max_height
=info_box_height
,
3145 width
=info_box_width
,
3146 runner
=self
._runner
)
3148 self
._log
= self
._window
.add(
3151 rely
=info_box_height
+ 1,
3153 width
=self
._window
.curses_pad
.getmaxyx()[1] - 2,
3154 runner
=self
._runner
,
3157 self
._help
= self
._window
.add(
3166 def while_waiting(self
):
3167 self
._general
.update()
3168 self
._window
.display(clear
=True)
3172 # .--Main----------------------------------------------------------------.
3174 # | | \/ | __ _(_)_ __ |
3175 # | | |\/| |/ _` | | '_ \ |
3176 # | | | | | (_| | | | | | |
3177 # | |_| |_|\__,_|_|_| |_| |
3179 # +----------------------------------------------------------------------+
3181 # '----------------------------------------------------------------------'
3188 register_signal_handlers()
3190 set_cmdline(" ".join(["mkbench"] + sys
.argv
[1:]))
3194 num_hosts
, num_helpers
= 100, 20
3198 "help", "version", "debug", "report", "cleanup-data", "num-hosts=", "num-helpers="
3202 opts
, args
= getopt
.getopt(sys
.argv
[1:], short_options
, long_options
)
3203 except getopt
.GetoptError
, e
:
3209 if o
in ["-h", "--help"]:
3211 elif o
== "--version":
3212 sys
.stdout
.write("mkbench %s\n" % __version__
)
3214 elif o
== "--debug":
3217 elif o
== "--report":
3220 elif o
== "--cleanup-data":
3221 mode
= "cleanup-data"
3223 elif o
== "--num-hosts":
3227 raise MKGeneralException("--num-hosts must be given as number")
3229 if num_hosts
not in TestRunner
.host_steps
:
3230 raise MKGeneralException(
3231 "--num-hosts must be one of: %s" % ", ".join(map(str, TestRunner
.host_steps
)))
3233 elif o
== "--num-helpers":
3235 num_helpers
= int(a
)
3237 raise MKGeneralException("--num-helpers must be given as number")
3240 run_performance_test(num_hosts
, num_helpers
)
3241 elif mode
== "report":
3246 elif mode
== "cleanup-data":
3249 raise NotImplementedError()
3252 def verify_is_site():
3253 if "OMD_ROOT" not in os
.environ
:
3254 raise MKGeneralException("mkbench can only be run as Check_MK site user")
3257 def verify_site_is_running():
3258 site_state
= subprocess
.call(["omd", "status", "--bare"],
3259 stdout
=file(os
.devnull
, "w"),
3262 raise MKGeneralException("Site needs to be running")
3265 def verify_site_is_empty():
3266 hosts
= LocalConnection().query_column("GET hosts\nColumns: host_name")
3269 # - created with "omd-vonheute"
3270 # - previously terminated mkbench runs
3271 if not hosts
or hosts
== ["heute"] or all([h
.startswith("std-agent-host-") for h
in hosts
]):
3274 raise MKGeneralException("Site needs to be empty")
3277 if __name__
== "__main__":
3280 except KeyboardInterrupt:
3281 sys
.stderr
.write("Terminated.\n")
3284 except MKGeneralException
, e
:
3285 sys
.stderr
.write("%s\n" % e
)