cmk_base/inventory.py

   1 #!/usr/bin/python
   2 # -*- encoding: utf-8; py-indent-offset: 4 -*-
   3 # +------------------------------------------------------------------+
   4 # |             ____ _               _        __  __ _  __           |
   5 # |            / ___| |__   ___  ___| | __   |  \/  | |/ /           |
   6 # |           | |   | '_ \ / _ \/ __| |/ /   | |\/| | ' /            |
   7 # |           | |___| | | |  __/ (__|   <    | |  | | . \            |
   8 # |            \____|_| |_|\___|\___|_|\_\___|_|  |_|_|\_\           |
   9 # |                                                                  |
  10 # | Copyright Mathias Kettner 2014             mk@mathias-kettner.de |
  11 # +------------------------------------------------------------------+
  12 #
  13 # This file is part of Check_MK.
  14 # The official homepage is at http://mathias-kettner.de/check_mk.
  15 #
  16 # check_mk is free software;  you can redistribute it and/or modify it
  17 # under the  terms of the  GNU General Public License  as published by
  18 # the Free Software Foundation in version 2.  check_mk is  distributed
  19 # in the hope that it will be useful, but WITHOUT ANY WARRANTY;  with-
  20 # out even the implied warranty of  MERCHANTABILITY  or  FITNESS FOR A
  21 # PARTICULAR PURPOSE. See the  GNU General Public License for more de-
  22 # tails. You should have  received  a copy of the  GNU  General Public
  23 # License along with GNU Make; see the file  COPYING.  If  not,  write
  24 # to the Free Software Foundation, Inc., 51 Franklin St,  Fifth Floor,
  25 # Boston, MA 02110-1301 USA.
  26 """Currently this module manages the inventory tree which is built
  27 while the inventory is performed for one host.
  28
  29 In the future all inventory code should be moved to this module."""
  30
  31 import inspect
  32 import os
  33 from typing import Tuple, Optional  # pylint: disable=unused-import
  34
  35 import cmk
  36 import cmk.utils.paths
  37 import cmk.utils.store
  38 import cmk.utils.tty as tty
  39 from cmk.utils.exceptions import MKGeneralException
  40 from cmk.utils.structured_data import StructuredDataTree
  41 import cmk.utils.debug
  42
  43 import cmk_base.utils
  44 import cmk_base.console as console
  45 import cmk_base.config as config
  46 import cmk_base.check_api_utils as check_api_utils
  47 import cmk_base.snmp_scan as snmp_scan
  48 import cmk_base.ip_lookup as ip_lookup
  49 import cmk_base.data_sources as data_sources
  50 import cmk_base.cleanup
  51 import cmk_base.decorator
  52 import cmk_base.check_api as check_api
  53 from cmk_base.discovered_labels import DiscoveredHostLabels, DiscoveredHostLabelsStore
  54
  55 #.
  56 #   .--Inventory-----------------------------------------------------------.
  57 #   |            ___                      _                                |
  58 #   |           |_ _|_ ____   _____ _ __ | |_ ___  _ __ _   _              |
  59 #   |            | || '_ \ \ / / _ \ '_ \| __/ _ \| '__| | | |             |
  60 #   |            | || | | \ V /  __/ | | | || (_) | |  | |_| |             |
  61 #   |           |___|_| |_|\_/ \___|_| |_|\__\___/|_|   \__, |             |
  62 #   |                                                   |___/              |
  63 #   +----------------------------------------------------------------------+
  64 #   | Code for doing the actual inventory                                  |
  65 #   '----------------------------------------------------------------------'
  66
  67
  68 def do_inv(hostnames):
  69     cmk.utils.store.makedirs(cmk.utils.paths.inventory_output_dir)
  70     cmk.utils.store.makedirs(cmk.utils.paths.inventory_archive_dir)
  71
  72     for hostname in hostnames:
  73         console.section_begin(hostname)
  74         try:
  75             config_cache = config.get_config_cache()
  76             host_config = config_cache.get_host_config(hostname)
  77
  78             if host_config.is_cluster:
  79                 ipaddress = None
  80             else:
  81                 ipaddress = ip_lookup.lookup_ip_address(hostname)
  82
  83             sources = data_sources.DataSources(hostname, ipaddress)
  84             _do_inv_for(
  85                 sources,
  86                 multi_host_sections=None,
  87                 host_config=host_config,
  88                 ipaddress=ipaddress,
  89                 do_status_data_inv=config.do_status_data_inventory_for(hostname),
  90                 do_host_label_discovery=config.do_host_label_discovery_for(hostname),
  91             )
  92         except Exception as e:
  93             if cmk.utils.debug.enabled():
  94                 raise
  95
  96             console.section_error("%s" % e)
  97         finally:
  98             cmk_base.cleanup.cleanup_globals()
  99
 100
 101 @cmk_base.decorator.handle_check_mk_check_result("check_mk_active-cmk_inv",
 102                                                  "Check_MK HW/SW Inventory")
 103 def do_inv_check(hostname, options):
 104     _inv_hw_changes = options.get("hw-changes", 0)
 105     _inv_sw_changes = options.get("sw-changes", 0)
 106     _inv_sw_missing = options.get("sw-missing", 0)
 107     _inv_fail_status = options.get("inv-fail-status",
 108                                    1)  # State in case of an error (default: WARN)
 109
 110     config_cache = config.get_config_cache()
 111     host_config = config_cache.get_host_config(hostname)
 112
 113     if host_config.is_cluster:
 114         ipaddress = None
 115     else:
 116         ipaddress = ip_lookup.lookup_ip_address(hostname)
 117
 118     status, infotexts, long_infotexts, perfdata = 0, [], [], []
 119
 120     sources = data_sources.DataSources(hostname, ipaddress)
 121     old_timestamp, inventory_tree, status_data_tree, discovered_host_labels = _do_inv_for(
 122         sources,
 123         multi_host_sections=None,
 124         host_config=host_config,
 125         ipaddress=ipaddress,
 126         do_status_data_inv=config.do_status_data_inventory_for(hostname),
 127         do_host_label_discovery=config.do_host_label_discovery_for(hostname),
 128     )
 129
 130     if (inventory_tree.is_empty() and status_data_tree.is_empty() and
 131             discovered_host_labels.is_empty()):
 132         infotexts.append("Found no data")
 133
 134     else:
 135         infotexts.append("Found %d inventory entries" % inventory_tree.count_entries())
 136
 137         if not inventory_tree.has_edge("software") and _inv_sw_missing:
 138             infotexts.append("software information is missing" +
 139                              check_api_utils.state_markers[_inv_sw_missing])
 140             status = max(status, _inv_sw_missing)
 141
 142         if old_timestamp:
 143             path = "%s/%s/%d" % (cmk.utils.paths.inventory_archive_dir, hostname, old_timestamp)
 144             old_tree = StructuredDataTree().load_from(path)
 145
 146             if not old_tree.is_equal(inventory_tree, edges=["software"]):
 147                 infotext = "software changes"
 148                 if _inv_sw_changes:
 149                     status = max(status, _inv_sw_changes)
 150                     infotext += check_api_utils.state_markers[_inv_sw_changes]
 151                 infotexts.append(infotext)
 152
 153             if not old_tree.is_equal(inventory_tree, edges=["hardware"]):
 154                 infotext = "hardware changes"
 155                 if _inv_hw_changes:
 156                     status = max(status, _inv_hw_changes)
 157                     infotext += check_api_utils.state_markers[_inv_hw_changes]
 158
 159                 infotexts.append(infotext)
 160
 161         if not status_data_tree.is_empty():
 162             infotexts.append("Found %s status entries" % status_data_tree.count_entries())
 163
 164         if not discovered_host_labels.is_empty():
 165             infotexts.append("Found %s host labels" % len(discovered_host_labels))
 166
 167     for source in sources.get_data_sources():
 168         source_state, source_output, _source_perfdata = source.get_summary_result_for_inventory()
 169         # Do not output informational (state = 0) things. These information are shown by the "Check_MK" service
 170         if source_state != 0:
 171             status = max(source_state, status)
 172             infotexts.append("[%s] %s" % (source.id(), source_output))
 173
 174     return status, infotexts, long_infotexts, perfdata
 175
 176
 177 def do_inventory_actions_during_checking_for(sources, multi_host_sections, host_config, ipaddress):
 178     # type: (data_sources.DataSources, data_sources.MultiHostSections, config.HostConfig, Optional[str]) -> None
 179     hostname = host_config.hostname
 180     do_status_data_inventory = not host_config.is_cluster \
 181         and config.do_status_data_inventory_for(hostname)
 182
 183     do_host_label_discovery = config.do_host_label_discovery_for(hostname)
 184
 185     if not do_status_data_inventory:
 186         _cleanup_status_data(hostname)
 187
 188     if not do_status_data_inventory and not do_host_label_discovery:
 189         return  # nothing to do here
 190
 191     # This is called during checking, but the inventory plugins are not loaded yet
 192     import cmk_base.inventory_plugins as inventory_plugins
 193     inventory_plugins.load_plugins(check_api.get_check_api_context, get_inventory_context)
 194
 195     config_cache = config.get_config_cache()
 196     host_config = config_cache.get_host_config(hostname)
 197
 198     _do_inv_for(
 199         sources,
 200         multi_host_sections=multi_host_sections,
 201         host_config=host_config,
 202         ipaddress=ipaddress,
 203         do_status_data_inv=do_status_data_inventory,
 204         do_host_label_discovery=do_host_label_discovery,
 205     )
 206
 207
 208 def _cleanup_status_data(hostname):
 209     # type: (str) -> None
 210     filepath = "%s/%s" % (cmk.utils.paths.status_data_dir, hostname)
 211     if os.path.exists(filepath):  # Remove empty status data files.
 212         os.remove(filepath)
 213     if os.path.exists(filepath + ".gz"):
 214         os.remove(filepath + ".gz")
 215
 216
 217 def _do_inv_for(sources, multi_host_sections, host_config, ipaddress, do_status_data_inv,
 218                 do_host_label_discovery):
 219     # type: (data_sources.DataSources, data_sources.MultiHostSections, config.HostConfig, Optional[str], bool, bool) -> Tuple[Optional[float], StructuredDataTree, StructuredDataTree, DiscoveredHostLabels]
 220     hostname = host_config.hostname
 221
 222     _initialize_inventory_tree()
 223     inventory_tree = g_inv_tree
 224     status_data_tree = StructuredDataTree()
 225     discovered_host_labels = DiscoveredHostLabels(inventory_tree)
 226
 227     node = inventory_tree.get_dict("software.applications.check_mk.cluster.")
 228     if host_config.is_cluster:
 229         node["is_cluster"] = True
 230         _do_inv_for_cluster(host_config, inventory_tree)
 231     else:
 232         node["is_cluster"] = False
 233         _do_inv_for_realhost(sources, multi_host_sections, hostname, ipaddress, inventory_tree,
 234                              status_data_tree, discovered_host_labels)
 235
 236     inventory_tree.normalize_nodes()
 237     old_timestamp = _save_inventory_tree(hostname, inventory_tree)
 238     _run_inventory_export_hooks(hostname, inventory_tree)
 239
 240     success_msg = [
 241         "Found %s%s%d%s inventory entries" % (tty.bold, tty.yellow, inventory_tree.count_entries(),
 242                                               tty.normal)
 243     ]
 244
 245     if do_host_label_discovery:
 246         DiscoveredHostLabelsStore(hostname).save(discovered_host_labels.to_dict())
 247         success_msg.append("and %s%s%d%s host labels" % (tty.bold, tty.yellow,
 248                                                          len(discovered_host_labels), tty.normal))
 249
 250     console.section_success(", ".join(success_msg))
 251
 252     if do_status_data_inv:
 253         status_data_tree.normalize_nodes()
 254         _save_status_data_tree(hostname, status_data_tree)
 255
 256         console.section_success(
 257             "Found %s%s%d%s status entries" % (tty.bold, tty.yellow,
 258                                                status_data_tree.count_entries(), tty.normal))
 259
 260     return old_timestamp, inventory_tree, status_data_tree, discovered_host_labels
 261
 262
 263 def _do_inv_for_cluster(host_config, inventory_tree):
 264     # type: (config.HostConfig, StructuredDataTree) -> None
 265     if host_config.nodes is None:
 266         return
 267
 268     inv_node = inventory_tree.get_list("software.applications.check_mk.cluster.nodes:")
 269     for node_name in host_config.nodes:
 270         inv_node.append({
 271             "name": node_name,
 272         })
 273
 274
 275 def _do_inv_for_realhost(sources, multi_host_sections, hostname, ipaddress, inventory_tree,
 276                          status_data_tree, discovered_host_labels):
 277     for source in sources.get_data_sources():
 278         if isinstance(source, data_sources.SNMPDataSource):
 279             source.set_on_error("raise")
 280             source.set_do_snmp_scan(True)
 281             source.disable_data_source_cache()
 282             source.set_use_snmpwalk_cache(False)
 283             source.set_ignore_check_interval(True)
 284             source.set_check_plugin_name_filter(_gather_snmp_check_plugin_names_inventory)
 285             if multi_host_sections is not None:
 286                 # Status data inventory already provides filled multi_host_sections object.
 287                 # SNMP data source: If 'do_status_data_inv' is enabled there may be
 288                 # sections for inventory plugins which were not fetched yet.
 289                 source.enforce_check_plugin_names(None)
 290                 host_sections = multi_host_sections.add_or_get_host_sections(hostname, ipaddress)
 291                 source.set_fetched_check_plugin_names(host_sections.sections.keys())
 292                 host_sections_from_source = source.run()
 293                 host_sections.update(host_sections_from_source)
 294
 295     if multi_host_sections is None:
 296         multi_host_sections = sources.get_host_sections()
 297
 298     console.step("Executing inventory plugins")
 299     import cmk_base.inventory_plugins as inventory_plugins
 300     console.verbose("Plugins:")
 301     for section_name, plugin in inventory_plugins.sorted_inventory_plugins():
 302         section_content = multi_host_sections.get_section_content(
 303             hostname, ipaddress, section_name, for_discovery=False)
 304         # TODO: Don't we need to take config.check_info[check_plugin_name]["handle_empty_info"]:
 305         #       like it is done in checking.execute_check()? Standardize this!
 306         if not section_content:  # section not present (None or [])
 307             # Note: this also excludes existing sections without info..
 308             continue
 309
 310         if all([x in [[], {}, None] for x in section_content]):
 311             # Inventory plugins which get parsed info from related
 312             # check plugin may have more than one return value, eg
 313             # parse function of oracle_tablespaces returns ({}, {})
 314             continue
 315
 316         console.verbose(" %s%s%s%s" % (tty.green, tty.bold, section_name, tty.normal))
 317
 318         # Inventory functions can optionally have a second argument: parameters.
 319         # These are configured via rule sets (much like check parameters).
 320         inv_function = plugin["inv_function"]
 321         inv_function_args = inspect.getargspec(inv_function).args
 322
 323         kwargs = {}
 324         for dynamic_arg_name, dynamic_arg_value in [
 325             ("inventory_tree", inventory_tree),
 326             ("status_data_tree", status_data_tree),
 327             ("discovered_host_labels", discovered_host_labels),
 328         ]:
 329             if dynamic_arg_name in inv_function_args:
 330                 inv_function_args.remove(dynamic_arg_name)
 331                 kwargs[dynamic_arg_name] = dynamic_arg_value
 332
 333         if len(inv_function_args) == 2:
 334             params = _get_inv_params(hostname, section_name)
 335             args = [section_content, params]
 336         else:
 337             args = [section_content]
 338         inv_function(*args, **kwargs)
 339     console.verbose("\n")
 340
 341
 342 def _gather_snmp_check_plugin_names_inventory(host_config,
 343                                               on_error,
 344                                               do_snmp_scan,
 345                                               for_mgmt_board=False):
 346     return snmp_scan.gather_snmp_check_plugin_names(
 347         host_config, on_error, do_snmp_scan, for_inventory=True, for_mgmt_board=for_mgmt_board)
 348
 349
 350 def _get_inv_params(hostname, section_name):
 351     return config.get_config_cache().host_extra_conf_merged(
 352         hostname, config.inv_parameters.get(section_name, []))
 353
 354
 355 #.
 356 #   .--Inventory Tree------------------------------------------------------.
 357 #   |  ___                      _                     _____                |
 358 #   | |_ _|_ ____   _____ _ __ | |_ ___  _ __ _   _  |_   _| __ ___  ___   |
 359 #   |  | || '_ \ \ / / _ \ '_ \| __/ _ \| '__| | | |   | || '__/ _ \/ _ \  |
 360 #   |  | || | | \ V /  __/ | | | || (_) | |  | |_| |   | || | |  __/  __/  |
 361 #   | |___|_| |_|\_/ \___|_| |_|\__\___/|_|   \__, |   |_||_|  \___|\___|  |
 362 #   |                                         |___/                        |
 363 #   +----------------------------------------------------------------------+
 364 #   | Managing the inventory tree of a host                                |
 365 #   '----------------------------------------------------------------------'
 366
 367 g_inv_tree = StructuredDataTree()  # TODO Remove one day. Deprecated with version 1.5.0i3??
 368
 369
 370 def _initialize_inventory_tree():  # TODO Remove one day. Deprecated with version 1.5.0i3??
 371     global g_inv_tree
 372     g_inv_tree = StructuredDataTree()
 373
 374
 375 # Dict based
 376 def inv_tree(path):  # TODO Remove one day. Deprecated with version 1.5.0i3??
 377     return g_inv_tree.get_dict(path)
 378
 379
 380 # List based
 381 def inv_tree_list(path):  # TODO Remove one day. Deprecated with version 1.5.0i3??
 382     return g_inv_tree.get_list(path)
 383
 384
 385 def _save_inventory_tree(hostname, inventory_tree):
 386     # type: (str, StructuredDataTree) -> Optional[float]
 387     cmk.utils.store.makedirs(cmk.utils.paths.inventory_output_dir)
 388
 389     old_time = None
 390     filepath = cmk.utils.paths.inventory_output_dir + "/" + hostname
 391     if inventory_tree:
 392         old_tree = StructuredDataTree().load_from(filepath)
 393         old_tree.normalize_nodes()
 394         if old_tree.is_equal(inventory_tree):
 395             console.verbose("Inventory was unchanged\n")
 396         else:
 397             if old_tree.is_empty():
 398                 console.verbose("New inventory tree\n")
 399             else:
 400                 console.verbose("Inventory tree has changed\n")
 401                 old_time = os.stat(filepath).st_mtime
 402                 arcdir = "%s/%s" % (cmk.utils.paths.inventory_archive_dir, hostname)
 403                 cmk.utils.store.makedirs(arcdir)
 404                 os.rename(filepath, arcdir + ("/%d" % old_time))
 405             inventory_tree.save_to(cmk.utils.paths.inventory_output_dir, hostname)
 406
 407     else:
 408         if os.path.exists(
 409                 filepath):  # Remove empty inventory files. Important for host inventory icon
 410             os.remove(filepath)
 411         if os.path.exists(filepath + ".gz"):
 412             os.remove(filepath + ".gz")
 413
 414     return old_time
 415
 416
 417 def _save_status_data_tree(hostname, status_data_tree):
 418     if status_data_tree and not status_data_tree.is_empty():
 419         cmk.utils.store.makedirs(cmk.utils.paths.status_data_dir)
 420         status_data_tree.save_to(cmk.utils.paths.status_data_dir, hostname)
 421
 422
 423 def _run_inventory_export_hooks(hostname, inventory_tree):
 424     import cmk_base.inventory_plugins as inventory_plugins
 425     hooks = []
 426     config_cache = config.get_config_cache()
 427     for hookname, ruleset in config.inv_exports.items():
 428         entries = config_cache.host_extra_conf(hostname, ruleset)
 429         if entries:
 430             hooks.append((hookname, entries[0]))
 431
 432     if not hooks:
 433         return
 434
 435     console.step("Execute inventory export hooks")
 436     for hookname, params in hooks:
 437         console.verbose(
 438             "Execute export hook: %s%s%s%s" % (tty.blue, tty.bold, hookname, tty.normal))
 439         try:
 440             func = inventory_plugins.inv_export[hookname]["export_function"]
 441             func(hostname, params, inventory_tree.get_raw_tree())
 442         except Exception as e:
 443             if cmk.utils.debug.enabled():
 444                 raise
 445             raise MKGeneralException("Failed to execute export hook %s: %s" % (hookname, e))
 446
 447
 448 #.
 449 #   .--Plugin API----------------------------------------------------------.
 450 #   |           ____  _             _            _    ____ ___             |
 451 #   |          |  _ \| |_   _  __ _(_)_ __      / \  |  _ \_ _|            |
 452 #   |          | |_) | | | | |/ _` | | '_ \    / _ \ | |_) | |             |
 453 #   |          |  __/| | |_| | (_| | | | | |  / ___ \|  __/| |             |
 454 #   |          |_|   |_|\__,_|\__, |_|_| |_| /_/   \_\_|  |___|            |
 455 #   |                         |___/                                        |
 456 #   +----------------------------------------------------------------------+
 457 #   | Helper API for being used in inventory plugins. Plugins have access  |
 458 #   | to all things defined by the regular Check_MK check API and all the  |
 459 #   | things declared here.                                                |
 460 #   '----------------------------------------------------------------------'
 461
 462
 463 def get_inventory_context():
 464     return {
 465         "inv_tree_list": inv_tree_list,
 466         "inv_tree": inv_tree,
 467     }