cmk_base/inventory.py

   1 #!/usr/bin/python
   2 # -*- encoding: utf-8; py-indent-offset: 4 -*-
   3 # +------------------------------------------------------------------+
   4 # |             ____ _               _        __  __ _  __           |
   5 # |            / ___| |__   ___  ___| | __   |  \/  | |/ /           |
   6 # |           | |   | '_ \ / _ \/ __| |/ /   | |\/| | ' /            |
   7 # |           | |___| | | |  __/ (__|   <    | |  | | . \            |
   8 # |            \____|_| |_|\___|\___|_|\_\___|_|  |_|_|\_\           |
   9 # |                                                                  |
  10 # | Copyright Mathias Kettner 2014             mk@mathias-kettner.de |
  11 # +------------------------------------------------------------------+
  12 #
  13 # This file is part of Check_MK.
  14 # The official homepage is at http://mathias-kettner.de/check_mk.
  15 #
  16 # check_mk is free software;  you can redistribute it and/or modify it
  17 # under the  terms of the  GNU General Public License  as published by
  18 # the Free Software Foundation in version 2.  check_mk is  distributed
  19 # in the hope that it will be useful, but WITHOUT ANY WARRANTY;  with-
  20 # out even the implied warranty of  MERCHANTABILITY  or  FITNESS FOR A
  21 # PARTICULAR PURPOSE. See the  GNU General Public License for more de-
  22 # tails. You should have  received  a copy of the  GNU  General Public
  23 # License along with GNU Make; see the file  COPYING.  If  not,  write
  24 # to the Free Software Foundation, Inc., 51 Franklin St,  Fifth Floor,
  25 # Boston, MA 02110-1301 USA.
  26 """Currently this module manages the inventory tree which is built
  27 while the inventory is performed for one host.
  28
  29 In the future all inventory code should be moved to this module."""
  30
  31 import inspect
  32 import os
  33
  34 import cmk
  35 import cmk.utils.paths
  36 import cmk.utils.store
  37 import cmk.utils.tty as tty
  38 from cmk.utils.exceptions import MKGeneralException
  39 from cmk.utils.structured_data import StructuredDataTree
  40 import cmk.utils.debug
  41
  42 import cmk_base.utils
  43 import cmk_base.console as console
  44 import cmk_base.config as config
  45 import cmk_base.check_api_utils as check_api_utils
  46 import cmk_base.snmp_scan as snmp_scan
  47 import cmk_base.ip_lookup as ip_lookup
  48 import cmk_base.data_sources as data_sources
  49 import cmk_base.cleanup
  50 import cmk_base.decorator
  51 import cmk_base.check_api as check_api
  52 from cmk_base.discovered_labels import DiscoveredHostLabels, DiscoveredHostLabelsStore
  53
  54 #.
  55 #   .--Inventory-----------------------------------------------------------.
  56 #   |            ___                      _                                |
  57 #   |           |_ _|_ ____   _____ _ __ | |_ ___  _ __ _   _              |
  58 #   |            | || '_ \ \ / / _ \ '_ \| __/ _ \| '__| | | |             |
  59 #   |            | || | | \ V /  __/ | | | || (_) | |  | |_| |             |
  60 #   |           |___|_| |_|\_/ \___|_| |_|\__\___/|_|   \__, |             |
  61 #   |                                                   |___/              |
  62 #   +----------------------------------------------------------------------+
  63 #   | Code for doing the actual inventory                                  |
  64 #   '----------------------------------------------------------------------'
  65
  66
  67 def do_inv(hostnames):
  68     cmk.utils.store.makedirs(cmk.utils.paths.inventory_output_dir)
  69     cmk.utils.store.makedirs(cmk.utils.paths.inventory_archive_dir)
  70
  71     for hostname in hostnames:
  72         console.section_begin(hostname)
  73         try:
  74             if config.is_cluster(hostname):
  75                 ipaddress = None
  76             else:
  77                 ipaddress = ip_lookup.lookup_ip_address(hostname)
  78
  79             sources = data_sources.DataSources(hostname, ipaddress)
  80             _do_inv_for(
  81                 sources,
  82                 multi_host_sections=None,
  83                 hostname=hostname,
  84                 ipaddress=ipaddress,
  85                 do_status_data_inv=config.do_status_data_inventory_for(hostname),
  86                 do_host_label_discovery=config.do_host_label_discovery_for(hostname),
  87             )
  88         except Exception as e:
  89             if cmk.utils.debug.enabled():
  90                 raise
  91
  92             console.section_error("%s" % e)
  93         finally:
  94             cmk_base.cleanup.cleanup_globals()
  95
  96
  97 @cmk_base.decorator.handle_check_mk_check_result("check_mk_active-cmk_inv",
  98                                                  "Check_MK HW/SW Inventory")
  99 def do_inv_check(hostname, options):
 100     _inv_hw_changes = options.get("hw-changes", 0)
 101     _inv_sw_changes = options.get("sw-changes", 0)
 102     _inv_sw_missing = options.get("sw-missing", 0)
 103     _inv_fail_status = options.get("inv-fail-status",
 104                                    1)  # State in case of an error (default: WARN)
 105
 106     if config.is_cluster(hostname):
 107         ipaddress = None
 108     else:
 109         ipaddress = ip_lookup.lookup_ip_address(hostname)
 110
 111     status, infotexts, long_infotexts, perfdata = 0, [], [], []
 112
 113     sources = data_sources.DataSources(hostname, ipaddress)
 114     old_timestamp, inventory_tree, status_data_tree, discovered_host_labels = _do_inv_for(
 115         sources,
 116         multi_host_sections=None,
 117         hostname=hostname,
 118         ipaddress=ipaddress,
 119         do_status_data_inv=config.do_status_data_inventory_for(hostname),
 120         do_host_label_discovery=config.do_host_label_discovery_for(hostname),
 121     )
 122
 123     if (inventory_tree.is_empty() and status_data_tree.is_empty() and
 124             discovered_host_labels.is_empty()):
 125         infotexts.append("Found no data")
 126
 127     else:
 128         infotexts.append("Found %d inventory entries" % inventory_tree.count_entries())
 129
 130         if not inventory_tree.has_edge("software") and _inv_sw_missing:
 131             infotexts.append("software information is missing" +
 132                              check_api_utils.state_markers[_inv_sw_missing])
 133             status = max(status, _inv_sw_missing)
 134
 135         if old_timestamp:
 136             path = "%s/%s/%d" % (cmk.utils.paths.inventory_archive_dir, hostname, old_timestamp)
 137             old_tree = StructuredDataTree().load_from(path)
 138
 139             if not old_tree.is_equal(inventory_tree, edges=["software"]):
 140                 infotext = "software changes"
 141                 if _inv_sw_changes:
 142                     status = max(status, _inv_sw_changes)
 143                     infotext += check_api_utils.state_markers[_inv_sw_changes]
 144                 infotexts.append(infotext)
 145
 146             if not old_tree.is_equal(inventory_tree, edges=["hardware"]):
 147                 infotext = "hardware changes"
 148                 if _inv_hw_changes:
 149                     status = max(status, _inv_hw_changes)
 150                     infotext += check_api_utils.state_markers[_inv_hw_changes]
 151
 152                 infotexts.append(infotext)
 153
 154         if not status_data_tree.is_empty():
 155             infotexts.append("Found %s status entries" % status_data_tree.count_entries())
 156
 157         if not discovered_host_labels.is_empty():
 158             infotexts.append("Found %s host labels" % len(discovered_host_labels))
 159
 160     for source in sources.get_data_sources():
 161         source_state, source_output, _source_perfdata = source.get_summary_result_for_inventory()
 162         # Do not output informational (state = 0) things. These information are shown by the "Check_MK" service
 163         if source_state != 0:
 164             status = max(source_state, status)
 165             infotexts.append("[%s] %s" % (source.id(), source_output))
 166
 167     return status, infotexts, long_infotexts, perfdata
 168
 169
 170 def do_inventory_actions_during_checking_for(sources, multi_host_sections, hostname, ipaddress):
 171     do_status_data_inventory = not config.is_cluster(hostname) \
 172         and config.do_status_data_inventory_for(hostname)
 173
 174     do_host_label_discovery = config.do_host_label_discovery_for(hostname)
 175
 176     if not do_status_data_inventory:
 177         _cleanup_status_data(hostname)
 178
 179     if not do_status_data_inventory and not do_host_label_discovery:
 180         return  # nothing to do here
 181
 182     # This is called during checking, but the inventory plugins are not loaded yet
 183     import cmk_base.inventory_plugins as inventory_plugins
 184     inventory_plugins.load_plugins(check_api.get_check_api_context, get_inventory_context)
 185
 186     _do_inv_for(
 187         sources,
 188         multi_host_sections=multi_host_sections,
 189         hostname=hostname,
 190         ipaddress=ipaddress,
 191         do_status_data_inv=do_status_data_inventory,
 192         do_host_label_discovery=do_host_label_discovery,
 193     )
 194
 195
 196 def _cleanup_status_data(hostname):
 197     filepath = "%s/%s" % (cmk.utils.paths.status_data_dir, hostname)
 198     if os.path.exists(filepath):  # Remove empty status data files.
 199         os.remove(filepath)
 200     if os.path.exists(filepath + ".gz"):
 201         os.remove(filepath + ".gz")
 202
 203
 204 def _do_inv_for(sources, multi_host_sections, hostname, ipaddress, do_status_data_inv,
 205                 do_host_label_discovery):
 206     _initialize_inventory_tree()
 207     inventory_tree = g_inv_tree
 208     status_data_tree = StructuredDataTree()
 209     discovered_host_labels = DiscoveredHostLabels(inventory_tree)
 210
 211     node = inventory_tree.get_dict("software.applications.check_mk.cluster.")
 212     if config.is_cluster(hostname):
 213         node["is_cluster"] = True
 214         _do_inv_for_cluster(hostname, inventory_tree)
 215     else:
 216         node["is_cluster"] = False
 217         _do_inv_for_realhost(sources, multi_host_sections, hostname, ipaddress, inventory_tree,
 218                              status_data_tree, discovered_host_labels)
 219
 220     inventory_tree.normalize_nodes()
 221     old_timestamp = _save_inventory_tree(hostname, inventory_tree)
 222     _run_inventory_export_hooks(hostname, inventory_tree)
 223
 224     success_msg = [
 225         "Found %s%s%d%s inventory entries" % (tty.bold, tty.yellow, inventory_tree.count_entries(),
 226                                               tty.normal)
 227     ]
 228
 229     if do_host_label_discovery:
 230         DiscoveredHostLabelsStore(hostname).save(discovered_host_labels.to_dict())
 231         success_msg.append("and %s%s%d%s host labels" % (tty.bold, tty.yellow,
 232                                                          len(discovered_host_labels), tty.normal))
 233
 234     console.section_success(", ".join(success_msg))
 235
 236     if do_status_data_inv:
 237         status_data_tree.normalize_nodes()
 238         _save_status_data_tree(hostname, status_data_tree)
 239
 240         console.section_success(
 241             "Found %s%s%d%s status entries" % (tty.bold, tty.yellow,
 242                                                status_data_tree.count_entries(), tty.normal))
 243
 244     return old_timestamp, inventory_tree, status_data_tree, discovered_host_labels
 245
 246
 247 def _do_inv_for_cluster(hostname, inventory_tree):
 248     inv_node = inventory_tree.get_list("software.applications.check_mk.cluster.nodes:")
 249     for node_name in config.nodes_of(hostname):
 250         inv_node.append({
 251             "name": node_name,
 252         })
 253
 254
 255 def _do_inv_for_realhost(sources, multi_host_sections, hostname, ipaddress, inventory_tree,
 256                          status_data_tree, discovered_host_labels):
 257     for source in sources.get_data_sources():
 258         if isinstance(source, data_sources.SNMPDataSource):
 259             source.set_on_error("raise")
 260             source.set_do_snmp_scan(True)
 261             source.disable_data_source_cache()
 262             source.set_use_snmpwalk_cache(False)
 263             source.set_ignore_check_interval(True)
 264             source.set_check_plugin_name_filter(_gather_snmp_check_plugin_names_inventory)
 265             if multi_host_sections is not None:
 266                 # Status data inventory already provides filled multi_host_sections object.
 267                 # SNMP data source: If 'do_status_data_inv' is enabled there may be
 268                 # sections for inventory plugins which were not fetched yet.
 269                 source.enforce_check_plugin_names(None)
 270                 host_sections = multi_host_sections.add_or_get_host_sections(hostname, ipaddress)
 271                 source.set_fetched_check_plugin_names(host_sections.sections.keys())
 272                 host_sections_from_source = source.run()
 273                 host_sections.update(host_sections_from_source)
 274
 275     if multi_host_sections is None:
 276         multi_host_sections = sources.get_host_sections()
 277
 278     console.step("Executing inventory plugins")
 279     import cmk_base.inventory_plugins as inventory_plugins
 280     console.verbose("Plugins:")
 281     for section_name, plugin in inventory_plugins.sorted_inventory_plugins():
 282         section_content = multi_host_sections.get_section_content(
 283             hostname, ipaddress, section_name, for_discovery=False)
 284         # TODO: Don't we need to take config.check_info[check_plugin_name]["handle_empty_info"]:
 285         #       like it is done in checking.execute_check()? Standardize this!
 286         if not section_content:  # section not present (None or [])
 287             # Note: this also excludes existing sections without info..
 288             continue
 289
 290         if all([x in [[], {}, None] for x in section_content]):
 291             # Inventory plugins which get parsed info from related
 292             # check plugin may have more than one return value, eg
 293             # parse function of oracle_tablespaces returns ({}, {})
 294             continue
 295
 296         console.verbose(" %s%s%s%s" % (tty.green, tty.bold, section_name, tty.normal))
 297
 298         # Inventory functions can optionally have a second argument: parameters.
 299         # These are configured via rule sets (much like check parameters).
 300         inv_function = plugin["inv_function"]
 301         inv_function_args = inspect.getargspec(inv_function).args
 302
 303         kwargs = {}
 304         for dynamic_arg_name, dynamic_arg_value in [
 305             ("inventory_tree", inventory_tree),
 306             ("status_data_tree", status_data_tree),
 307             ("discovered_host_labels", discovered_host_labels),
 308         ]:
 309             if dynamic_arg_name in inv_function_args:
 310                 inv_function_args.remove(dynamic_arg_name)
 311                 kwargs[dynamic_arg_name] = dynamic_arg_value
 312
 313         if len(inv_function_args) == 2:
 314             params = _get_inv_params(hostname, section_name)
 315             args = [section_content, params]
 316         else:
 317             args = [section_content]
 318         inv_function(*args, **kwargs)
 319     console.verbose("\n")
 320
 321
 322 def _gather_snmp_check_plugin_names_inventory(host_config,
 323                                               on_error,
 324                                               do_snmp_scan,
 325                                               for_mgmt_board=False):
 326     return snmp_scan.gather_snmp_check_plugin_names(
 327         host_config, on_error, do_snmp_scan, for_inventory=True, for_mgmt_board=for_mgmt_board)
 328
 329
 330 def _get_inv_params(hostname, section_name):
 331     return config.host_extra_conf_merged(hostname, config.inv_parameters.get(section_name, []))
 332
 333
 334 #.
 335 #   .--Inventory Tree------------------------------------------------------.
 336 #   |  ___                      _                     _____                |
 337 #   | |_ _|_ ____   _____ _ __ | |_ ___  _ __ _   _  |_   _| __ ___  ___   |
 338 #   |  | || '_ \ \ / / _ \ '_ \| __/ _ \| '__| | | |   | || '__/ _ \/ _ \  |
 339 #   |  | || | | \ V /  __/ | | | || (_) | |  | |_| |   | || | |  __/  __/  |
 340 #   | |___|_| |_|\_/ \___|_| |_|\__\___/|_|   \__, |   |_||_|  \___|\___|  |
 341 #   |                                         |___/                        |
 342 #   +----------------------------------------------------------------------+
 343 #   | Managing the inventory tree of a host                                |
 344 #   '----------------------------------------------------------------------'
 345
 346 g_inv_tree = StructuredDataTree()  # TODO Remove one day. Deprecated with version 1.5.0i3??
 347
 348
 349 def _initialize_inventory_tree():  # TODO Remove one day. Deprecated with version 1.5.0i3??
 350     global g_inv_tree
 351     g_inv_tree = StructuredDataTree()
 352
 353
 354 # Dict based
 355 def inv_tree(path):  # TODO Remove one day. Deprecated with version 1.5.0i3??
 356     return g_inv_tree.get_dict(path)
 357
 358
 359 # List based
 360 def inv_tree_list(path):  # TODO Remove one day. Deprecated with version 1.5.0i3??
 361     return g_inv_tree.get_list(path)
 362
 363
 364 def _save_inventory_tree(hostname, inventory_tree):
 365     cmk.utils.store.makedirs(cmk.utils.paths.inventory_output_dir)
 366
 367     old_time = None
 368     filepath = cmk.utils.paths.inventory_output_dir + "/" + hostname
 369     if inventory_tree:
 370         old_tree = StructuredDataTree().load_from(filepath)
 371         old_tree.normalize_nodes()
 372         if old_tree.is_equal(inventory_tree):
 373             console.verbose("Inventory was unchanged\n")
 374         else:
 375             if old_tree.is_empty():
 376                 console.verbose("New inventory tree\n")
 377             else:
 378                 console.verbose("Inventory tree has changed\n")
 379                 old_time = os.stat(filepath).st_mtime
 380                 arcdir = "%s/%s" % (cmk.utils.paths.inventory_archive_dir, hostname)
 381                 cmk.utils.store.makedirs(arcdir)
 382                 os.rename(filepath, arcdir + ("/%d" % old_time))
 383             inventory_tree.save_to(cmk.utils.paths.inventory_output_dir, hostname)
 384
 385     else:
 386         if os.path.exists(
 387                 filepath):  # Remove empty inventory files. Important for host inventory icon
 388             os.remove(filepath)
 389         if os.path.exists(filepath + ".gz"):
 390             os.remove(filepath + ".gz")
 391
 392     return old_time
 393
 394
 395 def _save_status_data_tree(hostname, status_data_tree):
 396     if status_data_tree and not status_data_tree.is_empty():
 397         cmk.utils.store.makedirs(cmk.utils.paths.status_data_dir)
 398         status_data_tree.save_to(cmk.utils.paths.status_data_dir, hostname)
 399
 400
 401 def _run_inventory_export_hooks(hostname, inventory_tree):
 402     import cmk_base.inventory_plugins as inventory_plugins
 403     hooks = []
 404     for hookname, ruleset in config.inv_exports.items():
 405         entries = config.host_extra_conf(hostname, ruleset)
 406         if entries:
 407             hooks.append((hookname, entries[0]))
 408
 409     if not hooks:
 410         return
 411
 412     console.step("Execute inventory export hooks")
 413     for hookname, params in hooks:
 414         console.verbose(
 415             "Execute export hook: %s%s%s%s" % (tty.blue, tty.bold, hookname, tty.normal))
 416         try:
 417             func = inventory_plugins.inv_export[hookname]["export_function"]
 418             func(hostname, params, inventory_tree.get_raw_tree())
 419         except Exception as e:
 420             if cmk.utils.debug.enabled():
 421                 raise
 422             raise MKGeneralException("Failed to execute export hook %s: %s" % (hookname, e))
 423
 424
 425 #.
 426 #   .--Plugin API----------------------------------------------------------.
 427 #   |           ____  _             _            _    ____ ___             |
 428 #   |          |  _ \| |_   _  __ _(_)_ __      / \  |  _ \_ _|            |
 429 #   |          | |_) | | | | |/ _` | | '_ \    / _ \ | |_) | |             |
 430 #   |          |  __/| | |_| | (_| | | | | |  / ___ \|  __/| |             |
 431 #   |          |_|   |_|\__,_|\__, |_|_| |_| /_/   \_\_|  |___|            |
 432 #   |                         |___/                                        |
 433 #   +----------------------------------------------------------------------+
 434 #   | Helper API for being used in inventory plugins. Plugins have access  |
 435 #   | to all things defined by the regular Check_MK check API and all the  |
 436 #   | things declared here.                                                |
 437 #   '----------------------------------------------------------------------'
 438
 439
 440 def get_inventory_context():
 441     return {
 442         "inv_tree_list": inv_tree_list,
 443         "inv_tree": inv_tree,
 444     }