cmk_base/snmp.py

   1 #!/usr/bin/python
   2 # -*- encoding: utf-8; py-indent-offset: 4 -*-
   3 # +------------------------------------------------------------------+
   4 # |             ____ _               _        __  __ _  __           |
   5 # |            / ___| |__   ___  ___| | __   |  \/  | |/ /           |
   6 # |           | |   | '_ \ / _ \/ __| |/ /   | |\/| | ' /            |
   7 # |           | |___| | | |  __/ (__|   <    | |  | | . \            |
   8 # |            \____|_| |_|\___|\___|_|\_\___|_|  |_|_|\_\           |
   9 # |                                                                  |
  10 # | Copyright Mathias Kettner 2014             mk@mathias-kettner.de |
  11 # +------------------------------------------------------------------+
  12 #
  13 # This file is part of Check_MK.
  14 # The official homepage is at http://mathias-kettner.de/check_mk.
  15 #
  16 # check_mk is free software;  you can redistribute it and/or modify it
  17 # under the  terms of the  GNU General Public License  as published by
  18 # the Free Software Foundation in version 2.  check_mk is  distributed
  19 # in the hope that it will be useful, but WITHOUT ANY WARRANTY;  with-
  20 # out even the implied warranty of  MERCHANTABILITY  or  FITNESS FOR A
  21 # PARTICULAR PURPOSE. See the  GNU General Public License for more de-
  22 # tails. You should have  received  a copy of the  GNU  General Public
  23 # License along with GNU Make; see the file  COPYING.  If  not,  write
  24 # to the Free Software Foundation, Inc., 51 Franklin St,  Fifth Floor,
  25 # Boston, MA 02110-1301 USA.
  26
  27 import os
  28 import subprocess
  29 from typing import Tuple, Optional, Any, Dict, List  # pylint: disable=unused-import
  30
  31 import cmk.utils.debug
  32 import cmk.utils.tty as tty
  33 from cmk.utils.exceptions import MKGeneralException, MKBailOut
  34 import cmk.utils.store as store
  35
  36 import cmk_base.utils
  37 import cmk_base.config as config
  38 import cmk_base.console as console
  39 import cmk_base.classic_snmp as classic_snmp
  40 import cmk_base.ip_lookup as ip_lookup
  41 import cmk_base.agent_simulator
  42 from cmk_base.exceptions import MKSNMPError
  43 import cmk_base.cleanup
  44 import cmk_base.snmp_utils as snmp_utils
  45
  46 try:
  47     import cmk_base.cee.inline_snmp as inline_snmp
  48 except ImportError:
  49     inline_snmp = None  # type: ignore
  50
  51 _enforce_stored_walks = False
  52
  53 # TODO: Replace this by generic caching
  54 _g_single_oid_hostname = None
  55 _g_single_oid_ipaddress = None
  56 _g_single_oid_cache = None
  57 # TODO: Move to StoredWalkSNMPBackend?
  58 _g_walk_cache = {}  # type: Dict[str, List[str]]
  59
  60 #.
  61 #   .--caching-------------------------------------------------------------.
  62 #   |                                _     _                               |
  63 #   |                  ___ __ _  ___| |__ (_)_ __   __ _                   |
  64 #   |                 / __/ _` |/ __| '_ \| | '_ \ / _` |                  |
  65 #   |                | (_| (_| | (__| | | | | | | | (_| |                  |
  66 #   |                 \___\__,_|\___|_| |_|_|_| |_|\__, |                  |
  67 #   |                                              |___/                   |
  68 #   '----------------------------------------------------------------------'
  69
  70 #TODO CACHING
  71
  72
  73 def initialize_single_oid_cache(snmp_config, from_disk=False):
  74     # type: (snmp_utils.SNMPHostConfig, bool) -> None
  75     global _g_single_oid_cache, _g_single_oid_ipaddress, _g_single_oid_hostname
  76
  77     if not (_g_single_oid_hostname == snmp_config.hostname \
  78        and _g_single_oid_ipaddress == snmp_config.ipaddress) \
  79        or _g_single_oid_cache is None:
  80         _g_single_oid_hostname = snmp_config.hostname
  81         _g_single_oid_ipaddress = snmp_config.ipaddress
  82         if from_disk:
  83             _g_single_oid_cache = _load_single_oid_cache(snmp_config)
  84         else:
  85             _g_single_oid_cache = {}
  86
  87
  88 def write_single_oid_cache(snmp_config):
  89     # type: (snmp_utils.SNMPHostConfig) -> None
  90     if not _g_single_oid_cache:
  91         return
  92
  93     cache_dir = cmk.utils.paths.snmp_scan_cache_dir
  94     if not os.path.exists(cache_dir):
  95         os.makedirs(cache_dir)
  96     cache_path = "%s/%s.%s" % (cache_dir, snmp_config.hostname, snmp_config.ipaddress)
  97     store.save_data_to_file(cache_path, _g_single_oid_cache, pretty=False)
  98
  99
 100 def set_single_oid_cache(snmp_config, oid, value):
 101     _g_single_oid_cache[oid] = value
 102
 103
 104 def _is_in_single_oid_cache(snmp_config, oid):
 105     return oid in _g_single_oid_cache
 106
 107
 108 def _get_oid_from_single_oid_cache(snmp_config, oid):
 109     return _g_single_oid_cache.get(oid)
 110
 111
 112 def _load_single_oid_cache(snmp_config):
 113     # type: (snmp_utils.SNMPHostConfig) -> Dict[str, str]
 114     cache_path = "%s/%s.%s" % (cmk.utils.paths.snmp_scan_cache_dir, snmp_config.hostname,
 115                                snmp_config.ipaddress)
 116     return store.load_data_from_file(cache_path, {})
 117
 118
 119 def cleanup_host_caches():
 120     # type: () -> None
 121     global _g_walk_cache
 122     _g_walk_cache = {}
 123     _clear_other_hosts_oid_cache(None)
 124     if inline_snmp:
 125         inline_snmp.cleanup_inline_snmp_globals()
 126
 127
 128 def _clear_other_hosts_oid_cache(hostname):
 129     # type: (Optional[str]) -> None
 130     global _g_single_oid_cache, _g_single_oid_ipaddress, _g_single_oid_hostname
 131     if _g_single_oid_hostname != hostname:
 132         _g_single_oid_cache = None
 133         _g_single_oid_hostname = hostname
 134         _g_single_oid_ipaddress = None
 135
 136
 137 #.
 138 #   .--Generic SNMP--------------------------------------------------------.
 139 #   |     ____                      _        ____  _   _ __  __ ____       |
 140 #   |    / ___| ___ _ __   ___ _ __(_) ___  / ___|| \ | |  \/  |  _ \      |
 141 #   |   | |  _ / _ \ '_ \ / _ \ '__| |/ __| \___ \|  \| | |\/| | |_) |     |
 142 #   |   | |_| |  __/ | | |  __/ |  | | (__   ___) | |\  | |  | |  __/      |
 143 #   |    \____|\___|_| |_|\___|_|  |_|\___| |____/|_| \_|_|  |_|_|         |
 144 #   |                                                                      |
 145 #   +----------------------------------------------------------------------+
 146 #   | Top level functions to realize SNMP functionality for Check_MK.      |
 147 #   '----------------------------------------------------------------------'
 148
 149
 150 def create_snmp_host_config(hostname):
 151     # type: (str) -> snmp_utils.SNMPHostConfig
 152     host_config = config.get_config_cache().get_host_config(hostname)
 153
 154     # ip_lookup.lookup_ipv4_address() returns Optional[str] in general, but for
 155     # all cases that reach the code here we seem to have "str".
 156     address = ip_lookup.lookup_ip_address(hostname)
 157     if address is None:
 158         raise MKGeneralException("Failed to gather IP address of %s" % hostname)
 159
 160     return host_config.snmp_config(address)
 161
 162
 163 # TODO: OID_END_OCTET_STRING is not used at all. Drop it.
 164 def get_snmp_table(snmp_config, check_plugin_name, oid_info, use_snmpwalk_cache):
 165     # oid_info is either ( oid, columns ) or
 166     # ( oid, suboids, columns )
 167     # suboids is a list if OID-infixes that are put between baseoid
 168     # and the columns and also prefixed to the index column. This
 169     # allows to merge distinct SNMP subtrees with a similar structure
 170     # to one virtual new tree (look into cmctc_temp for an example)
 171     if len(oid_info) == 2:
 172         oid, targetcolumns = oid_info
 173         suboids = [None]
 174     else:
 175         oid, suboids, targetcolumns = oid_info
 176
 177     if not oid.startswith("."):
 178         raise MKGeneralException("OID definition '%s' does not begin with ." % oid)
 179
 180     index_column = -1
 181     index_format = None
 182     info = []
 183     for suboid in suboids:
 184         columns = []
 185         # Detect missing (empty columns)
 186         max_len = 0
 187         max_len_col = -1
 188
 189         for colno, column in enumerate(targetcolumns):
 190             fetchoid, value_encoding = _compute_fetch_oid(oid, suboid, column)
 191
 192             # column may be integer or string like "1.5.4.2.3"
 193             # if column is 0, we do not fetch any data from snmp, but use
 194             # a running counter as index. If the index column is the first one,
 195             # we do not know the number of entries right now. We need to fill
 196             # in later. If the column is OID_STRING or OID_BIN we do something
 197             # similar: we fill in the complete OID of the entry, either as
 198             # string or as binary UTF-8 encoded number string
 199             if column in [
 200                     snmp_utils.OID_END, snmp_utils.OID_STRING, snmp_utils.OID_BIN,
 201                     snmp_utils.OID_END_BIN, snmp_utils.OID_END_OCTET_STRING
 202             ]:
 203                 if index_column >= 0 and index_column != colno:
 204                     raise MKGeneralException(
 205                         "Invalid SNMP OID specification in implementation of check. "
 206                         "You can only use one of OID_END, OID_STRING, OID_BIN, OID_END_BIN and OID_END_OCTET_STRING."
 207                     )
 208                 index_column = colno
 209                 columns.append((fetchoid, [], "string"))
 210                 index_format = column
 211                 continue
 212
 213             rowinfo = _get_snmpwalk(snmp_config, check_plugin_name, oid, fetchoid, column,
 214                                     use_snmpwalk_cache)
 215
 216             columns.append((fetchoid, rowinfo, value_encoding))
 217             number_of_rows = len(rowinfo)
 218             if number_of_rows > max_len:
 219                 max_len = number_of_rows
 220                 max_len_col = colno
 221
 222         if index_column != -1:
 223             index_rows = []
 224             # Take end-oids of non-index columns as indices
 225             fetchoid, max_column, value_encoding = columns[max_len_col]
 226             for o, _unused_value in max_column:
 227                 if index_format == snmp_utils.OID_END:
 228                     index_rows.append((o, _extract_end_oid(fetchoid, o)))
 229                 elif index_format == snmp_utils.OID_STRING:
 230                     index_rows.append((o, o))
 231                 elif index_format == snmp_utils.OID_BIN:
 232                     index_rows.append((o, _oid_to_bin(o)))
 233                 elif index_format == snmp_utils.OID_END_BIN:
 234                     index_rows.append((o, _oid_to_bin(_extract_end_oid(fetchoid, o))))
 235                 elif index_format == snmp_utils.OID_END_OCTET_STRING:
 236                     index_rows.append((o, _oid_to_bin(_extract_end_oid(fetchoid, o))[1:]))
 237                 else:
 238                     raise MKGeneralException("Invalid index format %s" % index_format)
 239
 240             index_encoding = columns[index_column][-1]
 241             columns[index_column] = fetchoid, index_rows, index_encoding
 242
 243         # prepend suboid to first column
 244         if suboid and columns:
 245             fetchoid, first_column, value_encoding = columns[0]
 246             new_first_column = []
 247             for o, val in first_column:
 248                 new_first_column.append((o, str(suboid) + "." + str(val)))
 249             columns[0] = fetchoid, new_first_column, value_encoding
 250
 251         # Here we have to deal with a nasty problem: Some brain-dead devices
 252         # omit entries in some sub OIDs. This happens e.g. for CISCO 3650
 253         # in the interfaces MIB with 64 bit counters. So we need to look at
 254         # the OIDs and watch out for gaps we need to fill with dummy values.
 255         new_columns = _sanitize_snmp_table_columns(columns)
 256
 257         # From all SNMP data sources (stored walk, classic SNMP, inline SNMP) we
 258         # get normal python strings. But for Check_MK we need unicode strings now.
 259         # Convert them by using the standard Check_MK approach for incoming data
 260         sanitized_columns = _sanitize_snmp_encoding(snmp_config, new_columns)
 261
 262         info += _construct_snmp_table_of_rows(sanitized_columns)
 263
 264     return info
 265
 266
 267 # Contextes can only be used when check_plugin_name is given.
 268 def get_single_oid(snmp_config, oid, check_plugin_name=None, do_snmp_scan=True):
 269     # type: (snmp_utils.SNMPHostConfig, str, Optional[str], bool) -> Optional[str]
 270     # The OID can end with ".*". In that case we do a snmpgetnext and try to
 271     # find an OID with the prefix in question. The *cache* is working including
 272     # the X, however.
 273     if oid[0] != '.':
 274         if cmk.utils.debug.enabled():
 275             raise MKGeneralException("OID definition '%s' does not begin with a '.'" % oid)
 276         else:
 277             oid = '.' + oid
 278
 279     # TODO: Use generic cache mechanism
 280     if _is_in_single_oid_cache(snmp_config, oid):
 281         console.vverbose("       Using cached OID %s: " % oid)
 282         value = _get_oid_from_single_oid_cache(snmp_config, oid)
 283         console.vverbose("%s%s%s%s\n" % (tty.bold, tty.green, value, tty.normal))
 284         return value
 285
 286     # get_single_oid() can only return a single value. When SNMPv3 is used with multiple
 287     # SNMP contexts, all contextes will be queried until the first answer is received.
 288     if check_plugin_name is not None and snmp_utils.is_snmpv3_host(snmp_config):
 289         snmp_contexts = _snmpv3_contexts_of(snmp_config, check_plugin_name)
 290     else:
 291         snmp_contexts = [None]
 292
 293     console.vverbose("       Getting OID %s: " % oid)
 294     for context_name in snmp_contexts:
 295         try:
 296             snmp_backend = SNMPBackendFactory().factory(
 297                 snmp_config, enforce_stored_walks=_enforce_stored_walks)
 298             value = snmp_backend.get(snmp_config, oid, context_name)
 299
 300             if value is not None:
 301                 break  # Use first received answer in case of multiple contextes
 302         except:
 303             if cmk.utils.debug.enabled():
 304                 raise
 305             value = None
 306
 307     if value is not None:
 308         console.vverbose("%s%s%s%s\n" % (tty.bold, tty.green, value, tty.normal))
 309     else:
 310         console.vverbose("failed.\n")
 311
 312     set_single_oid_cache(snmp_config, oid, value)
 313     return value
 314
 315
 316 class SNMPBackendFactory(object):
 317     @staticmethod
 318     def factory(snmp_config, enforce_stored_walks):
 319         # type: (snmp_utils.SNMPHostConfig, bool) -> snmp_utils.ABCSNMPBackend
 320         if enforce_stored_walks or snmp_config.is_usewalk_host:
 321             return StoredWalkSNMPBackend()
 322
 323         if snmp_config.is_inline_snmp_host:
 324             return inline_snmp.InlineSNMPBackend()
 325
 326         return classic_snmp.ClassicSNMPBackend()
 327
 328
 329 class StoredWalkSNMPBackend(snmp_utils.ABCSNMPBackend):
 330     def get(self, snmp_config, oid, context_name=None):
 331         walk = self.walk(snmp_config, oid)
 332
 333         # get_stored_snmpwalk returns all oids that start with oid but here
 334         # we need an exact match
 335         if len(walk) == 1 and oid == walk[0][0]:
 336             return walk[0][1]
 337
 338         elif oid.endswith(".*") and len(walk) > 0:
 339             return walk[0][1]
 340
 341         return None
 342
 343     def walk(self, snmp_config, oid, check_plugin_name=None, table_base_oid=None,
 344              context_name=None):
 345         # type: (snmp_utils.SNMPHostConfig, str, Optional[str], Optional[str], Optional[str]) -> snmp_utils.SNMPRowInfo
 346         if oid.startswith("."):
 347             oid = oid[1:]
 348
 349         if oid.endswith(".*"):
 350             oid_prefix = oid[:-2]
 351             dot_star = True
 352         else:
 353             oid_prefix = oid
 354             dot_star = False
 355
 356         path = cmk.utils.paths.snmpwalks_dir + "/" + snmp_config.hostname
 357
 358         console.vverbose("  Loading %s from %s\n" % (oid, path))
 359
 360         rowinfo = []  # type: List[Tuple[str, str]]
 361
 362         if snmp_config.hostname in _g_walk_cache:
 363             lines = _g_walk_cache[snmp_config.hostname]
 364         else:
 365             try:
 366                 lines = file(path).readlines()
 367             except IOError:
 368                 raise MKSNMPError("No snmpwalk file %s" % path)
 369             _g_walk_cache[snmp_config.hostname] = lines
 370
 371         begin = 0
 372         end = len(lines)
 373         hit = None
 374         while end - begin > 0:
 375             current = (begin + end) / 2
 376             parts = lines[current].split(None, 1)
 377             comp = parts[0]
 378             hit = self._compare_oids(oid_prefix, comp)
 379             if hit == 0:
 380                 break
 381             elif hit == 1:  # we are too low
 382                 begin = current + 1
 383             else:
 384                 end = current
 385
 386         if hit != 0:
 387             return []  # not found
 388
 389         rowinfo = self._collect_until(oid, oid_prefix, lines, current, -1)
 390         rowinfo.reverse()
 391         rowinfo += self._collect_until(oid, oid_prefix, lines, current + 1, 1)
 392
 393         if dot_star:
 394             return [rowinfo[0]]
 395
 396         return rowinfo
 397
 398     def _compare_oids(self, a, b):
 399         aa = self._to_bin_string(a)
 400         bb = self._to_bin_string(b)
 401         if len(aa) <= len(bb) and bb[:len(aa)] == aa:
 402             result = 0
 403         else:
 404             result = cmp(aa, bb)
 405         return result
 406
 407     def _to_bin_string(self, oid):
 408         try:
 409             return tuple(map(int, oid.strip(".").split(".")))
 410         except:
 411             raise MKGeneralException("Invalid OID %s" % oid)
 412
 413     def _collect_until(self, oid, oid_prefix, lines, index, direction):
 414         rows = []
 415         # Handle case, where we run after the end of the lines list
 416         if index >= len(lines):
 417             if direction > 0:
 418                 return []
 419             else:
 420                 index -= 1
 421         while True:
 422             line = lines[index]
 423             parts = line.split(None, 1)
 424             o = parts[0]
 425             if o.startswith('.'):
 426                 o = o[1:]
 427             if o == oid or o.startswith(oid_prefix + "."):
 428                 if len(parts) > 1:
 429                     try:
 430                         value = cmk_base.agent_simulator.process(parts[1])
 431                     except:
 432                         value = parts[1]  # agent simulator missing in precompiled mode
 433                 else:
 434                     value = ""
 435                 # Fix for missing starting oids
 436                 rows.append(('.' + o, classic_snmp.strip_snmp_value(value)))
 437                 index += direction
 438                 if index < 0 or index >= len(lines):
 439                     break
 440             else:
 441                 break
 442         return rows
 443
 444
 445 def walk_for_export(snmp_config, oid):
 446     # type: (snmp_utils.SNMPHostConfig, str) -> List[Tuple[str, str]]
 447     if snmp_config.is_inline_snmp_host:
 448         backend = inline_snmp.InlineSNMPBackend()  # type: snmp_utils.ABCSNMPBackend
 449     else:
 450         backend = classic_snmp.ClassicSNMPBackend()
 451
 452     rows = backend.walk(snmp_config, oid)
 453     return _convert_rows_for_stored_walk(rows)
 454
 455
 456 def enforce_use_stored_walks():
 457     global _enforce_stored_walks
 458     _enforce_stored_walks = True
 459
 460
 461 #.
 462 #   .--SNMP helpers--------------------------------------------------------.
 463 #   |     ____  _   _ __  __ ____    _          _                          |
 464 #   |    / ___|| \ | |  \/  |  _ \  | |__   ___| |_ __   ___ _ __ ___      |
 465 #   |    \___ \|  \| | |\/| | |_) | | '_ \ / _ \ | '_ \ / _ \ '__/ __|     |
 466 #   |     ___) | |\  | |  | |  __/  | | | |  __/ | |_) |  __/ |  \__ \     |
 467 #   |    |____/|_| \_|_|  |_|_|     |_| |_|\___|_| .__/ \___|_|  |___/     |
 468 #   |                                            |_|                       |
 469 #   +----------------------------------------------------------------------+
 470 #   | Internal helpers for processing SNMP things                          |
 471 #   '----------------------------------------------------------------------'
 472
 473
 474 def _convert_rows_for_stored_walk(rows):
 475     def should_be_encoded(v):
 476         for c in v:
 477             if ord(c) < 32 or ord(c) > 127:
 478                 return True
 479         return False
 480
 481     def hex_encode_value(v):
 482         encoded = ""
 483         for c in v:
 484             encoded += "%02X " % ord(c)
 485         return "\"%s\"" % encoded
 486
 487     new_rows = []
 488     for oid, value in rows:
 489         if value == "ENDOFMIBVIEW":
 490             continue
 491
 492         if should_be_encoded(value):
 493             new_rows.append((oid, hex_encode_value(value)))
 494         else:
 495             new_rows.append((oid, value))
 496     return new_rows
 497
 498
 499 def _oid_to_bin(oid):
 500     return u"".join([unichr(int(p)) for p in oid.strip(".").split(".")])
 501
 502
 503 def _extract_end_oid(prefix, complete):
 504     return complete[len(prefix):].lstrip('.')
 505
 506
 507 # sort OID strings numerically
 508 def _oid_to_intlist(oid):
 509     if oid:
 510         return map(int, oid.split('.'))
 511
 512     return []
 513
 514
 515 def _cmp_oids(o1, o2):
 516     return cmp(_oid_to_intlist(o1), _oid_to_intlist(o2))
 517
 518
 519 def _cmp_oid_pairs(pair1, pair2):
 520     return cmp(_oid_to_intlist(pair1[0].lstrip('.')), _oid_to_intlist(pair2[0].lstrip('.')))
 521
 522
 523 def _snmpv3_contexts_of(snmp_config, check_plugin_name):
 524     for ty, rules in snmp_config.snmpv3_contexts:
 525         if ty is None or ty == check_plugin_name:
 526             return rules
 527     return [None]
 528
 529
 530 def _get_snmpwalk(snmp_config, check_plugin_name, oid, fetchoid, column, use_snmpwalk_cache):
 531     is_cachable = _is_snmpwalk_cachable(column)
 532     rowinfo = None
 533     if is_cachable and use_snmpwalk_cache:
 534         # Returns either the cached SNMP walk or None when nothing is cached
 535         rowinfo = _get_cached_snmpwalk(snmp_config.hostname, fetchoid)
 536
 537     if rowinfo is None:
 538         rowinfo = _perform_snmpwalk(snmp_config, check_plugin_name, oid, fetchoid)
 539
 540         if is_cachable:
 541             _save_snmpwalk_cache(snmp_config.hostname, fetchoid, rowinfo)
 542
 543     return rowinfo
 544
 545
 546 def _perform_snmpwalk(snmp_config, check_plugin_name, base_oid, fetchoid):
 547     added_oids = set([])
 548     rowinfo = []
 549     if snmp_utils.is_snmpv3_host(snmp_config):
 550         snmp_contexts = _snmpv3_contexts_of(snmp_config, check_plugin_name)
 551     else:
 552         snmp_contexts = [None]
 553
 554     for context_name in snmp_contexts:
 555         snmp_backend = SNMPBackendFactory().factory(
 556             snmp_config, enforce_stored_walks=_enforce_stored_walks)
 557
 558         rows = snmp_backend.walk(
 559             snmp_config,
 560             fetchoid,
 561             check_plugin_name=check_plugin_name,
 562             table_base_oid=base_oid,
 563             context_name=context_name)
 564
 565         # I've seen a broken device (Mikrotik Router), that broke after an
 566         # update to RouterOS v6.22. It would return 9 time the same OID when
 567         # .1.3.6.1.2.1.1.1.0 was being walked. We try to detect these situations
 568         # by removing any duplicate OID information
 569         if len(rows) > 1 and rows[0][0] == rows[1][0]:
 570             console.vverbose(
 571                 "Detected broken SNMP agent. Ignoring duplicate OID %s.\n" % rows[0][0])
 572             rows = rows[:1]
 573
 574         for row_oid, val in rows:
 575             if row_oid in added_oids:
 576                 console.vverbose("Duplicate OID found: %s (%s)\n" % (row_oid, val))
 577             else:
 578                 rowinfo.append((row_oid, val))
 579                 added_oids.add(row_oid)
 580
 581     return rowinfo
 582
 583
 584 def _compute_fetch_oid(oid, suboid, column):
 585     fetchoid = oid
 586     value_encoding = "string"
 587
 588     if suboid:
 589         fetchoid += "." + str(suboid)
 590
 591     if column != "":
 592         if isinstance(column, tuple):
 593             fetchoid += "." + str(column[1])
 594             if column[0] == "binary":
 595                 value_encoding = "binary"
 596         else:
 597             fetchoid += "." + str(column)
 598
 599     return fetchoid, value_encoding
 600
 601
 602 def _sanitize_snmp_encoding(snmp_config, columns):
 603     decode_string_func = lambda s: _snmp_decode_string(snmp_config, s)
 604
 605     for index, (column, value_encoding) in enumerate(columns):
 606         if value_encoding == "string":
 607             columns[index] = map(decode_string_func, column)
 608         else:
 609             columns[index] = map(_snmp_decode_binary, column)
 610     return columns
 611
 612
 613 def _snmp_decode_string(snmp_config, text):
 614     encoding = snmp_config.character_encoding
 615     if encoding:
 616         return text.decode(encoding)
 617
 618     # Try to determine the current string encoding. In case a UTF-8 decoding fails, we decode latin1.
 619     try:
 620         return text.decode('utf-8')
 621     except:
 622         return text.decode('latin1')
 623
 624
 625 def _snmp_decode_binary(text):
 626     return map(ord, text)
 627
 628
 629 def _sanitize_snmp_table_columns(columns):
 630     # First compute the complete list of end-oids appearing in the output
 631     # by looping all results and putting the endoids to a flat list
 632     endoids = []
 633     for fetchoid, column, value_encoding in columns:
 634         for o, value in column:
 635             endoid = _extract_end_oid(fetchoid, o)
 636             if endoid not in endoids:
 637                 endoids.append(endoid)
 638
 639     # The list needs to be sorted to prevent problems when the first
 640     # column has missing values in the middle of the tree.
 641     if not _are_ascending_oids(endoids):
 642         endoids.sort(cmp=_cmp_oids)
 643         need_sort = True
 644     else:
 645         need_sort = False
 646
 647     # Now fill gaps in columns where some endois are missing
 648     new_columns = []
 649     for fetchoid, column, value_encoding in columns:
 650         # It might happen that end OIDs are not ordered. Fix the OID sorting to make
 651         # it comparable to the already sorted endoids list. Otherwise we would get
 652         # some mixups when filling gaps
 653         if need_sort:
 654             column.sort(cmp=_cmp_oid_pairs)
 655
 656         i = 0
 657         new_column = []
 658         # Loop all lines to fill holes in the middle of the list. All
 659         # columns check the following lines for the correct endoid. If
 660         # an endoid differs empty values are added until the hole is filled
 661         for o, value in column:
 662             eo = _extract_end_oid(fetchoid, o)
 663             if len(column) != len(endoids):
 664                 while i < len(endoids) and endoids[i] != eo:
 665                     new_column.append("")  # (beginoid + '.' +endoids[i], "" ) )
 666                     i += 1
 667             new_column.append(value)
 668             i += 1
 669
 670         # At the end check if trailing OIDs are missing
 671         while i < len(endoids):
 672             new_column.append("")  # (beginoid + '.' +endoids[i], "") )
 673             i += 1
 674         new_columns.append((new_column, value_encoding))
 675
 676     return new_columns
 677
 678
 679 def _are_ascending_oids(oid_list):
 680     for a in range(len(oid_list) - 1):
 681         if _cmp_oids(oid_list[a], oid_list[a + 1]) > 0:  # == 0 should never happen
 682             return False
 683     return True
 684
 685
 686 def _construct_snmp_table_of_rows(columns):
 687     if not columns:
 688         return []
 689
 690     # Now construct table by swapping X and Y.
 691     new_info = []
 692     for index in range(len(columns[0])):
 693         row = [c[index] for c in columns]
 694         new_info.append(row)
 695     return new_info
 696
 697
 698 def _is_snmpwalk_cachable(column):
 699     return isinstance(column, tuple) and column[0] == "cached"
 700
 701
 702 def _get_cached_snmpwalk(hostname, fetchoid):
 703     path = _snmpwalk_cache_path(hostname, fetchoid)
 704     try:
 705         console.vverbose("  Loading %s from walk cache %s\n" % (fetchoid, path))
 706         return store.load_data_from_file(path)
 707     except:
 708         if cmk.utils.debug.enabled():
 709             raise
 710         console.verbose("  Failed loading walk cache. Continue without it.\n" % path)
 711         return None
 712
 713
 714 def _save_snmpwalk_cache(hostname, fetchoid, rowinfo):
 715     path = _snmpwalk_cache_path(hostname, fetchoid)
 716
 717     if not os.path.exists(os.path.dirname(path)):
 718         os.makedirs(os.path.dirname(path))
 719
 720     console.vverbose("  Saving walk of %s to walk cache %s\n" % (fetchoid, path))
 721     store.save_data_to_file(path, rowinfo, pretty=False)
 722
 723
 724 def _snmpwalk_cache_path(hostname, fetchoid):
 725     return os.path.join(cmk.utils.paths.var_dir, "snmp_cache", hostname, fetchoid)
 726
 727
 728 #.
 729 #   .--Main modes----------------------------------------------------------.
 730 #   |       __  __       _                             _                   |
 731 #   |      |  \/  | __ _(_)_ __    _ __ ___   ___   __| | ___  ___         |
 732 #   |      | |\/| |/ _` | | '_ \  | '_ ` _ \ / _ \ / _` |/ _ \/ __|        |
 733 #   |      | |  | | (_| | | | | | | | | | | | (_) | (_| |  __/\__ \        |
 734 #   |      |_|  |_|\__,_|_|_| |_| |_| |_| |_|\___/ \__,_|\___||___/        |
 735 #   |                                                                      |
 736 #   +----------------------------------------------------------------------+
 737 #   | Some main modes to help the user                                     |
 738 #   '----------------------------------------------------------------------'
 739
 740
 741 def do_snmptranslate(walk_filename):
 742     if not walk_filename:
 743         raise MKGeneralException("Please provide the name of a SNMP walk file")
 744
 745     walk_path = "%s/%s" % (cmk.utils.paths.snmpwalks_dir, walk_filename)
 746     if not os.path.exists(walk_path):
 747         raise MKGeneralException("The walk '%s' does not exist" % walk_path)
 748
 749     def translate(lines):
 750         result_lines = []
 751         try:
 752             oids_for_command = []
 753             for line in lines:
 754                 oids_for_command.append(line.split(" ")[0])
 755
 756             command = ["snmptranslate", "-m", "ALL",
 757                        "-M+%s" % cmk.utils.paths.local_mibs_dir] + oids_for_command
 758             p = subprocess.Popen(
 759                 command, stdout=subprocess.PIPE, stderr=open(os.devnull, "w"), close_fds=True)
 760             p.wait()
 761             output = p.stdout.read()
 762             result = output.split("\n")[0::2]
 763             for idx, line in enumerate(result):
 764                 result_lines.append((line.strip(), lines[idx].strip()))
 765
 766         except Exception as e:
 767             console.error("%s\n" % e)
 768
 769         return result_lines
 770
 771     # Translate n-oid's per cycle
 772     entries_per_cycle = 500
 773     translated_lines = []
 774
 775     walk_lines = file(walk_path).readlines()
 776     console.error("Processing %d lines.\n" % len(walk_lines))
 777
 778     i = 0
 779     while i < len(walk_lines):
 780         console.error("\r%d to go...    " % (len(walk_lines) - i))
 781         process_lines = walk_lines[i:i + entries_per_cycle]
 782         translated = translate(process_lines)
 783         i += len(translated)
 784         translated_lines += translated
 785     console.error("\rfinished.                \n")
 786
 787     # Output formatted
 788     for translation, line in translated_lines:
 789         console.output("%s --> %s\n" % (line, translation))
 790
 791
 792 def do_snmpwalk(options, hostnames):
 793     if "oids" in options and "extraoids" in options:
 794         raise MKGeneralException("You cannot specify --oid and --extraoid at the same time.")
 795
 796     if not hostnames:
 797         raise MKBailOut("Please specify host names to walk on.")
 798
 799     if not os.path.exists(cmk.utils.paths.snmpwalks_dir):
 800         os.makedirs(cmk.utils.paths.snmpwalks_dir)
 801
 802     for hostname in hostnames:
 803         #TODO: What about SNMP management boards?
 804         snmp_config = create_snmp_host_config(hostname)
 805
 806         try:
 807             _do_snmpwalk_on(snmp_config, options, cmk.utils.paths.snmpwalks_dir + "/" + hostname)
 808         except Exception as e:
 809             console.error("Error walking %s: %s\n" % (hostname, e))
 810             if cmk.utils.debug.enabled():
 811                 raise
 812         cmk_base.cleanup.cleanup_globals()
 813
 814
 815 def _do_snmpwalk_on(snmp_config, options, filename):
 816     console.verbose("%s:\n" % snmp_config.hostname)
 817
 818     oids = oids_to_walk(options)
 819
 820     with open(filename, "w") as out:
 821         for rows in _execute_walks_for_dump(snmp_config, oids):
 822             for oid, value in rows:
 823                 out.write("%s %s\n" % (oid, value))
 824             console.verbose("%d variables.\n" % len(rows))
 825
 826     console.verbose("Wrote fetched data to %s%s%s.\n" % (tty.bold, filename, tty.normal))
 827
 828
 829 def _execute_walks_for_dump(snmp_config, oids):
 830     for oid in oids:
 831         try:
 832             console.verbose("Walk on \"%s\"..." % oid)
 833             yield walk_for_export(snmp_config, oid)
 834         except Exception as e:
 835             console.error("Error: %s\n" % e)
 836             if cmk.utils.debug.enabled():
 837                 raise
 838
 839
 840 def oids_to_walk(options=None):
 841     if options is None:
 842         options = {}
 843
 844     oids = [
 845         ".1.3.6.1.2.1",  # SNMPv2-SMI::mib-2
 846         ".1.3.6.1.4.1"  # SNMPv2-SMI::enterprises
 847     ]
 848
 849     if "oids" in options:
 850         oids = options["oids"]
 851
 852     elif "extraoids" in options:
 853         oids += options["extraoids"]
 854
 855     return sorted(oids, key=lambda x: map(int, x.strip(".").split(".")))
 856
 857
 858 def do_snmpget(*args):
 859     if not args[0]:
 860         raise MKBailOut("You need to specify an OID.")
 861     oid = args[0][0]
 862
 863     config_cache = config.get_config_cache()
 864
 865     hostnames = args[0][1:]
 866     if not hostnames:
 867         hostnames = []
 868         for host in config_cache.all_active_realhosts():
 869             host_config = config_cache.get_host_config(host)
 870             if host_config.is_snmp_host:
 871                 hostnames.append(host)
 872
 873     for hostname in hostnames:
 874         #TODO what about SNMP management boards?
 875         snmp_config = create_snmp_host_config(hostname)
 876
 877         value = get_single_oid(snmp_config, oid)
 878         console.output("%s (%s): %r\n" % (hostname, snmp_config.ipaddress, value))
 879         cmk_base.cleanup.cleanup_globals()
 880
 881
 882 cmk_base.cleanup.register_cleanup(cleanup_host_caches)