cmk_base/piggyback.py

   1 #!/usr/bin/env python
   2 # -*- encoding: utf-8; py-indent-offset: 4 -*-
   3 # +------------------------------------------------------------------+
   4 # |             ____ _               _        __  __ _  __           |
   5 # |            / ___| |__   ___  ___| | __   |  \/  | |/ /           |
   6 # |           | |   | '_ \ / _ \/ __| |/ /   | |\/| | ' /            |
   7 # |           | |___| | | |  __/ (__|   <    | |  | | . \            |
   8 # |            \____|_| |_|\___|\___|_|\_\___|_|  |_|_|\_\           |
   9 # |                                                                  |
  10 # | Copyright Mathias Kettner 2014             mk@mathias-kettner.de |
  11 # +------------------------------------------------------------------+
  12 #
  13 # This file is part of Check_MK.
  14 # The official homepage is at http://mathias-kettner.de/check_mk.
  15 #
  16 # check_mk is free software;  you can redistribute it and/or modify it
  17 # under the  terms of the  GNU General Public License  as published by
  18 # the Free Software Foundation in version 2.  check_mk is  distributed
  19 # in the hope that it will be useful, but WITHOUT ANY WARRANTY;  with-
  20 # out even the implied warranty of  MERCHANTABILITY  or  FITNESS FOR A
  21 # PARTICULAR PURPOSE. See the  GNU General Public License for more de-
  22 # tails. You should have  received  a copy of the  GNU  General Public
  23 # License along with GNU Make; see the file  COPYING.  If  not,  write
  24 # to the Free Software Foundation, Inc., 51 Franklin St,  Fifth Floor,
  25 # Boston, MA 02110-1301 USA.
  26
  27 import os
  28 import tempfile
  29
  30 import cmk.utils.paths
  31 import cmk.utils.translations
  32 import cmk.utils.store as store
  33 from cmk.utils.exceptions import MKGeneralException
  34
  35 import cmk_base.utils
  36 import cmk_base.console as console
  37
  38
  39 def get_piggyback_raw_data(piggyback_max_cachefile_age, hostname):
  40     """Returns the usable piggyback data for the given host
  41
  42     A list of two element tuples where the first element is
  43     the source host name and the second element is the raw
  44     piggyback data (byte string)
  45     """
  46     if not hostname:
  47         return []
  48
  49     piggyback_data = []
  50     for source_host, piggyback_file_path in _get_piggyback_files(piggyback_max_cachefile_age,
  51                                                                  hostname):
  52         try:
  53             raw_data = file(piggyback_file_path).read()
  54         except IOError as e:
  55             console.verbose("Cannot read piggyback raw data from host %s: %s\n" % (source_host, e))
  56             continue
  57
  58         console.verbose("Using piggyback raw data from host %s.\n" % source_host)
  59         piggyback_data.append((source_host, raw_data))
  60
  61     return piggyback_data
  62
  63
  64 def has_piggyback_raw_data(piggyback_max_cachefile_age, hostname):
  65     return _get_piggyback_files(piggyback_max_cachefile_age, hostname) != []
  66
  67
  68 def _get_piggyback_files(piggyback_max_cachefile_age, hostname):
  69     """Gather a list of piggyback files to read for further processing.
  70
  71     Please note that there may be multiple parallel calls executing the
  72     _get_piggyback_files(), store_piggyback_raw_data() or cleanup_piggyback_files()
  73     functions. Therefor all these functions needs to deal with suddenly vanishing or
  74     updated files/directories.
  75     """
  76     files = []
  77     piggyback_dir = os.path.join(cmk.utils.paths.tmp_dir, "piggyback", hostname)
  78
  79     # cleanup_piggyback_files() may remove stale piggyback files of one source
  80     # host and also the directory "hostname" when the last piggyback file for the
  81     # current host was removed. This may cause the os.listdir() to fail. We treat
  82     # this as regular case: No piggyback files for the current host.
  83     try:
  84         source_host_names = os.listdir(piggyback_dir)
  85     except OSError as e:
  86         if e.errno == 2:  # No such file or directory
  87             return files
  88         else:
  89             raise
  90
  91     for source_host in source_host_names:
  92         if source_host.startswith("."):
  93             continue
  94
  95         piggyback_file_path = os.path.join(piggyback_dir, source_host)
  96
  97         try:
  98             file_age = cmk_base.utils.cachefile_age(piggyback_file_path)
  99         except MKGeneralException as e:
 100             continue  # File might've been deleted. That's ok.
 101
 102         # Skip piggyback files that are outdated at all
 103         if file_age > piggyback_max_cachefile_age:
 104             console.verbose("Piggyback file %s is outdated (%d seconds too old). Skip processing.\n"
 105                             % (piggyback_file_path, file_age - piggyback_max_cachefile_age))
 106             continue
 107
 108         status_file_path = _piggyback_source_status_path(source_host)
 109         if not os.path.exists(status_file_path):
 110             console.verbose(
 111                 "Piggyback file %s is outdated (Source not sending piggyback). Skip processing.\n" %
 112                 piggyback_file_path)
 113             continue
 114
 115         if _is_piggyback_file_outdated(status_file_path, piggyback_file_path):
 116             console.verbose(
 117                 "Piggyback file %s is outdated (Not updated by source). Skip processing.\n" %
 118                 piggyback_file_path)
 119             continue
 120
 121         files.append((source_host, piggyback_file_path))
 122
 123     return files
 124
 125
 126 def _is_piggyback_file_outdated(status_file_path, piggyback_file_path):
 127     try:
 128         # On POSIX platforms Python reads atime and mtime at nanosecond resolution
 129         # but only writes them at microsecond resolution.
 130         # (We're using os.utime() in _store_status_file_of())
 131         return os.stat(status_file_path)[8] > os.stat(piggyback_file_path)[8]
 132     except OSError as e:
 133         if e.errno == 2:  # No such file or directory
 134             return True
 135         else:
 136             raise
 137
 138
 139 def _piggyback_source_status_path(source_host):
 140     return os.path.join(cmk.utils.paths.tmp_dir, "piggyback_sources", source_host)
 141
 142
 143 def _remove_piggyback_file(piggyback_file_path):
 144     try:
 145         os.remove(piggyback_file_path)
 146         return True
 147     except OSError as e:
 148         if e.errno == 2:  # No such file or directory
 149             return False
 150         else:
 151             raise
 152
 153
 154 def remove_source_status_file(source_host):
 155     """Remove the source_status_file of this piggyback host which will
 156     mark the piggyback data from this source as outdated."""
 157     source_status_path = _piggyback_source_status_path(source_host)
 158     return _remove_piggyback_file(source_status_path)
 159
 160
 161 def store_piggyback_raw_data(source_host, piggybacked_raw_data):
 162     piggyback_file_paths = []
 163     for piggybacked_host, lines in piggybacked_raw_data.items():
 164         piggyback_file_path = os.path.join(cmk.utils.paths.tmp_dir, "piggyback", piggybacked_host,
 165                                            source_host)
 166         console.verbose("Storing piggyback data for: %s\n" % piggybacked_host)
 167         content = "\n".join(lines) + "\n"
 168         store.save_file(piggyback_file_path, content)
 169         piggyback_file_paths.append(piggyback_file_path)
 170
 171     # Store the last contact with this piggyback source to be able to filter outdated data later
 172     # We use the mtime of this file later for comparison.
 173     # Only do this for hosts that sent piggyback data this turn, cleanup the status file when no
 174     # piggyback data was sent this turn.
 175     if piggybacked_raw_data:
 176         status_file_path = _piggyback_source_status_path(source_host)
 177         _store_status_file_of(status_file_path, piggyback_file_paths)
 178     else:
 179         remove_source_status_file(source_host)
 180
 181
 182 def _store_status_file_of(status_file_path, piggyback_file_paths):
 183     with tempfile.NamedTemporaryFile(
 184             "w",
 185             dir=os.path.dirname(status_file_path),
 186             prefix=".%s.new" % os.path.basename(status_file_path),
 187             delete=False) as tmp:
 188         tmp_path = tmp.name
 189         os.chmod(tmp_path, 0660)
 190         tmp.write("")
 191
 192         tmp_stats = os.stat(tmp_path)
 193         status_file_times = (tmp_stats.st_atime, tmp_stats.st_mtime)
 194         for piggyback_file_path in piggyback_file_paths:
 195             try:
 196                 os.utime(piggyback_file_path, status_file_times)
 197             except OSError as e:
 198                 if e.errno == 2:  # No such file or directory
 199                     continue
 200                 else:
 201                     raise
 202     os.rename(tmp_path, status_file_path)
 203
 204
 205 def cleanup_piggyback_files(piggyback_max_cachefile_age):
 206     """This is a housekeeping job to clean up different old files from the
 207     piggyback directories.
 208
 209     # Cleanup piggyback data of hosts that are not sending piggyback data anymore
 210     # a) hosts that have a file below piggyback_sources:
 211     #    -> check age of the file and remove it once it reached piggyback_max_cachefile_age
 212     # b) hosts that don't have a file below piggyback_sources (old version or removed by step "a)"):
 213     #    -> remove all piggyback_raw_data files created by this source
 214
 215     # Cleanup empty backed host directories below "piggyback"
 216
 217     Please note that there may be multiple parallel calls executing the
 218     _get_piggyback_files(), store_piggyback_raw_data() or cleanup_piggyback_files()
 219     functions. Therefor all these functions needs to deal with suddenly vanishing or
 220     updated files/directories.
 221     """
 222     _cleanup_old_source_status_files(piggyback_max_cachefile_age)
 223     _cleanup_old_piggybacked_files(piggyback_max_cachefile_age)
 224
 225
 226 def _cleanup_old_source_status_files(piggyback_max_cachefile_age):
 227     base_dir = os.path.join(cmk.utils.paths.tmp_dir, "piggyback_sources")
 228     for entry in os.listdir(base_dir):
 229         if entry[0] == ".":
 230             continue
 231
 232         piggyback_file_path = os.path.join(base_dir, entry)
 233
 234         try:
 235             file_age = cmk_base.utils.cachefile_age(piggyback_file_path)
 236         except MKGeneralException:
 237             continue  # File might've been deleted. That's ok.
 238
 239         if file_age > piggyback_max_cachefile_age:
 240             console.verbose(
 241                 "Removing outdated piggyback source status file %s\n" % piggyback_file_path)
 242             _remove_piggyback_file(piggyback_file_path)
 243
 244
 245 def _cleanup_old_piggybacked_files(piggyback_max_cachefile_age):
 246     """Remove piggyback data that is not needed anymore
 247
 248     The monitoring (_get_piggyback_files()) is already skipping these files,
 249     but we need some cleanup mechanism.
 250
 251     - Remove all piggyback files created by sources without status file
 252     - Remove all piggyback files that are older that the current status file of the source host
 253     - Cleanup empty backed host directories below "piggyback"
 254     """
 255     keep_sources = set(os.listdir(os.path.join(cmk.utils.paths.tmp_dir, "piggyback_sources")))
 256
 257     base_dir = os.path.join(cmk.utils.paths.tmp_dir, "piggyback")
 258     for backed_host_name in os.listdir(base_dir):
 259         if backed_host_name[0] == ".":
 260             continue
 261
 262         # Cleanup piggyback files from sources that we have no status file for
 263         backed_host_dir_path = os.path.join(base_dir, backed_host_name)
 264         for source_host_name in os.listdir(backed_host_dir_path):
 265             if source_host_name[0] == ".":
 266                 continue
 267
 268             piggyback_file_path = os.path.join(backed_host_dir_path, source_host_name)
 269
 270             delete_reason = _shall_cleanup_piggyback_file(
 271                 piggyback_max_cachefile_age, piggyback_file_path, source_host_name, keep_sources)
 272             if delete_reason:
 273                 console.verbose("Removing outdated piggyback file (%s) %s\n" %
 274                                 (delete_reason, piggyback_file_path))
 275                 _remove_piggyback_file(piggyback_file_path)
 276
 277         # Remove empty backed host directory
 278         try:
 279             os.rmdir(backed_host_dir_path)
 280         except OSError as e:
 281             if e.errno == 39:  #Directory not empty
 282                 pass
 283             else:
 284                 raise
 285
 286
 287 def _shall_cleanup_piggyback_file(piggyback_max_cachefile_age, piggyback_file_path,
 288                                   source_host_name, keep_sources):
 289     if source_host_name not in keep_sources:
 290         return "Source not sending piggyback data"
 291
 292     try:
 293         file_age = cmk_base.utils.cachefile_age(piggyback_file_path)
 294     except MKGeneralException:
 295         return None  # File might've been deleted. That's ok.
 296
 297     # Skip piggyback files that are outdated at all
 298     if file_age > piggyback_max_cachefile_age:
 299         return "%d seconds too old" % (file_age - piggyback_max_cachefile_age)
 300
 301     status_file_path = _piggyback_source_status_path(source_host_name)
 302     if not os.path.exists(status_file_path):
 303         return "Source not sending piggyback"
 304
 305     if _is_piggyback_file_outdated(status_file_path, piggyback_file_path):
 306         return "Not updated by source"
 307
 308     return None