doc/helpers/validate_checks

   1 #!/usr/bin/python
   2
   3 # Ideen:
   4 # - Header
   5 # - Copyright drin?
   6 # - Globale Variablen
   7 #   - Zu viele?
   8 #   - Check Namen in der Variable?
   9
  10 import os, sys, re, getopt, operator
  11 import reindent
  12
  13 on_tty = sys.stdout.isatty()
  14
  15 if on_tty:
  16     tty_black = '\033[30m'
  17     tty_red = '\033[31m'
  18     tty_green = '\033[32m'
  19     tty_yellow = '\033[33m'
  20     tty_blue = '\033[34m'
  21     tty_magenta = '\033[35m'
  22     tty_cyan = '\033[36m'
  23     tty_white = '\033[37m'
  24     tty_bgred = '\033[41m'
  25     tty_bggreen = '\033[42m'
  26     tty_bgyellow = '\033[43m'
  27     tty_bgblue = '\033[44m'
  28     tty_bgmagenta = '\033[45m'
  29     tty_bgcyan = '\033[46m'
  30     tty_bgwhite = '\033[47m'
  31     tty_bold = '\033[1m'
  32     tty_underline = '\033[4m'
  33     tty_normal = '\033[0m'
  34     tty_ok = tty_green + tty_bold + 'OK' + tty_normal
  35     tty_error = tty_red + tty_bold + 'ERROR' + tty_normal
  36 else:
  37     tty_black = ''
  38     tty_red = ''
  39     tty_green = ''
  40     tty_yellow = ''
  41     tty_blue = ''
  42     tty_magenta = ''
  43     tty_cyan = ''
  44     tty_white = ''
  45     tty_bgred = ''
  46     tty_bggreen = ''
  47     tty_bgyellow = ''
  48     tty_bgblue = ''
  49     tty_bgmagenta = ''
  50     tty_bgcyan = ''
  51     tty_bold = ''
  52     tty_underline = ''
  53     tty_normal = ''
  54     tty_ok = 'OK'
  55     tty_error = 'ERROR'
  56
  57 __name__ = 'asd'
  58 if os.path.exists("validate_checks"):
  59     os.chdir('../../')
  60 defaults_path = 'defaults'
  61 #autochecks = []
  62 execfile('check_mk')
  63 __name__ = '__main__'
  64
  65
  66 def get_checks():
  67     files = os.listdir('checks')
  68     return [ f for f in files if f.endswith(".include") and not f.startswith('.') ] + \
  69        [ f for f in files if not f.endswith(".include") and not f.startswith('.') ]
  70
  71
  72 def get_perfometer_files():
  73     files = os.listdir('web/plugins/perfometer')
  74     return [f for f in files if f.endswith(".py") and not f.startswith('.')]
  75
  76
  77 def check_has_perfdata(check):
  78     return check_info[check].get("has_perfdata", False)
  79
  80
  81 def get_all_checks():
  82     k = check_info.keys()
  83     k.sort()
  84     return k
  85
  86
  87 def is_snmp_check(check):
  88     return check in snmp_info
  89
  90
  91 def all_nonfunction_vars():
  92     return set([
  93         name for name, value in globals().items()
  94         if name[0] != '_' and type(value) != type(lambda: 0)
  95     ])
  96
  97
  98 def get_manpage(check):
  99     if not check in manpage:
 100         try:
 101             manpage[check] = open('checkman/%s' % check.split('.')[0]).read()
 102         except IOError, e:
 103             manpage[check] = ""
 104     return manpage[check]
 105
 106
 107 def get_checkfile(check):
 108     if not check in checkfiles:
 109         try:
 110             checkfiles[check] = open('checks/%s' % check.split('.')[0]).read()
 111         except IOError, e:
 112             checkfiles[check] = ""
 113     return checkfiles[check]
 114
 115
 116 def grep_manpage(check, match, section=None):
 117     manpage = get_manpage(check)
 118     if section is None:
 119         return not re.search(match, manpage) is None
 120     else:
 121         sec_started = False
 122         for line in [l.strip() for l in manpage.split("\n")]:
 123             if line == '[%s]' % section:
 124                 # Block starts with this line
 125                 sec_started = True
 126             elif sec_started == True and (line == '' or line.startswith('[')):
 127                 # Reached nex section. Didn't find a match ... so return false
 128                 return False
 129             elif sec_started == True:
 130                 if not re.search(match, line) is None:
 131                     return True
 132         return False
 133
 134
 135 def usage(msg=''):
 136     if msg:
 137         sys.stderr.write("%s\n" % msg)
 138     sys.stderr.write("\n")
 139     for s, l, t in _args:
 140         sys.stderr.write('  %3s, --%-15s    %s\n' % ('-' + s.rstrip(':'), l, t))
 141     sys.stderr.write("\n")
 142     sys.exit(0)
 143
 144
 145 ignored_variables = [
 146     # Python Module
 147     'datetime',
 148     'timedelta',
 149     # Misc vars
 150     'vars_before_check',
 151 ]
 152
 153 # Load all checks and record global var definitions
 154 # Also read the man pages
 155 global_vars = {}
 156 invalid_global_vars = {}
 157 for check in get_checks():
 158     vars_before_check = all_nonfunction_vars()
 159     execfile('checks/%s' % check)
 160     vars_after_check = all_nonfunction_vars()
 161
 162     global_vars[check] = []
 163     for name in vars_after_check:
 164         if name not in ignored_variables and name not in vars_before_check:
 165             global_vars[check] += [name]
 166
 167 # Load all perfometers
 168 perfometers = {}
 169 for f in get_perfometer_files():
 170     execfile('web/plugins/perfometer/' + f)
 171
 172 SCORE_START = 10
 173
 174 C_OK = 1
 175 C_FAILED = 2
 176 C_INVALID = 3
 177
 178 LINELENGTH_LIMIT = 140
 179
 180 TESTS = {
 181     'manpage': C_OK,
 182     'snmp_scan': C_OK,
 183     'pnp_tmpl': C_OK,
 184     'perfometer': C_OK,
 185     'snmp_scan': C_OK,
 186     'reindent': C_OK,
 187     'global_vars': C_OK,
 188     'linelength': C_OK,
 189     'wato': C_OK,
 190     'debug': C_OK,
 191 }
 192
 193 WEIGHT = {
 194     'manpage': 2,
 195     'global_vars': 2,
 196     'reindent': 2,
 197     'linelength': 2,
 198     'debug': 10,
 199 }
 200
 201 manpage = {}
 202 checkfiles = {}
 203
 204 # #############################################################################
 205 # Check definitions
 206 # #############################################################################
 207
 208 # Verify global vars:
 209 # - Are the global configuration vars mentioned in the manpage
 210 # - Are the other (internal) helper vars namend correctly
 211
 212
 213 def is_valid_global_vars(check):
 214     return True
 215
 216
 217 def verify_global_vars(check):
 218     check_file = check.split('.')[0]
 219
 220     # No global vars registered by this check
 221     if not check_file in global_vars:
 222         return True
 223
 224     # Loop all global vars of this check and verify them
 225     for var in global_vars[check_file]:
 226         invalid = False
 227         if not var.islower():
 228             invalid = True
 229
 230         # Documented configuration vars can be skipped
 231         if not invalid and grep_manpage(check, "^%s" % var, 'configuration'):
 232             continue
 233
 234         # Check unknown vars for correct prefix
 235         if not var.startswith('%s_' % check_file):
 236             invalid = True
 237
 238         if invalid:
 239             if not check_file in invalid_global_vars:
 240                 invalid_global_vars[check_file] = [var]
 241             else:
 242                 invalid_global_vars[check_file] += [var]
 243
 244     return not check_file in invalid_global_vars
 245
 246
 247 # Reindent code:
 248 # - Is there some code to be reindented?
 249
 250
 251 def is_valid_reindent(check):
 252     return True
 253
 254
 255 def verify_reindent(check):
 256     f = open('checks/%s' % check.split('.')[0])
 257     r = reindent.Reindenter(f)
 258     f.close()
 259     return not r.run()
 260
 261
 262 # Debug:
 263 # - are there 'print' statements in the code?
 264 # - are there 'sys.*.write' statements?
 265
 266
 267 def is_valid_debug(check):
 268     return True
 269
 270
 271 def verify_debug(check):
 272     f = open('checks/%s' % check.split('.')[0])
 273     for l in f.read().split("\n"):
 274         l = l.strip()
 275         if l.startswith('print ') or l.startswith('print('):
 276             return False
 277         elif l.startswith('sys.stdout.write'):
 278             return False
 279         elif l.startswith('sys.stderr.write'):
 280             return False
 281     f.close()
 282     return True
 283
 284
 285 # Toooo long lines:
 286 # - Are there too long lines in the check?
 287
 288
 289 def is_valid_linelength(check):
 290     return True
 291
 292
 293 def verify_linelength(check):
 294     """ Read the checkfile and compare all lines with the maximum line length """
 295     for line in get_checkfile(check).split("\n"):
 296         if len(line) > LINELENGTH_LIMIT:
 297             return False
 298     return True
 299
 300
 301 # Manpage:
 302 # - Does the check have a manpage?
 303
 304
 305 def is_valid_manpage(check):
 306     return True
 307
 308
 309 def verify_manpage(check):
 310     return os.path.exists('checkman/%s' % check)
 311
 312
 313 # SNMP scan function:
 314 # - Does the snmp check have a scan function?
 315
 316
 317 def is_valid_snmp_scan(check):
 318     return is_snmp_check(check)
 319
 320
 321 def verify_snmp_scan(check):
 322     return check in snmp_scan_functions
 323
 324
 325 # PNP-Template:
 326 # Does the chekc which produces perfdata have a pnp template?
 327
 328
 329 def is_valid_pnp_tmpl(check):
 330     return check_has_perfdata(check)
 331
 332
 333 def verify_pnp_tmpl(check):
 334     return os.path.exists('pnp-templates/check_mk-%s.php' % check)
 335
 336
 337 # Perfometer:
 338 # Checks with perfdata should provide a perfometer
 339
 340
 341 def is_valid_perfometer(check):
 342     return check_has_perfdata(check)
 343
 344
 345 def verify_perfometer(check):
 346     return 'check_mk-' + check in perfometers
 347
 348
 349 # Has this check a group assigned to?
 350 def is_valid_wato(check):
 351     return True
 352
 353
 354 def verify_wato(check):
 355     group = check_info[check]["group"]
 356     if group == "obsolete":
 357         return True
 358
 359     now = False
 360     for line in file("web/plugins/wato/check_parameters.py"):
 361         line = line.strip()
 362         if line.startswith("checkgroups.append(("):
 363             now = True
 364         elif now:
 365             this_group = line.strip(",").strip('"').strip("'")
 366             if group == this_group:
 367                 return True
 368             now = False
 369         else:
 370             now = False
 371     return False
 372
 373
 374 # #############################################################################
 375 # MAIN
 376 # #############################################################################
 377
 378 _args = [
 379     ('t:', 'tests=', 'Put one or more tests (comma separated) to limit the validations to perform'),
 380     ('c:', 'checks=', 'Put one or more checks (comma separated) to limit the checks to validate'),
 381     ('l:', 'score-limit=',
 382      'Set an upper score limit. Only checks with a lower score will be shown'),
 383     ('a', 'alternative-output', 'Choose an alternative output format (only listing the problems)'),
 384     ('v', 'verbose', 'Enable verbose output'),
 385     ('h', 'help', 'Show help message'),
 386 ]
 387
 388 _verbose = 0
 389 _score_limit = None
 390 _checks = []
 391 _tests = []
 392 _alternative_output = False
 393
 394 try:
 395     opts, args = getopt.getopt(sys.argv[1:], ''.join([x[0] for x in _args]), [x[1] for x in _args])
 396 except getopt.error, msg:
 397     usage(msg)
 398 for o, a in opts:
 399     if o in ('-v', '--verbose'):
 400         _verbose += 1
 401     elif o in ('-l', '--score-limit'):
 402         _score_limit = int(a)
 403     elif o in ('-a', '--alternative-output'):
 404         _alternative_output = True
 405
 406     elif o in ('-c', '--checks'):
 407         _checks = a.split(',')
 408     elif o in ('-t', '--tests'):
 409         _tests = a.split(',')
 410         TESTS = dict([(k, v) for k, v in TESTS.iteritems() if k in _tests])
 411     elif o in ('-h', '--help'):
 412         usage()
 413     else:
 414         usage('Unhandled parameter: %s' % str(o))
 415
 416 convert_check_info()
 417
 418 # 1) Perform checks
 419 #
 420 results = {}
 421 for check in get_all_checks():
 422     if _checks and not check in _checks:
 423         continue
 424
 425     score = SCORE_START
 426     check_results = TESTS.copy()
 427
 428     for test in TESTS.keys():
 429         if eval('is_valid_%s(\'%s\')' % (test, check)):
 430             if not eval('verify_%s(\'%s\')' % (test, check)):
 431                 check_results[test] = C_FAILED
 432                 score -= WEIGHT.get(test, 1)
 433         else:
 434             check_results[test] = C_INVALID
 435
 436     if _score_limit is None or score < _score_limit:
 437         results[check] = (score, check_results)
 438
 439 #
 440 # 2) Sort by score
 441 #
 442 results = sorted(results.iteritems(), key=lambda x: (x[1], x[0]))
 443
 444 #
 445 # 3) Output the results
 446 #
 447 if not _alternative_output:
 448     sys.stdout.write('%-25s' % 'Check')
 449     for _f in TESTS.keys():
 450         sys.stdout.write('%-15s' % (_f + ' (%d)' % WEIGHT.get(_f, 1)))
 451     sys.stdout.write('%-8s' % 'Score')
 452     sys.stdout.write("\n")
 453
 454 for check, (score, check_results) in results:
 455     if score == SCORE_START:
 456         color = tty_green
 457     elif score == 9:
 458         color = tty_yellow
 459     else:
 460         color = tty_red
 461
 462     if not _alternative_output:
 463         sys.stdout.write('%s%-25s%s' % (color, check, tty_normal))
 464
 465     first = True
 466     for _f in TESTS.keys():
 467         if _alternative_output:
 468             if check_results[_f] == C_FAILED:
 469                 if first:
 470                     sys.stdout.write("%-25s" % check)
 471                     first = False
 472                 sys.stdout.write("%s " % _f)
 473         else:
 474             if check_results[_f] == C_FAILED:
 475                 sub_color = tty_red
 476                 label = ':-('
 477             elif check_results[_f] == C_INVALID:
 478                 sub_color = tty_normal
 479                 label = ''
 480             else:
 481                 sub_color = tty_green
 482                 label = ':-)'
 483
 484             sys.stdout.write('%s%-15s%s' % (sub_color, label, tty_normal))
 485
 486     if not _alternative_output:
 487         sys.stdout.write('%s%-8s%s' % (color, score, tty_normal))
 488     if not _alternative_output or not first:
 489         sys.stdout.write("\n")
 490
 491 if _verbose > 0:
 492     sys.stdout.write("\n")
 493     sys.stdout.write(
 494         "===========================================================================\n")
 495     sys.stdout.write("Invalid global vars by check:\n")
 496     sys.stdout.write(
 497         "===========================================================================\n")
 498     # sort checks by name
 499     for check, vars in sorted(invalid_global_vars.iteritems(), key=operator.itemgetter(0)):
 500         for var in vars:
 501             sys.stdout.write("%-25s: %s\n" % (check, var))