tests/testlib/pylint_cmk.py

   1 #!/usr/bin/python
   2 # Library for pylint checks of Check_MK
   3
   4 from __future__ import print_function
   5
   6 import os
   7 import sys
   8 import glob
   9 import multiprocessing
  10 import shutil
  11 import subprocess
  12 import tempfile
  13
  14 from pylint.reporters.text import ColorizedTextReporter, ParseableTextReporter
  15 from pylint.utils import Message
  16
  17 from testlib import repo_path, cmk_path, cmc_path, cme_path
  18
  19 def check_files(base_dir):
  20     filelist = sorted([ base_dir + "/" + f for f in os.listdir(base_dir)
  21                          if not f.startswith(".") ])
  22
  23     # Sort: first includes, then other
  24     filelist = [ f for f in filelist if f.endswith(".include") ] + \
  25                [ f for f in filelist if not f.endswith(".include") ]
  26
  27     return filelist
  28
  29
  30 def add_file(f, path):
  31     relpath = os.path.relpath(os.path.realpath(path), cmk_path())
  32     f.write("# -*- encoding: utf-8 -*-")
  33     f.write("#\n")
  34     f.write("# ORIG-FILE: " + relpath + "\n")
  35     f.write("#\n")
  36     f.write("\n")
  37     f.write(file(path).read())
  38
  39
  40 def run_pylint(base_path, check_files=None): #, cleanup_test_dir=False):
  41     args = os.environ.get("PYLINT_ARGS", "")
  42     if args:
  43         pylint_args = args.split(" ")
  44     else:
  45         pylint_args = []
  46
  47     pylint_cfg = repo_path() + "/.pylintrc"
  48
  49     if not check_files:
  50         check_files = get_pylint_files(base_path, "*")
  51         if not check_files:
  52             print("Nothing to do...")
  53             return 0 # nothing to do
  54
  55     cmd = [
  56         "python", "-m", "pylint",
  57         "--rcfile", pylint_cfg,
  58         "--jobs=%d" % num_jobs_to_use(),
  59     ] + pylint_args + check_files
  60
  61     os.putenv("TEST_PATH", repo_path() + "/tests")
  62     print("Running pylint in '%s' with: %s" % (base_path, subprocess.list2cmdline(cmd)))
  63     p = subprocess.Popen(cmd, shell=False, cwd=base_path)
  64     exit_code = p.wait()
  65     print("Finished with exit code: %d" % exit_code)
  66
  67     #if exit_code == 0 and cleanup_test_dir:
  68     #    # Don't remove directory when specified via WORKDIR env
  69     #    if not os.environ.get("WORKDIR"):
  70     #        print("Removing build path...")
  71     #        shutil.rmtree(base_path)
  72
  73     return exit_code
  74
  75 def num_jobs_to_use():
  76     # Naive heuristic, but looks OK for our use cases: Normal quad core CPUs
  77     # with HT report 8 CPUs (=> 6 jobs), our server 24-core CPU reports 48 CPUs
  78     # (=> 11 jobs). Just using 0 (meaning: use all reported CPUs) might just
  79     # work, too, but it's probably a bit too much.
  80     return multiprocessing.cpu_count() / 8 + 5
  81
  82
  83 def get_pylint_files(base_path, file_pattern):
  84     files = []
  85     for path in glob.glob("%s/%s" % (base_path, file_pattern)):
  86         f = path[len(base_path)+1:]
  87
  88         if is_python_file(path):
  89             files.append(f)
  90
  91     return files
  92
  93
  94 def is_python_file(path):
  95     if not os.path.isfile(path) or os.path.islink(path):
  96         return False
  97
  98     if path.endswith(".py"):
  99         return True
 100
 101     # Only add python files
 102     shebang = file(path, "r").readline()
 103     if shebang.startswith("#!") and "python" in shebang:
 104         return True
 105
 106     return False
 107
 108
 109 # Check_MK currently uses a packed version of it's files to
 110 # run the pylint tests because it's not well structured in
 111 # python modules. This custom reporter rewrites the found
 112 # messages to tell the users the original location in the
 113 # python sources
 114 # TODO: This can be dropped once we have refactored checks/inventory/bakery plugins
 115 # to real modules
 116 class CMKFixFileMixin(object):
 117     def handle_message(self, msg):
 118         new_path, new_line = self._orig_location_from_compiled_file(msg)
 119
 120         if new_path is None:
 121             new_path = self._change_path_to_repo_path(msg)
 122
 123         if new_path is not None:
 124             msg = msg._replace(path=new_path)
 125         if new_line is not None:
 126             msg = msg._replace(line=new_line)
 127
 128         super(CMKFixFileMixin, self).handle_message(msg)
 129
 130
 131     def _change_path_to_repo_path(self, msg):
 132         return os.path.relpath(msg.abspath, cmk_path())
 133
 134
 135     def _orig_location_from_compiled_file(self, msg):
 136         lines = file(msg.abspath).readlines()
 137         line_nr = msg.line
 138         orig_file, went_back = None, -3
 139         while line_nr > 0:
 140             line_nr -= 1
 141             went_back += 1
 142             line = lines[line_nr]
 143             if line.startswith("# ORIG-FILE: "):
 144                 orig_file = line.split(": ", 1)[1].strip()
 145                 break
 146
 147         if orig_file is None:
 148             went_back = None
 149
 150         return orig_file, went_back
 151
 152
 153
 154 class CMKColorizedTextReporter(CMKFixFileMixin, ColorizedTextReporter):
 155     name = "cmk_colorized"
 156
 157
 158
 159 class CMKParseableTextReporter(CMKFixFileMixin, ParseableTextReporter):
 160     name = "cmk_parseable"
 161
 162
 163 def verify_pylint_version():
 164     import pylint
 165     if tuple(map(int, pylint.__version__.split("."))) < (1, 5, 5):
 166         raise Exception("You need to use at least pylint 1.5.5. Run \"make setup\" in "
 167                         "pylint directory to get the current version.")
 168
 169
 170 # Is called by pylint to load this plugin
 171 def register(linter):
 172     verify_pylint_version()
 173
 174     linter.register_reporter(CMKColorizedTextReporter)
 175     linter.register_reporter(CMKParseableTextReporter)