utils/check_patch.py

   1 #!/usr/bin/python
   2 """
   3 Script to verify errors on autotest code contributions (patches).
   4 The workflow is as follows:
   5
   6  * Patch will be applied and eventual problems will be notified.
   7  * If there are new files created, remember user to add them to VCS.
   8  * If any added file looks like a executable file, remember user to make them
   9    executable.
  10  * If any of the files added or modified introduces trailing whitespaces, tabs
  11    or incorrect indentation, report problems.
  12  * If any of the files have problems during pylint validation, report failures.
  13  * If any of the files changed have a unittest suite, run the unittest suite
  14    and report any failures.
  15
  16 Usage: check_patch.py -p [/path/to/patch]
  17        check_patch.py -i [patchwork id]
  18
  19 @copyright: Red Hat Inc, 2009.
  20 @author: Lucas Meneghel Rodrigues <lmr@redhat.com>
  21 """
  22
  23 import os, stat, logging, sys, optparse, time
  24 import common
  25 from autotest_lib.client.common_lib import utils, error, logging_config
  26 from autotest_lib.client.common_lib import logging_manager
  27
  28
  29 class CheckPatchLoggingConfig(logging_config.LoggingConfig):
  30     def configure_logging(self, results_dir=None, verbose=False):
  31         super(CheckPatchLoggingConfig, self).configure_logging(use_console=True,
  32                                                                verbose=verbose)
  33
  34
  35 def ask(question, auto=False):
  36     """
  37     Raw input with a prompt that emulates logging.
  38
  39     @param question: Question to be asked
  40     @param auto: Whether to return "y" instead of asking the question
  41     """
  42     if auto:
  43         logging.info("%s (y/n) y" % question)
  44         return "y"
  45     return raw_input("%s INFO | %s (y/n) " %
  46                      (time.strftime("%H:%M:%S", time.localtime()), question))
  47
  48
  49 class VCS(object):
  50     """
  51     Abstraction layer to the version control system.
  52     """
  53     def __init__(self):
  54         """
  55         Class constructor. Guesses the version control name and instantiates it
  56         as a backend.
  57         """
  58         backend_name = self.guess_vcs_name()
  59         if backend_name == "SVN":
  60             self.backend = SubVersionBackend()
  61
  62
  63     def guess_vcs_name(self):
  64         if os.path.isdir(".svn"):
  65             return "SVN"
  66         else:
  67             logging.error("Could not figure version control system. Are you "
  68                           "on a working directory? Aborting.")
  69             sys.exit(1)
  70
  71
  72     def get_unknown_files(self):
  73         """
  74         Return a list of files unknown to the VCS.
  75         """
  76         return self.backend.get_unknown_files()
  77
  78
  79     def get_modified_files(self):
  80         """
  81         Return a list of files that were modified, according to the VCS.
  82         """
  83         return self.backend.get_modified_files()
  84
  85
  86     def add_untracked_file(self, file):
  87         """
  88         Add an untracked file to version control.
  89         """
  90         return self.backend.add_untracked_file(file)
  91
  92
  93     def revert_file(self, file):
  94         """
  95         Restore file according to the latest state on the reference repo.
  96         """
  97         return self.backend.revert_file(file)
  98
  99
 100     def apply_patch(self, patch):
 101         """
 102         Applies a patch using the most appropriate method to the particular VCS.
 103         """
 104         return self.backend.apply_patch(patch)
 105
 106
 107     def update(self):
 108         """
 109         Updates the tree according to the latest state of the public tree
 110         """
 111         return self.backend.update()
 112
 113
 114 class SubVersionBackend(object):
 115     """
 116     Implementation of a subversion backend for use with the VCS abstraction
 117     layer.
 118     """
 119     def __init__(self):
 120         logging.debug("Subversion VCS backend initialized.")
 121         self.ignored_extension_list = ['.orig', '.bak']
 122
 123
 124     def get_unknown_files(self):
 125         status = utils.system_output("svn status --ignore-externals")
 126         unknown_files = []
 127         for line in status.split("\n"):
 128             status_flag = line[0]
 129             if line and status_flag == "?":
 130                 for extension in self.ignored_extension_list:
 131                     if not line.endswith(extension):
 132                         unknown_files.append(line[1:].strip())
 133         return unknown_files
 134
 135
 136     def get_modified_files(self):
 137         status = utils.system_output("svn status --ignore-externals")
 138         modified_files = []
 139         for line in status.split("\n"):
 140             status_flag = line[0]
 141             if line and status_flag == "M" or status_flag == "A":
 142                 modified_files.append(line[1:].strip())
 143         return modified_files
 144
 145
 146     def add_untracked_file(self, file):
 147         """
 148         Add an untracked file under revision control.
 149
 150         @param file: Path to untracked file.
 151         """
 152         try:
 153             utils.run('svn add %s' % file)
 154         except error.CmdError, e:
 155             logging.error("Problem adding file %s to svn: %s", file, e)
 156             sys.exit(1)
 157
 158
 159     def revert_file(self, file):
 160         """
 161         Revert file against last revision.
 162
 163         @param file: Path to file to be reverted.
 164         """
 165         try:
 166             utils.run('svn revert %s' % file)
 167         except error.CmdError, e:
 168             logging.error("Problem reverting file %s: %s", file, e)
 169             sys.exit(1)
 170
 171
 172     def apply_patch(self, patch):
 173         """
 174         Apply a patch to the code base. Patches are expected to be made using
 175         level -p1, and taken according to the code base top level.
 176
 177         @param patch: Path to the patch file.
 178         """
 179         try:
 180             utils.system_output("patch -p1 < %s" % patch)
 181         except:
 182             logging.error("Patch applied incorrectly. Possible causes: ")
 183             logging.error("1 - Patch might not be -p1")
 184             logging.error("2 - You are not at the top of the autotest tree")
 185             logging.error("3 - Patch was made using an older tree")
 186             logging.error("4 - Mailer might have messed the patch")
 187             sys.exit(1)
 188
 189     def update(self):
 190         try:
 191             utils.system("svn update", ignore_status=True)
 192         except error.CmdError, e:
 193             logging.error("SVN tree update failed: %s" % e)
 194
 195
 196 class FileChecker(object):
 197     """
 198     Picks up a given file and performs various checks, looking after problems
 199     and eventually suggesting solutions.
 200     """
 201     def __init__(self, path, confirm=False):
 202         """
 203         Class constructor, sets the path attribute.
 204
 205         @param path: Path to the file that will be checked.
 206         @param confirm: Whether to answer yes to all questions asked without
 207                 prompting the user.
 208         """
 209         self.path = path
 210         self.confirm = confirm
 211         self.basename = os.path.basename(self.path)
 212         if self.basename.endswith('.py'):
 213             self.is_python = True
 214         else:
 215             self.is_python = False
 216
 217         mode = os.stat(self.path)[stat.ST_MODE]
 218         if mode & stat.S_IXUSR:
 219             self.is_executable = True
 220         else:
 221             self.is_executable = False
 222
 223         checked_file = open(self.path, "r")
 224         self.first_line = checked_file.readline()
 225         checked_file.close()
 226         self.corrective_actions = []
 227         self.indentation_exceptions = ['job_unittest.py']
 228
 229
 230     def _check_indent(self):
 231         """
 232         Verifies the file with reindent.py. This tool performs the following
 233         checks on python files:
 234
 235           * Trailing whitespaces
 236           * Tabs
 237           * End of line
 238           * Incorrect indentation
 239
 240         For the purposes of checking, the dry run mode is used and no changes
 241         are made. It is up to the user to decide if he wants to run reindent
 242         to correct the issues.
 243         """
 244         reindent_raw = utils.system_output('reindent.py -v -d %s | head -1' %
 245                                            self.path)
 246         reindent_results = reindent_raw.split(" ")[-1].strip(".")
 247         if reindent_results == "changed":
 248             if self.basename not in self.indentation_exceptions:
 249                 self.corrective_actions.append("reindent.py -v %s" % self.path)
 250
 251
 252     def _check_code(self):
 253         """
 254         Verifies the file with run_pylint.py. This tool will call the static
 255         code checker pylint using the special autotest conventions and warn
 256         only on problems. If problems are found, a report will be generated.
 257         Some of the problems reported might be bogus, but it's allways good
 258         to look at them.
 259         """
 260         c_cmd = 'run_pylint.py %s' % self.path
 261         rc = utils.system(c_cmd, ignore_status=True)
 262         if rc != 0:
 263             logging.error("Syntax issues found during '%s'", c_cmd)
 264
 265
 266     def _check_unittest(self):
 267         """
 268         Verifies if the file in question has a unittest suite, if so, run the
 269         unittest and report on any failures. This is important to keep our
 270         unit tests up to date.
 271         """
 272         if "unittest" not in self.basename:
 273             stripped_name = self.basename.strip(".py")
 274             unittest_name = stripped_name + "_unittest.py"
 275             unittest_path = self.path.replace(self.basename, unittest_name)
 276             if os.path.isfile(unittest_path):
 277                 unittest_cmd = 'python %s' % unittest_path
 278                 rc = utils.system(unittest_cmd, ignore_status=True)
 279                 if rc != 0:
 280                     logging.error("Unittest issues found during '%s'",
 281                                   unittest_cmd)
 282
 283
 284     def _check_permissions(self):
 285         """
 286         Verifies the execution permissions, specifically:
 287           * Files with no shebang and execution permissions are reported.
 288           * Files with shebang and no execution permissions are reported.
 289         """
 290         if self.first_line.startswith("#!"):
 291             if not self.is_executable:
 292                 self.corrective_actions.append("svn propset svn:executable ON %s" % self.path)
 293         else:
 294             if self.is_executable:
 295                 self.corrective_actions.append("svn propdel svn:executable %s" % self.path)
 296
 297
 298     def report(self):
 299         """
 300         Executes all required checks, if problems are found, the possible
 301         corrective actions are listed.
 302         """
 303         self._check_permissions()
 304         if self.is_python:
 305             self._check_indent()
 306             self._check_code()
 307             self._check_unittest()
 308         if self.corrective_actions:
 309             for action in self.corrective_actions:
 310                 answer = ask("Would you like to execute %s?" % action,
 311                              auto=self.confirm)
 312                 if answer == "y":
 313                     rc = utils.system(action, ignore_status=True)
 314                     if rc != 0:
 315                         logging.error("Error executing %s" % action)
 316
 317
 318 class PatchChecker(object):
 319     def __init__(self, patch=None, patchwork_id=None, confirm=False):
 320         self.confirm = confirm
 321         self.base_dir = os.getcwd()
 322         if patch:
 323             self.patch = os.path.abspath(patch)
 324         if patchwork_id:
 325             self.patch = self._fetch_from_patchwork(patchwork_id)
 326
 327         if not os.path.isfile(self.patch):
 328             logging.error("Invalid patch file %s provided. Aborting.",
 329                           self.patch)
 330             sys.exit(1)
 331
 332         self.vcs = VCS()
 333         changed_files_before = self.vcs.get_modified_files()
 334         if changed_files_before:
 335             logging.error("Repository has changed files prior to patch "
 336                           "application. ")
 337             answer = ask("Would you like to revert them?", auto=self.confirm)
 338             if answer == "n":
 339                 logging.error("Not safe to proceed without reverting files.")
 340                 sys.exit(1)
 341             else:
 342                 for changed_file in changed_files_before:
 343                     self.vcs.revert_file(changed_file)
 344
 345         self.untracked_files_before = self.vcs.get_unknown_files()
 346         self.vcs.update()
 347
 348
 349     def _fetch_from_patchwork(self, id):
 350         """
 351         Gets a patch file from patchwork and puts it under the cwd so it can
 352         be applied.
 353
 354         @param id: Patchwork patch id.
 355         """
 356         patch_url = "http://patchwork.test.kernel.org/patch/%s/mbox/" % id
 357         patch_dest = os.path.join(self.base_dir, 'patchwork-%s.patch' % id)
 358         patch = utils.get_file(patch_url, patch_dest)
 359         # Patchwork sometimes puts garbage on the path, such as long
 360         # sequences of underscores (_______). Get rid of those.
 361         patch_ro = open(patch, 'r')
 362         patch_contents = patch_ro.readlines()
 363         patch_ro.close()
 364         patch_rw = open(patch, 'w')
 365         for line in patch_contents:
 366             if not line.startswith("___"):
 367                 patch_rw.write(line)
 368         patch_rw.close()
 369         return patch
 370
 371
 372     def _check_files_modified_patch(self):
 373         untracked_files_after = self.vcs.get_unknown_files()
 374         modified_files_after = self.vcs.get_modified_files()
 375         add_to_vcs = []
 376         for untracked_file in untracked_files_after:
 377             if untracked_file not in self.untracked_files_before:
 378                 add_to_vcs.append(untracked_file)
 379
 380         if add_to_vcs:
 381             logging.info("The files: ")
 382             for untracked_file in add_to_vcs:
 383                 logging.info(untracked_file)
 384             logging.info("Might need to be added to VCS")
 385             answer = ask("Would you like to add them to VCS ?")
 386             if answer == "y":
 387                 for untracked_file in add_to_vcs:
 388                     self.vcs.add_untracked_file(untracked_file)
 389                     modified_files_after.append(untracked_file)
 390             elif answer == "n":
 391                 pass
 392
 393         for modified_file in modified_files_after:
 394             # Additional safety check, new commits might introduce
 395             # new directories
 396             if os.path.isfile(modified_file):
 397                 file_checker = FileChecker(modified_file)
 398                 file_checker.report()
 399
 400
 401     def check(self):
 402         self.vcs.apply_patch(self.patch)
 403         self._check_files_modified_patch()
 404
 405
 406 if __name__ == "__main__":
 407     parser = optparse.OptionParser()
 408     parser.add_option('-p', '--patch', dest="local_patch", action='store',
 409                       help='path to a patch file that will be checked')
 410     parser.add_option('-i', '--patchwork-id', dest="id", action='store',
 411                       help='id of a given patchwork patch')
 412     parser.add_option('--verbose', dest="debug", action='store_true',
 413                       help='include debug messages in console output')
 414     parser.add_option('-f', '--full-check', dest="full_check",
 415                       action='store_true',
 416                       help='check the full tree for corrective actions')
 417     parser.add_option('-y', '--yes', dest="confirm",
 418                       action='store_true',
 419                       help='Answer yes to all questions')
 420
 421     options, args = parser.parse_args()
 422     local_patch = options.local_patch
 423     id = options.id
 424     debug = options.debug
 425     full_check = options.full_check
 426     confirm = options.confirm
 427
 428     logging_manager.configure_logging(CheckPatchLoggingConfig(), verbose=debug)
 429
 430     ignore_file_list = ['common.py']
 431     if full_check:
 432         for root, dirs, files in os.walk('.'):
 433             if not '.svn' in root:
 434                 for file in files:
 435                     if file not in ignore_file_list:
 436                         path = os.path.join(root, file)
 437                         file_checker = FileChecker(path, confirm=confirm)
 438                         file_checker.report()
 439     else:
 440         if local_patch:
 441             patch_checker = PatchChecker(patch=local_patch, confirm=confirm)
 442         elif id:
 443             patch_checker = PatchChecker(patchwork_id=id, confirm=confirm)
 444         else:
 445             logging.error('No patch or patchwork id specified. Aborting.')
 446             sys.exit(1)
 447         patch_checker.check()