tools/checkperms/checkperms.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Makes sure files have the right permissions.
   7
   8 Some developers have broken SCM configurations that flip the svn:executable
   9 permission on for no good reason. Unix developers who run ls --color will then
  10 see .cc files in green and get confused.
  11
  12 - For file extensions that must be executable, add it to EXECUTABLE_EXTENSIONS.
  13 - For file extensions that must not be executable, add it to
  14   NOT_EXECUTABLE_EXTENSIONS.
  15 - To ignore all the files inside a directory, add it to IGNORED_PATHS.
  16 - For file base name with ambiguous state and that should not be checked for
  17   shebang, add it to IGNORED_FILENAMES.
  18
  19 Any file not matching the above will be opened and looked if it has a shebang.
  20 It this doesn't match the executable bit on the file, the file will be flagged.
  21
  22 Note that all directory separators must be slashes (Unix-style) and not
  23 backslashes. All directories should be relative to the source root and all
  24 file paths should be only lowercase.
  25 """
  26
  27 import logging
  28 import optparse
  29 import os
  30 import stat
  31 import subprocess
  32 import sys
  33
  34 #### USER EDITABLE SECTION STARTS HERE ####
  35
  36 # Files with these extensions must have executable bit set.
  37 EXECUTABLE_EXTENSIONS = (
  38   'bat',
  39   'dll',
  40   'dylib',
  41   'exe',
  42 )
  43
  44 # These files must have executable bit set.
  45 EXECUTABLE_PATHS = (
  46   # TODO(maruel): Detect ELF files.
  47   'chrome/test/data/webrtc/linux/peerconnection_server',
  48   'chrome/installer/mac/sign_app.sh.in',
  49   'chrome/installer/mac/sign_versioned_dir.sh.in',
  50 )
  51
  52 # These files must not have the executable bit set. This is mainly a performance
  53 # optimization as these files are not checked for shebang. The list was
  54 # partially generated from:
  55 # git ls-files | grep "\\." | sed 's/.*\.//' | sort | uniq -c | sort -b -g
  56 NON_EXECUTABLE_EXTENSIONS = (
  57   '1',
  58   '3ds',
  59   'S',
  60   'am',
  61   'applescript',
  62   'asm',
  63   'c',
  64   'cc',
  65   'cfg',
  66   'chromium',
  67   'cpp',
  68   'crx',
  69   'cs',
  70   'css',
  71   'cur',
  72   'def',
  73   'der',
  74   'expected',
  75   'gif',
  76   'grd',
  77   'gyp',
  78   'gypi',
  79   'h',
  80   'hh',
  81   'htm',
  82   'html',
  83   'hyph',
  84   'ico',
  85   'idl',
  86   'java',
  87   'jpg',
  88   'js',
  89   'json',
  90   'm',
  91   'm4',
  92   'mm',
  93   'mms',
  94   'mock-http-headers',
  95   'nmf',
  96   'onc',
  97   'pat',
  98   'patch',
  99   'pdf',
 100   'pem',
 101   'plist',
 102   'png',
 103   'proto',
 104   'rc',
 105   'rfx',
 106   'rgs',
 107   'rules',
 108   'spec',
 109   'sql',
 110   'srpc',
 111   'svg',
 112   'tcl',
 113   'test',
 114   'tga',
 115   'txt',
 116   'vcproj',
 117   'vsprops',
 118   'webm',
 119   'word',
 120   'xib',
 121   'xml',
 122   'xtb',
 123   'zip',
 124 )
 125
 126 # File names that are always whitelisted.  (These are all autoconf spew.)
 127 IGNORED_FILENAMES = (
 128   'config.guess',
 129   'config.sub',
 130   'configure',
 131   'depcomp',
 132   'install-sh',
 133   'missing',
 134   'mkinstalldirs',
 135   'naclsdk',
 136   'scons',
 137 )
 138
 139 # File paths starting with one of these will be ignored as well.
 140 IGNORED_PATHS = (
 141   # TODO(maruel): Detect ELF files.
 142   'chrome/test/data/extensions/uitest/plugins',
 143   'chrome/test/data/extensions/uitest/plugins_private',
 144   'chrome/test/data/webrtc/linux/peerconnection_server',
 145   'chrome/installer/mac/sign_app.sh.in',
 146   'chrome/installer/mac/sign_versioned_dir.sh.in',
 147   'native_client_sdk/src/build_tools/sdk_tools/third_party/',
 148   'out/',
 149   # TODO(maruel): Fix these.
 150   'third_party/android_testrunner/',
 151   'third_party/closure_linter/',
 152   'third_party/devscripts/licensecheck.pl.vanilla',
 153   'third_party/hyphen/',
 154   'third_party/jemalloc/',
 155   'third_party/lcov-1.9/contrib/galaxy/conglomerate_functions.pl',
 156   'third_party/lcov-1.9/contrib/galaxy/gen_makefile.sh',
 157   'third_party/lcov/contrib/galaxy/conglomerate_functions.pl',
 158   'third_party/lcov/contrib/galaxy/gen_makefile.sh',
 159   'third_party/libevent/autogen.sh',
 160   'third_party/libevent/test/test.sh',
 161   'third_party/libxml/linux/xml2-config',
 162   'third_party/libxml/src/ltmain.sh',
 163   'third_party/mesa/',
 164   'third_party/protobuf/',
 165   'third_party/python_gflags/gflags.py',
 166   'third_party/sqlite/',
 167   'third_party/talloc/script/mksyms.sh',
 168   'third_party/tcmalloc/',
 169   'third_party/tlslite/setup.py',
 170 )
 171
 172 #### USER EDITABLE SECTION ENDS HERE ####
 173
 174 assert set(EXECUTABLE_EXTENSIONS) & set(NON_EXECUTABLE_EXTENSIONS) == set()
 175
 176
 177 def capture(cmd, cwd):
 178   """Returns the output of a command.
 179
 180   Ignores the error code or stderr.
 181   """
 182   logging.debug('%s; cwd=%s' % (' '.join(cmd), cwd))
 183   env = os.environ.copy()
 184   env['LANGUAGE'] = 'en_US.UTF-8'
 185   p = subprocess.Popen(
 186       cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd, env=env)
 187   return p.communicate()[0]
 188
 189
 190 def get_svn_info(dir_path):
 191   """Returns svn meta-data for a svn checkout."""
 192   if not os.path.isdir(dir_path):
 193     return {}
 194   out = capture(['svn', 'info', '.', '--non-interactive'], dir_path)
 195   return dict(l.split(': ', 1) for l in out.splitlines() if l)
 196
 197
 198 def get_svn_url(dir_path):
 199   return get_svn_info(dir_path).get('URL')
 200
 201
 202 def get_svn_root(dir_path):
 203   """Returns the svn checkout root or None."""
 204   svn_url = get_svn_url(dir_path)
 205   if not svn_url:
 206     return None
 207   logging.info('svn url: %s' % svn_url)
 208   while True:
 209     parent = os.path.dirname(dir_path)
 210     if parent == dir_path:
 211       return None
 212     svn_url = svn_url.rsplit('/', 1)[0]
 213     if svn_url != get_svn_url(parent):
 214       return dir_path
 215     dir_path = parent
 216
 217
 218 def get_git_root(dir_path):
 219   """Returns the git checkout root or None."""
 220   root = capture(['git', 'rev-parse', '--show-toplevel'], dir_path).strip()
 221   if root:
 222     return root
 223
 224
 225 def is_ignored(rel_path):
 226   """Returns True if rel_path is in our whitelist of files to ignore."""
 227   rel_path = rel_path.lower()
 228   return (
 229       os.path.basename(rel_path) in IGNORED_FILENAMES or
 230       rel_path.startswith(IGNORED_PATHS))
 231
 232
 233 def must_be_executable(rel_path):
 234   """The file name represents a file type that must have the executable bit
 235   set.
 236   """
 237   return (
 238       os.path.splitext(rel_path)[1][1:].lower() in EXECUTABLE_EXTENSIONS or
 239       rel_path in EXECUTABLE_PATHS)
 240
 241
 242 def must_not_be_executable(rel_path):
 243   """The file name represents a file type that must not have the executable
 244   bit set.
 245   """
 246   return os.path.splitext(rel_path)[1][1:].lower() in NON_EXECUTABLE_EXTENSIONS
 247
 248
 249 def has_executable_bit(full_path):
 250   """Returns if any executable bit is set."""
 251   permission = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
 252   return bool(permission & os.stat(full_path).st_mode)
 253
 254
 255 def has_shebang(full_path):
 256   """Returns if the file starts with #!/.
 257
 258   file_path is the absolute path to the file.
 259   """
 260   with open(full_path, 'rb') as f:
 261     return f.read(3) == '#!/'
 262
 263
 264 class ApiBase(object):
 265   def __init__(self, root_dir, bare_output):
 266     self.root_dir = root_dir
 267     self.bare_output = bare_output
 268     self.count = 0
 269     self.count_shebang = 0
 270
 271   def check_file(self, rel_path):
 272     """Checks file_path's permissions and returns an error if it is
 273     inconsistent.
 274
 275     It is assumed that the file is not ignored by is_ignored().
 276
 277     If the file name is matched with must_be_executable() or
 278     must_not_be_executable(), only its executable bit is checked.
 279     Otherwise, the 3 first bytes of the file are read to verify if it has a
 280     shebang and compares this with the executable bit on the file.
 281     """
 282     logging.debug('check_file(%s)' % rel_path)
 283     self.count += 1
 284
 285     full_path = os.path.join(self.root_dir, rel_path)
 286     try:
 287       bit = has_executable_bit(full_path)
 288     except OSError:
 289       # It's faster to catch exception than call os.path.islink(). Chromium
 290       # tree happens to have invalid symlinks under
 291       # third_party/openssl/openssl/test/.
 292       return None
 293
 294     if must_be_executable(rel_path):
 295       if not bit:
 296         if self.bare_output:
 297           return rel_path
 298         return '%s: Must have executable bit set' % rel_path
 299       return
 300     if must_not_be_executable(rel_path):
 301       if bit:
 302         if self.bare_output:
 303           return rel_path
 304         return '%s: Must not have executable bit set' % rel_path
 305       return
 306
 307     # For the others, it depends on the shebang.
 308     shebang = has_shebang(full_path)
 309     self.count_shebang += 1
 310     if bit != shebang:
 311       if self.bare_output:
 312         return rel_path
 313       if bit:
 314         return '%s: Has executable bit but not shebang' % rel_path
 315       else:
 316         return '%s: Has shebang but not executable bit' % rel_path
 317
 318   def check_dir(self, rel_path):
 319     return self.check(rel_path)
 320
 321   def check(self, start_dir):
 322     """Check the files in start_dir, recursively check its subdirectories."""
 323     errors = []
 324     items = self.list_dir(start_dir)
 325     logging.info('check(%s) -> %d' % (start_dir, len(items)))
 326     for item in items:
 327       full_path = os.path.join(self.root_dir, start_dir, item)
 328       rel_path = full_path[len(self.root_dir) + 1:]
 329       if is_ignored(rel_path):
 330         continue
 331       if os.path.isdir(full_path):
 332         # Depth first.
 333         errors.extend(self.check_dir(rel_path))
 334       else:
 335         error = self.check_file(rel_path)
 336         if error:
 337           errors.append(error)
 338     return errors
 339
 340   def list_dir(self, start_dir):
 341     """Lists all the files and directory inside start_dir."""
 342     return sorted(
 343       x for x in os.listdir(os.path.join(self.root_dir, start_dir))
 344       if not x.startswith('.')
 345     )
 346
 347
 348 class ApiSvnQuick(ApiBase):
 349   """Returns all files in svn-versioned directories, independent of the fact if
 350   they are versionned.
 351
 352   Uses svn info in each directory to determine which directories should be
 353   crawled.
 354   """
 355   def __init__(self, *args):
 356     super(ApiSvnQuick, self).__init__(*args)
 357     self.url = get_svn_url(self.root_dir)
 358
 359   def check_dir(self, rel_path):
 360     url = self.url + '/' + rel_path
 361     if get_svn_url(os.path.join(self.root_dir, rel_path)) != url:
 362       return []
 363     return super(ApiSvnQuick, self).check_dir(rel_path)
 364
 365
 366 class ApiAllFilesAtOnceBase(ApiBase):
 367   _files = None
 368
 369   def list_dir(self, start_dir):
 370     """Lists all the files and directory inside start_dir."""
 371     if self._files is None:
 372       self._files = sorted(self._get_all_files())
 373       if not self.bare_output:
 374         print 'Found %s files' % len(self._files)
 375     start_dir = start_dir[len(self.root_dir) + 1:]
 376     return [
 377       x[len(start_dir):] for x in self._files if x.startswith(start_dir)
 378     ]
 379
 380   def _get_all_files(self):
 381     """Lists all the files and directory inside self._root_dir."""
 382     raise NotImplementedError()
 383
 384
 385 class ApiSvn(ApiAllFilesAtOnceBase):
 386   """Returns all the subversion controlled files.
 387
 388   Warning: svn ls is abnormally slow.
 389   """
 390   def _get_all_files(self):
 391     cmd = ['svn', 'ls', '--non-interactive', '--recursive']
 392     return (
 393         x for x in capture(cmd, self.root_dir).splitlines()
 394         if not x.endswith(os.path.sep))
 395
 396
 397 class ApiGit(ApiAllFilesAtOnceBase):
 398   def _get_all_files(self):
 399     return capture(['git', 'ls-files'], cwd=self.root_dir).splitlines()
 400
 401
 402 def get_scm(dir_path, bare):
 403   """Returns a properly configured ApiBase instance."""
 404   cwd = os.getcwd()
 405   root = get_svn_root(dir_path or cwd)
 406   if root:
 407     if not bare:
 408       print('Found subversion checkout at %s' % root)
 409     return ApiSvnQuick(dir_path or root, bare)
 410   root = get_git_root(dir_path or cwd)
 411   if root:
 412     if not bare:
 413       print('Found git repository at %s' % root)
 414     return ApiGit(dir_path or root, bare)
 415
 416   # Returns a non-scm aware checker.
 417   if not bare:
 418     print('Failed to determine the SCM for %s' % dir_path)
 419   return ApiBase(dir_path or cwd, bare)
 420
 421
 422 def main():
 423   usage = """Usage: python %prog [--root <root>] [tocheck]
 424   tocheck  Specifies the directory, relative to root, to check. This defaults
 425            to "." so it checks everything.
 426
 427 Examples:
 428   python %prog
 429   python %prog --root /path/to/source chrome"""
 430
 431   parser = optparse.OptionParser(usage=usage)
 432   parser.add_option(
 433       '--root',
 434       help='Specifies the repository root. This defaults '
 435            'to the checkout repository root')
 436   parser.add_option(
 437       '-v', '--verbose', action='count', default=0, help='Print debug logging')
 438   parser.add_option(
 439       '--bare',
 440       action='store_true',
 441       default=False,
 442       help='Prints the bare filename triggering the checks')
 443   options, args = parser.parse_args()
 444
 445   levels = [logging.ERROR, logging.INFO, logging.DEBUG]
 446   logging.basicConfig(level=levels[min(len(levels) - 1, options.verbose)])
 447
 448   if len(args) > 1:
 449     parser.error('Too many arguments used')
 450
 451   if options.root:
 452     options.root = os.path.abspath(options.root)
 453
 454   api = get_scm(options.root, options.bare)
 455   if args:
 456     start_dir = args[0]
 457   else:
 458     start_dir = api.root_dir
 459
 460   errors = api.check(start_dir)
 461
 462   if not options.bare:
 463     print 'Processed %s files, %d files where tested for shebang' % (
 464         api.count, api.count_shebang)
 465
 466   if errors:
 467     if not options.bare:
 468       print '\nFAILED\n'
 469     print '\n'.join(errors)
 470     return 1
 471   if not options.bare:
 472     print '\nSUCCESS\n'
 473   return 0
 474
 475
 476 if '__main__' == __name__:
 477   sys.exit(main())