Lib/pyclbr.py

   1 """Parse a Python module and describe its classes and methods.
   2
   3 Parse enough of a Python file to recognize imports and class and
   4 method definitions, and to find out the superclasses of a class.
   5
   6 The interface consists of a single function:
   7         readmodule_ex(module [, path])
   8 where module is the name of a Python module, and path is an optional
   9 list of directories where the module is to be searched.  If present,
  10 path is prepended to the system search path sys.path.  The return
  11 value is a dictionary.  The keys of the dictionary are the names of
  12 the classes defined in the module (including classes that are defined
  13 via the from XXX import YYY construct).  The values are class
  14 instances of the class Class defined here.  One special key/value pair
  15 is present for packages: the key '__path__' has a list as its value
  16 which contains the package search path.
  17
  18 A class is described by the class Class in this module.  Instances
  19 of this class have the following instance variables:
  20         module -- the module name
  21         name -- the name of the class
  22         super -- a list of super classes (Class instances)
  23         methods -- a dictionary of methods
  24         file -- the file in which the class was defined
  25         lineno -- the line in the file on which the class statement occurred
  26 The dictionary of methods uses the method names as keys and the line
  27 numbers on which the method was defined as values.
  28 If the name of a super class is not recognized, the corresponding
  29 entry in the list of super classes is not a class instance but a
  30 string giving the name of the super class.  Since import statements
  31 are recognized and imported modules are scanned as well, this
  32 shouldn't happen often.
  33
  34 A function is described by the class Function in this module.
  35 Instances of this class have the following instance variables:
  36         module -- the module name
  37         name -- the name of the class
  38         file -- the file in which the class was defined
  39         lineno -- the line in the file on which the class statement occurred
  40 """
  41
  42 import sys
  43 import imp
  44 import tokenize # Python tokenizer
  45 from token import NAME, DEDENT, NEWLINE
  46 from operator import itemgetter
  47
  48 __all__ = ["readmodule", "readmodule_ex", "Class", "Function"]
  49
  50 _modules = {}                           # cache of modules we've seen
  51
  52 # each Python class is represented by an instance of this class
  53 class Class:
  54     '''Class to represent a Python class.'''
  55     def __init__(self, module, name, super, file, lineno):
  56         self.module = module
  57         self.name = name
  58         if super is None:
  59             super = []
  60         self.super = super
  61         self.methods = {}
  62         self.file = file
  63         self.lineno = lineno
  64
  65     def _addmethod(self, name, lineno):
  66         self.methods[name] = lineno
  67
  68 class Function:
  69     '''Class to represent a top-level Python function'''
  70     def __init__(self, module, name, file, lineno):
  71         self.module = module
  72         self.name = name
  73         self.file = file
  74         self.lineno = lineno
  75
  76 def readmodule(module, path=[]):
  77     '''Backwards compatible interface.
  78
  79     Call readmodule_ex() and then only keep Class objects from the
  80     resulting dictionary.'''
  81
  82     dict = _readmodule(module, path)
  83     res = {}
  84     for key, value in dict.items():
  85         if isinstance(value, Class):
  86             res[key] = value
  87     return res
  88
  89 def readmodule_ex(module, path=[]):
  90     '''Read a module file and return a dictionary of classes.
  91
  92     Search for MODULE in PATH and sys.path, read and parse the
  93     module and return a dictionary with one entry for each class
  94     found in the module.
  95
  96     If INPACKAGE is true, it must be the dotted name of the package in
  97     which we are searching for a submodule, and then PATH must be the
  98     package search path; otherwise, we are searching for a top-level
  99     module, and PATH is combined with sys.path.
 100     '''
 101     return _readmodule(module, path)
 102
 103 def _readmodule(module, path, inpackage=None):
 104     '''Do the hard work for readmodule[_ex].'''
 105     # Compute the full module name (prepending inpackage if set)
 106     if inpackage:
 107         fullmodule = "%s.%s" % (inpackage, module)
 108     else:
 109         fullmodule = module
 110
 111     # Check in the cache
 112     if fullmodule in _modules:
 113         return _modules[fullmodule]
 114
 115     # Initialize the dict for this module's contents
 116     dict = {}
 117
 118     # Check if it is a built-in module; we don't do much for these
 119     if module in sys.builtin_module_names and not inpackage:
 120         _modules[module] = dict
 121         return dict
 122
 123     # Check for a dotted module name
 124     i = module.rfind('.')
 125     if i >= 0:
 126         package = module[:i]
 127         submodule = module[i+1:]
 128         parent = _readmodule(package, path, inpackage)
 129         if inpackage:
 130             package = "%s.%s" % (inpackage, package)
 131         return _readmodule(submodule, parent['__path__'], package)
 132
 133     # Search the path for the module
 134     f = None
 135     if inpackage:
 136         f, file, (suff, mode, type) = imp.find_module(module, path)
 137     else:
 138         f, file, (suff, mode, type) = imp.find_module(module, path + sys.path)
 139     if type == imp.PKG_DIRECTORY:
 140         dict['__path__'] = [file]
 141         path = [file] + path
 142         f, file, (suff, mode, type) = imp.find_module('__init__', [file])
 143     _modules[fullmodule] = dict
 144     if type != imp.PY_SOURCE:
 145         # not Python source, can't do anything with this module
 146         f.close()
 147         return dict
 148
 149     stack = [] # stack of (class, indent) pairs
 150
 151     g = tokenize.generate_tokens(f.readline)
 152     try:
 153         for tokentype, token, start, end, line in g:
 154             if tokentype == DEDENT:
 155                 lineno, thisindent = start
 156                 # close nested classes and defs
 157                 while stack and stack[-1][1] >= thisindent:
 158                     del stack[-1]
 159             elif token == 'def':
 160                 lineno, thisindent = start
 161                 # close previous nested classes and defs
 162                 while stack and stack[-1][1] >= thisindent:
 163                     del stack[-1]
 164                 tokentype, meth_name, start, end, line = g.next()
 165                 if tokentype != NAME:
 166                     continue # Syntax error
 167                 if stack:
 168                     cur_class = stack[-1][0]
 169                     if isinstance(cur_class, Class):
 170                         # it's a method
 171                         cur_class._addmethod(meth_name, lineno)
 172                     # else it's a nested def
 173                 else:
 174                     # it's a function
 175                     dict[meth_name] = Function(module, meth_name, file, lineno)
 176                 stack.append((None, thisindent)) # Marker for nested fns
 177             elif token == 'class':
 178                 lineno, thisindent = start
 179                 # close previous nested classes and defs
 180                 while stack and stack[-1][1] >= thisindent:
 181                     del stack[-1]
 182                 tokentype, class_name, start, end, line = g.next()
 183                 if tokentype != NAME:
 184                     continue # Syntax error
 185                 # parse what follows the class name
 186                 tokentype, token, start, end, line = g.next()
 187                 inherit = None
 188                 if token == '(':
 189                     names = [] # List of superclasses
 190                     # there's a list of superclasses
 191                     level = 1
 192                     super = [] # Tokens making up current superclass
 193                     while True:
 194                         tokentype, token, start, end, line = g.next()
 195                         if token in (')', ',') and level == 1:
 196                             n = "".join(super)
 197                             if n in dict:
 198                                 # we know this super class
 199                                 n = dict[n]
 200                             else:
 201                                 c = n.split('.')
 202                                 if len(c) > 1:
 203                                     # super class is of the form
 204                                     # module.class: look in module for
 205                                     # class
 206                                     m = c[-2]
 207                                     c = c[-1]
 208                                     if m in _modules:
 209                                         d = _modules[m]
 210                                         if c in d:
 211                                             n = d[c]
 212                             names.append(n)
 213                             super = []
 214                         if token == '(':
 215                             level += 1
 216                         elif token == ')':
 217                             level -= 1
 218                             if level == 0:
 219                                 break
 220                         elif token == ',' and level == 1:
 221                             pass
 222                         else:
 223                             super.append(token)
 224                     inherit = names
 225                 cur_class = Class(fullmodule, class_name, inherit, file, lineno)
 226                 if not stack:
 227                     dict[class_name] = cur_class
 228                 stack.append((cur_class, thisindent))
 229             elif token == 'import' and start[1] == 0:
 230                 modules = _getnamelist(g)
 231                 for mod, mod2 in modules:
 232                     try:
 233                         # Recursively read the imported module
 234                         if not inpackage:
 235                             _readmodule(mod, path)
 236                         else:
 237                             try:
 238                                 _readmodule(mod, path, inpackage)
 239                             except ImportError:
 240                                 _readmodule(mod, [])
 241                     except:
 242                         # If we can't find or parse the imported module,
 243                         # too bad -- don't die here.
 244                         pass
 245             elif token == 'from' and start[1] == 0:
 246                 mod, token = _getname(g)
 247                 if not mod or token != "import":
 248                     continue
 249                 names = _getnamelist(g)
 250                 try:
 251                     # Recursively read the imported module
 252                     d = _readmodule(mod, path, inpackage)
 253                 except:
 254                     # If we can't find or parse the imported module,
 255                     # too bad -- don't die here.
 256                     continue
 257                 # add any classes that were defined in the imported module
 258                 # to our name space if they were mentioned in the list
 259                 for n, n2 in names:
 260                     if n in d:
 261                         dict[n2 or n] = d[n]
 262                     elif n == '*':
 263                         # don't add names that start with _
 264                         for n in d:
 265                             if n[0] != '_':
 266                                 dict[n] = d[n]
 267     except StopIteration:
 268         pass
 269
 270     f.close()
 271     return dict
 272
 273 def _getnamelist(g):
 274     # Helper to get a comma-separated list of dotted names plus 'as'
 275     # clauses.  Return a list of pairs (name, name2) where name2 is
 276     # the 'as' name, or None if there is no 'as' clause.
 277     names = []
 278     while True:
 279         name, token = _getname(g)
 280         if not name:
 281             break
 282         if token == 'as':
 283             name2, token = _getname(g)
 284         else:
 285             name2 = None
 286         names.append((name, name2))
 287         while token != "," and "\n" not in token:
 288             tokentype, token, start, end, line = g.next()
 289         if token != ",":
 290             break
 291     return names
 292
 293 def _getname(g):
 294     # Helper to get a dotted name, return a pair (name, token) where
 295     # name is the dotted name, or None if there was no dotted name,
 296     # and token is the next input token.
 297     parts = []
 298     tokentype, token, start, end, line = g.next()
 299     if tokentype != NAME and token != '*':
 300         return (None, token)
 301     parts.append(token)
 302     while True:
 303         tokentype, token, start, end, line = g.next()
 304         if token != '.':
 305             break
 306         tokentype, token, start, end, line = g.next()
 307         if tokentype != NAME:
 308             break
 309         parts.append(token)
 310     return (".".join(parts), token)
 311
 312 def _main():
 313     # Main program for testing.
 314     import os
 315     mod = sys.argv[1]
 316     if os.path.exists(mod):
 317         path = [os.path.dirname(mod)]
 318         mod = os.path.basename(mod)
 319         if mod.lower().endswith(".py"):
 320             mod = mod[:-3]
 321     else:
 322         path = []
 323     dict = readmodule_ex(mod, path)
 324     objs = dict.values()
 325     objs.sort(lambda a, b: cmp(getattr(a, 'lineno', 0),
 326                                getattr(b, 'lineno', 0)))
 327     for obj in objs:
 328         if isinstance(obj, Class):
 329             print "class", obj.name, obj.super, obj.lineno
 330             methods = sorted(obj.methods.iteritems(), key=itemgetter(1))
 331             for name, lineno in methods:
 332                 if name != "__path__":
 333                     print "  def", name, lineno
 334         elif isinstance(obj, Function):
 335             print "def", obj.name, obj.lineno
 336
 337 if __name__ == "__main__":
 338     _main()