sandbox/tibs/pysource/utils.py

   1 """Utilities for pysource
   2 """
   3
   4 import types
   5 import string
   6 import compiler
   7
   8 # We'd better have at least *one* module in this package that demonstrates
   9 # *not* using reST for our docstrings...
  10 __docformat__ = "none"
  11
  12 \f
  13 # ----------------------------------------------------------------------
  14 PLAINTEXT = "plaintext"
  15 RESTRUCTUREDTEXT = "restructuredtext"
  16
  17 canonical_format = { "plaintext" : PLAINTEXT,
  18                      "plain"     : PLAINTEXT,
  19                      "none"      : PLAINTEXT,
  20                      "rst"              : RESTRUCTUREDTEXT,
  21                      "rest"             : RESTRUCTUREDTEXT,
  22                      "rtxt"             : RESTRUCTUREDTEXT,
  23                      "restructuredtext" : RESTRUCTUREDTEXT,
  24                      }
  25
  26 def docformat(text):
  27     """Interpret a module's __docformat__ string.
  28
  29     Returns a tuple of (format,language)
  30     """
  31     if text == None:
  32         return PLAINTEXT,"en"
  33
  34     words = string.split(string.lower(text))
  35
  36     #print words
  37
  38     if len(words) == 0:
  39         return PLAINTEXT,"en"
  40     elif len(words) > 2:
  41         raise ValueError,"__docformat__ may be at most two 'words'"
  42
  43     if len(words) == 2:
  44         language = string.lower(words[1])
  45     else:
  46         language = "en"
  47
  48     try:
  49         format = canonical_format[string.lower(words[0])]
  50     except KeyError:
  51         legal = canonical_format.keys()
  52         legal.sort()
  53         raise ValueError,"__docformat__ should be one of %s"%legal
  54
  55     return format,language
  56
  57 \f
  58 # ----------------------------------------------------------------------
  59 def flatten(item):
  60     """Retrieve some simpler representation of our AST.
  61
  62     (and it's not meant to be 'theoretically' wonderful, just something
  63     I can look at to work out how an AST works...)
  64     """
  65     if isinstance(item,compiler.ast.Node):
  66         things = [item.__class__.__name__]
  67         children = item.getChildren()
  68         for child in children:
  69             things.append(flatten(child))
  70         return things
  71     else:
  72         return [item]
  73
  74 \f
  75 # ----------------------------------------------------------------------
  76 def treeprint(stream,item,indent=0):
  77     """Simple pretty printer for the AST."""
  78     if isinstance(item,compiler.ast.Node):
  79         stream.write("\n%s<%s>"%(" "*indent,item.__class__.__name__))
  80
  81         children = item.getChildren()
  82         for child in children:
  83             treeprint(stream,child,indent+2)
  84
  85         # Fake our docstring as a sub-node (it's *really* more an attribute)
  86         if hasattr(item,"docstring"):
  87             stream.write("\n%s  <docstring> %s"%(" "*indent,item.docstring))
  88
  89         # And ditto for a remembered assignment expression
  90         if hasattr(item,"assign_expr"):
  91             stream.write("\n%s  <assign_expr>"%(" "*indent))
  92             treeprint(stream,item.assign_expr,indent+4)
  93     else:
  94         stream.write(" ")
  95         stream.write(`item`)
  96
  97 \f
  98 # ----------------------------------------------------------------------
  99 def find_attr_docs(tree,verbose=0):
 100     """Find candidates for documented attributes
 101
 102     Note that after this, it may be that the AST will not garbage collect
 103     its own nodes properly anymore, as we are adding in cross-linkages.
 104     """
 105
 106     if not isinstance(tree,compiler.ast.Node):
 107         return
 108
 109     children = tree.getChildren()
 110
 111     # Might as well get our recursion done with first...
 112     for child in children:
 113         find_attr_docs(child,verbose)
 114
 115     # I believe that only Stmt nodes can have Assign and Discard
 116     # nodes as children
 117     if not isinstance(tree,compiler.ast.Stmt):
 118         return
 119
 120     if len(children) == 0:
 121         return
 122
 123     pairs = []
 124     last  = children[0]
 125     for item in children[1:]:
 126         pairs.append((last,item))
 127         last = item
 128
 129     for this,next in pairs:
 130         if isinstance(this,compiler.ast.Assign) and \
 131            isinstance(next,compiler.ast.Discard):
 132             if verbose:
 133                 print
 134                 print
 135                 print "*** Attribute docstring candidate"
 136                 treeprint(this,4)
 137                 treeprint(next,4)
 138                 print
 139
 140             nextexpr = next.expr
 141             if isinstance(nextexpr,compiler.ast.Const):
 142                 if type(nextexpr.value) == types.StringType:
 143                     docstring = nextexpr.value
 144                 else:
 145                     if verbose:
 146                         print
 147                         print "... Discarded constant is not a string"
 148                     continue
 149             else:
 150                 if verbose:
 151                     print
 152                     print "... Discarded expression is not a constant"
 153                 continue
 154
 155             # If there is more than one assignment attached to
 156             # the <Assign> node, we are not interested
 157             if len(this.nodes) > 1:
 158                 if verbose:
 159                     print
 160                     print "... (but there are too many assignments in the <Assign>)"
 161                 continue
 162
 163             target = this.nodes[0]
 164             if isinstance(target,compiler.ast.AssName):
 165                 # Let's be cheeky and glue the docstring on...
 166                 target.docstring = docstring
 167             elif isinstance(target,compiler.ast.AssAttr):
 168                 # Let's be cheeky and glue the docstring on...
 169                 target.docstring = docstring
 170             else:
 171                 if verbose:
 172                     print
 173                     print "... (but the assignment is to a tuple/list/etc.)"
 174                 continue
 175
 176             if verbose:
 177                 print
 178                 print "Becomes:"
 179                 treeprint(this,4)
 180
 181 \f
 182 # ----------------------------------------------------------------------
 183 def find_attr_vals(tree,verbose=0):
 184     """Find attributes whose values we're interested in.
 185
 186     Clearly, when this is working, it could do with being "folded" into
 187     `find_attr_docs()`.
 188
 189     Note that after this, it may be that the AST will not garbage collect
 190     its own nodes properly anymore, as we are adding in cross-linkages.
 191     """
 192
 193     if not isinstance(tree,compiler.ast.Node):
 194         return
 195
 196     children = tree.getChildren()
 197
 198     # Might as well get our recursion done with first...
 199     for child in children:
 200         find_attr_vals(child,verbose)
 201
 202     # I believe that only Stmt nodes can have Assign and Discard
 203     # nodes as children
 204     if not isinstance(tree,compiler.ast.Stmt):
 205         return
 206
 207     for this in children:
 208         if isinstance(this,compiler.ast.Assign):
 209             if verbose:
 210                 print
 211                 print
 212                 print "*** Assignment - name/value candidate"
 213                 treeprint(this,4)
 214                 print
 215
 216             # If there is more than one assignment attached to
 217             # the <Assign> node, we are not interested
 218             if len(this.nodes) > 1:
 219                 if verbose:
 220                     print
 221                     print "... (but there are too many assignments in the <Assign>)"
 222                 continue
 223
 224             target = this.nodes[0]
 225             if isinstance(target,compiler.ast.AssName) or \
 226                isinstance(target,compiler.ast.AssAttr):
 227                 # Let's be cheeky and glue the associated expression on...
 228                 target.assign_expr = this.expr
 229             else:
 230                 if verbose:
 231                     print
 232                     print "... (but the assignment is to a tuple/list/etc.)"
 233                 continue
 234
 235             if verbose:
 236                 print
 237                 print "Becomes:"
 238                 treeprint(this,4)
 239
 240 \f
 241 # ----------------------------------------------------------------------
 242 def stringify_arg(thing):
 243     """Return a string representation of a function argument.
 244
 245     This just works for tuples of (strings or tuples (of strings ...) ...)
 246     """
 247     if type(thing) == types.StringType:
 248         return thing
 249     elif type(thing) == types.TupleType:
 250         innards = []
 251         for item in thing:
 252             innards.append(stringify_arg(item))
 253         return "(" + string.join(innards,",") + ")"
 254     else:
 255         raise ValueError,"Tried to stringify type %s"%type(thing)
 256
 257 \f
 258 # ----------------------------------------------------------------------
 259 def merge_args(args,defaults):
 260     """Merge together arguments and defaults from an argument list.
 261
 262     Returns a list of argument strings.
 263     """
 264     if args == None:
 265         return []
 266
 267     if defaults == None:
 268         defaults = []
 269
 270     # This is horrible - do it nicely later on!
 271     argstrs = []
 272     for item in args:
 273         argstrs.append(stringify_arg(item))
 274
 275     pos = len(args) - len(defaults)
 276     next = 0
 277     for index in range(pos,len(args)):
 278         thing = defaults[next]
 279         thing = stringify_expr(thing)
 280         argstrs[index] = "%s=%s"%(argstrs[index],thing)
 281         next = next + 1
 282     return argstrs
 283
 284 \f
 285 # ----------------------------------------------------------------------
 286 def stringify_expr(thing):
 287     """Return a very simple string representation of an expression node
 288
 289     Specifically, this function aims to support stringifying things
 290     which can be on the RHS of an assignment - is that *actually* the
 291     same as stringifying expression nodes?
 292     """
 293
 294     # Humph - saving typing may be a good thing...
 295     strify = stringify_expr
 296
 297     #print thing.__class__.__name__
 298
 299     if thing == None:
 300         return 'None'
 301     elif isinstance(thing,compiler.ast.Add):
 302         return strify(thing.left) + " + " + strify(thing.right)
 303     elif isinstance(thing,compiler.ast.And):
 304         exprs = []
 305         for node in thing.nodes:
 306             exprs.append(strify(node))
 307         return string.join(exprs," && ")
 308     elif isinstance(thing,compiler.ast.AssAttr):
 309         # Attribute as target of assignment
 310         if thing.flags == compiler.consts.OP_ASSIGN:
 311             return strify(thing.expr) + "." + thing.attrname
 312         else:
 313             raise ValueError,"Unexpected flag %d in %s"%(thing.flags,`thing`)
 314     elif isinstance(thing,compiler.ast.AssName):
 315         # Name as target of assignment, but this also means name
 316         # as target of "in" assignment (e.g., "x in [1,2,3]"),
 317         # which is why *we're* interested in it (since this can
 318         # occur in list comprehensions, which can occur as the
 319         # RHS of assignments)
 320         if thing.flags == compiler.consts.OP_ASSIGN:
 321             return thing.name
 322         else:
 323             raise ValueError,"Unexpected flag %d in %s"%(thing.flags,`thing`)
 324     elif isinstance(thing,compiler.ast.Backquote):
 325         return "`" + strify(thing.expr) + "`"
 326     elif isinstance(thing,compiler.ast.Bitand):
 327         exprs = []
 328         for node in thing.nodes:
 329             exprs.append(strify(node))
 330         return string.join(exprs," & ")
 331     elif isinstance(thing,compiler.ast.Bitor):
 332         exprs = []
 333         for node in thing.nodes:
 334             exprs.append(strify(node))
 335         return string.join(exprs," | ")
 336     elif isinstance(thing,compiler.ast.Bitxor):
 337         exprs = []
 338         for node in thing.nodes:
 339             exprs.append(strify(node))
 340         return string.join(exprs," ^ ")
 341     elif isinstance(thing,compiler.ast.CallFunc):
 342         # Yuck - this is getting complicated!
 343         # (for an example, see method `hyperlink_target` in
 344         # restructuredtext/states.py)
 345         str = strify(thing.node) + "("
 346         arglist = []
 347         if thing.args:
 348             for arg in thing.args:
 349                 arglist.append(strify(arg))
 350         if thing.star_args:
 351             arglist.append("*"+strify(thing.star_args))
 352         if thing.dstar_args:
 353             arglist.append("**"+strify(thing.dstar_args))
 354         if arglist:
 355             str += string.join(arglist,", ")
 356         return str+")"
 357     elif isinstance(thing,compiler.ast.Compare):
 358         str = strify(thing.expr) + " "
 359         for op,val in thing.ops:
 360             str += op + " " + strify(val)
 361         return str
 362     elif isinstance(thing,compiler.ast.Const):
 363         # Try not to let long strings take up too much room...
 364         value = thing.value
 365         if type(value) == type("") and len(value) > 50:
 366             value = value[:47] + "..."
 367         # Make Python decide for us if it needs quotes round it
 368         # (and, if so, what sort)
 369         return `value`
 370     elif isinstance(thing,compiler.ast.Dict):
 371         innards = []
 372         for key,val in thing.items:
 373             key = strify(key)
 374             val = strify(val)
 375             innards.append(key+":"+val)
 376         return "{" + string.join(innards,", ") + "}"
 377     elif isinstance(thing,compiler.ast.Div):
 378         return strify(thing.left) + " / " + strify(thing.right)
 379     elif isinstance(thing,compiler.ast.Ellipsis):
 380         return "..."
 381     elif isinstance(thing,compiler.ast.Getattr):
 382         return strify(thing.expr) + "." + thing.attrname
 383     elif isinstance(thing,compiler.ast.Invert):
 384         # Bitwise negation
 385         return "~" + strify(thing.expr)
 386     elif isinstance(thing,compiler.ast.Keyword):
 387         # An 'arg=value' within a function call
 388         return thing.name + "=" + strify(thing.expr)
 389     elif isinstance(thing,compiler.ast.Lambda):
 390         str = "lambda "
 391         if thing.flags != 0:
 392             str += " <flag %d> "%thing.flags
 393         str += string.join(merge_args(thing.argnames,thing.defaults),", ")
 394         str += ": "
 395         str += strify(thing.code)
 396         return str
 397     elif isinstance(thing,compiler.ast.LeftShift):
 398         return strify(thing.left) + " << " + strify(thing.right)
 399     elif isinstance(thing,compiler.ast.List):
 400         innards = []
 401         for item in thing.nodes:
 402             innards.append(strify(item))
 403         return "[" + string.join(innards,", ") + "]"
 404     elif isinstance(thing,compiler.ast.ListComp):
 405         str = "["+strify(thing.expr)
 406         for node in thing.quals:
 407             str += " "+strify(node)
 408         return str+"]"
 409     elif isinstance(thing,compiler.ast.ListCompFor):
 410         str = "for "+strify(thing.assign)
 411         str += " in "+strify(thing.list)
 412         if thing.ifs:
 413             for node in thing.ifs:
 414                 str += " "+strify(node)
 415         return str
 416     elif isinstance(thing,compiler.ast.ListCompIf):
 417         return "if "+strify(thing.test)
 418     elif isinstance(thing,compiler.ast.Mod):
 419         return strify(thing.left) + "%" + strify(thing.right)
 420     elif isinstance(thing,compiler.ast.Mul):
 421         return strify(thing.left) + " * " + strify(thing.right)
 422     elif isinstance(thing,compiler.ast.Name):
 423         return thing.name
 424     elif isinstance(thing,compiler.ast.Not):
 425         return "not " + strify(thing.expr)
 426     elif isinstance(thing,compiler.ast.Or):
 427         exprs = []
 428         for node in thing.nodes:
 429             exprs.append(strify(node))
 430         return string.join(exprs," || ")
 431     elif isinstance(thing,compiler.ast.Power):
 432         return strify(thing.left) + " ** " + strify(thing.right)
 433     elif isinstance(thing,compiler.ast.RightShift):
 434         return strify(thing.left) + " >> " + strify(thing.right)
 435     elif isinstance(thing,compiler.ast.Slice):
 436         if thing.flags != compiler.consts.OP_APPLY:
 437             raise ValueError,"Unexpected flag %d in %s"%(thing.flags,`thing`)
 438         return strify(thing.expr) + "[" + \
 439                strify(thing.lower) + ":" + strify(thing.upper) + "]"
 440     elif isinstance(thing,compiler.ast.Sliceobj):
 441         slicelist = []
 442         for idx in thing.nodes:
 443             slicelist.append(strify(idx))
 444         return string.join(slicelist,":")
 445     elif isinstance(thing,compiler.ast.Sub):
 446         return strify(thing.left) + " - " + strify(thing.right)
 447     elif isinstance(thing,compiler.ast.Subscript):
 448         if thing.flags != compiler.consts.OP_APPLY:
 449             raise ValueError,"Unexpected flag %d in %s"%(thing.flags,`thing`)
 450         str = strify(thing.expr) + "["
 451         sublist = []
 452         for sub in thing.subs:
 453             sublist.append(strify(sub))
 454         return str + string.join(sublist,", ") + "]"
 455     elif isinstance(thing,compiler.ast.Tuple):
 456         innards = []
 457         for item in thing.nodes:
 458             innards.append(strify(item))
 459         return "(" + string.join(innards,", ") + ")"
 460     elif isinstance(thing,compiler.ast.UnaryAdd):
 461         return "+" + strify(thing.expr)
 462     elif isinstance(thing,compiler.ast.UnarySub):
 463         return "-" + strify(thing.expr)
 464     else:
 465         return _whatsthis(thing)
 466
 467 def _whatsthis(thing):
 468     # Wrong, but what else can we do?
 469     import sys
 470     print >>sys.stderr,"stringify_expr - don't recognise %s %s"%\
 471           (thing.__class__.__name__,thing)
 472     return `thing`
 473
 474 \f