Tools/scripts/pindent.py

   1 #! /usr/bin/env python
   2
   3 # This file contains a class and a main program that perform three
   4 # related (though complimentary) formatting operations on Python
   5 # programs.  When called as "pindent -c", it takes a valid Python
   6 # program as input and outputs a version augmented with block-closing
   7 # comments.  When called as "pindent -d", it assumes its input is a
   8 # Python program with block-closing comments and outputs a commentless
   9 # version.   When called as "pindent -r" it assumes its input is a
  10 # Python program with block-closing comments but with its indentation
  11 # messed up, and outputs a properly indented version.
  12
  13 # A "block-closing comment" is a comment of the form '# end <keyword>'
  14 # where <keyword> is the keyword that opened the block.  If the
  15 # opening keyword is 'def' or 'class', the function or class name may
  16 # be repeated in the block-closing comment as well.  Here is an
  17 # example of a program fully augmented with block-closing comments:
  18
  19 # def foobar(a, b):
  20 #    if a == b:
  21 #        a = a+1
  22 #    elif a < b:
  23 #        b = b-1
  24 #        if b > a: a = a-1
  25 #        # end if
  26 #    else:
  27 #        print 'oops!'
  28 #    # end if
  29 # # end def foobar
  30
  31 # Note that only the last part of an if...elif...else... block needs a
  32 # block-closing comment; the same is true for other compound
  33 # statements (e.g. try...except).  Also note that "short-form" blocks
  34 # like the second 'if' in the example must be closed as well;
  35 # otherwise the 'else' in the example would be ambiguous (remember
  36 # that indentation is not significant when interpreting block-closing
  37 # comments).
  38
  39 # The operations are idempotent (i.e. applied to their own output
  40 # they yield an identical result).  Running first "pindent -c" and
  41 # then "pindent -r" on a valid Python program produces a program that
  42 # is semantically identical to the input (though its indentation may
  43 # be different). Running "pindent -e" on that output produces a
  44 # program that only differs from the original in indentation.
  45
  46 # Other options:
  47 # -s stepsize: set the indentation step size (default 8)
  48 # -t tabsize : set the number of spaces a tab character is worth (default 8)
  49 # -e         : expand TABs into spaces
  50 # file ...   : input file(s) (default standard input)
  51 # The results always go to standard output
  52
  53 # Caveats:
  54 # - comments ending in a backslash will be mistaken for continued lines
  55 # - continuations using backslash are always left unchanged
  56 # - continuations inside parentheses are not extra indented by -r
  57 #   but must be indented for -c to work correctly (this breaks
  58 #   idempotency!)
  59 # - continued lines inside triple-quoted strings are totally garbled
  60
  61 # Secret feature:
  62 # - On input, a block may also be closed with an "end statement" --
  63 #   this is a block-closing comment without the '#' sign.
  64
  65 # Possible improvements:
  66 # - check syntax based on transitions in 'next' table
  67 # - better error reporting
  68 # - better error recovery
  69 # - check identifier after class/def
  70
  71 # The following wishes need a more complete tokenization of the source:
  72 # - Don't get fooled by comments ending in backslash
  73 # - reindent continuation lines indicated by backslash
  74 # - handle continuation lines inside parentheses/braces/brackets
  75 # - handle triple quoted strings spanning lines
  76 # - realign comments
  77 # - optionally do much more thorough reformatting, a la C indent
  78
  79 # Defaults
  80 STEPSIZE = 8
  81 TABSIZE = 8
  82 EXPANDTABS = 0
  83
  84 import re
  85 import sys
  86
  87 next = {}
  88 next['if'] = next['elif'] = 'elif', 'else', 'end'
  89 next['while'] = next['for'] = 'else', 'end'
  90 next['try'] = 'except', 'finally'
  91 next['except'] = 'except', 'else', 'finally', 'end'
  92 next['else'] = next['finally'] = next['def'] = next['class'] = 'end'
  93 next['end'] = ()
  94 start = 'if', 'while', 'for', 'try', 'with', 'def', 'class'
  95
  96 class PythonIndenter:
  97
  98     def __init__(self, fpi = sys.stdin, fpo = sys.stdout,
  99                  indentsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
 100         self.fpi = fpi
 101         self.fpo = fpo
 102         self.indentsize = indentsize
 103         self.tabsize = tabsize
 104         self.lineno = 0
 105         self.expandtabs = expandtabs
 106         self._write = fpo.write
 107         self.kwprog = re.compile(
 108                 r'^\s*(?P<kw>[a-z]+)'
 109                 r'(\s+(?P<id>[a-zA-Z_]\w*))?'
 110                 r'[^\w]')
 111         self.endprog = re.compile(
 112                 r'^\s*#?\s*end\s+(?P<kw>[a-z]+)'
 113                 r'(\s+(?P<id>[a-zA-Z_]\w*))?'
 114                 r'[^\w]')
 115         self.wsprog = re.compile(r'^[ \t]*')
 116     # end def __init__
 117
 118     def write(self, line):
 119         if self.expandtabs:
 120             self._write(line.expandtabs(self.tabsize))
 121         else:
 122             self._write(line)
 123         # end if
 124     # end def write
 125
 126     def readline(self):
 127         line = self.fpi.readline()
 128         if line: self.lineno = self.lineno + 1
 129         # end if
 130         return line
 131     # end def readline
 132
 133     def error(self, fmt, *args):
 134         if args: fmt = fmt % args
 135         # end if
 136         sys.stderr.write('Error at line %d: %s\n' % (self.lineno, fmt))
 137         self.write('### %s ###\n' % fmt)
 138     # end def error
 139
 140     def getline(self):
 141         line = self.readline()
 142         while line[-2:] == '\\\n':
 143             line2 = self.readline()
 144             if not line2: break
 145             # end if
 146             line = line + line2
 147         # end while
 148         return line
 149     # end def getline
 150
 151     def putline(self, line, indent = None):
 152         if indent is None:
 153             self.write(line)
 154             return
 155         # end if
 156         tabs, spaces = divmod(indent*self.indentsize, self.tabsize)
 157         i = 0
 158         m = self.wsprog.match(line)
 159         if m: i = m.end()
 160         # end if
 161         self.write('\t'*tabs + ' '*spaces + line[i:])
 162     # end def putline
 163
 164     def reformat(self):
 165         stack = []
 166         while 1:
 167             line = self.getline()
 168             if not line: break      # EOF
 169             # end if
 170             m = self.endprog.match(line)
 171             if m:
 172                 kw = 'end'
 173                 kw2 = m.group('kw')
 174                 if not stack:
 175                     self.error('unexpected end')
 176                 elif stack[-1][0] != kw2:
 177                     self.error('unmatched end')
 178                 # end if
 179                 del stack[-1:]
 180                 self.putline(line, len(stack))
 181                 continue
 182             # end if
 183             m = self.kwprog.match(line)
 184             if m:
 185                 kw = m.group('kw')
 186                 if kw in start:
 187                     self.putline(line, len(stack))
 188                     stack.append((kw, kw))
 189                     continue
 190                 # end if
 191                 if next.has_key(kw) and stack:
 192                     self.putline(line, len(stack)-1)
 193                     kwa, kwb = stack[-1]
 194                     stack[-1] = kwa, kw
 195                     continue
 196                 # end if
 197             # end if
 198             self.putline(line, len(stack))
 199         # end while
 200         if stack:
 201             self.error('unterminated keywords')
 202             for kwa, kwb in stack:
 203                 self.write('\t%s\n' % kwa)
 204             # end for
 205         # end if
 206     # end def reformat
 207
 208     def delete(self):
 209         begin_counter = 0
 210         end_counter = 0
 211         while 1:
 212             line = self.getline()
 213             if not line: break      # EOF
 214             # end if
 215             m = self.endprog.match(line)
 216             if m:
 217                 end_counter = end_counter + 1
 218                 continue
 219             # end if
 220             m = self.kwprog.match(line)
 221             if m:
 222                 kw = m.group('kw')
 223                 if kw in start:
 224                     begin_counter = begin_counter + 1
 225                 # end if
 226             # end if
 227             self.putline(line)
 228         # end while
 229         if begin_counter - end_counter < 0:
 230             sys.stderr.write('Warning: input contained more end tags than expected\n')
 231         elif begin_counter - end_counter > 0:
 232             sys.stderr.write('Warning: input contained less end tags than expected\n')
 233         # end if
 234     # end def delete
 235
 236     def complete(self):
 237         self.indentsize = 1
 238         stack = []
 239         todo = []
 240         thisid = ''
 241         current, firstkw, lastkw, topid = 0, '', '', ''
 242         while 1:
 243             line = self.getline()
 244             i = 0
 245             m = self.wsprog.match(line)
 246             if m: i = m.end()
 247             # end if
 248             m = self.endprog.match(line)
 249             if m:
 250                 thiskw = 'end'
 251                 endkw = m.group('kw')
 252                 thisid = m.group('id')
 253             else:
 254                 m = self.kwprog.match(line)
 255                 if m:
 256                     thiskw = m.group('kw')
 257                     if not next.has_key(thiskw):
 258                         thiskw = ''
 259                     # end if
 260                     if thiskw in ('def', 'class'):
 261                         thisid = m.group('id')
 262                     else:
 263                         thisid = ''
 264                     # end if
 265                 elif line[i:i+1] in ('\n', '#'):
 266                     todo.append(line)
 267                     continue
 268                 else:
 269                     thiskw = ''
 270                 # end if
 271             # end if
 272             indent = len(line[:i].expandtabs(self.tabsize))
 273             while indent < current:
 274                 if firstkw:
 275                     if topid:
 276                         s = '# end %s %s\n' % (
 277                                 firstkw, topid)
 278                     else:
 279                         s = '# end %s\n' % firstkw
 280                     # end if
 281                     self.putline(s, current)
 282                     firstkw = lastkw = ''
 283                 # end if
 284                 current, firstkw, lastkw, topid = stack[-1]
 285                 del stack[-1]
 286             # end while
 287             if indent == current and firstkw:
 288                 if thiskw == 'end':
 289                     if endkw != firstkw:
 290                         self.error('mismatched end')
 291                     # end if
 292                     firstkw = lastkw = ''
 293                 elif not thiskw or thiskw in start:
 294                     if topid:
 295                         s = '# end %s %s\n' % (
 296                                 firstkw, topid)
 297                     else:
 298                         s = '# end %s\n' % firstkw
 299                     # end if
 300                     self.putline(s, current)
 301                     firstkw = lastkw = topid = ''
 302                 # end if
 303             # end if
 304             if indent > current:
 305                 stack.append((current, firstkw, lastkw, topid))
 306                 if thiskw and thiskw not in start:
 307                     # error
 308                     thiskw = ''
 309                 # end if
 310                 current, firstkw, lastkw, topid = \
 311                          indent, thiskw, thiskw, thisid
 312             # end if
 313             if thiskw:
 314                 if thiskw in start:
 315                     firstkw = lastkw = thiskw
 316                     topid = thisid
 317                 else:
 318                     lastkw = thiskw
 319                 # end if
 320             # end if
 321             for l in todo: self.write(l)
 322             # end for
 323             todo = []
 324             if not line: break
 325             # end if
 326             self.write(line)
 327         # end while
 328     # end def complete
 329
 330 # end class PythonIndenter
 331
 332 # Simplified user interface
 333 # - xxx_filter(input, output): read and write file objects
 334 # - xxx_string(s): take and return string object
 335 # - xxx_file(filename): process file in place, return true iff changed
 336
 337 def complete_filter(input = sys.stdin, output = sys.stdout,
 338                     stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
 339     pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
 340     pi.complete()
 341 # end def complete_filter
 342
 343 def delete_filter(input= sys.stdin, output = sys.stdout,
 344                         stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
 345     pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
 346     pi.delete()
 347 # end def delete_filter
 348
 349 def reformat_filter(input = sys.stdin, output = sys.stdout,
 350                     stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
 351     pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
 352     pi.reformat()
 353 # end def reformat_filter
 354
 355 class StringReader:
 356     def __init__(self, buf):
 357         self.buf = buf
 358         self.pos = 0
 359         self.len = len(self.buf)
 360     # end def __init__
 361     def read(self, n = 0):
 362         if n <= 0:
 363             n = self.len - self.pos
 364         else:
 365             n = min(n, self.len - self.pos)
 366         # end if
 367         r = self.buf[self.pos : self.pos + n]
 368         self.pos = self.pos + n
 369         return r
 370     # end def read
 371     def readline(self):
 372         i = self.buf.find('\n', self.pos)
 373         return self.read(i + 1 - self.pos)
 374     # end def readline
 375     def readlines(self):
 376         lines = []
 377         line = self.readline()
 378         while line:
 379             lines.append(line)
 380             line = self.readline()
 381         # end while
 382         return lines
 383     # end def readlines
 384     # seek/tell etc. are left as an exercise for the reader
 385 # end class StringReader
 386
 387 class StringWriter:
 388     def __init__(self):
 389         self.buf = ''
 390     # end def __init__
 391     def write(self, s):
 392         self.buf = self.buf + s
 393     # end def write
 394     def getvalue(self):
 395         return self.buf
 396     # end def getvalue
 397 # end class StringWriter
 398
 399 def complete_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
 400     input = StringReader(source)
 401     output = StringWriter()
 402     pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
 403     pi.complete()
 404     return output.getvalue()
 405 # end def complete_string
 406
 407 def delete_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
 408     input = StringReader(source)
 409     output = StringWriter()
 410     pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
 411     pi.delete()
 412     return output.getvalue()
 413 # end def delete_string
 414
 415 def reformat_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
 416     input = StringReader(source)
 417     output = StringWriter()
 418     pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
 419     pi.reformat()
 420     return output.getvalue()
 421 # end def reformat_string
 422
 423 def complete_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
 424     source = open(filename, 'r').read()
 425     result = complete_string(source, stepsize, tabsize, expandtabs)
 426     if source == result: return 0
 427     # end if
 428     import os
 429     try: os.rename(filename, filename + '~')
 430     except os.error: pass
 431     # end try
 432     f = open(filename, 'w')
 433     f.write(result)
 434     f.close()
 435     return 1
 436 # end def complete_file
 437
 438 def delete_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
 439     source = open(filename, 'r').read()
 440     result = delete_string(source, stepsize, tabsize, expandtabs)
 441     if source == result: return 0
 442     # end if
 443     import os
 444     try: os.rename(filename, filename + '~')
 445     except os.error: pass
 446     # end try
 447     f = open(filename, 'w')
 448     f.write(result)
 449     f.close()
 450     return 1
 451 # end def delete_file
 452
 453 def reformat_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
 454     source = open(filename, 'r').read()
 455     result = reformat_string(source, stepsize, tabsize, expandtabs)
 456     if source == result: return 0
 457     # end if
 458     import os
 459     try: os.rename(filename, filename + '~')
 460     except os.error: pass
 461     # end try
 462     f = open(filename, 'w')
 463     f.write(result)
 464     f.close()
 465     return 1
 466 # end def reformat_file
 467
 468 # Test program when called as a script
 469
 470 usage = """
 471 usage: pindent (-c|-d|-r) [-s stepsize] [-t tabsize] [-e] [file] ...
 472 -c         : complete a correctly indented program (add #end directives)
 473 -d         : delete #end directives
 474 -r         : reformat a completed program (use #end directives)
 475 -s stepsize: indentation step (default %(STEPSIZE)d)
 476 -t tabsize : the worth in spaces of a tab (default %(TABSIZE)d)
 477 -e         : expand TABs into spaces (defailt OFF)
 478 [file] ... : files are changed in place, with backups in file~
 479 If no files are specified or a single - is given,
 480 the program acts as a filter (reads stdin, writes stdout).
 481 """ % vars()
 482
 483 def error_both(op1, op2):
 484     sys.stderr.write('Error: You can not specify both '+op1+' and -'+op2[0]+' at the same time\n')
 485     sys.stderr.write(usage)
 486     sys.exit(2)
 487 # end def error_both
 488
 489 def test():
 490     import getopt
 491     try:
 492         opts, args = getopt.getopt(sys.argv[1:], 'cdrs:t:e')
 493     except getopt.error, msg:
 494         sys.stderr.write('Error: %s\n' % msg)
 495         sys.stderr.write(usage)
 496         sys.exit(2)
 497     # end try
 498     action = None
 499     stepsize = STEPSIZE
 500     tabsize = TABSIZE
 501     expandtabs = EXPANDTABS
 502     for o, a in opts:
 503         if o == '-c':
 504             if action: error_both(o, action)
 505             # end if
 506             action = 'complete'
 507         elif o == '-d':
 508             if action: error_both(o, action)
 509             # end if
 510             action = 'delete'
 511         elif o == '-r':
 512             if action: error_both(o, action)
 513             # end if
 514             action = 'reformat'
 515         elif o == '-s':
 516             stepsize = int(a)
 517         elif o == '-t':
 518             tabsize = int(a)
 519         elif o == '-e':
 520             expandtabs = 1
 521         # end if
 522     # end for
 523     if not action:
 524         sys.stderr.write(
 525                 'You must specify -c(omplete), -d(elete) or -r(eformat)\n')
 526         sys.stderr.write(usage)
 527         sys.exit(2)
 528     # end if
 529     if not args or args == ['-']:
 530         action = eval(action + '_filter')
 531         action(sys.stdin, sys.stdout, stepsize, tabsize, expandtabs)
 532     else:
 533         action = eval(action + '_file')
 534         for filename in args:
 535             action(filename, stepsize, tabsize, expandtabs)
 536         # end for
 537     # end if
 538 # end def test
 539
 540 if __name__ == '__main__':
 541     test()
 542 # end if