Tools/scripts/cleanfuture.py

   1 #! /usr/bin/env python
   2
   3 """cleanfuture [-d][-r][-v] path ...
   4
   5 -d  Dry run.  Analyze, but don't make any changes to, files.
   6 -r  Recurse.  Search for all .py files in subdirectories too.
   7 -v  Verbose.  Print informative msgs.
   8
   9 Search Python (.py) files for future statements, and remove the features
  10 from such statements that are already mandatory in the version of Python
  11 you're using.
  12
  13 Pass one or more file and/or directory paths.  When a directory path, all
  14 .py files within the directory will be examined, and, if the -r option is
  15 given, likewise recursively for subdirectories.
  16
  17 Overwrites files in place, renaming the originals with a .bak extension. If
  18 cleanfuture finds nothing to change, the file is left alone.  If cleanfuture
  19 does change a file, the changed file is a fixed-point (i.e., running
  20 cleanfuture on the resulting .py file won't change it again, at least not
  21 until you try it again with a later Python release).
  22
  23 Limitations:  You can do these things, but this tool won't help you then:
  24
  25 + A future statement cannot be mixed with any other statement on the same
  26   physical line (separated by semicolon).
  27
  28 + A future statement cannot contain an "as" clause.
  29
  30 Example:  Assuming you're using Python 2.2, if a file containing
  31
  32 from __future__ import nested_scopes, generators
  33
  34 is analyzed by cleanfuture, the line is rewritten to
  35
  36 from __future__ import generators
  37
  38 because nested_scopes is no longer optional in 2.2 but generators is.
  39 """
  40
  41 import __future__
  42 import tokenize
  43 import os
  44 import sys
  45
  46 dryrun  = 0
  47 recurse = 0
  48 verbose = 0
  49
  50 def errprint(*args):
  51     strings = map(str, args)
  52     msg = ' '.join(strings)
  53     if msg[-1:] != '\n':
  54         msg += '\n'
  55     sys.stderr.write(msg)
  56
  57 def main():
  58     import getopt
  59     global verbose, recurse, dryrun
  60     try:
  61         opts, args = getopt.getopt(sys.argv[1:], "drv")
  62     except getopt.error, msg:
  63         errprint(msg)
  64         return
  65     for o, a in opts:
  66         if o == '-d':
  67             dryrun += 1
  68         elif o == '-r':
  69             recurse += 1
  70         elif o == '-v':
  71             verbose += 1
  72     if not args:
  73         errprint("Usage:", __doc__)
  74         return
  75     for arg in args:
  76         check(arg)
  77
  78 def check(file):
  79     if os.path.isdir(file) and not os.path.islink(file):
  80         if verbose:
  81             print "listing directory", file
  82         names = os.listdir(file)
  83         for name in names:
  84             fullname = os.path.join(file, name)
  85             if ((recurse and os.path.isdir(fullname) and
  86                  not os.path.islink(fullname))
  87                 or name.lower().endswith(".py")):
  88                 check(fullname)
  89         return
  90
  91     if verbose:
  92         print "checking", file, "...",
  93     try:
  94         f = open(file)
  95     except IOError, msg:
  96         errprint("%r: I/O Error: %s" % (file, str(msg)))
  97         return
  98
  99     ff = FutureFinder(f, file)
 100     changed = ff.run()
 101     if changed:
 102         ff.gettherest()
 103     f.close()
 104     if changed:
 105         if verbose:
 106             print "changed."
 107             if dryrun:
 108                 print "But this is a dry run, so leaving it alone."
 109         for s, e, line in changed:
 110             print "%r lines %d-%d" % (file, s+1, e+1)
 111             for i in range(s, e+1):
 112                 print ff.lines[i],
 113             if line is None:
 114                 print "-- deleted"
 115             else:
 116                 print "-- change to:"
 117                 print line,
 118         if not dryrun:
 119             bak = file + ".bak"
 120             if os.path.exists(bak):
 121                 os.remove(bak)
 122             os.rename(file, bak)
 123             if verbose:
 124                 print "renamed", file, "to", bak
 125             g = open(file, "w")
 126             ff.write(g)
 127             g.close()
 128             if verbose:
 129                 print "wrote new", file
 130     else:
 131         if verbose:
 132             print "unchanged."
 133
 134 class FutureFinder:
 135
 136     def __init__(self, f, fname):
 137         self.f = f
 138         self.fname = fname
 139         self.ateof = 0
 140         self.lines = [] # raw file lines
 141
 142         # List of (start_index, end_index, new_line) triples.
 143         self.changed = []
 144
 145     # Line-getter for tokenize.
 146     def getline(self):
 147         if self.ateof:
 148             return ""
 149         line = self.f.readline()
 150         if line == "":
 151             self.ateof = 1
 152         else:
 153             self.lines.append(line)
 154         return line
 155
 156     def run(self):
 157         STRING = tokenize.STRING
 158         NL = tokenize.NL
 159         NEWLINE = tokenize.NEWLINE
 160         COMMENT = tokenize.COMMENT
 161         NAME = tokenize.NAME
 162         OP = tokenize.OP
 163
 164         changed = self.changed
 165         get = tokenize.generate_tokens(self.getline).next
 166         type, token, (srow, scol), (erow, ecol), line = get()
 167
 168         # Chew up initial comments and blank lines (if any).
 169         while type in (COMMENT, NL, NEWLINE):
 170             type, token, (srow, scol), (erow, ecol), line = get()
 171
 172         # Chew up docstring (if any -- and it may be implicitly catenated!).
 173         while type is STRING:
 174             type, token, (srow, scol), (erow, ecol), line = get()
 175
 176         # Analyze the future stmts.
 177         while 1:
 178             # Chew up comments and blank lines (if any).
 179             while type in (COMMENT, NL, NEWLINE):
 180                 type, token, (srow, scol), (erow, ecol), line = get()
 181
 182             if not (type is NAME and token == "from"):
 183                 break
 184             startline = srow - 1    # tokenize is one-based
 185             type, token, (srow, scol), (erow, ecol), line = get()
 186
 187             if not (type is NAME and token == "__future__"):
 188                 break
 189             type, token, (srow, scol), (erow, ecol), line = get()
 190
 191             if not (type is NAME and token == "import"):
 192                 break
 193             type, token, (srow, scol), (erow, ecol), line = get()
 194
 195             # Get the list of features.
 196             features = []
 197             while type is NAME:
 198                 features.append(token)
 199                 type, token, (srow, scol), (erow, ecol), line = get()
 200
 201                 if not (type is OP and token == ','):
 202                     break
 203                 type, token, (srow, scol), (erow, ecol), line = get()
 204
 205             # A trailing comment?
 206             comment = None
 207             if type is COMMENT:
 208                 comment = token
 209                 type, token, (srow, scol), (erow, ecol), line = get()
 210
 211             if type is not NEWLINE:
 212                 errprint("Skipping file %r; can't parse line %d:\n%s" %
 213                          (self.fname, srow, line))
 214                 return []
 215
 216             endline = srow - 1
 217
 218             # Check for obsolete features.
 219             okfeatures = []
 220             for f in features:
 221                 object = getattr(__future__, f, None)
 222                 if object is None:
 223                     # A feature we don't know about yet -- leave it in.
 224                     # They'll get a compile-time error when they compile
 225                     # this program, but that's not our job to sort out.
 226                     okfeatures.append(f)
 227                 else:
 228                     released = object.getMandatoryRelease()
 229                     if released is None or released <= sys.version_info:
 230                         # Withdrawn or obsolete.
 231                         pass
 232                     else:
 233                         okfeatures.append(f)
 234
 235             # Rewrite the line if at least one future-feature is obsolete.
 236             if len(okfeatures) < len(features):
 237                 if len(okfeatures) == 0:
 238                     line = None
 239                 else:
 240                     line = "from __future__ import "
 241                     line += ', '.join(okfeatures)
 242                     if comment is not None:
 243                         line += ' ' + comment
 244                     line += '\n'
 245                 changed.append((startline, endline, line))
 246
 247             # Loop back for more future statements.
 248
 249         return changed
 250
 251     def gettherest(self):
 252         if self.ateof:
 253             self.therest = ''
 254         else:
 255             self.therest = self.f.read()
 256
 257     def write(self, f):
 258         changed = self.changed
 259         assert changed
 260         # Prevent calling this again.
 261         self.changed = []
 262         # Apply changes in reverse order.
 263         changed.reverse()
 264         for s, e, line in changed:
 265             if line is None:
 266                 # pure deletion
 267                 del self.lines[s:e+1]
 268             else:
 269                 self.lines[s:e+1] = [line]
 270         f.writelines(self.lines)
 271         # Copy over the remainder of the file.
 272         if self.therest:
 273             f.write(self.therest)
 274
 275 if __name__ == '__main__':
 276     main()