Lib/distutils/filelist.py

   1 """distutils.filelist
   2
   3 Provides the FileList class, used for poking about the filesystem
   4 and building lists of files.
   5 """
   6
   7 __revision__ = "$Id$"
   8
   9 import os, re
  10 import fnmatch
  11 from distutils.util import convert_path
  12 from distutils.errors import DistutilsTemplateError, DistutilsInternalError
  13 from distutils import log
  14
  15 class FileList:
  16     """A list of files built by on exploring the filesystem and filtered by
  17     applying various patterns to what we find there.
  18
  19     Instance attributes:
  20       dir
  21         directory from which files will be taken -- only used if
  22         'allfiles' not supplied to constructor
  23       files
  24         list of filenames currently being built/filtered/manipulated
  25       allfiles
  26         complete list of files under consideration (ie. without any
  27         filtering applied)
  28     """
  29
  30     def __init__(self, warn=None, debug_print=None):
  31         # ignore argument to FileList, but keep them for backwards
  32         # compatibility
  33         self.allfiles = None
  34         self.files = []
  35
  36     def set_allfiles(self, allfiles):
  37         self.allfiles = allfiles
  38
  39     def findall(self, dir=os.curdir):
  40         self.allfiles = findall(dir)
  41
  42     def debug_print(self, msg):
  43         """Print 'msg' to stdout if the global DEBUG (taken from the
  44         DISTUTILS_DEBUG environment variable) flag is true.
  45         """
  46         from distutils.debug import DEBUG
  47         if DEBUG:
  48             print msg
  49
  50     # -- List-like methods ---------------------------------------------
  51
  52     def append(self, item):
  53         self.files.append(item)
  54
  55     def extend(self, items):
  56         self.files.extend(items)
  57
  58     def sort(self):
  59         # Not a strict lexical sort!
  60         sortable_files = map(os.path.split, self.files)
  61         sortable_files.sort()
  62         self.files = []
  63         for sort_tuple in sortable_files:
  64             self.files.append(os.path.join(*sort_tuple))
  65
  66
  67     # -- Other miscellaneous utility methods ---------------------------
  68
  69     def remove_duplicates(self):
  70         # Assumes list has been sorted!
  71         for i in range(len(self.files) - 1, 0, -1):
  72             if self.files[i] == self.files[i - 1]:
  73                 del self.files[i]
  74
  75
  76     # -- "File template" methods ---------------------------------------
  77
  78     def _parse_template_line(self, line):
  79         words = line.split()
  80         action = words[0]
  81
  82         patterns = dir = dir_pattern = None
  83
  84         if action in ('include', 'exclude',
  85                       'global-include', 'global-exclude'):
  86             if len(words) < 2:
  87                 raise DistutilsTemplateError, \
  88                       "'%s' expects <pattern1> <pattern2> ..." % action
  89
  90             patterns = map(convert_path, words[1:])
  91
  92         elif action in ('recursive-include', 'recursive-exclude'):
  93             if len(words) < 3:
  94                 raise DistutilsTemplateError, \
  95                       "'%s' expects <dir> <pattern1> <pattern2> ..." % action
  96
  97             dir = convert_path(words[1])
  98             patterns = map(convert_path, words[2:])
  99
 100         elif action in ('graft', 'prune'):
 101             if len(words) != 2:
 102                 raise DistutilsTemplateError, \
 103                      "'%s' expects a single <dir_pattern>" % action
 104
 105             dir_pattern = convert_path(words[1])
 106
 107         else:
 108             raise DistutilsTemplateError, "unknown action '%s'" % action
 109
 110         return (action, patterns, dir, dir_pattern)
 111
 112     def process_template_line(self, line):
 113         # Parse the line: split it up, make sure the right number of words
 114         # is there, and return the relevant words.  'action' is always
 115         # defined: it's the first word of the line.  Which of the other
 116         # three are defined depends on the action; it'll be either
 117         # patterns, (dir and patterns), or (dir_pattern).
 118         action, patterns, dir, dir_pattern = self._parse_template_line(line)
 119
 120         # OK, now we know that the action is valid and we have the
 121         # right number of words on the line for that action -- so we
 122         # can proceed with minimal error-checking.
 123         if action == 'include':
 124             self.debug_print("include " + ' '.join(patterns))
 125             for pattern in patterns:
 126                 if not self.include_pattern(pattern, anchor=1):
 127                     log.warn("warning: no files found matching '%s'",
 128                              pattern)
 129
 130         elif action == 'exclude':
 131             self.debug_print("exclude " + ' '.join(patterns))
 132             for pattern in patterns:
 133                 if not self.exclude_pattern(pattern, anchor=1):
 134                     log.warn(("warning: no previously-included files "
 135                               "found matching '%s'"), pattern)
 136
 137         elif action == 'global-include':
 138             self.debug_print("global-include " + ' '.join(patterns))
 139             for pattern in patterns:
 140                 if not self.include_pattern(pattern, anchor=0):
 141                     log.warn(("warning: no files found matching '%s' " +
 142                               "anywhere in distribution"), pattern)
 143
 144         elif action == 'global-exclude':
 145             self.debug_print("global-exclude " + ' '.join(patterns))
 146             for pattern in patterns:
 147                 if not self.exclude_pattern(pattern, anchor=0):
 148                     log.warn(("warning: no previously-included files matching "
 149                               "'%s' found anywhere in distribution"),
 150                              pattern)
 151
 152         elif action == 'recursive-include':
 153             self.debug_print("recursive-include %s %s" %
 154                              (dir, ' '.join(patterns)))
 155             for pattern in patterns:
 156                 if not self.include_pattern(pattern, prefix=dir):
 157                     log.warn(("warning: no files found matching '%s' " +
 158                                 "under directory '%s'"),
 159                              pattern, dir)
 160
 161         elif action == 'recursive-exclude':
 162             self.debug_print("recursive-exclude %s %s" %
 163                              (dir, ' '.join(patterns)))
 164             for pattern in patterns:
 165                 if not self.exclude_pattern(pattern, prefix=dir):
 166                     log.warn(("warning: no previously-included files matching "
 167                               "'%s' found under directory '%s'"),
 168                              pattern, dir)
 169
 170         elif action == 'graft':
 171             self.debug_print("graft " + dir_pattern)
 172             if not self.include_pattern(None, prefix=dir_pattern):
 173                 log.warn("warning: no directories found matching '%s'",
 174                          dir_pattern)
 175
 176         elif action == 'prune':
 177             self.debug_print("prune " + dir_pattern)
 178             if not self.exclude_pattern(None, prefix=dir_pattern):
 179                 log.warn(("no previously-included directories found " +
 180                           "matching '%s'"), dir_pattern)
 181         else:
 182             raise DistutilsInternalError, \
 183                   "this cannot happen: invalid action '%s'" % action
 184
 185     # -- Filtering/selection methods -----------------------------------
 186
 187     def include_pattern(self, pattern, anchor=1, prefix=None, is_regex=0):
 188         """Select strings (presumably filenames) from 'self.files' that
 189         match 'pattern', a Unix-style wildcard (glob) pattern.
 190
 191         Patterns are not quite the same as implemented by the 'fnmatch'
 192         module: '*' and '?'  match non-special characters, where "special"
 193         is platform-dependent: slash on Unix; colon, slash, and backslash on
 194         DOS/Windows; and colon on Mac OS.
 195
 196         If 'anchor' is true (the default), then the pattern match is more
 197         stringent: "*.py" will match "foo.py" but not "foo/bar.py".  If
 198         'anchor' is false, both of these will match.
 199
 200         If 'prefix' is supplied, then only filenames starting with 'prefix'
 201         (itself a pattern) and ending with 'pattern', with anything in between
 202         them, will match.  'anchor' is ignored in this case.
 203
 204         If 'is_regex' is true, 'anchor' and 'prefix' are ignored, and
 205         'pattern' is assumed to be either a string containing a regex or a
 206         regex object -- no translation is done, the regex is just compiled
 207         and used as-is.
 208
 209         Selected strings will be added to self.files.
 210
 211         Return 1 if files are found.
 212         """
 213         files_found = 0
 214         pattern_re = translate_pattern(pattern, anchor, prefix, is_regex)
 215         self.debug_print("include_pattern: applying regex r'%s'" %
 216                          pattern_re.pattern)
 217
 218         # delayed loading of allfiles list
 219         if self.allfiles is None:
 220             self.findall()
 221
 222         for name in self.allfiles:
 223             if pattern_re.search(name):
 224                 self.debug_print(" adding " + name)
 225                 self.files.append(name)
 226                 files_found = 1
 227
 228         return files_found
 229
 230
 231     def exclude_pattern(self, pattern, anchor=1, prefix=None, is_regex=0):
 232         """Remove strings (presumably filenames) from 'files' that match
 233         'pattern'.
 234
 235         Other parameters are the same as for 'include_pattern()', above.
 236         The list 'self.files' is modified in place. Return 1 if files are
 237         found.
 238         """
 239         files_found = 0
 240         pattern_re = translate_pattern(pattern, anchor, prefix, is_regex)
 241         self.debug_print("exclude_pattern: applying regex r'%s'" %
 242                          pattern_re.pattern)
 243         for i in range(len(self.files)-1, -1, -1):
 244             if pattern_re.search(self.files[i]):
 245                 self.debug_print(" removing " + self.files[i])
 246                 del self.files[i]
 247                 files_found = 1
 248
 249         return files_found
 250
 251
 252 # ----------------------------------------------------------------------
 253 # Utility functions
 254
 255 def findall(dir = os.curdir):
 256     """Find all files under 'dir' and return the list of full filenames
 257     (relative to 'dir').
 258     """
 259     from stat import ST_MODE, S_ISREG, S_ISDIR, S_ISLNK
 260
 261     list = []
 262     stack = [dir]
 263     pop = stack.pop
 264     push = stack.append
 265
 266     while stack:
 267         dir = pop()
 268         names = os.listdir(dir)
 269
 270         for name in names:
 271             if dir != os.curdir:        # avoid the dreaded "./" syndrome
 272                 fullname = os.path.join(dir, name)
 273             else:
 274                 fullname = name
 275
 276             # Avoid excess stat calls -- just one will do, thank you!
 277             stat = os.stat(fullname)
 278             mode = stat[ST_MODE]
 279             if S_ISREG(mode):
 280                 list.append(fullname)
 281             elif S_ISDIR(mode) and not S_ISLNK(mode):
 282                 push(fullname)
 283
 284     return list
 285
 286
 287 def glob_to_re(pattern):
 288     """Translate a shell-like glob pattern to a regular expression.
 289
 290     Return a string containing the regex.  Differs from
 291     'fnmatch.translate()' in that '*' does not match "special characters"
 292     (which are platform-specific).
 293     """
 294     pattern_re = fnmatch.translate(pattern)
 295
 296     # '?' and '*' in the glob pattern become '.' and '.*' in the RE, which
 297     # IMHO is wrong -- '?' and '*' aren't supposed to match slash in Unix,
 298     # and by extension they shouldn't match such "special characters" under
 299     # any OS.  So change all non-escaped dots in the RE to match any
 300     # character except the special characters.
 301     # XXX currently the "special characters" are just slash -- i.e. this is
 302     # Unix-only.
 303     pattern_re = re.sub(r'((?<!\\)(\\\\)*)\.', r'\1[^/]', pattern_re)
 304
 305     return pattern_re
 306
 307
 308 def translate_pattern(pattern, anchor=1, prefix=None, is_regex=0):
 309     """Translate a shell-like wildcard pattern to a compiled regular
 310     expression.
 311
 312     Return the compiled regex.  If 'is_regex' true,
 313     then 'pattern' is directly compiled to a regex (if it's a string)
 314     or just returned as-is (assumes it's a regex object).
 315     """
 316     if is_regex:
 317         if isinstance(pattern, str):
 318             return re.compile(pattern)
 319         else:
 320             return pattern
 321
 322     if pattern:
 323         pattern_re = glob_to_re(pattern)
 324     else:
 325         pattern_re = ''
 326
 327     if prefix is not None:
 328         # ditch end of pattern character
 329         empty_pattern = glob_to_re('')
 330         prefix_re = glob_to_re(prefix)[:-len(empty_pattern)]
 331         pattern_re = "^" + os.path.join(prefix_re, ".*" + pattern_re)
 332     else:                               # no prefix -- respect anchor flag
 333         if anchor:
 334             pattern_re = "^" + pattern_re
 335
 336     return re.compile(pattern_re)