python/google/appengine/tools/devappserver2/inotify_file_watcher.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17 """Monitors a directory tree for changes using the inotify API.
  18
  19 See http://linux.die.net/man/7/inotify.
  20 """
  21
  22
  23 import ctypes
  24 import ctypes.util
  25 import errno
  26 import itertools
  27 import logging
  28 import os
  29 import select
  30 import struct
  31 import sys
  32
  33 from google.appengine.tools.devappserver2 import watcher_common
  34
  35 IN_MODIFY = 0x00000002
  36 IN_ATTRIB = 0x00000004
  37 IN_MOVED_FROM = 0x00000040
  38 IN_MOVED_TO = 0x00000080
  39 IN_CREATE = 0x00000100
  40 IN_DELETE = 0x00000200
  41
  42 IN_IGNORED = 0x00008000
  43 IN_ISDIR = 0x40000000
  44
  45 _INOTIFY_EVENT = struct.Struct('iIII')
  46 _INOTIFY_EVENT_SIZE = _INOTIFY_EVENT.size
  47 _INTERESTING_INOTIFY_EVENTS = (
  48     IN_ATTRIB|IN_MODIFY|IN_MOVED_FROM|IN_MOVED_TO|IN_CREATE|IN_DELETE)
  49
  50 # inotify only available on Linux and a ctypes.CDLL will raise if code tries to
  51 # specify the arg types or return type for a non-existent function.
  52 if sys.platform.startswith('linux'):
  53   _libc = ctypes.CDLL(ctypes.util.find_library('c'), use_errno=True)
  54   _libc.inotify_init.argtypes = []
  55   _libc.inotify_init.restype = ctypes.c_int
  56   _libc.inotify_add_watch.argtypes = [ctypes.c_int,
  57                                       ctypes.c_char_p,
  58                                       ctypes.c_uint32]
  59   _libc.inotify_add_watch.restype = ctypes.c_int
  60   _libc.inotify_rm_watch.argtypes = [ctypes.c_int,
  61                                      ctypes.c_int]
  62   _libc.inotify_rm_watch.restype = ctypes.c_int
  63 else:
  64   _libc = None
  65
  66
  67 class InotifyFileWatcher(object):
  68   """Monitors a directory tree for changes using inotify."""
  69
  70   SUPPORTS_MULTIPLE_DIRECTORIES = True
  71
  72   def __init__(self, directories):
  73     """Initializer for InotifyFileWatcher.
  74
  75     Args:
  76       directories: An iterable of strings representing the path to a directory
  77           that should be monitored for changes i.e. files and directories added,
  78           renamed, deleted or changed.
  79
  80     Raises:
  81       OSError: if there are no inotify instances available.
  82     """
  83     assert _libc is not None, 'InotifyFileWatcher only available on Linux.'
  84     self._directories = [os.path.abspath(d) for d in directories]
  85     self._watch_to_directory = {}
  86     self._directory_to_watch_descriptor = {}
  87     self._directory_to_subdirs = {}
  88     self._inotify_events = ''
  89     self._inotify_fd = _libc.inotify_init()
  90     if self._inotify_fd < 0:
  91       error = OSError('failed call to inotify_init')
  92       error.errno = ctypes.get_errno()
  93       error.strerror = errno.errorcode[ctypes.get_errno()]
  94       raise error
  95     self._inotify_poll = select.poll()
  96
  97
  98   def _remove_watch_for_path(self, path):
  99     logging.debug('_remove_watch_for_path(%r)', path)
 100     wd = self._directory_to_watch_descriptor[path]
 101
 102     if _libc.inotify_rm_watch(self._inotify_fd, wd) < 0:
 103       # If the directory is deleted then the watch will removed automatically
 104       # and inotify_rm_watch will fail. Just log the error.
 105       logging.debug('inotify_rm_watch failed for %r: %d [%r]',
 106                     path,
 107                     ctypes.get_errno(),
 108                     errno.errorcode[ctypes.get_errno()])
 109
 110     parent_path = os.path.dirname(path)
 111     if parent_path in self._directory_to_subdirs:
 112       self._directory_to_subdirs[parent_path].remove(path)
 113
 114     # _directory_to_subdirs must be copied because it is mutated in the
 115     # recursive call.
 116     for subdir in frozenset(self._directory_to_subdirs[path]):
 117       self._remove_watch_for_path(subdir)
 118
 119     del self._watch_to_directory[wd]
 120     del self._directory_to_watch_descriptor[path]
 121     del self._directory_to_subdirs[path]
 122
 123   def _add_watch_for_path(self, path):
 124     logging.debug('_add_watch_for_path(%r)', path)
 125
 126     for dirpath, directories, _ in itertools.chain(
 127         [(os.path.dirname(path), [os.path.basename(path)], None)],
 128         os.walk(path, topdown=True, followlinks=True)):
 129       watcher_common.remove_ignored_dirs(directories)
 130       for directory in directories:
 131         directory_path = os.path.join(dirpath, directory)
 132         # dirpath cannot be used as the parent directory path because it is the
 133         # empty string for symlinks :-(
 134         parent_path = os.path.dirname(directory_path)
 135
 136         watch_descriptor = _libc.inotify_add_watch(
 137             self._inotify_fd,
 138             ctypes.create_string_buffer(directory_path),
 139             _INTERESTING_INOTIFY_EVENTS)
 140         if watch_descriptor < 0:
 141           if ctypes.get_errno() == errno.ENOSPC:
 142             logging.warning(
 143                 'There are too many directories in your application for '
 144                 'changes in all of them to be monitored. You may have to '
 145                 'restart the development server to see some changes to your '
 146                 'files.')
 147             return
 148           error = OSError('could not add watch for %r' % directory_path)
 149           error.errno = ctypes.get_errno()
 150           error.strerror = errno.errorcode[ctypes.get_errno()]
 151           error.filename = directory_path
 152           raise error
 153
 154         if parent_path in self._directory_to_subdirs:
 155           self._directory_to_subdirs[parent_path].add(directory_path)
 156         self._watch_to_directory[watch_descriptor] = directory_path
 157         self._directory_to_watch_descriptor[directory_path] = watch_descriptor
 158         self._directory_to_subdirs[directory_path] = set()
 159
 160   def start(self):
 161     """Start watching the directory for changes."""
 162     self._inotify_poll.register(self._inotify_fd, select.POLLIN)
 163     for directory in self._directories:
 164       self._add_watch_for_path(directory)
 165
 166   def quit(self):
 167     """Stop watching the directory for changes."""
 168     os.close(self._inotify_fd)
 169
 170   def _get_changed_paths(self):
 171     """Return paths for changed files and directories.
 172
 173     start() must be called before this method.
 174
 175     Returns:
 176       A set of strings representing file and directory paths that have changed
 177       since the last call to get_changed_paths.
 178     """
 179     paths = set()
 180     while True:
 181       if not self._inotify_poll.poll(0):
 182         break
 183
 184       self._inotify_events += os.read(self._inotify_fd, 1024)
 185       while len(self._inotify_events) > _INOTIFY_EVENT_SIZE:
 186         wd, mask, cookie, length = _INOTIFY_EVENT.unpack(
 187             self._inotify_events[:_INOTIFY_EVENT_SIZE])
 188         if len(self._inotify_events) < _INOTIFY_EVENT_SIZE + length:
 189           break
 190
 191         name = self._inotify_events[
 192             _INOTIFY_EVENT_SIZE:_INOTIFY_EVENT_SIZE+length]
 193         name = name.rstrip('\0')
 194
 195         logging.debug('wd=%s, mask=%s, cookie=%s, length=%s, name=%r',
 196                       wd, hex(mask), cookie, length, name)
 197
 198         self._inotify_events = self._inotify_events[_INOTIFY_EVENT_SIZE+length:]
 199
 200         if mask & IN_IGNORED:
 201           continue
 202         try:
 203           directory = self._watch_to_directory[wd]
 204         except KeyError:
 205           logging.debug('Watch deleted for watch descriptor=%d', wd)
 206           continue
 207
 208         path = os.path.join(directory, name)
 209         if os.path.isdir(path) or path in self._directory_to_watch_descriptor:
 210           if mask & IN_DELETE:
 211             self._remove_watch_for_path(path)
 212           elif mask & IN_MOVED_FROM:
 213             self._remove_watch_for_path(path)
 214           elif mask & IN_CREATE:
 215             self._add_watch_for_path(path)
 216           elif mask & IN_MOVED_TO:
 217             self._add_watch_for_path(path)
 218         if path not in paths and not watcher_common.ignore_file(path):
 219           paths.add(path)
 220     return paths
 221
 222   def has_changes(self):
 223     return bool(self._get_changed_paths())