cola/fsmonitor.py

   1 # Copyright (c) 2008 David Aguilar
   2 # Copyright (c) 2015 Daniel Harding
   3 """Provides an filesystem monitoring for Linux (via inotify) and for Windows
   4 (via pywin32 and the ReadDirectoryChanges function)"""
   5 from __future__ import division, absolute_import, unicode_literals
   6
   7 import errno
   8 import os
   9 import os.path
  10 import select
  11 from threading import Lock
  12
  13 from cola import utils
  14 from cola.decorators import memoize
  15
  16 AVAILABLE = None
  17
  18 if utils.is_win32():
  19     try:
  20         import pywintypes
  21         import win32con
  22         import win32event
  23         import win32file
  24     except ImportError:
  25         pass
  26     else:
  27         AVAILABLE = 'pywin32'
  28 elif utils.is_linux():
  29     try:
  30         from cola import inotify
  31     except ImportError:
  32         pass
  33     else:
  34         AVAILABLE = 'inotify'
  35
  36 from PyQt4 import QtCore
  37 from PyQt4.QtCore import SIGNAL
  38
  39 from cola import core
  40 from cola import gitcfg
  41 from cola import gitcmds
  42 from cola.compat import bchr
  43 from cola.git import git
  44 from cola.i18n import N_
  45 from cola.interaction import Interaction
  46
  47
  48 class _Monitor(QtCore.QObject):
  49     def __init__(self, thread_class):
  50         QtCore.QObject.__init__(self)
  51         self._thread_class = thread_class
  52         self._thread = None
  53
  54     def start(self, refs_only=False):
  55         if self._thread_class is not None:
  56             assert self._thread is None
  57             self._thread = self._thread_class(self, refs_only)
  58             self._thread.start()
  59
  60     def stop(self):
  61         if self._thread_class is not None:
  62             assert self._thread is not None
  63             self._thread.stop()
  64             self._thread.wait()
  65             self._thread = None
  66
  67     def refresh(self):
  68         if self._thread is not None:
  69             self._thread.refresh()
  70
  71
  72 class _BaseThread(QtCore.QThread):
  73     #: The delay, in milliseconds, between detecting file system modification
  74     #: and triggering the 'files_changed' signal, to coalesce multiple
  75     #: modifications into a single signal.
  76     _NOTIFICATION_DELAY = 888
  77
  78     def __init__(self, monitor, refs_only):
  79         QtCore.QThread.__init__(self)
  80         self._monitor = monitor
  81         self._refs_only = refs_only
  82         self._running = True
  83         self._pending = False
  84
  85     def refresh(self):
  86         """Do any housekeeping necessary in response to repository changes."""
  87         pass
  88
  89     def notify(self):
  90         """Notifies all observers"""
  91         self._pending = False
  92         self._monitor.emit(SIGNAL('files_changed'))
  93
  94     @staticmethod
  95     def _log_enabled_message():
  96         msg = N_('File system change monitoring: enabled.\n')
  97         Interaction.safe_log(msg)
  98
  99
 100 if AVAILABLE == 'inotify':
 101     class _InotifyThread(_BaseThread):
 102         _TRIGGER_MASK = (
 103                 inotify.IN_ATTRIB |
 104                 inotify.IN_CLOSE_WRITE |
 105                 inotify.IN_CREATE |
 106                 inotify.IN_DELETE |
 107                 inotify.IN_MODIFY |
 108                 inotify.IN_MOVED_FROM |
 109                 inotify.IN_MOVED_TO
 110         )
 111         _ADD_MASK = (
 112                 _TRIGGER_MASK |
 113                 inotify.IN_EXCL_UNLINK |
 114                 inotify.IN_ONLYDIR
 115         )
 116
 117         def __init__(self, monitor, refs_only):
 118             _BaseThread.__init__(self, monitor, refs_only)
 119             if refs_only:
 120                 worktree = None
 121             else:
 122                 worktree = git.worktree()
 123                 if worktree is not None:
 124                     worktree = core.abspath(worktree)
 125             self._worktree = worktree
 126             self._git_dir = git.git_path()
 127             self._lock = Lock()
 128             self._inotify_fd = None
 129             self._pipe_r = None
 130             self._pipe_w = None
 131             self._worktree_wds = set()
 132             self._worktree_wd_map = {}
 133             self._git_dir_wds = set()
 134             self._git_dir_wd_map = {}
 135             self._git_dir_wd = None
 136
 137         @staticmethod
 138         def _log_out_of_wds_message():
 139             msg = N_('File system change monitoring: disabled because the'
 140                      ' limit on the total number of inotify watches was'
 141                      ' reached.  You may be able to increase the limit on'
 142                      ' the number of watches by running:\n'
 143                      '\n'
 144                      '    echo fs.inotify.max_user_watches=100000 |'
 145                      ' sudo tee -a /etc/sysctl.conf &&'
 146                      ' sudo sysctl -p\n')
 147             Interaction.safe_log(msg)
 148
 149         def run(self):
 150             try:
 151                 with self._lock:
 152                     self._inotify_fd = inotify.init()
 153                     self._pipe_r, self._pipe_w = os.pipe()
 154
 155                 poll_obj = select.poll()
 156                 poll_obj.register(self._inotify_fd, select.POLLIN)
 157                 poll_obj.register(self._pipe_r, select.POLLIN)
 158
 159                 self.refresh()
 160
 161                 self._log_enabled_message()
 162
 163                 while self._running:
 164                     if self._pending:
 165                         timeout = self._NOTIFICATION_DELAY
 166                     else:
 167                         timeout = None
 168                     try:
 169                         events = poll_obj.poll(timeout)
 170                     except OSError as e:
 171                         if e.errno == errno.EINTR:
 172                             continue
 173                         else:
 174                             raise
 175                     else:
 176                         if not self._running:
 177                             break
 178                         elif not events:
 179                             self.notify()
 180                         else:
 181                             for fd, event in events:
 182                                 if fd == self._inotify_fd:
 183                                     self._handle_events()
 184             finally:
 185                 with self._lock:
 186                     if self._inotify_fd is not None:
 187                         os.close(self._inotify_fd)
 188                         self._inotify_fd = None
 189                     if self._pipe_r is not None:
 190                         os.close(self._pipe_r)
 191                         self._pipe_r = None
 192                         os.close(self._pipe_w)
 193                         self._pipe_w = None
 194
 195         def refresh(self):
 196             with self._lock:
 197                 if self._inotify_fd is None:
 198                     return
 199                 try:
 200                     if self._worktree is not None:
 201                         tracked_dirs = set(
 202                                 os.path.dirname(os.path.join(self._worktree,
 203                                                              path))
 204                                 for path in gitcmds.tracked_files())
 205                         self._refresh_watches(tracked_dirs, self._worktree_wds,
 206                                               self._worktree_wd_map)
 207                     git_dirs = set()
 208                     git_dirs.add(self._git_dir)
 209                     for dirpath, dirnames, filenames in core.walk(
 210                             os.path.join(self._git_dir, 'refs')):
 211                         git_dirs.add(dirpath)
 212                     self._refresh_watches(git_dirs, self._git_dir_wds,
 213                                           self._git_dir_wd_map)
 214                     self._git_dir_wd = self._git_dir_wd_map[self._git_dir]
 215                 except OSError as e:
 216                     if e.errno == errno.ENOSPC:
 217                         self._log_out_of_wds_message()
 218                         self._running = False
 219                     else:
 220                         raise
 221
 222         def _refresh_watches(self, paths_to_watch, wd_set, wd_map):
 223             watched_paths = set(wd_map)
 224             for path in watched_paths - paths_to_watch:
 225                 wd = wd_map.pop(path)
 226                 wd_set.remove(wd)
 227                 try:
 228                     inotify.rm_watch(self._inotify_fd, wd)
 229                 except OSError as e:
 230                     if e.errno == errno.EINVAL:
 231                         # This error can occur if the target of the wd was
 232                         # removed on the filesystem before we call
 233                         # inotify.rm_watch() so ignore it.
 234                         pass
 235                     else:
 236                         raise
 237             for path in paths_to_watch - watched_paths:
 238                 try:
 239                     wd = inotify.add_watch(self._inotify_fd, core.encode(path),
 240                                            self._ADD_MASK)
 241                 except OSError as e:
 242                     if e.errno in (errno.ENOENT, errno.ENOTDIR):
 243                         # These two errors should only occur as a result of
 244                         # race conditions:  the first if the directory
 245                         # referenced by path was removed or renamed before the
 246                         # call to inotify.add_watch(); the second if the
 247                         # directory referenced by path was replaced with a file
 248                         # before the call to inotify.add_watch().  Therefore we
 249                         # simply ignore them.
 250                         pass
 251                     else:
 252                         raise
 253                 else:
 254                     wd_set.add(wd)
 255                     wd_map[path] = wd
 256
 257         def _event_is_relevant(self, wd, mask, name):
 258             if mask & inotify.IN_Q_OVERFLOW:
 259                 return True
 260             elif not mask & self._TRIGGER_MASK:
 261                 return False
 262             elif wd in self._worktree_wds:
 263                 return True
 264             elif mask & inotify.IN_ISDIR:
 265                 return False
 266             elif wd == self._git_dir_wd:
 267                 name = core.decode(name)
 268                 if name == 'HEAD':
 269                     return True
 270                 elif not self._refs_only and name == 'index':
 271                     return True
 272             elif (wd in self._git_dir_wds and
 273                     not core.decode(name).endswith('.lock')):
 274                 return True
 275             else:
 276                 return False
 277
 278         def _handle_events(self):
 279             for wd, mask, cookie, name in \
 280                     inotify.read_events(self._inotify_fd):
 281                 if self._event_is_relevant(wd, mask, name):
 282                     self._pending = True
 283
 284         def stop(self):
 285             self._running = False
 286             with self._lock:
 287                 if self._pipe_w is not None:
 288                     os.write(self._pipe_w, bchr(0))
 289             self.wait()
 290
 291
 292 if AVAILABLE == 'pywin32':
 293
 294     class _Win32Watch(object):
 295
 296         def __init__(self, path, flags):
 297             self.flags = flags
 298
 299             self.handle = None
 300             self.event = None
 301
 302             try:
 303                 self.handle = win32file.CreateFileW(
 304                         path,
 305                         0x0001,  # FILE_LIST_DIRECTORY
 306                         win32con.FILE_SHARE_READ | win32con.FILE_SHARE_WRITE,
 307                         None,
 308                         win32con.OPEN_EXISTING,
 309                         win32con.FILE_FLAG_BACKUP_SEMANTICS |
 310                         win32con.FILE_FLAG_OVERLAPPED,
 311                         None)
 312
 313                 self.buffer = win32file.AllocateReadBuffer(8192)
 314                 self.event = win32event.CreateEvent(None, True, False, None)
 315                 self.overlapped = pywintypes.OVERLAPPED()
 316                 self.overlapped.hEvent = self.event
 317                 self._start()
 318             except:
 319                 self.close()
 320                 raise
 321
 322         def _start(self):
 323             win32file.ReadDirectoryChangesW(self.handle, self.buffer, True,
 324                                             self.flags, self.overlapped)
 325
 326         def read(self):
 327             if win32event.WaitForSingleObject(self.event, 0) \
 328                     == win32event.WAIT_TIMEOUT:
 329                 result = []
 330             else:
 331                 nbytes = win32file.GetOverlappedResult(self.handle,
 332                                                        self.overlapped, False)
 333                 result = win32file.FILE_NOTIFY_INFORMATION(self.buffer, nbytes)
 334                 self._start()
 335             return result
 336
 337         def close(self):
 338             if self.handle is not None:
 339                 win32file.CancelIo(self.handle)
 340                 win32file.CloseHandle(self.handle)
 341             if self.event is not None:
 342                 win32file.CloseHandle(self.event)
 343
 344     class _Win32Thread(_BaseThread):
 345         _FLAGS = (win32con.FILE_NOTIFY_CHANGE_FILE_NAME |
 346                   win32con.FILE_NOTIFY_CHANGE_DIR_NAME |
 347                   win32con.FILE_NOTIFY_CHANGE_ATTRIBUTES |
 348                   win32con.FILE_NOTIFY_CHANGE_SIZE |
 349                   win32con.FILE_NOTIFY_CHANGE_LAST_WRITE |
 350                   win32con.FILE_NOTIFY_CHANGE_SECURITY)
 351
 352         def __init__(self, monitor, refs_only):
 353             _BaseThread.__init__(self, monitor, refs_only)
 354             if refs_only:
 355                 worktree = None
 356             else:
 357                 worktree = git.worktree()
 358                 if worktree is not None:
 359                     worktree = self._transform_path(core.abspath(worktree))
 360             self._worktree = worktree
 361             self._worktree_watch = None
 362             self._git_dir = self._transform_path(core.abspath(git.git_path()))
 363             self._git_dir_watch = None
 364             self._stop_event_lock = Lock()
 365             self._stop_event = None
 366
 367         @staticmethod
 368         def _transform_path(path):
 369             return path.replace('\\', '/').lower()
 370
 371         def _read_watch(self, watch):
 372             if win32event.WaitForSingleObject(watch.event, 0) \
 373                     == win32event.WAIT_TIMEOUT:
 374                 nbytes = 0
 375             else:
 376                 nbytes = win32file.GetOverlappedResult(watch.handle,
 377                                                        watch.overlapped, False)
 378             return win32file.FILE_NOTIFY_INFORMATION(watch.buffer, nbytes)
 379
 380         def run(self):
 381             try:
 382                 with self._stop_event_lock:
 383                     self._stop_event = win32event.CreateEvent(None, True,
 384                                                               False, None)
 385
 386                 events = [self._stop_event]
 387
 388                 if self._worktree is not None:
 389                     self._worktree_watch = _Win32Watch(self._worktree,
 390                                                        self._FLAGS)
 391                     events.append(self._worktree_watch.event)
 392
 393                 self._git_dir_watch = _Win32Watch(self._git_dir, self._FLAGS)
 394                 events.append(self._git_dir_watch.event)
 395
 396                 self._log_enabled_message()
 397
 398                 while self._running:
 399                     if self._pending:
 400                         timeout = self._NOTIFICATION_DELAY
 401                     else:
 402                         timeout = win32event.INFINITE
 403                     rc = win32event.WaitForMultipleObjects(events, False,
 404                                                            timeout)
 405                     if not self._running:
 406                         break
 407                     elif rc == win32event.WAIT_TIMEOUT:
 408                         self.notify()
 409                     else:
 410                         self._handle_results()
 411             finally:
 412                 with self._stop_event_lock:
 413                     if self._stop_event is not None:
 414                         win32file.CloseHandle(self._stop_event)
 415                         self._stop_event = None
 416                 if self._worktree_watch is not None:
 417                     self._worktree_watch.close()
 418                 if self._git_dir_watch is not None:
 419                     self._git_dir_watch.close()
 420
 421         def _handle_results(self):
 422             if self._worktree_watch is not None:
 423                 for action, path in self._worktree_watch.read():
 424                     if not self._running:
 425                         break
 426                     path = self._worktree + '/' + self._transform_path(path)
 427                     if (path != self._git_dir and
 428                             not path.startswith(self._git_dir + '/')):
 429                         self._pending = True
 430             for action, path in self._git_dir_watch.read():
 431                 if not self._running:
 432                     break
 433                 path = self._transform_path(path)
 434                 if path.endswith('.lock'):
 435                     continue
 436                 if path == 'head' or path.startswith('refs/'):
 437                     self._pending = True
 438                 elif not self._refs_only and path == 'index':
 439                     self._pending = True
 440
 441         def stop(self):
 442             self._running = False
 443             with self._stop_event_lock:
 444                 if self._stop_event is not None:
 445                     win32event.SetEvent(self._stop_event)
 446             self.wait()
 447
 448
 449 @memoize
 450 def current():
 451     return _create_instance()
 452
 453
 454 def _create_instance():
 455     thread_class = None
 456     cfg = gitcfg.current()
 457     if not cfg.get('cola.inotify', True):
 458         msg = N_('File system change monitoring: disabled because'
 459                  ' "cola.inotify" is false.\n')
 460         Interaction.log(msg)
 461     elif AVAILABLE == 'inotify':
 462         thread_class = _InotifyThread
 463     elif AVAILABLE == 'pywin32':
 464         thread_class = _Win32Thread
 465     else:
 466         if utils.is_win32():
 467             msg = N_('File system change monitoring: disabled because pywin32'
 468                      ' is not installed.\n')
 469             Interaction.log(msg)
 470         elif utils.is_linux():
 471             msg = N_('File system change monitoring: disabled because libc'
 472                      ' does not support the inotify system calls.\n')
 473             Interaction.log(msg)
 474     return _Monitor(thread_class)