cola/fsmonitor.py

   1 # Copyright (c) 2008 David Aguilar
   2 # Copyright (c) 2015 Daniel Harding
   3 """Provides an filesystem monitoring for Linux (via inotify) and for Windows
   4 (via pywin32 and the ReadDirectoryChanges function)"""
   5 from __future__ import division, absolute_import, unicode_literals
   6
   7 import errno
   8 import os
   9 import os.path
  10 import select
  11 from threading import Lock
  12
  13 from cola import utils
  14
  15 AVAILABLE = None
  16
  17 if utils.is_win32():
  18     try:
  19         import pywintypes
  20         import win32con
  21         import win32event
  22         import win32file
  23     except ImportError:
  24         pass
  25     else:
  26         AVAILABLE = 'pywin32'
  27 elif utils.is_linux():
  28     try:
  29         from cola import inotify
  30     except ImportError:
  31         pass
  32     else:
  33         AVAILABLE = 'inotify'
  34
  35 from PyQt4 import QtCore
  36 from PyQt4.QtCore import SIGNAL
  37
  38 from cola import core
  39 from cola import gitcfg
  40 from cola import gitcmds
  41 from cola.compat import bchr
  42 from cola.git import git
  43 from cola.i18n import N_
  44 from cola.interaction import Interaction
  45
  46
  47 class _Monitor(QtCore.QObject):
  48     def __init__(self, thread_class):
  49         QtCore.QObject.__init__(self)
  50         self._thread_class = thread_class
  51         self._thread = None
  52
  53     def start(self):
  54         if self._thread_class is not None:
  55             assert self._thread is None
  56             self._thread = self._thread_class(self)
  57             self._thread.start()
  58
  59     def stop(self):
  60         if self._thread_class is not None:
  61             assert self._thread is not None
  62             self._thread.stop()
  63             self._thread.wait()
  64             self._thread = None
  65
  66     def refresh(self):
  67         if self._thread is not None:
  68             self._thread.refresh()
  69
  70
  71 class _BaseThread(QtCore.QThread):
  72     #: The delay, in milliseconds, between detecting file system modification
  73     #: and triggering the 'files_changed' signal, to coalesce multiple
  74     #: modifications into a single signal.
  75     _NOTIFICATION_DELAY = 888
  76
  77     def __init__(self, monitor):
  78         QtCore.QThread.__init__(self)
  79         self._monitor = monitor
  80         self._running = True
  81         self._pending = False
  82
  83     def refresh(self):
  84         """Do any housekeeping necessary in response to repository changes."""
  85         pass
  86
  87     def notify(self):
  88         """Notifies all observers"""
  89         self._pending = False
  90         self._monitor.emit(SIGNAL('files_changed'))
  91
  92     @staticmethod
  93     def _log_enabled_message():
  94         msg = N_('File system change monitoring: enabled.\n')
  95         Interaction.safe_log(msg)
  96
  97
  98 if AVAILABLE == 'inotify':
  99     class _InotifyThread(_BaseThread):
 100         _TRIGGER_MASK = (
 101                 inotify.IN_ATTRIB |
 102                 inotify.IN_CLOSE_WRITE |
 103                 inotify.IN_CREATE |
 104                 inotify.IN_DELETE |
 105                 inotify.IN_MODIFY |
 106                 inotify.IN_MOVED_FROM |
 107                 inotify.IN_MOVED_TO
 108         )
 109         _ADD_MASK = (
 110                 _TRIGGER_MASK |
 111                 inotify.IN_EXCL_UNLINK |
 112                 inotify.IN_ONLYDIR
 113         )
 114
 115         def __init__(self, monitor):
 116             _BaseThread.__init__(self, monitor)
 117             self._worktree = core.abspath(git.worktree())
 118             self._git_dir = git.git_dir()
 119             self._lock = Lock()
 120             self._inotify_fd = None
 121             self._pipe_r = None
 122             self._pipe_w = None
 123             self._worktree_wds = set()
 124             self._worktree_wd_map = {}
 125             self._git_dir_wds = set()
 126             self._git_dir_wd_map = {}
 127
 128         @staticmethod
 129         def _log_out_of_wds_message():
 130             msg = N_('File system change monitoring: disabled because the'
 131                          ' limit on the total number of inotify watches was'
 132                          ' reached.  You may be able to increase the limit on'
 133                          ' the number of watches by running:\n'
 134                      '\n'
 135                      '    echo fs.inotify.max_user_watches=100000 |'
 136                          ' sudo tee -a /etc/sysctl.conf &&'
 137                          ' sudo sysctl -p\n')
 138             Interaction.safe_log(msg)
 139
 140         def run(self):
 141             try:
 142                 with self._lock:
 143                     self._inotify_fd = inotify.init()
 144                     self._pipe_r, self._pipe_w = os.pipe()
 145
 146                 poll_obj = select.poll()
 147                 poll_obj.register(self._inotify_fd, select.POLLIN)
 148                 poll_obj.register(self._pipe_r, select.POLLIN)
 149
 150                 self.refresh()
 151
 152                 self._log_enabled_message()
 153
 154                 while self._running:
 155                     if self._pending:
 156                         timeout = self._NOTIFICATION_DELAY
 157                     else:
 158                         timeout = None
 159                     try:
 160                         events = poll_obj.poll(timeout)
 161                     except OSError as e:
 162                         if e.errno == errno.EINTR:
 163                             continue
 164                         else:
 165                             raise
 166                     else:
 167                         if not self._running:
 168                             break
 169                         elif not events:
 170                             self.notify()
 171                         else:
 172                             for fd, event in events:
 173                                 if fd == self._inotify_fd:
 174                                     self._handle_events()
 175             finally:
 176                 with self._lock:
 177                     if self._inotify_fd is not None:
 178                         os.close(self._inotify_fd)
 179                         self._inotify_fd = None
 180                     if self._pipe_r is not None:
 181                         os.close(self._pipe_r)
 182                         self._pipe_r = None
 183                         os.close(self._pipe_w)
 184                         self._pipe_w = None
 185
 186         def refresh(self):
 187             with self._lock:
 188                 if self._inotify_fd is None:
 189                     return
 190                 try:
 191                     tracked_dirs = set(os.path.dirname(
 192                                            os.path.join(self._worktree, path))
 193                                        for path in gitcmds.tracked_files())
 194                     self._refresh_watches(tracked_dirs, self._worktree_wds,
 195                                           self._worktree_wd_map)
 196                     git_dirs = set()
 197                     git_dirs.add(self._git_dir)
 198                     for dirpath, dirnames, filenames in core.walk(
 199                             os.path.join(self._git_dir, 'refs')):
 200                         git_dirs.add(dirpath)
 201                     self._refresh_watches(git_dirs, self._git_dir_wds,
 202                                           self._git_dir_wd_map)
 203                 except OSError as e:
 204                     if e.errno == errno.ENOSPC:
 205                         self._log_out_of_wds_message()
 206                         self._running = False
 207                     else:
 208                         raise
 209
 210         def _refresh_watches(self, paths_to_watch, wd_set, wd_map):
 211             watched_paths = set(wd_map)
 212             for path in watched_paths - paths_to_watch:
 213                 wd = wd_map.pop(path)
 214                 wd_set.remove(wd)
 215                 try:
 216                     inotify.rm_watch(self._inotify_fd, wd)
 217                 except OSError as e:
 218                     if e.errno == errno.EINVAL:
 219                         # This error can occur if the target of the wd was
 220                         # removed on the filesystem before we call
 221                         # inotify.rm_watch() so ignore it.
 222                         pass
 223                     else:
 224                         raise
 225             for path in paths_to_watch - watched_paths:
 226                 try:
 227                     wd = inotify.add_watch(self._inotify_fd, core.encode(path),
 228                                            self._ADD_MASK)
 229                 except OSError as e:
 230                     if e.errno in (errno.ENOENT, errno.ENOTDIR):
 231                         # These two errors should only occur as a result of
 232                         # race conditions:  the first if the directory
 233                         # referenced by path was removed or renamed before the
 234                         # call to inotify.add_watch(); the second if the
 235                         # directory referenced by path was replaced with a file
 236                         # before the call to inotify.add_watch().  Therefore we
 237                         # simply ignore them.
 238                         pass
 239                     else:
 240                         raise
 241                 else:
 242                     wd_set.add(wd)
 243                     wd_map[path] = wd
 244
 245         def _filter_event(self, wd, mask, name):
 246             # An event is relevant iff:
 247             # 1) it is an event queue overflow
 248             # 2) the wd is for the worktree
 249             # 3) the wd is for the git dir and
 250             #    a) the event is for a file, and
 251             #    b) the file name does not end with ".lock"
 252             if mask & inotify.IN_Q_OVERFLOW:
 253                 return True
 254             if mask & self._TRIGGER_MASK:
 255                 if wd in self._worktree_wds:
 256                     return True
 257                 if (wd in self._git_dir_wds
 258                         and not mask & inotify.IN_ISDIR
 259                         and not core.decode(name).endswith('.lock')):
 260                     return True
 261             return False
 262
 263         def _handle_events(self):
 264             for wd, mask, cookie, name in \
 265                     inotify.read_events(self._inotify_fd):
 266                 if self._filter_event(wd, mask, name):
 267                     self._pending = True
 268
 269         def stop(self):
 270             self._running = False
 271             with self._lock:
 272                 if self._pipe_w is not None:
 273                     os.write(self._pipe_w, bchr(0))
 274             self.wait()
 275
 276
 277 if AVAILABLE == 'pywin32':
 278     class _Win32Watch(object):
 279         def __init__(self, path, flags):
 280             self.flags = flags
 281
 282             self.handle = None
 283             self.event = None
 284
 285             try:
 286                 self.handle = win32file.CreateFileW(
 287                         path,
 288                         0x0001, # FILE_LIST_DIRECTORY
 289                         win32con.FILE_SHARE_READ | win32con.FILE_SHARE_WRITE,
 290                         None,
 291                         win32con.OPEN_EXISTING,
 292                         win32con.FILE_FLAG_BACKUP_SEMANTICS |
 293                             win32con.FILE_FLAG_OVERLAPPED,
 294                         None)
 295
 296                 self.buffer = win32file.AllocateReadBuffer(8192)
 297                 self.event = win32event.CreateEvent(None, True, False, None)
 298                 self.overlapped = pywintypes.OVERLAPPED()
 299                 self.overlapped.hEvent = self.event
 300                 self._start()
 301             except:
 302                 self.close()
 303                 raise
 304
 305         def _start(self):
 306             win32file.ReadDirectoryChangesW(self.handle, self.buffer, True,
 307                                             self.flags, self.overlapped)
 308
 309         def read(self):
 310             if win32event.WaitForSingleObject(self.event, 0) \
 311                     == win32event.WAIT_TIMEOUT:
 312                 result = []
 313             else:
 314                 nbytes = win32file.GetOverlappedResult(self.handle,
 315                                                        self.overlapped, False)
 316                 result = win32file.FILE_NOTIFY_INFORMATION(self.buffer, nbytes)
 317                 self._start()
 318             return result
 319
 320         def close(self):
 321             if self.handle is not None:
 322                 win32file.CancelIo(self.handle)
 323                 win32file.CloseHandle(self.handle)
 324             if self.event is not None:
 325                 win32file.CloseHandle(self.event)
 326
 327
 328     class _Win32Thread(_BaseThread):
 329         _FLAGS = (win32con.FILE_NOTIFY_CHANGE_FILE_NAME |
 330                   win32con.FILE_NOTIFY_CHANGE_DIR_NAME |
 331                   win32con.FILE_NOTIFY_CHANGE_ATTRIBUTES |
 332                   win32con.FILE_NOTIFY_CHANGE_SIZE |
 333                   win32con.FILE_NOTIFY_CHANGE_LAST_WRITE |
 334                   win32con.FILE_NOTIFY_CHANGE_SECURITY)
 335
 336         def __init__(self, monitor):
 337             _BaseThread.__init__(self, monitor)
 338             self._worktree = self._transform_path(core.abspath(git.worktree()))
 339             self._worktree_watch = None
 340             self._git_dir = self._transform_path(core.abspath(git.git_dir()))
 341             self._git_dir_watch = None
 342             self._stop_event_lock = Lock()
 343             self._stop_event = None
 344
 345         @staticmethod
 346         def _transform_path(path):
 347             return path.replace('\\', '/').lower()
 348
 349         def _read_watch(self, watch):
 350             if win32event.WaitForSingleObject(watch.event, 0) \
 351                     == win32event.WAIT_TIMEOUT:
 352                 nbytes = 0
 353             else:
 354                 nbytes = win32file.GetOverlappedResult(watch.handle,
 355                                                        watch.overlapped, False)
 356             return win32file.FILE_NOTIFY_INFORMATION(watch.buffer, nbytes)
 357
 358         def run(self):
 359             try:
 360                 with self._stop_event_lock:
 361                     self._stop_event = win32event.CreateEvent(None, True,
 362                                                               False, None)
 363
 364                 self._worktree_watch = _Win32Watch(self._worktree, self._FLAGS)
 365                 self._git_dir_watch = _Win32Watch(self._git_dir, self._FLAGS)
 366
 367                 self._log_enabled_message()
 368
 369                 events = [self._worktree_watch.event,
 370                           self._git_dir_watch.event,
 371                           self._stop_event]
 372                 while self._running:
 373                     if self._pending:
 374                         timeout = self._NOTIFICATION_DELAY
 375                     else:
 376                         timeout = win32event.INFINITE
 377                     rc = win32event.WaitForMultipleObjects(events, False,
 378                                                            timeout)
 379                     if not self._running:
 380                         break
 381                     elif rc == win32event.WAIT_TIMEOUT:
 382                         self.notify()
 383                     else:
 384                         self._handle_results()
 385             finally:
 386                 with self._stop_event_lock:
 387                     if self._stop_event is not None:
 388                         win32file.CloseHandle(self._stop_event)
 389                         self._stop_event = None
 390                 if self._worktree_watch is not None:
 391                     self._worktree_watch.close()
 392                 if self._git_dir_watch is not None:
 393                     self._git_dir_watch.close()
 394
 395         def _handle_results(self):
 396             for action, path in self._worktree_watch.read():
 397                 if not self._running:
 398                     break
 399                 path = self._worktree + '/' + self._transform_path(path)
 400                 if (path != self._git_dir
 401                         and not path.startswith(self._git_dir + '/')):
 402                     self._pending = True
 403             for action, path in self._git_dir_watch.read():
 404                 if not self._running:
 405                     break
 406                 if not path.endswith('.lock'):
 407                     self._pending = True
 408
 409         def stop(self):
 410             self._running = False
 411             with self._stop_event_lock:
 412                 if self._stop_event is not None:
 413                     win32event.SetEvent(self._stop_event)
 414             self.wait()
 415
 416
 417 _instance = None
 418
 419 def instance():
 420     global _instance
 421     if _instance is None:
 422         _instance = _create_instance()
 423     return _instance
 424
 425
 426 def _create_instance():
 427     thread_class = None
 428     cfg = gitcfg.current()
 429     if not cfg.get('cola.inotify', True):
 430         msg = N_('File system change monitoring: disabled because'
 431                  ' "cola.inotify" is false.\n')
 432         Interaction.log(msg)
 433     elif AVAILABLE == 'inotify':
 434         thread_class = _InotifyThread
 435     elif AVAILABLE == 'pywin32':
 436         thread_class = _Win32Thread
 437     else:
 438         if utils.is_win32():
 439             msg = N_('File system change monitoring: disabled because pywin32'
 440                      ' is not installed.\n')
 441             Interaction.log(msg)
 442         elif utils.is_linux():
 443             msg = N_('File system change monitoring: disabled because libc'
 444                      ' does not support the inotify system calls.\n')
 445             Interaction.log(msg)
 446     return _Monitor(thread_class)