cola/fsmonitor.py

   1 # Copyright (C) 2008-2017 David Aguilar
   2 # Copyright (C) 2015 Daniel Harding
   3 """Filesystem monitor for Linux and Windows
   4
   5 Linux monitoring uses using inotify.
   6 Windows monitoring uses pywin32 and the ReadDirectoryChanges function.
   7
   8 """
   9 from __future__ import division, absolute_import, unicode_literals
  10 import errno
  11 import os
  12 import os.path
  13 import select
  14 from threading import Lock
  15
  16 from qtpy import QtCore
  17 from qtpy.QtCore import Signal
  18
  19 from . import utils
  20 from . import core
  21 from . import gitcmds
  22 from . import version
  23 from .compat import bchr
  24 from .i18n import N_
  25 from .interaction import Interaction
  26
  27 AVAILABLE = None
  28
  29 if utils.is_win32():
  30     try:
  31         import pywintypes
  32         import win32con
  33         import win32event
  34         import win32file
  35     except ImportError:
  36         pass
  37     else:
  38         AVAILABLE = 'pywin32'
  39 elif utils.is_linux():
  40     try:
  41         from . import inotify
  42     except ImportError:
  43         pass
  44     else:
  45         AVAILABLE = 'inotify'
  46
  47
  48 class _Monitor(QtCore.QObject):
  49
  50     files_changed = Signal()
  51     config_changed = Signal()
  52
  53     def __init__(self, context, thread_class):
  54         QtCore.QObject.__init__(self)
  55         self.context = context
  56         self._thread_class = thread_class
  57         self._thread = None
  58
  59     def start(self):
  60         if self._thread_class is not None:
  61             assert self._thread is None
  62             self._thread = self._thread_class(self.context, self)
  63             self._thread.start()
  64
  65     def stop(self):
  66         if self._thread_class is not None:
  67             assert self._thread is not None
  68             self._thread.stop()
  69             self._thread.wait()
  70             self._thread = None
  71
  72     def refresh(self):
  73         if self._thread is not None:
  74             self._thread.refresh()
  75
  76
  77 class _BaseThread(QtCore.QThread):
  78     #: The delay, in milliseconds, between detecting file system modification
  79     #: and triggering the 'files_changed' signal, to coalesce multiple
  80     #: modifications into a single signal.
  81     _NOTIFICATION_DELAY = 888
  82
  83     def __init__(self, context, monitor):
  84         QtCore.QThread.__init__(self)
  85         self.context = context
  86         self._monitor = monitor
  87         self._running = True
  88         self._use_check_ignore = version.check_git(context, 'check-ignore')
  89         self._force_notify = False
  90         self._force_config = False
  91         self._file_paths = set()
  92
  93     @property
  94     def _pending(self):
  95         return self._force_notify or self._file_paths or self._force_config
  96
  97     # pylint: disable=no-self-use
  98     def refresh(self):
  99         """Do any housekeeping necessary in response to repository changes."""
 100         return
 101
 102     def notify(self):
 103         """Notifies all observers"""
 104         do_notify = False
 105         do_config = False
 106         if self._force_config:
 107             do_config = True
 108         if self._force_notify:
 109             do_notify = True
 110         elif self._file_paths:
 111             proc = core.start_command(['git', 'check-ignore', '--verbose',
 112                                        '--non-matching', '-z', '--stdin'])
 113             path_list = bchr(0).join(core.encode(path)
 114                                      for path in self._file_paths)
 115             out, _ = proc.communicate(path_list)
 116             if proc.returncode:
 117                 do_notify = True
 118             else:
 119                 # Each output record is four fields separated by NULL
 120                 # characters (records are also separated by NULL characters):
 121                 # <source> <NULL> <linenum> <NULL> <pattern> <NULL> <pathname>
 122                 # For paths which are not ignored, all fields will be empty
 123                 # except for <pathname>.  So to see if we have any non-ignored
 124                 # files, we simply check every fourth field to see if any of
 125                 # them are empty.
 126                 source_fields = out.split(bchr(0))[0:-1:4]
 127                 do_notify = not all(source_fields)
 128         self._force_notify = False
 129         self._force_config = False
 130         self._file_paths = set()
 131
 132         # "files changed" is a bigger hammer than "config changed".
 133         # and is a superset relative to what is done in response to the
 134         # signal.  Thus, the "elif" below avoids repeated work that
 135         # would be done if it were a simple "if" check instead.
 136         if do_notify:
 137             self._monitor.files_changed.emit()
 138         elif do_config:
 139             self._monitor.config_changed.emit()
 140
 141     @staticmethod
 142     def _log_enabled_message():
 143         msg = N_('File system change monitoring: enabled.\n')
 144         Interaction.log(msg)
 145
 146
 147 if AVAILABLE == 'inotify':
 148
 149     class _InotifyThread(_BaseThread):
 150         _TRIGGER_MASK = (
 151             inotify.IN_ATTRIB |
 152             inotify.IN_CLOSE_WRITE |
 153             inotify.IN_CREATE |
 154             inotify.IN_DELETE |
 155             inotify.IN_MODIFY |
 156             inotify.IN_MOVED_FROM |
 157             inotify.IN_MOVED_TO
 158         )
 159         _ADD_MASK = (
 160             _TRIGGER_MASK |
 161             inotify.IN_EXCL_UNLINK |
 162             inotify.IN_ONLYDIR
 163         )
 164
 165         def __init__(self, context, monitor):
 166             _BaseThread.__init__(self, context, monitor)
 167             git = context.git
 168             worktree = git.worktree()
 169             if worktree is not None:
 170                 worktree = core.abspath(worktree)
 171             self._worktree = worktree
 172             self._git_dir = git.git_path()
 173             self._lock = Lock()
 174             self._inotify_fd = None
 175             self._pipe_r = None
 176             self._pipe_w = None
 177             self._worktree_wd_to_path_map = {}
 178             self._worktree_path_to_wd_map = {}
 179             self._git_dir_wd_to_path_map = {}
 180             self._git_dir_path_to_wd_map = {}
 181             self._git_dir_wd = None
 182
 183         @staticmethod
 184         def _log_out_of_wds_message():
 185             msg = N_('File system change monitoring: disabled because the'
 186                      ' limit on the total number of inotify watches was'
 187                      ' reached.  You may be able to increase the limit on'
 188                      ' the number of watches by running:\n'
 189                      '\n'
 190                      '    echo fs.inotify.max_user_watches=100000 |'
 191                      ' sudo tee -a /etc/sysctl.conf &&'
 192                      ' sudo sysctl -p\n')
 193             Interaction.log(msg)
 194
 195         def run(self):
 196             try:
 197                 with self._lock:
 198                     self._inotify_fd = inotify.init()
 199                     self._pipe_r, self._pipe_w = os.pipe()
 200
 201                 poll_obj = select.poll()
 202                 poll_obj.register(self._inotify_fd, select.POLLIN)
 203                 poll_obj.register(self._pipe_r, select.POLLIN)
 204
 205                 self.refresh()
 206
 207                 self._log_enabled_message()
 208                 self._process_events(poll_obj)
 209             finally:
 210                 self._close_fds()
 211
 212         def _process_events(self, poll_obj):
 213             while self._running:
 214                 if self._pending:
 215                     timeout = self._NOTIFICATION_DELAY
 216                 else:
 217                     timeout = None
 218                 try:
 219                     events = poll_obj.poll(timeout)
 220                 # pylint: disable=duplicate-except
 221                 except (OSError, select.error):
 222                     continue
 223                 else:
 224                     if not self._running:
 225                         break
 226                     elif not events:
 227                         self.notify()
 228                     else:
 229                         for (fd, _) in events:
 230                             if fd == self._inotify_fd:
 231                                 self._handle_events()
 232
 233         def _close_fds(self):
 234             with self._lock:
 235                 if self._inotify_fd is not None:
 236                     os.close(self._inotify_fd)
 237                     self._inotify_fd = None
 238                 if self._pipe_r is not None:
 239                     os.close(self._pipe_r)
 240                     self._pipe_r = None
 241                     os.close(self._pipe_w)
 242                     self._pipe_w = None
 243
 244         def refresh(self):
 245             with self._lock:
 246                 self._refresh()
 247
 248         def _refresh(self):
 249             if self._inotify_fd is None:
 250                 return
 251             context = self.context
 252             try:
 253                 if self._worktree is not None:
 254                     tracked_dirs = set([
 255                         os.path.dirname(os.path.join(self._worktree, path))
 256                         for path in gitcmds.tracked_files(context)])
 257                     self._refresh_watches(tracked_dirs,
 258                                           self._worktree_wd_to_path_map,
 259                                           self._worktree_path_to_wd_map)
 260                 git_dirs = set()
 261                 git_dirs.add(self._git_dir)
 262                 for dirpath, _, _ in core.walk(
 263                         os.path.join(self._git_dir, 'refs')):
 264                     git_dirs.add(dirpath)
 265                 self._refresh_watches(git_dirs,
 266                                       self._git_dir_wd_to_path_map,
 267                                       self._git_dir_path_to_wd_map)
 268                 self._git_dir_wd = \
 269                     self._git_dir_path_to_wd_map.get(self._git_dir)
 270             except OSError as e:
 271                 if e.errno == errno.ENOSPC:
 272                     self._log_out_of_wds_message()
 273                     self._running = False
 274                 else:
 275                     raise
 276
 277         def _refresh_watches(self, paths_to_watch, wd_to_path_map,
 278                              path_to_wd_map):
 279             watched_paths = set(path_to_wd_map)
 280             for path in watched_paths - paths_to_watch:
 281                 wd = path_to_wd_map.pop(path)
 282                 wd_to_path_map.pop(wd)
 283                 try:
 284                     inotify.rm_watch(self._inotify_fd, wd)
 285                 except OSError as e:
 286                     if e.errno == errno.EINVAL:
 287                         # This error can occur if the target of the wd was
 288                         # removed on the filesystem before we call
 289                         # inotify.rm_watch() so ignore it.
 290                         continue
 291                     raise e
 292             for path in paths_to_watch - watched_paths:
 293                 try:
 294                     wd = inotify.add_watch(self._inotify_fd, core.encode(path),
 295                                            self._ADD_MASK)
 296                 except OSError as e:
 297                     if e.errno in (errno.ENOENT, errno.ENOTDIR):
 298                         # These two errors should only occur as a result of
 299                         # race conditions:  the first if the directory
 300                         # referenced by path was removed or renamed before the
 301                         # call to inotify.add_watch(); the second if the
 302                         # directory referenced by path was replaced with a file
 303                         # before the call to inotify.add_watch().  Therefore we
 304                         # simply ignore them.
 305                         continue
 306                     raise e
 307                 else:
 308                     wd_to_path_map[wd] = path
 309                     path_to_wd_map[path] = wd
 310
 311         def _check_event(self, wd, mask, name):
 312             if mask & inotify.IN_Q_OVERFLOW:
 313                 self._force_notify = True
 314             elif not mask & self._TRIGGER_MASK:
 315                 pass
 316             elif mask & inotify.IN_ISDIR:
 317                 pass
 318             elif wd in self._worktree_wd_to_path_map:
 319                 if self._use_check_ignore and name:
 320                     path = os.path.join(self._worktree_wd_to_path_map[wd],
 321                                         core.decode(name))
 322                     self._file_paths.add(path)
 323                 else:
 324                     self._force_notify = True
 325             elif wd == self._git_dir_wd:
 326                 name = core.decode(name)
 327                 if name in ('HEAD', 'index'):
 328                     self._force_notify = True
 329                 elif name == 'config':
 330                     self._force_config = True
 331             elif (wd in self._git_dir_wd_to_path_map
 332                   and not core.decode(name).endswith('.lock')):
 333                 self._force_notify = True
 334
 335         def _handle_events(self):
 336             for wd, mask, _, name in \
 337                     inotify.read_events(self._inotify_fd):
 338                 if not self._force_notify:
 339                     self._check_event(wd, mask, name)
 340
 341         def stop(self):
 342             self._running = False
 343             with self._lock:
 344                 if self._pipe_w is not None:
 345                     os.write(self._pipe_w, bchr(0))
 346             self.wait()
 347
 348
 349 if AVAILABLE == 'pywin32':
 350
 351     class _Win32Watch(object):
 352
 353         def __init__(self, path, flags):
 354             self.flags = flags
 355
 356             self.handle = None
 357             self.event = None
 358
 359             try:
 360                 self.handle = win32file.CreateFileW(
 361                     path,
 362                     0x0001,  # FILE_LIST_DIRECTORY
 363                     win32con.FILE_SHARE_READ | win32con.FILE_SHARE_WRITE,
 364                     None,
 365                     win32con.OPEN_EXISTING,
 366                     win32con.FILE_FLAG_BACKUP_SEMANTICS |
 367                     win32con.FILE_FLAG_OVERLAPPED,
 368                     None)
 369
 370                 self.buffer = win32file.AllocateReadBuffer(8192)
 371                 self.event = win32event.CreateEvent(None, True, False, None)
 372                 self.overlapped = pywintypes.OVERLAPPED()
 373                 self.overlapped.hEvent = self.event
 374                 self._start()
 375             except Exception:
 376                 self.close()
 377                 raise
 378
 379         def _start(self):
 380             win32file.ReadDirectoryChangesW(self.handle, self.buffer, True,
 381                                             self.flags, self.overlapped)
 382
 383         def read(self):
 384             if win32event.WaitForSingleObject(self.event, 0) \
 385                     == win32event.WAIT_TIMEOUT:
 386                 result = []
 387             else:
 388                 nbytes = win32file.GetOverlappedResult(self.handle,
 389                                                        self.overlapped, False)
 390                 result = win32file.FILE_NOTIFY_INFORMATION(self.buffer, nbytes)
 391                 self._start()
 392             return result
 393
 394         def close(self):
 395             if self.handle is not None:
 396                 win32file.CancelIo(self.handle)
 397                 win32file.CloseHandle(self.handle)
 398             if self.event is not None:
 399                 win32file.CloseHandle(self.event)
 400
 401     class _Win32Thread(_BaseThread):
 402         _FLAGS = (win32con.FILE_NOTIFY_CHANGE_FILE_NAME |
 403                   win32con.FILE_NOTIFY_CHANGE_DIR_NAME |
 404                   win32con.FILE_NOTIFY_CHANGE_ATTRIBUTES |
 405                   win32con.FILE_NOTIFY_CHANGE_SIZE |
 406                   win32con.FILE_NOTIFY_CHANGE_LAST_WRITE |
 407                   win32con.FILE_NOTIFY_CHANGE_SECURITY)
 408
 409         def __init__(self, context, monitor):
 410             _BaseThread.__init__(self, context, monitor)
 411             git = context.git
 412             worktree = git.worktree()
 413             if worktree is not None:
 414                 worktree = self._transform_path(core.abspath(worktree))
 415             self._worktree = worktree
 416             self._worktree_watch = None
 417             self._git_dir = self._transform_path(core.abspath(git.git_path()))
 418             self._git_dir_watch = None
 419             self._stop_event_lock = Lock()
 420             self._stop_event = None
 421
 422         @staticmethod
 423         def _transform_path(path):
 424             return path.replace('\\', '/').lower()
 425
 426         def run(self):
 427             try:
 428                 with self._stop_event_lock:
 429                     self._stop_event = win32event.CreateEvent(None, True,
 430                                                               False, None)
 431
 432                 events = [self._stop_event]
 433
 434                 if self._worktree is not None:
 435                     self._worktree_watch = _Win32Watch(self._worktree,
 436                                                        self._FLAGS)
 437                     events.append(self._worktree_watch.event)
 438
 439                 self._git_dir_watch = _Win32Watch(self._git_dir, self._FLAGS)
 440                 events.append(self._git_dir_watch.event)
 441
 442                 self._log_enabled_message()
 443
 444                 while self._running:
 445                     if self._pending:
 446                         timeout = self._NOTIFICATION_DELAY
 447                     else:
 448                         timeout = win32event.INFINITE
 449                     rc = win32event.WaitForMultipleObjects(events, False,
 450                                                            timeout)
 451                     if not self._running:
 452                         break
 453                     elif rc == win32event.WAIT_TIMEOUT:
 454                         self.notify()
 455                     else:
 456                         self._handle_results()
 457             finally:
 458                 with self._stop_event_lock:
 459                     if self._stop_event is not None:
 460                         win32file.CloseHandle(self._stop_event)
 461                         self._stop_event = None
 462                 if self._worktree_watch is not None:
 463                     self._worktree_watch.close()
 464                 if self._git_dir_watch is not None:
 465                     self._git_dir_watch.close()
 466
 467         def _handle_results(self):
 468             if self._worktree_watch is not None:
 469                 for _, path in self._worktree_watch.read():
 470                     if not self._running:
 471                         break
 472                     if self._force_notify:
 473                         continue
 474                     path = self._worktree + '/' + self._transform_path(path)
 475                     if (path != self._git_dir
 476                             and not path.startswith(self._git_dir + '/')
 477                             and not os.path.isdir(path)):
 478                         if self._use_check_ignore:
 479                             self._file_paths.add(path)
 480                         else:
 481                             self._force_notify = True
 482             for _, path in self._git_dir_watch.read():
 483                 if not self._running:
 484                     break
 485                 if self._force_notify:
 486                     continue
 487                 path = self._transform_path(path)
 488                 if path.endswith('.lock'):
 489                     continue
 490                 if path == 'config':
 491                     self._force_config = True
 492                     continue
 493                 if (path == 'head'
 494                         or path == 'index'
 495                         or path.startswith('refs/')):
 496                     self._force_notify = True
 497
 498         def stop(self):
 499             self._running = False
 500             with self._stop_event_lock:
 501                 if self._stop_event is not None:
 502                     win32event.SetEvent(self._stop_event)
 503             self.wait()
 504
 505
 506 def create(context):
 507     thread_class = None
 508     cfg = context.cfg
 509     if not cfg.get('cola.inotify', default=True):
 510         msg = N_('File system change monitoring: disabled because'
 511                  ' "cola.inotify" is false.\n')
 512         Interaction.log(msg)
 513     elif AVAILABLE == 'inotify':
 514         thread_class = _InotifyThread
 515     elif AVAILABLE == 'pywin32':
 516         thread_class = _Win32Thread
 517     else:
 518         if utils.is_win32():
 519             msg = N_('File system change monitoring: disabled because pywin32'
 520                      ' is not installed.\n')
 521             Interaction.log(msg)
 522         elif utils.is_linux():
 523             msg = N_('File system change monitoring: disabled because libc'
 524                      ' does not support the inotify system calls.\n')
 525             Interaction.log(msg)
 526     return _Monitor(context, thread_class)