cola/fsmonitor.py

   1 # Copyright (C) 2008-2017 David Aguilar
   2 # Copyright (C) 2015 Daniel Harding
   3 """Filesystem monitor for Linux and Windows
   4
   5 Linux monitoring uses using inotify.
   6 Windows monitoring uses pywin32 and the ReadDirectoryChanges function.
   7
   8 """
   9 from __future__ import division, absolute_import, unicode_literals
  10 import errno
  11 import os
  12 import os.path
  13 import select
  14 from threading import Lock
  15
  16 from qtpy import QtCore
  17 from qtpy.QtCore import Signal
  18
  19 from . import utils
  20 from . import core
  21 from . import gitcmds
  22 from . import version
  23 from .compat import bchr
  24 from .i18n import N_
  25 from .interaction import Interaction
  26
  27 AVAILABLE = None
  28
  29 if utils.is_win32():
  30     try:
  31         import pywintypes
  32         import win32con
  33         import win32event
  34         import win32file
  35     except ImportError:
  36         pass
  37     else:
  38         AVAILABLE = 'pywin32'
  39 elif utils.is_linux():
  40     try:
  41         from . import inotify
  42     except ImportError:
  43         pass
  44     else:
  45         AVAILABLE = 'inotify'
  46
  47
  48 class _Monitor(QtCore.QObject):
  49
  50     files_changed = Signal()
  51     config_changed = Signal()
  52
  53     def __init__(self, context, thread_class):
  54         QtCore.QObject.__init__(self)
  55         self.context = context
  56         self._thread_class = thread_class
  57         self._thread = None
  58
  59     def start(self):
  60         if self._thread_class is not None:
  61             assert self._thread is None
  62             self._thread = self._thread_class(self.context, self)
  63             self._thread.start()
  64
  65     def stop(self):
  66         if self._thread_class is not None:
  67             assert self._thread is not None
  68             self._thread.stop()
  69             self._thread.wait()
  70             self._thread = None
  71
  72     def refresh(self):
  73         if self._thread is not None:
  74             self._thread.refresh()
  75
  76
  77 class _BaseThread(QtCore.QThread):
  78     #: The delay, in milliseconds, between detecting file system modification
  79     #: and triggering the 'files_changed' signal, to coalesce multiple
  80     #: modifications into a single signal.
  81     _NOTIFICATION_DELAY = 888
  82
  83     def __init__(self, context, monitor):
  84         QtCore.QThread.__init__(self)
  85         self.context = context
  86         self._monitor = monitor
  87         self._running = True
  88         self._use_check_ignore = version.check_git(context, 'check-ignore')
  89         self._force_notify = False
  90         self._force_config = False
  91         self._file_paths = set()
  92
  93     @property
  94     def _pending(self):
  95         return self._force_notify or self._file_paths or self._force_config
  96
  97     # pylint: disable=no-self-use
  98     def refresh(self):
  99         """Do any housekeeping necessary in response to repository changes."""
 100         return
 101
 102     def notify(self):
 103         """Notifies all observers"""
 104         do_notify = False
 105         do_config = False
 106         if self._force_config:
 107             do_config = True
 108         if self._force_notify:
 109             do_notify = True
 110         elif self._file_paths:
 111             proc = core.start_command(
 112                 ['git', 'check-ignore', '--verbose', '--non-matching', '-z', '--stdin']
 113             )
 114             path_list = bchr(0).join(core.encode(path) for path in self._file_paths)
 115             out, _ = proc.communicate(path_list)
 116             if proc.returncode:
 117                 do_notify = True
 118             else:
 119                 # Each output record is four fields separated by NULL
 120                 # characters (records are also separated by NULL characters):
 121                 # <source> <NULL> <linenum> <NULL> <pattern> <NULL> <pathname>
 122                 # For paths which are not ignored, all fields will be empty
 123                 # except for <pathname>.  So to see if we have any non-ignored
 124                 # files, we simply check every fourth field to see if any of
 125                 # them are empty.
 126                 source_fields = out.split(bchr(0))[0:-1:4]
 127                 do_notify = not all(source_fields)
 128         self._force_notify = False
 129         self._force_config = False
 130         self._file_paths = set()
 131
 132         # "files changed" is a bigger hammer than "config changed".
 133         # and is a superset relative to what is done in response to the
 134         # signal.  Thus, the "elif" below avoids repeated work that
 135         # would be done if it were a simple "if" check instead.
 136         if do_notify:
 137             self._monitor.files_changed.emit()
 138         elif do_config:
 139             self._monitor.config_changed.emit()
 140
 141     @staticmethod
 142     def _log_enabled_message():
 143         msg = N_('File system change monitoring: enabled.\n')
 144         Interaction.log(msg)
 145
 146
 147 if AVAILABLE == 'inotify':
 148
 149     class _InotifyThread(_BaseThread):
 150         _TRIGGER_MASK = (
 151             inotify.IN_ATTRIB
 152             | inotify.IN_CLOSE_WRITE
 153             | inotify.IN_CREATE
 154             | inotify.IN_DELETE
 155             | inotify.IN_MODIFY
 156             | inotify.IN_MOVED_FROM
 157             | inotify.IN_MOVED_TO
 158         )
 159         _ADD_MASK = _TRIGGER_MASK | inotify.IN_EXCL_UNLINK | inotify.IN_ONLYDIR
 160
 161         def __init__(self, context, monitor):
 162             _BaseThread.__init__(self, context, monitor)
 163             git = context.git
 164             worktree = git.worktree()
 165             if worktree is not None:
 166                 worktree = core.abspath(worktree)
 167             self._worktree = worktree
 168             self._git_dir = git.git_path()
 169             self._lock = Lock()
 170             self._inotify_fd = None
 171             self._pipe_r = None
 172             self._pipe_w = None
 173             self._worktree_wd_to_path_map = {}
 174             self._worktree_path_to_wd_map = {}
 175             self._git_dir_wd_to_path_map = {}
 176             self._git_dir_path_to_wd_map = {}
 177             self._git_dir_wd = None
 178
 179         @staticmethod
 180         def _log_out_of_wds_message():
 181             msg = N_(
 182                 'File system change monitoring: disabled because the'
 183                 ' limit on the total number of inotify watches was'
 184                 ' reached.  You may be able to increase the limit on'
 185                 ' the number of watches by running:\n'
 186                 '\n'
 187                 '    echo fs.inotify.max_user_watches=100000 |'
 188                 ' sudo tee -a /etc/sysctl.conf &&'
 189                 ' sudo sysctl -p\n'
 190             )
 191             Interaction.log(msg)
 192
 193         def run(self):
 194             try:
 195                 with self._lock:
 196                     try:
 197                         self._inotify_fd = inotify.init()
 198                     except OSError as e:
 199                         self._inotify_fd = None
 200                         self._running = False
 201                         if e.errno == errno.EMFILE:
 202                             self._log_out_of_wds_message()
 203                         return
 204                     self._pipe_r, self._pipe_w = os.pipe()
 205
 206                 # pylint: disable=no-member
 207                 poll_obj = select.poll()
 208                 poll_obj.register(self._inotify_fd, select.POLLIN)
 209                 poll_obj.register(self._pipe_r, select.POLLIN)
 210
 211                 self.refresh()
 212
 213                 if self._running:
 214                     self._log_enabled_message()
 215                     self._process_events(poll_obj)
 216             finally:
 217                 self._close_fds()
 218
 219         def _process_events(self, poll_obj):
 220             while self._running:
 221                 if self._pending:
 222                     timeout = self._NOTIFICATION_DELAY
 223                 else:
 224                     timeout = None
 225                 try:
 226                     events = poll_obj.poll(timeout)
 227                 # pylint: disable=duplicate-except
 228                 except (OSError, select.error):
 229                     continue
 230                 else:
 231                     if not self._running:
 232                         break
 233                     if not events:
 234                         self.notify()
 235                     else:
 236                         for (fd, _) in events:
 237                             if fd == self._inotify_fd:
 238                                 self._handle_events()
 239
 240         def _close_fds(self):
 241             with self._lock:
 242                 if self._inotify_fd is not None:
 243                     os.close(self._inotify_fd)
 244                     self._inotify_fd = None
 245                 if self._pipe_r is not None:
 246                     os.close(self._pipe_r)
 247                     self._pipe_r = None
 248                     os.close(self._pipe_w)
 249                     self._pipe_w = None
 250
 251         def refresh(self):
 252             with self._lock:
 253                 self._refresh()
 254
 255         def _refresh(self):
 256             if self._inotify_fd is None:
 257                 return
 258             context = self.context
 259             try:
 260                 if self._worktree is not None:
 261                     tracked_dirs = set(
 262                         [
 263                             os.path.dirname(os.path.join(self._worktree, path))
 264                             for path in gitcmds.tracked_files(context)
 265                         ]
 266                     )
 267                     self._refresh_watches(
 268                         tracked_dirs,
 269                         self._worktree_wd_to_path_map,
 270                         self._worktree_path_to_wd_map,
 271                     )
 272                 git_dirs = set()
 273                 git_dirs.add(self._git_dir)
 274                 for dirpath, _, _ in core.walk(os.path.join(self._git_dir, 'refs')):
 275                     git_dirs.add(dirpath)
 276                 self._refresh_watches(
 277                     git_dirs, self._git_dir_wd_to_path_map, self._git_dir_path_to_wd_map
 278                 )
 279                 self._git_dir_wd = self._git_dir_path_to_wd_map.get(self._git_dir)
 280             except OSError as e:
 281                 if e.errno in (errno.ENOSPC, errno.EMFILE):
 282                     self._log_out_of_wds_message()
 283                     self._running = False
 284                 else:
 285                     raise
 286
 287         def _refresh_watches(self, paths_to_watch, wd_to_path_map, path_to_wd_map):
 288             watched_paths = set(path_to_wd_map)
 289             for path in watched_paths - paths_to_watch:
 290                 wd = path_to_wd_map.pop(path)
 291                 wd_to_path_map.pop(wd)
 292                 try:
 293                     inotify.rm_watch(self._inotify_fd, wd)
 294                 except OSError as e:
 295                     if e.errno == errno.EINVAL:
 296                         # This error can occur if the target of the wd was
 297                         # removed on the filesystem before we call
 298                         # inotify.rm_watch() so ignore it.
 299                         continue
 300                     raise e
 301             for path in paths_to_watch - watched_paths:
 302                 try:
 303                     wd = inotify.add_watch(
 304                         self._inotify_fd, core.encode(path), self._ADD_MASK
 305                     )
 306                 except OSError as e:
 307                     if e.errno in (errno.ENOENT, errno.ENOTDIR):
 308                         # These two errors should only occur as a result of
 309                         # race conditions:  the first if the directory
 310                         # referenced by path was removed or renamed before the
 311                         # call to inotify.add_watch(); the second if the
 312                         # directory referenced by path was replaced with a file
 313                         # before the call to inotify.add_watch().  Therefore we
 314                         # simply ignore them.
 315                         continue
 316                     raise e
 317                 else:
 318                     wd_to_path_map[wd] = path
 319                     path_to_wd_map[path] = wd
 320
 321         def _check_event(self, wd, mask, name):
 322             if mask & inotify.IN_Q_OVERFLOW:
 323                 self._force_notify = True
 324             elif not mask & self._TRIGGER_MASK:
 325                 pass
 326             elif mask & inotify.IN_ISDIR:
 327                 pass
 328             elif wd in self._worktree_wd_to_path_map:
 329                 if self._use_check_ignore and name:
 330                     path = os.path.join(
 331                         self._worktree_wd_to_path_map[wd], core.decode(name)
 332                     )
 333                     self._file_paths.add(path)
 334                 else:
 335                     self._force_notify = True
 336             elif wd == self._git_dir_wd:
 337                 name = core.decode(name)
 338                 if name in ('HEAD', 'index'):
 339                     self._force_notify = True
 340                 elif name == 'config':
 341                     self._force_config = True
 342             elif wd in self._git_dir_wd_to_path_map and not core.decode(name).endswith(
 343                 '.lock'
 344             ):
 345                 self._force_notify = True
 346
 347         def _handle_events(self):
 348             for wd, mask, _, name in inotify.read_events(self._inotify_fd):
 349                 if not self._force_notify:
 350                     self._check_event(wd, mask, name)
 351
 352         def stop(self):
 353             self._running = False
 354             with self._lock:
 355                 if self._pipe_w is not None:
 356                     os.write(self._pipe_w, bchr(0))
 357             self.wait()
 358
 359
 360 if AVAILABLE == 'pywin32':
 361
 362     class _Win32Watch(object):
 363         def __init__(self, path, flags):
 364             self.flags = flags
 365
 366             self.handle = None
 367             self.event = None
 368
 369             try:
 370                 self.handle = win32file.CreateFileW(
 371                     path,
 372                     0x0001,  # FILE_LIST_DIRECTORY
 373                     win32con.FILE_SHARE_READ | win32con.FILE_SHARE_WRITE,
 374                     None,
 375                     win32con.OPEN_EXISTING,
 376                     win32con.FILE_FLAG_BACKUP_SEMANTICS | win32con.FILE_FLAG_OVERLAPPED,
 377                     None,
 378                 )
 379
 380                 self.buffer = win32file.AllocateReadBuffer(8192)
 381                 self.event = win32event.CreateEvent(None, True, False, None)
 382                 self.overlapped = pywintypes.OVERLAPPED()
 383                 self.overlapped.hEvent = self.event
 384                 self._start()
 385             except Exception:
 386                 self.close()
 387                 raise
 388
 389         def _start(self):
 390             win32file.ReadDirectoryChangesW(
 391                 self.handle, self.buffer, True, self.flags, self.overlapped
 392             )
 393
 394         def read(self):
 395             if win32event.WaitForSingleObject(self.event, 0) == win32event.WAIT_TIMEOUT:
 396                 result = []
 397             else:
 398                 nbytes = win32file.GetOverlappedResult(
 399                     self.handle, self.overlapped, False
 400                 )
 401                 result = win32file.FILE_NOTIFY_INFORMATION(self.buffer, nbytes)
 402                 self._start()
 403             return result
 404
 405         def close(self):
 406             if self.handle is not None:
 407                 win32file.CancelIo(self.handle)
 408                 win32file.CloseHandle(self.handle)
 409             if self.event is not None:
 410                 win32file.CloseHandle(self.event)
 411
 412     class _Win32Thread(_BaseThread):
 413         _FLAGS = (
 414             win32con.FILE_NOTIFY_CHANGE_FILE_NAME
 415             | win32con.FILE_NOTIFY_CHANGE_DIR_NAME
 416             | win32con.FILE_NOTIFY_CHANGE_ATTRIBUTES
 417             | win32con.FILE_NOTIFY_CHANGE_SIZE
 418             | win32con.FILE_NOTIFY_CHANGE_LAST_WRITE
 419             | win32con.FILE_NOTIFY_CHANGE_SECURITY
 420         )
 421
 422         def __init__(self, context, monitor):
 423             _BaseThread.__init__(self, context, monitor)
 424             git = context.git
 425             worktree = git.worktree()
 426             if worktree is not None:
 427                 worktree = self._transform_path(core.abspath(worktree))
 428             self._worktree = worktree
 429             self._worktree_watch = None
 430             self._git_dir = self._transform_path(core.abspath(git.git_path()))
 431             self._git_dir_watch = None
 432             self._stop_event_lock = Lock()
 433             self._stop_event = None
 434
 435         @staticmethod
 436         def _transform_path(path):
 437             return path.replace('\\', '/').lower()
 438
 439         def run(self):
 440             try:
 441                 with self._stop_event_lock:
 442                     self._stop_event = win32event.CreateEvent(None, True, False, None)
 443
 444                 events = [self._stop_event]
 445
 446                 if self._worktree is not None:
 447                     self._worktree_watch = _Win32Watch(self._worktree, self._FLAGS)
 448                     events.append(self._worktree_watch.event)
 449
 450                 self._git_dir_watch = _Win32Watch(self._git_dir, self._FLAGS)
 451                 events.append(self._git_dir_watch.event)
 452
 453                 self._log_enabled_message()
 454
 455                 while self._running:
 456                     if self._pending:
 457                         timeout = self._NOTIFICATION_DELAY
 458                     else:
 459                         timeout = win32event.INFINITE
 460                     rc = win32event.WaitForMultipleObjects(events, False, timeout)
 461                     if not self._running:
 462                         break
 463                     if rc == win32event.WAIT_TIMEOUT:
 464                         self.notify()
 465                     else:
 466                         self._handle_results()
 467             finally:
 468                 with self._stop_event_lock:
 469                     if self._stop_event is not None:
 470                         win32file.CloseHandle(self._stop_event)
 471                         self._stop_event = None
 472                 if self._worktree_watch is not None:
 473                     self._worktree_watch.close()
 474                 if self._git_dir_watch is not None:
 475                     self._git_dir_watch.close()
 476
 477         def _handle_results(self):
 478             if self._worktree_watch is not None:
 479                 for _, path in self._worktree_watch.read():
 480                     if not self._running:
 481                         break
 482                     if self._force_notify:
 483                         continue
 484                     path = self._worktree + '/' + self._transform_path(path)
 485                     if (
 486                         path != self._git_dir
 487                         and not path.startswith(self._git_dir + '/')
 488                         and not os.path.isdir(path)
 489                     ):
 490                         if self._use_check_ignore:
 491                             self._file_paths.add(path)
 492                         else:
 493                             self._force_notify = True
 494             for _, path in self._git_dir_watch.read():
 495                 if not self._running:
 496                     break
 497                 if self._force_notify:
 498                     continue
 499                 path = self._transform_path(path)
 500                 if path.endswith('.lock'):
 501                     continue
 502                 if path == 'config':
 503                     self._force_config = True
 504                     continue
 505                 if path == 'head' or path == 'index' or path.startswith('refs/'):
 506                     self._force_notify = True
 507
 508         def stop(self):
 509             self._running = False
 510             with self._stop_event_lock:
 511                 if self._stop_event is not None:
 512                     win32event.SetEvent(self._stop_event)
 513             self.wait()
 514
 515
 516 def create(context):
 517     thread_class = None
 518     cfg = context.cfg
 519     if not cfg.get('cola.inotify', default=True):
 520         msg = N_(
 521             'File system change monitoring: disabled because'
 522             ' "cola.inotify" is false.\n'
 523         )
 524         Interaction.log(msg)
 525     elif AVAILABLE == 'inotify':
 526         thread_class = _InotifyThread
 527     elif AVAILABLE == 'pywin32':
 528         thread_class = _Win32Thread
 529     else:
 530         if utils.is_win32():
 531             msg = N_(
 532                 'File system change monitoring: disabled because pywin32'
 533                 ' is not installed.\n'
 534             )
 535             Interaction.log(msg)
 536         elif utils.is_linux():
 537             msg = N_(
 538                 'File system change monitoring: disabled because libc'
 539                 ' does not support the inotify system calls.\n'
 540             )
 541             Interaction.log(msg)
 542     return _Monitor(context, thread_class)