cola/fsmonitor.py

   1 # Copyright (C) 2008-2017 David Aguilar
   2 # Copyright (C) 2015 Daniel Harding
   3 """Filesystem monitor for Linux and Windows
   4
   5 Linux monitoring uses using inotify.
   6 Windows monitoring uses pywin32 and the ReadDirectoryChanges function.
   7
   8 """
   9 import errno
  10 import os
  11 import os.path
  12 import select
  13 from threading import Lock
  14
  15 from qtpy import QtCore
  16 from qtpy.QtCore import Signal
  17
  18 from . import utils
  19 from . import core
  20 from . import gitcmds
  21 from . import version
  22 from .compat import bchr
  23 from .i18n import N_
  24 from .interaction import Interaction
  25
  26 AVAILABLE = None
  27
  28 if utils.is_win32():
  29     try:
  30         import pywintypes
  31         import win32con
  32         import win32event
  33         import win32file
  34     except ImportError:
  35         pass
  36     else:
  37         AVAILABLE = 'pywin32'
  38 elif utils.is_linux():
  39     try:
  40         from . import inotify
  41     except ImportError:
  42         pass
  43     else:
  44         AVAILABLE = 'inotify'
  45
  46
  47 class _Monitor(QtCore.QObject):
  48     files_changed = Signal()
  49     config_changed = Signal()
  50
  51     def __init__(self, context, thread_class):
  52         QtCore.QObject.__init__(self)
  53         self.context = context
  54         self._thread_class = thread_class
  55         self._thread = None
  56
  57     def start(self):
  58         if self._thread_class is not None:
  59             assert self._thread is None
  60             self._thread = self._thread_class(self.context, self)
  61             self._thread.start()
  62
  63     def stop(self):
  64         if self._thread_class is not None:
  65             assert self._thread is not None
  66             self._thread.stop()
  67             self._thread.wait()
  68             self._thread = None
  69
  70     def refresh(self):
  71         if self._thread is not None:
  72             self._thread.refresh()
  73
  74
  75 class _BaseThread(QtCore.QThread):
  76     #: The delay, in milliseconds, between detecting file system modification
  77     #: and triggering the 'files_changed' signal, to coalesce multiple
  78     #: modifications into a single signal.
  79     _NOTIFICATION_DELAY = 888
  80
  81     def __init__(self, context, monitor):
  82         QtCore.QThread.__init__(self)
  83         self.context = context
  84         self._monitor = monitor
  85         self._running = True
  86         self._use_check_ignore = version.check_git(context, 'check-ignore')
  87         self._force_notify = False
  88         self._force_config = False
  89         self._file_paths = set()
  90
  91     @property
  92     def _pending(self):
  93         return self._force_notify or self._file_paths or self._force_config
  94
  95     def refresh(self):
  96         """Do any housekeeping necessary in response to repository changes."""
  97         return
  98
  99     def notify(self):
 100         """Notifies all observers"""
 101         do_notify = False
 102         do_config = False
 103         if self._force_config:
 104             do_config = True
 105         if self._force_notify:
 106             do_notify = True
 107         elif self._file_paths:
 108             proc = core.start_command(
 109                 ['git', 'check-ignore', '--verbose', '--non-matching', '-z', '--stdin']
 110             )
 111             path_list = bchr(0).join(core.encode(path) for path in self._file_paths)
 112             out, _ = proc.communicate(path_list)
 113             if proc.returncode:
 114                 do_notify = True
 115             else:
 116                 # Each output record is four fields separated by NULL
 117                 # characters (records are also separated by NULL characters):
 118                 # <source> <NULL> <linenum> <NULL> <pattern> <NULL> <pathname>
 119                 # For paths which are not ignored, all fields will be empty
 120                 # except for <pathname>.  So to see if we have any non-ignored
 121                 # files, we simply check every fourth field to see if any of
 122                 # them are empty.
 123                 source_fields = out.split(bchr(0))[0:-1:4]
 124                 do_notify = not all(source_fields)
 125         self._force_notify = False
 126         self._force_config = False
 127         self._file_paths = set()
 128
 129         # "files changed" is a bigger hammer than "config changed".
 130         # and is a superset relative to what is done in response to the
 131         # signal.  Thus, the "elif" below avoids repeated work that
 132         # would be done if it were a simple "if" check instead.
 133         if do_notify:
 134             self._monitor.files_changed.emit()
 135         elif do_config:
 136             self._monitor.config_changed.emit()
 137
 138     @staticmethod
 139     def _log_enabled_message():
 140         msg = N_('File system change monitoring: enabled.\n')
 141         Interaction.log(msg)
 142
 143
 144 if AVAILABLE == 'inotify':
 145
 146     class _InotifyThread(_BaseThread):
 147         _TRIGGER_MASK = (
 148             inotify.IN_ATTRIB
 149             | inotify.IN_CLOSE_WRITE
 150             | inotify.IN_CREATE
 151             | inotify.IN_DELETE
 152             | inotify.IN_MODIFY
 153             | inotify.IN_MOVED_FROM
 154             | inotify.IN_MOVED_TO
 155         )
 156         _ADD_MASK = _TRIGGER_MASK | inotify.IN_EXCL_UNLINK | inotify.IN_ONLYDIR
 157
 158         def __init__(self, context, monitor):
 159             _BaseThread.__init__(self, context, monitor)
 160             git = context.git
 161             worktree = git.worktree()
 162             if worktree is not None:
 163                 worktree = core.abspath(worktree)
 164             self._worktree = worktree
 165             self._git_dir = git.git_path()
 166             self._lock = Lock()
 167             self._inotify_fd = None
 168             self._pipe_r = None
 169             self._pipe_w = None
 170             self._worktree_wd_to_path_map = {}
 171             self._worktree_path_to_wd_map = {}
 172             self._git_dir_wd_to_path_map = {}
 173             self._git_dir_path_to_wd_map = {}
 174             self._git_dir_wd = None
 175
 176         @staticmethod
 177         def _log_out_of_wds_message():
 178             msg = N_(
 179                 'File system change monitoring: disabled because the'
 180                 ' limit on the total number of inotify watches was'
 181                 ' reached.  You may be able to increase the limit on'
 182                 ' the number of watches by running:\n'
 183                 '\n'
 184                 '    echo fs.inotify.max_user_watches=100000 |'
 185                 ' sudo tee -a /etc/sysctl.conf &&'
 186                 ' sudo sysctl -p\n'
 187             )
 188             Interaction.log(msg)
 189
 190         def run(self):
 191             try:
 192                 with self._lock:
 193                     try:
 194                         self._inotify_fd = inotify.init()
 195                     except OSError as e:
 196                         self._inotify_fd = None
 197                         self._running = False
 198                         if e.errno == errno.EMFILE:
 199                             self._log_out_of_wds_message()
 200                         return
 201                     self._pipe_r, self._pipe_w = os.pipe()
 202
 203                 # pylint: disable=no-member
 204                 poll_obj = select.poll()
 205                 poll_obj.register(self._inotify_fd, select.POLLIN)
 206                 poll_obj.register(self._pipe_r, select.POLLIN)
 207
 208                 self.refresh()
 209
 210                 if self._running:
 211                     self._log_enabled_message()
 212                     self._process_events(poll_obj)
 213             finally:
 214                 self._close_fds()
 215
 216         def _process_events(self, poll_obj):
 217             while self._running:
 218                 if self._pending:
 219                     timeout = self._NOTIFICATION_DELAY
 220                 else:
 221                     timeout = None
 222                 try:
 223                     events = poll_obj.poll(timeout)
 224                 # pylint: disable=duplicate-except
 225                 except OSError:
 226                     continue
 227                 else:
 228                     if not self._running:
 229                         break
 230                     if not events:
 231                         self.notify()
 232                     else:
 233                         for fd, _ in events:
 234                             if fd == self._inotify_fd:
 235                                 self._handle_events()
 236
 237         def _close_fds(self):
 238             with self._lock:
 239                 if self._inotify_fd is not None:
 240                     os.close(self._inotify_fd)
 241                     self._inotify_fd = None
 242                 if self._pipe_r is not None:
 243                     os.close(self._pipe_r)
 244                     self._pipe_r = None
 245                     os.close(self._pipe_w)
 246                     self._pipe_w = None
 247
 248         def refresh(self):
 249             with self._lock:
 250                 self._refresh()
 251
 252         def _refresh(self):
 253             if self._inotify_fd is None:
 254                 return
 255             context = self.context
 256             try:
 257                 if self._worktree is not None:
 258                     tracked_dirs = {
 259                         os.path.dirname(os.path.join(self._worktree, path))
 260                         for path in gitcmds.tracked_files(context)
 261                     }
 262                     self._refresh_watches(
 263                         tracked_dirs,
 264                         self._worktree_wd_to_path_map,
 265                         self._worktree_path_to_wd_map,
 266                     )
 267                 git_dirs = set()
 268                 git_dirs.add(self._git_dir)
 269                 for dirpath, _, _ in core.walk(os.path.join(self._git_dir, 'refs')):
 270                     git_dirs.add(dirpath)
 271                 self._refresh_watches(
 272                     git_dirs, self._git_dir_wd_to_path_map, self._git_dir_path_to_wd_map
 273                 )
 274                 self._git_dir_wd = self._git_dir_path_to_wd_map.get(self._git_dir)
 275             except OSError as e:
 276                 if e.errno in (errno.ENOSPC, errno.EMFILE):
 277                     self._log_out_of_wds_message()
 278                     self._running = False
 279                 else:
 280                     raise
 281
 282         def _refresh_watches(self, paths_to_watch, wd_to_path_map, path_to_wd_map):
 283             watched_paths = set(path_to_wd_map)
 284             for path in watched_paths - paths_to_watch:
 285                 wd = path_to_wd_map.pop(path)
 286                 wd_to_path_map.pop(wd)
 287                 try:
 288                     inotify.rm_watch(self._inotify_fd, wd)
 289                 except OSError as e:
 290                     if e.errno == errno.EINVAL:
 291                         # This error can occur if the target of the wd was
 292                         # removed on the filesystem before we call
 293                         # inotify.rm_watch() so ignore it.
 294                         continue
 295                     raise e
 296             for path in paths_to_watch - watched_paths:
 297                 try:
 298                     wd = inotify.add_watch(
 299                         self._inotify_fd, core.encode(path), self._ADD_MASK
 300                     )
 301                 except OSError as e:
 302                     if e.errno in (errno.ENOENT, errno.ENOTDIR):
 303                         # These two errors should only occur as a result of
 304                         # race conditions:  the first if the directory
 305                         # referenced by path was removed or renamed before the
 306                         # call to inotify.add_watch(); the second if the
 307                         # directory referenced by path was replaced with a file
 308                         # before the call to inotify.add_watch().  Therefore we
 309                         # simply ignore them.
 310                         continue
 311                     raise e
 312                 wd_to_path_map[wd] = path
 313                 path_to_wd_map[path] = wd
 314
 315         def _check_event(self, wd, mask, name):
 316             if mask & inotify.IN_Q_OVERFLOW:
 317                 self._force_notify = True
 318             elif not mask & self._TRIGGER_MASK:
 319                 pass
 320             elif mask & inotify.IN_ISDIR:
 321                 pass
 322             elif wd in self._worktree_wd_to_path_map:
 323                 if self._use_check_ignore and name:
 324                     path = os.path.join(
 325                         self._worktree_wd_to_path_map[wd], core.decode(name)
 326                     )
 327                     self._file_paths.add(path)
 328                 else:
 329                     self._force_notify = True
 330             elif wd == self._git_dir_wd:
 331                 name = core.decode(name)
 332                 if name in ('HEAD', 'index'):
 333                     self._force_notify = True
 334                 elif name == 'config':
 335                     self._force_config = True
 336             elif wd in self._git_dir_wd_to_path_map and not core.decode(name).endswith(
 337                 '.lock'
 338             ):
 339                 self._force_notify = True
 340
 341         def _handle_events(self):
 342             for wd, mask, _, name in inotify.read_events(self._inotify_fd):
 343                 if not self._force_notify:
 344                     self._check_event(wd, mask, name)
 345
 346         def stop(self):
 347             self._running = False
 348             with self._lock:
 349                 if self._pipe_w is not None:
 350                     os.write(self._pipe_w, bchr(0))
 351             self.wait()
 352
 353
 354 if AVAILABLE == 'pywin32':
 355
 356     class _Win32Watch:
 357         def __init__(self, path, flags):
 358             self.flags = flags
 359
 360             self.handle = None
 361             self.event = None
 362
 363             try:
 364                 self.handle = win32file.CreateFileW(
 365                     path,
 366                     0x0001,  # FILE_LIST_DIRECTORY
 367                     win32con.FILE_SHARE_READ | win32con.FILE_SHARE_WRITE,
 368                     None,
 369                     win32con.OPEN_EXISTING,
 370                     win32con.FILE_FLAG_BACKUP_SEMANTICS | win32con.FILE_FLAG_OVERLAPPED,
 371                     None,
 372                 )
 373
 374                 self.buffer = win32file.AllocateReadBuffer(8192)
 375                 self.event = win32event.CreateEvent(None, True, False, None)
 376                 self.overlapped = pywintypes.OVERLAPPED()
 377                 self.overlapped.hEvent = self.event
 378                 self._start()
 379             except Exception:
 380                 self.close()
 381                 raise
 382
 383         def _start(self):
 384             win32file.ReadDirectoryChangesW(
 385                 self.handle, self.buffer, True, self.flags, self.overlapped
 386             )
 387
 388         def read(self):
 389             if win32event.WaitForSingleObject(self.event, 0) == win32event.WAIT_TIMEOUT:
 390                 result = []
 391             else:
 392                 nbytes = win32file.GetOverlappedResult(
 393                     self.handle, self.overlapped, False
 394                 )
 395                 result = win32file.FILE_NOTIFY_INFORMATION(self.buffer, nbytes)
 396                 self._start()
 397             return result
 398
 399         def close(self):
 400             if self.handle is not None:
 401                 win32file.CancelIo(self.handle)
 402                 win32file.CloseHandle(self.handle)
 403             if self.event is not None:
 404                 win32file.CloseHandle(self.event)
 405
 406     class _Win32Thread(_BaseThread):
 407         _FLAGS = (
 408             win32con.FILE_NOTIFY_CHANGE_FILE_NAME
 409             | win32con.FILE_NOTIFY_CHANGE_DIR_NAME
 410             | win32con.FILE_NOTIFY_CHANGE_ATTRIBUTES
 411             | win32con.FILE_NOTIFY_CHANGE_SIZE
 412             | win32con.FILE_NOTIFY_CHANGE_LAST_WRITE
 413             | win32con.FILE_NOTIFY_CHANGE_SECURITY
 414         )
 415
 416         def __init__(self, context, monitor):
 417             _BaseThread.__init__(self, context, monitor)
 418             git = context.git
 419             worktree = git.worktree()
 420             if worktree is not None:
 421                 worktree = self._transform_path(core.abspath(worktree))
 422             self._worktree = worktree
 423             self._worktree_watch = None
 424             self._git_dir = self._transform_path(core.abspath(git.git_path()))
 425             self._git_dir_watch = None
 426             self._stop_event_lock = Lock()
 427             self._stop_event = None
 428
 429         @staticmethod
 430         def _transform_path(path):
 431             return path.replace('\\', '/').lower()
 432
 433         def run(self):
 434             try:
 435                 with self._stop_event_lock:
 436                     self._stop_event = win32event.CreateEvent(None, True, False, None)
 437
 438                 events = [self._stop_event]
 439
 440                 if self._worktree is not None:
 441                     self._worktree_watch = _Win32Watch(self._worktree, self._FLAGS)
 442                     events.append(self._worktree_watch.event)
 443
 444                 self._git_dir_watch = _Win32Watch(self._git_dir, self._FLAGS)
 445                 events.append(self._git_dir_watch.event)
 446
 447                 self._log_enabled_message()
 448
 449                 while self._running:
 450                     if self._pending:
 451                         timeout = self._NOTIFICATION_DELAY
 452                     else:
 453                         timeout = win32event.INFINITE
 454                     status = win32event.WaitForMultipleObjects(events, False, timeout)
 455                     if not self._running:
 456                         break
 457                     if status == win32event.WAIT_TIMEOUT:
 458                         self.notify()
 459                     else:
 460                         self._handle_results()
 461             finally:
 462                 with self._stop_event_lock:
 463                     if self._stop_event is not None:
 464                         win32file.CloseHandle(self._stop_event)
 465                         self._stop_event = None
 466                 if self._worktree_watch is not None:
 467                     self._worktree_watch.close()
 468                 if self._git_dir_watch is not None:
 469                     self._git_dir_watch.close()
 470
 471         def _handle_results(self):
 472             if self._worktree_watch is not None:
 473                 for _, path in self._worktree_watch.read():
 474                     if not self._running:
 475                         break
 476                     if self._force_notify:
 477                         continue
 478                     path = self._worktree + '/' + self._transform_path(path)
 479                     if (
 480                         path != self._git_dir
 481                         and not path.startswith(self._git_dir + '/')
 482                         and not os.path.isdir(path)
 483                     ):
 484                         if self._use_check_ignore:
 485                             self._file_paths.add(path)
 486                         else:
 487                             self._force_notify = True
 488             for _, path in self._git_dir_watch.read():
 489                 if not self._running:
 490                     break
 491                 if self._force_notify:
 492                     continue
 493                 path = self._transform_path(path)
 494                 if path.endswith('.lock'):
 495                     continue
 496                 if path == 'config':
 497                     self._force_config = True
 498                     continue
 499                 if path == 'head' or path == 'index' or path.startswith('refs/'):
 500                     self._force_notify = True
 501
 502         def stop(self):
 503             self._running = False
 504             with self._stop_event_lock:
 505                 if self._stop_event is not None:
 506                     win32event.SetEvent(self._stop_event)
 507             self.wait()
 508
 509
 510 def create(context):
 511     thread_class = None
 512     cfg = context.cfg
 513     if not cfg.get('cola.inotify', default=True):
 514         msg = N_(
 515             'File system change monitoring: disabled because'
 516             ' "cola.inotify" is false.\n'
 517         )
 518         Interaction.log(msg)
 519     elif AVAILABLE == 'inotify':
 520         thread_class = _InotifyThread
 521     elif AVAILABLE == 'pywin32':
 522         thread_class = _Win32Thread
 523     else:
 524         if utils.is_win32():
 525             msg = N_(
 526                 'File system change monitoring: disabled because pywin32'
 527                 ' is not installed.\n'
 528             )
 529             Interaction.log(msg)
 530         elif utils.is_linux():
 531             msg = N_(
 532                 'File system change monitoring: disabled because libc'
 533                 ' does not support the inotify system calls.\n'
 534             )
 535             Interaction.log(msg)
 536     return _Monitor(context, thread_class)