cola/fsmonitor.py

   1 # Copyright (C) 2008-2017 David Aguilar
   2 # Copyright (C) 2015 Daniel Harding
   3 """Filesystem monitor for Linux and Windows
   4
   5 Linux monitoring uses using inotify.
   6 Windows monitoring uses pywin32 and the ReadDirectoryChanges function.
   7
   8 """
   9 from __future__ import absolute_import, division, print_function, unicode_literals
  10 import errno
  11 import os
  12 import os.path
  13 import select
  14 from threading import Lock
  15
  16 from qtpy import QtCore
  17 from qtpy.QtCore import Signal
  18
  19 from . import utils
  20 from . import core
  21 from . import gitcmds
  22 from . import version
  23 from .compat import bchr
  24 from .i18n import N_
  25 from .interaction import Interaction
  26
  27 AVAILABLE = None
  28
  29 if utils.is_win32():
  30     try:
  31         import pywintypes
  32         import win32con
  33         import win32event
  34         import win32file
  35     except ImportError:
  36         pass
  37     else:
  38         AVAILABLE = 'pywin32'
  39 elif utils.is_linux():
  40     try:
  41         from . import inotify
  42     except ImportError:
  43         pass
  44     else:
  45         AVAILABLE = 'inotify'
  46
  47
  48 class _Monitor(QtCore.QObject):
  49
  50     files_changed = Signal()
  51     config_changed = Signal()
  52
  53     def __init__(self, context, thread_class):
  54         QtCore.QObject.__init__(self)
  55         self.context = context
  56         self._thread_class = thread_class
  57         self._thread = None
  58
  59     def start(self):
  60         if self._thread_class is not None:
  61             assert self._thread is None
  62             self._thread = self._thread_class(self.context, self)
  63             self._thread.start()
  64
  65     def stop(self):
  66         if self._thread_class is not None:
  67             assert self._thread is not None
  68             self._thread.stop()
  69             self._thread.wait()
  70             self._thread = None
  71
  72     def refresh(self):
  73         if self._thread is not None:
  74             self._thread.refresh()
  75
  76
  77 class _BaseThread(QtCore.QThread):
  78     #: The delay, in milliseconds, between detecting file system modification
  79     #: and triggering the 'files_changed' signal, to coalesce multiple
  80     #: modifications into a single signal.
  81     _NOTIFICATION_DELAY = 888
  82
  83     def __init__(self, context, monitor):
  84         QtCore.QThread.__init__(self)
  85         self.context = context
  86         self._monitor = monitor
  87         self._running = True
  88         self._use_check_ignore = version.check_git(context, 'check-ignore')
  89         self._force_notify = False
  90         self._force_config = False
  91         self._file_paths = set()
  92
  93     @property
  94     def _pending(self):
  95         return self._force_notify or self._file_paths or self._force_config
  96
  97     def refresh(self):
  98         """Do any housekeeping necessary in response to repository changes."""
  99         return
 100
 101     def notify(self):
 102         """Notifies all observers"""
 103         do_notify = False
 104         do_config = False
 105         if self._force_config:
 106             do_config = True
 107         if self._force_notify:
 108             do_notify = True
 109         elif self._file_paths:
 110             proc = core.start_command(
 111                 ['git', 'check-ignore', '--verbose', '--non-matching', '-z', '--stdin']
 112             )
 113             path_list = bchr(0).join(core.encode(path) for path in self._file_paths)
 114             out, _ = proc.communicate(path_list)
 115             if proc.returncode:
 116                 do_notify = True
 117             else:
 118                 # Each output record is four fields separated by NULL
 119                 # characters (records are also separated by NULL characters):
 120                 # <source> <NULL> <linenum> <NULL> <pattern> <NULL> <pathname>
 121                 # For paths which are not ignored, all fields will be empty
 122                 # except for <pathname>.  So to see if we have any non-ignored
 123                 # files, we simply check every fourth field to see if any of
 124                 # them are empty.
 125                 source_fields = out.split(bchr(0))[0:-1:4]
 126                 do_notify = not all(source_fields)
 127         self._force_notify = False
 128         self._force_config = False
 129         self._file_paths = set()
 130
 131         # "files changed" is a bigger hammer than "config changed".
 132         # and is a superset relative to what is done in response to the
 133         # signal.  Thus, the "elif" below avoids repeated work that
 134         # would be done if it were a simple "if" check instead.
 135         if do_notify:
 136             self._monitor.files_changed.emit()
 137         elif do_config:
 138             self._monitor.config_changed.emit()
 139
 140     @staticmethod
 141     def _log_enabled_message():
 142         msg = N_('File system change monitoring: enabled.\n')
 143         Interaction.log(msg)
 144
 145
 146 if AVAILABLE == 'inotify':
 147
 148     class _InotifyThread(_BaseThread):
 149         _TRIGGER_MASK = (
 150             inotify.IN_ATTRIB
 151             | inotify.IN_CLOSE_WRITE
 152             | inotify.IN_CREATE
 153             | inotify.IN_DELETE
 154             | inotify.IN_MODIFY
 155             | inotify.IN_MOVED_FROM
 156             | inotify.IN_MOVED_TO
 157         )
 158         _ADD_MASK = _TRIGGER_MASK | inotify.IN_EXCL_UNLINK | inotify.IN_ONLYDIR
 159
 160         def __init__(self, context, monitor):
 161             _BaseThread.__init__(self, context, monitor)
 162             git = context.git
 163             worktree = git.worktree()
 164             if worktree is not None:
 165                 worktree = core.abspath(worktree)
 166             self._worktree = worktree
 167             self._git_dir = git.git_path()
 168             self._lock = Lock()
 169             self._inotify_fd = None
 170             self._pipe_r = None
 171             self._pipe_w = None
 172             self._worktree_wd_to_path_map = {}
 173             self._worktree_path_to_wd_map = {}
 174             self._git_dir_wd_to_path_map = {}
 175             self._git_dir_path_to_wd_map = {}
 176             self._git_dir_wd = None
 177
 178         @staticmethod
 179         def _log_out_of_wds_message():
 180             msg = N_(
 181                 'File system change monitoring: disabled because the'
 182                 ' limit on the total number of inotify watches was'
 183                 ' reached.  You may be able to increase the limit on'
 184                 ' the number of watches by running:\n'
 185                 '\n'
 186                 '    echo fs.inotify.max_user_watches=100000 |'
 187                 ' sudo tee -a /etc/sysctl.conf &&'
 188                 ' sudo sysctl -p\n'
 189             )
 190             Interaction.log(msg)
 191
 192         def run(self):
 193             try:
 194                 with self._lock:
 195                     try:
 196                         self._inotify_fd = inotify.init()
 197                     except OSError as e:
 198                         self._inotify_fd = None
 199                         self._running = False
 200                         if e.errno == errno.EMFILE:
 201                             self._log_out_of_wds_message()
 202                         return
 203                     self._pipe_r, self._pipe_w = os.pipe()
 204
 205                 # pylint: disable=no-member
 206                 poll_obj = select.poll()
 207                 poll_obj.register(self._inotify_fd, select.POLLIN)
 208                 poll_obj.register(self._pipe_r, select.POLLIN)
 209
 210                 self.refresh()
 211
 212                 if self._running:
 213                     self._log_enabled_message()
 214                     self._process_events(poll_obj)
 215             finally:
 216                 self._close_fds()
 217
 218         def _process_events(self, poll_obj):
 219             while self._running:
 220                 if self._pending:
 221                     timeout = self._NOTIFICATION_DELAY
 222                 else:
 223                     timeout = None
 224                 try:
 225                     events = poll_obj.poll(timeout)
 226                 # pylint: disable=duplicate-except
 227                 except (OSError, select.error):
 228                     continue
 229                 else:
 230                     if not self._running:
 231                         break
 232                     if not events:
 233                         self.notify()
 234                     else:
 235                         for (fd, _) in events:
 236                             if fd == self._inotify_fd:
 237                                 self._handle_events()
 238
 239         def _close_fds(self):
 240             with self._lock:
 241                 if self._inotify_fd is not None:
 242                     os.close(self._inotify_fd)
 243                     self._inotify_fd = None
 244                 if self._pipe_r is not None:
 245                     os.close(self._pipe_r)
 246                     self._pipe_r = None
 247                     os.close(self._pipe_w)
 248                     self._pipe_w = None
 249
 250         def refresh(self):
 251             with self._lock:
 252                 self._refresh()
 253
 254         def _refresh(self):
 255             if self._inotify_fd is None:
 256                 return
 257             context = self.context
 258             try:
 259                 if self._worktree is not None:
 260                     tracked_dirs = {
 261                         os.path.dirname(os.path.join(self._worktree, path))
 262                         for path in gitcmds.tracked_files(context)
 263                     }
 264                     self._refresh_watches(
 265                         tracked_dirs,
 266                         self._worktree_wd_to_path_map,
 267                         self._worktree_path_to_wd_map,
 268                     )
 269                 git_dirs = set()
 270                 git_dirs.add(self._git_dir)
 271                 for dirpath, _, _ in core.walk(os.path.join(self._git_dir, 'refs')):
 272                     git_dirs.add(dirpath)
 273                 self._refresh_watches(
 274                     git_dirs, self._git_dir_wd_to_path_map, self._git_dir_path_to_wd_map
 275                 )
 276                 self._git_dir_wd = self._git_dir_path_to_wd_map.get(self._git_dir)
 277             except OSError as e:
 278                 if e.errno in (errno.ENOSPC, errno.EMFILE):
 279                     self._log_out_of_wds_message()
 280                     self._running = False
 281                 else:
 282                     raise
 283
 284         def _refresh_watches(self, paths_to_watch, wd_to_path_map, path_to_wd_map):
 285             watched_paths = set(path_to_wd_map)
 286             for path in watched_paths - paths_to_watch:
 287                 wd = path_to_wd_map.pop(path)
 288                 wd_to_path_map.pop(wd)
 289                 try:
 290                     inotify.rm_watch(self._inotify_fd, wd)
 291                 except OSError as e:
 292                     if e.errno == errno.EINVAL:
 293                         # This error can occur if the target of the wd was
 294                         # removed on the filesystem before we call
 295                         # inotify.rm_watch() so ignore it.
 296                         continue
 297                     raise e
 298             for path in paths_to_watch - watched_paths:
 299                 try:
 300                     wd = inotify.add_watch(
 301                         self._inotify_fd, core.encode(path), self._ADD_MASK
 302                     )
 303                 except OSError as e:
 304                     if e.errno in (errno.ENOENT, errno.ENOTDIR):
 305                         # These two errors should only occur as a result of
 306                         # race conditions:  the first if the directory
 307                         # referenced by path was removed or renamed before the
 308                         # call to inotify.add_watch(); the second if the
 309                         # directory referenced by path was replaced with a file
 310                         # before the call to inotify.add_watch().  Therefore we
 311                         # simply ignore them.
 312                         continue
 313                     raise e
 314                 else:
 315                     wd_to_path_map[wd] = path
 316                     path_to_wd_map[path] = wd
 317
 318         def _check_event(self, wd, mask, name):
 319             if mask & inotify.IN_Q_OVERFLOW:
 320                 self._force_notify = True
 321             elif not mask & self._TRIGGER_MASK:
 322                 pass
 323             elif mask & inotify.IN_ISDIR:
 324                 pass
 325             elif wd in self._worktree_wd_to_path_map:
 326                 if self._use_check_ignore and name:
 327                     path = os.path.join(
 328                         self._worktree_wd_to_path_map[wd], core.decode(name)
 329                     )
 330                     self._file_paths.add(path)
 331                 else:
 332                     self._force_notify = True
 333             elif wd == self._git_dir_wd:
 334                 name = core.decode(name)
 335                 if name in ('HEAD', 'index'):
 336                     self._force_notify = True
 337                 elif name == 'config':
 338                     self._force_config = True
 339             elif wd in self._git_dir_wd_to_path_map and not core.decode(name).endswith(
 340                 '.lock'
 341             ):
 342                 self._force_notify = True
 343
 344         def _handle_events(self):
 345             for wd, mask, _, name in inotify.read_events(self._inotify_fd):
 346                 if not self._force_notify:
 347                     self._check_event(wd, mask, name)
 348
 349         def stop(self):
 350             self._running = False
 351             with self._lock:
 352                 if self._pipe_w is not None:
 353                     os.write(self._pipe_w, bchr(0))
 354             self.wait()
 355
 356
 357 if AVAILABLE == 'pywin32':
 358
 359     class _Win32Watch(object):
 360         def __init__(self, path, flags):
 361             self.flags = flags
 362
 363             self.handle = None
 364             self.event = None
 365
 366             try:
 367                 self.handle = win32file.CreateFileW(
 368                     path,
 369                     0x0001,  # FILE_LIST_DIRECTORY
 370                     win32con.FILE_SHARE_READ | win32con.FILE_SHARE_WRITE,
 371                     None,
 372                     win32con.OPEN_EXISTING,
 373                     win32con.FILE_FLAG_BACKUP_SEMANTICS | win32con.FILE_FLAG_OVERLAPPED,
 374                     None,
 375                 )
 376
 377                 self.buffer = win32file.AllocateReadBuffer(8192)
 378                 self.event = win32event.CreateEvent(None, True, False, None)
 379                 self.overlapped = pywintypes.OVERLAPPED()
 380                 self.overlapped.hEvent = self.event
 381                 self._start()
 382             except Exception:
 383                 self.close()
 384                 raise
 385
 386         def _start(self):
 387             win32file.ReadDirectoryChangesW(
 388                 self.handle, self.buffer, True, self.flags, self.overlapped
 389             )
 390
 391         def read(self):
 392             if win32event.WaitForSingleObject(self.event, 0) == win32event.WAIT_TIMEOUT:
 393                 result = []
 394             else:
 395                 nbytes = win32file.GetOverlappedResult(
 396                     self.handle, self.overlapped, False
 397                 )
 398                 result = win32file.FILE_NOTIFY_INFORMATION(self.buffer, nbytes)
 399                 self._start()
 400             return result
 401
 402         def close(self):
 403             if self.handle is not None:
 404                 win32file.CancelIo(self.handle)
 405                 win32file.CloseHandle(self.handle)
 406             if self.event is not None:
 407                 win32file.CloseHandle(self.event)
 408
 409     class _Win32Thread(_BaseThread):
 410         _FLAGS = (
 411             win32con.FILE_NOTIFY_CHANGE_FILE_NAME
 412             | win32con.FILE_NOTIFY_CHANGE_DIR_NAME
 413             | win32con.FILE_NOTIFY_CHANGE_ATTRIBUTES
 414             | win32con.FILE_NOTIFY_CHANGE_SIZE
 415             | win32con.FILE_NOTIFY_CHANGE_LAST_WRITE
 416             | win32con.FILE_NOTIFY_CHANGE_SECURITY
 417         )
 418
 419         def __init__(self, context, monitor):
 420             _BaseThread.__init__(self, context, monitor)
 421             git = context.git
 422             worktree = git.worktree()
 423             if worktree is not None:
 424                 worktree = self._transform_path(core.abspath(worktree))
 425             self._worktree = worktree
 426             self._worktree_watch = None
 427             self._git_dir = self._transform_path(core.abspath(git.git_path()))
 428             self._git_dir_watch = None
 429             self._stop_event_lock = Lock()
 430             self._stop_event = None
 431
 432         @staticmethod
 433         def _transform_path(path):
 434             return path.replace('\\', '/').lower()
 435
 436         def run(self):
 437             try:
 438                 with self._stop_event_lock:
 439                     self._stop_event = win32event.CreateEvent(None, True, False, None)
 440
 441                 events = [self._stop_event]
 442
 443                 if self._worktree is not None:
 444                     self._worktree_watch = _Win32Watch(self._worktree, self._FLAGS)
 445                     events.append(self._worktree_watch.event)
 446
 447                 self._git_dir_watch = _Win32Watch(self._git_dir, self._FLAGS)
 448                 events.append(self._git_dir_watch.event)
 449
 450                 self._log_enabled_message()
 451
 452                 while self._running:
 453                     if self._pending:
 454                         timeout = self._NOTIFICATION_DELAY
 455                     else:
 456                         timeout = win32event.INFINITE
 457                     status = win32event.WaitForMultipleObjects(events, False, timeout)
 458                     if not self._running:
 459                         break
 460                     if status == win32event.WAIT_TIMEOUT:
 461                         self.notify()
 462                     else:
 463                         self._handle_results()
 464             finally:
 465                 with self._stop_event_lock:
 466                     if self._stop_event is not None:
 467                         win32file.CloseHandle(self._stop_event)
 468                         self._stop_event = None
 469                 if self._worktree_watch is not None:
 470                     self._worktree_watch.close()
 471                 if self._git_dir_watch is not None:
 472                     self._git_dir_watch.close()
 473
 474         def _handle_results(self):
 475             if self._worktree_watch is not None:
 476                 for _, path in self._worktree_watch.read():
 477                     if not self._running:
 478                         break
 479                     if self._force_notify:
 480                         continue
 481                     path = self._worktree + '/' + self._transform_path(path)
 482                     if (
 483                         path != self._git_dir
 484                         and not path.startswith(self._git_dir + '/')
 485                         and not os.path.isdir(path)
 486                     ):
 487                         if self._use_check_ignore:
 488                             self._file_paths.add(path)
 489                         else:
 490                             self._force_notify = True
 491             for _, path in self._git_dir_watch.read():
 492                 if not self._running:
 493                     break
 494                 if self._force_notify:
 495                     continue
 496                 path = self._transform_path(path)
 497                 if path.endswith('.lock'):
 498                     continue
 499                 if path == 'config':
 500                     self._force_config = True
 501                     continue
 502                 if path == 'head' or path == 'index' or path.startswith('refs/'):
 503                     self._force_notify = True
 504
 505         def stop(self):
 506             self._running = False
 507             with self._stop_event_lock:
 508                 if self._stop_event is not None:
 509                     win32event.SetEvent(self._stop_event)
 510             self.wait()
 511
 512
 513 def create(context):
 514     thread_class = None
 515     cfg = context.cfg
 516     if not cfg.get('cola.inotify', default=True):
 517         msg = N_(
 518             'File system change monitoring: disabled because'
 519             ' "cola.inotify" is false.\n'
 520         )
 521         Interaction.log(msg)
 522     elif AVAILABLE == 'inotify':
 523         thread_class = _InotifyThread
 524     elif AVAILABLE == 'pywin32':
 525         thread_class = _Win32Thread
 526     else:
 527         if utils.is_win32():
 528             msg = N_(
 529                 'File system change monitoring: disabled because pywin32'
 530                 ' is not installed.\n'
 531             )
 532             Interaction.log(msg)
 533         elif utils.is_linux():
 534             msg = N_(
 535                 'File system change monitoring: disabled because libc'
 536                 ' does not support the inotify system calls.\n'
 537             )
 538             Interaction.log(msg)
 539     return _Monitor(context, thread_class)