cola/fsmonitor.py

   1 # Copyright (C) 2008-2017 David Aguilar
   2 # Copyright (C) 2015 Daniel Harding
   3 """Filesystem monitor for Linux and Windows
   4
   5 Linux monitoring uses using inotify.
   6 Windows monitoring uses pywin32 and the ReadDirectoryChanges function.
   7
   8 """
   9 from __future__ import absolute_import, division, print_function, unicode_literals
  10 import errno
  11 import os
  12 import os.path
  13 import select
  14 from threading import Lock
  15
  16 from qtpy import QtCore
  17 from qtpy.QtCore import Signal
  18
  19 from . import utils
  20 from . import core
  21 from . import gitcmds
  22 from . import version
  23 from .compat import bchr
  24 from .i18n import N_
  25 from .interaction import Interaction
  26
  27 AVAILABLE = None
  28
  29 if utils.is_win32():
  30     try:
  31         import pywintypes
  32         import win32con
  33         import win32event
  34         import win32file
  35     except ImportError:
  36         pass
  37     else:
  38         AVAILABLE = 'pywin32'
  39 elif utils.is_linux():
  40     try:
  41         from . import inotify
  42     except ImportError:
  43         pass
  44     else:
  45         AVAILABLE = 'inotify'
  46
  47
  48 class _Monitor(QtCore.QObject):
  49
  50     files_changed = Signal()
  51     config_changed = Signal()
  52
  53     def __init__(self, context, thread_class):
  54         QtCore.QObject.__init__(self)
  55         self.context = context
  56         self._thread_class = thread_class
  57         self._thread = None
  58
  59     def start(self):
  60         if self._thread_class is not None:
  61             assert self._thread is None
  62             self._thread = self._thread_class(self.context, self)
  63             self._thread.start()
  64
  65     def stop(self):
  66         if self._thread_class is not None:
  67             assert self._thread is not None
  68             self._thread.stop()
  69             self._thread.wait()
  70             self._thread = None
  71
  72     def refresh(self):
  73         if self._thread is not None:
  74             self._thread.refresh()
  75
  76
  77 class _BaseThread(QtCore.QThread):
  78     #: The delay, in milliseconds, between detecting file system modification
  79     #: and triggering the 'files_changed' signal, to coalesce multiple
  80     #: modifications into a single signal.
  81     _NOTIFICATION_DELAY = 888
  82
  83     def __init__(self, context, monitor):
  84         QtCore.QThread.__init__(self)
  85         self.context = context
  86         self._monitor = monitor
  87         self._running = True
  88         self._use_check_ignore = version.check_git(context, 'check-ignore')
  89         self._force_notify = False
  90         self._force_config = False
  91         self._file_paths = set()
  92
  93     @property
  94     def _pending(self):
  95         return self._force_notify or self._file_paths or self._force_config
  96
  97     # pylint: disable=no-self-use
  98     def refresh(self):
  99         """Do any housekeeping necessary in response to repository changes."""
 100         return
 101
 102     def notify(self):
 103         """Notifies all observers"""
 104         do_notify = False
 105         do_config = False
 106         if self._force_config:
 107             do_config = True
 108         if self._force_notify:
 109             do_notify = True
 110         elif self._file_paths:
 111             proc = core.start_command(
 112                 ['git', 'check-ignore', '--verbose', '--non-matching', '-z', '--stdin']
 113             )
 114             path_list = bchr(0).join(core.encode(path) for path in self._file_paths)
 115             out, _ = proc.communicate(path_list)
 116             if proc.returncode:
 117                 do_notify = True
 118             else:
 119                 # Each output record is four fields separated by NULL
 120                 # characters (records are also separated by NULL characters):
 121                 # <source> <NULL> <linenum> <NULL> <pattern> <NULL> <pathname>
 122                 # For paths which are not ignored, all fields will be empty
 123                 # except for <pathname>.  So to see if we have any non-ignored
 124                 # files, we simply check every fourth field to see if any of
 125                 # them are empty.
 126                 source_fields = out.split(bchr(0))[0:-1:4]
 127                 do_notify = not all(source_fields)
 128         self._force_notify = False
 129         self._force_config = False
 130         self._file_paths = set()
 131
 132         # "files changed" is a bigger hammer than "config changed".
 133         # and is a superset relative to what is done in response to the
 134         # signal.  Thus, the "elif" below avoids repeated work that
 135         # would be done if it were a simple "if" check instead.
 136         if do_notify:
 137             self._monitor.files_changed.emit()
 138         elif do_config:
 139             self._monitor.config_changed.emit()
 140
 141     @staticmethod
 142     def _log_enabled_message():
 143         msg = N_('File system change monitoring: enabled.\n')
 144         Interaction.log(msg)
 145
 146
 147 if AVAILABLE == 'inotify':
 148
 149     class _InotifyThread(_BaseThread):
 150         _TRIGGER_MASK = (
 151             inotify.IN_ATTRIB
 152             | inotify.IN_CLOSE_WRITE
 153             | inotify.IN_CREATE
 154             | inotify.IN_DELETE
 155             | inotify.IN_MODIFY
 156             | inotify.IN_MOVED_FROM
 157             | inotify.IN_MOVED_TO
 158         )
 159         _ADD_MASK = _TRIGGER_MASK | inotify.IN_EXCL_UNLINK | inotify.IN_ONLYDIR
 160
 161         def __init__(self, context, monitor):
 162             _BaseThread.__init__(self, context, monitor)
 163             git = context.git
 164             worktree = git.worktree()
 165             if worktree is not None:
 166                 worktree = core.abspath(worktree)
 167             self._worktree = worktree
 168             self._git_dir = git.git_path()
 169             self._lock = Lock()
 170             self._inotify_fd = None
 171             self._pipe_r = None
 172             self._pipe_w = None
 173             self._worktree_wd_to_path_map = {}
 174             self._worktree_path_to_wd_map = {}
 175             self._git_dir_wd_to_path_map = {}
 176             self._git_dir_path_to_wd_map = {}
 177             self._git_dir_wd = None
 178
 179         @staticmethod
 180         def _log_out_of_wds_message():
 181             msg = N_(
 182                 'File system change monitoring: disabled because the'
 183                 ' limit on the total number of inotify watches was'
 184                 ' reached.  You may be able to increase the limit on'
 185                 ' the number of watches by running:\n'
 186                 '\n'
 187                 '    echo fs.inotify.max_user_watches=100000 |'
 188                 ' sudo tee -a /etc/sysctl.conf &&'
 189                 ' sudo sysctl -p\n'
 190             )
 191             Interaction.log(msg)
 192
 193         def run(self):
 194             try:
 195                 with self._lock:
 196                     try:
 197                         self._inotify_fd = inotify.init()
 198                     except OSError as e:
 199                         self._inotify_fd = None
 200                         self._running = False
 201                         if e.errno == errno.EMFILE:
 202                             self._log_out_of_wds_message()
 203                         return
 204                     self._pipe_r, self._pipe_w = os.pipe()
 205
 206                 # pylint: disable=no-member
 207                 poll_obj = select.poll()
 208                 poll_obj.register(self._inotify_fd, select.POLLIN)
 209                 poll_obj.register(self._pipe_r, select.POLLIN)
 210
 211                 self.refresh()
 212
 213                 if self._running:
 214                     self._log_enabled_message()
 215                     self._process_events(poll_obj)
 216             finally:
 217                 self._close_fds()
 218
 219         def _process_events(self, poll_obj):
 220             while self._running:
 221                 if self._pending:
 222                     timeout = self._NOTIFICATION_DELAY
 223                 else:
 224                     timeout = None
 225                 try:
 226                     events = poll_obj.poll(timeout)
 227                 # pylint: disable=duplicate-except
 228                 except (OSError, select.error):
 229                     continue
 230                 else:
 231                     if not self._running:
 232                         break
 233                     if not events:
 234                         self.notify()
 235                     else:
 236                         for (fd, _) in events:
 237                             if fd == self._inotify_fd:
 238                                 self._handle_events()
 239
 240         def _close_fds(self):
 241             with self._lock:
 242                 if self._inotify_fd is not None:
 243                     os.close(self._inotify_fd)
 244                     self._inotify_fd = None
 245                 if self._pipe_r is not None:
 246                     os.close(self._pipe_r)
 247                     self._pipe_r = None
 248                     os.close(self._pipe_w)
 249                     self._pipe_w = None
 250
 251         def refresh(self):
 252             with self._lock:
 253                 self._refresh()
 254
 255         def _refresh(self):
 256             if self._inotify_fd is None:
 257                 return
 258             context = self.context
 259             try:
 260                 if self._worktree is not None:
 261                     tracked_dirs = {
 262                         os.path.dirname(os.path.join(self._worktree, path))
 263                         for path in gitcmds.tracked_files(context)
 264                     }
 265                     self._refresh_watches(
 266                         tracked_dirs,
 267                         self._worktree_wd_to_path_map,
 268                         self._worktree_path_to_wd_map,
 269                     )
 270                 git_dirs = set()
 271                 git_dirs.add(self._git_dir)
 272                 for dirpath, _, _ in core.walk(os.path.join(self._git_dir, 'refs')):
 273                     git_dirs.add(dirpath)
 274                 self._refresh_watches(
 275                     git_dirs, self._git_dir_wd_to_path_map, self._git_dir_path_to_wd_map
 276                 )
 277                 self._git_dir_wd = self._git_dir_path_to_wd_map.get(self._git_dir)
 278             except OSError as e:
 279                 if e.errno in (errno.ENOSPC, errno.EMFILE):
 280                     self._log_out_of_wds_message()
 281                     self._running = False
 282                 else:
 283                     raise
 284
 285         def _refresh_watches(self, paths_to_watch, wd_to_path_map, path_to_wd_map):
 286             watched_paths = set(path_to_wd_map)
 287             for path in watched_paths - paths_to_watch:
 288                 wd = path_to_wd_map.pop(path)
 289                 wd_to_path_map.pop(wd)
 290                 try:
 291                     inotify.rm_watch(self._inotify_fd, wd)
 292                 except OSError as e:
 293                     if e.errno == errno.EINVAL:
 294                         # This error can occur if the target of the wd was
 295                         # removed on the filesystem before we call
 296                         # inotify.rm_watch() so ignore it.
 297                         continue
 298                     raise e
 299             for path in paths_to_watch - watched_paths:
 300                 try:
 301                     wd = inotify.add_watch(
 302                         self._inotify_fd, core.encode(path), self._ADD_MASK
 303                     )
 304                 except OSError as e:
 305                     if e.errno in (errno.ENOENT, errno.ENOTDIR):
 306                         # These two errors should only occur as a result of
 307                         # race conditions:  the first if the directory
 308                         # referenced by path was removed or renamed before the
 309                         # call to inotify.add_watch(); the second if the
 310                         # directory referenced by path was replaced with a file
 311                         # before the call to inotify.add_watch().  Therefore we
 312                         # simply ignore them.
 313                         continue
 314                     raise e
 315                 else:
 316                     wd_to_path_map[wd] = path
 317                     path_to_wd_map[path] = wd
 318
 319         def _check_event(self, wd, mask, name):
 320             if mask & inotify.IN_Q_OVERFLOW:
 321                 self._force_notify = True
 322             elif not mask & self._TRIGGER_MASK:
 323                 pass
 324             elif mask & inotify.IN_ISDIR:
 325                 pass
 326             elif wd in self._worktree_wd_to_path_map:
 327                 if self._use_check_ignore and name:
 328                     path = os.path.join(
 329                         self._worktree_wd_to_path_map[wd], core.decode(name)
 330                     )
 331                     self._file_paths.add(path)
 332                 else:
 333                     self._force_notify = True
 334             elif wd == self._git_dir_wd:
 335                 name = core.decode(name)
 336                 if name in ('HEAD', 'index'):
 337                     self._force_notify = True
 338                 elif name == 'config':
 339                     self._force_config = True
 340             elif wd in self._git_dir_wd_to_path_map and not core.decode(name).endswith(
 341                 '.lock'
 342             ):
 343                 self._force_notify = True
 344
 345         def _handle_events(self):
 346             for wd, mask, _, name in inotify.read_events(self._inotify_fd):
 347                 if not self._force_notify:
 348                     self._check_event(wd, mask, name)
 349
 350         def stop(self):
 351             self._running = False
 352             with self._lock:
 353                 if self._pipe_w is not None:
 354                     os.write(self._pipe_w, bchr(0))
 355             self.wait()
 356
 357
 358 if AVAILABLE == 'pywin32':
 359
 360     class _Win32Watch(object):
 361         def __init__(self, path, flags):
 362             self.flags = flags
 363
 364             self.handle = None
 365             self.event = None
 366
 367             try:
 368                 self.handle = win32file.CreateFileW(
 369                     path,
 370                     0x0001,  # FILE_LIST_DIRECTORY
 371                     win32con.FILE_SHARE_READ | win32con.FILE_SHARE_WRITE,
 372                     None,
 373                     win32con.OPEN_EXISTING,
 374                     win32con.FILE_FLAG_BACKUP_SEMANTICS | win32con.FILE_FLAG_OVERLAPPED,
 375                     None,
 376                 )
 377
 378                 self.buffer = win32file.AllocateReadBuffer(8192)
 379                 self.event = win32event.CreateEvent(None, True, False, None)
 380                 self.overlapped = pywintypes.OVERLAPPED()
 381                 self.overlapped.hEvent = self.event
 382                 self._start()
 383             except Exception:
 384                 self.close()
 385                 raise
 386
 387         def _start(self):
 388             win32file.ReadDirectoryChangesW(
 389                 self.handle, self.buffer, True, self.flags, self.overlapped
 390             )
 391
 392         def read(self):
 393             if win32event.WaitForSingleObject(self.event, 0) == win32event.WAIT_TIMEOUT:
 394                 result = []
 395             else:
 396                 nbytes = win32file.GetOverlappedResult(
 397                     self.handle, self.overlapped, False
 398                 )
 399                 result = win32file.FILE_NOTIFY_INFORMATION(self.buffer, nbytes)
 400                 self._start()
 401             return result
 402
 403         def close(self):
 404             if self.handle is not None:
 405                 win32file.CancelIo(self.handle)
 406                 win32file.CloseHandle(self.handle)
 407             if self.event is not None:
 408                 win32file.CloseHandle(self.event)
 409
 410     class _Win32Thread(_BaseThread):
 411         _FLAGS = (
 412             win32con.FILE_NOTIFY_CHANGE_FILE_NAME
 413             | win32con.FILE_NOTIFY_CHANGE_DIR_NAME
 414             | win32con.FILE_NOTIFY_CHANGE_ATTRIBUTES
 415             | win32con.FILE_NOTIFY_CHANGE_SIZE
 416             | win32con.FILE_NOTIFY_CHANGE_LAST_WRITE
 417             | win32con.FILE_NOTIFY_CHANGE_SECURITY
 418         )
 419
 420         def __init__(self, context, monitor):
 421             _BaseThread.__init__(self, context, monitor)
 422             git = context.git
 423             worktree = git.worktree()
 424             if worktree is not None:
 425                 worktree = self._transform_path(core.abspath(worktree))
 426             self._worktree = worktree
 427             self._worktree_watch = None
 428             self._git_dir = self._transform_path(core.abspath(git.git_path()))
 429             self._git_dir_watch = None
 430             self._stop_event_lock = Lock()
 431             self._stop_event = None
 432
 433         @staticmethod
 434         def _transform_path(path):
 435             return path.replace('\\', '/').lower()
 436
 437         def run(self):
 438             try:
 439                 with self._stop_event_lock:
 440                     self._stop_event = win32event.CreateEvent(None, True, False, None)
 441
 442                 events = [self._stop_event]
 443
 444                 if self._worktree is not None:
 445                     self._worktree_watch = _Win32Watch(self._worktree, self._FLAGS)
 446                     events.append(self._worktree_watch.event)
 447
 448                 self._git_dir_watch = _Win32Watch(self._git_dir, self._FLAGS)
 449                 events.append(self._git_dir_watch.event)
 450
 451                 self._log_enabled_message()
 452
 453                 while self._running:
 454                     if self._pending:
 455                         timeout = self._NOTIFICATION_DELAY
 456                     else:
 457                         timeout = win32event.INFINITE
 458                     rc = win32event.WaitForMultipleObjects(events, False, timeout)
 459                     if not self._running:
 460                         break
 461                     if rc == win32event.WAIT_TIMEOUT:
 462                         self.notify()
 463                     else:
 464                         self._handle_results()
 465             finally:
 466                 with self._stop_event_lock:
 467                     if self._stop_event is not None:
 468                         win32file.CloseHandle(self._stop_event)
 469                         self._stop_event = None
 470                 if self._worktree_watch is not None:
 471                     self._worktree_watch.close()
 472                 if self._git_dir_watch is not None:
 473                     self._git_dir_watch.close()
 474
 475         def _handle_results(self):
 476             if self._worktree_watch is not None:
 477                 for _, path in self._worktree_watch.read():
 478                     if not self._running:
 479                         break
 480                     if self._force_notify:
 481                         continue
 482                     path = self._worktree + '/' + self._transform_path(path)
 483                     if (
 484                         path != self._git_dir
 485                         and not path.startswith(self._git_dir + '/')
 486                         and not os.path.isdir(path)
 487                     ):
 488                         if self._use_check_ignore:
 489                             self._file_paths.add(path)
 490                         else:
 491                             self._force_notify = True
 492             for _, path in self._git_dir_watch.read():
 493                 if not self._running:
 494                     break
 495                 if self._force_notify:
 496                     continue
 497                 path = self._transform_path(path)
 498                 if path.endswith('.lock'):
 499                     continue
 500                 if path == 'config':
 501                     self._force_config = True
 502                     continue
 503                 if path == 'head' or path == 'index' or path.startswith('refs/'):
 504                     self._force_notify = True
 505
 506         def stop(self):
 507             self._running = False
 508             with self._stop_event_lock:
 509                 if self._stop_event is not None:
 510                     win32event.SetEvent(self._stop_event)
 511             self.wait()
 512
 513
 514 def create(context):
 515     thread_class = None
 516     cfg = context.cfg
 517     if not cfg.get('cola.inotify', default=True):
 518         msg = N_(
 519             'File system change monitoring: disabled because'
 520             ' "cola.inotify" is false.\n'
 521         )
 522         Interaction.log(msg)
 523     elif AVAILABLE == 'inotify':
 524         thread_class = _InotifyThread
 525     elif AVAILABLE == 'pywin32':
 526         thread_class = _Win32Thread
 527     else:
 528         if utils.is_win32():
 529             msg = N_(
 530                 'File system change monitoring: disabled because pywin32'
 531                 ' is not installed.\n'
 532             )
 533             Interaction.log(msg)
 534         elif utils.is_linux():
 535             msg = N_(
 536                 'File system change monitoring: disabled because libc'
 537                 ' does not support the inotify system calls.\n'
 538             )
 539             Interaction.log(msg)
 540     return _Monitor(context, thread_class)