cola/fsmonitor.py

   1 # Copyright (C) 2008-2024 David Aguilar
   2 # Copyright (C) 2015 Daniel Harding
   3 """Filesystem monitor for Linux and Windows
   4
   5 Linux monitoring uses using inotify.
   6 Windows monitoring uses pywin32 and the ReadDirectoryChanges function.
   7
   8 """
   9 import errno
  10 import os
  11 import os.path
  12 import select
  13 from threading import Lock
  14
  15 from qtpy import QtCore
  16 from qtpy.QtCore import Signal
  17
  18 from . import utils
  19 from . import core
  20 from . import gitcmds
  21 from . import version
  22 from .compat import bchr
  23 from .i18n import N_
  24 from .interaction import Interaction
  25
  26 AVAILABLE = None
  27
  28 pywintypes = None
  29 win32file = None
  30 win32con = None
  31 win32event = None
  32 if utils.is_win32():
  33     try:
  34         import pywintypes
  35         import win32con
  36         import win32event
  37         import win32file
  38
  39         AVAILABLE = 'pywin32'
  40     except ImportError:
  41         pass
  42
  43 elif utils.is_linux():
  44     try:
  45         from . import inotify
  46     except ImportError:
  47         pass
  48     else:
  49         AVAILABLE = 'inotify'
  50
  51
  52 class _Monitor(QtCore.QObject):
  53     files_changed = Signal()
  54     config_changed = Signal()
  55
  56     def __init__(self, context, thread_class):
  57         QtCore.QObject.__init__(self)
  58         self.context = context
  59         self._thread_class = thread_class
  60         self._thread = None
  61
  62     def start(self):
  63         if self._thread_class is not None:
  64             assert self._thread is None
  65             self._thread = self._thread_class(self.context, self)
  66             self._thread.start()
  67
  68     def stop(self):
  69         if self._thread_class is not None:
  70             assert self._thread is not None
  71             self._thread.stop()
  72             self._thread.wait()
  73             self._thread = None
  74
  75     def refresh(self):
  76         if self._thread is not None:
  77             self._thread.refresh()
  78
  79
  80 class _BaseThread(QtCore.QThread):
  81     #: The delay, in milliseconds, between detecting file system modification
  82     #: and triggering the 'files_changed' signal, to coalesce multiple
  83     #: modifications into a single signal.
  84     _NOTIFICATION_DELAY = 888
  85
  86     def __init__(self, context, monitor):
  87         QtCore.QThread.__init__(self)
  88         self.context = context
  89         self._monitor = monitor
  90         self._running = True
  91         self._use_check_ignore = version.check_git(context, 'check-ignore')
  92         self._force_notify = False
  93         self._force_config = False
  94         self._file_paths = set()
  95
  96     @property
  97     def _pending(self):
  98         return self._force_notify or self._file_paths or self._force_config
  99
 100     def refresh(self):
 101         """Do any housekeeping necessary in response to repository changes."""
 102         return
 103
 104     def notify(self):
 105         """Notifies all observers"""
 106         do_notify = False
 107         do_config = False
 108         if self._force_config:
 109             do_config = True
 110         if self._force_notify:
 111             do_notify = True
 112         elif self._file_paths:
 113             proc = core.start_command(
 114                 ['git', 'check-ignore', '--verbose', '--non-matching', '-z', '--stdin']
 115             )
 116             path_list = bchr(0).join(core.encode(path) for path in self._file_paths)
 117             out, _ = proc.communicate(path_list)
 118             if proc.returncode:
 119                 do_notify = True
 120             else:
 121                 # Each output record is four fields separated by NULL
 122                 # characters (records are also separated by NULL characters):
 123                 # <source> <NULL> <linenum> <NULL> <pattern> <NULL> <pathname>
 124                 # For paths which are not ignored, all fields will be empty
 125                 # except for <pathname>.  So to see if we have any non-ignored
 126                 # files, we simply check every fourth field to see if any of
 127                 # them are empty.
 128                 source_fields = out.split(bchr(0))[0:-1:4]
 129                 do_notify = not all(source_fields)
 130         self._force_notify = False
 131         self._force_config = False
 132         self._file_paths = set()
 133
 134         # "files changed" is a bigger hammer than "config changed".
 135         # and is a superset relative to what is done in response to the
 136         # signal.  Thus, the "elif" below avoids repeated work that
 137         # would be done if it were a simple "if" check instead.
 138         if do_notify:
 139             self._monitor.files_changed.emit()
 140         elif do_config:
 141             self._monitor.config_changed.emit()
 142
 143     @staticmethod
 144     def _log_enabled_message():
 145         msg = N_('File system change monitoring: enabled.\n')
 146         Interaction.log(msg)
 147
 148
 149 if AVAILABLE == 'inotify':
 150
 151     class _InotifyThread(_BaseThread):
 152         _TRIGGER_MASK = (
 153             inotify.IN_ATTRIB
 154             | inotify.IN_CLOSE_WRITE
 155             | inotify.IN_CREATE
 156             | inotify.IN_DELETE
 157             | inotify.IN_MODIFY
 158             | inotify.IN_MOVED_FROM
 159             | inotify.IN_MOVED_TO
 160         )
 161         _ADD_MASK = _TRIGGER_MASK | inotify.IN_EXCL_UNLINK | inotify.IN_ONLYDIR
 162
 163         def __init__(self, context, monitor):
 164             _BaseThread.__init__(self, context, monitor)
 165             git = context.git
 166             worktree = git.worktree()
 167             if worktree is not None:
 168                 worktree = core.abspath(worktree)
 169             self._worktree = worktree
 170             self._git_dir = git.git_path()
 171             self._lock = Lock()
 172             self._inotify_fd = None
 173             self._pipe_r = None
 174             self._pipe_w = None
 175             self._worktree_wd_to_path_map = {}
 176             self._worktree_path_to_wd_map = {}
 177             self._git_dir_wd_to_path_map = {}
 178             self._git_dir_path_to_wd_map = {}
 179             self._git_dir_wd = None
 180
 181         @staticmethod
 182         def _log_out_of_wds_message():
 183             msg = N_(
 184                 'File system change monitoring: disabled because the'
 185                 ' limit on the total number of inotify watches was'
 186                 ' reached.  You may be able to increase the limit on'
 187                 ' the number of watches by running:\n'
 188                 '\n'
 189                 '    echo fs.inotify.max_user_watches=100000 |'
 190                 ' sudo tee -a /etc/sysctl.conf &&'
 191                 ' sudo sysctl -p\n'
 192             )
 193             Interaction.log(msg)
 194
 195         def run(self):
 196             try:
 197                 with self._lock:
 198                     try:
 199                         self._inotify_fd = inotify.init()
 200                     except OSError as e:
 201                         self._inotify_fd = None
 202                         self._running = False
 203                         if e.errno == errno.EMFILE:
 204                             self._log_out_of_wds_message()
 205                         return
 206                     self._pipe_r, self._pipe_w = os.pipe()
 207
 208                 # pylint: disable=no-member
 209                 poll_obj = select.poll()
 210                 poll_obj.register(self._inotify_fd, select.POLLIN)
 211                 poll_obj.register(self._pipe_r, select.POLLIN)
 212
 213                 self.refresh()
 214
 215                 if self._running:
 216                     self._log_enabled_message()
 217                     self._process_events(poll_obj)
 218             finally:
 219                 self._close_fds()
 220
 221         def _process_events(self, poll_obj):
 222             while self._running:
 223                 if self._pending:
 224                     timeout = self._NOTIFICATION_DELAY
 225                 else:
 226                     timeout = None
 227                 try:
 228                     events = poll_obj.poll(timeout)
 229                 # pylint: disable=duplicate-except
 230                 except OSError:
 231                     continue
 232                 else:
 233                     if not self._running:
 234                         break
 235                     if not events:
 236                         self.notify()
 237                     else:
 238                         for fd, _ in events:
 239                             if fd == self._inotify_fd:
 240                                 self._handle_events()
 241
 242         def _close_fds(self):
 243             with self._lock:
 244                 if self._inotify_fd is not None:
 245                     os.close(self._inotify_fd)
 246                     self._inotify_fd = None
 247                 if self._pipe_r is not None:
 248                     os.close(self._pipe_r)
 249                     self._pipe_r = None
 250                     os.close(self._pipe_w)
 251                     self._pipe_w = None
 252
 253         def refresh(self):
 254             with self._lock:
 255                 self._refresh()
 256
 257         def _refresh(self):
 258             if self._inotify_fd is None:
 259                 return
 260             context = self.context
 261             try:
 262                 if self._worktree is not None:
 263                     tracked_dirs = {
 264                         os.path.dirname(os.path.join(self._worktree, path))
 265                         for path in gitcmds.tracked_files(context)
 266                     }
 267                     self._refresh_watches(
 268                         tracked_dirs,
 269                         self._worktree_wd_to_path_map,
 270                         self._worktree_path_to_wd_map,
 271                     )
 272                 git_dirs = set()
 273                 git_dirs.add(self._git_dir)
 274                 for dirpath, _, _ in core.walk(os.path.join(self._git_dir, 'refs')):
 275                     git_dirs.add(dirpath)
 276                 self._refresh_watches(
 277                     git_dirs, self._git_dir_wd_to_path_map, self._git_dir_path_to_wd_map
 278                 )
 279                 self._git_dir_wd = self._git_dir_path_to_wd_map.get(self._git_dir)
 280             except OSError as e:
 281                 if e.errno in (errno.ENOSPC, errno.EMFILE):
 282                     self._log_out_of_wds_message()
 283                     self._running = False
 284                 else:
 285                     raise
 286
 287         def _refresh_watches(self, paths_to_watch, wd_to_path_map, path_to_wd_map):
 288             watched_paths = set(path_to_wd_map)
 289             for path in watched_paths - paths_to_watch:
 290                 wd = path_to_wd_map.pop(path)
 291                 wd_to_path_map.pop(wd)
 292                 try:
 293                     inotify.rm_watch(self._inotify_fd, wd)
 294                 except OSError as e:
 295                     if e.errno == errno.EINVAL:
 296                         # This error can occur if the target of the watch was
 297                         # removed on the filesystem before we call
 298                         # inotify.rm_watch() so ignore it.
 299                         continue
 300                     raise e
 301             for path in paths_to_watch - watched_paths:
 302                 try:
 303                     wd = inotify.add_watch(
 304                         self._inotify_fd, core.encode(path), self._ADD_MASK
 305                     )
 306                 except PermissionError:
 307                     continue
 308                 except OSError as e:
 309                     if e.errno in (errno.ENOENT, errno.ENOTDIR):
 310                         # These two errors should only occur as a result of
 311                         # race conditions:  the first if the directory
 312                         # referenced by path was removed or renamed before the
 313                         # call to inotify.add_watch(); the second if the
 314                         # directory referenced by path was replaced with a file
 315                         # before the call to inotify.add_watch().  Therefore we
 316                         # simply ignore them.
 317                         continue
 318                     raise e
 319                 wd_to_path_map[wd] = path
 320                 path_to_wd_map[path] = wd
 321
 322         def _check_event(self, wd, mask, name):
 323             if mask & inotify.IN_Q_OVERFLOW:
 324                 self._force_notify = True
 325             elif not mask & self._TRIGGER_MASK:
 326                 pass
 327             elif mask & inotify.IN_ISDIR:
 328                 pass
 329             elif wd in self._worktree_wd_to_path_map:
 330                 if self._use_check_ignore and name:
 331                     path = os.path.join(
 332                         self._worktree_wd_to_path_map[wd], core.decode(name)
 333                     )
 334                     self._file_paths.add(path)
 335                 else:
 336                     self._force_notify = True
 337             elif wd == self._git_dir_wd:
 338                 name = core.decode(name)
 339                 if name in ('HEAD', 'index'):
 340                     self._force_notify = True
 341                 elif name == 'config':
 342                     self._force_config = True
 343             elif wd in self._git_dir_wd_to_path_map and not core.decode(name).endswith(
 344                 '.lock'
 345             ):
 346                 self._force_notify = True
 347
 348         def _handle_events(self):
 349             for wd, mask, _, name in inotify.read_events(self._inotify_fd):
 350                 if not self._force_notify:
 351                     self._check_event(wd, mask, name)
 352
 353         def stop(self):
 354             self._running = False
 355             with self._lock:
 356                 if self._pipe_w is not None:
 357                     os.write(self._pipe_w, bchr(0))
 358             self.wait()
 359
 360
 361 if AVAILABLE == 'pywin32':
 362
 363     class _Win32Watch:
 364         def __init__(self, path, flags):
 365             self.flags = flags
 366
 367             self.handle = None
 368             self.event = None
 369
 370             try:
 371                 self.handle = win32file.CreateFileW(
 372                     path,
 373                     0x0001,  # FILE_LIST_DIRECTORY
 374                     win32con.FILE_SHARE_READ | win32con.FILE_SHARE_WRITE,
 375                     None,
 376                     win32con.OPEN_EXISTING,
 377                     win32con.FILE_FLAG_BACKUP_SEMANTICS | win32con.FILE_FLAG_OVERLAPPED,
 378                     None,
 379                 )
 380
 381                 self.buffer = win32file.AllocateReadBuffer(8192)
 382                 self.event = win32event.CreateEvent(None, True, False, None)
 383                 self.overlapped = pywintypes.OVERLAPPED()
 384                 self.overlapped.hEvent = self.event
 385                 self._start()
 386             except Exception:  # pylint: disable=broad-exception-caught,broad-except
 387                 self.close()
 388
 389         def append(self, events):
 390             """Append our event to the events list when valid"""
 391             if self.event is not None:
 392                 events.append(self.event)
 393
 394         def _start(self):
 395             if self.handle is None:
 396                 return
 397             win32file.ReadDirectoryChangesW(
 398                 self.handle, self.buffer, True, self.flags, self.overlapped
 399             )
 400
 401         def read(self):
 402             if self.handle is None or self.event is None:
 403                 return []
 404             if win32event.WaitForSingleObject(self.event, 0) == win32event.WAIT_TIMEOUT:
 405                 result = []
 406             else:
 407                 nbytes = win32file.GetOverlappedResult(
 408                     self.handle, self.overlapped, False
 409                 )
 410                 result = win32file.FILE_NOTIFY_INFORMATION(self.buffer, nbytes)
 411                 self._start()
 412             return result
 413
 414         def close(self):
 415             if self.handle is not None:
 416                 win32file.CancelIo(self.handle)
 417                 win32file.CloseHandle(self.handle)
 418             if self.event is not None:
 419                 win32file.CloseHandle(self.event)
 420
 421     class _Win32Thread(_BaseThread):
 422         _FLAGS = (
 423             win32con.FILE_NOTIFY_CHANGE_FILE_NAME
 424             | win32con.FILE_NOTIFY_CHANGE_DIR_NAME
 425             | win32con.FILE_NOTIFY_CHANGE_ATTRIBUTES
 426             | win32con.FILE_NOTIFY_CHANGE_SIZE
 427             | win32con.FILE_NOTIFY_CHANGE_LAST_WRITE
 428             | win32con.FILE_NOTIFY_CHANGE_SECURITY
 429         )
 430
 431         def __init__(self, context, monitor):
 432             _BaseThread.__init__(self, context, monitor)
 433             git = context.git
 434             worktree = git.worktree()
 435             if worktree is not None:
 436                 worktree = self._transform_path(core.abspath(worktree))
 437             self._worktree = worktree
 438             self._worktree_watch = None
 439             self._git_dir = self._transform_path(core.abspath(git.git_path()))
 440             self._git_dir_watch = None
 441             self._stop_event_lock = Lock()
 442             self._stop_event = None
 443
 444         @staticmethod
 445         def _transform_path(path):
 446             return path.replace('\\', '/').lower()
 447
 448         def run(self):
 449             try:
 450                 with self._stop_event_lock:
 451                     self._stop_event = win32event.CreateEvent(None, True, False, None)
 452
 453                 events = [self._stop_event]
 454
 455                 if self._worktree is not None:
 456                     self._worktree_watch = _Win32Watch(self._worktree, self._FLAGS)
 457                     self._worktree_watch.append(events)
 458
 459                 self._git_dir_watch = _Win32Watch(self._git_dir, self._FLAGS)
 460                 self._git_dir_watch.append(events)
 461
 462                 self._log_enabled_message()
 463
 464                 while self._running:
 465                     if self._pending:
 466                         timeout = self._NOTIFICATION_DELAY
 467                     else:
 468                         timeout = win32event.INFINITE
 469                     status = win32event.WaitForMultipleObjects(events, False, timeout)
 470                     if not self._running:
 471                         break
 472                     if status == win32event.WAIT_TIMEOUT:
 473                         self.notify()
 474                     else:
 475                         self._handle_results()
 476             finally:
 477                 with self._stop_event_lock:
 478                     if self._stop_event is not None:
 479                         win32file.CloseHandle(self._stop_event)
 480                         self._stop_event = None
 481                 if self._worktree_watch is not None:
 482                     self._worktree_watch.close()
 483                 if self._git_dir_watch is not None:
 484                     self._git_dir_watch.close()
 485
 486         def _handle_results(self):
 487             if self._worktree_watch is not None:
 488                 for _, path in self._worktree_watch.read():
 489                     if not self._running:
 490                         break
 491                     if self._force_notify:
 492                         continue
 493                     path = self._worktree + '/' + self._transform_path(path)
 494                     if (
 495                         path != self._git_dir
 496                         and not path.startswith(self._git_dir + '/')
 497                         and not os.path.isdir(path)
 498                     ):
 499                         if self._use_check_ignore:
 500                             self._file_paths.add(path)
 501                         else:
 502                             self._force_notify = True
 503             for _, path in self._git_dir_watch.read():
 504                 if not self._running:
 505                     break
 506                 if self._force_notify:
 507                     continue
 508                 path = self._transform_path(path)
 509                 if path.endswith('.lock'):
 510                     continue
 511                 if path == 'config':
 512                     self._force_config = True
 513                     continue
 514                 if path == 'head' or path == 'index' or path.startswith('refs/'):
 515                     self._force_notify = True
 516
 517         def stop(self):
 518             self._running = False
 519             with self._stop_event_lock:
 520                 if self._stop_event is not None:
 521                     win32event.SetEvent(self._stop_event)
 522             self.wait()
 523
 524
 525 def create(context):
 526     thread_class = None
 527     cfg = context.cfg
 528     if not cfg.get('cola.inotify', default=True):
 529         msg = N_(
 530             'File system change monitoring: disabled because'
 531             ' "cola.inotify" is false.\n'
 532         )
 533         Interaction.log(msg)
 534     elif AVAILABLE == 'inotify':
 535         thread_class = _InotifyThread
 536     elif AVAILABLE == 'pywin32':
 537         thread_class = _Win32Thread
 538     else:
 539         if utils.is_win32():
 540             msg = N_(
 541                 'File system change monitoring: disabled because pywin32'
 542                 ' is not installed.\n'
 543             )
 544             Interaction.log(msg)
 545         elif utils.is_linux():
 546             msg = N_(
 547                 'File system change monitoring: disabled because libc'
 548                 ' does not support the inotify system calls.\n'
 549             )
 550             Interaction.log(msg)
 551     return _Monitor(context, thread_class)