cola/fsmonitor.py

   1 # Copyright (C) 2008-2024 David Aguilar
   2 # Copyright (C) 2015 Daniel Harding
   3 """Filesystem monitor for Linux and Windows
   4
   5 Linux monitoring uses using inotify.
   6 Windows monitoring uses pywin32 and the ReadDirectoryChanges function.
   7
   8 """
   9 import errno
  10 import os
  11 import os.path
  12 import select
  13 from threading import Lock
  14
  15 from qtpy import QtCore
  16 from qtpy.QtCore import Signal
  17
  18 from . import utils
  19 from . import core
  20 from . import gitcmds
  21 from . import version
  22 from .compat import bchr
  23 from .i18n import N_
  24 from .interaction import Interaction
  25
  26 AVAILABLE = None
  27
  28 pywintypes = None
  29 win32file = None
  30 win32con = None
  31 win32event = None
  32 if utils.is_win32():
  33     try:
  34         import pywintypes
  35         import win32con
  36         import win32event
  37         import win32file
  38
  39         AVAILABLE = 'pywin32'
  40     except ImportError:
  41         pass
  42
  43 elif utils.is_linux():
  44     try:
  45         from . import inotify
  46     except ImportError:
  47         pass
  48     else:
  49         AVAILABLE = 'inotify'
  50
  51
  52 class _Monitor(QtCore.QObject):
  53     files_changed = Signal()
  54     config_changed = Signal()
  55
  56     def __init__(self, context, thread_class):
  57         QtCore.QObject.__init__(self)
  58         self.context = context
  59         self._thread_class = thread_class
  60         self._thread = None
  61
  62     def start(self):
  63         if self._thread_class is not None:
  64             assert self._thread is None
  65             self._thread = self._thread_class(self.context, self)
  66             self._thread.start()
  67
  68     def stop(self):
  69         if self._thread_class is not None:
  70             assert self._thread is not None
  71             self._thread.stop()
  72             self._thread.wait()
  73             self._thread = None
  74
  75     def refresh(self):
  76         if self._thread is not None:
  77             self._thread.refresh()
  78
  79
  80 class _BaseThread(QtCore.QThread):
  81     #: The delay, in milliseconds, between detecting file system modification
  82     #: and triggering the 'files_changed' signal, to coalesce multiple
  83     #: modifications into a single signal.
  84     _NOTIFICATION_DELAY = 888
  85
  86     def __init__(self, context, monitor):
  87         QtCore.QThread.__init__(self)
  88         self.context = context
  89         self._monitor = monitor
  90         self._running = True
  91         self._use_check_ignore = version.check_git(context, 'check-ignore')
  92         self._force_notify = False
  93         self._force_config = False
  94         self._file_paths = set()
  95
  96     @property
  97     def _pending(self):
  98         return self._force_notify or self._file_paths or self._force_config
  99
 100     def refresh(self):
 101         """Do any housekeeping necessary in response to repository changes."""
 102         return
 103
 104     def notify(self):
 105         """Notifies all observers"""
 106         do_notify = False
 107         do_config = False
 108         if self._force_config:
 109             do_config = True
 110         if self._force_notify:
 111             do_notify = True
 112         elif self._file_paths:
 113             proc = core.start_command(
 114                 ['git', 'check-ignore', '--verbose', '--non-matching', '-z', '--stdin']
 115             )
 116             path_list = bchr(0).join(core.encode(path) for path in self._file_paths)
 117             out, _ = proc.communicate(path_list)
 118             if proc.returncode:
 119                 do_notify = True
 120             else:
 121                 # Each output record is four fields separated by NULL
 122                 # characters (records are also separated by NULL characters):
 123                 # <source> <NULL> <linenum> <NULL> <pattern> <NULL> <pathname>
 124                 # For paths which are not ignored, all fields will be empty
 125                 # except for <pathname>.  So to see if we have any non-ignored
 126                 # files, we simply check every fourth field to see if any of
 127                 # them are empty.
 128                 source_fields = out.split(bchr(0))[0:-1:4]
 129                 do_notify = not all(source_fields)
 130         self._force_notify = False
 131         self._force_config = False
 132         self._file_paths = set()
 133
 134         # "files changed" is a bigger hammer than "config changed".
 135         # and is a superset relative to what is done in response to the
 136         # signal.  Thus, the "elif" below avoids repeated work that
 137         # would be done if it were a simple "if" check instead.
 138         if do_notify:
 139             self._monitor.files_changed.emit()
 140         elif do_config:
 141             self._monitor.config_changed.emit()
 142
 143     @staticmethod
 144     def _log_enabled_message():
 145         msg = N_('File system change monitoring: enabled.\n')
 146         Interaction.log(msg)
 147
 148
 149 if AVAILABLE == 'inotify':
 150
 151     class _InotifyThread(_BaseThread):
 152         _TRIGGER_MASK = (
 153             inotify.IN_ATTRIB
 154             | inotify.IN_CLOSE_WRITE
 155             | inotify.IN_CREATE
 156             | inotify.IN_DELETE
 157             | inotify.IN_MODIFY
 158             | inotify.IN_MOVED_FROM
 159             | inotify.IN_MOVED_TO
 160         )
 161         _ADD_MASK = _TRIGGER_MASK | inotify.IN_EXCL_UNLINK | inotify.IN_ONLYDIR
 162
 163         def __init__(self, context, monitor):
 164             _BaseThread.__init__(self, context, monitor)
 165             git = context.git
 166             worktree = git.worktree()
 167             if worktree is not None:
 168                 worktree = core.abspath(worktree)
 169             self._worktree = worktree
 170             self._git_dir = git.git_path()
 171             self._lock = Lock()
 172             self._inotify_fd = None
 173             self._pipe_r = None
 174             self._pipe_w = None
 175             self._worktree_wd_to_path_map = {}
 176             self._worktree_path_to_wd_map = {}
 177             self._git_dir_wd_to_path_map = {}
 178             self._git_dir_path_to_wd_map = {}
 179             self._git_dir_wd = None
 180
 181         @staticmethod
 182         def _log_out_of_wds_message():
 183             msg = N_(
 184                 'File system change monitoring: disabled because the'
 185                 ' limit on the total number of inotify watches was'
 186                 ' reached.  You may be able to increase the limit on'
 187                 ' the number of watches by running:\n'
 188                 '\n'
 189                 '    echo fs.inotify.max_user_watches=100000 |'
 190                 ' sudo tee -a /etc/sysctl.conf &&'
 191                 ' sudo sysctl -p\n'
 192             )
 193             Interaction.log(msg)
 194
 195         def run(self):
 196             try:
 197                 with self._lock:
 198                     try:
 199                         self._inotify_fd = inotify.init()
 200                     except OSError as e:
 201                         self._inotify_fd = None
 202                         self._running = False
 203                         if e.errno == errno.EMFILE:
 204                             self._log_out_of_wds_message()
 205                         return
 206                     self._pipe_r, self._pipe_w = os.pipe()
 207
 208                 poll_obj = select.poll()
 209                 poll_obj.register(self._inotify_fd, select.POLLIN)
 210                 poll_obj.register(self._pipe_r, select.POLLIN)
 211
 212                 self.refresh()
 213
 214                 if self._running:
 215                     self._log_enabled_message()
 216                     self._process_events(poll_obj)
 217             finally:
 218                 self._close_fds()
 219
 220         def _process_events(self, poll_obj):
 221             while self._running:
 222                 if self._pending:
 223                     timeout = self._NOTIFICATION_DELAY
 224                 else:
 225                     timeout = None
 226                 try:
 227                     events = poll_obj.poll(timeout)
 228                 except OSError:
 229                     continue
 230                 else:
 231                     if not self._running:
 232                         break
 233                     if not events:
 234                         self.notify()
 235                     else:
 236                         for fd, _ in events:
 237                             if fd == self._inotify_fd:
 238                                 self._handle_events()
 239
 240         def _close_fds(self):
 241             with self._lock:
 242                 if self._inotify_fd is not None:
 243                     os.close(self._inotify_fd)
 244                     self._inotify_fd = None
 245                 if self._pipe_r is not None:
 246                     os.close(self._pipe_r)
 247                     self._pipe_r = None
 248                     os.close(self._pipe_w)
 249                     self._pipe_w = None
 250
 251         def refresh(self):
 252             with self._lock:
 253                 self._refresh()
 254
 255         def _refresh(self):
 256             if self._inotify_fd is None:
 257                 return
 258             context = self.context
 259             try:
 260                 if self._worktree is not None:
 261                     tracked_dirs = {
 262                         os.path.dirname(os.path.join(self._worktree, path))
 263                         for path in gitcmds.tracked_files(context)
 264                     }
 265                     self._refresh_watches(
 266                         tracked_dirs,
 267                         self._worktree_wd_to_path_map,
 268                         self._worktree_path_to_wd_map,
 269                     )
 270                 git_dirs = set()
 271                 git_dirs.add(self._git_dir)
 272                 for dirpath, _, _ in core.walk(os.path.join(self._git_dir, 'refs')):
 273                     git_dirs.add(dirpath)
 274                 self._refresh_watches(
 275                     git_dirs, self._git_dir_wd_to_path_map, self._git_dir_path_to_wd_map
 276                 )
 277                 self._git_dir_wd = self._git_dir_path_to_wd_map.get(self._git_dir)
 278             except OSError as e:
 279                 if e.errno in (errno.ENOSPC, errno.EMFILE):
 280                     self._log_out_of_wds_message()
 281                     self._running = False
 282                 else:
 283                     raise
 284
 285         def _refresh_watches(self, paths_to_watch, wd_to_path_map, path_to_wd_map):
 286             watched_paths = set(path_to_wd_map)
 287             for path in watched_paths - paths_to_watch:
 288                 wd = path_to_wd_map.pop(path)
 289                 wd_to_path_map.pop(wd)
 290                 try:
 291                     inotify.rm_watch(self._inotify_fd, wd)
 292                 except OSError as e:
 293                     if e.errno == errno.EINVAL:
 294                         # This error can occur if the target of the watch was
 295                         # removed on the filesystem before we call
 296                         # inotify.rm_watch() so ignore it.
 297                         continue
 298                     raise e
 299             for path in paths_to_watch - watched_paths:
 300                 try:
 301                     wd = inotify.add_watch(
 302                         self._inotify_fd, core.encode(path), self._ADD_MASK
 303                     )
 304                 except PermissionError:
 305                     continue
 306                 except OSError as e:
 307                     if e.errno in (errno.ENOENT, errno.ENOTDIR):
 308                         # These two errors should only occur as a result of
 309                         # race conditions:  the first if the directory
 310                         # referenced by path was removed or renamed before the
 311                         # call to inotify.add_watch(); the second if the
 312                         # directory referenced by path was replaced with a file
 313                         # before the call to inotify.add_watch().  Therefore we
 314                         # simply ignore them.
 315                         continue
 316                     raise e
 317                 wd_to_path_map[wd] = path
 318                 path_to_wd_map[path] = wd
 319
 320         def _check_event(self, wd, mask, name):
 321             if mask & inotify.IN_Q_OVERFLOW:
 322                 self._force_notify = True
 323             elif not mask & self._TRIGGER_MASK:
 324                 pass
 325             elif mask & inotify.IN_ISDIR:
 326                 pass
 327             elif wd in self._worktree_wd_to_path_map:
 328                 if self._use_check_ignore and name:
 329                     path = os.path.join(
 330                         self._worktree_wd_to_path_map[wd], core.decode(name)
 331                     )
 332                     self._file_paths.add(path)
 333                 else:
 334                     self._force_notify = True
 335             elif wd == self._git_dir_wd:
 336                 name = core.decode(name)
 337                 if name in ('HEAD', 'index'):
 338                     self._force_notify = True
 339                 elif name == 'config':
 340                     self._force_config = True
 341             elif wd in self._git_dir_wd_to_path_map and not core.decode(name).endswith(
 342                 '.lock'
 343             ):
 344                 self._force_notify = True
 345
 346         def _handle_events(self):
 347             for wd, mask, _, name in inotify.read_events(self._inotify_fd):
 348                 if not self._force_notify:
 349                     self._check_event(wd, mask, name)
 350
 351         def stop(self):
 352             self._running = False
 353             with self._lock:
 354                 if self._pipe_w is not None:
 355                     os.write(self._pipe_w, bchr(0))
 356             self.wait()
 357
 358
 359 if AVAILABLE == 'pywin32':
 360
 361     class _Win32Watch:
 362         def __init__(self, path, flags):
 363             self.flags = flags
 364
 365             self.handle = None
 366             self.event = None
 367
 368             try:
 369                 self.handle = win32file.CreateFileW(
 370                     path,
 371                     0x0001,  # FILE_LIST_DIRECTORY
 372                     win32con.FILE_SHARE_READ | win32con.FILE_SHARE_WRITE,
 373                     None,
 374                     win32con.OPEN_EXISTING,
 375                     win32con.FILE_FLAG_BACKUP_SEMANTICS | win32con.FILE_FLAG_OVERLAPPED,
 376                     None,
 377                 )
 378
 379                 self.buffer = win32file.AllocateReadBuffer(8192)
 380                 self.event = win32event.CreateEvent(None, True, False, None)
 381                 self.overlapped = pywintypes.OVERLAPPED()
 382                 self.overlapped.hEvent = self.event
 383                 self._start()
 384             except Exception:
 385                 self.close()
 386
 387         def append(self, events):
 388             """Append our event to the events list when valid"""
 389             if self.event is not None:
 390                 events.append(self.event)
 391
 392         def _start(self):
 393             if self.handle is None:
 394                 return
 395             win32file.ReadDirectoryChangesW(
 396                 self.handle, self.buffer, True, self.flags, self.overlapped
 397             )
 398
 399         def read(self):
 400             if self.handle is None or self.event is None:
 401                 return []
 402             if win32event.WaitForSingleObject(self.event, 0) == win32event.WAIT_TIMEOUT:
 403                 result = []
 404             else:
 405                 nbytes = win32file.GetOverlappedResult(
 406                     self.handle, self.overlapped, False
 407                 )
 408                 result = win32file.FILE_NOTIFY_INFORMATION(self.buffer, nbytes)
 409                 self._start()
 410             return result
 411
 412         def close(self):
 413             if self.handle is not None:
 414                 win32file.CancelIo(self.handle)
 415                 win32file.CloseHandle(self.handle)
 416             if self.event is not None:
 417                 win32file.CloseHandle(self.event)
 418
 419     class _Win32Thread(_BaseThread):
 420         _FLAGS = (
 421             win32con.FILE_NOTIFY_CHANGE_FILE_NAME
 422             | win32con.FILE_NOTIFY_CHANGE_DIR_NAME
 423             | win32con.FILE_NOTIFY_CHANGE_ATTRIBUTES
 424             | win32con.FILE_NOTIFY_CHANGE_SIZE
 425             | win32con.FILE_NOTIFY_CHANGE_LAST_WRITE
 426             | win32con.FILE_NOTIFY_CHANGE_SECURITY
 427         )
 428
 429         def __init__(self, context, monitor):
 430             _BaseThread.__init__(self, context, monitor)
 431             git = context.git
 432             worktree = git.worktree()
 433             if worktree is not None:
 434                 worktree = self._transform_path(core.abspath(worktree))
 435             self._worktree = worktree
 436             self._worktree_watch = None
 437             self._git_dir = self._transform_path(core.abspath(git.git_path()))
 438             self._git_dir_watch = None
 439             self._stop_event_lock = Lock()
 440             self._stop_event = None
 441
 442         @staticmethod
 443         def _transform_path(path):
 444             return path.replace('\\', '/').lower()
 445
 446         def run(self):
 447             try:
 448                 with self._stop_event_lock:
 449                     self._stop_event = win32event.CreateEvent(None, True, False, None)
 450
 451                 events = [self._stop_event]
 452
 453                 if self._worktree is not None:
 454                     self._worktree_watch = _Win32Watch(self._worktree, self._FLAGS)
 455                     self._worktree_watch.append(events)
 456
 457                 self._git_dir_watch = _Win32Watch(self._git_dir, self._FLAGS)
 458                 self._git_dir_watch.append(events)
 459
 460                 self._log_enabled_message()
 461
 462                 while self._running:
 463                     if self._pending:
 464                         timeout = self._NOTIFICATION_DELAY
 465                     else:
 466                         timeout = win32event.INFINITE
 467                     status = win32event.WaitForMultipleObjects(events, False, timeout)
 468                     if not self._running:
 469                         break
 470                     if status == win32event.WAIT_TIMEOUT:
 471                         self.notify()
 472                     else:
 473                         self._handle_results()
 474             finally:
 475                 with self._stop_event_lock:
 476                     if self._stop_event is not None:
 477                         win32file.CloseHandle(self._stop_event)
 478                         self._stop_event = None
 479                 if self._worktree_watch is not None:
 480                     self._worktree_watch.close()
 481                 if self._git_dir_watch is not None:
 482                     self._git_dir_watch.close()
 483
 484         def _handle_results(self):
 485             if self._worktree_watch is not None:
 486                 for _, path in self._worktree_watch.read():
 487                     if not self._running:
 488                         break
 489                     if self._force_notify:
 490                         continue
 491                     path = self._worktree + '/' + self._transform_path(path)
 492                     if (
 493                         path != self._git_dir
 494                         and not path.startswith(self._git_dir + '/')
 495                         and not os.path.isdir(path)
 496                     ):
 497                         if self._use_check_ignore:
 498                             self._file_paths.add(path)
 499                         else:
 500                             self._force_notify = True
 501             for _, path in self._git_dir_watch.read():
 502                 if not self._running:
 503                     break
 504                 if self._force_notify:
 505                     continue
 506                 path = self._transform_path(path)
 507                 if path.endswith('.lock'):
 508                     continue
 509                 if path == 'config':
 510                     self._force_config = True
 511                     continue
 512                 if path == 'head' or path == 'index' or path.startswith('refs/'):
 513                     self._force_notify = True
 514
 515         def stop(self):
 516             self._running = False
 517             with self._stop_event_lock:
 518                 if self._stop_event is not None:
 519                     win32event.SetEvent(self._stop_event)
 520             self.wait()
 521
 522
 523 def create(context):
 524     thread_class = None
 525     cfg = context.cfg
 526     if not cfg.get('cola.inotify', default=True):
 527         msg = N_(
 528             'File system change monitoring: disabled because'
 529             ' "cola.inotify" is false.\n'
 530         )
 531         Interaction.log(msg)
 532     elif AVAILABLE == 'inotify':
 533         thread_class = _InotifyThread
 534     elif AVAILABLE == 'pywin32':
 535         thread_class = _Win32Thread
 536     else:
 537         if utils.is_win32():
 538             msg = N_(
 539                 'File system change monitoring: disabled because pywin32'
 540                 ' is not installed.\n'
 541             )
 542             Interaction.log(msg)
 543         elif utils.is_linux():
 544             msg = N_(
 545                 'File system change monitoring: disabled because libc'
 546                 ' does not support the inotify system calls.\n'
 547             )
 548             Interaction.log(msg)
 549     return _Monitor(context, thread_class)