cola/fsmonitor.py

   1 # Copyright (C) 2008-2017 David Aguilar
   2 # Copyright (C) 2015 Daniel Harding
   3 """Filesystem monitor for Linux and Windows
   4
   5 Linux monitoring uses using inotify.
   6 Windows monitoring uses pywin32 and the ReadDirectoryChanges function.
   7
   8 """
   9 from __future__ import division, absolute_import, unicode_literals
  10 import errno
  11 import os
  12 import os.path
  13 import select
  14 from threading import Lock
  15
  16 from qtpy import QtCore
  17 from qtpy.QtCore import Signal
  18
  19 from . import utils
  20 from . import core
  21 from . import gitcmds
  22 from . import version
  23 from .compat import bchr
  24 from .i18n import N_
  25 from .interaction import Interaction
  26
  27 AVAILABLE = None
  28
  29 if utils.is_win32():
  30     try:
  31         import pywintypes
  32         import win32con
  33         import win32event
  34         import win32file
  35     except ImportError:
  36         pass
  37     else:
  38         AVAILABLE = 'pywin32'
  39 elif utils.is_linux():
  40     try:
  41         from . import inotify
  42     except ImportError:
  43         pass
  44     else:
  45         AVAILABLE = 'inotify'
  46
  47
  48 class _Monitor(QtCore.QObject):
  49
  50     files_changed = Signal()
  51     config_changed = Signal()
  52
  53     def __init__(self, context, thread_class):
  54         QtCore.QObject.__init__(self)
  55         self.context = context
  56         self._thread_class = thread_class
  57         self._thread = None
  58
  59     def start(self):
  60         if self._thread_class is not None:
  61             assert self._thread is None
  62             self._thread = self._thread_class(self.context, self)
  63             self._thread.start()
  64
  65     def stop(self):
  66         if self._thread_class is not None:
  67             assert self._thread is not None
  68             self._thread.stop()
  69             self._thread.wait()
  70             self._thread = None
  71
  72     def refresh(self):
  73         if self._thread is not None:
  74             self._thread.refresh()
  75
  76
  77 class _BaseThread(QtCore.QThread):
  78     #: The delay, in milliseconds, between detecting file system modification
  79     #: and triggering the 'files_changed' signal, to coalesce multiple
  80     #: modifications into a single signal.
  81     _NOTIFICATION_DELAY = 888
  82
  83     def __init__(self, context, monitor):
  84         QtCore.QThread.__init__(self)
  85         self.context = context
  86         self._monitor = monitor
  87         self._running = True
  88         self._use_check_ignore = version.check_git(context, 'check-ignore')
  89         self._force_notify = False
  90         self._force_config = False
  91         self._file_paths = set()
  92
  93     @property
  94     def _pending(self):
  95         return self._force_notify or self._file_paths or self._force_config
  96
  97     def refresh(self):
  98         """Do any housekeeping necessary in response to repository changes."""
  99         pass
 100
 101     def notify(self):
 102         """Notifies all observers"""
 103         do_notify = False
 104         do_config = False
 105         if self._force_config:
 106             do_config = True
 107         if self._force_notify:
 108             do_notify = True
 109         elif self._file_paths:
 110             proc = core.start_command(['git', 'check-ignore', '--verbose',
 111                                        '--non-matching', '-z', '--stdin'])
 112             path_list = bchr(0).join(core.encode(path)
 113                                      for path in self._file_paths)
 114             out, _ = proc.communicate(path_list)
 115             if proc.returncode:
 116                 do_notify = True
 117             else:
 118                 # Each output record is four fields separated by NULL
 119                 # characters (records are also separated by NULL characters):
 120                 # <source> <NULL> <linenum> <NULL> <pattern> <NULL> <pathname>
 121                 # For paths which are not ignored, all fields will be empty
 122                 # except for <pathname>.  So to see if we have any non-ignored
 123                 # files, we simply check every fourth field to see if any of
 124                 # them are empty.
 125                 source_fields = out.split(bchr(0))[0:-1:4]
 126                 do_notify = not all(source_fields)
 127         self._force_notify = False
 128         self._force_config = False
 129         self._file_paths = set()
 130
 131         # "files changed" is a bigger hammer than "config changed".
 132         # and is a superset relative to what is done in response to the
 133         # signal.  Thus, the "elif" below avoids repeated work that
 134         # would be done if it were a simple "if" check instead.
 135         if do_notify:
 136             self._monitor.files_changed.emit()
 137         elif do_config:
 138             self._monitor.config_changed.emit()
 139
 140     @staticmethod
 141     def _log_enabled_message():
 142         msg = N_('File system change monitoring: enabled.\n')
 143         Interaction.log(msg)
 144
 145
 146 if AVAILABLE == 'inotify':
 147
 148     class _InotifyThread(_BaseThread):
 149         _TRIGGER_MASK = (
 150             inotify.IN_ATTRIB |
 151             inotify.IN_CLOSE_WRITE |
 152             inotify.IN_CREATE |
 153             inotify.IN_DELETE |
 154             inotify.IN_MODIFY |
 155             inotify.IN_MOVED_FROM |
 156             inotify.IN_MOVED_TO
 157         )
 158         _ADD_MASK = (
 159             _TRIGGER_MASK |
 160             inotify.IN_EXCL_UNLINK |
 161             inotify.IN_ONLYDIR
 162         )
 163
 164         def __init__(self, context, monitor):
 165             _BaseThread.__init__(self, context, monitor)
 166             git = context.git
 167             worktree = git.worktree()
 168             if worktree is not None:
 169                 worktree = core.abspath(worktree)
 170             self._worktree = worktree
 171             self._git_dir = git.git_path()
 172             self._lock = Lock()
 173             self._inotify_fd = None
 174             self._pipe_r = None
 175             self._pipe_w = None
 176             self._worktree_wd_to_path_map = {}
 177             self._worktree_path_to_wd_map = {}
 178             self._git_dir_wd_to_path_map = {}
 179             self._git_dir_path_to_wd_map = {}
 180             self._git_dir_wd = None
 181
 182         @staticmethod
 183         def _log_out_of_wds_message():
 184             msg = N_('File system change monitoring: disabled because the'
 185                      ' limit on the total number of inotify watches was'
 186                      ' reached.  You may be able to increase the limit on'
 187                      ' the number of watches by running:\n'
 188                      '\n'
 189                      '    echo fs.inotify.max_user_watches=100000 |'
 190                      ' sudo tee -a /etc/sysctl.conf &&'
 191                      ' sudo sysctl -p\n')
 192             Interaction.log(msg)
 193
 194         def run(self):
 195             try:
 196                 with self._lock:
 197                     self._inotify_fd = inotify.init()
 198                     self._pipe_r, self._pipe_w = os.pipe()
 199
 200                 poll_obj = select.poll()
 201                 poll_obj.register(self._inotify_fd, select.POLLIN)
 202                 poll_obj.register(self._pipe_r, select.POLLIN)
 203
 204                 self.refresh()
 205
 206                 self._log_enabled_message()
 207
 208                 while self._running:
 209                     if self._pending:
 210                         timeout = self._NOTIFICATION_DELAY
 211                     else:
 212                         timeout = None
 213                     try:
 214                         events = poll_obj.poll(timeout)
 215                     except OSError as e:
 216                         if e.errno == errno.EINTR:
 217                             continue
 218                         else:
 219                             raise
 220                     except select.error:
 221                         continue
 222                     else:
 223                         if not self._running:
 224                             break
 225                         elif not events:
 226                             self.notify()
 227                         else:
 228                             for (fd, _) in events:
 229                                 if fd == self._inotify_fd:
 230                                     self._handle_events()
 231             finally:
 232                 with self._lock:
 233                     if self._inotify_fd is not None:
 234                         os.close(self._inotify_fd)
 235                         self._inotify_fd = None
 236                     if self._pipe_r is not None:
 237                         os.close(self._pipe_r)
 238                         self._pipe_r = None
 239                         os.close(self._pipe_w)
 240                         self._pipe_w = None
 241
 242         def refresh(self):
 243             with self._lock:
 244                 self._refresh()
 245
 246         def _refresh(self):
 247             if self._inotify_fd is None:
 248                 return
 249             context = self.context
 250             try:
 251                 if self._worktree is not None:
 252                     tracked_dirs = set([
 253                         os.path.dirname(os.path.join(self._worktree, path))
 254                         for path in gitcmds.tracked_files(context)])
 255                     self._refresh_watches(tracked_dirs,
 256                                           self._worktree_wd_to_path_map,
 257                                           self._worktree_path_to_wd_map)
 258                 git_dirs = set()
 259                 git_dirs.add(self._git_dir)
 260                 for dirpath, _, _ in core.walk(
 261                         os.path.join(self._git_dir, 'refs')):
 262                     git_dirs.add(dirpath)
 263                 self._refresh_watches(git_dirs,
 264                                       self._git_dir_wd_to_path_map,
 265                                       self._git_dir_path_to_wd_map)
 266                 self._git_dir_wd = \
 267                     self._git_dir_path_to_wd_map.get(self._git_dir)
 268             except OSError as e:
 269                 if e.errno == errno.ENOSPC:
 270                     self._log_out_of_wds_message()
 271                     self._running = False
 272                 else:
 273                     raise
 274
 275         def _refresh_watches(self, paths_to_watch, wd_to_path_map,
 276                              path_to_wd_map):
 277             watched_paths = set(path_to_wd_map)
 278             for path in watched_paths - paths_to_watch:
 279                 wd = path_to_wd_map.pop(path)
 280                 wd_to_path_map.pop(wd)
 281                 try:
 282                     inotify.rm_watch(self._inotify_fd, wd)
 283                 except OSError as e:
 284                     if e.errno == errno.EINVAL:
 285                         # This error can occur if the target of the wd was
 286                         # removed on the filesystem before we call
 287                         # inotify.rm_watch() so ignore it.
 288                         pass
 289                     else:
 290                         raise
 291             for path in paths_to_watch - watched_paths:
 292                 try:
 293                     wd = inotify.add_watch(self._inotify_fd, core.encode(path),
 294                                            self._ADD_MASK)
 295                 except OSError as e:
 296                     if e.errno in (errno.ENOENT, errno.ENOTDIR):
 297                         # These two errors should only occur as a result of
 298                         # race conditions:  the first if the directory
 299                         # referenced by path was removed or renamed before the
 300                         # call to inotify.add_watch(); the second if the
 301                         # directory referenced by path was replaced with a file
 302                         # before the call to inotify.add_watch().  Therefore we
 303                         # simply ignore them.
 304                         pass
 305                     else:
 306                         raise
 307                 else:
 308                     wd_to_path_map[wd] = path
 309                     path_to_wd_map[path] = wd
 310
 311         def _check_event(self, wd, mask, name):
 312             if mask & inotify.IN_Q_OVERFLOW:
 313                 self._force_notify = True
 314             elif not mask & self._TRIGGER_MASK:
 315                 pass
 316             elif mask & inotify.IN_ISDIR:
 317                 pass
 318             elif wd in self._worktree_wd_to_path_map:
 319                 if self._use_check_ignore and name:
 320                     path = os.path.join(self._worktree_wd_to_path_map[wd],
 321                                         core.decode(name))
 322                     self._file_paths.add(path)
 323                 else:
 324                     self._force_notify = True
 325             elif wd == self._git_dir_wd:
 326                 name = core.decode(name)
 327                 if name == 'HEAD' or name == 'index':
 328                     self._force_notify = True
 329                 elif name == 'config':
 330                     self._force_config = True
 331             elif (wd in self._git_dir_wd_to_path_map
 332                   and not core.decode(name).endswith('.lock')):
 333                 self._force_notify = True
 334
 335         def _handle_events(self):
 336             for wd, mask, _, name in \
 337                     inotify.read_events(self._inotify_fd):
 338                 if not self._force_notify:
 339                     self._check_event(wd, mask, name)
 340
 341         def stop(self):
 342             self._running = False
 343             with self._lock:
 344                 if self._pipe_w is not None:
 345                     os.write(self._pipe_w, bchr(0))
 346             self.wait()
 347
 348
 349 if AVAILABLE == 'pywin32':
 350
 351     class _Win32Watch(object):
 352
 353         def __init__(self, path, flags):
 354             self.flags = flags
 355
 356             self.handle = None
 357             self.event = None
 358
 359             try:
 360                 self.handle = win32file.CreateFileW(
 361                     path,
 362                     0x0001,  # FILE_LIST_DIRECTORY
 363                     win32con.FILE_SHARE_READ | win32con.FILE_SHARE_WRITE,
 364                     None,
 365                     win32con.OPEN_EXISTING,
 366                     win32con.FILE_FLAG_BACKUP_SEMANTICS |
 367                     win32con.FILE_FLAG_OVERLAPPED,
 368                     None)
 369
 370                 self.buffer = win32file.AllocateReadBuffer(8192)
 371                 self.event = win32event.CreateEvent(None, True, False, None)
 372                 self.overlapped = pywintypes.OVERLAPPED()
 373                 self.overlapped.hEvent = self.event
 374                 self._start()
 375             except Exception:
 376                 self.close()
 377                 raise
 378
 379         def _start(self):
 380             win32file.ReadDirectoryChangesW(self.handle, self.buffer, True,
 381                                             self.flags, self.overlapped)
 382
 383         def read(self):
 384             if win32event.WaitForSingleObject(self.event, 0) \
 385                     == win32event.WAIT_TIMEOUT:
 386                 result = []
 387             else:
 388                 nbytes = win32file.GetOverlappedResult(self.handle,
 389                                                        self.overlapped, False)
 390                 result = win32file.FILE_NOTIFY_INFORMATION(self.buffer, nbytes)
 391                 self._start()
 392             return result
 393
 394         def close(self):
 395             if self.handle is not None:
 396                 win32file.CancelIo(self.handle)
 397                 win32file.CloseHandle(self.handle)
 398             if self.event is not None:
 399                 win32file.CloseHandle(self.event)
 400
 401     class _Win32Thread(_BaseThread):
 402         _FLAGS = (win32con.FILE_NOTIFY_CHANGE_FILE_NAME |
 403                   win32con.FILE_NOTIFY_CHANGE_DIR_NAME |
 404                   win32con.FILE_NOTIFY_CHANGE_ATTRIBUTES |
 405                   win32con.FILE_NOTIFY_CHANGE_SIZE |
 406                   win32con.FILE_NOTIFY_CHANGE_LAST_WRITE |
 407                   win32con.FILE_NOTIFY_CHANGE_SECURITY)
 408
 409         def __init__(self, context, monitor):
 410             _BaseThread.__init__(self, context, monitor)
 411             git = context.git
 412             worktree = git.worktree()
 413             if worktree is not None:
 414                 worktree = self._transform_path(core.abspath(worktree))
 415             self._worktree = worktree
 416             self._worktree_watch = None
 417             self._git_dir = self._transform_path(core.abspath(git.git_path()))
 418             self._git_dir_watch = None
 419             self._stop_event_lock = Lock()
 420             self._stop_event = None
 421
 422         @staticmethod
 423         def _transform_path(path):
 424             return path.replace('\\', '/').lower()
 425
 426         def _read_watch(self, watch):
 427             if win32event.WaitForSingleObject(watch.event, 0) \
 428                     == win32event.WAIT_TIMEOUT:
 429                 nbytes = 0
 430             else:
 431                 nbytes = win32file.GetOverlappedResult(watch.handle,
 432                                                        watch.overlapped, False)
 433             return win32file.FILE_NOTIFY_INFORMATION(watch.buffer, nbytes)
 434
 435         def run(self):
 436             try:
 437                 with self._stop_event_lock:
 438                     self._stop_event = win32event.CreateEvent(None, True,
 439                                                               False, None)
 440
 441                 events = [self._stop_event]
 442
 443                 if self._worktree is not None:
 444                     self._worktree_watch = _Win32Watch(self._worktree,
 445                                                        self._FLAGS)
 446                     events.append(self._worktree_watch.event)
 447
 448                 self._git_dir_watch = _Win32Watch(self._git_dir, self._FLAGS)
 449                 events.append(self._git_dir_watch.event)
 450
 451                 self._log_enabled_message()
 452
 453                 while self._running:
 454                     if self._pending:
 455                         timeout = self._NOTIFICATION_DELAY
 456                     else:
 457                         timeout = win32event.INFINITE
 458                     rc = win32event.WaitForMultipleObjects(events, False,
 459                                                            timeout)
 460                     if not self._running:
 461                         break
 462                     elif rc == win32event.WAIT_TIMEOUT:
 463                         self.notify()
 464                     else:
 465                         self._handle_results()
 466             finally:
 467                 with self._stop_event_lock:
 468                     if self._stop_event is not None:
 469                         win32file.CloseHandle(self._stop_event)
 470                         self._stop_event = None
 471                 if self._worktree_watch is not None:
 472                     self._worktree_watch.close()
 473                 if self._git_dir_watch is not None:
 474                     self._git_dir_watch.close()
 475
 476         def _handle_results(self):
 477             if self._worktree_watch is not None:
 478                 for _, path in self._worktree_watch.read():
 479                     if not self._running:
 480                         break
 481                     if self._force_notify:
 482                         continue
 483                     path = self._worktree + '/' + self._transform_path(path)
 484                     if (path != self._git_dir
 485                             and not path.startswith(self._git_dir + '/')
 486                             and not os.path.isdir(path)):
 487                         if self._use_check_ignore:
 488                             self._file_paths.add(path)
 489                         else:
 490                             self._force_notify = True
 491             for _, path in self._git_dir_watch.read():
 492                 if not self._running:
 493                     break
 494                 if self._force_notify:
 495                     continue
 496                 path = self._transform_path(path)
 497                 if path.endswith('.lock'):
 498                     continue
 499                 if path == 'config':
 500                     self._force_config = True
 501                     continue
 502                 if (path == 'head'
 503                         or path == 'index'
 504                         or path.startswith('refs/')):
 505                     self._force_notify = True
 506
 507         def stop(self):
 508             self._running = False
 509             with self._stop_event_lock:
 510                 if self._stop_event is not None:
 511                     win32event.SetEvent(self._stop_event)
 512             self.wait()
 513
 514
 515 def create(context):
 516     thread_class = None
 517     cfg = context.cfg
 518     if not cfg.get('cola.inotify', default=True):
 519         msg = N_('File system change monitoring: disabled because'
 520                  ' "cola.inotify" is false.\n')
 521         Interaction.log(msg)
 522     elif AVAILABLE == 'inotify':
 523         thread_class = _InotifyThread
 524     elif AVAILABLE == 'pywin32':
 525         thread_class = _Win32Thread
 526     else:
 527         if utils.is_win32():
 528             msg = N_('File system change monitoring: disabled because pywin32'
 529                      ' is not installed.\n')
 530             Interaction.log(msg)
 531         elif utils.is_linux():
 532             msg = N_('File system change monitoring: disabled because libc'
 533                      ' does not support the inotify system calls.\n')
 534             Interaction.log(msg)
 535     return _Monitor(context, thread_class)