cola/fsmonitor.py

   1 # Copyright (c) 2008 David Aguilar
   2 # Copyright (c) 2015 Daniel Harding
   3 """Provides an filesystem monitoring for Linux (via inotify) and for Windows
   4 (via pywin32 and the ReadDirectoryChanges function)"""
   5 from __future__ import division, absolute_import, unicode_literals
   6
   7 import errno
   8 import os
   9 import os.path
  10 import select
  11 from threading import Lock
  12
  13 from . import utils
  14 from . import version
  15 from .decorators import memoize
  16
  17 AVAILABLE = None
  18
  19 if utils.is_win32():
  20     try:
  21         import pywintypes
  22         import win32con
  23         import win32event
  24         import win32file
  25     except ImportError:
  26         pass
  27     else:
  28         AVAILABLE = 'pywin32'
  29 elif utils.is_linux():
  30     try:
  31         from . import inotify
  32     except ImportError:
  33         pass
  34     else:
  35         AVAILABLE = 'inotify'
  36
  37 from qtpy import QtCore
  38 from qtpy.QtCore import Signal
  39
  40 from . import core
  41 from . import gitcfg
  42 from . import gitcmds
  43 from .compat import bchr
  44 from .git import git
  45 from .i18n import N_
  46 from .interaction import Interaction
  47
  48
  49 class _Monitor(QtCore.QObject):
  50
  51     files_changed = Signal()
  52
  53     def __init__(self, thread_class):
  54         QtCore.QObject.__init__(self)
  55         self._thread_class = thread_class
  56         self._thread = None
  57
  58     def start(self):
  59         if self._thread_class is not None:
  60             assert self._thread is None
  61             self._thread = self._thread_class(self)
  62             self._thread.start()
  63
  64     def stop(self):
  65         if self._thread_class is not None:
  66             assert self._thread is not None
  67             self._thread.stop()
  68             self._thread.wait()
  69             self._thread = None
  70
  71     def refresh(self):
  72         if self._thread is not None:
  73             self._thread.refresh()
  74
  75
  76 class _BaseThread(QtCore.QThread):
  77     #: The delay, in milliseconds, between detecting file system modification
  78     #: and triggering the 'files_changed' signal, to coalesce multiple
  79     #: modifications into a single signal.
  80     _NOTIFICATION_DELAY = 888
  81
  82     def __init__(self, monitor):
  83         QtCore.QThread.__init__(self)
  84         self._monitor = monitor
  85         self._running = True
  86         self._use_check_ignore = version.check('check-ignore',
  87                                                version.git_version())
  88         self._force_notify = False
  89         self._file_paths = set()
  90
  91     @property
  92     def _pending(self):
  93         return self._force_notify or self._file_paths
  94
  95     def refresh(self):
  96         """Do any housekeeping necessary in response to repository changes."""
  97         pass
  98
  99     def notify(self):
 100         """Notifies all observers"""
 101         do_notify = False
 102         if self._force_notify:
 103             do_notify = True
 104         elif self._file_paths:
 105             proc = core.start_command(['git', 'check-ignore', '--verbose',
 106                                        '--non-matching', '-z', '--stdin'])
 107             path_list = bchr(0).join(core.encode(path)
 108                                      for path in self._file_paths)
 109             out, err = proc.communicate(path_list)
 110             if proc.returncode:
 111                 do_notify = True
 112             else:
 113                 # Each output record is four fields separated by NULL
 114                 # characters (records are also separated by NULL characters):
 115                 # <source> <NULL> <linenum> <NULL> <pattern> <NULL> <pathname>
 116                 # For paths which are not ignored, all fields will be empty
 117                 # except for <pathname>.  So to see if we have any non-ignored
 118                 # files, we simply check every fourth field to see if any of
 119                 # them are empty.
 120                 source_fields = out.split(bchr(0))[0:-1:4]
 121                 do_notify = not all(source_fields)
 122         self._force_notify = False
 123         self._file_paths = set()
 124         if do_notify:
 125             self._monitor.files_changed.emit()
 126
 127     @staticmethod
 128     def _log_enabled_message():
 129         msg = N_('File system change monitoring: enabled.\n')
 130         Interaction.safe_log(msg)
 131
 132
 133 if AVAILABLE == 'inotify':
 134
 135     class _InotifyThread(_BaseThread):
 136         _TRIGGER_MASK = (
 137                 inotify.IN_ATTRIB |
 138                 inotify.IN_CLOSE_WRITE |
 139                 inotify.IN_CREATE |
 140                 inotify.IN_DELETE |
 141                 inotify.IN_MODIFY |
 142                 inotify.IN_MOVED_FROM |
 143                 inotify.IN_MOVED_TO
 144         )
 145         _ADD_MASK = (
 146                 _TRIGGER_MASK |
 147                 inotify.IN_EXCL_UNLINK |
 148                 inotify.IN_ONLYDIR
 149         )
 150
 151         def __init__(self, monitor):
 152             _BaseThread.__init__(self, monitor)
 153             worktree = git.worktree()
 154             if worktree is not None:
 155                 worktree = core.abspath(worktree)
 156             self._worktree = worktree
 157             self._git_dir = git.git_path()
 158             self._lock = Lock()
 159             self._inotify_fd = None
 160             self._pipe_r = None
 161             self._pipe_w = None
 162             self._worktree_wd_to_path_map = {}
 163             self._worktree_path_to_wd_map = {}
 164             self._git_dir_wd_to_path_map = {}
 165             self._git_dir_path_to_wd_map = {}
 166             self._git_dir_wd = None
 167
 168         @staticmethod
 169         def _log_out_of_wds_message():
 170             msg = N_('File system change monitoring: disabled because the'
 171                      ' limit on the total number of inotify watches was'
 172                      ' reached.  You may be able to increase the limit on'
 173                      ' the number of watches by running:\n'
 174                      '\n'
 175                      '    echo fs.inotify.max_user_watches=100000 |'
 176                      ' sudo tee -a /etc/sysctl.conf &&'
 177                      ' sudo sysctl -p\n')
 178             Interaction.safe_log(msg)
 179
 180         def run(self):
 181             try:
 182                 with self._lock:
 183                     self._inotify_fd = inotify.init()
 184                     self._pipe_r, self._pipe_w = os.pipe()
 185
 186                 poll_obj = select.poll()
 187                 poll_obj.register(self._inotify_fd, select.POLLIN)
 188                 poll_obj.register(self._pipe_r, select.POLLIN)
 189
 190                 self.refresh()
 191
 192                 self._log_enabled_message()
 193
 194                 while self._running:
 195                     if self._pending:
 196                         timeout = self._NOTIFICATION_DELAY
 197                     else:
 198                         timeout = None
 199                     try:
 200                         events = poll_obj.poll(timeout)
 201                     except OSError as e:
 202                         if e.errno == errno.EINTR:
 203                             continue
 204                         else:
 205                             raise
 206                     except select.error:
 207                         continue
 208                     else:
 209                         if not self._running:
 210                             break
 211                         elif not events:
 212                             self.notify()
 213                         else:
 214                             for fd, event in events:
 215                                 if fd == self._inotify_fd:
 216                                     self._handle_events()
 217             finally:
 218                 with self._lock:
 219                     if self._inotify_fd is not None:
 220                         os.close(self._inotify_fd)
 221                         self._inotify_fd = None
 222                     if self._pipe_r is not None:
 223                         os.close(self._pipe_r)
 224                         self._pipe_r = None
 225                         os.close(self._pipe_w)
 226                         self._pipe_w = None
 227
 228         def refresh(self):
 229             with self._lock:
 230                 if self._inotify_fd is None:
 231                     return
 232                 try:
 233                     if self._worktree is not None:
 234                         tracked_dirs = set(
 235                                 os.path.dirname(os.path.join(self._worktree,
 236                                                              path))
 237                                 for path in gitcmds.tracked_files())
 238                         self._refresh_watches(tracked_dirs,
 239                                               self._worktree_wd_to_path_map,
 240                                               self._worktree_path_to_wd_map)
 241                     git_dirs = set()
 242                     git_dirs.add(self._git_dir)
 243                     for dirpath, dirnames, filenames in core.walk(
 244                             os.path.join(self._git_dir, 'refs')):
 245                         git_dirs.add(dirpath)
 246                     self._refresh_watches(git_dirs,
 247                                           self._git_dir_wd_to_path_map,
 248                                           self._git_dir_path_to_wd_map)
 249                     self._git_dir_wd = \
 250                             self._git_dir_path_to_wd_map[self._git_dir]
 251                 except OSError as e:
 252                     if e.errno == errno.ENOSPC:
 253                         self._log_out_of_wds_message()
 254                         self._running = False
 255                     else:
 256                         raise
 257
 258         def _refresh_watches(self, paths_to_watch, wd_to_path_map,
 259                              path_to_wd_map):
 260             watched_paths = set(path_to_wd_map)
 261             for path in watched_paths - paths_to_watch:
 262                 wd = path_to_wd_map.pop(path)
 263                 wd_to_path_map.pop(wd)
 264                 try:
 265                     inotify.rm_watch(self._inotify_fd, wd)
 266                 except OSError as e:
 267                     if e.errno == errno.EINVAL:
 268                         # This error can occur if the target of the wd was
 269                         # removed on the filesystem before we call
 270                         # inotify.rm_watch() so ignore it.
 271                         pass
 272                     else:
 273                         raise
 274             for path in paths_to_watch - watched_paths:
 275                 try:
 276                     wd = inotify.add_watch(self._inotify_fd, core.encode(path),
 277                                            self._ADD_MASK)
 278                 except OSError as e:
 279                     if e.errno in (errno.ENOENT, errno.ENOTDIR):
 280                         # These two errors should only occur as a result of
 281                         # race conditions:  the first if the directory
 282                         # referenced by path was removed or renamed before the
 283                         # call to inotify.add_watch(); the second if the
 284                         # directory referenced by path was replaced with a file
 285                         # before the call to inotify.add_watch().  Therefore we
 286                         # simply ignore them.
 287                         pass
 288                     else:
 289                         raise
 290                 else:
 291                     wd_to_path_map[wd] = path
 292                     path_to_wd_map[path] = wd
 293
 294         def _check_event(self, wd, mask, name):
 295             if mask & inotify.IN_Q_OVERFLOW:
 296                 self._force_notify = True
 297             elif not mask & self._TRIGGER_MASK:
 298                 pass
 299             elif mask & inotify.IN_ISDIR:
 300                 pass
 301             elif wd in self._worktree_wd_to_path_map:
 302                 if self._use_check_ignore:
 303                     self._file_paths.add(
 304                             os.path.join(self._worktree_wd_to_path_map[wd],
 305                                          core.decode(name)))
 306                 else:
 307                     self._force_notify = True
 308             elif wd == self._git_dir_wd:
 309                 name = core.decode(name)
 310                 if name == 'HEAD' or name == 'index':
 311                     self._force_notify = True
 312             elif (wd in self._git_dir_wd_to_path_map
 313                     and not core.decode(name).endswith('.lock')):
 314                 self._force_notify = True
 315
 316         def _handle_events(self):
 317             for wd, mask, cookie, name in \
 318                     inotify.read_events(self._inotify_fd):
 319                 if not self._force_notify:
 320                     self._check_event(wd, mask, name)
 321
 322         def stop(self):
 323             self._running = False
 324             with self._lock:
 325                 if self._pipe_w is not None:
 326                     os.write(self._pipe_w, bchr(0))
 327             self.wait()
 328
 329
 330 if AVAILABLE == 'pywin32':
 331
 332     class _Win32Watch(object):
 333
 334         def __init__(self, path, flags):
 335             self.flags = flags
 336
 337             self.handle = None
 338             self.event = None
 339
 340             try:
 341                 self.handle = win32file.CreateFileW(
 342                         path,
 343                         0x0001,  # FILE_LIST_DIRECTORY
 344                         win32con.FILE_SHARE_READ | win32con.FILE_SHARE_WRITE,
 345                         None,
 346                         win32con.OPEN_EXISTING,
 347                         win32con.FILE_FLAG_BACKUP_SEMANTICS |
 348                         win32con.FILE_FLAG_OVERLAPPED,
 349                         None)
 350
 351                 self.buffer = win32file.AllocateReadBuffer(8192)
 352                 self.event = win32event.CreateEvent(None, True, False, None)
 353                 self.overlapped = pywintypes.OVERLAPPED()
 354                 self.overlapped.hEvent = self.event
 355                 self._start()
 356             except:
 357                 self.close()
 358                 raise
 359
 360         def _start(self):
 361             win32file.ReadDirectoryChangesW(self.handle, self.buffer, True,
 362                                             self.flags, self.overlapped)
 363
 364         def read(self):
 365             if win32event.WaitForSingleObject(self.event, 0) \
 366                     == win32event.WAIT_TIMEOUT:
 367                 result = []
 368             else:
 369                 nbytes = win32file.GetOverlappedResult(self.handle,
 370                                                        self.overlapped, False)
 371                 result = win32file.FILE_NOTIFY_INFORMATION(self.buffer, nbytes)
 372                 self._start()
 373             return result
 374
 375         def close(self):
 376             if self.handle is not None:
 377                 win32file.CancelIo(self.handle)
 378                 win32file.CloseHandle(self.handle)
 379             if self.event is not None:
 380                 win32file.CloseHandle(self.event)
 381
 382     class _Win32Thread(_BaseThread):
 383         _FLAGS = (win32con.FILE_NOTIFY_CHANGE_FILE_NAME |
 384                   win32con.FILE_NOTIFY_CHANGE_DIR_NAME |
 385                   win32con.FILE_NOTIFY_CHANGE_ATTRIBUTES |
 386                   win32con.FILE_NOTIFY_CHANGE_SIZE |
 387                   win32con.FILE_NOTIFY_CHANGE_LAST_WRITE |
 388                   win32con.FILE_NOTIFY_CHANGE_SECURITY)
 389
 390         def __init__(self, monitor):
 391             _BaseThread.__init__(self, monitor)
 392             worktree = git.worktree()
 393             if worktree is not None:
 394                 worktree = self._transform_path(core.abspath(worktree))
 395             self._worktree = worktree
 396             self._worktree_watch = None
 397             self._git_dir = self._transform_path(core.abspath(git.git_path()))
 398             self._git_dir_watch = None
 399             self._stop_event_lock = Lock()
 400             self._stop_event = None
 401
 402         @staticmethod
 403         def _transform_path(path):
 404             return path.replace('\\', '/').lower()
 405
 406         def _read_watch(self, watch):
 407             if win32event.WaitForSingleObject(watch.event, 0) \
 408                     == win32event.WAIT_TIMEOUT:
 409                 nbytes = 0
 410             else:
 411                 nbytes = win32file.GetOverlappedResult(watch.handle,
 412                                                        watch.overlapped, False)
 413             return win32file.FILE_NOTIFY_INFORMATION(watch.buffer, nbytes)
 414
 415         def run(self):
 416             try:
 417                 with self._stop_event_lock:
 418                     self._stop_event = win32event.CreateEvent(None, True,
 419                                                               False, None)
 420
 421                 events = [self._stop_event]
 422
 423                 if self._worktree is not None:
 424                     self._worktree_watch = _Win32Watch(self._worktree,
 425                                                        self._FLAGS)
 426                     events.append(self._worktree_watch.event)
 427
 428                 self._git_dir_watch = _Win32Watch(self._git_dir, self._FLAGS)
 429                 events.append(self._git_dir_watch.event)
 430
 431                 self._log_enabled_message()
 432
 433                 while self._running:
 434                     if self._pending:
 435                         timeout = self._NOTIFICATION_DELAY
 436                     else:
 437                         timeout = win32event.INFINITE
 438                     rc = win32event.WaitForMultipleObjects(events, False,
 439                                                            timeout)
 440                     if not self._running:
 441                         break
 442                     elif rc == win32event.WAIT_TIMEOUT:
 443                         self.notify()
 444                     else:
 445                         self._handle_results()
 446             finally:
 447                 with self._stop_event_lock:
 448                     if self._stop_event is not None:
 449                         win32file.CloseHandle(self._stop_event)
 450                         self._stop_event = None
 451                 if self._worktree_watch is not None:
 452                     self._worktree_watch.close()
 453                 if self._git_dir_watch is not None:
 454                     self._git_dir_watch.close()
 455
 456         def _handle_results(self):
 457             if self._worktree_watch is not None:
 458                 for action, path in self._worktree_watch.read():
 459                     if not self._running:
 460                         break
 461                     if self._force_notify:
 462                         continue
 463                     path = self._worktree + '/' + self._transform_path(path)
 464                     if (path != self._git_dir
 465                         and not path.startswith(self._git_dir + '/')
 466                         and not os.path.isdir(path)
 467                        ):
 468                         if self._use_check_ignore:
 469                             self._file_paths.add(path)
 470                         else:
 471                             self._force_notify = True
 472             for action, path in self._git_dir_watch.read():
 473                 if not self._running:
 474                     break
 475                 if self._force_notify:
 476                     continue
 477                 path = self._transform_path(path)
 478                 if path.endswith('.lock'):
 479                     continue
 480                 if (path == 'head'
 481                     or path == 'index'
 482                     or path.startswith('refs/')
 483                    ):
 484                     self._force_notify = True
 485
 486         def stop(self):
 487             self._running = False
 488             with self._stop_event_lock:
 489                 if self._stop_event is not None:
 490                     win32event.SetEvent(self._stop_event)
 491             self.wait()
 492
 493
 494 @memoize
 495 def current():
 496     return _create_instance()
 497
 498
 499 def _create_instance():
 500     thread_class = None
 501     cfg = gitcfg.current()
 502     if not cfg.get('cola.inotify', True):
 503         msg = N_('File system change monitoring: disabled because'
 504                  ' "cola.inotify" is false.\n')
 505         Interaction.log(msg)
 506     elif AVAILABLE == 'inotify':
 507         thread_class = _InotifyThread
 508     elif AVAILABLE == 'pywin32':
 509         thread_class = _Win32Thread
 510     else:
 511         if utils.is_win32():
 512             msg = N_('File system change monitoring: disabled because pywin32'
 513                      ' is not installed.\n')
 514             Interaction.log(msg)
 515         elif utils.is_linux():
 516             msg = N_('File system change monitoring: disabled because libc'
 517                      ' does not support the inotify system calls.\n')
 518             Interaction.log(msg)
 519     return _Monitor(thread_class)