src/TGitCache/FolderCrawler.cpp

   1 // TortoiseGit - a Windows shell extension for easy version control
   2
   3 // External Cache Copyright (C) 2005-2008,2011,2014 - TortoiseSVN
   4 // Copyright (C) 2008-2014, 2016 - TortoiseGit
   5
   6 // This program is free software; you can redistribute it and/or
   7 // modify it under the terms of the GNU General Public License
   8 // as published by the Free Software Foundation; either version 2
   9 // of the License, or (at your option) any later version.
  10
  11 // This program is distributed in the hope that it will be useful,
  12 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 // GNU General Public License for more details.
  15
  16 // You should have received a copy of the GNU General Public License
  17 // along with this program; if not, write to the Free Software Foundation,
  18 // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  19 //
  20
  21 #include "stdafx.h"
  22 #include "FolderCrawler.h"
  23 #include "GitStatusCache.h"
  24 #include "registry.h"
  25 #include "TGitCache.h"
  26 #include <ShlObj.h>
  27
  28 CFolderCrawler::CFolderCrawler(void)
  29 {
  30         m_hWakeEvent = CreateEvent(nullptr, FALSE, FALSE, nullptr);
  31         m_hTerminationEvent = CreateEvent(nullptr, TRUE, FALSE, nullptr);
  32         m_lCrawlInhibitSet = 0;
  33         m_crawlHoldoffReleasesAt = (LONGLONG)GetTickCount64();
  34         m_bRun = false;
  35         m_bPathsAddedSinceLastCrawl = false;
  36         m_bItemsAddedSinceLastCrawl = false;
  37         m_blockReleasesAt = 0;
  38 }
  39
  40 CFolderCrawler::~CFolderCrawler(void)
  41 {
  42         Stop();
  43 }
  44
  45 void CFolderCrawler::Stop()
  46 {
  47         m_bRun = false;
  48         if (m_hTerminationEvent)
  49         {
  50                 SetEvent(m_hTerminationEvent);
  51                 if (m_hThread.IsValid() && WaitForSingleObject(m_hThread, 4000) != WAIT_OBJECT_0)
  52                         CTraceToOutputDebugString::Instance()(__FUNCTION__ ": Error terminating crawler thread\n");
  53         }
  54         m_hThread.CloseHandle();
  55         m_hTerminationEvent.CloseHandle();
  56         m_hWakeEvent.CloseHandle();
  57 }
  58
  59 void CFolderCrawler::Initialise()
  60 {
  61         // Don't call Initialize more than once
  62         ATLASSERT(!m_hThread);
  63
  64         // Just start the worker thread.
  65         // It will wait for event being signaled.
  66         // If m_hWakeEvent is already signaled the worker thread
  67         // will behave properly (with normal priority at worst).
  68
  69         m_bRun = true;
  70         unsigned int threadId;
  71         m_hThread = (HANDLE)_beginthreadex(nullptr, 0, ThreadEntry, this, 0, &threadId);
  72         SetThreadPriority(m_hThread, THREAD_PRIORITY_BELOW_NORMAL);
  73 }
  74
  75 void CFolderCrawler::RemoveDuplicate(std::deque<CTGitPath> &list,const CTGitPath &path)
  76 {
  77         for (auto it = list.cbegin(); it != list.cend(); ++it)
  78         {
  79                 if(*it == path)
  80                 {
  81                         list.erase(it);
  82                         it = list.cbegin(); /* search again*/
  83                         if (it == list.cend())
  84                                 break;
  85                 }
  86         }
  87 }
  88 void CFolderCrawler::AddDirectoryForUpdate(const CTGitPath& path)
  89 {
  90         /* Index file changing*/
  91         if( GitStatus::IsExistIndexLockFile((CString&)path.GetWinPathString()))
  92                 return;
  93
  94         if (!CGitStatusCache::Instance().IsPathGood(path))
  95                 return;
  96         {
  97                 CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) L": AddDirectoryForUpdate %s\n", path.GetWinPath());
  98
  99                 AutoLocker lock(m_critSec);
 100
 101                 m_foldersToUpdate.Push(path);
 102
 103                 //ATLASSERT(path.IsDirectory() || !path.Exists());
 104                 // set this flag while we are sync'ed
 105                 // with the worker thread
 106                 m_bItemsAddedSinceLastCrawl = true;
 107         }
 108         //if (SetHoldoff())
 109                 SetEvent(m_hWakeEvent);
 110 }
 111
 112 void CFolderCrawler::AddPathForUpdate(const CTGitPath& path)
 113 {
 114         /* Index file changing*/
 115         if( GitStatus::IsExistIndexLockFile((CString&)path.GetWinPathString()))
 116                 return;
 117
 118         {
 119                 AutoLocker lock(m_critSec);
 120
 121                 m_pathsToUpdate.Push(path);
 122                 m_bPathsAddedSinceLastCrawl = true;
 123         }
 124         //if (SetHoldoff())
 125                 SetEvent(m_hWakeEvent);
 126 }
 127
 128 void CFolderCrawler::ReleasePathForUpdate(const CTGitPath& path)
 129 {
 130         CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) L": ReleasePathForUpdate %s\n", path.GetWinPath());
 131         AutoLocker lock(m_critSec);
 132
 133         m_pathsToRelease.Push(path);
 134         SetEvent(m_hWakeEvent);
 135 }
 136
 137 unsigned int CFolderCrawler::ThreadEntry(void* pContext)
 138 {
 139         reinterpret_cast<CFolderCrawler*>(pContext)->WorkerThread();
 140         return 0;
 141 }
 142
 143 void CFolderCrawler::WorkerThread()
 144 {
 145         HANDLE hWaitHandles[2];
 146         hWaitHandles[0] = m_hTerminationEvent;
 147         hWaitHandles[1] = m_hWakeEvent;
 148         CTGitPath workingPath;
 149         ULONGLONG currentTicks = 0;
 150
 151         for(;;)
 152         {
 153                 bool bRecursive = !!(DWORD)CRegStdDWORD(L"Software\\TortoiseGit\\RecursiveOverlay", TRUE);
 154
 155                 SetThreadPriority(GetCurrentThread(), THREAD_MODE_BACKGROUND_END);
 156
 157                 DWORD waitResult = WaitForMultipleObjects(_countof(hWaitHandles), hWaitHandles, FALSE, INFINITE);
 158
 159                 // exit event/working loop if the first event (m_hTerminationEvent)
 160                 // has been signaled or if one of the events has been abandoned
 161                 // (i.e. ~CFolderCrawler() is being executed)
 162                 if(m_bRun == false || waitResult == WAIT_OBJECT_0 || waitResult == WAIT_ABANDONED_0 || waitResult == WAIT_ABANDONED_0+1)
 163                 {
 164                         // Termination event
 165                         break;
 166                 }
 167
 168                 SetThreadPriority(GetCurrentThread(), THREAD_MODE_BACKGROUND_BEGIN);
 169
 170                 // If we get here, we've been woken up by something being added to the queue.
 171                 // However, it's important that we don't do our crawling while
 172                 // the shell is still asking for items
 173                 bool bFirstRunAfterWakeup = true;
 174                 for(;;)
 175                 {
 176                         if (!m_bRun)
 177                                 break;
 178                         // Any locks today?
 179                         if (CGitStatusCache::Instance().m_bClearMemory)
 180                         {
 181                                 CAutoWriteLock writeLock(CGitStatusCache::Instance().GetGuard());
 182                                 CGitStatusCache::Instance().ClearCache();
 183                                 CGitStatusCache::Instance().m_bClearMemory = false;
 184                         }
 185                         if(m_lCrawlInhibitSet > 0)
 186                         {
 187                                 // We're in crawl hold-off
 188                                 CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) L": Crawl hold-off\n");
 189                                 Sleep(50);
 190                                 continue;
 191                         }
 192                         if (bFirstRunAfterWakeup)
 193                         {
 194                                 Sleep(20);
 195                                 CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) L": Crawl bFirstRunAfterWakeup\n");
 196                                 bFirstRunAfterWakeup = false;
 197                                 continue;
 198                         }
 199                         if ((m_blockReleasesAt < GetTickCount64()) && (!m_blockedPath.IsEmpty()))
 200                         {
 201                                 CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) L": Crawl stop blocking path %s\n", m_blockedPath.GetWinPath());
 202                                 m_blockedPath.Reset();
 203                         }
 204                         CGitStatusCache::Instance().RemoveTimedoutBlocks();
 205
 206                         while (!m_pathsToRelease.empty())
 207                         {
 208                                 AutoLocker lock(m_critSec);
 209                                 CTGitPath path = m_pathsToRelease.Pop();
 210                                 GitStatus::ReleasePath(path.GetWinPathString());
 211                         }
 212
 213                         if (m_foldersToUpdate.empty() && m_pathsToUpdate.empty())
 214                         {
 215                                 // Nothing left to do
 216                                 break;
 217                         }
 218                         currentTicks = GetTickCount64();
 219                         if (!m_pathsToUpdate.empty())
 220                         {
 221                                 {
 222                                         AutoLocker lock(m_critSec);
 223
 224                                         m_bPathsAddedSinceLastCrawl = false;
 225
 226                                         workingPath = m_pathsToUpdate.Pop();
 227                                         if ((!m_blockedPath.IsEmpty()) && (m_blockedPath.IsAncestorOf(workingPath)))
 228                                         {
 229                                                 // move the path to the end of the list
 230                                                 m_pathsToUpdate.Push(workingPath);
 231                                                 if (m_pathsToUpdate.size() < 3)
 232                                                         Sleep(50);
 233                                                 continue;
 234                                         }
 235                                 }
 236
 237                                 // don't crawl paths that are excluded
 238                                 if (!CGitStatusCache::Instance().IsPathAllowed(workingPath))
 239                                         continue;
 240                                 // check if the changed path is inside an .git folder
 241                                 CString projectroot;
 242                                 if ((workingPath.HasAdminDir(&projectroot)&&workingPath.IsDirectory()) || workingPath.IsAdminDir())
 243                                 {
 244                                         // we don't crawl for paths changed in a tmp folder inside an .git folder.
 245                                         // Because we also get notifications for those even if we just ask for the status!
 246                                         // And changes there don't affect the file status at all, so it's safe
 247                                         // to ignore notifications on those paths.
 248                                         if (workingPath.IsAdminDir())
 249                                         {
 250                                                 // TODO: add git specific filters here. is there really any change besides index file in .git
 251                                                 //       that is relevant for overlays?
 252                                                 /*CString lowerpath = workingPath.GetWinPathString();
 253                                                 lowerpath.MakeLower();
 254                                                 if (lowerpath.Find(L"\\tmp\\") > 0)
 255                                                         continue;
 256                                                 if (CStringUtils::EndsWith(lowerpath, L"\\tmp"))
 257                                                         continue;
 258                                                 if (lowerpath.Find(L"\\log") > 0)
 259                                                         continue;*/
 260                                                 // Here's a little problem:
 261                                                 // the lock file is also created for fetching the status
 262                                                 // and not just when committing.
 263                                                 // If we could find out why the lock file was changed
 264                                                 // we could decide to crawl the folder again or not.
 265                                                 // But for now, we have to crawl the parent folder
 266                                                 // no matter what.
 267
 268                                                 //if (lowerpath.Find(L"\\lock") > 0)
 269                                                 //      continue;
 270                                                 // only go back to wc root if we are in .git-dir
 271                                                 do
 272                                                 {
 273                                                         workingPath = workingPath.GetContainingDirectory();
 274                                                 } while(workingPath.IsAdminDir());
 275                                         }
 276                                         else if (!workingPath.Exists())
 277                                         {
 278                                                 CAutoWriteLock writeLock(CGitStatusCache::Instance().GetGuard());
 279                                                 CGitStatusCache::Instance().RemoveCacheForPath(workingPath);
 280                                                 continue;
 281                                         }
 282
 283                                         if (!CGitStatusCache::Instance().IsPathGood(workingPath))
 284                                         {
 285                                                 AutoLocker lock(m_critSec);
 286                                                 // move the path, the root of the repository, to the end of the list
 287                                                 if (projectroot.IsEmpty())
 288                                                         m_pathsToUpdate.Push(workingPath);
 289                                                 else
 290                                                         m_pathsToUpdate.Push(CTGitPath(projectroot));
 291                                                 if (m_pathsToUpdate.size() < 3)
 292                                                         Sleep(50);
 293                                                 continue;
 294                                         }
 295
 296                                         CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) L": Invalidating and refreshing folder: %s\n", workingPath.GetWinPath());
 297                                         {
 298                                                 AutoLocker print(critSec);
 299                                                 _sntprintf_s(szCurrentCrawledPath[nCurrentCrawledpathIndex], MAX_CRAWLEDPATHSLEN, _TRUNCATE, L"Invalidating and refreshing folder: %s", workingPath.GetWinPath());
 300                                                 ++nCurrentCrawledpathIndex;
 301                                                 if (nCurrentCrawledpathIndex >= MAX_CRAWLEDPATHS)
 302                                                         nCurrentCrawledpathIndex = 0;
 303                                         }
 304                                         InvalidateRect(hWndHidden, nullptr, FALSE);
 305                                         {
 306                                                 CAutoReadLock readLock(CGitStatusCache::Instance().GetGuard());
 307                                                 // Invalidate the cache of this folder, to make sure its status is fetched again.
 308                                                 CCachedDirectory * pCachedDir = CGitStatusCache::Instance().GetDirectoryCacheEntry(workingPath);
 309                                                 if (pCachedDir)
 310                                                 {
 311                                                         git_wc_status_kind status = pCachedDir->GetCurrentFullStatus();
 312                                                         pCachedDir->Invalidate();
 313                                                         if (workingPath.Exists())
 314                                                         {
 315                                                                 pCachedDir->RefreshStatus(bRecursive);
 316                                                                 // if the previous status wasn't normal and now it is, then
 317                                                                 // send a notification too.
 318                                                                 // We do this here because GetCurrentFullStatus() doesn't send
 319                                                                 // notifications for 'normal' status - if it would, we'd get tons
 320                                                                 // of notifications when crawling a working copy not yet in the cache.
 321                                                                 if ((status != git_wc_status_normal) && (pCachedDir->GetCurrentFullStatus() != status))
 322                                                                 {
 323                                                                         CGitStatusCache::Instance().UpdateShell(workingPath);
 324                                                                         CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) L": shell update in crawler for %s\n", workingPath.GetWinPath());
 325                                                                 }
 326                                                         }
 327                                                         else
 328                                                         {
 329                                                                 CAutoWriteLock writeLock(CGitStatusCache::Instance().GetGuard());
 330                                                                 CGitStatusCache::Instance().RemoveCacheForPath(workingPath);
 331                                                         }
 332                                                 }
 333                                         }
 334                                         //In case that svn_client_stat() modified a file and we got
 335                                         //a notification about that in the directory watcher,
 336                                         //remove that here again - this is to prevent an endless loop
 337                                         AutoLocker lock(m_critSec);
 338                                         m_pathsToUpdate.erase(workingPath);
 339                                 }
 340                                 else if (workingPath.HasAdminDir())
 341                                 {
 342                                         if (!workingPath.Exists())
 343                                         {
 344                                                 CAutoWriteLock writeLock(CGitStatusCache::Instance().GetGuard());
 345                                                 CGitStatusCache::Instance().RemoveCacheForPath(workingPath);
 346                                                 if (!workingPath.GetContainingDirectory().Exists())
 347                                                         continue;
 348                                                 else
 349                                                         workingPath = workingPath.GetContainingDirectory();
 350                                         }
 351                                         CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) L": Updating path: %s\n", workingPath.GetWinPath());
 352                                         {
 353                                                 AutoLocker print(critSec);
 354                                                 _sntprintf_s(szCurrentCrawledPath[nCurrentCrawledpathIndex], MAX_CRAWLEDPATHSLEN, _TRUNCATE, L"Updating path: %s", workingPath.GetWinPath());
 355                                                 ++nCurrentCrawledpathIndex;
 356                                                 if (nCurrentCrawledpathIndex >= MAX_CRAWLEDPATHS)
 357                                                         nCurrentCrawledpathIndex = 0;
 358                                         }
 359                                         InvalidateRect(hWndHidden, nullptr, FALSE);
 360                                         // HasAdminDir() already checks if the path points to a dir
 361                                         DWORD flags = TGITCACHE_FLAGS_FOLDERISKNOWN;
 362                                         flags |= (workingPath.IsDirectory() ? TGITCACHE_FLAGS_ISFOLDER : 0);
 363                                         flags |= (bRecursive ? TGITCACHE_FLAGS_RECUSIVE_STATUS : 0);
 364                                         {
 365                                                 CAutoReadLock readLock(CGitStatusCache::Instance().GetGuard());
 366                                                 // Invalidate the cache of folders manually. The cache of files is invalidated
 367                                                 // automatically if the status is asked for it and the file times don't match
 368                                                 // anymore, so we don't need to manually invalidate those.
 369                                                 if (workingPath.IsDirectory())
 370                                                 {
 371                                                         CCachedDirectory * cachedDir = CGitStatusCache::Instance().GetDirectoryCacheEntry(workingPath);
 372                                                         if (cachedDir)
 373                                                                 cachedDir->Invalidate();
 374                                                 }
 375                                                 CStatusCacheEntry ce = CGitStatusCache::Instance().GetStatusForPath(workingPath, flags);
 376                                                 if (ce.GetEffectiveStatus() > git_wc_status_unversioned)
 377                                                 {
 378                                                         CGitStatusCache::Instance().UpdateShell(workingPath);
 379                                                         CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) L": shell update in folder crawler for %s\n", workingPath.GetWinPath());
 380                                                 }
 381                                         }
 382                                         AutoLocker lock(m_critSec);
 383                                         m_pathsToUpdate.erase(workingPath);
 384                                 }
 385                                 else
 386                                 {
 387                                         if (!workingPath.Exists())
 388                                         {
 389                                                 CAutoWriteLock writeLock(CGitStatusCache::Instance().GetGuard());
 390                                                 CGitStatusCache::Instance().RemoveCacheForPath(workingPath);
 391                                         }
 392                                 }
 393                         }
 394                         else if (!m_foldersToUpdate.empty())
 395                         {
 396                                 {
 397                                         AutoLocker lock(m_critSec);
 398                                         m_bItemsAddedSinceLastCrawl = false;
 399
 400                                         // create a new CTGitPath object to make sure the cached flags are requested again.
 401                                         // without this, a missing file/folder is still treated as missing even if it is available
 402                                         // now when crawling.
 403                                         workingPath = CTGitPath(m_foldersToUpdate.Pop().GetWinPath());
 404
 405                                         if ((!m_blockedPath.IsEmpty())&&(m_blockedPath.IsAncestorOf(workingPath)))
 406                                         {
 407                                                 // move the path to the end of the list
 408                                                 m_foldersToUpdate.Push(workingPath);
 409                                                 if (m_foldersToUpdate.size() < 3)
 410                                                         Sleep(50);
 411                                                 continue;
 412                                         }
 413                                 }
 414                                 if ((!m_blockedPath.IsEmpty())&&(m_blockedPath.IsAncestorOf(workingPath)))
 415                                         continue;
 416                                 if (!CGitStatusCache::Instance().IsPathAllowed(workingPath))
 417                                         continue;
 418                                 if (!CGitStatusCache::Instance().IsPathGood(workingPath))
 419                                         continue;
 420
 421                                 CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) L": Crawling folder: %s\n", workingPath.GetWinPath());
 422                                 {
 423                                         AutoLocker print(critSec);
 424                                         _sntprintf_s(szCurrentCrawledPath[nCurrentCrawledpathIndex], MAX_CRAWLEDPATHSLEN, _TRUNCATE, L"Crawling folder: %s", workingPath.GetWinPath());
 425                                         ++nCurrentCrawledpathIndex;
 426                                         if (nCurrentCrawledpathIndex >= MAX_CRAWLEDPATHS)
 427                                                 nCurrentCrawledpathIndex = 0;
 428                                 }
 429                                 InvalidateRect(hWndHidden, nullptr, FALSE);
 430                                 {
 431                                         CAutoReadLock readLock(CGitStatusCache::Instance().GetGuard());
 432                                         // Now, we need to visit this folder, to make sure that we know its 'most important' status
 433                                         CCachedDirectory * cachedDir = CGitStatusCache::Instance().GetDirectoryCacheEntry(workingPath.GetDirectory());
 434                                         // check if the path is monitored by the watcher. If it isn't, then we have to invalidate the cache
 435                                         // for that path and add it to the watcher.
 436                                         if (!CGitStatusCache::Instance().IsPathWatched(workingPath))
 437                                         {
 438                                                 if (workingPath.HasAdminDir())
 439                                                 {
 440                                                         CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) L": Add watch path %s\n", workingPath.GetWinPath());
 441                                                         CGitStatusCache::Instance().AddPathToWatch(workingPath);
 442                                                 }
 443                                                 if (cachedDir)
 444                                                         cachedDir->Invalidate();
 445                                                 else
 446                                                 {
 447                                                         CAutoWriteLock writeLock(CGitStatusCache::Instance().GetGuard());
 448                                                         CGitStatusCache::Instance().RemoveCacheForPath(workingPath);
 449                                                         // now cacheDir is invalid because it got deleted in the RemoveCacheForPath() call above.
 450                                                         cachedDir = nullptr;
 451                                                 }
 452                                         }
 453                                         if (cachedDir)
 454                                                 cachedDir->RefreshStatus(bRecursive);
 455                                 }
 456
 457                                 // While refreshing the status, we could get another crawl request for the same folder.
 458                                 // This can happen if the crawled folder has a lower status than one of the child folders
 459                                 // (recursively). To avoid double crawlings, remove such a crawl request here
 460                                 AutoLocker lock(m_critSec);
 461                                 if (m_bItemsAddedSinceLastCrawl)
 462                                 {
 463                                         m_foldersToUpdate.erase(workingPath);
 464                                 }
 465                         }
 466                 }
 467         }
 468         _endthread();
 469 }
 470
 471 bool CFolderCrawler::SetHoldoff(DWORD milliseconds /* = 100*/)
 472 {
 473         LONGLONG tick = (LONGLONG)GetTickCount64();
 474         bool ret = ((tick - m_crawlHoldoffReleasesAt) > 0);
 475         m_crawlHoldoffReleasesAt = tick + milliseconds;
 476         return ret;
 477 }
 478
 479 void CFolderCrawler::BlockPath(const CTGitPath& path, DWORD ticks)
 480 {
 481         CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) L": block path %s from being crawled\n", path.GetWinPath());
 482         m_blockedPath = path;
 483         if (ticks == 0)
 484                 m_blockReleasesAt = GetTickCount64() + 10000;
 485         else
 486                 m_blockReleasesAt = GetTickCount64() + ticks;
 487 }