src/TGitCache/FolderCrawler.cpp

   1 // TortoiseGit - a Windows shell extension for easy version control
   2
   3 // External Cache Copyright (C) 2005-2008,2011 - TortoiseSVN
   4 // Copyright (C) 2008-2014 - TortoiseGit
   5
   6 // This program is free software; you can redistribute it and/or
   7 // modify it under the terms of the GNU General Public License
   8 // as published by the Free Software Foundation; either version 2
   9 // of the License, or (at your option) any later version.
  10
  11 // This program is distributed in the hope that it will be useful,
  12 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 // GNU General Public License for more details.
  15
  16 // You should have received a copy of the GNU General Public License
  17 // along with this program; if not, write to the Free Software Foundation,
  18 // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  19 //
  20
  21 #include "stdafx.h"
  22 #include "FolderCrawler.h"
  23 #include "GitStatusCache.h"
  24 #include "registry.h"
  25 #include "TGitCache.h"
  26 #include <ShlObj.h>
  27 #include "SysInfo.h"
  28
  29
  30 CFolderCrawler::CFolderCrawler(void)
  31 {
  32         m_hWakeEvent = CreateEvent(NULL,FALSE,FALSE,NULL);
  33         m_hTerminationEvent = CreateEvent(NULL,TRUE,FALSE,NULL);
  34         m_lCrawlInhibitSet = 0;
  35         m_crawlHoldoffReleasesAt = (long)GetTickCount();
  36         m_bRun = false;
  37         m_bPathsAddedSinceLastCrawl = false;
  38         m_bItemsAddedSinceLastCrawl = false;
  39         m_blockReleasesAt = 0;
  40 }
  41
  42 CFolderCrawler::~CFolderCrawler(void)
  43 {
  44         Stop();
  45 }
  46
  47 void CFolderCrawler::Stop()
  48 {
  49         m_bRun = false;
  50         if (m_hTerminationEvent)
  51         {
  52                 SetEvent(m_hTerminationEvent);
  53                 if (m_hThread.IsValid() && WaitForSingleObject(m_hThread, 4000) != WAIT_OBJECT_0)
  54                 {
  55                         CTraceToOutputDebugString::Instance()(__FUNCTION__ ": Error terminating crawler thread\n");
  56                 }
  57         }
  58         m_hThread.CloseHandle();
  59         m_hTerminationEvent.CloseHandle();
  60         m_hWakeEvent.CloseHandle();
  61 }
  62
  63 void CFolderCrawler::Initialise()
  64 {
  65         // Don't call Initialize more than once
  66         ATLASSERT(!m_hThread);
  67
  68         // Just start the worker thread.
  69         // It will wait for event being signaled.
  70         // If m_hWakeEvent is already signaled the worker thread
  71         // will behave properly (with normal priority at worst).
  72
  73         m_bRun = true;
  74         unsigned int threadId;
  75         m_hThread = (HANDLE)_beginthreadex(NULL,0,ThreadEntry,this,0,&threadId);
  76         SetThreadPriority(m_hThread, THREAD_PRIORITY_BELOW_NORMAL);
  77 }
  78
  79 void CFolderCrawler::RemoveDuplicate(std::deque<CTGitPath> &list,const CTGitPath &path)
  80 {
  81         std::deque<CTGitPath>::iterator it;
  82         for(it = list.begin(); it != list.end(); ++it)
  83         {
  84                 if(*it == path)
  85                 {
  86                         list.erase(it);
  87                         it = list.begin(); /* search again*/
  88                         if(it == list.end())
  89                                 break;
  90                 }
  91         }
  92 }
  93 void CFolderCrawler::AddDirectoryForUpdate(const CTGitPath& path)
  94 {
  95         /* Index file changing*/
  96         if( GitStatus::IsExistIndexLockFile((CString&)path.GetWinPathString()))
  97                 return;
  98
  99         if (!CGitStatusCache::Instance().IsPathGood(path))
 100                 return;
 101         {
 102                 CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) _T(": AddDirectoryForUpdate %s\n"), path.GetWinPath());
 103
 104                 AutoLocker lock(m_critSec);
 105
 106                 m_foldersToUpdate.Push(path);
 107
 108                 //ATLASSERT(path.IsDirectory() || !path.Exists());
 109                 // set this flag while we are sync'ed
 110                 // with the worker thread
 111                 m_bItemsAddedSinceLastCrawl = true;
 112         }
 113         //if (SetHoldoff())
 114                 SetEvent(m_hWakeEvent);
 115 }
 116
 117 void CFolderCrawler::AddPathForUpdate(const CTGitPath& path)
 118 {
 119         /* Index file changing*/
 120         if( GitStatus::IsExistIndexLockFile((CString&)path.GetWinPathString()))
 121                 return;
 122
 123         {
 124                 AutoLocker lock(m_critSec);
 125
 126                 m_pathsToUpdate.Push(path);
 127                 m_bPathsAddedSinceLastCrawl = true;
 128         }
 129         //if (SetHoldoff())
 130                 SetEvent(m_hWakeEvent);
 131 }
 132
 133 void CFolderCrawler::ReleasePathForUpdate(const CTGitPath& path)
 134 {
 135         CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) _T(": ReleasePathForUpdate %s\n"), path.GetWinPath());
 136         AutoLocker lock(m_critSec);
 137
 138         m_pathsToRelease.Push(path);
 139         SetEvent(m_hWakeEvent);
 140 }
 141
 142 unsigned int CFolderCrawler::ThreadEntry(void* pContext)
 143 {
 144         ((CFolderCrawler*)pContext)->WorkerThread();
 145         return 0;
 146 }
 147
 148 void CFolderCrawler::WorkerThread()
 149 {
 150         HANDLE hWaitHandles[2];
 151         hWaitHandles[0] = m_hTerminationEvent;
 152         hWaitHandles[1] = m_hWakeEvent;
 153         CTGitPath workingPath;
 154         DWORD currentTicks = 0;
 155
 156         for(;;)
 157         {
 158                 bool bRecursive = !!(DWORD)CRegStdDWORD(_T("Software\\TortoiseGit\\RecursiveOverlay"), TRUE);
 159
 160                 if (SysInfo::Instance().IsVistaOrLater())
 161                 {
 162                         SetThreadPriority(GetCurrentThread(), THREAD_MODE_BACKGROUND_END);
 163                 }
 164                 DWORD waitResult = WaitForMultipleObjects(_countof(hWaitHandles), hWaitHandles, FALSE, INFINITE);
 165
 166                 // exit event/working loop if the first event (m_hTerminationEvent)
 167                 // has been signaled or if one of the events has been abandoned
 168                 // (i.e. ~CFolderCrawler() is being executed)
 169                 if(m_bRun == false || waitResult == WAIT_OBJECT_0 || waitResult == WAIT_ABANDONED_0 || waitResult == WAIT_ABANDONED_0+1)
 170                 {
 171                         // Termination event
 172                         break;
 173                 }
 174
 175                 if (SysInfo::Instance().IsVistaOrLater())
 176                 {
 177                         SetThreadPriority(GetCurrentThread(), THREAD_MODE_BACKGROUND_BEGIN);
 178                 }
 179
 180                 // If we get here, we've been woken up by something being added to the queue.
 181                 // However, it's important that we don't do our crawling while
 182                 // the shell is still asking for items
 183                 bool bFirstRunAfterWakeup = true;
 184                 for(;;)
 185                 {
 186                         if (!m_bRun)
 187                                 break;
 188                         // Any locks today?
 189                         if (CGitStatusCache::Instance().m_bClearMemory)
 190                         {
 191                                 CAutoWriteLock writeLock(CGitStatusCache::Instance().GetGuard());
 192                                 CGitStatusCache::Instance().ClearCache();
 193                                 CGitStatusCache::Instance().m_bClearMemory = false;
 194                         }
 195                         if(m_lCrawlInhibitSet > 0)
 196                         {
 197                                 // We're in crawl hold-off
 198                                 CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) _T(": Crawl hold-off\n"));
 199                                 Sleep(50);
 200                                 continue;
 201                         }
 202                         if (bFirstRunAfterWakeup)
 203                         {
 204                                 Sleep(20);
 205                                 CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) _T(": Crawl bFirstRunAfterWakeup\n"));
 206                                 bFirstRunAfterWakeup = false;
 207                                 continue;
 208                         }
 209                         if ((m_blockReleasesAt < GetTickCount())&&(!m_blockedPath.IsEmpty()))
 210                         {
 211                                 CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) _T(": Crawl stop blocking path %s\n"), m_blockedPath.GetWinPath());
 212                                 m_blockedPath.Reset();
 213                         }
 214                         CGitStatusCache::Instance().RemoveTimedoutBlocks();
 215
 216                         while (!m_pathsToRelease.empty())
 217                         {
 218                                 AutoLocker lock(m_critSec);
 219                                 CTGitPath path = m_pathsToRelease.Pop();
 220                                 GitStatus::ReleasePath(path.GetWinPathString());
 221                         }
 222
 223                         if (m_foldersToUpdate.empty() && m_pathsToUpdate.empty())
 224                         {
 225                                 // Nothing left to do
 226                                 break;
 227                         }
 228                         currentTicks = GetTickCount();
 229                         if (!m_pathsToUpdate.empty())
 230                         {
 231                                 {
 232                                         AutoLocker lock(m_critSec);
 233
 234                                         m_bPathsAddedSinceLastCrawl = false;
 235
 236                                         workingPath = m_pathsToUpdate.Pop();
 237                                         if ((!m_blockedPath.IsEmpty()) && (m_blockedPath.IsAncestorOf(workingPath)))
 238                                         {
 239                                                 // move the path to the end of the list
 240                                                 m_pathsToUpdate.Push(workingPath);
 241                                                 if (m_pathsToUpdate.size() < 3)
 242                                                         Sleep(50);
 243                                                 continue;
 244                                         }
 245                                 }
 246
 247                                 // don't crawl paths that are excluded
 248                                 if (!CGitStatusCache::Instance().IsPathAllowed(workingPath))
 249                                         continue;
 250                                 // check if the changed path is inside an .git folder
 251                                 CString projectroot;
 252                                 if ((workingPath.HasAdminDir(&projectroot)&&workingPath.IsDirectory()) || workingPath.IsAdminDir())
 253                                 {
 254                                         // we don't crawl for paths changed in a tmp folder inside an .git folder.
 255                                         // Because we also get notifications for those even if we just ask for the status!
 256                                         // And changes there don't affect the file status at all, so it's safe
 257                                         // to ignore notifications on those paths.
 258                                         if (workingPath.IsAdminDir())
 259                                         {
 260                                                 // TODO: add git specific filters here. is there really any change besides index file in .git
 261                                                 //       that is relevant for overlays?
 262                                                 /*CString lowerpath = workingPath.GetWinPathString();
 263                                                 lowerpath.MakeLower();
 264                                                 if (lowerpath.Find(_T("\\tmp\\"))>0)
 265                                                         continue;
 266                                                 if (lowerpath.Find(_T("\\tmp")) == (lowerpath.GetLength()-4))
 267                                                         continue;
 268                                                 if (lowerpath.Find(_T("\\log"))>0)
 269                                                         continue;*/
 270                                                 // Here's a little problem:
 271                                                 // the lock file is also created for fetching the status
 272                                                 // and not just when committing.
 273                                                 // If we could find out why the lock file was changed
 274                                                 // we could decide to crawl the folder again or not.
 275                                                 // But for now, we have to crawl the parent folder
 276                                                 // no matter what.
 277
 278                                                 //if (lowerpath.Find(_T("\\lock"))>0)
 279                                                 //      continue;
 280                                                 // only go back to wc root if we are in .git-dir
 281                                                 do
 282                                                 {
 283                                                         workingPath = workingPath.GetContainingDirectory();
 284                                                 } while(workingPath.IsAdminDir());
 285                                         }
 286                                         else if (!workingPath.Exists())
 287                                         {
 288                                                 CAutoWriteLock writeLock(CGitStatusCache::Instance().GetGuard());
 289                                                 CGitStatusCache::Instance().RemoveCacheForPath(workingPath);
 290                                                 continue;
 291                                         }
 292
 293                                         if (!CGitStatusCache::Instance().IsPathGood(workingPath))
 294                                         {
 295                                                 AutoLocker lock(m_critSec);
 296                                                 // move the path, the root of the repository, to the end of the list
 297                                                 if (projectroot.IsEmpty())
 298                                                         m_pathsToUpdate.Push(workingPath);
 299                                                 else
 300                                                         m_pathsToUpdate.Push(CTGitPath(projectroot));
 301                                                 if (m_pathsToUpdate.size() < 3)
 302                                                         Sleep(50);
 303                                                 continue;
 304                                         }
 305
 306                                         CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) _T(": Invalidating and refreshing folder: %s\n"), workingPath.GetWinPath());
 307                                         {
 308                                                 AutoLocker print(critSec);
 309                                                 _sntprintf_s(szCurrentCrawledPath[nCurrentCrawledpathIndex], MAX_CRAWLEDPATHSLEN, _TRUNCATE, _T("Invalidating and refreshing folder: %s"), workingPath.GetWinPath());
 310                                                 ++nCurrentCrawledpathIndex;
 311                                                 if (nCurrentCrawledpathIndex >= MAX_CRAWLEDPATHS)
 312                                                         nCurrentCrawledpathIndex = 0;
 313                                         }
 314                                         InvalidateRect(hWnd, NULL, FALSE);
 315                                         {
 316                                                 CAutoReadLock readLock(CGitStatusCache::Instance().GetGuard());
 317                                                 // Invalidate the cache of this folder, to make sure its status is fetched again.
 318                                                 CCachedDirectory * pCachedDir = CGitStatusCache::Instance().GetDirectoryCacheEntry(workingPath);
 319                                                 if (pCachedDir)
 320                                                 {
 321                                                         git_wc_status_kind status = pCachedDir->GetCurrentFullStatus();
 322                                                         pCachedDir->Invalidate();
 323                                                         if (workingPath.Exists())
 324                                                         {
 325                                                                 pCachedDir->RefreshStatus(bRecursive);
 326                                                                 // if the previous status wasn't normal and now it is, then
 327                                                                 // send a notification too.
 328                                                                 // We do this here because GetCurrentFullStatus() doesn't send
 329                                                                 // notifications for 'normal' status - if it would, we'd get tons
 330                                                                 // of notifications when crawling a working copy not yet in the cache.
 331                                                                 if ((status != git_wc_status_normal) && (pCachedDir->GetCurrentFullStatus() != status))
 332                                                                 {
 333                                                                         CGitStatusCache::Instance().UpdateShell(workingPath);
 334                                                                         CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) _T(": shell update in crawler for %s\n"), workingPath.GetWinPath());
 335                                                                 }
 336                                                         }
 337                                                         else
 338                                                         {
 339                                                                 CAutoWriteLock writeLock(CGitStatusCache::Instance().GetGuard());
 340                                                                 CGitStatusCache::Instance().RemoveCacheForPath(workingPath);
 341                                                         }
 342                                                 }
 343                                         }
 344                                         //In case that svn_client_stat() modified a file and we got
 345                                         //a notification about that in the directory watcher,
 346                                         //remove that here again - this is to prevent an endless loop
 347                                         AutoLocker lock(m_critSec);
 348                                         m_pathsToUpdate.erase(workingPath);
 349                                 }
 350                                 else if (workingPath.HasAdminDir())
 351                                 {
 352                                         if (!workingPath.Exists())
 353                                         {
 354                                                 CAutoWriteLock writeLock(CGitStatusCache::Instance().GetGuard());
 355                                                 CGitStatusCache::Instance().RemoveCacheForPath(workingPath);
 356                                                 if (!workingPath.GetContainingDirectory().Exists())
 357                                                         continue;
 358                                                 else
 359                                                         workingPath = workingPath.GetContainingDirectory();
 360                                         }
 361                                         CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) _T(": Updating path: %s\n"), workingPath.GetWinPath());
 362                                         {
 363                                                 AutoLocker print(critSec);
 364                                                 _sntprintf_s(szCurrentCrawledPath[nCurrentCrawledpathIndex], MAX_CRAWLEDPATHSLEN, _TRUNCATE, _T("Updating path: %s"), workingPath.GetWinPath());
 365                                                 ++nCurrentCrawledpathIndex;
 366                                                 if (nCurrentCrawledpathIndex >= MAX_CRAWLEDPATHS)
 367                                                         nCurrentCrawledpathIndex = 0;
 368                                         }
 369                                         InvalidateRect(hWnd, NULL, FALSE);
 370                                         // HasAdminDir() already checks if the path points to a dir
 371                                         DWORD flags = TGITCACHE_FLAGS_FOLDERISKNOWN;
 372                                         flags |= (workingPath.IsDirectory() ? TGITCACHE_FLAGS_ISFOLDER : 0);
 373                                         flags |= (bRecursive ? TGITCACHE_FLAGS_RECUSIVE_STATUS : 0);
 374                                         {
 375                                                 CAutoReadLock readLock(CGitStatusCache::Instance().GetGuard());
 376                                                 // Invalidate the cache of folders manually. The cache of files is invalidated
 377                                                 // automatically if the status is asked for it and the file times don't match
 378                                                 // anymore, so we don't need to manually invalidate those.
 379                                                 if (workingPath.IsDirectory())
 380                                                 {
 381                                                         CCachedDirectory * cachedDir = CGitStatusCache::Instance().GetDirectoryCacheEntry(workingPath);
 382                                                         if (cachedDir)
 383                                                                 cachedDir->Invalidate();
 384                                                 }
 385                                                 CStatusCacheEntry ce = CGitStatusCache::Instance().GetStatusForPath(workingPath, flags);
 386                                                 if (ce.GetEffectiveStatus() > git_wc_status_unversioned)
 387                                                 {
 388                                                         CGitStatusCache::Instance().UpdateShell(workingPath);
 389                                                         CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) _T(": shell update in folder crawler for %s\n"), workingPath.GetWinPath());
 390                                                 }
 391                                         }
 392                                         AutoLocker lock(m_critSec);
 393                                         m_pathsToUpdate.erase(workingPath);
 394                                 }
 395                                 else
 396                                 {
 397                                         if (!workingPath.Exists())
 398                                         {
 399                                                 CAutoWriteLock writeLock(CGitStatusCache::Instance().GetGuard());
 400                                                 CGitStatusCache::Instance().RemoveCacheForPath(workingPath);
 401                                         }
 402                                 }
 403                         }
 404                         else if (!m_foldersToUpdate.empty())
 405                         {
 406                                 {
 407                                         AutoLocker lock(m_critSec);
 408                                         m_bItemsAddedSinceLastCrawl = false;
 409
 410                                         // create a new CTGitPath object to make sure the cached flags are requested again.
 411                                         // without this, a missing file/folder is still treated as missing even if it is available
 412                                         // now when crawling.
 413                                         workingPath = CTGitPath(m_foldersToUpdate.Pop().GetWinPath());
 414
 415                                         if ((!m_blockedPath.IsEmpty())&&(m_blockedPath.IsAncestorOf(workingPath)))
 416                                         {
 417                                                 // move the path to the end of the list
 418                                                 m_foldersToUpdate.Push(workingPath);
 419                                                 if (m_foldersToUpdate.size() < 3)
 420                                                         Sleep(50);
 421                                                 continue;
 422                                         }
 423                                 }
 424                                 if (DWORD(workingPath.GetCustomData()) >= currentTicks)
 425                                 {
 426                                         Sleep(50);
 427                                         continue;
 428                                 }
 429                                 if ((!m_blockedPath.IsEmpty())&&(m_blockedPath.IsAncestorOf(workingPath)))
 430                                         continue;
 431                                 if (!CGitStatusCache::Instance().IsPathAllowed(workingPath))
 432                                         continue;
 433                                 if (!CGitStatusCache::Instance().IsPathGood(workingPath))
 434                                         continue;
 435
 436                                 CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) _T(": Crawling folder: %s\n"), workingPath.GetWinPath());
 437                                 {
 438                                         AutoLocker print(critSec);
 439                                         _sntprintf_s(szCurrentCrawledPath[nCurrentCrawledpathIndex], MAX_CRAWLEDPATHSLEN, _TRUNCATE, _T("Crawling folder: %s"), workingPath.GetWinPath());
 440                                         ++nCurrentCrawledpathIndex;
 441                                         if (nCurrentCrawledpathIndex >= MAX_CRAWLEDPATHS)
 442                                                 nCurrentCrawledpathIndex = 0;
 443                                 }
 444                                 InvalidateRect(hWnd, NULL, FALSE);
 445                                 {
 446                                         CAutoReadLock readLock(CGitStatusCache::Instance().GetGuard());
 447                                         // Now, we need to visit this folder, to make sure that we know its 'most important' status
 448                                         CCachedDirectory * cachedDir = CGitStatusCache::Instance().GetDirectoryCacheEntry(workingPath.GetDirectory());
 449                                         // check if the path is monitored by the watcher. If it isn't, then we have to invalidate the cache
 450                                         // for that path and add it to the watcher.
 451                                         if (!CGitStatusCache::Instance().IsPathWatched(workingPath))
 452                                         {
 453                                                 if (workingPath.HasAdminDir())
 454                                                 {
 455                                                         CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) _T(": Add watch path %s\n"), workingPath.GetWinPath());
 456                                                         CGitStatusCache::Instance().AddPathToWatch(workingPath);
 457                                                 }
 458                                                 if (cachedDir)
 459                                                         cachedDir->Invalidate();
 460                                                 else
 461                                                 {
 462                                                         CAutoWriteLock writeLock(CGitStatusCache::Instance().GetGuard());
 463                                                         CGitStatusCache::Instance().RemoveCacheForPath(workingPath);
 464                                                         // now cacheDir is invalid because it got deleted in the RemoveCacheForPath() call above.
 465                                                         cachedDir = NULL;
 466                                                 }
 467                                         }
 468                                         if (cachedDir)
 469                                                 cachedDir->RefreshStatus(bRecursive);
 470                                 }
 471
 472                                 // While refreshing the status, we could get another crawl request for the same folder.
 473                                 // This can happen if the crawled folder has a lower status than one of the child folders
 474                                 // (recursively). To avoid double crawlings, remove such a crawl request here
 475                                 AutoLocker lock(m_critSec);
 476                                 if (m_bItemsAddedSinceLastCrawl)
 477                                 {
 478                                         m_foldersToUpdate.erase(workingPath);
 479                                 }
 480                         }
 481                 }
 482         }
 483         _endthread();
 484 }
 485
 486 bool CFolderCrawler::SetHoldoff(DWORD milliseconds /* = 100*/)
 487 {
 488         long tick = (long)GetTickCount();
 489         bool ret = ((tick - m_crawlHoldoffReleasesAt) > 0);
 490         m_crawlHoldoffReleasesAt = tick + milliseconds;
 491         return ret;
 492 }
 493
 494 void CFolderCrawler::BlockPath(const CTGitPath& path, DWORD ticks)
 495 {
 496         CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) _T(": block path %s from being crawled\n"), path.GetWinPath());
 497         m_blockedPath = path;
 498         if (ticks == 0)
 499                 m_blockReleasesAt = GetTickCount()+10000;
 500         else
 501                 m_blockReleasesAt = GetTickCount()+ticks;
 502 }