src/TGitCache/FolderCrawler.cpp

   1 // TortoiseGit - a Windows shell extension for easy version control
   2
   3 // External Cache Copyright (C) 2005-2008 - TortoiseSVN
   4 // Copyright (C) 2008-2011 - TortoiseGit
   5
   6 // This program is free software; you can redistribute it and/or
   7 // modify it under the terms of the GNU General Public License
   8 // as published by the Free Software Foundation; either version 2
   9 // of the License, or (at your option) any later version.
  10
  11 // This program is distributed in the hope that it will be useful,
  12 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 // GNU General Public License for more details.
  15
  16 // You should have received a copy of the GNU General Public License
  17 // along with this program; if not, write to the Free Software Foundation,
  18 // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  19 //
  20
  21 #include "StdAfx.h"
  22 #include ".\foldercrawler.h"
  23 #include "GitStatusCache.h"
  24 #include "registry.h"
  25 #include "TSVNCache.h"
  26 #include "shlobj.h"
  27
  28
  29 CFolderCrawler::CFolderCrawler(void)
  30 {
  31         m_hWakeEvent = CreateEvent(NULL,FALSE,FALSE,NULL);
  32         m_hTerminationEvent = CreateEvent(NULL,TRUE,FALSE,NULL);
  33         m_hThread = INVALID_HANDLE_VALUE;
  34         m_lCrawlInhibitSet = 0;
  35         m_crawlHoldoffReleasesAt = (long)GetTickCount();
  36         m_bRun = false;
  37         m_bPathsAddedSinceLastCrawl = false;
  38         m_bItemsAddedSinceLastCrawl = false;
  39 }
  40
  41 CFolderCrawler::~CFolderCrawler(void)
  42 {
  43         Stop();
  44 }
  45
  46 void CFolderCrawler::Stop()
  47 {
  48         m_bRun = false;
  49         if (m_hTerminationEvent != INVALID_HANDLE_VALUE)
  50         {
  51                 SetEvent(m_hTerminationEvent);
  52                 if(WaitForSingleObject(m_hThread, 4000) != WAIT_OBJECT_0)
  53                 {
  54                         ATLTRACE("Error terminating crawler thread\n");
  55                 }
  56                 CloseHandle(m_hThread);
  57                 m_hThread = INVALID_HANDLE_VALUE;
  58                 CloseHandle(m_hTerminationEvent);
  59                 m_hTerminationEvent = INVALID_HANDLE_VALUE;
  60                 CloseHandle(m_hWakeEvent);
  61                 m_hWakeEvent = INVALID_HANDLE_VALUE;
  62         }
  63 }
  64
  65 void CFolderCrawler::Initialise()
  66 {
  67         // Don't call Initialize more than once
  68         ATLASSERT(m_hThread == INVALID_HANDLE_VALUE);
  69
  70         // Just start the worker thread.
  71         // It will wait for event being signaled.
  72         // If m_hWakeEvent is already signaled the worker thread
  73         // will behave properly (with normal priority at worst).
  74
  75         m_bRun = true;
  76         unsigned int threadId;
  77         m_hThread = (HANDLE)_beginthreadex(NULL,0,ThreadEntry,this,0,&threadId);
  78         SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_IDLE);
  79 }
  80
  81 void CFolderCrawler::RemoveDuplicate(std::deque<CTGitPath> &list,const CTGitPath &path)
  82 {
  83         std::deque<CTGitPath>::iterator it, lastit;
  84         for(it = list.begin(); it != list.end(); ++it)
  85         {
  86                 if(*it == path)
  87                 {
  88                         list.erase(it);
  89                         it = list.begin(); /* search again*/
  90                         if(it == list.end())
  91                                 break;
  92                 }
  93         }
  94 }
  95 void CFolderCrawler::AddDirectoryForUpdate(const CTGitPath& path)
  96 {
  97         /* Index file changing*/
  98         if( GitStatus::IsExistIndexLockFile((CString&)path.GetWinPathString()))
  99                 return;
 100
 101         if (!CGitStatusCache::Instance().IsPathGood(path))
 102                 return;
 103         {
 104                 ATLTRACE(_T("AddDirectoryForUpdate %s\n"),path.GetWinPath());
 105
 106                 AutoLocker lock(m_critSec);
 107
 108                 RemoveDuplicate(m_foldersToUpdate, path);
 109
 110                 m_foldersToUpdate.push_back(path);
 111                 m_foldersToUpdate.back().SetCustomData(GetTickCount()+10);
 112
 113                 //ATLASSERT(path.IsDirectory() || !path.Exists());
 114                 // set this flag while we are sync'ed
 115                 // with the worker thread
 116                 m_bItemsAddedSinceLastCrawl = true;
 117         }
 118         //if (SetHoldoff())
 119                 SetEvent(m_hWakeEvent);
 120 }
 121
 122 void CFolderCrawler::AddPathForUpdate(const CTGitPath& path)
 123 {
 124         /* Index file changing*/
 125         if( GitStatus::IsExistIndexLockFile((CString&)path.GetWinPathString()))
 126                 return;
 127
 128         if (!CGitStatusCache::Instance().IsPathGood(path))
 129                 return;
 130
 131         {
 132                 AutoLocker lock(m_critSec);
 133
 134                 RemoveDuplicate(m_pathsToUpdate, path);
 135                 m_pathsToUpdate.push_back(path);
 136                 m_pathsToUpdate.back().SetCustomData(GetTickCount()+1000);
 137                 m_bPathsAddedSinceLastCrawl = true;
 138         }
 139         //if (SetHoldoff())
 140                 SetEvent(m_hWakeEvent);
 141 }
 142
 143 unsigned int CFolderCrawler::ThreadEntry(void* pContext)
 144 {
 145         ((CFolderCrawler*)pContext)->WorkerThread();
 146         return 0;
 147 }
 148
 149 void CFolderCrawler::WorkerThread()
 150 {
 151         HANDLE hWaitHandles[2];
 152         hWaitHandles[0] = m_hTerminationEvent;
 153         hWaitHandles[1] = m_hWakeEvent;
 154         CTGitPath workingPath;
 155         bool bFirstRunAfterWakeup = false;
 156         DWORD currentTicks = 0;
 157
 158         // Quick check if we're on Vista
 159         OSVERSIONINFOEX inf;
 160         SecureZeroMemory(&inf, sizeof(OSVERSIONINFOEX));
 161         inf.dwOSVersionInfoSize = sizeof(OSVERSIONINFOEX);
 162         GetVersionEx((OSVERSIONINFO *)&inf);
 163         WORD fullver = MAKEWORD(inf.dwMinorVersion, inf.dwMajorVersion);
 164
 165         for(;;)
 166         {
 167                 bool bRecursive = !!(DWORD)CRegStdWORD(_T("Software\\TortoiseGit\\RecursiveOverlay"), TRUE);
 168
 169                 if (fullver >= 0x0600)
 170                 {
 171                         SetThreadPriority(GetCurrentThread(), THREAD_MODE_BACKGROUND_END);
 172                 }
 173                 DWORD waitResult = WaitForMultipleObjects(sizeof(hWaitHandles)/sizeof(hWaitHandles[0]), hWaitHandles, FALSE, INFINITE);
 174
 175                 // exit event/working loop if the first event (m_hTerminationEvent)
 176                 // has been signaled or if one of the events has been abandoned
 177                 // (i.e. ~CFolderCrawler() is being executed)
 178                 if(m_bRun == false || waitResult == WAIT_OBJECT_0 || waitResult == WAIT_ABANDONED_0 || waitResult == WAIT_ABANDONED_0+1)
 179                 {
 180                         // Termination event
 181                         break;
 182                 }
 183
 184                 if (fullver >= 0x0600)
 185                 {
 186                         SetThreadPriority(GetCurrentThread(), THREAD_MODE_BACKGROUND_BEGIN);
 187                 }
 188
 189                 // If we get here, we've been woken up by something being added to the queue.
 190                 // However, it's important that we don't do our crawling while
 191                 // the shell is still asking for items
 192                 bFirstRunAfterWakeup = true;
 193                 for(;;)
 194                 {
 195                         if (!m_bRun)
 196                                 break;
 197                         // Any locks today?
 198                         if (CGitStatusCache::Instance().m_bClearMemory)
 199                         {
 200                                 CGitStatusCache::Instance().WaitToWrite();
 201                                 CGitStatusCache::Instance().ClearCache();
 202                                 CGitStatusCache::Instance().Done();
 203                                 CGitStatusCache::Instance().m_bClearMemory = false;
 204                         }
 205                         if(m_lCrawlInhibitSet > 0)
 206                         {
 207                                 // We're in crawl hold-off
 208                                 ATLTRACE("Crawl hold-off\n");
 209                                 Sleep(50);
 210                                 continue;
 211                         }
 212                         if (bFirstRunAfterWakeup)
 213                         {
 214                                 Sleep(20);
 215                                 ATLTRACE("Crawl bFirstRunAfterWakeup\n");
 216                                 bFirstRunAfterWakeup = false;
 217                                 continue;
 218                         }
 219                         if ((m_blockReleasesAt < GetTickCount())&&(!m_blockedPath.IsEmpty()))
 220                         {
 221                                 ATLTRACE(_T("Crawl stop blocking path %s\n"), m_blockedPath.GetWinPath());
 222                                 m_blockedPath.Reset();
 223                         }
 224
 225                         if ((m_foldersToUpdate.empty())&&(m_pathsToUpdate.empty()))
 226                         {
 227                                 // Nothing left to do
 228                                 break;
 229                         }
 230                         currentTicks = GetTickCount();
 231                         if (!m_pathsToUpdate.empty())
 232                         {
 233                                 {
 234                                         AutoLocker lock(m_critSec);
 235
 236                                         m_bPathsAddedSinceLastCrawl = false;
 237
 238                                         workingPath = m_pathsToUpdate.front();
 239                                         //m_pathsToUpdateUnique.erase (workingPath);
 240                                         m_pathsToUpdate.pop_front();
 241                                         if ((DWORD(workingPath.GetCustomData()) >= currentTicks) ||
 242                                                 ((!m_blockedPath.IsEmpty())&&(m_blockedPath.IsAncestorOf(workingPath))))
 243                                         {
 244                                                 // move the path to the end of the list
 245                                                 //m_pathsToUpdateUnique.insert (workingPath);
 246                                                 m_pathsToUpdate.push_back(workingPath);
 247                                                 if (m_pathsToUpdate.size() < 3)
 248                                                         Sleep(50);
 249                                                 continue;
 250                                         }
 251                                 }
 252
 253                                 // don't crawl paths that are excluded
 254                                 if (!CGitStatusCache::Instance().IsPathAllowed(workingPath))
 255                                         continue;
 256                                 // check if the changed path is inside an .git folder
 257                                 CString projectroot;
 258                                 if ((workingPath.HasAdminDir(&projectroot)&&workingPath.IsDirectory()) || workingPath.IsAdminDir())
 259                                 {
 260                                         // we don't crawl for paths changed in a tmp folder inside an .git folder.
 261                                         // Because we also get notifications for those even if we just ask for the status!
 262                                         // And changes there don't affect the file status at all, so it's safe
 263                                         // to ignore notifications on those paths.
 264                                         if (workingPath.IsAdminDir())
 265                                         {
 266                                                 // TODO: add git specific filters here. is there really any change besides index file in .git
 267                                                 //       that is relevant for overlays?
 268                                                 /*CString lowerpath = workingPath.GetWinPathString();
 269                                                 lowerpath.MakeLower();
 270                                                 if (lowerpath.Find(_T("\\tmp\\"))>0)
 271                                                         continue;
 272                                                 if (lowerpath.Find(_T("\\tmp")) == (lowerpath.GetLength()-4))
 273                                                         continue;
 274                                                 if (lowerpath.Find(_T("\\log"))>0)
 275                                                         continue;*/
 276                                                 // Here's a little problem:
 277                                                 // the lock file is also created for fetching the status
 278                                                 // and not just when committing.
 279                                                 // If we could find out why the lock file was changed
 280                                                 // we could decide to crawl the folder again or not.
 281                                                 // But for now, we have to crawl the parent folder
 282                                                 // no matter what.
 283
 284                                                 //if (lowerpath.Find(_T("\\lock"))>0)
 285                                                 //      continue;
 286                                         }
 287                                         else if (!workingPath.Exists())
 288                                         {
 289                                                 CGitStatusCache::Instance().WaitToWrite();
 290                                                 CGitStatusCache::Instance().RemoveCacheForPath(workingPath);
 291                                                 CGitStatusCache::Instance().Done();
 292                                                 continue;
 293                                         }
 294
 295                                         do
 296                                         {
 297                                                 workingPath = workingPath.GetContainingDirectory();
 298                                         } while(workingPath.IsAdminDir());
 299
 300                                         ATLTRACE(_T("Invalidating and refreshing folder: %s\n"), workingPath.GetWinPath());
 301                                         {
 302                                                 AutoLocker print(critSec);
 303                                                 _stprintf_s(szCurrentCrawledPath[nCurrentCrawledpathIndex], MAX_CRAWLEDPATHSLEN, _T("Invalidating and refreshing folder: %s"), workingPath.GetWinPath());
 304                                                 nCurrentCrawledpathIndex++;
 305                                                 if (nCurrentCrawledpathIndex >= MAX_CRAWLEDPATHS)
 306                                                         nCurrentCrawledpathIndex = 0;
 307                                         }
 308                                         InvalidateRect(hWnd, NULL, FALSE);
 309                                         CGitStatusCache::Instance().WaitToRead();
 310                                         // Invalidate the cache of this folder, to make sure its status is fetched again.
 311                                         CCachedDirectory * pCachedDir = CGitStatusCache::Instance().GetDirectoryCacheEntry(workingPath);
 312                                         if (pCachedDir)
 313                                         {
 314                                                 git_wc_status_kind status = pCachedDir->GetCurrentFullStatus();
 315                                                 pCachedDir->Invalidate();
 316                                                 if (workingPath.Exists())
 317                                                 {
 318                                                         pCachedDir->RefreshStatus(bRecursive);
 319                                                         // if the previous status wasn't normal and now it is, then
 320                                                         // send a notification too.
 321                                                         // We do this here because GetCurrentFullStatus() doesn't send
 322                                                         // notifications for 'normal' status - if it would, we'd get tons
 323                                                         // of notifications when crawling a working copy not yet in the cache.
 324                                                         if ((status != git_wc_status_normal)&&(pCachedDir->GetCurrentFullStatus() != status))
 325                                                         {
 326                                                                 CGitStatusCache::Instance().UpdateShell(workingPath);
 327                                                                 ATLTRACE(_T("shell update in crawler for %s\n"), workingPath.GetWinPath());
 328                                                         }
 329                                                 }
 330                                                 else
 331                                                 {
 332                                                         CGitStatusCache::Instance().Done();
 333                                                         CGitStatusCache::Instance().WaitToWrite();
 334                                                         CGitStatusCache::Instance().RemoveCacheForPath(workingPath);
 335                                                 }
 336                                         }
 337                                         CGitStatusCache::Instance().Done();
 338                                         //In case that svn_client_stat() modified a file and we got
 339                                         //a notification about that in the directory watcher,
 340                                         //remove that here again - this is to prevent an endless loop
 341                                         AutoLocker lock(m_critSec);
 342                                         m_pathsToUpdate.erase(std::remove(m_pathsToUpdate.begin(), m_pathsToUpdate.end(), workingPath), m_pathsToUpdate.end());
 343                                 }
 344                                 else if (workingPath.HasAdminDir())
 345                                 {
 346                                         if (!workingPath.Exists())
 347                                         {
 348                                                 CGitStatusCache::Instance().WaitToWrite();
 349                                                 CGitStatusCache::Instance().RemoveCacheForPath(workingPath);
 350                                                 CGitStatusCache::Instance().Done();
 351                                                 continue;
 352                                         }
 353                                         if (!workingPath.Exists())
 354                                                 continue;
 355                                         ATLTRACE(_T("Updating path: %s\n"), workingPath.GetWinPath());
 356                                         {
 357                                                 AutoLocker print(critSec);
 358                                                 _stprintf_s(szCurrentCrawledPath[nCurrentCrawledpathIndex], MAX_CRAWLEDPATHSLEN, _T("Updating path: %s"), workingPath.GetWinPath());
 359                                                 nCurrentCrawledpathIndex++;
 360                                                 if (nCurrentCrawledpathIndex >= MAX_CRAWLEDPATHS)
 361                                                         nCurrentCrawledpathIndex = 0;
 362                                         }
 363                                         InvalidateRect(hWnd, NULL, FALSE);
 364                                         // HasAdminDir() already checks if the path points to a dir
 365                                         DWORD flags = TSVNCACHE_FLAGS_FOLDERISKNOWN;
 366                                         flags |= (workingPath.IsDirectory() ? TSVNCACHE_FLAGS_ISFOLDER : 0);
 367                                         flags |= (bRecursive ? TSVNCACHE_FLAGS_RECUSIVE_STATUS : 0);
 368                                         CGitStatusCache::Instance().WaitToRead();
 369                                         // Invalidate the cache of folders manually. The cache of files is invalidated
 370                                         // automatically if the status is asked for it and the file times don't match
 371                                         // anymore, so we don't need to manually invalidate those.
 372                                         if (workingPath.IsDirectory())
 373                                         {
 374                                                 CCachedDirectory * cachedDir = CGitStatusCache::Instance().GetDirectoryCacheEntry(workingPath);
 375                                                 if (cachedDir)
 376                                                         cachedDir->Invalidate();
 377                                         }
 378                                         CStatusCacheEntry ce = CGitStatusCache::Instance().GetStatusForPath(workingPath, flags);
 379                                         if (ce.GetEffectiveStatus() > git_wc_status_unversioned)
 380                                         {
 381                                                 CGitStatusCache::Instance().UpdateShell(workingPath);
 382                                                 ATLTRACE(_T("shell update in folder crawler for %s\n"), workingPath.GetWinPath());
 383                                         }
 384                                         CGitStatusCache::Instance().Done();
 385                                         AutoLocker lock(m_critSec);
 386                                         m_pathsToUpdate.erase(std::remove(m_pathsToUpdate.begin(), m_pathsToUpdate.end(), workingPath), m_pathsToUpdate.end());
 387                                 }
 388                                 else
 389                                 {
 390                                         if (!workingPath.Exists())
 391                                         {
 392                                                 CGitStatusCache::Instance().WaitToWrite();
 393                                                 CGitStatusCache::Instance().RemoveCacheForPath(workingPath);
 394                                                 CGitStatusCache::Instance().Done();
 395                                         }
 396                                 }
 397                         }
 398                         else if (!m_foldersToUpdate.empty())
 399                         {
 400                                 {
 401                                         AutoLocker lock(m_critSec);
 402                                         m_bItemsAddedSinceLastCrawl = false;
 403
 404                                         // create a new CTSVNPath object to make sure the cached flags are requested again.
 405                                         // without this, a missing file/folder is still treated as missing even if it is available
 406                                         // now when crawling.
 407                                         CTGitPath& folderToUpdate = m_foldersToUpdate.front();
 408                                         workingPath = CTGitPath(folderToUpdate.GetWinPath());
 409                                         workingPath.SetCustomData(folderToUpdate.GetCustomData());
 410                                         m_foldersToUpdate.pop_front();
 411
 412                                         if ((DWORD(workingPath.GetCustomData()) >= currentTicks) ||
 413                                                 ((!m_blockedPath.IsEmpty())&&(m_blockedPath.IsAncestorOf(workingPath))))
 414                                         {
 415                                                 // move the path to the end of the list
 416                                                 m_foldersToUpdate.push_back (workingPath);
 417                                                 if (m_foldersToUpdate.size() < 3)
 418                                                         Sleep(50);
 419                                                 continue;
 420                                         }
 421                                 }
 422                                 if (DWORD(workingPath.GetCustomData()) >= currentTicks)
 423                                 {
 424                                         Sleep(50);
 425                                         continue;
 426                                 }
 427                                 if ((!m_blockedPath.IsEmpty())&&(m_blockedPath.IsAncestorOf(workingPath)))
 428                                         continue;
 429                                 if (!CGitStatusCache::Instance().IsPathAllowed(workingPath))
 430                                         continue;
 431
 432                                 ATLTRACE(_T("Crawling folder: %s\n"), workingPath.GetWinPath());
 433                                 {
 434                                         AutoLocker print(critSec);
 435                                         _stprintf_s(szCurrentCrawledPath[nCurrentCrawledpathIndex], MAX_CRAWLEDPATHSLEN, _T("Crawling folder: %s"), workingPath.GetWinPath());
 436                                         nCurrentCrawledpathIndex++;
 437                                         if (nCurrentCrawledpathIndex >= MAX_CRAWLEDPATHS)
 438                                                 nCurrentCrawledpathIndex = 0;
 439                                 }
 440                                 InvalidateRect(hWnd, NULL, FALSE);
 441                                 CGitStatusCache::Instance().WaitToRead();
 442                                 // Now, we need to visit this folder, to make sure that we know its 'most important' status
 443                                 CCachedDirectory * cachedDir = CGitStatusCache::Instance().GetDirectoryCacheEntry(workingPath.GetDirectory());
 444                                 // check if the path is monitored by the watcher. If it isn't, then we have to invalidate the cache
 445                                 // for that path and add it to the watcher.
 446                                 if (!CGitStatusCache::Instance().IsPathWatched(workingPath))
 447                                 {
 448                                         if (workingPath.HasAdminDir())
 449                                         {
 450                                                 ATLTRACE(_T("Add watch path %s\n"), workingPath.GetWinPath());
 451                                                 CGitStatusCache::Instance().AddPathToWatch(workingPath);
 452                                         }
 453                                         if (cachedDir)
 454                                                 cachedDir->Invalidate();
 455                                         else
 456                                         {
 457                                                 CGitStatusCache::Instance().Done();
 458                                                 CGitStatusCache::Instance().WaitToWrite();
 459                                                 CGitStatusCache::Instance().RemoveCacheForPath(workingPath);
 460                                         }
 461                                 }
 462                                 if (cachedDir)
 463                                         cachedDir->RefreshStatus(bRecursive);
 464
 465 #if 0
 466                                 // While refreshing the status, we could get another crawl request for the same folder.
 467                                 // This can happen if the crawled folder has a lower status than one of the child folders
 468                                 // (recursively). To avoid double crawlings, remove such a crawl request here
 469                                 AutoLocker lock(m_critSec);
 470                                 if (m_bItemsAddedSinceLastCrawl)
 471                                 {
 472                                         if (m_foldersToUpdate.back().IsEquivalentToWithoutCase(workingPath))
 473                                         {
 474                                                 m_foldersToUpdate.pop_back();
 475                                                 m_bItemsAddedSinceLastCrawl = false;
 476                                         }
 477                                 }
 478 #endif
 479                                 CGitStatusCache::Instance().Done();
 480                         }
 481                 }
 482         }
 483         _endthread();
 484 }
 485
 486 bool CFolderCrawler::SetHoldoff(DWORD milliseconds /* = 100*/)
 487 {
 488         long tick = (long)GetTickCount();
 489         bool ret = ((tick - m_crawlHoldoffReleasesAt) > 0);
 490         m_crawlHoldoffReleasesAt = tick + milliseconds;
 491         return ret;
 492 }
 493
 494 void CFolderCrawler::BlockPath(const CTGitPath& path, DWORD ticks)
 495 {
 496         ATLTRACE(_T("block path %s from being crawled\n"), path.GetWinPath());
 497         m_blockedPath = path;
 498         if (ticks == 0)
 499                 m_blockReleasesAt = GetTickCount()+10000;
 500         else
 501                 m_blockReleasesAt = GetTickCount()+ticks;
 502 }