src/TGitCache/FolderCrawler.cpp

   1 // TortoiseGit - a Windows shell extension for easy version control
   2
   3 // External Cache Copyright (C) 2005-2008,2011 - TortoiseSVN
   4 // Copyright (C) 2008-2012 - TortoiseGit
   5
   6 // This program is free software; you can redistribute it and/or
   7 // modify it under the terms of the GNU General Public License
   8 // as published by the Free Software Foundation; either version 2
   9 // of the License, or (at your option) any later version.
  10
  11 // This program is distributed in the hope that it will be useful,
  12 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 // GNU General Public License for more details.
  15
  16 // You should have received a copy of the GNU General Public License
  17 // along with this program; if not, write to the Free Software Foundation,
  18 // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  19 //
  20
  21 #include "stdafx.h"
  22 #include "foldercrawler.h"
  23 #include "GitStatusCache.h"
  24 #include "registry.h"
  25 #include "TGitCache.h"
  26 #include "shlobj.h"
  27 #include "SysInfo.h"
  28
  29
  30 CFolderCrawler::CFolderCrawler(void)
  31 {
  32         m_hWakeEvent = CreateEvent(NULL,FALSE,FALSE,NULL);
  33         m_hTerminationEvent = CreateEvent(NULL,TRUE,FALSE,NULL);
  34         m_lCrawlInhibitSet = 0;
  35         m_crawlHoldoffReleasesAt = (long)GetTickCount();
  36         m_bRun = false;
  37         m_bPathsAddedSinceLastCrawl = false;
  38         m_bItemsAddedSinceLastCrawl = false;
  39 }
  40
  41 CFolderCrawler::~CFolderCrawler(void)
  42 {
  43         Stop();
  44 }
  45
  46 void CFolderCrawler::Stop()
  47 {
  48         m_bRun = false;
  49         if (m_hTerminationEvent)
  50         {
  51                 SetEvent(m_hTerminationEvent);
  52                 if(WaitForSingleObject(m_hThread, 4000) != WAIT_OBJECT_0)
  53                 {
  54                         ATLTRACE("Error terminating crawler thread\n");
  55                 }
  56         }
  57         m_hThread.CloseHandle();
  58         m_hTerminationEvent.CloseHandle();
  59         m_hWakeEvent.CloseHandle();
  60 }
  61
  62 void CFolderCrawler::Initialise()
  63 {
  64         // Don't call Initialize more than once
  65         ATLASSERT(!m_hThread);
  66
  67         // Just start the worker thread.
  68         // It will wait for event being signaled.
  69         // If m_hWakeEvent is already signaled the worker thread
  70         // will behave properly (with normal priority at worst).
  71
  72         m_bRun = true;
  73         unsigned int threadId;
  74         m_hThread = (HANDLE)_beginthreadex(NULL,0,ThreadEntry,this,0,&threadId);
  75         SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_IDLE);
  76 }
  77
  78 void CFolderCrawler::RemoveDuplicate(std::deque<CTGitPath> &list,const CTGitPath &path)
  79 {
  80         std::deque<CTGitPath>::iterator it;
  81         for(it = list.begin(); it != list.end(); ++it)
  82         {
  83                 if(*it == path)
  84                 {
  85                         list.erase(it);
  86                         it = list.begin(); /* search again*/
  87                         if(it == list.end())
  88                                 break;
  89                 }
  90         }
  91 }
  92 void CFolderCrawler::AddDirectoryForUpdate(const CTGitPath& path)
  93 {
  94         /* Index file changing*/
  95         if( GitStatus::IsExistIndexLockFile((CString&)path.GetWinPathString()))
  96                 return;
  97
  98         if (!CGitStatusCache::Instance().IsPathGood(path))
  99                 return;
 100         {
 101                 ATLTRACE(_T("AddDirectoryForUpdate %s\n"),path.GetWinPath());
 102
 103                 AutoLocker lock(m_critSec);
 104
 105                 m_foldersToUpdate.Push(path);
 106
 107                 //ATLASSERT(path.IsDirectory() || !path.Exists());
 108                 // set this flag while we are sync'ed
 109                 // with the worker thread
 110                 m_bItemsAddedSinceLastCrawl = true;
 111         }
 112         //if (SetHoldoff())
 113                 SetEvent(m_hWakeEvent);
 114 }
 115
 116 void CFolderCrawler::AddPathForUpdate(const CTGitPath& path)
 117 {
 118         /* Index file changing*/
 119         if( GitStatus::IsExistIndexLockFile((CString&)path.GetWinPathString()))
 120                 return;
 121
 122         {
 123                 AutoLocker lock(m_critSec);
 124
 125                 m_pathsToUpdate.Push(path);
 126                 m_bPathsAddedSinceLastCrawl = true;
 127         }
 128         //if (SetHoldoff())
 129                 SetEvent(m_hWakeEvent);
 130 }
 131
 132 unsigned int CFolderCrawler::ThreadEntry(void* pContext)
 133 {
 134         ((CFolderCrawler*)pContext)->WorkerThread();
 135         return 0;
 136 }
 137
 138 void CFolderCrawler::WorkerThread()
 139 {
 140         HANDLE hWaitHandles[2];
 141         hWaitHandles[0] = m_hTerminationEvent;
 142         hWaitHandles[1] = m_hWakeEvent;
 143         CTGitPath workingPath;
 144         bool bFirstRunAfterWakeup = false;
 145         DWORD currentTicks = 0;
 146
 147         for(;;)
 148         {
 149                 bool bRecursive = !!(DWORD)CRegStdDWORD(_T("Software\\TortoiseGit\\RecursiveOverlay"), TRUE);
 150
 151                 if (SysInfo::Instance().IsVistaOrLater())
 152                 {
 153                         SetThreadPriority(GetCurrentThread(), THREAD_MODE_BACKGROUND_END);
 154                 }
 155                 DWORD waitResult = WaitForMultipleObjects(_countof(hWaitHandles), hWaitHandles, FALSE, INFINITE);
 156
 157                 // exit event/working loop if the first event (m_hTerminationEvent)
 158                 // has been signaled or if one of the events has been abandoned
 159                 // (i.e. ~CFolderCrawler() is being executed)
 160                 if(m_bRun == false || waitResult == WAIT_OBJECT_0 || waitResult == WAIT_ABANDONED_0 || waitResult == WAIT_ABANDONED_0+1)
 161                 {
 162                         // Termination event
 163                         break;
 164                 }
 165
 166                 if (SysInfo::Instance().IsVistaOrLater())
 167                 {
 168                         SetThreadPriority(GetCurrentThread(), THREAD_MODE_BACKGROUND_BEGIN);
 169                 }
 170
 171                 // If we get here, we've been woken up by something being added to the queue.
 172                 // However, it's important that we don't do our crawling while
 173                 // the shell is still asking for items
 174                 bFirstRunAfterWakeup = true;
 175                 for(;;)
 176                 {
 177                         if (!m_bRun)
 178                                 break;
 179                         // Any locks today?
 180                         if (CGitStatusCache::Instance().m_bClearMemory)
 181                         {
 182                                 CAutoWriteLock writeLock(CGitStatusCache::Instance().GetGuard());
 183                                 CGitStatusCache::Instance().ClearCache();
 184                                 CGitStatusCache::Instance().m_bClearMemory = false;
 185                         }
 186                         if(m_lCrawlInhibitSet > 0)
 187                         {
 188                                 // We're in crawl hold-off
 189                                 ATLTRACE("Crawl hold-off\n");
 190                                 Sleep(50);
 191                                 continue;
 192                         }
 193                         if (bFirstRunAfterWakeup)
 194                         {
 195                                 Sleep(20);
 196                                 ATLTRACE("Crawl bFirstRunAfterWakeup\n");
 197                                 bFirstRunAfterWakeup = false;
 198                                 continue;
 199                         }
 200                         if ((m_blockReleasesAt < GetTickCount())&&(!m_blockedPath.IsEmpty()))
 201                         {
 202                                 ATLTRACE(_T("Crawl stop blocking path %s\n"), m_blockedPath.GetWinPath());
 203                                 m_blockedPath.Reset();
 204                         }
 205                         CGitStatusCache::Instance().RemoveTimedoutBlocks();
 206
 207                         if (m_foldersToUpdate.empty() && m_pathsToUpdate.empty())
 208                         {
 209                                 // Nothing left to do
 210                                 break;
 211                         }
 212                         currentTicks = GetTickCount();
 213                         if (!m_pathsToUpdate.empty())
 214                         {
 215                                 {
 216                                         AutoLocker lock(m_critSec);
 217
 218                                         m_bPathsAddedSinceLastCrawl = false;
 219
 220                                         workingPath = m_pathsToUpdate.Pop();
 221                                         if ((!m_blockedPath.IsEmpty()) && (m_blockedPath.IsAncestorOf(workingPath)))
 222                                         {
 223                                                 // move the path to the end of the list
 224                                                 m_pathsToUpdate.Push(workingPath);
 225                                                 if (m_pathsToUpdate.size() < 3)
 226                                                         Sleep(50);
 227                                                 continue;
 228                                         }
 229                                 }
 230
 231                                 // don't crawl paths that are excluded
 232                                 if (!CGitStatusCache::Instance().IsPathAllowed(workingPath))
 233                                         continue;
 234                                 // check if the changed path is inside an .git folder
 235                                 CString projectroot;
 236                                 if ((workingPath.HasAdminDir(&projectroot)&&workingPath.IsDirectory()) || workingPath.IsAdminDir())
 237                                 {
 238                                         // we don't crawl for paths changed in a tmp folder inside an .git folder.
 239                                         // Because we also get notifications for those even if we just ask for the status!
 240                                         // And changes there don't affect the file status at all, so it's safe
 241                                         // to ignore notifications on those paths.
 242                                         if (workingPath.IsAdminDir())
 243                                         {
 244                                                 // TODO: add git specific filters here. is there really any change besides index file in .git
 245                                                 //       that is relevant for overlays?
 246                                                 /*CString lowerpath = workingPath.GetWinPathString();
 247                                                 lowerpath.MakeLower();
 248                                                 if (lowerpath.Find(_T("\\tmp\\"))>0)
 249                                                         continue;
 250                                                 if (lowerpath.Find(_T("\\tmp")) == (lowerpath.GetLength()-4))
 251                                                         continue;
 252                                                 if (lowerpath.Find(_T("\\log"))>0)
 253                                                         continue;*/
 254                                                 // Here's a little problem:
 255                                                 // the lock file is also created for fetching the status
 256                                                 // and not just when committing.
 257                                                 // If we could find out why the lock file was changed
 258                                                 // we could decide to crawl the folder again or not.
 259                                                 // But for now, we have to crawl the parent folder
 260                                                 // no matter what.
 261
 262                                                 //if (lowerpath.Find(_T("\\lock"))>0)
 263                                                 //      continue;
 264                                                 // only go back to wc root if we are in .git-dir
 265                                                 do
 266                                                 {
 267                                                         workingPath = workingPath.GetContainingDirectory();
 268                                                 } while(workingPath.IsAdminDir());
 269                                         }
 270                                         else if (!workingPath.Exists())
 271                                         {
 272                                                 CAutoWriteLock writeLock(CGitStatusCache::Instance().GetGuard());
 273                                                 CGitStatusCache::Instance().RemoveCacheForPath(workingPath);
 274                                                 continue;
 275                                         }
 276
 277                                         if (!CGitStatusCache::Instance().IsPathGood(workingPath))
 278                                         {
 279                                                 AutoLocker lock(m_critSec);
 280                                                 // move the path, the root of the repository, to the end of the list
 281                                                 if (projectroot.IsEmpty())
 282                                                         m_pathsToUpdate.Push(workingPath);
 283                                                 else
 284                                                         m_pathsToUpdate.Push(CTGitPath(projectroot));
 285                                                 if (m_pathsToUpdate.size() < 3)
 286                                                         Sleep(50);
 287                                                 continue;
 288                                         }
 289
 290                                         ATLTRACE(_T("Invalidating and refreshing folder: %s\n"), workingPath.GetWinPath());
 291                                         {
 292                                                 AutoLocker print(critSec);
 293                                                 _sntprintf_s(szCurrentCrawledPath[nCurrentCrawledpathIndex], MAX_CRAWLEDPATHSLEN, _TRUNCATE, _T("Invalidating and refreshing folder: %s"), workingPath.GetWinPath());
 294                                                 nCurrentCrawledpathIndex++;
 295                                                 if (nCurrentCrawledpathIndex >= MAX_CRAWLEDPATHS)
 296                                                         nCurrentCrawledpathIndex = 0;
 297                                         }
 298                                         InvalidateRect(hWnd, NULL, FALSE);
 299                                         {
 300                                                 CAutoReadLock readLock(CGitStatusCache::Instance().GetGuard());
 301                                                 // Invalidate the cache of this folder, to make sure its status is fetched again.
 302                                                 CCachedDirectory * pCachedDir = CGitStatusCache::Instance().GetDirectoryCacheEntry(workingPath);
 303                                                 if (pCachedDir)
 304                                                 {
 305                                                         git_wc_status_kind status = pCachedDir->GetCurrentFullStatus();
 306                                                         pCachedDir->Invalidate();
 307                                                         if (workingPath.Exists())
 308                                                         {
 309                                                                 pCachedDir->RefreshStatus(bRecursive);
 310                                                                 // if the previous status wasn't normal and now it is, then
 311                                                                 // send a notification too.
 312                                                                 // We do this here because GetCurrentFullStatus() doesn't send
 313                                                                 // notifications for 'normal' status - if it would, we'd get tons
 314                                                                 // of notifications when crawling a working copy not yet in the cache.
 315                                                                 if ((status != git_wc_status_normal) && (pCachedDir->GetCurrentFullStatus() != status))
 316                                                                 {
 317                                                                         CGitStatusCache::Instance().UpdateShell(workingPath);
 318                                                                         ATLTRACE(_T("shell update in crawler for %s\n"), workingPath.GetWinPath());
 319                                                                 }
 320                                                         }
 321                                                         else
 322                                                         {
 323                                                                 CAutoWriteLock writeLock(CGitStatusCache::Instance().GetGuard());
 324                                                                 CGitStatusCache::Instance().RemoveCacheForPath(workingPath);
 325                                                         }
 326                                                 }
 327                                         }
 328                                         //In case that svn_client_stat() modified a file and we got
 329                                         //a notification about that in the directory watcher,
 330                                         //remove that here again - this is to prevent an endless loop
 331                                         AutoLocker lock(m_critSec);
 332                                         m_pathsToUpdate.erase(workingPath);
 333                                 }
 334                                 else if (workingPath.HasAdminDir())
 335                                 {
 336                                         if (!workingPath.Exists())
 337                                         {
 338                                                 CAutoWriteLock writeLock(CGitStatusCache::Instance().GetGuard());
 339                                                 CGitStatusCache::Instance().RemoveCacheForPath(workingPath);
 340                                                 if (!workingPath.GetContainingDirectory().Exists())
 341                                                         continue;
 342                                                 else
 343                                                         workingPath = workingPath.GetContainingDirectory();
 344                                         }
 345                                         ATLTRACE(_T("Updating path: %s\n"), workingPath.GetWinPath());
 346                                         {
 347                                                 AutoLocker print(critSec);
 348                                                 _sntprintf_s(szCurrentCrawledPath[nCurrentCrawledpathIndex], MAX_CRAWLEDPATHSLEN, _TRUNCATE, _T("Updating path: %s"), workingPath.GetWinPath());
 349                                                 nCurrentCrawledpathIndex++;
 350                                                 if (nCurrentCrawledpathIndex >= MAX_CRAWLEDPATHS)
 351                                                         nCurrentCrawledpathIndex = 0;
 352                                         }
 353                                         InvalidateRect(hWnd, NULL, FALSE);
 354                                         // HasAdminDir() already checks if the path points to a dir
 355                                         DWORD flags = TGITCACHE_FLAGS_FOLDERISKNOWN;
 356                                         flags |= (workingPath.IsDirectory() ? TGITCACHE_FLAGS_ISFOLDER : 0);
 357                                         flags |= (bRecursive ? TGITCACHE_FLAGS_RECUSIVE_STATUS : 0);
 358                                         {
 359                                                 CAutoReadLock readLock(CGitStatusCache::Instance().GetGuard());
 360                                                 // Invalidate the cache of folders manually. The cache of files is invalidated
 361                                                 // automatically if the status is asked for it and the file times don't match
 362                                                 // anymore, so we don't need to manually invalidate those.
 363                                                 if (workingPath.IsDirectory())
 364                                                 {
 365                                                         CCachedDirectory * cachedDir = CGitStatusCache::Instance().GetDirectoryCacheEntry(workingPath);
 366                                                         if (cachedDir)
 367                                                                 cachedDir->Invalidate();
 368                                                 }
 369                                                 CStatusCacheEntry ce = CGitStatusCache::Instance().GetStatusForPath(workingPath, flags);
 370                                                 if (ce.GetEffectiveStatus() > git_wc_status_unversioned)
 371                                                 {
 372                                                         CGitStatusCache::Instance().UpdateShell(workingPath);
 373                                                         ATLTRACE(_T("shell update in folder crawler for %s\n"), workingPath.GetWinPath());
 374                                                 }
 375                                         }
 376                                         AutoLocker lock(m_critSec);
 377                                         m_pathsToUpdate.erase(workingPath);
 378                                 }
 379                                 else
 380                                 {
 381                                         if (!workingPath.Exists())
 382                                         {
 383                                                 CAutoWriteLock writeLock(CGitStatusCache::Instance().GetGuard());
 384                                                 CGitStatusCache::Instance().RemoveCacheForPath(workingPath);
 385                                         }
 386                                 }
 387                         }
 388                         else if (!m_foldersToUpdate.empty())
 389                         {
 390                                 {
 391                                         AutoLocker lock(m_critSec);
 392                                         m_bItemsAddedSinceLastCrawl = false;
 393
 394                                         // create a new CTSVNPath object to make sure the cached flags are requested again.
 395                                         // without this, a missing file/folder is still treated as missing even if it is available
 396                                         // now when crawling.
 397                                         workingPath = CTGitPath(m_foldersToUpdate.Pop().GetWinPath());
 398
 399                                         if ((!m_blockedPath.IsEmpty())&&(m_blockedPath.IsAncestorOf(workingPath)))
 400                                         {
 401                                                 // move the path to the end of the list
 402                                                 m_foldersToUpdate.Push(workingPath);
 403                                                 if (m_foldersToUpdate.size() < 3)
 404                                                         Sleep(50);
 405                                                 continue;
 406                                         }
 407                                 }
 408                                 if (DWORD(workingPath.GetCustomData()) >= currentTicks)
 409                                 {
 410                                         Sleep(50);
 411                                         continue;
 412                                 }
 413                                 if ((!m_blockedPath.IsEmpty())&&(m_blockedPath.IsAncestorOf(workingPath)))
 414                                         continue;
 415                                 if (!CGitStatusCache::Instance().IsPathAllowed(workingPath))
 416                                         continue;
 417                                 if (!CGitStatusCache::Instance().IsPathGood(workingPath))
 418                                         continue;
 419
 420                                 ATLTRACE(_T("Crawling folder: %s\n"), workingPath.GetWinPath());
 421                                 {
 422                                         AutoLocker print(critSec);
 423                                         _sntprintf_s(szCurrentCrawledPath[nCurrentCrawledpathIndex], MAX_CRAWLEDPATHSLEN, _TRUNCATE, _T("Crawling folder: %s"), workingPath.GetWinPath());
 424                                         nCurrentCrawledpathIndex++;
 425                                         if (nCurrentCrawledpathIndex >= MAX_CRAWLEDPATHS)
 426                                                 nCurrentCrawledpathIndex = 0;
 427                                 }
 428                                 InvalidateRect(hWnd, NULL, FALSE);
 429                                 {
 430                                         CAutoReadLock readLock(CGitStatusCache::Instance().GetGuard());
 431                                         // Now, we need to visit this folder, to make sure that we know its 'most important' status
 432                                         CCachedDirectory * cachedDir = CGitStatusCache::Instance().GetDirectoryCacheEntry(workingPath.GetDirectory());
 433                                         // check if the path is monitored by the watcher. If it isn't, then we have to invalidate the cache
 434                                         // for that path and add it to the watcher.
 435                                         if (!CGitStatusCache::Instance().IsPathWatched(workingPath))
 436                                         {
 437                                                 if (workingPath.HasAdminDir())
 438                                                 {
 439                                                         ATLTRACE(_T("Add watch path %s\n"), workingPath.GetWinPath());
 440                                                         CGitStatusCache::Instance().AddPathToWatch(workingPath);
 441                                                 }
 442                                                 if (cachedDir)
 443                                                         cachedDir->Invalidate();
 444                                                 else
 445                                                 {
 446                                                         CAutoWriteLock writeLock(CGitStatusCache::Instance().GetGuard());
 447                                                         CGitStatusCache::Instance().RemoveCacheForPath(workingPath);
 448                                                         // now cacheDir is invalid because it got deleted in the RemoveCacheForPath() call above.
 449                                                         cachedDir = NULL;
 450                                                 }
 451                                         }
 452                                         if (cachedDir)
 453                                                 cachedDir->RefreshStatus(bRecursive);
 454                                 }
 455
 456                                 // While refreshing the status, we could get another crawl request for the same folder.
 457                                 // This can happen if the crawled folder has a lower status than one of the child folders
 458                                 // (recursively). To avoid double crawlings, remove such a crawl request here
 459                                 AutoLocker lock(m_critSec);
 460                                 if (m_bItemsAddedSinceLastCrawl)
 461                                 {
 462                                         m_foldersToUpdate.erase(workingPath);
 463                                 }
 464                         }
 465                 }
 466         }
 467         _endthread();
 468 }
 469
 470 bool CFolderCrawler::SetHoldoff(DWORD milliseconds /* = 100*/)
 471 {
 472         long tick = (long)GetTickCount();
 473         bool ret = ((tick - m_crawlHoldoffReleasesAt) > 0);
 474         m_crawlHoldoffReleasesAt = tick + milliseconds;
 475         return ret;
 476 }
 477
 478 void CFolderCrawler::BlockPath(const CTGitPath& path, DWORD ticks)
 479 {
 480         ATLTRACE(_T("block path %s from being crawled\n"), path.GetWinPath());
 481         m_blockedPath = path;
 482         if (ticks == 0)
 483                 m_blockReleasesAt = GetTickCount()+10000;
 484         else
 485                 m_blockReleasesAt = GetTickCount()+ticks;
 486 }