Unhide variables
[TortoiseGit.git] / src / TGitCache / GITStatusCache.cpp
blob581639a8b8386ee5edf785d028db1ddeed3f173b
1 // TortoiseGit - a Windows shell extension for easy version control
3 // External Cache Copyright (C) 2005-2006,2008,2010,2014 - TortoiseSVN
4 // Copyright (C) 2008-2017 - TortoiseGit
6 // This program is free software; you can redistribute it and/or
7 // modify it under the terms of the GNU General Public License
8 // as published by the Free Software Foundation; either version 2
9 // of the License, or (at your option) any later version.
11 // This program is distributed in the hope that it will be useful,
12 // but WITHOUT ANY WARRANTY; without even the implied warranty of
13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 // GNU General Public License for more details.
16 // You should have received a copy of the GNU General Public License
17 // along with this program; if not, write to the Free Software Foundation,
18 // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21 #include "stdafx.h"
22 #include "GitAdminDir.h"
23 #include "GitStatus.h"
24 #include "GitStatusCache.h"
25 #include "CacheInterface.h"
26 #include <ShlObj.h>
27 #include "PathUtils.h"
29 //////////////////////////////////////////////////////////////////////////
31 #define BLOCK_PATH_DEFAULT_TIMEOUT 600 // 10 minutes
32 #define BLOCK_PATH_MAX_TIMEOUT 1200 // 20 minutes
34 #define CACHEDISKVERSION 2
36 #ifdef _WIN64
37 #define STATUSCACHEFILENAME L"cache64"
38 #else
39 #define STATUSCACHEFILENAME L"cache"
40 #endif
42 CGitStatusCache* CGitStatusCache::m_pInstance;
44 CGitStatusCache& CGitStatusCache::Instance()
46 ATLASSERT(m_pInstance);
47 return *m_pInstance;
50 void CGitStatusCache::Create()
52 ATLASSERT(!m_pInstance);
53 m_pInstance = new CGitStatusCache;
55 m_pInstance->watcher.SetFolderCrawler(&m_pInstance->m_folderCrawler);
57 if (!CRegStdDWORD(L"Software\\TortoiseGit\\CacheSave", TRUE))
58 return;
60 #define LOADVALUEFROMFILE(x) if (fread(&x, sizeof(x), 1, pFile)!=1) goto exit;
61 #define LOADVALUEFROMFILE2(x) if (fread(&x, sizeof(x), 1, pFile)!=1) goto error;
62 unsigned int value = (unsigned int)-1;
63 // find the location of the cache
64 CString path = CPathUtils::GetLocalAppDataDirectory();
65 CString path2;
66 if (!path.IsEmpty())
68 path += STATUSCACHEFILENAME;
69 // in case the cache file is corrupt, we could crash while
70 // reading it! To prevent crashing every time once that happens,
71 // we make a copy of the cache file and use that copy to read from.
72 // if that copy is corrupt, the original file won't exist anymore
73 // and the second time we start up and try to read the file,
74 // it's not there anymore and we start from scratch without a crash.
75 path2 = path;
76 path2 += L'2';
77 DeleteFile(path2);
78 CopyFile(path, path2, FALSE);
79 DeleteFile(path);
80 CAutoFILE pFile = _wfsopen(path2, L"rb", _SH_DENYWR);
81 if (pFile)
83 try
85 LOADVALUEFROMFILE(value);
86 if (value != CACHEDISKVERSION)
88 goto error;
90 int mapsize = 0;
91 LOADVALUEFROMFILE(mapsize);
92 for (int i=0; i<mapsize; ++i)
94 LOADVALUEFROMFILE2(value);
95 if (value > MAX_PATH)
96 goto error;
97 if (value)
99 CString sKey;
100 if (fread(sKey.GetBuffer(value+1), sizeof(TCHAR), value, pFile)!=value)
102 sKey.ReleaseBuffer(0);
103 goto error;
105 sKey.ReleaseBuffer(value);
106 auto cacheddir = std::make_unique<CCachedDirectory>();
107 if (!cacheddir.get() || !cacheddir->LoadFromDisk(pFile))
109 cacheddir.reset();
110 goto error;
112 CTGitPath KeyPath = CTGitPath(sKey);
113 if (m_pInstance->IsPathAllowed(KeyPath))
115 // only add the path to the watch list if it is versioned
116 if ((cacheddir->GetCurrentFullStatus() != git_wc_status_unversioned)&&(cacheddir->GetCurrentFullStatus() != git_wc_status_none))
117 m_pInstance->watcher.AddPath(KeyPath, false);
119 m_pInstance->m_directoryCache[KeyPath] = cacheddir.release();
121 // do *not* add the paths for crawling!
122 // because crawled paths will trigger a shell
123 // notification, which makes the desktop flash constantly
124 // until the whole first time crawling is over
125 // m_pInstance->AddFolderForCrawling(KeyPath);
130 catch (CAtlException)
132 goto error;
136 exit:
137 DeleteFile(path2);
138 m_pInstance->watcher.ClearInfoMap();
139 CTraceToOutputDebugString::Instance()(__FUNCTION__ ": cache loaded from disk successfully!\n");
140 return;
141 error:
142 DeleteFile(path2);
143 m_pInstance->watcher.ClearInfoMap();
144 Destroy();
145 m_pInstance = new CGitStatusCache;
146 CTraceToOutputDebugString::Instance()(__FUNCTION__ ": cache not loaded from disk\n");
149 bool CGitStatusCache::SaveCache()
151 if (!CRegStdDWORD(L"Software\\TortoiseGit\\CacheSave", TRUE))
152 return false;
154 #define WRITEVALUETOFILE(x) if (fwrite(&x, sizeof(x), 1, pFile)!=1) goto error;
155 unsigned int value = 0;
156 // save the cache to disk
157 // find a location to write the cache to
158 CString path = CPathUtils::GetLocalAppDataDirectory();
159 if (!path.IsEmpty())
161 path += STATUSCACHEFILENAME;
162 CAutoFILE pFile = _wfsopen(path, L"wb", SH_DENYRW);
163 if (pFile)
165 value = CACHEDISKVERSION;
166 WRITEVALUETOFILE(value);
167 value = (int)m_pInstance->m_directoryCache.size();
168 WRITEVALUETOFILE(value);
169 for (auto I = m_pInstance->m_directoryCache.cbegin(); I != m_pInstance->m_directoryCache.cend(); ++I)
171 if (!I->second)
173 value = 0;
174 WRITEVALUETOFILE(value);
175 continue;
177 const CString& key = I->first.GetWinPathString();
178 value = key.GetLength();
179 WRITEVALUETOFILE(value);
180 if (value)
182 if (fwrite((LPCTSTR)key, sizeof(TCHAR), value, pFile)!=value)
183 goto error;
184 if (!I->second->SaveToDisk(pFile))
185 goto error;
190 CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) L": cache saved to disk at %s\n", (LPCTSTR)path);
191 return true;
192 error:
193 Destroy();
194 DeleteFile(path);
195 return false;
198 void CGitStatusCache::Destroy()
200 if (m_pInstance)
202 m_pInstance->Stop();
203 Sleep(100);
204 delete m_pInstance;
205 m_pInstance = nullptr;
209 void CGitStatusCache::Stop()
211 // m_svnHelp.Cancel(true);
212 watcher.Stop();
213 m_folderCrawler.Stop();
214 m_shellUpdater.Stop();
217 void CGitStatusCache::Init()
219 m_folderCrawler.Initialise();
220 m_shellUpdater.Initialise();
223 CGitStatusCache::CGitStatusCache(void)
225 #define forever DWORD(-1)
226 AutoLocker lock(m_NoWatchPathCritSec);
227 KNOWNFOLDERID folderids[] = { FOLDERID_Cookies, FOLDERID_History, FOLDERID_InternetCache, FOLDERID_Windows, FOLDERID_CDBurning, FOLDERID_Fonts, FOLDERID_RecycleBinFolder }; //FOLDERID_SearchHistory
228 for (KNOWNFOLDERID folderid : folderids)
229 m_NoWatchPaths[CTGitPath(GetSpecialFolder(folderid))] = forever;
230 m_bClearMemory = false;
231 m_mostRecentExpiresAt = 0;
234 CGitStatusCache::~CGitStatusCache(void)
236 CAutoWriteLock writeLock(m_guard);
237 ClearCache();
240 void CGitStatusCache::Refresh()
242 m_shellCache.RefreshIfNeeded();
243 // m_pInstance->m_svnHelp.ReloadConfig();
244 if (!m_pInstance->m_directoryCache.empty())
246 auto I = m_pInstance->m_directoryCache.cbegin();
247 for (/* no init */; I != m_pInstance->m_directoryCache.cend(); ++I)
249 if (m_shellCache.IsPathAllowed(I->first.GetWinPath()))
250 I->second->RefreshMostImportant();
251 else
253 CGitStatusCache::Instance().RemoveCacheForPath(I->first);
254 I = m_pInstance->m_directoryCache.cbegin();
255 if (I == m_pInstance->m_directoryCache.cend())
256 break;
262 bool CGitStatusCache::IsPathGood(const CTGitPath& path)
264 AutoLocker lock(m_NoWatchPathCritSec);
265 for (auto it = m_NoWatchPaths.cbegin(); it != m_NoWatchPaths.cend(); ++it)
267 // the ticks check is necessary here, because RemoveTimedoutBlocks is only called within the FolderCrawler loop
268 // and we might miss update calls
269 // TODO: maybe we also need to check if index.lock and HEAD.lock still exists after a specific timeout (if we miss update notifications for these files too often)
270 if (GetTickCount64() < it->second && it->first.IsAncestorOf(path))
272 CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) L": path not good: %s\n", it->first.GetWinPath());
273 return false;
276 return true;
279 bool CGitStatusCache::BlockPath(const CTGitPath& path, ULONGLONG timeout /* = 0 */)
281 if (timeout == 0)
282 timeout = BLOCK_PATH_DEFAULT_TIMEOUT;
284 if (timeout > BLOCK_PATH_MAX_TIMEOUT)
285 timeout = BLOCK_PATH_MAX_TIMEOUT;
287 timeout = GetTickCount64() + (timeout * 1000); // timeout is in seconds, but we need the milliseconds
289 AutoLocker lock(m_NoWatchPathCritSec);
290 m_NoWatchPaths[path.GetDirectory()] = timeout;
292 return true;
295 bool CGitStatusCache::UnBlockPath(const CTGitPath& path)
297 bool ret = false;
298 AutoLocker lock(m_NoWatchPathCritSec);
299 std::map<CTGitPath, ULONGLONG>::iterator it = m_NoWatchPaths.find(path.GetDirectory());
300 if (it != m_NoWatchPaths.end())
302 CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) L": path removed from no good: %s\n", it->first.GetWinPath());
303 m_NoWatchPaths.erase(it);
304 ret = true;
306 AddFolderForCrawling(path.GetDirectory());
308 return ret;
311 bool CGitStatusCache::RemoveTimedoutBlocks()
313 bool ret = false;
314 ULONGLONG currentTicks = GetTickCount64();
315 AutoLocker lock(m_NoWatchPathCritSec);
316 std::vector<CTGitPath> toRemove;
317 for (auto it = m_NoWatchPaths.cbegin(); it != m_NoWatchPaths.cend(); ++it)
319 if (currentTicks > it->second)
320 toRemove.push_back(it->first);
322 if (!toRemove.empty())
324 for (auto it = toRemove.cbegin(); it != toRemove.cend(); ++it)
325 ret = ret || UnBlockPath(*it);
328 return ret;
331 void CGitStatusCache::UpdateShell(const CTGitPath& path)
333 if (path.IsEquivalentToWithoutCase(m_mostRecentPath))
334 m_mostRecentExpiresAt = 0;
335 m_shellUpdater.AddPathForUpdate(path);
338 void CGitStatusCache::ClearCache()
340 CAutoWriteLock writeLock(m_guard);
341 for (CCachedDirectory::CachedDirMap::iterator I = m_directoryCache.begin(); I != m_directoryCache.end(); ++I)
343 delete I->second;
344 I->second = nullptr;
346 m_directoryCache.clear();
349 bool CGitStatusCache::RemoveCacheForDirectory(CCachedDirectory * cdir)
351 if (!cdir)
352 return false;
354 CAutoWriteLock writeLock(m_guard);
355 if (!cdir->m_childDirectories.empty())
357 for (auto it = cdir->m_childDirectories.begin(); it != cdir->m_childDirectories.end();)
359 CCachedDirectory * childdir = CGitStatusCache::Instance().GetDirectoryCacheEntryNoCreate(it->first);
360 if ((childdir) && (!cdir->m_directoryPath.IsEquivalentTo(childdir->m_directoryPath)) && (cdir->m_directoryPath.GetFileOrDirectoryName() != L".."))
361 RemoveCacheForDirectory(childdir);
362 cdir->m_childDirectories.erase(it->first);
363 it = cdir->m_childDirectories.begin();
366 cdir->m_childDirectories.clear();
367 m_directoryCache.erase(cdir->m_directoryPath);
369 // we could have entries versioned and/or stored in our cache which are
370 // children of the specified directory, but not in the m_childDirectories
371 // member
372 CCachedDirectory::ItDir itMap = m_directoryCache.lower_bound(cdir->m_directoryPath);
375 if (itMap != m_directoryCache.end())
377 if (cdir->m_directoryPath.IsAncestorOf(itMap->first))
379 // just in case (see TortoiseSVN issue #255)
380 if (itMap->second == cdir)
381 m_directoryCache.erase(itMap);
382 else
383 RemoveCacheForDirectory(itMap->second);
386 itMap = m_directoryCache.lower_bound(cdir->m_directoryPath);
387 } while (itMap != m_directoryCache.end() && cdir->m_directoryPath.IsAncestorOf(itMap->first));
389 CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) L": removed from cache %s\n", cdir->m_directoryPath.GetWinPath());
390 delete cdir;
391 return true;
394 void CGitStatusCache::RemoveCacheForPath(const CTGitPath& path)
396 // Stop the crawler starting on a new folder
397 CCrawlInhibitor crawlInhibit(&m_folderCrawler);
398 CCachedDirectory::ItDir itMap;
399 CCachedDirectory* dirtoremove = nullptr;
401 itMap = m_directoryCache.find(path);
402 if ((itMap != m_directoryCache.end())&&(itMap->second))
403 dirtoremove = itMap->second;
404 if (!dirtoremove)
405 return;
406 ATLASSERT(path.IsEquivalentToWithoutCase(dirtoremove->m_directoryPath));
407 RemoveCacheForDirectory(dirtoremove);
410 CCachedDirectory * CGitStatusCache::GetDirectoryCacheEntry(const CTGitPath& path, bool isAddToWatch)
412 ATLASSERT(path.IsDirectory() || !PathFileExists(path.GetWinPath()));
414 CAutoReadLock readLock(m_guardcacheddirectories);
415 CCachedDirectory::ItDir itMap;
416 itMap = m_directoryCache.find(path);
417 if ((itMap != m_directoryCache.end())&&(itMap->second))
419 // We've found this directory in the cache
420 return itMap->second;
422 else
424 // if the CCachedDirectory is nullptr but the path is in our cache,
425 // that means that path got invalidated and needs to be treated
426 // as if it never was in our cache. So we remove the last remains
427 // from the cache and start from scratch.
429 CAutoWriteLock writeLock(m_guardcacheddirectories);
430 // Since above there's a small chance that before we can upgrade to
431 // writer state some other thread gained writer state and changed
432 // the data, we have to recreate the iterator here again.
433 itMap = m_directoryCache.find(path);
434 if (itMap!=m_directoryCache.end())
436 CAutoWriteLock writeLock2(m_guard); // needed? Can this happen?
437 delete itMap->second;
438 m_directoryCache.erase(itMap);
440 // We don't know anything about this directory yet - lets add it to our cache
441 // but only if it exists!
442 if (path.Exists() && m_shellCache.IsPathAllowed(path.GetWinPath()) && !GitAdminDir::IsAdminDirPath(path.GetWinPath()))
444 // some notifications are for files which got removed/moved around.
445 // In such cases, the CTGitPath::IsDirectory() will return true (it assumes a directory if
446 // the path doesn't exist). Which means we can get here with a path to a file
447 // instead of a directory.
448 // Since we're here most likely called from the crawler thread, the file could exist
449 // again. If that's the case, just do nothing
450 if (path.IsDirectory()||(!path.Exists()))
452 CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) L": adding %s to our cache\n", path.GetWinPath());
453 CCachedDirectory * newcdir = new CCachedDirectory(path);
454 if (newcdir)
456 CCachedDirectory * cdir = m_directoryCache.insert(m_directoryCache.lower_bound(path), std::make_pair(path, newcdir))->second;
457 CString gitdir;
458 if ((!path.IsEmpty())&&(path.HasAdminDir(&gitdir))&&isAddToWatch)
460 /* Just watch version path */
461 watcher.AddPath(gitdir);
462 watcher.AddPath(path);
464 return cdir;
466 m_bClearMemory = true;
469 return nullptr;
473 CCachedDirectory * CGitStatusCache::GetDirectoryCacheEntryNoCreate(const CTGitPath& path)
475 ATLASSERT(path.IsDirectory() || !PathFileExists(path.GetWinPath()));
477 CAutoReadLock readLock(m_guardcacheddirectories);
478 CCachedDirectory::ItDir itMap;
479 itMap = m_directoryCache.find(path);
480 if(itMap != m_directoryCache.end())
482 // We've found this directory in the cache
483 return itMap->second;
485 return nullptr;
488 /* Fetch is true, means fetch status from */
489 /* Fetch is false, means fetch status from cache */
490 CStatusCacheEntry CGitStatusCache::GetStatusForPath(const CTGitPath& path, DWORD flags, bool bFetch /* = true */)
492 bool bRecursive = !!(flags & TGITCACHE_FLAGS_RECUSIVE_STATUS);
494 // Check a very short-lived 'mini-cache' of the last thing we were asked for.
495 LONGLONG now = (LONGLONG)GetTickCount64();
496 if(now-m_mostRecentExpiresAt < 0)
498 if (path.IsEquivalentTo(m_mostRecentPath))
499 return m_mostRecentStatus;
502 AutoLocker lock(m_critSec);
503 m_mostRecentPath = path;
504 m_mostRecentExpiresAt = now + 1000;
507 if (IsPathGood(path))
509 // Stop the crawler starting on a new folder while we're doing this much more important task...
510 // Please note, that this may be a second "lock" used concurrently to the one in RemoveCacheForPath().
511 CCrawlInhibitor crawlInhibit(&m_folderCrawler);
513 CTGitPath dirpath = path.GetContainingDirectory();
514 if ((dirpath.IsEmpty()) || (!m_shellCache.IsPathAllowed(dirpath.GetWinPath())))
515 dirpath = path.GetDirectory();
516 CCachedDirectory * cachedDir = GetDirectoryCacheEntry(dirpath);
517 if (cachedDir)
519 //CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) L": GetStatusForMember %d\n", bFetch);
520 CStatusCacheEntry entry = cachedDir->GetStatusForMember(path, bRecursive, bFetch);
522 AutoLocker lock(m_critSec);
523 m_mostRecentStatus = entry;
524 return m_mostRecentStatus;
528 else
530 // path is blocked for some reason: return the cached status if we have one
531 // we do here only a cache search, absolutely no disk access is allowed!
532 auto cachedDir = GetDirectoryCacheEntryNoCreate(path.GetDirectory());
533 if (cachedDir)
535 if (path.IsDirectory())
537 CStatusCacheEntry entry = cachedDir->GetOwnStatus(false);
538 AutoLocker lock(m_critSec);
539 m_mostRecentStatus = entry;
540 return m_mostRecentStatus;
542 else
544 // We've found this directory in the cache
545 CStatusCacheEntry entry = cachedDir->GetCacheStatusForMember(path);
547 AutoLocker lock(m_critSec);
548 m_mostRecentStatus = entry;
549 return m_mostRecentStatus;
554 AutoLocker lock(m_critSec);
555 CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) L": ignored no good path %s\n", path.GetWinPath());
556 m_mostRecentStatus = CStatusCacheEntry();
557 if (m_shellCache.ShowExcludedAsNormal() && path.IsDirectory() && m_shellCache.HasGITAdminDir(path.GetWinPath(), true))
559 CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) L": force status %s\n", path.GetWinPath());
560 m_mostRecentStatus.ForceStatus(git_wc_status_normal);
562 return m_mostRecentStatus;
565 void CGitStatusCache::AddFolderForCrawling(const CTGitPath& path)
567 m_folderCrawler.AddDirectoryForUpdate(path);
570 void CGitStatusCache::CloseWatcherHandles(HANDLE hFile)
572 CTGitPath path = watcher.CloseInfoMap(hFile);
573 if (!path.IsEmpty())
574 m_folderCrawler.BlockPath(path);
575 CGitStatusCache::Instance().m_GitStatus.ReleasePathsRecursively(path.GetWinPathString());
578 void CGitStatusCache::CloseWatcherHandles(const CTGitPath& path)
580 watcher.CloseHandlesForPath(path);
581 m_folderCrawler.ReleasePathForUpdate(path);
582 CGitStatusCache::Instance().m_GitStatus.ReleasePathsRecursively(path.GetWinPathString());
585 CString CGitStatusCache::GetSpecialFolder(REFKNOWNFOLDERID rfid)
587 PWSTR pszPath = nullptr;
588 if (SHGetKnownFolderPath(rfid, KF_FLAG_CREATE, nullptr, &pszPath) != S_OK)
589 return CString();
591 CString path = pszPath;
592 CoTaskMemFree(pszPath);
593 return path;