1 // TortoiseGit - a Windows shell extension for easy version control
3 // External Cache Copyright (C) 2005-2006,2008,2010,2014 - TortoiseSVN
4 // Copyright (C) 2008-2017 - TortoiseGit
6 // This program is free software; you can redistribute it and/or
7 // modify it under the terms of the GNU General Public License
8 // as published by the Free Software Foundation; either version 2
9 // of the License, or (at your option) any later version.
11 // This program is distributed in the hope that it will be useful,
12 // but WITHOUT ANY WARRANTY; without even the implied warranty of
13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 // GNU General Public License for more details.
16 // You should have received a copy of the GNU General Public License
17 // along with this program; if not, write to the Free Software Foundation,
18 // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
22 #include "GitAdminDir.h"
23 #include "GitStatus.h"
24 #include "GitStatusCache.h"
25 #include "CacheInterface.h"
27 #include "PathUtils.h"
29 //////////////////////////////////////////////////////////////////////////
31 #define BLOCK_PATH_DEFAULT_TIMEOUT 600 // 10 minutes
32 #define BLOCK_PATH_MAX_TIMEOUT 1200 // 20 minutes
34 #define CACHEDISKVERSION 2
37 #define STATUSCACHEFILENAME L"cache64"
39 #define STATUSCACHEFILENAME L"cache"
42 CGitStatusCache
* CGitStatusCache::m_pInstance
;
44 CGitStatusCache
& CGitStatusCache::Instance()
46 ATLASSERT(m_pInstance
);
50 void CGitStatusCache::Create()
52 ATLASSERT(!m_pInstance
);
53 m_pInstance
= new CGitStatusCache
;
55 m_pInstance
->watcher
.SetFolderCrawler(&m_pInstance
->m_folderCrawler
);
57 if (!CRegStdDWORD(L
"Software\\TortoiseGit\\CacheSave", TRUE
))
60 #define LOADVALUEFROMFILE(x) if (fread(&x, sizeof(x), 1, pFile)!=1) goto exit;
61 #define LOADVALUEFROMFILE2(x) if (fread(&x, sizeof(x), 1, pFile)!=1) goto error;
62 unsigned int value
= (unsigned int)-1;
63 // find the location of the cache
64 CString path
= CPathUtils::GetLocalAppDataDirectory();
68 path
+= STATUSCACHEFILENAME
;
69 // in case the cache file is corrupt, we could crash while
70 // reading it! To prevent crashing every time once that happens,
71 // we make a copy of the cache file and use that copy to read from.
72 // if that copy is corrupt, the original file won't exist anymore
73 // and the second time we start up and try to read the file,
74 // it's not there anymore and we start from scratch without a crash.
78 CopyFile(path
, path2
, FALSE
);
80 CAutoFILE pFile
= _wfsopen(path2
, L
"rb", _SH_DENYWR
);
85 LOADVALUEFROMFILE(value
);
86 if (value
!= CACHEDISKVERSION
)
91 LOADVALUEFROMFILE(mapsize
);
92 for (int i
=0; i
<mapsize
; ++i
)
94 LOADVALUEFROMFILE2(value
);
100 if (fread(sKey
.GetBuffer(value
+1), sizeof(TCHAR
), value
, pFile
)!=value
)
102 sKey
.ReleaseBuffer(0);
105 sKey
.ReleaseBuffer(value
);
106 auto cacheddir
= std::make_unique
<CCachedDirectory
>();
107 if (!cacheddir
.get() || !cacheddir
->LoadFromDisk(pFile
))
112 CTGitPath KeyPath
= CTGitPath(sKey
);
113 if (m_pInstance
->IsPathAllowed(KeyPath
))
115 // only add the path to the watch list if it is versioned
116 if ((cacheddir
->GetCurrentFullStatus() != git_wc_status_unversioned
)&&(cacheddir
->GetCurrentFullStatus() != git_wc_status_none
))
117 m_pInstance
->watcher
.AddPath(KeyPath
, false);
119 m_pInstance
->m_directoryCache
[KeyPath
] = cacheddir
.release();
121 // do *not* add the paths for crawling!
122 // because crawled paths will trigger a shell
123 // notification, which makes the desktop flash constantly
124 // until the whole first time crawling is over
125 // m_pInstance->AddFolderForCrawling(KeyPath);
130 catch (CAtlException
)
138 m_pInstance
->watcher
.ClearInfoMap();
139 CTraceToOutputDebugString::Instance()(__FUNCTION__
": cache loaded from disk successfully!\n");
143 m_pInstance
->watcher
.ClearInfoMap();
145 m_pInstance
= new CGitStatusCache
;
146 CTraceToOutputDebugString::Instance()(__FUNCTION__
": cache not loaded from disk\n");
149 bool CGitStatusCache::SaveCache()
151 if (!CRegStdDWORD(L
"Software\\TortoiseGit\\CacheSave", TRUE
))
154 #define WRITEVALUETOFILE(x) if (fwrite(&x, sizeof(x), 1, pFile)!=1) goto error;
155 unsigned int value
= 0;
156 // save the cache to disk
157 // find a location to write the cache to
158 CString path
= CPathUtils::GetLocalAppDataDirectory();
161 path
+= STATUSCACHEFILENAME
;
162 CAutoFILE pFile
= _wfsopen(path
, L
"wb", SH_DENYRW
);
165 value
= CACHEDISKVERSION
;
166 WRITEVALUETOFILE(value
);
167 value
= (int)m_pInstance
->m_directoryCache
.size();
168 WRITEVALUETOFILE(value
);
169 for (auto I
= m_pInstance
->m_directoryCache
.cbegin(); I
!= m_pInstance
->m_directoryCache
.cend(); ++I
)
174 WRITEVALUETOFILE(value
);
177 const CString
& key
= I
->first
.GetWinPathString();
178 value
= key
.GetLength();
179 WRITEVALUETOFILE(value
);
182 if (fwrite((LPCTSTR
)key
, sizeof(TCHAR
), value
, pFile
)!=value
)
184 if (!I
->second
->SaveToDisk(pFile
))
190 CTraceToOutputDebugString::Instance()(_T(__FUNCTION__
) L
": cache saved to disk at %s\n", (LPCTSTR
)path
);
198 void CGitStatusCache::Destroy()
205 m_pInstance
= nullptr;
209 void CGitStatusCache::Stop()
211 // m_svnHelp.Cancel(true);
213 m_folderCrawler
.Stop();
214 m_shellUpdater
.Stop();
217 void CGitStatusCache::Init()
219 m_folderCrawler
.Initialise();
220 m_shellUpdater
.Initialise();
223 CGitStatusCache::CGitStatusCache(void)
225 #define forever DWORD(-1)
226 AutoLocker
lock(m_NoWatchPathCritSec
);
227 KNOWNFOLDERID folderids
[] = { FOLDERID_Cookies
, FOLDERID_History
, FOLDERID_InternetCache
, FOLDERID_Windows
, FOLDERID_CDBurning
, FOLDERID_Fonts
, FOLDERID_RecycleBinFolder
}; //FOLDERID_SearchHistory
228 for (KNOWNFOLDERID folderid
: folderids
)
229 m_NoWatchPaths
[CTGitPath(GetSpecialFolder(folderid
))] = forever
;
230 m_bClearMemory
= false;
231 m_mostRecentExpiresAt
= 0;
234 CGitStatusCache::~CGitStatusCache(void)
236 CAutoWriteLock
writeLock(m_guard
);
240 void CGitStatusCache::Refresh()
242 m_shellCache
.RefreshIfNeeded();
243 // m_pInstance->m_svnHelp.ReloadConfig();
244 if (!m_pInstance
->m_directoryCache
.empty())
246 auto I
= m_pInstance
->m_directoryCache
.cbegin();
247 for (/* no init */; I
!= m_pInstance
->m_directoryCache
.cend(); ++I
)
249 if (m_shellCache
.IsPathAllowed(I
->first
.GetWinPath()))
250 I
->second
->RefreshMostImportant();
253 CGitStatusCache::Instance().RemoveCacheForPath(I
->first
);
254 I
= m_pInstance
->m_directoryCache
.cbegin();
255 if (I
== m_pInstance
->m_directoryCache
.cend())
262 bool CGitStatusCache::IsPathGood(const CTGitPath
& path
)
264 AutoLocker
lock(m_NoWatchPathCritSec
);
265 for (auto it
= m_NoWatchPaths
.cbegin(); it
!= m_NoWatchPaths
.cend(); ++it
)
267 // the ticks check is necessary here, because RemoveTimedoutBlocks is only called within the FolderCrawler loop
268 // and we might miss update calls
269 // TODO: maybe we also need to check if index.lock and HEAD.lock still exists after a specific timeout (if we miss update notifications for these files too often)
270 if (GetTickCount64() < it
->second
&& it
->first
.IsAncestorOf(path
))
272 CTraceToOutputDebugString::Instance()(_T(__FUNCTION__
) L
": path not good: %s\n", it
->first
.GetWinPath());
279 bool CGitStatusCache::BlockPath(const CTGitPath
& path
, ULONGLONG timeout
/* = 0 */)
282 timeout
= BLOCK_PATH_DEFAULT_TIMEOUT
;
284 if (timeout
> BLOCK_PATH_MAX_TIMEOUT
)
285 timeout
= BLOCK_PATH_MAX_TIMEOUT
;
287 timeout
= GetTickCount64() + (timeout
* 1000); // timeout is in seconds, but we need the milliseconds
289 AutoLocker
lock(m_NoWatchPathCritSec
);
290 m_NoWatchPaths
[path
.GetDirectory()] = timeout
;
295 bool CGitStatusCache::UnBlockPath(const CTGitPath
& path
)
298 AutoLocker
lock(m_NoWatchPathCritSec
);
299 std::map
<CTGitPath
, ULONGLONG
>::iterator it
= m_NoWatchPaths
.find(path
.GetDirectory());
300 if (it
!= m_NoWatchPaths
.end())
302 CTraceToOutputDebugString::Instance()(_T(__FUNCTION__
) L
": path removed from no good: %s\n", it
->first
.GetWinPath());
303 m_NoWatchPaths
.erase(it
);
306 AddFolderForCrawling(path
.GetDirectory());
311 bool CGitStatusCache::RemoveTimedoutBlocks()
314 ULONGLONG currentTicks
= GetTickCount64();
315 AutoLocker
lock(m_NoWatchPathCritSec
);
316 std::vector
<CTGitPath
> toRemove
;
317 for (auto it
= m_NoWatchPaths
.cbegin(); it
!= m_NoWatchPaths
.cend(); ++it
)
319 if (currentTicks
> it
->second
)
320 toRemove
.push_back(it
->first
);
322 if (!toRemove
.empty())
324 for (auto it
= toRemove
.cbegin(); it
!= toRemove
.cend(); ++it
)
325 ret
= ret
|| UnBlockPath(*it
);
331 void CGitStatusCache::UpdateShell(const CTGitPath
& path
)
333 if (path
.IsEquivalentToWithoutCase(m_mostRecentPath
))
334 m_mostRecentExpiresAt
= 0;
335 m_shellUpdater
.AddPathForUpdate(path
);
338 void CGitStatusCache::ClearCache()
340 CAutoWriteLock
writeLock(m_guard
);
341 for (CCachedDirectory::CachedDirMap::iterator I
= m_directoryCache
.begin(); I
!= m_directoryCache
.end(); ++I
)
346 m_directoryCache
.clear();
349 bool CGitStatusCache::RemoveCacheForDirectory(CCachedDirectory
* cdir
)
354 CAutoWriteLock
writeLock(m_guard
);
355 if (!cdir
->m_childDirectories
.empty())
357 for (auto it
= cdir
->m_childDirectories
.begin(); it
!= cdir
->m_childDirectories
.end();)
359 CCachedDirectory
* childdir
= CGitStatusCache::Instance().GetDirectoryCacheEntryNoCreate(it
->first
);
360 if ((childdir
) && (!cdir
->m_directoryPath
.IsEquivalentTo(childdir
->m_directoryPath
)) && (cdir
->m_directoryPath
.GetFileOrDirectoryName() != L
".."))
361 RemoveCacheForDirectory(childdir
);
362 cdir
->m_childDirectories
.erase(it
->first
);
363 it
= cdir
->m_childDirectories
.begin();
366 cdir
->m_childDirectories
.clear();
367 m_directoryCache
.erase(cdir
->m_directoryPath
);
369 // we could have entries versioned and/or stored in our cache which are
370 // children of the specified directory, but not in the m_childDirectories
372 CCachedDirectory::ItDir itMap
= m_directoryCache
.lower_bound(cdir
->m_directoryPath
);
375 if (itMap
!= m_directoryCache
.end())
377 if (cdir
->m_directoryPath
.IsAncestorOf(itMap
->first
))
379 // just in case (see TortoiseSVN issue #255)
380 if (itMap
->second
== cdir
)
381 m_directoryCache
.erase(itMap
);
383 RemoveCacheForDirectory(itMap
->second
);
386 itMap
= m_directoryCache
.lower_bound(cdir
->m_directoryPath
);
387 } while (itMap
!= m_directoryCache
.end() && cdir
->m_directoryPath
.IsAncestorOf(itMap
->first
));
389 CTraceToOutputDebugString::Instance()(_T(__FUNCTION__
) L
": removed from cache %s\n", cdir
->m_directoryPath
.GetWinPath());
394 void CGitStatusCache::RemoveCacheForPath(const CTGitPath
& path
)
396 // Stop the crawler starting on a new folder
397 CCrawlInhibitor
crawlInhibit(&m_folderCrawler
);
398 CCachedDirectory::ItDir itMap
;
399 CCachedDirectory
* dirtoremove
= nullptr;
401 itMap
= m_directoryCache
.find(path
);
402 if ((itMap
!= m_directoryCache
.end())&&(itMap
->second
))
403 dirtoremove
= itMap
->second
;
406 ATLASSERT(path
.IsEquivalentToWithoutCase(dirtoremove
->m_directoryPath
));
407 RemoveCacheForDirectory(dirtoremove
);
410 CCachedDirectory
* CGitStatusCache::GetDirectoryCacheEntry(const CTGitPath
& path
, bool isAddToWatch
)
412 ATLASSERT(path
.IsDirectory() || !PathFileExists(path
.GetWinPath()));
414 CAutoReadLock
readLock(m_guardcacheddirectories
);
415 CCachedDirectory::ItDir itMap
;
416 itMap
= m_directoryCache
.find(path
);
417 if ((itMap
!= m_directoryCache
.end())&&(itMap
->second
))
419 // We've found this directory in the cache
420 return itMap
->second
;
424 // if the CCachedDirectory is nullptr but the path is in our cache,
425 // that means that path got invalidated and needs to be treated
426 // as if it never was in our cache. So we remove the last remains
427 // from the cache and start from scratch.
429 CAutoWriteLock
writeLock(m_guardcacheddirectories
);
430 // Since above there's a small chance that before we can upgrade to
431 // writer state some other thread gained writer state and changed
432 // the data, we have to recreate the iterator here again.
433 itMap
= m_directoryCache
.find(path
);
434 if (itMap
!=m_directoryCache
.end())
436 CAutoWriteLock
writeLock2(m_guard
); // needed? Can this happen?
437 delete itMap
->second
;
438 m_directoryCache
.erase(itMap
);
440 // We don't know anything about this directory yet - lets add it to our cache
441 // but only if it exists!
442 if (path
.Exists() && m_shellCache
.IsPathAllowed(path
.GetWinPath()) && !GitAdminDir::IsAdminDirPath(path
.GetWinPath()))
444 // some notifications are for files which got removed/moved around.
445 // In such cases, the CTGitPath::IsDirectory() will return true (it assumes a directory if
446 // the path doesn't exist). Which means we can get here with a path to a file
447 // instead of a directory.
448 // Since we're here most likely called from the crawler thread, the file could exist
449 // again. If that's the case, just do nothing
450 if (path
.IsDirectory()||(!path
.Exists()))
452 CTraceToOutputDebugString::Instance()(_T(__FUNCTION__
) L
": adding %s to our cache\n", path
.GetWinPath());
453 CCachedDirectory
* newcdir
= new CCachedDirectory(path
);
456 CCachedDirectory
* cdir
= m_directoryCache
.insert(m_directoryCache
.lower_bound(path
), std::make_pair(path
, newcdir
))->second
;
458 if ((!path
.IsEmpty())&&(path
.HasAdminDir(&gitdir
))&&isAddToWatch
)
460 /* Just watch version path */
461 watcher
.AddPath(gitdir
);
462 watcher
.AddPath(path
);
466 m_bClearMemory
= true;
473 CCachedDirectory
* CGitStatusCache::GetDirectoryCacheEntryNoCreate(const CTGitPath
& path
)
475 ATLASSERT(path
.IsDirectory() || !PathFileExists(path
.GetWinPath()));
477 CAutoReadLock
readLock(m_guardcacheddirectories
);
478 CCachedDirectory::ItDir itMap
;
479 itMap
= m_directoryCache
.find(path
);
480 if(itMap
!= m_directoryCache
.end())
482 // We've found this directory in the cache
483 return itMap
->second
;
488 /* Fetch is true, means fetch status from */
489 /* Fetch is false, means fetch status from cache */
490 CStatusCacheEntry
CGitStatusCache::GetStatusForPath(const CTGitPath
& path
, DWORD flags
, bool bFetch
/* = true */)
492 bool bRecursive
= !!(flags
& TGITCACHE_FLAGS_RECUSIVE_STATUS
);
494 // Check a very short-lived 'mini-cache' of the last thing we were asked for.
495 LONGLONG now
= (LONGLONG
)GetTickCount64();
496 if(now
-m_mostRecentExpiresAt
< 0)
498 if (path
.IsEquivalentTo(m_mostRecentPath
))
499 return m_mostRecentStatus
;
502 AutoLocker
lock(m_critSec
);
503 m_mostRecentPath
= path
;
504 m_mostRecentExpiresAt
= now
+ 1000;
507 if (IsPathGood(path
))
509 // Stop the crawler starting on a new folder while we're doing this much more important task...
510 // Please note, that this may be a second "lock" used concurrently to the one in RemoveCacheForPath().
511 CCrawlInhibitor
crawlInhibit(&m_folderCrawler
);
513 CTGitPath dirpath
= path
.GetContainingDirectory();
514 if ((dirpath
.IsEmpty()) || (!m_shellCache
.IsPathAllowed(dirpath
.GetWinPath())))
515 dirpath
= path
.GetDirectory();
516 CCachedDirectory
* cachedDir
= GetDirectoryCacheEntry(dirpath
);
519 //CTraceToOutputDebugString::Instance()(_T(__FUNCTION__) L": GetStatusForMember %d\n", bFetch);
520 CStatusCacheEntry entry
= cachedDir
->GetStatusForMember(path
, bRecursive
, bFetch
);
522 AutoLocker
lock(m_critSec
);
523 m_mostRecentStatus
= entry
;
524 return m_mostRecentStatus
;
530 // path is blocked for some reason: return the cached status if we have one
531 // we do here only a cache search, absolutely no disk access is allowed!
532 auto cachedDir
= GetDirectoryCacheEntryNoCreate(path
.GetDirectory());
535 if (path
.IsDirectory())
537 CStatusCacheEntry entry
= cachedDir
->GetOwnStatus(false);
538 AutoLocker
lock(m_critSec
);
539 m_mostRecentStatus
= entry
;
540 return m_mostRecentStatus
;
544 // We've found this directory in the cache
545 CStatusCacheEntry entry
= cachedDir
->GetCacheStatusForMember(path
);
547 AutoLocker
lock(m_critSec
);
548 m_mostRecentStatus
= entry
;
549 return m_mostRecentStatus
;
554 AutoLocker
lock(m_critSec
);
555 CTraceToOutputDebugString::Instance()(_T(__FUNCTION__
) L
": ignored no good path %s\n", path
.GetWinPath());
556 m_mostRecentStatus
= CStatusCacheEntry();
557 if (m_shellCache
.ShowExcludedAsNormal() && path
.IsDirectory() && m_shellCache
.HasGITAdminDir(path
.GetWinPath(), true))
559 CTraceToOutputDebugString::Instance()(_T(__FUNCTION__
) L
": force status %s\n", path
.GetWinPath());
560 m_mostRecentStatus
.ForceStatus(git_wc_status_normal
);
562 return m_mostRecentStatus
;
565 void CGitStatusCache::AddFolderForCrawling(const CTGitPath
& path
)
567 m_folderCrawler
.AddDirectoryForUpdate(path
);
570 void CGitStatusCache::CloseWatcherHandles(HANDLE hFile
)
572 CTGitPath path
= watcher
.CloseInfoMap(hFile
);
574 m_folderCrawler
.BlockPath(path
);
575 CGitStatusCache::Instance().m_GitStatus
.ReleasePathsRecursively(path
.GetWinPathString());
578 void CGitStatusCache::CloseWatcherHandles(const CTGitPath
& path
)
580 watcher
.CloseHandlesForPath(path
);
581 m_folderCrawler
.ReleasePathForUpdate(path
);
582 CGitStatusCache::Instance().m_GitStatus
.ReleasePathsRecursively(path
.GetWinPathString());
585 CString
CGitStatusCache::GetSpecialFolder(REFKNOWNFOLDERID rfid
)
587 PWSTR pszPath
= nullptr;
588 if (SHGetKnownFolderPath(rfid
, KF_FLAG_CREATE
, nullptr, &pszPath
) != S_OK
)
591 CString path
= pszPath
;
592 CoTaskMemFree(pszPath
);