Bumping manifests a=b2g-bump
[gecko.git] / netwerk / cache2 / CacheStorageService.cpp
blobea3efc4b69447f77fdd7d0a35030e032653fcd0e
1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
5 #include "CacheLog.h"
6 #include "CacheStorageService.h"
7 #include "CacheFileIOManager.h"
8 #include "CacheObserver.h"
9 #include "CacheIndex.h"
10 #include "CacheIndexIterator.h"
11 #include "CacheStorage.h"
12 #include "AppCacheStorage.h"
13 #include "CacheEntry.h"
14 #include "CacheFileUtils.h"
16 #include "OldWrappers.h"
17 #include "nsCacheService.h"
18 #include "nsDeleteDir.h"
20 #include "nsICacheStorageVisitor.h"
21 #include "nsIObserverService.h"
22 #include "nsIFile.h"
23 #include "nsIURI.h"
24 #include "nsCOMPtr.h"
25 #include "nsAutoPtr.h"
26 #include "nsNetCID.h"
27 #include "nsNetUtil.h"
28 #include "nsServiceManagerUtils.h"
29 #include "nsWeakReference.h"
30 #include "mozilla/TimeStamp.h"
31 #include "mozilla/DebugOnly.h"
32 #include "mozilla/VisualEventTracer.h"
33 #include "mozilla/Services.h"
35 namespace mozilla {
36 namespace net {
38 namespace {
40 void AppendMemoryStorageID(nsAutoCString &key)
42 key.Append('/');
43 key.Append('M');
48 // Not defining as static or class member of CacheStorageService since
49 // it would otherwise need to include CacheEntry.h and that then would
50 // need to be exported to make nsNetModule.cpp compilable.
51 typedef nsClassHashtable<nsCStringHashKey, CacheEntryTable>
52 GlobalEntryTables;
54 /**
55 * Keeps tables of entries. There is one entries table for each distinct load
56 * context type. The distinction is based on following load context info states:
57 * <isPrivate|isAnon|appId|inBrowser> which builds a mapping key.
59 * Thread-safe to access, protected by the service mutex.
61 static GlobalEntryTables* sGlobalEntryTables;
63 CacheMemoryConsumer::CacheMemoryConsumer(uint32_t aFlags)
64 : mReportedMemoryConsumption(0)
65 , mFlags(aFlags)
69 void
70 CacheMemoryConsumer::DoMemoryReport(uint32_t aCurrentSize)
72 if (!(mFlags & DONT_REPORT) && CacheStorageService::Self()) {
73 CacheStorageService::Self()->OnMemoryConsumptionChange(this, aCurrentSize);
77 CacheStorageService::MemoryPool::MemoryPool(EType aType)
78 : mType(aType)
79 , mMemorySize(0)
83 CacheStorageService::MemoryPool::~MemoryPool()
85 if (mMemorySize != 0) {
86 NS_ERROR("Network cache reported memory consumption is not at 0, probably leaking?");
90 uint32_t const
91 CacheStorageService::MemoryPool::Limit() const
93 switch (mType) {
94 case DISK:
95 return CacheObserver::MetadataMemoryLimit();
96 case MEMORY:
97 return CacheObserver::MemoryCacheCapacity();
100 MOZ_CRASH("Bad pool type");
101 return 0;
104 NS_IMPL_ISUPPORTS(CacheStorageService,
105 nsICacheStorageService,
106 nsIMemoryReporter,
107 nsITimerCallback)
109 CacheStorageService* CacheStorageService::sSelf = nullptr;
111 CacheStorageService::CacheStorageService()
112 : mLock("CacheStorageService.mLock")
113 , mForcedValidEntriesLock("CacheStorageService.mForcedValidEntriesLock")
114 , mShutdown(false)
115 , mDiskPool(MemoryPool::DISK)
116 , mMemoryPool(MemoryPool::MEMORY)
118 CacheFileIOManager::Init();
120 MOZ_ASSERT(!sSelf);
122 sSelf = this;
123 sGlobalEntryTables = new GlobalEntryTables();
125 RegisterStrongMemoryReporter(this);
128 CacheStorageService::~CacheStorageService()
130 LOG(("CacheStorageService::~CacheStorageService"));
131 sSelf = nullptr;
134 void CacheStorageService::Shutdown()
136 if (mShutdown)
137 return;
139 LOG(("CacheStorageService::Shutdown - start"));
141 mShutdown = true;
143 nsCOMPtr<nsIRunnable> event =
144 NS_NewRunnableMethod(this, &CacheStorageService::ShutdownBackground);
145 Dispatch(event);
147 mozilla::MutexAutoLock lock(mLock);
148 sGlobalEntryTables->Clear();
149 delete sGlobalEntryTables;
150 sGlobalEntryTables = nullptr;
152 LOG(("CacheStorageService::Shutdown - done"));
155 void CacheStorageService::ShutdownBackground()
157 MOZ_ASSERT(IsOnManagementThread());
159 // Cancel purge timer to avoid leaking.
160 if (mPurgeTimer) {
161 mPurgeTimer->Cancel();
164 Pool(false).mFrecencyArray.Clear();
165 Pool(false).mExpirationArray.Clear();
166 Pool(true).mFrecencyArray.Clear();
167 Pool(true).mExpirationArray.Clear();
170 // Internal management methods
172 namespace { // anon
174 // WalkCacheRunnable
175 // Base class for particular storage entries visiting
176 class WalkCacheRunnable : public nsRunnable
177 , public CacheStorageService::EntryInfoCallback
179 protected:
180 WalkCacheRunnable(nsICacheStorageVisitor* aVisitor,
181 bool aVisitEntries)
182 : mService(CacheStorageService::Self())
183 , mCallback(aVisitor)
184 , mSize(0)
185 , mNotifyStorage(true)
186 , mVisitEntries(aVisitEntries)
188 MOZ_ASSERT(NS_IsMainThread());
191 virtual ~WalkCacheRunnable()
193 if (mCallback) {
194 ProxyReleaseMainThread(mCallback);
198 nsRefPtr<CacheStorageService> mService;
199 nsCOMPtr<nsICacheStorageVisitor> mCallback;
201 uint64_t mSize;
203 bool mNotifyStorage : 1;
204 bool mVisitEntries : 1;
207 // WalkMemoryCacheRunnable
208 // Responsible to visit memory storage and walk
209 // all entries on it asynchronously.
210 class WalkMemoryCacheRunnable : public WalkCacheRunnable
212 public:
213 WalkMemoryCacheRunnable(nsILoadContextInfo *aLoadInfo,
214 bool aVisitEntries,
215 nsICacheStorageVisitor* aVisitor)
216 : WalkCacheRunnable(aVisitor, aVisitEntries)
218 CacheFileUtils::AppendKeyPrefix(aLoadInfo, mContextKey);
219 MOZ_ASSERT(NS_IsMainThread());
222 nsresult Walk()
224 return mService->Dispatch(this);
227 private:
228 NS_IMETHODIMP Run()
230 if (CacheStorageService::IsOnManagementThread()) {
231 LOG(("WalkMemoryCacheRunnable::Run - collecting [this=%p]", this));
232 // First, walk, count and grab all entries from the storage
234 mozilla::MutexAutoLock lock(CacheStorageService::Self()->Lock());
236 if (!CacheStorageService::IsRunning())
237 return NS_ERROR_NOT_INITIALIZED;
239 CacheEntryTable* entries;
240 if (sGlobalEntryTables->Get(mContextKey, &entries))
241 entries->EnumerateRead(&WalkMemoryCacheRunnable::WalkStorage, this);
243 // Next, we dispatch to the main thread
244 } else if (NS_IsMainThread()) {
245 LOG(("WalkMemoryCacheRunnable::Run - notifying [this=%p]", this));
247 if (mNotifyStorage) {
248 LOG((" storage"));
250 // Second, notify overall storage info
251 mCallback->OnCacheStorageInfo(mEntryArray.Length(), mSize,
252 CacheObserver::MemoryCacheCapacity(), nullptr);
253 if (!mVisitEntries)
254 return NS_OK; // done
256 mNotifyStorage = false;
258 } else {
259 LOG((" entry [left=%d]", mEntryArray.Length()));
261 // Third, notify each entry until depleted
262 if (!mEntryArray.Length()) {
263 mCallback->OnCacheEntryVisitCompleted();
264 return NS_OK; // done
267 // Grab the next entry
268 nsRefPtr<CacheEntry> entry = mEntryArray[0];
269 mEntryArray.RemoveElementAt(0);
271 // Invokes this->OnEntryInfo, that calls the callback with all
272 // information of the entry.
273 CacheStorageService::GetCacheEntryInfo(entry, this);
275 } else {
276 MOZ_CRASH("Bad thread");
277 return NS_ERROR_FAILURE;
280 NS_DispatchToMainThread(this);
281 return NS_OK;
284 virtual ~WalkMemoryCacheRunnable()
286 if (mCallback)
287 ProxyReleaseMainThread(mCallback);
290 static PLDHashOperator
291 WalkStorage(const nsACString& aKey,
292 CacheEntry* aEntry,
293 void* aClosure)
295 WalkMemoryCacheRunnable* walker =
296 static_cast<WalkMemoryCacheRunnable*>(aClosure);
298 // Ignore disk entries
299 if (aEntry->IsUsingDisk())
300 return PL_DHASH_NEXT;
302 walker->mSize += aEntry->GetMetadataMemoryConsumption();
304 int64_t size;
305 if (NS_SUCCEEDED(aEntry->GetDataSize(&size)))
306 walker->mSize += size;
308 walker->mEntryArray.AppendElement(aEntry);
309 return PL_DHASH_NEXT;
312 virtual void OnEntryInfo(const nsACString & aURISpec, const nsACString & aIdEnhance,
313 int64_t aDataSize, int32_t aFetchCount,
314 uint32_t aLastModifiedTime, uint32_t aExpirationTime)
316 nsCOMPtr<nsIURI> uri;
317 nsresult rv = NS_NewURI(getter_AddRefs(uri), aURISpec);
318 if (NS_FAILED(rv))
319 return;
321 mCallback->OnCacheEntryInfo(uri, aIdEnhance, aDataSize, aFetchCount,
322 aLastModifiedTime, aExpirationTime);
325 private:
326 nsCString mContextKey;
327 nsTArray<nsRefPtr<CacheEntry> > mEntryArray;
330 // WalkDiskCacheRunnable
331 // Using the cache index information to get the list of files per context.
332 class WalkDiskCacheRunnable : public WalkCacheRunnable
334 public:
335 WalkDiskCacheRunnable(nsILoadContextInfo *aLoadInfo,
336 bool aVisitEntries,
337 nsICacheStorageVisitor* aVisitor)
338 : WalkCacheRunnable(aVisitor, aVisitEntries)
339 , mLoadInfo(aLoadInfo)
340 , mPass(COLLECT_STATS)
344 nsresult Walk()
346 // TODO, bug 998693
347 // Initial index build should be forced here so that about:cache soon
348 // after startup gives some meaningfull results.
350 // Dispatch to the INDEX level in hope that very recent cache entries
351 // information gets to the index list before we grab the index iterator
352 // for the first time. This tries to avoid miss of entries that has
353 // been created right before the visit is required.
354 nsRefPtr<CacheIOThread> thread = CacheFileIOManager::IOThread();
355 NS_ENSURE_TRUE(thread, NS_ERROR_NOT_INITIALIZED);
357 return thread->Dispatch(this, CacheIOThread::INDEX);
360 private:
361 // Invokes OnCacheEntryInfo callback for each single found entry.
362 // There is one instance of this class per one entry.
363 class OnCacheEntryInfoRunnable : public nsRunnable
365 public:
366 explicit OnCacheEntryInfoRunnable(WalkDiskCacheRunnable* aWalker)
367 : mWalker(aWalker)
371 NS_IMETHODIMP Run()
373 MOZ_ASSERT(NS_IsMainThread());
375 nsCOMPtr<nsIURI> uri;
376 nsresult rv = NS_NewURI(getter_AddRefs(uri), mURISpec);
377 if (NS_FAILED(rv))
378 return NS_OK;
380 mWalker->mCallback->OnCacheEntryInfo(
381 uri, mIdEnhance, mDataSize, mFetchCount,
382 mLastModifiedTime, mExpirationTime);
383 return NS_OK;
386 nsRefPtr<WalkDiskCacheRunnable> mWalker;
388 nsCString mURISpec;
389 nsCString mIdEnhance;
390 int64_t mDataSize;
391 int32_t mFetchCount;
392 uint32_t mLastModifiedTime;
393 uint32_t mExpirationTime;
396 NS_IMETHODIMP Run()
398 // The main loop
399 nsresult rv;
401 if (CacheStorageService::IsOnManagementThread()) {
402 switch (mPass) {
403 case COLLECT_STATS:
404 // Get quickly the cache stats.
405 uint32_t size;
406 rv = CacheIndex::GetCacheStats(mLoadInfo, &size, &mCount);
407 if (NS_FAILED(rv)) {
408 if (mVisitEntries) {
409 // both onStorageInfo and onCompleted are expected
410 NS_DispatchToMainThread(this);
412 return NS_DispatchToMainThread(this);
415 mSize = size << 10;
417 // Invoke onCacheStorageInfo with valid information.
418 NS_DispatchToMainThread(this);
420 if (!mVisitEntries) {
421 return NS_OK; // done
424 mPass = ITERATE_METADATA;
425 // no break
427 case ITERATE_METADATA:
428 // Now grab the context iterator.
429 if (!mIter) {
430 rv = CacheIndex::GetIterator(mLoadInfo, true, getter_AddRefs(mIter));
431 if (NS_FAILED(rv)) {
432 // Invoke onCacheEntryVisitCompleted now
433 return NS_DispatchToMainThread(this);
437 while (true) {
438 if (CacheIOThread::YieldAndRerun())
439 return NS_OK;
441 SHA1Sum::Hash hash;
442 rv = mIter->GetNextHash(&hash);
443 if (NS_FAILED(rv))
444 break; // done (or error?)
446 // This synchronously invokes onCacheEntryInfo on this class where we
447 // redispatch to the main thread for the consumer callback.
448 CacheFileIOManager::GetEntryInfo(&hash, this);
451 // Invoke onCacheEntryVisitCompleted on the main thread
452 NS_DispatchToMainThread(this);
454 } else if (NS_IsMainThread()) {
455 if (mNotifyStorage) {
456 nsCOMPtr<nsIFile> dir;
457 CacheFileIOManager::GetCacheDirectory(getter_AddRefs(dir));
458 mCallback->OnCacheStorageInfo(mCount, mSize, CacheObserver::DiskCacheCapacity(), dir);
459 mNotifyStorage = false;
460 } else {
461 mCallback->OnCacheEntryVisitCompleted();
463 } else {
464 MOZ_CRASH("Bad thread");
465 return NS_ERROR_FAILURE;
468 return NS_OK;
471 virtual void OnEntryInfo(const nsACString & aURISpec, const nsACString & aIdEnhance,
472 int64_t aDataSize, int32_t aFetchCount,
473 uint32_t aLastModifiedTime, uint32_t aExpirationTime)
475 // Called directly from CacheFileIOManager::GetEntryInfo.
477 // Invoke onCacheEntryInfo on the main thread for this entry.
478 nsRefPtr<OnCacheEntryInfoRunnable> info = new OnCacheEntryInfoRunnable(this);
479 info->mURISpec = aURISpec;
480 info->mIdEnhance = aIdEnhance;
481 info->mDataSize = aDataSize;
482 info->mFetchCount = aFetchCount;
483 info->mLastModifiedTime = aLastModifiedTime;
484 info->mExpirationTime = aExpirationTime;
486 NS_DispatchToMainThread(info);
489 nsRefPtr<nsILoadContextInfo> mLoadInfo;
490 enum {
491 // First, we collect stats for the load context.
492 COLLECT_STATS,
494 // Second, if demanded, we iterate over the entries gethered
495 // from the iterator and call CacheFileIOManager::GetEntryInfo
496 // for each found entry.
497 ITERATE_METADATA,
498 } mPass;
500 nsRefPtr<CacheIndexIterator> mIter;
501 uint32_t mCount;
504 PLDHashOperator CollectPrivateContexts(const nsACString& aKey,
505 CacheEntryTable* aTable,
506 void* aClosure)
508 nsCOMPtr<nsILoadContextInfo> info = CacheFileUtils::ParseKey(aKey);
509 if (info && info->IsPrivate()) {
510 nsTArray<nsCString>* keys = static_cast<nsTArray<nsCString>*>(aClosure);
511 keys->AppendElement(aKey);
514 return PL_DHASH_NEXT;
517 PLDHashOperator CollectContexts(const nsACString& aKey,
518 CacheEntryTable* aTable,
519 void* aClosure)
521 nsTArray<nsCString>* keys = static_cast<nsTArray<nsCString>*>(aClosure);
522 keys->AppendElement(aKey);
524 return PL_DHASH_NEXT;
527 } // anon
529 void CacheStorageService::DropPrivateBrowsingEntries()
531 mozilla::MutexAutoLock lock(mLock);
533 if (mShutdown)
534 return;
536 nsTArray<nsCString> keys;
537 sGlobalEntryTables->EnumerateRead(&CollectPrivateContexts, &keys);
539 for (uint32_t i = 0; i < keys.Length(); ++i)
540 DoomStorageEntries(keys[i], nullptr, true, nullptr);
543 namespace { // anon
545 class CleaupCacheDirectoriesRunnable : public nsRunnable
547 public:
548 NS_DECL_NSIRUNNABLE
549 static bool Post(uint32_t aVersion, uint32_t aActive);
551 private:
552 CleaupCacheDirectoriesRunnable(uint32_t aVersion, uint32_t aActive)
553 : mVersion(aVersion), mActive(aActive)
555 nsCacheService::GetDiskCacheDirectory(getter_AddRefs(mCache1Dir));
556 CacheFileIOManager::GetCacheDirectory(getter_AddRefs(mCache2Dir));
557 #if defined(MOZ_WIDGET_ANDROID)
558 CacheFileIOManager::GetProfilelessCacheDirectory(getter_AddRefs(mCache2Profileless));
559 #endif
562 virtual ~CleaupCacheDirectoriesRunnable() {}
563 uint32_t mVersion, mActive;
564 nsCOMPtr<nsIFile> mCache1Dir, mCache2Dir;
565 #if defined(MOZ_WIDGET_ANDROID)
566 nsCOMPtr<nsIFile> mCache2Profileless;
567 #endif
570 // static
571 bool CleaupCacheDirectoriesRunnable::Post(uint32_t aVersion, uint32_t aActive)
573 // CleaupCacheDirectories is called regardless what cache version is set up to use.
574 // To obtain the cache1 directory we must unfortunatelly instantiate the old cache
575 // service despite it may not be used at all... This also initialize nsDeleteDir.
576 nsCOMPtr<nsICacheService> service = do_GetService(NS_CACHESERVICE_CONTRACTID);
577 if (!service)
578 return false;
580 nsCOMPtr<nsIEventTarget> thread;
581 service->GetCacheIOTarget(getter_AddRefs(thread));
582 if (!thread)
583 return false;
585 nsRefPtr<CleaupCacheDirectoriesRunnable> r =
586 new CleaupCacheDirectoriesRunnable(aVersion, aActive);
587 thread->Dispatch(r, NS_DISPATCH_NORMAL);
588 return true;
591 NS_IMETHODIMP CleaupCacheDirectoriesRunnable::Run()
593 MOZ_ASSERT(!NS_IsMainThread());
595 if (mCache1Dir) {
596 nsDeleteDir::RemoveOldTrashes(mCache1Dir);
598 if (mCache2Dir) {
599 nsDeleteDir::RemoveOldTrashes(mCache2Dir);
601 #if defined(MOZ_WIDGET_ANDROID)
602 if (mCache2Profileless) {
603 nsDeleteDir::RemoveOldTrashes(mCache2Profileless);
604 // Always delete the profileless cache on Android
605 nsDeleteDir::DeleteDir(mCache2Profileless, true, 30000);
607 #endif
609 // Delete the non-active version cache data right now
610 if (mVersion == mActive) {
611 return NS_OK;
614 switch (mVersion) {
615 case 0:
616 if (mCache1Dir) {
617 nsDeleteDir::DeleteDir(mCache1Dir, true, 30000);
619 break;
620 case 1:
621 if (mCache2Dir) {
622 nsDeleteDir::DeleteDir(mCache2Dir, true, 30000);
624 break;
627 return NS_OK;
630 } // anon
632 // static
633 void CacheStorageService::CleaupCacheDirectories(uint32_t aVersion, uint32_t aActive)
635 // Make sure we schedule just once in case CleaupCacheDirectories gets called
636 // multiple times from some reason.
637 static bool runOnce = CleaupCacheDirectoriesRunnable::Post(aVersion, aActive);
638 if (!runOnce) {
639 NS_WARNING("Could not start cache trashes cleanup");
643 // Helper methods
645 // static
646 bool CacheStorageService::IsOnManagementThread()
648 nsRefPtr<CacheStorageService> service = Self();
649 if (!service)
650 return false;
652 nsCOMPtr<nsIEventTarget> target = service->Thread();
653 if (!target)
654 return false;
656 bool currentThread;
657 nsresult rv = target->IsOnCurrentThread(&currentThread);
658 return NS_SUCCEEDED(rv) && currentThread;
661 already_AddRefed<nsIEventTarget> CacheStorageService::Thread() const
663 return CacheFileIOManager::IOTarget();
666 nsresult CacheStorageService::Dispatch(nsIRunnable* aEvent)
668 nsRefPtr<CacheIOThread> cacheIOThread = CacheFileIOManager::IOThread();
669 if (!cacheIOThread)
670 return NS_ERROR_NOT_AVAILABLE;
672 return cacheIOThread->Dispatch(aEvent, CacheIOThread::MANAGEMENT);
675 // nsICacheStorageService
677 NS_IMETHODIMP CacheStorageService::MemoryCacheStorage(nsILoadContextInfo *aLoadContextInfo,
678 nsICacheStorage * *_retval)
680 NS_ENSURE_ARG(aLoadContextInfo);
681 NS_ENSURE_ARG(_retval);
683 nsCOMPtr<nsICacheStorage> storage;
684 if (CacheObserver::UseNewCache()) {
685 storage = new CacheStorage(aLoadContextInfo, false, false);
687 else {
688 storage = new _OldStorage(aLoadContextInfo, false, false, false, nullptr);
691 storage.forget(_retval);
692 return NS_OK;
695 NS_IMETHODIMP CacheStorageService::DiskCacheStorage(nsILoadContextInfo *aLoadContextInfo,
696 bool aLookupAppCache,
697 nsICacheStorage * *_retval)
699 NS_ENSURE_ARG(aLoadContextInfo);
700 NS_ENSURE_ARG(_retval);
702 // TODO save some heap granularity - cache commonly used storages.
704 // When disk cache is disabled, still provide a storage, but just keep stuff
705 // in memory.
706 bool useDisk = CacheObserver::UseDiskCache();
708 nsCOMPtr<nsICacheStorage> storage;
709 if (CacheObserver::UseNewCache()) {
710 storage = new CacheStorage(aLoadContextInfo, useDisk, aLookupAppCache);
712 else {
713 storage = new _OldStorage(aLoadContextInfo, useDisk, aLookupAppCache, false, nullptr);
716 storage.forget(_retval);
717 return NS_OK;
720 NS_IMETHODIMP CacheStorageService::AppCacheStorage(nsILoadContextInfo *aLoadContextInfo,
721 nsIApplicationCache *aApplicationCache,
722 nsICacheStorage * *_retval)
724 NS_ENSURE_ARG(aLoadContextInfo);
725 NS_ENSURE_ARG(_retval);
727 nsCOMPtr<nsICacheStorage> storage;
728 if (CacheObserver::UseNewCache()) {
729 // Using classification since cl believes we want to instantiate this method
730 // having the same name as the desired class...
731 storage = new mozilla::net::AppCacheStorage(aLoadContextInfo, aApplicationCache);
733 else {
734 storage = new _OldStorage(aLoadContextInfo, true, false, true, aApplicationCache);
737 storage.forget(_retval);
738 return NS_OK;
741 NS_IMETHODIMP CacheStorageService::Clear()
743 nsresult rv;
745 if (CacheObserver::UseNewCache()) {
747 mozilla::MutexAutoLock lock(mLock);
749 NS_ENSURE_TRUE(!mShutdown, NS_ERROR_NOT_INITIALIZED);
751 nsTArray<nsCString> keys;
752 sGlobalEntryTables->EnumerateRead(&CollectContexts, &keys);
754 for (uint32_t i = 0; i < keys.Length(); ++i)
755 DoomStorageEntries(keys[i], nullptr, true, nullptr);
758 rv = CacheFileIOManager::EvictAll();
759 NS_ENSURE_SUCCESS(rv, rv);
760 } else {
761 nsCOMPtr<nsICacheService> serv =
762 do_GetService(NS_CACHESERVICE_CONTRACTID, &rv);
763 NS_ENSURE_SUCCESS(rv, rv);
765 rv = serv->EvictEntries(nsICache::STORE_ANYWHERE);
766 NS_ENSURE_SUCCESS(rv, rv);
769 return NS_OK;
772 NS_IMETHODIMP CacheStorageService::PurgeFromMemory(uint32_t aWhat)
774 uint32_t what;
776 switch (aWhat) {
777 case PURGE_DISK_DATA_ONLY:
778 what = CacheEntry::PURGE_DATA_ONLY_DISK_BACKED;
779 break;
781 case PURGE_DISK_ALL:
782 what = CacheEntry::PURGE_WHOLE_ONLY_DISK_BACKED;
783 break;
785 case PURGE_EVERYTHING:
786 what = CacheEntry::PURGE_WHOLE;
787 break;
789 default:
790 return NS_ERROR_INVALID_ARG;
793 nsCOMPtr<nsIRunnable> event =
794 new PurgeFromMemoryRunnable(this, what);
796 return Dispatch(event);
799 NS_IMETHODIMP CacheStorageService::AsyncGetDiskConsumption(
800 nsICacheStorageConsumptionObserver* aObserver)
802 NS_ENSURE_ARG(aObserver);
804 nsresult rv;
806 if (CacheObserver::UseNewCache()) {
807 rv = CacheIndex::AsyncGetDiskConsumption(aObserver);
808 NS_ENSURE_SUCCESS(rv, rv);
809 } else {
810 rv = _OldGetDiskConsumption::Get(aObserver);
811 NS_ENSURE_SUCCESS(rv, rv);
814 return NS_OK;
817 NS_IMETHODIMP CacheStorageService::GetIoTarget(nsIEventTarget** aEventTarget)
819 NS_ENSURE_ARG(aEventTarget);
821 if (CacheObserver::UseNewCache()) {
822 nsCOMPtr<nsIEventTarget> ioTarget = CacheFileIOManager::IOTarget();
823 ioTarget.forget(aEventTarget);
825 else {
826 nsresult rv;
828 nsCOMPtr<nsICacheService> serv =
829 do_GetService(NS_CACHESERVICE_CONTRACTID, &rv);
830 NS_ENSURE_SUCCESS(rv, rv);
832 rv = serv->GetCacheIOTarget(aEventTarget);
833 NS_ENSURE_SUCCESS(rv, rv);
836 return NS_OK;
839 // Methods used by CacheEntry for management of in-memory structures.
841 namespace { // anon
843 class FrecencyComparator
845 public:
846 bool Equals(CacheEntry* a, CacheEntry* b) const {
847 return a->GetFrecency() == b->GetFrecency();
849 bool LessThan(CacheEntry* a, CacheEntry* b) const {
850 return a->GetFrecency() < b->GetFrecency();
854 class ExpirationComparator
856 public:
857 bool Equals(CacheEntry* a, CacheEntry* b) const {
858 return a->GetExpirationTime() == b->GetExpirationTime();
860 bool LessThan(CacheEntry* a, CacheEntry* b) const {
861 return a->GetExpirationTime() < b->GetExpirationTime();
865 } // anon
867 void
868 CacheStorageService::RegisterEntry(CacheEntry* aEntry)
870 MOZ_ASSERT(IsOnManagementThread());
872 if (mShutdown || !aEntry->CanRegister())
873 return;
875 TelemetryRecordEntryCreation(aEntry);
877 LOG(("CacheStorageService::RegisterEntry [entry=%p]", aEntry));
879 MemoryPool& pool = Pool(aEntry->IsUsingDisk());
880 pool.mFrecencyArray.InsertElementSorted(aEntry, FrecencyComparator());
881 pool.mExpirationArray.InsertElementSorted(aEntry, ExpirationComparator());
883 aEntry->SetRegistered(true);
886 void
887 CacheStorageService::UnregisterEntry(CacheEntry* aEntry)
889 MOZ_ASSERT(IsOnManagementThread());
891 if (!aEntry->IsRegistered())
892 return;
894 TelemetryRecordEntryRemoval(aEntry);
896 LOG(("CacheStorageService::UnregisterEntry [entry=%p]", aEntry));
898 MemoryPool& pool = Pool(aEntry->IsUsingDisk());
899 mozilla::DebugOnly<bool> removedFrecency = pool.mFrecencyArray.RemoveElement(aEntry);
900 mozilla::DebugOnly<bool> removedExpiration = pool.mExpirationArray.RemoveElement(aEntry);
902 MOZ_ASSERT(mShutdown || (removedFrecency && removedExpiration));
904 // Note: aEntry->CanRegister() since now returns false
905 aEntry->SetRegistered(false);
908 static bool
909 AddExactEntry(CacheEntryTable* aEntries,
910 nsCString const& aKey,
911 CacheEntry* aEntry,
912 bool aOverwrite)
914 nsRefPtr<CacheEntry> existingEntry;
915 if (!aOverwrite && aEntries->Get(aKey, getter_AddRefs(existingEntry))) {
916 bool equals = existingEntry == aEntry;
917 LOG(("AddExactEntry [entry=%p equals=%d]", aEntry, equals));
918 return equals; // Already there...
921 LOG(("AddExactEntry [entry=%p put]", aEntry));
922 aEntries->Put(aKey, aEntry);
923 return true;
926 static bool
927 RemoveExactEntry(CacheEntryTable* aEntries,
928 nsCString const& aKey,
929 CacheEntry* aEntry,
930 bool aOverwrite)
932 nsRefPtr<CacheEntry> existingEntry;
933 if (!aEntries->Get(aKey, getter_AddRefs(existingEntry))) {
934 LOG(("RemoveExactEntry [entry=%p already gone]", aEntry));
935 return false; // Already removed...
938 if (!aOverwrite && existingEntry != aEntry) {
939 LOG(("RemoveExactEntry [entry=%p already replaced]", aEntry));
940 return false; // Already replaced...
943 LOG(("RemoveExactEntry [entry=%p removed]", aEntry));
944 aEntries->Remove(aKey);
945 return true;
948 bool
949 CacheStorageService::RemoveEntry(CacheEntry* aEntry, bool aOnlyUnreferenced)
951 LOG(("CacheStorageService::RemoveEntry [entry=%p]", aEntry));
953 nsAutoCString entryKey;
954 nsresult rv = aEntry->HashingKey(entryKey);
955 if (NS_FAILED(rv)) {
956 NS_ERROR("aEntry->HashingKey() failed?");
957 return false;
960 mozilla::MutexAutoLock lock(mLock);
962 if (mShutdown) {
963 LOG((" after shutdown"));
964 return false;
967 if (aOnlyUnreferenced) {
968 if (aEntry->IsReferenced()) {
969 LOG((" still referenced, not removing"));
970 return false;
973 if (!aEntry->IsUsingDisk() && IsForcedValidEntry(entryKey)) {
974 LOG((" forced valid, not removing"));
975 return false;
979 CacheEntryTable* entries;
980 if (sGlobalEntryTables->Get(aEntry->GetStorageID(), &entries))
981 RemoveExactEntry(entries, entryKey, aEntry, false /* don't overwrite */);
983 nsAutoCString memoryStorageID(aEntry->GetStorageID());
984 AppendMemoryStorageID(memoryStorageID);
986 if (sGlobalEntryTables->Get(memoryStorageID, &entries))
987 RemoveExactEntry(entries, entryKey, aEntry, false /* don't overwrite */);
989 return true;
992 void
993 CacheStorageService::RecordMemoryOnlyEntry(CacheEntry* aEntry,
994 bool aOnlyInMemory,
995 bool aOverwrite)
997 LOG(("CacheStorageService::RecordMemoryOnlyEntry [entry=%p, memory=%d, overwrite=%d]",
998 aEntry, aOnlyInMemory, aOverwrite));
999 // This method is responsible to put this entry to a special record hashtable
1000 // that contains only entries that are stored in memory.
1001 // Keep in mind that every entry, regardless of whether is in-memory-only or not
1002 // is always recorded in the storage master hash table, the one identified by
1003 // CacheEntry.StorageID().
1005 mLock.AssertCurrentThreadOwns();
1007 if (mShutdown) {
1008 LOG((" after shutdown"));
1009 return;
1012 nsresult rv;
1014 nsAutoCString entryKey;
1015 rv = aEntry->HashingKey(entryKey);
1016 if (NS_FAILED(rv)) {
1017 NS_ERROR("aEntry->HashingKey() failed?");
1018 return;
1021 CacheEntryTable* entries = nullptr;
1022 nsAutoCString memoryStorageID(aEntry->GetStorageID());
1023 AppendMemoryStorageID(memoryStorageID);
1025 if (!sGlobalEntryTables->Get(memoryStorageID, &entries)) {
1026 if (!aOnlyInMemory) {
1027 LOG((" not recorded as memory only"));
1028 return;
1031 entries = new CacheEntryTable(CacheEntryTable::MEMORY_ONLY);
1032 sGlobalEntryTables->Put(memoryStorageID, entries);
1033 LOG((" new memory-only storage table for %s", memoryStorageID.get()));
1036 if (aOnlyInMemory) {
1037 AddExactEntry(entries, entryKey, aEntry, aOverwrite);
1039 else {
1040 RemoveExactEntry(entries, entryKey, aEntry, aOverwrite);
1044 // Checks if a cache entry is forced valid (will be loaded directly from cache
1045 // without further validation) - see nsICacheEntry.idl for further details
1046 bool CacheStorageService::IsForcedValidEntry(nsACString &aCacheEntryKey)
1048 mozilla::MutexAutoLock lock(mForcedValidEntriesLock);
1050 TimeStamp validUntil;
1052 if (!mForcedValidEntries.Get(aCacheEntryKey, &validUntil)) {
1053 return false;
1056 if (validUntil.IsNull()) {
1057 return false;
1060 // Entry timeout not reached yet
1061 if (TimeStamp::NowLoRes() <= validUntil) {
1062 return true;
1065 // Entry timeout has been reached
1066 mForcedValidEntries.Remove(aCacheEntryKey);
1067 return false;
1070 // Allows a cache entry to be loaded directly from cache without further
1071 // validation - see nsICacheEntry.idl for further details
1072 void CacheStorageService::ForceEntryValidFor(nsACString &aCacheEntryKey,
1073 uint32_t aSecondsToTheFuture)
1075 mozilla::MutexAutoLock lock(mForcedValidEntriesLock);
1077 TimeStamp now = TimeStamp::NowLoRes();
1078 ForcedValidEntriesPrune(now);
1080 // This will be the timeout
1081 TimeStamp validUntil = now + TimeDuration::FromSeconds(aSecondsToTheFuture);
1083 mForcedValidEntries.Put(aCacheEntryKey, validUntil);
1086 namespace { // anon
1088 PLDHashOperator PruneForcedValidEntries(
1089 const nsACString& aKey, TimeStamp& aTimeStamp, void* aClosure)
1091 TimeStamp* now = static_cast<TimeStamp*>(aClosure);
1092 if (aTimeStamp < *now) {
1093 return PL_DHASH_REMOVE;
1096 return PL_DHASH_NEXT;
1099 } // anon
1101 // Cleans out the old entries in mForcedValidEntries
1102 void CacheStorageService::ForcedValidEntriesPrune(TimeStamp &now)
1104 static TimeDuration const oneMinute = TimeDuration::FromSeconds(60);
1105 static TimeStamp dontPruneUntil = now + oneMinute;
1106 if (now < dontPruneUntil)
1107 return;
1109 mForcedValidEntries.Enumerate(PruneForcedValidEntries, &now);
1110 dontPruneUntil = now + oneMinute;
1113 void
1114 CacheStorageService::OnMemoryConsumptionChange(CacheMemoryConsumer* aConsumer,
1115 uint32_t aCurrentMemoryConsumption)
1117 LOG(("CacheStorageService::OnMemoryConsumptionChange [consumer=%p, size=%u]",
1118 aConsumer, aCurrentMemoryConsumption));
1120 uint32_t savedMemorySize = aConsumer->mReportedMemoryConsumption;
1121 if (savedMemorySize == aCurrentMemoryConsumption)
1122 return;
1124 // Exchange saved size with current one.
1125 aConsumer->mReportedMemoryConsumption = aCurrentMemoryConsumption;
1127 bool usingDisk = !(aConsumer->mFlags & CacheMemoryConsumer::MEMORY_ONLY);
1128 bool overLimit = Pool(usingDisk).OnMemoryConsumptionChange(
1129 savedMemorySize, aCurrentMemoryConsumption);
1131 if (!overLimit)
1132 return;
1134 // It's likely the timer has already been set when we get here,
1135 // check outside the lock to save resources.
1136 if (mPurgeTimer)
1137 return;
1139 // We don't know if this is called under the service lock or not,
1140 // hence rather dispatch.
1141 nsRefPtr<nsIEventTarget> cacheIOTarget = Thread();
1142 if (!cacheIOTarget)
1143 return;
1145 // Dispatch as a priority task, we want to set the purge timer
1146 // ASAP to prevent vain redispatch of this event.
1147 nsCOMPtr<nsIRunnable> event =
1148 NS_NewRunnableMethod(this, &CacheStorageService::SchedulePurgeOverMemoryLimit);
1149 cacheIOTarget->Dispatch(event, nsIEventTarget::DISPATCH_NORMAL);
1152 bool
1153 CacheStorageService::MemoryPool::OnMemoryConsumptionChange(uint32_t aSavedMemorySize,
1154 uint32_t aCurrentMemoryConsumption)
1156 mMemorySize -= aSavedMemorySize;
1157 mMemorySize += aCurrentMemoryConsumption;
1159 LOG((" mMemorySize=%u (+%u,-%u)", uint32_t(mMemorySize), aCurrentMemoryConsumption, aSavedMemorySize));
1161 // Bypass purging when memory has not grew up significantly
1162 if (aCurrentMemoryConsumption <= aSavedMemorySize)
1163 return false;
1165 return mMemorySize > Limit();
1168 void
1169 CacheStorageService::SchedulePurgeOverMemoryLimit()
1171 mozilla::MutexAutoLock lock(mLock);
1173 if (mPurgeTimer)
1174 return;
1176 mPurgeTimer = do_CreateInstance(NS_TIMER_CONTRACTID);
1177 if (mPurgeTimer)
1178 mPurgeTimer->InitWithCallback(this, 1000, nsITimer::TYPE_ONE_SHOT);
1181 NS_IMETHODIMP
1182 CacheStorageService::Notify(nsITimer* aTimer)
1184 if (aTimer == mPurgeTimer) {
1185 mPurgeTimer = nullptr;
1187 nsCOMPtr<nsIRunnable> event =
1188 NS_NewRunnableMethod(this, &CacheStorageService::PurgeOverMemoryLimit);
1189 Dispatch(event);
1192 return NS_OK;
1195 void
1196 CacheStorageService::PurgeOverMemoryLimit()
1198 MOZ_ASSERT(IsOnManagementThread());
1200 LOG(("CacheStorageService::PurgeOverMemoryLimit"));
1202 Pool(true).PurgeOverMemoryLimit();
1203 Pool(false).PurgeOverMemoryLimit();
1206 void
1207 CacheStorageService::MemoryPool::PurgeOverMemoryLimit()
1209 #ifdef PR_LOGGING
1210 TimeStamp start(TimeStamp::Now());
1211 #endif
1213 uint32_t const memoryLimit = Limit();
1214 if (mMemorySize > memoryLimit) {
1215 LOG((" memory data consumption over the limit, abandon expired entries"));
1216 PurgeExpired();
1219 bool frecencyNeedsSort = true;
1221 // No longer makes sense since:
1222 // Memory entries are never purged partially, only as a whole when the memory
1223 // cache limit is overreached.
1224 // Disk entries throw the data away ASAP so that only metadata are kept.
1225 // TODO when this concept of two separate pools is found working, the code should
1226 // clean up.
1227 #if 0
1228 if (mMemorySize > memoryLimit) {
1229 LOG((" memory data consumption over the limit, abandon disk backed data"));
1230 PurgeByFrecency(frecencyNeedsSort, CacheEntry::PURGE_DATA_ONLY_DISK_BACKED);
1233 if (mMemorySize > memoryLimit) {
1234 LOG((" metadata consumtion over the limit, abandon disk backed entries"));
1235 PurgeByFrecency(frecencyNeedsSort, CacheEntry::PURGE_WHOLE_ONLY_DISK_BACKED);
1237 #endif
1239 if (mMemorySize > memoryLimit) {
1240 LOG((" memory data consumption over the limit, abandon any entry"));
1241 PurgeByFrecency(frecencyNeedsSort, CacheEntry::PURGE_WHOLE);
1244 LOG((" purging took %1.2fms", (TimeStamp::Now() - start).ToMilliseconds()));
1247 void
1248 CacheStorageService::MemoryPool::PurgeExpired()
1250 MOZ_ASSERT(IsOnManagementThread());
1252 mExpirationArray.Sort(ExpirationComparator());
1253 uint32_t now = NowInSeconds();
1255 uint32_t const memoryLimit = Limit();
1257 for (uint32_t i = 0; mMemorySize > memoryLimit && i < mExpirationArray.Length();) {
1258 if (CacheIOThread::YieldAndRerun())
1259 return;
1261 nsRefPtr<CacheEntry> entry = mExpirationArray[i];
1263 uint32_t expirationTime = entry->GetExpirationTime();
1264 if (expirationTime > 0 && expirationTime <= now &&
1265 entry->Purge(CacheEntry::PURGE_WHOLE)) {
1266 LOG((" purged expired, entry=%p, exptime=%u (now=%u)",
1267 entry.get(), entry->GetExpirationTime(), now));
1268 continue;
1271 // not purged, move to the next one
1272 ++i;
1276 void
1277 CacheStorageService::MemoryPool::PurgeByFrecency(bool &aFrecencyNeedsSort, uint32_t aWhat)
1279 MOZ_ASSERT(IsOnManagementThread());
1281 if (aFrecencyNeedsSort) {
1282 mFrecencyArray.Sort(FrecencyComparator());
1283 aFrecencyNeedsSort = false;
1286 uint32_t const memoryLimit = Limit();
1288 for (uint32_t i = 0; mMemorySize > memoryLimit && i < mFrecencyArray.Length();) {
1289 if (CacheIOThread::YieldAndRerun())
1290 return;
1292 nsRefPtr<CacheEntry> entry = mFrecencyArray[i];
1294 if (entry->Purge(aWhat)) {
1295 LOG((" abandoned (%d), entry=%p, frecency=%1.10f",
1296 aWhat, entry.get(), entry->GetFrecency()));
1297 continue;
1300 // not purged, move to the next one
1301 ++i;
1305 void
1306 CacheStorageService::MemoryPool::PurgeAll(uint32_t aWhat)
1308 LOG(("CacheStorageService::MemoryPool::PurgeAll aWhat=%d", aWhat));
1309 MOZ_ASSERT(IsOnManagementThread());
1311 for (uint32_t i = 0; i < mFrecencyArray.Length();) {
1312 if (CacheIOThread::YieldAndRerun())
1313 return;
1315 nsRefPtr<CacheEntry> entry = mFrecencyArray[i];
1317 if (entry->Purge(aWhat)) {
1318 LOG((" abandoned entry=%p", entry.get()));
1319 continue;
1322 // not purged, move to the next one
1323 ++i;
1327 // Methods exposed to and used by CacheStorage.
1329 nsresult
1330 CacheStorageService::AddStorageEntry(CacheStorage const* aStorage,
1331 nsIURI* aURI,
1332 const nsACString & aIdExtension,
1333 bool aCreateIfNotExist,
1334 bool aReplace,
1335 CacheEntryHandle** aResult)
1337 NS_ENSURE_FALSE(mShutdown, NS_ERROR_NOT_INITIALIZED);
1339 NS_ENSURE_ARG(aStorage);
1341 nsAutoCString contextKey;
1342 CacheFileUtils::AppendKeyPrefix(aStorage->LoadInfo(), contextKey);
1344 return AddStorageEntry(contextKey, aURI, aIdExtension,
1345 aStorage->WriteToDisk(), aCreateIfNotExist, aReplace,
1346 aResult);
1349 nsresult
1350 CacheStorageService::AddStorageEntry(nsCSubstring const& aContextKey,
1351 nsIURI* aURI,
1352 const nsACString & aIdExtension,
1353 bool aWriteToDisk,
1354 bool aCreateIfNotExist,
1355 bool aReplace,
1356 CacheEntryHandle** aResult)
1358 NS_ENSURE_ARG(aURI);
1360 nsresult rv;
1362 nsAutoCString entryKey;
1363 rv = CacheEntry::HashingKey(EmptyCString(), aIdExtension, aURI, entryKey);
1364 NS_ENSURE_SUCCESS(rv, rv);
1366 LOG(("CacheStorageService::AddStorageEntry [entryKey=%s, contextKey=%s]",
1367 entryKey.get(), aContextKey.BeginReading()));
1369 nsRefPtr<CacheEntry> entry;
1370 nsRefPtr<CacheEntryHandle> handle;
1373 mozilla::MutexAutoLock lock(mLock);
1375 NS_ENSURE_FALSE(mShutdown, NS_ERROR_NOT_INITIALIZED);
1377 // Ensure storage table
1378 CacheEntryTable* entries;
1379 if (!sGlobalEntryTables->Get(aContextKey, &entries)) {
1380 entries = new CacheEntryTable(CacheEntryTable::ALL_ENTRIES);
1381 sGlobalEntryTables->Put(aContextKey, entries);
1382 LOG((" new storage entries table for context '%s'", aContextKey.BeginReading()));
1385 bool entryExists = entries->Get(entryKey, getter_AddRefs(entry));
1387 if (entryExists && !aReplace) {
1388 // check whether the file is already doomed or we want to turn this entry
1389 // to a memory-only.
1390 if (MOZ_UNLIKELY(entry->IsFileDoomed())) {
1391 LOG((" file already doomed, replacing the entry"));
1392 aReplace = true;
1393 } else if (MOZ_UNLIKELY(!aWriteToDisk) && MOZ_LIKELY(entry->IsUsingDisk())) {
1394 LOG((" entry is persistnet but we want mem-only, replacing it"));
1395 aReplace = true;
1399 // If truncate is demanded, delete and doom the current entry
1400 if (entryExists && aReplace) {
1401 entries->Remove(entryKey);
1403 LOG((" dooming entry %p for %s because of OPEN_TRUNCATE", entry.get(), entryKey.get()));
1404 // On purpose called under the lock to prevent races of doom and open on I/O thread
1405 // No need to remove from both memory-only and all-entries tables. The new entry
1406 // will overwrite the shadow entry in its ctor.
1407 entry->DoomAlreadyRemoved();
1409 entry = nullptr;
1410 entryExists = false;
1413 // Ensure entry for the particular URL, if not read/only
1414 if (!entryExists && (aCreateIfNotExist || aReplace)) {
1415 // Entry is not in the hashtable or has just been truncated...
1416 entry = new CacheEntry(aContextKey, aURI, aIdExtension, aWriteToDisk);
1417 entries->Put(entryKey, entry);
1418 LOG((" new entry %p for %s", entry.get(), entryKey.get()));
1421 if (entry) {
1422 // Here, if this entry was not for a long time referenced by any consumer,
1423 // gets again first 'handles count' reference.
1424 handle = entry->NewHandle();
1428 handle.forget(aResult);
1429 return NS_OK;
1432 nsresult
1433 CacheStorageService::CheckStorageEntry(CacheStorage const* aStorage,
1434 nsIURI* aURI,
1435 const nsACString & aIdExtension,
1436 bool* aResult)
1438 nsresult rv;
1440 nsAutoCString contextKey;
1441 CacheFileUtils::AppendKeyPrefix(aStorage->LoadInfo(), contextKey);
1443 if (!aStorage->WriteToDisk()) {
1444 AppendMemoryStorageID(contextKey);
1447 #ifdef PR_LOGGING
1448 nsAutoCString uriSpec;
1449 aURI->GetAsciiSpec(uriSpec);
1450 LOG(("CacheStorageService::CheckStorageEntry [uri=%s, eid=%s, contextKey=%s]",
1451 uriSpec.get(), aIdExtension.BeginReading(), contextKey.get()));
1452 #endif
1455 mozilla::MutexAutoLock lock(mLock);
1457 NS_ENSURE_FALSE(mShutdown, NS_ERROR_NOT_INITIALIZED);
1459 nsAutoCString entryKey;
1460 rv = CacheEntry::HashingKey(EmptyCString(), aIdExtension, aURI, entryKey);
1461 NS_ENSURE_SUCCESS(rv, rv);
1463 CacheEntryTable* entries;
1464 if ((*aResult = sGlobalEntryTables->Get(contextKey, &entries)) &&
1465 entries->GetWeak(entryKey, aResult)) {
1466 LOG((" found in hash tables"));
1467 return NS_OK;
1471 if (!aStorage->WriteToDisk()) {
1472 // Memory entry, nothing more to do.
1473 LOG((" not found in hash tables"));
1474 return NS_OK;
1477 // Disk entry, not found in the hashtable, check the index.
1478 nsAutoCString fileKey;
1479 rv = CacheEntry::HashingKey(contextKey, aIdExtension, aURI, fileKey);
1481 CacheIndex::EntryStatus status;
1482 rv = CacheIndex::HasEntry(fileKey, &status);
1483 if (NS_FAILED(rv) || status == CacheIndex::DO_NOT_KNOW) {
1484 LOG((" index doesn't know, rv=0x%08x", rv));
1485 return NS_ERROR_NOT_AVAILABLE;
1488 *aResult = status == CacheIndex::EXISTS;
1489 LOG((" %sfound in index", *aResult ? "" : "not "));
1490 return NS_OK;
1493 namespace { // anon
1495 class CacheEntryDoomByKeyCallback : public CacheFileIOListener
1496 , public nsIRunnable
1498 public:
1499 NS_DECL_THREADSAFE_ISUPPORTS
1500 NS_DECL_NSIRUNNABLE
1502 explicit CacheEntryDoomByKeyCallback(nsICacheEntryDoomCallback* aCallback)
1503 : mCallback(aCallback) { }
1505 private:
1506 virtual ~CacheEntryDoomByKeyCallback();
1508 NS_IMETHOD OnFileOpened(CacheFileHandle *aHandle, nsresult aResult) MOZ_OVERRIDE { return NS_OK; }
1509 NS_IMETHOD OnDataWritten(CacheFileHandle *aHandle, const char *aBuf, nsresult aResult) MOZ_OVERRIDE { return NS_OK; }
1510 NS_IMETHOD OnDataRead(CacheFileHandle *aHandle, char *aBuf, nsresult aResult) MOZ_OVERRIDE { return NS_OK; }
1511 NS_IMETHOD OnFileDoomed(CacheFileHandle *aHandle, nsresult aResult) MOZ_OVERRIDE;
1512 NS_IMETHOD OnEOFSet(CacheFileHandle *aHandle, nsresult aResult) MOZ_OVERRIDE { return NS_OK; }
1513 NS_IMETHOD OnFileRenamed(CacheFileHandle *aHandle, nsresult aResult) MOZ_OVERRIDE { return NS_OK; }
1515 nsCOMPtr<nsICacheEntryDoomCallback> mCallback;
1516 nsresult mResult;
1519 CacheEntryDoomByKeyCallback::~CacheEntryDoomByKeyCallback()
1521 if (mCallback)
1522 ProxyReleaseMainThread(mCallback);
1525 NS_IMETHODIMP CacheEntryDoomByKeyCallback::OnFileDoomed(CacheFileHandle *aHandle,
1526 nsresult aResult)
1528 if (!mCallback)
1529 return NS_OK;
1531 mResult = aResult;
1532 if (NS_IsMainThread()) {
1533 Run();
1534 } else {
1535 NS_DispatchToMainThread(this);
1538 return NS_OK;
1541 NS_IMETHODIMP CacheEntryDoomByKeyCallback::Run()
1543 mCallback->OnCacheEntryDoomed(mResult);
1544 return NS_OK;
1547 NS_IMPL_ISUPPORTS(CacheEntryDoomByKeyCallback, CacheFileIOListener, nsIRunnable);
1549 } // anon
1551 nsresult
1552 CacheStorageService::DoomStorageEntry(CacheStorage const* aStorage,
1553 nsIURI *aURI,
1554 const nsACString & aIdExtension,
1555 nsICacheEntryDoomCallback* aCallback)
1557 LOG(("CacheStorageService::DoomStorageEntry"));
1559 NS_ENSURE_ARG(aStorage);
1560 NS_ENSURE_ARG(aURI);
1562 nsAutoCString contextKey;
1563 CacheFileUtils::AppendKeyPrefix(aStorage->LoadInfo(), contextKey);
1565 nsAutoCString entryKey;
1566 nsresult rv = CacheEntry::HashingKey(EmptyCString(), aIdExtension, aURI, entryKey);
1567 NS_ENSURE_SUCCESS(rv, rv);
1569 nsRefPtr<CacheEntry> entry;
1571 mozilla::MutexAutoLock lock(mLock);
1573 NS_ENSURE_FALSE(mShutdown, NS_ERROR_NOT_INITIALIZED);
1575 CacheEntryTable* entries;
1576 if (sGlobalEntryTables->Get(contextKey, &entries)) {
1577 if (entries->Get(entryKey, getter_AddRefs(entry))) {
1578 if (aStorage->WriteToDisk() || !entry->IsUsingDisk()) {
1579 // When evicting from disk storage, purge
1580 // When evicting from memory storage and the entry is memory-only, purge
1581 LOG((" purging entry %p for %s [storage use disk=%d, entry use disk=%d]",
1582 entry.get(), entryKey.get(), aStorage->WriteToDisk(), entry->IsUsingDisk()));
1583 entries->Remove(entryKey);
1585 else {
1586 // Otherwise, leave it
1587 LOG((" leaving entry %p for %s [storage use disk=%d, entry use disk=%d]",
1588 entry.get(), entryKey.get(), aStorage->WriteToDisk(), entry->IsUsingDisk()));
1589 entry = nullptr;
1595 if (entry) {
1596 LOG((" dooming entry %p for %s", entry.get(), entryKey.get()));
1597 return entry->AsyncDoom(aCallback);
1600 LOG((" no entry loaded for %s", entryKey.get()));
1602 if (aStorage->WriteToDisk()) {
1603 nsAutoCString contextKey;
1604 CacheFileUtils::AppendKeyPrefix(aStorage->LoadInfo(), contextKey);
1606 rv = CacheEntry::HashingKey(contextKey, aIdExtension, aURI, entryKey);
1607 NS_ENSURE_SUCCESS(rv, rv);
1609 LOG((" dooming file only for %s", entryKey.get()));
1611 nsRefPtr<CacheEntryDoomByKeyCallback> callback(
1612 new CacheEntryDoomByKeyCallback(aCallback));
1613 rv = CacheFileIOManager::DoomFileByKey(entryKey, callback);
1614 NS_ENSURE_SUCCESS(rv, rv);
1616 return NS_OK;
1619 class Callback : public nsRunnable
1621 public:
1622 explicit Callback(nsICacheEntryDoomCallback* aCallback) : mCallback(aCallback) { }
1623 NS_IMETHODIMP Run()
1625 mCallback->OnCacheEntryDoomed(NS_ERROR_NOT_AVAILABLE);
1626 return NS_OK;
1628 nsCOMPtr<nsICacheEntryDoomCallback> mCallback;
1631 if (aCallback) {
1632 nsRefPtr<nsRunnable> callback = new Callback(aCallback);
1633 return NS_DispatchToMainThread(callback);
1636 return NS_OK;
1639 nsresult
1640 CacheStorageService::DoomStorageEntries(CacheStorage const* aStorage,
1641 nsICacheEntryDoomCallback* aCallback)
1643 LOG(("CacheStorageService::DoomStorageEntries"));
1645 NS_ENSURE_FALSE(mShutdown, NS_ERROR_NOT_INITIALIZED);
1646 NS_ENSURE_ARG(aStorage);
1648 nsAutoCString contextKey;
1649 CacheFileUtils::AppendKeyPrefix(aStorage->LoadInfo(), contextKey);
1651 mozilla::MutexAutoLock lock(mLock);
1653 return DoomStorageEntries(contextKey, aStorage->LoadInfo(),
1654 aStorage->WriteToDisk(), aCallback);
1657 nsresult
1658 CacheStorageService::DoomStorageEntries(nsCSubstring const& aContextKey,
1659 nsILoadContextInfo* aContext,
1660 bool aDiskStorage,
1661 nsICacheEntryDoomCallback* aCallback)
1663 mLock.AssertCurrentThreadOwns();
1665 NS_ENSURE_TRUE(!mShutdown, NS_ERROR_NOT_INITIALIZED);
1667 nsAutoCString memoryStorageID(aContextKey);
1668 AppendMemoryStorageID(memoryStorageID);
1670 if (aDiskStorage) {
1671 LOG((" dooming disk+memory storage of %s", aContextKey.BeginReading()));
1673 // Just remove all entries, CacheFileIOManager will take care of the files.
1674 sGlobalEntryTables->Remove(aContextKey);
1675 sGlobalEntryTables->Remove(memoryStorageID);
1677 if (aContext && !aContext->IsPrivate()) {
1678 LOG((" dooming disk entries"));
1679 CacheFileIOManager::EvictByContext(aContext);
1681 } else {
1682 LOG((" dooming memory-only storage of %s", aContextKey.BeginReading()));
1684 class MemoryEntriesRemoval {
1685 public:
1686 static PLDHashOperator EvictEntry(const nsACString& aKey,
1687 CacheEntry* aEntry,
1688 void* aClosure)
1690 CacheEntryTable* entries = static_cast<CacheEntryTable*>(aClosure);
1691 nsCString key(aKey);
1692 RemoveExactEntry(entries, key, aEntry, false);
1693 return PL_DHASH_NEXT;
1697 // Remove the memory entries table from the global tables.
1698 // Since we store memory entries also in the disk entries table
1699 // we need to remove the memory entries from the disk table one
1700 // by one manually.
1701 nsAutoPtr<CacheEntryTable> memoryEntries;
1702 sGlobalEntryTables->RemoveAndForget(memoryStorageID, memoryEntries);
1704 CacheEntryTable* entries;
1705 sGlobalEntryTables->Get(aContextKey, &entries);
1706 if (memoryEntries && entries)
1707 memoryEntries->EnumerateRead(&MemoryEntriesRemoval::EvictEntry, entries);
1710 // An artificial callback. This is a candidate for removal tho. In the new
1711 // cache any 'doom' or 'evict' function ensures that the entry or entries
1712 // being doomed is/are not accessible after the function returns. So there is
1713 // probably no need for a callback - has no meaning. But for compatibility
1714 // with the old cache that is still in the tree we keep the API similar to be
1715 // able to make tests as well as other consumers work for now.
1716 class Callback : public nsRunnable
1718 public:
1719 explicit Callback(nsICacheEntryDoomCallback* aCallback) : mCallback(aCallback) { }
1720 NS_IMETHODIMP Run()
1722 mCallback->OnCacheEntryDoomed(NS_OK);
1723 return NS_OK;
1725 nsCOMPtr<nsICacheEntryDoomCallback> mCallback;
1728 if (aCallback) {
1729 nsRefPtr<nsRunnable> callback = new Callback(aCallback);
1730 return NS_DispatchToMainThread(callback);
1733 return NS_OK;
1736 nsresult
1737 CacheStorageService::WalkStorageEntries(CacheStorage const* aStorage,
1738 bool aVisitEntries,
1739 nsICacheStorageVisitor* aVisitor)
1741 LOG(("CacheStorageService::WalkStorageEntries [cb=%p, visitentries=%d]", aVisitor, aVisitEntries));
1742 NS_ENSURE_FALSE(mShutdown, NS_ERROR_NOT_INITIALIZED);
1744 NS_ENSURE_ARG(aStorage);
1746 if (aStorage->WriteToDisk()) {
1747 nsRefPtr<WalkDiskCacheRunnable> event =
1748 new WalkDiskCacheRunnable(aStorage->LoadInfo(), aVisitEntries, aVisitor);
1749 return event->Walk();
1752 nsRefPtr<WalkMemoryCacheRunnable> event =
1753 new WalkMemoryCacheRunnable(aStorage->LoadInfo(), aVisitEntries, aVisitor);
1754 return event->Walk();
1757 void
1758 CacheStorageService::CacheFileDoomed(nsILoadContextInfo* aLoadContextInfo,
1759 const nsACString & aIdExtension,
1760 const nsACString & aURISpec)
1762 nsAutoCString contextKey;
1763 CacheFileUtils::AppendKeyPrefix(aLoadContextInfo, contextKey);
1765 nsAutoCString entryKey;
1766 CacheEntry::HashingKey(EmptyCString(), aIdExtension, aURISpec, entryKey);
1768 mozilla::MutexAutoLock lock(mLock);
1770 if (mShutdown)
1771 return;
1773 CacheEntryTable* entries;
1774 if (!sGlobalEntryTables->Get(contextKey, &entries))
1775 return;
1777 nsRefPtr<CacheEntry> entry;
1778 if (!entries->Get(entryKey, getter_AddRefs(entry)))
1779 return;
1781 if (!entry->IsFileDoomed())
1782 return;
1784 if (entry->IsReferenced())
1785 return;
1787 // Need to remove under the lock to avoid possible race leading
1788 // to duplication of the entry per its key.
1789 RemoveExactEntry(entries, entryKey, entry, false);
1790 entry->DoomAlreadyRemoved();
1793 bool
1794 CacheStorageService::GetCacheEntryInfo(nsILoadContextInfo* aLoadContextInfo,
1795 const nsACString & aIdExtension,
1796 const nsACString & aURISpec,
1797 EntryInfoCallback *aCallback)
1799 nsAutoCString contextKey;
1800 CacheFileUtils::AppendKeyPrefix(aLoadContextInfo, contextKey);
1802 nsAutoCString entryKey;
1803 CacheEntry::HashingKey(EmptyCString(), aIdExtension, aURISpec, entryKey);
1805 nsRefPtr<CacheEntry> entry;
1807 mozilla::MutexAutoLock lock(mLock);
1809 if (mShutdown) {
1810 return false;
1813 CacheEntryTable* entries;
1814 if (!sGlobalEntryTables->Get(contextKey, &entries)) {
1815 return false;
1818 if (!entries->Get(entryKey, getter_AddRefs(entry))) {
1819 return false;
1823 GetCacheEntryInfo(entry, aCallback);
1824 return true;
1827 // static
1828 void
1829 CacheStorageService::GetCacheEntryInfo(CacheEntry* aEntry,
1830 EntryInfoCallback *aCallback)
1832 nsIURI* uri = aEntry->GetURI();
1833 nsAutoCString uriSpec;
1834 if (uri) {
1835 uri->GetAsciiSpec(uriSpec);
1838 nsCString const enhanceId = aEntry->GetEnhanceID();
1839 uint32_t dataSize;
1840 if (NS_FAILED(aEntry->GetStorageDataSize(&dataSize))) {
1841 dataSize = 0;
1843 int32_t fetchCount;
1844 if (NS_FAILED(aEntry->GetFetchCount(&fetchCount))) {
1845 fetchCount = 0;
1847 uint32_t lastModified;
1848 if (NS_FAILED(aEntry->GetLastModified(&lastModified))) {
1849 lastModified = 0;
1851 uint32_t expirationTime;
1852 if (NS_FAILED(aEntry->GetExpirationTime(&expirationTime))) {
1853 expirationTime = 0;
1856 aCallback->OnEntryInfo(uriSpec, enhanceId, dataSize,
1857 fetchCount, lastModified, expirationTime);
1860 // Telementry collection
1862 namespace { // anon
1864 bool TelemetryEntryKey(CacheEntry const* entry, nsAutoCString& key)
1866 nsAutoCString entryKey;
1867 nsresult rv = entry->HashingKey(entryKey);
1868 if (NS_FAILED(rv))
1869 return false;
1871 if (entry->GetStorageID().IsEmpty()) {
1872 // Hopefully this will be const-copied, saves some memory
1873 key = entryKey;
1874 } else {
1875 key.Assign(entry->GetStorageID());
1876 key.Append(':');
1877 key.Append(entryKey);
1880 return true;
1883 PLDHashOperator PrunePurgeTimeStamps(
1884 const nsACString& aKey, TimeStamp& aTimeStamp, void* aClosure)
1886 TimeStamp* now = static_cast<TimeStamp*>(aClosure);
1887 static TimeDuration const fifteenMinutes = TimeDuration::FromSeconds(900);
1889 if (*now - aTimeStamp > fifteenMinutes) {
1890 // We are not interested in resurrection of entries after 15 minutes
1891 // of time. This is also the limit for the telemetry.
1892 return PL_DHASH_REMOVE;
1895 return PL_DHASH_NEXT;
1898 } // anon
1900 void
1901 CacheStorageService::TelemetryPrune(TimeStamp &now)
1903 static TimeDuration const oneMinute = TimeDuration::FromSeconds(60);
1904 static TimeStamp dontPruneUntil = now + oneMinute;
1905 if (now < dontPruneUntil)
1906 return;
1908 mPurgeTimeStamps.Enumerate(PrunePurgeTimeStamps, &now);
1909 dontPruneUntil = now + oneMinute;
1912 void
1913 CacheStorageService::TelemetryRecordEntryCreation(CacheEntry const* entry)
1915 MOZ_ASSERT(CacheStorageService::IsOnManagementThread());
1917 nsAutoCString key;
1918 if (!TelemetryEntryKey(entry, key))
1919 return;
1921 TimeStamp now = TimeStamp::NowLoRes();
1922 TelemetryPrune(now);
1924 // When an entry is craeted (registered actually) we check if there is
1925 // a timestamp marked when this very same cache entry has been removed
1926 // (deregistered) because of over-memory-limit purging. If there is such
1927 // a timestamp found accumulate telemetry on how long the entry was away.
1928 TimeStamp timeStamp;
1929 if (!mPurgeTimeStamps.Get(key, &timeStamp))
1930 return;
1932 mPurgeTimeStamps.Remove(key);
1934 Telemetry::AccumulateTimeDelta(Telemetry::HTTP_CACHE_ENTRY_RELOAD_TIME,
1935 timeStamp, TimeStamp::NowLoRes());
1938 void
1939 CacheStorageService::TelemetryRecordEntryRemoval(CacheEntry const* entry)
1941 MOZ_ASSERT(CacheStorageService::IsOnManagementThread());
1943 // Doomed entries must not be considered, we are only interested in purged
1944 // entries. Note that the mIsDoomed flag is always set before deregistration
1945 // happens.
1946 if (entry->IsDoomed())
1947 return;
1949 nsAutoCString key;
1950 if (!TelemetryEntryKey(entry, key))
1951 return;
1953 // When an entry is removed (deregistered actually) we put a timestamp for this
1954 // entry to the hashtable so that when the entry is created (registered) again
1955 // we know how long it was away. Also accumulate number of AsyncOpen calls on
1956 // the entry, this tells us how efficiently the pool actually works.
1958 TimeStamp now = TimeStamp::NowLoRes();
1959 TelemetryPrune(now);
1960 mPurgeTimeStamps.Put(key, now);
1962 Telemetry::Accumulate(Telemetry::HTTP_CACHE_ENTRY_REUSE_COUNT, entry->UseCount());
1963 Telemetry::AccumulateTimeDelta(Telemetry::HTTP_CACHE_ENTRY_ALIVE_TIME,
1964 entry->LoadStart(), TimeStamp::NowLoRes());
1967 // nsIMemoryReporter
1969 size_t
1970 CacheStorageService::SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const
1972 CacheStorageService::Self()->Lock().AssertCurrentThreadOwns();
1974 size_t n = 0;
1975 // The elemets are referenced by sGlobalEntryTables and are reported from there
1976 n += Pool(true).mFrecencyArray.SizeOfExcludingThis(mallocSizeOf);
1977 n += Pool(true).mExpirationArray.SizeOfExcludingThis(mallocSizeOf);
1978 n += Pool(false).mFrecencyArray.SizeOfExcludingThis(mallocSizeOf);
1979 n += Pool(false).mExpirationArray.SizeOfExcludingThis(mallocSizeOf);
1980 // Entries reported manually in CacheStorageService::CollectReports callback
1981 if (sGlobalEntryTables) {
1982 n += sGlobalEntryTables->SizeOfIncludingThis(nullptr, mallocSizeOf);
1985 return n;
1988 size_t
1989 CacheStorageService::SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const
1991 return mallocSizeOf(this) + SizeOfExcludingThis(mallocSizeOf);
1994 namespace { // anon
1996 class ReportStorageMemoryData
1998 public:
1999 nsIMemoryReporterCallback *mHandleReport;
2000 nsISupports *mData;
2003 size_t CollectEntryMemory(nsACString const & aKey,
2004 nsRefPtr<mozilla::net::CacheEntry> const & aEntry,
2005 mozilla::MallocSizeOf mallocSizeOf,
2006 void * aClosure)
2008 CacheStorageService::Self()->Lock().AssertCurrentThreadOwns();
2010 CacheEntryTable* aTable = static_cast<CacheEntryTable*>(aClosure);
2012 size_t n = 0;
2013 n += aKey.SizeOfExcludingThisIfUnshared(mallocSizeOf);
2015 // Bypass memory-only entries, those will be reported when iterating
2016 // the memory only table. Memory-only entries are stored in both ALL_ENTRIES
2017 // and MEMORY_ONLY hashtables.
2018 if (aTable->Type() == CacheEntryTable::MEMORY_ONLY || aEntry->IsUsingDisk())
2019 n += aEntry->SizeOfIncludingThis(mallocSizeOf);
2021 return n;
2024 PLDHashOperator ReportStorageMemory(const nsACString& aKey,
2025 CacheEntryTable* aTable,
2026 void* aClosure)
2028 CacheStorageService::Self()->Lock().AssertCurrentThreadOwns();
2030 size_t size = aTable->SizeOfIncludingThis(&CollectEntryMemory,
2031 CacheStorageService::MallocSizeOf,
2032 aTable);
2034 ReportStorageMemoryData& data =
2035 *static_cast<ReportStorageMemoryData*>(aClosure);
2036 // These key names are not privacy-sensitive.
2037 data.mHandleReport->Callback(
2038 EmptyCString(),
2039 nsPrintfCString("explicit/network/cache2/%s-storage(%s)",
2040 aTable->Type() == CacheEntryTable::MEMORY_ONLY ? "memory" : "disk",
2041 aKey.BeginReading()),
2042 nsIMemoryReporter::KIND_HEAP,
2043 nsIMemoryReporter::UNITS_BYTES,
2044 size,
2045 NS_LITERAL_CSTRING("Memory used by the cache storage."),
2046 data.mData);
2048 return PL_DHASH_NEXT;
2051 } // anon
2053 NS_IMETHODIMP
2054 CacheStorageService::CollectReports(nsIMemoryReporterCallback* aHandleReport,
2055 nsISupports* aData, bool aAnonymize)
2057 nsresult rv;
2059 rv = MOZ_COLLECT_REPORT(
2060 "explicit/network/cache2/io", KIND_HEAP, UNITS_BYTES,
2061 CacheFileIOManager::SizeOfIncludingThis(MallocSizeOf),
2062 "Memory used by the cache IO manager.");
2063 if (NS_WARN_IF(NS_FAILED(rv)))
2064 return rv;
2066 rv = MOZ_COLLECT_REPORT(
2067 "explicit/network/cache2/index", KIND_HEAP, UNITS_BYTES,
2068 CacheIndex::SizeOfIncludingThis(MallocSizeOf),
2069 "Memory used by the cache index.");
2070 if (NS_WARN_IF(NS_FAILED(rv)))
2071 return rv;
2073 MutexAutoLock lock(mLock);
2075 // Report the service instance, this doesn't report entries, done lower
2076 rv = MOZ_COLLECT_REPORT(
2077 "explicit/network/cache2/service", KIND_HEAP, UNITS_BYTES,
2078 SizeOfIncludingThis(MallocSizeOf),
2079 "Memory used by the cache storage service.");
2080 if (NS_WARN_IF(NS_FAILED(rv)))
2081 return rv;
2083 // Report all entries, each storage separately (by the context key)
2085 // References are:
2086 // sGlobalEntryTables to N CacheEntryTable
2087 // CacheEntryTable to N CacheEntry
2088 // CacheEntry to 1 CacheFile
2089 // CacheFile to
2090 // N CacheFileChunk (keeping the actual data)
2091 // 1 CacheFileMetadata (keeping http headers etc.)
2092 // 1 CacheFileOutputStream
2093 // N CacheFileInputStream
2094 ReportStorageMemoryData data;
2095 data.mHandleReport = aHandleReport;
2096 data.mData = aData;
2097 sGlobalEntryTables->EnumerateRead(&ReportStorageMemory, &data);
2099 return NS_OK;
2102 } // net
2103 } // mozilla