1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
5 #include "CacheIndex.h"
8 #include "CacheFileIOManager.h"
9 #include "CacheFileMetadata.h"
10 #include "CacheIndexIterator.h"
11 #include "CacheIndexContextIterator.h"
12 #include "nsThreadUtils.h"
13 #include "nsISizeOf.h"
14 #include "nsPrintfCString.h"
15 #include "mozilla/DebugOnly.h"
19 #include "mozilla/AutoRestore.h"
21 #include "mozilla/Telemetry.h"
22 #include "mozilla/Unused.h"
24 #define kMinUnwrittenChanges 300
25 #define kMinDumpInterval 20000 // in milliseconds
26 #define kMaxBufSize 16384
27 #define kIndexVersion 0x00000009
28 #define kUpdateIndexStartDelay 50000 // in milliseconds
29 #define kTelemetryReportBytesLimit (2U * 1024U * 1024U * 1024U) // 2GB
31 #define INDEX_NAME "index"
32 #define TEMP_INDEX_NAME "index.tmp"
33 #define JOURNAL_NAME "index.log"
40 class FrecencyComparator
{
42 bool Equals(CacheIndexRecord
* a
, CacheIndexRecord
* b
) const {
47 return a
->mFrecency
== b
->mFrecency
;
49 bool LessThan(CacheIndexRecord
* a
, CacheIndexRecord
* b
) const {
50 // Removed (=null) entries must be at the end of the array.
58 // Place entries with frecency 0 at the end of the non-removed entries.
59 if (a
->mFrecency
== 0) {
62 if (b
->mFrecency
== 0) {
66 return a
->mFrecency
< b
->mFrecency
;
73 * This helper class is responsible for keeping CacheIndex::mIndexStats and
74 * CacheIndex::mFrecencyArray up to date.
76 class CacheIndexEntryAutoManage
{
78 CacheIndexEntryAutoManage(const SHA1Sum::Hash
* aHash
, CacheIndex
* aIndex
)
82 mDoNotSearchInIndex(false),
83 mDoNotSearchInUpdates(false) {
84 CacheIndex::sLock
.AssertCurrentThreadOwns();
87 const CacheIndexEntry
* entry
= FindEntry();
88 mIndex
->mIndexStats
.BeforeChange(entry
);
89 if (entry
&& entry
->IsInitialized() && !entry
->IsRemoved()) {
90 mOldRecord
= entry
->mRec
.get();
91 mOldFrecency
= entry
->mRec
->mFrecency
;
95 ~CacheIndexEntryAutoManage() {
96 CacheIndex::sLock
.AssertCurrentThreadOwns();
98 const CacheIndexEntry
* entry
= FindEntry();
99 mIndex
->mIndexStats
.AfterChange(entry
);
100 if (!entry
|| !entry
->IsInitialized() || entry
->IsRemoved()) {
104 if (entry
&& !mOldRecord
) {
105 mIndex
->mFrecencyArray
.AppendRecord(entry
->mRec
.get());
106 mIndex
->AddRecordToIterators(entry
->mRec
.get());
107 } else if (!entry
&& mOldRecord
) {
108 mIndex
->mFrecencyArray
.RemoveRecord(mOldRecord
);
109 mIndex
->RemoveRecordFromIterators(mOldRecord
);
110 } else if (entry
&& mOldRecord
) {
111 auto rec
= entry
->mRec
.get();
112 if (rec
!= mOldRecord
) {
113 // record has a different address, we have to replace it
114 mIndex
->ReplaceRecordInIterators(mOldRecord
, rec
);
116 if (entry
->mRec
->mFrecency
== mOldFrecency
) {
117 // If frecency hasn't changed simply replace the pointer
118 mIndex
->mFrecencyArray
.ReplaceRecord(mOldRecord
, rec
);
120 // Remove old pointer and insert the new one at the end of the array
121 mIndex
->mFrecencyArray
.RemoveRecord(mOldRecord
);
122 mIndex
->mFrecencyArray
.AppendRecord(rec
);
124 } else if (entry
->mRec
->mFrecency
!= mOldFrecency
) {
125 // Move the element at the end of the array
126 mIndex
->mFrecencyArray
.RemoveRecord(rec
);
127 mIndex
->mFrecencyArray
.AppendRecord(rec
);
130 // both entries were removed or not initialized, do nothing
134 // We cannot rely on nsTHashtable::GetEntry() in case we are removing entries
135 // while iterating. Destructor is called before the entry is removed. Caller
136 // must call one of following methods to skip lookup in the hashtable.
137 void DoNotSearchInIndex() { mDoNotSearchInIndex
= true; }
138 void DoNotSearchInUpdates() { mDoNotSearchInUpdates
= true; }
141 const CacheIndexEntry
* FindEntry() {
142 const CacheIndexEntry
* entry
= nullptr;
144 switch (mIndex
->mState
) {
145 case CacheIndex::READING
:
146 case CacheIndex::WRITING
:
147 if (!mDoNotSearchInUpdates
) {
148 entry
= mIndex
->mPendingUpdates
.GetEntry(*mHash
);
151 case CacheIndex::BUILDING
:
152 case CacheIndex::UPDATING
:
153 case CacheIndex::READY
:
154 if (!entry
&& !mDoNotSearchInIndex
) {
155 entry
= mIndex
->mIndex
.GetEntry(*mHash
);
158 case CacheIndex::INITIAL
:
159 case CacheIndex::SHUTDOWN
:
161 MOZ_ASSERT(false, "Unexpected state!");
167 const SHA1Sum::Hash
* mHash
;
168 RefPtr
<CacheIndex
> mIndex
;
169 CacheIndexRecord
* mOldRecord
;
170 uint32_t mOldFrecency
;
171 bool mDoNotSearchInIndex
;
172 bool mDoNotSearchInUpdates
;
175 class FileOpenHelper final
: public CacheFileIOListener
{
177 NS_DECL_THREADSAFE_ISUPPORTS
179 explicit FileOpenHelper(CacheIndex
* aIndex
)
180 : mIndex(aIndex
), mCanceled(false) {}
183 CacheIndex::sLock
.AssertCurrentThreadOwns();
188 virtual ~FileOpenHelper() = default;
190 NS_IMETHOD
OnFileOpened(CacheFileHandle
* aHandle
, nsresult aResult
) override
;
191 NS_IMETHOD
OnDataWritten(CacheFileHandle
* aHandle
, const char* aBuf
,
192 nsresult aResult
) override
{
193 MOZ_CRASH("FileOpenHelper::OnDataWritten should not be called!");
194 return NS_ERROR_UNEXPECTED
;
196 NS_IMETHOD
OnDataRead(CacheFileHandle
* aHandle
, char* aBuf
,
197 nsresult aResult
) override
{
198 MOZ_CRASH("FileOpenHelper::OnDataRead should not be called!");
199 return NS_ERROR_UNEXPECTED
;
201 NS_IMETHOD
OnFileDoomed(CacheFileHandle
* aHandle
, nsresult aResult
) override
{
202 MOZ_CRASH("FileOpenHelper::OnFileDoomed should not be called!");
203 return NS_ERROR_UNEXPECTED
;
205 NS_IMETHOD
OnEOFSet(CacheFileHandle
* aHandle
, nsresult aResult
) override
{
206 MOZ_CRASH("FileOpenHelper::OnEOFSet should not be called!");
207 return NS_ERROR_UNEXPECTED
;
209 NS_IMETHOD
OnFileRenamed(CacheFileHandle
* aHandle
,
210 nsresult aResult
) override
{
211 MOZ_CRASH("FileOpenHelper::OnFileRenamed should not be called!");
212 return NS_ERROR_UNEXPECTED
;
215 RefPtr
<CacheIndex
> mIndex
;
219 NS_IMETHODIMP
FileOpenHelper::OnFileOpened(CacheFileHandle
* aHandle
,
221 StaticMutexAutoLock
lock(CacheIndex::sLock
);
225 CacheFileIOManager::DoomFile(aHandle
, nullptr);
231 mIndex
->OnFileOpenedInternal(this, aHandle
, aResult
);
236 NS_IMPL_ISUPPORTS(FileOpenHelper
, CacheFileIOListener
);
238 StaticRefPtr
<CacheIndex
> CacheIndex::gInstance
;
239 StaticMutex
CacheIndex::sLock
;
241 NS_IMPL_ADDREF(CacheIndex
)
242 NS_IMPL_RELEASE(CacheIndex
)
244 NS_INTERFACE_MAP_BEGIN(CacheIndex
)
245 NS_INTERFACE_MAP_ENTRY(mozilla::net::CacheFileIOListener
)
246 NS_INTERFACE_MAP_ENTRY(nsIRunnable
)
249 CacheIndex::CacheIndex()
251 mShuttingDown(false),
252 mIndexNeedsUpdate(false),
254 mIndexOnDiskIsValid(false),
255 mDontMarkIndexClean(false),
257 mUpdateEventPending(false),
264 mJournalReadSuccessfully(false),
265 mAsyncGetDiskConsumptionBlocked(false),
266 mTotalBytesWritten(0) {
267 sLock
.AssertCurrentThreadOwns();
268 LOG(("CacheIndex::CacheIndex [this=%p]", this));
269 MOZ_ASSERT(!gInstance
, "multiple CacheIndex instances!");
272 CacheIndex::~CacheIndex() {
273 sLock
.AssertCurrentThreadOwns();
274 LOG(("CacheIndex::~CacheIndex [this=%p]", this));
280 nsresult
CacheIndex::Init(nsIFile
* aCacheDirectory
) {
281 LOG(("CacheIndex::Init()"));
283 MOZ_ASSERT(NS_IsMainThread());
285 StaticMutexAutoLock
lock(sLock
);
288 return NS_ERROR_ALREADY_INITIALIZED
;
291 RefPtr
<CacheIndex
> idx
= new CacheIndex();
293 nsresult rv
= idx
->InitInternal(aCacheDirectory
);
294 NS_ENSURE_SUCCESS(rv
, rv
);
296 gInstance
= std::move(idx
);
300 nsresult
CacheIndex::InitInternal(nsIFile
* aCacheDirectory
) {
303 rv
= aCacheDirectory
->Clone(getter_AddRefs(mCacheDirectory
));
304 NS_ENSURE_SUCCESS(rv
, rv
);
306 mStartTime
= TimeStamp::NowLoRes();
308 mTotalBytesWritten
= CacheObserver::CacheAmountWritten();
309 mTotalBytesWritten
<<= 10;
317 nsresult
CacheIndex::PreShutdown() {
318 MOZ_ASSERT(NS_IsMainThread());
320 StaticMutexAutoLock
lock(sLock
);
322 LOG(("CacheIndex::PreShutdown() [gInstance=%p]", gInstance
.get()));
325 RefPtr
<CacheIndex
> index
= gInstance
;
328 return NS_ERROR_NOT_INITIALIZED
;
332 ("CacheIndex::PreShutdown() - [state=%d, indexOnDiskIsValid=%d, "
333 "dontMarkIndexClean=%d]",
334 index
->mState
, index
->mIndexOnDiskIsValid
, index
->mDontMarkIndexClean
));
336 LOG(("CacheIndex::PreShutdown() - Closing iterators."));
337 for (uint32_t i
= 0; i
< index
->mIterators
.Length();) {
338 rv
= index
->mIterators
[i
]->CloseInternal(NS_ERROR_FAILURE
);
340 // CacheIndexIterator::CloseInternal() removes itself from mIteratos iff
341 // it returns success.
343 ("CacheIndex::PreShutdown() - Failed to remove iterator %p. "
344 "[rv=0x%08" PRIx32
"]",
345 index
->mIterators
[i
], static_cast<uint32_t>(rv
)));
350 index
->mShuttingDown
= true;
352 if (index
->mState
== READY
) {
353 return NS_OK
; // nothing to do
356 nsCOMPtr
<nsIRunnable
> event
;
357 event
= NewRunnableMethod("net::CacheIndex::PreShutdownInternal", index
,
358 &CacheIndex::PreShutdownInternal
);
360 nsCOMPtr
<nsIEventTarget
> ioTarget
= CacheFileIOManager::IOTarget();
361 MOZ_ASSERT(ioTarget
);
363 // PreShutdownInternal() will be executed before any queued event on INDEX
364 // level. That's OK since we don't want to wait for any operation in progess.
365 rv
= ioTarget
->Dispatch(event
, nsIEventTarget::DISPATCH_NORMAL
);
367 NS_WARNING("CacheIndex::PreShutdown() - Can't dispatch event");
368 LOG(("CacheIndex::PreShutdown() - Can't dispatch event"));
375 void CacheIndex::PreShutdownInternal() {
376 StaticMutexAutoLock
lock(sLock
);
379 ("CacheIndex::PreShutdownInternal() - [state=%d, indexOnDiskIsValid=%d, "
380 "dontMarkIndexClean=%d]",
381 mState
, mIndexOnDiskIsValid
, mDontMarkIndexClean
));
383 MOZ_ASSERT(mShuttingDown
);
386 mUpdateTimer
->Cancel();
387 mUpdateTimer
= nullptr;
395 // nothing to do, write the journal in Shutdown()
405 MOZ_ASSERT(false, "Implement me!");
408 // We should end up in READY state
409 MOZ_ASSERT(mState
== READY
);
413 nsresult
CacheIndex::Shutdown() {
414 MOZ_ASSERT(NS_IsMainThread());
416 StaticMutexAutoLock
lock(sLock
);
418 LOG(("CacheIndex::Shutdown() [gInstance=%p]", gInstance
.get()));
420 RefPtr
<CacheIndex
> index
= gInstance
.forget();
423 return NS_ERROR_NOT_INITIALIZED
;
426 bool sanitize
= CacheObserver::ClearCacheOnShutdown();
428 CacheObserver::SetCacheAmountWritten(index
->mTotalBytesWritten
>> 10);
431 ("CacheIndex::Shutdown() - [state=%d, indexOnDiskIsValid=%d, "
432 "dontMarkIndexClean=%d, sanitize=%d]",
433 index
->mState
, index
->mIndexOnDiskIsValid
, index
->mDontMarkIndexClean
,
436 MOZ_ASSERT(index
->mShuttingDown
);
438 EState oldState
= index
->mState
;
439 index
->ChangeState(SHUTDOWN
);
441 if (oldState
!= READY
) {
443 ("CacheIndex::Shutdown() - Unexpected state. Did posting of "
444 "PreShutdownInternal() fail?"));
449 index
->FinishWrite(false);
452 if (index
->mIndexOnDiskIsValid
&& !index
->mDontMarkIndexClean
) {
453 if (!sanitize
&& NS_FAILED(index
->WriteLogToDisk())) {
454 index
->RemoveJournalAndTempFile();
457 index
->RemoveJournalAndTempFile();
461 index
->FinishRead(false);
465 index
->FinishUpdate(false);
468 MOZ_ASSERT(false, "Unexpected state!");
472 index
->RemoveAllIndexFiles();
479 nsresult
CacheIndex::AddEntry(const SHA1Sum::Hash
* aHash
) {
480 LOG(("CacheIndex::AddEntry() [hash=%08x%08x%08x%08x%08x]", LOGSHA1(aHash
)));
482 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
484 StaticMutexAutoLock
lock(sLock
);
486 RefPtr
<CacheIndex
> index
= gInstance
;
489 return NS_ERROR_NOT_INITIALIZED
;
492 if (!index
->IsIndexUsable()) {
493 return NS_ERROR_NOT_AVAILABLE
;
496 // Getters in CacheIndexStats assert when mStateLogged is true since the
497 // information is incomplete between calls to BeforeChange() and AfterChange()
498 // (i.e. while CacheIndexEntryAutoManage exists). We need to check whether
499 // non-fresh entries exists outside the scope of CacheIndexEntryAutoManage.
500 bool updateIfNonFreshEntriesExist
= false;
503 CacheIndexEntryAutoManage
entryMng(aHash
, index
);
505 CacheIndexEntry
* entry
= index
->mIndex
.GetEntry(*aHash
);
506 bool entryRemoved
= entry
&& entry
->IsRemoved();
507 CacheIndexEntryUpdate
* updated
= nullptr;
509 if (index
->mState
== READY
|| index
->mState
== UPDATING
||
510 index
->mState
== BUILDING
) {
511 MOZ_ASSERT(index
->mPendingUpdates
.Count() == 0);
513 if (entry
&& !entryRemoved
) {
514 // Found entry in index that shouldn't exist.
516 if (entry
->IsFresh()) {
517 // Someone removed the file on disk while FF is running. Update
518 // process can fix only non-fresh entries (i.e. entries that were not
519 // added within this session). Start update only if we have such
522 // TODO: This should be very rare problem. If it turns out not to be
523 // true, change the update process so that it also iterates all
524 // initialized non-empty entries and checks whether the file exists.
527 ("CacheIndex::AddEntry() - Cache file was removed outside FF "
530 updateIfNonFreshEntriesExist
= true;
531 } else if (index
->mState
== READY
) {
532 // Index is outdated, update it.
534 ("CacheIndex::AddEntry() - Found entry that shouldn't exist, "
535 "update is needed"));
536 index
->mIndexNeedsUpdate
= true;
538 // We cannot be here when building index since all entries are fresh
540 MOZ_ASSERT(index
->mState
== UPDATING
);
545 entry
= index
->mIndex
.PutEntry(*aHash
);
547 } else { // WRITING, READING
548 updated
= index
->mPendingUpdates
.GetEntry(*aHash
);
549 bool updatedRemoved
= updated
&& updated
->IsRemoved();
551 if ((updated
&& !updatedRemoved
) ||
552 (!updated
&& entry
&& !entryRemoved
&& entry
->IsFresh())) {
553 // Fresh entry found, so the file was removed outside FF
555 ("CacheIndex::AddEntry() - Cache file was removed outside FF "
558 updateIfNonFreshEntriesExist
= true;
559 } else if (!updated
&& entry
&& !entryRemoved
) {
560 if (index
->mState
== WRITING
) {
562 ("CacheIndex::AddEntry() - Found entry that shouldn't exist, "
563 "update is needed"));
564 index
->mIndexNeedsUpdate
= true;
566 // Ignore if state is READING since the index information is partial
569 updated
= index
->mPendingUpdates
.PutEntry(*aHash
);
574 updated
->MarkDirty();
575 updated
->MarkFresh();
583 if (updateIfNonFreshEntriesExist
&&
584 index
->mIndexStats
.Count() != index
->mIndexStats
.Fresh()) {
585 index
->mIndexNeedsUpdate
= true;
588 index
->StartUpdatingIndexIfNeeded();
589 index
->WriteIndexToDiskIfNeeded();
595 nsresult
CacheIndex::EnsureEntryExists(const SHA1Sum::Hash
* aHash
) {
596 LOG(("CacheIndex::EnsureEntryExists() [hash=%08x%08x%08x%08x%08x]",
599 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
601 StaticMutexAutoLock
lock(sLock
);
603 RefPtr
<CacheIndex
> index
= gInstance
;
606 return NS_ERROR_NOT_INITIALIZED
;
609 if (!index
->IsIndexUsable()) {
610 return NS_ERROR_NOT_AVAILABLE
;
614 CacheIndexEntryAutoManage
entryMng(aHash
, index
);
616 CacheIndexEntry
* entry
= index
->mIndex
.GetEntry(*aHash
);
617 bool entryRemoved
= entry
&& entry
->IsRemoved();
619 if (index
->mState
== READY
|| index
->mState
== UPDATING
||
620 index
->mState
== BUILDING
) {
621 MOZ_ASSERT(index
->mPendingUpdates
.Count() == 0);
623 if (!entry
|| entryRemoved
) {
624 if (entryRemoved
&& entry
->IsFresh()) {
625 // This could happen only if somebody copies files to the entries
626 // directory while FF is running.
628 ("CacheIndex::EnsureEntryExists() - Cache file was added outside "
629 "FF process! Update is needed."));
630 index
->mIndexNeedsUpdate
= true;
631 } else if (index
->mState
== READY
||
632 (entryRemoved
&& !entry
->IsFresh())) {
633 // Removed non-fresh entries can be present as a result of
636 ("CacheIndex::EnsureEntryExists() - Didn't find entry that should"
637 " exist, update is needed"));
638 index
->mIndexNeedsUpdate
= true;
642 entry
= index
->mIndex
.PutEntry(*aHash
);
648 } else { // WRITING, READING
649 CacheIndexEntryUpdate
* updated
= index
->mPendingUpdates
.GetEntry(*aHash
);
650 bool updatedRemoved
= updated
&& updated
->IsRemoved();
652 if (updatedRemoved
|| (!updated
&& entryRemoved
&& entry
->IsFresh())) {
653 // Fresh information about missing entry found. This could happen only
654 // if somebody copies files to the entries directory while FF is
657 ("CacheIndex::EnsureEntryExists() - Cache file was added outside "
658 "FF process! Update is needed."));
659 index
->mIndexNeedsUpdate
= true;
660 } else if (!updated
&& (!entry
|| entryRemoved
)) {
661 if (index
->mState
== WRITING
) {
663 ("CacheIndex::EnsureEntryExists() - Didn't find entry that should"
664 " exist, update is needed"));
665 index
->mIndexNeedsUpdate
= true;
667 // Ignore if state is READING since the index information is partial
670 // We don't need entryRemoved and updatedRemoved info anymore
671 if (entryRemoved
) entry
= nullptr;
672 if (updatedRemoved
) updated
= nullptr;
675 updated
->MarkFresh();
678 // Create a new entry
679 updated
= index
->mPendingUpdates
.PutEntry(*aHash
);
681 updated
->MarkFresh();
682 updated
->MarkDirty();
684 if (!entry
->IsFresh()) {
685 // To mark the entry fresh we must make a copy of index entry
686 // since the index is read-only.
687 updated
= index
->mPendingUpdates
.PutEntry(*aHash
);
689 updated
->MarkFresh();
696 index
->StartUpdatingIndexIfNeeded();
697 index
->WriteIndexToDiskIfNeeded();
703 nsresult
CacheIndex::InitEntry(const SHA1Sum::Hash
* aHash
,
704 OriginAttrsHash aOriginAttrsHash
,
705 bool aAnonymous
, bool aPinned
) {
707 ("CacheIndex::InitEntry() [hash=%08x%08x%08x%08x%08x, "
708 "originAttrsHash=%" PRIx64
", anonymous=%d, pinned=%d]",
709 LOGSHA1(aHash
), aOriginAttrsHash
, aAnonymous
, aPinned
));
711 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
713 StaticMutexAutoLock
lock(sLock
);
715 RefPtr
<CacheIndex
> index
= gInstance
;
718 return NS_ERROR_NOT_INITIALIZED
;
721 if (!index
->IsIndexUsable()) {
722 return NS_ERROR_NOT_AVAILABLE
;
726 CacheIndexEntryAutoManage
entryMng(aHash
, index
);
728 CacheIndexEntry
* entry
= index
->mIndex
.GetEntry(*aHash
);
729 CacheIndexEntryUpdate
* updated
= nullptr;
730 bool reinitEntry
= false;
732 if (entry
&& entry
->IsRemoved()) {
736 if (index
->mState
== READY
|| index
->mState
== UPDATING
||
737 index
->mState
== BUILDING
) {
738 MOZ_ASSERT(index
->mPendingUpdates
.Count() == 0);
740 MOZ_ASSERT(entry
->IsFresh());
743 LOG(("CacheIndex::InitEntry() - Entry was not found in mIndex!"));
745 ("CacheIndex::InitEntry() - Entry was not found in mIndex!"));
746 return NS_ERROR_UNEXPECTED
;
749 if (IsCollision(entry
, aOriginAttrsHash
, aAnonymous
)) {
750 index
->mIndexNeedsUpdate
=
751 true; // TODO Does this really help in case of collision?
754 if (entry
->IsInitialized()) {
759 updated
= index
->mPendingUpdates
.GetEntry(*aHash
);
760 DebugOnly
<bool> removed
= updated
&& updated
->IsRemoved();
762 MOZ_ASSERT(updated
|| !removed
);
763 MOZ_ASSERT(updated
|| entry
);
765 if (!updated
&& !entry
) {
767 ("CacheIndex::InitEntry() - Entry was found neither in mIndex nor "
768 "in mPendingUpdates!"));
770 ("CacheIndex::InitEntry() - Entry was found neither in "
771 "mIndex nor in mPendingUpdates!"));
772 return NS_ERROR_UNEXPECTED
;
776 MOZ_ASSERT(updated
->IsFresh());
778 if (IsCollision(updated
, aOriginAttrsHash
, aAnonymous
)) {
779 index
->mIndexNeedsUpdate
= true;
782 if (updated
->IsInitialized()) {
787 MOZ_ASSERT(entry
->IsFresh());
789 if (IsCollision(entry
, aOriginAttrsHash
, aAnonymous
)) {
790 index
->mIndexNeedsUpdate
= true;
793 if (entry
->IsInitialized()) {
798 // make a copy of a read-only entry
799 updated
= index
->mPendingUpdates
.PutEntry(*aHash
);
805 // There is a collision and we are going to rewrite this entry. Initialize
806 // it as a new entry.
809 updated
->MarkFresh();
817 updated
->Init(aOriginAttrsHash
, aAnonymous
, aPinned
);
818 updated
->MarkDirty();
820 entry
->Init(aOriginAttrsHash
, aAnonymous
, aPinned
);
825 index
->StartUpdatingIndexIfNeeded();
826 index
->WriteIndexToDiskIfNeeded();
832 nsresult
CacheIndex::RemoveEntry(const SHA1Sum::Hash
* aHash
) {
833 LOG(("CacheIndex::RemoveEntry() [hash=%08x%08x%08x%08x%08x]",
836 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
838 StaticMutexAutoLock
lock(sLock
);
840 RefPtr
<CacheIndex
> index
= gInstance
;
843 return NS_ERROR_NOT_INITIALIZED
;
846 if (!index
->IsIndexUsable()) {
847 return NS_ERROR_NOT_AVAILABLE
;
851 CacheIndexEntryAutoManage
entryMng(aHash
, index
);
853 CacheIndexEntry
* entry
= index
->mIndex
.GetEntry(*aHash
);
854 bool entryRemoved
= entry
&& entry
->IsRemoved();
856 if (index
->mState
== READY
|| index
->mState
== UPDATING
||
857 index
->mState
== BUILDING
) {
858 MOZ_ASSERT(index
->mPendingUpdates
.Count() == 0);
860 if (!entry
|| entryRemoved
) {
861 if (entryRemoved
&& entry
->IsFresh()) {
862 // This could happen only if somebody copies files to the entries
863 // directory while FF is running.
865 ("CacheIndex::RemoveEntry() - Cache file was added outside FF "
866 "process! Update is needed."));
867 index
->mIndexNeedsUpdate
= true;
868 } else if (index
->mState
== READY
||
869 (entryRemoved
&& !entry
->IsFresh())) {
870 // Removed non-fresh entries can be present as a result of
873 ("CacheIndex::RemoveEntry() - Didn't find entry that should exist"
874 ", update is needed"));
875 index
->mIndexNeedsUpdate
= true;
879 if (!entry
->IsDirty() && entry
->IsFileEmpty()) {
880 index
->mIndex
.RemoveEntry(entry
);
883 entry
->MarkRemoved();
889 } else { // WRITING, READING
890 CacheIndexEntryUpdate
* updated
= index
->mPendingUpdates
.GetEntry(*aHash
);
891 bool updatedRemoved
= updated
&& updated
->IsRemoved();
893 if (updatedRemoved
|| (!updated
&& entryRemoved
&& entry
->IsFresh())) {
894 // Fresh information about missing entry found. This could happen only
895 // if somebody copies files to the entries directory while FF is
898 ("CacheIndex::RemoveEntry() - Cache file was added outside FF "
899 "process! Update is needed."));
900 index
->mIndexNeedsUpdate
= true;
901 } else if (!updated
&& (!entry
|| entryRemoved
)) {
902 if (index
->mState
== WRITING
) {
904 ("CacheIndex::RemoveEntry() - Didn't find entry that should exist"
905 ", update is needed"));
906 index
->mIndexNeedsUpdate
= true;
908 // Ignore if state is READING since the index information is partial
912 updated
= index
->mPendingUpdates
.PutEntry(*aHash
);
916 updated
->MarkRemoved();
917 updated
->MarkDirty();
918 updated
->MarkFresh();
922 index
->StartUpdatingIndexIfNeeded();
923 index
->WriteIndexToDiskIfNeeded();
929 nsresult
CacheIndex::UpdateEntry(const SHA1Sum::Hash
* aHash
,
930 const uint32_t* aFrecency
,
931 const bool* aHasAltData
,
932 const uint16_t* aOnStartTime
,
933 const uint16_t* aOnStopTime
,
934 const uint8_t* aContentType
,
935 const uint32_t* aSize
) {
937 ("CacheIndex::UpdateEntry() [hash=%08x%08x%08x%08x%08x, "
938 "frecency=%s, hasAltData=%s, onStartTime=%s, onStopTime=%s, "
939 "contentType=%s, size=%s]",
940 LOGSHA1(aHash
), aFrecency
? nsPrintfCString("%u", *aFrecency
).get() : "",
941 aHasAltData
? (*aHasAltData
? "true" : "false") : "",
942 aOnStartTime
? nsPrintfCString("%u", *aOnStartTime
).get() : "",
943 aOnStopTime
? nsPrintfCString("%u", *aOnStopTime
).get() : "",
944 aContentType
? nsPrintfCString("%u", *aContentType
).get() : "",
945 aSize
? nsPrintfCString("%u", *aSize
).get() : ""));
947 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
949 StaticMutexAutoLock
lock(sLock
);
951 RefPtr
<CacheIndex
> index
= gInstance
;
954 return NS_ERROR_NOT_INITIALIZED
;
957 if (!index
->IsIndexUsable()) {
958 return NS_ERROR_NOT_AVAILABLE
;
962 CacheIndexEntryAutoManage
entryMng(aHash
, index
);
964 CacheIndexEntry
* entry
= index
->mIndex
.GetEntry(*aHash
);
966 if (entry
&& entry
->IsRemoved()) {
970 if (index
->mState
== READY
|| index
->mState
== UPDATING
||
971 index
->mState
== BUILDING
) {
972 MOZ_ASSERT(index
->mPendingUpdates
.Count() == 0);
976 LOG(("CacheIndex::UpdateEntry() - Entry was not found in mIndex!"));
978 ("CacheIndex::UpdateEntry() - Entry was not found in mIndex!"));
979 return NS_ERROR_UNEXPECTED
;
982 if (!HasEntryChanged(entry
, aFrecency
, aHasAltData
, aOnStartTime
,
983 aOnStopTime
, aContentType
, aSize
)) {
987 MOZ_ASSERT(entry
->IsFresh());
988 MOZ_ASSERT(entry
->IsInitialized());
992 entry
->SetFrecency(*aFrecency
);
996 entry
->SetHasAltData(*aHasAltData
);
1000 entry
->SetOnStartTime(*aOnStartTime
);
1004 entry
->SetOnStopTime(*aOnStopTime
);
1008 entry
->SetContentType(*aContentType
);
1012 entry
->SetFileSize(*aSize
);
1015 CacheIndexEntryUpdate
* updated
= index
->mPendingUpdates
.GetEntry(*aHash
);
1016 DebugOnly
<bool> removed
= updated
&& updated
->IsRemoved();
1018 MOZ_ASSERT(updated
|| !removed
);
1019 MOZ_ASSERT(updated
|| entry
);
1024 ("CacheIndex::UpdateEntry() - Entry was found neither in mIndex "
1025 "nor in mPendingUpdates!"));
1027 ("CacheIndex::UpdateEntry() - Entry was found neither in "
1028 "mIndex nor in mPendingUpdates!"));
1029 return NS_ERROR_UNEXPECTED
;
1032 // make a copy of a read-only entry
1033 updated
= index
->mPendingUpdates
.PutEntry(*aHash
);
1037 MOZ_ASSERT(updated
->IsFresh());
1038 MOZ_ASSERT(updated
->IsInitialized());
1039 updated
->MarkDirty();
1042 updated
->SetFrecency(*aFrecency
);
1046 updated
->SetHasAltData(*aHasAltData
);
1050 updated
->SetOnStartTime(*aOnStartTime
);
1054 updated
->SetOnStopTime(*aOnStopTime
);
1058 updated
->SetContentType(*aContentType
);
1062 updated
->SetFileSize(*aSize
);
1067 index
->WriteIndexToDiskIfNeeded();
1073 nsresult
CacheIndex::RemoveAll() {
1074 LOG(("CacheIndex::RemoveAll()"));
1076 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
1078 nsCOMPtr
<nsIFile
> file
;
1081 StaticMutexAutoLock
lock(sLock
);
1083 RefPtr
<CacheIndex
> index
= gInstance
;
1086 return NS_ERROR_NOT_INITIALIZED
;
1089 MOZ_ASSERT(!index
->mRemovingAll
);
1091 if (!index
->IsIndexUsable()) {
1092 return NS_ERROR_NOT_AVAILABLE
;
1095 AutoRestore
<bool> saveRemovingAll(index
->mRemovingAll
);
1096 index
->mRemovingAll
= true;
1098 // Doom index and journal handles but don't null them out since this will be
1099 // done in FinishWrite/FinishRead methods.
1100 if (index
->mIndexHandle
) {
1101 CacheFileIOManager::DoomFile(index
->mIndexHandle
, nullptr);
1103 // We don't have a handle to index file, so get the file here, but delete
1104 // it outside the lock. Ignore the result since this is not fatal.
1105 index
->GetFile(nsLiteralCString(INDEX_NAME
), getter_AddRefs(file
));
1108 if (index
->mJournalHandle
) {
1109 CacheFileIOManager::DoomFile(index
->mJournalHandle
, nullptr);
1112 switch (index
->mState
) {
1114 index
->FinishWrite(false);
1120 index
->FinishRead(false);
1124 index
->FinishUpdate(false);
1127 MOZ_ASSERT(false, "Unexpected state!");
1130 // We should end up in READY state
1131 MOZ_ASSERT(index
->mState
== READY
);
1133 // There should not be any handle
1134 MOZ_ASSERT(!index
->mIndexHandle
);
1135 MOZ_ASSERT(!index
->mJournalHandle
);
1137 index
->mIndexOnDiskIsValid
= false;
1138 index
->mIndexNeedsUpdate
= false;
1140 index
->mIndexStats
.Clear();
1141 index
->mFrecencyArray
.Clear();
1142 index
->mIndex
.Clear();
1144 for (uint32_t i
= 0; i
< index
->mIterators
.Length();) {
1145 nsresult rv
= index
->mIterators
[i
]->CloseInternal(NS_ERROR_NOT_AVAILABLE
);
1146 if (NS_FAILED(rv
)) {
1147 // CacheIndexIterator::CloseInternal() removes itself from mIterators
1148 // iff it returns success.
1150 ("CacheIndex::RemoveAll() - Failed to remove iterator %p. "
1151 "[rv=0x%08" PRIx32
"]",
1152 index
->mIterators
[i
], static_cast<uint32_t>(rv
)));
1159 // Ignore the result. The file might not exist and the failure is not fatal.
1160 file
->Remove(false);
1167 nsresult
CacheIndex::HasEntry(
1168 const nsACString
& aKey
, EntryStatus
* _retval
,
1169 const std::function
<void(const CacheIndexEntry
*)>& aCB
) {
1170 LOG(("CacheIndex::HasEntry() [key=%s]", PromiseFlatCString(aKey
).get()));
1174 sum
.update(aKey
.BeginReading(), aKey
.Length());
1177 return HasEntry(hash
, _retval
, aCB
);
1181 nsresult
CacheIndex::HasEntry(
1182 const SHA1Sum::Hash
& hash
, EntryStatus
* _retval
,
1183 const std::function
<void(const CacheIndexEntry
*)>& aCB
) {
1184 StaticMutexAutoLock
lock(sLock
);
1186 RefPtr
<CacheIndex
> index
= gInstance
;
1189 return NS_ERROR_NOT_INITIALIZED
;
1192 if (!index
->IsIndexUsable()) {
1193 return NS_ERROR_NOT_AVAILABLE
;
1196 const CacheIndexEntry
* entry
= nullptr;
1198 switch (index
->mState
) {
1201 entry
= index
->mPendingUpdates
.GetEntry(hash
);
1207 entry
= index
->mIndex
.GetEntry(hash
);
1212 MOZ_ASSERT(false, "Unexpected state!");
1216 if (index
->mState
== READY
|| index
->mState
== WRITING
) {
1217 *_retval
= DOES_NOT_EXIST
;
1219 *_retval
= DO_NOT_KNOW
;
1222 if (entry
->IsRemoved()) {
1223 if (entry
->IsFresh()) {
1224 *_retval
= DOES_NOT_EXIST
;
1226 *_retval
= DO_NOT_KNOW
;
1236 LOG(("CacheIndex::HasEntry() - result is %u", *_retval
));
1241 nsresult
CacheIndex::GetEntryForEviction(bool aIgnoreEmptyEntries
,
1242 SHA1Sum::Hash
* aHash
, uint32_t* aCnt
) {
1243 LOG(("CacheIndex::GetEntryForEviction()"));
1245 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
1247 StaticMutexAutoLock
lock(sLock
);
1249 RefPtr
<CacheIndex
> index
= gInstance
;
1251 if (!index
) return NS_ERROR_NOT_INITIALIZED
;
1253 if (!index
->IsIndexUsable()) {
1254 return NS_ERROR_NOT_AVAILABLE
;
1257 if (index
->mIndexStats
.Size() == 0) {
1258 return NS_ERROR_NOT_AVAILABLE
;
1261 int32_t mediaUsage
=
1262 round(static_cast<double>(index
->mIndexStats
.SizeByType(
1263 nsICacheEntry::CONTENT_TYPE_MEDIA
)) *
1264 100.0 / static_cast<double>(index
->mIndexStats
.Size()));
1265 int32_t mediaUsageLimit
=
1266 StaticPrefs::browser_cache_disk_content_type_media_limit();
1267 bool evictMedia
= false;
1268 if (mediaUsage
> mediaUsageLimit
) {
1270 ("CacheIndex::GetEntryForEviction() - media content type is over the "
1271 "limit [mediaUsage=%d, mediaUsageLimit=%d]",
1272 mediaUsage
, mediaUsageLimit
));
1277 CacheIndexRecord
* foundRecord
= nullptr;
1278 uint32_t skipped
= 0;
1280 // find first non-forced valid and unpinned entry with the lowest frecency
1281 index
->mFrecencyArray
.SortIfNeeded();
1283 for (auto iter
= index
->mFrecencyArray
.Iter(); !iter
.Done(); iter
.Next()) {
1284 CacheIndexRecord
* rec
= iter
.Get();
1286 memcpy(&hash
, rec
->mHash
, sizeof(SHA1Sum::Hash
));
1290 if (evictMedia
&& CacheIndexEntry::GetContentType(rec
) !=
1291 nsICacheEntry::CONTENT_TYPE_MEDIA
) {
1295 if (IsForcedValidEntry(&hash
)) {
1299 if (CacheIndexEntry::IsPinned(rec
)) {
1303 if (aIgnoreEmptyEntries
&& !CacheIndexEntry::GetFileSize(*rec
)) {
1312 if (!foundRecord
) return NS_ERROR_NOT_AVAILABLE
;
1317 ("CacheIndex::GetEntryForEviction() - returning entry "
1318 "[hash=%08x%08x%08x%08x%08x, cnt=%u, frecency=%u, contentType=%u]",
1319 LOGSHA1(&hash
), *aCnt
, foundRecord
->mFrecency
,
1320 CacheIndexEntry::GetContentType(foundRecord
)));
1322 memcpy(aHash
, &hash
, sizeof(SHA1Sum::Hash
));
1328 bool CacheIndex::IsForcedValidEntry(const SHA1Sum::Hash
* aHash
) {
1329 RefPtr
<CacheFileHandle
> handle
;
1331 CacheFileIOManager::gInstance
->mHandles
.GetHandle(aHash
,
1332 getter_AddRefs(handle
));
1334 if (!handle
) return false;
1336 nsCString hashKey
= handle
->Key();
1337 return CacheStorageService::Self()->IsForcedValidEntry(hashKey
);
1341 nsresult
CacheIndex::GetCacheSize(uint32_t* _retval
) {
1342 LOG(("CacheIndex::GetCacheSize()"));
1344 StaticMutexAutoLock
lock(sLock
);
1346 RefPtr
<CacheIndex
> index
= gInstance
;
1348 if (!index
) return NS_ERROR_NOT_INITIALIZED
;
1350 if (!index
->IsIndexUsable()) {
1351 return NS_ERROR_NOT_AVAILABLE
;
1354 *_retval
= index
->mIndexStats
.Size();
1355 LOG(("CacheIndex::GetCacheSize() - returning %u", *_retval
));
1360 nsresult
CacheIndex::GetEntryFileCount(uint32_t* _retval
) {
1361 LOG(("CacheIndex::GetEntryFileCount()"));
1363 StaticMutexAutoLock
lock(sLock
);
1365 RefPtr
<CacheIndex
> index
= gInstance
;
1368 return NS_ERROR_NOT_INITIALIZED
;
1371 if (!index
->IsIndexUsable()) {
1372 return NS_ERROR_NOT_AVAILABLE
;
1375 *_retval
= index
->mIndexStats
.ActiveEntriesCount();
1376 LOG(("CacheIndex::GetEntryFileCount() - returning %u", *_retval
));
1381 nsresult
CacheIndex::GetCacheStats(nsILoadContextInfo
* aInfo
, uint32_t* aSize
,
1383 LOG(("CacheIndex::GetCacheStats() [info=%p]", aInfo
));
1385 StaticMutexAutoLock
lock(sLock
);
1387 RefPtr
<CacheIndex
> index
= gInstance
;
1390 return NS_ERROR_NOT_INITIALIZED
;
1393 if (!index
->IsIndexUsable()) {
1394 return NS_ERROR_NOT_AVAILABLE
;
1400 for (auto iter
= index
->mFrecencyArray
.Iter(); !iter
.Done(); iter
.Next()) {
1401 CacheIndexRecord
* record
= iter
.Get();
1402 if (aInfo
&& !CacheIndexEntry::RecordMatchesLoadContextInfo(record
, aInfo
))
1405 *aSize
+= CacheIndexEntry::GetFileSize(*record
);
1413 nsresult
CacheIndex::AsyncGetDiskConsumption(
1414 nsICacheStorageConsumptionObserver
* aObserver
) {
1415 LOG(("CacheIndex::AsyncGetDiskConsumption()"));
1417 StaticMutexAutoLock
lock(sLock
);
1419 RefPtr
<CacheIndex
> index
= gInstance
;
1422 return NS_ERROR_NOT_INITIALIZED
;
1425 if (!index
->IsIndexUsable()) {
1426 return NS_ERROR_NOT_AVAILABLE
;
1429 RefPtr
<DiskConsumptionObserver
> observer
=
1430 DiskConsumptionObserver::Init(aObserver
);
1432 NS_ENSURE_ARG(observer
);
1434 if ((index
->mState
== READY
|| index
->mState
== WRITING
) &&
1435 !index
->mAsyncGetDiskConsumptionBlocked
) {
1436 LOG(("CacheIndex::AsyncGetDiskConsumption - calling immediately"));
1437 // Safe to call the callback under the lock,
1438 // we always post to the main thread.
1439 observer
->OnDiskConsumption(index
->mIndexStats
.Size() << 10);
1443 LOG(("CacheIndex::AsyncGetDiskConsumption - remembering callback"));
1444 // Will be called when the index get to the READY state.
1445 index
->mDiskConsumptionObservers
.AppendElement(observer
);
1447 // Move forward with index re/building if it is pending
1448 RefPtr
<CacheIOThread
> ioThread
= CacheFileIOManager::IOThread();
1451 NS_NewRunnableFunction("net::CacheIndex::AsyncGetDiskConsumption",
1453 StaticMutexAutoLock
lock(sLock
);
1455 RefPtr
<CacheIndex
> index
= gInstance
;
1456 if (index
&& index
->mUpdateTimer
) {
1457 index
->mUpdateTimer
->Cancel();
1458 index
->DelayedUpdateLocked();
1461 CacheIOThread::INDEX
);
1468 nsresult
CacheIndex::GetIterator(nsILoadContextInfo
* aInfo
, bool aAddNew
,
1469 CacheIndexIterator
** _retval
) {
1470 LOG(("CacheIndex::GetIterator() [info=%p, addNew=%d]", aInfo
, aAddNew
));
1472 StaticMutexAutoLock
lock(sLock
);
1474 RefPtr
<CacheIndex
> index
= gInstance
;
1477 return NS_ERROR_NOT_INITIALIZED
;
1480 if (!index
->IsIndexUsable()) {
1481 return NS_ERROR_NOT_AVAILABLE
;
1484 RefPtr
<CacheIndexIterator
> idxIter
;
1486 idxIter
= new CacheIndexContextIterator(index
, aAddNew
, aInfo
);
1488 idxIter
= new CacheIndexIterator(index
, aAddNew
);
1491 index
->mFrecencyArray
.SortIfNeeded();
1493 for (auto iter
= index
->mFrecencyArray
.Iter(); !iter
.Done(); iter
.Next()) {
1494 idxIter
->AddRecord(iter
.Get());
1497 index
->mIterators
.AppendElement(idxIter
);
1498 idxIter
.swap(*_retval
);
1503 nsresult
CacheIndex::IsUpToDate(bool* _retval
) {
1504 LOG(("CacheIndex::IsUpToDate()"));
1506 StaticMutexAutoLock
lock(sLock
);
1508 RefPtr
<CacheIndex
> index
= gInstance
;
1511 return NS_ERROR_NOT_INITIALIZED
;
1514 if (!index
->IsIndexUsable()) {
1515 return NS_ERROR_NOT_AVAILABLE
;
1518 *_retval
= (index
->mState
== READY
|| index
->mState
== WRITING
) &&
1519 !index
->mIndexNeedsUpdate
&& !index
->mShuttingDown
;
1521 LOG(("CacheIndex::IsUpToDate() - returning %d", *_retval
));
1525 bool CacheIndex::IsIndexUsable() {
1526 MOZ_ASSERT(mState
!= INITIAL
);
1545 bool CacheIndex::IsCollision(CacheIndexEntry
* aEntry
,
1546 OriginAttrsHash aOriginAttrsHash
,
1548 if (!aEntry
->IsInitialized()) {
1552 if (aEntry
->Anonymous() != aAnonymous
||
1553 aEntry
->OriginAttrsHash() != aOriginAttrsHash
) {
1555 ("CacheIndex::IsCollision() - Collision detected for entry hash=%08x"
1556 "%08x%08x%08x%08x, expected values: originAttrsHash=%" PRIu64
", "
1557 "anonymous=%d; actual values: originAttrsHash=%" PRIu64
1559 LOGSHA1(aEntry
->Hash()), aOriginAttrsHash
, aAnonymous
,
1560 aEntry
->OriginAttrsHash(), aEntry
->Anonymous()));
1568 bool CacheIndex::HasEntryChanged(
1569 CacheIndexEntry
* aEntry
, const uint32_t* aFrecency
, const bool* aHasAltData
,
1570 const uint16_t* aOnStartTime
, const uint16_t* aOnStopTime
,
1571 const uint8_t* aContentType
, const uint32_t* aSize
) {
1572 if (aFrecency
&& *aFrecency
!= aEntry
->GetFrecency()) {
1576 if (aHasAltData
&& *aHasAltData
!= aEntry
->GetHasAltData()) {
1580 if (aOnStartTime
&& *aOnStartTime
!= aEntry
->GetOnStartTime()) {
1584 if (aOnStopTime
&& *aOnStopTime
!= aEntry
->GetOnStopTime()) {
1588 if (aContentType
&& *aContentType
!= aEntry
->GetContentType()) {
1593 (*aSize
& CacheIndexEntry::kFileSizeMask
) != aEntry
->GetFileSize()) {
1600 void CacheIndex::ProcessPendingOperations() {
1601 LOG(("CacheIndex::ProcessPendingOperations()"));
1603 sLock
.AssertCurrentThreadOwns();
1605 for (auto iter
= mPendingUpdates
.Iter(); !iter
.Done(); iter
.Next()) {
1606 CacheIndexEntryUpdate
* update
= iter
.Get();
1608 LOG(("CacheIndex::ProcessPendingOperations() [hash=%08x%08x%08x%08x%08x]",
1609 LOGSHA1(update
->Hash())));
1611 MOZ_ASSERT(update
->IsFresh());
1613 CacheIndexEntry
* entry
= mIndex
.GetEntry(*update
->Hash());
1616 CacheIndexEntryAutoManage
emng(update
->Hash(), this);
1617 emng
.DoNotSearchInUpdates();
1619 if (update
->IsRemoved()) {
1621 if (entry
->IsRemoved()) {
1622 MOZ_ASSERT(entry
->IsFresh());
1623 MOZ_ASSERT(entry
->IsDirty());
1624 } else if (!entry
->IsDirty() && entry
->IsFileEmpty()) {
1625 // Entries with empty file are not stored in index on disk. Just
1626 // remove the entry, but only in case the entry is not dirty, i.e.
1627 // the entry file was empty when we wrote the index.
1628 mIndex
.RemoveEntry(entry
);
1631 entry
->MarkRemoved();
1637 // Some information in mIndex can be newer than in mPendingUpdates (see
1638 // bug 1074832). This will copy just those values that were really
1640 update
->ApplyUpdate(entry
);
1642 // There is no entry in mIndex, copy all information from
1643 // mPendingUpdates to mIndex.
1644 entry
= mIndex
.PutEntry(*update
->Hash());
1652 MOZ_ASSERT(mPendingUpdates
.Count() == 0);
1654 EnsureCorrectStats();
1657 bool CacheIndex::WriteIndexToDiskIfNeeded() {
1658 if (mState
!= READY
|| mShuttingDown
|| mRWPending
) {
1662 if (!mLastDumpTime
.IsNull() &&
1663 (TimeStamp::NowLoRes() - mLastDumpTime
).ToMilliseconds() <
1668 if (mIndexStats
.Dirty() < kMinUnwrittenChanges
) {
1676 void CacheIndex::WriteIndexToDisk() {
1677 LOG(("CacheIndex::WriteIndexToDisk()"));
1682 sLock
.AssertCurrentThreadOwns();
1683 MOZ_ASSERT(mState
== READY
);
1684 MOZ_ASSERT(!mRWBuf
);
1685 MOZ_ASSERT(!mRWHash
);
1686 MOZ_ASSERT(!mRWPending
);
1688 ChangeState(WRITING
);
1690 mProcessEntries
= mIndexStats
.ActiveEntriesCount();
1692 mIndexFileOpener
= new FileOpenHelper(this);
1693 rv
= CacheFileIOManager::OpenFile(
1694 nsLiteralCString(TEMP_INDEX_NAME
),
1695 CacheFileIOManager::SPECIAL_FILE
| CacheFileIOManager::CREATE
,
1697 if (NS_FAILED(rv
)) {
1698 LOG(("CacheIndex::WriteIndexToDisk() - Can't open file [rv=0x%08" PRIx32
1700 static_cast<uint32_t>(rv
)));
1705 // Write index header to a buffer, it will be written to disk together with
1706 // records in WriteRecords() once we open the file successfully.
1708 mRWHash
= new CacheHash();
1712 NetworkEndian::writeUint32(mRWBuf
+ mRWBufPos
, kIndexVersion
);
1713 mRWBufPos
+= sizeof(uint32_t);
1715 NetworkEndian::writeUint32(mRWBuf
+ mRWBufPos
,
1716 static_cast<uint32_t>(PR_Now() / PR_USEC_PER_SEC
));
1717 mRWBufPos
+= sizeof(uint32_t);
1719 NetworkEndian::writeUint32(mRWBuf
+ mRWBufPos
, 1);
1720 mRWBufPos
+= sizeof(uint32_t);
1725 void CacheIndex::WriteRecords() {
1726 LOG(("CacheIndex::WriteRecords()"));
1730 sLock
.AssertCurrentThreadOwns();
1731 MOZ_ASSERT(mState
== WRITING
);
1732 MOZ_ASSERT(!mRWPending
);
1737 MOZ_ASSERT(mRWBufPos
== 0);
1738 fileOffset
= sizeof(CacheIndexHeader
);
1739 fileOffset
+= sizeof(CacheIndexRecord
) * mSkipEntries
;
1741 MOZ_ASSERT(mRWBufPos
== sizeof(CacheIndexHeader
));
1744 uint32_t hashOffset
= mRWBufPos
;
1746 char* buf
= mRWBuf
+ mRWBufPos
;
1747 uint32_t skip
= mSkipEntries
;
1748 uint32_t processMax
= (mRWBufSize
- mRWBufPos
) / sizeof(CacheIndexRecord
);
1749 MOZ_ASSERT(processMax
!= 0 ||
1751 0); // TODO make sure we can write an empty index
1752 uint32_t processed
= 0;
1754 bool hasMore
= false;
1756 for (auto iter
= mIndex
.Iter(); !iter
.Done(); iter
.Next()) {
1757 CacheIndexEntry
* entry
= iter
.Get();
1758 if (entry
->IsRemoved() || !entry
->IsInitialized() || entry
->IsFileEmpty()) {
1767 if (processed
== processMax
) {
1774 entry
->WriteToBuf(buf
);
1775 buf
+= sizeof(CacheIndexRecord
);
1779 MOZ_ASSERT(mRWBufPos
!= static_cast<uint32_t>(buf
- mRWBuf
) ||
1780 mProcessEntries
== 0);
1781 mRWBufPos
= buf
- mRWBuf
;
1782 mSkipEntries
+= processed
;
1783 MOZ_ASSERT(mSkipEntries
<= mProcessEntries
);
1785 mRWHash
->Update(mRWBuf
+ hashOffset
, mRWBufPos
- hashOffset
);
1787 if (mSkipEntries
== mProcessEntries
) {
1788 MOZ_ASSERT(!hasMore
);
1790 // We've processed all records
1791 if (mRWBufPos
+ sizeof(CacheHash::Hash32_t
) > mRWBufSize
) {
1792 // realloc buffer to spare another write cycle
1793 mRWBufSize
= mRWBufPos
+ sizeof(CacheHash::Hash32_t
);
1794 mRWBuf
= static_cast<char*>(moz_xrealloc(mRWBuf
, mRWBufSize
));
1797 NetworkEndian::writeUint32(mRWBuf
+ mRWBufPos
, mRWHash
->GetHash());
1798 mRWBufPos
+= sizeof(CacheHash::Hash32_t
);
1800 MOZ_ASSERT(hasMore
);
1803 rv
= CacheFileIOManager::Write(mIndexHandle
, fileOffset
, mRWBuf
, mRWBufPos
,
1804 mSkipEntries
== mProcessEntries
, false, this);
1805 if (NS_FAILED(rv
)) {
1807 ("CacheIndex::WriteRecords() - CacheFileIOManager::Write() failed "
1808 "synchronously [rv=0x%08" PRIx32
"]",
1809 static_cast<uint32_t>(rv
)));
1818 void CacheIndex::FinishWrite(bool aSucceeded
) {
1819 LOG(("CacheIndex::FinishWrite() [succeeded=%d]", aSucceeded
));
1821 MOZ_ASSERT((!aSucceeded
&& mState
== SHUTDOWN
) || mState
== WRITING
);
1823 sLock
.AssertCurrentThreadOwns();
1825 // If there is write operation pending we must be cancelling writing of the
1826 // index when shutting down or removing the whole index.
1827 MOZ_ASSERT(!mRWPending
|| (!aSucceeded
&& (mShuttingDown
|| mRemovingAll
)));
1829 mIndexHandle
= nullptr;
1834 // Opening of the file must not be in progress if writing succeeded.
1835 MOZ_ASSERT(!mIndexFileOpener
);
1837 for (auto iter
= mIndex
.Iter(); !iter
.Done(); iter
.Next()) {
1838 CacheIndexEntry
* entry
= iter
.Get();
1840 bool remove
= false;
1842 CacheIndexEntryAutoManage
emng(entry
->Hash(), this);
1844 if (entry
->IsRemoved()) {
1845 emng
.DoNotSearchInIndex();
1847 } else if (entry
->IsDirty()) {
1848 entry
->ClearDirty();
1856 mIndexOnDiskIsValid
= true;
1858 if (mIndexFileOpener
) {
1859 // If opening of the file is still in progress (e.g. WRITE process was
1860 // canceled by RemoveAll()) then we need to cancel the opener to make sure
1861 // that OnFileOpenedInternal() won't be called.
1862 mIndexFileOpener
->Cancel();
1863 mIndexFileOpener
= nullptr;
1867 ProcessPendingOperations();
1870 if (mState
== WRITING
) {
1872 mLastDumpTime
= TimeStamp::NowLoRes();
1876 nsresult
CacheIndex::GetFile(const nsACString
& aName
, nsIFile
** _retval
) {
1879 nsCOMPtr
<nsIFile
> file
;
1880 rv
= mCacheDirectory
->Clone(getter_AddRefs(file
));
1881 NS_ENSURE_SUCCESS(rv
, rv
);
1883 rv
= file
->AppendNative(aName
);
1884 NS_ENSURE_SUCCESS(rv
, rv
);
1886 file
.swap(*_retval
);
1890 void CacheIndex::RemoveFile(const nsACString
& aName
) {
1891 MOZ_ASSERT(mState
== SHUTDOWN
);
1895 nsCOMPtr
<nsIFile
> file
;
1896 rv
= GetFile(aName
, getter_AddRefs(file
));
1897 NS_ENSURE_SUCCESS_VOID(rv
);
1899 rv
= file
->Remove(false);
1900 if (NS_FAILED(rv
) && rv
!= NS_ERROR_FILE_NOT_FOUND
) {
1902 ("CacheIndex::RemoveFile() - Cannot remove old entry file from disk "
1903 "[rv=0x%08" PRIx32
", name=%s]",
1904 static_cast<uint32_t>(rv
), PromiseFlatCString(aName
).get()));
1908 void CacheIndex::RemoveAllIndexFiles() {
1909 LOG(("CacheIndex::RemoveAllIndexFiles()"));
1910 RemoveFile(nsLiteralCString(INDEX_NAME
));
1911 RemoveJournalAndTempFile();
1914 void CacheIndex::RemoveJournalAndTempFile() {
1915 LOG(("CacheIndex::RemoveJournalAndTempFile()"));
1916 RemoveFile(nsLiteralCString(TEMP_INDEX_NAME
));
1917 RemoveFile(nsLiteralCString(JOURNAL_NAME
));
1920 class WriteLogHelper
{
1922 explicit WriteLogHelper(PRFileDesc
* aFD
)
1923 : mFD(aFD
), mBufSize(kMaxBufSize
), mBufPos(0) {
1924 mHash
= new CacheHash();
1925 mBuf
= static_cast<char*>(moz_xmalloc(mBufSize
));
1928 ~WriteLogHelper() { free(mBuf
); }
1930 nsresult
AddEntry(CacheIndexEntry
* aEntry
);
1934 nsresult
FlushBuffer();
1940 RefPtr
<CacheHash
> mHash
;
1943 nsresult
WriteLogHelper::AddEntry(CacheIndexEntry
* aEntry
) {
1946 if (mBufPos
+ sizeof(CacheIndexRecord
) > mBufSize
) {
1947 mHash
->Update(mBuf
, mBufPos
);
1950 NS_ENSURE_SUCCESS(rv
, rv
);
1951 MOZ_ASSERT(mBufPos
+ sizeof(CacheIndexRecord
) <= mBufSize
);
1954 aEntry
->WriteToBuf(mBuf
+ mBufPos
);
1955 mBufPos
+= sizeof(CacheIndexRecord
);
1960 nsresult
WriteLogHelper::Finish() {
1963 mHash
->Update(mBuf
, mBufPos
);
1964 if (mBufPos
+ sizeof(CacheHash::Hash32_t
) > mBufSize
) {
1966 NS_ENSURE_SUCCESS(rv
, rv
);
1967 MOZ_ASSERT(mBufPos
+ sizeof(CacheHash::Hash32_t
) <= mBufSize
);
1970 NetworkEndian::writeUint32(mBuf
+ mBufPos
, mHash
->GetHash());
1971 mBufPos
+= sizeof(CacheHash::Hash32_t
);
1974 NS_ENSURE_SUCCESS(rv
, rv
);
1979 nsresult
WriteLogHelper::FlushBuffer() {
1980 if (CacheObserver::IsPastShutdownIOLag()) {
1981 LOG(("WriteLogHelper::FlushBuffer() - Interrupting writing journal."));
1982 return NS_ERROR_FAILURE
;
1985 int32_t bytesWritten
= PR_Write(mFD
, mBuf
, mBufPos
);
1987 if (bytesWritten
!= mBufPos
) {
1988 return NS_ERROR_FAILURE
;
1995 nsresult
CacheIndex::WriteLogToDisk() {
1996 LOG(("CacheIndex::WriteLogToDisk()"));
2000 MOZ_ASSERT(mPendingUpdates
.Count() == 0);
2001 MOZ_ASSERT(mState
== SHUTDOWN
);
2003 if (CacheObserver::IsPastShutdownIOLag()) {
2004 LOG(("CacheIndex::WriteLogToDisk() - Skipping writing journal."));
2005 return NS_ERROR_FAILURE
;
2008 RemoveFile(nsLiteralCString(TEMP_INDEX_NAME
));
2010 nsCOMPtr
<nsIFile
> indexFile
;
2011 rv
= GetFile(nsLiteralCString(INDEX_NAME
), getter_AddRefs(indexFile
));
2012 NS_ENSURE_SUCCESS(rv
, rv
);
2014 nsCOMPtr
<nsIFile
> logFile
;
2015 rv
= GetFile(nsLiteralCString(JOURNAL_NAME
), getter_AddRefs(logFile
));
2016 NS_ENSURE_SUCCESS(rv
, rv
);
2020 PRFileDesc
* fd
= nullptr;
2021 rv
= logFile
->OpenNSPRFileDesc(PR_RDWR
| PR_CREATE_FILE
| PR_TRUNCATE
, 0600,
2023 NS_ENSURE_SUCCESS(rv
, rv
);
2025 WriteLogHelper
wlh(fd
);
2026 for (auto iter
= mIndex
.Iter(); !iter
.Done(); iter
.Next()) {
2027 CacheIndexEntry
* entry
= iter
.Get();
2028 if (entry
->IsRemoved() || entry
->IsDirty()) {
2029 rv
= wlh
.AddEntry(entry
);
2030 if (NS_WARN_IF(NS_FAILED(rv
))) {
2038 NS_ENSURE_SUCCESS(rv
, rv
);
2040 rv
= indexFile
->OpenNSPRFileDesc(PR_RDWR
, 0600, &fd
);
2041 NS_ENSURE_SUCCESS(rv
, rv
);
2043 // Seek to dirty flag in the index header and clear it.
2044 static_assert(2 * sizeof(uint32_t) == offsetof(CacheIndexHeader
, mIsDirty
),
2045 "Unexpected offset of CacheIndexHeader::mIsDirty");
2046 int64_t offset
= PR_Seek64(fd
, 2 * sizeof(uint32_t), PR_SEEK_SET
);
2049 return NS_ERROR_FAILURE
;
2052 uint32_t isDirty
= 0;
2053 int32_t bytesWritten
= PR_Write(fd
, &isDirty
, sizeof(isDirty
));
2055 if (bytesWritten
!= sizeof(isDirty
)) {
2056 return NS_ERROR_FAILURE
;
2062 void CacheIndex::ReadIndexFromDisk() {
2063 LOG(("CacheIndex::ReadIndexFromDisk()"));
2067 sLock
.AssertCurrentThreadOwns();
2068 MOZ_ASSERT(mState
== INITIAL
);
2070 ChangeState(READING
);
2072 mIndexFileOpener
= new FileOpenHelper(this);
2073 rv
= CacheFileIOManager::OpenFile(
2074 nsLiteralCString(INDEX_NAME
),
2075 CacheFileIOManager::SPECIAL_FILE
| CacheFileIOManager::OPEN
,
2077 if (NS_FAILED(rv
)) {
2079 ("CacheIndex::ReadIndexFromDisk() - CacheFileIOManager::OpenFile() "
2080 "failed [rv=0x%08" PRIx32
", file=%s]",
2081 static_cast<uint32_t>(rv
), INDEX_NAME
));
2086 mJournalFileOpener
= new FileOpenHelper(this);
2087 rv
= CacheFileIOManager::OpenFile(
2088 nsLiteralCString(JOURNAL_NAME
),
2089 CacheFileIOManager::SPECIAL_FILE
| CacheFileIOManager::OPEN
,
2090 mJournalFileOpener
);
2091 if (NS_FAILED(rv
)) {
2093 ("CacheIndex::ReadIndexFromDisk() - CacheFileIOManager::OpenFile() "
2094 "failed [rv=0x%08" PRIx32
", file=%s]",
2095 static_cast<uint32_t>(rv
), JOURNAL_NAME
));
2099 mTmpFileOpener
= new FileOpenHelper(this);
2100 rv
= CacheFileIOManager::OpenFile(
2101 nsLiteralCString(TEMP_INDEX_NAME
),
2102 CacheFileIOManager::SPECIAL_FILE
| CacheFileIOManager::OPEN
,
2104 if (NS_FAILED(rv
)) {
2106 ("CacheIndex::ReadIndexFromDisk() - CacheFileIOManager::OpenFile() "
2107 "failed [rv=0x%08" PRIx32
", file=%s]",
2108 static_cast<uint32_t>(rv
), TEMP_INDEX_NAME
));
2113 void CacheIndex::StartReadingIndex() {
2114 LOG(("CacheIndex::StartReadingIndex()"));
2118 sLock
.AssertCurrentThreadOwns();
2120 MOZ_ASSERT(mIndexHandle
);
2121 MOZ_ASSERT(mState
== READING
);
2122 MOZ_ASSERT(!mIndexOnDiskIsValid
);
2123 MOZ_ASSERT(!mDontMarkIndexClean
);
2124 MOZ_ASSERT(!mJournalReadSuccessfully
);
2125 MOZ_ASSERT(mIndexHandle
->FileSize() >= 0);
2126 MOZ_ASSERT(!mRWPending
);
2128 int64_t entriesSize
= mIndexHandle
->FileSize() - sizeof(CacheIndexHeader
) -
2129 sizeof(CacheHash::Hash32_t
);
2131 if (entriesSize
< 0 || entriesSize
% sizeof(CacheIndexRecord
)) {
2132 LOG(("CacheIndex::StartReadingIndex() - Index is corrupted"));
2139 mRWHash
= new CacheHash();
2142 std::min(mRWBufSize
, static_cast<uint32_t>(mIndexHandle
->FileSize()));
2144 rv
= CacheFileIOManager::Read(mIndexHandle
, 0, mRWBuf
, mRWBufPos
, this);
2145 if (NS_FAILED(rv
)) {
2147 ("CacheIndex::StartReadingIndex() - CacheFileIOManager::Read() failed "
2148 "synchronously [rv=0x%08" PRIx32
"]",
2149 static_cast<uint32_t>(rv
)));
2156 void CacheIndex::ParseRecords() {
2157 LOG(("CacheIndex::ParseRecords()"));
2161 sLock
.AssertCurrentThreadOwns();
2163 MOZ_ASSERT(!mRWPending
);
2165 uint32_t entryCnt
= (mIndexHandle
->FileSize() - sizeof(CacheIndexHeader
) -
2166 sizeof(CacheHash::Hash32_t
)) /
2167 sizeof(CacheIndexRecord
);
2170 if (!mSkipEntries
) {
2171 if (NetworkEndian::readUint32(mRWBuf
+ pos
) != kIndexVersion
) {
2175 pos
+= sizeof(uint32_t);
2177 mIndexTimeStamp
= NetworkEndian::readUint32(mRWBuf
+ pos
);
2178 pos
+= sizeof(uint32_t);
2180 if (NetworkEndian::readUint32(mRWBuf
+ pos
)) {
2181 if (mJournalHandle
) {
2182 CacheFileIOManager::DoomFile(mJournalHandle
, nullptr);
2183 mJournalHandle
= nullptr;
2187 reinterpret_cast<uint32_t*>(moz_xmalloc(sizeof(uint32_t)));
2188 NetworkEndian::writeUint32(isDirty
, 1);
2190 // Mark index dirty. The buffer is freed by CacheFileIOManager when
2191 // nullptr is passed as the listener and the call doesn't fail
2193 rv
= CacheFileIOManager::Write(mIndexHandle
, 2 * sizeof(uint32_t),
2194 reinterpret_cast<char*>(isDirty
),
2195 sizeof(uint32_t), true, false, nullptr);
2196 if (NS_FAILED(rv
)) {
2197 // This is not fatal, just free the memory
2201 pos
+= sizeof(uint32_t);
2204 uint32_t hashOffset
= pos
;
2206 while (pos
+ sizeof(CacheIndexRecord
) <= mRWBufPos
&&
2207 mSkipEntries
!= entryCnt
) {
2208 CacheIndexRecord
* rec
= reinterpret_cast<CacheIndexRecord
*>(mRWBuf
+ pos
);
2209 CacheIndexEntry
tmpEntry(&rec
->mHash
);
2210 tmpEntry
.ReadFromBuf(mRWBuf
+ pos
);
2212 if (tmpEntry
.IsDirty() || !tmpEntry
.IsInitialized() ||
2213 tmpEntry
.IsFileEmpty() || tmpEntry
.IsFresh() || tmpEntry
.IsRemoved()) {
2215 ("CacheIndex::ParseRecords() - Invalid entry found in index, removing"
2216 " whole index [dirty=%d, initialized=%d, fileEmpty=%d, fresh=%d, "
2218 tmpEntry
.IsDirty(), tmpEntry
.IsInitialized(), tmpEntry
.IsFileEmpty(),
2219 tmpEntry
.IsFresh(), tmpEntry
.IsRemoved()));
2224 CacheIndexEntryAutoManage
emng(tmpEntry
.Hash(), this);
2226 CacheIndexEntry
* entry
= mIndex
.PutEntry(*tmpEntry
.Hash());
2229 pos
+= sizeof(CacheIndexRecord
);
2233 mRWHash
->Update(mRWBuf
+ hashOffset
, pos
- hashOffset
);
2235 if (pos
!= mRWBufPos
) {
2236 memmove(mRWBuf
, mRWBuf
+ pos
, mRWBufPos
- pos
);
2242 int64_t fileOffset
= sizeof(CacheIndexHeader
) +
2243 mSkipEntries
* sizeof(CacheIndexRecord
) + mRWBufPos
;
2245 MOZ_ASSERT(fileOffset
<= mIndexHandle
->FileSize());
2246 if (fileOffset
== mIndexHandle
->FileSize()) {
2247 uint32_t expectedHash
= NetworkEndian::readUint32(mRWBuf
);
2248 if (mRWHash
->GetHash() != expectedHash
) {
2249 LOG(("CacheIndex::ParseRecords() - Hash mismatch, [is %x, should be %x]",
2250 mRWHash
->GetHash(), expectedHash
));
2255 mIndexOnDiskIsValid
= true;
2256 mJournalReadSuccessfully
= false;
2258 if (mJournalHandle
) {
2259 StartReadingJournal();
2269 std::min(mRWBufSize
- pos
,
2270 static_cast<uint32_t>(mIndexHandle
->FileSize() - fileOffset
));
2271 mRWBufPos
= pos
+ toRead
;
2273 rv
= CacheFileIOManager::Read(mIndexHandle
, fileOffset
, mRWBuf
+ pos
, toRead
,
2275 if (NS_FAILED(rv
)) {
2277 ("CacheIndex::ParseRecords() - CacheFileIOManager::Read() failed "
2278 "synchronously [rv=0x%08" PRIx32
"]",
2279 static_cast<uint32_t>(rv
)));
2286 void CacheIndex::StartReadingJournal() {
2287 LOG(("CacheIndex::StartReadingJournal()"));
2291 sLock
.AssertCurrentThreadOwns();
2293 MOZ_ASSERT(mJournalHandle
);
2294 MOZ_ASSERT(mIndexOnDiskIsValid
);
2295 MOZ_ASSERT(mTmpJournal
.Count() == 0);
2296 MOZ_ASSERT(mJournalHandle
->FileSize() >= 0);
2297 MOZ_ASSERT(!mRWPending
);
2299 int64_t entriesSize
=
2300 mJournalHandle
->FileSize() - sizeof(CacheHash::Hash32_t
);
2302 if (entriesSize
< 0 || entriesSize
% sizeof(CacheIndexRecord
)) {
2303 LOG(("CacheIndex::StartReadingJournal() - Journal is corrupted"));
2309 mRWHash
= new CacheHash();
2312 std::min(mRWBufSize
, static_cast<uint32_t>(mJournalHandle
->FileSize()));
2314 rv
= CacheFileIOManager::Read(mJournalHandle
, 0, mRWBuf
, mRWBufPos
, this);
2315 if (NS_FAILED(rv
)) {
2317 ("CacheIndex::StartReadingJournal() - CacheFileIOManager::Read() failed"
2318 " synchronously [rv=0x%08" PRIx32
"]",
2319 static_cast<uint32_t>(rv
)));
2326 void CacheIndex::ParseJournal() {
2327 LOG(("CacheIndex::ParseJournal()"));
2331 sLock
.AssertCurrentThreadOwns();
2333 MOZ_ASSERT(!mRWPending
);
2336 (mJournalHandle
->FileSize() - sizeof(CacheHash::Hash32_t
)) /
2337 sizeof(CacheIndexRecord
);
2341 while (pos
+ sizeof(CacheIndexRecord
) <= mRWBufPos
&&
2342 mSkipEntries
!= entryCnt
) {
2343 CacheIndexEntry
tmpEntry(reinterpret_cast<SHA1Sum::Hash
*>(mRWBuf
+ pos
));
2344 tmpEntry
.ReadFromBuf(mRWBuf
+ pos
);
2346 CacheIndexEntry
* entry
= mTmpJournal
.PutEntry(*tmpEntry
.Hash());
2349 if (entry
->IsDirty() || entry
->IsFresh()) {
2351 ("CacheIndex::ParseJournal() - Invalid entry found in journal, "
2352 "ignoring whole journal [dirty=%d, fresh=%d]",
2353 entry
->IsDirty(), entry
->IsFresh()));
2358 pos
+= sizeof(CacheIndexRecord
);
2362 mRWHash
->Update(mRWBuf
, pos
);
2364 if (pos
!= mRWBufPos
) {
2365 memmove(mRWBuf
, mRWBuf
+ pos
, mRWBufPos
- pos
);
2371 int64_t fileOffset
= mSkipEntries
* sizeof(CacheIndexRecord
) + mRWBufPos
;
2373 MOZ_ASSERT(fileOffset
<= mJournalHandle
->FileSize());
2374 if (fileOffset
== mJournalHandle
->FileSize()) {
2375 uint32_t expectedHash
= NetworkEndian::readUint32(mRWBuf
);
2376 if (mRWHash
->GetHash() != expectedHash
) {
2377 LOG(("CacheIndex::ParseJournal() - Hash mismatch, [is %x, should be %x]",
2378 mRWHash
->GetHash(), expectedHash
));
2383 mJournalReadSuccessfully
= true;
2390 std::min(mRWBufSize
- pos
,
2391 static_cast<uint32_t>(mJournalHandle
->FileSize() - fileOffset
));
2392 mRWBufPos
= pos
+ toRead
;
2394 rv
= CacheFileIOManager::Read(mJournalHandle
, fileOffset
, mRWBuf
+ pos
,
2396 if (NS_FAILED(rv
)) {
2398 ("CacheIndex::ParseJournal() - CacheFileIOManager::Read() failed "
2399 "synchronously [rv=0x%08" PRIx32
"]",
2400 static_cast<uint32_t>(rv
)));
2407 void CacheIndex::MergeJournal() {
2408 LOG(("CacheIndex::MergeJournal()"));
2410 sLock
.AssertCurrentThreadOwns();
2412 for (auto iter
= mTmpJournal
.Iter(); !iter
.Done(); iter
.Next()) {
2413 CacheIndexEntry
* entry
= iter
.Get();
2415 LOG(("CacheIndex::MergeJournal() [hash=%08x%08x%08x%08x%08x]",
2416 LOGSHA1(entry
->Hash())));
2418 CacheIndexEntry
* entry2
= mIndex
.GetEntry(*entry
->Hash());
2420 CacheIndexEntryAutoManage
emng(entry
->Hash(), this);
2421 if (entry
->IsRemoved()) {
2423 entry2
->MarkRemoved();
2424 entry2
->MarkDirty();
2428 entry2
= mIndex
.PutEntry(*entry
->Hash());
2432 entry2
->MarkDirty();
2438 MOZ_ASSERT(mTmpJournal
.Count() == 0);
2441 void CacheIndex::EnsureNoFreshEntry() {
2443 CacheIndexStats debugStats
;
2444 debugStats
.DisableLogging();
2445 for (auto iter
= mIndex
.Iter(); !iter
.Done(); iter
.Next()) {
2446 debugStats
.BeforeChange(nullptr);
2447 debugStats
.AfterChange(iter
.Get());
2449 MOZ_ASSERT(debugStats
.Fresh() == 0);
2453 void CacheIndex::EnsureCorrectStats() {
2455 MOZ_ASSERT(mPendingUpdates
.Count() == 0);
2456 CacheIndexStats debugStats
;
2457 debugStats
.DisableLogging();
2458 for (auto iter
= mIndex
.Iter(); !iter
.Done(); iter
.Next()) {
2459 debugStats
.BeforeChange(nullptr);
2460 debugStats
.AfterChange(iter
.Get());
2462 MOZ_ASSERT(debugStats
== mIndexStats
);
2466 void CacheIndex::FinishRead(bool aSucceeded
) {
2467 LOG(("CacheIndex::FinishRead() [succeeded=%d]", aSucceeded
));
2468 sLock
.AssertCurrentThreadOwns();
2470 MOZ_ASSERT((!aSucceeded
&& mState
== SHUTDOWN
) || mState
== READING
);
2474 (!aSucceeded
&& !mIndexOnDiskIsValid
&& !mJournalReadSuccessfully
) ||
2476 (!aSucceeded
&& mIndexOnDiskIsValid
&& !mJournalReadSuccessfully
) ||
2478 (aSucceeded
&& mIndexOnDiskIsValid
&& mJournalReadSuccessfully
));
2480 // If there is read operation pending we must be cancelling reading of the
2481 // index when shutting down or removing the whole index.
2482 MOZ_ASSERT(!mRWPending
|| (!aSucceeded
&& (mShuttingDown
|| mRemovingAll
)));
2484 if (mState
== SHUTDOWN
) {
2485 RemoveFile(nsLiteralCString(TEMP_INDEX_NAME
));
2486 RemoveFile(nsLiteralCString(JOURNAL_NAME
));
2488 if (mIndexHandle
&& !mIndexOnDiskIsValid
) {
2489 CacheFileIOManager::DoomFile(mIndexHandle
, nullptr);
2492 if (mJournalHandle
) {
2493 CacheFileIOManager::DoomFile(mJournalHandle
, nullptr);
2497 if (mIndexFileOpener
) {
2498 mIndexFileOpener
->Cancel();
2499 mIndexFileOpener
= nullptr;
2501 if (mJournalFileOpener
) {
2502 mJournalFileOpener
->Cancel();
2503 mJournalFileOpener
= nullptr;
2505 if (mTmpFileOpener
) {
2506 mTmpFileOpener
->Cancel();
2507 mTmpFileOpener
= nullptr;
2510 mIndexHandle
= nullptr;
2511 mJournalHandle
= nullptr;
2515 if (mState
== SHUTDOWN
) {
2519 if (!mIndexOnDiskIsValid
) {
2520 MOZ_ASSERT(mTmpJournal
.Count() == 0);
2521 EnsureNoFreshEntry();
2522 ProcessPendingOperations();
2523 // Remove all entries that we haven't seen during this session
2524 RemoveNonFreshEntries();
2525 StartUpdatingIndex(true);
2529 if (!mJournalReadSuccessfully
) {
2530 mTmpJournal
.Clear();
2531 EnsureNoFreshEntry();
2532 ProcessPendingOperations();
2533 StartUpdatingIndex(false);
2538 EnsureNoFreshEntry();
2539 ProcessPendingOperations();
2543 mLastDumpTime
= TimeStamp::NowLoRes(); // Do not dump new index immediately
2547 void CacheIndex::DelayedUpdate(nsITimer
* aTimer
, void* aClosure
) {
2548 LOG(("CacheIndex::DelayedUpdate()"));
2550 StaticMutexAutoLock
lock(sLock
);
2551 RefPtr
<CacheIndex
> index
= gInstance
;
2557 index
->DelayedUpdateLocked();
2561 void CacheIndex::DelayedUpdateLocked() {
2562 LOG(("CacheIndex::DelayedUpdateLocked()"));
2564 sLock
.AssertCurrentThreadOwns();
2568 mUpdateTimer
= nullptr;
2570 if (!IsIndexUsable()) {
2574 if (mState
== READY
&& mShuttingDown
) {
2578 // mUpdateEventPending must be false here since StartUpdatingIndex() won't
2579 // schedule timer if it is true.
2580 MOZ_ASSERT(!mUpdateEventPending
);
2581 if (mState
!= BUILDING
&& mState
!= UPDATING
) {
2582 LOG(("CacheIndex::DelayedUpdateLocked() - Update was canceled"));
2586 // We need to redispatch to run with lower priority
2587 RefPtr
<CacheIOThread
> ioThread
= CacheFileIOManager::IOThread();
2588 MOZ_ASSERT(ioThread
);
2590 mUpdateEventPending
= true;
2591 rv
= ioThread
->Dispatch(this, CacheIOThread::INDEX
);
2592 if (NS_FAILED(rv
)) {
2593 mUpdateEventPending
= false;
2594 NS_WARNING("CacheIndex::DelayedUpdateLocked() - Can't dispatch event");
2595 LOG(("CacheIndex::DelayedUpdate() - Can't dispatch event"));
2596 FinishUpdate(false);
2600 nsresult
CacheIndex::ScheduleUpdateTimer(uint32_t aDelay
) {
2601 LOG(("CacheIndex::ScheduleUpdateTimer() [delay=%u]", aDelay
));
2603 MOZ_ASSERT(!mUpdateTimer
);
2605 nsCOMPtr
<nsIEventTarget
> ioTarget
= CacheFileIOManager::IOTarget();
2606 MOZ_ASSERT(ioTarget
);
2608 return NS_NewTimerWithFuncCallback(
2609 getter_AddRefs(mUpdateTimer
), CacheIndex::DelayedUpdate
, nullptr, aDelay
,
2610 nsITimer::TYPE_ONE_SHOT
, "net::CacheIndex::ScheduleUpdateTimer",
2614 nsresult
CacheIndex::SetupDirectoryEnumerator() {
2615 MOZ_ASSERT(!NS_IsMainThread());
2616 MOZ_ASSERT(!mDirEnumerator
);
2619 nsCOMPtr
<nsIFile
> file
;
2621 rv
= mCacheDirectory
->Clone(getter_AddRefs(file
));
2622 NS_ENSURE_SUCCESS(rv
, rv
);
2624 rv
= file
->AppendNative(nsLiteralCString(ENTRIES_DIR
));
2625 NS_ENSURE_SUCCESS(rv
, rv
);
2628 rv
= file
->Exists(&exists
);
2629 NS_ENSURE_SUCCESS(rv
, rv
);
2633 "CacheIndex::SetupDirectoryEnumerator() - Entries directory "
2636 ("CacheIndex::SetupDirectoryEnumerator() - Entries directory doesn't "
2638 return NS_ERROR_UNEXPECTED
;
2641 rv
= file
->GetDirectoryEntries(getter_AddRefs(mDirEnumerator
));
2642 NS_ENSURE_SUCCESS(rv
, rv
);
2647 nsresult
CacheIndex::InitEntryFromDiskData(CacheIndexEntry
* aEntry
,
2648 CacheFileMetadata
* aMetaData
,
2649 int64_t aFileSize
) {
2653 aEntry
->MarkDirty();
2654 aEntry
->MarkFresh();
2656 aEntry
->Init(GetOriginAttrsHash(aMetaData
->OriginAttributes()),
2657 aMetaData
->IsAnonymous(), aMetaData
->Pinned());
2659 aEntry
->SetFrecency(aMetaData
->GetFrecency());
2661 const char* altData
= aMetaData
->GetElement(CacheFileUtils::kAltDataKey
);
2662 bool hasAltData
= altData
? true : false;
2663 if (hasAltData
&& NS_FAILED(CacheFileUtils::ParseAlternativeDataInfo(
2664 altData
, nullptr, nullptr))) {
2665 return NS_ERROR_FAILURE
;
2667 aEntry
->SetHasAltData(hasAltData
);
2669 static auto toUint16
= [](const char* aUint16String
) -> uint16_t {
2670 if (!aUint16String
) {
2671 return kIndexTimeNotAvailable
;
2674 uint64_t n64
= nsDependentCString(aUint16String
).ToInteger64(&rv
);
2675 MOZ_ASSERT(NS_SUCCEEDED(rv
));
2676 return n64
<= kIndexTimeOutOfBound
? n64
: kIndexTimeOutOfBound
;
2679 aEntry
->SetOnStartTime(
2680 toUint16(aMetaData
->GetElement("net-response-time-onstart")));
2681 aEntry
->SetOnStopTime(
2682 toUint16(aMetaData
->GetElement("net-response-time-onstop")));
2684 const char* contentTypeStr
= aMetaData
->GetElement("ctid");
2685 uint8_t contentType
= nsICacheEntry::CONTENT_TYPE_UNKNOWN
;
2686 if (contentTypeStr
) {
2687 int64_t n64
= nsDependentCString(contentTypeStr
).ToInteger64(&rv
);
2688 if (NS_FAILED(rv
) || n64
< nsICacheEntry::CONTENT_TYPE_UNKNOWN
||
2689 n64
>= nsICacheEntry::CONTENT_TYPE_LAST
) {
2690 n64
= nsICacheEntry::CONTENT_TYPE_UNKNOWN
;
2694 aEntry
->SetContentType(contentType
);
2696 aEntry
->SetFileSize(static_cast<uint32_t>(std::min(
2697 static_cast<int64_t>(PR_UINT32_MAX
), (aFileSize
+ 0x3FF) >> 10)));
2701 bool CacheIndex::IsUpdatePending() {
2702 sLock
.AssertCurrentThreadOwns();
2704 if (mUpdateTimer
|| mUpdateEventPending
) {
2711 void CacheIndex::BuildIndex() {
2712 LOG(("CacheIndex::BuildIndex()"));
2714 sLock
.AssertCurrentThreadOwns();
2716 MOZ_ASSERT(mPendingUpdates
.Count() == 0);
2720 if (!mDirEnumerator
) {
2722 // Do not do IO under the lock.
2723 StaticMutexAutoUnlock
unlock(sLock
);
2724 rv
= SetupDirectoryEnumerator();
2726 if (mState
== SHUTDOWN
) {
2727 // The index was shut down while we released the lock. FinishUpdate() was
2728 // already called from Shutdown(), so just simply return here.
2732 if (NS_FAILED(rv
)) {
2733 FinishUpdate(false);
2739 if (CacheIOThread::YieldAndRerun()) {
2741 "CacheIndex::BuildIndex() - Breaking loop for higher level events."));
2742 mUpdateEventPending
= true;
2746 bool fileExists
= false;
2747 nsCOMPtr
<nsIFile
> file
;
2749 // Do not do IO under the lock.
2750 StaticMutexAutoUnlock
unlock(sLock
);
2751 rv
= mDirEnumerator
->GetNextFile(getter_AddRefs(file
));
2754 file
->Exists(&fileExists
);
2757 if (mState
== SHUTDOWN
) {
2761 FinishUpdate(NS_SUCCEEDED(rv
));
2766 rv
= file
->GetNativeLeafName(leaf
);
2767 if (NS_FAILED(rv
)) {
2769 ("CacheIndex::BuildIndex() - GetNativeLeafName() failed! Skipping "
2771 mDontMarkIndexClean
= true;
2777 ("CacheIndex::BuildIndex() - File returned by the iterator was "
2778 "removed in the meantime [name=%s]",
2784 rv
= CacheFileIOManager::StrToHash(leaf
, &hash
);
2785 if (NS_FAILED(rv
)) {
2787 ("CacheIndex::BuildIndex() - Filename is not a hash, removing file. "
2790 file
->Remove(false);
2794 CacheIndexEntry
* entry
= mIndex
.GetEntry(hash
);
2795 if (entry
&& entry
->IsRemoved()) {
2797 ("CacheIndex::BuildIndex() - Found file that should not exist. "
2801 MOZ_ASSERT(entry
->IsFresh());
2806 RefPtr
<CacheFileHandle
> handle
;
2807 CacheFileIOManager::gInstance
->mHandles
.GetHandle(&hash
,
2808 getter_AddRefs(handle
));
2812 // the entry is up to date
2814 ("CacheIndex::BuildIndex() - Skipping file because the entry is up to"
2818 MOZ_ASSERT(entry
->IsFresh()); // The entry must be from this session
2819 // there must be an active CacheFile if the entry is not initialized
2820 MOZ_ASSERT(entry
->IsInitialized() || handle
);
2824 MOZ_ASSERT(!handle
);
2826 RefPtr
<CacheFileMetadata
> meta
= new CacheFileMetadata();
2830 // Do not do IO under the lock.
2831 StaticMutexAutoUnlock
unlock(sLock
);
2832 rv
= meta
->SyncReadMetadata(file
);
2834 if (NS_SUCCEEDED(rv
)) {
2835 rv
= file
->GetFileSize(&size
);
2836 if (NS_FAILED(rv
)) {
2838 ("CacheIndex::BuildIndex() - Cannot get filesize of file that was"
2839 " successfully parsed. [name=%s]",
2844 if (mState
== SHUTDOWN
) {
2848 // Nobody could add the entry while the lock was released since we modify
2849 // the index only on IO thread and this loop is executed on IO thread too.
2850 entry
= mIndex
.GetEntry(hash
);
2851 MOZ_ASSERT(!entry
|| entry
->IsRemoved());
2853 if (NS_FAILED(rv
)) {
2855 ("CacheIndex::BuildIndex() - CacheFileMetadata::SyncReadMetadata() "
2856 "failed, removing file. [name=%s]",
2858 file
->Remove(false);
2860 CacheIndexEntryAutoManage
entryMng(&hash
, this);
2861 entry
= mIndex
.PutEntry(hash
);
2862 if (NS_FAILED(InitEntryFromDiskData(entry
, meta
, size
))) {
2864 ("CacheIndex::BuildIndex() - CacheFile::InitEntryFromDiskData() "
2865 "failed, removing file. [name=%s]",
2867 file
->Remove(false);
2868 entry
->MarkRemoved();
2870 LOG(("CacheIndex::BuildIndex() - Added entry to index. [name=%s]",
2877 MOZ_ASSERT_UNREACHABLE("We should never get here");
2880 bool CacheIndex::StartUpdatingIndexIfNeeded(bool aSwitchingToReadyState
) {
2881 // Start updating process when we are in or we are switching to READY state
2882 // and index needs update, but not during shutdown or when removing all
2884 if ((mState
== READY
|| aSwitchingToReadyState
) && mIndexNeedsUpdate
&&
2885 !mShuttingDown
&& !mRemovingAll
) {
2886 LOG(("CacheIndex::StartUpdatingIndexIfNeeded() - starting update process"));
2887 mIndexNeedsUpdate
= false;
2888 StartUpdatingIndex(false);
2895 void CacheIndex::StartUpdatingIndex(bool aRebuild
) {
2896 LOG(("CacheIndex::StartUpdatingIndex() [rebuild=%d]", aRebuild
));
2898 sLock
.AssertCurrentThreadOwns();
2904 ChangeState(aRebuild
? BUILDING
: UPDATING
);
2905 mDontMarkIndexClean
= false;
2907 if (mShuttingDown
|| mRemovingAll
) {
2908 FinishUpdate(false);
2912 if (IsUpdatePending()) {
2913 LOG(("CacheIndex::StartUpdatingIndex() - Update is already pending"));
2917 uint32_t elapsed
= (TimeStamp::NowLoRes() - mStartTime
).ToMilliseconds();
2918 if (elapsed
< kUpdateIndexStartDelay
) {
2920 ("CacheIndex::StartUpdatingIndex() - %u ms elapsed since startup, "
2921 "scheduling timer to fire in %u ms.",
2922 elapsed
, kUpdateIndexStartDelay
- elapsed
));
2923 rv
= ScheduleUpdateTimer(kUpdateIndexStartDelay
- elapsed
);
2924 if (NS_SUCCEEDED(rv
)) {
2929 ("CacheIndex::StartUpdatingIndex() - ScheduleUpdateTimer() failed. "
2930 "Starting update immediately."));
2933 ("CacheIndex::StartUpdatingIndex() - %u ms elapsed since startup, "
2934 "starting update now.",
2938 RefPtr
<CacheIOThread
> ioThread
= CacheFileIOManager::IOThread();
2939 MOZ_ASSERT(ioThread
);
2941 // We need to dispatch an event even if we are on IO thread since we need to
2942 // update the index with the correct priority.
2943 mUpdateEventPending
= true;
2944 rv
= ioThread
->Dispatch(this, CacheIOThread::INDEX
);
2945 if (NS_FAILED(rv
)) {
2946 mUpdateEventPending
= false;
2947 NS_WARNING("CacheIndex::StartUpdatingIndex() - Can't dispatch event");
2948 LOG(("CacheIndex::StartUpdatingIndex() - Can't dispatch event"));
2949 FinishUpdate(false);
2953 void CacheIndex::UpdateIndex() {
2954 LOG(("CacheIndex::UpdateIndex()"));
2956 sLock
.AssertCurrentThreadOwns();
2958 MOZ_ASSERT(mPendingUpdates
.Count() == 0);
2962 if (!mDirEnumerator
) {
2964 // Do not do IO under the lock.
2965 StaticMutexAutoUnlock
unlock(sLock
);
2966 rv
= SetupDirectoryEnumerator();
2968 if (mState
== SHUTDOWN
) {
2969 // The index was shut down while we released the lock. FinishUpdate() was
2970 // already called from Shutdown(), so just simply return here.
2974 if (NS_FAILED(rv
)) {
2975 FinishUpdate(false);
2981 if (CacheIOThread::YieldAndRerun()) {
2983 ("CacheIndex::UpdateIndex() - Breaking loop for higher level "
2985 mUpdateEventPending
= true;
2989 bool fileExists
= false;
2990 nsCOMPtr
<nsIFile
> file
;
2992 // Do not do IO under the lock.
2993 StaticMutexAutoUnlock
unlock(sLock
);
2994 rv
= mDirEnumerator
->GetNextFile(getter_AddRefs(file
));
2997 file
->Exists(&fileExists
);
3000 if (mState
== SHUTDOWN
) {
3004 FinishUpdate(NS_SUCCEEDED(rv
));
3009 rv
= file
->GetNativeLeafName(leaf
);
3010 if (NS_FAILED(rv
)) {
3012 ("CacheIndex::UpdateIndex() - GetNativeLeafName() failed! Skipping "
3014 mDontMarkIndexClean
= true;
3020 ("CacheIndex::UpdateIndex() - File returned by the iterator was "
3021 "removed in the meantime [name=%s]",
3027 rv
= CacheFileIOManager::StrToHash(leaf
, &hash
);
3028 if (NS_FAILED(rv
)) {
3030 ("CacheIndex::UpdateIndex() - Filename is not a hash, removing file. "
3033 file
->Remove(false);
3037 CacheIndexEntry
* entry
= mIndex
.GetEntry(hash
);
3038 if (entry
&& entry
->IsRemoved()) {
3039 if (entry
->IsFresh()) {
3041 ("CacheIndex::UpdateIndex() - Found file that should not exist. "
3050 RefPtr
<CacheFileHandle
> handle
;
3051 CacheFileIOManager::gInstance
->mHandles
.GetHandle(&hash
,
3052 getter_AddRefs(handle
));
3055 if (entry
&& entry
->IsFresh()) {
3056 // the entry is up to date
3058 ("CacheIndex::UpdateIndex() - Skipping file because the entry is up "
3059 " to date. [name=%s]",
3062 // there must be an active CacheFile if the entry is not initialized
3063 MOZ_ASSERT(entry
->IsInitialized() || handle
);
3067 MOZ_ASSERT(!handle
);
3070 PRTime lastModifiedTime
;
3072 // Do not do IO under the lock.
3073 StaticMutexAutoUnlock
unlock(sLock
);
3074 rv
= file
->GetLastModifiedTime(&lastModifiedTime
);
3076 if (mState
== SHUTDOWN
) {
3079 if (NS_FAILED(rv
)) {
3081 ("CacheIndex::UpdateIndex() - Cannot get lastModifiedTime. "
3084 // Assume the file is newer than index
3086 if (mIndexTimeStamp
> (lastModifiedTime
/ PR_MSEC_PER_SEC
)) {
3088 ("CacheIndex::UpdateIndex() - Skipping file because of last "
3089 "modified time. [name=%s, indexTimeStamp=%" PRIu32
", "
3090 "lastModifiedTime=%" PRId64
"]",
3091 leaf
.get(), mIndexTimeStamp
,
3092 lastModifiedTime
/ PR_MSEC_PER_SEC
));
3094 CacheIndexEntryAutoManage
entryMng(&hash
, this);
3101 RefPtr
<CacheFileMetadata
> meta
= new CacheFileMetadata();
3105 // Do not do IO under the lock.
3106 StaticMutexAutoUnlock
unlock(sLock
);
3107 rv
= meta
->SyncReadMetadata(file
);
3109 if (NS_SUCCEEDED(rv
)) {
3110 rv
= file
->GetFileSize(&size
);
3111 if (NS_FAILED(rv
)) {
3113 ("CacheIndex::UpdateIndex() - Cannot get filesize of file that "
3114 "was successfully parsed. [name=%s]",
3119 if (mState
== SHUTDOWN
) {
3123 // Nobody could add the entry while the lock was released since we modify
3124 // the index only on IO thread and this loop is executed on IO thread too.
3125 entry
= mIndex
.GetEntry(hash
);
3126 MOZ_ASSERT(!entry
|| !entry
->IsFresh());
3128 CacheIndexEntryAutoManage
entryMng(&hash
, this);
3130 if (NS_FAILED(rv
)) {
3132 ("CacheIndex::UpdateIndex() - CacheFileMetadata::SyncReadMetadata() "
3133 "failed, removing file. [name=%s]",
3136 entry
= mIndex
.PutEntry(hash
);
3137 rv
= InitEntryFromDiskData(entry
, meta
, size
);
3138 if (NS_FAILED(rv
)) {
3140 ("CacheIndex::UpdateIndex() - CacheIndex::InitEntryFromDiskData "
3141 "failed, removing file. [name=%s]",
3146 if (NS_FAILED(rv
)) {
3147 file
->Remove(false);
3149 entry
->MarkRemoved();
3155 ("CacheIndex::UpdateIndex() - Added/updated entry to/in index. "
3162 MOZ_ASSERT_UNREACHABLE("We should never get here");
3165 void CacheIndex::FinishUpdate(bool aSucceeded
) {
3166 LOG(("CacheIndex::FinishUpdate() [succeeded=%d]", aSucceeded
));
3168 MOZ_ASSERT(mState
== UPDATING
|| mState
== BUILDING
||
3169 (!aSucceeded
&& mState
== SHUTDOWN
));
3171 sLock
.AssertCurrentThreadOwns();
3173 if (mDirEnumerator
) {
3174 if (NS_IsMainThread()) {
3176 ("CacheIndex::FinishUpdate() - posting of PreShutdownInternal failed?"
3177 " Cannot safely release mDirEnumerator, leaking it!"));
3178 NS_WARNING(("CacheIndex::FinishUpdate() - Leaking mDirEnumerator!"));
3179 // This can happen only in case dispatching event to IO thread failed in
3180 // CacheIndex::PreShutdown().
3181 Unused
<< mDirEnumerator
.forget(); // Leak it since dir enumerator is not
3184 mDirEnumerator
->Close();
3185 mDirEnumerator
= nullptr;
3190 mDontMarkIndexClean
= true;
3193 if (mState
== SHUTDOWN
) {
3197 if (mState
== UPDATING
&& aSucceeded
) {
3198 // If we've iterated over all entries successfully then all entries that
3199 // really exist on the disk are now marked as fresh. All non-fresh entries
3200 // don't exist anymore and must be removed from the index.
3201 RemoveNonFreshEntries();
3204 // Make sure we won't start update. If the build or update failed, there is no
3205 // reason to believe that it will succeed next time.
3206 mIndexNeedsUpdate
= false;
3209 mLastDumpTime
= TimeStamp::NowLoRes(); // Do not dump new index immediately
3212 void CacheIndex::RemoveNonFreshEntries() {
3213 for (auto iter
= mIndex
.Iter(); !iter
.Done(); iter
.Next()) {
3214 CacheIndexEntry
* entry
= iter
.Get();
3215 if (entry
->IsFresh()) {
3220 ("CacheIndex::RemoveNonFreshEntries() - Removing entry. "
3221 "[hash=%08x%08x%08x%08x%08x]",
3222 LOGSHA1(entry
->Hash())));
3225 CacheIndexEntryAutoManage
emng(entry
->Hash(), this);
3226 emng
.DoNotSearchInIndex();
3234 char const* CacheIndex::StateString(EState aState
) {
3252 MOZ_ASSERT(false, "Unexpected state!");
3256 void CacheIndex::ChangeState(EState aNewState
) {
3257 LOG(("CacheIndex::ChangeState() changing state %s -> %s", StateString(mState
),
3258 StateString(aNewState
)));
3260 // All pending updates should be processed before changing state
3261 MOZ_ASSERT(mPendingUpdates
.Count() == 0);
3263 // PreShutdownInternal() should change the state to READY from every state. It
3264 // may go through different states, but once we are in READY state the only
3265 // possible transition is to SHUTDOWN state.
3266 MOZ_ASSERT(!mShuttingDown
|| mState
!= READY
|| aNewState
== SHUTDOWN
);
3268 // Start updating process when switching to READY state if needed
3269 if (aNewState
== READY
&& StartUpdatingIndexIfNeeded(true)) {
3273 if ((mState
== READING
|| mState
== BUILDING
|| mState
== UPDATING
) &&
3274 aNewState
== READY
) {
3278 // Try to evict entries over limit everytime we're leaving state READING,
3279 // BUILDING or UPDATING, but not during shutdown or when removing all
3281 if (!mShuttingDown
&& !mRemovingAll
&& aNewState
!= SHUTDOWN
&&
3282 (mState
== READING
|| mState
== BUILDING
|| mState
== UPDATING
)) {
3283 CacheFileIOManager::EvictIfOverLimit();
3288 if (mState
!= SHUTDOWN
) {
3289 CacheFileIOManager::CacheIndexStateChanged();
3292 NotifyAsyncGetDiskConsumptionCallbacks();
3295 void CacheIndex::NotifyAsyncGetDiskConsumptionCallbacks() {
3296 if ((mState
== READY
|| mState
== WRITING
) &&
3297 !mAsyncGetDiskConsumptionBlocked
&& mDiskConsumptionObservers
.Length()) {
3298 for (uint32_t i
= 0; i
< mDiskConsumptionObservers
.Length(); ++i
) {
3299 DiskConsumptionObserver
* o
= mDiskConsumptionObservers
[i
];
3300 // Safe to call under the lock. We always post to the main thread.
3301 o
->OnDiskConsumption(mIndexStats
.Size() << 10);
3304 mDiskConsumptionObservers
.Clear();
3308 void CacheIndex::AllocBuffer() {
3311 mRWBufSize
= sizeof(CacheIndexHeader
) + sizeof(CacheHash::Hash32_t
) +
3312 mProcessEntries
* sizeof(CacheIndexRecord
);
3313 if (mRWBufSize
> kMaxBufSize
) {
3314 mRWBufSize
= kMaxBufSize
;
3318 mRWBufSize
= kMaxBufSize
;
3321 MOZ_ASSERT(false, "Unexpected state!");
3324 mRWBuf
= static_cast<char*>(moz_xmalloc(mRWBufSize
));
3327 void CacheIndex::ReleaseBuffer() {
3328 sLock
.AssertCurrentThreadOwns();
3330 if (!mRWBuf
|| mRWPending
) {
3334 LOG(("CacheIndex::ReleaseBuffer() releasing buffer"));
3342 void CacheIndex::FrecencyArray::AppendRecord(CacheIndexRecord
* aRecord
) {
3344 ("CacheIndex::FrecencyArray::AppendRecord() [record=%p, hash=%08x%08x%08x"
3346 aRecord
, LOGSHA1(aRecord
->mHash
)));
3348 MOZ_ASSERT(!mRecs
.Contains(aRecord
));
3349 mRecs
.AppendElement(aRecord
);
3351 // If the new frecency is 0, the element should be at the end of the array,
3352 // i.e. this change doesn't affect order of the array
3353 if (aRecord
->mFrecency
!= 0) {
3354 ++mUnsortedElements
;
3358 void CacheIndex::FrecencyArray::RemoveRecord(CacheIndexRecord
* aRecord
) {
3359 LOG(("CacheIndex::FrecencyArray::RemoveRecord() [record=%p]", aRecord
));
3361 decltype(mRecs
)::index_type idx
;
3362 idx
= mRecs
.IndexOf(aRecord
);
3363 MOZ_RELEASE_ASSERT(idx
!= mRecs
.NoIndex
);
3364 mRecs
[idx
] = nullptr;
3367 // Calling SortIfNeeded ensures that we get rid of removed elements in the
3368 // array once we hit the limit.
3372 void CacheIndex::FrecencyArray::ReplaceRecord(CacheIndexRecord
* aOldRecord
,
3373 CacheIndexRecord
* aNewRecord
) {
3375 ("CacheIndex::FrecencyArray::ReplaceRecord() [oldRecord=%p, "
3377 aOldRecord
, aNewRecord
));
3379 decltype(mRecs
)::index_type idx
;
3380 idx
= mRecs
.IndexOf(aOldRecord
);
3381 MOZ_RELEASE_ASSERT(idx
!= mRecs
.NoIndex
);
3382 mRecs
[idx
] = aNewRecord
;
3385 void CacheIndex::FrecencyArray::SortIfNeeded() {
3386 const uint32_t kMaxUnsortedCount
= 512;
3387 const uint32_t kMaxUnsortedPercent
= 10;
3388 const uint32_t kMaxRemovedCount
= 512;
3390 uint32_t unsortedLimit
= std::min
<uint32_t>(
3391 kMaxUnsortedCount
, Length() * kMaxUnsortedPercent
/ 100);
3393 if (mUnsortedElements
> unsortedLimit
||
3394 mRemovedElements
> kMaxRemovedCount
) {
3396 ("CacheIndex::FrecencyArray::SortIfNeeded() - Sorting array "
3397 "[unsortedElements=%u, unsortedLimit=%u, removedElements=%u, "
3398 "maxRemovedCount=%u]",
3399 mUnsortedElements
, unsortedLimit
, mRemovedElements
, kMaxRemovedCount
));
3401 mRecs
.Sort(FrecencyComparator());
3402 mUnsortedElements
= 0;
3403 if (mRemovedElements
) {
3405 for (uint32_t i
= Length(); i
< mRecs
.Length(); ++i
) {
3406 MOZ_ASSERT(!mRecs
[i
]);
3409 // Removed elements are at the end after sorting.
3410 mRecs
.RemoveElementsAt(Length(), mRemovedElements
);
3411 mRemovedElements
= 0;
3416 void CacheIndex::AddRecordToIterators(CacheIndexRecord
* aRecord
) {
3417 sLock
.AssertCurrentThreadOwns();
3419 for (uint32_t i
= 0; i
< mIterators
.Length(); ++i
) {
3420 // Add a new record only when iterator is supposed to be updated.
3421 if (mIterators
[i
]->ShouldBeNewAdded()) {
3422 mIterators
[i
]->AddRecord(aRecord
);
3427 void CacheIndex::RemoveRecordFromIterators(CacheIndexRecord
* aRecord
) {
3428 sLock
.AssertCurrentThreadOwns();
3430 for (uint32_t i
= 0; i
< mIterators
.Length(); ++i
) {
3431 // Remove the record from iterator always, it makes no sence to return
3432 // non-existing entries. Also the pointer to the record is no longer valid
3433 // once the entry is removed from index.
3434 mIterators
[i
]->RemoveRecord(aRecord
);
3438 void CacheIndex::ReplaceRecordInIterators(CacheIndexRecord
* aOldRecord
,
3439 CacheIndexRecord
* aNewRecord
) {
3440 sLock
.AssertCurrentThreadOwns();
3442 for (uint32_t i
= 0; i
< mIterators
.Length(); ++i
) {
3443 // We have to replace the record always since the pointer is no longer
3444 // valid after this point. NOTE: Replacing the record doesn't mean that
3445 // a new entry was added, it just means that the data in the entry was
3446 // changed (e.g. a file size) and we had to track this change in
3447 // mPendingUpdates since mIndex was read-only.
3448 mIterators
[i
]->ReplaceRecord(aOldRecord
, aNewRecord
);
3452 nsresult
CacheIndex::Run() {
3453 LOG(("CacheIndex::Run()"));
3455 StaticMutexAutoLock
lock(sLock
);
3457 if (!IsIndexUsable()) {
3458 return NS_ERROR_NOT_AVAILABLE
;
3461 if (mState
== READY
&& mShuttingDown
) {
3465 mUpdateEventPending
= false;
3475 LOG(("CacheIndex::Run() - Update/Build was canceled"));
3481 void CacheIndex::OnFileOpenedInternal(FileOpenHelper
* aOpener
,
3482 CacheFileHandle
* aHandle
,
3485 ("CacheIndex::OnFileOpenedInternal() [opener=%p, handle=%p, "
3486 "result=0x%08" PRIx32
"]",
3487 aOpener
, aHandle
, static_cast<uint32_t>(aResult
)));
3488 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
3492 sLock
.AssertCurrentThreadOwns();
3494 MOZ_RELEASE_ASSERT(IsIndexUsable());
3496 if (mState
== READY
&& mShuttingDown
) {
3502 MOZ_ASSERT(aOpener
== mIndexFileOpener
);
3503 mIndexFileOpener
= nullptr;
3505 if (NS_FAILED(aResult
)) {
3507 ("CacheIndex::OnFileOpenedInternal() - Can't open index file for "
3508 "writing [rv=0x%08" PRIx32
"]",
3509 static_cast<uint32_t>(aResult
)));
3512 mIndexHandle
= aHandle
;
3517 if (aOpener
== mIndexFileOpener
) {
3518 mIndexFileOpener
= nullptr;
3520 if (NS_SUCCEEDED(aResult
)) {
3521 if (aHandle
->FileSize() == 0) {
3523 CacheFileIOManager::DoomFile(aHandle
, nullptr);
3526 mIndexHandle
= aHandle
;
3531 } else if (aOpener
== mJournalFileOpener
) {
3532 mJournalFileOpener
= nullptr;
3533 mJournalHandle
= aHandle
;
3534 } else if (aOpener
== mTmpFileOpener
) {
3535 mTmpFileOpener
= nullptr;
3536 mTmpHandle
= aHandle
;
3538 MOZ_ASSERT(false, "Unexpected state!");
3541 if (mIndexFileOpener
|| mJournalFileOpener
|| mTmpFileOpener
) {
3542 // Some opener still didn't finish
3546 // We fail and cancel all other openers when we opening index file fails.
3547 MOZ_ASSERT(mIndexHandle
);
3550 CacheFileIOManager::DoomFile(mTmpHandle
, nullptr);
3551 mTmpHandle
= nullptr;
3553 if (mJournalHandle
) { // this shouldn't normally happen
3555 ("CacheIndex::OnFileOpenedInternal() - Unexpected state, all "
3556 "files [%s, %s, %s] should never exist. Removing whole index.",
3557 INDEX_NAME
, JOURNAL_NAME
, TEMP_INDEX_NAME
));
3563 if (mJournalHandle
) {
3564 // Rename journal to make sure we update index on next start in case
3566 rv
= CacheFileIOManager::RenameFile(
3567 mJournalHandle
, nsLiteralCString(TEMP_INDEX_NAME
), this);
3568 if (NS_FAILED(rv
)) {
3570 ("CacheIndex::OnFileOpenedInternal() - CacheFileIOManager::"
3571 "RenameFile() failed synchronously [rv=0x%08" PRIx32
"]",
3572 static_cast<uint32_t>(rv
)));
3577 StartReadingIndex();
3582 MOZ_ASSERT(false, "Unexpected state!");
3586 nsresult
CacheIndex::OnFileOpened(CacheFileHandle
* aHandle
, nsresult aResult
) {
3587 MOZ_CRASH("CacheIndex::OnFileOpened should not be called!");
3588 return NS_ERROR_UNEXPECTED
;
3591 nsresult
CacheIndex::OnDataWritten(CacheFileHandle
* aHandle
, const char* aBuf
,
3593 LOG(("CacheIndex::OnDataWritten() [handle=%p, result=0x%08" PRIx32
"]",
3594 aHandle
, static_cast<uint32_t>(aResult
)));
3596 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
3600 StaticMutexAutoLock
lock(sLock
);
3602 MOZ_RELEASE_ASSERT(IsIndexUsable());
3603 MOZ_RELEASE_ASSERT(mRWPending
);
3606 if (mState
== READY
&& mShuttingDown
) {
3612 MOZ_ASSERT(mIndexHandle
== aHandle
);
3614 if (NS_FAILED(aResult
)) {
3617 if (mSkipEntries
== mProcessEntries
) {
3618 rv
= CacheFileIOManager::RenameFile(
3619 mIndexHandle
, nsLiteralCString(INDEX_NAME
), this);
3620 if (NS_FAILED(rv
)) {
3622 ("CacheIndex::OnDataWritten() - CacheFileIOManager::"
3623 "RenameFile() failed synchronously [rv=0x%08" PRIx32
"]",
3624 static_cast<uint32_t>(rv
)));
3633 // Writing was canceled.
3635 ("CacheIndex::OnDataWritten() - ignoring notification since the "
3636 "operation was previously canceled [state=%d]",
3644 nsresult
CacheIndex::OnDataRead(CacheFileHandle
* aHandle
, char* aBuf
,
3646 LOG(("CacheIndex::OnDataRead() [handle=%p, result=0x%08" PRIx32
"]", aHandle
,
3647 static_cast<uint32_t>(aResult
)));
3649 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
3651 StaticMutexAutoLock
lock(sLock
);
3653 MOZ_RELEASE_ASSERT(IsIndexUsable());
3654 MOZ_RELEASE_ASSERT(mRWPending
);
3659 MOZ_ASSERT(mIndexHandle
== aHandle
|| mJournalHandle
== aHandle
);
3661 if (NS_FAILED(aResult
)) {
3664 if (!mIndexOnDiskIsValid
) {
3672 // Reading was canceled.
3674 ("CacheIndex::OnDataRead() - ignoring notification since the "
3675 "operation was previously canceled [state=%d]",
3683 nsresult
CacheIndex::OnFileDoomed(CacheFileHandle
* aHandle
, nsresult aResult
) {
3684 MOZ_CRASH("CacheIndex::OnFileDoomed should not be called!");
3685 return NS_ERROR_UNEXPECTED
;
3688 nsresult
CacheIndex::OnEOFSet(CacheFileHandle
* aHandle
, nsresult aResult
) {
3689 MOZ_CRASH("CacheIndex::OnEOFSet should not be called!");
3690 return NS_ERROR_UNEXPECTED
;
3693 nsresult
CacheIndex::OnFileRenamed(CacheFileHandle
* aHandle
, nsresult aResult
) {
3694 LOG(("CacheIndex::OnFileRenamed() [handle=%p, result=0x%08" PRIx32
"]",
3695 aHandle
, static_cast<uint32_t>(aResult
)));
3697 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
3699 StaticMutexAutoLock
lock(sLock
);
3701 MOZ_RELEASE_ASSERT(IsIndexUsable());
3703 if (mState
== READY
&& mShuttingDown
) {
3709 // This is a result of renaming the new index written to tmpfile to index
3710 // file. This is the last step when writing the index and the whole
3711 // writing process is successful iff renaming was successful.
3713 if (mIndexHandle
!= aHandle
) {
3715 ("CacheIndex::OnFileRenamed() - ignoring notification since it "
3716 "belongs to previously canceled operation [state=%d]",
3721 FinishWrite(NS_SUCCEEDED(aResult
));
3724 // This is a result of renaming journal file to tmpfile. It is renamed
3725 // before we start reading index and journal file and it should normally
3726 // succeed. If it fails give up reading of index.
3728 if (mJournalHandle
!= aHandle
) {
3730 ("CacheIndex::OnFileRenamed() - ignoring notification since it "
3731 "belongs to previously canceled operation [state=%d]",
3736 if (NS_FAILED(aResult
)) {
3739 StartReadingIndex();
3743 // Reading/writing was canceled.
3745 ("CacheIndex::OnFileRenamed() - ignoring notification since the "
3746 "operation was previously canceled [state=%d]",
3755 size_t CacheIndex::SizeOfExcludingThisInternal(
3756 mozilla::MallocSizeOf mallocSizeOf
) const {
3757 sLock
.AssertCurrentThreadOwns();
3760 nsCOMPtr
<nsISizeOf
> sizeOf
;
3762 // mIndexHandle and mJournalHandle are reported via SizeOfHandlesRunnable
3763 // in CacheFileIOManager::SizeOfExcludingThisInternal as part of special
3766 sizeOf
= do_QueryInterface(mCacheDirectory
);
3768 n
+= sizeOf
->SizeOfIncludingThis(mallocSizeOf
);
3771 sizeOf
= do_QueryInterface(mUpdateTimer
);
3773 n
+= sizeOf
->SizeOfIncludingThis(mallocSizeOf
);
3776 n
+= mallocSizeOf(mRWBuf
);
3777 n
+= mallocSizeOf(mRWHash
);
3779 n
+= mIndex
.SizeOfExcludingThis(mallocSizeOf
);
3780 n
+= mPendingUpdates
.SizeOfExcludingThis(mallocSizeOf
);
3781 n
+= mTmpJournal
.SizeOfExcludingThis(mallocSizeOf
);
3783 // mFrecencyArray items are reported by mIndex/mPendingUpdates
3784 n
+= mFrecencyArray
.mRecs
.ShallowSizeOfExcludingThis(mallocSizeOf
);
3785 n
+= mDiskConsumptionObservers
.ShallowSizeOfExcludingThis(mallocSizeOf
);
3791 size_t CacheIndex::SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf
) {
3792 sLock
.AssertCurrentThreadOwns();
3794 if (!gInstance
) return 0;
3796 return gInstance
->SizeOfExcludingThisInternal(mallocSizeOf
);
3800 size_t CacheIndex::SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf
) {
3801 StaticMutexAutoLock
lock(sLock
);
3803 return mallocSizeOf(gInstance
) + SizeOfExcludingThis(mallocSizeOf
);
3808 class HashComparator
{
3810 bool Equals(CacheIndexRecord
* a
, CacheIndexRecord
* b
) const {
3811 return memcmp(&a
->mHash
, &b
->mHash
, sizeof(SHA1Sum::Hash
)) == 0;
3813 bool LessThan(CacheIndexRecord
* a
, CacheIndexRecord
* b
) const {
3814 return memcmp(&a
->mHash
, &b
->mHash
, sizeof(SHA1Sum::Hash
)) < 0;
3818 void ReportHashSizeMatch(const SHA1Sum::Hash
* aHash1
,
3819 const SHA1Sum::Hash
* aHash2
) {
3820 const uint32_t* h1
= reinterpret_cast<const uint32_t*>(aHash1
);
3821 const uint32_t* h2
= reinterpret_cast<const uint32_t*>(aHash2
);
3823 for (uint32_t i
= 0; i
< 5; ++i
) {
3824 if (h1
[i
] != h2
[i
]) {
3825 uint32_t bitsDiff
= h1
[i
] ^ h2
[i
];
3826 bitsDiff
= NetworkEndian::readUint32(&bitsDiff
);
3828 // count leading zeros in bitsDiff
3829 static const uint8_t debruijn32
[32] = {
3830 0, 31, 9, 30, 3, 8, 13, 29, 2, 5, 7, 21, 12, 24, 28, 19,
3831 1, 10, 4, 14, 6, 22, 25, 20, 11, 15, 23, 26, 16, 27, 17, 18};
3833 bitsDiff
|= bitsDiff
>> 1;
3834 bitsDiff
|= bitsDiff
>> 2;
3835 bitsDiff
|= bitsDiff
>> 4;
3836 bitsDiff
|= bitsDiff
>> 8;
3837 bitsDiff
|= bitsDiff
>> 16;
3840 uint8_t hashSizeMatch
=
3841 debruijn32
[bitsDiff
* 0x076be629 >> 27] + (i
<< 5);
3842 Telemetry::Accumulate(Telemetry::NETWORK_CACHE_HASH_STATS
, hashSizeMatch
);
3848 MOZ_ASSERT(false, "Found a collision in the index!");
3853 void CacheIndex::ReportHashStats() {
3854 // We're gathering the hash stats only once, exclude too small caches.
3855 if (CacheObserver::HashStatsReported() || mFrecencyArray
.Length() < 15000) {
3859 nsTArray
<CacheIndexRecord
*> records
;
3860 for (auto iter
= mFrecencyArray
.Iter(); !iter
.Done(); iter
.Next()) {
3861 records
.AppendElement(iter
.Get());
3864 records
.Sort(HashComparator());
3866 for (uint32_t i
= 1; i
< records
.Length(); i
++) {
3867 ReportHashSizeMatch(&records
[i
- 1]->mHash
, &records
[i
]->mHash
);
3870 CacheObserver::SetHashStatsReported();
3874 void CacheIndex::UpdateTotalBytesWritten(uint32_t aBytesWritten
) {
3875 StaticMutexAutoLock
lock(sLock
);
3877 RefPtr
<CacheIndex
> index
= gInstance
;
3882 index
->mTotalBytesWritten
+= aBytesWritten
;
3884 // Do telemetry report if enough data has been written and the index is
3885 // in READY state. The data is available also in WRITING state, but we would
3886 // need to deal with pending updates.
3887 if (index
->mTotalBytesWritten
>= kTelemetryReportBytesLimit
&&
3888 index
->mState
== READY
&& !index
->mIndexNeedsUpdate
&&
3889 !index
->mShuttingDown
) {
3890 index
->DoTelemetryReport();
3892 index
->mTotalBytesWritten
= 0;
3893 CacheObserver::SetCacheAmountWritten(0);
3897 uint64_t writtenKB
= index
->mTotalBytesWritten
>> 10;
3898 // Store number of written kilobytes to prefs after writing at least 10MB.
3899 if ((writtenKB
- CacheObserver::CacheAmountWritten()) > (10 * 1024)) {
3900 CacheObserver::SetCacheAmountWritten(writtenKB
);
3904 void CacheIndex::DoTelemetryReport() {
3905 static const nsLiteralCString
3906 contentTypeNames
[nsICacheEntry::CONTENT_TYPE_LAST
] = {
3907 "UNKNOWN"_ns
, "OTHER"_ns
, "JAVASCRIPT"_ns
, "IMAGE"_ns
,
3908 "MEDIA"_ns
, "STYLESHEET"_ns
, "WASM"_ns
};
3910 for (uint32_t i
= 0; i
< nsICacheEntry::CONTENT_TYPE_LAST
; ++i
) {
3911 if (mIndexStats
.Size() > 0) {
3912 Telemetry::Accumulate(
3913 Telemetry::NETWORK_CACHE_SIZE_SHARE
, contentTypeNames
[i
],
3914 round(static_cast<double>(mIndexStats
.SizeByType(i
)) * 100.0 /
3915 static_cast<double>(mIndexStats
.Size())));
3918 if (mIndexStats
.Count() > 0) {
3919 Telemetry::Accumulate(
3920 Telemetry::NETWORK_CACHE_ENTRY_COUNT_SHARE
, contentTypeNames
[i
],
3921 round(static_cast<double>(mIndexStats
.CountByType(i
)) * 100.0 /
3922 static_cast<double>(mIndexStats
.Count())));
3927 if (CacheObserver::SmartCacheSizeEnabled()) {
3928 probeKey
= "SMARTSIZE"_ns
;
3930 probeKey
= "USERDEFINEDSIZE"_ns
;
3932 Telemetry::Accumulate(Telemetry::NETWORK_CACHE_ENTRY_COUNT
, probeKey
,
3933 mIndexStats
.Count());
3934 Telemetry::Accumulate(Telemetry::NETWORK_CACHE_SIZE
, probeKey
,
3935 mIndexStats
.Size() >> 10);
3939 void CacheIndex::OnAsyncEviction(bool aEvicting
) {
3940 StaticMutexAutoLock
lock(sLock
);
3942 RefPtr
<CacheIndex
> index
= gInstance
;
3947 index
->mAsyncGetDiskConsumptionBlocked
= aEvicting
;
3949 index
->NotifyAsyncGetDiskConsumptionCallbacks();
3954 } // namespace mozilla