1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
5 #include "CacheIndex.h"
8 #include "CacheFileIOManager.h"
9 #include "CacheFileMetadata.h"
10 #include "CacheFileUtils.h"
11 #include "CacheIndexIterator.h"
12 #include "CacheIndexContextIterator.h"
13 #include "nsThreadUtils.h"
14 #include "nsISizeOf.h"
15 #include "nsPrintfCString.h"
16 #include "mozilla/DebugOnly.h"
20 #include "mozilla/AutoRestore.h"
22 #include "mozilla/StaticPrefs_network.h"
23 #include "mozilla/Telemetry.h"
24 #include "mozilla/Unused.h"
26 #define kMinUnwrittenChanges 300
27 #define kMinDumpInterval 20000 // in milliseconds
28 #define kMaxBufSize 16384
29 #define kIndexVersion 0x0000000A
30 #define kUpdateIndexStartDelay 50000 // in milliseconds
31 #define kTelemetryReportBytesLimit (2U * 1024U * 1024U * 1024U) // 2GB
33 #define INDEX_NAME "index"
34 #define TEMP_INDEX_NAME "index.tmp"
35 #define JOURNAL_NAME "index.log"
37 namespace mozilla::net
{
41 class FrecencyComparator
{
43 bool Equals(const RefPtr
<CacheIndexRecordWrapper
>& a
,
44 const RefPtr
<CacheIndexRecordWrapper
>& b
) const {
49 return a
->Get()->mFrecency
== b
->Get()->mFrecency
;
51 bool LessThan(const RefPtr
<CacheIndexRecordWrapper
>& a
,
52 const RefPtr
<CacheIndexRecordWrapper
>& b
) const {
53 // Removed (=null) entries must be at the end of the array.
61 // Place entries with frecency 0 at the end of the non-removed entries.
62 if (a
->Get()->mFrecency
== 0) {
65 if (b
->Get()->mFrecency
== 0) {
69 return a
->Get()->mFrecency
< b
->Get()->mFrecency
;
75 // used to dispatch a wrapper deletion the caller's thread
76 // cannot be used on IOThread after shutdown begins
77 class DeleteCacheIndexRecordWrapper
: public Runnable
{
78 CacheIndexRecordWrapper
* mWrapper
;
81 explicit DeleteCacheIndexRecordWrapper(CacheIndexRecordWrapper
* wrapper
)
82 : Runnable("net::CacheIndex::DeleteCacheIndexRecordWrapper"),
84 NS_IMETHOD
Run() override
{
85 StaticMutexAutoLock
lock(CacheIndex::sLock
);
87 // if somehow the item is still in the frecency array, remove it
88 RefPtr
<CacheIndex
> index
= CacheIndex::gInstance
;
90 bool found
= index
->mFrecencyArray
.RecordExistedUnlocked(mWrapper
);
93 ("DeleteCacheIndexRecordWrapper::Run() - \
94 record wrapper found in frecency array during deletion"));
95 index
->mFrecencyArray
.RemoveRecord(mWrapper
, lock
);
104 void CacheIndexRecordWrapper::DispatchDeleteSelfToCurrentThread() {
105 // Dispatch during shutdown will not trigger DeleteCacheIndexRecordWrapper
106 nsCOMPtr
<nsIRunnable
> event
= new DeleteCacheIndexRecordWrapper(this);
107 MOZ_ALWAYS_SUCCEEDS(NS_DispatchToCurrentThread(event
));
110 CacheIndexRecordWrapper::~CacheIndexRecordWrapper() {
111 #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
112 CacheIndex::sLock
.AssertCurrentThreadOwns();
113 RefPtr
<CacheIndex
> index
= CacheIndex::gInstance
;
115 bool found
= index
->mFrecencyArray
.RecordExistedUnlocked(this);
116 MOZ_DIAGNOSTIC_ASSERT(!found
);
122 * This helper class is responsible for keeping CacheIndex::mIndexStats and
123 * CacheIndex::mFrecencyArray up to date.
125 class MOZ_RAII CacheIndexEntryAutoManage
{
127 CacheIndexEntryAutoManage(const SHA1Sum::Hash
* aHash
, CacheIndex
* aIndex
,
128 const StaticMutexAutoLock
& aProofOfLock
)
129 MOZ_REQUIRES(CacheIndex::sLock
)
130 : mIndex(aIndex
), mProofOfLock(aProofOfLock
) {
132 const CacheIndexEntry
* entry
= FindEntry();
133 mIndex
->mIndexStats
.BeforeChange(entry
);
134 if (entry
&& entry
->IsInitialized() && !entry
->IsRemoved()) {
135 mOldRecord
= entry
->mRec
;
136 mOldFrecency
= entry
->mRec
->Get()->mFrecency
;
140 ~CacheIndexEntryAutoManage() MOZ_REQUIRES(CacheIndex::sLock
) {
141 const CacheIndexEntry
* entry
= FindEntry();
142 mIndex
->mIndexStats
.AfterChange(entry
);
143 if (!entry
|| !entry
->IsInitialized() || entry
->IsRemoved()) {
147 if (entry
&& !mOldRecord
) {
148 mIndex
->mFrecencyArray
.AppendRecord(entry
->mRec
, mProofOfLock
);
149 mIndex
->AddRecordToIterators(entry
->mRec
, mProofOfLock
);
150 } else if (!entry
&& mOldRecord
) {
151 mIndex
->mFrecencyArray
.RemoveRecord(mOldRecord
, mProofOfLock
);
152 mIndex
->RemoveRecordFromIterators(mOldRecord
, mProofOfLock
);
153 } else if (entry
&& mOldRecord
) {
154 if (entry
->mRec
!= mOldRecord
) {
155 // record has a different address, we have to replace it
156 mIndex
->ReplaceRecordInIterators(mOldRecord
, entry
->mRec
, mProofOfLock
);
158 if (entry
->mRec
->Get()->mFrecency
== mOldFrecency
) {
159 // If frecency hasn't changed simply replace the pointer
160 mIndex
->mFrecencyArray
.ReplaceRecord(mOldRecord
, entry
->mRec
,
163 // Remove old pointer and insert the new one at the end of the array
164 mIndex
->mFrecencyArray
.RemoveRecord(mOldRecord
, mProofOfLock
);
165 mIndex
->mFrecencyArray
.AppendRecord(entry
->mRec
, mProofOfLock
);
167 } else if (entry
->mRec
->Get()->mFrecency
!= mOldFrecency
) {
168 // Move the element at the end of the array
169 mIndex
->mFrecencyArray
.RemoveRecord(entry
->mRec
, mProofOfLock
);
170 mIndex
->mFrecencyArray
.AppendRecord(entry
->mRec
, mProofOfLock
);
173 // both entries were removed or not initialized, do nothing
177 // We cannot rely on nsTHashtable::GetEntry() in case we are removing entries
178 // while iterating. Destructor is called before the entry is removed. Caller
179 // must call one of following methods to skip lookup in the hashtable.
180 void DoNotSearchInIndex() { mDoNotSearchInIndex
= true; }
181 void DoNotSearchInUpdates() { mDoNotSearchInUpdates
= true; }
184 const CacheIndexEntry
* FindEntry() MOZ_REQUIRES(CacheIndex::sLock
) {
185 const CacheIndexEntry
* entry
= nullptr;
187 switch (mIndex
->mState
) {
188 case CacheIndex::READING
:
189 case CacheIndex::WRITING
:
190 if (!mDoNotSearchInUpdates
) {
191 entry
= mIndex
->mPendingUpdates
.GetEntry(*mHash
);
194 case CacheIndex::BUILDING
:
195 case CacheIndex::UPDATING
:
196 case CacheIndex::READY
:
197 if (!entry
&& !mDoNotSearchInIndex
) {
198 entry
= mIndex
->mIndex
.GetEntry(*mHash
);
201 case CacheIndex::INITIAL
:
202 case CacheIndex::SHUTDOWN
:
204 MOZ_ASSERT(false, "Unexpected state!");
210 const SHA1Sum::Hash
* mHash
;
211 RefPtr
<CacheIndex
> mIndex
;
212 RefPtr
<CacheIndexRecordWrapper
> mOldRecord
;
213 uint32_t mOldFrecency
{0};
214 bool mDoNotSearchInIndex
{false};
215 bool mDoNotSearchInUpdates
{false};
216 const StaticMutexAutoLock
& mProofOfLock
;
219 class FileOpenHelper final
: public CacheFileIOListener
{
221 NS_DECL_THREADSAFE_ISUPPORTS
223 explicit FileOpenHelper(CacheIndex
* aIndex
)
224 : mIndex(aIndex
), mCanceled(false) {}
227 CacheIndex::sLock
.AssertCurrentThreadOwns();
232 virtual ~FileOpenHelper() = default;
234 NS_IMETHOD
OnFileOpened(CacheFileHandle
* aHandle
, nsresult aResult
) override
;
235 NS_IMETHOD
OnDataWritten(CacheFileHandle
* aHandle
, const char* aBuf
,
236 nsresult aResult
) override
{
237 MOZ_CRASH("FileOpenHelper::OnDataWritten should not be called!");
238 return NS_ERROR_UNEXPECTED
;
240 NS_IMETHOD
OnDataRead(CacheFileHandle
* aHandle
, char* aBuf
,
241 nsresult aResult
) override
{
242 MOZ_CRASH("FileOpenHelper::OnDataRead should not be called!");
243 return NS_ERROR_UNEXPECTED
;
245 NS_IMETHOD
OnFileDoomed(CacheFileHandle
* aHandle
, nsresult aResult
) override
{
246 MOZ_CRASH("FileOpenHelper::OnFileDoomed should not be called!");
247 return NS_ERROR_UNEXPECTED
;
249 NS_IMETHOD
OnEOFSet(CacheFileHandle
* aHandle
, nsresult aResult
) override
{
250 MOZ_CRASH("FileOpenHelper::OnEOFSet should not be called!");
251 return NS_ERROR_UNEXPECTED
;
253 NS_IMETHOD
OnFileRenamed(CacheFileHandle
* aHandle
,
254 nsresult aResult
) override
{
255 MOZ_CRASH("FileOpenHelper::OnFileRenamed should not be called!");
256 return NS_ERROR_UNEXPECTED
;
259 RefPtr
<CacheIndex
> mIndex
;
263 NS_IMETHODIMP
FileOpenHelper::OnFileOpened(CacheFileHandle
* aHandle
,
265 StaticMutexAutoLock
lock(CacheIndex::sLock
);
269 CacheFileIOManager::DoomFile(aHandle
, nullptr);
275 mIndex
->OnFileOpenedInternal(this, aHandle
, aResult
, lock
);
280 NS_IMPL_ISUPPORTS(FileOpenHelper
, CacheFileIOListener
);
282 StaticRefPtr
<CacheIndex
> CacheIndex::gInstance
;
283 StaticMutex
CacheIndex::sLock
;
285 NS_IMPL_ADDREF(CacheIndex
)
286 NS_IMPL_RELEASE(CacheIndex
)
288 NS_INTERFACE_MAP_BEGIN(CacheIndex
)
289 NS_INTERFACE_MAP_ENTRY(mozilla::net::CacheFileIOListener
)
290 NS_INTERFACE_MAP_ENTRY(nsIRunnable
)
293 CacheIndex::CacheIndex() {
294 sLock
.AssertCurrentThreadOwns();
295 LOG(("CacheIndex::CacheIndex [this=%p]", this));
296 MOZ_ASSERT(!gInstance
, "multiple CacheIndex instances!");
299 CacheIndex::~CacheIndex() {
300 sLock
.AssertCurrentThreadOwns();
301 LOG(("CacheIndex::~CacheIndex [this=%p]", this));
307 nsresult
CacheIndex::Init(nsIFile
* aCacheDirectory
) {
308 LOG(("CacheIndex::Init()"));
310 MOZ_ASSERT(NS_IsMainThread());
312 StaticMutexAutoLock
lock(sLock
);
315 return NS_ERROR_ALREADY_INITIALIZED
;
318 RefPtr
<CacheIndex
> idx
= new CacheIndex();
320 nsresult rv
= idx
->InitInternal(aCacheDirectory
, lock
);
321 NS_ENSURE_SUCCESS(rv
, rv
);
323 gInstance
= std::move(idx
);
327 nsresult
CacheIndex::InitInternal(nsIFile
* aCacheDirectory
,
328 const StaticMutexAutoLock
& aProofOfLock
) {
330 sLock
.AssertCurrentThreadOwns();
332 rv
= aCacheDirectory
->Clone(getter_AddRefs(mCacheDirectory
));
333 NS_ENSURE_SUCCESS(rv
, rv
);
335 mStartTime
= TimeStamp::NowLoRes();
337 ReadIndexFromDisk(aProofOfLock
);
343 nsresult
CacheIndex::PreShutdown() {
344 MOZ_ASSERT(NS_IsMainThread());
346 StaticMutexAutoLock
lock(sLock
);
348 LOG(("CacheIndex::PreShutdown() [gInstance=%p]", gInstance
.get()));
351 RefPtr
<CacheIndex
> index
= gInstance
;
354 return NS_ERROR_NOT_INITIALIZED
;
358 ("CacheIndex::PreShutdown() - [state=%d, indexOnDiskIsValid=%d, "
359 "dontMarkIndexClean=%d]",
360 index
->mState
, index
->mIndexOnDiskIsValid
, index
->mDontMarkIndexClean
));
362 LOG(("CacheIndex::PreShutdown() - Closing iterators."));
363 for (uint32_t i
= 0; i
< index
->mIterators
.Length();) {
364 rv
= index
->mIterators
[i
]->CloseInternal(NS_ERROR_FAILURE
);
366 // CacheIndexIterator::CloseInternal() removes itself from mIteratos iff
367 // it returns success.
369 ("CacheIndex::PreShutdown() - Failed to remove iterator %p. "
370 "[rv=0x%08" PRIx32
"]",
371 index
->mIterators
[i
], static_cast<uint32_t>(rv
)));
376 index
->mShuttingDown
= true;
378 if (index
->mState
== READY
) {
379 return NS_OK
; // nothing to do
382 nsCOMPtr
<nsIRunnable
> event
;
383 event
= NewRunnableMethod("net::CacheIndex::PreShutdownInternal", index
,
384 &CacheIndex::PreShutdownInternal
);
386 nsCOMPtr
<nsIEventTarget
> ioTarget
= CacheFileIOManager::IOTarget();
387 MOZ_ASSERT(ioTarget
);
389 // PreShutdownInternal() will be executed before any queued event on INDEX
390 // level. That's OK since we don't want to wait for any operation in progess.
391 rv
= ioTarget
->Dispatch(event
, nsIEventTarget::DISPATCH_NORMAL
);
393 NS_WARNING("CacheIndex::PreShutdown() - Can't dispatch event");
394 LOG(("CacheIndex::PreShutdown() - Can't dispatch event"));
401 void CacheIndex::PreShutdownInternal() {
402 StaticMutexAutoLock
lock(sLock
);
405 ("CacheIndex::PreShutdownInternal() - [state=%d, indexOnDiskIsValid=%d, "
406 "dontMarkIndexClean=%d]",
407 mState
, mIndexOnDiskIsValid
, mDontMarkIndexClean
));
409 MOZ_ASSERT(mShuttingDown
);
412 mUpdateTimer
->Cancel();
413 mUpdateTimer
= nullptr;
418 FinishWrite(false, lock
);
421 // nothing to do, write the journal in Shutdown()
424 FinishRead(false, lock
);
428 FinishUpdate(false, lock
);
431 MOZ_ASSERT(false, "Implement me!");
434 // We should end up in READY state
435 MOZ_ASSERT(mState
== READY
);
439 nsresult
CacheIndex::Shutdown() {
440 MOZ_ASSERT(NS_IsMainThread());
442 StaticMutexAutoLock
lock(sLock
);
444 LOG(("CacheIndex::Shutdown() [gInstance=%p]", gInstance
.get()));
446 RefPtr
<CacheIndex
> index
= gInstance
.forget();
449 return NS_ERROR_NOT_INITIALIZED
;
452 bool sanitize
= CacheObserver::ClearCacheOnShutdown();
455 ("CacheIndex::Shutdown() - [state=%d, indexOnDiskIsValid=%d, "
456 "dontMarkIndexClean=%d, sanitize=%d]",
457 index
->mState
, index
->mIndexOnDiskIsValid
, index
->mDontMarkIndexClean
,
460 MOZ_ASSERT(index
->mShuttingDown
);
462 EState oldState
= index
->mState
;
463 index
->ChangeState(SHUTDOWN
, lock
);
465 if (oldState
!= READY
) {
467 ("CacheIndex::Shutdown() - Unexpected state. Did posting of "
468 "PreShutdownInternal() fail?"));
473 index
->FinishWrite(false, lock
);
476 if (index
->mIndexOnDiskIsValid
&& !index
->mDontMarkIndexClean
) {
477 if (!sanitize
&& NS_FAILED(index
->WriteLogToDisk())) {
478 index
->RemoveJournalAndTempFile();
481 index
->RemoveJournalAndTempFile();
485 index
->FinishRead(false, lock
);
489 index
->FinishUpdate(false, lock
);
492 MOZ_ASSERT(false, "Unexpected state!");
496 index
->RemoveAllIndexFiles();
503 nsresult
CacheIndex::AddEntry(const SHA1Sum::Hash
* aHash
) {
504 LOG(("CacheIndex::AddEntry() [hash=%08x%08x%08x%08x%08x]", LOGSHA1(aHash
)));
506 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
508 StaticMutexAutoLock
lock(sLock
);
510 RefPtr
<CacheIndex
> index
= gInstance
;
513 return NS_ERROR_NOT_INITIALIZED
;
516 if (!index
->IsIndexUsable()) {
517 return NS_ERROR_NOT_AVAILABLE
;
520 // Getters in CacheIndexStats assert when mStateLogged is true since the
521 // information is incomplete between calls to BeforeChange() and AfterChange()
522 // (i.e. while CacheIndexEntryAutoManage exists). We need to check whether
523 // non-fresh entries exists outside the scope of CacheIndexEntryAutoManage.
524 bool updateIfNonFreshEntriesExist
= false;
527 CacheIndexEntryAutoManage
entryMng(aHash
, index
, lock
);
529 CacheIndexEntry
* entry
= index
->mIndex
.GetEntry(*aHash
);
530 bool entryRemoved
= entry
&& entry
->IsRemoved();
531 CacheIndexEntryUpdate
* updated
= nullptr;
533 if (index
->mState
== READY
|| index
->mState
== UPDATING
||
534 index
->mState
== BUILDING
) {
535 MOZ_ASSERT(index
->mPendingUpdates
.Count() == 0);
537 if (entry
&& !entryRemoved
) {
538 // Found entry in index that shouldn't exist.
540 if (entry
->IsFresh()) {
541 // Someone removed the file on disk while FF is running. Update
542 // process can fix only non-fresh entries (i.e. entries that were not
543 // added within this session). Start update only if we have such
546 // TODO: This should be very rare problem. If it turns out not to be
547 // true, change the update process so that it also iterates all
548 // initialized non-empty entries and checks whether the file exists.
551 ("CacheIndex::AddEntry() - Cache file was removed outside FF "
554 updateIfNonFreshEntriesExist
= true;
555 } else if (index
->mState
== READY
) {
556 // Index is outdated, update it.
558 ("CacheIndex::AddEntry() - Found entry that shouldn't exist, "
559 "update is needed"));
560 index
->mIndexNeedsUpdate
= true;
562 // We cannot be here when building index since all entries are fresh
564 MOZ_ASSERT(index
->mState
== UPDATING
);
569 entry
= index
->mIndex
.PutEntry(*aHash
);
571 } else { // WRITING, READING
572 updated
= index
->mPendingUpdates
.GetEntry(*aHash
);
573 bool updatedRemoved
= updated
&& updated
->IsRemoved();
575 if ((updated
&& !updatedRemoved
) ||
576 (!updated
&& entry
&& !entryRemoved
&& entry
->IsFresh())) {
577 // Fresh entry found, so the file was removed outside FF
579 ("CacheIndex::AddEntry() - Cache file was removed outside FF "
582 updateIfNonFreshEntriesExist
= true;
583 } else if (!updated
&& entry
&& !entryRemoved
) {
584 if (index
->mState
== WRITING
) {
586 ("CacheIndex::AddEntry() - Found entry that shouldn't exist, "
587 "update is needed"));
588 index
->mIndexNeedsUpdate
= true;
590 // Ignore if state is READING since the index information is partial
593 updated
= index
->mPendingUpdates
.PutEntry(*aHash
);
598 updated
->MarkDirty();
599 updated
->MarkFresh();
607 if (updateIfNonFreshEntriesExist
&&
608 index
->mIndexStats
.Count() != index
->mIndexStats
.Fresh()) {
609 index
->mIndexNeedsUpdate
= true;
612 index
->StartUpdatingIndexIfNeeded(lock
);
613 index
->WriteIndexToDiskIfNeeded(lock
);
619 nsresult
CacheIndex::EnsureEntryExists(const SHA1Sum::Hash
* aHash
) {
620 LOG(("CacheIndex::EnsureEntryExists() [hash=%08x%08x%08x%08x%08x]",
623 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
625 StaticMutexAutoLock
lock(sLock
);
627 RefPtr
<CacheIndex
> index
= gInstance
;
630 return NS_ERROR_NOT_INITIALIZED
;
633 if (!index
->IsIndexUsable()) {
634 return NS_ERROR_NOT_AVAILABLE
;
638 CacheIndexEntryAutoManage
entryMng(aHash
, index
, lock
);
640 CacheIndexEntry
* entry
= index
->mIndex
.GetEntry(*aHash
);
641 bool entryRemoved
= entry
&& entry
->IsRemoved();
643 if (index
->mState
== READY
|| index
->mState
== UPDATING
||
644 index
->mState
== BUILDING
) {
645 MOZ_ASSERT(index
->mPendingUpdates
.Count() == 0);
647 if (!entry
|| entryRemoved
) {
648 if (entryRemoved
&& entry
->IsFresh()) {
649 // This could happen only if somebody copies files to the entries
650 // directory while FF is running.
652 ("CacheIndex::EnsureEntryExists() - Cache file was added outside "
653 "FF process! Update is needed."));
654 index
->mIndexNeedsUpdate
= true;
655 } else if (index
->mState
== READY
||
656 (entryRemoved
&& !entry
->IsFresh())) {
657 // Removed non-fresh entries can be present as a result of
660 ("CacheIndex::EnsureEntryExists() - Didn't find entry that should"
661 " exist, update is needed"));
662 index
->mIndexNeedsUpdate
= true;
666 entry
= index
->mIndex
.PutEntry(*aHash
);
672 } else { // WRITING, READING
673 CacheIndexEntryUpdate
* updated
= index
->mPendingUpdates
.GetEntry(*aHash
);
674 bool updatedRemoved
= updated
&& updated
->IsRemoved();
676 if (updatedRemoved
|| (!updated
&& entryRemoved
&& entry
->IsFresh())) {
677 // Fresh information about missing entry found. This could happen only
678 // if somebody copies files to the entries directory while FF is
681 ("CacheIndex::EnsureEntryExists() - Cache file was added outside "
682 "FF process! Update is needed."));
683 index
->mIndexNeedsUpdate
= true;
684 } else if (!updated
&& (!entry
|| entryRemoved
)) {
685 if (index
->mState
== WRITING
) {
687 ("CacheIndex::EnsureEntryExists() - Didn't find entry that should"
688 " exist, update is needed"));
689 index
->mIndexNeedsUpdate
= true;
691 // Ignore if state is READING since the index information is partial
694 // We don't need entryRemoved and updatedRemoved info anymore
695 if (entryRemoved
) entry
= nullptr;
696 if (updatedRemoved
) updated
= nullptr;
699 updated
->MarkFresh();
702 // Create a new entry
703 updated
= index
->mPendingUpdates
.PutEntry(*aHash
);
705 updated
->MarkFresh();
706 updated
->MarkDirty();
708 if (!entry
->IsFresh()) {
709 // To mark the entry fresh we must make a copy of index entry
710 // since the index is read-only.
711 updated
= index
->mPendingUpdates
.PutEntry(*aHash
);
713 updated
->MarkFresh();
720 index
->StartUpdatingIndexIfNeeded(lock
);
721 index
->WriteIndexToDiskIfNeeded(lock
);
727 nsresult
CacheIndex::InitEntry(const SHA1Sum::Hash
* aHash
,
728 OriginAttrsHash aOriginAttrsHash
,
729 bool aAnonymous
, bool aPinned
) {
731 ("CacheIndex::InitEntry() [hash=%08x%08x%08x%08x%08x, "
732 "originAttrsHash=%" PRIx64
", anonymous=%d, pinned=%d]",
733 LOGSHA1(aHash
), aOriginAttrsHash
, aAnonymous
, aPinned
));
735 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
737 StaticMutexAutoLock
lock(sLock
);
739 RefPtr
<CacheIndex
> index
= gInstance
;
742 return NS_ERROR_NOT_INITIALIZED
;
745 if (!index
->IsIndexUsable()) {
746 return NS_ERROR_NOT_AVAILABLE
;
750 CacheIndexEntryAutoManage
entryMng(aHash
, index
, lock
);
752 CacheIndexEntry
* entry
= index
->mIndex
.GetEntry(*aHash
);
753 CacheIndexEntryUpdate
* updated
= nullptr;
754 bool reinitEntry
= false;
756 if (entry
&& entry
->IsRemoved()) {
760 if (index
->mState
== READY
|| index
->mState
== UPDATING
||
761 index
->mState
== BUILDING
) {
762 MOZ_ASSERT(index
->mPendingUpdates
.Count() == 0);
764 MOZ_ASSERT(entry
->IsFresh());
767 LOG(("CacheIndex::InitEntry() - Entry was not found in mIndex!"));
769 ("CacheIndex::InitEntry() - Entry was not found in mIndex!"));
770 return NS_ERROR_UNEXPECTED
;
773 if (IsCollision(entry
, aOriginAttrsHash
, aAnonymous
)) {
774 index
->mIndexNeedsUpdate
=
775 true; // TODO Does this really help in case of collision?
778 if (entry
->IsInitialized()) {
783 updated
= index
->mPendingUpdates
.GetEntry(*aHash
);
784 DebugOnly
<bool> removed
= updated
&& updated
->IsRemoved();
786 MOZ_ASSERT(updated
|| !removed
);
787 MOZ_ASSERT(updated
|| entry
);
789 if (!updated
&& !entry
) {
791 ("CacheIndex::InitEntry() - Entry was found neither in mIndex nor "
792 "in mPendingUpdates!"));
794 ("CacheIndex::InitEntry() - Entry was found neither in "
795 "mIndex nor in mPendingUpdates!"));
796 return NS_ERROR_UNEXPECTED
;
800 MOZ_ASSERT(updated
->IsFresh());
802 if (IsCollision(updated
, aOriginAttrsHash
, aAnonymous
)) {
803 index
->mIndexNeedsUpdate
= true;
806 if (updated
->IsInitialized()) {
811 MOZ_ASSERT(entry
->IsFresh());
813 if (IsCollision(entry
, aOriginAttrsHash
, aAnonymous
)) {
814 index
->mIndexNeedsUpdate
= true;
817 if (entry
->IsInitialized()) {
822 // make a copy of a read-only entry
823 updated
= index
->mPendingUpdates
.PutEntry(*aHash
);
829 // There is a collision and we are going to rewrite this entry. Initialize
830 // it as a new entry.
833 updated
->MarkFresh();
841 updated
->Init(aOriginAttrsHash
, aAnonymous
, aPinned
);
842 updated
->MarkDirty();
844 entry
->Init(aOriginAttrsHash
, aAnonymous
, aPinned
);
849 index
->StartUpdatingIndexIfNeeded(lock
);
850 index
->WriteIndexToDiskIfNeeded(lock
);
856 nsresult
CacheIndex::RemoveEntry(const SHA1Sum::Hash
* aHash
) {
857 LOG(("CacheIndex::RemoveEntry() [hash=%08x%08x%08x%08x%08x]",
860 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
862 StaticMutexAutoLock
lock(sLock
);
864 RefPtr
<CacheIndex
> index
= gInstance
;
867 return NS_ERROR_NOT_INITIALIZED
;
870 if (!index
->IsIndexUsable()) {
871 return NS_ERROR_NOT_AVAILABLE
;
875 CacheIndexEntryAutoManage
entryMng(aHash
, index
, lock
);
877 CacheIndexEntry
* entry
= index
->mIndex
.GetEntry(*aHash
);
878 bool entryRemoved
= entry
&& entry
->IsRemoved();
880 if (index
->mState
== READY
|| index
->mState
== UPDATING
||
881 index
->mState
== BUILDING
) {
882 MOZ_ASSERT(index
->mPendingUpdates
.Count() == 0);
884 if (!entry
|| entryRemoved
) {
885 if (entryRemoved
&& entry
->IsFresh()) {
886 // This could happen only if somebody copies files to the entries
887 // directory while FF is running.
889 ("CacheIndex::RemoveEntry() - Cache file was added outside FF "
890 "process! Update is needed."));
891 index
->mIndexNeedsUpdate
= true;
892 } else if (index
->mState
== READY
||
893 (entryRemoved
&& !entry
->IsFresh())) {
894 // Removed non-fresh entries can be present as a result of
897 ("CacheIndex::RemoveEntry() - Didn't find entry that should exist"
898 ", update is needed"));
899 index
->mIndexNeedsUpdate
= true;
903 if (!entry
->IsDirty() && entry
->IsFileEmpty()) {
904 index
->mIndex
.RemoveEntry(entry
);
907 entry
->MarkRemoved();
913 } else { // WRITING, READING
914 CacheIndexEntryUpdate
* updated
= index
->mPendingUpdates
.GetEntry(*aHash
);
915 bool updatedRemoved
= updated
&& updated
->IsRemoved();
917 if (updatedRemoved
|| (!updated
&& entryRemoved
&& entry
->IsFresh())) {
918 // Fresh information about missing entry found. This could happen only
919 // if somebody copies files to the entries directory while FF is
922 ("CacheIndex::RemoveEntry() - Cache file was added outside FF "
923 "process! Update is needed."));
924 index
->mIndexNeedsUpdate
= true;
925 } else if (!updated
&& (!entry
|| entryRemoved
)) {
926 if (index
->mState
== WRITING
) {
928 ("CacheIndex::RemoveEntry() - Didn't find entry that should exist"
929 ", update is needed"));
930 index
->mIndexNeedsUpdate
= true;
932 // Ignore if state is READING since the index information is partial
936 updated
= index
->mPendingUpdates
.PutEntry(*aHash
);
940 updated
->MarkRemoved();
941 updated
->MarkDirty();
942 updated
->MarkFresh();
945 index
->StartUpdatingIndexIfNeeded(lock
);
946 index
->WriteIndexToDiskIfNeeded(lock
);
952 nsresult
CacheIndex::UpdateEntry(const SHA1Sum::Hash
* aHash
,
953 const uint32_t* aFrecency
,
954 const bool* aHasAltData
,
955 const uint16_t* aOnStartTime
,
956 const uint16_t* aOnStopTime
,
957 const uint8_t* aContentType
,
958 const uint32_t* aSize
) {
960 ("CacheIndex::UpdateEntry() [hash=%08x%08x%08x%08x%08x, "
961 "frecency=%s, hasAltData=%s, onStartTime=%s, onStopTime=%s, "
962 "contentType=%s, size=%s]",
963 LOGSHA1(aHash
), aFrecency
? nsPrintfCString("%u", *aFrecency
).get() : "",
964 aHasAltData
? (*aHasAltData
? "true" : "false") : "",
965 aOnStartTime
? nsPrintfCString("%u", *aOnStartTime
).get() : "",
966 aOnStopTime
? nsPrintfCString("%u", *aOnStopTime
).get() : "",
967 aContentType
? nsPrintfCString("%u", *aContentType
).get() : "",
968 aSize
? nsPrintfCString("%u", *aSize
).get() : ""));
970 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
972 StaticMutexAutoLock
lock(sLock
);
974 RefPtr
<CacheIndex
> index
= gInstance
;
977 return NS_ERROR_NOT_INITIALIZED
;
980 if (!index
->IsIndexUsable()) {
981 return NS_ERROR_NOT_AVAILABLE
;
985 CacheIndexEntryAutoManage
entryMng(aHash
, index
, lock
);
987 CacheIndexEntry
* entry
= index
->mIndex
.GetEntry(*aHash
);
989 if (entry
&& entry
->IsRemoved()) {
993 if (index
->mState
== READY
|| index
->mState
== UPDATING
||
994 index
->mState
== BUILDING
) {
995 MOZ_ASSERT(index
->mPendingUpdates
.Count() == 0);
999 LOG(("CacheIndex::UpdateEntry() - Entry was not found in mIndex!"));
1001 ("CacheIndex::UpdateEntry() - Entry was not found in mIndex!"));
1002 return NS_ERROR_UNEXPECTED
;
1005 if (!HasEntryChanged(entry
, aFrecency
, aHasAltData
, aOnStartTime
,
1006 aOnStopTime
, aContentType
, aSize
)) {
1010 MOZ_ASSERT(entry
->IsFresh());
1011 MOZ_ASSERT(entry
->IsInitialized());
1015 entry
->SetFrecency(*aFrecency
);
1019 entry
->SetHasAltData(*aHasAltData
);
1023 entry
->SetOnStartTime(*aOnStartTime
);
1027 entry
->SetOnStopTime(*aOnStopTime
);
1031 entry
->SetContentType(*aContentType
);
1035 entry
->SetFileSize(*aSize
);
1038 CacheIndexEntryUpdate
* updated
= index
->mPendingUpdates
.GetEntry(*aHash
);
1039 DebugOnly
<bool> removed
= updated
&& updated
->IsRemoved();
1041 MOZ_ASSERT(updated
|| !removed
);
1042 MOZ_ASSERT(updated
|| entry
);
1047 ("CacheIndex::UpdateEntry() - Entry was found neither in mIndex "
1048 "nor in mPendingUpdates!"));
1050 ("CacheIndex::UpdateEntry() - Entry was found neither in "
1051 "mIndex nor in mPendingUpdates!"));
1052 return NS_ERROR_UNEXPECTED
;
1055 // make a copy of a read-only entry
1056 updated
= index
->mPendingUpdates
.PutEntry(*aHash
);
1060 MOZ_ASSERT(updated
->IsFresh());
1061 MOZ_ASSERT(updated
->IsInitialized());
1062 updated
->MarkDirty();
1065 updated
->SetFrecency(*aFrecency
);
1069 updated
->SetHasAltData(*aHasAltData
);
1073 updated
->SetOnStartTime(*aOnStartTime
);
1077 updated
->SetOnStopTime(*aOnStopTime
);
1081 updated
->SetContentType(*aContentType
);
1085 updated
->SetFileSize(*aSize
);
1090 index
->WriteIndexToDiskIfNeeded(lock
);
1096 nsresult
CacheIndex::RemoveAll() {
1097 LOG(("CacheIndex::RemoveAll()"));
1099 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
1101 nsCOMPtr
<nsIFile
> file
;
1104 StaticMutexAutoLock
lock(sLock
);
1106 RefPtr
<CacheIndex
> index
= gInstance
;
1109 return NS_ERROR_NOT_INITIALIZED
;
1112 MOZ_ASSERT(!index
->mRemovingAll
);
1114 if (!index
->IsIndexUsable()) {
1115 return NS_ERROR_NOT_AVAILABLE
;
1118 AutoRestore
<bool> saveRemovingAll(index
->mRemovingAll
);
1119 index
->mRemovingAll
= true;
1121 // Doom index and journal handles but don't null them out since this will be
1122 // done in FinishWrite/FinishRead methods.
1123 if (index
->mIndexHandle
) {
1124 CacheFileIOManager::DoomFile(index
->mIndexHandle
, nullptr);
1126 // We don't have a handle to index file, so get the file here, but delete
1127 // it outside the lock. Ignore the result since this is not fatal.
1128 index
->GetFile(nsLiteralCString(INDEX_NAME
), getter_AddRefs(file
));
1131 if (index
->mJournalHandle
) {
1132 CacheFileIOManager::DoomFile(index
->mJournalHandle
, nullptr);
1135 switch (index
->mState
) {
1137 index
->FinishWrite(false, lock
);
1143 index
->FinishRead(false, lock
);
1147 index
->FinishUpdate(false, lock
);
1150 MOZ_ASSERT(false, "Unexpected state!");
1153 // We should end up in READY state
1154 MOZ_ASSERT(index
->mState
== READY
);
1156 // There should not be any handle
1157 MOZ_ASSERT(!index
->mIndexHandle
);
1158 MOZ_ASSERT(!index
->mJournalHandle
);
1160 index
->mIndexOnDiskIsValid
= false;
1161 index
->mIndexNeedsUpdate
= false;
1163 index
->mIndexStats
.Clear();
1164 index
->mFrecencyArray
.Clear(lock
);
1165 index
->mIndex
.Clear();
1167 for (uint32_t i
= 0; i
< index
->mIterators
.Length();) {
1168 nsresult rv
= index
->mIterators
[i
]->CloseInternal(NS_ERROR_NOT_AVAILABLE
);
1169 if (NS_FAILED(rv
)) {
1170 // CacheIndexIterator::CloseInternal() removes itself from mIterators
1171 // iff it returns success.
1173 ("CacheIndex::RemoveAll() - Failed to remove iterator %p. "
1174 "[rv=0x%08" PRIx32
"]",
1175 index
->mIterators
[i
], static_cast<uint32_t>(rv
)));
1182 // Ignore the result. The file might not exist and the failure is not fatal.
1183 file
->Remove(false);
1190 nsresult
CacheIndex::HasEntry(
1191 const nsACString
& aKey
, EntryStatus
* _retval
,
1192 const std::function
<void(const CacheIndexEntry
*)>& aCB
) {
1193 LOG(("CacheIndex::HasEntry() [key=%s]", PromiseFlatCString(aKey
).get()));
1197 sum
.update(aKey
.BeginReading(), aKey
.Length());
1200 return HasEntry(hash
, _retval
, aCB
);
1204 nsresult
CacheIndex::HasEntry(
1205 const SHA1Sum::Hash
& hash
, EntryStatus
* _retval
,
1206 const std::function
<void(const CacheIndexEntry
*)>& aCB
) {
1207 StaticMutexAutoLock
lock(sLock
);
1209 RefPtr
<CacheIndex
> index
= gInstance
;
1212 return NS_ERROR_NOT_INITIALIZED
;
1215 if (!index
->IsIndexUsable()) {
1216 return NS_ERROR_NOT_AVAILABLE
;
1219 const CacheIndexEntry
* entry
= nullptr;
1221 switch (index
->mState
) {
1224 entry
= index
->mPendingUpdates
.GetEntry(hash
);
1230 entry
= index
->mIndex
.GetEntry(hash
);
1235 MOZ_ASSERT(false, "Unexpected state!");
1239 if (index
->mState
== READY
|| index
->mState
== WRITING
) {
1240 *_retval
= DOES_NOT_EXIST
;
1242 *_retval
= DO_NOT_KNOW
;
1245 if (entry
->IsRemoved()) {
1246 if (entry
->IsFresh()) {
1247 *_retval
= DOES_NOT_EXIST
;
1249 *_retval
= DO_NOT_KNOW
;
1259 LOG(("CacheIndex::HasEntry() - result is %u", *_retval
));
1264 nsresult
CacheIndex::GetEntryForEviction(bool aIgnoreEmptyEntries
,
1265 SHA1Sum::Hash
* aHash
, uint32_t* aCnt
) {
1266 LOG(("CacheIndex::GetEntryForEviction()"));
1268 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
1270 StaticMutexAutoLock
lock(sLock
);
1272 RefPtr
<CacheIndex
> index
= gInstance
;
1274 if (!index
) return NS_ERROR_NOT_INITIALIZED
;
1276 if (!index
->IsIndexUsable()) {
1277 return NS_ERROR_NOT_AVAILABLE
;
1280 if (index
->mIndexStats
.Size() == 0) {
1281 return NS_ERROR_NOT_AVAILABLE
;
1284 int32_t mediaUsage
=
1285 round(static_cast<double>(index
->mIndexStats
.SizeByType(
1286 nsICacheEntry::CONTENT_TYPE_MEDIA
)) *
1287 100.0 / static_cast<double>(index
->mIndexStats
.Size()));
1288 int32_t mediaUsageLimit
=
1289 StaticPrefs::browser_cache_disk_content_type_media_limit();
1290 bool evictMedia
= false;
1291 if (mediaUsage
> mediaUsageLimit
) {
1293 ("CacheIndex::GetEntryForEviction() - media content type is over the "
1294 "limit [mediaUsage=%d, mediaUsageLimit=%d]",
1295 mediaUsage
, mediaUsageLimit
));
1300 CacheIndexRecord
* foundRecord
= nullptr;
1301 uint32_t skipped
= 0;
1303 // find first non-forced valid and unpinned entry with the lowest frecency
1304 index
->mFrecencyArray
.SortIfNeeded(lock
);
1306 for (auto iter
= index
->mFrecencyArray
.Iter(); !iter
.Done(); iter
.Next()) {
1307 CacheIndexRecord
* rec
= iter
.Get()->Get();
1309 memcpy(&hash
, rec
->mHash
, sizeof(SHA1Sum::Hash
));
1313 if (evictMedia
&& CacheIndexEntry::GetContentType(rec
) !=
1314 nsICacheEntry::CONTENT_TYPE_MEDIA
) {
1318 if (IsForcedValidEntry(&hash
)) {
1322 if (CacheIndexEntry::IsPinned(rec
)) {
1326 if (aIgnoreEmptyEntries
&& !CacheIndexEntry::GetFileSize(*rec
)) {
1335 if (!foundRecord
) return NS_ERROR_NOT_AVAILABLE
;
1340 ("CacheIndex::GetEntryForEviction() - returning entry "
1341 "[hash=%08x%08x%08x%08x%08x, cnt=%u, frecency=%u, contentType=%u]",
1342 LOGSHA1(&hash
), *aCnt
, foundRecord
->mFrecency
,
1343 CacheIndexEntry::GetContentType(foundRecord
)));
1345 memcpy(aHash
, &hash
, sizeof(SHA1Sum::Hash
));
1351 bool CacheIndex::IsForcedValidEntry(const SHA1Sum::Hash
* aHash
) {
1352 RefPtr
<CacheFileHandle
> handle
;
1354 CacheFileIOManager::gInstance
->mHandles
.GetHandle(aHash
,
1355 getter_AddRefs(handle
));
1357 if (!handle
) return false;
1359 nsCString hashKey
= handle
->Key();
1360 return CacheStorageService::Self()->IsForcedValidEntry(hashKey
);
1364 nsresult
CacheIndex::GetCacheSize(uint32_t* _retval
) {
1365 LOG(("CacheIndex::GetCacheSize()"));
1367 StaticMutexAutoLock
lock(sLock
);
1369 RefPtr
<CacheIndex
> index
= gInstance
;
1371 if (!index
) return NS_ERROR_NOT_INITIALIZED
;
1373 if (!index
->IsIndexUsable()) {
1374 return NS_ERROR_NOT_AVAILABLE
;
1377 *_retval
= index
->mIndexStats
.Size();
1378 LOG(("CacheIndex::GetCacheSize() - returning %u", *_retval
));
1383 nsresult
CacheIndex::GetEntryFileCount(uint32_t* _retval
) {
1384 LOG(("CacheIndex::GetEntryFileCount()"));
1386 StaticMutexAutoLock
lock(sLock
);
1388 RefPtr
<CacheIndex
> index
= gInstance
;
1391 return NS_ERROR_NOT_INITIALIZED
;
1394 if (!index
->IsIndexUsable()) {
1395 return NS_ERROR_NOT_AVAILABLE
;
1398 *_retval
= index
->mIndexStats
.ActiveEntriesCount();
1399 LOG(("CacheIndex::GetEntryFileCount() - returning %u", *_retval
));
1404 nsresult
CacheIndex::GetCacheStats(nsILoadContextInfo
* aInfo
, uint32_t* aSize
,
1406 LOG(("CacheIndex::GetCacheStats() [info=%p]", aInfo
));
1408 StaticMutexAutoLock
lock(sLock
);
1410 RefPtr
<CacheIndex
> index
= gInstance
;
1413 return NS_ERROR_NOT_INITIALIZED
;
1416 if (!index
->IsIndexUsable()) {
1417 return NS_ERROR_NOT_AVAILABLE
;
1423 for (auto iter
= index
->mFrecencyArray
.Iter(); !iter
.Done(); iter
.Next()) {
1425 !CacheIndexEntry::RecordMatchesLoadContextInfo(iter
.Get(), aInfo
)) {
1429 *aSize
+= CacheIndexEntry::GetFileSize(*(iter
.Get()->Get()));
1437 nsresult
CacheIndex::AsyncGetDiskConsumption(
1438 nsICacheStorageConsumptionObserver
* aObserver
) {
1439 LOG(("CacheIndex::AsyncGetDiskConsumption()"));
1441 StaticMutexAutoLock
lock(sLock
);
1443 RefPtr
<CacheIndex
> index
= gInstance
;
1446 return NS_ERROR_NOT_INITIALIZED
;
1449 if (!index
->IsIndexUsable()) {
1450 return NS_ERROR_NOT_AVAILABLE
;
1453 RefPtr
<DiskConsumptionObserver
> observer
=
1454 DiskConsumptionObserver::Init(aObserver
);
1456 NS_ENSURE_ARG(observer
);
1458 if ((index
->mState
== READY
|| index
->mState
== WRITING
) &&
1459 !index
->mAsyncGetDiskConsumptionBlocked
) {
1460 LOG(("CacheIndex::AsyncGetDiskConsumption - calling immediately"));
1461 // Safe to call the callback under the lock,
1462 // we always post to the main thread.
1463 observer
->OnDiskConsumption(index
->mIndexStats
.Size() << 10);
1467 LOG(("CacheIndex::AsyncGetDiskConsumption - remembering callback"));
1468 // Will be called when the index get to the READY state.
1469 index
->mDiskConsumptionObservers
.AppendElement(observer
);
1471 // Move forward with index re/building if it is pending
1472 RefPtr
<CacheIOThread
> ioThread
= CacheFileIOManager::IOThread();
1475 NS_NewRunnableFunction("net::CacheIndex::AsyncGetDiskConsumption",
1477 StaticMutexAutoLock
lock(sLock
);
1479 RefPtr
<CacheIndex
> index
= gInstance
;
1480 if (index
&& index
->mUpdateTimer
) {
1481 index
->mUpdateTimer
->Cancel();
1482 index
->DelayedUpdateLocked(lock
);
1485 CacheIOThread::INDEX
);
1492 nsresult
CacheIndex::GetIterator(nsILoadContextInfo
* aInfo
, bool aAddNew
,
1493 CacheIndexIterator
** _retval
) {
1494 LOG(("CacheIndex::GetIterator() [info=%p, addNew=%d]", aInfo
, aAddNew
));
1496 StaticMutexAutoLock
lock(sLock
);
1498 RefPtr
<CacheIndex
> index
= gInstance
;
1501 return NS_ERROR_NOT_INITIALIZED
;
1504 if (!index
->IsIndexUsable()) {
1505 return NS_ERROR_NOT_AVAILABLE
;
1508 RefPtr
<CacheIndexIterator
> idxIter
;
1510 idxIter
= new CacheIndexContextIterator(index
, aAddNew
, aInfo
);
1512 idxIter
= new CacheIndexIterator(index
, aAddNew
);
1515 index
->mFrecencyArray
.SortIfNeeded(lock
);
1517 for (auto iter
= index
->mFrecencyArray
.Iter(); !iter
.Done(); iter
.Next()) {
1518 idxIter
->AddRecord(iter
.Get(), lock
);
1521 index
->mIterators
.AppendElement(idxIter
);
1522 idxIter
.swap(*_retval
);
1527 nsresult
CacheIndex::IsUpToDate(bool* _retval
) {
1528 LOG(("CacheIndex::IsUpToDate()"));
1530 StaticMutexAutoLock
lock(sLock
);
1532 RefPtr
<CacheIndex
> index
= gInstance
;
1535 return NS_ERROR_NOT_INITIALIZED
;
1538 if (!index
->IsIndexUsable()) {
1539 return NS_ERROR_NOT_AVAILABLE
;
1542 *_retval
= (index
->mState
== READY
|| index
->mState
== WRITING
) &&
1543 !index
->mIndexNeedsUpdate
&& !index
->mShuttingDown
;
1545 LOG(("CacheIndex::IsUpToDate() - returning %d", *_retval
));
1549 bool CacheIndex::IsIndexUsable() {
1550 MOZ_ASSERT(mState
!= INITIAL
);
1569 bool CacheIndex::IsCollision(CacheIndexEntry
* aEntry
,
1570 OriginAttrsHash aOriginAttrsHash
,
1572 if (!aEntry
->IsInitialized()) {
1576 if (aEntry
->Anonymous() != aAnonymous
||
1577 aEntry
->OriginAttrsHash() != aOriginAttrsHash
) {
1579 ("CacheIndex::IsCollision() - Collision detected for entry hash=%08x"
1580 "%08x%08x%08x%08x, expected values: originAttrsHash=%" PRIu64
", "
1581 "anonymous=%d; actual values: originAttrsHash=%" PRIu64
1583 LOGSHA1(aEntry
->Hash()), aOriginAttrsHash
, aAnonymous
,
1584 aEntry
->OriginAttrsHash(), aEntry
->Anonymous()));
1592 bool CacheIndex::HasEntryChanged(
1593 CacheIndexEntry
* aEntry
, const uint32_t* aFrecency
, const bool* aHasAltData
,
1594 const uint16_t* aOnStartTime
, const uint16_t* aOnStopTime
,
1595 const uint8_t* aContentType
, const uint32_t* aSize
) {
1596 if (aFrecency
&& *aFrecency
!= aEntry
->GetFrecency()) {
1600 if (aHasAltData
&& *aHasAltData
!= aEntry
->GetHasAltData()) {
1604 if (aOnStartTime
&& *aOnStartTime
!= aEntry
->GetOnStartTime()) {
1608 if (aOnStopTime
&& *aOnStopTime
!= aEntry
->GetOnStopTime()) {
1612 if (aContentType
&& *aContentType
!= aEntry
->GetContentType()) {
1617 (*aSize
& CacheIndexEntry::kFileSizeMask
) != aEntry
->GetFileSize()) {
1624 void CacheIndex::ProcessPendingOperations(
1625 const StaticMutexAutoLock
& aProofOfLock
) {
1626 sLock
.AssertCurrentThreadOwns();
1627 LOG(("CacheIndex::ProcessPendingOperations()"));
1629 for (auto iter
= mPendingUpdates
.Iter(); !iter
.Done(); iter
.Next()) {
1630 CacheIndexEntryUpdate
* update
= iter
.Get();
1632 LOG(("CacheIndex::ProcessPendingOperations() [hash=%08x%08x%08x%08x%08x]",
1633 LOGSHA1(update
->Hash())));
1635 MOZ_ASSERT(update
->IsFresh());
1637 CacheIndexEntry
* entry
= mIndex
.GetEntry(*update
->Hash());
1639 CacheIndexEntryAutoManage
emng(update
->Hash(), this, aProofOfLock
);
1640 emng
.DoNotSearchInUpdates();
1642 if (update
->IsRemoved()) {
1644 if (entry
->IsRemoved()) {
1645 MOZ_ASSERT(entry
->IsFresh());
1646 MOZ_ASSERT(entry
->IsDirty());
1647 } else if (!entry
->IsDirty() && entry
->IsFileEmpty()) {
1648 // Entries with empty file are not stored in index on disk. Just
1649 // remove the entry, but only in case the entry is not dirty, i.e.
1650 // the entry file was empty when we wrote the index.
1651 mIndex
.RemoveEntry(entry
);
1654 entry
->MarkRemoved();
1660 // Some information in mIndex can be newer than in mPendingUpdates (see
1661 // bug 1074832). This will copy just those values that were really
1663 update
->ApplyUpdate(entry
);
1665 // There is no entry in mIndex, copy all information from
1666 // mPendingUpdates to mIndex.
1667 entry
= mIndex
.PutEntry(*update
->Hash());
1674 MOZ_ASSERT(mPendingUpdates
.Count() == 0);
1676 EnsureCorrectStats();
1679 bool CacheIndex::WriteIndexToDiskIfNeeded(
1680 const StaticMutexAutoLock
& aProofOfLock
) {
1681 sLock
.AssertCurrentThreadOwns();
1682 if (mState
!= READY
|| mShuttingDown
|| mRWPending
) {
1686 if (!mLastDumpTime
.IsNull() &&
1687 (TimeStamp::NowLoRes() - mLastDumpTime
).ToMilliseconds() <
1692 if (mIndexStats
.Dirty() < kMinUnwrittenChanges
) {
1696 WriteIndexToDisk(aProofOfLock
);
1700 void CacheIndex::WriteIndexToDisk(const StaticMutexAutoLock
& aProofOfLock
) {
1701 sLock
.AssertCurrentThreadOwns();
1702 LOG(("CacheIndex::WriteIndexToDisk()"));
1707 MOZ_ASSERT(mState
== READY
);
1708 MOZ_ASSERT(!mRWBuf
);
1709 MOZ_ASSERT(!mRWHash
);
1710 MOZ_ASSERT(!mRWPending
);
1712 ChangeState(WRITING
, aProofOfLock
);
1714 mProcessEntries
= mIndexStats
.ActiveEntriesCount();
1716 mIndexFileOpener
= new FileOpenHelper(this);
1717 rv
= CacheFileIOManager::OpenFile(
1718 nsLiteralCString(TEMP_INDEX_NAME
),
1719 CacheFileIOManager::SPECIAL_FILE
| CacheFileIOManager::CREATE
,
1721 if (NS_FAILED(rv
)) {
1722 LOG(("CacheIndex::WriteIndexToDisk() - Can't open file [rv=0x%08" PRIx32
1724 static_cast<uint32_t>(rv
)));
1725 FinishWrite(false, aProofOfLock
);
1729 // Write index header to a buffer, it will be written to disk together with
1730 // records in WriteRecords() once we open the file successfully.
1732 mRWHash
= new CacheHash();
1736 NetworkEndian::writeUint32(mRWBuf
+ mRWBufPos
, kIndexVersion
);
1737 mRWBufPos
+= sizeof(uint32_t);
1739 NetworkEndian::writeUint32(mRWBuf
+ mRWBufPos
,
1740 static_cast<uint32_t>(PR_Now() / PR_USEC_PER_SEC
));
1741 mRWBufPos
+= sizeof(uint32_t);
1743 NetworkEndian::writeUint32(mRWBuf
+ mRWBufPos
, 1);
1744 mRWBufPos
+= sizeof(uint32_t);
1745 // amount of data written to the cache
1746 NetworkEndian::writeUint32(mRWBuf
+ mRWBufPos
,
1747 static_cast<uint32_t>(mTotalBytesWritten
>> 10));
1748 mRWBufPos
+= sizeof(uint32_t);
1753 void CacheIndex::WriteRecords(const StaticMutexAutoLock
& aProofOfLock
) {
1754 sLock
.AssertCurrentThreadOwns();
1755 LOG(("CacheIndex::WriteRecords()"));
1759 MOZ_ASSERT(mState
== WRITING
);
1760 MOZ_ASSERT(!mRWPending
);
1765 MOZ_ASSERT(mRWBufPos
== 0);
1766 fileOffset
= sizeof(CacheIndexHeader
);
1767 fileOffset
+= sizeof(CacheIndexRecord
) * mSkipEntries
;
1769 MOZ_ASSERT(mRWBufPos
== sizeof(CacheIndexHeader
));
1772 uint32_t hashOffset
= mRWBufPos
;
1774 char* buf
= mRWBuf
+ mRWBufPos
;
1775 uint32_t skip
= mSkipEntries
;
1776 uint32_t processMax
= (mRWBufSize
- mRWBufPos
) / sizeof(CacheIndexRecord
);
1777 MOZ_ASSERT(processMax
!= 0 ||
1779 0); // TODO make sure we can write an empty index
1780 uint32_t processed
= 0;
1782 bool hasMore
= false;
1784 for (auto iter
= mIndex
.Iter(); !iter
.Done(); iter
.Next()) {
1785 CacheIndexEntry
* entry
= iter
.Get();
1786 if (entry
->IsRemoved() || !entry
->IsInitialized() || entry
->IsFileEmpty()) {
1795 if (processed
== processMax
) {
1802 entry
->WriteToBuf(buf
);
1803 buf
+= sizeof(CacheIndexRecord
);
1807 MOZ_ASSERT(mRWBufPos
!= static_cast<uint32_t>(buf
- mRWBuf
) ||
1808 mProcessEntries
== 0);
1809 mRWBufPos
= buf
- mRWBuf
;
1810 mSkipEntries
+= processed
;
1811 MOZ_ASSERT(mSkipEntries
<= mProcessEntries
);
1813 mRWHash
->Update(mRWBuf
+ hashOffset
, mRWBufPos
- hashOffset
);
1815 if (mSkipEntries
== mProcessEntries
) {
1816 MOZ_ASSERT(!hasMore
);
1818 // We've processed all records
1819 if (mRWBufPos
+ sizeof(CacheHash::Hash32_t
) > mRWBufSize
) {
1820 // realloc buffer to spare another write cycle
1821 mRWBufSize
= mRWBufPos
+ sizeof(CacheHash::Hash32_t
);
1822 mRWBuf
= static_cast<char*>(moz_xrealloc(mRWBuf
, mRWBufSize
));
1825 NetworkEndian::writeUint32(mRWBuf
+ mRWBufPos
, mRWHash
->GetHash());
1826 mRWBufPos
+= sizeof(CacheHash::Hash32_t
);
1828 MOZ_ASSERT(hasMore
);
1831 rv
= CacheFileIOManager::Write(mIndexHandle
, fileOffset
, mRWBuf
, mRWBufPos
,
1832 mSkipEntries
== mProcessEntries
, false, this);
1833 if (NS_FAILED(rv
)) {
1835 ("CacheIndex::WriteRecords() - CacheFileIOManager::Write() failed "
1836 "synchronously [rv=0x%08" PRIx32
"]",
1837 static_cast<uint32_t>(rv
)));
1838 FinishWrite(false, aProofOfLock
);
1846 void CacheIndex::FinishWrite(bool aSucceeded
,
1847 const StaticMutexAutoLock
& aProofOfLock
) {
1848 sLock
.AssertCurrentThreadOwns();
1849 LOG(("CacheIndex::FinishWrite() [succeeded=%d]", aSucceeded
));
1851 MOZ_ASSERT((!aSucceeded
&& mState
== SHUTDOWN
) || mState
== WRITING
);
1853 // If there is write operation pending we must be cancelling writing of the
1854 // index when shutting down or removing the whole index.
1855 MOZ_ASSERT(!mRWPending
|| (!aSucceeded
&& (mShuttingDown
|| mRemovingAll
)));
1857 mIndexHandle
= nullptr;
1862 // Opening of the file must not be in progress if writing succeeded.
1863 MOZ_ASSERT(!mIndexFileOpener
);
1865 for (auto iter
= mIndex
.Iter(); !iter
.Done(); iter
.Next()) {
1866 CacheIndexEntry
* entry
= iter
.Get();
1868 bool remove
= false;
1870 CacheIndexEntryAutoManage
emng(entry
->Hash(), this, aProofOfLock
);
1872 if (entry
->IsRemoved()) {
1873 emng
.DoNotSearchInIndex();
1875 } else if (entry
->IsDirty()) {
1876 entry
->ClearDirty();
1884 mIndexOnDiskIsValid
= true;
1886 if (mIndexFileOpener
) {
1887 // If opening of the file is still in progress (e.g. WRITE process was
1888 // canceled by RemoveAll()) then we need to cancel the opener to make sure
1889 // that OnFileOpenedInternal() won't be called.
1890 mIndexFileOpener
->Cancel();
1891 mIndexFileOpener
= nullptr;
1895 ProcessPendingOperations(aProofOfLock
);
1898 if (mState
== WRITING
) {
1899 ChangeState(READY
, aProofOfLock
);
1900 mLastDumpTime
= TimeStamp::NowLoRes();
1904 nsresult
CacheIndex::GetFile(const nsACString
& aName
, nsIFile
** _retval
) {
1907 nsCOMPtr
<nsIFile
> file
;
1908 rv
= mCacheDirectory
->Clone(getter_AddRefs(file
));
1909 NS_ENSURE_SUCCESS(rv
, rv
);
1911 rv
= file
->AppendNative(aName
);
1912 NS_ENSURE_SUCCESS(rv
, rv
);
1914 file
.swap(*_retval
);
1918 void CacheIndex::RemoveFile(const nsACString
& aName
) {
1919 MOZ_ASSERT(mState
== SHUTDOWN
);
1923 nsCOMPtr
<nsIFile
> file
;
1924 rv
= GetFile(aName
, getter_AddRefs(file
));
1925 NS_ENSURE_SUCCESS_VOID(rv
);
1927 rv
= file
->Remove(false);
1928 if (NS_FAILED(rv
) && rv
!= NS_ERROR_FILE_NOT_FOUND
) {
1930 ("CacheIndex::RemoveFile() - Cannot remove old entry file from disk "
1931 "[rv=0x%08" PRIx32
", name=%s]",
1932 static_cast<uint32_t>(rv
), PromiseFlatCString(aName
).get()));
1936 void CacheIndex::RemoveAllIndexFiles() {
1937 LOG(("CacheIndex::RemoveAllIndexFiles()"));
1938 RemoveFile(nsLiteralCString(INDEX_NAME
));
1939 RemoveJournalAndTempFile();
1942 void CacheIndex::RemoveJournalAndTempFile() {
1943 LOG(("CacheIndex::RemoveJournalAndTempFile()"));
1944 RemoveFile(nsLiteralCString(TEMP_INDEX_NAME
));
1945 RemoveFile(nsLiteralCString(JOURNAL_NAME
));
1948 class WriteLogHelper
{
1950 explicit WriteLogHelper(PRFileDesc
* aFD
)
1951 : mFD(aFD
), mBufSize(kMaxBufSize
), mBufPos(0) {
1952 mHash
= new CacheHash();
1953 mBuf
= static_cast<char*>(moz_xmalloc(mBufSize
));
1956 ~WriteLogHelper() { free(mBuf
); }
1958 nsresult
AddEntry(CacheIndexEntry
* aEntry
);
1962 nsresult
FlushBuffer();
1968 RefPtr
<CacheHash
> mHash
;
1971 nsresult
WriteLogHelper::AddEntry(CacheIndexEntry
* aEntry
) {
1974 if (mBufPos
+ sizeof(CacheIndexRecord
) > mBufSize
) {
1975 mHash
->Update(mBuf
, mBufPos
);
1978 NS_ENSURE_SUCCESS(rv
, rv
);
1979 MOZ_ASSERT(mBufPos
+ sizeof(CacheIndexRecord
) <= mBufSize
);
1982 aEntry
->WriteToBuf(mBuf
+ mBufPos
);
1983 mBufPos
+= sizeof(CacheIndexRecord
);
1988 nsresult
WriteLogHelper::Finish() {
1991 mHash
->Update(mBuf
, mBufPos
);
1992 if (mBufPos
+ sizeof(CacheHash::Hash32_t
) > mBufSize
) {
1994 NS_ENSURE_SUCCESS(rv
, rv
);
1995 MOZ_ASSERT(mBufPos
+ sizeof(CacheHash::Hash32_t
) <= mBufSize
);
1998 NetworkEndian::writeUint32(mBuf
+ mBufPos
, mHash
->GetHash());
1999 mBufPos
+= sizeof(CacheHash::Hash32_t
);
2002 NS_ENSURE_SUCCESS(rv
, rv
);
2007 nsresult
WriteLogHelper::FlushBuffer() {
2008 if (CacheObserver::IsPastShutdownIOLag()) {
2009 LOG(("WriteLogHelper::FlushBuffer() - Interrupting writing journal."));
2010 return NS_ERROR_FAILURE
;
2013 int32_t bytesWritten
= PR_Write(mFD
, mBuf
, mBufPos
);
2015 if (bytesWritten
!= mBufPos
) {
2016 return NS_ERROR_FAILURE
;
2023 nsresult
CacheIndex::WriteLogToDisk() {
2024 LOG(("CacheIndex::WriteLogToDisk()"));
2028 MOZ_ASSERT(mPendingUpdates
.Count() == 0);
2029 MOZ_ASSERT(mState
== SHUTDOWN
);
2031 if (CacheObserver::IsPastShutdownIOLag()) {
2032 LOG(("CacheIndex::WriteLogToDisk() - Skipping writing journal."));
2033 return NS_ERROR_FAILURE
;
2036 RemoveFile(nsLiteralCString(TEMP_INDEX_NAME
));
2038 nsCOMPtr
<nsIFile
> indexFile
;
2039 rv
= GetFile(nsLiteralCString(INDEX_NAME
), getter_AddRefs(indexFile
));
2040 NS_ENSURE_SUCCESS(rv
, rv
);
2042 nsCOMPtr
<nsIFile
> logFile
;
2043 rv
= GetFile(nsLiteralCString(JOURNAL_NAME
), getter_AddRefs(logFile
));
2044 NS_ENSURE_SUCCESS(rv
, rv
);
2048 PRFileDesc
* fd
= nullptr;
2049 rv
= logFile
->OpenNSPRFileDesc(PR_RDWR
| PR_CREATE_FILE
| PR_TRUNCATE
, 0600,
2051 NS_ENSURE_SUCCESS(rv
, rv
);
2053 WriteLogHelper
wlh(fd
);
2054 for (auto iter
= mIndex
.Iter(); !iter
.Done(); iter
.Next()) {
2055 CacheIndexEntry
* entry
= iter
.Get();
2056 if (entry
->IsRemoved() || entry
->IsDirty()) {
2057 rv
= wlh
.AddEntry(entry
);
2058 if (NS_WARN_IF(NS_FAILED(rv
))) {
2066 NS_ENSURE_SUCCESS(rv
, rv
);
2068 rv
= indexFile
->OpenNSPRFileDesc(PR_RDWR
, 0600, &fd
);
2069 NS_ENSURE_SUCCESS(rv
, rv
);
2071 // Seek to dirty flag in the index header and clear it.
2072 static_assert(2 * sizeof(uint32_t) == offsetof(CacheIndexHeader
, mIsDirty
),
2073 "Unexpected offset of CacheIndexHeader::mIsDirty");
2074 int64_t offset
= PR_Seek64(fd
, 2 * sizeof(uint32_t), PR_SEEK_SET
);
2077 return NS_ERROR_FAILURE
;
2080 uint32_t isDirty
= 0;
2081 int32_t bytesWritten
= PR_Write(fd
, &isDirty
, sizeof(isDirty
));
2083 if (bytesWritten
!= sizeof(isDirty
)) {
2084 return NS_ERROR_FAILURE
;
2090 void CacheIndex::ReadIndexFromDisk(const StaticMutexAutoLock
& aProofOfLock
) {
2091 sLock
.AssertCurrentThreadOwns();
2092 LOG(("CacheIndex::ReadIndexFromDisk()"));
2096 MOZ_ASSERT(mState
== INITIAL
);
2098 ChangeState(READING
, aProofOfLock
);
2100 mIndexFileOpener
= new FileOpenHelper(this);
2101 rv
= CacheFileIOManager::OpenFile(
2102 nsLiteralCString(INDEX_NAME
),
2103 CacheFileIOManager::SPECIAL_FILE
| CacheFileIOManager::OPEN
,
2105 if (NS_FAILED(rv
)) {
2107 ("CacheIndex::ReadIndexFromDisk() - CacheFileIOManager::OpenFile() "
2108 "failed [rv=0x%08" PRIx32
", file=%s]",
2109 static_cast<uint32_t>(rv
), INDEX_NAME
));
2110 FinishRead(false, aProofOfLock
);
2114 mJournalFileOpener
= new FileOpenHelper(this);
2115 rv
= CacheFileIOManager::OpenFile(
2116 nsLiteralCString(JOURNAL_NAME
),
2117 CacheFileIOManager::SPECIAL_FILE
| CacheFileIOManager::OPEN
,
2118 mJournalFileOpener
);
2119 if (NS_FAILED(rv
)) {
2121 ("CacheIndex::ReadIndexFromDisk() - CacheFileIOManager::OpenFile() "
2122 "failed [rv=0x%08" PRIx32
", file=%s]",
2123 static_cast<uint32_t>(rv
), JOURNAL_NAME
));
2124 FinishRead(false, aProofOfLock
);
2127 mTmpFileOpener
= new FileOpenHelper(this);
2128 rv
= CacheFileIOManager::OpenFile(
2129 nsLiteralCString(TEMP_INDEX_NAME
),
2130 CacheFileIOManager::SPECIAL_FILE
| CacheFileIOManager::OPEN
,
2132 if (NS_FAILED(rv
)) {
2134 ("CacheIndex::ReadIndexFromDisk() - CacheFileIOManager::OpenFile() "
2135 "failed [rv=0x%08" PRIx32
", file=%s]",
2136 static_cast<uint32_t>(rv
), TEMP_INDEX_NAME
));
2137 FinishRead(false, aProofOfLock
);
2141 void CacheIndex::StartReadingIndex(const StaticMutexAutoLock
& aProofOfLock
) {
2142 sLock
.AssertCurrentThreadOwns();
2143 LOG(("CacheIndex::StartReadingIndex()"));
2147 MOZ_ASSERT(mIndexHandle
);
2148 MOZ_ASSERT(mState
== READING
);
2149 MOZ_ASSERT(!mIndexOnDiskIsValid
);
2150 MOZ_ASSERT(!mDontMarkIndexClean
);
2151 MOZ_ASSERT(!mJournalReadSuccessfully
);
2152 MOZ_ASSERT(mIndexHandle
->FileSize() >= 0);
2153 MOZ_ASSERT(!mRWPending
);
2155 int64_t entriesSize
= mIndexHandle
->FileSize() - sizeof(CacheIndexHeader
) -
2156 sizeof(CacheHash::Hash32_t
);
2158 if (entriesSize
< 0 || entriesSize
% sizeof(CacheIndexRecord
)) {
2159 LOG(("CacheIndex::StartReadingIndex() - Index is corrupted"));
2160 FinishRead(false, aProofOfLock
);
2166 mRWHash
= new CacheHash();
2169 std::min(mRWBufSize
, static_cast<uint32_t>(mIndexHandle
->FileSize()));
2171 rv
= CacheFileIOManager::Read(mIndexHandle
, 0, mRWBuf
, mRWBufPos
, this);
2172 if (NS_FAILED(rv
)) {
2174 ("CacheIndex::StartReadingIndex() - CacheFileIOManager::Read() failed "
2175 "synchronously [rv=0x%08" PRIx32
"]",
2176 static_cast<uint32_t>(rv
)));
2177 FinishRead(false, aProofOfLock
);
2183 void CacheIndex::ParseRecords(const StaticMutexAutoLock
& aProofOfLock
) {
2184 sLock
.AssertCurrentThreadOwns();
2185 LOG(("CacheIndex::ParseRecords()"));
2189 MOZ_ASSERT(!mRWPending
);
2191 uint32_t entryCnt
= (mIndexHandle
->FileSize() - sizeof(CacheIndexHeader
) -
2192 sizeof(CacheHash::Hash32_t
)) /
2193 sizeof(CacheIndexRecord
);
2196 if (!mSkipEntries
) {
2197 if (NetworkEndian::readUint32(mRWBuf
+ pos
) != kIndexVersion
) {
2198 FinishRead(false, aProofOfLock
);
2201 pos
+= sizeof(uint32_t);
2203 mIndexTimeStamp
= NetworkEndian::readUint32(mRWBuf
+ pos
);
2204 pos
+= sizeof(uint32_t);
2206 if (NetworkEndian::readUint32(mRWBuf
+ pos
)) {
2207 if (mJournalHandle
) {
2208 CacheFileIOManager::DoomFile(mJournalHandle
, nullptr);
2209 mJournalHandle
= nullptr;
2213 reinterpret_cast<uint32_t*>(moz_xmalloc(sizeof(uint32_t)));
2214 NetworkEndian::writeUint32(isDirty
, 1);
2216 // Mark index dirty. The buffer is freed by CacheFileIOManager when
2217 // nullptr is passed as the listener and the call doesn't fail
2219 rv
= CacheFileIOManager::Write(mIndexHandle
, 2 * sizeof(uint32_t),
2220 reinterpret_cast<char*>(isDirty
),
2221 sizeof(uint32_t), true, false, nullptr);
2222 if (NS_FAILED(rv
)) {
2223 // This is not fatal, just free the memory
2227 pos
+= sizeof(uint32_t);
2229 uint64_t dataWritten
= NetworkEndian::readUint32(mRWBuf
+ pos
);
2230 pos
+= sizeof(uint32_t);
2232 mTotalBytesWritten
+= dataWritten
;
2235 uint32_t hashOffset
= pos
;
2237 while (pos
+ sizeof(CacheIndexRecord
) <= mRWBufPos
&&
2238 mSkipEntries
!= entryCnt
) {
2239 CacheIndexRecord
* rec
= reinterpret_cast<CacheIndexRecord
*>(mRWBuf
+ pos
);
2240 CacheIndexEntry
tmpEntry(&rec
->mHash
);
2241 tmpEntry
.ReadFromBuf(mRWBuf
+ pos
);
2243 if (tmpEntry
.IsDirty() || !tmpEntry
.IsInitialized() ||
2244 tmpEntry
.IsFileEmpty() || tmpEntry
.IsFresh() || tmpEntry
.IsRemoved()) {
2246 ("CacheIndex::ParseRecords() - Invalid entry found in index, removing"
2247 " whole index [dirty=%d, initialized=%d, fileEmpty=%d, fresh=%d, "
2249 tmpEntry
.IsDirty(), tmpEntry
.IsInitialized(), tmpEntry
.IsFileEmpty(),
2250 tmpEntry
.IsFresh(), tmpEntry
.IsRemoved()));
2251 FinishRead(false, aProofOfLock
);
2255 CacheIndexEntryAutoManage
emng(tmpEntry
.Hash(), this, aProofOfLock
);
2257 CacheIndexEntry
* entry
= mIndex
.PutEntry(*tmpEntry
.Hash());
2260 pos
+= sizeof(CacheIndexRecord
);
2264 mRWHash
->Update(mRWBuf
+ hashOffset
, pos
- hashOffset
);
2266 if (pos
!= mRWBufPos
) {
2267 memmove(mRWBuf
, mRWBuf
+ pos
, mRWBufPos
- pos
);
2273 int64_t fileOffset
= sizeof(CacheIndexHeader
) +
2274 mSkipEntries
* sizeof(CacheIndexRecord
) + mRWBufPos
;
2276 MOZ_ASSERT(fileOffset
<= mIndexHandle
->FileSize());
2277 if (fileOffset
== mIndexHandle
->FileSize()) {
2278 uint32_t expectedHash
= NetworkEndian::readUint32(mRWBuf
);
2279 if (mRWHash
->GetHash() != expectedHash
) {
2280 LOG(("CacheIndex::ParseRecords() - Hash mismatch, [is %x, should be %x]",
2281 mRWHash
->GetHash(), expectedHash
));
2282 FinishRead(false, aProofOfLock
);
2286 mIndexOnDiskIsValid
= true;
2287 mJournalReadSuccessfully
= false;
2289 if (mJournalHandle
) {
2290 StartReadingJournal(aProofOfLock
);
2292 FinishRead(false, aProofOfLock
);
2300 std::min(mRWBufSize
- pos
,
2301 static_cast<uint32_t>(mIndexHandle
->FileSize() - fileOffset
));
2302 mRWBufPos
= pos
+ toRead
;
2304 rv
= CacheFileIOManager::Read(mIndexHandle
, fileOffset
, mRWBuf
+ pos
, toRead
,
2306 if (NS_FAILED(rv
)) {
2308 ("CacheIndex::ParseRecords() - CacheFileIOManager::Read() failed "
2309 "synchronously [rv=0x%08" PRIx32
"]",
2310 static_cast<uint32_t>(rv
)));
2311 FinishRead(false, aProofOfLock
);
2317 void CacheIndex::StartReadingJournal(const StaticMutexAutoLock
& aProofOfLock
) {
2318 sLock
.AssertCurrentThreadOwns();
2319 LOG(("CacheIndex::StartReadingJournal()"));
2323 MOZ_ASSERT(mJournalHandle
);
2324 MOZ_ASSERT(mIndexOnDiskIsValid
);
2325 MOZ_ASSERT(mTmpJournal
.Count() == 0);
2326 MOZ_ASSERT(mJournalHandle
->FileSize() >= 0);
2327 MOZ_ASSERT(!mRWPending
);
2329 int64_t entriesSize
=
2330 mJournalHandle
->FileSize() - sizeof(CacheHash::Hash32_t
);
2332 if (entriesSize
< 0 || entriesSize
% sizeof(CacheIndexRecord
)) {
2333 LOG(("CacheIndex::StartReadingJournal() - Journal is corrupted"));
2334 FinishRead(false, aProofOfLock
);
2339 mRWHash
= new CacheHash();
2342 std::min(mRWBufSize
, static_cast<uint32_t>(mJournalHandle
->FileSize()));
2344 rv
= CacheFileIOManager::Read(mJournalHandle
, 0, mRWBuf
, mRWBufPos
, this);
2345 if (NS_FAILED(rv
)) {
2347 ("CacheIndex::StartReadingJournal() - CacheFileIOManager::Read() failed"
2348 " synchronously [rv=0x%08" PRIx32
"]",
2349 static_cast<uint32_t>(rv
)));
2350 FinishRead(false, aProofOfLock
);
2356 void CacheIndex::ParseJournal(const StaticMutexAutoLock
& aProofOfLock
) {
2357 sLock
.AssertCurrentThreadOwns();
2358 LOG(("CacheIndex::ParseJournal()"));
2362 MOZ_ASSERT(!mRWPending
);
2365 (mJournalHandle
->FileSize() - sizeof(CacheHash::Hash32_t
)) /
2366 sizeof(CacheIndexRecord
);
2370 while (pos
+ sizeof(CacheIndexRecord
) <= mRWBufPos
&&
2371 mSkipEntries
!= entryCnt
) {
2372 CacheIndexEntry
tmpEntry(reinterpret_cast<SHA1Sum::Hash
*>(mRWBuf
+ pos
));
2373 tmpEntry
.ReadFromBuf(mRWBuf
+ pos
);
2375 CacheIndexEntry
* entry
= mTmpJournal
.PutEntry(*tmpEntry
.Hash());
2378 if (entry
->IsDirty() || entry
->IsFresh()) {
2380 ("CacheIndex::ParseJournal() - Invalid entry found in journal, "
2381 "ignoring whole journal [dirty=%d, fresh=%d]",
2382 entry
->IsDirty(), entry
->IsFresh()));
2383 FinishRead(false, aProofOfLock
);
2387 pos
+= sizeof(CacheIndexRecord
);
2391 mRWHash
->Update(mRWBuf
, pos
);
2393 if (pos
!= mRWBufPos
) {
2394 memmove(mRWBuf
, mRWBuf
+ pos
, mRWBufPos
- pos
);
2400 int64_t fileOffset
= mSkipEntries
* sizeof(CacheIndexRecord
) + mRWBufPos
;
2402 MOZ_ASSERT(fileOffset
<= mJournalHandle
->FileSize());
2403 if (fileOffset
== mJournalHandle
->FileSize()) {
2404 uint32_t expectedHash
= NetworkEndian::readUint32(mRWBuf
);
2405 if (mRWHash
->GetHash() != expectedHash
) {
2406 LOG(("CacheIndex::ParseJournal() - Hash mismatch, [is %x, should be %x]",
2407 mRWHash
->GetHash(), expectedHash
));
2408 FinishRead(false, aProofOfLock
);
2412 mJournalReadSuccessfully
= true;
2413 FinishRead(true, aProofOfLock
);
2419 std::min(mRWBufSize
- pos
,
2420 static_cast<uint32_t>(mJournalHandle
->FileSize() - fileOffset
));
2421 mRWBufPos
= pos
+ toRead
;
2423 rv
= CacheFileIOManager::Read(mJournalHandle
, fileOffset
, mRWBuf
+ pos
,
2425 if (NS_FAILED(rv
)) {
2427 ("CacheIndex::ParseJournal() - CacheFileIOManager::Read() failed "
2428 "synchronously [rv=0x%08" PRIx32
"]",
2429 static_cast<uint32_t>(rv
)));
2430 FinishRead(false, aProofOfLock
);
2436 void CacheIndex::MergeJournal(const StaticMutexAutoLock
& aProofOfLock
) {
2437 sLock
.AssertCurrentThreadOwns();
2438 LOG(("CacheIndex::MergeJournal()"));
2440 for (auto iter
= mTmpJournal
.Iter(); !iter
.Done(); iter
.Next()) {
2441 CacheIndexEntry
* entry
= iter
.Get();
2443 LOG(("CacheIndex::MergeJournal() [hash=%08x%08x%08x%08x%08x]",
2444 LOGSHA1(entry
->Hash())));
2446 CacheIndexEntry
* entry2
= mIndex
.GetEntry(*entry
->Hash());
2448 CacheIndexEntryAutoManage
emng(entry
->Hash(), this, aProofOfLock
);
2449 if (entry
->IsRemoved()) {
2451 entry2
->MarkRemoved();
2452 entry2
->MarkDirty();
2456 entry2
= mIndex
.PutEntry(*entry
->Hash());
2460 entry2
->MarkDirty();
2466 MOZ_ASSERT(mTmpJournal
.Count() == 0);
2469 void CacheIndex::EnsureNoFreshEntry() {
2471 CacheIndexStats debugStats
;
2472 debugStats
.DisableLogging();
2473 for (auto iter
= mIndex
.Iter(); !iter
.Done(); iter
.Next()) {
2474 debugStats
.BeforeChange(nullptr);
2475 debugStats
.AfterChange(iter
.Get());
2477 MOZ_ASSERT(debugStats
.Fresh() == 0);
2481 void CacheIndex::EnsureCorrectStats() {
2483 MOZ_ASSERT(mPendingUpdates
.Count() == 0);
2484 CacheIndexStats debugStats
;
2485 debugStats
.DisableLogging();
2486 for (auto iter
= mIndex
.Iter(); !iter
.Done(); iter
.Next()) {
2487 debugStats
.BeforeChange(nullptr);
2488 debugStats
.AfterChange(iter
.Get());
2490 MOZ_ASSERT(debugStats
== mIndexStats
);
2494 void CacheIndex::FinishRead(bool aSucceeded
,
2495 const StaticMutexAutoLock
& aProofOfLock
) {
2496 sLock
.AssertCurrentThreadOwns();
2497 LOG(("CacheIndex::FinishRead() [succeeded=%d]", aSucceeded
));
2499 MOZ_ASSERT((!aSucceeded
&& mState
== SHUTDOWN
) || mState
== READING
);
2503 (!aSucceeded
&& !mIndexOnDiskIsValid
&& !mJournalReadSuccessfully
) ||
2505 (!aSucceeded
&& mIndexOnDiskIsValid
&& !mJournalReadSuccessfully
) ||
2507 (aSucceeded
&& mIndexOnDiskIsValid
&& mJournalReadSuccessfully
));
2509 // If there is read operation pending we must be cancelling reading of the
2510 // index when shutting down or removing the whole index.
2511 MOZ_ASSERT(!mRWPending
|| (!aSucceeded
&& (mShuttingDown
|| mRemovingAll
)));
2513 if (mState
== SHUTDOWN
) {
2514 RemoveFile(nsLiteralCString(TEMP_INDEX_NAME
));
2515 RemoveFile(nsLiteralCString(JOURNAL_NAME
));
2517 if (mIndexHandle
&& !mIndexOnDiskIsValid
) {
2518 CacheFileIOManager::DoomFile(mIndexHandle
, nullptr);
2521 if (mJournalHandle
) {
2522 CacheFileIOManager::DoomFile(mJournalHandle
, nullptr);
2526 if (mIndexFileOpener
) {
2527 mIndexFileOpener
->Cancel();
2528 mIndexFileOpener
= nullptr;
2530 if (mJournalFileOpener
) {
2531 mJournalFileOpener
->Cancel();
2532 mJournalFileOpener
= nullptr;
2534 if (mTmpFileOpener
) {
2535 mTmpFileOpener
->Cancel();
2536 mTmpFileOpener
= nullptr;
2539 mIndexHandle
= nullptr;
2540 mJournalHandle
= nullptr;
2544 if (mState
== SHUTDOWN
) {
2548 if (!mIndexOnDiskIsValid
) {
2549 MOZ_ASSERT(mTmpJournal
.Count() == 0);
2550 EnsureNoFreshEntry();
2551 ProcessPendingOperations(aProofOfLock
);
2552 // Remove all entries that we haven't seen during this session
2553 RemoveNonFreshEntries(aProofOfLock
);
2554 StartUpdatingIndex(true, aProofOfLock
);
2558 if (!mJournalReadSuccessfully
) {
2559 mTmpJournal
.Clear();
2560 EnsureNoFreshEntry();
2561 ProcessPendingOperations(aProofOfLock
);
2562 StartUpdatingIndex(false, aProofOfLock
);
2566 MergeJournal(aProofOfLock
);
2567 EnsureNoFreshEntry();
2568 ProcessPendingOperations(aProofOfLock
);
2571 ChangeState(READY
, aProofOfLock
);
2572 mLastDumpTime
= TimeStamp::NowLoRes(); // Do not dump new index immediately
2576 void CacheIndex::DelayedUpdate(nsITimer
* aTimer
, void* aClosure
) {
2577 LOG(("CacheIndex::DelayedUpdate()"));
2579 StaticMutexAutoLock
lock(sLock
);
2580 RefPtr
<CacheIndex
> index
= gInstance
;
2586 index
->DelayedUpdateLocked(lock
);
2590 void CacheIndex::DelayedUpdateLocked(const StaticMutexAutoLock
& aProofOfLock
) {
2591 sLock
.AssertCurrentThreadOwns();
2592 LOG(("CacheIndex::DelayedUpdateLocked()"));
2596 mUpdateTimer
= nullptr;
2598 if (!IsIndexUsable()) {
2602 if (mState
== READY
&& mShuttingDown
) {
2606 // mUpdateEventPending must be false here since StartUpdatingIndex() won't
2607 // schedule timer if it is true.
2608 MOZ_ASSERT(!mUpdateEventPending
);
2609 if (mState
!= BUILDING
&& mState
!= UPDATING
) {
2610 LOG(("CacheIndex::DelayedUpdateLocked() - Update was canceled"));
2614 // We need to redispatch to run with lower priority
2615 RefPtr
<CacheIOThread
> ioThread
= CacheFileIOManager::IOThread();
2616 MOZ_ASSERT(ioThread
);
2618 mUpdateEventPending
= true;
2619 rv
= ioThread
->Dispatch(this, CacheIOThread::INDEX
);
2620 if (NS_FAILED(rv
)) {
2621 mUpdateEventPending
= false;
2622 NS_WARNING("CacheIndex::DelayedUpdateLocked() - Can't dispatch event");
2623 LOG(("CacheIndex::DelayedUpdate() - Can't dispatch event"));
2624 FinishUpdate(false, aProofOfLock
);
2628 nsresult
CacheIndex::ScheduleUpdateTimer(uint32_t aDelay
) {
2629 LOG(("CacheIndex::ScheduleUpdateTimer() [delay=%u]", aDelay
));
2631 MOZ_ASSERT(!mUpdateTimer
);
2633 nsCOMPtr
<nsIEventTarget
> ioTarget
= CacheFileIOManager::IOTarget();
2634 MOZ_ASSERT(ioTarget
);
2636 return NS_NewTimerWithFuncCallback(
2637 getter_AddRefs(mUpdateTimer
), CacheIndex::DelayedUpdate
, nullptr, aDelay
,
2638 nsITimer::TYPE_ONE_SHOT
, "net::CacheIndex::ScheduleUpdateTimer",
2642 nsresult
CacheIndex::SetupDirectoryEnumerator() {
2643 MOZ_ASSERT(!NS_IsMainThread());
2644 MOZ_ASSERT(!mDirEnumerator
);
2647 nsCOMPtr
<nsIFile
> file
;
2649 rv
= mCacheDirectory
->Clone(getter_AddRefs(file
));
2650 NS_ENSURE_SUCCESS(rv
, rv
);
2652 rv
= file
->AppendNative(nsLiteralCString(ENTRIES_DIR
));
2653 NS_ENSURE_SUCCESS(rv
, rv
);
2656 rv
= file
->Exists(&exists
);
2657 NS_ENSURE_SUCCESS(rv
, rv
);
2661 "CacheIndex::SetupDirectoryEnumerator() - Entries directory "
2664 ("CacheIndex::SetupDirectoryEnumerator() - Entries directory doesn't "
2666 return NS_ERROR_UNEXPECTED
;
2669 // Do not do IO under the lock.
2670 nsCOMPtr
<nsIDirectoryEnumerator
> dirEnumerator
;
2672 StaticMutexAutoUnlock
unlock(sLock
);
2673 rv
= file
->GetDirectoryEntries(getter_AddRefs(dirEnumerator
));
2675 mDirEnumerator
= dirEnumerator
.forget();
2676 NS_ENSURE_SUCCESS(rv
, rv
);
2681 nsresult
CacheIndex::InitEntryFromDiskData(CacheIndexEntry
* aEntry
,
2682 CacheFileMetadata
* aMetaData
,
2683 int64_t aFileSize
) {
2687 aEntry
->MarkDirty();
2688 aEntry
->MarkFresh();
2690 aEntry
->Init(GetOriginAttrsHash(aMetaData
->OriginAttributes()),
2691 aMetaData
->IsAnonymous(), aMetaData
->Pinned());
2693 aEntry
->SetFrecency(aMetaData
->GetFrecency());
2695 const char* altData
= aMetaData
->GetElement(CacheFileUtils::kAltDataKey
);
2696 bool hasAltData
= altData
!= nullptr;
2697 if (hasAltData
&& NS_FAILED(CacheFileUtils::ParseAlternativeDataInfo(
2698 altData
, nullptr, nullptr))) {
2699 return NS_ERROR_FAILURE
;
2701 aEntry
->SetHasAltData(hasAltData
);
2703 static auto toUint16
= [](const char* aUint16String
) -> uint16_t {
2704 if (!aUint16String
) {
2705 return kIndexTimeNotAvailable
;
2708 uint64_t n64
= nsDependentCString(aUint16String
).ToInteger64(&rv
);
2709 MOZ_ASSERT(NS_SUCCEEDED(rv
));
2710 return n64
<= kIndexTimeOutOfBound
? n64
: kIndexTimeOutOfBound
;
2713 aEntry
->SetOnStartTime(
2714 toUint16(aMetaData
->GetElement("net-response-time-onstart")));
2715 aEntry
->SetOnStopTime(
2716 toUint16(aMetaData
->GetElement("net-response-time-onstop")));
2718 const char* contentTypeStr
= aMetaData
->GetElement("ctid");
2719 uint8_t contentType
= nsICacheEntry::CONTENT_TYPE_UNKNOWN
;
2720 if (contentTypeStr
) {
2721 int64_t n64
= nsDependentCString(contentTypeStr
).ToInteger64(&rv
);
2722 if (NS_FAILED(rv
) || n64
< nsICacheEntry::CONTENT_TYPE_UNKNOWN
||
2723 n64
>= nsICacheEntry::CONTENT_TYPE_LAST
) {
2724 n64
= nsICacheEntry::CONTENT_TYPE_UNKNOWN
;
2728 aEntry
->SetContentType(contentType
);
2730 aEntry
->SetFileSize(static_cast<uint32_t>(std::min(
2731 static_cast<int64_t>(PR_UINT32_MAX
), (aFileSize
+ 0x3FF) >> 10)));
2735 bool CacheIndex::IsUpdatePending() {
2736 sLock
.AssertCurrentThreadOwns();
2738 return mUpdateTimer
|| mUpdateEventPending
;
2741 void CacheIndex::BuildIndex(const StaticMutexAutoLock
& aProofOfLock
) {
2742 sLock
.AssertCurrentThreadOwns();
2743 LOG(("CacheIndex::BuildIndex()"));
2745 MOZ_ASSERT(mPendingUpdates
.Count() == 0);
2749 if (!mDirEnumerator
) {
2750 rv
= SetupDirectoryEnumerator();
2751 if (mState
== SHUTDOWN
) {
2752 // The index was shut down while we released the lock. FinishUpdate() was
2753 // already called from Shutdown(), so just simply return here.
2757 if (NS_FAILED(rv
)) {
2758 FinishUpdate(false, aProofOfLock
);
2764 if (CacheIOThread::YieldAndRerun()) {
2766 "CacheIndex::BuildIndex() - Breaking loop for higher level events."));
2767 mUpdateEventPending
= true;
2771 bool fileExists
= false;
2772 nsCOMPtr
<nsIFile
> file
;
2774 // Do not do IO under the lock.
2775 nsCOMPtr
<nsIDirectoryEnumerator
> dirEnumerator(mDirEnumerator
);
2776 sLock
.AssertCurrentThreadOwns();
2777 StaticMutexAutoUnlock
unlock(sLock
);
2778 rv
= dirEnumerator
->GetNextFile(getter_AddRefs(file
));
2781 file
->Exists(&fileExists
);
2784 if (mState
== SHUTDOWN
) {
2788 FinishUpdate(NS_SUCCEEDED(rv
), aProofOfLock
);
2793 rv
= file
->GetNativeLeafName(leaf
);
2794 if (NS_FAILED(rv
)) {
2796 ("CacheIndex::BuildIndex() - GetNativeLeafName() failed! Skipping "
2798 mDontMarkIndexClean
= true;
2804 ("CacheIndex::BuildIndex() - File returned by the iterator was "
2805 "removed in the meantime [name=%s]",
2811 rv
= CacheFileIOManager::StrToHash(leaf
, &hash
);
2812 if (NS_FAILED(rv
)) {
2814 ("CacheIndex::BuildIndex() - Filename is not a hash, removing file. "
2817 file
->Remove(false);
2821 CacheIndexEntry
* entry
= mIndex
.GetEntry(hash
);
2822 if (entry
&& entry
->IsRemoved()) {
2824 ("CacheIndex::BuildIndex() - Found file that should not exist. "
2828 MOZ_ASSERT(entry
->IsFresh());
2833 RefPtr
<CacheFileHandle
> handle
;
2834 CacheFileIOManager::gInstance
->mHandles
.GetHandle(&hash
,
2835 getter_AddRefs(handle
));
2839 // the entry is up to date
2841 ("CacheIndex::BuildIndex() - Skipping file because the entry is up to"
2845 MOZ_ASSERT(entry
->IsFresh()); // The entry must be from this session
2846 // there must be an active CacheFile if the entry is not initialized
2847 MOZ_ASSERT(entry
->IsInitialized() || handle
);
2851 MOZ_ASSERT(!handle
);
2853 RefPtr
<CacheFileMetadata
> meta
= new CacheFileMetadata();
2857 // Do not do IO under the lock.
2858 StaticMutexAutoUnlock
unlock(sLock
);
2859 rv
= meta
->SyncReadMetadata(file
);
2861 if (NS_SUCCEEDED(rv
)) {
2862 rv
= file
->GetFileSize(&size
);
2863 if (NS_FAILED(rv
)) {
2865 ("CacheIndex::BuildIndex() - Cannot get filesize of file that was"
2866 " successfully parsed. [name=%s]",
2871 if (mState
== SHUTDOWN
) {
2875 // Nobody could add the entry while the lock was released since we modify
2876 // the index only on IO thread and this loop is executed on IO thread too.
2877 entry
= mIndex
.GetEntry(hash
);
2878 MOZ_ASSERT(!entry
|| entry
->IsRemoved());
2880 if (NS_FAILED(rv
)) {
2882 ("CacheIndex::BuildIndex() - CacheFileMetadata::SyncReadMetadata() "
2883 "failed, removing file. [name=%s]",
2885 file
->Remove(false);
2887 CacheIndexEntryAutoManage
entryMng(&hash
, this, aProofOfLock
);
2888 entry
= mIndex
.PutEntry(hash
);
2889 if (NS_FAILED(InitEntryFromDiskData(entry
, meta
, size
))) {
2891 ("CacheIndex::BuildIndex() - CacheFile::InitEntryFromDiskData() "
2892 "failed, removing file. [name=%s]",
2894 file
->Remove(false);
2895 entry
->MarkRemoved();
2897 LOG(("CacheIndex::BuildIndex() - Added entry to index. [name=%s]",
2904 MOZ_ASSERT_UNREACHABLE("We should never get here");
2907 bool CacheIndex::StartUpdatingIndexIfNeeded(
2908 const StaticMutexAutoLock
& aProofOfLock
, bool aSwitchingToReadyState
) {
2909 sLock
.AssertCurrentThreadOwns();
2910 // Start updating process when we are in or we are switching to READY state
2911 // and index needs update, but not during shutdown or when removing all
2913 if ((mState
== READY
|| aSwitchingToReadyState
) && mIndexNeedsUpdate
&&
2914 !mShuttingDown
&& !mRemovingAll
) {
2915 LOG(("CacheIndex::StartUpdatingIndexIfNeeded() - starting update process"));
2916 mIndexNeedsUpdate
= false;
2917 StartUpdatingIndex(false, aProofOfLock
);
2924 void CacheIndex::StartUpdatingIndex(bool aRebuild
,
2925 const StaticMutexAutoLock
& aProofOfLock
) {
2926 sLock
.AssertCurrentThreadOwns();
2927 LOG(("CacheIndex::StartUpdatingIndex() [rebuild=%d]", aRebuild
));
2933 ChangeState(aRebuild
? BUILDING
: UPDATING
, aProofOfLock
);
2934 mDontMarkIndexClean
= false;
2936 if (mShuttingDown
|| mRemovingAll
) {
2937 FinishUpdate(false, aProofOfLock
);
2941 if (IsUpdatePending()) {
2942 LOG(("CacheIndex::StartUpdatingIndex() - Update is already pending"));
2946 uint32_t elapsed
= (TimeStamp::NowLoRes() - mStartTime
).ToMilliseconds();
2947 if (elapsed
< kUpdateIndexStartDelay
) {
2949 ("CacheIndex::StartUpdatingIndex() - %u ms elapsed since startup, "
2950 "scheduling timer to fire in %u ms.",
2951 elapsed
, kUpdateIndexStartDelay
- elapsed
));
2952 rv
= ScheduleUpdateTimer(kUpdateIndexStartDelay
- elapsed
);
2953 if (NS_SUCCEEDED(rv
)) {
2958 ("CacheIndex::StartUpdatingIndex() - ScheduleUpdateTimer() failed. "
2959 "Starting update immediately."));
2962 ("CacheIndex::StartUpdatingIndex() - %u ms elapsed since startup, "
2963 "starting update now.",
2967 RefPtr
<CacheIOThread
> ioThread
= CacheFileIOManager::IOThread();
2968 MOZ_ASSERT(ioThread
);
2970 // We need to dispatch an event even if we are on IO thread since we need to
2971 // update the index with the correct priority.
2972 mUpdateEventPending
= true;
2973 rv
= ioThread
->Dispatch(this, CacheIOThread::INDEX
);
2974 if (NS_FAILED(rv
)) {
2975 mUpdateEventPending
= false;
2976 NS_WARNING("CacheIndex::StartUpdatingIndex() - Can't dispatch event");
2977 LOG(("CacheIndex::StartUpdatingIndex() - Can't dispatch event"));
2978 FinishUpdate(false, aProofOfLock
);
2982 void CacheIndex::UpdateIndex(const StaticMutexAutoLock
& aProofOfLock
) {
2983 sLock
.AssertCurrentThreadOwns();
2984 LOG(("CacheIndex::UpdateIndex()"));
2986 MOZ_ASSERT(mPendingUpdates
.Count() == 0);
2987 sLock
.AssertCurrentThreadOwns();
2991 if (!mDirEnumerator
) {
2992 rv
= SetupDirectoryEnumerator();
2993 if (mState
== SHUTDOWN
) {
2994 // The index was shut down while we released the lock. FinishUpdate() was
2995 // already called from Shutdown(), so just simply return here.
2999 if (NS_FAILED(rv
)) {
3000 FinishUpdate(false, aProofOfLock
);
3006 if (CacheIOThread::YieldAndRerun()) {
3008 ("CacheIndex::UpdateIndex() - Breaking loop for higher level "
3010 mUpdateEventPending
= true;
3014 bool fileExists
= false;
3015 nsCOMPtr
<nsIFile
> file
;
3017 // Do not do IO under the lock.
3018 nsCOMPtr
<nsIDirectoryEnumerator
> dirEnumerator(mDirEnumerator
);
3019 StaticMutexAutoUnlock
unlock(sLock
);
3020 rv
= dirEnumerator
->GetNextFile(getter_AddRefs(file
));
3023 file
->Exists(&fileExists
);
3026 if (mState
== SHUTDOWN
) {
3030 FinishUpdate(NS_SUCCEEDED(rv
), aProofOfLock
);
3035 rv
= file
->GetNativeLeafName(leaf
);
3036 if (NS_FAILED(rv
)) {
3038 ("CacheIndex::UpdateIndex() - GetNativeLeafName() failed! Skipping "
3040 mDontMarkIndexClean
= true;
3046 ("CacheIndex::UpdateIndex() - File returned by the iterator was "
3047 "removed in the meantime [name=%s]",
3053 rv
= CacheFileIOManager::StrToHash(leaf
, &hash
);
3054 if (NS_FAILED(rv
)) {
3056 ("CacheIndex::UpdateIndex() - Filename is not a hash, removing file. "
3059 file
->Remove(false);
3063 CacheIndexEntry
* entry
= mIndex
.GetEntry(hash
);
3064 if (entry
&& entry
->IsRemoved()) {
3065 if (entry
->IsFresh()) {
3067 ("CacheIndex::UpdateIndex() - Found file that should not exist. "
3076 RefPtr
<CacheFileHandle
> handle
;
3077 CacheFileIOManager::gInstance
->mHandles
.GetHandle(&hash
,
3078 getter_AddRefs(handle
));
3081 if (entry
&& entry
->IsFresh()) {
3082 // the entry is up to date
3084 ("CacheIndex::UpdateIndex() - Skipping file because the entry is up "
3085 " to date. [name=%s]",
3088 // there must be an active CacheFile if the entry is not initialized
3089 MOZ_ASSERT(entry
->IsInitialized() || handle
);
3093 MOZ_ASSERT(!handle
);
3096 PRTime lastModifiedTime
;
3098 // Do not do IO under the lock.
3099 StaticMutexAutoUnlock
unlock(sLock
);
3100 rv
= file
->GetLastModifiedTime(&lastModifiedTime
);
3102 if (mState
== SHUTDOWN
) {
3105 if (NS_FAILED(rv
)) {
3107 ("CacheIndex::UpdateIndex() - Cannot get lastModifiedTime. "
3110 // Assume the file is newer than index
3112 if (mIndexTimeStamp
> (lastModifiedTime
/ PR_MSEC_PER_SEC
)) {
3114 ("CacheIndex::UpdateIndex() - Skipping file because of last "
3115 "modified time. [name=%s, indexTimeStamp=%" PRIu32
", "
3116 "lastModifiedTime=%" PRId64
"]",
3117 leaf
.get(), mIndexTimeStamp
,
3118 lastModifiedTime
/ PR_MSEC_PER_SEC
));
3120 CacheIndexEntryAutoManage
entryMng(&hash
, this, aProofOfLock
);
3127 RefPtr
<CacheFileMetadata
> meta
= new CacheFileMetadata();
3131 // Do not do IO under the lock.
3132 StaticMutexAutoUnlock
unlock(sLock
);
3133 rv
= meta
->SyncReadMetadata(file
);
3135 if (NS_SUCCEEDED(rv
)) {
3136 rv
= file
->GetFileSize(&size
);
3137 if (NS_FAILED(rv
)) {
3139 ("CacheIndex::UpdateIndex() - Cannot get filesize of file that "
3140 "was successfully parsed. [name=%s]",
3145 if (mState
== SHUTDOWN
) {
3149 // Nobody could add the entry while the lock was released since we modify
3150 // the index only on IO thread and this loop is executed on IO thread too.
3151 entry
= mIndex
.GetEntry(hash
);
3152 MOZ_ASSERT(!entry
|| !entry
->IsFresh());
3154 CacheIndexEntryAutoManage
entryMng(&hash
, this, aProofOfLock
);
3156 if (NS_FAILED(rv
)) {
3158 ("CacheIndex::UpdateIndex() - CacheFileMetadata::SyncReadMetadata() "
3159 "failed, removing file. [name=%s]",
3162 entry
= mIndex
.PutEntry(hash
);
3163 rv
= InitEntryFromDiskData(entry
, meta
, size
);
3164 if (NS_FAILED(rv
)) {
3166 ("CacheIndex::UpdateIndex() - CacheIndex::InitEntryFromDiskData "
3167 "failed, removing file. [name=%s]",
3172 if (NS_FAILED(rv
)) {
3173 file
->Remove(false);
3175 entry
->MarkRemoved();
3181 ("CacheIndex::UpdateIndex() - Added/updated entry to/in index. "
3188 MOZ_ASSERT_UNREACHABLE("We should never get here");
3191 void CacheIndex::FinishUpdate(bool aSucceeded
,
3192 const StaticMutexAutoLock
& aProofOfLock
) {
3193 LOG(("CacheIndex::FinishUpdate() [succeeded=%d]", aSucceeded
));
3195 MOZ_ASSERT(mState
== UPDATING
|| mState
== BUILDING
||
3196 (!aSucceeded
&& mState
== SHUTDOWN
));
3198 if (mDirEnumerator
) {
3199 if (NS_IsMainThread()) {
3201 ("CacheIndex::FinishUpdate() - posting of PreShutdownInternal failed?"
3202 " Cannot safely release mDirEnumerator, leaking it!"));
3203 NS_WARNING(("CacheIndex::FinishUpdate() - Leaking mDirEnumerator!"));
3204 // This can happen only in case dispatching event to IO thread failed in
3205 // CacheIndex::PreShutdown().
3206 Unused
<< mDirEnumerator
.forget(); // Leak it since dir enumerator is not
3209 mDirEnumerator
->Close();
3210 mDirEnumerator
= nullptr;
3215 mDontMarkIndexClean
= true;
3218 if (mState
== SHUTDOWN
) {
3222 if (mState
== UPDATING
&& aSucceeded
) {
3223 // If we've iterated over all entries successfully then all entries that
3224 // really exist on the disk are now marked as fresh. All non-fresh entries
3225 // don't exist anymore and must be removed from the index.
3226 RemoveNonFreshEntries(aProofOfLock
);
3229 // Make sure we won't start update. If the build or update failed, there is no
3230 // reason to believe that it will succeed next time.
3231 mIndexNeedsUpdate
= false;
3233 ChangeState(READY
, aProofOfLock
);
3234 mLastDumpTime
= TimeStamp::NowLoRes(); // Do not dump new index immediately
3237 void CacheIndex::RemoveNonFreshEntries(
3238 const StaticMutexAutoLock
& aProofOfLock
) {
3239 sLock
.AssertCurrentThreadOwns();
3240 for (auto iter
= mIndex
.Iter(); !iter
.Done(); iter
.Next()) {
3241 CacheIndexEntry
* entry
= iter
.Get();
3242 if (entry
->IsFresh()) {
3247 ("CacheIndex::RemoveNonFreshEntries() - Removing entry. "
3248 "[hash=%08x%08x%08x%08x%08x]",
3249 LOGSHA1(entry
->Hash())));
3252 CacheIndexEntryAutoManage
emng(entry
->Hash(), this, aProofOfLock
);
3253 emng
.DoNotSearchInIndex();
3261 char const* CacheIndex::StateString(EState aState
) {
3279 MOZ_ASSERT(false, "Unexpected state!");
3283 void CacheIndex::ChangeState(EState aNewState
,
3284 const StaticMutexAutoLock
& aProofOfLock
) {
3285 sLock
.AssertCurrentThreadOwns();
3286 LOG(("CacheIndex::ChangeState() changing state %s -> %s", StateString(mState
),
3287 StateString(aNewState
)));
3289 // All pending updates should be processed before changing state
3290 MOZ_ASSERT(mPendingUpdates
.Count() == 0);
3292 // PreShutdownInternal() should change the state to READY from every state. It
3293 // may go through different states, but once we are in READY state the only
3294 // possible transition is to SHUTDOWN state.
3295 MOZ_ASSERT(!mShuttingDown
|| mState
!= READY
|| aNewState
== SHUTDOWN
);
3297 // Start updating process when switching to READY state if needed
3298 if (aNewState
== READY
&& StartUpdatingIndexIfNeeded(aProofOfLock
, true)) {
3302 // Try to evict entries over limit everytime we're leaving state READING,
3303 // BUILDING or UPDATING, but not during shutdown or when removing all
3305 if (!mShuttingDown
&& !mRemovingAll
&& aNewState
!= SHUTDOWN
&&
3306 (mState
== READING
|| mState
== BUILDING
|| mState
== UPDATING
)) {
3307 CacheFileIOManager::EvictIfOverLimit();
3312 if (mState
!= SHUTDOWN
) {
3313 CacheFileIOManager::CacheIndexStateChanged();
3316 NotifyAsyncGetDiskConsumptionCallbacks();
3319 void CacheIndex::NotifyAsyncGetDiskConsumptionCallbacks() {
3320 if ((mState
== READY
|| mState
== WRITING
) &&
3321 !mAsyncGetDiskConsumptionBlocked
&& mDiskConsumptionObservers
.Length()) {
3322 for (uint32_t i
= 0; i
< mDiskConsumptionObservers
.Length(); ++i
) {
3323 DiskConsumptionObserver
* o
= mDiskConsumptionObservers
[i
];
3324 // Safe to call under the lock. We always post to the main thread.
3325 o
->OnDiskConsumption(mIndexStats
.Size() << 10);
3328 mDiskConsumptionObservers
.Clear();
3332 void CacheIndex::AllocBuffer() {
3335 mRWBufSize
= sizeof(CacheIndexHeader
) + sizeof(CacheHash::Hash32_t
) +
3336 mProcessEntries
* sizeof(CacheIndexRecord
);
3337 if (mRWBufSize
> kMaxBufSize
) {
3338 mRWBufSize
= kMaxBufSize
;
3342 mRWBufSize
= kMaxBufSize
;
3345 MOZ_ASSERT(false, "Unexpected state!");
3348 mRWBuf
= static_cast<char*>(moz_xmalloc(mRWBufSize
));
3351 void CacheIndex::ReleaseBuffer() {
3352 sLock
.AssertCurrentThreadOwns();
3354 if (!mRWBuf
|| mRWPending
) {
3358 LOG(("CacheIndex::ReleaseBuffer() releasing buffer"));
3366 void CacheIndex::FrecencyArray::AppendRecord(
3367 CacheIndexRecordWrapper
* aRecord
, const StaticMutexAutoLock
& aProofOfLock
) {
3368 sLock
.AssertCurrentThreadOwns();
3370 ("CacheIndex::FrecencyArray::AppendRecord() [record=%p, hash=%08x%08x%08x"
3372 aRecord
, LOGSHA1(aRecord
->Get()->mHash
)));
3374 MOZ_DIAGNOSTIC_ASSERT(!mRecs
.Contains(aRecord
));
3375 mRecs
.AppendElement(aRecord
);
3377 // If the new frecency is 0, the element should be at the end of the array,
3378 // i.e. this change doesn't affect order of the array
3379 if (aRecord
->Get()->mFrecency
!= 0) {
3380 ++mUnsortedElements
;
3384 void CacheIndex::FrecencyArray::RemoveRecord(
3385 CacheIndexRecordWrapper
* aRecord
, const StaticMutexAutoLock
& aProofOfLock
) {
3386 sLock
.AssertCurrentThreadOwns();
3387 LOG(("CacheIndex::FrecencyArray::RemoveRecord() [record=%p]", aRecord
));
3389 decltype(mRecs
)::index_type idx
;
3390 idx
= mRecs
.IndexOf(aRecord
);
3391 MOZ_RELEASE_ASSERT(idx
!= mRecs
.NoIndex
);
3392 // sanity check to ensure correct record removal
3393 MOZ_RELEASE_ASSERT(mRecs
[idx
] == aRecord
);
3394 mRecs
[idx
] = nullptr;
3397 // Calling SortIfNeeded ensures that we get rid of removed elements in the
3398 // array once we hit the limit.
3399 SortIfNeeded(aProofOfLock
);
3402 void CacheIndex::FrecencyArray::ReplaceRecord(
3403 CacheIndexRecordWrapper
* aOldRecord
, CacheIndexRecordWrapper
* aNewRecord
,
3404 const StaticMutexAutoLock
& aProofOfLock
) {
3405 sLock
.AssertCurrentThreadOwns();
3407 ("CacheIndex::FrecencyArray::ReplaceRecord() [oldRecord=%p, "
3409 aOldRecord
, aNewRecord
));
3411 decltype(mRecs
)::index_type idx
;
3412 idx
= mRecs
.IndexOf(aOldRecord
);
3413 MOZ_RELEASE_ASSERT(idx
!= mRecs
.NoIndex
);
3414 // sanity check to ensure correct record replaced
3415 MOZ_RELEASE_ASSERT(mRecs
[idx
] == aOldRecord
);
3416 mRecs
[idx
] = aNewRecord
;
3419 void CacheIndex::FrecencyArray::SortIfNeeded(
3420 const StaticMutexAutoLock
& aProofOfLock
) {
3421 sLock
.AssertCurrentThreadOwns();
3422 const uint32_t kMaxUnsortedCount
= 512;
3423 const uint32_t kMaxUnsortedPercent
= 10;
3424 const uint32_t kMaxRemovedCount
= 512;
3426 uint32_t unsortedLimit
= std::min
<uint32_t>(
3427 kMaxUnsortedCount
, Length() * kMaxUnsortedPercent
/ 100);
3429 if (mUnsortedElements
> unsortedLimit
||
3430 mRemovedElements
> kMaxRemovedCount
) {
3432 ("CacheIndex::FrecencyArray::SortIfNeeded() - Sorting array "
3433 "[unsortedElements=%u, unsortedLimit=%u, removedElements=%u, "
3434 "maxRemovedCount=%u]",
3435 mUnsortedElements
, unsortedLimit
, mRemovedElements
, kMaxRemovedCount
));
3437 mRecs
.Sort(FrecencyComparator());
3438 mUnsortedElements
= 0;
3439 if (mRemovedElements
) {
3440 #if defined(EARLY_BETA_OR_EARLIER)
3441 // validate only null items are removed
3442 for (uint32_t i
= Length(); i
< mRecs
.Length(); ++i
) {
3443 MOZ_DIAGNOSTIC_ASSERT(!mRecs
[i
]);
3446 // Removed elements are at the end after sorting.
3447 mRecs
.RemoveElementsAt(Length(), mRemovedElements
);
3448 mRemovedElements
= 0;
3453 bool CacheIndex::FrecencyArray::RecordExistedUnlocked(
3454 CacheIndexRecordWrapper
* aRecord
) {
3455 return mRecs
.Contains(aRecord
);
3458 void CacheIndex::AddRecordToIterators(CacheIndexRecordWrapper
* aRecord
,
3459 const StaticMutexAutoLock
& aProofOfLock
) {
3460 sLock
.AssertCurrentThreadOwns();
3461 for (uint32_t i
= 0; i
< mIterators
.Length(); ++i
) {
3462 // Add a new record only when iterator is supposed to be updated.
3463 if (mIterators
[i
]->ShouldBeNewAdded()) {
3464 mIterators
[i
]->AddRecord(aRecord
, aProofOfLock
);
3469 void CacheIndex::RemoveRecordFromIterators(
3470 CacheIndexRecordWrapper
* aRecord
, const StaticMutexAutoLock
& aProofOfLock
) {
3471 sLock
.AssertCurrentThreadOwns();
3472 for (uint32_t i
= 0; i
< mIterators
.Length(); ++i
) {
3473 // Remove the record from iterator always, it makes no sence to return
3474 // non-existing entries. Also the pointer to the record is no longer valid
3475 // once the entry is removed from index.
3476 mIterators
[i
]->RemoveRecord(aRecord
, aProofOfLock
);
3480 void CacheIndex::ReplaceRecordInIterators(
3481 CacheIndexRecordWrapper
* aOldRecord
, CacheIndexRecordWrapper
* aNewRecord
,
3482 const StaticMutexAutoLock
& aProofOfLock
) {
3483 sLock
.AssertCurrentThreadOwns();
3484 for (uint32_t i
= 0; i
< mIterators
.Length(); ++i
) {
3485 // We have to replace the record always since the pointer is no longer
3486 // valid after this point. NOTE: Replacing the record doesn't mean that
3487 // a new entry was added, it just means that the data in the entry was
3488 // changed (e.g. a file size) and we had to track this change in
3489 // mPendingUpdates since mIndex was read-only.
3490 mIterators
[i
]->ReplaceRecord(aOldRecord
, aNewRecord
, aProofOfLock
);
3494 nsresult
CacheIndex::Run() {
3495 LOG(("CacheIndex::Run()"));
3497 StaticMutexAutoLock
lock(sLock
);
3499 if (!IsIndexUsable()) {
3500 return NS_ERROR_NOT_AVAILABLE
;
3503 if (mState
== READY
&& mShuttingDown
) {
3507 mUpdateEventPending
= false;
3517 LOG(("CacheIndex::Run() - Update/Build was canceled"));
3523 void CacheIndex::OnFileOpenedInternal(FileOpenHelper
* aOpener
,
3524 CacheFileHandle
* aHandle
,
3526 const StaticMutexAutoLock
& aProofOfLock
) {
3527 sLock
.AssertCurrentThreadOwns();
3529 ("CacheIndex::OnFileOpenedInternal() [opener=%p, handle=%p, "
3530 "result=0x%08" PRIx32
"]",
3531 aOpener
, aHandle
, static_cast<uint32_t>(aResult
)));
3532 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
3536 MOZ_RELEASE_ASSERT(IsIndexUsable());
3538 if (mState
== READY
&& mShuttingDown
) {
3544 MOZ_ASSERT(aOpener
== mIndexFileOpener
);
3545 mIndexFileOpener
= nullptr;
3547 if (NS_FAILED(aResult
)) {
3549 ("CacheIndex::OnFileOpenedInternal() - Can't open index file for "
3550 "writing [rv=0x%08" PRIx32
"]",
3551 static_cast<uint32_t>(aResult
)));
3552 FinishWrite(false, aProofOfLock
);
3554 mIndexHandle
= aHandle
;
3555 WriteRecords(aProofOfLock
);
3559 if (aOpener
== mIndexFileOpener
) {
3560 mIndexFileOpener
= nullptr;
3562 if (NS_SUCCEEDED(aResult
)) {
3563 if (aHandle
->FileSize() == 0) {
3564 FinishRead(false, aProofOfLock
);
3565 CacheFileIOManager::DoomFile(aHandle
, nullptr);
3568 mIndexHandle
= aHandle
;
3570 FinishRead(false, aProofOfLock
);
3573 } else if (aOpener
== mJournalFileOpener
) {
3574 mJournalFileOpener
= nullptr;
3575 mJournalHandle
= aHandle
;
3576 } else if (aOpener
== mTmpFileOpener
) {
3577 mTmpFileOpener
= nullptr;
3578 mTmpHandle
= aHandle
;
3580 MOZ_ASSERT(false, "Unexpected state!");
3583 if (mIndexFileOpener
|| mJournalFileOpener
|| mTmpFileOpener
) {
3584 // Some opener still didn't finish
3588 // We fail and cancel all other openers when we opening index file fails.
3589 MOZ_ASSERT(mIndexHandle
);
3592 CacheFileIOManager::DoomFile(mTmpHandle
, nullptr);
3593 mTmpHandle
= nullptr;
3595 if (mJournalHandle
) { // this shouldn't normally happen
3597 ("CacheIndex::OnFileOpenedInternal() - Unexpected state, all "
3598 "files [%s, %s, %s] should never exist. Removing whole index.",
3599 INDEX_NAME
, JOURNAL_NAME
, TEMP_INDEX_NAME
));
3600 FinishRead(false, aProofOfLock
);
3605 if (mJournalHandle
) {
3606 // Rename journal to make sure we update index on next start in case
3608 rv
= CacheFileIOManager::RenameFile(
3609 mJournalHandle
, nsLiteralCString(TEMP_INDEX_NAME
), this);
3610 if (NS_FAILED(rv
)) {
3612 ("CacheIndex::OnFileOpenedInternal() - CacheFileIOManager::"
3613 "RenameFile() failed synchronously [rv=0x%08" PRIx32
"]",
3614 static_cast<uint32_t>(rv
)));
3615 FinishRead(false, aProofOfLock
);
3619 StartReadingIndex(aProofOfLock
);
3624 MOZ_ASSERT(false, "Unexpected state!");
3628 nsresult
CacheIndex::OnFileOpened(CacheFileHandle
* aHandle
, nsresult aResult
) {
3629 MOZ_CRASH("CacheIndex::OnFileOpened should not be called!");
3630 return NS_ERROR_UNEXPECTED
;
3633 nsresult
CacheIndex::OnDataWritten(CacheFileHandle
* aHandle
, const char* aBuf
,
3635 LOG(("CacheIndex::OnDataWritten() [handle=%p, result=0x%08" PRIx32
"]",
3636 aHandle
, static_cast<uint32_t>(aResult
)));
3638 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
3642 StaticMutexAutoLock
lock(sLock
);
3644 MOZ_RELEASE_ASSERT(IsIndexUsable());
3645 MOZ_RELEASE_ASSERT(mRWPending
);
3648 if (mState
== READY
&& mShuttingDown
) {
3654 MOZ_ASSERT(mIndexHandle
== aHandle
);
3656 if (NS_FAILED(aResult
)) {
3657 FinishWrite(false, lock
);
3659 if (mSkipEntries
== mProcessEntries
) {
3660 rv
= CacheFileIOManager::RenameFile(
3661 mIndexHandle
, nsLiteralCString(INDEX_NAME
), this);
3662 if (NS_FAILED(rv
)) {
3664 ("CacheIndex::OnDataWritten() - CacheFileIOManager::"
3665 "RenameFile() failed synchronously [rv=0x%08" PRIx32
"]",
3666 static_cast<uint32_t>(rv
)));
3667 FinishWrite(false, lock
);
3675 // Writing was canceled.
3677 ("CacheIndex::OnDataWritten() - ignoring notification since the "
3678 "operation was previously canceled [state=%d]",
3686 nsresult
CacheIndex::OnDataRead(CacheFileHandle
* aHandle
, char* aBuf
,
3688 LOG(("CacheIndex::OnDataRead() [handle=%p, result=0x%08" PRIx32
"]", aHandle
,
3689 static_cast<uint32_t>(aResult
)));
3691 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
3693 StaticMutexAutoLock
lock(sLock
);
3695 MOZ_RELEASE_ASSERT(IsIndexUsable());
3696 MOZ_RELEASE_ASSERT(mRWPending
);
3701 MOZ_ASSERT(mIndexHandle
== aHandle
|| mJournalHandle
== aHandle
);
3703 if (NS_FAILED(aResult
)) {
3704 FinishRead(false, lock
);
3706 if (!mIndexOnDiskIsValid
) {
3714 // Reading was canceled.
3716 ("CacheIndex::OnDataRead() - ignoring notification since the "
3717 "operation was previously canceled [state=%d]",
3725 nsresult
CacheIndex::OnFileDoomed(CacheFileHandle
* aHandle
, nsresult aResult
) {
3726 MOZ_CRASH("CacheIndex::OnFileDoomed should not be called!");
3727 return NS_ERROR_UNEXPECTED
;
3730 nsresult
CacheIndex::OnEOFSet(CacheFileHandle
* aHandle
, nsresult aResult
) {
3731 MOZ_CRASH("CacheIndex::OnEOFSet should not be called!");
3732 return NS_ERROR_UNEXPECTED
;
3735 nsresult
CacheIndex::OnFileRenamed(CacheFileHandle
* aHandle
, nsresult aResult
) {
3736 LOG(("CacheIndex::OnFileRenamed() [handle=%p, result=0x%08" PRIx32
"]",
3737 aHandle
, static_cast<uint32_t>(aResult
)));
3739 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
3741 StaticMutexAutoLock
lock(sLock
);
3743 MOZ_RELEASE_ASSERT(IsIndexUsable());
3745 if (mState
== READY
&& mShuttingDown
) {
3751 // This is a result of renaming the new index written to tmpfile to index
3752 // file. This is the last step when writing the index and the whole
3753 // writing process is successful iff renaming was successful.
3755 if (mIndexHandle
!= aHandle
) {
3757 ("CacheIndex::OnFileRenamed() - ignoring notification since it "
3758 "belongs to previously canceled operation [state=%d]",
3763 FinishWrite(NS_SUCCEEDED(aResult
), lock
);
3766 // This is a result of renaming journal file to tmpfile. It is renamed
3767 // before we start reading index and journal file and it should normally
3768 // succeed. If it fails give up reading of index.
3770 if (mJournalHandle
!= aHandle
) {
3772 ("CacheIndex::OnFileRenamed() - ignoring notification since it "
3773 "belongs to previously canceled operation [state=%d]",
3778 if (NS_FAILED(aResult
)) {
3779 FinishRead(false, lock
);
3781 StartReadingIndex(lock
);
3785 // Reading/writing was canceled.
3787 ("CacheIndex::OnFileRenamed() - ignoring notification since the "
3788 "operation was previously canceled [state=%d]",
3797 size_t CacheIndex::SizeOfExcludingThisInternal(
3798 mozilla::MallocSizeOf mallocSizeOf
) const {
3799 sLock
.AssertCurrentThreadOwns();
3802 nsCOMPtr
<nsISizeOf
> sizeOf
;
3804 // mIndexHandle and mJournalHandle are reported via SizeOfHandlesRunnable
3805 // in CacheFileIOManager::SizeOfExcludingThisInternal as part of special
3808 sizeOf
= do_QueryInterface(mCacheDirectory
);
3810 n
+= sizeOf
->SizeOfIncludingThis(mallocSizeOf
);
3813 sizeOf
= do_QueryInterface(mUpdateTimer
);
3815 n
+= sizeOf
->SizeOfIncludingThis(mallocSizeOf
);
3818 n
+= mallocSizeOf(mRWBuf
);
3819 n
+= mallocSizeOf(mRWHash
);
3821 n
+= mIndex
.SizeOfExcludingThis(mallocSizeOf
);
3822 n
+= mPendingUpdates
.SizeOfExcludingThis(mallocSizeOf
);
3823 n
+= mTmpJournal
.SizeOfExcludingThis(mallocSizeOf
);
3825 // mFrecencyArray items are reported by mIndex/mPendingUpdates
3826 n
+= mFrecencyArray
.mRecs
.ShallowSizeOfExcludingThis(mallocSizeOf
);
3827 n
+= mDiskConsumptionObservers
.ShallowSizeOfExcludingThis(mallocSizeOf
);
3833 size_t CacheIndex::SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf
) {
3834 StaticMutexAutoLock
lock(sLock
);
3836 if (!gInstance
) return 0;
3838 return gInstance
->SizeOfExcludingThisInternal(mallocSizeOf
);
3842 size_t CacheIndex::SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf
) {
3843 StaticMutexAutoLock
lock(sLock
);
3845 return mallocSizeOf(gInstance
) +
3846 (gInstance
? gInstance
->SizeOfExcludingThisInternal(mallocSizeOf
) : 0);
3850 void CacheIndex::UpdateTotalBytesWritten(uint32_t aBytesWritten
) {
3851 StaticMutexAutoLock
lock(sLock
);
3853 RefPtr
<CacheIndex
> index
= gInstance
;
3858 index
->mTotalBytesWritten
+= aBytesWritten
;
3860 // Do telemetry report if enough data has been written and the index is
3861 // in READY state. The data is available also in WRITING state, but we would
3862 // need to deal with pending updates.
3863 if (index
->mTotalBytesWritten
>= kTelemetryReportBytesLimit
&&
3864 index
->mState
== READY
&& !index
->mIndexNeedsUpdate
&&
3865 !index
->mShuttingDown
) {
3866 index
->DoTelemetryReport();
3867 index
->mTotalBytesWritten
= 0;
3872 void CacheIndex::DoTelemetryReport() {
3873 static const nsLiteralCString
3874 contentTypeNames
[nsICacheEntry::CONTENT_TYPE_LAST
] = {
3875 "UNKNOWN"_ns
, "OTHER"_ns
, "JAVASCRIPT"_ns
, "IMAGE"_ns
,
3876 "MEDIA"_ns
, "STYLESHEET"_ns
, "WASM"_ns
};
3878 for (uint32_t i
= 0; i
< nsICacheEntry::CONTENT_TYPE_LAST
; ++i
) {
3879 if (mIndexStats
.Size() > 0) {
3880 Telemetry::Accumulate(
3881 Telemetry::NETWORK_CACHE_SIZE_SHARE
, contentTypeNames
[i
],
3882 round(static_cast<double>(mIndexStats
.SizeByType(i
)) * 100.0 /
3883 static_cast<double>(mIndexStats
.Size())));
3886 if (mIndexStats
.Count() > 0) {
3887 Telemetry::Accumulate(
3888 Telemetry::NETWORK_CACHE_ENTRY_COUNT_SHARE
, contentTypeNames
[i
],
3889 round(static_cast<double>(mIndexStats
.CountByType(i
)) * 100.0 /
3890 static_cast<double>(mIndexStats
.Count())));
3895 if (CacheObserver::SmartCacheSizeEnabled()) {
3896 probeKey
= "SMARTSIZE"_ns
;
3898 probeKey
= "USERDEFINEDSIZE"_ns
;
3900 Telemetry::Accumulate(Telemetry::NETWORK_CACHE_ENTRY_COUNT
, probeKey
,
3901 mIndexStats
.Count());
3902 Telemetry::Accumulate(Telemetry::NETWORK_CACHE_SIZE
, probeKey
,
3903 mIndexStats
.Size() >> 10);
3907 void CacheIndex::OnAsyncEviction(bool aEvicting
) {
3908 StaticMutexAutoLock
lock(sLock
);
3910 RefPtr
<CacheIndex
> index
= gInstance
;
3915 index
->mAsyncGetDiskConsumptionBlocked
= aEvicting
;
3917 index
->NotifyAsyncGetDiskConsumptionCallbacks();
3921 } // namespace mozilla::net