1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
5 #ifndef CacheIndex__h__
6 #define CacheIndex__h__
9 #include "CacheFileIOManager.h"
10 #include "nsIRunnable.h"
11 #include "CacheHashUtils.h"
12 #include "nsICacheStorageService.h"
13 #include "nsICacheEntry.h"
14 #include "nsILoadContextInfo.h"
15 #include "nsTHashtable.h"
16 #include "nsThreadUtils.h"
17 #include "nsWeakReference.h"
18 #include "mozilla/SHA1.h"
19 #include "mozilla/Mutex.h"
20 #include "mozilla/Endian.h"
21 #include "mozilla/TimeStamp.h"
24 class nsIDirectoryEnumerator
;
35 class CacheFileMetadata
;
37 class CacheIndexIterator
;
40 // Version of the index. The index must be ignored and deleted when the file
41 // on disk was written with a newer version.
44 // Timestamp of time when the last successful write of the index started.
45 // During update process we use this timestamp for a quick validation of entry
46 // files. If last modified time of the file is lower than this timestamp, we
47 // skip parsing of such file since the information in index should be up to
51 // We set this flag as soon as possible after parsing index during startup
52 // and clean it after we write journal to disk during shutdown. We ignore the
53 // journal and start update process whenever this flag is set during index
58 struct CacheIndexRecord
{
61 uint32_t mExpirationTime
;
65 * 1000 0000 0000 0000 0000 0000 0000 0000 : initialized
66 * 0100 0000 0000 0000 0000 0000 0000 0000 : anonymous
67 * 0010 0000 0000 0000 0000 0000 0000 0000 : inBrowser
68 * 0001 0000 0000 0000 0000 0000 0000 0000 : removed
69 * 0000 1000 0000 0000 0000 0000 0000 0000 : dirty
70 * 0000 0100 0000 0000 0000 0000 0000 0000 : fresh
71 * 0000 0011 0000 0000 0000 0000 0000 0000 : reserved
72 * 0000 0000 1111 1111 1111 1111 1111 1111 : file size (in kB)
78 , mExpirationTime(nsICacheEntry::NO_EXPIRATION_TIME
)
79 , mAppId(nsILoadContextInfo::NO_APP_ID
)
84 class CacheIndexEntry
: public PLDHashEntryHdr
87 typedef const SHA1Sum::Hash
& KeyType
;
88 typedef const SHA1Sum::Hash
* KeyTypePointer
;
90 explicit CacheIndexEntry(KeyTypePointer aKey
)
92 MOZ_COUNT_CTOR(CacheIndexEntry
);
93 mRec
= new CacheIndexRecord();
94 LOG(("CacheIndexEntry::CacheIndexEntry() - Created record [rec=%p]", mRec
.get()));
95 memcpy(&mRec
->mHash
, aKey
, sizeof(SHA1Sum::Hash
));
97 CacheIndexEntry(const CacheIndexEntry
& aOther
)
99 NS_NOTREACHED("CacheIndexEntry copy constructor is forbidden!");
103 MOZ_COUNT_DTOR(CacheIndexEntry
);
104 LOG(("CacheIndexEntry::~CacheIndexEntry() - Deleting record [rec=%p]",
108 // KeyEquals(): does this entry match this key?
109 bool KeyEquals(KeyTypePointer aKey
) const
111 return memcmp(&mRec
->mHash
, aKey
, sizeof(SHA1Sum::Hash
)) == 0;
114 // KeyToPointer(): Convert KeyType to KeyTypePointer
115 static KeyTypePointer
KeyToPointer(KeyType aKey
) { return &aKey
; }
117 // HashKey(): calculate the hash number
118 static PLDHashNumber
HashKey(KeyTypePointer aKey
)
120 return (reinterpret_cast<const uint32_t *>(aKey
))[0];
123 // ALLOW_MEMMOVE can we move this class with memmove(), or do we have
124 // to use the copy constructor?
125 enum { ALLOW_MEMMOVE
= true };
127 bool operator==(const CacheIndexEntry
& aOther
) const
129 return KeyEquals(&aOther
.mRec
->mHash
);
132 CacheIndexEntry
& operator=(const CacheIndexEntry
& aOther
)
134 MOZ_ASSERT(memcmp(&mRec
->mHash
, &aOther
.mRec
->mHash
,
135 sizeof(SHA1Sum::Hash
)) == 0);
136 mRec
->mFrecency
= aOther
.mRec
->mFrecency
;
137 mRec
->mExpirationTime
= aOther
.mRec
->mExpirationTime
;
138 mRec
->mAppId
= aOther
.mRec
->mAppId
;
139 mRec
->mFlags
= aOther
.mRec
->mFlags
;
146 mRec
->mExpirationTime
= nsICacheEntry::NO_EXPIRATION_TIME
;
147 mRec
->mAppId
= nsILoadContextInfo::NO_APP_ID
;
151 void Init(uint32_t aAppId
, bool aAnonymous
, bool aInBrowser
)
153 MOZ_ASSERT(mRec
->mFrecency
== 0);
154 MOZ_ASSERT(mRec
->mExpirationTime
== nsICacheEntry::NO_EXPIRATION_TIME
);
155 MOZ_ASSERT(mRec
->mAppId
== nsILoadContextInfo::NO_APP_ID
);
156 // When we init the entry it must be fresh and may be dirty
157 MOZ_ASSERT((mRec
->mFlags
& ~kDirtyMask
) == kFreshMask
);
159 mRec
->mAppId
= aAppId
;
160 mRec
->mFlags
|= kInitializedMask
;
162 mRec
->mFlags
|= kAnonymousMask
;
165 mRec
->mFlags
|= kInBrowserMask
;
169 const SHA1Sum::Hash
* Hash() const { return &mRec
->mHash
; }
171 bool IsInitialized() const { return !!(mRec
->mFlags
& kInitializedMask
); }
173 uint32_t AppId() const { return mRec
->mAppId
; }
174 bool Anonymous() const { return !!(mRec
->mFlags
& kAnonymousMask
); }
175 bool InBrowser() const { return !!(mRec
->mFlags
& kInBrowserMask
); }
177 bool IsRemoved() const { return !!(mRec
->mFlags
& kRemovedMask
); }
178 void MarkRemoved() { mRec
->mFlags
|= kRemovedMask
; }
180 bool IsDirty() const { return !!(mRec
->mFlags
& kDirtyMask
); }
181 void MarkDirty() { mRec
->mFlags
|= kDirtyMask
; }
182 void ClearDirty() { mRec
->mFlags
&= ~kDirtyMask
; }
184 bool IsFresh() const { return !!(mRec
->mFlags
& kFreshMask
); }
185 void MarkFresh() { mRec
->mFlags
|= kFreshMask
; }
187 void SetFrecency(uint32_t aFrecency
) { mRec
->mFrecency
= aFrecency
; }
188 uint32_t GetFrecency() const { return mRec
->mFrecency
; }
190 void SetExpirationTime(uint32_t aExpirationTime
)
192 mRec
->mExpirationTime
= aExpirationTime
;
194 uint32_t GetExpirationTime() const { return mRec
->mExpirationTime
; }
196 // Sets filesize in kilobytes.
197 void SetFileSize(uint32_t aFileSize
)
199 if (aFileSize
> kFileSizeMask
) {
200 LOG(("CacheIndexEntry::SetFileSize() - FileSize is too large, "
201 "truncating to %u", kFileSizeMask
));
202 aFileSize
= kFileSizeMask
;
204 mRec
->mFlags
&= ~kFileSizeMask
;
205 mRec
->mFlags
|= aFileSize
;
207 // Returns filesize in kilobytes.
208 uint32_t GetFileSize() const { return GetFileSize(mRec
); }
209 static uint32_t GetFileSize(CacheIndexRecord
*aRec
)
211 return aRec
->mFlags
& kFileSizeMask
;
213 bool IsFileEmpty() const { return GetFileSize() == 0; }
215 void WriteToBuf(void *aBuf
)
217 CacheIndexRecord
*dst
= reinterpret_cast<CacheIndexRecord
*>(aBuf
);
219 // Copy the whole record to the buffer.
220 memcpy(aBuf
, mRec
, sizeof(CacheIndexRecord
));
222 // Dirty and fresh flags should never go to disk, since they make sense only
223 // during current session.
224 dst
->mFlags
&= ~kDirtyMask
;
225 dst
->mFlags
&= ~kFreshMask
;
227 #if defined(IS_LITTLE_ENDIAN)
228 // Data in the buffer are in machine byte order and we want them in network
230 NetworkEndian::writeUint32(&dst
->mFrecency
, dst
->mFrecency
);
231 NetworkEndian::writeUint32(&dst
->mExpirationTime
, dst
->mExpirationTime
);
232 NetworkEndian::writeUint32(&dst
->mAppId
, dst
->mAppId
);
233 NetworkEndian::writeUint32(&dst
->mFlags
, dst
->mFlags
);
237 void ReadFromBuf(void *aBuf
)
239 CacheIndexRecord
*src
= reinterpret_cast<CacheIndexRecord
*>(aBuf
);
240 MOZ_ASSERT(memcmp(&mRec
->mHash
, &src
->mHash
,
241 sizeof(SHA1Sum::Hash
)) == 0);
243 mRec
->mFrecency
= NetworkEndian::readUint32(&src
->mFrecency
);
244 mRec
->mExpirationTime
= NetworkEndian::readUint32(&src
->mExpirationTime
);
245 mRec
->mAppId
= NetworkEndian::readUint32(&src
->mAppId
);
246 mRec
->mFlags
= NetworkEndian::readUint32(&src
->mFlags
);
250 LOG(("CacheIndexEntry::Log() [this=%p, hash=%08x%08x%08x%08x%08x, fresh=%u,"
251 " initialized=%u, removed=%u, dirty=%u, anonymous=%u, inBrowser=%u, "
252 "appId=%u, frecency=%u, expirationTime=%u, size=%u]",
253 this, LOGSHA1(mRec
->mHash
), IsFresh(), IsInitialized(), IsRemoved(),
254 IsDirty(), Anonymous(), InBrowser(), AppId(), GetFrecency(),
255 GetExpirationTime(), GetFileSize()));
258 static bool RecordMatchesLoadContextInfo(CacheIndexRecord
*aRec
,
259 nsILoadContextInfo
*aInfo
)
261 if (!aInfo
->IsPrivate() &&
262 aInfo
->AppId() == aRec
->mAppId
&&
263 aInfo
->IsAnonymous() == !!(aRec
->mFlags
& kAnonymousMask
) &&
264 aInfo
->IsInBrowserElement() == !!(aRec
->mFlags
& kInBrowserMask
)) {
272 size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf
) const
274 return mallocSizeOf(mRec
.get());
277 size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf
) const
279 return mallocSizeOf(this) + SizeOfExcludingThis(mallocSizeOf
);
283 friend class CacheIndexEntryUpdate
;
284 friend class CacheIndex
;
285 friend class CacheIndexEntryAutoManage
;
287 static const uint32_t kInitializedMask
= 0x80000000;
288 static const uint32_t kAnonymousMask
= 0x40000000;
289 static const uint32_t kInBrowserMask
= 0x20000000;
291 // This flag is set when the entry was removed. We need to keep this
292 // information in memory until we write the index file.
293 static const uint32_t kRemovedMask
= 0x10000000;
295 // This flag is set when the information in memory is not in sync with the
296 // information in index file on disk.
297 static const uint32_t kDirtyMask
= 0x08000000;
299 // This flag is set when the information about the entry is fresh, i.e.
300 // we've created or opened this entry during this session, or we've seen
301 // this entry during update or build process.
302 static const uint32_t kFreshMask
= 0x04000000;
304 static const uint32_t kReservedMask
= 0x03000000;
306 // FileSize in kilobytes
307 static const uint32_t kFileSizeMask
= 0x00FFFFFF;
309 nsAutoPtr
<CacheIndexRecord
> mRec
;
312 class CacheIndexEntryUpdate
: public CacheIndexEntry
315 explicit CacheIndexEntryUpdate(CacheIndexEntry::KeyTypePointer aKey
)
316 : CacheIndexEntry(aKey
)
319 MOZ_COUNT_CTOR(CacheIndexEntryUpdate
);
320 LOG(("CacheIndexEntryUpdate::CacheIndexEntryUpdate()"));
322 ~CacheIndexEntryUpdate()
324 MOZ_COUNT_DTOR(CacheIndexEntryUpdate
);
325 LOG(("CacheIndexEntryUpdate::~CacheIndexEntryUpdate()"));
328 CacheIndexEntryUpdate
& operator=(const CacheIndexEntry
& aOther
)
330 MOZ_ASSERT(memcmp(&mRec
->mHash
, &aOther
.mRec
->mHash
,
331 sizeof(SHA1Sum::Hash
)) == 0);
333 *(static_cast<CacheIndexEntry
*>(this)) = aOther
;
339 mUpdateFlags
= kFrecencyUpdatedMask
| kExpirationUpdatedMask
|
340 kFileSizeUpdatedMask
;
341 CacheIndexEntry::InitNew();
344 void SetFrecency(uint32_t aFrecency
)
346 mUpdateFlags
|= kFrecencyUpdatedMask
;
347 CacheIndexEntry::SetFrecency(aFrecency
);
350 void SetExpirationTime(uint32_t aExpirationTime
)
352 mUpdateFlags
|= kExpirationUpdatedMask
;
353 CacheIndexEntry::SetExpirationTime(aExpirationTime
);
356 void SetFileSize(uint32_t aFileSize
)
358 mUpdateFlags
|= kFileSizeUpdatedMask
;
359 CacheIndexEntry::SetFileSize(aFileSize
);
362 void ApplyUpdate(CacheIndexEntry
*aDst
) {
363 MOZ_ASSERT(memcmp(&mRec
->mHash
, &aDst
->mRec
->mHash
,
364 sizeof(SHA1Sum::Hash
)) == 0);
365 if (mUpdateFlags
& kFrecencyUpdatedMask
) {
366 aDst
->mRec
->mFrecency
= mRec
->mFrecency
;
368 if (mUpdateFlags
& kExpirationUpdatedMask
) {
369 aDst
->mRec
->mExpirationTime
= mRec
->mExpirationTime
;
371 aDst
->mRec
->mAppId
= mRec
->mAppId
;
372 if (mUpdateFlags
& kFileSizeUpdatedMask
) {
373 aDst
->mRec
->mFlags
= mRec
->mFlags
;
375 // Copy all flags except file size.
376 aDst
->mRec
->mFlags
&= kFileSizeMask
;
377 aDst
->mRec
->mFlags
|= (mRec
->mFlags
& ~kFileSizeMask
);
382 static const uint32_t kFrecencyUpdatedMask
= 0x00000001;
383 static const uint32_t kExpirationUpdatedMask
= 0x00000002;
384 static const uint32_t kFileSizeUpdatedMask
= 0x00000004;
386 uint32_t mUpdateFlags
;
389 class CacheIndexStats
401 , mStateLogged(false)
402 , mDisableLogging(false)
407 bool operator==(const CacheIndexStats
& aOther
) const
411 aOther
.mStateLogged
== mStateLogged
&&
413 aOther
.mCount
== mCount
&&
414 aOther
.mNotInitialized
== mNotInitialized
&&
415 aOther
.mRemoved
== mRemoved
&&
416 aOther
.mDirty
== mDirty
&&
417 aOther
.mFresh
== mFresh
&&
418 aOther
.mEmpty
== mEmpty
&&
419 aOther
.mSize
== mSize
;
423 void DisableLogging() {
424 mDisableLogging
= true;
429 LOG(("CacheIndexStats::Log() [count=%u, notInitialized=%u, removed=%u, "
430 "dirty=%u, fresh=%u, empty=%u, size=%u]", mCount
, mNotInitialized
,
431 mRemoved
, mDirty
, mFresh
, mEmpty
, mSize
));
435 MOZ_ASSERT(!mStateLogged
, "CacheIndexStats::Clear() - state logged!");
453 MOZ_ASSERT(!mStateLogged
, "CacheIndexStats::Count() - state logged!");
458 MOZ_ASSERT(!mStateLogged
, "CacheIndexStats::Dirty() - state logged!");
463 MOZ_ASSERT(!mStateLogged
, "CacheIndexStats::Fresh() - state logged!");
467 uint32_t ActiveEntriesCount() {
468 MOZ_ASSERT(!mStateLogged
, "CacheIndexStats::ActiveEntriesCount() - state "
470 return mCount
- mRemoved
- mNotInitialized
- mEmpty
;
474 MOZ_ASSERT(!mStateLogged
, "CacheIndexStats::Size() - state logged!");
478 void BeforeChange(const CacheIndexEntry
*aEntry
) {
480 if (!mDisableLogging
) {
481 LOG(("CacheIndexStats::BeforeChange()"));
486 MOZ_ASSERT(!mStateLogged
, "CacheIndexStats::BeforeChange() - state "
494 if (aEntry
->IsDirty()) {
498 if (aEntry
->IsFresh()) {
502 if (aEntry
->IsRemoved()) {
503 MOZ_ASSERT(mRemoved
);
506 if (!aEntry
->IsInitialized()) {
507 MOZ_ASSERT(mNotInitialized
);
510 if (aEntry
->IsFileEmpty()) {
514 MOZ_ASSERT(mSize
>= aEntry
->GetFileSize());
515 mSize
-= aEntry
->GetFileSize();
522 void AfterChange(const CacheIndexEntry
*aEntry
) {
523 MOZ_ASSERT(mStateLogged
, "CacheIndexStats::AfterChange() - state not "
526 mStateLogged
= false;
530 if (aEntry
->IsDirty()) {
533 if (aEntry
->IsFresh()) {
536 if (aEntry
->IsRemoved()) {
539 if (!aEntry
->IsInitialized()) {
542 if (aEntry
->IsFileEmpty()) {
545 mSize
+= aEntry
->GetFileSize();
552 if (!mDisableLogging
) {
553 LOG(("CacheIndexStats::AfterChange()"));
561 uint32_t mNotInitialized
;
568 // We completely remove the data about an entry from the stats in
569 // BeforeChange() and set this flag to true. The entry is then modified,
570 // deleted or created and the data is again put into the stats and this flag
571 // set to false. Statistics must not be read during this time since the
572 // information is not correct.
575 // Disables logging in this instance of CacheIndexStats
576 bool mDisableLogging
;
580 class CacheIndex
: public CacheFileIOListener
584 NS_DECL_THREADSAFE_ISUPPORTS
589 static nsresult
Init(nsIFile
*aCacheDirectory
);
590 static nsresult
PreShutdown();
591 static nsresult
Shutdown();
593 // Following methods can be called only on IO thread.
595 // Add entry to the index. The entry shouldn't be present in index. This
596 // method is called whenever a new handle for a new entry file is created. The
597 // newly created entry is not initialized and it must be either initialized
598 // with InitEntry() or removed with RemoveEntry().
599 static nsresult
AddEntry(const SHA1Sum::Hash
*aHash
);
601 // Inform index about an existing entry that should be present in index. This
602 // method is called whenever a new handle for an existing entry file is
603 // created. Like in case of AddEntry(), either InitEntry() or RemoveEntry()
604 // must be called on the entry, since the entry is not initizlized if the
605 // index is outdated.
606 static nsresult
EnsureEntryExists(const SHA1Sum::Hash
*aHash
);
608 // Initialize the entry. It MUST be present in index. Call to AddEntry() or
609 // EnsureEntryExists() must precede the call to this method.
610 static nsresult
InitEntry(const SHA1Sum::Hash
*aHash
,
615 // Remove entry from index. The entry should be present in index.
616 static nsresult
RemoveEntry(const SHA1Sum::Hash
*aHash
);
618 // Update some information in entry. The entry MUST be present in index and
619 // MUST be initialized. Call to AddEntry() or EnsureEntryExists() and to
620 // InitEntry() must precede the call to this method.
621 // Pass nullptr if the value didn't change.
622 static nsresult
UpdateEntry(const SHA1Sum::Hash
*aHash
,
623 const uint32_t *aFrecency
,
624 const uint32_t *aExpirationTime
,
625 const uint32_t *aSize
);
627 // Remove all entries from the index. Called when clearing the whole cache.
628 static nsresult
RemoveAll();
636 // Returns status of the entry in index for the given key. It can be called
638 static nsresult
HasEntry(const nsACString
&aKey
, EntryStatus
*_retval
);
640 // Returns a hash of the least important entry that should be evicted if the
641 // cache size is over limit and also returns a total number of all entries in
642 // the index minus the number of forced valid entries that we encounter
643 // when searching (see below)
644 static nsresult
GetEntryForEviction(bool aIgnoreEmptyEntries
, SHA1Sum::Hash
*aHash
, uint32_t *aCnt
);
646 // Checks if a cache entry is currently forced valid. Used to prevent an entry
647 // (that has been forced valid) from being evicted when the cache size reaches
649 static bool IsForcedValidEntry(const SHA1Sum::Hash
*aHash
);
651 // Returns cache size in kB.
652 static nsresult
GetCacheSize(uint32_t *_retval
);
654 // Synchronously returns the disk occupation and number of entries per-context.
655 // Callable on any thread.
656 static nsresult
GetCacheStats(nsILoadContextInfo
*aInfo
, uint32_t *aSize
, uint32_t *aCount
);
658 // Asynchronously gets the disk cache size, used for display in the UI.
659 static nsresult
AsyncGetDiskConsumption(nsICacheStorageConsumptionObserver
* aObserver
);
661 // Returns an iterator that returns entries matching a given context that were
662 // present in the index at the time this method was called. If aAddNew is true
663 // then the iterator will also return entries created after this call.
664 // NOTE: When some entry is removed from index it is removed also from the
665 // iterator regardless what aAddNew was passed.
666 static nsresult
GetIterator(nsILoadContextInfo
*aInfo
, bool aAddNew
,
667 CacheIndexIterator
**_retval
);
669 // Returns true if we _think_ that the index is up to date. I.e. the state is
670 // READY or WRITING and mIndexNeedsUpdate as well as mShuttingDown is false.
671 static nsresult
IsUpToDate(bool *_retval
);
674 static size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf
);
675 static size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf
);
678 friend class CacheIndexEntryAutoManage
;
679 friend class CacheIndexAutoLock
;
680 friend class CacheIndexAutoUnlock
;
681 friend class FileOpenHelper
;
682 friend class CacheIndexIterator
;
684 virtual ~CacheIndex();
686 NS_IMETHOD
OnFileOpened(CacheFileHandle
*aHandle
, nsresult aResult
) MOZ_OVERRIDE
;
687 nsresult
OnFileOpenedInternal(FileOpenHelper
*aOpener
,
688 CacheFileHandle
*aHandle
, nsresult aResult
);
689 NS_IMETHOD
OnDataWritten(CacheFileHandle
*aHandle
, const char *aBuf
,
690 nsresult aResult
) MOZ_OVERRIDE
;
691 NS_IMETHOD
OnDataRead(CacheFileHandle
*aHandle
, char *aBuf
, nsresult aResult
) MOZ_OVERRIDE
;
692 NS_IMETHOD
OnFileDoomed(CacheFileHandle
*aHandle
, nsresult aResult
) MOZ_OVERRIDE
;
693 NS_IMETHOD
OnEOFSet(CacheFileHandle
*aHandle
, nsresult aResult
) MOZ_OVERRIDE
;
694 NS_IMETHOD
OnFileRenamed(CacheFileHandle
*aHandle
, nsresult aResult
) MOZ_OVERRIDE
;
698 void AssertOwnsLock();
700 nsresult
InitInternal(nsIFile
*aCacheDirectory
);
701 void PreShutdownInternal();
703 // This method returns false when index is not initialized or is shut down.
704 bool IsIndexUsable();
706 // This method checks whether the entry has the same values of appId,
707 // isAnonymous and isInBrowser. We don't expect to find a collision since
708 // these values are part of the key that we hash and we use a strong hash
710 static bool IsCollision(CacheIndexEntry
*aEntry
,
715 // Checks whether any of the information about the entry has changed.
716 static bool HasEntryChanged(CacheIndexEntry
*aEntry
,
717 const uint32_t *aFrecency
,
718 const uint32_t *aExpirationTime
,
719 const uint32_t *aSize
);
721 // Merge all pending operations from mPendingUpdates into mIndex.
722 void ProcessPendingOperations();
723 static PLDHashOperator
UpdateEntryInIndex(CacheIndexEntryUpdate
*aEntry
,
726 // Following methods perform writing of the index file.
728 // The index is written periodically, but not earlier than once in
729 // kMinDumpInterval and there must be at least kMinUnwrittenChanges
730 // differences between index on disk and in memory. Index is always first
731 // written to a temporary file and the old index file is replaced when the
732 // writing process succeeds.
734 // Starts writing of index when both limits (minimal delay between writes and
735 // minimum number of changes in index) were exceeded.
736 bool WriteIndexToDiskIfNeeded();
737 // Starts writing of index file.
738 void WriteIndexToDisk();
739 // Serializes part of mIndex hashtable to the write buffer a writes the buffer
742 // Finalizes writing process.
743 void FinishWrite(bool aSucceeded
);
745 static PLDHashOperator
CopyRecordsToRWBuf(CacheIndexEntry
*aEntry
,
747 static PLDHashOperator
ApplyIndexChanges(CacheIndexEntry
*aEntry
,
750 // Following methods perform writing of the journal during shutdown. All these
751 // methods must be called only during shutdown since they write/delete files
752 // directly on the main thread instead of using CacheFileIOManager that does
753 // it asynchronously on IO thread. Journal contains only entries that are
754 // dirty, i.e. changes that are not present in the index file on the disk.
755 // When the log is written successfully, the dirty flag in index file is
757 nsresult
GetFile(const nsACString
&aName
, nsIFile
**_retval
);
758 nsresult
RemoveFile(const nsACString
&aName
);
759 void RemoveIndexFromDisk();
760 // Writes journal to the disk and clears dirty flag in index header.
761 nsresult
WriteLogToDisk();
763 static PLDHashOperator
WriteEntryToLog(CacheIndexEntry
*aEntry
,
766 // Following methods perform reading of the index from the disk.
768 // Index is read at startup just after initializing the CacheIndex. There are
769 // 3 files used when manipulating with index: index file, journal file and
770 // a temporary file. All files contain the hash of the data, so we can check
771 // whether the content is valid and complete. Index file contains also a dirty
772 // flag in the index header which is unset on a clean shutdown. During opening
773 // and reading of the files we determine the status of the whole index from
774 // the states of the separate files. Following table shows all possible
777 // index, journal, tmpfile
778 // M * * - index is missing -> BUILD
779 // I * * - index is invalid -> BUILD
780 // D * * - index is dirty -> UPDATE
781 // C M * - index is dirty -> UPDATE
782 // C I * - unexpected state -> UPDATE
783 // C V E - unexpected state -> UPDATE
784 // C V M - index is up to date -> READY
786 // where the letters mean:
789 // M - file is missing
790 // I - data is invalid (parsing failed or hash didn't match)
791 // D - dirty (data in index file is correct, but dirty flag is set)
792 // C - clean (index file is clean)
793 // V - valid (data in journal file is correct)
795 // Note: We accept the data from journal only when the index is up to date as
796 // a whole (i.e. C,V,M state).
798 // We rename the journal file to the temporary file as soon as possible after
799 // initial test to ensure that we start update process on the next startup if
800 // FF crashes during parsing of the index.
802 // Initiates reading index from disk.
803 void ReadIndexFromDisk();
804 // Starts reading data from index file.
805 void StartReadingIndex();
806 // Parses data read from index file.
808 // Starts reading data from journal file.
809 void StartReadingJournal();
810 // Parses data read from journal file.
812 // Merges entries from journal into mIndex.
814 // In debug build this method checks that we have no fresh entry in mIndex
815 // after we finish reading index and before we process pending operations.
816 void EnsureNoFreshEntry();
817 // In debug build this method is called after processing pending operations
818 // to make sure mIndexStats contains correct information.
819 void EnsureCorrectStats();
820 static PLDHashOperator
SumIndexStats(CacheIndexEntry
*aEntry
, void* aClosure
);
821 // Finalizes reading process.
822 void FinishRead(bool aSucceeded
);
824 static PLDHashOperator
ProcessJournalEntry(CacheIndexEntry
*aEntry
,
827 // Following methods perform updating and building of the index.
828 // Timer callback that starts update or build process.
829 static void DelayedUpdate(nsITimer
*aTimer
, void *aClosure
);
830 // Posts timer event that start update or build process.
831 nsresult
ScheduleUpdateTimer(uint32_t aDelay
);
832 nsresult
SetupDirectoryEnumerator();
833 void InitEntryFromDiskData(CacheIndexEntry
*aEntry
,
834 CacheFileMetadata
*aMetaData
,
836 // Returns true when either a timer is scheduled or event is posted.
837 bool IsUpdatePending();
838 // Iterates through all files in entries directory that we didn't create/open
839 // during this session, parses them and adds the entries to the index.
842 bool StartUpdatingIndexIfNeeded(bool aSwitchingToReadyState
= false);
843 // Starts update or build process or fires a timer when it is too early after
845 void StartUpdatingIndex(bool aRebuild
);
846 // Iterates through all files in entries directory that we didn't create/open
847 // during this session and theirs last modified time is newer than timestamp
848 // in the index header. Parses the files and adds the entries to the index.
850 // Finalizes update or build process.
851 void FinishUpdate(bool aSucceeded
);
853 static PLDHashOperator
RemoveNonFreshEntries(CacheIndexEntry
*aEntry
,
857 // Initial state in which the index is not usable
858 // Possible transitions:
862 // Index is being read from the disk.
863 // Possible transitions:
864 // -> INITIAL - We failed to dispatch a read event.
865 // -> BUILDING - No or corrupted index file was found.
866 // -> UPDATING - No or corrupted journal file was found.
867 // - Dirty flag was set in index header.
868 // -> READY - Index was read successfully or was interrupted by
870 // -> SHUTDOWN - This could happen only in case of pre-shutdown failure.
873 // Index is being written to the disk.
874 // Possible transitions:
875 // -> READY - Writing of index finished or was interrupted by
877 // -> UPDATING - Writing of index finished, but index was found outdated
879 // -> SHUTDOWN - This could happen only in case of pre-shutdown failure.
882 // Index is being build.
883 // Possible transitions:
884 // -> READY - Building of index finished or was interrupted by
886 // -> SHUTDOWN - This could happen only in case of pre-shutdown failure.
889 // Index is being updated.
890 // Possible transitions:
891 // -> READY - Updating of index finished or was interrupted by
893 // -> SHUTDOWN - This could happen only in case of pre-shutdown failure.
897 // Possible transitions:
898 // -> UPDATING - Index was found outdated.
899 // -> SHUTDOWN - Index is shutting down.
902 // Index is shutting down.
907 static char const * StateString(EState aState
);
909 void ChangeState(EState aNewState
);
911 // Allocates and releases buffer used for reading and writing index.
913 void ReleaseBuffer();
915 // Methods used by CacheIndexEntryAutoManage to keep the arrays up to date.
916 void InsertRecordToFrecencyArray(CacheIndexRecord
*aRecord
);
917 void InsertRecordToExpirationArray(CacheIndexRecord
*aRecord
);
918 void RemoveRecordFromFrecencyArray(CacheIndexRecord
*aRecord
);
919 void RemoveRecordFromExpirationArray(CacheIndexRecord
*aRecord
);
921 // Methods used by CacheIndexEntryAutoManage to keep the iterators up to date.
922 void AddRecordToIterators(CacheIndexRecord
*aRecord
);
923 void RemoveRecordFromIterators(CacheIndexRecord
*aRecord
);
924 void ReplaceRecordInIterators(CacheIndexRecord
*aOldRecord
,
925 CacheIndexRecord
*aNewRecord
);
927 // Memory reporting (private part)
928 size_t SizeOfExcludingThisInternal(mozilla::MallocSizeOf mallocSizeOf
) const;
930 static CacheIndex
*gInstance
;
932 nsCOMPtr
<nsIFile
> mCacheDirectory
;
934 mozilla::Mutex mLock
;
936 // Timestamp of time when the index was initialized. We use it to delay
937 // initial update or build of index.
938 TimeStamp mStartTime
;
939 // Set to true in PreShutdown(), it is checked on variaous places to prevent
940 // starting any process (write, update, etc.) during shutdown.
942 // When set to true, update process should start as soon as possible. This
943 // flag is set whenever we find some inconsistency which would be fixed by
944 // update process. The flag is checked always when switching to READY state.
945 // To make sure we start the update process as soon as possible, methods that
946 // set this flag should also call StartUpdatingIndexIfNeeded() to cover the
947 // case when we are currently in READY state.
948 bool mIndexNeedsUpdate
;
949 // Set at the beginning of RemoveAll() which clears the whole index. When
950 // removing all entries we must stop any pending reading, writing, updating or
951 // building operation. This flag is checked at various places and it prevents
952 // we won't start another operation (e.g. canceling reading of the index would
953 // normally start update or build process)
955 // Whether the index file on disk exists and is valid.
956 bool mIndexOnDiskIsValid
;
957 // When something goes wrong during updating or building process, we don't
958 // mark index clean (and also don't write journal) to ensure that update or
959 // build will be initiated on the next start.
960 bool mDontMarkIndexClean
;
961 // Timestamp value from index file. It is used during update process to skip
962 // entries that were last modified before this timestamp.
963 uint32_t mIndexTimeStamp
;
964 // Timestamp of last time the index was dumped to disk.
965 // NOTE: The index might not be necessarily dumped at this time. The value
966 // is used to schedule next dump of the index.
967 TimeStamp mLastDumpTime
;
969 // Timer of delayed update/build.
970 nsCOMPtr
<nsITimer
> mUpdateTimer
;
971 // True when build or update event is posted
972 bool mUpdateEventPending
;
974 // Helper members used when reading/writing index from/to disk.
975 // Contains number of entries that should be skipped:
976 // - in hashtable when writing index because they were already written
977 // - in index file when reading index because they were already read
978 uint32_t mSkipEntries
;
979 // Number of entries that should be written to disk. This is number of entries
980 // in hashtable that are initialized and are not marked as removed when writing
982 uint32_t mProcessEntries
;
986 nsRefPtr
<CacheHash
> mRWHash
;
988 // Reading of journal succeeded if true.
989 bool mJournalReadSuccessfully
;
991 // Handle used for writing and reading index file.
992 nsRefPtr
<CacheFileHandle
> mIndexHandle
;
993 // Handle used for reading journal file.
994 nsRefPtr
<CacheFileHandle
> mJournalHandle
;
995 // Used to check the existence of the file during reading process.
996 nsRefPtr
<CacheFileHandle
> mTmpHandle
;
998 nsRefPtr
<FileOpenHelper
> mIndexFileOpener
;
999 nsRefPtr
<FileOpenHelper
> mJournalFileOpener
;
1000 nsRefPtr
<FileOpenHelper
> mTmpFileOpener
;
1002 // Directory enumerator used when building and updating index.
1003 nsCOMPtr
<nsIDirectoryEnumerator
> mDirEnumerator
;
1005 // Main index hashtable.
1006 nsTHashtable
<CacheIndexEntry
> mIndex
;
1008 // We cannot add, remove or change any entry in mIndex in states READING and
1009 // WRITING. We track all changes in mPendingUpdates during these states.
1010 nsTHashtable
<CacheIndexEntryUpdate
> mPendingUpdates
;
1012 // Contains information statistics for mIndex + mPendingUpdates.
1013 CacheIndexStats mIndexStats
;
1015 // When reading journal, we must first parse the whole file and apply the
1016 // changes iff the journal was read successfully. mTmpJournal is used to store
1017 // entries from the journal file. We throw away all these entries if parsing
1018 // of the journal fails or the hash does not match.
1019 nsTHashtable
<CacheIndexEntry
> mTmpJournal
;
1021 // Arrays that keep entry records ordered by eviction preference. When looking
1022 // for an entry to evict, we first try to find an expired entry. If there is
1023 // no expired entry, we take the entry with lowest valid frecency. Zero
1024 // frecency is an initial value and such entries are stored at the end of the
1025 // array. Uninitialized entries and entries marked as deleted are not present
1027 nsTArray
<CacheIndexRecord
*> mFrecencyArray
;
1028 nsTArray
<CacheIndexRecord
*> mExpirationArray
;
1030 nsTArray
<CacheIndexIterator
*> mIterators
;
1032 class DiskConsumptionObserver
: public nsRunnable
1035 static DiskConsumptionObserver
* Init(nsICacheStorageConsumptionObserver
* aObserver
)
1037 nsWeakPtr observer
= do_GetWeakReference(aObserver
);
1041 return new DiskConsumptionObserver(observer
);
1044 void OnDiskConsumption(int64_t aSize
)
1047 NS_DispatchToMainThread(this);
1051 explicit DiskConsumptionObserver(nsWeakPtr
const &aWeakObserver
)
1052 : mObserver(aWeakObserver
) { }
1053 virtual ~DiskConsumptionObserver() { }
1057 MOZ_ASSERT(NS_IsMainThread());
1059 nsCOMPtr
<nsICacheStorageConsumptionObserver
> observer
=
1060 do_QueryReferent(mObserver
);
1063 observer
->OnNetworkCacheDiskConsumption(mSize
);
1069 nsWeakPtr mObserver
;
1073 // List of async observers that want to get disk consumption information
1074 nsTArray
<nsRefPtr
<DiskConsumptionObserver
> > mDiskConsumptionObservers
;
1077 class CacheIndexAutoLock
{
1079 explicit CacheIndexAutoLock(CacheIndex
*aIndex
)
1085 ~CacheIndexAutoLock()
1093 MOZ_ASSERT(!mLocked
);
1099 MOZ_ASSERT(mLocked
);
1105 nsRefPtr
<CacheIndex
> mIndex
;
1109 class CacheIndexAutoUnlock
{
1111 explicit CacheIndexAutoUnlock(CacheIndex
*aIndex
)
1117 ~CacheIndexAutoUnlock()
1125 MOZ_ASSERT(!mLocked
);
1131 MOZ_ASSERT(mLocked
);
1137 nsRefPtr
<CacheIndex
> mIndex
;