netwerk/cache2/CacheIndex.h

   1 /* This Source Code Form is subject to the terms of the Mozilla Public
   2  * License, v. 2.0. If a copy of the MPL was not distributed with this
   3  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   4
   5 #ifndef CacheIndex__h__
   6 #define CacheIndex__h__
   7
   8 #include "CacheLog.h"
   9 #include "CacheFileIOManager.h"
  10 #include "nsIRunnable.h"
  11 #include "CacheHashUtils.h"
  12 #include "nsICacheStorageService.h"
  13 #include "nsICacheEntry.h"
  14 #include "nsILoadContextInfo.h"
  15 #include "nsIWeakReferenceUtils.h"
  16 #include "nsTHashtable.h"
  17 #include "nsThreadUtils.h"
  18 #include "mozilla/IntegerPrintfMacros.h"
  19 #include "mozilla/SHA1.h"
  20 #include "mozilla/StaticMutex.h"
  21 #include "mozilla/StaticPtr.h"
  22 #include "mozilla/EndianUtils.h"
  23 #include "mozilla/TimeStamp.h"
  24 #include "mozilla/UniquePtr.h"
  25
  26 class nsIFile;
  27 class nsIDirectoryEnumerator;
  28 class nsITimer;
  29
  30 #ifdef DEBUG
  31 #  define DEBUG_STATS 1
  32 #endif
  33
  34 namespace mozilla {
  35 namespace net {
  36
  37 class CacheFileMetadata;
  38 class FileOpenHelper;
  39 class CacheIndexIterator;
  40
  41 const uint16_t kIndexTimeNotAvailable = 0xFFFFU;
  42 const uint16_t kIndexTimeOutOfBound = 0xFFFEU;
  43
  44 using CacheIndexHeader = struct {
  45   // Version of the index. The index must be ignored and deleted when the file
  46   // on disk was written with a newer version.
  47   uint32_t mVersion;
  48
  49   // Timestamp of time when the last successful write of the index started.
  50   // During update process we use this timestamp for a quick validation of entry
  51   // files. If last modified time of the file is lower than this timestamp, we
  52   // skip parsing of such file since the information in index should be up to
  53   // date.
  54   uint32_t mTimeStamp;
  55
  56   // We set this flag as soon as possible after parsing index during startup
  57   // and clean it after we write journal to disk during shutdown. We ignore the
  58   // journal and start update process whenever this flag is set during index
  59   // parsing.
  60   uint32_t mIsDirty;
  61
  62   // The amount of data written to the cache. When it reaches
  63   // kTelemetryReportBytesLimit a telemetry report is sent and the counter is
  64   // reset.
  65   uint32_t mKBWritten;
  66 };
  67
  68 static_assert(sizeof(CacheIndexHeader::mVersion) +
  69                       sizeof(CacheIndexHeader::mTimeStamp) +
  70                       sizeof(CacheIndexHeader::mIsDirty) +
  71                       sizeof(CacheIndexHeader::mKBWritten) ==
  72                   sizeof(CacheIndexHeader),
  73               "Unexpected sizeof(CacheIndexHeader)!");
  74
  75 #pragma pack(push, 1)
  76 struct CacheIndexRecord {
  77   SHA1Sum::Hash mHash{};
  78   uint32_t mFrecency{0};
  79   OriginAttrsHash mOriginAttrsHash{0};
  80   uint16_t mOnStartTime{kIndexTimeNotAvailable};
  81   uint16_t mOnStopTime{kIndexTimeNotAvailable};
  82   uint8_t mContentType{nsICacheEntry::CONTENT_TYPE_UNKNOWN};
  83
  84   /*
  85    *    1000 0000 0000 0000 0000 0000 0000 0000 : initialized
  86    *    0100 0000 0000 0000 0000 0000 0000 0000 : anonymous
  87    *    0010 0000 0000 0000 0000 0000 0000 0000 : removed
  88    *    0001 0000 0000 0000 0000 0000 0000 0000 : dirty
  89    *    0000 1000 0000 0000 0000 0000 0000 0000 : fresh
  90    *    0000 0100 0000 0000 0000 0000 0000 0000 : pinned
  91    *    0000 0010 0000 0000 0000 0000 0000 0000 : has cached alt data
  92    *    0000 0001 0000 0000 0000 0000 0000 0000 : reserved
  93    *    0000 0000 1111 1111 1111 1111 1111 1111 : file size (in kB)
  94    */
  95   uint32_t mFlags{0};
  96
  97   CacheIndexRecord() = default;
  98 };
  99 #pragma pack(pop)
 100
 101 static_assert(sizeof(CacheIndexRecord::mHash) +
 102                       sizeof(CacheIndexRecord::mFrecency) +
 103                       sizeof(CacheIndexRecord::mOriginAttrsHash) +
 104                       sizeof(CacheIndexRecord::mOnStartTime) +
 105                       sizeof(CacheIndexRecord::mOnStopTime) +
 106                       sizeof(CacheIndexRecord::mContentType) +
 107                       sizeof(CacheIndexRecord::mFlags) ==
 108                   sizeof(CacheIndexRecord),
 109               "Unexpected sizeof(CacheIndexRecord)!");
 110
 111 class CacheIndexRecordWrapper final {
 112  public:
 113   NS_INLINE_DECL_THREADSAFE_REFCOUNTING_WITH_DESTROY(
 114       CacheIndexRecordWrapper, DispatchDeleteSelfToCurrentThread());
 115
 116   CacheIndexRecordWrapper() : mRec(MakeUnique<CacheIndexRecord>()) {}
 117   CacheIndexRecord* Get() { return mRec.get(); }
 118
 119  private:
 120   ~CacheIndexRecordWrapper();
 121   void DispatchDeleteSelfToCurrentThread();
 122   UniquePtr<CacheIndexRecord> mRec;
 123   friend class DeleteCacheIndexRecordWrapper;
 124 };
 125
 126 class CacheIndexEntry : public PLDHashEntryHdr {
 127  public:
 128   using KeyType = const SHA1Sum::Hash&;
 129   using KeyTypePointer = const SHA1Sum::Hash*;
 130
 131   explicit CacheIndexEntry(KeyTypePointer aKey) {
 132     MOZ_COUNT_CTOR(CacheIndexEntry);
 133     mRec = new CacheIndexRecordWrapper();
 134     LOG(("CacheIndexEntry::CacheIndexEntry() - Created record [rec=%p]",
 135          mRec->Get()));
 136     memcpy(&mRec->Get()->mHash, aKey, sizeof(SHA1Sum::Hash));
 137   }
 138   CacheIndexEntry(const CacheIndexEntry& aOther) {
 139     MOZ_ASSERT_UNREACHABLE("CacheIndexEntry copy constructor is forbidden!");
 140   }
 141   ~CacheIndexEntry() {
 142     MOZ_COUNT_DTOR(CacheIndexEntry);
 143     LOG(("CacheIndexEntry::~CacheIndexEntry() - Deleting record [rec=%p]",
 144          mRec->Get()));
 145   }
 146
 147   // KeyEquals(): does this entry match this key?
 148   bool KeyEquals(KeyTypePointer aKey) const {
 149     return memcmp(&mRec->Get()->mHash, aKey, sizeof(SHA1Sum::Hash)) == 0;
 150   }
 151
 152   // KeyToPointer(): Convert KeyType to KeyTypePointer
 153   static KeyTypePointer KeyToPointer(KeyType aKey) { return &aKey; }
 154
 155   // HashKey(): calculate the hash number
 156   static PLDHashNumber HashKey(KeyTypePointer aKey) {
 157     return (reinterpret_cast<const uint32_t*>(aKey))[0];
 158   }
 159
 160   // ALLOW_MEMMOVE can we move this class with memmove(), or do we have
 161   // to use the copy constructor?
 162   enum { ALLOW_MEMMOVE = true };
 163
 164   bool operator==(const CacheIndexEntry& aOther) const {
 165     return KeyEquals(&aOther.mRec->Get()->mHash);
 166   }
 167
 168   CacheIndexEntry& operator=(const CacheIndexEntry& aOther) {
 169     MOZ_ASSERT(memcmp(&mRec->Get()->mHash, &aOther.mRec->Get()->mHash,
 170                       sizeof(SHA1Sum::Hash)) == 0);
 171     mRec->Get()->mFrecency = aOther.mRec->Get()->mFrecency;
 172     mRec->Get()->mOriginAttrsHash = aOther.mRec->Get()->mOriginAttrsHash;
 173     mRec->Get()->mOnStartTime = aOther.mRec->Get()->mOnStartTime;
 174     mRec->Get()->mOnStopTime = aOther.mRec->Get()->mOnStopTime;
 175     mRec->Get()->mContentType = aOther.mRec->Get()->mContentType;
 176     mRec->Get()->mFlags = aOther.mRec->Get()->mFlags;
 177     return *this;
 178   }
 179
 180   void InitNew() {
 181     mRec->Get()->mFrecency = 0;
 182     mRec->Get()->mOriginAttrsHash = 0;
 183     mRec->Get()->mOnStartTime = kIndexTimeNotAvailable;
 184     mRec->Get()->mOnStopTime = kIndexTimeNotAvailable;
 185     mRec->Get()->mContentType = nsICacheEntry::CONTENT_TYPE_UNKNOWN;
 186     mRec->Get()->mFlags = 0;
 187   }
 188
 189   void Init(OriginAttrsHash aOriginAttrsHash, bool aAnonymous, bool aPinned) {
 190     MOZ_ASSERT(mRec->Get()->mFrecency == 0);
 191     MOZ_ASSERT(mRec->Get()->mOriginAttrsHash == 0);
 192     MOZ_ASSERT(mRec->Get()->mOnStartTime == kIndexTimeNotAvailable);
 193     MOZ_ASSERT(mRec->Get()->mOnStopTime == kIndexTimeNotAvailable);
 194     MOZ_ASSERT(mRec->Get()->mContentType ==
 195                nsICacheEntry::CONTENT_TYPE_UNKNOWN);
 196     // When we init the entry it must be fresh and may be dirty
 197     MOZ_ASSERT((mRec->Get()->mFlags & ~kDirtyMask) == kFreshMask);
 198
 199     mRec->Get()->mOriginAttrsHash = aOriginAttrsHash;
 200     mRec->Get()->mFlags |= kInitializedMask;
 201     if (aAnonymous) {
 202       mRec->Get()->mFlags |= kAnonymousMask;
 203     }
 204     if (aPinned) {
 205       mRec->Get()->mFlags |= kPinnedMask;
 206     }
 207   }
 208
 209   const SHA1Sum::Hash* Hash() const { return &mRec->Get()->mHash; }
 210
 211   bool IsInitialized() const {
 212     return !!(mRec->Get()->mFlags & kInitializedMask);
 213   }
 214
 215   mozilla::net::OriginAttrsHash OriginAttrsHash() const {
 216     return mRec->Get()->mOriginAttrsHash;
 217   }
 218
 219   bool Anonymous() const { return !!(mRec->Get()->mFlags & kAnonymousMask); }
 220
 221   bool IsRemoved() const { return !!(mRec->Get()->mFlags & kRemovedMask); }
 222   void MarkRemoved() { mRec->Get()->mFlags |= kRemovedMask; }
 223
 224   bool IsDirty() const { return !!(mRec->Get()->mFlags & kDirtyMask); }
 225   void MarkDirty() { mRec->Get()->mFlags |= kDirtyMask; }
 226   void ClearDirty() { mRec->Get()->mFlags &= ~kDirtyMask; }
 227
 228   bool IsFresh() const { return !!(mRec->Get()->mFlags & kFreshMask); }
 229   void MarkFresh() { mRec->Get()->mFlags |= kFreshMask; }
 230
 231   bool IsPinned() const { return !!(mRec->Get()->mFlags & kPinnedMask); }
 232
 233   void SetFrecency(uint32_t aFrecency) { mRec->Get()->mFrecency = aFrecency; }
 234   uint32_t GetFrecency() const { return mRec->Get()->mFrecency; }
 235
 236   void SetHasAltData(bool aHasAltData) {
 237     aHasAltData ? mRec->Get()->mFlags |= kHasAltDataMask
 238                 : mRec->Get()->mFlags &= ~kHasAltDataMask;
 239   }
 240   bool GetHasAltData() const {
 241     return !!(mRec->Get()->mFlags & kHasAltDataMask);
 242   }
 243
 244   void SetOnStartTime(uint16_t aTime) { mRec->Get()->mOnStartTime = aTime; }
 245   uint16_t GetOnStartTime() const { return mRec->Get()->mOnStartTime; }
 246
 247   void SetOnStopTime(uint16_t aTime) { mRec->Get()->mOnStopTime = aTime; }
 248   uint16_t GetOnStopTime() const { return mRec->Get()->mOnStopTime; }
 249
 250   void SetContentType(uint8_t aType) { mRec->Get()->mContentType = aType; }
 251   uint8_t GetContentType() const { return GetContentType(mRec->Get()); }
 252   static uint8_t GetContentType(CacheIndexRecord* aRec) {
 253     if (aRec->mContentType >= nsICacheEntry::CONTENT_TYPE_LAST) {
 254       LOG(
 255           ("CacheIndexEntry::GetContentType() - Found invalid content type "
 256            "[hash=%08x%08x%08x%08x%08x, contentType=%u]",
 257            LOGSHA1(aRec->mHash), aRec->mContentType));
 258       return nsICacheEntry::CONTENT_TYPE_UNKNOWN;
 259     }
 260     return aRec->mContentType;
 261   }
 262
 263   // Sets filesize in kilobytes.
 264   void SetFileSize(uint32_t aFileSize) {
 265     if (aFileSize > kFileSizeMask) {
 266       LOG(
 267           ("CacheIndexEntry::SetFileSize() - FileSize is too large, "
 268            "truncating to %u",
 269            kFileSizeMask));
 270       aFileSize = kFileSizeMask;
 271     }
 272     mRec->Get()->mFlags &= ~kFileSizeMask;
 273     mRec->Get()->mFlags |= aFileSize;
 274   }
 275   // Returns filesize in kilobytes.
 276   uint32_t GetFileSize() const { return GetFileSize(*(mRec->Get())); }
 277   static uint32_t GetFileSize(const CacheIndexRecord& aRec) {
 278     return aRec.mFlags & kFileSizeMask;
 279   }
 280   static uint32_t IsPinned(CacheIndexRecord* aRec) {
 281     return aRec->mFlags & kPinnedMask;
 282   }
 283   bool IsFileEmpty() const { return GetFileSize() == 0; }
 284
 285   void WriteToBuf(void* aBuf) {
 286     uint8_t* ptr = static_cast<uint8_t*>(aBuf);
 287     memcpy(ptr, mRec->Get()->mHash, sizeof(SHA1Sum::Hash));
 288     ptr += sizeof(SHA1Sum::Hash);
 289     NetworkEndian::writeUint32(ptr, mRec->Get()->mFrecency);
 290     ptr += sizeof(uint32_t);
 291     NetworkEndian::writeUint64(ptr, mRec->Get()->mOriginAttrsHash);
 292     ptr += sizeof(uint64_t);
 293     NetworkEndian::writeUint16(ptr, mRec->Get()->mOnStartTime);
 294     ptr += sizeof(uint16_t);
 295     NetworkEndian::writeUint16(ptr, mRec->Get()->mOnStopTime);
 296     ptr += sizeof(uint16_t);
 297     *ptr = mRec->Get()->mContentType;
 298     ptr += sizeof(uint8_t);
 299     // Dirty and fresh flags should never go to disk, since they make sense only
 300     // during current session.
 301     NetworkEndian::writeUint32(
 302         ptr, mRec->Get()->mFlags & ~(kDirtyMask | kFreshMask));
 303   }
 304
 305   void ReadFromBuf(void* aBuf) {
 306     const uint8_t* ptr = static_cast<const uint8_t*>(aBuf);
 307     MOZ_ASSERT(memcmp(&mRec->Get()->mHash, ptr, sizeof(SHA1Sum::Hash)) == 0);
 308     ptr += sizeof(SHA1Sum::Hash);
 309     mRec->Get()->mFrecency = NetworkEndian::readUint32(ptr);
 310     ptr += sizeof(uint32_t);
 311     mRec->Get()->mOriginAttrsHash = NetworkEndian::readUint64(ptr);
 312     ptr += sizeof(uint64_t);
 313     mRec->Get()->mOnStartTime = NetworkEndian::readUint16(ptr);
 314     ptr += sizeof(uint16_t);
 315     mRec->Get()->mOnStopTime = NetworkEndian::readUint16(ptr);
 316     ptr += sizeof(uint16_t);
 317     mRec->Get()->mContentType = *ptr;
 318     ptr += sizeof(uint8_t);
 319     mRec->Get()->mFlags = NetworkEndian::readUint32(ptr);
 320   }
 321
 322   void Log() const {
 323     LOG(
 324         ("CacheIndexEntry::Log() [this=%p, hash=%08x%08x%08x%08x%08x, fresh=%u,"
 325          " initialized=%u, removed=%u, dirty=%u, anonymous=%u, "
 326          "originAttrsHash=%" PRIx64 ", frecency=%u, hasAltData=%u, "
 327          "onStartTime=%u, onStopTime=%u, contentType=%u, size=%u]",
 328          this, LOGSHA1(mRec->Get()->mHash), IsFresh(), IsInitialized(),
 329          IsRemoved(), IsDirty(), Anonymous(), OriginAttrsHash(), GetFrecency(),
 330          GetHasAltData(), GetOnStartTime(), GetOnStopTime(), GetContentType(),
 331          GetFileSize()));
 332   }
 333
 334   static bool RecordMatchesLoadContextInfo(CacheIndexRecordWrapper* aRec,
 335                                            nsILoadContextInfo* aInfo) {
 336     MOZ_ASSERT(aInfo);
 337
 338     return !aInfo->IsPrivate() &&
 339            GetOriginAttrsHash(*aInfo->OriginAttributesPtr()) ==
 340                aRec->Get()->mOriginAttrsHash &&
 341            aInfo->IsAnonymous() == !!(aRec->Get()->mFlags & kAnonymousMask);
 342   }
 343
 344   // Memory reporting
 345   size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const {
 346     return mallocSizeOf(mRec->Get());
 347   }
 348
 349   size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const {
 350     return mallocSizeOf(this) + SizeOfExcludingThis(mallocSizeOf);
 351   }
 352
 353  private:
 354   friend class CacheIndexEntryUpdate;
 355   friend class CacheIndex;
 356   friend class CacheIndexEntryAutoManage;
 357   friend struct CacheIndexRecord;
 358
 359   static const uint32_t kInitializedMask = 0x80000000;
 360   static const uint32_t kAnonymousMask = 0x40000000;
 361
 362   // This flag is set when the entry was removed. We need to keep this
 363   // information in memory until we write the index file.
 364   static const uint32_t kRemovedMask = 0x20000000;
 365
 366   // This flag is set when the information in memory is not in sync with the
 367   // information in index file on disk.
 368   static const uint32_t kDirtyMask = 0x10000000;
 369
 370   // This flag is set when the information about the entry is fresh, i.e.
 371   // we've created or opened this entry during this session, or we've seen
 372   // this entry during update or build process.
 373   static const uint32_t kFreshMask = 0x08000000;
 374
 375   // Indicates a pinned entry.
 376   static const uint32_t kPinnedMask = 0x04000000;
 377
 378   // Indicates there is cached alternative data in the entry.
 379   static const uint32_t kHasAltDataMask = 0x02000000;
 380   static const uint32_t kReservedMask = 0x01000000;
 381
 382   // FileSize in kilobytes
 383   static const uint32_t kFileSizeMask = 0x00FFFFFF;
 384
 385   RefPtr<CacheIndexRecordWrapper> mRec;
 386 };
 387
 388 class CacheIndexEntryUpdate : public CacheIndexEntry {
 389  public:
 390   explicit CacheIndexEntryUpdate(CacheIndexEntry::KeyTypePointer aKey)
 391       : CacheIndexEntry(aKey), mUpdateFlags(0) {
 392     MOZ_COUNT_CTOR(CacheIndexEntryUpdate);
 393     LOG(("CacheIndexEntryUpdate::CacheIndexEntryUpdate()"));
 394   }
 395   ~CacheIndexEntryUpdate() {
 396     MOZ_COUNT_DTOR(CacheIndexEntryUpdate);
 397     LOG(("CacheIndexEntryUpdate::~CacheIndexEntryUpdate()"));
 398   }
 399
 400   CacheIndexEntryUpdate& operator=(const CacheIndexEntry& aOther) {
 401     MOZ_ASSERT(memcmp(&mRec->Get()->mHash, &aOther.mRec->Get()->mHash,
 402                       sizeof(SHA1Sum::Hash)) == 0);
 403     mUpdateFlags = 0;
 404     *(static_cast<CacheIndexEntry*>(this)) = aOther;
 405     return *this;
 406   }
 407
 408   void InitNew() {
 409     mUpdateFlags = kFrecencyUpdatedMask | kHasAltDataUpdatedMask |
 410                    kOnStartTimeUpdatedMask | kOnStopTimeUpdatedMask |
 411                    kContentTypeUpdatedMask | kFileSizeUpdatedMask;
 412     CacheIndexEntry::InitNew();
 413   }
 414
 415   void SetFrecency(uint32_t aFrecency) {
 416     mUpdateFlags |= kFrecencyUpdatedMask;
 417     CacheIndexEntry::SetFrecency(aFrecency);
 418   }
 419
 420   void SetHasAltData(bool aHasAltData) {
 421     mUpdateFlags |= kHasAltDataUpdatedMask;
 422     CacheIndexEntry::SetHasAltData(aHasAltData);
 423   }
 424
 425   void SetOnStartTime(uint16_t aTime) {
 426     mUpdateFlags |= kOnStartTimeUpdatedMask;
 427     CacheIndexEntry::SetOnStartTime(aTime);
 428   }
 429
 430   void SetOnStopTime(uint16_t aTime) {
 431     mUpdateFlags |= kOnStopTimeUpdatedMask;
 432     CacheIndexEntry::SetOnStopTime(aTime);
 433   }
 434
 435   void SetContentType(uint8_t aType) {
 436     mUpdateFlags |= kContentTypeUpdatedMask;
 437     CacheIndexEntry::SetContentType(aType);
 438   }
 439
 440   void SetFileSize(uint32_t aFileSize) {
 441     mUpdateFlags |= kFileSizeUpdatedMask;
 442     CacheIndexEntry::SetFileSize(aFileSize);
 443   }
 444
 445   void ApplyUpdate(CacheIndexEntry* aDst) {
 446     MOZ_ASSERT(memcmp(&mRec->Get()->mHash, &aDst->mRec->Get()->mHash,
 447                       sizeof(SHA1Sum::Hash)) == 0);
 448     if (mUpdateFlags & kFrecencyUpdatedMask) {
 449       aDst->mRec->Get()->mFrecency = mRec->Get()->mFrecency;
 450     }
 451     aDst->mRec->Get()->mOriginAttrsHash = mRec->Get()->mOriginAttrsHash;
 452     if (mUpdateFlags & kOnStartTimeUpdatedMask) {
 453       aDst->mRec->Get()->mOnStartTime = mRec->Get()->mOnStartTime;
 454     }
 455     if (mUpdateFlags & kOnStopTimeUpdatedMask) {
 456       aDst->mRec->Get()->mOnStopTime = mRec->Get()->mOnStopTime;
 457     }
 458     if (mUpdateFlags & kContentTypeUpdatedMask) {
 459       aDst->mRec->Get()->mContentType = mRec->Get()->mContentType;
 460     }
 461     if (mUpdateFlags & kHasAltDataUpdatedMask &&
 462         ((aDst->mRec->Get()->mFlags ^ mRec->Get()->mFlags) & kHasAltDataMask)) {
 463       // Toggle the bit if we need to.
 464       aDst->mRec->Get()->mFlags ^= kHasAltDataMask;
 465     }
 466
 467     if (mUpdateFlags & kFileSizeUpdatedMask) {
 468       // Copy all flags except |HasAltData|.
 469       aDst->mRec->Get()->mFlags |= (mRec->Get()->mFlags & ~kHasAltDataMask);
 470     } else {
 471       // Copy all flags except |HasAltData| and file size.
 472       aDst->mRec->Get()->mFlags &= kFileSizeMask;
 473       aDst->mRec->Get()->mFlags |=
 474           (mRec->Get()->mFlags & ~kHasAltDataMask & ~kFileSizeMask);
 475     }
 476   }
 477
 478  private:
 479   static const uint32_t kFrecencyUpdatedMask = 0x00000001;
 480   static const uint32_t kContentTypeUpdatedMask = 0x00000002;
 481   static const uint32_t kFileSizeUpdatedMask = 0x00000004;
 482   static const uint32_t kHasAltDataUpdatedMask = 0x00000008;
 483   static const uint32_t kOnStartTimeUpdatedMask = 0x00000010;
 484   static const uint32_t kOnStopTimeUpdatedMask = 0x00000020;
 485
 486   uint32_t mUpdateFlags;
 487 };
 488
 489 class CacheIndexStats {
 490  public:
 491   CacheIndexStats() {
 492     for (uint32_t i = 0; i < nsICacheEntry::CONTENT_TYPE_LAST; ++i) {
 493       mCountByType[i] = 0;
 494       mSizeByType[i] = 0;
 495     }
 496   }
 497
 498   bool operator==(const CacheIndexStats& aOther) const {
 499     for (uint32_t i = 0; i < nsICacheEntry::CONTENT_TYPE_LAST; ++i) {
 500       if (mCountByType[i] != aOther.mCountByType[i] ||
 501           mSizeByType[i] != aOther.mSizeByType[i]) {
 502         return false;
 503       }
 504     }
 505
 506     return
 507 #ifdef DEBUG
 508         aOther.mStateLogged == mStateLogged &&
 509 #endif
 510         aOther.mCount == mCount && aOther.mNotInitialized == mNotInitialized &&
 511         aOther.mRemoved == mRemoved && aOther.mDirty == mDirty &&
 512         aOther.mFresh == mFresh && aOther.mEmpty == mEmpty &&
 513         aOther.mSize == mSize;
 514   }
 515
 516 #ifdef DEBUG
 517   void DisableLogging() { mDisableLogging = true; }
 518 #endif
 519
 520   void Log() {
 521     LOG(
 522         ("CacheIndexStats::Log() [count=%u, notInitialized=%u, removed=%u, "
 523          "dirty=%u, fresh=%u, empty=%u, size=%u]",
 524          mCount, mNotInitialized, mRemoved, mDirty, mFresh, mEmpty, mSize));
 525   }
 526
 527   void Clear() {
 528     MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Clear() - state logged!");
 529
 530     mCount = 0;
 531     mNotInitialized = 0;
 532     mRemoved = 0;
 533     mDirty = 0;
 534     mFresh = 0;
 535     mEmpty = 0;
 536     mSize = 0;
 537     for (uint32_t i = 0; i < nsICacheEntry::CONTENT_TYPE_LAST; ++i) {
 538       mCountByType[i] = 0;
 539       mSizeByType[i] = 0;
 540     }
 541   }
 542
 543 #ifdef DEBUG
 544   bool StateLogged() { return mStateLogged; }
 545 #endif
 546
 547   uint32_t Count() {
 548     MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Count() - state logged!");
 549     return mCount;
 550   }
 551
 552   uint32_t CountByType(uint8_t aContentType) {
 553     MOZ_ASSERT(!mStateLogged, "CacheIndexStats::CountByType() - state logged!");
 554     MOZ_RELEASE_ASSERT(aContentType < nsICacheEntry::CONTENT_TYPE_LAST);
 555     return mCountByType[aContentType];
 556   }
 557
 558   uint32_t Dirty() {
 559     MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Dirty() - state logged!");
 560     return mDirty;
 561   }
 562
 563   uint32_t Fresh() {
 564     MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Fresh() - state logged!");
 565     return mFresh;
 566   }
 567
 568   uint32_t ActiveEntriesCount() {
 569     MOZ_ASSERT(!mStateLogged,
 570                "CacheIndexStats::ActiveEntriesCount() - state "
 571                "logged!");
 572     return mCount - mRemoved - mNotInitialized - mEmpty;
 573   }
 574
 575   uint32_t Size() {
 576     MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Size() - state logged!");
 577     return mSize;
 578   }
 579
 580   uint32_t SizeByType(uint8_t aContentType) {
 581     MOZ_ASSERT(!mStateLogged, "CacheIndexStats::SizeByType() - state logged!");
 582     MOZ_RELEASE_ASSERT(aContentType < nsICacheEntry::CONTENT_TYPE_LAST);
 583     return mSizeByType[aContentType];
 584   }
 585
 586   void BeforeChange(const CacheIndexEntry* aEntry) {
 587 #ifdef DEBUG_STATS
 588     if (!mDisableLogging) {
 589       LOG(("CacheIndexStats::BeforeChange()"));
 590       Log();
 591     }
 592 #endif
 593
 594     MOZ_ASSERT(!mStateLogged,
 595                "CacheIndexStats::BeforeChange() - state "
 596                "logged!");
 597 #ifdef DEBUG
 598     mStateLogged = true;
 599 #endif
 600     if (aEntry) {
 601       MOZ_ASSERT(mCount);
 602       uint8_t contentType = aEntry->GetContentType();
 603       mCount--;
 604       mCountByType[contentType]--;
 605       if (aEntry->IsDirty()) {
 606         MOZ_ASSERT(mDirty);
 607         mDirty--;
 608       }
 609       if (aEntry->IsFresh()) {
 610         MOZ_ASSERT(mFresh);
 611         mFresh--;
 612       }
 613       if (aEntry->IsRemoved()) {
 614         MOZ_ASSERT(mRemoved);
 615         mRemoved--;
 616       } else {
 617         if (!aEntry->IsInitialized()) {
 618           MOZ_ASSERT(mNotInitialized);
 619           mNotInitialized--;
 620         } else {
 621           if (aEntry->IsFileEmpty()) {
 622             MOZ_ASSERT(mEmpty);
 623             mEmpty--;
 624           } else {
 625             MOZ_ASSERT(mSize >= aEntry->GetFileSize());
 626             mSize -= aEntry->GetFileSize();
 627             mSizeByType[contentType] -= aEntry->GetFileSize();
 628           }
 629         }
 630       }
 631     }
 632   }
 633
 634   void AfterChange(const CacheIndexEntry* aEntry) {
 635     MOZ_ASSERT(mStateLogged,
 636                "CacheIndexStats::AfterChange() - state not "
 637                "logged!");
 638 #ifdef DEBUG
 639     mStateLogged = false;
 640 #endif
 641     if (aEntry) {
 642       uint8_t contentType = aEntry->GetContentType();
 643       ++mCount;
 644       ++mCountByType[contentType];
 645       if (aEntry->IsDirty()) {
 646         mDirty++;
 647       }
 648       if (aEntry->IsFresh()) {
 649         mFresh++;
 650       }
 651       if (aEntry->IsRemoved()) {
 652         mRemoved++;
 653       } else {
 654         if (!aEntry->IsInitialized()) {
 655           mNotInitialized++;
 656         } else {
 657           if (aEntry->IsFileEmpty()) {
 658             mEmpty++;
 659           } else {
 660             mSize += aEntry->GetFileSize();
 661             mSizeByType[contentType] += aEntry->GetFileSize();
 662           }
 663         }
 664       }
 665     }
 666
 667 #ifdef DEBUG_STATS
 668     if (!mDisableLogging) {
 669       LOG(("CacheIndexStats::AfterChange()"));
 670       Log();
 671     }
 672 #endif
 673   }
 674
 675  private:
 676   uint32_t mCount{0};
 677   uint32_t mCountByType[nsICacheEntry::CONTENT_TYPE_LAST]{0};
 678   uint32_t mNotInitialized{0};
 679   uint32_t mRemoved{0};
 680   uint32_t mDirty{0};
 681   uint32_t mFresh{0};
 682   uint32_t mEmpty{0};
 683   uint32_t mSize{0};
 684   uint32_t mSizeByType[nsICacheEntry::CONTENT_TYPE_LAST]{0};
 685 #ifdef DEBUG
 686   // We completely remove the data about an entry from the stats in
 687   // BeforeChange() and set this flag to true. The entry is then modified,
 688   // deleted or created and the data is again put into the stats and this flag
 689   // set to false. Statistics must not be read during this time since the
 690   // information is not correct.
 691   bool mStateLogged{false};
 692
 693   // Disables logging in this instance of CacheIndexStats
 694   bool mDisableLogging{false};
 695 #endif
 696 };
 697
 698 class CacheIndex final : public CacheFileIOListener, public nsIRunnable {
 699  public:
 700   NS_DECL_THREADSAFE_ISUPPORTS
 701   NS_DECL_NSIRUNNABLE
 702
 703   CacheIndex();
 704
 705   static nsresult Init(nsIFile* aCacheDirectory);
 706   static nsresult PreShutdown();
 707   static nsresult Shutdown();
 708
 709   // Following methods can be called only on IO thread.
 710
 711   // Add entry to the index. The entry shouldn't be present in index. This
 712   // method is called whenever a new handle for a new entry file is created. The
 713   // newly created entry is not initialized and it must be either initialized
 714   // with InitEntry() or removed with RemoveEntry().
 715   static nsresult AddEntry(const SHA1Sum::Hash* aHash);
 716
 717   // Inform index about an existing entry that should be present in index. This
 718   // method is called whenever a new handle for an existing entry file is
 719   // created. Like in case of AddEntry(), either InitEntry() or RemoveEntry()
 720   // must be called on the entry, since the entry is not initizlized if the
 721   // index is outdated.
 722   static nsresult EnsureEntryExists(const SHA1Sum::Hash* aHash);
 723
 724   // Initialize the entry. It MUST be present in index. Call to AddEntry() or
 725   // EnsureEntryExists() must precede the call to this method.
 726   static nsresult InitEntry(const SHA1Sum::Hash* aHash,
 727                             OriginAttrsHash aOriginAttrsHash, bool aAnonymous,
 728                             bool aPinned);
 729
 730   // Remove entry from index. The entry should be present in index.
 731   static nsresult RemoveEntry(const SHA1Sum::Hash* aHash);
 732
 733   // Update some information in entry. The entry MUST be present in index and
 734   // MUST be initialized. Call to AddEntry() or EnsureEntryExists() and to
 735   // InitEntry() must precede the call to this method.
 736   // Pass nullptr if the value didn't change.
 737   static nsresult UpdateEntry(const SHA1Sum::Hash* aHash,
 738                               const uint32_t* aFrecency,
 739                               const bool* aHasAltData,
 740                               const uint16_t* aOnStartTime,
 741                               const uint16_t* aOnStopTime,
 742                               const uint8_t* aContentType,
 743                               const uint32_t* aSize);
 744
 745   // Remove all entries from the index. Called when clearing the whole cache.
 746   static nsresult RemoveAll();
 747
 748   enum EntryStatus { EXISTS = 0, DOES_NOT_EXIST = 1, DO_NOT_KNOW = 2 };
 749
 750   // Returns status of the entry in index for the given key. It can be called
 751   // on any thread.
 752   // If the optional aCB callback is given, the it will be called with a
 753   // CacheIndexEntry only if _retval is EXISTS when the method returns.
 754   static nsresult HasEntry(
 755       const nsACString& aKey, EntryStatus* _retval,
 756       const std::function<void(const CacheIndexEntry*)>& aCB = nullptr);
 757   static nsresult HasEntry(
 758       const SHA1Sum::Hash& hash, EntryStatus* _retval,
 759       const std::function<void(const CacheIndexEntry*)>& aCB = nullptr);
 760
 761   // Returns a hash of the least important entry that should be evicted if the
 762   // cache size is over limit and also returns a total number of all entries in
 763   // the index minus the number of forced valid entries and unpinned entries
 764   // that we encounter when searching (see below)
 765   static nsresult GetEntryForEviction(bool aIgnoreEmptyEntries,
 766                                       SHA1Sum::Hash* aHash, uint32_t* aCnt);
 767
 768   // Checks if a cache entry is currently forced valid. Used to prevent an entry
 769   // (that has been forced valid) from being evicted when the cache size reaches
 770   // its limit.
 771   static bool IsForcedValidEntry(const SHA1Sum::Hash* aHash);
 772
 773   // Returns cache size in kB.
 774   static nsresult GetCacheSize(uint32_t* _retval);
 775
 776   // Returns number of entry files in the cache
 777   static nsresult GetEntryFileCount(uint32_t* _retval);
 778
 779   // Synchronously returns the disk occupation and number of entries
 780   // per-context. Callable on any thread. It will ignore loadContextInfo and get
 781   // stats for all entries if the aInfo is a nullptr.
 782   static nsresult GetCacheStats(nsILoadContextInfo* aInfo, uint32_t* aSize,
 783                                 uint32_t* aCount);
 784
 785   // Asynchronously gets the disk cache size, used for display in the UI.
 786   static nsresult AsyncGetDiskConsumption(
 787       nsICacheStorageConsumptionObserver* aObserver);
 788
 789   // Returns an iterator that returns entries matching a given context that were
 790   // present in the index at the time this method was called. If aAddNew is true
 791   // then the iterator will also return entries created after this call.
 792   // NOTE: When some entry is removed from index it is removed also from the
 793   // iterator regardless what aAddNew was passed.
 794   static nsresult GetIterator(nsILoadContextInfo* aInfo, bool aAddNew,
 795                               CacheIndexIterator** _retval);
 796
 797   // Returns true if we _think_ that the index is up to date. I.e. the state is
 798   // READY or WRITING and mIndexNeedsUpdate as well as mShuttingDown is false.
 799   static nsresult IsUpToDate(bool* _retval);
 800
 801   // Called from CacheStorageService::Clear() and
 802   // CacheFileContextEvictor::EvictEntries(), sets a flag that blocks
 803   // notification to AsyncGetDiskConsumption.
 804   static void OnAsyncEviction(bool aEvicting);
 805
 806   // We keep track of total bytes written to the cache to be able to do
 807   // a telemetry report after writting certain amount of data to the cache.
 808   static void UpdateTotalBytesWritten(uint32_t aBytesWritten);
 809
 810   // Memory reporting
 811   static size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf);
 812   static size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf);
 813
 814  private:
 815   friend class CacheIndexEntryAutoManage;
 816   friend class FileOpenHelper;
 817   friend class CacheIndexIterator;
 818   friend class CacheIndexRecordWrapper;
 819   friend class DeleteCacheIndexRecordWrapper;
 820
 821   virtual ~CacheIndex();
 822
 823   NS_IMETHOD OnFileOpened(CacheFileHandle* aHandle, nsresult aResult) override;
 824   void OnFileOpenedInternal(FileOpenHelper* aOpener, CacheFileHandle* aHandle,
 825                             nsresult aResult,
 826                             const StaticMutexAutoLock& aProofOfLock)
 827       MOZ_REQUIRES(sLock);
 828   NS_IMETHOD OnDataWritten(CacheFileHandle* aHandle, const char* aBuf,
 829                            nsresult aResult) override;
 830   NS_IMETHOD OnDataRead(CacheFileHandle* aHandle, char* aBuf,
 831                         nsresult aResult) override;
 832   NS_IMETHOD OnFileDoomed(CacheFileHandle* aHandle, nsresult aResult) override;
 833   NS_IMETHOD OnEOFSet(CacheFileHandle* aHandle, nsresult aResult) override;
 834   NS_IMETHOD OnFileRenamed(CacheFileHandle* aHandle, nsresult aResult) override;
 835
 836   nsresult InitInternal(nsIFile* aCacheDirectory,
 837                         const StaticMutexAutoLock& aProofOfLock);
 838   void PreShutdownInternal();
 839
 840   // This method returns false when index is not initialized or is shut down.
 841   bool IsIndexUsable() MOZ_REQUIRES(sLock);
 842
 843   // This method checks whether the entry has the same values of
 844   // originAttributes and isAnonymous. We don't expect to find a collision
 845   // since these values are part of the key that we hash and we use a strong
 846   // hash function.
 847   static bool IsCollision(CacheIndexEntry* aEntry,
 848                           OriginAttrsHash aOriginAttrsHash, bool aAnonymous);
 849
 850   // Checks whether any of the information about the entry has changed.
 851   static bool HasEntryChanged(CacheIndexEntry* aEntry,
 852                               const uint32_t* aFrecency,
 853                               const bool* aHasAltData,
 854                               const uint16_t* aOnStartTime,
 855                               const uint16_t* aOnStopTime,
 856                               const uint8_t* aContentType,
 857                               const uint32_t* aSize);
 858
 859   // Merge all pending operations from mPendingUpdates into mIndex.
 860   void ProcessPendingOperations(const StaticMutexAutoLock& aProofOfLock)
 861       MOZ_REQUIRES(sLock);
 862
 863   // Following methods perform writing of the index file.
 864   //
 865   // The index is written periodically, but not earlier than once in
 866   // kMinDumpInterval and there must be at least kMinUnwrittenChanges
 867   // differences between index on disk and in memory. Index is always first
 868   // written to a temporary file and the old index file is replaced when the
 869   // writing process succeeds.
 870   //
 871   // Starts writing of index when both limits (minimal delay between writes and
 872   // minimum number of changes in index) were exceeded.
 873   bool WriteIndexToDiskIfNeeded(const StaticMutexAutoLock& aProofOfLock)
 874       MOZ_REQUIRES(sLock);
 875   // Starts writing of index file.
 876   void WriteIndexToDisk(const StaticMutexAutoLock& aProofOfLock)
 877       MOZ_REQUIRES(sLock);
 878   // Serializes part of mIndex hashtable to the write buffer a writes the buffer
 879   // to the file.
 880   void WriteRecords(const StaticMutexAutoLock& aProofOfLock)
 881       MOZ_REQUIRES(sLock);
 882   // Finalizes writing process.
 883   void FinishWrite(bool aSucceeded, const StaticMutexAutoLock& aProofOfLock)
 884       MOZ_REQUIRES(sLock);
 885
 886   // Following methods perform writing of the journal during shutdown. All these
 887   // methods must be called only during shutdown since they write/delete files
 888   // directly on the main thread instead of using CacheFileIOManager that does
 889   // it asynchronously on IO thread. Journal contains only entries that are
 890   // dirty, i.e. changes that are not present in the index file on the disk.
 891   // When the log is written successfully, the dirty flag in index file is
 892   // cleared.
 893   nsresult GetFile(const nsACString& aName, nsIFile** _retval);
 894   void RemoveFile(const nsACString& aName) MOZ_REQUIRES(sLock);
 895   void RemoveAllIndexFiles() MOZ_REQUIRES(sLock);
 896   void RemoveJournalAndTempFile() MOZ_REQUIRES(sLock);
 897   // Writes journal to the disk and clears dirty flag in index header.
 898   nsresult WriteLogToDisk() MOZ_REQUIRES(sLock);
 899
 900   // Following methods perform reading of the index from the disk.
 901   //
 902   // Index is read at startup just after initializing the CacheIndex. There are
 903   // 3 files used when manipulating with index: index file, journal file and
 904   // a temporary file. All files contain the hash of the data, so we can check
 905   // whether the content is valid and complete. Index file contains also a dirty
 906   // flag in the index header which is unset on a clean shutdown. During opening
 907   // and reading of the files we determine the status of the whole index from
 908   // the states of the separate files. Following table shows all possible
 909   // combinations:
 910   //
 911   // index, journal, tmpfile
 912   // M      *        *       - index is missing    -> BUILD
 913   // I      *        *       - index is invalid    -> BUILD
 914   // D      *        *       - index is dirty      -> UPDATE
 915   // C      M        *       - index is dirty      -> UPDATE
 916   // C      I        *       - unexpected state    -> UPDATE
 917   // C      V        E       - unexpected state    -> UPDATE
 918   // C      V        M       - index is up to date -> READY
 919   //
 920   // where the letters mean:
 921   //   * - any state
 922   //   E - file exists
 923   //   M - file is missing
 924   //   I - data is invalid (parsing failed or hash didn't match)
 925   //   D - dirty (data in index file is correct, but dirty flag is set)
 926   //   C - clean (index file is clean)
 927   //   V - valid (data in journal file is correct)
 928   //
 929   // Note: We accept the data from journal only when the index is up to date as
 930   // a whole (i.e. C,V,M state).
 931   //
 932   // We rename the journal file to the temporary file as soon as possible after
 933   // initial test to ensure that we start update process on the next startup if
 934   // FF crashes during parsing of the index.
 935   //
 936   // Initiates reading index from disk.
 937   void ReadIndexFromDisk(const StaticMutexAutoLock& aProofOfLock)
 938       MOZ_REQUIRES(sLock);
 939   // Starts reading data from index file.
 940   void StartReadingIndex(const StaticMutexAutoLock& aProofOfLock)
 941       MOZ_REQUIRES(sLock);
 942   // Parses data read from index file.
 943   void ParseRecords(const StaticMutexAutoLock& aProofOfLock)
 944       MOZ_REQUIRES(sLock);
 945   // Starts reading data from journal file.
 946   void StartReadingJournal(const StaticMutexAutoLock& aProofOfLock)
 947       MOZ_REQUIRES(sLock);
 948   // Parses data read from journal file.
 949   void ParseJournal(const StaticMutexAutoLock& aProofOfLock)
 950       MOZ_REQUIRES(sLock);
 951   // Merges entries from journal into mIndex.
 952   void MergeJournal(const StaticMutexAutoLock& aProofOfLock)
 953       MOZ_REQUIRES(sLock);
 954   // In debug build this method checks that we have no fresh entry in mIndex
 955   // after we finish reading index and before we process pending operations.
 956   void EnsureNoFreshEntry() MOZ_REQUIRES(sLock);
 957   // In debug build this method is called after processing pending operations
 958   // to make sure mIndexStats contains correct information.
 959   void EnsureCorrectStats() MOZ_REQUIRES(sLock);
 960
 961   // Finalizes reading process.
 962   void FinishRead(bool aSucceeded, const StaticMutexAutoLock& aProofOfLock)
 963       MOZ_REQUIRES(sLock);
 964
 965   // Following methods perform updating and building of the index.
 966   // Timer callback that starts update or build process.
 967   static void DelayedUpdate(nsITimer* aTimer, void* aClosure);
 968   void DelayedUpdateLocked(const StaticMutexAutoLock& aProofOfLock)
 969       MOZ_REQUIRES(sLock);
 970   // Posts timer event that start update or build process.
 971   nsresult ScheduleUpdateTimer(uint32_t aDelay) MOZ_REQUIRES(sLock);
 972   nsresult SetupDirectoryEnumerator() MOZ_REQUIRES(sLock);
 973   nsresult InitEntryFromDiskData(CacheIndexEntry* aEntry,
 974                                  CacheFileMetadata* aMetaData,
 975                                  int64_t aFileSize);
 976   // Returns true when either a timer is scheduled or event is posted.
 977   bool IsUpdatePending() MOZ_REQUIRES(sLock);
 978   // Iterates through all files in entries directory that we didn't create/open
 979   // during this session, parses them and adds the entries to the index.
 980   void BuildIndex(const StaticMutexAutoLock& aProofOfLock) MOZ_REQUIRES(sLock);
 981
 982   bool StartUpdatingIndexIfNeeded(const StaticMutexAutoLock& aProofOfLock,
 983                                   bool aSwitchingToReadyState = false);
 984   // Starts update or build process or fires a timer when it is too early after
 985   // startup.
 986   void StartUpdatingIndex(bool aRebuild,
 987                           const StaticMutexAutoLock& aProofOfLock)
 988       MOZ_REQUIRES(sLock);
 989   // Iterates through all files in entries directory that we didn't create/open
 990   // during this session and theirs last modified time is newer than timestamp
 991   // in the index header. Parses the files and adds the entries to the index.
 992   void UpdateIndex(const StaticMutexAutoLock& aProofOfLock) MOZ_REQUIRES(sLock);
 993   // Finalizes update or build process.
 994   void FinishUpdate(bool aSucceeded, const StaticMutexAutoLock& aProofOfLock)
 995       MOZ_REQUIRES(sLock);
 996
 997   void RemoveNonFreshEntries(const StaticMutexAutoLock& aProofOfLock)
 998       MOZ_REQUIRES(sLock);
 999
1000   enum EState {
1001     // Initial state in which the index is not usable
1002     // Possible transitions:
1003     //  -> READING
1004     INITIAL = 0,
1005
1006     // Index is being read from the disk.
1007     // Possible transitions:
1008     //  -> INITIAL  - We failed to dispatch a read event.
1009     //  -> BUILDING - No or corrupted index file was found.
1010     //  -> UPDATING - No or corrupted journal file was found.
1011     //              - Dirty flag was set in index header.
1012     //  -> READY    - Index was read successfully or was interrupted by
1013     //                pre-shutdown.
1014     //  -> SHUTDOWN - This could happen only in case of pre-shutdown failure.
1015     READING = 1,
1016
1017     // Index is being written to the disk.
1018     // Possible transitions:
1019     //  -> READY    - Writing of index finished or was interrupted by
1020     //                pre-shutdown..
1021     //  -> UPDATING - Writing of index finished, but index was found outdated
1022     //                during writing.
1023     //  -> SHUTDOWN - This could happen only in case of pre-shutdown failure.
1024     WRITING = 2,
1025
1026     // Index is being build.
1027     // Possible transitions:
1028     //  -> READY    - Building of index finished or was interrupted by
1029     //                pre-shutdown.
1030     //  -> SHUTDOWN - This could happen only in case of pre-shutdown failure.
1031     BUILDING = 3,
1032
1033     // Index is being updated.
1034     // Possible transitions:
1035     //  -> READY    - Updating of index finished or was interrupted by
1036     //                pre-shutdown.
1037     //  -> SHUTDOWN - This could happen only in case of pre-shutdown failure.
1038     UPDATING = 4,
1039
1040     // Index is ready.
1041     // Possible transitions:
1042     //  -> UPDATING - Index was found outdated.
1043     //  -> SHUTDOWN - Index is shutting down.
1044     READY = 5,
1045
1046     // Index is shutting down.
1047     SHUTDOWN = 6
1048   };
1049
1050   static char const* StateString(EState aState);
1051   void ChangeState(EState aNewState, const StaticMutexAutoLock& aProofOfLock);
1052   void NotifyAsyncGetDiskConsumptionCallbacks() MOZ_REQUIRES(sLock);
1053
1054   // Allocates and releases buffer used for reading and writing index.
1055   void AllocBuffer() MOZ_REQUIRES(sLock);
1056   void ReleaseBuffer() MOZ_REQUIRES(sLock);
1057
1058   // Methods used by CacheIndexEntryAutoManage to keep the iterators up to date.
1059   void AddRecordToIterators(CacheIndexRecordWrapper* aRecord,
1060                             const StaticMutexAutoLock& aProofOfLock)
1061       MOZ_REQUIRES(sLock);
1062   void RemoveRecordFromIterators(CacheIndexRecordWrapper* aRecord,
1063                                  const StaticMutexAutoLock& aProofOfLock)
1064       MOZ_REQUIRES(sLock);
1065   void ReplaceRecordInIterators(CacheIndexRecordWrapper* aOldRecord,
1066                                 CacheIndexRecordWrapper* aNewRecord,
1067                                 const StaticMutexAutoLock& aProofOfLock)
1068       MOZ_REQUIRES(sLock);
1069
1070   // Memory reporting (private part)
1071   size_t SizeOfExcludingThisInternal(mozilla::MallocSizeOf mallocSizeOf) const
1072       MOZ_REQUIRES(sLock);
1073
1074   // Reports telemetry about cache, i.e. size, entry count and content type
1075   // stats.
1076   void DoTelemetryReport() MOZ_REQUIRES(sLock);
1077
1078   static mozilla::StaticRefPtr<CacheIndex> gInstance MOZ_GUARDED_BY(sLock);
1079
1080   // sLock guards almost everything here...
1081   // Also guards FileOpenHelper::mCanceled
1082   static StaticMutex sLock;
1083
1084   nsCOMPtr<nsIFile> mCacheDirectory;
1085
1086   EState mState MOZ_GUARDED_BY(sLock){INITIAL};
1087   // Timestamp of time when the index was initialized. We use it to delay
1088   // initial update or build of index.
1089   TimeStamp mStartTime MOZ_GUARDED_BY(sLock);
1090   // Set to true in PreShutdown(), it is checked on variaous places to prevent
1091   // starting any process (write, update, etc.) during shutdown.
1092   bool mShuttingDown MOZ_GUARDED_BY(sLock){false};
1093   // When set to true, update process should start as soon as possible. This
1094   // flag is set whenever we find some inconsistency which would be fixed by
1095   // update process. The flag is checked always when switching to READY state.
1096   // To make sure we start the update process as soon as possible, methods that
1097   // set this flag should also call StartUpdatingIndexIfNeeded() to cover the
1098   // case when we are currently in READY state.
1099   bool mIndexNeedsUpdate MOZ_GUARDED_BY(sLock){false};
1100   // Set at the beginning of RemoveAll() which clears the whole index. When
1101   // removing all entries we must stop any pending reading, writing, updating or
1102   // building operation. This flag is checked at various places and it prevents
1103   // we won't start another operation (e.g. canceling reading of the index would
1104   // normally start update or build process)
1105   bool mRemovingAll MOZ_GUARDED_BY(sLock){false};
1106   // Whether the index file on disk exists and is valid.
1107   bool mIndexOnDiskIsValid MOZ_GUARDED_BY(sLock){false};
1108   // When something goes wrong during updating or building process, we don't
1109   // mark index clean (and also don't write journal) to ensure that update or
1110   // build will be initiated on the next start.
1111   bool mDontMarkIndexClean MOZ_GUARDED_BY(sLock){false};
1112   // Timestamp value from index file. It is used during update process to skip
1113   // entries that were last modified before this timestamp.
1114   uint32_t mIndexTimeStamp MOZ_GUARDED_BY(sLock){0};
1115   // Timestamp of last time the index was dumped to disk.
1116   // NOTE: The index might not be necessarily dumped at this time. The value
1117   // is used to schedule next dump of the index.
1118   TimeStamp mLastDumpTime MOZ_GUARDED_BY(sLock);
1119
1120   // Timer of delayed update/build.
1121   nsCOMPtr<nsITimer> mUpdateTimer MOZ_GUARDED_BY(sLock);
1122   // True when build or update event is posted
1123   bool mUpdateEventPending MOZ_GUARDED_BY(sLock){false};
1124
1125   // Helper members used when reading/writing index from/to disk.
1126   // Contains number of entries that should be skipped:
1127   //  - in hashtable when writing index because they were already written
1128   //  - in index file when reading index because they were already read
1129   uint32_t mSkipEntries MOZ_GUARDED_BY(sLock){0};
1130   // Number of entries that should be written to disk. This is number of entries
1131   // in hashtable that are initialized and are not marked as removed when
1132   // writing begins.
1133   uint32_t mProcessEntries MOZ_GUARDED_BY(sLock){0};
1134   char* mRWBuf MOZ_GUARDED_BY(sLock){nullptr};
1135   uint32_t mRWBufSize MOZ_GUARDED_BY(sLock){0};
1136   uint32_t mRWBufPos MOZ_GUARDED_BY(sLock){0};
1137   RefPtr<CacheHash> mRWHash MOZ_GUARDED_BY(sLock);
1138
1139   // True if read or write operation is pending. It is used to ensure that
1140   // mRWBuf is not freed until OnDataRead or OnDataWritten is called.
1141   bool mRWPending MOZ_GUARDED_BY(sLock){false};
1142
1143   // Reading of journal succeeded if true.
1144   bool mJournalReadSuccessfully MOZ_GUARDED_BY(sLock){false};
1145
1146   // Handle used for writing and reading index file.
1147   RefPtr<CacheFileHandle> mIndexHandle MOZ_GUARDED_BY(sLock);
1148   // Handle used for reading journal file.
1149   RefPtr<CacheFileHandle> mJournalHandle MOZ_GUARDED_BY(sLock);
1150   // Used to check the existence of the file during reading process.
1151   RefPtr<CacheFileHandle> mTmpHandle MOZ_GUARDED_BY(sLock);
1152
1153   RefPtr<FileOpenHelper> mIndexFileOpener MOZ_GUARDED_BY(sLock);
1154   RefPtr<FileOpenHelper> mJournalFileOpener MOZ_GUARDED_BY(sLock);
1155   RefPtr<FileOpenHelper> mTmpFileOpener MOZ_GUARDED_BY(sLock);
1156
1157   // Directory enumerator used when building and updating index.
1158   nsCOMPtr<nsIDirectoryEnumerator> mDirEnumerator MOZ_GUARDED_BY(sLock);
1159
1160   // Main index hashtable.
1161   nsTHashtable<CacheIndexEntry> mIndex MOZ_GUARDED_BY(sLock);
1162
1163   // We cannot add, remove or change any entry in mIndex in states READING and
1164   // WRITING. We track all changes in mPendingUpdates during these states.
1165   nsTHashtable<CacheIndexEntryUpdate> mPendingUpdates MOZ_GUARDED_BY(sLock);
1166
1167   // Contains information statistics for mIndex + mPendingUpdates.
1168   CacheIndexStats mIndexStats MOZ_GUARDED_BY(sLock);
1169
1170   // When reading journal, we must first parse the whole file and apply the
1171   // changes iff the journal was read successfully. mTmpJournal is used to store
1172   // entries from the journal file. We throw away all these entries if parsing
1173   // of the journal fails or the hash does not match.
1174   nsTHashtable<CacheIndexEntry> mTmpJournal MOZ_GUARDED_BY(sLock);
1175
1176   // FrecencyArray maintains order of entry records for eviction. Ideally, the
1177   // records would be ordered by frecency all the time, but since this would be
1178   // quite expensive, we allow certain amount of entries to be out of order.
1179   // When the frecency is updated the new value is always bigger than the old
1180   // one. Instead of keeping updated entries at the same position, we move them
1181   // at the end of the array. This protects recently updated entries from
1182   // eviction. The array is sorted once we hit the limit of maximum unsorted
1183   // entries.
1184   class FrecencyArray {
1185     class Iterator {
1186      public:
1187       explicit Iterator(nsTArray<RefPtr<CacheIndexRecordWrapper>>* aRecs)
1188           : mRecs(aRecs), mIdx(0) {
1189         while (!Done() && !(*mRecs)[mIdx]) {
1190           mIdx++;
1191         }
1192       }
1193
1194       bool Done() const { return mIdx == mRecs->Length(); }
1195
1196       CacheIndexRecordWrapper* Get() const {
1197         MOZ_ASSERT(!Done());
1198         return (*mRecs)[mIdx];
1199       }
1200
1201       void Next() {
1202         MOZ_ASSERT(!Done());
1203         ++mIdx;
1204         while (!Done() && !(*mRecs)[mIdx]) {
1205           mIdx++;
1206         }
1207       }
1208
1209      private:
1210       nsTArray<RefPtr<CacheIndexRecordWrapper>>* mRecs;
1211       uint32_t mIdx;
1212     };
1213
1214    public:
1215     Iterator Iter() { return Iterator(&mRecs); }
1216
1217     FrecencyArray() = default;
1218
1219     // Methods used by CacheIndexEntryAutoManage to keep the array up to date.
1220     void AppendRecord(CacheIndexRecordWrapper* aRecord,
1221                       const StaticMutexAutoLock& aProofOfLock);
1222     void RemoveRecord(CacheIndexRecordWrapper* aRecord,
1223                       const StaticMutexAutoLock& aProofOfLock);
1224     void ReplaceRecord(CacheIndexRecordWrapper* aOldRecord,
1225                        CacheIndexRecordWrapper* aNewRecord,
1226                        const StaticMutexAutoLock& aProofOfLock);
1227     void SortIfNeeded(const StaticMutexAutoLock& aProofOfLock);
1228     bool RecordExistedUnlocked(CacheIndexRecordWrapper* aRecord);
1229
1230     size_t Length() const { return mRecs.Length() - mRemovedElements; }
1231     void Clear(const StaticMutexAutoLock& aProofOfLock) { mRecs.Clear(); }
1232
1233    private:
1234     friend class CacheIndex;
1235
1236     nsTArray<RefPtr<CacheIndexRecordWrapper>> mRecs;
1237     uint32_t mUnsortedElements{0};
1238     // Instead of removing elements from the array immediately, we null them out
1239     // and the iterator skips them when accessing the array. The null pointers
1240     // are placed at the end during sorting and we strip them out all at once.
1241     // This saves moving a lot of memory in nsTArray::RemoveElementsAt.
1242     uint32_t mRemovedElements{0};
1243   };
1244
1245   FrecencyArray mFrecencyArray MOZ_GUARDED_BY(sLock);
1246
1247   nsTArray<CacheIndexIterator*> mIterators MOZ_GUARDED_BY(sLock);
1248
1249   // This flag is true iff we are between CacheStorageService:Clear() and
1250   // processing all contexts to be evicted.  It will make UI to show
1251   // "calculating" instead of any intermediate cache size.
1252   bool mAsyncGetDiskConsumptionBlocked MOZ_GUARDED_BY(sLock){false};
1253
1254   class DiskConsumptionObserver : public Runnable {
1255    public:
1256     static DiskConsumptionObserver* Init(
1257         nsICacheStorageConsumptionObserver* aObserver) {
1258       nsWeakPtr observer = do_GetWeakReference(aObserver);
1259       if (!observer) return nullptr;
1260
1261       return new DiskConsumptionObserver(observer);
1262     }
1263
1264     void OnDiskConsumption(int64_t aSize) {
1265       mSize = aSize;
1266       NS_DispatchToMainThread(this);
1267     }
1268
1269    private:
1270     explicit DiskConsumptionObserver(nsWeakPtr const& aWeakObserver)
1271         : Runnable("net::CacheIndex::DiskConsumptionObserver"),
1272           mObserver(aWeakObserver),
1273           mSize(0) {}
1274     virtual ~DiskConsumptionObserver() {
1275       if (mObserver && !NS_IsMainThread()) {
1276         NS_ReleaseOnMainThread("DiskConsumptionObserver::mObserver",
1277                                mObserver.forget());
1278       }
1279     }
1280
1281     NS_IMETHOD Run() override {
1282       MOZ_ASSERT(NS_IsMainThread());
1283
1284       nsCOMPtr<nsICacheStorageConsumptionObserver> observer =
1285           do_QueryReferent(mObserver);
1286
1287       mObserver = nullptr;
1288
1289       if (observer) {
1290         observer->OnNetworkCacheDiskConsumption(mSize);
1291       }
1292
1293       return NS_OK;
1294     }
1295
1296     nsWeakPtr mObserver;
1297     int64_t mSize;
1298   };
1299
1300   // List of async observers that want to get disk consumption information
1301   nsTArray<RefPtr<DiskConsumptionObserver>> mDiskConsumptionObservers
1302       MOZ_GUARDED_BY(sLock);
1303
1304   // Number of bytes written to the cache since the last telemetry report
1305   uint64_t mTotalBytesWritten MOZ_GUARDED_BY(sLock){0};
1306 };
1307
1308 }  // namespace net
1309 }  // namespace mozilla
1310
1311 #endif