netwerk/cache2/CacheIndex.cpp

   1 /* This Source Code Form is subject to the terms of the Mozilla Public
   2  * License, v. 2.0. If a copy of the MPL was not distributed with this
   3  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   4
   5 #include "CacheIndex.h"
   6
   7 #include "CacheLog.h"
   8 #include "CacheFileIOManager.h"
   9 #include "CacheFileMetadata.h"
  10 #include "CacheFileUtils.h"
  11 #include "CacheIndexIterator.h"
  12 #include "CacheIndexContextIterator.h"
  13 #include "nsThreadUtils.h"
  14 #include "nsISizeOf.h"
  15 #include "nsPrintfCString.h"
  16 #include "mozilla/DebugOnly.h"
  17 #include "prinrval.h"
  18 #include "nsIFile.h"
  19 #include "nsITimer.h"
  20 #include "mozilla/AutoRestore.h"
  21 #include <algorithm>
  22 #include "mozilla/StaticPrefs_network.h"
  23 #include "mozilla/Telemetry.h"
  24 #include "mozilla/Unused.h"
  25
  26 #define kMinUnwrittenChanges 300
  27 #define kMinDumpInterval 20000  // in milliseconds
  28 #define kMaxBufSize 16384
  29 #define kIndexVersion 0x0000000A
  30 #define kUpdateIndexStartDelay 50000  // in milliseconds
  31 #define kTelemetryReportBytesLimit (2U * 1024U * 1024U * 1024U)  // 2GB
  32
  33 #define INDEX_NAME "index"
  34 #define TEMP_INDEX_NAME "index.tmp"
  35 #define JOURNAL_NAME "index.log"
  36
  37 namespace mozilla::net {
  38
  39 namespace {
  40
  41 class FrecencyComparator {
  42  public:
  43   bool Equals(const RefPtr<CacheIndexRecordWrapper>& a,
  44               const RefPtr<CacheIndexRecordWrapper>& b) const {
  45     if (!a || !b) {
  46       return false;
  47     }
  48
  49     return a->Get()->mFrecency == b->Get()->mFrecency;
  50   }
  51   bool LessThan(const RefPtr<CacheIndexRecordWrapper>& a,
  52                 const RefPtr<CacheIndexRecordWrapper>& b) const {
  53     // Removed (=null) entries must be at the end of the array.
  54     if (!a) {
  55       return false;
  56     }
  57     if (!b) {
  58       return true;
  59     }
  60
  61     // Place entries with frecency 0 at the end of the non-removed entries.
  62     if (a->Get()->mFrecency == 0) {
  63       return false;
  64     }
  65     if (b->Get()->mFrecency == 0) {
  66       return true;
  67     }
  68
  69     return a->Get()->mFrecency < b->Get()->mFrecency;
  70   }
  71 };
  72
  73 }  // namespace
  74
  75 // used to dispatch a wrapper deletion the caller's thread
  76 // cannot be used on IOThread after shutdown begins
  77 class DeleteCacheIndexRecordWrapper : public Runnable {
  78   CacheIndexRecordWrapper* mWrapper;
  79
  80  public:
  81   explicit DeleteCacheIndexRecordWrapper(CacheIndexRecordWrapper* wrapper)
  82       : Runnable("net::CacheIndex::DeleteCacheIndexRecordWrapper"),
  83         mWrapper(wrapper) {}
  84   NS_IMETHOD Run() override {
  85     StaticMutexAutoLock lock(CacheIndex::sLock);
  86
  87     // if somehow the item is still in the frecency array, remove it
  88     RefPtr<CacheIndex> index = CacheIndex::gInstance;
  89     if (index) {
  90       bool found = index->mFrecencyArray.RecordExistedUnlocked(mWrapper);
  91       if (found) {
  92         LOG(
  93             ("DeleteCacheIndexRecordWrapper::Run() - \
  94             record wrapper found in frecency array during deletion"));
  95         index->mFrecencyArray.RemoveRecord(mWrapper, lock);
  96       }
  97     }
  98
  99     delete mWrapper;
 100     return NS_OK;
 101   }
 102 };
 103
 104 void CacheIndexRecordWrapper::DispatchDeleteSelfToCurrentThread() {
 105   // Dispatch during shutdown will not trigger DeleteCacheIndexRecordWrapper
 106   nsCOMPtr<nsIRunnable> event = new DeleteCacheIndexRecordWrapper(this);
 107   MOZ_ALWAYS_SUCCEEDS(NS_DispatchToCurrentThread(event));
 108 }
 109
 110 CacheIndexRecordWrapper::~CacheIndexRecordWrapper() {
 111 #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
 112   CacheIndex::sLock.AssertCurrentThreadOwns();
 113   RefPtr<CacheIndex> index = CacheIndex::gInstance;
 114   if (index) {
 115     bool found = index->mFrecencyArray.RecordExistedUnlocked(this);
 116     MOZ_DIAGNOSTIC_ASSERT(!found);
 117   }
 118 #endif
 119 }
 120
 121 /**
 122  * This helper class is responsible for keeping CacheIndex::mIndexStats and
 123  * CacheIndex::mFrecencyArray up to date.
 124  */
 125 class MOZ_RAII CacheIndexEntryAutoManage {
 126  public:
 127   CacheIndexEntryAutoManage(const SHA1Sum::Hash* aHash, CacheIndex* aIndex,
 128                             const StaticMutexAutoLock& aProofOfLock)
 129       MOZ_REQUIRES(CacheIndex::sLock)
 130       : mIndex(aIndex), mProofOfLock(aProofOfLock) {
 131     mHash = aHash;
 132     const CacheIndexEntry* entry = FindEntry();
 133     mIndex->mIndexStats.BeforeChange(entry);
 134     if (entry && entry->IsInitialized() && !entry->IsRemoved()) {
 135       mOldRecord = entry->mRec;
 136       mOldFrecency = entry->mRec->Get()->mFrecency;
 137     }
 138   }
 139
 140   ~CacheIndexEntryAutoManage() MOZ_REQUIRES(CacheIndex::sLock) {
 141     const CacheIndexEntry* entry = FindEntry();
 142     mIndex->mIndexStats.AfterChange(entry);
 143     if (!entry || !entry->IsInitialized() || entry->IsRemoved()) {
 144       entry = nullptr;
 145     }
 146
 147     if (entry && !mOldRecord) {
 148       mIndex->mFrecencyArray.AppendRecord(entry->mRec, mProofOfLock);
 149       mIndex->AddRecordToIterators(entry->mRec, mProofOfLock);
 150     } else if (!entry && mOldRecord) {
 151       mIndex->mFrecencyArray.RemoveRecord(mOldRecord, mProofOfLock);
 152       mIndex->RemoveRecordFromIterators(mOldRecord, mProofOfLock);
 153     } else if (entry && mOldRecord) {
 154       if (entry->mRec != mOldRecord) {
 155         // record has a different address, we have to replace it
 156         mIndex->ReplaceRecordInIterators(mOldRecord, entry->mRec, mProofOfLock);
 157
 158         if (entry->mRec->Get()->mFrecency == mOldFrecency) {
 159           // If frecency hasn't changed simply replace the pointer
 160           mIndex->mFrecencyArray.ReplaceRecord(mOldRecord, entry->mRec,
 161                                                mProofOfLock);
 162         } else {
 163           // Remove old pointer and insert the new one at the end of the array
 164           mIndex->mFrecencyArray.RemoveRecord(mOldRecord, mProofOfLock);
 165           mIndex->mFrecencyArray.AppendRecord(entry->mRec, mProofOfLock);
 166         }
 167       } else if (entry->mRec->Get()->mFrecency != mOldFrecency) {
 168         // Move the element at the end of the array
 169         mIndex->mFrecencyArray.RemoveRecord(entry->mRec, mProofOfLock);
 170         mIndex->mFrecencyArray.AppendRecord(entry->mRec, mProofOfLock);
 171       }
 172     } else {
 173       // both entries were removed or not initialized, do nothing
 174     }
 175   }
 176
 177   // We cannot rely on nsTHashtable::GetEntry() in case we are removing entries
 178   // while iterating. Destructor is called before the entry is removed. Caller
 179   // must call one of following methods to skip lookup in the hashtable.
 180   void DoNotSearchInIndex() { mDoNotSearchInIndex = true; }
 181   void DoNotSearchInUpdates() { mDoNotSearchInUpdates = true; }
 182
 183  private:
 184   const CacheIndexEntry* FindEntry() MOZ_REQUIRES(CacheIndex::sLock) {
 185     const CacheIndexEntry* entry = nullptr;
 186
 187     switch (mIndex->mState) {
 188       case CacheIndex::READING:
 189       case CacheIndex::WRITING:
 190         if (!mDoNotSearchInUpdates) {
 191           entry = mIndex->mPendingUpdates.GetEntry(*mHash);
 192         }
 193         [[fallthrough]];
 194       case CacheIndex::BUILDING:
 195       case CacheIndex::UPDATING:
 196       case CacheIndex::READY:
 197         if (!entry && !mDoNotSearchInIndex) {
 198           entry = mIndex->mIndex.GetEntry(*mHash);
 199         }
 200         break;
 201       case CacheIndex::INITIAL:
 202       case CacheIndex::SHUTDOWN:
 203       default:
 204         MOZ_ASSERT(false, "Unexpected state!");
 205     }
 206
 207     return entry;
 208   }
 209
 210   const SHA1Sum::Hash* mHash;
 211   RefPtr<CacheIndex> mIndex;
 212   RefPtr<CacheIndexRecordWrapper> mOldRecord;
 213   uint32_t mOldFrecency{0};
 214   bool mDoNotSearchInIndex{false};
 215   bool mDoNotSearchInUpdates{false};
 216   const StaticMutexAutoLock& mProofOfLock;
 217 };
 218
 219 class FileOpenHelper final : public CacheFileIOListener {
 220  public:
 221   NS_DECL_THREADSAFE_ISUPPORTS
 222
 223   explicit FileOpenHelper(CacheIndex* aIndex)
 224       : mIndex(aIndex), mCanceled(false) {}
 225
 226   void Cancel() {
 227     CacheIndex::sLock.AssertCurrentThreadOwns();
 228     mCanceled = true;
 229   }
 230
 231  private:
 232   virtual ~FileOpenHelper() = default;
 233
 234   NS_IMETHOD OnFileOpened(CacheFileHandle* aHandle, nsresult aResult) override;
 235   NS_IMETHOD OnDataWritten(CacheFileHandle* aHandle, const char* aBuf,
 236                            nsresult aResult) override {
 237     MOZ_CRASH("FileOpenHelper::OnDataWritten should not be called!");
 238     return NS_ERROR_UNEXPECTED;
 239   }
 240   NS_IMETHOD OnDataRead(CacheFileHandle* aHandle, char* aBuf,
 241                         nsresult aResult) override {
 242     MOZ_CRASH("FileOpenHelper::OnDataRead should not be called!");
 243     return NS_ERROR_UNEXPECTED;
 244   }
 245   NS_IMETHOD OnFileDoomed(CacheFileHandle* aHandle, nsresult aResult) override {
 246     MOZ_CRASH("FileOpenHelper::OnFileDoomed should not be called!");
 247     return NS_ERROR_UNEXPECTED;
 248   }
 249   NS_IMETHOD OnEOFSet(CacheFileHandle* aHandle, nsresult aResult) override {
 250     MOZ_CRASH("FileOpenHelper::OnEOFSet should not be called!");
 251     return NS_ERROR_UNEXPECTED;
 252   }
 253   NS_IMETHOD OnFileRenamed(CacheFileHandle* aHandle,
 254                            nsresult aResult) override {
 255     MOZ_CRASH("FileOpenHelper::OnFileRenamed should not be called!");
 256     return NS_ERROR_UNEXPECTED;
 257   }
 258
 259   RefPtr<CacheIndex> mIndex;
 260   bool mCanceled;
 261 };
 262
 263 NS_IMETHODIMP FileOpenHelper::OnFileOpened(CacheFileHandle* aHandle,
 264                                            nsresult aResult) {
 265   StaticMutexAutoLock lock(CacheIndex::sLock);
 266
 267   if (mCanceled) {
 268     if (aHandle) {
 269       CacheFileIOManager::DoomFile(aHandle, nullptr);
 270     }
 271
 272     return NS_OK;
 273   }
 274
 275   mIndex->OnFileOpenedInternal(this, aHandle, aResult, lock);
 276
 277   return NS_OK;
 278 }
 279
 280 NS_IMPL_ISUPPORTS(FileOpenHelper, CacheFileIOListener);
 281
 282 StaticRefPtr<CacheIndex> CacheIndex::gInstance;
 283 StaticMutex CacheIndex::sLock;
 284
 285 NS_IMPL_ADDREF(CacheIndex)
 286 NS_IMPL_RELEASE(CacheIndex)
 287
 288 NS_INTERFACE_MAP_BEGIN(CacheIndex)
 289   NS_INTERFACE_MAP_ENTRY(mozilla::net::CacheFileIOListener)
 290   NS_INTERFACE_MAP_ENTRY(nsIRunnable)
 291 NS_INTERFACE_MAP_END
 292
 293 CacheIndex::CacheIndex() {
 294   sLock.AssertCurrentThreadOwns();
 295   LOG(("CacheIndex::CacheIndex [this=%p]", this));
 296   MOZ_ASSERT(!gInstance, "multiple CacheIndex instances!");
 297 }
 298
 299 CacheIndex::~CacheIndex() {
 300   sLock.AssertCurrentThreadOwns();
 301   LOG(("CacheIndex::~CacheIndex [this=%p]", this));
 302
 303   ReleaseBuffer();
 304 }
 305
 306 // static
 307 nsresult CacheIndex::Init(nsIFile* aCacheDirectory) {
 308   LOG(("CacheIndex::Init()"));
 309
 310   MOZ_ASSERT(NS_IsMainThread());
 311
 312   StaticMutexAutoLock lock(sLock);
 313
 314   if (gInstance) {
 315     return NS_ERROR_ALREADY_INITIALIZED;
 316   }
 317
 318   RefPtr<CacheIndex> idx = new CacheIndex();
 319
 320   nsresult rv = idx->InitInternal(aCacheDirectory, lock);
 321   NS_ENSURE_SUCCESS(rv, rv);
 322
 323   gInstance = std::move(idx);
 324   return NS_OK;
 325 }
 326
 327 nsresult CacheIndex::InitInternal(nsIFile* aCacheDirectory,
 328                                   const StaticMutexAutoLock& aProofOfLock) {
 329   nsresult rv;
 330   sLock.AssertCurrentThreadOwns();
 331
 332   rv = aCacheDirectory->Clone(getter_AddRefs(mCacheDirectory));
 333   NS_ENSURE_SUCCESS(rv, rv);
 334
 335   mStartTime = TimeStamp::NowLoRes();
 336
 337   ReadIndexFromDisk(aProofOfLock);
 338
 339   return NS_OK;
 340 }
 341
 342 // static
 343 nsresult CacheIndex::PreShutdown() {
 344   MOZ_ASSERT(NS_IsMainThread());
 345
 346   StaticMutexAutoLock lock(sLock);
 347
 348   LOG(("CacheIndex::PreShutdown() [gInstance=%p]", gInstance.get()));
 349
 350   nsresult rv;
 351   RefPtr<CacheIndex> index = gInstance;
 352
 353   if (!index) {
 354     return NS_ERROR_NOT_INITIALIZED;
 355   }
 356
 357   LOG(
 358       ("CacheIndex::PreShutdown() - [state=%d, indexOnDiskIsValid=%d, "
 359        "dontMarkIndexClean=%d]",
 360        index->mState, index->mIndexOnDiskIsValid, index->mDontMarkIndexClean));
 361
 362   LOG(("CacheIndex::PreShutdown() - Closing iterators."));
 363   for (uint32_t i = 0; i < index->mIterators.Length();) {
 364     rv = index->mIterators[i]->CloseInternal(NS_ERROR_FAILURE);
 365     if (NS_FAILED(rv)) {
 366       // CacheIndexIterator::CloseInternal() removes itself from mIteratos iff
 367       // it returns success.
 368       LOG(
 369           ("CacheIndex::PreShutdown() - Failed to remove iterator %p. "
 370            "[rv=0x%08" PRIx32 "]",
 371            index->mIterators[i], static_cast<uint32_t>(rv)));
 372       i++;
 373     }
 374   }
 375
 376   index->mShuttingDown = true;
 377
 378   if (index->mState == READY) {
 379     return NS_OK;  // nothing to do
 380   }
 381
 382   nsCOMPtr<nsIRunnable> event;
 383   event = NewRunnableMethod("net::CacheIndex::PreShutdownInternal", index,
 384                             &CacheIndex::PreShutdownInternal);
 385
 386   nsCOMPtr<nsIEventTarget> ioTarget = CacheFileIOManager::IOTarget();
 387   MOZ_ASSERT(ioTarget);
 388
 389   // PreShutdownInternal() will be executed before any queued event on INDEX
 390   // level. That's OK since we don't want to wait for any operation in progess.
 391   rv = ioTarget->Dispatch(event, nsIEventTarget::DISPATCH_NORMAL);
 392   if (NS_FAILED(rv)) {
 393     NS_WARNING("CacheIndex::PreShutdown() - Can't dispatch event");
 394     LOG(("CacheIndex::PreShutdown() - Can't dispatch event"));
 395     return rv;
 396   }
 397
 398   return NS_OK;
 399 }
 400
 401 void CacheIndex::PreShutdownInternal() {
 402   StaticMutexAutoLock lock(sLock);
 403
 404   LOG(
 405       ("CacheIndex::PreShutdownInternal() - [state=%d, indexOnDiskIsValid=%d, "
 406        "dontMarkIndexClean=%d]",
 407        mState, mIndexOnDiskIsValid, mDontMarkIndexClean));
 408
 409   MOZ_ASSERT(mShuttingDown);
 410
 411   if (mUpdateTimer) {
 412     mUpdateTimer->Cancel();
 413     mUpdateTimer = nullptr;
 414   }
 415
 416   switch (mState) {
 417     case WRITING:
 418       FinishWrite(false, lock);
 419       break;
 420     case READY:
 421       // nothing to do, write the journal in Shutdown()
 422       break;
 423     case READING:
 424       FinishRead(false, lock);
 425       break;
 426     case BUILDING:
 427     case UPDATING:
 428       FinishUpdate(false, lock);
 429       break;
 430     default:
 431       MOZ_ASSERT(false, "Implement me!");
 432   }
 433
 434   // We should end up in READY state
 435   MOZ_ASSERT(mState == READY);
 436 }
 437
 438 // static
 439 nsresult CacheIndex::Shutdown() {
 440   MOZ_ASSERT(NS_IsMainThread());
 441
 442   StaticMutexAutoLock lock(sLock);
 443
 444   LOG(("CacheIndex::Shutdown() [gInstance=%p]", gInstance.get()));
 445
 446   RefPtr<CacheIndex> index = gInstance.forget();
 447
 448   if (!index) {
 449     return NS_ERROR_NOT_INITIALIZED;
 450   }
 451
 452   bool sanitize = CacheObserver::ClearCacheOnShutdown();
 453
 454   LOG(
 455       ("CacheIndex::Shutdown() - [state=%d, indexOnDiskIsValid=%d, "
 456        "dontMarkIndexClean=%d, sanitize=%d]",
 457        index->mState, index->mIndexOnDiskIsValid, index->mDontMarkIndexClean,
 458        sanitize));
 459
 460   MOZ_ASSERT(index->mShuttingDown);
 461
 462   EState oldState = index->mState;
 463   index->ChangeState(SHUTDOWN, lock);
 464
 465   if (oldState != READY) {
 466     LOG(
 467         ("CacheIndex::Shutdown() - Unexpected state. Did posting of "
 468          "PreShutdownInternal() fail?"));
 469   }
 470
 471   switch (oldState) {
 472     case WRITING:
 473       index->FinishWrite(false, lock);
 474       [[fallthrough]];
 475     case READY:
 476       if (index->mIndexOnDiskIsValid && !index->mDontMarkIndexClean) {
 477         if (!sanitize && NS_FAILED(index->WriteLogToDisk())) {
 478           index->RemoveJournalAndTempFile();
 479         }
 480       } else {
 481         index->RemoveJournalAndTempFile();
 482       }
 483       break;
 484     case READING:
 485       index->FinishRead(false, lock);
 486       break;
 487     case BUILDING:
 488     case UPDATING:
 489       index->FinishUpdate(false, lock);
 490       break;
 491     default:
 492       MOZ_ASSERT(false, "Unexpected state!");
 493   }
 494
 495   if (sanitize) {
 496     index->RemoveAllIndexFiles();
 497   }
 498
 499   return NS_OK;
 500 }
 501
 502 // static
 503 nsresult CacheIndex::AddEntry(const SHA1Sum::Hash* aHash) {
 504   LOG(("CacheIndex::AddEntry() [hash=%08x%08x%08x%08x%08x]", LOGSHA1(aHash)));
 505
 506   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
 507
 508   StaticMutexAutoLock lock(sLock);
 509
 510   RefPtr<CacheIndex> index = gInstance;
 511
 512   if (!index) {
 513     return NS_ERROR_NOT_INITIALIZED;
 514   }
 515
 516   if (!index->IsIndexUsable()) {
 517     return NS_ERROR_NOT_AVAILABLE;
 518   }
 519
 520   // Getters in CacheIndexStats assert when mStateLogged is true since the
 521   // information is incomplete between calls to BeforeChange() and AfterChange()
 522   // (i.e. while CacheIndexEntryAutoManage exists). We need to check whether
 523   // non-fresh entries exists outside the scope of CacheIndexEntryAutoManage.
 524   bool updateIfNonFreshEntriesExist = false;
 525
 526   {
 527     CacheIndexEntryAutoManage entryMng(aHash, index, lock);
 528
 529     CacheIndexEntry* entry = index->mIndex.GetEntry(*aHash);
 530     bool entryRemoved = entry && entry->IsRemoved();
 531     CacheIndexEntryUpdate* updated = nullptr;
 532
 533     if (index->mState == READY || index->mState == UPDATING ||
 534         index->mState == BUILDING) {
 535       MOZ_ASSERT(index->mPendingUpdates.Count() == 0);
 536
 537       if (entry && !entryRemoved) {
 538         // Found entry in index that shouldn't exist.
 539
 540         if (entry->IsFresh()) {
 541           // Someone removed the file on disk while FF is running. Update
 542           // process can fix only non-fresh entries (i.e. entries that were not
 543           // added within this session). Start update only if we have such
 544           // entries.
 545           //
 546           // TODO: This should be very rare problem. If it turns out not to be
 547           // true, change the update process so that it also iterates all
 548           // initialized non-empty entries and checks whether the file exists.
 549
 550           LOG(
 551               ("CacheIndex::AddEntry() - Cache file was removed outside FF "
 552                "process!"));
 553
 554           updateIfNonFreshEntriesExist = true;
 555         } else if (index->mState == READY) {
 556           // Index is outdated, update it.
 557           LOG(
 558               ("CacheIndex::AddEntry() - Found entry that shouldn't exist, "
 559                "update is needed"));
 560           index->mIndexNeedsUpdate = true;
 561         } else {
 562           // We cannot be here when building index since all entries are fresh
 563           // during building.
 564           MOZ_ASSERT(index->mState == UPDATING);
 565         }
 566       }
 567
 568       if (!entry) {
 569         entry = index->mIndex.PutEntry(*aHash);
 570       }
 571     } else {  // WRITING, READING
 572       updated = index->mPendingUpdates.GetEntry(*aHash);
 573       bool updatedRemoved = updated && updated->IsRemoved();
 574
 575       if ((updated && !updatedRemoved) ||
 576           (!updated && entry && !entryRemoved && entry->IsFresh())) {
 577         // Fresh entry found, so the file was removed outside FF
 578         LOG(
 579             ("CacheIndex::AddEntry() - Cache file was removed outside FF "
 580              "process!"));
 581
 582         updateIfNonFreshEntriesExist = true;
 583       } else if (!updated && entry && !entryRemoved) {
 584         if (index->mState == WRITING) {
 585           LOG(
 586               ("CacheIndex::AddEntry() - Found entry that shouldn't exist, "
 587                "update is needed"));
 588           index->mIndexNeedsUpdate = true;
 589         }
 590         // Ignore if state is READING since the index information is partial
 591       }
 592
 593       updated = index->mPendingUpdates.PutEntry(*aHash);
 594     }
 595
 596     if (updated) {
 597       updated->InitNew();
 598       updated->MarkDirty();
 599       updated->MarkFresh();
 600     } else {
 601       entry->InitNew();
 602       entry->MarkDirty();
 603       entry->MarkFresh();
 604     }
 605   }
 606
 607   if (updateIfNonFreshEntriesExist &&
 608       index->mIndexStats.Count() != index->mIndexStats.Fresh()) {
 609     index->mIndexNeedsUpdate = true;
 610   }
 611
 612   index->StartUpdatingIndexIfNeeded(lock);
 613   index->WriteIndexToDiskIfNeeded(lock);
 614
 615   return NS_OK;
 616 }
 617
 618 // static
 619 nsresult CacheIndex::EnsureEntryExists(const SHA1Sum::Hash* aHash) {
 620   LOG(("CacheIndex::EnsureEntryExists() [hash=%08x%08x%08x%08x%08x]",
 621        LOGSHA1(aHash)));
 622
 623   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
 624
 625   StaticMutexAutoLock lock(sLock);
 626
 627   RefPtr<CacheIndex> index = gInstance;
 628
 629   if (!index) {
 630     return NS_ERROR_NOT_INITIALIZED;
 631   }
 632
 633   if (!index->IsIndexUsable()) {
 634     return NS_ERROR_NOT_AVAILABLE;
 635   }
 636
 637   {
 638     CacheIndexEntryAutoManage entryMng(aHash, index, lock);
 639
 640     CacheIndexEntry* entry = index->mIndex.GetEntry(*aHash);
 641     bool entryRemoved = entry && entry->IsRemoved();
 642
 643     if (index->mState == READY || index->mState == UPDATING ||
 644         index->mState == BUILDING) {
 645       MOZ_ASSERT(index->mPendingUpdates.Count() == 0);
 646
 647       if (!entry || entryRemoved) {
 648         if (entryRemoved && entry->IsFresh()) {
 649           // This could happen only if somebody copies files to the entries
 650           // directory while FF is running.
 651           LOG(
 652               ("CacheIndex::EnsureEntryExists() - Cache file was added outside "
 653                "FF process! Update is needed."));
 654           index->mIndexNeedsUpdate = true;
 655         } else if (index->mState == READY ||
 656                    (entryRemoved && !entry->IsFresh())) {
 657           // Removed non-fresh entries can be present as a result of
 658           // MergeJournal()
 659           LOG(
 660               ("CacheIndex::EnsureEntryExists() - Didn't find entry that should"
 661                " exist, update is needed"));
 662           index->mIndexNeedsUpdate = true;
 663         }
 664
 665         if (!entry) {
 666           entry = index->mIndex.PutEntry(*aHash);
 667         }
 668         entry->InitNew();
 669         entry->MarkDirty();
 670       }
 671       entry->MarkFresh();
 672     } else {  // WRITING, READING
 673       CacheIndexEntryUpdate* updated = index->mPendingUpdates.GetEntry(*aHash);
 674       bool updatedRemoved = updated && updated->IsRemoved();
 675
 676       if (updatedRemoved || (!updated && entryRemoved && entry->IsFresh())) {
 677         // Fresh information about missing entry found. This could happen only
 678         // if somebody copies files to the entries directory while FF is
 679         // running.
 680         LOG(
 681             ("CacheIndex::EnsureEntryExists() - Cache file was added outside "
 682              "FF process! Update is needed."));
 683         index->mIndexNeedsUpdate = true;
 684       } else if (!updated && (!entry || entryRemoved)) {
 685         if (index->mState == WRITING) {
 686           LOG(
 687               ("CacheIndex::EnsureEntryExists() - Didn't find entry that should"
 688                " exist, update is needed"));
 689           index->mIndexNeedsUpdate = true;
 690         }
 691         // Ignore if state is READING since the index information is partial
 692       }
 693
 694       // We don't need entryRemoved and updatedRemoved info anymore
 695       if (entryRemoved) entry = nullptr;
 696       if (updatedRemoved) updated = nullptr;
 697
 698       if (updated) {
 699         updated->MarkFresh();
 700       } else {
 701         if (!entry) {
 702           // Create a new entry
 703           updated = index->mPendingUpdates.PutEntry(*aHash);
 704           updated->InitNew();
 705           updated->MarkFresh();
 706           updated->MarkDirty();
 707         } else {
 708           if (!entry->IsFresh()) {
 709             // To mark the entry fresh we must make a copy of index entry
 710             // since the index is read-only.
 711             updated = index->mPendingUpdates.PutEntry(*aHash);
 712             *updated = *entry;
 713             updated->MarkFresh();
 714           }
 715         }
 716       }
 717     }
 718   }
 719
 720   index->StartUpdatingIndexIfNeeded(lock);
 721   index->WriteIndexToDiskIfNeeded(lock);
 722
 723   return NS_OK;
 724 }
 725
 726 // static
 727 nsresult CacheIndex::InitEntry(const SHA1Sum::Hash* aHash,
 728                                OriginAttrsHash aOriginAttrsHash,
 729                                bool aAnonymous, bool aPinned) {
 730   LOG(
 731       ("CacheIndex::InitEntry() [hash=%08x%08x%08x%08x%08x, "
 732        "originAttrsHash=%" PRIx64 ", anonymous=%d, pinned=%d]",
 733        LOGSHA1(aHash), aOriginAttrsHash, aAnonymous, aPinned));
 734
 735   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
 736
 737   StaticMutexAutoLock lock(sLock);
 738
 739   RefPtr<CacheIndex> index = gInstance;
 740
 741   if (!index) {
 742     return NS_ERROR_NOT_INITIALIZED;
 743   }
 744
 745   if (!index->IsIndexUsable()) {
 746     return NS_ERROR_NOT_AVAILABLE;
 747   }
 748
 749   {
 750     CacheIndexEntryAutoManage entryMng(aHash, index, lock);
 751
 752     CacheIndexEntry* entry = index->mIndex.GetEntry(*aHash);
 753     CacheIndexEntryUpdate* updated = nullptr;
 754     bool reinitEntry = false;
 755
 756     if (entry && entry->IsRemoved()) {
 757       entry = nullptr;
 758     }
 759
 760     if (index->mState == READY || index->mState == UPDATING ||
 761         index->mState == BUILDING) {
 762       MOZ_ASSERT(index->mPendingUpdates.Count() == 0);
 763       MOZ_ASSERT(entry);
 764       MOZ_ASSERT(entry->IsFresh());
 765
 766       if (!entry) {
 767         LOG(("CacheIndex::InitEntry() - Entry was not found in mIndex!"));
 768         NS_WARNING(
 769             ("CacheIndex::InitEntry() - Entry was not found in mIndex!"));
 770         return NS_ERROR_UNEXPECTED;
 771       }
 772
 773       if (IsCollision(entry, aOriginAttrsHash, aAnonymous)) {
 774         index->mIndexNeedsUpdate =
 775             true;  // TODO Does this really help in case of collision?
 776         reinitEntry = true;
 777       } else {
 778         if (entry->IsInitialized()) {
 779           return NS_OK;
 780         }
 781       }
 782     } else {
 783       updated = index->mPendingUpdates.GetEntry(*aHash);
 784       DebugOnly<bool> removed = updated && updated->IsRemoved();
 785
 786       MOZ_ASSERT(updated || !removed);
 787       MOZ_ASSERT(updated || entry);
 788
 789       if (!updated && !entry) {
 790         LOG(
 791             ("CacheIndex::InitEntry() - Entry was found neither in mIndex nor "
 792              "in mPendingUpdates!"));
 793         NS_WARNING(
 794             ("CacheIndex::InitEntry() - Entry was found neither in "
 795              "mIndex nor in mPendingUpdates!"));
 796         return NS_ERROR_UNEXPECTED;
 797       }
 798
 799       if (updated) {
 800         MOZ_ASSERT(updated->IsFresh());
 801
 802         if (IsCollision(updated, aOriginAttrsHash, aAnonymous)) {
 803           index->mIndexNeedsUpdate = true;
 804           reinitEntry = true;
 805         } else {
 806           if (updated->IsInitialized()) {
 807             return NS_OK;
 808           }
 809         }
 810       } else {
 811         MOZ_ASSERT(entry->IsFresh());
 812
 813         if (IsCollision(entry, aOriginAttrsHash, aAnonymous)) {
 814           index->mIndexNeedsUpdate = true;
 815           reinitEntry = true;
 816         } else {
 817           if (entry->IsInitialized()) {
 818             return NS_OK;
 819           }
 820         }
 821
 822         // make a copy of a read-only entry
 823         updated = index->mPendingUpdates.PutEntry(*aHash);
 824         *updated = *entry;
 825       }
 826     }
 827
 828     if (reinitEntry) {
 829       // There is a collision and we are going to rewrite this entry. Initialize
 830       // it as a new entry.
 831       if (updated) {
 832         updated->InitNew();
 833         updated->MarkFresh();
 834       } else {
 835         entry->InitNew();
 836         entry->MarkFresh();
 837       }
 838     }
 839
 840     if (updated) {
 841       updated->Init(aOriginAttrsHash, aAnonymous, aPinned);
 842       updated->MarkDirty();
 843     } else {
 844       entry->Init(aOriginAttrsHash, aAnonymous, aPinned);
 845       entry->MarkDirty();
 846     }
 847   }
 848
 849   index->StartUpdatingIndexIfNeeded(lock);
 850   index->WriteIndexToDiskIfNeeded(lock);
 851
 852   return NS_OK;
 853 }
 854
 855 // static
 856 nsresult CacheIndex::RemoveEntry(const SHA1Sum::Hash* aHash) {
 857   LOG(("CacheIndex::RemoveEntry() [hash=%08x%08x%08x%08x%08x]",
 858        LOGSHA1(aHash)));
 859
 860   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
 861
 862   StaticMutexAutoLock lock(sLock);
 863
 864   RefPtr<CacheIndex> index = gInstance;
 865
 866   if (!index) {
 867     return NS_ERROR_NOT_INITIALIZED;
 868   }
 869
 870   if (!index->IsIndexUsable()) {
 871     return NS_ERROR_NOT_AVAILABLE;
 872   }
 873
 874   {
 875     CacheIndexEntryAutoManage entryMng(aHash, index, lock);
 876
 877     CacheIndexEntry* entry = index->mIndex.GetEntry(*aHash);
 878     bool entryRemoved = entry && entry->IsRemoved();
 879
 880     if (index->mState == READY || index->mState == UPDATING ||
 881         index->mState == BUILDING) {
 882       MOZ_ASSERT(index->mPendingUpdates.Count() == 0);
 883
 884       if (!entry || entryRemoved) {
 885         if (entryRemoved && entry->IsFresh()) {
 886           // This could happen only if somebody copies files to the entries
 887           // directory while FF is running.
 888           LOG(
 889               ("CacheIndex::RemoveEntry() - Cache file was added outside FF "
 890                "process! Update is needed."));
 891           index->mIndexNeedsUpdate = true;
 892         } else if (index->mState == READY ||
 893                    (entryRemoved && !entry->IsFresh())) {
 894           // Removed non-fresh entries can be present as a result of
 895           // MergeJournal()
 896           LOG(
 897               ("CacheIndex::RemoveEntry() - Didn't find entry that should exist"
 898                ", update is needed"));
 899           index->mIndexNeedsUpdate = true;
 900         }
 901       } else {
 902         if (entry) {
 903           if (!entry->IsDirty() && entry->IsFileEmpty()) {
 904             index->mIndex.RemoveEntry(entry);
 905             entry = nullptr;
 906           } else {
 907             entry->MarkRemoved();
 908             entry->MarkDirty();
 909             entry->MarkFresh();
 910           }
 911         }
 912       }
 913     } else {  // WRITING, READING
 914       CacheIndexEntryUpdate* updated = index->mPendingUpdates.GetEntry(*aHash);
 915       bool updatedRemoved = updated && updated->IsRemoved();
 916
 917       if (updatedRemoved || (!updated && entryRemoved && entry->IsFresh())) {
 918         // Fresh information about missing entry found. This could happen only
 919         // if somebody copies files to the entries directory while FF is
 920         // running.
 921         LOG(
 922             ("CacheIndex::RemoveEntry() - Cache file was added outside FF "
 923              "process! Update is needed."));
 924         index->mIndexNeedsUpdate = true;
 925       } else if (!updated && (!entry || entryRemoved)) {
 926         if (index->mState == WRITING) {
 927           LOG(
 928               ("CacheIndex::RemoveEntry() - Didn't find entry that should exist"
 929                ", update is needed"));
 930           index->mIndexNeedsUpdate = true;
 931         }
 932         // Ignore if state is READING since the index information is partial
 933       }
 934
 935       if (!updated) {
 936         updated = index->mPendingUpdates.PutEntry(*aHash);
 937         updated->InitNew();
 938       }
 939
 940       updated->MarkRemoved();
 941       updated->MarkDirty();
 942       updated->MarkFresh();
 943     }
 944   }
 945   index->StartUpdatingIndexIfNeeded(lock);
 946   index->WriteIndexToDiskIfNeeded(lock);
 947
 948   return NS_OK;
 949 }
 950
 951 // static
 952 nsresult CacheIndex::UpdateEntry(const SHA1Sum::Hash* aHash,
 953                                  const uint32_t* aFrecency,
 954                                  const bool* aHasAltData,
 955                                  const uint16_t* aOnStartTime,
 956                                  const uint16_t* aOnStopTime,
 957                                  const uint8_t* aContentType,
 958                                  const uint32_t* aSize) {
 959   LOG(
 960       ("CacheIndex::UpdateEntry() [hash=%08x%08x%08x%08x%08x, "
 961        "frecency=%s, hasAltData=%s, onStartTime=%s, onStopTime=%s, "
 962        "contentType=%s, size=%s]",
 963        LOGSHA1(aHash), aFrecency ? nsPrintfCString("%u", *aFrecency).get() : "",
 964        aHasAltData ? (*aHasAltData ? "true" : "false") : "",
 965        aOnStartTime ? nsPrintfCString("%u", *aOnStartTime).get() : "",
 966        aOnStopTime ? nsPrintfCString("%u", *aOnStopTime).get() : "",
 967        aContentType ? nsPrintfCString("%u", *aContentType).get() : "",
 968        aSize ? nsPrintfCString("%u", *aSize).get() : ""));
 969
 970   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
 971
 972   StaticMutexAutoLock lock(sLock);
 973
 974   RefPtr<CacheIndex> index = gInstance;
 975
 976   if (!index) {
 977     return NS_ERROR_NOT_INITIALIZED;
 978   }
 979
 980   if (!index->IsIndexUsable()) {
 981     return NS_ERROR_NOT_AVAILABLE;
 982   }
 983
 984   {
 985     CacheIndexEntryAutoManage entryMng(aHash, index, lock);
 986
 987     CacheIndexEntry* entry = index->mIndex.GetEntry(*aHash);
 988
 989     if (entry && entry->IsRemoved()) {
 990       entry = nullptr;
 991     }
 992
 993     if (index->mState == READY || index->mState == UPDATING ||
 994         index->mState == BUILDING) {
 995       MOZ_ASSERT(index->mPendingUpdates.Count() == 0);
 996       MOZ_ASSERT(entry);
 997
 998       if (!entry) {
 999         LOG(("CacheIndex::UpdateEntry() - Entry was not found in mIndex!"));
1000         NS_WARNING(
1001             ("CacheIndex::UpdateEntry() - Entry was not found in mIndex!"));
1002         return NS_ERROR_UNEXPECTED;
1003       }
1004
1005       if (!HasEntryChanged(entry, aFrecency, aHasAltData, aOnStartTime,
1006                            aOnStopTime, aContentType, aSize)) {
1007         return NS_OK;
1008       }
1009
1010       MOZ_ASSERT(entry->IsFresh());
1011       MOZ_ASSERT(entry->IsInitialized());
1012       entry->MarkDirty();
1013
1014       if (aFrecency) {
1015         entry->SetFrecency(*aFrecency);
1016       }
1017
1018       if (aHasAltData) {
1019         entry->SetHasAltData(*aHasAltData);
1020       }
1021
1022       if (aOnStartTime) {
1023         entry->SetOnStartTime(*aOnStartTime);
1024       }
1025
1026       if (aOnStopTime) {
1027         entry->SetOnStopTime(*aOnStopTime);
1028       }
1029
1030       if (aContentType) {
1031         entry->SetContentType(*aContentType);
1032       }
1033
1034       if (aSize) {
1035         entry->SetFileSize(*aSize);
1036       }
1037     } else {
1038       CacheIndexEntryUpdate* updated = index->mPendingUpdates.GetEntry(*aHash);
1039       DebugOnly<bool> removed = updated && updated->IsRemoved();
1040
1041       MOZ_ASSERT(updated || !removed);
1042       MOZ_ASSERT(updated || entry);
1043
1044       if (!updated) {
1045         if (!entry) {
1046           LOG(
1047               ("CacheIndex::UpdateEntry() - Entry was found neither in mIndex "
1048                "nor in mPendingUpdates!"));
1049           NS_WARNING(
1050               ("CacheIndex::UpdateEntry() - Entry was found neither in "
1051                "mIndex nor in mPendingUpdates!"));
1052           return NS_ERROR_UNEXPECTED;
1053         }
1054
1055         // make a copy of a read-only entry
1056         updated = index->mPendingUpdates.PutEntry(*aHash);
1057         *updated = *entry;
1058       }
1059
1060       MOZ_ASSERT(updated->IsFresh());
1061       MOZ_ASSERT(updated->IsInitialized());
1062       updated->MarkDirty();
1063
1064       if (aFrecency) {
1065         updated->SetFrecency(*aFrecency);
1066       }
1067
1068       if (aHasAltData) {
1069         updated->SetHasAltData(*aHasAltData);
1070       }
1071
1072       if (aOnStartTime) {
1073         updated->SetOnStartTime(*aOnStartTime);
1074       }
1075
1076       if (aOnStopTime) {
1077         updated->SetOnStopTime(*aOnStopTime);
1078       }
1079
1080       if (aContentType) {
1081         updated->SetContentType(*aContentType);
1082       }
1083
1084       if (aSize) {
1085         updated->SetFileSize(*aSize);
1086       }
1087     }
1088   }
1089
1090   index->WriteIndexToDiskIfNeeded(lock);
1091
1092   return NS_OK;
1093 }
1094
1095 // static
1096 nsresult CacheIndex::RemoveAll() {
1097   LOG(("CacheIndex::RemoveAll()"));
1098
1099   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
1100
1101   nsCOMPtr<nsIFile> file;
1102
1103   {
1104     StaticMutexAutoLock lock(sLock);
1105
1106     RefPtr<CacheIndex> index = gInstance;
1107
1108     if (!index) {
1109       return NS_ERROR_NOT_INITIALIZED;
1110     }
1111
1112     MOZ_ASSERT(!index->mRemovingAll);
1113
1114     if (!index->IsIndexUsable()) {
1115       return NS_ERROR_NOT_AVAILABLE;
1116     }
1117
1118     AutoRestore<bool> saveRemovingAll(index->mRemovingAll);
1119     index->mRemovingAll = true;
1120
1121     // Doom index and journal handles but don't null them out since this will be
1122     // done in FinishWrite/FinishRead methods.
1123     if (index->mIndexHandle) {
1124       CacheFileIOManager::DoomFile(index->mIndexHandle, nullptr);
1125     } else {
1126       // We don't have a handle to index file, so get the file here, but delete
1127       // it outside the lock. Ignore the result since this is not fatal.
1128       index->GetFile(nsLiteralCString(INDEX_NAME), getter_AddRefs(file));
1129     }
1130
1131     if (index->mJournalHandle) {
1132       CacheFileIOManager::DoomFile(index->mJournalHandle, nullptr);
1133     }
1134
1135     switch (index->mState) {
1136       case WRITING:
1137         index->FinishWrite(false, lock);
1138         break;
1139       case READY:
1140         // nothing to do
1141         break;
1142       case READING:
1143         index->FinishRead(false, lock);
1144         break;
1145       case BUILDING:
1146       case UPDATING:
1147         index->FinishUpdate(false, lock);
1148         break;
1149       default:
1150         MOZ_ASSERT(false, "Unexpected state!");
1151     }
1152
1153     // We should end up in READY state
1154     MOZ_ASSERT(index->mState == READY);
1155
1156     // There should not be any handle
1157     MOZ_ASSERT(!index->mIndexHandle);
1158     MOZ_ASSERT(!index->mJournalHandle);
1159
1160     index->mIndexOnDiskIsValid = false;
1161     index->mIndexNeedsUpdate = false;
1162
1163     index->mIndexStats.Clear();
1164     index->mFrecencyArray.Clear(lock);
1165     index->mIndex.Clear();
1166
1167     for (uint32_t i = 0; i < index->mIterators.Length();) {
1168       nsresult rv = index->mIterators[i]->CloseInternal(NS_ERROR_NOT_AVAILABLE);
1169       if (NS_FAILED(rv)) {
1170         // CacheIndexIterator::CloseInternal() removes itself from mIterators
1171         // iff it returns success.
1172         LOG(
1173             ("CacheIndex::RemoveAll() - Failed to remove iterator %p. "
1174              "[rv=0x%08" PRIx32 "]",
1175              index->mIterators[i], static_cast<uint32_t>(rv)));
1176         i++;
1177       }
1178     }
1179   }
1180
1181   if (file) {
1182     // Ignore the result. The file might not exist and the failure is not fatal.
1183     file->Remove(false);
1184   }
1185
1186   return NS_OK;
1187 }
1188
1189 // static
1190 nsresult CacheIndex::HasEntry(
1191     const nsACString& aKey, EntryStatus* _retval,
1192     const std::function<void(const CacheIndexEntry*)>& aCB) {
1193   LOG(("CacheIndex::HasEntry() [key=%s]", PromiseFlatCString(aKey).get()));
1194
1195   SHA1Sum sum;
1196   SHA1Sum::Hash hash;
1197   sum.update(aKey.BeginReading(), aKey.Length());
1198   sum.finish(hash);
1199
1200   return HasEntry(hash, _retval, aCB);
1201 }
1202
1203 // static
1204 nsresult CacheIndex::HasEntry(
1205     const SHA1Sum::Hash& hash, EntryStatus* _retval,
1206     const std::function<void(const CacheIndexEntry*)>& aCB) {
1207   StaticMutexAutoLock lock(sLock);
1208
1209   RefPtr<CacheIndex> index = gInstance;
1210
1211   if (!index) {
1212     return NS_ERROR_NOT_INITIALIZED;
1213   }
1214
1215   if (!index->IsIndexUsable()) {
1216     return NS_ERROR_NOT_AVAILABLE;
1217   }
1218
1219   const CacheIndexEntry* entry = nullptr;
1220
1221   switch (index->mState) {
1222     case READING:
1223     case WRITING:
1224       entry = index->mPendingUpdates.GetEntry(hash);
1225       [[fallthrough]];
1226     case BUILDING:
1227     case UPDATING:
1228     case READY:
1229       if (!entry) {
1230         entry = index->mIndex.GetEntry(hash);
1231       }
1232       break;
1233     case INITIAL:
1234     case SHUTDOWN:
1235       MOZ_ASSERT(false, "Unexpected state!");
1236   }
1237
1238   if (!entry) {
1239     if (index->mState == READY || index->mState == WRITING) {
1240       *_retval = DOES_NOT_EXIST;
1241     } else {
1242       *_retval = DO_NOT_KNOW;
1243     }
1244   } else {
1245     if (entry->IsRemoved()) {
1246       if (entry->IsFresh()) {
1247         *_retval = DOES_NOT_EXIST;
1248       } else {
1249         *_retval = DO_NOT_KNOW;
1250       }
1251     } else {
1252       *_retval = EXISTS;
1253       if (aCB) {
1254         aCB(entry);
1255       }
1256     }
1257   }
1258
1259   LOG(("CacheIndex::HasEntry() - result is %u", *_retval));
1260   return NS_OK;
1261 }
1262
1263 // static
1264 nsresult CacheIndex::GetEntryForEviction(bool aIgnoreEmptyEntries,
1265                                          SHA1Sum::Hash* aHash, uint32_t* aCnt) {
1266   LOG(("CacheIndex::GetEntryForEviction()"));
1267
1268   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
1269
1270   StaticMutexAutoLock lock(sLock);
1271
1272   RefPtr<CacheIndex> index = gInstance;
1273
1274   if (!index) return NS_ERROR_NOT_INITIALIZED;
1275
1276   if (!index->IsIndexUsable()) {
1277     return NS_ERROR_NOT_AVAILABLE;
1278   }
1279
1280   if (index->mIndexStats.Size() == 0) {
1281     return NS_ERROR_NOT_AVAILABLE;
1282   }
1283
1284   int32_t mediaUsage =
1285       round(static_cast<double>(index->mIndexStats.SizeByType(
1286                 nsICacheEntry::CONTENT_TYPE_MEDIA)) *
1287             100.0 / static_cast<double>(index->mIndexStats.Size()));
1288   int32_t mediaUsageLimit =
1289       StaticPrefs::browser_cache_disk_content_type_media_limit();
1290   bool evictMedia = false;
1291   if (mediaUsage > mediaUsageLimit) {
1292     LOG(
1293         ("CacheIndex::GetEntryForEviction() - media content type is over the "
1294          "limit [mediaUsage=%d, mediaUsageLimit=%d]",
1295          mediaUsage, mediaUsageLimit));
1296     evictMedia = true;
1297   }
1298
1299   SHA1Sum::Hash hash;
1300   CacheIndexRecord* foundRecord = nullptr;
1301   uint32_t skipped = 0;
1302
1303   // find first non-forced valid and unpinned entry with the lowest frecency
1304   index->mFrecencyArray.SortIfNeeded(lock);
1305
1306   for (auto iter = index->mFrecencyArray.Iter(); !iter.Done(); iter.Next()) {
1307     CacheIndexRecord* rec = iter.Get()->Get();
1308
1309     memcpy(&hash, rec->mHash, sizeof(SHA1Sum::Hash));
1310
1311     ++skipped;
1312
1313     if (evictMedia && CacheIndexEntry::GetContentType(rec) !=
1314                           nsICacheEntry::CONTENT_TYPE_MEDIA) {
1315       continue;
1316     }
1317
1318     if (IsForcedValidEntry(&hash)) {
1319       continue;
1320     }
1321
1322     if (CacheIndexEntry::IsPinned(rec)) {
1323       continue;
1324     }
1325
1326     if (aIgnoreEmptyEntries && !CacheIndexEntry::GetFileSize(*rec)) {
1327       continue;
1328     }
1329
1330     --skipped;
1331     foundRecord = rec;
1332     break;
1333   }
1334
1335   if (!foundRecord) return NS_ERROR_NOT_AVAILABLE;
1336
1337   *aCnt = skipped;
1338
1339   LOG(
1340       ("CacheIndex::GetEntryForEviction() - returning entry "
1341        "[hash=%08x%08x%08x%08x%08x, cnt=%u, frecency=%u, contentType=%u]",
1342        LOGSHA1(&hash), *aCnt, foundRecord->mFrecency,
1343        CacheIndexEntry::GetContentType(foundRecord)));
1344
1345   memcpy(aHash, &hash, sizeof(SHA1Sum::Hash));
1346
1347   return NS_OK;
1348 }
1349
1350 // static
1351 bool CacheIndex::IsForcedValidEntry(const SHA1Sum::Hash* aHash) {
1352   RefPtr<CacheFileHandle> handle;
1353
1354   CacheFileIOManager::gInstance->mHandles.GetHandle(aHash,
1355                                                     getter_AddRefs(handle));
1356
1357   if (!handle) return false;
1358
1359   nsCString hashKey = handle->Key();
1360   return CacheStorageService::Self()->IsForcedValidEntry(hashKey);
1361 }
1362
1363 // static
1364 nsresult CacheIndex::GetCacheSize(uint32_t* _retval) {
1365   LOG(("CacheIndex::GetCacheSize()"));
1366
1367   StaticMutexAutoLock lock(sLock);
1368
1369   RefPtr<CacheIndex> index = gInstance;
1370
1371   if (!index) return NS_ERROR_NOT_INITIALIZED;
1372
1373   if (!index->IsIndexUsable()) {
1374     return NS_ERROR_NOT_AVAILABLE;
1375   }
1376
1377   *_retval = index->mIndexStats.Size();
1378   LOG(("CacheIndex::GetCacheSize() - returning %u", *_retval));
1379   return NS_OK;
1380 }
1381
1382 // static
1383 nsresult CacheIndex::GetEntryFileCount(uint32_t* _retval) {
1384   LOG(("CacheIndex::GetEntryFileCount()"));
1385
1386   StaticMutexAutoLock lock(sLock);
1387
1388   RefPtr<CacheIndex> index = gInstance;
1389
1390   if (!index) {
1391     return NS_ERROR_NOT_INITIALIZED;
1392   }
1393
1394   if (!index->IsIndexUsable()) {
1395     return NS_ERROR_NOT_AVAILABLE;
1396   }
1397
1398   *_retval = index->mIndexStats.ActiveEntriesCount();
1399   LOG(("CacheIndex::GetEntryFileCount() - returning %u", *_retval));
1400   return NS_OK;
1401 }
1402
1403 // static
1404 nsresult CacheIndex::GetCacheStats(nsILoadContextInfo* aInfo, uint32_t* aSize,
1405                                    uint32_t* aCount) {
1406   LOG(("CacheIndex::GetCacheStats() [info=%p]", aInfo));
1407
1408   StaticMutexAutoLock lock(sLock);
1409
1410   RefPtr<CacheIndex> index = gInstance;
1411
1412   if (!index) {
1413     return NS_ERROR_NOT_INITIALIZED;
1414   }
1415
1416   if (!index->IsIndexUsable()) {
1417     return NS_ERROR_NOT_AVAILABLE;
1418   }
1419
1420   *aSize = 0;
1421   *aCount = 0;
1422
1423   for (auto iter = index->mFrecencyArray.Iter(); !iter.Done(); iter.Next()) {
1424     if (aInfo &&
1425         !CacheIndexEntry::RecordMatchesLoadContextInfo(iter.Get(), aInfo)) {
1426       continue;
1427     }
1428
1429     *aSize += CacheIndexEntry::GetFileSize(*(iter.Get()->Get()));
1430     ++*aCount;
1431   }
1432
1433   return NS_OK;
1434 }
1435
1436 // static
1437 nsresult CacheIndex::AsyncGetDiskConsumption(
1438     nsICacheStorageConsumptionObserver* aObserver) {
1439   LOG(("CacheIndex::AsyncGetDiskConsumption()"));
1440
1441   StaticMutexAutoLock lock(sLock);
1442
1443   RefPtr<CacheIndex> index = gInstance;
1444
1445   if (!index) {
1446     return NS_ERROR_NOT_INITIALIZED;
1447   }
1448
1449   if (!index->IsIndexUsable()) {
1450     return NS_ERROR_NOT_AVAILABLE;
1451   }
1452
1453   RefPtr<DiskConsumptionObserver> observer =
1454       DiskConsumptionObserver::Init(aObserver);
1455
1456   NS_ENSURE_ARG(observer);
1457
1458   if ((index->mState == READY || index->mState == WRITING) &&
1459       !index->mAsyncGetDiskConsumptionBlocked) {
1460     LOG(("CacheIndex::AsyncGetDiskConsumption - calling immediately"));
1461     // Safe to call the callback under the lock,
1462     // we always post to the main thread.
1463     observer->OnDiskConsumption(index->mIndexStats.Size() << 10);
1464     return NS_OK;
1465   }
1466
1467   LOG(("CacheIndex::AsyncGetDiskConsumption - remembering callback"));
1468   // Will be called when the index get to the READY state.
1469   index->mDiskConsumptionObservers.AppendElement(observer);
1470
1471   // Move forward with index re/building if it is pending
1472   RefPtr<CacheIOThread> ioThread = CacheFileIOManager::IOThread();
1473   if (ioThread) {
1474     ioThread->Dispatch(
1475         NS_NewRunnableFunction("net::CacheIndex::AsyncGetDiskConsumption",
1476                                []() -> void {
1477                                  StaticMutexAutoLock lock(sLock);
1478
1479                                  RefPtr<CacheIndex> index = gInstance;
1480                                  if (index && index->mUpdateTimer) {
1481                                    index->mUpdateTimer->Cancel();
1482                                    index->DelayedUpdateLocked(lock);
1483                                  }
1484                                }),
1485         CacheIOThread::INDEX);
1486   }
1487
1488   return NS_OK;
1489 }
1490
1491 // static
1492 nsresult CacheIndex::GetIterator(nsILoadContextInfo* aInfo, bool aAddNew,
1493                                  CacheIndexIterator** _retval) {
1494   LOG(("CacheIndex::GetIterator() [info=%p, addNew=%d]", aInfo, aAddNew));
1495
1496   StaticMutexAutoLock lock(sLock);
1497
1498   RefPtr<CacheIndex> index = gInstance;
1499
1500   if (!index) {
1501     return NS_ERROR_NOT_INITIALIZED;
1502   }
1503
1504   if (!index->IsIndexUsable()) {
1505     return NS_ERROR_NOT_AVAILABLE;
1506   }
1507
1508   RefPtr<CacheIndexIterator> idxIter;
1509   if (aInfo) {
1510     idxIter = new CacheIndexContextIterator(index, aAddNew, aInfo);
1511   } else {
1512     idxIter = new CacheIndexIterator(index, aAddNew);
1513   }
1514
1515   index->mFrecencyArray.SortIfNeeded(lock);
1516
1517   for (auto iter = index->mFrecencyArray.Iter(); !iter.Done(); iter.Next()) {
1518     idxIter->AddRecord(iter.Get(), lock);
1519   }
1520
1521   index->mIterators.AppendElement(idxIter);
1522   idxIter.swap(*_retval);
1523   return NS_OK;
1524 }
1525
1526 // static
1527 nsresult CacheIndex::IsUpToDate(bool* _retval) {
1528   LOG(("CacheIndex::IsUpToDate()"));
1529
1530   StaticMutexAutoLock lock(sLock);
1531
1532   RefPtr<CacheIndex> index = gInstance;
1533
1534   if (!index) {
1535     return NS_ERROR_NOT_INITIALIZED;
1536   }
1537
1538   if (!index->IsIndexUsable()) {
1539     return NS_ERROR_NOT_AVAILABLE;
1540   }
1541
1542   *_retval = (index->mState == READY || index->mState == WRITING) &&
1543              !index->mIndexNeedsUpdate && !index->mShuttingDown;
1544
1545   LOG(("CacheIndex::IsUpToDate() - returning %d", *_retval));
1546   return NS_OK;
1547 }
1548
1549 bool CacheIndex::IsIndexUsable() {
1550   MOZ_ASSERT(mState != INITIAL);
1551
1552   switch (mState) {
1553     case INITIAL:
1554     case SHUTDOWN:
1555       return false;
1556
1557     case READING:
1558     case WRITING:
1559     case BUILDING:
1560     case UPDATING:
1561     case READY:
1562       break;
1563   }
1564
1565   return true;
1566 }
1567
1568 // static
1569 bool CacheIndex::IsCollision(CacheIndexEntry* aEntry,
1570                              OriginAttrsHash aOriginAttrsHash,
1571                              bool aAnonymous) {
1572   if (!aEntry->IsInitialized()) {
1573     return false;
1574   }
1575
1576   if (aEntry->Anonymous() != aAnonymous ||
1577       aEntry->OriginAttrsHash() != aOriginAttrsHash) {
1578     LOG(
1579         ("CacheIndex::IsCollision() - Collision detected for entry hash=%08x"
1580          "%08x%08x%08x%08x, expected values: originAttrsHash=%" PRIu64 ", "
1581          "anonymous=%d; actual values: originAttrsHash=%" PRIu64
1582          ", anonymous=%d]",
1583          LOGSHA1(aEntry->Hash()), aOriginAttrsHash, aAnonymous,
1584          aEntry->OriginAttrsHash(), aEntry->Anonymous()));
1585     return true;
1586   }
1587
1588   return false;
1589 }
1590
1591 // static
1592 bool CacheIndex::HasEntryChanged(
1593     CacheIndexEntry* aEntry, const uint32_t* aFrecency, const bool* aHasAltData,
1594     const uint16_t* aOnStartTime, const uint16_t* aOnStopTime,
1595     const uint8_t* aContentType, const uint32_t* aSize) {
1596   if (aFrecency && *aFrecency != aEntry->GetFrecency()) {
1597     return true;
1598   }
1599
1600   if (aHasAltData && *aHasAltData != aEntry->GetHasAltData()) {
1601     return true;
1602   }
1603
1604   if (aOnStartTime && *aOnStartTime != aEntry->GetOnStartTime()) {
1605     return true;
1606   }
1607
1608   if (aOnStopTime && *aOnStopTime != aEntry->GetOnStopTime()) {
1609     return true;
1610   }
1611
1612   if (aContentType && *aContentType != aEntry->GetContentType()) {
1613     return true;
1614   }
1615
1616   if (aSize &&
1617       (*aSize & CacheIndexEntry::kFileSizeMask) != aEntry->GetFileSize()) {
1618     return true;
1619   }
1620
1621   return false;
1622 }
1623
1624 void CacheIndex::ProcessPendingOperations(
1625     const StaticMutexAutoLock& aProofOfLock) {
1626   sLock.AssertCurrentThreadOwns();
1627   LOG(("CacheIndex::ProcessPendingOperations()"));
1628
1629   for (auto iter = mPendingUpdates.Iter(); !iter.Done(); iter.Next()) {
1630     CacheIndexEntryUpdate* update = iter.Get();
1631
1632     LOG(("CacheIndex::ProcessPendingOperations() [hash=%08x%08x%08x%08x%08x]",
1633          LOGSHA1(update->Hash())));
1634
1635     MOZ_ASSERT(update->IsFresh());
1636
1637     CacheIndexEntry* entry = mIndex.GetEntry(*update->Hash());
1638     {
1639       CacheIndexEntryAutoManage emng(update->Hash(), this, aProofOfLock);
1640       emng.DoNotSearchInUpdates();
1641
1642       if (update->IsRemoved()) {
1643         if (entry) {
1644           if (entry->IsRemoved()) {
1645             MOZ_ASSERT(entry->IsFresh());
1646             MOZ_ASSERT(entry->IsDirty());
1647           } else if (!entry->IsDirty() && entry->IsFileEmpty()) {
1648             // Entries with empty file are not stored in index on disk. Just
1649             // remove the entry, but only in case the entry is not dirty, i.e.
1650             // the entry file was empty when we wrote the index.
1651             mIndex.RemoveEntry(entry);
1652             entry = nullptr;
1653           } else {
1654             entry->MarkRemoved();
1655             entry->MarkDirty();
1656             entry->MarkFresh();
1657           }
1658         }
1659       } else if (entry) {
1660         // Some information in mIndex can be newer than in mPendingUpdates (see
1661         // bug 1074832). This will copy just those values that were really
1662         // updated.
1663         update->ApplyUpdate(entry);
1664       } else {
1665         // There is no entry in mIndex, copy all information from
1666         // mPendingUpdates to mIndex.
1667         entry = mIndex.PutEntry(*update->Hash());
1668         *entry = *update;
1669       }
1670     }
1671     iter.Remove();
1672   }
1673
1674   MOZ_ASSERT(mPendingUpdates.Count() == 0);
1675
1676   EnsureCorrectStats();
1677 }
1678
1679 bool CacheIndex::WriteIndexToDiskIfNeeded(
1680     const StaticMutexAutoLock& aProofOfLock) {
1681   sLock.AssertCurrentThreadOwns();
1682   if (mState != READY || mShuttingDown || mRWPending) {
1683     return false;
1684   }
1685
1686   if (!mLastDumpTime.IsNull() &&
1687       (TimeStamp::NowLoRes() - mLastDumpTime).ToMilliseconds() <
1688           kMinDumpInterval) {
1689     return false;
1690   }
1691
1692   if (mIndexStats.Dirty() < kMinUnwrittenChanges) {
1693     return false;
1694   }
1695
1696   WriteIndexToDisk(aProofOfLock);
1697   return true;
1698 }
1699
1700 void CacheIndex::WriteIndexToDisk(const StaticMutexAutoLock& aProofOfLock) {
1701   sLock.AssertCurrentThreadOwns();
1702   LOG(("CacheIndex::WriteIndexToDisk()"));
1703   mIndexStats.Log();
1704
1705   nsresult rv;
1706
1707   MOZ_ASSERT(mState == READY);
1708   MOZ_ASSERT(!mRWBuf);
1709   MOZ_ASSERT(!mRWHash);
1710   MOZ_ASSERT(!mRWPending);
1711
1712   ChangeState(WRITING, aProofOfLock);
1713
1714   mProcessEntries = mIndexStats.ActiveEntriesCount();
1715
1716   mIndexFileOpener = new FileOpenHelper(this);
1717   rv = CacheFileIOManager::OpenFile(
1718       nsLiteralCString(TEMP_INDEX_NAME),
1719       CacheFileIOManager::SPECIAL_FILE | CacheFileIOManager::CREATE,
1720       mIndexFileOpener);
1721   if (NS_FAILED(rv)) {
1722     LOG(("CacheIndex::WriteIndexToDisk() - Can't open file [rv=0x%08" PRIx32
1723          "]",
1724          static_cast<uint32_t>(rv)));
1725     FinishWrite(false, aProofOfLock);
1726     return;
1727   }
1728
1729   // Write index header to a buffer, it will be written to disk together with
1730   // records in WriteRecords() once we open the file successfully.
1731   AllocBuffer();
1732   mRWHash = new CacheHash();
1733
1734   mRWBufPos = 0;
1735   // index version
1736   NetworkEndian::writeUint32(mRWBuf + mRWBufPos, kIndexVersion);
1737   mRWBufPos += sizeof(uint32_t);
1738   // timestamp
1739   NetworkEndian::writeUint32(mRWBuf + mRWBufPos,
1740                              static_cast<uint32_t>(PR_Now() / PR_USEC_PER_SEC));
1741   mRWBufPos += sizeof(uint32_t);
1742   // dirty flag
1743   NetworkEndian::writeUint32(mRWBuf + mRWBufPos, 1);
1744   mRWBufPos += sizeof(uint32_t);
1745   // amount of data written to the cache
1746   NetworkEndian::writeUint32(mRWBuf + mRWBufPos,
1747                              static_cast<uint32_t>(mTotalBytesWritten >> 10));
1748   mRWBufPos += sizeof(uint32_t);
1749
1750   mSkipEntries = 0;
1751 }
1752
1753 void CacheIndex::WriteRecords(const StaticMutexAutoLock& aProofOfLock) {
1754   sLock.AssertCurrentThreadOwns();
1755   LOG(("CacheIndex::WriteRecords()"));
1756
1757   nsresult rv;
1758
1759   MOZ_ASSERT(mState == WRITING);
1760   MOZ_ASSERT(!mRWPending);
1761
1762   int64_t fileOffset;
1763
1764   if (mSkipEntries) {
1765     MOZ_ASSERT(mRWBufPos == 0);
1766     fileOffset = sizeof(CacheIndexHeader);
1767     fileOffset += sizeof(CacheIndexRecord) * mSkipEntries;
1768   } else {
1769     MOZ_ASSERT(mRWBufPos == sizeof(CacheIndexHeader));
1770     fileOffset = 0;
1771   }
1772   uint32_t hashOffset = mRWBufPos;
1773
1774   char* buf = mRWBuf + mRWBufPos;
1775   uint32_t skip = mSkipEntries;
1776   uint32_t processMax = (mRWBufSize - mRWBufPos) / sizeof(CacheIndexRecord);
1777   MOZ_ASSERT(processMax != 0 ||
1778              mProcessEntries ==
1779                  0);  // TODO make sure we can write an empty index
1780   uint32_t processed = 0;
1781 #ifdef DEBUG
1782   bool hasMore = false;
1783 #endif
1784   for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) {
1785     CacheIndexEntry* entry = iter.Get();
1786     if (entry->IsRemoved() || !entry->IsInitialized() || entry->IsFileEmpty()) {
1787       continue;
1788     }
1789
1790     if (skip) {
1791       skip--;
1792       continue;
1793     }
1794
1795     if (processed == processMax) {
1796 #ifdef DEBUG
1797       hasMore = true;
1798 #endif
1799       break;
1800     }
1801
1802     entry->WriteToBuf(buf);
1803     buf += sizeof(CacheIndexRecord);
1804     processed++;
1805   }
1806
1807   MOZ_ASSERT(mRWBufPos != static_cast<uint32_t>(buf - mRWBuf) ||
1808              mProcessEntries == 0);
1809   mRWBufPos = buf - mRWBuf;
1810   mSkipEntries += processed;
1811   MOZ_ASSERT(mSkipEntries <= mProcessEntries);
1812
1813   mRWHash->Update(mRWBuf + hashOffset, mRWBufPos - hashOffset);
1814
1815   if (mSkipEntries == mProcessEntries) {
1816     MOZ_ASSERT(!hasMore);
1817
1818     // We've processed all records
1819     if (mRWBufPos + sizeof(CacheHash::Hash32_t) > mRWBufSize) {
1820       // realloc buffer to spare another write cycle
1821       mRWBufSize = mRWBufPos + sizeof(CacheHash::Hash32_t);
1822       mRWBuf = static_cast<char*>(moz_xrealloc(mRWBuf, mRWBufSize));
1823     }
1824
1825     NetworkEndian::writeUint32(mRWBuf + mRWBufPos, mRWHash->GetHash());
1826     mRWBufPos += sizeof(CacheHash::Hash32_t);
1827   } else {
1828     MOZ_ASSERT(hasMore);
1829   }
1830
1831   rv = CacheFileIOManager::Write(mIndexHandle, fileOffset, mRWBuf, mRWBufPos,
1832                                  mSkipEntries == mProcessEntries, false, this);
1833   if (NS_FAILED(rv)) {
1834     LOG(
1835         ("CacheIndex::WriteRecords() - CacheFileIOManager::Write() failed "
1836          "synchronously [rv=0x%08" PRIx32 "]",
1837          static_cast<uint32_t>(rv)));
1838     FinishWrite(false, aProofOfLock);
1839   } else {
1840     mRWPending = true;
1841   }
1842
1843   mRWBufPos = 0;
1844 }
1845
1846 void CacheIndex::FinishWrite(bool aSucceeded,
1847                              const StaticMutexAutoLock& aProofOfLock) {
1848   sLock.AssertCurrentThreadOwns();
1849   LOG(("CacheIndex::FinishWrite() [succeeded=%d]", aSucceeded));
1850
1851   MOZ_ASSERT((!aSucceeded && mState == SHUTDOWN) || mState == WRITING);
1852
1853   // If there is write operation pending we must be cancelling writing of the
1854   // index when shutting down or removing the whole index.
1855   MOZ_ASSERT(!mRWPending || (!aSucceeded && (mShuttingDown || mRemovingAll)));
1856
1857   mIndexHandle = nullptr;
1858   mRWHash = nullptr;
1859   ReleaseBuffer();
1860
1861   if (aSucceeded) {
1862     // Opening of the file must not be in progress if writing succeeded.
1863     MOZ_ASSERT(!mIndexFileOpener);
1864
1865     for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) {
1866       CacheIndexEntry* entry = iter.Get();
1867
1868       bool remove = false;
1869       {
1870         CacheIndexEntryAutoManage emng(entry->Hash(), this, aProofOfLock);
1871
1872         if (entry->IsRemoved()) {
1873           emng.DoNotSearchInIndex();
1874           remove = true;
1875         } else if (entry->IsDirty()) {
1876           entry->ClearDirty();
1877         }
1878       }
1879       if (remove) {
1880         iter.Remove();
1881       }
1882     }
1883
1884     mIndexOnDiskIsValid = true;
1885   } else {
1886     if (mIndexFileOpener) {
1887       // If opening of the file is still in progress (e.g. WRITE process was
1888       // canceled by RemoveAll()) then we need to cancel the opener to make sure
1889       // that OnFileOpenedInternal() won't be called.
1890       mIndexFileOpener->Cancel();
1891       mIndexFileOpener = nullptr;
1892     }
1893   }
1894
1895   ProcessPendingOperations(aProofOfLock);
1896   mIndexStats.Log();
1897
1898   if (mState == WRITING) {
1899     ChangeState(READY, aProofOfLock);
1900     mLastDumpTime = TimeStamp::NowLoRes();
1901   }
1902 }
1903
1904 nsresult CacheIndex::GetFile(const nsACString& aName, nsIFile** _retval) {
1905   nsresult rv;
1906
1907   nsCOMPtr<nsIFile> file;
1908   rv = mCacheDirectory->Clone(getter_AddRefs(file));
1909   NS_ENSURE_SUCCESS(rv, rv);
1910
1911   rv = file->AppendNative(aName);
1912   NS_ENSURE_SUCCESS(rv, rv);
1913
1914   file.swap(*_retval);
1915   return NS_OK;
1916 }
1917
1918 void CacheIndex::RemoveFile(const nsACString& aName) {
1919   MOZ_ASSERT(mState == SHUTDOWN);
1920
1921   nsresult rv;
1922
1923   nsCOMPtr<nsIFile> file;
1924   rv = GetFile(aName, getter_AddRefs(file));
1925   NS_ENSURE_SUCCESS_VOID(rv);
1926
1927   rv = file->Remove(false);
1928   if (NS_FAILED(rv) && rv != NS_ERROR_FILE_NOT_FOUND) {
1929     LOG(
1930         ("CacheIndex::RemoveFile() - Cannot remove old entry file from disk "
1931          "[rv=0x%08" PRIx32 ", name=%s]",
1932          static_cast<uint32_t>(rv), PromiseFlatCString(aName).get()));
1933   }
1934 }
1935
1936 void CacheIndex::RemoveAllIndexFiles() {
1937   LOG(("CacheIndex::RemoveAllIndexFiles()"));
1938   RemoveFile(nsLiteralCString(INDEX_NAME));
1939   RemoveJournalAndTempFile();
1940 }
1941
1942 void CacheIndex::RemoveJournalAndTempFile() {
1943   LOG(("CacheIndex::RemoveJournalAndTempFile()"));
1944   RemoveFile(nsLiteralCString(TEMP_INDEX_NAME));
1945   RemoveFile(nsLiteralCString(JOURNAL_NAME));
1946 }
1947
1948 class WriteLogHelper {
1949  public:
1950   explicit WriteLogHelper(PRFileDesc* aFD)
1951       : mFD(aFD), mBufSize(kMaxBufSize), mBufPos(0) {
1952     mHash = new CacheHash();
1953     mBuf = static_cast<char*>(moz_xmalloc(mBufSize));
1954   }
1955
1956   ~WriteLogHelper() { free(mBuf); }
1957
1958   nsresult AddEntry(CacheIndexEntry* aEntry);
1959   nsresult Finish();
1960
1961  private:
1962   nsresult FlushBuffer();
1963
1964   PRFileDesc* mFD;
1965   char* mBuf;
1966   uint32_t mBufSize;
1967   int32_t mBufPos;
1968   RefPtr<CacheHash> mHash;
1969 };
1970
1971 nsresult WriteLogHelper::AddEntry(CacheIndexEntry* aEntry) {
1972   nsresult rv;
1973
1974   if (mBufPos + sizeof(CacheIndexRecord) > mBufSize) {
1975     mHash->Update(mBuf, mBufPos);
1976
1977     rv = FlushBuffer();
1978     NS_ENSURE_SUCCESS(rv, rv);
1979     MOZ_ASSERT(mBufPos + sizeof(CacheIndexRecord) <= mBufSize);
1980   }
1981
1982   aEntry->WriteToBuf(mBuf + mBufPos);
1983   mBufPos += sizeof(CacheIndexRecord);
1984
1985   return NS_OK;
1986 }
1987
1988 nsresult WriteLogHelper::Finish() {
1989   nsresult rv;
1990
1991   mHash->Update(mBuf, mBufPos);
1992   if (mBufPos + sizeof(CacheHash::Hash32_t) > mBufSize) {
1993     rv = FlushBuffer();
1994     NS_ENSURE_SUCCESS(rv, rv);
1995     MOZ_ASSERT(mBufPos + sizeof(CacheHash::Hash32_t) <= mBufSize);
1996   }
1997
1998   NetworkEndian::writeUint32(mBuf + mBufPos, mHash->GetHash());
1999   mBufPos += sizeof(CacheHash::Hash32_t);
2000
2001   rv = FlushBuffer();
2002   NS_ENSURE_SUCCESS(rv, rv);
2003
2004   return NS_OK;
2005 }
2006
2007 nsresult WriteLogHelper::FlushBuffer() {
2008   if (CacheObserver::IsPastShutdownIOLag()) {
2009     LOG(("WriteLogHelper::FlushBuffer() - Interrupting writing journal."));
2010     return NS_ERROR_FAILURE;
2011   }
2012
2013   int32_t bytesWritten = PR_Write(mFD, mBuf, mBufPos);
2014
2015   if (bytesWritten != mBufPos) {
2016     return NS_ERROR_FAILURE;
2017   }
2018
2019   mBufPos = 0;
2020   return NS_OK;
2021 }
2022
2023 nsresult CacheIndex::WriteLogToDisk() {
2024   LOG(("CacheIndex::WriteLogToDisk()"));
2025
2026   nsresult rv;
2027
2028   MOZ_ASSERT(mPendingUpdates.Count() == 0);
2029   MOZ_ASSERT(mState == SHUTDOWN);
2030
2031   if (CacheObserver::IsPastShutdownIOLag()) {
2032     LOG(("CacheIndex::WriteLogToDisk() - Skipping writing journal."));
2033     return NS_ERROR_FAILURE;
2034   }
2035
2036   RemoveFile(nsLiteralCString(TEMP_INDEX_NAME));
2037
2038   nsCOMPtr<nsIFile> indexFile;
2039   rv = GetFile(nsLiteralCString(INDEX_NAME), getter_AddRefs(indexFile));
2040   NS_ENSURE_SUCCESS(rv, rv);
2041
2042   nsCOMPtr<nsIFile> logFile;
2043   rv = GetFile(nsLiteralCString(JOURNAL_NAME), getter_AddRefs(logFile));
2044   NS_ENSURE_SUCCESS(rv, rv);
2045
2046   mIndexStats.Log();
2047
2048   PRFileDesc* fd = nullptr;
2049   rv = logFile->OpenNSPRFileDesc(PR_RDWR | PR_CREATE_FILE | PR_TRUNCATE, 0600,
2050                                  &fd);
2051   NS_ENSURE_SUCCESS(rv, rv);
2052
2053   WriteLogHelper wlh(fd);
2054   for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) {
2055     CacheIndexEntry* entry = iter.Get();
2056     if (entry->IsRemoved() || entry->IsDirty()) {
2057       rv = wlh.AddEntry(entry);
2058       if (NS_WARN_IF(NS_FAILED(rv))) {
2059         return rv;
2060       }
2061     }
2062   }
2063
2064   rv = wlh.Finish();
2065   PR_Close(fd);
2066   NS_ENSURE_SUCCESS(rv, rv);
2067
2068   rv = indexFile->OpenNSPRFileDesc(PR_RDWR, 0600, &fd);
2069   NS_ENSURE_SUCCESS(rv, rv);
2070
2071   // Seek to dirty flag in the index header and clear it.
2072   static_assert(2 * sizeof(uint32_t) == offsetof(CacheIndexHeader, mIsDirty),
2073                 "Unexpected offset of CacheIndexHeader::mIsDirty");
2074   int64_t offset = PR_Seek64(fd, 2 * sizeof(uint32_t), PR_SEEK_SET);
2075   if (offset == -1) {
2076     PR_Close(fd);
2077     return NS_ERROR_FAILURE;
2078   }
2079
2080   uint32_t isDirty = 0;
2081   int32_t bytesWritten = PR_Write(fd, &isDirty, sizeof(isDirty));
2082   PR_Close(fd);
2083   if (bytesWritten != sizeof(isDirty)) {
2084     return NS_ERROR_FAILURE;
2085   }
2086
2087   return NS_OK;
2088 }
2089
2090 void CacheIndex::ReadIndexFromDisk(const StaticMutexAutoLock& aProofOfLock) {
2091   sLock.AssertCurrentThreadOwns();
2092   LOG(("CacheIndex::ReadIndexFromDisk()"));
2093
2094   nsresult rv;
2095
2096   MOZ_ASSERT(mState == INITIAL);
2097
2098   ChangeState(READING, aProofOfLock);
2099
2100   mIndexFileOpener = new FileOpenHelper(this);
2101   rv = CacheFileIOManager::OpenFile(
2102       nsLiteralCString(INDEX_NAME),
2103       CacheFileIOManager::SPECIAL_FILE | CacheFileIOManager::OPEN,
2104       mIndexFileOpener);
2105   if (NS_FAILED(rv)) {
2106     LOG(
2107         ("CacheIndex::ReadIndexFromDisk() - CacheFileIOManager::OpenFile() "
2108          "failed [rv=0x%08" PRIx32 ", file=%s]",
2109          static_cast<uint32_t>(rv), INDEX_NAME));
2110     FinishRead(false, aProofOfLock);
2111     return;
2112   }
2113
2114   mJournalFileOpener = new FileOpenHelper(this);
2115   rv = CacheFileIOManager::OpenFile(
2116       nsLiteralCString(JOURNAL_NAME),
2117       CacheFileIOManager::SPECIAL_FILE | CacheFileIOManager::OPEN,
2118       mJournalFileOpener);
2119   if (NS_FAILED(rv)) {
2120     LOG(
2121         ("CacheIndex::ReadIndexFromDisk() - CacheFileIOManager::OpenFile() "
2122          "failed [rv=0x%08" PRIx32 ", file=%s]",
2123          static_cast<uint32_t>(rv), JOURNAL_NAME));
2124     FinishRead(false, aProofOfLock);
2125   }
2126
2127   mTmpFileOpener = new FileOpenHelper(this);
2128   rv = CacheFileIOManager::OpenFile(
2129       nsLiteralCString(TEMP_INDEX_NAME),
2130       CacheFileIOManager::SPECIAL_FILE | CacheFileIOManager::OPEN,
2131       mTmpFileOpener);
2132   if (NS_FAILED(rv)) {
2133     LOG(
2134         ("CacheIndex::ReadIndexFromDisk() - CacheFileIOManager::OpenFile() "
2135          "failed [rv=0x%08" PRIx32 ", file=%s]",
2136          static_cast<uint32_t>(rv), TEMP_INDEX_NAME));
2137     FinishRead(false, aProofOfLock);
2138   }
2139 }
2140
2141 void CacheIndex::StartReadingIndex(const StaticMutexAutoLock& aProofOfLock) {
2142   sLock.AssertCurrentThreadOwns();
2143   LOG(("CacheIndex::StartReadingIndex()"));
2144
2145   nsresult rv;
2146
2147   MOZ_ASSERT(mIndexHandle);
2148   MOZ_ASSERT(mState == READING);
2149   MOZ_ASSERT(!mIndexOnDiskIsValid);
2150   MOZ_ASSERT(!mDontMarkIndexClean);
2151   MOZ_ASSERT(!mJournalReadSuccessfully);
2152   MOZ_ASSERT(mIndexHandle->FileSize() >= 0);
2153   MOZ_ASSERT(!mRWPending);
2154
2155   int64_t entriesSize = mIndexHandle->FileSize() - sizeof(CacheIndexHeader) -
2156                         sizeof(CacheHash::Hash32_t);
2157
2158   if (entriesSize < 0 || entriesSize % sizeof(CacheIndexRecord)) {
2159     LOG(("CacheIndex::StartReadingIndex() - Index is corrupted"));
2160     FinishRead(false, aProofOfLock);
2161     return;
2162   }
2163
2164   AllocBuffer();
2165   mSkipEntries = 0;
2166   mRWHash = new CacheHash();
2167
2168   mRWBufPos =
2169       std::min(mRWBufSize, static_cast<uint32_t>(mIndexHandle->FileSize()));
2170
2171   rv = CacheFileIOManager::Read(mIndexHandle, 0, mRWBuf, mRWBufPos, this);
2172   if (NS_FAILED(rv)) {
2173     LOG(
2174         ("CacheIndex::StartReadingIndex() - CacheFileIOManager::Read() failed "
2175          "synchronously [rv=0x%08" PRIx32 "]",
2176          static_cast<uint32_t>(rv)));
2177     FinishRead(false, aProofOfLock);
2178   } else {
2179     mRWPending = true;
2180   }
2181 }
2182
2183 void CacheIndex::ParseRecords(const StaticMutexAutoLock& aProofOfLock) {
2184   sLock.AssertCurrentThreadOwns();
2185   LOG(("CacheIndex::ParseRecords()"));
2186
2187   nsresult rv;
2188
2189   MOZ_ASSERT(!mRWPending);
2190
2191   uint32_t entryCnt = (mIndexHandle->FileSize() - sizeof(CacheIndexHeader) -
2192                        sizeof(CacheHash::Hash32_t)) /
2193                       sizeof(CacheIndexRecord);
2194   uint32_t pos = 0;
2195
2196   if (!mSkipEntries) {
2197     if (NetworkEndian::readUint32(mRWBuf + pos) != kIndexVersion) {
2198       FinishRead(false, aProofOfLock);
2199       return;
2200     }
2201     pos += sizeof(uint32_t);
2202
2203     mIndexTimeStamp = NetworkEndian::readUint32(mRWBuf + pos);
2204     pos += sizeof(uint32_t);
2205
2206     if (NetworkEndian::readUint32(mRWBuf + pos)) {
2207       if (mJournalHandle) {
2208         CacheFileIOManager::DoomFile(mJournalHandle, nullptr);
2209         mJournalHandle = nullptr;
2210       }
2211     } else {
2212       uint32_t* isDirty =
2213           reinterpret_cast<uint32_t*>(moz_xmalloc(sizeof(uint32_t)));
2214       NetworkEndian::writeUint32(isDirty, 1);
2215
2216       // Mark index dirty. The buffer is freed by CacheFileIOManager when
2217       // nullptr is passed as the listener and the call doesn't fail
2218       // synchronously.
2219       rv = CacheFileIOManager::Write(mIndexHandle, 2 * sizeof(uint32_t),
2220                                      reinterpret_cast<char*>(isDirty),
2221                                      sizeof(uint32_t), true, false, nullptr);
2222       if (NS_FAILED(rv)) {
2223         // This is not fatal, just free the memory
2224         free(isDirty);
2225       }
2226     }
2227     pos += sizeof(uint32_t);
2228
2229     uint64_t dataWritten = NetworkEndian::readUint32(mRWBuf + pos);
2230     pos += sizeof(uint32_t);
2231     dataWritten <<= 10;
2232     mTotalBytesWritten += dataWritten;
2233   }
2234
2235   uint32_t hashOffset = pos;
2236
2237   while (pos + sizeof(CacheIndexRecord) <= mRWBufPos &&
2238          mSkipEntries != entryCnt) {
2239     CacheIndexRecord* rec = reinterpret_cast<CacheIndexRecord*>(mRWBuf + pos);
2240     CacheIndexEntry tmpEntry(&rec->mHash);
2241     tmpEntry.ReadFromBuf(mRWBuf + pos);
2242
2243     if (tmpEntry.IsDirty() || !tmpEntry.IsInitialized() ||
2244         tmpEntry.IsFileEmpty() || tmpEntry.IsFresh() || tmpEntry.IsRemoved()) {
2245       LOG(
2246           ("CacheIndex::ParseRecords() - Invalid entry found in index, removing"
2247            " whole index [dirty=%d, initialized=%d, fileEmpty=%d, fresh=%d, "
2248            "removed=%d]",
2249            tmpEntry.IsDirty(), tmpEntry.IsInitialized(), tmpEntry.IsFileEmpty(),
2250            tmpEntry.IsFresh(), tmpEntry.IsRemoved()));
2251       FinishRead(false, aProofOfLock);
2252       return;
2253     }
2254
2255     CacheIndexEntryAutoManage emng(tmpEntry.Hash(), this, aProofOfLock);
2256
2257     CacheIndexEntry* entry = mIndex.PutEntry(*tmpEntry.Hash());
2258     *entry = tmpEntry;
2259
2260     pos += sizeof(CacheIndexRecord);
2261     mSkipEntries++;
2262   }
2263
2264   mRWHash->Update(mRWBuf + hashOffset, pos - hashOffset);
2265
2266   if (pos != mRWBufPos) {
2267     memmove(mRWBuf, mRWBuf + pos, mRWBufPos - pos);
2268   }
2269
2270   mRWBufPos -= pos;
2271   pos = 0;
2272
2273   int64_t fileOffset = sizeof(CacheIndexHeader) +
2274                        mSkipEntries * sizeof(CacheIndexRecord) + mRWBufPos;
2275
2276   MOZ_ASSERT(fileOffset <= mIndexHandle->FileSize());
2277   if (fileOffset == mIndexHandle->FileSize()) {
2278     uint32_t expectedHash = NetworkEndian::readUint32(mRWBuf);
2279     if (mRWHash->GetHash() != expectedHash) {
2280       LOG(("CacheIndex::ParseRecords() - Hash mismatch, [is %x, should be %x]",
2281            mRWHash->GetHash(), expectedHash));
2282       FinishRead(false, aProofOfLock);
2283       return;
2284     }
2285
2286     mIndexOnDiskIsValid = true;
2287     mJournalReadSuccessfully = false;
2288
2289     if (mJournalHandle) {
2290       StartReadingJournal(aProofOfLock);
2291     } else {
2292       FinishRead(false, aProofOfLock);
2293     }
2294
2295     return;
2296   }
2297
2298   pos = mRWBufPos;
2299   uint32_t toRead =
2300       std::min(mRWBufSize - pos,
2301                static_cast<uint32_t>(mIndexHandle->FileSize() - fileOffset));
2302   mRWBufPos = pos + toRead;
2303
2304   rv = CacheFileIOManager::Read(mIndexHandle, fileOffset, mRWBuf + pos, toRead,
2305                                 this);
2306   if (NS_FAILED(rv)) {
2307     LOG(
2308         ("CacheIndex::ParseRecords() - CacheFileIOManager::Read() failed "
2309          "synchronously [rv=0x%08" PRIx32 "]",
2310          static_cast<uint32_t>(rv)));
2311     FinishRead(false, aProofOfLock);
2312     return;
2313   }
2314   mRWPending = true;
2315 }
2316
2317 void CacheIndex::StartReadingJournal(const StaticMutexAutoLock& aProofOfLock) {
2318   sLock.AssertCurrentThreadOwns();
2319   LOG(("CacheIndex::StartReadingJournal()"));
2320
2321   nsresult rv;
2322
2323   MOZ_ASSERT(mJournalHandle);
2324   MOZ_ASSERT(mIndexOnDiskIsValid);
2325   MOZ_ASSERT(mTmpJournal.Count() == 0);
2326   MOZ_ASSERT(mJournalHandle->FileSize() >= 0);
2327   MOZ_ASSERT(!mRWPending);
2328
2329   int64_t entriesSize =
2330       mJournalHandle->FileSize() - sizeof(CacheHash::Hash32_t);
2331
2332   if (entriesSize < 0 || entriesSize % sizeof(CacheIndexRecord)) {
2333     LOG(("CacheIndex::StartReadingJournal() - Journal is corrupted"));
2334     FinishRead(false, aProofOfLock);
2335     return;
2336   }
2337
2338   mSkipEntries = 0;
2339   mRWHash = new CacheHash();
2340
2341   mRWBufPos =
2342       std::min(mRWBufSize, static_cast<uint32_t>(mJournalHandle->FileSize()));
2343
2344   rv = CacheFileIOManager::Read(mJournalHandle, 0, mRWBuf, mRWBufPos, this);
2345   if (NS_FAILED(rv)) {
2346     LOG(
2347         ("CacheIndex::StartReadingJournal() - CacheFileIOManager::Read() failed"
2348          " synchronously [rv=0x%08" PRIx32 "]",
2349          static_cast<uint32_t>(rv)));
2350     FinishRead(false, aProofOfLock);
2351   } else {
2352     mRWPending = true;
2353   }
2354 }
2355
2356 void CacheIndex::ParseJournal(const StaticMutexAutoLock& aProofOfLock) {
2357   sLock.AssertCurrentThreadOwns();
2358   LOG(("CacheIndex::ParseJournal()"));
2359
2360   nsresult rv;
2361
2362   MOZ_ASSERT(!mRWPending);
2363
2364   uint32_t entryCnt =
2365       (mJournalHandle->FileSize() - sizeof(CacheHash::Hash32_t)) /
2366       sizeof(CacheIndexRecord);
2367
2368   uint32_t pos = 0;
2369
2370   while (pos + sizeof(CacheIndexRecord) <= mRWBufPos &&
2371          mSkipEntries != entryCnt) {
2372     CacheIndexEntry tmpEntry(reinterpret_cast<SHA1Sum::Hash*>(mRWBuf + pos));
2373     tmpEntry.ReadFromBuf(mRWBuf + pos);
2374
2375     CacheIndexEntry* entry = mTmpJournal.PutEntry(*tmpEntry.Hash());
2376     *entry = tmpEntry;
2377
2378     if (entry->IsDirty() || entry->IsFresh()) {
2379       LOG(
2380           ("CacheIndex::ParseJournal() - Invalid entry found in journal, "
2381            "ignoring whole journal [dirty=%d, fresh=%d]",
2382            entry->IsDirty(), entry->IsFresh()));
2383       FinishRead(false, aProofOfLock);
2384       return;
2385     }
2386
2387     pos += sizeof(CacheIndexRecord);
2388     mSkipEntries++;
2389   }
2390
2391   mRWHash->Update(mRWBuf, pos);
2392
2393   if (pos != mRWBufPos) {
2394     memmove(mRWBuf, mRWBuf + pos, mRWBufPos - pos);
2395   }
2396
2397   mRWBufPos -= pos;
2398   pos = 0;
2399
2400   int64_t fileOffset = mSkipEntries * sizeof(CacheIndexRecord) + mRWBufPos;
2401
2402   MOZ_ASSERT(fileOffset <= mJournalHandle->FileSize());
2403   if (fileOffset == mJournalHandle->FileSize()) {
2404     uint32_t expectedHash = NetworkEndian::readUint32(mRWBuf);
2405     if (mRWHash->GetHash() != expectedHash) {
2406       LOG(("CacheIndex::ParseJournal() - Hash mismatch, [is %x, should be %x]",
2407            mRWHash->GetHash(), expectedHash));
2408       FinishRead(false, aProofOfLock);
2409       return;
2410     }
2411
2412     mJournalReadSuccessfully = true;
2413     FinishRead(true, aProofOfLock);
2414     return;
2415   }
2416
2417   pos = mRWBufPos;
2418   uint32_t toRead =
2419       std::min(mRWBufSize - pos,
2420                static_cast<uint32_t>(mJournalHandle->FileSize() - fileOffset));
2421   mRWBufPos = pos + toRead;
2422
2423   rv = CacheFileIOManager::Read(mJournalHandle, fileOffset, mRWBuf + pos,
2424                                 toRead, this);
2425   if (NS_FAILED(rv)) {
2426     LOG(
2427         ("CacheIndex::ParseJournal() - CacheFileIOManager::Read() failed "
2428          "synchronously [rv=0x%08" PRIx32 "]",
2429          static_cast<uint32_t>(rv)));
2430     FinishRead(false, aProofOfLock);
2431     return;
2432   }
2433   mRWPending = true;
2434 }
2435
2436 void CacheIndex::MergeJournal(const StaticMutexAutoLock& aProofOfLock) {
2437   sLock.AssertCurrentThreadOwns();
2438   LOG(("CacheIndex::MergeJournal()"));
2439
2440   for (auto iter = mTmpJournal.Iter(); !iter.Done(); iter.Next()) {
2441     CacheIndexEntry* entry = iter.Get();
2442
2443     LOG(("CacheIndex::MergeJournal() [hash=%08x%08x%08x%08x%08x]",
2444          LOGSHA1(entry->Hash())));
2445
2446     CacheIndexEntry* entry2 = mIndex.GetEntry(*entry->Hash());
2447     {
2448       CacheIndexEntryAutoManage emng(entry->Hash(), this, aProofOfLock);
2449       if (entry->IsRemoved()) {
2450         if (entry2) {
2451           entry2->MarkRemoved();
2452           entry2->MarkDirty();
2453         }
2454       } else {
2455         if (!entry2) {
2456           entry2 = mIndex.PutEntry(*entry->Hash());
2457         }
2458
2459         *entry2 = *entry;
2460         entry2->MarkDirty();
2461       }
2462     }
2463     iter.Remove();
2464   }
2465
2466   MOZ_ASSERT(mTmpJournal.Count() == 0);
2467 }
2468
2469 void CacheIndex::EnsureNoFreshEntry() {
2470 #ifdef DEBUG_STATS
2471   CacheIndexStats debugStats;
2472   debugStats.DisableLogging();
2473   for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) {
2474     debugStats.BeforeChange(nullptr);
2475     debugStats.AfterChange(iter.Get());
2476   }
2477   MOZ_ASSERT(debugStats.Fresh() == 0);
2478 #endif
2479 }
2480
2481 void CacheIndex::EnsureCorrectStats() {
2482 #ifdef DEBUG_STATS
2483   MOZ_ASSERT(mPendingUpdates.Count() == 0);
2484   CacheIndexStats debugStats;
2485   debugStats.DisableLogging();
2486   for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) {
2487     debugStats.BeforeChange(nullptr);
2488     debugStats.AfterChange(iter.Get());
2489   }
2490   MOZ_ASSERT(debugStats == mIndexStats);
2491 #endif
2492 }
2493
2494 void CacheIndex::FinishRead(bool aSucceeded,
2495                             const StaticMutexAutoLock& aProofOfLock) {
2496   sLock.AssertCurrentThreadOwns();
2497   LOG(("CacheIndex::FinishRead() [succeeded=%d]", aSucceeded));
2498
2499   MOZ_ASSERT((!aSucceeded && mState == SHUTDOWN) || mState == READING);
2500
2501   MOZ_ASSERT(
2502       // -> rebuild
2503       (!aSucceeded && !mIndexOnDiskIsValid && !mJournalReadSuccessfully) ||
2504       // -> update
2505       (!aSucceeded && mIndexOnDiskIsValid && !mJournalReadSuccessfully) ||
2506       // -> ready
2507       (aSucceeded && mIndexOnDiskIsValid && mJournalReadSuccessfully));
2508
2509   // If there is read operation pending we must be cancelling reading of the
2510   // index when shutting down or removing the whole index.
2511   MOZ_ASSERT(!mRWPending || (!aSucceeded && (mShuttingDown || mRemovingAll)));
2512
2513   if (mState == SHUTDOWN) {
2514     RemoveFile(nsLiteralCString(TEMP_INDEX_NAME));
2515     RemoveFile(nsLiteralCString(JOURNAL_NAME));
2516   } else {
2517     if (mIndexHandle && !mIndexOnDiskIsValid) {
2518       CacheFileIOManager::DoomFile(mIndexHandle, nullptr);
2519     }
2520
2521     if (mJournalHandle) {
2522       CacheFileIOManager::DoomFile(mJournalHandle, nullptr);
2523     }
2524   }
2525
2526   if (mIndexFileOpener) {
2527     mIndexFileOpener->Cancel();
2528     mIndexFileOpener = nullptr;
2529   }
2530   if (mJournalFileOpener) {
2531     mJournalFileOpener->Cancel();
2532     mJournalFileOpener = nullptr;
2533   }
2534   if (mTmpFileOpener) {
2535     mTmpFileOpener->Cancel();
2536     mTmpFileOpener = nullptr;
2537   }
2538
2539   mIndexHandle = nullptr;
2540   mJournalHandle = nullptr;
2541   mRWHash = nullptr;
2542   ReleaseBuffer();
2543
2544   if (mState == SHUTDOWN) {
2545     return;
2546   }
2547
2548   if (!mIndexOnDiskIsValid) {
2549     MOZ_ASSERT(mTmpJournal.Count() == 0);
2550     EnsureNoFreshEntry();
2551     ProcessPendingOperations(aProofOfLock);
2552     // Remove all entries that we haven't seen during this session
2553     RemoveNonFreshEntries(aProofOfLock);
2554     StartUpdatingIndex(true, aProofOfLock);
2555     return;
2556   }
2557
2558   if (!mJournalReadSuccessfully) {
2559     mTmpJournal.Clear();
2560     EnsureNoFreshEntry();
2561     ProcessPendingOperations(aProofOfLock);
2562     StartUpdatingIndex(false, aProofOfLock);
2563     return;
2564   }
2565
2566   MergeJournal(aProofOfLock);
2567   EnsureNoFreshEntry();
2568   ProcessPendingOperations(aProofOfLock);
2569   mIndexStats.Log();
2570
2571   ChangeState(READY, aProofOfLock);
2572   mLastDumpTime = TimeStamp::NowLoRes();  // Do not dump new index immediately
2573 }
2574
2575 // static
2576 void CacheIndex::DelayedUpdate(nsITimer* aTimer, void* aClosure) {
2577   LOG(("CacheIndex::DelayedUpdate()"));
2578
2579   StaticMutexAutoLock lock(sLock);
2580   RefPtr<CacheIndex> index = gInstance;
2581
2582   if (!index) {
2583     return;
2584   }
2585
2586   index->DelayedUpdateLocked(lock);
2587 }
2588
2589 // static
2590 void CacheIndex::DelayedUpdateLocked(const StaticMutexAutoLock& aProofOfLock) {
2591   sLock.AssertCurrentThreadOwns();
2592   LOG(("CacheIndex::DelayedUpdateLocked()"));
2593
2594   nsresult rv;
2595
2596   mUpdateTimer = nullptr;
2597
2598   if (!IsIndexUsable()) {
2599     return;
2600   }
2601
2602   if (mState == READY && mShuttingDown) {
2603     return;
2604   }
2605
2606   // mUpdateEventPending must be false here since StartUpdatingIndex() won't
2607   // schedule timer if it is true.
2608   MOZ_ASSERT(!mUpdateEventPending);
2609   if (mState != BUILDING && mState != UPDATING) {
2610     LOG(("CacheIndex::DelayedUpdateLocked() - Update was canceled"));
2611     return;
2612   }
2613
2614   // We need to redispatch to run with lower priority
2615   RefPtr<CacheIOThread> ioThread = CacheFileIOManager::IOThread();
2616   MOZ_ASSERT(ioThread);
2617
2618   mUpdateEventPending = true;
2619   rv = ioThread->Dispatch(this, CacheIOThread::INDEX);
2620   if (NS_FAILED(rv)) {
2621     mUpdateEventPending = false;
2622     NS_WARNING("CacheIndex::DelayedUpdateLocked() - Can't dispatch event");
2623     LOG(("CacheIndex::DelayedUpdate() - Can't dispatch event"));
2624     FinishUpdate(false, aProofOfLock);
2625   }
2626 }
2627
2628 nsresult CacheIndex::ScheduleUpdateTimer(uint32_t aDelay) {
2629   LOG(("CacheIndex::ScheduleUpdateTimer() [delay=%u]", aDelay));
2630
2631   MOZ_ASSERT(!mUpdateTimer);
2632
2633   nsCOMPtr<nsIEventTarget> ioTarget = CacheFileIOManager::IOTarget();
2634   MOZ_ASSERT(ioTarget);
2635
2636   return NS_NewTimerWithFuncCallback(
2637       getter_AddRefs(mUpdateTimer), CacheIndex::DelayedUpdate, nullptr, aDelay,
2638       nsITimer::TYPE_ONE_SHOT, "net::CacheIndex::ScheduleUpdateTimer",
2639       ioTarget);
2640 }
2641
2642 nsresult CacheIndex::SetupDirectoryEnumerator() {
2643   MOZ_ASSERT(!NS_IsMainThread());
2644   MOZ_ASSERT(!mDirEnumerator);
2645
2646   nsresult rv;
2647   nsCOMPtr<nsIFile> file;
2648
2649   rv = mCacheDirectory->Clone(getter_AddRefs(file));
2650   NS_ENSURE_SUCCESS(rv, rv);
2651
2652   rv = file->AppendNative(nsLiteralCString(ENTRIES_DIR));
2653   NS_ENSURE_SUCCESS(rv, rv);
2654
2655   bool exists;
2656   rv = file->Exists(&exists);
2657   NS_ENSURE_SUCCESS(rv, rv);
2658
2659   if (!exists) {
2660     NS_WARNING(
2661         "CacheIndex::SetupDirectoryEnumerator() - Entries directory "
2662         "doesn't exist!");
2663     LOG(
2664         ("CacheIndex::SetupDirectoryEnumerator() - Entries directory doesn't "
2665          "exist!"));
2666     return NS_ERROR_UNEXPECTED;
2667   }
2668
2669   // Do not do IO under the lock.
2670   nsCOMPtr<nsIDirectoryEnumerator> dirEnumerator;
2671   {
2672     StaticMutexAutoUnlock unlock(sLock);
2673     rv = file->GetDirectoryEntries(getter_AddRefs(dirEnumerator));
2674   }
2675   mDirEnumerator = dirEnumerator.forget();
2676   NS_ENSURE_SUCCESS(rv, rv);
2677
2678   return NS_OK;
2679 }
2680
2681 nsresult CacheIndex::InitEntryFromDiskData(CacheIndexEntry* aEntry,
2682                                            CacheFileMetadata* aMetaData,
2683                                            int64_t aFileSize) {
2684   nsresult rv;
2685
2686   aEntry->InitNew();
2687   aEntry->MarkDirty();
2688   aEntry->MarkFresh();
2689
2690   aEntry->Init(GetOriginAttrsHash(aMetaData->OriginAttributes()),
2691                aMetaData->IsAnonymous(), aMetaData->Pinned());
2692
2693   aEntry->SetFrecency(aMetaData->GetFrecency());
2694
2695   const char* altData = aMetaData->GetElement(CacheFileUtils::kAltDataKey);
2696   bool hasAltData = altData != nullptr;
2697   if (hasAltData && NS_FAILED(CacheFileUtils::ParseAlternativeDataInfo(
2698                         altData, nullptr, nullptr))) {
2699     return NS_ERROR_FAILURE;
2700   }
2701   aEntry->SetHasAltData(hasAltData);
2702
2703   static auto toUint16 = [](const char* aUint16String) -> uint16_t {
2704     if (!aUint16String) {
2705       return kIndexTimeNotAvailable;
2706     }
2707     nsresult rv;
2708     uint64_t n64 = nsDependentCString(aUint16String).ToInteger64(&rv);
2709     MOZ_ASSERT(NS_SUCCEEDED(rv));
2710     return n64 <= kIndexTimeOutOfBound ? n64 : kIndexTimeOutOfBound;
2711   };
2712
2713   aEntry->SetOnStartTime(
2714       toUint16(aMetaData->GetElement("net-response-time-onstart")));
2715   aEntry->SetOnStopTime(
2716       toUint16(aMetaData->GetElement("net-response-time-onstop")));
2717
2718   const char* contentTypeStr = aMetaData->GetElement("ctid");
2719   uint8_t contentType = nsICacheEntry::CONTENT_TYPE_UNKNOWN;
2720   if (contentTypeStr) {
2721     int64_t n64 = nsDependentCString(contentTypeStr).ToInteger64(&rv);
2722     if (NS_FAILED(rv) || n64 < nsICacheEntry::CONTENT_TYPE_UNKNOWN ||
2723         n64 >= nsICacheEntry::CONTENT_TYPE_LAST) {
2724       n64 = nsICacheEntry::CONTENT_TYPE_UNKNOWN;
2725     }
2726     contentType = n64;
2727   }
2728   aEntry->SetContentType(contentType);
2729
2730   aEntry->SetFileSize(static_cast<uint32_t>(std::min(
2731       static_cast<int64_t>(PR_UINT32_MAX), (aFileSize + 0x3FF) >> 10)));
2732   return NS_OK;
2733 }
2734
2735 bool CacheIndex::IsUpdatePending() {
2736   sLock.AssertCurrentThreadOwns();
2737
2738   return mUpdateTimer || mUpdateEventPending;
2739 }
2740
2741 void CacheIndex::BuildIndex(const StaticMutexAutoLock& aProofOfLock) {
2742   sLock.AssertCurrentThreadOwns();
2743   LOG(("CacheIndex::BuildIndex()"));
2744
2745   MOZ_ASSERT(mPendingUpdates.Count() == 0);
2746
2747   nsresult rv;
2748
2749   if (!mDirEnumerator) {
2750     rv = SetupDirectoryEnumerator();
2751     if (mState == SHUTDOWN) {
2752       // The index was shut down while we released the lock. FinishUpdate() was
2753       // already called from Shutdown(), so just simply return here.
2754       return;
2755     }
2756
2757     if (NS_FAILED(rv)) {
2758       FinishUpdate(false, aProofOfLock);
2759       return;
2760     }
2761   }
2762
2763   while (true) {
2764     if (CacheIOThread::YieldAndRerun()) {
2765       LOG((
2766           "CacheIndex::BuildIndex() - Breaking loop for higher level events."));
2767       mUpdateEventPending = true;
2768       return;
2769     }
2770
2771     bool fileExists = false;
2772     nsCOMPtr<nsIFile> file;
2773     {
2774       // Do not do IO under the lock.
2775       nsCOMPtr<nsIDirectoryEnumerator> dirEnumerator(mDirEnumerator);
2776       sLock.AssertCurrentThreadOwns();
2777       StaticMutexAutoUnlock unlock(sLock);
2778       rv = dirEnumerator->GetNextFile(getter_AddRefs(file));
2779
2780       if (file) {
2781         file->Exists(&fileExists);
2782       }
2783     }
2784     if (mState == SHUTDOWN) {
2785       return;
2786     }
2787     if (!file) {
2788       FinishUpdate(NS_SUCCEEDED(rv), aProofOfLock);
2789       return;
2790     }
2791
2792     nsAutoCString leaf;
2793     rv = file->GetNativeLeafName(leaf);
2794     if (NS_FAILED(rv)) {
2795       LOG(
2796           ("CacheIndex::BuildIndex() - GetNativeLeafName() failed! Skipping "
2797            "file."));
2798       mDontMarkIndexClean = true;
2799       continue;
2800     }
2801
2802     if (!fileExists) {
2803       LOG(
2804           ("CacheIndex::BuildIndex() - File returned by the iterator was "
2805            "removed in the meantime [name=%s]",
2806            leaf.get()));
2807       continue;
2808     }
2809
2810     SHA1Sum::Hash hash;
2811     rv = CacheFileIOManager::StrToHash(leaf, &hash);
2812     if (NS_FAILED(rv)) {
2813       LOG(
2814           ("CacheIndex::BuildIndex() - Filename is not a hash, removing file. "
2815            "[name=%s]",
2816            leaf.get()));
2817       file->Remove(false);
2818       continue;
2819     }
2820
2821     CacheIndexEntry* entry = mIndex.GetEntry(hash);
2822     if (entry && entry->IsRemoved()) {
2823       LOG(
2824           ("CacheIndex::BuildIndex() - Found file that should not exist. "
2825            "[name=%s]",
2826            leaf.get()));
2827       entry->Log();
2828       MOZ_ASSERT(entry->IsFresh());
2829       entry = nullptr;
2830     }
2831
2832 #ifdef DEBUG
2833     RefPtr<CacheFileHandle> handle;
2834     CacheFileIOManager::gInstance->mHandles.GetHandle(&hash,
2835                                                       getter_AddRefs(handle));
2836 #endif
2837
2838     if (entry) {
2839       // the entry is up to date
2840       LOG(
2841           ("CacheIndex::BuildIndex() - Skipping file because the entry is up to"
2842            " date. [name=%s]",
2843            leaf.get()));
2844       entry->Log();
2845       MOZ_ASSERT(entry->IsFresh());  // The entry must be from this session
2846       // there must be an active CacheFile if the entry is not initialized
2847       MOZ_ASSERT(entry->IsInitialized() || handle);
2848       continue;
2849     }
2850
2851     MOZ_ASSERT(!handle);
2852
2853     RefPtr<CacheFileMetadata> meta = new CacheFileMetadata();
2854     int64_t size = 0;
2855
2856     {
2857       // Do not do IO under the lock.
2858       StaticMutexAutoUnlock unlock(sLock);
2859       rv = meta->SyncReadMetadata(file);
2860
2861       if (NS_SUCCEEDED(rv)) {
2862         rv = file->GetFileSize(&size);
2863         if (NS_FAILED(rv)) {
2864           LOG(
2865               ("CacheIndex::BuildIndex() - Cannot get filesize of file that was"
2866                " successfully parsed. [name=%s]",
2867                leaf.get()));
2868         }
2869       }
2870     }
2871     if (mState == SHUTDOWN) {
2872       return;
2873     }
2874
2875     // Nobody could add the entry while the lock was released since we modify
2876     // the index only on IO thread and this loop is executed on IO thread too.
2877     entry = mIndex.GetEntry(hash);
2878     MOZ_ASSERT(!entry || entry->IsRemoved());
2879
2880     if (NS_FAILED(rv)) {
2881       LOG(
2882           ("CacheIndex::BuildIndex() - CacheFileMetadata::SyncReadMetadata() "
2883            "failed, removing file. [name=%s]",
2884            leaf.get()));
2885       file->Remove(false);
2886     } else {
2887       CacheIndexEntryAutoManage entryMng(&hash, this, aProofOfLock);
2888       entry = mIndex.PutEntry(hash);
2889       if (NS_FAILED(InitEntryFromDiskData(entry, meta, size))) {
2890         LOG(
2891             ("CacheIndex::BuildIndex() - CacheFile::InitEntryFromDiskData() "
2892              "failed, removing file. [name=%s]",
2893              leaf.get()));
2894         file->Remove(false);
2895         entry->MarkRemoved();
2896       } else {
2897         LOG(("CacheIndex::BuildIndex() - Added entry to index. [name=%s]",
2898              leaf.get()));
2899         entry->Log();
2900       }
2901     }
2902   }
2903
2904   MOZ_ASSERT_UNREACHABLE("We should never get here");
2905 }
2906
2907 bool CacheIndex::StartUpdatingIndexIfNeeded(
2908     const StaticMutexAutoLock& aProofOfLock, bool aSwitchingToReadyState) {
2909   sLock.AssertCurrentThreadOwns();
2910   // Start updating process when we are in or we are switching to READY state
2911   // and index needs update, but not during shutdown or when removing all
2912   // entries.
2913   if ((mState == READY || aSwitchingToReadyState) && mIndexNeedsUpdate &&
2914       !mShuttingDown && !mRemovingAll) {
2915     LOG(("CacheIndex::StartUpdatingIndexIfNeeded() - starting update process"));
2916     mIndexNeedsUpdate = false;
2917     StartUpdatingIndex(false, aProofOfLock);
2918     return true;
2919   }
2920
2921   return false;
2922 }
2923
2924 void CacheIndex::StartUpdatingIndex(bool aRebuild,
2925                                     const StaticMutexAutoLock& aProofOfLock) {
2926   sLock.AssertCurrentThreadOwns();
2927   LOG(("CacheIndex::StartUpdatingIndex() [rebuild=%d]", aRebuild));
2928
2929   nsresult rv;
2930
2931   mIndexStats.Log();
2932
2933   ChangeState(aRebuild ? BUILDING : UPDATING, aProofOfLock);
2934   mDontMarkIndexClean = false;
2935
2936   if (mShuttingDown || mRemovingAll) {
2937     FinishUpdate(false, aProofOfLock);
2938     return;
2939   }
2940
2941   if (IsUpdatePending()) {
2942     LOG(("CacheIndex::StartUpdatingIndex() - Update is already pending"));
2943     return;
2944   }
2945
2946   uint32_t elapsed = (TimeStamp::NowLoRes() - mStartTime).ToMilliseconds();
2947   if (elapsed < kUpdateIndexStartDelay) {
2948     LOG(
2949         ("CacheIndex::StartUpdatingIndex() - %u ms elapsed since startup, "
2950          "scheduling timer to fire in %u ms.",
2951          elapsed, kUpdateIndexStartDelay - elapsed));
2952     rv = ScheduleUpdateTimer(kUpdateIndexStartDelay - elapsed);
2953     if (NS_SUCCEEDED(rv)) {
2954       return;
2955     }
2956
2957     LOG(
2958         ("CacheIndex::StartUpdatingIndex() - ScheduleUpdateTimer() failed. "
2959          "Starting update immediately."));
2960   } else {
2961     LOG(
2962         ("CacheIndex::StartUpdatingIndex() - %u ms elapsed since startup, "
2963          "starting update now.",
2964          elapsed));
2965   }
2966
2967   RefPtr<CacheIOThread> ioThread = CacheFileIOManager::IOThread();
2968   MOZ_ASSERT(ioThread);
2969
2970   // We need to dispatch an event even if we are on IO thread since we need to
2971   // update the index with the correct priority.
2972   mUpdateEventPending = true;
2973   rv = ioThread->Dispatch(this, CacheIOThread::INDEX);
2974   if (NS_FAILED(rv)) {
2975     mUpdateEventPending = false;
2976     NS_WARNING("CacheIndex::StartUpdatingIndex() - Can't dispatch event");
2977     LOG(("CacheIndex::StartUpdatingIndex() - Can't dispatch event"));
2978     FinishUpdate(false, aProofOfLock);
2979   }
2980 }
2981
2982 void CacheIndex::UpdateIndex(const StaticMutexAutoLock& aProofOfLock) {
2983   sLock.AssertCurrentThreadOwns();
2984   LOG(("CacheIndex::UpdateIndex()"));
2985
2986   MOZ_ASSERT(mPendingUpdates.Count() == 0);
2987   sLock.AssertCurrentThreadOwns();
2988
2989   nsresult rv;
2990
2991   if (!mDirEnumerator) {
2992     rv = SetupDirectoryEnumerator();
2993     if (mState == SHUTDOWN) {
2994       // The index was shut down while we released the lock. FinishUpdate() was
2995       // already called from Shutdown(), so just simply return here.
2996       return;
2997     }
2998
2999     if (NS_FAILED(rv)) {
3000       FinishUpdate(false, aProofOfLock);
3001       return;
3002     }
3003   }
3004
3005   while (true) {
3006     if (CacheIOThread::YieldAndRerun()) {
3007       LOG(
3008           ("CacheIndex::UpdateIndex() - Breaking loop for higher level "
3009            "events."));
3010       mUpdateEventPending = true;
3011       return;
3012     }
3013
3014     bool fileExists = false;
3015     nsCOMPtr<nsIFile> file;
3016     {
3017       // Do not do IO under the lock.
3018       nsCOMPtr<nsIDirectoryEnumerator> dirEnumerator(mDirEnumerator);
3019       StaticMutexAutoUnlock unlock(sLock);
3020       rv = dirEnumerator->GetNextFile(getter_AddRefs(file));
3021
3022       if (file) {
3023         file->Exists(&fileExists);
3024       }
3025     }
3026     if (mState == SHUTDOWN) {
3027       return;
3028     }
3029     if (!file) {
3030       FinishUpdate(NS_SUCCEEDED(rv), aProofOfLock);
3031       return;
3032     }
3033
3034     nsAutoCString leaf;
3035     rv = file->GetNativeLeafName(leaf);
3036     if (NS_FAILED(rv)) {
3037       LOG(
3038           ("CacheIndex::UpdateIndex() - GetNativeLeafName() failed! Skipping "
3039            "file."));
3040       mDontMarkIndexClean = true;
3041       continue;
3042     }
3043
3044     if (!fileExists) {
3045       LOG(
3046           ("CacheIndex::UpdateIndex() - File returned by the iterator was "
3047            "removed in the meantime [name=%s]",
3048            leaf.get()));
3049       continue;
3050     }
3051
3052     SHA1Sum::Hash hash;
3053     rv = CacheFileIOManager::StrToHash(leaf, &hash);
3054     if (NS_FAILED(rv)) {
3055       LOG(
3056           ("CacheIndex::UpdateIndex() - Filename is not a hash, removing file. "
3057            "[name=%s]",
3058            leaf.get()));
3059       file->Remove(false);
3060       continue;
3061     }
3062
3063     CacheIndexEntry* entry = mIndex.GetEntry(hash);
3064     if (entry && entry->IsRemoved()) {
3065       if (entry->IsFresh()) {
3066         LOG(
3067             ("CacheIndex::UpdateIndex() - Found file that should not exist. "
3068              "[name=%s]",
3069              leaf.get()));
3070         entry->Log();
3071       }
3072       entry = nullptr;
3073     }
3074
3075 #ifdef DEBUG
3076     RefPtr<CacheFileHandle> handle;
3077     CacheFileIOManager::gInstance->mHandles.GetHandle(&hash,
3078                                                       getter_AddRefs(handle));
3079 #endif
3080
3081     if (entry && entry->IsFresh()) {
3082       // the entry is up to date
3083       LOG(
3084           ("CacheIndex::UpdateIndex() - Skipping file because the entry is up "
3085            " to date. [name=%s]",
3086            leaf.get()));
3087       entry->Log();
3088       // there must be an active CacheFile if the entry is not initialized
3089       MOZ_ASSERT(entry->IsInitialized() || handle);
3090       continue;
3091     }
3092
3093     MOZ_ASSERT(!handle);
3094
3095     if (entry) {
3096       PRTime lastModifiedTime;
3097       {
3098         // Do not do IO under the lock.
3099         StaticMutexAutoUnlock unlock(sLock);
3100         rv = file->GetLastModifiedTime(&lastModifiedTime);
3101       }
3102       if (mState == SHUTDOWN) {
3103         return;
3104       }
3105       if (NS_FAILED(rv)) {
3106         LOG(
3107             ("CacheIndex::UpdateIndex() - Cannot get lastModifiedTime. "
3108              "[name=%s]",
3109              leaf.get()));
3110         // Assume the file is newer than index
3111       } else {
3112         if (mIndexTimeStamp > (lastModifiedTime / PR_MSEC_PER_SEC)) {
3113           LOG(
3114               ("CacheIndex::UpdateIndex() - Skipping file because of last "
3115                "modified time. [name=%s, indexTimeStamp=%" PRIu32 ", "
3116                "lastModifiedTime=%" PRId64 "]",
3117                leaf.get(), mIndexTimeStamp,
3118                lastModifiedTime / PR_MSEC_PER_SEC));
3119
3120           CacheIndexEntryAutoManage entryMng(&hash, this, aProofOfLock);
3121           entry->MarkFresh();
3122           continue;
3123         }
3124       }
3125     }
3126
3127     RefPtr<CacheFileMetadata> meta = new CacheFileMetadata();
3128     int64_t size = 0;
3129
3130     {
3131       // Do not do IO under the lock.
3132       StaticMutexAutoUnlock unlock(sLock);
3133       rv = meta->SyncReadMetadata(file);
3134
3135       if (NS_SUCCEEDED(rv)) {
3136         rv = file->GetFileSize(&size);
3137         if (NS_FAILED(rv)) {
3138           LOG(
3139               ("CacheIndex::UpdateIndex() - Cannot get filesize of file that "
3140                "was successfully parsed. [name=%s]",
3141                leaf.get()));
3142         }
3143       }
3144     }
3145     if (mState == SHUTDOWN) {
3146       return;
3147     }
3148
3149     // Nobody could add the entry while the lock was released since we modify
3150     // the index only on IO thread and this loop is executed on IO thread too.
3151     entry = mIndex.GetEntry(hash);
3152     MOZ_ASSERT(!entry || !entry->IsFresh());
3153
3154     CacheIndexEntryAutoManage entryMng(&hash, this, aProofOfLock);
3155
3156     if (NS_FAILED(rv)) {
3157       LOG(
3158           ("CacheIndex::UpdateIndex() - CacheFileMetadata::SyncReadMetadata() "
3159            "failed, removing file. [name=%s]",
3160            leaf.get()));
3161     } else {
3162       entry = mIndex.PutEntry(hash);
3163       rv = InitEntryFromDiskData(entry, meta, size);
3164       if (NS_FAILED(rv)) {
3165         LOG(
3166             ("CacheIndex::UpdateIndex() - CacheIndex::InitEntryFromDiskData "
3167              "failed, removing file. [name=%s]",
3168              leaf.get()));
3169       }
3170     }
3171
3172     if (NS_FAILED(rv)) {
3173       file->Remove(false);
3174       if (entry) {
3175         entry->MarkRemoved();
3176         entry->MarkFresh();
3177         entry->MarkDirty();
3178       }
3179     } else {
3180       LOG(
3181           ("CacheIndex::UpdateIndex() - Added/updated entry to/in index. "
3182            "[name=%s]",
3183            leaf.get()));
3184       entry->Log();
3185     }
3186   }
3187
3188   MOZ_ASSERT_UNREACHABLE("We should never get here");
3189 }
3190
3191 void CacheIndex::FinishUpdate(bool aSucceeded,
3192                               const StaticMutexAutoLock& aProofOfLock) {
3193   LOG(("CacheIndex::FinishUpdate() [succeeded=%d]", aSucceeded));
3194
3195   MOZ_ASSERT(mState == UPDATING || mState == BUILDING ||
3196              (!aSucceeded && mState == SHUTDOWN));
3197
3198   if (mDirEnumerator) {
3199     if (NS_IsMainThread()) {
3200       LOG(
3201           ("CacheIndex::FinishUpdate() - posting of PreShutdownInternal failed?"
3202            " Cannot safely release mDirEnumerator, leaking it!"));
3203       NS_WARNING(("CacheIndex::FinishUpdate() - Leaking mDirEnumerator!"));
3204       // This can happen only in case dispatching event to IO thread failed in
3205       // CacheIndex::PreShutdown().
3206       Unused << mDirEnumerator.forget();  // Leak it since dir enumerator is not
3207                                           // threadsafe
3208     } else {
3209       mDirEnumerator->Close();
3210       mDirEnumerator = nullptr;
3211     }
3212   }
3213
3214   if (!aSucceeded) {
3215     mDontMarkIndexClean = true;
3216   }
3217
3218   if (mState == SHUTDOWN) {
3219     return;
3220   }
3221
3222   if (mState == UPDATING && aSucceeded) {
3223     // If we've iterated over all entries successfully then all entries that
3224     // really exist on the disk are now marked as fresh. All non-fresh entries
3225     // don't exist anymore and must be removed from the index.
3226     RemoveNonFreshEntries(aProofOfLock);
3227   }
3228
3229   // Make sure we won't start update. If the build or update failed, there is no
3230   // reason to believe that it will succeed next time.
3231   mIndexNeedsUpdate = false;
3232
3233   ChangeState(READY, aProofOfLock);
3234   mLastDumpTime = TimeStamp::NowLoRes();  // Do not dump new index immediately
3235 }
3236
3237 void CacheIndex::RemoveNonFreshEntries(
3238     const StaticMutexAutoLock& aProofOfLock) {
3239   sLock.AssertCurrentThreadOwns();
3240   for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) {
3241     CacheIndexEntry* entry = iter.Get();
3242     if (entry->IsFresh()) {
3243       continue;
3244     }
3245
3246     LOG(
3247         ("CacheIndex::RemoveNonFreshEntries() - Removing entry. "
3248          "[hash=%08x%08x%08x%08x%08x]",
3249          LOGSHA1(entry->Hash())));
3250
3251     {
3252       CacheIndexEntryAutoManage emng(entry->Hash(), this, aProofOfLock);
3253       emng.DoNotSearchInIndex();
3254     }
3255
3256     iter.Remove();
3257   }
3258 }
3259
3260 // static
3261 char const* CacheIndex::StateString(EState aState) {
3262   switch (aState) {
3263     case INITIAL:
3264       return "INITIAL";
3265     case READING:
3266       return "READING";
3267     case WRITING:
3268       return "WRITING";
3269     case BUILDING:
3270       return "BUILDING";
3271     case UPDATING:
3272       return "UPDATING";
3273     case READY:
3274       return "READY";
3275     case SHUTDOWN:
3276       return "SHUTDOWN";
3277   }
3278
3279   MOZ_ASSERT(false, "Unexpected state!");
3280   return "?";
3281 }
3282
3283 void CacheIndex::ChangeState(EState aNewState,
3284                              const StaticMutexAutoLock& aProofOfLock) {
3285   sLock.AssertCurrentThreadOwns();
3286   LOG(("CacheIndex::ChangeState() changing state %s -> %s", StateString(mState),
3287        StateString(aNewState)));
3288
3289   // All pending updates should be processed before changing state
3290   MOZ_ASSERT(mPendingUpdates.Count() == 0);
3291
3292   // PreShutdownInternal() should change the state to READY from every state. It
3293   // may go through different states, but once we are in READY state the only
3294   // possible transition is to SHUTDOWN state.
3295   MOZ_ASSERT(!mShuttingDown || mState != READY || aNewState == SHUTDOWN);
3296
3297   // Start updating process when switching to READY state if needed
3298   if (aNewState == READY && StartUpdatingIndexIfNeeded(aProofOfLock, true)) {
3299     return;
3300   }
3301
3302   // Try to evict entries over limit everytime we're leaving state READING,
3303   // BUILDING or UPDATING, but not during shutdown or when removing all
3304   // entries.
3305   if (!mShuttingDown && !mRemovingAll && aNewState != SHUTDOWN &&
3306       (mState == READING || mState == BUILDING || mState == UPDATING)) {
3307     CacheFileIOManager::EvictIfOverLimit();
3308   }
3309
3310   mState = aNewState;
3311
3312   if (mState != SHUTDOWN) {
3313     CacheFileIOManager::CacheIndexStateChanged();
3314   }
3315
3316   NotifyAsyncGetDiskConsumptionCallbacks();
3317 }
3318
3319 void CacheIndex::NotifyAsyncGetDiskConsumptionCallbacks() {
3320   if ((mState == READY || mState == WRITING) &&
3321       !mAsyncGetDiskConsumptionBlocked && mDiskConsumptionObservers.Length()) {
3322     for (uint32_t i = 0; i < mDiskConsumptionObservers.Length(); ++i) {
3323       DiskConsumptionObserver* o = mDiskConsumptionObservers[i];
3324       // Safe to call under the lock.  We always post to the main thread.
3325       o->OnDiskConsumption(mIndexStats.Size() << 10);
3326     }
3327
3328     mDiskConsumptionObservers.Clear();
3329   }
3330 }
3331
3332 void CacheIndex::AllocBuffer() {
3333   switch (mState) {
3334     case WRITING:
3335       mRWBufSize = sizeof(CacheIndexHeader) + sizeof(CacheHash::Hash32_t) +
3336                    mProcessEntries * sizeof(CacheIndexRecord);
3337       if (mRWBufSize > kMaxBufSize) {
3338         mRWBufSize = kMaxBufSize;
3339       }
3340       break;
3341     case READING:
3342       mRWBufSize = kMaxBufSize;
3343       break;
3344     default:
3345       MOZ_ASSERT(false, "Unexpected state!");
3346   }
3347
3348   mRWBuf = static_cast<char*>(moz_xmalloc(mRWBufSize));
3349 }
3350
3351 void CacheIndex::ReleaseBuffer() {
3352   sLock.AssertCurrentThreadOwns();
3353
3354   if (!mRWBuf || mRWPending) {
3355     return;
3356   }
3357
3358   LOG(("CacheIndex::ReleaseBuffer() releasing buffer"));
3359
3360   free(mRWBuf);
3361   mRWBuf = nullptr;
3362   mRWBufSize = 0;
3363   mRWBufPos = 0;
3364 }
3365
3366 void CacheIndex::FrecencyArray::AppendRecord(
3367     CacheIndexRecordWrapper* aRecord, const StaticMutexAutoLock& aProofOfLock) {
3368   sLock.AssertCurrentThreadOwns();
3369   LOG(
3370       ("CacheIndex::FrecencyArray::AppendRecord() [record=%p, hash=%08x%08x%08x"
3371        "%08x%08x]",
3372        aRecord, LOGSHA1(aRecord->Get()->mHash)));
3373
3374   MOZ_DIAGNOSTIC_ASSERT(!mRecs.Contains(aRecord));
3375   mRecs.AppendElement(aRecord);
3376
3377   // If the new frecency is 0, the element should be at the end of the array,
3378   // i.e. this change doesn't affect order of the array
3379   if (aRecord->Get()->mFrecency != 0) {
3380     ++mUnsortedElements;
3381   }
3382 }
3383
3384 void CacheIndex::FrecencyArray::RemoveRecord(
3385     CacheIndexRecordWrapper* aRecord, const StaticMutexAutoLock& aProofOfLock) {
3386   sLock.AssertCurrentThreadOwns();
3387   LOG(("CacheIndex::FrecencyArray::RemoveRecord() [record=%p]", aRecord));
3388
3389   decltype(mRecs)::index_type idx;
3390   idx = mRecs.IndexOf(aRecord);
3391   MOZ_RELEASE_ASSERT(idx != mRecs.NoIndex);
3392   // sanity check to ensure correct record removal
3393   MOZ_RELEASE_ASSERT(mRecs[idx] == aRecord);
3394   mRecs[idx] = nullptr;
3395   ++mRemovedElements;
3396
3397   // Calling SortIfNeeded ensures that we get rid of removed elements in the
3398   // array once we hit the limit.
3399   SortIfNeeded(aProofOfLock);
3400 }
3401
3402 void CacheIndex::FrecencyArray::ReplaceRecord(
3403     CacheIndexRecordWrapper* aOldRecord, CacheIndexRecordWrapper* aNewRecord,
3404     const StaticMutexAutoLock& aProofOfLock) {
3405   sLock.AssertCurrentThreadOwns();
3406   LOG(
3407       ("CacheIndex::FrecencyArray::ReplaceRecord() [oldRecord=%p, "
3408        "newRecord=%p]",
3409        aOldRecord, aNewRecord));
3410
3411   decltype(mRecs)::index_type idx;
3412   idx = mRecs.IndexOf(aOldRecord);
3413   MOZ_RELEASE_ASSERT(idx != mRecs.NoIndex);
3414   // sanity check to ensure correct record replaced
3415   MOZ_RELEASE_ASSERT(mRecs[idx] == aOldRecord);
3416   mRecs[idx] = aNewRecord;
3417 }
3418
3419 void CacheIndex::FrecencyArray::SortIfNeeded(
3420     const StaticMutexAutoLock& aProofOfLock) {
3421   sLock.AssertCurrentThreadOwns();
3422   const uint32_t kMaxUnsortedCount = 512;
3423   const uint32_t kMaxUnsortedPercent = 10;
3424   const uint32_t kMaxRemovedCount = 512;
3425
3426   uint32_t unsortedLimit = std::min<uint32_t>(
3427       kMaxUnsortedCount, Length() * kMaxUnsortedPercent / 100);
3428
3429   if (mUnsortedElements > unsortedLimit ||
3430       mRemovedElements > kMaxRemovedCount) {
3431     LOG(
3432         ("CacheIndex::FrecencyArray::SortIfNeeded() - Sorting array "
3433          "[unsortedElements=%u, unsortedLimit=%u, removedElements=%u, "
3434          "maxRemovedCount=%u]",
3435          mUnsortedElements, unsortedLimit, mRemovedElements, kMaxRemovedCount));
3436
3437     mRecs.Sort(FrecencyComparator());
3438     mUnsortedElements = 0;
3439     if (mRemovedElements) {
3440 #if defined(EARLY_BETA_OR_EARLIER)
3441       // validate only null items are removed
3442       for (uint32_t i = Length(); i < mRecs.Length(); ++i) {
3443         MOZ_DIAGNOSTIC_ASSERT(!mRecs[i]);
3444       }
3445 #endif
3446       // Removed elements are at the end after sorting.
3447       mRecs.RemoveElementsAt(Length(), mRemovedElements);
3448       mRemovedElements = 0;
3449     }
3450   }
3451 }
3452
3453 bool CacheIndex::FrecencyArray::RecordExistedUnlocked(
3454     CacheIndexRecordWrapper* aRecord) {
3455   return mRecs.Contains(aRecord);
3456 }
3457
3458 void CacheIndex::AddRecordToIterators(CacheIndexRecordWrapper* aRecord,
3459                                       const StaticMutexAutoLock& aProofOfLock) {
3460   sLock.AssertCurrentThreadOwns();
3461   for (uint32_t i = 0; i < mIterators.Length(); ++i) {
3462     // Add a new record only when iterator is supposed to be updated.
3463     if (mIterators[i]->ShouldBeNewAdded()) {
3464       mIterators[i]->AddRecord(aRecord, aProofOfLock);
3465     }
3466   }
3467 }
3468
3469 void CacheIndex::RemoveRecordFromIterators(
3470     CacheIndexRecordWrapper* aRecord, const StaticMutexAutoLock& aProofOfLock) {
3471   sLock.AssertCurrentThreadOwns();
3472   for (uint32_t i = 0; i < mIterators.Length(); ++i) {
3473     // Remove the record from iterator always, it makes no sence to return
3474     // non-existing entries. Also the pointer to the record is no longer valid
3475     // once the entry is removed from index.
3476     mIterators[i]->RemoveRecord(aRecord, aProofOfLock);
3477   }
3478 }
3479
3480 void CacheIndex::ReplaceRecordInIterators(
3481     CacheIndexRecordWrapper* aOldRecord, CacheIndexRecordWrapper* aNewRecord,
3482     const StaticMutexAutoLock& aProofOfLock) {
3483   sLock.AssertCurrentThreadOwns();
3484   for (uint32_t i = 0; i < mIterators.Length(); ++i) {
3485     // We have to replace the record always since the pointer is no longer
3486     // valid after this point. NOTE: Replacing the record doesn't mean that
3487     // a new entry was added, it just means that the data in the entry was
3488     // changed (e.g. a file size) and we had to track this change in
3489     // mPendingUpdates since mIndex was read-only.
3490     mIterators[i]->ReplaceRecord(aOldRecord, aNewRecord, aProofOfLock);
3491   }
3492 }
3493
3494 nsresult CacheIndex::Run() {
3495   LOG(("CacheIndex::Run()"));
3496
3497   StaticMutexAutoLock lock(sLock);
3498
3499   if (!IsIndexUsable()) {
3500     return NS_ERROR_NOT_AVAILABLE;
3501   }
3502
3503   if (mState == READY && mShuttingDown) {
3504     return NS_OK;
3505   }
3506
3507   mUpdateEventPending = false;
3508
3509   switch (mState) {
3510     case BUILDING:
3511       BuildIndex(lock);
3512       break;
3513     case UPDATING:
3514       UpdateIndex(lock);
3515       break;
3516     default:
3517       LOG(("CacheIndex::Run() - Update/Build was canceled"));
3518   }
3519
3520   return NS_OK;
3521 }
3522
3523 void CacheIndex::OnFileOpenedInternal(FileOpenHelper* aOpener,
3524                                       CacheFileHandle* aHandle,
3525                                       nsresult aResult,
3526                                       const StaticMutexAutoLock& aProofOfLock) {
3527   sLock.AssertCurrentThreadOwns();
3528   LOG(
3529       ("CacheIndex::OnFileOpenedInternal() [opener=%p, handle=%p, "
3530        "result=0x%08" PRIx32 "]",
3531        aOpener, aHandle, static_cast<uint32_t>(aResult)));
3532   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
3533
3534   nsresult rv;
3535
3536   MOZ_RELEASE_ASSERT(IsIndexUsable());
3537
3538   if (mState == READY && mShuttingDown) {
3539     return;
3540   }
3541
3542   switch (mState) {
3543     case WRITING:
3544       MOZ_ASSERT(aOpener == mIndexFileOpener);
3545       mIndexFileOpener = nullptr;
3546
3547       if (NS_FAILED(aResult)) {
3548         LOG(
3549             ("CacheIndex::OnFileOpenedInternal() - Can't open index file for "
3550              "writing [rv=0x%08" PRIx32 "]",
3551              static_cast<uint32_t>(aResult)));
3552         FinishWrite(false, aProofOfLock);
3553       } else {
3554         mIndexHandle = aHandle;
3555         WriteRecords(aProofOfLock);
3556       }
3557       break;
3558     case READING:
3559       if (aOpener == mIndexFileOpener) {
3560         mIndexFileOpener = nullptr;
3561
3562         if (NS_SUCCEEDED(aResult)) {
3563           if (aHandle->FileSize() == 0) {
3564             FinishRead(false, aProofOfLock);
3565             CacheFileIOManager::DoomFile(aHandle, nullptr);
3566             break;
3567           }
3568           mIndexHandle = aHandle;
3569         } else {
3570           FinishRead(false, aProofOfLock);
3571           break;
3572         }
3573       } else if (aOpener == mJournalFileOpener) {
3574         mJournalFileOpener = nullptr;
3575         mJournalHandle = aHandle;
3576       } else if (aOpener == mTmpFileOpener) {
3577         mTmpFileOpener = nullptr;
3578         mTmpHandle = aHandle;
3579       } else {
3580         MOZ_ASSERT(false, "Unexpected state!");
3581       }
3582
3583       if (mIndexFileOpener || mJournalFileOpener || mTmpFileOpener) {
3584         // Some opener still didn't finish
3585         break;
3586       }
3587
3588       // We fail and cancel all other openers when we opening index file fails.
3589       MOZ_ASSERT(mIndexHandle);
3590
3591       if (mTmpHandle) {
3592         CacheFileIOManager::DoomFile(mTmpHandle, nullptr);
3593         mTmpHandle = nullptr;
3594
3595         if (mJournalHandle) {  // this shouldn't normally happen
3596           LOG(
3597               ("CacheIndex::OnFileOpenedInternal() - Unexpected state, all "
3598                "files [%s, %s, %s] should never exist. Removing whole index.",
3599                INDEX_NAME, JOURNAL_NAME, TEMP_INDEX_NAME));
3600           FinishRead(false, aProofOfLock);
3601           break;
3602         }
3603       }
3604
3605       if (mJournalHandle) {
3606         // Rename journal to make sure we update index on next start in case
3607         // firefox crashes
3608         rv = CacheFileIOManager::RenameFile(
3609             mJournalHandle, nsLiteralCString(TEMP_INDEX_NAME), this);
3610         if (NS_FAILED(rv)) {
3611           LOG(
3612               ("CacheIndex::OnFileOpenedInternal() - CacheFileIOManager::"
3613                "RenameFile() failed synchronously [rv=0x%08" PRIx32 "]",
3614                static_cast<uint32_t>(rv)));
3615           FinishRead(false, aProofOfLock);
3616           break;
3617         }
3618       } else {
3619         StartReadingIndex(aProofOfLock);
3620       }
3621
3622       break;
3623     default:
3624       MOZ_ASSERT(false, "Unexpected state!");
3625   }
3626 }
3627
3628 nsresult CacheIndex::OnFileOpened(CacheFileHandle* aHandle, nsresult aResult) {
3629   MOZ_CRASH("CacheIndex::OnFileOpened should not be called!");
3630   return NS_ERROR_UNEXPECTED;
3631 }
3632
3633 nsresult CacheIndex::OnDataWritten(CacheFileHandle* aHandle, const char* aBuf,
3634                                    nsresult aResult) {
3635   LOG(("CacheIndex::OnDataWritten() [handle=%p, result=0x%08" PRIx32 "]",
3636        aHandle, static_cast<uint32_t>(aResult)));
3637
3638   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
3639
3640   nsresult rv;
3641
3642   StaticMutexAutoLock lock(sLock);
3643
3644   MOZ_RELEASE_ASSERT(IsIndexUsable());
3645   MOZ_RELEASE_ASSERT(mRWPending);
3646   mRWPending = false;
3647
3648   if (mState == READY && mShuttingDown) {
3649     return NS_OK;
3650   }
3651
3652   switch (mState) {
3653     case WRITING:
3654       MOZ_ASSERT(mIndexHandle == aHandle);
3655
3656       if (NS_FAILED(aResult)) {
3657         FinishWrite(false, lock);
3658       } else {
3659         if (mSkipEntries == mProcessEntries) {
3660           rv = CacheFileIOManager::RenameFile(
3661               mIndexHandle, nsLiteralCString(INDEX_NAME), this);
3662           if (NS_FAILED(rv)) {
3663             LOG(
3664                 ("CacheIndex::OnDataWritten() - CacheFileIOManager::"
3665                  "RenameFile() failed synchronously [rv=0x%08" PRIx32 "]",
3666                  static_cast<uint32_t>(rv)));
3667             FinishWrite(false, lock);
3668           }
3669         } else {
3670           WriteRecords(lock);
3671         }
3672       }
3673       break;
3674     default:
3675       // Writing was canceled.
3676       LOG(
3677           ("CacheIndex::OnDataWritten() - ignoring notification since the "
3678            "operation was previously canceled [state=%d]",
3679            mState));
3680       ReleaseBuffer();
3681   }
3682
3683   return NS_OK;
3684 }
3685
3686 nsresult CacheIndex::OnDataRead(CacheFileHandle* aHandle, char* aBuf,
3687                                 nsresult aResult) {
3688   LOG(("CacheIndex::OnDataRead() [handle=%p, result=0x%08" PRIx32 "]", aHandle,
3689        static_cast<uint32_t>(aResult)));
3690
3691   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
3692
3693   StaticMutexAutoLock lock(sLock);
3694
3695   MOZ_RELEASE_ASSERT(IsIndexUsable());
3696   MOZ_RELEASE_ASSERT(mRWPending);
3697   mRWPending = false;
3698
3699   switch (mState) {
3700     case READING:
3701       MOZ_ASSERT(mIndexHandle == aHandle || mJournalHandle == aHandle);
3702
3703       if (NS_FAILED(aResult)) {
3704         FinishRead(false, lock);
3705       } else {
3706         if (!mIndexOnDiskIsValid) {
3707           ParseRecords(lock);
3708         } else {
3709           ParseJournal(lock);
3710         }
3711       }
3712       break;
3713     default:
3714       // Reading was canceled.
3715       LOG(
3716           ("CacheIndex::OnDataRead() - ignoring notification since the "
3717            "operation was previously canceled [state=%d]",
3718            mState));
3719       ReleaseBuffer();
3720   }
3721
3722   return NS_OK;
3723 }
3724
3725 nsresult CacheIndex::OnFileDoomed(CacheFileHandle* aHandle, nsresult aResult) {
3726   MOZ_CRASH("CacheIndex::OnFileDoomed should not be called!");
3727   return NS_ERROR_UNEXPECTED;
3728 }
3729
3730 nsresult CacheIndex::OnEOFSet(CacheFileHandle* aHandle, nsresult aResult) {
3731   MOZ_CRASH("CacheIndex::OnEOFSet should not be called!");
3732   return NS_ERROR_UNEXPECTED;
3733 }
3734
3735 nsresult CacheIndex::OnFileRenamed(CacheFileHandle* aHandle, nsresult aResult) {
3736   LOG(("CacheIndex::OnFileRenamed() [handle=%p, result=0x%08" PRIx32 "]",
3737        aHandle, static_cast<uint32_t>(aResult)));
3738
3739   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
3740
3741   StaticMutexAutoLock lock(sLock);
3742
3743   MOZ_RELEASE_ASSERT(IsIndexUsable());
3744
3745   if (mState == READY && mShuttingDown) {
3746     return NS_OK;
3747   }
3748
3749   switch (mState) {
3750     case WRITING:
3751       // This is a result of renaming the new index written to tmpfile to index
3752       // file. This is the last step when writing the index and the whole
3753       // writing process is successful iff renaming was successful.
3754
3755       if (mIndexHandle != aHandle) {
3756         LOG(
3757             ("CacheIndex::OnFileRenamed() - ignoring notification since it "
3758              "belongs to previously canceled operation [state=%d]",
3759              mState));
3760         break;
3761       }
3762
3763       FinishWrite(NS_SUCCEEDED(aResult), lock);
3764       break;
3765     case READING:
3766       // This is a result of renaming journal file to tmpfile. It is renamed
3767       // before we start reading index and journal file and it should normally
3768       // succeed. If it fails give up reading of index.
3769
3770       if (mJournalHandle != aHandle) {
3771         LOG(
3772             ("CacheIndex::OnFileRenamed() - ignoring notification since it "
3773              "belongs to previously canceled operation [state=%d]",
3774              mState));
3775         break;
3776       }
3777
3778       if (NS_FAILED(aResult)) {
3779         FinishRead(false, lock);
3780       } else {
3781         StartReadingIndex(lock);
3782       }
3783       break;
3784     default:
3785       // Reading/writing was canceled.
3786       LOG(
3787           ("CacheIndex::OnFileRenamed() - ignoring notification since the "
3788            "operation was previously canceled [state=%d]",
3789            mState));
3790   }
3791
3792   return NS_OK;
3793 }
3794
3795 // Memory reporting
3796
3797 size_t CacheIndex::SizeOfExcludingThisInternal(
3798     mozilla::MallocSizeOf mallocSizeOf) const {
3799   sLock.AssertCurrentThreadOwns();
3800
3801   size_t n = 0;
3802   nsCOMPtr<nsISizeOf> sizeOf;
3803
3804   // mIndexHandle and mJournalHandle are reported via SizeOfHandlesRunnable
3805   // in CacheFileIOManager::SizeOfExcludingThisInternal as part of special
3806   // handles array.
3807
3808   sizeOf = do_QueryInterface(mCacheDirectory);
3809   if (sizeOf) {
3810     n += sizeOf->SizeOfIncludingThis(mallocSizeOf);
3811   }
3812
3813   sizeOf = do_QueryInterface(mUpdateTimer);
3814   if (sizeOf) {
3815     n += sizeOf->SizeOfIncludingThis(mallocSizeOf);
3816   }
3817
3818   n += mallocSizeOf(mRWBuf);
3819   n += mallocSizeOf(mRWHash);
3820
3821   n += mIndex.SizeOfExcludingThis(mallocSizeOf);
3822   n += mPendingUpdates.SizeOfExcludingThis(mallocSizeOf);
3823   n += mTmpJournal.SizeOfExcludingThis(mallocSizeOf);
3824
3825   // mFrecencyArray items are reported by mIndex/mPendingUpdates
3826   n += mFrecencyArray.mRecs.ShallowSizeOfExcludingThis(mallocSizeOf);
3827   n += mDiskConsumptionObservers.ShallowSizeOfExcludingThis(mallocSizeOf);
3828
3829   return n;
3830 }
3831
3832 // static
3833 size_t CacheIndex::SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) {
3834   StaticMutexAutoLock lock(sLock);
3835
3836   if (!gInstance) return 0;
3837
3838   return gInstance->SizeOfExcludingThisInternal(mallocSizeOf);
3839 }
3840
3841 // static
3842 size_t CacheIndex::SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) {
3843   StaticMutexAutoLock lock(sLock);
3844
3845   return mallocSizeOf(gInstance) +
3846          (gInstance ? gInstance->SizeOfExcludingThisInternal(mallocSizeOf) : 0);
3847 }
3848
3849 // static
3850 void CacheIndex::UpdateTotalBytesWritten(uint32_t aBytesWritten) {
3851   StaticMutexAutoLock lock(sLock);
3852
3853   RefPtr<CacheIndex> index = gInstance;
3854   if (!index) {
3855     return;
3856   }
3857
3858   index->mTotalBytesWritten += aBytesWritten;
3859
3860   // Do telemetry report if enough data has been written and the index is
3861   // in READY state. The data is available also in WRITING state, but we would
3862   // need to deal with pending updates.
3863   if (index->mTotalBytesWritten >= kTelemetryReportBytesLimit &&
3864       index->mState == READY && !index->mIndexNeedsUpdate &&
3865       !index->mShuttingDown) {
3866     index->DoTelemetryReport();
3867     index->mTotalBytesWritten = 0;
3868     return;
3869   }
3870 }
3871
3872 void CacheIndex::DoTelemetryReport() {
3873   static const nsLiteralCString
3874       contentTypeNames[nsICacheEntry::CONTENT_TYPE_LAST] = {
3875           "UNKNOWN"_ns, "OTHER"_ns,      "JAVASCRIPT"_ns, "IMAGE"_ns,
3876           "MEDIA"_ns,   "STYLESHEET"_ns, "WASM"_ns};
3877
3878   for (uint32_t i = 0; i < nsICacheEntry::CONTENT_TYPE_LAST; ++i) {
3879     if (mIndexStats.Size() > 0) {
3880       Telemetry::Accumulate(
3881           Telemetry::NETWORK_CACHE_SIZE_SHARE, contentTypeNames[i],
3882           round(static_cast<double>(mIndexStats.SizeByType(i)) * 100.0 /
3883                 static_cast<double>(mIndexStats.Size())));
3884     }
3885
3886     if (mIndexStats.Count() > 0) {
3887       Telemetry::Accumulate(
3888           Telemetry::NETWORK_CACHE_ENTRY_COUNT_SHARE, contentTypeNames[i],
3889           round(static_cast<double>(mIndexStats.CountByType(i)) * 100.0 /
3890                 static_cast<double>(mIndexStats.Count())));
3891     }
3892   }
3893
3894   nsCString probeKey;
3895   if (CacheObserver::SmartCacheSizeEnabled()) {
3896     probeKey = "SMARTSIZE"_ns;
3897   } else {
3898     probeKey = "USERDEFINEDSIZE"_ns;
3899   }
3900   Telemetry::Accumulate(Telemetry::NETWORK_CACHE_ENTRY_COUNT, probeKey,
3901                         mIndexStats.Count());
3902   Telemetry::Accumulate(Telemetry::NETWORK_CACHE_SIZE, probeKey,
3903                         mIndexStats.Size() >> 10);
3904 }
3905
3906 // static
3907 void CacheIndex::OnAsyncEviction(bool aEvicting) {
3908   StaticMutexAutoLock lock(sLock);
3909
3910   RefPtr<CacheIndex> index = gInstance;
3911   if (!index) {
3912     return;
3913   }
3914
3915   index->mAsyncGetDiskConsumptionBlocked = aEvicting;
3916   if (!aEvicting) {
3917     index->NotifyAsyncGetDiskConsumptionCallbacks();
3918   }
3919 }
3920
3921 }  // namespace mozilla::net