netwerk/cache2/CacheIndex.cpp

   1 /* This Source Code Form is subject to the terms of the Mozilla Public
   2  * License, v. 2.0. If a copy of the MPL was not distributed with this
   3  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   4
   5 #include "CacheIndex.h"
   6
   7 #include "CacheLog.h"
   8 #include "CacheFileIOManager.h"
   9 #include "CacheFileMetadata.h"
  10 #include "CacheIndexIterator.h"
  11 #include "CacheIndexContextIterator.h"
  12 #include "nsThreadUtils.h"
  13 #include "nsISizeOf.h"
  14 #include "nsPrintfCString.h"
  15 #include "mozilla/DebugOnly.h"
  16 #include "prinrval.h"
  17 #include "nsIFile.h"
  18 #include "nsITimer.h"
  19 #include "mozilla/AutoRestore.h"
  20 #include <algorithm>
  21 #include "mozilla/Telemetry.h"
  22 #include "mozilla/Unused.h"
  23
  24 #define kMinUnwrittenChanges 300
  25 #define kMinDumpInterval 20000  // in milliseconds
  26 #define kMaxBufSize 16384
  27 #define kIndexVersion 0x00000009
  28 #define kUpdateIndexStartDelay 50000  // in milliseconds
  29 #define kTelemetryReportBytesLimit (2U * 1024U * 1024U * 1024U)  // 2GB
  30
  31 #define INDEX_NAME "index"
  32 #define TEMP_INDEX_NAME "index.tmp"
  33 #define JOURNAL_NAME "index.log"
  34
  35 namespace mozilla {
  36 namespace net {
  37
  38 namespace {
  39
  40 class FrecencyComparator {
  41  public:
  42   bool Equals(CacheIndexRecord* a, CacheIndexRecord* b) const {
  43     if (!a || !b) {
  44       return false;
  45     }
  46
  47     return a->mFrecency == b->mFrecency;
  48   }
  49   bool LessThan(CacheIndexRecord* a, CacheIndexRecord* b) const {
  50     // Removed (=null) entries must be at the end of the array.
  51     if (!a) {
  52       return false;
  53     }
  54     if (!b) {
  55       return true;
  56     }
  57
  58     // Place entries with frecency 0 at the end of the non-removed entries.
  59     if (a->mFrecency == 0) {
  60       return false;
  61     }
  62     if (b->mFrecency == 0) {
  63       return true;
  64     }
  65
  66     return a->mFrecency < b->mFrecency;
  67   }
  68 };
  69
  70 }  // namespace
  71
  72 /**
  73  * This helper class is responsible for keeping CacheIndex::mIndexStats and
  74  * CacheIndex::mFrecencyArray up to date.
  75  */
  76 class CacheIndexEntryAutoManage {
  77  public:
  78   CacheIndexEntryAutoManage(const SHA1Sum::Hash* aHash, CacheIndex* aIndex)
  79       : mIndex(aIndex),
  80         mOldRecord(nullptr),
  81         mOldFrecency(0),
  82         mDoNotSearchInIndex(false),
  83         mDoNotSearchInUpdates(false) {
  84     CacheIndex::sLock.AssertCurrentThreadOwns();
  85
  86     mHash = aHash;
  87     const CacheIndexEntry* entry = FindEntry();
  88     mIndex->mIndexStats.BeforeChange(entry);
  89     if (entry && entry->IsInitialized() && !entry->IsRemoved()) {
  90       mOldRecord = entry->mRec.get();
  91       mOldFrecency = entry->mRec->mFrecency;
  92     }
  93   }
  94
  95   ~CacheIndexEntryAutoManage() {
  96     CacheIndex::sLock.AssertCurrentThreadOwns();
  97
  98     const CacheIndexEntry* entry = FindEntry();
  99     mIndex->mIndexStats.AfterChange(entry);
 100     if (!entry || !entry->IsInitialized() || entry->IsRemoved()) {
 101       entry = nullptr;
 102     }
 103
 104     if (entry && !mOldRecord) {
 105       mIndex->mFrecencyArray.AppendRecord(entry->mRec.get());
 106       mIndex->AddRecordToIterators(entry->mRec.get());
 107     } else if (!entry && mOldRecord) {
 108       mIndex->mFrecencyArray.RemoveRecord(mOldRecord);
 109       mIndex->RemoveRecordFromIterators(mOldRecord);
 110     } else if (entry && mOldRecord) {
 111       auto rec = entry->mRec.get();
 112       if (rec != mOldRecord) {
 113         // record has a different address, we have to replace it
 114         mIndex->ReplaceRecordInIterators(mOldRecord, rec);
 115
 116         if (entry->mRec->mFrecency == mOldFrecency) {
 117           // If frecency hasn't changed simply replace the pointer
 118           mIndex->mFrecencyArray.ReplaceRecord(mOldRecord, rec);
 119         } else {
 120           // Remove old pointer and insert the new one at the end of the array
 121           mIndex->mFrecencyArray.RemoveRecord(mOldRecord);
 122           mIndex->mFrecencyArray.AppendRecord(rec);
 123         }
 124       } else if (entry->mRec->mFrecency != mOldFrecency) {
 125         // Move the element at the end of the array
 126         mIndex->mFrecencyArray.RemoveRecord(rec);
 127         mIndex->mFrecencyArray.AppendRecord(rec);
 128       }
 129     } else {
 130       // both entries were removed or not initialized, do nothing
 131     }
 132   }
 133
 134   // We cannot rely on nsTHashtable::GetEntry() in case we are removing entries
 135   // while iterating. Destructor is called before the entry is removed. Caller
 136   // must call one of following methods to skip lookup in the hashtable.
 137   void DoNotSearchInIndex() { mDoNotSearchInIndex = true; }
 138   void DoNotSearchInUpdates() { mDoNotSearchInUpdates = true; }
 139
 140  private:
 141   const CacheIndexEntry* FindEntry() {
 142     const CacheIndexEntry* entry = nullptr;
 143
 144     switch (mIndex->mState) {
 145       case CacheIndex::READING:
 146       case CacheIndex::WRITING:
 147         if (!mDoNotSearchInUpdates) {
 148           entry = mIndex->mPendingUpdates.GetEntry(*mHash);
 149         }
 150         [[fallthrough]];
 151       case CacheIndex::BUILDING:
 152       case CacheIndex::UPDATING:
 153       case CacheIndex::READY:
 154         if (!entry && !mDoNotSearchInIndex) {
 155           entry = mIndex->mIndex.GetEntry(*mHash);
 156         }
 157         break;
 158       case CacheIndex::INITIAL:
 159       case CacheIndex::SHUTDOWN:
 160       default:
 161         MOZ_ASSERT(false, "Unexpected state!");
 162     }
 163
 164     return entry;
 165   }
 166
 167   const SHA1Sum::Hash* mHash;
 168   RefPtr<CacheIndex> mIndex;
 169   CacheIndexRecord* mOldRecord;
 170   uint32_t mOldFrecency;
 171   bool mDoNotSearchInIndex;
 172   bool mDoNotSearchInUpdates;
 173 };
 174
 175 class FileOpenHelper final : public CacheFileIOListener {
 176  public:
 177   NS_DECL_THREADSAFE_ISUPPORTS
 178
 179   explicit FileOpenHelper(CacheIndex* aIndex)
 180       : mIndex(aIndex), mCanceled(false) {}
 181
 182   void Cancel() {
 183     CacheIndex::sLock.AssertCurrentThreadOwns();
 184     mCanceled = true;
 185   }
 186
 187  private:
 188   virtual ~FileOpenHelper() = default;
 189
 190   NS_IMETHOD OnFileOpened(CacheFileHandle* aHandle, nsresult aResult) override;
 191   NS_IMETHOD OnDataWritten(CacheFileHandle* aHandle, const char* aBuf,
 192                            nsresult aResult) override {
 193     MOZ_CRASH("FileOpenHelper::OnDataWritten should not be called!");
 194     return NS_ERROR_UNEXPECTED;
 195   }
 196   NS_IMETHOD OnDataRead(CacheFileHandle* aHandle, char* aBuf,
 197                         nsresult aResult) override {
 198     MOZ_CRASH("FileOpenHelper::OnDataRead should not be called!");
 199     return NS_ERROR_UNEXPECTED;
 200   }
 201   NS_IMETHOD OnFileDoomed(CacheFileHandle* aHandle, nsresult aResult) override {
 202     MOZ_CRASH("FileOpenHelper::OnFileDoomed should not be called!");
 203     return NS_ERROR_UNEXPECTED;
 204   }
 205   NS_IMETHOD OnEOFSet(CacheFileHandle* aHandle, nsresult aResult) override {
 206     MOZ_CRASH("FileOpenHelper::OnEOFSet should not be called!");
 207     return NS_ERROR_UNEXPECTED;
 208   }
 209   NS_IMETHOD OnFileRenamed(CacheFileHandle* aHandle,
 210                            nsresult aResult) override {
 211     MOZ_CRASH("FileOpenHelper::OnFileRenamed should not be called!");
 212     return NS_ERROR_UNEXPECTED;
 213   }
 214
 215   RefPtr<CacheIndex> mIndex;
 216   bool mCanceled;
 217 };
 218
 219 NS_IMETHODIMP FileOpenHelper::OnFileOpened(CacheFileHandle* aHandle,
 220                                            nsresult aResult) {
 221   StaticMutexAutoLock lock(CacheIndex::sLock);
 222
 223   if (mCanceled) {
 224     if (aHandle) {
 225       CacheFileIOManager::DoomFile(aHandle, nullptr);
 226     }
 227
 228     return NS_OK;
 229   }
 230
 231   mIndex->OnFileOpenedInternal(this, aHandle, aResult);
 232
 233   return NS_OK;
 234 }
 235
 236 NS_IMPL_ISUPPORTS(FileOpenHelper, CacheFileIOListener);
 237
 238 StaticRefPtr<CacheIndex> CacheIndex::gInstance;
 239 StaticMutex CacheIndex::sLock;
 240
 241 NS_IMPL_ADDREF(CacheIndex)
 242 NS_IMPL_RELEASE(CacheIndex)
 243
 244 NS_INTERFACE_MAP_BEGIN(CacheIndex)
 245   NS_INTERFACE_MAP_ENTRY(mozilla::net::CacheFileIOListener)
 246   NS_INTERFACE_MAP_ENTRY(nsIRunnable)
 247 NS_INTERFACE_MAP_END
 248
 249 CacheIndex::CacheIndex()
 250     : mState(INITIAL),
 251       mShuttingDown(false),
 252       mIndexNeedsUpdate(false),
 253       mRemovingAll(false),
 254       mIndexOnDiskIsValid(false),
 255       mDontMarkIndexClean(false),
 256       mIndexTimeStamp(0),
 257       mUpdateEventPending(false),
 258       mSkipEntries(0),
 259       mProcessEntries(0),
 260       mRWBuf(nullptr),
 261       mRWBufSize(0),
 262       mRWBufPos(0),
 263       mRWPending(false),
 264       mJournalReadSuccessfully(false),
 265       mAsyncGetDiskConsumptionBlocked(false),
 266       mTotalBytesWritten(0) {
 267   sLock.AssertCurrentThreadOwns();
 268   LOG(("CacheIndex::CacheIndex [this=%p]", this));
 269   MOZ_ASSERT(!gInstance, "multiple CacheIndex instances!");
 270 }
 271
 272 CacheIndex::~CacheIndex() {
 273   sLock.AssertCurrentThreadOwns();
 274   LOG(("CacheIndex::~CacheIndex [this=%p]", this));
 275
 276   ReleaseBuffer();
 277 }
 278
 279 // static
 280 nsresult CacheIndex::Init(nsIFile* aCacheDirectory) {
 281   LOG(("CacheIndex::Init()"));
 282
 283   MOZ_ASSERT(NS_IsMainThread());
 284
 285   StaticMutexAutoLock lock(sLock);
 286
 287   if (gInstance) {
 288     return NS_ERROR_ALREADY_INITIALIZED;
 289   }
 290
 291   RefPtr<CacheIndex> idx = new CacheIndex();
 292
 293   nsresult rv = idx->InitInternal(aCacheDirectory);
 294   NS_ENSURE_SUCCESS(rv, rv);
 295
 296   gInstance = std::move(idx);
 297   return NS_OK;
 298 }
 299
 300 nsresult CacheIndex::InitInternal(nsIFile* aCacheDirectory) {
 301   nsresult rv;
 302
 303   rv = aCacheDirectory->Clone(getter_AddRefs(mCacheDirectory));
 304   NS_ENSURE_SUCCESS(rv, rv);
 305
 306   mStartTime = TimeStamp::NowLoRes();
 307
 308   mTotalBytesWritten = CacheObserver::CacheAmountWritten();
 309   mTotalBytesWritten <<= 10;
 310
 311   ReadIndexFromDisk();
 312
 313   return NS_OK;
 314 }
 315
 316 // static
 317 nsresult CacheIndex::PreShutdown() {
 318   MOZ_ASSERT(NS_IsMainThread());
 319
 320   StaticMutexAutoLock lock(sLock);
 321
 322   LOG(("CacheIndex::PreShutdown() [gInstance=%p]", gInstance.get()));
 323
 324   nsresult rv;
 325   RefPtr<CacheIndex> index = gInstance;
 326
 327   if (!index) {
 328     return NS_ERROR_NOT_INITIALIZED;
 329   }
 330
 331   LOG(
 332       ("CacheIndex::PreShutdown() - [state=%d, indexOnDiskIsValid=%d, "
 333        "dontMarkIndexClean=%d]",
 334        index->mState, index->mIndexOnDiskIsValid, index->mDontMarkIndexClean));
 335
 336   LOG(("CacheIndex::PreShutdown() - Closing iterators."));
 337   for (uint32_t i = 0; i < index->mIterators.Length();) {
 338     rv = index->mIterators[i]->CloseInternal(NS_ERROR_FAILURE);
 339     if (NS_FAILED(rv)) {
 340       // CacheIndexIterator::CloseInternal() removes itself from mIteratos iff
 341       // it returns success.
 342       LOG(
 343           ("CacheIndex::PreShutdown() - Failed to remove iterator %p. "
 344            "[rv=0x%08" PRIx32 "]",
 345            index->mIterators[i], static_cast<uint32_t>(rv)));
 346       i++;
 347     }
 348   }
 349
 350   index->mShuttingDown = true;
 351
 352   if (index->mState == READY) {
 353     return NS_OK;  // nothing to do
 354   }
 355
 356   nsCOMPtr<nsIRunnable> event;
 357   event = NewRunnableMethod("net::CacheIndex::PreShutdownInternal", index,
 358                             &CacheIndex::PreShutdownInternal);
 359
 360   nsCOMPtr<nsIEventTarget> ioTarget = CacheFileIOManager::IOTarget();
 361   MOZ_ASSERT(ioTarget);
 362
 363   // PreShutdownInternal() will be executed before any queued event on INDEX
 364   // level. That's OK since we don't want to wait for any operation in progess.
 365   rv = ioTarget->Dispatch(event, nsIEventTarget::DISPATCH_NORMAL);
 366   if (NS_FAILED(rv)) {
 367     NS_WARNING("CacheIndex::PreShutdown() - Can't dispatch event");
 368     LOG(("CacheIndex::PreShutdown() - Can't dispatch event"));
 369     return rv;
 370   }
 371
 372   return NS_OK;
 373 }
 374
 375 void CacheIndex::PreShutdownInternal() {
 376   StaticMutexAutoLock lock(sLock);
 377
 378   LOG(
 379       ("CacheIndex::PreShutdownInternal() - [state=%d, indexOnDiskIsValid=%d, "
 380        "dontMarkIndexClean=%d]",
 381        mState, mIndexOnDiskIsValid, mDontMarkIndexClean));
 382
 383   MOZ_ASSERT(mShuttingDown);
 384
 385   if (mUpdateTimer) {
 386     mUpdateTimer->Cancel();
 387     mUpdateTimer = nullptr;
 388   }
 389
 390   switch (mState) {
 391     case WRITING:
 392       FinishWrite(false);
 393       break;
 394     case READY:
 395       // nothing to do, write the journal in Shutdown()
 396       break;
 397     case READING:
 398       FinishRead(false);
 399       break;
 400     case BUILDING:
 401     case UPDATING:
 402       FinishUpdate(false);
 403       break;
 404     default:
 405       MOZ_ASSERT(false, "Implement me!");
 406   }
 407
 408   // We should end up in READY state
 409   MOZ_ASSERT(mState == READY);
 410 }
 411
 412 // static
 413 nsresult CacheIndex::Shutdown() {
 414   MOZ_ASSERT(NS_IsMainThread());
 415
 416   StaticMutexAutoLock lock(sLock);
 417
 418   LOG(("CacheIndex::Shutdown() [gInstance=%p]", gInstance.get()));
 419
 420   RefPtr<CacheIndex> index = gInstance.forget();
 421
 422   if (!index) {
 423     return NS_ERROR_NOT_INITIALIZED;
 424   }
 425
 426   bool sanitize = CacheObserver::ClearCacheOnShutdown();
 427
 428   CacheObserver::SetCacheAmountWritten(index->mTotalBytesWritten >> 10);
 429
 430   LOG(
 431       ("CacheIndex::Shutdown() - [state=%d, indexOnDiskIsValid=%d, "
 432        "dontMarkIndexClean=%d, sanitize=%d]",
 433        index->mState, index->mIndexOnDiskIsValid, index->mDontMarkIndexClean,
 434        sanitize));
 435
 436   MOZ_ASSERT(index->mShuttingDown);
 437
 438   EState oldState = index->mState;
 439   index->ChangeState(SHUTDOWN);
 440
 441   if (oldState != READY) {
 442     LOG(
 443         ("CacheIndex::Shutdown() - Unexpected state. Did posting of "
 444          "PreShutdownInternal() fail?"));
 445   }
 446
 447   switch (oldState) {
 448     case WRITING:
 449       index->FinishWrite(false);
 450       [[fallthrough]];
 451     case READY:
 452       if (index->mIndexOnDiskIsValid && !index->mDontMarkIndexClean) {
 453         if (!sanitize && NS_FAILED(index->WriteLogToDisk())) {
 454           index->RemoveJournalAndTempFile();
 455         }
 456       } else {
 457         index->RemoveJournalAndTempFile();
 458       }
 459       break;
 460     case READING:
 461       index->FinishRead(false);
 462       break;
 463     case BUILDING:
 464     case UPDATING:
 465       index->FinishUpdate(false);
 466       break;
 467     default:
 468       MOZ_ASSERT(false, "Unexpected state!");
 469   }
 470
 471   if (sanitize) {
 472     index->RemoveAllIndexFiles();
 473   }
 474
 475   return NS_OK;
 476 }
 477
 478 // static
 479 nsresult CacheIndex::AddEntry(const SHA1Sum::Hash* aHash) {
 480   LOG(("CacheIndex::AddEntry() [hash=%08x%08x%08x%08x%08x]", LOGSHA1(aHash)));
 481
 482   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
 483
 484   StaticMutexAutoLock lock(sLock);
 485
 486   RefPtr<CacheIndex> index = gInstance;
 487
 488   if (!index) {
 489     return NS_ERROR_NOT_INITIALIZED;
 490   }
 491
 492   if (!index->IsIndexUsable()) {
 493     return NS_ERROR_NOT_AVAILABLE;
 494   }
 495
 496   // Getters in CacheIndexStats assert when mStateLogged is true since the
 497   // information is incomplete between calls to BeforeChange() and AfterChange()
 498   // (i.e. while CacheIndexEntryAutoManage exists). We need to check whether
 499   // non-fresh entries exists outside the scope of CacheIndexEntryAutoManage.
 500   bool updateIfNonFreshEntriesExist = false;
 501
 502   {
 503     CacheIndexEntryAutoManage entryMng(aHash, index);
 504
 505     CacheIndexEntry* entry = index->mIndex.GetEntry(*aHash);
 506     bool entryRemoved = entry && entry->IsRemoved();
 507     CacheIndexEntryUpdate* updated = nullptr;
 508
 509     if (index->mState == READY || index->mState == UPDATING ||
 510         index->mState == BUILDING) {
 511       MOZ_ASSERT(index->mPendingUpdates.Count() == 0);
 512
 513       if (entry && !entryRemoved) {
 514         // Found entry in index that shouldn't exist.
 515
 516         if (entry->IsFresh()) {
 517           // Someone removed the file on disk while FF is running. Update
 518           // process can fix only non-fresh entries (i.e. entries that were not
 519           // added within this session). Start update only if we have such
 520           // entries.
 521           //
 522           // TODO: This should be very rare problem. If it turns out not to be
 523           // true, change the update process so that it also iterates all
 524           // initialized non-empty entries and checks whether the file exists.
 525
 526           LOG(
 527               ("CacheIndex::AddEntry() - Cache file was removed outside FF "
 528                "process!"));
 529
 530           updateIfNonFreshEntriesExist = true;
 531         } else if (index->mState == READY) {
 532           // Index is outdated, update it.
 533           LOG(
 534               ("CacheIndex::AddEntry() - Found entry that shouldn't exist, "
 535                "update is needed"));
 536           index->mIndexNeedsUpdate = true;
 537         } else {
 538           // We cannot be here when building index since all entries are fresh
 539           // during building.
 540           MOZ_ASSERT(index->mState == UPDATING);
 541         }
 542       }
 543
 544       if (!entry) {
 545         entry = index->mIndex.PutEntry(*aHash);
 546       }
 547     } else {  // WRITING, READING
 548       updated = index->mPendingUpdates.GetEntry(*aHash);
 549       bool updatedRemoved = updated && updated->IsRemoved();
 550
 551       if ((updated && !updatedRemoved) ||
 552           (!updated && entry && !entryRemoved && entry->IsFresh())) {
 553         // Fresh entry found, so the file was removed outside FF
 554         LOG(
 555             ("CacheIndex::AddEntry() - Cache file was removed outside FF "
 556              "process!"));
 557
 558         updateIfNonFreshEntriesExist = true;
 559       } else if (!updated && entry && !entryRemoved) {
 560         if (index->mState == WRITING) {
 561           LOG(
 562               ("CacheIndex::AddEntry() - Found entry that shouldn't exist, "
 563                "update is needed"));
 564           index->mIndexNeedsUpdate = true;
 565         }
 566         // Ignore if state is READING since the index information is partial
 567       }
 568
 569       updated = index->mPendingUpdates.PutEntry(*aHash);
 570     }
 571
 572     if (updated) {
 573       updated->InitNew();
 574       updated->MarkDirty();
 575       updated->MarkFresh();
 576     } else {
 577       entry->InitNew();
 578       entry->MarkDirty();
 579       entry->MarkFresh();
 580     }
 581   }
 582
 583   if (updateIfNonFreshEntriesExist &&
 584       index->mIndexStats.Count() != index->mIndexStats.Fresh()) {
 585     index->mIndexNeedsUpdate = true;
 586   }
 587
 588   index->StartUpdatingIndexIfNeeded();
 589   index->WriteIndexToDiskIfNeeded();
 590
 591   return NS_OK;
 592 }
 593
 594 // static
 595 nsresult CacheIndex::EnsureEntryExists(const SHA1Sum::Hash* aHash) {
 596   LOG(("CacheIndex::EnsureEntryExists() [hash=%08x%08x%08x%08x%08x]",
 597        LOGSHA1(aHash)));
 598
 599   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
 600
 601   StaticMutexAutoLock lock(sLock);
 602
 603   RefPtr<CacheIndex> index = gInstance;
 604
 605   if (!index) {
 606     return NS_ERROR_NOT_INITIALIZED;
 607   }
 608
 609   if (!index->IsIndexUsable()) {
 610     return NS_ERROR_NOT_AVAILABLE;
 611   }
 612
 613   {
 614     CacheIndexEntryAutoManage entryMng(aHash, index);
 615
 616     CacheIndexEntry* entry = index->mIndex.GetEntry(*aHash);
 617     bool entryRemoved = entry && entry->IsRemoved();
 618
 619     if (index->mState == READY || index->mState == UPDATING ||
 620         index->mState == BUILDING) {
 621       MOZ_ASSERT(index->mPendingUpdates.Count() == 0);
 622
 623       if (!entry || entryRemoved) {
 624         if (entryRemoved && entry->IsFresh()) {
 625           // This could happen only if somebody copies files to the entries
 626           // directory while FF is running.
 627           LOG(
 628               ("CacheIndex::EnsureEntryExists() - Cache file was added outside "
 629                "FF process! Update is needed."));
 630           index->mIndexNeedsUpdate = true;
 631         } else if (index->mState == READY ||
 632                    (entryRemoved && !entry->IsFresh())) {
 633           // Removed non-fresh entries can be present as a result of
 634           // MergeJournal()
 635           LOG(
 636               ("CacheIndex::EnsureEntryExists() - Didn't find entry that should"
 637                " exist, update is needed"));
 638           index->mIndexNeedsUpdate = true;
 639         }
 640
 641         if (!entry) {
 642           entry = index->mIndex.PutEntry(*aHash);
 643         }
 644         entry->InitNew();
 645         entry->MarkDirty();
 646       }
 647       entry->MarkFresh();
 648     } else {  // WRITING, READING
 649       CacheIndexEntryUpdate* updated = index->mPendingUpdates.GetEntry(*aHash);
 650       bool updatedRemoved = updated && updated->IsRemoved();
 651
 652       if (updatedRemoved || (!updated && entryRemoved && entry->IsFresh())) {
 653         // Fresh information about missing entry found. This could happen only
 654         // if somebody copies files to the entries directory while FF is
 655         // running.
 656         LOG(
 657             ("CacheIndex::EnsureEntryExists() - Cache file was added outside "
 658              "FF process! Update is needed."));
 659         index->mIndexNeedsUpdate = true;
 660       } else if (!updated && (!entry || entryRemoved)) {
 661         if (index->mState == WRITING) {
 662           LOG(
 663               ("CacheIndex::EnsureEntryExists() - Didn't find entry that should"
 664                " exist, update is needed"));
 665           index->mIndexNeedsUpdate = true;
 666         }
 667         // Ignore if state is READING since the index information is partial
 668       }
 669
 670       // We don't need entryRemoved and updatedRemoved info anymore
 671       if (entryRemoved) entry = nullptr;
 672       if (updatedRemoved) updated = nullptr;
 673
 674       if (updated) {
 675         updated->MarkFresh();
 676       } else {
 677         if (!entry) {
 678           // Create a new entry
 679           updated = index->mPendingUpdates.PutEntry(*aHash);
 680           updated->InitNew();
 681           updated->MarkFresh();
 682           updated->MarkDirty();
 683         } else {
 684           if (!entry->IsFresh()) {
 685             // To mark the entry fresh we must make a copy of index entry
 686             // since the index is read-only.
 687             updated = index->mPendingUpdates.PutEntry(*aHash);
 688             *updated = *entry;
 689             updated->MarkFresh();
 690           }
 691         }
 692       }
 693     }
 694   }
 695
 696   index->StartUpdatingIndexIfNeeded();
 697   index->WriteIndexToDiskIfNeeded();
 698
 699   return NS_OK;
 700 }
 701
 702 // static
 703 nsresult CacheIndex::InitEntry(const SHA1Sum::Hash* aHash,
 704                                OriginAttrsHash aOriginAttrsHash,
 705                                bool aAnonymous, bool aPinned) {
 706   LOG(
 707       ("CacheIndex::InitEntry() [hash=%08x%08x%08x%08x%08x, "
 708        "originAttrsHash=%" PRIx64 ", anonymous=%d, pinned=%d]",
 709        LOGSHA1(aHash), aOriginAttrsHash, aAnonymous, aPinned));
 710
 711   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
 712
 713   StaticMutexAutoLock lock(sLock);
 714
 715   RefPtr<CacheIndex> index = gInstance;
 716
 717   if (!index) {
 718     return NS_ERROR_NOT_INITIALIZED;
 719   }
 720
 721   if (!index->IsIndexUsable()) {
 722     return NS_ERROR_NOT_AVAILABLE;
 723   }
 724
 725   {
 726     CacheIndexEntryAutoManage entryMng(aHash, index);
 727
 728     CacheIndexEntry* entry = index->mIndex.GetEntry(*aHash);
 729     CacheIndexEntryUpdate* updated = nullptr;
 730     bool reinitEntry = false;
 731
 732     if (entry && entry->IsRemoved()) {
 733       entry = nullptr;
 734     }
 735
 736     if (index->mState == READY || index->mState == UPDATING ||
 737         index->mState == BUILDING) {
 738       MOZ_ASSERT(index->mPendingUpdates.Count() == 0);
 739       MOZ_ASSERT(entry);
 740       MOZ_ASSERT(entry->IsFresh());
 741
 742       if (!entry) {
 743         LOG(("CacheIndex::InitEntry() - Entry was not found in mIndex!"));
 744         NS_WARNING(
 745             ("CacheIndex::InitEntry() - Entry was not found in mIndex!"));
 746         return NS_ERROR_UNEXPECTED;
 747       }
 748
 749       if (IsCollision(entry, aOriginAttrsHash, aAnonymous)) {
 750         index->mIndexNeedsUpdate =
 751             true;  // TODO Does this really help in case of collision?
 752         reinitEntry = true;
 753       } else {
 754         if (entry->IsInitialized()) {
 755           return NS_OK;
 756         }
 757       }
 758     } else {
 759       updated = index->mPendingUpdates.GetEntry(*aHash);
 760       DebugOnly<bool> removed = updated && updated->IsRemoved();
 761
 762       MOZ_ASSERT(updated || !removed);
 763       MOZ_ASSERT(updated || entry);
 764
 765       if (!updated && !entry) {
 766         LOG(
 767             ("CacheIndex::InitEntry() - Entry was found neither in mIndex nor "
 768              "in mPendingUpdates!"));
 769         NS_WARNING(
 770             ("CacheIndex::InitEntry() - Entry was found neither in "
 771              "mIndex nor in mPendingUpdates!"));
 772         return NS_ERROR_UNEXPECTED;
 773       }
 774
 775       if (updated) {
 776         MOZ_ASSERT(updated->IsFresh());
 777
 778         if (IsCollision(updated, aOriginAttrsHash, aAnonymous)) {
 779           index->mIndexNeedsUpdate = true;
 780           reinitEntry = true;
 781         } else {
 782           if (updated->IsInitialized()) {
 783             return NS_OK;
 784           }
 785         }
 786       } else {
 787         MOZ_ASSERT(entry->IsFresh());
 788
 789         if (IsCollision(entry, aOriginAttrsHash, aAnonymous)) {
 790           index->mIndexNeedsUpdate = true;
 791           reinitEntry = true;
 792         } else {
 793           if (entry->IsInitialized()) {
 794             return NS_OK;
 795           }
 796         }
 797
 798         // make a copy of a read-only entry
 799         updated = index->mPendingUpdates.PutEntry(*aHash);
 800         *updated = *entry;
 801       }
 802     }
 803
 804     if (reinitEntry) {
 805       // There is a collision and we are going to rewrite this entry. Initialize
 806       // it as a new entry.
 807       if (updated) {
 808         updated->InitNew();
 809         updated->MarkFresh();
 810       } else {
 811         entry->InitNew();
 812         entry->MarkFresh();
 813       }
 814     }
 815
 816     if (updated) {
 817       updated->Init(aOriginAttrsHash, aAnonymous, aPinned);
 818       updated->MarkDirty();
 819     } else {
 820       entry->Init(aOriginAttrsHash, aAnonymous, aPinned);
 821       entry->MarkDirty();
 822     }
 823   }
 824
 825   index->StartUpdatingIndexIfNeeded();
 826   index->WriteIndexToDiskIfNeeded();
 827
 828   return NS_OK;
 829 }
 830
 831 // static
 832 nsresult CacheIndex::RemoveEntry(const SHA1Sum::Hash* aHash) {
 833   LOG(("CacheIndex::RemoveEntry() [hash=%08x%08x%08x%08x%08x]",
 834        LOGSHA1(aHash)));
 835
 836   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
 837
 838   StaticMutexAutoLock lock(sLock);
 839
 840   RefPtr<CacheIndex> index = gInstance;
 841
 842   if (!index) {
 843     return NS_ERROR_NOT_INITIALIZED;
 844   }
 845
 846   if (!index->IsIndexUsable()) {
 847     return NS_ERROR_NOT_AVAILABLE;
 848   }
 849
 850   {
 851     CacheIndexEntryAutoManage entryMng(aHash, index);
 852
 853     CacheIndexEntry* entry = index->mIndex.GetEntry(*aHash);
 854     bool entryRemoved = entry && entry->IsRemoved();
 855
 856     if (index->mState == READY || index->mState == UPDATING ||
 857         index->mState == BUILDING) {
 858       MOZ_ASSERT(index->mPendingUpdates.Count() == 0);
 859
 860       if (!entry || entryRemoved) {
 861         if (entryRemoved && entry->IsFresh()) {
 862           // This could happen only if somebody copies files to the entries
 863           // directory while FF is running.
 864           LOG(
 865               ("CacheIndex::RemoveEntry() - Cache file was added outside FF "
 866                "process! Update is needed."));
 867           index->mIndexNeedsUpdate = true;
 868         } else if (index->mState == READY ||
 869                    (entryRemoved && !entry->IsFresh())) {
 870           // Removed non-fresh entries can be present as a result of
 871           // MergeJournal()
 872           LOG(
 873               ("CacheIndex::RemoveEntry() - Didn't find entry that should exist"
 874                ", update is needed"));
 875           index->mIndexNeedsUpdate = true;
 876         }
 877       } else {
 878         if (entry) {
 879           if (!entry->IsDirty() && entry->IsFileEmpty()) {
 880             index->mIndex.RemoveEntry(entry);
 881             entry = nullptr;
 882           } else {
 883             entry->MarkRemoved();
 884             entry->MarkDirty();
 885             entry->MarkFresh();
 886           }
 887         }
 888       }
 889     } else {  // WRITING, READING
 890       CacheIndexEntryUpdate* updated = index->mPendingUpdates.GetEntry(*aHash);
 891       bool updatedRemoved = updated && updated->IsRemoved();
 892
 893       if (updatedRemoved || (!updated && entryRemoved && entry->IsFresh())) {
 894         // Fresh information about missing entry found. This could happen only
 895         // if somebody copies files to the entries directory while FF is
 896         // running.
 897         LOG(
 898             ("CacheIndex::RemoveEntry() - Cache file was added outside FF "
 899              "process! Update is needed."));
 900         index->mIndexNeedsUpdate = true;
 901       } else if (!updated && (!entry || entryRemoved)) {
 902         if (index->mState == WRITING) {
 903           LOG(
 904               ("CacheIndex::RemoveEntry() - Didn't find entry that should exist"
 905                ", update is needed"));
 906           index->mIndexNeedsUpdate = true;
 907         }
 908         // Ignore if state is READING since the index information is partial
 909       }
 910
 911       if (!updated) {
 912         updated = index->mPendingUpdates.PutEntry(*aHash);
 913         updated->InitNew();
 914       }
 915
 916       updated->MarkRemoved();
 917       updated->MarkDirty();
 918       updated->MarkFresh();
 919     }
 920   }
 921
 922   index->StartUpdatingIndexIfNeeded();
 923   index->WriteIndexToDiskIfNeeded();
 924
 925   return NS_OK;
 926 }
 927
 928 // static
 929 nsresult CacheIndex::UpdateEntry(const SHA1Sum::Hash* aHash,
 930                                  const uint32_t* aFrecency,
 931                                  const bool* aHasAltData,
 932                                  const uint16_t* aOnStartTime,
 933                                  const uint16_t* aOnStopTime,
 934                                  const uint8_t* aContentType,
 935                                  const uint32_t* aSize) {
 936   LOG(
 937       ("CacheIndex::UpdateEntry() [hash=%08x%08x%08x%08x%08x, "
 938        "frecency=%s, hasAltData=%s, onStartTime=%s, onStopTime=%s, "
 939        "contentType=%s, size=%s]",
 940        LOGSHA1(aHash), aFrecency ? nsPrintfCString("%u", *aFrecency).get() : "",
 941        aHasAltData ? (*aHasAltData ? "true" : "false") : "",
 942        aOnStartTime ? nsPrintfCString("%u", *aOnStartTime).get() : "",
 943        aOnStopTime ? nsPrintfCString("%u", *aOnStopTime).get() : "",
 944        aContentType ? nsPrintfCString("%u", *aContentType).get() : "",
 945        aSize ? nsPrintfCString("%u", *aSize).get() : ""));
 946
 947   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
 948
 949   StaticMutexAutoLock lock(sLock);
 950
 951   RefPtr<CacheIndex> index = gInstance;
 952
 953   if (!index) {
 954     return NS_ERROR_NOT_INITIALIZED;
 955   }
 956
 957   if (!index->IsIndexUsable()) {
 958     return NS_ERROR_NOT_AVAILABLE;
 959   }
 960
 961   {
 962     CacheIndexEntryAutoManage entryMng(aHash, index);
 963
 964     CacheIndexEntry* entry = index->mIndex.GetEntry(*aHash);
 965
 966     if (entry && entry->IsRemoved()) {
 967       entry = nullptr;
 968     }
 969
 970     if (index->mState == READY || index->mState == UPDATING ||
 971         index->mState == BUILDING) {
 972       MOZ_ASSERT(index->mPendingUpdates.Count() == 0);
 973       MOZ_ASSERT(entry);
 974
 975       if (!entry) {
 976         LOG(("CacheIndex::UpdateEntry() - Entry was not found in mIndex!"));
 977         NS_WARNING(
 978             ("CacheIndex::UpdateEntry() - Entry was not found in mIndex!"));
 979         return NS_ERROR_UNEXPECTED;
 980       }
 981
 982       if (!HasEntryChanged(entry, aFrecency, aHasAltData, aOnStartTime,
 983                            aOnStopTime, aContentType, aSize)) {
 984         return NS_OK;
 985       }
 986
 987       MOZ_ASSERT(entry->IsFresh());
 988       MOZ_ASSERT(entry->IsInitialized());
 989       entry->MarkDirty();
 990
 991       if (aFrecency) {
 992         entry->SetFrecency(*aFrecency);
 993       }
 994
 995       if (aHasAltData) {
 996         entry->SetHasAltData(*aHasAltData);
 997       }
 998
 999       if (aOnStartTime) {
1000         entry->SetOnStartTime(*aOnStartTime);
1001       }
1002
1003       if (aOnStopTime) {
1004         entry->SetOnStopTime(*aOnStopTime);
1005       }
1006
1007       if (aContentType) {
1008         entry->SetContentType(*aContentType);
1009       }
1010
1011       if (aSize) {
1012         entry->SetFileSize(*aSize);
1013       }
1014     } else {
1015       CacheIndexEntryUpdate* updated = index->mPendingUpdates.GetEntry(*aHash);
1016       DebugOnly<bool> removed = updated && updated->IsRemoved();
1017
1018       MOZ_ASSERT(updated || !removed);
1019       MOZ_ASSERT(updated || entry);
1020
1021       if (!updated) {
1022         if (!entry) {
1023           LOG(
1024               ("CacheIndex::UpdateEntry() - Entry was found neither in mIndex "
1025                "nor in mPendingUpdates!"));
1026           NS_WARNING(
1027               ("CacheIndex::UpdateEntry() - Entry was found neither in "
1028                "mIndex nor in mPendingUpdates!"));
1029           return NS_ERROR_UNEXPECTED;
1030         }
1031
1032         // make a copy of a read-only entry
1033         updated = index->mPendingUpdates.PutEntry(*aHash);
1034         *updated = *entry;
1035       }
1036
1037       MOZ_ASSERT(updated->IsFresh());
1038       MOZ_ASSERT(updated->IsInitialized());
1039       updated->MarkDirty();
1040
1041       if (aFrecency) {
1042         updated->SetFrecency(*aFrecency);
1043       }
1044
1045       if (aHasAltData) {
1046         updated->SetHasAltData(*aHasAltData);
1047       }
1048
1049       if (aOnStartTime) {
1050         updated->SetOnStartTime(*aOnStartTime);
1051       }
1052
1053       if (aOnStopTime) {
1054         updated->SetOnStopTime(*aOnStopTime);
1055       }
1056
1057       if (aContentType) {
1058         updated->SetContentType(*aContentType);
1059       }
1060
1061       if (aSize) {
1062         updated->SetFileSize(*aSize);
1063       }
1064     }
1065   }
1066
1067   index->WriteIndexToDiskIfNeeded();
1068
1069   return NS_OK;
1070 }
1071
1072 // static
1073 nsresult CacheIndex::RemoveAll() {
1074   LOG(("CacheIndex::RemoveAll()"));
1075
1076   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
1077
1078   nsCOMPtr<nsIFile> file;
1079
1080   {
1081     StaticMutexAutoLock lock(sLock);
1082
1083     RefPtr<CacheIndex> index = gInstance;
1084
1085     if (!index) {
1086       return NS_ERROR_NOT_INITIALIZED;
1087     }
1088
1089     MOZ_ASSERT(!index->mRemovingAll);
1090
1091     if (!index->IsIndexUsable()) {
1092       return NS_ERROR_NOT_AVAILABLE;
1093     }
1094
1095     AutoRestore<bool> saveRemovingAll(index->mRemovingAll);
1096     index->mRemovingAll = true;
1097
1098     // Doom index and journal handles but don't null them out since this will be
1099     // done in FinishWrite/FinishRead methods.
1100     if (index->mIndexHandle) {
1101       CacheFileIOManager::DoomFile(index->mIndexHandle, nullptr);
1102     } else {
1103       // We don't have a handle to index file, so get the file here, but delete
1104       // it outside the lock. Ignore the result since this is not fatal.
1105       index->GetFile(nsLiteralCString(INDEX_NAME), getter_AddRefs(file));
1106     }
1107
1108     if (index->mJournalHandle) {
1109       CacheFileIOManager::DoomFile(index->mJournalHandle, nullptr);
1110     }
1111
1112     switch (index->mState) {
1113       case WRITING:
1114         index->FinishWrite(false);
1115         break;
1116       case READY:
1117         // nothing to do
1118         break;
1119       case READING:
1120         index->FinishRead(false);
1121         break;
1122       case BUILDING:
1123       case UPDATING:
1124         index->FinishUpdate(false);
1125         break;
1126       default:
1127         MOZ_ASSERT(false, "Unexpected state!");
1128     }
1129
1130     // We should end up in READY state
1131     MOZ_ASSERT(index->mState == READY);
1132
1133     // There should not be any handle
1134     MOZ_ASSERT(!index->mIndexHandle);
1135     MOZ_ASSERT(!index->mJournalHandle);
1136
1137     index->mIndexOnDiskIsValid = false;
1138     index->mIndexNeedsUpdate = false;
1139
1140     index->mIndexStats.Clear();
1141     index->mFrecencyArray.Clear();
1142     index->mIndex.Clear();
1143
1144     for (uint32_t i = 0; i < index->mIterators.Length();) {
1145       nsresult rv = index->mIterators[i]->CloseInternal(NS_ERROR_NOT_AVAILABLE);
1146       if (NS_FAILED(rv)) {
1147         // CacheIndexIterator::CloseInternal() removes itself from mIterators
1148         // iff it returns success.
1149         LOG(
1150             ("CacheIndex::RemoveAll() - Failed to remove iterator %p. "
1151              "[rv=0x%08" PRIx32 "]",
1152              index->mIterators[i], static_cast<uint32_t>(rv)));
1153         i++;
1154       }
1155     }
1156   }
1157
1158   if (file) {
1159     // Ignore the result. The file might not exist and the failure is not fatal.
1160     file->Remove(false);
1161   }
1162
1163   return NS_OK;
1164 }
1165
1166 // static
1167 nsresult CacheIndex::HasEntry(
1168     const nsACString& aKey, EntryStatus* _retval,
1169     const std::function<void(const CacheIndexEntry*)>& aCB) {
1170   LOG(("CacheIndex::HasEntry() [key=%s]", PromiseFlatCString(aKey).get()));
1171
1172   SHA1Sum sum;
1173   SHA1Sum::Hash hash;
1174   sum.update(aKey.BeginReading(), aKey.Length());
1175   sum.finish(hash);
1176
1177   return HasEntry(hash, _retval, aCB);
1178 }
1179
1180 // static
1181 nsresult CacheIndex::HasEntry(
1182     const SHA1Sum::Hash& hash, EntryStatus* _retval,
1183     const std::function<void(const CacheIndexEntry*)>& aCB) {
1184   StaticMutexAutoLock lock(sLock);
1185
1186   RefPtr<CacheIndex> index = gInstance;
1187
1188   if (!index) {
1189     return NS_ERROR_NOT_INITIALIZED;
1190   }
1191
1192   if (!index->IsIndexUsable()) {
1193     return NS_ERROR_NOT_AVAILABLE;
1194   }
1195
1196   const CacheIndexEntry* entry = nullptr;
1197
1198   switch (index->mState) {
1199     case READING:
1200     case WRITING:
1201       entry = index->mPendingUpdates.GetEntry(hash);
1202       [[fallthrough]];
1203     case BUILDING:
1204     case UPDATING:
1205     case READY:
1206       if (!entry) {
1207         entry = index->mIndex.GetEntry(hash);
1208       }
1209       break;
1210     case INITIAL:
1211     case SHUTDOWN:
1212       MOZ_ASSERT(false, "Unexpected state!");
1213   }
1214
1215   if (!entry) {
1216     if (index->mState == READY || index->mState == WRITING) {
1217       *_retval = DOES_NOT_EXIST;
1218     } else {
1219       *_retval = DO_NOT_KNOW;
1220     }
1221   } else {
1222     if (entry->IsRemoved()) {
1223       if (entry->IsFresh()) {
1224         *_retval = DOES_NOT_EXIST;
1225       } else {
1226         *_retval = DO_NOT_KNOW;
1227       }
1228     } else {
1229       *_retval = EXISTS;
1230       if (aCB) {
1231         aCB(entry);
1232       }
1233     }
1234   }
1235
1236   LOG(("CacheIndex::HasEntry() - result is %u", *_retval));
1237   return NS_OK;
1238 }
1239
1240 // static
1241 nsresult CacheIndex::GetEntryForEviction(bool aIgnoreEmptyEntries,
1242                                          SHA1Sum::Hash* aHash, uint32_t* aCnt) {
1243   LOG(("CacheIndex::GetEntryForEviction()"));
1244
1245   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
1246
1247   StaticMutexAutoLock lock(sLock);
1248
1249   RefPtr<CacheIndex> index = gInstance;
1250
1251   if (!index) return NS_ERROR_NOT_INITIALIZED;
1252
1253   if (!index->IsIndexUsable()) {
1254     return NS_ERROR_NOT_AVAILABLE;
1255   }
1256
1257   if (index->mIndexStats.Size() == 0) {
1258     return NS_ERROR_NOT_AVAILABLE;
1259   }
1260
1261   int32_t mediaUsage =
1262       round(static_cast<double>(index->mIndexStats.SizeByType(
1263                 nsICacheEntry::CONTENT_TYPE_MEDIA)) *
1264             100.0 / static_cast<double>(index->mIndexStats.Size()));
1265   int32_t mediaUsageLimit =
1266       StaticPrefs::browser_cache_disk_content_type_media_limit();
1267   bool evictMedia = false;
1268   if (mediaUsage > mediaUsageLimit) {
1269     LOG(
1270         ("CacheIndex::GetEntryForEviction() - media content type is over the "
1271          "limit [mediaUsage=%d, mediaUsageLimit=%d]",
1272          mediaUsage, mediaUsageLimit));
1273     evictMedia = true;
1274   }
1275
1276   SHA1Sum::Hash hash;
1277   CacheIndexRecord* foundRecord = nullptr;
1278   uint32_t skipped = 0;
1279
1280   // find first non-forced valid and unpinned entry with the lowest frecency
1281   index->mFrecencyArray.SortIfNeeded();
1282
1283   for (auto iter = index->mFrecencyArray.Iter(); !iter.Done(); iter.Next()) {
1284     CacheIndexRecord* rec = iter.Get();
1285
1286     memcpy(&hash, rec->mHash, sizeof(SHA1Sum::Hash));
1287
1288     ++skipped;
1289
1290     if (evictMedia && CacheIndexEntry::GetContentType(rec) !=
1291                           nsICacheEntry::CONTENT_TYPE_MEDIA) {
1292       continue;
1293     }
1294
1295     if (IsForcedValidEntry(&hash)) {
1296       continue;
1297     }
1298
1299     if (CacheIndexEntry::IsPinned(rec)) {
1300       continue;
1301     }
1302
1303     if (aIgnoreEmptyEntries && !CacheIndexEntry::GetFileSize(*rec)) {
1304       continue;
1305     }
1306
1307     --skipped;
1308     foundRecord = rec;
1309     break;
1310   }
1311
1312   if (!foundRecord) return NS_ERROR_NOT_AVAILABLE;
1313
1314   *aCnt = skipped;
1315
1316   LOG(
1317       ("CacheIndex::GetEntryForEviction() - returning entry "
1318        "[hash=%08x%08x%08x%08x%08x, cnt=%u, frecency=%u, contentType=%u]",
1319        LOGSHA1(&hash), *aCnt, foundRecord->mFrecency,
1320        CacheIndexEntry::GetContentType(foundRecord)));
1321
1322   memcpy(aHash, &hash, sizeof(SHA1Sum::Hash));
1323
1324   return NS_OK;
1325 }
1326
1327 // static
1328 bool CacheIndex::IsForcedValidEntry(const SHA1Sum::Hash* aHash) {
1329   RefPtr<CacheFileHandle> handle;
1330
1331   CacheFileIOManager::gInstance->mHandles.GetHandle(aHash,
1332                                                     getter_AddRefs(handle));
1333
1334   if (!handle) return false;
1335
1336   nsCString hashKey = handle->Key();
1337   return CacheStorageService::Self()->IsForcedValidEntry(hashKey);
1338 }
1339
1340 // static
1341 nsresult CacheIndex::GetCacheSize(uint32_t* _retval) {
1342   LOG(("CacheIndex::GetCacheSize()"));
1343
1344   StaticMutexAutoLock lock(sLock);
1345
1346   RefPtr<CacheIndex> index = gInstance;
1347
1348   if (!index) return NS_ERROR_NOT_INITIALIZED;
1349
1350   if (!index->IsIndexUsable()) {
1351     return NS_ERROR_NOT_AVAILABLE;
1352   }
1353
1354   *_retval = index->mIndexStats.Size();
1355   LOG(("CacheIndex::GetCacheSize() - returning %u", *_retval));
1356   return NS_OK;
1357 }
1358
1359 // static
1360 nsresult CacheIndex::GetEntryFileCount(uint32_t* _retval) {
1361   LOG(("CacheIndex::GetEntryFileCount()"));
1362
1363   StaticMutexAutoLock lock(sLock);
1364
1365   RefPtr<CacheIndex> index = gInstance;
1366
1367   if (!index) {
1368     return NS_ERROR_NOT_INITIALIZED;
1369   }
1370
1371   if (!index->IsIndexUsable()) {
1372     return NS_ERROR_NOT_AVAILABLE;
1373   }
1374
1375   *_retval = index->mIndexStats.ActiveEntriesCount();
1376   LOG(("CacheIndex::GetEntryFileCount() - returning %u", *_retval));
1377   return NS_OK;
1378 }
1379
1380 // static
1381 nsresult CacheIndex::GetCacheStats(nsILoadContextInfo* aInfo, uint32_t* aSize,
1382                                    uint32_t* aCount) {
1383   LOG(("CacheIndex::GetCacheStats() [info=%p]", aInfo));
1384
1385   StaticMutexAutoLock lock(sLock);
1386
1387   RefPtr<CacheIndex> index = gInstance;
1388
1389   if (!index) {
1390     return NS_ERROR_NOT_INITIALIZED;
1391   }
1392
1393   if (!index->IsIndexUsable()) {
1394     return NS_ERROR_NOT_AVAILABLE;
1395   }
1396
1397   *aSize = 0;
1398   *aCount = 0;
1399
1400   for (auto iter = index->mFrecencyArray.Iter(); !iter.Done(); iter.Next()) {
1401     CacheIndexRecord* record = iter.Get();
1402     if (aInfo && !CacheIndexEntry::RecordMatchesLoadContextInfo(record, aInfo))
1403       continue;
1404
1405     *aSize += CacheIndexEntry::GetFileSize(*record);
1406     ++*aCount;
1407   }
1408
1409   return NS_OK;
1410 }
1411
1412 // static
1413 nsresult CacheIndex::AsyncGetDiskConsumption(
1414     nsICacheStorageConsumptionObserver* aObserver) {
1415   LOG(("CacheIndex::AsyncGetDiskConsumption()"));
1416
1417   StaticMutexAutoLock lock(sLock);
1418
1419   RefPtr<CacheIndex> index = gInstance;
1420
1421   if (!index) {
1422     return NS_ERROR_NOT_INITIALIZED;
1423   }
1424
1425   if (!index->IsIndexUsable()) {
1426     return NS_ERROR_NOT_AVAILABLE;
1427   }
1428
1429   RefPtr<DiskConsumptionObserver> observer =
1430       DiskConsumptionObserver::Init(aObserver);
1431
1432   NS_ENSURE_ARG(observer);
1433
1434   if ((index->mState == READY || index->mState == WRITING) &&
1435       !index->mAsyncGetDiskConsumptionBlocked) {
1436     LOG(("CacheIndex::AsyncGetDiskConsumption - calling immediately"));
1437     // Safe to call the callback under the lock,
1438     // we always post to the main thread.
1439     observer->OnDiskConsumption(index->mIndexStats.Size() << 10);
1440     return NS_OK;
1441   }
1442
1443   LOG(("CacheIndex::AsyncGetDiskConsumption - remembering callback"));
1444   // Will be called when the index get to the READY state.
1445   index->mDiskConsumptionObservers.AppendElement(observer);
1446
1447   // Move forward with index re/building if it is pending
1448   RefPtr<CacheIOThread> ioThread = CacheFileIOManager::IOThread();
1449   if (ioThread) {
1450     ioThread->Dispatch(
1451         NS_NewRunnableFunction("net::CacheIndex::AsyncGetDiskConsumption",
1452                                []() -> void {
1453                                  StaticMutexAutoLock lock(sLock);
1454
1455                                  RefPtr<CacheIndex> index = gInstance;
1456                                  if (index && index->mUpdateTimer) {
1457                                    index->mUpdateTimer->Cancel();
1458                                    index->DelayedUpdateLocked();
1459                                  }
1460                                }),
1461         CacheIOThread::INDEX);
1462   }
1463
1464   return NS_OK;
1465 }
1466
1467 // static
1468 nsresult CacheIndex::GetIterator(nsILoadContextInfo* aInfo, bool aAddNew,
1469                                  CacheIndexIterator** _retval) {
1470   LOG(("CacheIndex::GetIterator() [info=%p, addNew=%d]", aInfo, aAddNew));
1471
1472   StaticMutexAutoLock lock(sLock);
1473
1474   RefPtr<CacheIndex> index = gInstance;
1475
1476   if (!index) {
1477     return NS_ERROR_NOT_INITIALIZED;
1478   }
1479
1480   if (!index->IsIndexUsable()) {
1481     return NS_ERROR_NOT_AVAILABLE;
1482   }
1483
1484   RefPtr<CacheIndexIterator> idxIter;
1485   if (aInfo) {
1486     idxIter = new CacheIndexContextIterator(index, aAddNew, aInfo);
1487   } else {
1488     idxIter = new CacheIndexIterator(index, aAddNew);
1489   }
1490
1491   index->mFrecencyArray.SortIfNeeded();
1492
1493   for (auto iter = index->mFrecencyArray.Iter(); !iter.Done(); iter.Next()) {
1494     idxIter->AddRecord(iter.Get());
1495   }
1496
1497   index->mIterators.AppendElement(idxIter);
1498   idxIter.swap(*_retval);
1499   return NS_OK;
1500 }
1501
1502 // static
1503 nsresult CacheIndex::IsUpToDate(bool* _retval) {
1504   LOG(("CacheIndex::IsUpToDate()"));
1505
1506   StaticMutexAutoLock lock(sLock);
1507
1508   RefPtr<CacheIndex> index = gInstance;
1509
1510   if (!index) {
1511     return NS_ERROR_NOT_INITIALIZED;
1512   }
1513
1514   if (!index->IsIndexUsable()) {
1515     return NS_ERROR_NOT_AVAILABLE;
1516   }
1517
1518   *_retval = (index->mState == READY || index->mState == WRITING) &&
1519              !index->mIndexNeedsUpdate && !index->mShuttingDown;
1520
1521   LOG(("CacheIndex::IsUpToDate() - returning %d", *_retval));
1522   return NS_OK;
1523 }
1524
1525 bool CacheIndex::IsIndexUsable() {
1526   MOZ_ASSERT(mState != INITIAL);
1527
1528   switch (mState) {
1529     case INITIAL:
1530     case SHUTDOWN:
1531       return false;
1532
1533     case READING:
1534     case WRITING:
1535     case BUILDING:
1536     case UPDATING:
1537     case READY:
1538       break;
1539   }
1540
1541   return true;
1542 }
1543
1544 // static
1545 bool CacheIndex::IsCollision(CacheIndexEntry* aEntry,
1546                              OriginAttrsHash aOriginAttrsHash,
1547                              bool aAnonymous) {
1548   if (!aEntry->IsInitialized()) {
1549     return false;
1550   }
1551
1552   if (aEntry->Anonymous() != aAnonymous ||
1553       aEntry->OriginAttrsHash() != aOriginAttrsHash) {
1554     LOG(
1555         ("CacheIndex::IsCollision() - Collision detected for entry hash=%08x"
1556          "%08x%08x%08x%08x, expected values: originAttrsHash=%" PRIu64 ", "
1557          "anonymous=%d; actual values: originAttrsHash=%" PRIu64
1558          ", anonymous=%d]",
1559          LOGSHA1(aEntry->Hash()), aOriginAttrsHash, aAnonymous,
1560          aEntry->OriginAttrsHash(), aEntry->Anonymous()));
1561     return true;
1562   }
1563
1564   return false;
1565 }
1566
1567 // static
1568 bool CacheIndex::HasEntryChanged(
1569     CacheIndexEntry* aEntry, const uint32_t* aFrecency, const bool* aHasAltData,
1570     const uint16_t* aOnStartTime, const uint16_t* aOnStopTime,
1571     const uint8_t* aContentType, const uint32_t* aSize) {
1572   if (aFrecency && *aFrecency != aEntry->GetFrecency()) {
1573     return true;
1574   }
1575
1576   if (aHasAltData && *aHasAltData != aEntry->GetHasAltData()) {
1577     return true;
1578   }
1579
1580   if (aOnStartTime && *aOnStartTime != aEntry->GetOnStartTime()) {
1581     return true;
1582   }
1583
1584   if (aOnStopTime && *aOnStopTime != aEntry->GetOnStopTime()) {
1585     return true;
1586   }
1587
1588   if (aContentType && *aContentType != aEntry->GetContentType()) {
1589     return true;
1590   }
1591
1592   if (aSize &&
1593       (*aSize & CacheIndexEntry::kFileSizeMask) != aEntry->GetFileSize()) {
1594     return true;
1595   }
1596
1597   return false;
1598 }
1599
1600 void CacheIndex::ProcessPendingOperations() {
1601   LOG(("CacheIndex::ProcessPendingOperations()"));
1602
1603   sLock.AssertCurrentThreadOwns();
1604
1605   for (auto iter = mPendingUpdates.Iter(); !iter.Done(); iter.Next()) {
1606     CacheIndexEntryUpdate* update = iter.Get();
1607
1608     LOG(("CacheIndex::ProcessPendingOperations() [hash=%08x%08x%08x%08x%08x]",
1609          LOGSHA1(update->Hash())));
1610
1611     MOZ_ASSERT(update->IsFresh());
1612
1613     CacheIndexEntry* entry = mIndex.GetEntry(*update->Hash());
1614
1615     {
1616       CacheIndexEntryAutoManage emng(update->Hash(), this);
1617       emng.DoNotSearchInUpdates();
1618
1619       if (update->IsRemoved()) {
1620         if (entry) {
1621           if (entry->IsRemoved()) {
1622             MOZ_ASSERT(entry->IsFresh());
1623             MOZ_ASSERT(entry->IsDirty());
1624           } else if (!entry->IsDirty() && entry->IsFileEmpty()) {
1625             // Entries with empty file are not stored in index on disk. Just
1626             // remove the entry, but only in case the entry is not dirty, i.e.
1627             // the entry file was empty when we wrote the index.
1628             mIndex.RemoveEntry(entry);
1629             entry = nullptr;
1630           } else {
1631             entry->MarkRemoved();
1632             entry->MarkDirty();
1633             entry->MarkFresh();
1634           }
1635         }
1636       } else if (entry) {
1637         // Some information in mIndex can be newer than in mPendingUpdates (see
1638         // bug 1074832). This will copy just those values that were really
1639         // updated.
1640         update->ApplyUpdate(entry);
1641       } else {
1642         // There is no entry in mIndex, copy all information from
1643         // mPendingUpdates to mIndex.
1644         entry = mIndex.PutEntry(*update->Hash());
1645         *entry = *update;
1646       }
1647     }
1648
1649     iter.Remove();
1650   }
1651
1652   MOZ_ASSERT(mPendingUpdates.Count() == 0);
1653
1654   EnsureCorrectStats();
1655 }
1656
1657 bool CacheIndex::WriteIndexToDiskIfNeeded() {
1658   if (mState != READY || mShuttingDown || mRWPending) {
1659     return false;
1660   }
1661
1662   if (!mLastDumpTime.IsNull() &&
1663       (TimeStamp::NowLoRes() - mLastDumpTime).ToMilliseconds() <
1664           kMinDumpInterval) {
1665     return false;
1666   }
1667
1668   if (mIndexStats.Dirty() < kMinUnwrittenChanges) {
1669     return false;
1670   }
1671
1672   WriteIndexToDisk();
1673   return true;
1674 }
1675
1676 void CacheIndex::WriteIndexToDisk() {
1677   LOG(("CacheIndex::WriteIndexToDisk()"));
1678   mIndexStats.Log();
1679
1680   nsresult rv;
1681
1682   sLock.AssertCurrentThreadOwns();
1683   MOZ_ASSERT(mState == READY);
1684   MOZ_ASSERT(!mRWBuf);
1685   MOZ_ASSERT(!mRWHash);
1686   MOZ_ASSERT(!mRWPending);
1687
1688   ChangeState(WRITING);
1689
1690   mProcessEntries = mIndexStats.ActiveEntriesCount();
1691
1692   mIndexFileOpener = new FileOpenHelper(this);
1693   rv = CacheFileIOManager::OpenFile(
1694       nsLiteralCString(TEMP_INDEX_NAME),
1695       CacheFileIOManager::SPECIAL_FILE | CacheFileIOManager::CREATE,
1696       mIndexFileOpener);
1697   if (NS_FAILED(rv)) {
1698     LOG(("CacheIndex::WriteIndexToDisk() - Can't open file [rv=0x%08" PRIx32
1699          "]",
1700          static_cast<uint32_t>(rv)));
1701     FinishWrite(false);
1702     return;
1703   }
1704
1705   // Write index header to a buffer, it will be written to disk together with
1706   // records in WriteRecords() once we open the file successfully.
1707   AllocBuffer();
1708   mRWHash = new CacheHash();
1709
1710   mRWBufPos = 0;
1711   // index version
1712   NetworkEndian::writeUint32(mRWBuf + mRWBufPos, kIndexVersion);
1713   mRWBufPos += sizeof(uint32_t);
1714   // timestamp
1715   NetworkEndian::writeUint32(mRWBuf + mRWBufPos,
1716                              static_cast<uint32_t>(PR_Now() / PR_USEC_PER_SEC));
1717   mRWBufPos += sizeof(uint32_t);
1718   // dirty flag
1719   NetworkEndian::writeUint32(mRWBuf + mRWBufPos, 1);
1720   mRWBufPos += sizeof(uint32_t);
1721
1722   mSkipEntries = 0;
1723 }
1724
1725 void CacheIndex::WriteRecords() {
1726   LOG(("CacheIndex::WriteRecords()"));
1727
1728   nsresult rv;
1729
1730   sLock.AssertCurrentThreadOwns();
1731   MOZ_ASSERT(mState == WRITING);
1732   MOZ_ASSERT(!mRWPending);
1733
1734   int64_t fileOffset;
1735
1736   if (mSkipEntries) {
1737     MOZ_ASSERT(mRWBufPos == 0);
1738     fileOffset = sizeof(CacheIndexHeader);
1739     fileOffset += sizeof(CacheIndexRecord) * mSkipEntries;
1740   } else {
1741     MOZ_ASSERT(mRWBufPos == sizeof(CacheIndexHeader));
1742     fileOffset = 0;
1743   }
1744   uint32_t hashOffset = mRWBufPos;
1745
1746   char* buf = mRWBuf + mRWBufPos;
1747   uint32_t skip = mSkipEntries;
1748   uint32_t processMax = (mRWBufSize - mRWBufPos) / sizeof(CacheIndexRecord);
1749   MOZ_ASSERT(processMax != 0 ||
1750              mProcessEntries ==
1751                  0);  // TODO make sure we can write an empty index
1752   uint32_t processed = 0;
1753 #ifdef DEBUG
1754   bool hasMore = false;
1755 #endif
1756   for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) {
1757     CacheIndexEntry* entry = iter.Get();
1758     if (entry->IsRemoved() || !entry->IsInitialized() || entry->IsFileEmpty()) {
1759       continue;
1760     }
1761
1762     if (skip) {
1763       skip--;
1764       continue;
1765     }
1766
1767     if (processed == processMax) {
1768 #ifdef DEBUG
1769       hasMore = true;
1770 #endif
1771       break;
1772     }
1773
1774     entry->WriteToBuf(buf);
1775     buf += sizeof(CacheIndexRecord);
1776     processed++;
1777   }
1778
1779   MOZ_ASSERT(mRWBufPos != static_cast<uint32_t>(buf - mRWBuf) ||
1780              mProcessEntries == 0);
1781   mRWBufPos = buf - mRWBuf;
1782   mSkipEntries += processed;
1783   MOZ_ASSERT(mSkipEntries <= mProcessEntries);
1784
1785   mRWHash->Update(mRWBuf + hashOffset, mRWBufPos - hashOffset);
1786
1787   if (mSkipEntries == mProcessEntries) {
1788     MOZ_ASSERT(!hasMore);
1789
1790     // We've processed all records
1791     if (mRWBufPos + sizeof(CacheHash::Hash32_t) > mRWBufSize) {
1792       // realloc buffer to spare another write cycle
1793       mRWBufSize = mRWBufPos + sizeof(CacheHash::Hash32_t);
1794       mRWBuf = static_cast<char*>(moz_xrealloc(mRWBuf, mRWBufSize));
1795     }
1796
1797     NetworkEndian::writeUint32(mRWBuf + mRWBufPos, mRWHash->GetHash());
1798     mRWBufPos += sizeof(CacheHash::Hash32_t);
1799   } else {
1800     MOZ_ASSERT(hasMore);
1801   }
1802
1803   rv = CacheFileIOManager::Write(mIndexHandle, fileOffset, mRWBuf, mRWBufPos,
1804                                  mSkipEntries == mProcessEntries, false, this);
1805   if (NS_FAILED(rv)) {
1806     LOG(
1807         ("CacheIndex::WriteRecords() - CacheFileIOManager::Write() failed "
1808          "synchronously [rv=0x%08" PRIx32 "]",
1809          static_cast<uint32_t>(rv)));
1810     FinishWrite(false);
1811   } else {
1812     mRWPending = true;
1813   }
1814
1815   mRWBufPos = 0;
1816 }
1817
1818 void CacheIndex::FinishWrite(bool aSucceeded) {
1819   LOG(("CacheIndex::FinishWrite() [succeeded=%d]", aSucceeded));
1820
1821   MOZ_ASSERT((!aSucceeded && mState == SHUTDOWN) || mState == WRITING);
1822
1823   sLock.AssertCurrentThreadOwns();
1824
1825   // If there is write operation pending we must be cancelling writing of the
1826   // index when shutting down or removing the whole index.
1827   MOZ_ASSERT(!mRWPending || (!aSucceeded && (mShuttingDown || mRemovingAll)));
1828
1829   mIndexHandle = nullptr;
1830   mRWHash = nullptr;
1831   ReleaseBuffer();
1832
1833   if (aSucceeded) {
1834     // Opening of the file must not be in progress if writing succeeded.
1835     MOZ_ASSERT(!mIndexFileOpener);
1836
1837     for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) {
1838       CacheIndexEntry* entry = iter.Get();
1839
1840       bool remove = false;
1841       {
1842         CacheIndexEntryAutoManage emng(entry->Hash(), this);
1843
1844         if (entry->IsRemoved()) {
1845           emng.DoNotSearchInIndex();
1846           remove = true;
1847         } else if (entry->IsDirty()) {
1848           entry->ClearDirty();
1849         }
1850       }
1851       if (remove) {
1852         iter.Remove();
1853       }
1854     }
1855
1856     mIndexOnDiskIsValid = true;
1857   } else {
1858     if (mIndexFileOpener) {
1859       // If opening of the file is still in progress (e.g. WRITE process was
1860       // canceled by RemoveAll()) then we need to cancel the opener to make sure
1861       // that OnFileOpenedInternal() won't be called.
1862       mIndexFileOpener->Cancel();
1863       mIndexFileOpener = nullptr;
1864     }
1865   }
1866
1867   ProcessPendingOperations();
1868   mIndexStats.Log();
1869
1870   if (mState == WRITING) {
1871     ChangeState(READY);
1872     mLastDumpTime = TimeStamp::NowLoRes();
1873   }
1874 }
1875
1876 nsresult CacheIndex::GetFile(const nsACString& aName, nsIFile** _retval) {
1877   nsresult rv;
1878
1879   nsCOMPtr<nsIFile> file;
1880   rv = mCacheDirectory->Clone(getter_AddRefs(file));
1881   NS_ENSURE_SUCCESS(rv, rv);
1882
1883   rv = file->AppendNative(aName);
1884   NS_ENSURE_SUCCESS(rv, rv);
1885
1886   file.swap(*_retval);
1887   return NS_OK;
1888 }
1889
1890 void CacheIndex::RemoveFile(const nsACString& aName) {
1891   MOZ_ASSERT(mState == SHUTDOWN);
1892
1893   nsresult rv;
1894
1895   nsCOMPtr<nsIFile> file;
1896   rv = GetFile(aName, getter_AddRefs(file));
1897   NS_ENSURE_SUCCESS_VOID(rv);
1898
1899   rv = file->Remove(false);
1900   if (NS_FAILED(rv) && rv != NS_ERROR_FILE_NOT_FOUND) {
1901     LOG(
1902         ("CacheIndex::RemoveFile() - Cannot remove old entry file from disk "
1903          "[rv=0x%08" PRIx32 ", name=%s]",
1904          static_cast<uint32_t>(rv), PromiseFlatCString(aName).get()));
1905   }
1906 }
1907
1908 void CacheIndex::RemoveAllIndexFiles() {
1909   LOG(("CacheIndex::RemoveAllIndexFiles()"));
1910   RemoveFile(nsLiteralCString(INDEX_NAME));
1911   RemoveJournalAndTempFile();
1912 }
1913
1914 void CacheIndex::RemoveJournalAndTempFile() {
1915   LOG(("CacheIndex::RemoveJournalAndTempFile()"));
1916   RemoveFile(nsLiteralCString(TEMP_INDEX_NAME));
1917   RemoveFile(nsLiteralCString(JOURNAL_NAME));
1918 }
1919
1920 class WriteLogHelper {
1921  public:
1922   explicit WriteLogHelper(PRFileDesc* aFD)
1923       : mFD(aFD), mBufSize(kMaxBufSize), mBufPos(0) {
1924     mHash = new CacheHash();
1925     mBuf = static_cast<char*>(moz_xmalloc(mBufSize));
1926   }
1927
1928   ~WriteLogHelper() { free(mBuf); }
1929
1930   nsresult AddEntry(CacheIndexEntry* aEntry);
1931   nsresult Finish();
1932
1933  private:
1934   nsresult FlushBuffer();
1935
1936   PRFileDesc* mFD;
1937   char* mBuf;
1938   uint32_t mBufSize;
1939   int32_t mBufPos;
1940   RefPtr<CacheHash> mHash;
1941 };
1942
1943 nsresult WriteLogHelper::AddEntry(CacheIndexEntry* aEntry) {
1944   nsresult rv;
1945
1946   if (mBufPos + sizeof(CacheIndexRecord) > mBufSize) {
1947     mHash->Update(mBuf, mBufPos);
1948
1949     rv = FlushBuffer();
1950     NS_ENSURE_SUCCESS(rv, rv);
1951     MOZ_ASSERT(mBufPos + sizeof(CacheIndexRecord) <= mBufSize);
1952   }
1953
1954   aEntry->WriteToBuf(mBuf + mBufPos);
1955   mBufPos += sizeof(CacheIndexRecord);
1956
1957   return NS_OK;
1958 }
1959
1960 nsresult WriteLogHelper::Finish() {
1961   nsresult rv;
1962
1963   mHash->Update(mBuf, mBufPos);
1964   if (mBufPos + sizeof(CacheHash::Hash32_t) > mBufSize) {
1965     rv = FlushBuffer();
1966     NS_ENSURE_SUCCESS(rv, rv);
1967     MOZ_ASSERT(mBufPos + sizeof(CacheHash::Hash32_t) <= mBufSize);
1968   }
1969
1970   NetworkEndian::writeUint32(mBuf + mBufPos, mHash->GetHash());
1971   mBufPos += sizeof(CacheHash::Hash32_t);
1972
1973   rv = FlushBuffer();
1974   NS_ENSURE_SUCCESS(rv, rv);
1975
1976   return NS_OK;
1977 }
1978
1979 nsresult WriteLogHelper::FlushBuffer() {
1980   if (CacheObserver::IsPastShutdownIOLag()) {
1981     LOG(("WriteLogHelper::FlushBuffer() - Interrupting writing journal."));
1982     return NS_ERROR_FAILURE;
1983   }
1984
1985   int32_t bytesWritten = PR_Write(mFD, mBuf, mBufPos);
1986
1987   if (bytesWritten != mBufPos) {
1988     return NS_ERROR_FAILURE;
1989   }
1990
1991   mBufPos = 0;
1992   return NS_OK;
1993 }
1994
1995 nsresult CacheIndex::WriteLogToDisk() {
1996   LOG(("CacheIndex::WriteLogToDisk()"));
1997
1998   nsresult rv;
1999
2000   MOZ_ASSERT(mPendingUpdates.Count() == 0);
2001   MOZ_ASSERT(mState == SHUTDOWN);
2002
2003   if (CacheObserver::IsPastShutdownIOLag()) {
2004     LOG(("CacheIndex::WriteLogToDisk() - Skipping writing journal."));
2005     return NS_ERROR_FAILURE;
2006   }
2007
2008   RemoveFile(nsLiteralCString(TEMP_INDEX_NAME));
2009
2010   nsCOMPtr<nsIFile> indexFile;
2011   rv = GetFile(nsLiteralCString(INDEX_NAME), getter_AddRefs(indexFile));
2012   NS_ENSURE_SUCCESS(rv, rv);
2013
2014   nsCOMPtr<nsIFile> logFile;
2015   rv = GetFile(nsLiteralCString(JOURNAL_NAME), getter_AddRefs(logFile));
2016   NS_ENSURE_SUCCESS(rv, rv);
2017
2018   mIndexStats.Log();
2019
2020   PRFileDesc* fd = nullptr;
2021   rv = logFile->OpenNSPRFileDesc(PR_RDWR | PR_CREATE_FILE | PR_TRUNCATE, 0600,
2022                                  &fd);
2023   NS_ENSURE_SUCCESS(rv, rv);
2024
2025   WriteLogHelper wlh(fd);
2026   for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) {
2027     CacheIndexEntry* entry = iter.Get();
2028     if (entry->IsRemoved() || entry->IsDirty()) {
2029       rv = wlh.AddEntry(entry);
2030       if (NS_WARN_IF(NS_FAILED(rv))) {
2031         return rv;
2032       }
2033     }
2034   }
2035
2036   rv = wlh.Finish();
2037   PR_Close(fd);
2038   NS_ENSURE_SUCCESS(rv, rv);
2039
2040   rv = indexFile->OpenNSPRFileDesc(PR_RDWR, 0600, &fd);
2041   NS_ENSURE_SUCCESS(rv, rv);
2042
2043   // Seek to dirty flag in the index header and clear it.
2044   static_assert(2 * sizeof(uint32_t) == offsetof(CacheIndexHeader, mIsDirty),
2045                 "Unexpected offset of CacheIndexHeader::mIsDirty");
2046   int64_t offset = PR_Seek64(fd, 2 * sizeof(uint32_t), PR_SEEK_SET);
2047   if (offset == -1) {
2048     PR_Close(fd);
2049     return NS_ERROR_FAILURE;
2050   }
2051
2052   uint32_t isDirty = 0;
2053   int32_t bytesWritten = PR_Write(fd, &isDirty, sizeof(isDirty));
2054   PR_Close(fd);
2055   if (bytesWritten != sizeof(isDirty)) {
2056     return NS_ERROR_FAILURE;
2057   }
2058
2059   return NS_OK;
2060 }
2061
2062 void CacheIndex::ReadIndexFromDisk() {
2063   LOG(("CacheIndex::ReadIndexFromDisk()"));
2064
2065   nsresult rv;
2066
2067   sLock.AssertCurrentThreadOwns();
2068   MOZ_ASSERT(mState == INITIAL);
2069
2070   ChangeState(READING);
2071
2072   mIndexFileOpener = new FileOpenHelper(this);
2073   rv = CacheFileIOManager::OpenFile(
2074       nsLiteralCString(INDEX_NAME),
2075       CacheFileIOManager::SPECIAL_FILE | CacheFileIOManager::OPEN,
2076       mIndexFileOpener);
2077   if (NS_FAILED(rv)) {
2078     LOG(
2079         ("CacheIndex::ReadIndexFromDisk() - CacheFileIOManager::OpenFile() "
2080          "failed [rv=0x%08" PRIx32 ", file=%s]",
2081          static_cast<uint32_t>(rv), INDEX_NAME));
2082     FinishRead(false);
2083     return;
2084   }
2085
2086   mJournalFileOpener = new FileOpenHelper(this);
2087   rv = CacheFileIOManager::OpenFile(
2088       nsLiteralCString(JOURNAL_NAME),
2089       CacheFileIOManager::SPECIAL_FILE | CacheFileIOManager::OPEN,
2090       mJournalFileOpener);
2091   if (NS_FAILED(rv)) {
2092     LOG(
2093         ("CacheIndex::ReadIndexFromDisk() - CacheFileIOManager::OpenFile() "
2094          "failed [rv=0x%08" PRIx32 ", file=%s]",
2095          static_cast<uint32_t>(rv), JOURNAL_NAME));
2096     FinishRead(false);
2097   }
2098
2099   mTmpFileOpener = new FileOpenHelper(this);
2100   rv = CacheFileIOManager::OpenFile(
2101       nsLiteralCString(TEMP_INDEX_NAME),
2102       CacheFileIOManager::SPECIAL_FILE | CacheFileIOManager::OPEN,
2103       mTmpFileOpener);
2104   if (NS_FAILED(rv)) {
2105     LOG(
2106         ("CacheIndex::ReadIndexFromDisk() - CacheFileIOManager::OpenFile() "
2107          "failed [rv=0x%08" PRIx32 ", file=%s]",
2108          static_cast<uint32_t>(rv), TEMP_INDEX_NAME));
2109     FinishRead(false);
2110   }
2111 }
2112
2113 void CacheIndex::StartReadingIndex() {
2114   LOG(("CacheIndex::StartReadingIndex()"));
2115
2116   nsresult rv;
2117
2118   sLock.AssertCurrentThreadOwns();
2119
2120   MOZ_ASSERT(mIndexHandle);
2121   MOZ_ASSERT(mState == READING);
2122   MOZ_ASSERT(!mIndexOnDiskIsValid);
2123   MOZ_ASSERT(!mDontMarkIndexClean);
2124   MOZ_ASSERT(!mJournalReadSuccessfully);
2125   MOZ_ASSERT(mIndexHandle->FileSize() >= 0);
2126   MOZ_ASSERT(!mRWPending);
2127
2128   int64_t entriesSize = mIndexHandle->FileSize() - sizeof(CacheIndexHeader) -
2129                         sizeof(CacheHash::Hash32_t);
2130
2131   if (entriesSize < 0 || entriesSize % sizeof(CacheIndexRecord)) {
2132     LOG(("CacheIndex::StartReadingIndex() - Index is corrupted"));
2133     FinishRead(false);
2134     return;
2135   }
2136
2137   AllocBuffer();
2138   mSkipEntries = 0;
2139   mRWHash = new CacheHash();
2140
2141   mRWBufPos =
2142       std::min(mRWBufSize, static_cast<uint32_t>(mIndexHandle->FileSize()));
2143
2144   rv = CacheFileIOManager::Read(mIndexHandle, 0, mRWBuf, mRWBufPos, this);
2145   if (NS_FAILED(rv)) {
2146     LOG(
2147         ("CacheIndex::StartReadingIndex() - CacheFileIOManager::Read() failed "
2148          "synchronously [rv=0x%08" PRIx32 "]",
2149          static_cast<uint32_t>(rv)));
2150     FinishRead(false);
2151   } else {
2152     mRWPending = true;
2153   }
2154 }
2155
2156 void CacheIndex::ParseRecords() {
2157   LOG(("CacheIndex::ParseRecords()"));
2158
2159   nsresult rv;
2160
2161   sLock.AssertCurrentThreadOwns();
2162
2163   MOZ_ASSERT(!mRWPending);
2164
2165   uint32_t entryCnt = (mIndexHandle->FileSize() - sizeof(CacheIndexHeader) -
2166                        sizeof(CacheHash::Hash32_t)) /
2167                       sizeof(CacheIndexRecord);
2168   uint32_t pos = 0;
2169
2170   if (!mSkipEntries) {
2171     if (NetworkEndian::readUint32(mRWBuf + pos) != kIndexVersion) {
2172       FinishRead(false);
2173       return;
2174     }
2175     pos += sizeof(uint32_t);
2176
2177     mIndexTimeStamp = NetworkEndian::readUint32(mRWBuf + pos);
2178     pos += sizeof(uint32_t);
2179
2180     if (NetworkEndian::readUint32(mRWBuf + pos)) {
2181       if (mJournalHandle) {
2182         CacheFileIOManager::DoomFile(mJournalHandle, nullptr);
2183         mJournalHandle = nullptr;
2184       }
2185     } else {
2186       uint32_t* isDirty =
2187           reinterpret_cast<uint32_t*>(moz_xmalloc(sizeof(uint32_t)));
2188       NetworkEndian::writeUint32(isDirty, 1);
2189
2190       // Mark index dirty. The buffer is freed by CacheFileIOManager when
2191       // nullptr is passed as the listener and the call doesn't fail
2192       // synchronously.
2193       rv = CacheFileIOManager::Write(mIndexHandle, 2 * sizeof(uint32_t),
2194                                      reinterpret_cast<char*>(isDirty),
2195                                      sizeof(uint32_t), true, false, nullptr);
2196       if (NS_FAILED(rv)) {
2197         // This is not fatal, just free the memory
2198         free(isDirty);
2199       }
2200     }
2201     pos += sizeof(uint32_t);
2202   }
2203
2204   uint32_t hashOffset = pos;
2205
2206   while (pos + sizeof(CacheIndexRecord) <= mRWBufPos &&
2207          mSkipEntries != entryCnt) {
2208     CacheIndexRecord* rec = reinterpret_cast<CacheIndexRecord*>(mRWBuf + pos);
2209     CacheIndexEntry tmpEntry(&rec->mHash);
2210     tmpEntry.ReadFromBuf(mRWBuf + pos);
2211
2212     if (tmpEntry.IsDirty() || !tmpEntry.IsInitialized() ||
2213         tmpEntry.IsFileEmpty() || tmpEntry.IsFresh() || tmpEntry.IsRemoved()) {
2214       LOG(
2215           ("CacheIndex::ParseRecords() - Invalid entry found in index, removing"
2216            " whole index [dirty=%d, initialized=%d, fileEmpty=%d, fresh=%d, "
2217            "removed=%d]",
2218            tmpEntry.IsDirty(), tmpEntry.IsInitialized(), tmpEntry.IsFileEmpty(),
2219            tmpEntry.IsFresh(), tmpEntry.IsRemoved()));
2220       FinishRead(false);
2221       return;
2222     }
2223
2224     CacheIndexEntryAutoManage emng(tmpEntry.Hash(), this);
2225
2226     CacheIndexEntry* entry = mIndex.PutEntry(*tmpEntry.Hash());
2227     *entry = tmpEntry;
2228
2229     pos += sizeof(CacheIndexRecord);
2230     mSkipEntries++;
2231   }
2232
2233   mRWHash->Update(mRWBuf + hashOffset, pos - hashOffset);
2234
2235   if (pos != mRWBufPos) {
2236     memmove(mRWBuf, mRWBuf + pos, mRWBufPos - pos);
2237   }
2238
2239   mRWBufPos -= pos;
2240   pos = 0;
2241
2242   int64_t fileOffset = sizeof(CacheIndexHeader) +
2243                        mSkipEntries * sizeof(CacheIndexRecord) + mRWBufPos;
2244
2245   MOZ_ASSERT(fileOffset <= mIndexHandle->FileSize());
2246   if (fileOffset == mIndexHandle->FileSize()) {
2247     uint32_t expectedHash = NetworkEndian::readUint32(mRWBuf);
2248     if (mRWHash->GetHash() != expectedHash) {
2249       LOG(("CacheIndex::ParseRecords() - Hash mismatch, [is %x, should be %x]",
2250            mRWHash->GetHash(), expectedHash));
2251       FinishRead(false);
2252       return;
2253     }
2254
2255     mIndexOnDiskIsValid = true;
2256     mJournalReadSuccessfully = false;
2257
2258     if (mJournalHandle) {
2259       StartReadingJournal();
2260     } else {
2261       FinishRead(false);
2262     }
2263
2264     return;
2265   }
2266
2267   pos = mRWBufPos;
2268   uint32_t toRead =
2269       std::min(mRWBufSize - pos,
2270                static_cast<uint32_t>(mIndexHandle->FileSize() - fileOffset));
2271   mRWBufPos = pos + toRead;
2272
2273   rv = CacheFileIOManager::Read(mIndexHandle, fileOffset, mRWBuf + pos, toRead,
2274                                 this);
2275   if (NS_FAILED(rv)) {
2276     LOG(
2277         ("CacheIndex::ParseRecords() - CacheFileIOManager::Read() failed "
2278          "synchronously [rv=0x%08" PRIx32 "]",
2279          static_cast<uint32_t>(rv)));
2280     FinishRead(false);
2281     return;
2282   }
2283   mRWPending = true;
2284 }
2285
2286 void CacheIndex::StartReadingJournal() {
2287   LOG(("CacheIndex::StartReadingJournal()"));
2288
2289   nsresult rv;
2290
2291   sLock.AssertCurrentThreadOwns();
2292
2293   MOZ_ASSERT(mJournalHandle);
2294   MOZ_ASSERT(mIndexOnDiskIsValid);
2295   MOZ_ASSERT(mTmpJournal.Count() == 0);
2296   MOZ_ASSERT(mJournalHandle->FileSize() >= 0);
2297   MOZ_ASSERT(!mRWPending);
2298
2299   int64_t entriesSize =
2300       mJournalHandle->FileSize() - sizeof(CacheHash::Hash32_t);
2301
2302   if (entriesSize < 0 || entriesSize % sizeof(CacheIndexRecord)) {
2303     LOG(("CacheIndex::StartReadingJournal() - Journal is corrupted"));
2304     FinishRead(false);
2305     return;
2306   }
2307
2308   mSkipEntries = 0;
2309   mRWHash = new CacheHash();
2310
2311   mRWBufPos =
2312       std::min(mRWBufSize, static_cast<uint32_t>(mJournalHandle->FileSize()));
2313
2314   rv = CacheFileIOManager::Read(mJournalHandle, 0, mRWBuf, mRWBufPos, this);
2315   if (NS_FAILED(rv)) {
2316     LOG(
2317         ("CacheIndex::StartReadingJournal() - CacheFileIOManager::Read() failed"
2318          " synchronously [rv=0x%08" PRIx32 "]",
2319          static_cast<uint32_t>(rv)));
2320     FinishRead(false);
2321   } else {
2322     mRWPending = true;
2323   }
2324 }
2325
2326 void CacheIndex::ParseJournal() {
2327   LOG(("CacheIndex::ParseJournal()"));
2328
2329   nsresult rv;
2330
2331   sLock.AssertCurrentThreadOwns();
2332
2333   MOZ_ASSERT(!mRWPending);
2334
2335   uint32_t entryCnt =
2336       (mJournalHandle->FileSize() - sizeof(CacheHash::Hash32_t)) /
2337       sizeof(CacheIndexRecord);
2338
2339   uint32_t pos = 0;
2340
2341   while (pos + sizeof(CacheIndexRecord) <= mRWBufPos &&
2342          mSkipEntries != entryCnt) {
2343     CacheIndexEntry tmpEntry(reinterpret_cast<SHA1Sum::Hash*>(mRWBuf + pos));
2344     tmpEntry.ReadFromBuf(mRWBuf + pos);
2345
2346     CacheIndexEntry* entry = mTmpJournal.PutEntry(*tmpEntry.Hash());
2347     *entry = tmpEntry;
2348
2349     if (entry->IsDirty() || entry->IsFresh()) {
2350       LOG(
2351           ("CacheIndex::ParseJournal() - Invalid entry found in journal, "
2352            "ignoring whole journal [dirty=%d, fresh=%d]",
2353            entry->IsDirty(), entry->IsFresh()));
2354       FinishRead(false);
2355       return;
2356     }
2357
2358     pos += sizeof(CacheIndexRecord);
2359     mSkipEntries++;
2360   }
2361
2362   mRWHash->Update(mRWBuf, pos);
2363
2364   if (pos != mRWBufPos) {
2365     memmove(mRWBuf, mRWBuf + pos, mRWBufPos - pos);
2366   }
2367
2368   mRWBufPos -= pos;
2369   pos = 0;
2370
2371   int64_t fileOffset = mSkipEntries * sizeof(CacheIndexRecord) + mRWBufPos;
2372
2373   MOZ_ASSERT(fileOffset <= mJournalHandle->FileSize());
2374   if (fileOffset == mJournalHandle->FileSize()) {
2375     uint32_t expectedHash = NetworkEndian::readUint32(mRWBuf);
2376     if (mRWHash->GetHash() != expectedHash) {
2377       LOG(("CacheIndex::ParseJournal() - Hash mismatch, [is %x, should be %x]",
2378            mRWHash->GetHash(), expectedHash));
2379       FinishRead(false);
2380       return;
2381     }
2382
2383     mJournalReadSuccessfully = true;
2384     FinishRead(true);
2385     return;
2386   }
2387
2388   pos = mRWBufPos;
2389   uint32_t toRead =
2390       std::min(mRWBufSize - pos,
2391                static_cast<uint32_t>(mJournalHandle->FileSize() - fileOffset));
2392   mRWBufPos = pos + toRead;
2393
2394   rv = CacheFileIOManager::Read(mJournalHandle, fileOffset, mRWBuf + pos,
2395                                 toRead, this);
2396   if (NS_FAILED(rv)) {
2397     LOG(
2398         ("CacheIndex::ParseJournal() - CacheFileIOManager::Read() failed "
2399          "synchronously [rv=0x%08" PRIx32 "]",
2400          static_cast<uint32_t>(rv)));
2401     FinishRead(false);
2402     return;
2403   }
2404   mRWPending = true;
2405 }
2406
2407 void CacheIndex::MergeJournal() {
2408   LOG(("CacheIndex::MergeJournal()"));
2409
2410   sLock.AssertCurrentThreadOwns();
2411
2412   for (auto iter = mTmpJournal.Iter(); !iter.Done(); iter.Next()) {
2413     CacheIndexEntry* entry = iter.Get();
2414
2415     LOG(("CacheIndex::MergeJournal() [hash=%08x%08x%08x%08x%08x]",
2416          LOGSHA1(entry->Hash())));
2417
2418     CacheIndexEntry* entry2 = mIndex.GetEntry(*entry->Hash());
2419     {
2420       CacheIndexEntryAutoManage emng(entry->Hash(), this);
2421       if (entry->IsRemoved()) {
2422         if (entry2) {
2423           entry2->MarkRemoved();
2424           entry2->MarkDirty();
2425         }
2426       } else {
2427         if (!entry2) {
2428           entry2 = mIndex.PutEntry(*entry->Hash());
2429         }
2430
2431         *entry2 = *entry;
2432         entry2->MarkDirty();
2433       }
2434     }
2435     iter.Remove();
2436   }
2437
2438   MOZ_ASSERT(mTmpJournal.Count() == 0);
2439 }
2440
2441 void CacheIndex::EnsureNoFreshEntry() {
2442 #ifdef DEBUG_STATS
2443   CacheIndexStats debugStats;
2444   debugStats.DisableLogging();
2445   for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) {
2446     debugStats.BeforeChange(nullptr);
2447     debugStats.AfterChange(iter.Get());
2448   }
2449   MOZ_ASSERT(debugStats.Fresh() == 0);
2450 #endif
2451 }
2452
2453 void CacheIndex::EnsureCorrectStats() {
2454 #ifdef DEBUG_STATS
2455   MOZ_ASSERT(mPendingUpdates.Count() == 0);
2456   CacheIndexStats debugStats;
2457   debugStats.DisableLogging();
2458   for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) {
2459     debugStats.BeforeChange(nullptr);
2460     debugStats.AfterChange(iter.Get());
2461   }
2462   MOZ_ASSERT(debugStats == mIndexStats);
2463 #endif
2464 }
2465
2466 void CacheIndex::FinishRead(bool aSucceeded) {
2467   LOG(("CacheIndex::FinishRead() [succeeded=%d]", aSucceeded));
2468   sLock.AssertCurrentThreadOwns();
2469
2470   MOZ_ASSERT((!aSucceeded && mState == SHUTDOWN) || mState == READING);
2471
2472   MOZ_ASSERT(
2473       // -> rebuild
2474       (!aSucceeded && !mIndexOnDiskIsValid && !mJournalReadSuccessfully) ||
2475       // -> update
2476       (!aSucceeded && mIndexOnDiskIsValid && !mJournalReadSuccessfully) ||
2477       // -> ready
2478       (aSucceeded && mIndexOnDiskIsValid && mJournalReadSuccessfully));
2479
2480   // If there is read operation pending we must be cancelling reading of the
2481   // index when shutting down or removing the whole index.
2482   MOZ_ASSERT(!mRWPending || (!aSucceeded && (mShuttingDown || mRemovingAll)));
2483
2484   if (mState == SHUTDOWN) {
2485     RemoveFile(nsLiteralCString(TEMP_INDEX_NAME));
2486     RemoveFile(nsLiteralCString(JOURNAL_NAME));
2487   } else {
2488     if (mIndexHandle && !mIndexOnDiskIsValid) {
2489       CacheFileIOManager::DoomFile(mIndexHandle, nullptr);
2490     }
2491
2492     if (mJournalHandle) {
2493       CacheFileIOManager::DoomFile(mJournalHandle, nullptr);
2494     }
2495   }
2496
2497   if (mIndexFileOpener) {
2498     mIndexFileOpener->Cancel();
2499     mIndexFileOpener = nullptr;
2500   }
2501   if (mJournalFileOpener) {
2502     mJournalFileOpener->Cancel();
2503     mJournalFileOpener = nullptr;
2504   }
2505   if (mTmpFileOpener) {
2506     mTmpFileOpener->Cancel();
2507     mTmpFileOpener = nullptr;
2508   }
2509
2510   mIndexHandle = nullptr;
2511   mJournalHandle = nullptr;
2512   mRWHash = nullptr;
2513   ReleaseBuffer();
2514
2515   if (mState == SHUTDOWN) {
2516     return;
2517   }
2518
2519   if (!mIndexOnDiskIsValid) {
2520     MOZ_ASSERT(mTmpJournal.Count() == 0);
2521     EnsureNoFreshEntry();
2522     ProcessPendingOperations();
2523     // Remove all entries that we haven't seen during this session
2524     RemoveNonFreshEntries();
2525     StartUpdatingIndex(true);
2526     return;
2527   }
2528
2529   if (!mJournalReadSuccessfully) {
2530     mTmpJournal.Clear();
2531     EnsureNoFreshEntry();
2532     ProcessPendingOperations();
2533     StartUpdatingIndex(false);
2534     return;
2535   }
2536
2537   MergeJournal();
2538   EnsureNoFreshEntry();
2539   ProcessPendingOperations();
2540   mIndexStats.Log();
2541
2542   ChangeState(READY);
2543   mLastDumpTime = TimeStamp::NowLoRes();  // Do not dump new index immediately
2544 }
2545
2546 // static
2547 void CacheIndex::DelayedUpdate(nsITimer* aTimer, void* aClosure) {
2548   LOG(("CacheIndex::DelayedUpdate()"));
2549
2550   StaticMutexAutoLock lock(sLock);
2551   RefPtr<CacheIndex> index = gInstance;
2552
2553   if (!index) {
2554     return;
2555   }
2556
2557   index->DelayedUpdateLocked();
2558 }
2559
2560 // static
2561 void CacheIndex::DelayedUpdateLocked() {
2562   LOG(("CacheIndex::DelayedUpdateLocked()"));
2563
2564   sLock.AssertCurrentThreadOwns();
2565
2566   nsresult rv;
2567
2568   mUpdateTimer = nullptr;
2569
2570   if (!IsIndexUsable()) {
2571     return;
2572   }
2573
2574   if (mState == READY && mShuttingDown) {
2575     return;
2576   }
2577
2578   // mUpdateEventPending must be false here since StartUpdatingIndex() won't
2579   // schedule timer if it is true.
2580   MOZ_ASSERT(!mUpdateEventPending);
2581   if (mState != BUILDING && mState != UPDATING) {
2582     LOG(("CacheIndex::DelayedUpdateLocked() - Update was canceled"));
2583     return;
2584   }
2585
2586   // We need to redispatch to run with lower priority
2587   RefPtr<CacheIOThread> ioThread = CacheFileIOManager::IOThread();
2588   MOZ_ASSERT(ioThread);
2589
2590   mUpdateEventPending = true;
2591   rv = ioThread->Dispatch(this, CacheIOThread::INDEX);
2592   if (NS_FAILED(rv)) {
2593     mUpdateEventPending = false;
2594     NS_WARNING("CacheIndex::DelayedUpdateLocked() - Can't dispatch event");
2595     LOG(("CacheIndex::DelayedUpdate() - Can't dispatch event"));
2596     FinishUpdate(false);
2597   }
2598 }
2599
2600 nsresult CacheIndex::ScheduleUpdateTimer(uint32_t aDelay) {
2601   LOG(("CacheIndex::ScheduleUpdateTimer() [delay=%u]", aDelay));
2602
2603   MOZ_ASSERT(!mUpdateTimer);
2604
2605   nsCOMPtr<nsIEventTarget> ioTarget = CacheFileIOManager::IOTarget();
2606   MOZ_ASSERT(ioTarget);
2607
2608   return NS_NewTimerWithFuncCallback(
2609       getter_AddRefs(mUpdateTimer), CacheIndex::DelayedUpdate, nullptr, aDelay,
2610       nsITimer::TYPE_ONE_SHOT, "net::CacheIndex::ScheduleUpdateTimer",
2611       ioTarget);
2612 }
2613
2614 nsresult CacheIndex::SetupDirectoryEnumerator() {
2615   MOZ_ASSERT(!NS_IsMainThread());
2616   MOZ_ASSERT(!mDirEnumerator);
2617
2618   nsresult rv;
2619   nsCOMPtr<nsIFile> file;
2620
2621   rv = mCacheDirectory->Clone(getter_AddRefs(file));
2622   NS_ENSURE_SUCCESS(rv, rv);
2623
2624   rv = file->AppendNative(nsLiteralCString(ENTRIES_DIR));
2625   NS_ENSURE_SUCCESS(rv, rv);
2626
2627   bool exists;
2628   rv = file->Exists(&exists);
2629   NS_ENSURE_SUCCESS(rv, rv);
2630
2631   if (!exists) {
2632     NS_WARNING(
2633         "CacheIndex::SetupDirectoryEnumerator() - Entries directory "
2634         "doesn't exist!");
2635     LOG(
2636         ("CacheIndex::SetupDirectoryEnumerator() - Entries directory doesn't "
2637          "exist!"));
2638     return NS_ERROR_UNEXPECTED;
2639   }
2640
2641   rv = file->GetDirectoryEntries(getter_AddRefs(mDirEnumerator));
2642   NS_ENSURE_SUCCESS(rv, rv);
2643
2644   return NS_OK;
2645 }
2646
2647 nsresult CacheIndex::InitEntryFromDiskData(CacheIndexEntry* aEntry,
2648                                            CacheFileMetadata* aMetaData,
2649                                            int64_t aFileSize) {
2650   nsresult rv;
2651
2652   aEntry->InitNew();
2653   aEntry->MarkDirty();
2654   aEntry->MarkFresh();
2655
2656   aEntry->Init(GetOriginAttrsHash(aMetaData->OriginAttributes()),
2657                aMetaData->IsAnonymous(), aMetaData->Pinned());
2658
2659   aEntry->SetFrecency(aMetaData->GetFrecency());
2660
2661   const char* altData = aMetaData->GetElement(CacheFileUtils::kAltDataKey);
2662   bool hasAltData = altData ? true : false;
2663   if (hasAltData && NS_FAILED(CacheFileUtils::ParseAlternativeDataInfo(
2664                         altData, nullptr, nullptr))) {
2665     return NS_ERROR_FAILURE;
2666   }
2667   aEntry->SetHasAltData(hasAltData);
2668
2669   static auto toUint16 = [](const char* aUint16String) -> uint16_t {
2670     if (!aUint16String) {
2671       return kIndexTimeNotAvailable;
2672     }
2673     nsresult rv;
2674     uint64_t n64 = nsDependentCString(aUint16String).ToInteger64(&rv);
2675     MOZ_ASSERT(NS_SUCCEEDED(rv));
2676     return n64 <= kIndexTimeOutOfBound ? n64 : kIndexTimeOutOfBound;
2677   };
2678
2679   aEntry->SetOnStartTime(
2680       toUint16(aMetaData->GetElement("net-response-time-onstart")));
2681   aEntry->SetOnStopTime(
2682       toUint16(aMetaData->GetElement("net-response-time-onstop")));
2683
2684   const char* contentTypeStr = aMetaData->GetElement("ctid");
2685   uint8_t contentType = nsICacheEntry::CONTENT_TYPE_UNKNOWN;
2686   if (contentTypeStr) {
2687     int64_t n64 = nsDependentCString(contentTypeStr).ToInteger64(&rv);
2688     if (NS_FAILED(rv) || n64 < nsICacheEntry::CONTENT_TYPE_UNKNOWN ||
2689         n64 >= nsICacheEntry::CONTENT_TYPE_LAST) {
2690       n64 = nsICacheEntry::CONTENT_TYPE_UNKNOWN;
2691     }
2692     contentType = n64;
2693   }
2694   aEntry->SetContentType(contentType);
2695
2696   aEntry->SetFileSize(static_cast<uint32_t>(std::min(
2697       static_cast<int64_t>(PR_UINT32_MAX), (aFileSize + 0x3FF) >> 10)));
2698   return NS_OK;
2699 }
2700
2701 bool CacheIndex::IsUpdatePending() {
2702   sLock.AssertCurrentThreadOwns();
2703
2704   if (mUpdateTimer || mUpdateEventPending) {
2705     return true;
2706   }
2707
2708   return false;
2709 }
2710
2711 void CacheIndex::BuildIndex() {
2712   LOG(("CacheIndex::BuildIndex()"));
2713
2714   sLock.AssertCurrentThreadOwns();
2715
2716   MOZ_ASSERT(mPendingUpdates.Count() == 0);
2717
2718   nsresult rv;
2719
2720   if (!mDirEnumerator) {
2721     {
2722       // Do not do IO under the lock.
2723       StaticMutexAutoUnlock unlock(sLock);
2724       rv = SetupDirectoryEnumerator();
2725     }
2726     if (mState == SHUTDOWN) {
2727       // The index was shut down while we released the lock. FinishUpdate() was
2728       // already called from Shutdown(), so just simply return here.
2729       return;
2730     }
2731
2732     if (NS_FAILED(rv)) {
2733       FinishUpdate(false);
2734       return;
2735     }
2736   }
2737
2738   while (true) {
2739     if (CacheIOThread::YieldAndRerun()) {
2740       LOG((
2741           "CacheIndex::BuildIndex() - Breaking loop for higher level events."));
2742       mUpdateEventPending = true;
2743       return;
2744     }
2745
2746     bool fileExists = false;
2747     nsCOMPtr<nsIFile> file;
2748     {
2749       // Do not do IO under the lock.
2750       StaticMutexAutoUnlock unlock(sLock);
2751       rv = mDirEnumerator->GetNextFile(getter_AddRefs(file));
2752
2753       if (file) {
2754         file->Exists(&fileExists);
2755       }
2756     }
2757     if (mState == SHUTDOWN) {
2758       return;
2759     }
2760     if (!file) {
2761       FinishUpdate(NS_SUCCEEDED(rv));
2762       return;
2763     }
2764
2765     nsAutoCString leaf;
2766     rv = file->GetNativeLeafName(leaf);
2767     if (NS_FAILED(rv)) {
2768       LOG(
2769           ("CacheIndex::BuildIndex() - GetNativeLeafName() failed! Skipping "
2770            "file."));
2771       mDontMarkIndexClean = true;
2772       continue;
2773     }
2774
2775     if (!fileExists) {
2776       LOG(
2777           ("CacheIndex::BuildIndex() - File returned by the iterator was "
2778            "removed in the meantime [name=%s]",
2779            leaf.get()));
2780       continue;
2781     }
2782
2783     SHA1Sum::Hash hash;
2784     rv = CacheFileIOManager::StrToHash(leaf, &hash);
2785     if (NS_FAILED(rv)) {
2786       LOG(
2787           ("CacheIndex::BuildIndex() - Filename is not a hash, removing file. "
2788            "[name=%s]",
2789            leaf.get()));
2790       file->Remove(false);
2791       continue;
2792     }
2793
2794     CacheIndexEntry* entry = mIndex.GetEntry(hash);
2795     if (entry && entry->IsRemoved()) {
2796       LOG(
2797           ("CacheIndex::BuildIndex() - Found file that should not exist. "
2798            "[name=%s]",
2799            leaf.get()));
2800       entry->Log();
2801       MOZ_ASSERT(entry->IsFresh());
2802       entry = nullptr;
2803     }
2804
2805 #ifdef DEBUG
2806     RefPtr<CacheFileHandle> handle;
2807     CacheFileIOManager::gInstance->mHandles.GetHandle(&hash,
2808                                                       getter_AddRefs(handle));
2809 #endif
2810
2811     if (entry) {
2812       // the entry is up to date
2813       LOG(
2814           ("CacheIndex::BuildIndex() - Skipping file because the entry is up to"
2815            " date. [name=%s]",
2816            leaf.get()));
2817       entry->Log();
2818       MOZ_ASSERT(entry->IsFresh());  // The entry must be from this session
2819       // there must be an active CacheFile if the entry is not initialized
2820       MOZ_ASSERT(entry->IsInitialized() || handle);
2821       continue;
2822     }
2823
2824     MOZ_ASSERT(!handle);
2825
2826     RefPtr<CacheFileMetadata> meta = new CacheFileMetadata();
2827     int64_t size = 0;
2828
2829     {
2830       // Do not do IO under the lock.
2831       StaticMutexAutoUnlock unlock(sLock);
2832       rv = meta->SyncReadMetadata(file);
2833
2834       if (NS_SUCCEEDED(rv)) {
2835         rv = file->GetFileSize(&size);
2836         if (NS_FAILED(rv)) {
2837           LOG(
2838               ("CacheIndex::BuildIndex() - Cannot get filesize of file that was"
2839                " successfully parsed. [name=%s]",
2840                leaf.get()));
2841         }
2842       }
2843     }
2844     if (mState == SHUTDOWN) {
2845       return;
2846     }
2847
2848     // Nobody could add the entry while the lock was released since we modify
2849     // the index only on IO thread and this loop is executed on IO thread too.
2850     entry = mIndex.GetEntry(hash);
2851     MOZ_ASSERT(!entry || entry->IsRemoved());
2852
2853     if (NS_FAILED(rv)) {
2854       LOG(
2855           ("CacheIndex::BuildIndex() - CacheFileMetadata::SyncReadMetadata() "
2856            "failed, removing file. [name=%s]",
2857            leaf.get()));
2858       file->Remove(false);
2859     } else {
2860       CacheIndexEntryAutoManage entryMng(&hash, this);
2861       entry = mIndex.PutEntry(hash);
2862       if (NS_FAILED(InitEntryFromDiskData(entry, meta, size))) {
2863         LOG(
2864             ("CacheIndex::BuildIndex() - CacheFile::InitEntryFromDiskData() "
2865              "failed, removing file. [name=%s]",
2866              leaf.get()));
2867         file->Remove(false);
2868         entry->MarkRemoved();
2869       } else {
2870         LOG(("CacheIndex::BuildIndex() - Added entry to index. [name=%s]",
2871              leaf.get()));
2872         entry->Log();
2873       }
2874     }
2875   }
2876
2877   MOZ_ASSERT_UNREACHABLE("We should never get here");
2878 }
2879
2880 bool CacheIndex::StartUpdatingIndexIfNeeded(bool aSwitchingToReadyState) {
2881   // Start updating process when we are in or we are switching to READY state
2882   // and index needs update, but not during shutdown or when removing all
2883   // entries.
2884   if ((mState == READY || aSwitchingToReadyState) && mIndexNeedsUpdate &&
2885       !mShuttingDown && !mRemovingAll) {
2886     LOG(("CacheIndex::StartUpdatingIndexIfNeeded() - starting update process"));
2887     mIndexNeedsUpdate = false;
2888     StartUpdatingIndex(false);
2889     return true;
2890   }
2891
2892   return false;
2893 }
2894
2895 void CacheIndex::StartUpdatingIndex(bool aRebuild) {
2896   LOG(("CacheIndex::StartUpdatingIndex() [rebuild=%d]", aRebuild));
2897
2898   sLock.AssertCurrentThreadOwns();
2899
2900   nsresult rv;
2901
2902   mIndexStats.Log();
2903
2904   ChangeState(aRebuild ? BUILDING : UPDATING);
2905   mDontMarkIndexClean = false;
2906
2907   if (mShuttingDown || mRemovingAll) {
2908     FinishUpdate(false);
2909     return;
2910   }
2911
2912   if (IsUpdatePending()) {
2913     LOG(("CacheIndex::StartUpdatingIndex() - Update is already pending"));
2914     return;
2915   }
2916
2917   uint32_t elapsed = (TimeStamp::NowLoRes() - mStartTime).ToMilliseconds();
2918   if (elapsed < kUpdateIndexStartDelay) {
2919     LOG(
2920         ("CacheIndex::StartUpdatingIndex() - %u ms elapsed since startup, "
2921          "scheduling timer to fire in %u ms.",
2922          elapsed, kUpdateIndexStartDelay - elapsed));
2923     rv = ScheduleUpdateTimer(kUpdateIndexStartDelay - elapsed);
2924     if (NS_SUCCEEDED(rv)) {
2925       return;
2926     }
2927
2928     LOG(
2929         ("CacheIndex::StartUpdatingIndex() - ScheduleUpdateTimer() failed. "
2930          "Starting update immediately."));
2931   } else {
2932     LOG(
2933         ("CacheIndex::StartUpdatingIndex() - %u ms elapsed since startup, "
2934          "starting update now.",
2935          elapsed));
2936   }
2937
2938   RefPtr<CacheIOThread> ioThread = CacheFileIOManager::IOThread();
2939   MOZ_ASSERT(ioThread);
2940
2941   // We need to dispatch an event even if we are on IO thread since we need to
2942   // update the index with the correct priority.
2943   mUpdateEventPending = true;
2944   rv = ioThread->Dispatch(this, CacheIOThread::INDEX);
2945   if (NS_FAILED(rv)) {
2946     mUpdateEventPending = false;
2947     NS_WARNING("CacheIndex::StartUpdatingIndex() - Can't dispatch event");
2948     LOG(("CacheIndex::StartUpdatingIndex() - Can't dispatch event"));
2949     FinishUpdate(false);
2950   }
2951 }
2952
2953 void CacheIndex::UpdateIndex() {
2954   LOG(("CacheIndex::UpdateIndex()"));
2955
2956   sLock.AssertCurrentThreadOwns();
2957
2958   MOZ_ASSERT(mPendingUpdates.Count() == 0);
2959
2960   nsresult rv;
2961
2962   if (!mDirEnumerator) {
2963     {
2964       // Do not do IO under the lock.
2965       StaticMutexAutoUnlock unlock(sLock);
2966       rv = SetupDirectoryEnumerator();
2967     }
2968     if (mState == SHUTDOWN) {
2969       // The index was shut down while we released the lock. FinishUpdate() was
2970       // already called from Shutdown(), so just simply return here.
2971       return;
2972     }
2973
2974     if (NS_FAILED(rv)) {
2975       FinishUpdate(false);
2976       return;
2977     }
2978   }
2979
2980   while (true) {
2981     if (CacheIOThread::YieldAndRerun()) {
2982       LOG(
2983           ("CacheIndex::UpdateIndex() - Breaking loop for higher level "
2984            "events."));
2985       mUpdateEventPending = true;
2986       return;
2987     }
2988
2989     bool fileExists = false;
2990     nsCOMPtr<nsIFile> file;
2991     {
2992       // Do not do IO under the lock.
2993       StaticMutexAutoUnlock unlock(sLock);
2994       rv = mDirEnumerator->GetNextFile(getter_AddRefs(file));
2995
2996       if (file) {
2997         file->Exists(&fileExists);
2998       }
2999     }
3000     if (mState == SHUTDOWN) {
3001       return;
3002     }
3003     if (!file) {
3004       FinishUpdate(NS_SUCCEEDED(rv));
3005       return;
3006     }
3007
3008     nsAutoCString leaf;
3009     rv = file->GetNativeLeafName(leaf);
3010     if (NS_FAILED(rv)) {
3011       LOG(
3012           ("CacheIndex::UpdateIndex() - GetNativeLeafName() failed! Skipping "
3013            "file."));
3014       mDontMarkIndexClean = true;
3015       continue;
3016     }
3017
3018     if (!fileExists) {
3019       LOG(
3020           ("CacheIndex::UpdateIndex() - File returned by the iterator was "
3021            "removed in the meantime [name=%s]",
3022            leaf.get()));
3023       continue;
3024     }
3025
3026     SHA1Sum::Hash hash;
3027     rv = CacheFileIOManager::StrToHash(leaf, &hash);
3028     if (NS_FAILED(rv)) {
3029       LOG(
3030           ("CacheIndex::UpdateIndex() - Filename is not a hash, removing file. "
3031            "[name=%s]",
3032            leaf.get()));
3033       file->Remove(false);
3034       continue;
3035     }
3036
3037     CacheIndexEntry* entry = mIndex.GetEntry(hash);
3038     if (entry && entry->IsRemoved()) {
3039       if (entry->IsFresh()) {
3040         LOG(
3041             ("CacheIndex::UpdateIndex() - Found file that should not exist. "
3042              "[name=%s]",
3043              leaf.get()));
3044         entry->Log();
3045       }
3046       entry = nullptr;
3047     }
3048
3049 #ifdef DEBUG
3050     RefPtr<CacheFileHandle> handle;
3051     CacheFileIOManager::gInstance->mHandles.GetHandle(&hash,
3052                                                       getter_AddRefs(handle));
3053 #endif
3054
3055     if (entry && entry->IsFresh()) {
3056       // the entry is up to date
3057       LOG(
3058           ("CacheIndex::UpdateIndex() - Skipping file because the entry is up "
3059            " to date. [name=%s]",
3060            leaf.get()));
3061       entry->Log();
3062       // there must be an active CacheFile if the entry is not initialized
3063       MOZ_ASSERT(entry->IsInitialized() || handle);
3064       continue;
3065     }
3066
3067     MOZ_ASSERT(!handle);
3068
3069     if (entry) {
3070       PRTime lastModifiedTime;
3071       {
3072         // Do not do IO under the lock.
3073         StaticMutexAutoUnlock unlock(sLock);
3074         rv = file->GetLastModifiedTime(&lastModifiedTime);
3075       }
3076       if (mState == SHUTDOWN) {
3077         return;
3078       }
3079       if (NS_FAILED(rv)) {
3080         LOG(
3081             ("CacheIndex::UpdateIndex() - Cannot get lastModifiedTime. "
3082              "[name=%s]",
3083              leaf.get()));
3084         // Assume the file is newer than index
3085       } else {
3086         if (mIndexTimeStamp > (lastModifiedTime / PR_MSEC_PER_SEC)) {
3087           LOG(
3088               ("CacheIndex::UpdateIndex() - Skipping file because of last "
3089                "modified time. [name=%s, indexTimeStamp=%" PRIu32 ", "
3090                "lastModifiedTime=%" PRId64 "]",
3091                leaf.get(), mIndexTimeStamp,
3092                lastModifiedTime / PR_MSEC_PER_SEC));
3093
3094           CacheIndexEntryAutoManage entryMng(&hash, this);
3095           entry->MarkFresh();
3096           continue;
3097         }
3098       }
3099     }
3100
3101     RefPtr<CacheFileMetadata> meta = new CacheFileMetadata();
3102     int64_t size = 0;
3103
3104     {
3105       // Do not do IO under the lock.
3106       StaticMutexAutoUnlock unlock(sLock);
3107       rv = meta->SyncReadMetadata(file);
3108
3109       if (NS_SUCCEEDED(rv)) {
3110         rv = file->GetFileSize(&size);
3111         if (NS_FAILED(rv)) {
3112           LOG(
3113               ("CacheIndex::UpdateIndex() - Cannot get filesize of file that "
3114                "was successfully parsed. [name=%s]",
3115                leaf.get()));
3116         }
3117       }
3118     }
3119     if (mState == SHUTDOWN) {
3120       return;
3121     }
3122
3123     // Nobody could add the entry while the lock was released since we modify
3124     // the index only on IO thread and this loop is executed on IO thread too.
3125     entry = mIndex.GetEntry(hash);
3126     MOZ_ASSERT(!entry || !entry->IsFresh());
3127
3128     CacheIndexEntryAutoManage entryMng(&hash, this);
3129
3130     if (NS_FAILED(rv)) {
3131       LOG(
3132           ("CacheIndex::UpdateIndex() - CacheFileMetadata::SyncReadMetadata() "
3133            "failed, removing file. [name=%s]",
3134            leaf.get()));
3135     } else {
3136       entry = mIndex.PutEntry(hash);
3137       rv = InitEntryFromDiskData(entry, meta, size);
3138       if (NS_FAILED(rv)) {
3139         LOG(
3140             ("CacheIndex::UpdateIndex() - CacheIndex::InitEntryFromDiskData "
3141              "failed, removing file. [name=%s]",
3142              leaf.get()));
3143       }
3144     }
3145
3146     if (NS_FAILED(rv)) {
3147       file->Remove(false);
3148       if (entry) {
3149         entry->MarkRemoved();
3150         entry->MarkFresh();
3151         entry->MarkDirty();
3152       }
3153     } else {
3154       LOG(
3155           ("CacheIndex::UpdateIndex() - Added/updated entry to/in index. "
3156            "[name=%s]",
3157            leaf.get()));
3158       entry->Log();
3159     }
3160   }
3161
3162   MOZ_ASSERT_UNREACHABLE("We should never get here");
3163 }
3164
3165 void CacheIndex::FinishUpdate(bool aSucceeded) {
3166   LOG(("CacheIndex::FinishUpdate() [succeeded=%d]", aSucceeded));
3167
3168   MOZ_ASSERT(mState == UPDATING || mState == BUILDING ||
3169              (!aSucceeded && mState == SHUTDOWN));
3170
3171   sLock.AssertCurrentThreadOwns();
3172
3173   if (mDirEnumerator) {
3174     if (NS_IsMainThread()) {
3175       LOG(
3176           ("CacheIndex::FinishUpdate() - posting of PreShutdownInternal failed?"
3177            " Cannot safely release mDirEnumerator, leaking it!"));
3178       NS_WARNING(("CacheIndex::FinishUpdate() - Leaking mDirEnumerator!"));
3179       // This can happen only in case dispatching event to IO thread failed in
3180       // CacheIndex::PreShutdown().
3181       Unused << mDirEnumerator.forget();  // Leak it since dir enumerator is not
3182                                           // threadsafe
3183     } else {
3184       mDirEnumerator->Close();
3185       mDirEnumerator = nullptr;
3186     }
3187   }
3188
3189   if (!aSucceeded) {
3190     mDontMarkIndexClean = true;
3191   }
3192
3193   if (mState == SHUTDOWN) {
3194     return;
3195   }
3196
3197   if (mState == UPDATING && aSucceeded) {
3198     // If we've iterated over all entries successfully then all entries that
3199     // really exist on the disk are now marked as fresh. All non-fresh entries
3200     // don't exist anymore and must be removed from the index.
3201     RemoveNonFreshEntries();
3202   }
3203
3204   // Make sure we won't start update. If the build or update failed, there is no
3205   // reason to believe that it will succeed next time.
3206   mIndexNeedsUpdate = false;
3207
3208   ChangeState(READY);
3209   mLastDumpTime = TimeStamp::NowLoRes();  // Do not dump new index immediately
3210 }
3211
3212 void CacheIndex::RemoveNonFreshEntries() {
3213   for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) {
3214     CacheIndexEntry* entry = iter.Get();
3215     if (entry->IsFresh()) {
3216       continue;
3217     }
3218
3219     LOG(
3220         ("CacheIndex::RemoveNonFreshEntries() - Removing entry. "
3221          "[hash=%08x%08x%08x%08x%08x]",
3222          LOGSHA1(entry->Hash())));
3223
3224     {
3225       CacheIndexEntryAutoManage emng(entry->Hash(), this);
3226       emng.DoNotSearchInIndex();
3227     }
3228
3229     iter.Remove();
3230   }
3231 }
3232
3233 // static
3234 char const* CacheIndex::StateString(EState aState) {
3235   switch (aState) {
3236     case INITIAL:
3237       return "INITIAL";
3238     case READING:
3239       return "READING";
3240     case WRITING:
3241       return "WRITING";
3242     case BUILDING:
3243       return "BUILDING";
3244     case UPDATING:
3245       return "UPDATING";
3246     case READY:
3247       return "READY";
3248     case SHUTDOWN:
3249       return "SHUTDOWN";
3250   }
3251
3252   MOZ_ASSERT(false, "Unexpected state!");
3253   return "?";
3254 }
3255
3256 void CacheIndex::ChangeState(EState aNewState) {
3257   LOG(("CacheIndex::ChangeState() changing state %s -> %s", StateString(mState),
3258        StateString(aNewState)));
3259
3260   // All pending updates should be processed before changing state
3261   MOZ_ASSERT(mPendingUpdates.Count() == 0);
3262
3263   // PreShutdownInternal() should change the state to READY from every state. It
3264   // may go through different states, but once we are in READY state the only
3265   // possible transition is to SHUTDOWN state.
3266   MOZ_ASSERT(!mShuttingDown || mState != READY || aNewState == SHUTDOWN);
3267
3268   // Start updating process when switching to READY state if needed
3269   if (aNewState == READY && StartUpdatingIndexIfNeeded(true)) {
3270     return;
3271   }
3272
3273   if ((mState == READING || mState == BUILDING || mState == UPDATING) &&
3274       aNewState == READY) {
3275     ReportHashStats();
3276   }
3277
3278   // Try to evict entries over limit everytime we're leaving state READING,
3279   // BUILDING or UPDATING, but not during shutdown or when removing all
3280   // entries.
3281   if (!mShuttingDown && !mRemovingAll && aNewState != SHUTDOWN &&
3282       (mState == READING || mState == BUILDING || mState == UPDATING)) {
3283     CacheFileIOManager::EvictIfOverLimit();
3284   }
3285
3286   mState = aNewState;
3287
3288   if (mState != SHUTDOWN) {
3289     CacheFileIOManager::CacheIndexStateChanged();
3290   }
3291
3292   NotifyAsyncGetDiskConsumptionCallbacks();
3293 }
3294
3295 void CacheIndex::NotifyAsyncGetDiskConsumptionCallbacks() {
3296   if ((mState == READY || mState == WRITING) &&
3297       !mAsyncGetDiskConsumptionBlocked && mDiskConsumptionObservers.Length()) {
3298     for (uint32_t i = 0; i < mDiskConsumptionObservers.Length(); ++i) {
3299       DiskConsumptionObserver* o = mDiskConsumptionObservers[i];
3300       // Safe to call under the lock.  We always post to the main thread.
3301       o->OnDiskConsumption(mIndexStats.Size() << 10);
3302     }
3303
3304     mDiskConsumptionObservers.Clear();
3305   }
3306 }
3307
3308 void CacheIndex::AllocBuffer() {
3309   switch (mState) {
3310     case WRITING:
3311       mRWBufSize = sizeof(CacheIndexHeader) + sizeof(CacheHash::Hash32_t) +
3312                    mProcessEntries * sizeof(CacheIndexRecord);
3313       if (mRWBufSize > kMaxBufSize) {
3314         mRWBufSize = kMaxBufSize;
3315       }
3316       break;
3317     case READING:
3318       mRWBufSize = kMaxBufSize;
3319       break;
3320     default:
3321       MOZ_ASSERT(false, "Unexpected state!");
3322   }
3323
3324   mRWBuf = static_cast<char*>(moz_xmalloc(mRWBufSize));
3325 }
3326
3327 void CacheIndex::ReleaseBuffer() {
3328   sLock.AssertCurrentThreadOwns();
3329
3330   if (!mRWBuf || mRWPending) {
3331     return;
3332   }
3333
3334   LOG(("CacheIndex::ReleaseBuffer() releasing buffer"));
3335
3336   free(mRWBuf);
3337   mRWBuf = nullptr;
3338   mRWBufSize = 0;
3339   mRWBufPos = 0;
3340 }
3341
3342 void CacheIndex::FrecencyArray::AppendRecord(CacheIndexRecord* aRecord) {
3343   LOG(
3344       ("CacheIndex::FrecencyArray::AppendRecord() [record=%p, hash=%08x%08x%08x"
3345        "%08x%08x]",
3346        aRecord, LOGSHA1(aRecord->mHash)));
3347
3348   MOZ_ASSERT(!mRecs.Contains(aRecord));
3349   mRecs.AppendElement(aRecord);
3350
3351   // If the new frecency is 0, the element should be at the end of the array,
3352   // i.e. this change doesn't affect order of the array
3353   if (aRecord->mFrecency != 0) {
3354     ++mUnsortedElements;
3355   }
3356 }
3357
3358 void CacheIndex::FrecencyArray::RemoveRecord(CacheIndexRecord* aRecord) {
3359   LOG(("CacheIndex::FrecencyArray::RemoveRecord() [record=%p]", aRecord));
3360
3361   decltype(mRecs)::index_type idx;
3362   idx = mRecs.IndexOf(aRecord);
3363   MOZ_RELEASE_ASSERT(idx != mRecs.NoIndex);
3364   mRecs[idx] = nullptr;
3365   ++mRemovedElements;
3366
3367   // Calling SortIfNeeded ensures that we get rid of removed elements in the
3368   // array once we hit the limit.
3369   SortIfNeeded();
3370 }
3371
3372 void CacheIndex::FrecencyArray::ReplaceRecord(CacheIndexRecord* aOldRecord,
3373                                               CacheIndexRecord* aNewRecord) {
3374   LOG(
3375       ("CacheIndex::FrecencyArray::ReplaceRecord() [oldRecord=%p, "
3376        "newRecord=%p]",
3377        aOldRecord, aNewRecord));
3378
3379   decltype(mRecs)::index_type idx;
3380   idx = mRecs.IndexOf(aOldRecord);
3381   MOZ_RELEASE_ASSERT(idx != mRecs.NoIndex);
3382   mRecs[idx] = aNewRecord;
3383 }
3384
3385 void CacheIndex::FrecencyArray::SortIfNeeded() {
3386   const uint32_t kMaxUnsortedCount = 512;
3387   const uint32_t kMaxUnsortedPercent = 10;
3388   const uint32_t kMaxRemovedCount = 512;
3389
3390   uint32_t unsortedLimit = std::min<uint32_t>(
3391       kMaxUnsortedCount, Length() * kMaxUnsortedPercent / 100);
3392
3393   if (mUnsortedElements > unsortedLimit ||
3394       mRemovedElements > kMaxRemovedCount) {
3395     LOG(
3396         ("CacheIndex::FrecencyArray::SortIfNeeded() - Sorting array "
3397          "[unsortedElements=%u, unsortedLimit=%u, removedElements=%u, "
3398          "maxRemovedCount=%u]",
3399          mUnsortedElements, unsortedLimit, mRemovedElements, kMaxRemovedCount));
3400
3401     mRecs.Sort(FrecencyComparator());
3402     mUnsortedElements = 0;
3403     if (mRemovedElements) {
3404 #ifdef DEBUG
3405       for (uint32_t i = Length(); i < mRecs.Length(); ++i) {
3406         MOZ_ASSERT(!mRecs[i]);
3407       }
3408 #endif
3409       // Removed elements are at the end after sorting.
3410       mRecs.RemoveElementsAt(Length(), mRemovedElements);
3411       mRemovedElements = 0;
3412     }
3413   }
3414 }
3415
3416 void CacheIndex::AddRecordToIterators(CacheIndexRecord* aRecord) {
3417   sLock.AssertCurrentThreadOwns();
3418
3419   for (uint32_t i = 0; i < mIterators.Length(); ++i) {
3420     // Add a new record only when iterator is supposed to be updated.
3421     if (mIterators[i]->ShouldBeNewAdded()) {
3422       mIterators[i]->AddRecord(aRecord);
3423     }
3424   }
3425 }
3426
3427 void CacheIndex::RemoveRecordFromIterators(CacheIndexRecord* aRecord) {
3428   sLock.AssertCurrentThreadOwns();
3429
3430   for (uint32_t i = 0; i < mIterators.Length(); ++i) {
3431     // Remove the record from iterator always, it makes no sence to return
3432     // non-existing entries. Also the pointer to the record is no longer valid
3433     // once the entry is removed from index.
3434     mIterators[i]->RemoveRecord(aRecord);
3435   }
3436 }
3437
3438 void CacheIndex::ReplaceRecordInIterators(CacheIndexRecord* aOldRecord,
3439                                           CacheIndexRecord* aNewRecord) {
3440   sLock.AssertCurrentThreadOwns();
3441
3442   for (uint32_t i = 0; i < mIterators.Length(); ++i) {
3443     // We have to replace the record always since the pointer is no longer
3444     // valid after this point. NOTE: Replacing the record doesn't mean that
3445     // a new entry was added, it just means that the data in the entry was
3446     // changed (e.g. a file size) and we had to track this change in
3447     // mPendingUpdates since mIndex was read-only.
3448     mIterators[i]->ReplaceRecord(aOldRecord, aNewRecord);
3449   }
3450 }
3451
3452 nsresult CacheIndex::Run() {
3453   LOG(("CacheIndex::Run()"));
3454
3455   StaticMutexAutoLock lock(sLock);
3456
3457   if (!IsIndexUsable()) {
3458     return NS_ERROR_NOT_AVAILABLE;
3459   }
3460
3461   if (mState == READY && mShuttingDown) {
3462     return NS_OK;
3463   }
3464
3465   mUpdateEventPending = false;
3466
3467   switch (mState) {
3468     case BUILDING:
3469       BuildIndex();
3470       break;
3471     case UPDATING:
3472       UpdateIndex();
3473       break;
3474     default:
3475       LOG(("CacheIndex::Run() - Update/Build was canceled"));
3476   }
3477
3478   return NS_OK;
3479 }
3480
3481 void CacheIndex::OnFileOpenedInternal(FileOpenHelper* aOpener,
3482                                       CacheFileHandle* aHandle,
3483                                       nsresult aResult) {
3484   LOG(
3485       ("CacheIndex::OnFileOpenedInternal() [opener=%p, handle=%p, "
3486        "result=0x%08" PRIx32 "]",
3487        aOpener, aHandle, static_cast<uint32_t>(aResult)));
3488   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
3489
3490   nsresult rv;
3491
3492   sLock.AssertCurrentThreadOwns();
3493
3494   MOZ_RELEASE_ASSERT(IsIndexUsable());
3495
3496   if (mState == READY && mShuttingDown) {
3497     return;
3498   }
3499
3500   switch (mState) {
3501     case WRITING:
3502       MOZ_ASSERT(aOpener == mIndexFileOpener);
3503       mIndexFileOpener = nullptr;
3504
3505       if (NS_FAILED(aResult)) {
3506         LOG(
3507             ("CacheIndex::OnFileOpenedInternal() - Can't open index file for "
3508              "writing [rv=0x%08" PRIx32 "]",
3509              static_cast<uint32_t>(aResult)));
3510         FinishWrite(false);
3511       } else {
3512         mIndexHandle = aHandle;
3513         WriteRecords();
3514       }
3515       break;
3516     case READING:
3517       if (aOpener == mIndexFileOpener) {
3518         mIndexFileOpener = nullptr;
3519
3520         if (NS_SUCCEEDED(aResult)) {
3521           if (aHandle->FileSize() == 0) {
3522             FinishRead(false);
3523             CacheFileIOManager::DoomFile(aHandle, nullptr);
3524             break;
3525           }
3526           mIndexHandle = aHandle;
3527         } else {
3528           FinishRead(false);
3529           break;
3530         }
3531       } else if (aOpener == mJournalFileOpener) {
3532         mJournalFileOpener = nullptr;
3533         mJournalHandle = aHandle;
3534       } else if (aOpener == mTmpFileOpener) {
3535         mTmpFileOpener = nullptr;
3536         mTmpHandle = aHandle;
3537       } else {
3538         MOZ_ASSERT(false, "Unexpected state!");
3539       }
3540
3541       if (mIndexFileOpener || mJournalFileOpener || mTmpFileOpener) {
3542         // Some opener still didn't finish
3543         break;
3544       }
3545
3546       // We fail and cancel all other openers when we opening index file fails.
3547       MOZ_ASSERT(mIndexHandle);
3548
3549       if (mTmpHandle) {
3550         CacheFileIOManager::DoomFile(mTmpHandle, nullptr);
3551         mTmpHandle = nullptr;
3552
3553         if (mJournalHandle) {  // this shouldn't normally happen
3554           LOG(
3555               ("CacheIndex::OnFileOpenedInternal() - Unexpected state, all "
3556                "files [%s, %s, %s] should never exist. Removing whole index.",
3557                INDEX_NAME, JOURNAL_NAME, TEMP_INDEX_NAME));
3558           FinishRead(false);
3559           break;
3560         }
3561       }
3562
3563       if (mJournalHandle) {
3564         // Rename journal to make sure we update index on next start in case
3565         // firefox crashes
3566         rv = CacheFileIOManager::RenameFile(
3567             mJournalHandle, nsLiteralCString(TEMP_INDEX_NAME), this);
3568         if (NS_FAILED(rv)) {
3569           LOG(
3570               ("CacheIndex::OnFileOpenedInternal() - CacheFileIOManager::"
3571                "RenameFile() failed synchronously [rv=0x%08" PRIx32 "]",
3572                static_cast<uint32_t>(rv)));
3573           FinishRead(false);
3574           break;
3575         }
3576       } else {
3577         StartReadingIndex();
3578       }
3579
3580       break;
3581     default:
3582       MOZ_ASSERT(false, "Unexpected state!");
3583   }
3584 }
3585
3586 nsresult CacheIndex::OnFileOpened(CacheFileHandle* aHandle, nsresult aResult) {
3587   MOZ_CRASH("CacheIndex::OnFileOpened should not be called!");
3588   return NS_ERROR_UNEXPECTED;
3589 }
3590
3591 nsresult CacheIndex::OnDataWritten(CacheFileHandle* aHandle, const char* aBuf,
3592                                    nsresult aResult) {
3593   LOG(("CacheIndex::OnDataWritten() [handle=%p, result=0x%08" PRIx32 "]",
3594        aHandle, static_cast<uint32_t>(aResult)));
3595
3596   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
3597
3598   nsresult rv;
3599
3600   StaticMutexAutoLock lock(sLock);
3601
3602   MOZ_RELEASE_ASSERT(IsIndexUsable());
3603   MOZ_RELEASE_ASSERT(mRWPending);
3604   mRWPending = false;
3605
3606   if (mState == READY && mShuttingDown) {
3607     return NS_OK;
3608   }
3609
3610   switch (mState) {
3611     case WRITING:
3612       MOZ_ASSERT(mIndexHandle == aHandle);
3613
3614       if (NS_FAILED(aResult)) {
3615         FinishWrite(false);
3616       } else {
3617         if (mSkipEntries == mProcessEntries) {
3618           rv = CacheFileIOManager::RenameFile(
3619               mIndexHandle, nsLiteralCString(INDEX_NAME), this);
3620           if (NS_FAILED(rv)) {
3621             LOG(
3622                 ("CacheIndex::OnDataWritten() - CacheFileIOManager::"
3623                  "RenameFile() failed synchronously [rv=0x%08" PRIx32 "]",
3624                  static_cast<uint32_t>(rv)));
3625             FinishWrite(false);
3626           }
3627         } else {
3628           WriteRecords();
3629         }
3630       }
3631       break;
3632     default:
3633       // Writing was canceled.
3634       LOG(
3635           ("CacheIndex::OnDataWritten() - ignoring notification since the "
3636            "operation was previously canceled [state=%d]",
3637            mState));
3638       ReleaseBuffer();
3639   }
3640
3641   return NS_OK;
3642 }
3643
3644 nsresult CacheIndex::OnDataRead(CacheFileHandle* aHandle, char* aBuf,
3645                                 nsresult aResult) {
3646   LOG(("CacheIndex::OnDataRead() [handle=%p, result=0x%08" PRIx32 "]", aHandle,
3647        static_cast<uint32_t>(aResult)));
3648
3649   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
3650
3651   StaticMutexAutoLock lock(sLock);
3652
3653   MOZ_RELEASE_ASSERT(IsIndexUsable());
3654   MOZ_RELEASE_ASSERT(mRWPending);
3655   mRWPending = false;
3656
3657   switch (mState) {
3658     case READING:
3659       MOZ_ASSERT(mIndexHandle == aHandle || mJournalHandle == aHandle);
3660
3661       if (NS_FAILED(aResult)) {
3662         FinishRead(false);
3663       } else {
3664         if (!mIndexOnDiskIsValid) {
3665           ParseRecords();
3666         } else {
3667           ParseJournal();
3668         }
3669       }
3670       break;
3671     default:
3672       // Reading was canceled.
3673       LOG(
3674           ("CacheIndex::OnDataRead() - ignoring notification since the "
3675            "operation was previously canceled [state=%d]",
3676            mState));
3677       ReleaseBuffer();
3678   }
3679
3680   return NS_OK;
3681 }
3682
3683 nsresult CacheIndex::OnFileDoomed(CacheFileHandle* aHandle, nsresult aResult) {
3684   MOZ_CRASH("CacheIndex::OnFileDoomed should not be called!");
3685   return NS_ERROR_UNEXPECTED;
3686 }
3687
3688 nsresult CacheIndex::OnEOFSet(CacheFileHandle* aHandle, nsresult aResult) {
3689   MOZ_CRASH("CacheIndex::OnEOFSet should not be called!");
3690   return NS_ERROR_UNEXPECTED;
3691 }
3692
3693 nsresult CacheIndex::OnFileRenamed(CacheFileHandle* aHandle, nsresult aResult) {
3694   LOG(("CacheIndex::OnFileRenamed() [handle=%p, result=0x%08" PRIx32 "]",
3695        aHandle, static_cast<uint32_t>(aResult)));
3696
3697   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
3698
3699   StaticMutexAutoLock lock(sLock);
3700
3701   MOZ_RELEASE_ASSERT(IsIndexUsable());
3702
3703   if (mState == READY && mShuttingDown) {
3704     return NS_OK;
3705   }
3706
3707   switch (mState) {
3708     case WRITING:
3709       // This is a result of renaming the new index written to tmpfile to index
3710       // file. This is the last step when writing the index and the whole
3711       // writing process is successful iff renaming was successful.
3712
3713       if (mIndexHandle != aHandle) {
3714         LOG(
3715             ("CacheIndex::OnFileRenamed() - ignoring notification since it "
3716              "belongs to previously canceled operation [state=%d]",
3717              mState));
3718         break;
3719       }
3720
3721       FinishWrite(NS_SUCCEEDED(aResult));
3722       break;
3723     case READING:
3724       // This is a result of renaming journal file to tmpfile. It is renamed
3725       // before we start reading index and journal file and it should normally
3726       // succeed. If it fails give up reading of index.
3727
3728       if (mJournalHandle != aHandle) {
3729         LOG(
3730             ("CacheIndex::OnFileRenamed() - ignoring notification since it "
3731              "belongs to previously canceled operation [state=%d]",
3732              mState));
3733         break;
3734       }
3735
3736       if (NS_FAILED(aResult)) {
3737         FinishRead(false);
3738       } else {
3739         StartReadingIndex();
3740       }
3741       break;
3742     default:
3743       // Reading/writing was canceled.
3744       LOG(
3745           ("CacheIndex::OnFileRenamed() - ignoring notification since the "
3746            "operation was previously canceled [state=%d]",
3747            mState));
3748   }
3749
3750   return NS_OK;
3751 }
3752
3753 // Memory reporting
3754
3755 size_t CacheIndex::SizeOfExcludingThisInternal(
3756     mozilla::MallocSizeOf mallocSizeOf) const {
3757   sLock.AssertCurrentThreadOwns();
3758
3759   size_t n = 0;
3760   nsCOMPtr<nsISizeOf> sizeOf;
3761
3762   // mIndexHandle and mJournalHandle are reported via SizeOfHandlesRunnable
3763   // in CacheFileIOManager::SizeOfExcludingThisInternal as part of special
3764   // handles array.
3765
3766   sizeOf = do_QueryInterface(mCacheDirectory);
3767   if (sizeOf) {
3768     n += sizeOf->SizeOfIncludingThis(mallocSizeOf);
3769   }
3770
3771   sizeOf = do_QueryInterface(mUpdateTimer);
3772   if (sizeOf) {
3773     n += sizeOf->SizeOfIncludingThis(mallocSizeOf);
3774   }
3775
3776   n += mallocSizeOf(mRWBuf);
3777   n += mallocSizeOf(mRWHash);
3778
3779   n += mIndex.SizeOfExcludingThis(mallocSizeOf);
3780   n += mPendingUpdates.SizeOfExcludingThis(mallocSizeOf);
3781   n += mTmpJournal.SizeOfExcludingThis(mallocSizeOf);
3782
3783   // mFrecencyArray items are reported by mIndex/mPendingUpdates
3784   n += mFrecencyArray.mRecs.ShallowSizeOfExcludingThis(mallocSizeOf);
3785   n += mDiskConsumptionObservers.ShallowSizeOfExcludingThis(mallocSizeOf);
3786
3787   return n;
3788 }
3789
3790 // static
3791 size_t CacheIndex::SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) {
3792   sLock.AssertCurrentThreadOwns();
3793
3794   if (!gInstance) return 0;
3795
3796   return gInstance->SizeOfExcludingThisInternal(mallocSizeOf);
3797 }
3798
3799 // static
3800 size_t CacheIndex::SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) {
3801   StaticMutexAutoLock lock(sLock);
3802
3803   return mallocSizeOf(gInstance) + SizeOfExcludingThis(mallocSizeOf);
3804 }
3805
3806 namespace {
3807
3808 class HashComparator {
3809  public:
3810   bool Equals(CacheIndexRecord* a, CacheIndexRecord* b) const {
3811     return memcmp(&a->mHash, &b->mHash, sizeof(SHA1Sum::Hash)) == 0;
3812   }
3813   bool LessThan(CacheIndexRecord* a, CacheIndexRecord* b) const {
3814     return memcmp(&a->mHash, &b->mHash, sizeof(SHA1Sum::Hash)) < 0;
3815   }
3816 };
3817
3818 void ReportHashSizeMatch(const SHA1Sum::Hash* aHash1,
3819                          const SHA1Sum::Hash* aHash2) {
3820   const uint32_t* h1 = reinterpret_cast<const uint32_t*>(aHash1);
3821   const uint32_t* h2 = reinterpret_cast<const uint32_t*>(aHash2);
3822
3823   for (uint32_t i = 0; i < 5; ++i) {
3824     if (h1[i] != h2[i]) {
3825       uint32_t bitsDiff = h1[i] ^ h2[i];
3826       bitsDiff = NetworkEndian::readUint32(&bitsDiff);
3827
3828       // count leading zeros in bitsDiff
3829       static const uint8_t debruijn32[32] = {
3830           0, 31, 9, 30, 3, 8,  13, 29, 2,  5,  7,  21, 12, 24, 28, 19,
3831           1, 10, 4, 14, 6, 22, 25, 20, 11, 15, 23, 26, 16, 27, 17, 18};
3832
3833       bitsDiff |= bitsDiff >> 1;
3834       bitsDiff |= bitsDiff >> 2;
3835       bitsDiff |= bitsDiff >> 4;
3836       bitsDiff |= bitsDiff >> 8;
3837       bitsDiff |= bitsDiff >> 16;
3838       bitsDiff++;
3839
3840       uint8_t hashSizeMatch =
3841           debruijn32[bitsDiff * 0x076be629 >> 27] + (i << 5);
3842       Telemetry::Accumulate(Telemetry::NETWORK_CACHE_HASH_STATS, hashSizeMatch);
3843
3844       return;
3845     }
3846   }
3847
3848   MOZ_ASSERT(false, "Found a collision in the index!");
3849 }
3850
3851 }  // namespace
3852
3853 void CacheIndex::ReportHashStats() {
3854   // We're gathering the hash stats only once, exclude too small caches.
3855   if (CacheObserver::HashStatsReported() || mFrecencyArray.Length() < 15000) {
3856     return;
3857   }
3858
3859   nsTArray<CacheIndexRecord*> records;
3860   for (auto iter = mFrecencyArray.Iter(); !iter.Done(); iter.Next()) {
3861     records.AppendElement(iter.Get());
3862   }
3863
3864   records.Sort(HashComparator());
3865
3866   for (uint32_t i = 1; i < records.Length(); i++) {
3867     ReportHashSizeMatch(&records[i - 1]->mHash, &records[i]->mHash);
3868   }
3869
3870   CacheObserver::SetHashStatsReported();
3871 }
3872
3873 // static
3874 void CacheIndex::UpdateTotalBytesWritten(uint32_t aBytesWritten) {
3875   StaticMutexAutoLock lock(sLock);
3876
3877   RefPtr<CacheIndex> index = gInstance;
3878   if (!index) {
3879     return;
3880   }
3881
3882   index->mTotalBytesWritten += aBytesWritten;
3883
3884   // Do telemetry report if enough data has been written and the index is
3885   // in READY state. The data is available also in WRITING state, but we would
3886   // need to deal with pending updates.
3887   if (index->mTotalBytesWritten >= kTelemetryReportBytesLimit &&
3888       index->mState == READY && !index->mIndexNeedsUpdate &&
3889       !index->mShuttingDown) {
3890     index->DoTelemetryReport();
3891
3892     index->mTotalBytesWritten = 0;
3893     CacheObserver::SetCacheAmountWritten(0);
3894     return;
3895   }
3896
3897   uint64_t writtenKB = index->mTotalBytesWritten >> 10;
3898   // Store number of written kilobytes to prefs after writing at least 10MB.
3899   if ((writtenKB - CacheObserver::CacheAmountWritten()) > (10 * 1024)) {
3900     CacheObserver::SetCacheAmountWritten(writtenKB);
3901   }
3902 }
3903
3904 void CacheIndex::DoTelemetryReport() {
3905   static const nsLiteralCString
3906       contentTypeNames[nsICacheEntry::CONTENT_TYPE_LAST] = {
3907           "UNKNOWN"_ns, "OTHER"_ns,      "JAVASCRIPT"_ns, "IMAGE"_ns,
3908           "MEDIA"_ns,   "STYLESHEET"_ns, "WASM"_ns};
3909
3910   for (uint32_t i = 0; i < nsICacheEntry::CONTENT_TYPE_LAST; ++i) {
3911     if (mIndexStats.Size() > 0) {
3912       Telemetry::Accumulate(
3913           Telemetry::NETWORK_CACHE_SIZE_SHARE, contentTypeNames[i],
3914           round(static_cast<double>(mIndexStats.SizeByType(i)) * 100.0 /
3915                 static_cast<double>(mIndexStats.Size())));
3916     }
3917
3918     if (mIndexStats.Count() > 0) {
3919       Telemetry::Accumulate(
3920           Telemetry::NETWORK_CACHE_ENTRY_COUNT_SHARE, contentTypeNames[i],
3921           round(static_cast<double>(mIndexStats.CountByType(i)) * 100.0 /
3922                 static_cast<double>(mIndexStats.Count())));
3923     }
3924   }
3925
3926   nsCString probeKey;
3927   if (CacheObserver::SmartCacheSizeEnabled()) {
3928     probeKey = "SMARTSIZE"_ns;
3929   } else {
3930     probeKey = "USERDEFINEDSIZE"_ns;
3931   }
3932   Telemetry::Accumulate(Telemetry::NETWORK_CACHE_ENTRY_COUNT, probeKey,
3933                         mIndexStats.Count());
3934   Telemetry::Accumulate(Telemetry::NETWORK_CACHE_SIZE, probeKey,
3935                         mIndexStats.Size() >> 10);
3936 }
3937
3938 // static
3939 void CacheIndex::OnAsyncEviction(bool aEvicting) {
3940   StaticMutexAutoLock lock(sLock);
3941
3942   RefPtr<CacheIndex> index = gInstance;
3943   if (!index) {
3944     return;
3945   }
3946
3947   index->mAsyncGetDiskConsumptionBlocked = aEvicting;
3948   if (!aEvicting) {
3949     index->NotifyAsyncGetDiskConsumptionCallbacks();
3950   }
3951 }
3952
3953 }  // namespace net
3954 }  // namespace mozilla