netwerk/cache2/CacheIndex.cpp

   1 /* This Source Code Form is subject to the terms of the Mozilla Public
   2  * License, v. 2.0. If a copy of the MPL was not distributed with this
   3  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   4
   5 #include "CacheIndex.h"
   6
   7 #include "CacheLog.h"
   8 #include "CacheFileIOManager.h"
   9 #include "CacheFileMetadata.h"
  10 #include "CacheIndexIterator.h"
  11 #include "CacheIndexContextIterator.h"
  12 #include "nsThreadUtils.h"
  13 #include "nsISimpleEnumerator.h"
  14 #include "nsIDirectoryEnumerator.h"
  15 #include "nsISizeOf.h"
  16 #include "nsPrintfCString.h"
  17 #include "mozilla/DebugOnly.h"
  18 #include "prinrval.h"
  19 #include "nsIFile.h"
  20 #include "nsITimer.h"
  21 #include "mozilla/AutoRestore.h"
  22 #include <algorithm>
  23
  24
  25 #define kMinUnwrittenChanges   300
  26 #define kMinDumpInterval       20000 // in milliseconds
  27 #define kMaxBufSize            16384
  28 #define kIndexVersion          0x00000001
  29 #define kUpdateIndexStartDelay 50000 // in milliseconds
  30
  31 const char kIndexName[]     = "index";
  32 const char kTempIndexName[] = "index.tmp";
  33 const char kJournalName[]   = "index.log";
  34
  35 namespace mozilla {
  36 namespace net {
  37
  38 /**
  39  * This helper class is responsible for keeping CacheIndex::mIndexStats,
  40  * CacheIndex::mFrecencyArray and CacheIndex::mExpirationArray up to date.
  41  */
  42 class CacheIndexEntryAutoManage
  43 {
  44 public:
  45   CacheIndexEntryAutoManage(const SHA1Sum::Hash *aHash, CacheIndex *aIndex)
  46     : mIndex(aIndex)
  47     , mOldRecord(nullptr)
  48     , mOldFrecency(0)
  49     , mOldExpirationTime(nsICacheEntry::NO_EXPIRATION_TIME)
  50     , mDoNotSearchInIndex(false)
  51     , mDoNotSearchInUpdates(false)
  52   {
  53     mIndex->AssertOwnsLock();
  54
  55     mHash = aHash;
  56     CacheIndexEntry *entry = FindEntry();
  57     mIndex->mIndexStats.BeforeChange(entry);
  58     if (entry && entry->IsInitialized() && !entry->IsRemoved()) {
  59       mOldRecord = entry->mRec;
  60       mOldFrecency = entry->mRec->mFrecency;
  61       mOldExpirationTime = entry->mRec->mExpirationTime;
  62     }
  63   }
  64
  65   ~CacheIndexEntryAutoManage()
  66   {
  67     mIndex->AssertOwnsLock();
  68
  69     CacheIndexEntry *entry = FindEntry();
  70     mIndex->mIndexStats.AfterChange(entry);
  71     if (!entry || !entry->IsInitialized() || entry->IsRemoved()) {
  72       entry = nullptr;
  73     }
  74
  75     if (entry && !mOldRecord) {
  76       mIndex->InsertRecordToFrecencyArray(entry->mRec);
  77       mIndex->InsertRecordToExpirationArray(entry->mRec);
  78       mIndex->AddRecordToIterators(entry->mRec);
  79     } else if (!entry && mOldRecord) {
  80       mIndex->RemoveRecordFromFrecencyArray(mOldRecord);
  81       mIndex->RemoveRecordFromExpirationArray(mOldRecord);
  82       mIndex->RemoveRecordFromIterators(mOldRecord);
  83     } else if (entry && mOldRecord) {
  84       bool replaceFrecency = false;
  85       bool replaceExpiration = false;
  86
  87       if (entry->mRec != mOldRecord) {
  88         // record has a different address, we have to replace it
  89         replaceFrecency = replaceExpiration = true;
  90         mIndex->ReplaceRecordInIterators(mOldRecord, entry->mRec);
  91       } else {
  92         if (entry->mRec->mFrecency == 0 &&
  93             entry->mRec->mExpirationTime == nsICacheEntry::NO_EXPIRATION_TIME) {
  94           // This is a special case when we want to make sure that the entry is
  95           // placed at the end of the lists even when the values didn't change.
  96           replaceFrecency = replaceExpiration = true;
  97         } else {
  98           if (entry->mRec->mFrecency != mOldFrecency) {
  99             replaceFrecency = true;
 100           }
 101           if (entry->mRec->mExpirationTime != mOldExpirationTime) {
 102             replaceExpiration = true;
 103           }
 104         }
 105       }
 106
 107       if (replaceFrecency) {
 108         mIndex->RemoveRecordFromFrecencyArray(mOldRecord);
 109         mIndex->InsertRecordToFrecencyArray(entry->mRec);
 110       }
 111       if (replaceExpiration) {
 112         mIndex->RemoveRecordFromExpirationArray(mOldRecord);
 113         mIndex->InsertRecordToExpirationArray(entry->mRec);
 114       }
 115     } else {
 116       // both entries were removed or not initialized, do nothing
 117     }
 118   }
 119
 120   // We cannot rely on nsTHashtable::GetEntry() in case we are enumerating the
 121   // entries and returning PL_DHASH_REMOVE. Destructor is called before the
 122   // entry is removed. Caller must call one of following methods to skip
 123   // lookup in the hashtable.
 124   void DoNotSearchInIndex()   { mDoNotSearchInIndex = true; }
 125   void DoNotSearchInUpdates() { mDoNotSearchInUpdates = true; }
 126
 127 private:
 128   CacheIndexEntry * FindEntry()
 129   {
 130     CacheIndexEntry *entry = nullptr;
 131
 132     switch (mIndex->mState) {
 133       case CacheIndex::READING:
 134       case CacheIndex::WRITING:
 135         if (!mDoNotSearchInUpdates) {
 136           entry = mIndex->mPendingUpdates.GetEntry(*mHash);
 137         }
 138         // no break
 139       case CacheIndex::BUILDING:
 140       case CacheIndex::UPDATING:
 141       case CacheIndex::READY:
 142         if (!entry && !mDoNotSearchInIndex) {
 143           entry = mIndex->mIndex.GetEntry(*mHash);
 144         }
 145         break;
 146       case CacheIndex::INITIAL:
 147       case CacheIndex::SHUTDOWN:
 148       default:
 149         MOZ_ASSERT(false, "Unexpected state!");
 150     }
 151
 152     return entry;
 153   }
 154
 155   const SHA1Sum::Hash *mHash;
 156   nsRefPtr<CacheIndex> mIndex;
 157   CacheIndexRecord    *mOldRecord;
 158   uint32_t             mOldFrecency;
 159   uint32_t             mOldExpirationTime;
 160   bool                 mDoNotSearchInIndex;
 161   bool                 mDoNotSearchInUpdates;
 162 };
 163
 164 class FileOpenHelper : public CacheFileIOListener
 165 {
 166 public:
 167   NS_DECL_THREADSAFE_ISUPPORTS
 168
 169   explicit FileOpenHelper(CacheIndex* aIndex)
 170     : mIndex(aIndex)
 171     , mCanceled(false)
 172   {}
 173
 174   void Cancel() {
 175     mIndex->AssertOwnsLock();
 176     mCanceled = true;
 177   }
 178
 179 private:
 180   virtual ~FileOpenHelper() {}
 181
 182   NS_IMETHOD OnFileOpened(CacheFileHandle *aHandle, nsresult aResult);
 183   NS_IMETHOD OnDataWritten(CacheFileHandle *aHandle, const char *aBuf,
 184                            nsresult aResult) {
 185     MOZ_CRASH("FileOpenHelper::OnDataWritten should not be called!");
 186     return NS_ERROR_UNEXPECTED;
 187   }
 188   NS_IMETHOD OnDataRead(CacheFileHandle *aHandle, char *aBuf,
 189                         nsresult aResult) {
 190     MOZ_CRASH("FileOpenHelper::OnDataRead should not be called!");
 191     return NS_ERROR_UNEXPECTED;
 192   }
 193   NS_IMETHOD OnFileDoomed(CacheFileHandle *aHandle, nsresult aResult) {
 194     MOZ_CRASH("FileOpenHelper::OnFileDoomed should not be called!");
 195     return NS_ERROR_UNEXPECTED;
 196   }
 197   NS_IMETHOD OnEOFSet(CacheFileHandle *aHandle, nsresult aResult) {
 198     MOZ_CRASH("FileOpenHelper::OnEOFSet should not be called!");
 199     return NS_ERROR_UNEXPECTED;
 200   }
 201   NS_IMETHOD OnFileRenamed(CacheFileHandle *aHandle, nsresult aResult) {
 202     MOZ_CRASH("FileOpenHelper::OnFileRenamed should not be called!");
 203     return NS_ERROR_UNEXPECTED;
 204   }
 205
 206   nsRefPtr<CacheIndex> mIndex;
 207   bool                 mCanceled;
 208 };
 209
 210 NS_IMETHODIMP FileOpenHelper::OnFileOpened(CacheFileHandle *aHandle,
 211                                            nsresult aResult)
 212 {
 213   CacheIndexAutoLock lock(mIndex);
 214
 215   if (mCanceled) {
 216     if (aHandle) {
 217       CacheFileIOManager::DoomFile(aHandle, nullptr);
 218     }
 219
 220     return NS_OK;
 221   }
 222
 223   mIndex->OnFileOpenedInternal(this, aHandle, aResult);
 224
 225   return NS_OK;
 226 }
 227
 228 NS_IMPL_ISUPPORTS(FileOpenHelper, CacheFileIOListener);
 229
 230
 231 CacheIndex * CacheIndex::gInstance = nullptr;
 232
 233
 234 NS_IMPL_ADDREF(CacheIndex)
 235 NS_IMPL_RELEASE(CacheIndex)
 236
 237 NS_INTERFACE_MAP_BEGIN(CacheIndex)
 238   NS_INTERFACE_MAP_ENTRY(mozilla::net::CacheFileIOListener)
 239   NS_INTERFACE_MAP_ENTRY(nsIRunnable)
 240 NS_INTERFACE_MAP_END_THREADSAFE
 241
 242
 243 CacheIndex::CacheIndex()
 244   : mLock("CacheIndex.mLock")
 245   , mState(INITIAL)
 246   , mShuttingDown(false)
 247   , mIndexNeedsUpdate(false)
 248   , mRemovingAll(false)
 249   , mIndexOnDiskIsValid(false)
 250   , mDontMarkIndexClean(false)
 251   , mIndexTimeStamp(0)
 252   , mUpdateEventPending(false)
 253   , mSkipEntries(0)
 254   , mProcessEntries(0)
 255   , mRWBuf(nullptr)
 256   , mRWBufSize(0)
 257   , mRWBufPos(0)
 258   , mJournalReadSuccessfully(false)
 259 {
 260   LOG(("CacheIndex::CacheIndex [this=%p]", this));
 261   MOZ_COUNT_CTOR(CacheIndex);
 262   MOZ_ASSERT(!gInstance, "multiple CacheIndex instances!");
 263 }
 264
 265 CacheIndex::~CacheIndex()
 266 {
 267   LOG(("CacheIndex::~CacheIndex [this=%p]", this));
 268   MOZ_COUNT_DTOR(CacheIndex);
 269
 270   ReleaseBuffer();
 271 }
 272
 273 void
 274 CacheIndex::Lock()
 275 {
 276   mLock.Lock();
 277
 278   MOZ_ASSERT(!mIndexStats.StateLogged());
 279 }
 280
 281 void
 282 CacheIndex::Unlock()
 283 {
 284   MOZ_ASSERT(!mIndexStats.StateLogged());
 285
 286   mLock.Unlock();
 287 }
 288
 289 inline void
 290 CacheIndex::AssertOwnsLock()
 291 {
 292   mLock.AssertCurrentThreadOwns();
 293 }
 294
 295 // static
 296 nsresult
 297 CacheIndex::Init(nsIFile *aCacheDirectory)
 298 {
 299   LOG(("CacheIndex::Init()"));
 300
 301   MOZ_ASSERT(NS_IsMainThread());
 302
 303   if (gInstance) {
 304     return NS_ERROR_ALREADY_INITIALIZED;
 305   }
 306
 307   nsRefPtr<CacheIndex> idx = new CacheIndex();
 308
 309   CacheIndexAutoLock lock(idx);
 310
 311   nsresult rv = idx->InitInternal(aCacheDirectory);
 312   NS_ENSURE_SUCCESS(rv, rv);
 313
 314   idx.swap(gInstance);
 315   return NS_OK;
 316 }
 317
 318 nsresult
 319 CacheIndex::InitInternal(nsIFile *aCacheDirectory)
 320 {
 321   nsresult rv;
 322
 323   rv = aCacheDirectory->Clone(getter_AddRefs(mCacheDirectory));
 324   NS_ENSURE_SUCCESS(rv, rv);
 325
 326   mStartTime = TimeStamp::NowLoRes();
 327
 328   ReadIndexFromDisk();
 329
 330   return NS_OK;
 331 }
 332
 333 // static
 334 nsresult
 335 CacheIndex::PreShutdown()
 336 {
 337   LOG(("CacheIndex::PreShutdown() [gInstance=%p]", gInstance));
 338
 339   MOZ_ASSERT(NS_IsMainThread());
 340
 341   nsresult rv;
 342   nsRefPtr<CacheIndex> index = gInstance;
 343
 344   if (!index) {
 345     return NS_ERROR_NOT_INITIALIZED;
 346   }
 347
 348   CacheIndexAutoLock lock(index);
 349
 350   LOG(("CacheIndex::PreShutdown() - [state=%d, indexOnDiskIsValid=%d, "
 351        "dontMarkIndexClean=%d]", index->mState, index->mIndexOnDiskIsValid,
 352        index->mDontMarkIndexClean));
 353
 354   LOG(("CacheIndex::PreShutdown() - Closing iterators."));
 355   for (uint32_t i = 0; i < index->mIterators.Length(); ) {
 356     rv = index->mIterators[i]->CloseInternal(NS_ERROR_FAILURE);
 357     if (NS_FAILED(rv)) {
 358       // CacheIndexIterator::CloseInternal() removes itself from mIteratos iff
 359       // it returns success.
 360       LOG(("CacheIndex::PreShutdown() - Failed to remove iterator %p. "
 361            "[rv=0x%08x]", rv));
 362       i++;
 363     }
 364   }
 365
 366   index->mShuttingDown = true;
 367
 368   if (index->mState == READY) {
 369     return NS_OK; // nothing to do
 370   }
 371
 372   nsCOMPtr<nsIRunnable> event;
 373   event = NS_NewRunnableMethod(index, &CacheIndex::PreShutdownInternal);
 374
 375   nsCOMPtr<nsIEventTarget> ioTarget = CacheFileIOManager::IOTarget();
 376   MOZ_ASSERT(ioTarget);
 377
 378   // PreShutdownInternal() will be executed before any queued event on INDEX
 379   // level. That's OK since we don't want to wait for any operation in progess.
 380   // We need to interrupt it and save journal as quickly as possible.
 381   rv = ioTarget->Dispatch(event, nsIEventTarget::DISPATCH_NORMAL);
 382   if (NS_FAILED(rv)) {
 383     NS_WARNING("CacheIndex::PreShutdown() - Can't dispatch event");
 384     LOG(("CacheIndex::PreShutdown() - Can't dispatch event" ));
 385     return rv;
 386   }
 387
 388   return NS_OK;
 389 }
 390
 391 void
 392 CacheIndex::PreShutdownInternal()
 393 {
 394   CacheIndexAutoLock lock(this);
 395
 396   LOG(("CacheIndex::PreShutdownInternal() - [state=%d, indexOnDiskIsValid=%d, "
 397        "dontMarkIndexClean=%d]", mState, mIndexOnDiskIsValid,
 398        mDontMarkIndexClean));
 399
 400   MOZ_ASSERT(mShuttingDown);
 401
 402   if (mUpdateTimer) {
 403     mUpdateTimer = nullptr;
 404   }
 405
 406   switch (mState) {
 407     case WRITING:
 408       FinishWrite(false);
 409       break;
 410     case READY:
 411       // nothing to do, write the journal in Shutdown()
 412       break;
 413     case READING:
 414       FinishRead(false);
 415       break;
 416     case BUILDING:
 417     case UPDATING:
 418       FinishUpdate(false);
 419       break;
 420     default:
 421       MOZ_ASSERT(false, "Implement me!");
 422   }
 423
 424   // We should end up in READY state
 425   MOZ_ASSERT(mState == READY);
 426 }
 427
 428 // static
 429 nsresult
 430 CacheIndex::Shutdown()
 431 {
 432   LOG(("CacheIndex::Shutdown() [gInstance=%p]", gInstance));
 433
 434   MOZ_ASSERT(NS_IsMainThread());
 435
 436   nsRefPtr<CacheIndex> index;
 437   index.swap(gInstance);
 438
 439   if (!index) {
 440     return NS_ERROR_NOT_INITIALIZED;
 441   }
 442
 443   CacheIndexAutoLock lock(index);
 444
 445   bool sanitize = CacheObserver::ClearCacheOnShutdown();
 446
 447   LOG(("CacheIndex::Shutdown() - [state=%d, indexOnDiskIsValid=%d, "
 448        "dontMarkIndexClean=%d, sanitize=%d]", index->mState,
 449        index->mIndexOnDiskIsValid, index->mDontMarkIndexClean, sanitize));
 450
 451   MOZ_ASSERT(index->mShuttingDown);
 452
 453   EState oldState = index->mState;
 454   index->ChangeState(SHUTDOWN);
 455
 456   if (oldState != READY) {
 457     LOG(("CacheIndex::Shutdown() - Unexpected state. Did posting of "
 458          "PreShutdownInternal() fail?"));
 459   }
 460
 461   switch (oldState) {
 462     case WRITING:
 463       index->FinishWrite(false);
 464       // no break
 465     case READY:
 466       if (index->mIndexOnDiskIsValid && !index->mDontMarkIndexClean) {
 467         if (!sanitize && NS_FAILED(index->WriteLogToDisk())) {
 468           index->RemoveIndexFromDisk();
 469         }
 470       } else {
 471         index->RemoveIndexFromDisk();
 472       }
 473       break;
 474     case READING:
 475       index->FinishRead(false);
 476       break;
 477     case BUILDING:
 478     case UPDATING:
 479       index->FinishUpdate(false);
 480       break;
 481     default:
 482       MOZ_ASSERT(false, "Unexpected state!");
 483   }
 484
 485   if (sanitize) {
 486     index->RemoveIndexFromDisk();
 487   }
 488
 489   return NS_OK;
 490 }
 491
 492 // static
 493 nsresult
 494 CacheIndex::AddEntry(const SHA1Sum::Hash *aHash)
 495 {
 496   LOG(("CacheIndex::AddEntry() [hash=%08x%08x%08x%08x%08x]", LOGSHA1(aHash)));
 497
 498   nsRefPtr<CacheIndex> index = gInstance;
 499
 500   if (!index) {
 501     return NS_ERROR_NOT_INITIALIZED;
 502   }
 503
 504   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
 505
 506   CacheIndexAutoLock lock(index);
 507
 508   if (!index->IsIndexUsable()) {
 509     return NS_ERROR_NOT_AVAILABLE;
 510   }
 511
 512   // Getters in CacheIndexStats assert when mStateLogged is true since the
 513   // information is incomplete between calls to BeforeChange() and AfterChange()
 514   // (i.e. while CacheIndexEntryAutoManage exists). We need to check whether
 515   // non-fresh entries exists outside the scope of CacheIndexEntryAutoManage.
 516   bool updateIfNonFreshEntriesExist = false;
 517
 518   {
 519     CacheIndexEntryAutoManage entryMng(aHash, index);
 520
 521     CacheIndexEntry *entry = index->mIndex.GetEntry(*aHash);
 522     bool entryRemoved = entry && entry->IsRemoved();
 523
 524     if (index->mState == READY || index->mState == UPDATING ||
 525         index->mState == BUILDING) {
 526       MOZ_ASSERT(index->mPendingUpdates.Count() == 0);
 527
 528       if (entry && !entryRemoved) {
 529         // Found entry in index that shouldn't exist.
 530
 531         if (entry->IsFresh()) {
 532           // Someone removed the file on disk while FF is running. Update
 533           // process can fix only non-fresh entries (i.e. entries that were not
 534           // added within this session). Start update only if we have such
 535           // entries.
 536           //
 537           // TODO: This should be very rare problem. If it turns out not to be
 538           // true, change the update process so that it also iterates all
 539           // initialized non-empty entries and checks whether the file exists.
 540
 541           LOG(("CacheIndex::AddEntry() - Cache file was removed outside FF "
 542                "process!"));
 543
 544           updateIfNonFreshEntriesExist = true;
 545         } else if (index->mState == READY) {
 546           // Index is outdated, update it.
 547           LOG(("CacheIndex::AddEntry() - Found entry that shouldn't exist, "
 548                "update is needed"));
 549           index->mIndexNeedsUpdate = true;
 550         } else {
 551           // We cannot be here when building index since all entries are fresh
 552           // during building.
 553           MOZ_ASSERT(index->mState == UPDATING);
 554         }
 555       }
 556
 557       if (!entry) {
 558         entry = index->mIndex.PutEntry(*aHash);
 559       }
 560     } else { // WRITING, READING
 561       CacheIndexEntry *updated = index->mPendingUpdates.GetEntry(*aHash);
 562       bool updatedRemoved = updated && updated->IsRemoved();
 563
 564       if ((updated && !updatedRemoved) ||
 565           (!updated && entry && !entryRemoved && entry->IsFresh())) {
 566         // Fresh entry found, so the file was removed outside FF
 567         LOG(("CacheIndex::AddEntry() - Cache file was removed outside FF "
 568              "process!"));
 569
 570         updateIfNonFreshEntriesExist = true;
 571       } else if (!updated && entry && !entryRemoved) {
 572         if (index->mState == WRITING) {
 573           LOG(("CacheIndex::AddEntry() - Found entry that shouldn't exist, "
 574                "update is needed"));
 575           index->mIndexNeedsUpdate = true;
 576         }
 577         // Ignore if state is READING since the index information is partial
 578       }
 579
 580       updated = index->mPendingUpdates.PutEntry(*aHash);
 581       entry = updated;
 582     }
 583
 584     entry->InitNew();
 585     entry->MarkDirty();
 586     entry->MarkFresh();
 587   }
 588
 589   if (updateIfNonFreshEntriesExist &&
 590       index->mIndexStats.Count() != index->mIndexStats.Fresh()) {
 591     index->mIndexNeedsUpdate = true;
 592   }
 593
 594   index->StartUpdatingIndexIfNeeded();
 595   index->WriteIndexToDiskIfNeeded();
 596
 597   return NS_OK;
 598 }
 599
 600 // static
 601 nsresult
 602 CacheIndex::EnsureEntryExists(const SHA1Sum::Hash *aHash)
 603 {
 604   LOG(("CacheIndex::EnsureEntryExists() [hash=%08x%08x%08x%08x%08x]",
 605        LOGSHA1(aHash)));
 606
 607   nsRefPtr<CacheIndex> index = gInstance;
 608
 609   if (!index) {
 610     return NS_ERROR_NOT_INITIALIZED;
 611   }
 612
 613   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
 614
 615   CacheIndexAutoLock lock(index);
 616
 617   if (!index->IsIndexUsable()) {
 618     return NS_ERROR_NOT_AVAILABLE;
 619   }
 620
 621   {
 622     CacheIndexEntryAutoManage entryMng(aHash, index);
 623
 624     CacheIndexEntry *entry = index->mIndex.GetEntry(*aHash);
 625     bool entryRemoved = entry && entry->IsRemoved();
 626
 627     if (index->mState == READY || index->mState == UPDATING ||
 628         index->mState == BUILDING) {
 629       MOZ_ASSERT(index->mPendingUpdates.Count() == 0);
 630
 631       if (!entry || entryRemoved) {
 632         if (entryRemoved && entry->IsFresh()) {
 633           // This could happen only if somebody copies files to the entries
 634           // directory while FF is running.
 635           LOG(("CacheIndex::EnsureEntryExists() - Cache file was added outside "
 636                "FF process! Update is needed."));
 637           index->mIndexNeedsUpdate = true;
 638         } else if (index->mState == READY ||
 639                    (entryRemoved && !entry->IsFresh())) {
 640           // Removed non-fresh entries can be present as a result of
 641           // ProcessJournalEntry()
 642           LOG(("CacheIndex::EnsureEntryExists() - Didn't find entry that should"
 643                " exist, update is needed"));
 644           index->mIndexNeedsUpdate = true;
 645         }
 646
 647         if (!entry) {
 648           entry = index->mIndex.PutEntry(*aHash);
 649         }
 650         entry->InitNew();
 651         entry->MarkDirty();
 652       }
 653       entry->MarkFresh();
 654     } else { // WRITING, READING
 655       CacheIndexEntry *updated = index->mPendingUpdates.GetEntry(*aHash);
 656       bool updatedRemoved = updated && updated->IsRemoved();
 657
 658       if (updatedRemoved ||
 659           (!updated && entryRemoved && entry->IsFresh())) {
 660         // Fresh information about missing entry found. This could happen only
 661         // if somebody copies files to the entries directory while FF is running.
 662         LOG(("CacheIndex::EnsureEntryExists() - Cache file was added outside "
 663              "FF process! Update is needed."));
 664         index->mIndexNeedsUpdate = true;
 665       } else if (!updated && (!entry || entryRemoved)) {
 666         if (index->mState == WRITING) {
 667           LOG(("CacheIndex::EnsureEntryExists() - Didn't find entry that should"
 668                " exist, update is needed"));
 669           index->mIndexNeedsUpdate = true;
 670         }
 671         // Ignore if state is READING since the index information is partial
 672       }
 673
 674       // We don't need entryRemoved and updatedRemoved info anymore
 675       if (entryRemoved)   entry = nullptr;
 676       if (updatedRemoved) updated = nullptr;
 677
 678       if (updated) {
 679         updated->MarkFresh();
 680       } else {
 681         if (!entry) {
 682           // Create a new entry
 683           updated = index->mPendingUpdates.PutEntry(*aHash);
 684           updated->InitNew();
 685           updated->MarkFresh();
 686           updated->MarkDirty();
 687         } else {
 688           if (!entry->IsFresh()) {
 689             // To mark the entry fresh we must make a copy of index entry
 690             // since the index is read-only.
 691             updated = index->mPendingUpdates.PutEntry(*aHash);
 692             *updated = *entry;
 693             updated->MarkFresh();
 694           }
 695         }
 696       }
 697     }
 698   }
 699
 700   index->StartUpdatingIndexIfNeeded();
 701   index->WriteIndexToDiskIfNeeded();
 702
 703   return NS_OK;
 704 }
 705
 706 // static
 707 nsresult
 708 CacheIndex::InitEntry(const SHA1Sum::Hash *aHash,
 709                       uint32_t             aAppId,
 710                       bool                 aAnonymous,
 711                       bool                 aInBrowser)
 712 {
 713   LOG(("CacheIndex::InitEntry() [hash=%08x%08x%08x%08x%08x, appId=%u, "
 714        "anonymous=%d, inBrowser=%d]", LOGSHA1(aHash), aAppId, aAnonymous,
 715        aInBrowser));
 716
 717   nsRefPtr<CacheIndex> index = gInstance;
 718
 719   if (!index) {
 720     return NS_ERROR_NOT_INITIALIZED;
 721   }
 722
 723   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
 724
 725   CacheIndexAutoLock lock(index);
 726
 727   if (!index->IsIndexUsable()) {
 728     return NS_ERROR_NOT_AVAILABLE;
 729   }
 730
 731   {
 732     CacheIndexEntryAutoManage entryMng(aHash, index);
 733
 734     CacheIndexEntry *entry = index->mIndex.GetEntry(*aHash);
 735     bool reinitEntry = false;
 736
 737     if (entry && entry->IsRemoved()) {
 738       entry = nullptr;
 739     }
 740
 741     if (index->mState == READY || index->mState == UPDATING ||
 742         index->mState == BUILDING) {
 743       MOZ_ASSERT(index->mPendingUpdates.Count() == 0);
 744       MOZ_ASSERT(entry);
 745       MOZ_ASSERT(entry->IsFresh());
 746
 747       if (IsCollision(entry, aAppId, aAnonymous, aInBrowser)) {
 748         index->mIndexNeedsUpdate = true; // TODO Does this really help in case of collision?
 749         reinitEntry = true;
 750       } else {
 751         if (entry->IsInitialized()) {
 752           return NS_OK;
 753         }
 754       }
 755     } else {
 756       CacheIndexEntry *updated = index->mPendingUpdates.GetEntry(*aHash);
 757       DebugOnly<bool> removed = updated && updated->IsRemoved();
 758
 759       MOZ_ASSERT(updated || !removed);
 760       MOZ_ASSERT(updated || entry);
 761
 762       if (updated) {
 763         MOZ_ASSERT(updated->IsFresh());
 764
 765         if (IsCollision(updated, aAppId, aAnonymous, aInBrowser)) {
 766           index->mIndexNeedsUpdate = true;
 767           reinitEntry = true;
 768         } else {
 769           if (updated->IsInitialized()) {
 770             return NS_OK;
 771           }
 772         }
 773         entry = updated;
 774       } else {
 775         MOZ_ASSERT(entry->IsFresh());
 776
 777         if (IsCollision(entry, aAppId, aAnonymous, aInBrowser)) {
 778           index->mIndexNeedsUpdate = true;
 779           reinitEntry = true;
 780         } else {
 781           if (entry->IsInitialized()) {
 782             return NS_OK;
 783           }
 784         }
 785
 786         // make a copy of a read-only entry
 787         updated = index->mPendingUpdates.PutEntry(*aHash);
 788         *updated = *entry;
 789         entry = updated;
 790       }
 791     }
 792
 793     if (reinitEntry) {
 794       // There is a collision and we are going to rewrite this entry. Initialize
 795       // it as a new entry.
 796       entry->InitNew();
 797       entry->MarkFresh();
 798     }
 799     entry->Init(aAppId, aAnonymous, aInBrowser);
 800     entry->MarkDirty();
 801   }
 802
 803   index->StartUpdatingIndexIfNeeded();
 804   index->WriteIndexToDiskIfNeeded();
 805
 806   return NS_OK;
 807 }
 808
 809 // static
 810 nsresult
 811 CacheIndex::RemoveEntry(const SHA1Sum::Hash *aHash)
 812 {
 813   LOG(("CacheIndex::RemoveEntry() [hash=%08x%08x%08x%08x%08x]",
 814        LOGSHA1(aHash)));
 815
 816   nsRefPtr<CacheIndex> index = gInstance;
 817
 818   if (!index) {
 819     return NS_ERROR_NOT_INITIALIZED;
 820   }
 821
 822   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
 823
 824   CacheIndexAutoLock lock(index);
 825
 826   if (!index->IsIndexUsable()) {
 827     return NS_ERROR_NOT_AVAILABLE;
 828   }
 829
 830   {
 831     CacheIndexEntryAutoManage entryMng(aHash, index);
 832
 833     CacheIndexEntry *entry = index->mIndex.GetEntry(*aHash);
 834     bool entryRemoved = entry && entry->IsRemoved();
 835
 836     if (index->mState == READY || index->mState == UPDATING ||
 837         index->mState == BUILDING) {
 838       MOZ_ASSERT(index->mPendingUpdates.Count() == 0);
 839
 840       if (!entry || entryRemoved) {
 841         if (entryRemoved && entry->IsFresh()) {
 842           // This could happen only if somebody copies files to the entries
 843           // directory while FF is running.
 844           LOG(("CacheIndex::RemoveEntry() - Cache file was added outside FF "
 845                "process! Update is needed."));
 846           index->mIndexNeedsUpdate = true;
 847         } else if (index->mState == READY ||
 848                    (entryRemoved && !entry->IsFresh())) {
 849           // Removed non-fresh entries can be present as a result of
 850           // ProcessJournalEntry()
 851           LOG(("CacheIndex::RemoveEntry() - Didn't find entry that should exist"
 852                ", update is needed"));
 853           index->mIndexNeedsUpdate = true;
 854         }
 855       } else {
 856         if (entry) {
 857           if (!entry->IsDirty() && entry->IsFileEmpty()) {
 858             index->mIndex.RemoveEntry(*aHash);
 859             entry = nullptr;
 860           } else {
 861             entry->MarkRemoved();
 862             entry->MarkDirty();
 863             entry->MarkFresh();
 864           }
 865         }
 866       }
 867     } else { // WRITING, READING
 868       CacheIndexEntry *updated = index->mPendingUpdates.GetEntry(*aHash);
 869       bool updatedRemoved = updated && updated->IsRemoved();
 870
 871       if (updatedRemoved ||
 872           (!updated && entryRemoved && entry->IsFresh())) {
 873         // Fresh information about missing entry found. This could happen only
 874         // if somebody copies files to the entries directory while FF is running.
 875         LOG(("CacheIndex::RemoveEntry() - Cache file was added outside FF "
 876              "process! Update is needed."));
 877         index->mIndexNeedsUpdate = true;
 878       } else if (!updated && (!entry || entryRemoved)) {
 879         if (index->mState == WRITING) {
 880           LOG(("CacheIndex::RemoveEntry() - Didn't find entry that should exist"
 881                ", update is needed"));
 882           index->mIndexNeedsUpdate = true;
 883         }
 884         // Ignore if state is READING since the index information is partial
 885       }
 886
 887       if (!updated) {
 888         updated = index->mPendingUpdates.PutEntry(*aHash);
 889         updated->InitNew();
 890       }
 891
 892       updated->MarkRemoved();
 893       updated->MarkDirty();
 894       updated->MarkFresh();
 895     }
 896   }
 897
 898   index->StartUpdatingIndexIfNeeded();
 899   index->WriteIndexToDiskIfNeeded();
 900
 901   return NS_OK;
 902 }
 903
 904 // static
 905 nsresult
 906 CacheIndex::UpdateEntry(const SHA1Sum::Hash *aHash,
 907                         const uint32_t      *aFrecency,
 908                         const uint32_t      *aExpirationTime,
 909                         const uint32_t      *aSize)
 910 {
 911   LOG(("CacheIndex::UpdateEntry() [hash=%08x%08x%08x%08x%08x, "
 912        "frecency=%s, expirationTime=%s, size=%s]", LOGSHA1(aHash),
 913        aFrecency ? nsPrintfCString("%u", *aFrecency).get() : "",
 914        aExpirationTime ? nsPrintfCString("%u", *aExpirationTime).get() : "",
 915        aSize ? nsPrintfCString("%u", *aSize).get() : ""));
 916
 917   nsRefPtr<CacheIndex> index = gInstance;
 918
 919   if (!index) {
 920     return NS_ERROR_NOT_INITIALIZED;
 921   }
 922
 923   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
 924
 925   CacheIndexAutoLock lock(index);
 926
 927   if (!index->IsIndexUsable()) {
 928     return NS_ERROR_NOT_AVAILABLE;
 929   }
 930
 931   {
 932     CacheIndexEntryAutoManage entryMng(aHash, index);
 933
 934     CacheIndexEntry *entry = index->mIndex.GetEntry(*aHash);
 935
 936     if (entry && entry->IsRemoved()) {
 937       entry = nullptr;
 938     }
 939
 940     if (index->mState == READY || index->mState == UPDATING ||
 941         index->mState == BUILDING) {
 942       MOZ_ASSERT(index->mPendingUpdates.Count() == 0);
 943       MOZ_ASSERT(entry);
 944
 945       if (!HasEntryChanged(entry, aFrecency, aExpirationTime, aSize)) {
 946         return NS_OK;
 947       }
 948     } else {
 949       CacheIndexEntry *updated = index->mPendingUpdates.GetEntry(*aHash);
 950       DebugOnly<bool> removed = updated && updated->IsRemoved();
 951
 952       MOZ_ASSERT(updated || !removed);
 953       MOZ_ASSERT(updated || entry);
 954
 955       if (!updated) {
 956         if (entry &&
 957             HasEntryChanged(entry, aFrecency, aExpirationTime, aSize)) {
 958           // make a copy of a read-only entry
 959           updated = index->mPendingUpdates.PutEntry(*aHash);
 960           *updated = *entry;
 961           entry = updated;
 962         } else {
 963           return NS_ERROR_NOT_AVAILABLE;
 964         }
 965       } else {
 966         entry = updated;
 967       }
 968     }
 969
 970     MOZ_ASSERT(entry->IsFresh());
 971     MOZ_ASSERT(entry->IsInitialized());
 972     entry->MarkDirty();
 973
 974     if (aFrecency) {
 975       entry->SetFrecency(*aFrecency);
 976     }
 977
 978     if (aExpirationTime) {
 979       entry->SetExpirationTime(*aExpirationTime);
 980     }
 981
 982     if (aSize) {
 983       entry->SetFileSize(*aSize);
 984     }
 985   }
 986
 987   index->WriteIndexToDiskIfNeeded();
 988
 989   return NS_OK;
 990 }
 991
 992 // static
 993 nsresult
 994 CacheIndex::RemoveAll()
 995 {
 996   LOG(("CacheIndex::RemoveAll()"));
 997
 998   nsRefPtr<CacheIndex> index = gInstance;
 999
1000   if (!index) {
1001     return NS_ERROR_NOT_INITIALIZED;
1002   }
1003
1004   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
1005
1006   nsCOMPtr<nsIFile> file;
1007
1008   {
1009     CacheIndexAutoLock lock(index);
1010
1011     MOZ_ASSERT(!index->mRemovingAll);
1012
1013     if (!index->IsIndexUsable()) {
1014       return NS_ERROR_NOT_AVAILABLE;
1015     }
1016
1017     AutoRestore<bool> saveRemovingAll(index->mRemovingAll);
1018     index->mRemovingAll = true;
1019
1020     // Doom index and journal handles but don't null them out since this will be
1021     // done in FinishWrite/FinishRead methods.
1022     if (index->mIndexHandle) {
1023       CacheFileIOManager::DoomFile(index->mIndexHandle, nullptr);
1024     } else {
1025       // We don't have a handle to index file, so get the file here, but delete
1026       // it outside the lock. Ignore the result since this is not fatal.
1027       index->GetFile(NS_LITERAL_CSTRING(kIndexName), getter_AddRefs(file));
1028     }
1029
1030     if (index->mJournalHandle) {
1031       CacheFileIOManager::DoomFile(index->mJournalHandle, nullptr);
1032     }
1033
1034     switch (index->mState) {
1035       case WRITING:
1036         index->FinishWrite(false);
1037         break;
1038       case READY:
1039         // nothing to do
1040         break;
1041       case READING:
1042         index->FinishRead(false);
1043         break;
1044       case BUILDING:
1045       case UPDATING:
1046         index->FinishUpdate(false);
1047         break;
1048       default:
1049         MOZ_ASSERT(false, "Unexpected state!");
1050     }
1051
1052     // We should end up in READY state
1053     MOZ_ASSERT(index->mState == READY);
1054
1055     // There should not be any handle
1056     MOZ_ASSERT(!index->mIndexHandle);
1057     MOZ_ASSERT(!index->mJournalHandle);
1058
1059     index->mIndexOnDiskIsValid = false;
1060     index->mIndexNeedsUpdate = false;
1061
1062     index->mIndexStats.Clear();
1063     index->mFrecencyArray.Clear();
1064     index->mExpirationArray.Clear();
1065     index->mIndex.Clear();
1066   }
1067
1068   if (file) {
1069     // Ignore the result. The file might not exist and the failure is not fatal.
1070     file->Remove(false);
1071   }
1072
1073   return NS_OK;
1074 }
1075
1076 // static
1077 nsresult
1078 CacheIndex::HasEntry(const nsACString &aKey, EntryStatus *_retval)
1079 {
1080   LOG(("CacheIndex::HasEntry() [key=%s]", PromiseFlatCString(aKey).get()));
1081
1082   nsRefPtr<CacheIndex> index = gInstance;
1083
1084   if (!index) {
1085     return NS_ERROR_NOT_INITIALIZED;
1086   }
1087
1088   CacheIndexAutoLock lock(index);
1089
1090   if (!index->IsIndexUsable()) {
1091     return NS_ERROR_NOT_AVAILABLE;
1092   }
1093
1094   SHA1Sum sum;
1095   SHA1Sum::Hash hash;
1096   sum.update(aKey.BeginReading(), aKey.Length());
1097   sum.finish(hash);
1098
1099   CacheIndexEntry *entry = nullptr;
1100
1101   switch (index->mState) {
1102     case READING:
1103     case WRITING:
1104       entry = index->mPendingUpdates.GetEntry(hash);
1105       // no break
1106     case BUILDING:
1107     case UPDATING:
1108     case READY:
1109       if (!entry) {
1110         entry = index->mIndex.GetEntry(hash);
1111       }
1112       break;
1113     case INITIAL:
1114     case SHUTDOWN:
1115       MOZ_ASSERT(false, "Unexpected state!");
1116   }
1117
1118   if (!entry) {
1119     if (index->mState == READY || index->mState == WRITING) {
1120       *_retval = DOES_NOT_EXIST;
1121     } else {
1122       *_retval = DO_NOT_KNOW;
1123     }
1124   } else {
1125     if (entry->IsRemoved()) {
1126       if (entry->IsFresh()) {
1127         *_retval = DOES_NOT_EXIST;
1128       } else {
1129         *_retval = DO_NOT_KNOW;
1130       }
1131     } else {
1132       *_retval = EXISTS;
1133     }
1134   }
1135
1136   LOG(("CacheIndex::HasEntry() - result is %u", *_retval));
1137   return NS_OK;
1138 }
1139
1140 // static
1141 nsresult
1142 CacheIndex::GetEntryForEviction(SHA1Sum::Hash *aHash, uint32_t *aCnt)
1143 {
1144   LOG(("CacheIndex::GetEntryForEviction()"));
1145
1146   nsRefPtr<CacheIndex> index = gInstance;
1147
1148   if (!index)
1149     return NS_ERROR_NOT_INITIALIZED;
1150
1151   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
1152
1153   CacheIndexAutoLock lock(index);
1154
1155   if (!index->IsIndexUsable()) {
1156     return NS_ERROR_NOT_AVAILABLE;
1157   }
1158
1159   MOZ_ASSERT(index->mFrecencyArray.Length() ==
1160              index->mExpirationArray.Length());
1161
1162   if (index->mExpirationArray.Length() == 0)
1163     return NS_ERROR_NOT_AVAILABLE;
1164
1165   SHA1Sum::Hash hash;
1166   bool foundEntry = false;
1167   uint32_t i = 0, j = 0;
1168   uint32_t now = PR_Now() / PR_USEC_PER_SEC;
1169
1170   // find the first expired, non-forced valid entry
1171   for (i = 0; i < index->mExpirationArray.Length(); i++) {
1172     if (index->mExpirationArray[i]->mExpirationTime < now) {
1173       memcpy(&hash, &index->mExpirationArray[i]->mHash, sizeof(SHA1Sum::Hash));
1174
1175       if (!IsForcedValidEntry(&hash)) {
1176         foundEntry = true;
1177         break;
1178       }
1179
1180     } else {
1181       // all further entries have not expired yet
1182       break;
1183     }
1184   }
1185
1186   if (foundEntry) {
1187     *aCnt = index->mExpirationArray.Length() - i;
1188
1189     LOG(("CacheIndex::GetEntryForEviction() - returning entry from expiration "
1190          "array [hash=%08x%08x%08x%08x%08x, cnt=%u, expTime=%u, now=%u, "
1191          "frecency=%u]", LOGSHA1(&hash), *aCnt,
1192          index->mExpirationArray[i]->mExpirationTime, now,
1193          index->mExpirationArray[i]->mFrecency));
1194   }
1195   else {
1196     // check if we've already tried all the entries
1197     if (i == index->mExpirationArray.Length())
1198       return NS_ERROR_NOT_AVAILABLE;
1199
1200     // find first non-forced valid entry with the lowest frecency
1201     for (j = 0; j < index->mFrecencyArray.Length(); j++) {
1202       memcpy(&hash, &index->mFrecencyArray[j]->mHash, sizeof(SHA1Sum::Hash));
1203
1204       if (!IsForcedValidEntry(&hash)) {
1205         foundEntry = true;
1206         break;
1207       }
1208     }
1209
1210     if (!foundEntry)
1211       return NS_ERROR_NOT_AVAILABLE;
1212
1213     // forced valid entries skipped in both arrays could overlap, just use max
1214     *aCnt = index->mFrecencyArray.Length() - std::max(i, j);
1215
1216     LOG(("CacheIndex::GetEntryForEviction() - returning entry from frecency "
1217          "array [hash=%08x%08x%08x%08x%08x, cnt=%u, expTime=%u, now=%u, "
1218          "frecency=%u]", LOGSHA1(&hash), *aCnt,
1219          index->mFrecencyArray[j]->mExpirationTime, now,
1220          index->mFrecencyArray[j]->mFrecency));
1221   }
1222
1223   memcpy(aHash, &hash, sizeof(SHA1Sum::Hash));
1224
1225   return NS_OK;
1226 }
1227
1228
1229 // static
1230 bool CacheIndex::IsForcedValidEntry(const SHA1Sum::Hash *aHash)
1231 {
1232   nsRefPtr<CacheFileHandle> handle;
1233
1234   CacheFileIOManager::gInstance->mHandles.GetHandle(
1235     aHash, false, getter_AddRefs(handle));
1236
1237   if (!handle)
1238     return false;
1239
1240   nsCString hashKey = handle->Key();
1241   return CacheStorageService::Self()->IsForcedValidEntry(hashKey);
1242 }
1243
1244
1245 // static
1246 nsresult
1247 CacheIndex::GetCacheSize(uint32_t *_retval)
1248 {
1249   LOG(("CacheIndex::GetCacheSize()"));
1250
1251   nsRefPtr<CacheIndex> index = gInstance;
1252
1253   if (!index)
1254     return NS_ERROR_NOT_INITIALIZED;
1255
1256   CacheIndexAutoLock lock(index);
1257
1258   if (!index->IsIndexUsable()) {
1259     return NS_ERROR_NOT_AVAILABLE;
1260   }
1261
1262   *_retval = index->mIndexStats.Size();
1263   LOG(("CacheIndex::GetCacheSize() - returning %u", *_retval));
1264   return NS_OK;
1265 }
1266
1267 // static
1268 nsresult
1269 CacheIndex::GetCacheStats(nsILoadContextInfo *aInfo, uint32_t *aSize, uint32_t *aCount)
1270 {
1271   LOG(("CacheIndex::GetCacheStats() [info=%p]", aInfo));
1272
1273   nsRefPtr<CacheIndex> index = gInstance;
1274
1275   if (!index) {
1276     return NS_ERROR_NOT_INITIALIZED;
1277   }
1278
1279   CacheIndexAutoLock lock(index);
1280
1281   if (!index->IsIndexUsable()) {
1282     return NS_ERROR_NOT_AVAILABLE;
1283   }
1284
1285   if (!aInfo) {
1286     return NS_ERROR_INVALID_ARG;
1287   }
1288
1289   *aSize = 0;
1290   *aCount = 0;
1291
1292   for (uint32_t i = 0; i < index->mFrecencyArray.Length(); ++i) {
1293     CacheIndexRecord* record = index->mFrecencyArray[i];
1294     if (!CacheIndexEntry::RecordMatchesLoadContextInfo(record, aInfo))
1295       continue;
1296
1297     *aSize += CacheIndexEntry::GetFileSize(record);
1298     ++*aCount;
1299   }
1300
1301   return NS_OK;
1302 }
1303
1304 // static
1305 nsresult
1306 CacheIndex::AsyncGetDiskConsumption(nsICacheStorageConsumptionObserver* aObserver)
1307 {
1308   LOG(("CacheIndex::AsyncGetDiskConsumption()"));
1309
1310   nsRefPtr<CacheIndex> index = gInstance;
1311
1312   if (!index) {
1313     return NS_ERROR_NOT_INITIALIZED;
1314   }
1315
1316   CacheIndexAutoLock lock(index);
1317
1318   if (!index->IsIndexUsable()) {
1319     return NS_ERROR_NOT_AVAILABLE;
1320   }
1321
1322   nsRefPtr<DiskConsumptionObserver> observer =
1323     DiskConsumptionObserver::Init(aObserver);
1324
1325   NS_ENSURE_ARG(observer);
1326
1327   if (index->mState == READY || index->mState == WRITING) {
1328     LOG(("CacheIndex::AsyncGetDiskConsumption - calling immediately"));
1329     // Safe to call the callback under the lock,
1330     // we always post to the main thread.
1331     observer->OnDiskConsumption(index->mIndexStats.Size() << 10);
1332     return NS_OK;
1333   }
1334
1335   LOG(("CacheIndex::AsyncGetDiskConsumption - remembering callback"));
1336   // Will be called when the index get to the READY state.
1337   index->mDiskConsumptionObservers.AppendElement(observer);
1338
1339   return NS_OK;
1340 }
1341
1342 // static
1343 nsresult
1344 CacheIndex::GetIterator(nsILoadContextInfo *aInfo, bool aAddNew,
1345                         CacheIndexIterator **_retval)
1346 {
1347   LOG(("CacheIndex::GetIterator() [info=%p, addNew=%d]", aInfo, aAddNew));
1348
1349   nsRefPtr<CacheIndex> index = gInstance;
1350
1351   if (!index) {
1352     return NS_ERROR_NOT_INITIALIZED;
1353   }
1354
1355   CacheIndexAutoLock lock(index);
1356
1357   if (!index->IsIndexUsable()) {
1358     return NS_ERROR_NOT_AVAILABLE;
1359   }
1360
1361   nsRefPtr<CacheIndexIterator> iter;
1362   if (aInfo) {
1363     iter = new CacheIndexContextIterator(index, aAddNew, aInfo);
1364   } else {
1365     iter = new CacheIndexIterator(index, aAddNew);
1366   }
1367
1368   iter->AddRecords(index->mFrecencyArray);
1369
1370   index->mIterators.AppendElement(iter);
1371   iter.swap(*_retval);
1372   return NS_OK;
1373 }
1374
1375 // static
1376 nsresult
1377 CacheIndex::IsUpToDate(bool *_retval)
1378 {
1379   LOG(("CacheIndex::IsUpToDate()"));
1380
1381   nsRefPtr<CacheIndex> index = gInstance;
1382
1383   if (!index) {
1384     return NS_ERROR_NOT_INITIALIZED;
1385   }
1386
1387   CacheIndexAutoLock lock(index);
1388
1389   if (!index->IsIndexUsable()) {
1390     return NS_ERROR_NOT_AVAILABLE;
1391   }
1392
1393   *_retval = (index->mState == READY || index->mState == WRITING) &&
1394              !index->mIndexNeedsUpdate && !index->mShuttingDown;
1395
1396   LOG(("CacheIndex::IsUpToDate() - returning %p", *_retval));
1397   return NS_OK;
1398 }
1399
1400 bool
1401 CacheIndex::IsIndexUsable()
1402 {
1403   MOZ_ASSERT(mState != INITIAL);
1404
1405   switch (mState) {
1406     case INITIAL:
1407     case SHUTDOWN:
1408       return false;
1409
1410     case READING:
1411     case WRITING:
1412     case BUILDING:
1413     case UPDATING:
1414     case READY:
1415       break;
1416   }
1417
1418   return true;
1419 }
1420
1421 // static
1422 bool
1423 CacheIndex::IsCollision(CacheIndexEntry *aEntry,
1424                         uint32_t         aAppId,
1425                         bool             aAnonymous,
1426                         bool             aInBrowser)
1427 {
1428   if (!aEntry->IsInitialized()) {
1429     return false;
1430   }
1431
1432   if (aEntry->AppId() != aAppId || aEntry->Anonymous() != aAnonymous ||
1433       aEntry->InBrowser() != aInBrowser) {
1434     LOG(("CacheIndex::IsCollision() - Collision detected for entry hash=%08x"
1435          "%08x%08x%08x%08x, expected values: appId=%u, anonymous=%d, "
1436          "inBrowser=%d; actual values: appId=%u, anonymous=%d, inBrowser=%d]",
1437          LOGSHA1(aEntry->Hash()), aAppId, aAnonymous, aInBrowser,
1438          aEntry->AppId(), aEntry->Anonymous(), aEntry->InBrowser()));
1439     return true;
1440   }
1441
1442   return false;
1443 }
1444
1445 // static
1446 bool
1447 CacheIndex::HasEntryChanged(CacheIndexEntry *aEntry,
1448                             const uint32_t  *aFrecency,
1449                             const uint32_t  *aExpirationTime,
1450                             const uint32_t  *aSize)
1451 {
1452   if (aFrecency && *aFrecency != aEntry->GetFrecency()) {
1453     return true;
1454   }
1455
1456   if (aExpirationTime && *aExpirationTime != aEntry->GetExpirationTime()) {
1457     return true;
1458   }
1459
1460   if (aSize &&
1461       (*aSize & CacheIndexEntry::kFileSizeMask) != aEntry->GetFileSize()) {
1462     return true;
1463   }
1464
1465   return false;
1466 }
1467
1468 void
1469 CacheIndex::ProcessPendingOperations()
1470 {
1471   LOG(("CacheIndex::ProcessPendingOperations()"));
1472
1473   AssertOwnsLock();
1474
1475   mPendingUpdates.EnumerateEntries(&CacheIndex::UpdateEntryInIndex, this);
1476
1477   MOZ_ASSERT(mPendingUpdates.Count() == 0);
1478
1479   EnsureCorrectStats();
1480 }
1481
1482 // static
1483 PLDHashOperator
1484 CacheIndex::UpdateEntryInIndex(CacheIndexEntry *aEntry, void* aClosure)
1485 {
1486   CacheIndex *index = static_cast<CacheIndex *>(aClosure);
1487
1488   LOG(("CacheFile::UpdateEntryInIndex() [hash=%08x%08x%08x%08x%08x]",
1489        LOGSHA1(aEntry->Hash())));
1490
1491   MOZ_ASSERT(aEntry->IsFresh());
1492   MOZ_ASSERT(aEntry->IsDirty());
1493
1494   CacheIndexEntry *entry = index->mIndex.GetEntry(*aEntry->Hash());
1495
1496   CacheIndexEntryAutoManage emng(aEntry->Hash(), index);
1497   emng.DoNotSearchInUpdates();
1498
1499   if (aEntry->IsRemoved()) {
1500     if (entry) {
1501       if (entry->IsRemoved()) {
1502         MOZ_ASSERT(entry->IsFresh());
1503         MOZ_ASSERT(entry->IsDirty());
1504       } else if (!entry->IsDirty() && entry->IsFileEmpty()) {
1505         // Entries with empty file are not stored in index on disk. Just remove
1506         // the entry, but only in case the entry is not dirty, i.e. the entry
1507         // file was empty when we wrote the index.
1508         index->mIndex.RemoveEntry(*aEntry->Hash());
1509         entry = nullptr;
1510       } else {
1511         entry->MarkRemoved();
1512         entry->MarkDirty();
1513         entry->MarkFresh();
1514       }
1515     }
1516
1517     return PL_DHASH_REMOVE;
1518   }
1519
1520   entry = index->mIndex.PutEntry(*aEntry->Hash());
1521   *entry = *aEntry;
1522
1523   return PL_DHASH_REMOVE;
1524 }
1525
1526 bool
1527 CacheIndex::WriteIndexToDiskIfNeeded()
1528 {
1529   if (mState != READY || mShuttingDown) {
1530     return false;
1531   }
1532
1533   if (!mLastDumpTime.IsNull() &&
1534       (TimeStamp::NowLoRes() - mLastDumpTime).ToMilliseconds() <
1535       kMinDumpInterval) {
1536     return false;
1537   }
1538
1539   if (mIndexStats.Dirty() < kMinUnwrittenChanges) {
1540     return false;
1541   }
1542
1543   WriteIndexToDisk();
1544   return true;
1545 }
1546
1547 void
1548 CacheIndex::WriteIndexToDisk()
1549 {
1550   LOG(("CacheIndex::WriteIndexToDisk()"));
1551   mIndexStats.Log();
1552
1553   nsresult rv;
1554
1555   AssertOwnsLock();
1556   MOZ_ASSERT(mState == READY);
1557   MOZ_ASSERT(!mRWBuf);
1558   MOZ_ASSERT(!mRWHash);
1559
1560   ChangeState(WRITING);
1561
1562   mProcessEntries = mIndexStats.ActiveEntriesCount();
1563
1564   mIndexFileOpener = new FileOpenHelper(this);
1565   rv = CacheFileIOManager::OpenFile(NS_LITERAL_CSTRING(kTempIndexName),
1566                                     CacheFileIOManager::SPECIAL_FILE |
1567                                     CacheFileIOManager::CREATE,
1568                                     mIndexFileOpener);
1569   if (NS_FAILED(rv)) {
1570     LOG(("CacheIndex::WriteIndexToDisk() - Can't open file [rv=0x%08x]", rv));
1571     FinishWrite(false);
1572     return;
1573   }
1574
1575   // Write index header to a buffer, it will be written to disk together with
1576   // records in WriteRecords() once we open the file successfully.
1577   AllocBuffer();
1578   mRWHash = new CacheHash();
1579
1580   CacheIndexHeader *hdr = reinterpret_cast<CacheIndexHeader *>(mRWBuf);
1581   NetworkEndian::writeUint32(&hdr->mVersion, kIndexVersion);
1582   NetworkEndian::writeUint32(&hdr->mTimeStamp,
1583                              static_cast<uint32_t>(PR_Now() / PR_USEC_PER_SEC));
1584   NetworkEndian::writeUint32(&hdr->mIsDirty, 1);
1585
1586   mRWBufPos = sizeof(CacheIndexHeader);
1587   mSkipEntries = 0;
1588 }
1589
1590 namespace { // anon
1591
1592 struct WriteRecordsHelper
1593 {
1594   char    *mBuf;
1595   uint32_t mSkip;
1596   uint32_t mProcessMax;
1597   uint32_t mProcessed;
1598 #ifdef DEBUG
1599   bool     mHasMore;
1600 #endif
1601 };
1602
1603 } // anon
1604
1605 void
1606 CacheIndex::WriteRecords()
1607 {
1608   LOG(("CacheIndex::WriteRecords()"));
1609
1610   nsresult rv;
1611
1612   AssertOwnsLock();
1613   MOZ_ASSERT(mState == WRITING);
1614
1615   int64_t fileOffset;
1616
1617   if (mSkipEntries) {
1618     MOZ_ASSERT(mRWBufPos == 0);
1619     fileOffset = sizeof(CacheIndexHeader);
1620     fileOffset += sizeof(CacheIndexRecord) * mSkipEntries;
1621   } else {
1622     MOZ_ASSERT(mRWBufPos == sizeof(CacheIndexHeader));
1623     fileOffset = 0;
1624   }
1625   uint32_t hashOffset = mRWBufPos;
1626
1627   WriteRecordsHelper data;
1628   data.mBuf = mRWBuf + mRWBufPos;
1629   data.mSkip = mSkipEntries;
1630   data.mProcessMax = (mRWBufSize - mRWBufPos) / sizeof(CacheIndexRecord);
1631   MOZ_ASSERT(data.mProcessMax != 0 || mProcessEntries == 0); // TODO make sure we can write an empty index
1632   data.mProcessed = 0;
1633 #ifdef DEBUG
1634   data.mHasMore = false;
1635 #endif
1636
1637   mIndex.EnumerateEntries(&CacheIndex::CopyRecordsToRWBuf, &data);
1638   MOZ_ASSERT(mRWBufPos != static_cast<uint32_t>(data.mBuf - mRWBuf) ||
1639              mProcessEntries == 0);
1640   mRWBufPos = data.mBuf - mRWBuf;
1641   mSkipEntries += data.mProcessed;
1642   MOZ_ASSERT(mSkipEntries <= mProcessEntries);
1643
1644   mRWHash->Update(mRWBuf + hashOffset, mRWBufPos - hashOffset);
1645
1646   if (mSkipEntries == mProcessEntries) {
1647     MOZ_ASSERT(!data.mHasMore);
1648
1649     // We've processed all records
1650     if (mRWBufPos + sizeof(CacheHash::Hash32_t) > mRWBufSize) {
1651       // realloc buffer to spare another write cycle
1652       mRWBufSize = mRWBufPos + sizeof(CacheHash::Hash32_t);
1653       mRWBuf = static_cast<char *>(moz_xrealloc(mRWBuf, mRWBufSize));
1654     }
1655
1656     NetworkEndian::writeUint32(mRWBuf + mRWBufPos, mRWHash->GetHash());
1657     mRWBufPos += sizeof(CacheHash::Hash32_t);
1658   } else {
1659     MOZ_ASSERT(data.mHasMore);
1660   }
1661
1662   rv = CacheFileIOManager::Write(mIndexHandle, fileOffset, mRWBuf, mRWBufPos,
1663                                  mSkipEntries == mProcessEntries, this);
1664   if (NS_FAILED(rv)) {
1665     LOG(("CacheIndex::WriteRecords() - CacheFileIOManager::Write() failed "
1666          "synchronously [rv=0x%08x]", rv));
1667     FinishWrite(false);
1668   }
1669
1670   mRWBufPos = 0;
1671 }
1672
1673 void
1674 CacheIndex::FinishWrite(bool aSucceeded)
1675 {
1676   LOG(("CacheIndex::FinishWrite() [succeeded=%d]", aSucceeded));
1677
1678   MOZ_ASSERT((!aSucceeded && mState == SHUTDOWN) || mState == WRITING);
1679
1680   AssertOwnsLock();
1681
1682   mIndexHandle = nullptr;
1683   mRWHash = nullptr;
1684   ReleaseBuffer();
1685
1686   if (aSucceeded) {
1687     // Opening of the file must not be in progress if writing succeeded.
1688     MOZ_ASSERT(!mIndexFileOpener);
1689
1690     mIndex.EnumerateEntries(&CacheIndex::ApplyIndexChanges, this);
1691     mIndexOnDiskIsValid = true;
1692   } else {
1693     if (mIndexFileOpener) {
1694       // If opening of the file is still in progress (e.g. WRITE process was
1695       // canceled by RemoveAll()) then we need to cancel the opener to make sure
1696       // that OnFileOpenedInternal() won't be called.
1697       mIndexFileOpener->Cancel();
1698       mIndexFileOpener = nullptr;
1699     }
1700   }
1701
1702   ProcessPendingOperations();
1703   mIndexStats.Log();
1704
1705   if (mState == WRITING) {
1706     ChangeState(READY);
1707     mLastDumpTime = TimeStamp::NowLoRes();
1708   }
1709 }
1710
1711 // static
1712 PLDHashOperator
1713 CacheIndex::CopyRecordsToRWBuf(CacheIndexEntry *aEntry, void* aClosure)
1714 {
1715   if (aEntry->IsRemoved()) {
1716     return PL_DHASH_NEXT;
1717   }
1718
1719   if (!aEntry->IsInitialized()) {
1720     return PL_DHASH_NEXT;
1721   }
1722
1723   if (aEntry->IsFileEmpty()) {
1724     return PL_DHASH_NEXT;
1725   }
1726
1727   WriteRecordsHelper *data = static_cast<WriteRecordsHelper *>(aClosure);
1728   if (data->mSkip) {
1729     data->mSkip--;
1730     return PL_DHASH_NEXT;
1731   }
1732
1733   if (data->mProcessed == data->mProcessMax) {
1734 #ifdef DEBUG
1735     data->mHasMore = true;
1736 #endif
1737     return PL_DHASH_STOP;
1738   }
1739
1740   aEntry->WriteToBuf(data->mBuf);
1741   data->mBuf += sizeof(CacheIndexRecord);
1742   data->mProcessed++;
1743
1744   return PL_DHASH_NEXT;
1745 }
1746
1747 // static
1748 PLDHashOperator
1749 CacheIndex::ApplyIndexChanges(CacheIndexEntry *aEntry, void* aClosure)
1750 {
1751   CacheIndex *index = static_cast<CacheIndex *>(aClosure);
1752
1753   CacheIndexEntryAutoManage emng(aEntry->Hash(), index);
1754
1755   if (aEntry->IsRemoved()) {
1756     emng.DoNotSearchInIndex();
1757     return PL_DHASH_REMOVE;
1758   }
1759
1760   if (aEntry->IsDirty()) {
1761     aEntry->ClearDirty();
1762   }
1763
1764   return PL_DHASH_NEXT;
1765 }
1766
1767 nsresult
1768 CacheIndex::GetFile(const nsACString &aName, nsIFile **_retval)
1769 {
1770   nsresult rv;
1771
1772   nsCOMPtr<nsIFile> file;
1773   rv = mCacheDirectory->Clone(getter_AddRefs(file));
1774   NS_ENSURE_SUCCESS(rv, rv);
1775
1776   rv = file->AppendNative(aName);
1777   NS_ENSURE_SUCCESS(rv, rv);
1778
1779   file.swap(*_retval);
1780   return NS_OK;
1781 }
1782
1783 nsresult
1784 CacheIndex::RemoveFile(const nsACString &aName)
1785 {
1786   MOZ_ASSERT(mState == SHUTDOWN);
1787
1788   nsresult rv;
1789
1790   nsCOMPtr<nsIFile> file;
1791   rv = GetFile(aName, getter_AddRefs(file));
1792   NS_ENSURE_SUCCESS(rv, rv);
1793
1794   bool exists;
1795   rv = file->Exists(&exists);
1796   NS_ENSURE_SUCCESS(rv, rv);
1797
1798   if (exists) {
1799     rv = file->Remove(false);
1800     if (NS_FAILED(rv)) {
1801       LOG(("CacheIndex::RemoveFile() - Cannot remove old entry file from disk."
1802            "[name=%s]", PromiseFlatCString(aName).get()));
1803       NS_WARNING("Cannot remove old entry file from the disk");
1804       return rv;
1805     }
1806   }
1807
1808   return NS_OK;
1809 }
1810
1811 void
1812 CacheIndex::RemoveIndexFromDisk()
1813 {
1814   LOG(("CacheIndex::RemoveIndexFromDisk()"));
1815
1816   RemoveFile(NS_LITERAL_CSTRING(kIndexName));
1817   RemoveFile(NS_LITERAL_CSTRING(kTempIndexName));
1818   RemoveFile(NS_LITERAL_CSTRING(kJournalName));
1819 }
1820
1821 class WriteLogHelper
1822 {
1823 public:
1824   explicit WriteLogHelper(PRFileDesc *aFD)
1825     : mStatus(NS_OK)
1826     , mFD(aFD)
1827     , mBufSize(kMaxBufSize)
1828     , mBufPos(0)
1829   {
1830     mHash = new CacheHash();
1831     mBuf = static_cast<char *>(moz_xmalloc(mBufSize));
1832   }
1833
1834   ~WriteLogHelper() {
1835     free(mBuf);
1836   }
1837
1838   nsresult AddEntry(CacheIndexEntry *aEntry);
1839   nsresult Finish();
1840
1841 private:
1842
1843   nsresult FlushBuffer();
1844
1845   nsresult            mStatus;
1846   PRFileDesc         *mFD;
1847   char               *mBuf;
1848   uint32_t            mBufSize;
1849   int32_t             mBufPos;
1850   nsRefPtr<CacheHash> mHash;
1851 };
1852
1853 nsresult
1854 WriteLogHelper::AddEntry(CacheIndexEntry *aEntry)
1855 {
1856   nsresult rv;
1857
1858   if (NS_FAILED(mStatus)) {
1859     return mStatus;
1860   }
1861
1862   if (mBufPos + sizeof(CacheIndexRecord) > mBufSize) {
1863     mHash->Update(mBuf, mBufPos);
1864
1865     rv = FlushBuffer();
1866     if (NS_FAILED(rv)) {
1867       mStatus = rv;
1868       return rv;
1869     }
1870     MOZ_ASSERT(mBufPos + sizeof(CacheIndexRecord) <= mBufSize);
1871   }
1872
1873   aEntry->WriteToBuf(mBuf + mBufPos);
1874   mBufPos += sizeof(CacheIndexRecord);
1875
1876   return NS_OK;
1877 }
1878
1879 nsresult
1880 WriteLogHelper::Finish()
1881 {
1882   nsresult rv;
1883
1884   if (NS_FAILED(mStatus)) {
1885     return mStatus;
1886   }
1887
1888   mHash->Update(mBuf, mBufPos);
1889   if (mBufPos + sizeof(CacheHash::Hash32_t) > mBufSize) {
1890     rv = FlushBuffer();
1891     if (NS_FAILED(rv)) {
1892       mStatus = rv;
1893       return rv;
1894     }
1895     MOZ_ASSERT(mBufPos + sizeof(CacheHash::Hash32_t) <= mBufSize);
1896   }
1897
1898   NetworkEndian::writeUint32(mBuf + mBufPos, mHash->GetHash());
1899   mBufPos += sizeof(CacheHash::Hash32_t);
1900
1901   rv = FlushBuffer();
1902   NS_ENSURE_SUCCESS(rv, rv);
1903
1904   mStatus = NS_ERROR_UNEXPECTED; // Don't allow any other operation
1905   return NS_OK;
1906 }
1907
1908 nsresult
1909 WriteLogHelper::FlushBuffer()
1910 {
1911   MOZ_ASSERT(NS_SUCCEEDED(mStatus));
1912
1913   int32_t bytesWritten = PR_Write(mFD, mBuf, mBufPos);
1914
1915   if (bytesWritten != mBufPos) {
1916     return NS_ERROR_FAILURE;
1917   }
1918
1919   mBufPos = 0;
1920   return NS_OK;
1921 }
1922
1923 nsresult
1924 CacheIndex::WriteLogToDisk()
1925 {
1926   LOG(("CacheIndex::WriteLogToDisk()"));
1927
1928   nsresult rv;
1929
1930   MOZ_ASSERT(mPendingUpdates.Count() == 0);
1931   MOZ_ASSERT(mState == SHUTDOWN);
1932
1933   RemoveFile(NS_LITERAL_CSTRING(kTempIndexName));
1934
1935   nsCOMPtr<nsIFile> indexFile;
1936   rv = GetFile(NS_LITERAL_CSTRING(kIndexName), getter_AddRefs(indexFile));
1937   NS_ENSURE_SUCCESS(rv, rv);
1938
1939   nsCOMPtr<nsIFile> logFile;
1940   rv = GetFile(NS_LITERAL_CSTRING(kJournalName), getter_AddRefs(logFile));
1941   NS_ENSURE_SUCCESS(rv, rv);
1942
1943   mIndexStats.Log();
1944
1945   PRFileDesc *fd = nullptr;
1946   rv = logFile->OpenNSPRFileDesc(PR_RDWR | PR_CREATE_FILE | PR_TRUNCATE,
1947                                  0600, &fd);
1948   NS_ENSURE_SUCCESS(rv, rv);
1949
1950   WriteLogHelper wlh(fd);
1951   mIndex.EnumerateEntries(&CacheIndex::WriteEntryToLog, &wlh);
1952
1953   rv = wlh.Finish();
1954   PR_Close(fd);
1955   NS_ENSURE_SUCCESS(rv, rv);
1956
1957   rv = indexFile->OpenNSPRFileDesc(PR_RDWR, 0600, &fd);
1958   NS_ENSURE_SUCCESS(rv, rv);
1959
1960   CacheIndexHeader header;
1961   int32_t bytesRead = PR_Read(fd, &header, sizeof(CacheIndexHeader));
1962   if (bytesRead != sizeof(CacheIndexHeader)) {
1963     PR_Close(fd);
1964     return NS_ERROR_FAILURE;
1965   }
1966
1967   NetworkEndian::writeUint32(&header.mIsDirty, 0);
1968
1969   int64_t offset = PR_Seek64(fd, 0, PR_SEEK_SET);
1970   if (offset == -1) {
1971     PR_Close(fd);
1972     return NS_ERROR_FAILURE;
1973   }
1974
1975   int32_t bytesWritten = PR_Write(fd, &header, sizeof(CacheIndexHeader));
1976   PR_Close(fd);
1977   if (bytesWritten != sizeof(CacheIndexHeader)) {
1978     return NS_ERROR_FAILURE;
1979   }
1980
1981   return NS_OK;
1982 }
1983
1984 // static
1985 PLDHashOperator
1986 CacheIndex::WriteEntryToLog(CacheIndexEntry *aEntry, void* aClosure)
1987 {
1988   WriteLogHelper *wlh = static_cast<WriteLogHelper *>(aClosure);
1989
1990   if (aEntry->IsRemoved() || aEntry->IsDirty()) {
1991     wlh->AddEntry(aEntry);
1992   }
1993
1994   return PL_DHASH_REMOVE;
1995 }
1996
1997 void
1998 CacheIndex::ReadIndexFromDisk()
1999 {
2000   LOG(("CacheIndex::ReadIndexFromDisk()"));
2001
2002   nsresult rv;
2003
2004   AssertOwnsLock();
2005   MOZ_ASSERT(mState == INITIAL);
2006
2007   ChangeState(READING);
2008
2009   mIndexFileOpener = new FileOpenHelper(this);
2010   rv = CacheFileIOManager::OpenFile(NS_LITERAL_CSTRING(kIndexName),
2011                                     CacheFileIOManager::SPECIAL_FILE |
2012                                     CacheFileIOManager::OPEN,
2013                                     mIndexFileOpener);
2014   if (NS_FAILED(rv)) {
2015     LOG(("CacheIndex::ReadIndexFromDisk() - CacheFileIOManager::OpenFile() "
2016          "failed [rv=0x%08x, file=%s]", rv, kIndexName));
2017     FinishRead(false);
2018     return;
2019   }
2020
2021   mJournalFileOpener = new FileOpenHelper(this);
2022   rv = CacheFileIOManager::OpenFile(NS_LITERAL_CSTRING(kJournalName),
2023                                     CacheFileIOManager::SPECIAL_FILE |
2024                                     CacheFileIOManager::OPEN,
2025                                     mJournalFileOpener);
2026   if (NS_FAILED(rv)) {
2027     LOG(("CacheIndex::ReadIndexFromDisk() - CacheFileIOManager::OpenFile() "
2028          "failed [rv=0x%08x, file=%s]", rv, kJournalName));
2029     FinishRead(false);
2030   }
2031
2032   mTmpFileOpener = new FileOpenHelper(this);
2033   rv = CacheFileIOManager::OpenFile(NS_LITERAL_CSTRING(kTempIndexName),
2034                                     CacheFileIOManager::SPECIAL_FILE |
2035                                     CacheFileIOManager::OPEN,
2036                                     mTmpFileOpener);
2037   if (NS_FAILED(rv)) {
2038     LOG(("CacheIndex::ReadIndexFromDisk() - CacheFileIOManager::OpenFile() "
2039          "failed [rv=0x%08x, file=%s]", rv, kTempIndexName));
2040     FinishRead(false);
2041   }
2042 }
2043
2044 void
2045 CacheIndex::StartReadingIndex()
2046 {
2047   LOG(("CacheIndex::StartReadingIndex()"));
2048
2049   nsresult rv;
2050
2051   AssertOwnsLock();
2052
2053   MOZ_ASSERT(mIndexHandle);
2054   MOZ_ASSERT(mState == READING);
2055   MOZ_ASSERT(!mIndexOnDiskIsValid);
2056   MOZ_ASSERT(!mDontMarkIndexClean);
2057   MOZ_ASSERT(!mJournalReadSuccessfully);
2058   MOZ_ASSERT(mIndexHandle->FileSize() >= 0);
2059
2060   int64_t entriesSize = mIndexHandle->FileSize() - sizeof(CacheIndexHeader) -
2061                         sizeof(CacheHash::Hash32_t);
2062
2063   if (entriesSize < 0 || entriesSize % sizeof(CacheIndexRecord)) {
2064     LOG(("CacheIndex::StartReadingIndex() - Index is corrupted"));
2065     FinishRead(false);
2066     return;
2067   }
2068
2069   AllocBuffer();
2070   mSkipEntries = 0;
2071   mRWHash = new CacheHash();
2072
2073   mRWBufPos = std::min(mRWBufSize,
2074                        static_cast<uint32_t>(mIndexHandle->FileSize()));
2075
2076   rv = CacheFileIOManager::Read(mIndexHandle, 0, mRWBuf, mRWBufPos, this);
2077   if (NS_FAILED(rv)) {
2078     LOG(("CacheIndex::StartReadingIndex() - CacheFileIOManager::Read() failed "
2079          "synchronously [rv=0x%08x]", rv));
2080     FinishRead(false);
2081   }
2082 }
2083
2084 void
2085 CacheIndex::ParseRecords()
2086 {
2087   LOG(("CacheIndex::ParseRecords()"));
2088
2089   nsresult rv;
2090
2091   AssertOwnsLock();
2092
2093   uint32_t entryCnt = (mIndexHandle->FileSize() - sizeof(CacheIndexHeader) -
2094                      sizeof(CacheHash::Hash32_t)) / sizeof(CacheIndexRecord);
2095   uint32_t pos = 0;
2096
2097   if (!mSkipEntries) {
2098     CacheIndexHeader *hdr = reinterpret_cast<CacheIndexHeader *>(
2099                               moz_xmalloc(sizeof(CacheIndexHeader)));
2100     memcpy(hdr, mRWBuf, sizeof(CacheIndexHeader));
2101
2102     if (NetworkEndian::readUint32(&hdr->mVersion) != kIndexVersion) {
2103       free(hdr);
2104       FinishRead(false);
2105       return;
2106     }
2107
2108     mIndexTimeStamp = NetworkEndian::readUint32(&hdr->mTimeStamp);
2109
2110     if (NetworkEndian::readUint32(&hdr->mIsDirty)) {
2111       if (mJournalHandle) {
2112         CacheFileIOManager::DoomFile(mJournalHandle, nullptr);
2113         mJournalHandle = nullptr;
2114       }
2115       free(hdr);
2116     } else {
2117       NetworkEndian::writeUint32(&hdr->mIsDirty, 1);
2118
2119       // Mark index dirty. The buffer is freed by CacheFileIOManager when
2120       // nullptr is passed as the listener and the call doesn't fail
2121       // synchronously.
2122       rv = CacheFileIOManager::Write(mIndexHandle, 0,
2123                                      reinterpret_cast<char *>(hdr),
2124                                      sizeof(CacheIndexHeader), true, nullptr);
2125       if (NS_FAILED(rv)) {
2126         // This is not fatal, just free the memory
2127         free(hdr);
2128       }
2129     }
2130
2131     pos += sizeof(CacheIndexHeader);
2132   }
2133
2134   uint32_t hashOffset = pos;
2135
2136   while (pos + sizeof(CacheIndexRecord) <= mRWBufPos &&
2137          mSkipEntries != entryCnt) {
2138     CacheIndexRecord *rec = reinterpret_cast<CacheIndexRecord *>(mRWBuf + pos);
2139     CacheIndexEntry tmpEntry(&rec->mHash);
2140     tmpEntry.ReadFromBuf(mRWBuf + pos);
2141
2142     if (tmpEntry.IsDirty() || !tmpEntry.IsInitialized() ||
2143         tmpEntry.IsFileEmpty() || tmpEntry.IsFresh() || tmpEntry.IsRemoved()) {
2144       LOG(("CacheIndex::ParseRecords() - Invalid entry found in index, removing"
2145            " whole index [dirty=%d, initialized=%d, fileEmpty=%d, fresh=%d, "
2146            "removed=%d]", tmpEntry.IsDirty(), tmpEntry.IsInitialized(),
2147            tmpEntry.IsFileEmpty(), tmpEntry.IsFresh(), tmpEntry.IsRemoved()));
2148       FinishRead(false);
2149       return;
2150     }
2151
2152     CacheIndexEntryAutoManage emng(tmpEntry.Hash(), this);
2153
2154     CacheIndexEntry *entry = mIndex.PutEntry(*tmpEntry.Hash());
2155     *entry = tmpEntry;
2156
2157     pos += sizeof(CacheIndexRecord);
2158     mSkipEntries++;
2159   }
2160
2161   mRWHash->Update(mRWBuf + hashOffset, pos - hashOffset);
2162
2163   if (pos != mRWBufPos) {
2164     memmove(mRWBuf, mRWBuf + pos, mRWBufPos - pos);
2165     mRWBufPos -= pos;
2166     pos = 0;
2167   }
2168
2169   int64_t fileOffset = sizeof(CacheIndexHeader) +
2170                        mSkipEntries * sizeof(CacheIndexRecord) + mRWBufPos;
2171
2172   MOZ_ASSERT(fileOffset <= mIndexHandle->FileSize());
2173   if (fileOffset == mIndexHandle->FileSize()) {
2174     if (mRWHash->GetHash() != NetworkEndian::readUint32(mRWBuf)) {
2175       LOG(("CacheIndex::ParseRecords() - Hash mismatch, [is %x, should be %x]",
2176            mRWHash->GetHash(),
2177            NetworkEndian::readUint32(mRWBuf)));
2178       FinishRead(false);
2179       return;
2180     }
2181
2182     mIndexOnDiskIsValid = true;
2183     mJournalReadSuccessfully = false;
2184
2185     if (mJournalHandle) {
2186       StartReadingJournal();
2187     } else {
2188       FinishRead(false);
2189     }
2190
2191     return;
2192   }
2193
2194   pos = mRWBufPos;
2195   uint32_t toRead = std::min(mRWBufSize - pos,
2196                              static_cast<uint32_t>(mIndexHandle->FileSize() -
2197                                                    fileOffset));
2198   mRWBufPos = pos + toRead;
2199
2200   rv = CacheFileIOManager::Read(mIndexHandle, fileOffset, mRWBuf + pos, toRead,
2201                                 this);
2202   if (NS_FAILED(rv)) {
2203     LOG(("CacheIndex::ParseRecords() - CacheFileIOManager::Read() failed "
2204          "synchronously [rv=0x%08x]", rv));
2205     FinishRead(false);
2206     return;
2207   }
2208 }
2209
2210 void
2211 CacheIndex::StartReadingJournal()
2212 {
2213   LOG(("CacheIndex::StartReadingJournal()"));
2214
2215   nsresult rv;
2216
2217   AssertOwnsLock();
2218
2219   MOZ_ASSERT(mJournalHandle);
2220   MOZ_ASSERT(mIndexOnDiskIsValid);
2221   MOZ_ASSERT(mTmpJournal.Count() == 0);
2222   MOZ_ASSERT(mJournalHandle->FileSize() >= 0);
2223
2224   int64_t entriesSize = mJournalHandle->FileSize() -
2225                         sizeof(CacheHash::Hash32_t);
2226
2227   if (entriesSize < 0 || entriesSize % sizeof(CacheIndexRecord)) {
2228     LOG(("CacheIndex::StartReadingJournal() - Journal is corrupted"));
2229     FinishRead(false);
2230     return;
2231   }
2232
2233   mSkipEntries = 0;
2234   mRWHash = new CacheHash();
2235
2236   mRWBufPos = std::min(mRWBufSize,
2237                        static_cast<uint32_t>(mJournalHandle->FileSize()));
2238
2239   rv = CacheFileIOManager::Read(mJournalHandle, 0, mRWBuf, mRWBufPos, this);
2240   if (NS_FAILED(rv)) {
2241     LOG(("CacheIndex::StartReadingJournal() - CacheFileIOManager::Read() failed"
2242          " synchronously [rv=0x%08x]", rv));
2243     FinishRead(false);
2244   }
2245 }
2246
2247 void
2248 CacheIndex::ParseJournal()
2249 {
2250   LOG(("CacheIndex::ParseRecords()"));
2251
2252   nsresult rv;
2253
2254   AssertOwnsLock();
2255
2256   uint32_t entryCnt = (mJournalHandle->FileSize() -
2257                        sizeof(CacheHash::Hash32_t)) / sizeof(CacheIndexRecord);
2258
2259   uint32_t pos = 0;
2260
2261   while (pos + sizeof(CacheIndexRecord) <= mRWBufPos &&
2262          mSkipEntries != entryCnt) {
2263     CacheIndexRecord *rec = reinterpret_cast<CacheIndexRecord *>(mRWBuf + pos);
2264     CacheIndexEntry tmpEntry(&rec->mHash);
2265     tmpEntry.ReadFromBuf(mRWBuf + pos);
2266
2267     CacheIndexEntry *entry = mTmpJournal.PutEntry(*tmpEntry.Hash());
2268     *entry = tmpEntry;
2269
2270     if (entry->IsDirty() || entry->IsFresh()) {
2271       LOG(("CacheIndex::ParseJournal() - Invalid entry found in journal, "
2272            "ignoring whole journal [dirty=%d, fresh=%d]", entry->IsDirty(),
2273            entry->IsFresh()));
2274       FinishRead(false);
2275       return;
2276     }
2277
2278     pos += sizeof(CacheIndexRecord);
2279     mSkipEntries++;
2280   }
2281
2282   mRWHash->Update(mRWBuf, pos);
2283
2284   if (pos != mRWBufPos) {
2285     memmove(mRWBuf, mRWBuf + pos, mRWBufPos - pos);
2286     mRWBufPos -= pos;
2287     pos = 0;
2288   }
2289
2290   int64_t fileOffset = mSkipEntries * sizeof(CacheIndexRecord) + mRWBufPos;
2291
2292   MOZ_ASSERT(fileOffset <= mJournalHandle->FileSize());
2293   if (fileOffset == mJournalHandle->FileSize()) {
2294     if (mRWHash->GetHash() != NetworkEndian::readUint32(mRWBuf)) {
2295       LOG(("CacheIndex::ParseJournal() - Hash mismatch, [is %x, should be %x]",
2296            mRWHash->GetHash(),
2297            NetworkEndian::readUint32(mRWBuf)));
2298       FinishRead(false);
2299       return;
2300     }
2301
2302     mJournalReadSuccessfully = true;
2303     FinishRead(true);
2304     return;
2305   }
2306
2307   pos = mRWBufPos;
2308   uint32_t toRead = std::min(mRWBufSize - pos,
2309                              static_cast<uint32_t>(mJournalHandle->FileSize() -
2310                                                    fileOffset));
2311   mRWBufPos = pos + toRead;
2312
2313   rv = CacheFileIOManager::Read(mJournalHandle, fileOffset, mRWBuf + pos,
2314                                 toRead, this);
2315   if (NS_FAILED(rv)) {
2316     LOG(("CacheIndex::ParseJournal() - CacheFileIOManager::Read() failed "
2317          "synchronously [rv=0x%08x]", rv));
2318     FinishRead(false);
2319     return;
2320   }
2321 }
2322
2323 void
2324 CacheIndex::MergeJournal()
2325 {
2326   LOG(("CacheIndex::MergeJournal()"));
2327
2328   AssertOwnsLock();
2329
2330   mTmpJournal.EnumerateEntries(&CacheIndex::ProcessJournalEntry, this);
2331
2332   MOZ_ASSERT(mTmpJournal.Count() == 0);
2333 }
2334
2335 // static
2336 PLDHashOperator
2337 CacheIndex::ProcessJournalEntry(CacheIndexEntry *aEntry, void* aClosure)
2338 {
2339   CacheIndex *index = static_cast<CacheIndex *>(aClosure);
2340
2341   LOG(("CacheIndex::ProcessJournalEntry() [hash=%08x%08x%08x%08x%08x]",
2342        LOGSHA1(aEntry->Hash())));
2343
2344   CacheIndexEntry *entry = index->mIndex.GetEntry(*aEntry->Hash());
2345
2346   CacheIndexEntryAutoManage emng(aEntry->Hash(), index);
2347
2348   if (aEntry->IsRemoved()) {
2349     if (entry) {
2350       entry->MarkRemoved();
2351       entry->MarkDirty();
2352     }
2353   } else {
2354     if (!entry) {
2355       entry = index->mIndex.PutEntry(*aEntry->Hash());
2356     }
2357
2358     *entry = *aEntry;
2359     entry->MarkDirty();
2360   }
2361
2362   return PL_DHASH_REMOVE;
2363 }
2364
2365 void
2366 CacheIndex::EnsureNoFreshEntry()
2367 {
2368 #ifdef DEBUG_STATS
2369   CacheIndexStats debugStats;
2370   debugStats.DisableLogging();
2371   mIndex.EnumerateEntries(&CacheIndex::SumIndexStats, &debugStats);
2372   MOZ_ASSERT(debugStats.Fresh() == 0);
2373 #endif
2374 }
2375
2376 void
2377 CacheIndex::EnsureCorrectStats()
2378 {
2379 #ifdef DEBUG_STATS
2380   MOZ_ASSERT(mPendingUpdates.Count() == 0);
2381   CacheIndexStats debugStats;
2382   debugStats.DisableLogging();
2383   mIndex.EnumerateEntries(&CacheIndex::SumIndexStats, &debugStats);
2384   MOZ_ASSERT(debugStats == mIndexStats);
2385 #endif
2386 }
2387
2388 // static
2389 PLDHashOperator
2390 CacheIndex::SumIndexStats(CacheIndexEntry *aEntry, void* aClosure)
2391 {
2392   CacheIndexStats *stats = static_cast<CacheIndexStats *>(aClosure);
2393   stats->BeforeChange(nullptr);
2394   stats->AfterChange(aEntry);
2395   return PL_DHASH_NEXT;
2396 }
2397
2398 void
2399 CacheIndex::FinishRead(bool aSucceeded)
2400 {
2401   LOG(("CacheIndex::FinishRead() [succeeded=%d]", aSucceeded));
2402   AssertOwnsLock();
2403
2404   MOZ_ASSERT((!aSucceeded && mState == SHUTDOWN) || mState == READING);
2405
2406   MOZ_ASSERT(
2407     // -> rebuild
2408     (!aSucceeded && !mIndexOnDiskIsValid && !mJournalReadSuccessfully) ||
2409     // -> update
2410     (!aSucceeded && mIndexOnDiskIsValid && !mJournalReadSuccessfully) ||
2411     // -> ready
2412     (aSucceeded && mIndexOnDiskIsValid && mJournalReadSuccessfully));
2413
2414   if (mState == SHUTDOWN) {
2415     RemoveFile(NS_LITERAL_CSTRING(kTempIndexName));
2416     RemoveFile(NS_LITERAL_CSTRING(kJournalName));
2417   } else {
2418     if (mIndexHandle && !mIndexOnDiskIsValid) {
2419       CacheFileIOManager::DoomFile(mIndexHandle, nullptr);
2420     }
2421
2422     if (mJournalHandle) {
2423       CacheFileIOManager::DoomFile(mJournalHandle, nullptr);
2424     }
2425   }
2426
2427   if (mIndexFileOpener) {
2428     mIndexFileOpener->Cancel();
2429     mIndexFileOpener = nullptr;
2430   }
2431   if (mJournalFileOpener) {
2432     mJournalFileOpener->Cancel();
2433     mJournalFileOpener = nullptr;
2434   }
2435   if (mTmpFileOpener) {
2436     mTmpFileOpener->Cancel();
2437     mTmpFileOpener = nullptr;
2438   }
2439
2440   mIndexHandle = nullptr;
2441   mJournalHandle = nullptr;
2442   mRWHash = nullptr;
2443   ReleaseBuffer();
2444
2445   if (mState == SHUTDOWN) {
2446     return;
2447   }
2448
2449   if (!mIndexOnDiskIsValid) {
2450     MOZ_ASSERT(mTmpJournal.Count() == 0);
2451     EnsureNoFreshEntry();
2452     ProcessPendingOperations();
2453     // Remove all entries that we haven't seen during this session
2454     mIndex.EnumerateEntries(&CacheIndex::RemoveNonFreshEntries, this);
2455     StartUpdatingIndex(true);
2456     return;
2457   }
2458
2459   if (!mJournalReadSuccessfully) {
2460     mTmpJournal.Clear();
2461     EnsureNoFreshEntry();
2462     ProcessPendingOperations();
2463     StartUpdatingIndex(false);
2464     return;
2465   }
2466
2467   MergeJournal();
2468   EnsureNoFreshEntry();
2469   ProcessPendingOperations();
2470   mIndexStats.Log();
2471
2472   ChangeState(READY);
2473   mLastDumpTime = TimeStamp::NowLoRes(); // Do not dump new index immediately
2474 }
2475
2476 // static
2477 void
2478 CacheIndex::DelayedUpdate(nsITimer *aTimer, void *aClosure)
2479 {
2480   LOG(("CacheIndex::DelayedUpdate()"));
2481
2482   nsresult rv;
2483   nsRefPtr<CacheIndex> index = gInstance;
2484
2485   if (!index) {
2486     return;
2487   }
2488
2489   CacheIndexAutoLock lock(index);
2490
2491   index->mUpdateTimer = nullptr;
2492
2493   if (!index->IsIndexUsable()) {
2494     return;
2495   }
2496
2497   if (index->mState == READY && index->mShuttingDown) {
2498     return;
2499   }
2500
2501   // mUpdateEventPending must be false here since StartUpdatingIndex() won't
2502   // schedule timer if it is true.
2503   MOZ_ASSERT(!index->mUpdateEventPending);
2504   if (index->mState != BUILDING && index->mState != UPDATING) {
2505     LOG(("CacheIndex::DelayedUpdate() - Update was canceled"));
2506     return;
2507   }
2508
2509   // We need to redispatch to run with lower priority
2510   nsRefPtr<CacheIOThread> ioThread = CacheFileIOManager::IOThread();
2511   MOZ_ASSERT(ioThread);
2512
2513   index->mUpdateEventPending = true;
2514   rv = ioThread->Dispatch(index, CacheIOThread::INDEX);
2515   if (NS_FAILED(rv)) {
2516     index->mUpdateEventPending = false;
2517     NS_WARNING("CacheIndex::DelayedUpdate() - Can't dispatch event");
2518     LOG(("CacheIndex::DelayedUpdate() - Can't dispatch event" ));
2519     index->FinishUpdate(false);
2520   }
2521 }
2522
2523 nsresult
2524 CacheIndex::ScheduleUpdateTimer(uint32_t aDelay)
2525 {
2526   LOG(("CacheIndex::ScheduleUpdateTimer() [delay=%u]", aDelay));
2527
2528   MOZ_ASSERT(!mUpdateTimer);
2529
2530   nsresult rv;
2531
2532   nsCOMPtr<nsITimer> timer = do_CreateInstance("@mozilla.org/timer;1", &rv);
2533   NS_ENSURE_SUCCESS(rv, rv);
2534
2535   nsCOMPtr<nsIEventTarget> ioTarget = CacheFileIOManager::IOTarget();
2536   MOZ_ASSERT(ioTarget);
2537
2538   rv = timer->SetTarget(ioTarget);
2539   NS_ENSURE_SUCCESS(rv, rv);
2540
2541   rv = timer->InitWithFuncCallback(CacheIndex::DelayedUpdate, nullptr,
2542                                    aDelay, nsITimer::TYPE_ONE_SHOT);
2543   NS_ENSURE_SUCCESS(rv, rv);
2544
2545   mUpdateTimer.swap(timer);
2546   return NS_OK;
2547 }
2548
2549 nsresult
2550 CacheIndex::SetupDirectoryEnumerator()
2551 {
2552   MOZ_ASSERT(!NS_IsMainThread());
2553   MOZ_ASSERT(!mDirEnumerator);
2554
2555   nsresult rv;
2556   nsCOMPtr<nsIFile> file;
2557
2558   rv = mCacheDirectory->Clone(getter_AddRefs(file));
2559   NS_ENSURE_SUCCESS(rv, rv);
2560
2561   rv = file->AppendNative(NS_LITERAL_CSTRING(kEntriesDir));
2562   NS_ENSURE_SUCCESS(rv, rv);
2563
2564   bool exists;
2565   rv = file->Exists(&exists);
2566   NS_ENSURE_SUCCESS(rv, rv);
2567
2568   if (!exists) {
2569     NS_WARNING("CacheIndex::SetupDirectoryEnumerator() - Entries directory "
2570                "doesn't exist!");
2571     LOG(("CacheIndex::SetupDirectoryEnumerator() - Entries directory doesn't "
2572           "exist!" ));
2573     return NS_ERROR_UNEXPECTED;
2574   }
2575
2576   nsCOMPtr<nsISimpleEnumerator> enumerator;
2577   rv = file->GetDirectoryEntries(getter_AddRefs(enumerator));
2578   NS_ENSURE_SUCCESS(rv, rv);
2579
2580   mDirEnumerator = do_QueryInterface(enumerator, &rv);
2581   NS_ENSURE_SUCCESS(rv, rv);
2582
2583   return NS_OK;
2584 }
2585
2586 void
2587 CacheIndex::InitEntryFromDiskData(CacheIndexEntry *aEntry,
2588                                   CacheFileMetadata *aMetaData,
2589                                   int64_t aFileSize)
2590 {
2591   aEntry->InitNew();
2592   aEntry->MarkDirty();
2593   aEntry->MarkFresh();
2594   aEntry->Init(aMetaData->AppId(), aMetaData->IsAnonymous(),
2595                aMetaData->IsInBrowser());
2596
2597   uint32_t expirationTime;
2598   aMetaData->GetExpirationTime(&expirationTime);
2599   aEntry->SetExpirationTime(expirationTime);
2600
2601   uint32_t frecency;
2602   aMetaData->GetFrecency(&frecency);
2603   aEntry->SetFrecency(frecency);
2604
2605   aEntry->SetFileSize(static_cast<uint32_t>(
2606                         std::min(static_cast<int64_t>(PR_UINT32_MAX),
2607                                  (aFileSize + 0x3FF) >> 10)));
2608 }
2609
2610 bool
2611 CacheIndex::IsUpdatePending()
2612 {
2613   AssertOwnsLock();
2614
2615   if (mUpdateTimer || mUpdateEventPending) {
2616     return true;
2617   }
2618
2619   return false;
2620 }
2621
2622 void
2623 CacheIndex::BuildIndex()
2624 {
2625   LOG(("CacheIndex::BuildIndex()"));
2626
2627   AssertOwnsLock();
2628
2629   MOZ_ASSERT(mPendingUpdates.Count() == 0);
2630
2631   nsresult rv;
2632
2633   if (!mDirEnumerator) {
2634     {
2635       // Do not do IO under the lock.
2636       CacheIndexAutoUnlock unlock(this);
2637       rv = SetupDirectoryEnumerator();
2638     }
2639     if (mState == SHUTDOWN) {
2640       // The index was shut down while we released the lock. FinishUpdate() was
2641       // already called from Shutdown(), so just simply return here.
2642       return;
2643     }
2644
2645     if (NS_FAILED(rv)) {
2646       FinishUpdate(false);
2647       return;
2648     }
2649   }
2650
2651   while (true) {
2652     if (CacheIOThread::YieldAndRerun()) {
2653       LOG(("CacheIndex::BuildIndex() - Breaking loop for higher level events."));
2654       mUpdateEventPending = true;
2655       return;
2656     }
2657
2658     nsCOMPtr<nsIFile> file;
2659     {
2660       // Do not do IO under the lock.
2661       CacheIndexAutoUnlock unlock(this);
2662       rv = mDirEnumerator->GetNextFile(getter_AddRefs(file));
2663     }
2664     if (mState == SHUTDOWN) {
2665       return;
2666     }
2667     if (!file) {
2668       FinishUpdate(NS_SUCCEEDED(rv));
2669       return;
2670     }
2671
2672     nsAutoCString leaf;
2673     rv = file->GetNativeLeafName(leaf);
2674     if (NS_FAILED(rv)) {
2675       LOG(("CacheIndex::BuildIndex() - GetNativeLeafName() failed! Skipping "
2676            "file."));
2677       mDontMarkIndexClean = true;
2678       continue;
2679     }
2680
2681     SHA1Sum::Hash hash;
2682     rv = CacheFileIOManager::StrToHash(leaf, &hash);
2683     if (NS_FAILED(rv)) {
2684       LOG(("CacheIndex::BuildIndex() - Filename is not a hash, removing file. "
2685            "[name=%s]", leaf.get()));
2686       file->Remove(false);
2687       continue;
2688     }
2689
2690     CacheIndexEntry *entry = mIndex.GetEntry(hash);
2691     if (entry && entry->IsRemoved()) {
2692       LOG(("CacheIndex::BuildIndex() - Found file that should not exist. "
2693            "[name=%s]", leaf.get()));
2694       entry->Log();
2695       MOZ_ASSERT(entry->IsFresh());
2696       entry = nullptr;
2697     }
2698
2699 #ifdef DEBUG
2700     nsRefPtr<CacheFileHandle> handle;
2701     CacheFileIOManager::gInstance->mHandles.GetHandle(&hash, false,
2702                                                       getter_AddRefs(handle));
2703 #endif
2704
2705     if (entry) {
2706       // the entry is up to date
2707       LOG(("CacheIndex::BuildIndex() - Skipping file because the entry is up to"
2708            " date. [name=%s]", leaf.get()));
2709       entry->Log();
2710       MOZ_ASSERT(entry->IsFresh()); // The entry must be from this session
2711       // there must be an active CacheFile if the entry is not initialized
2712       MOZ_ASSERT(entry->IsInitialized() || handle);
2713       continue;
2714     }
2715
2716     MOZ_ASSERT(!handle);
2717
2718     nsRefPtr<CacheFileMetadata> meta = new CacheFileMetadata();
2719     int64_t size = 0;
2720
2721     {
2722       // Do not do IO under the lock.
2723       CacheIndexAutoUnlock unlock(this);
2724       rv = meta->SyncReadMetadata(file);
2725
2726       if (NS_SUCCEEDED(rv)) {
2727         rv = file->GetFileSize(&size);
2728         if (NS_FAILED(rv)) {
2729           LOG(("CacheIndex::BuildIndex() - Cannot get filesize of file that was"
2730                " successfully parsed. [name=%s]", leaf.get()));
2731         }
2732       }
2733     }
2734     if (mState == SHUTDOWN) {
2735       return;
2736     }
2737
2738     // Nobody could add the entry while the lock was released since we modify
2739     // the index only on IO thread and this loop is executed on IO thread too.
2740     entry = mIndex.GetEntry(hash);
2741     MOZ_ASSERT(!entry || entry->IsRemoved());
2742
2743     if (NS_FAILED(rv)) {
2744       LOG(("CacheIndex::BuildIndex() - CacheFileMetadata::SyncReadMetadata() "
2745            "failed, removing file. [name=%s]", leaf.get()));
2746       file->Remove(false);
2747     } else {
2748       CacheIndexEntryAutoManage entryMng(&hash, this);
2749       entry = mIndex.PutEntry(hash);
2750       InitEntryFromDiskData(entry, meta, size);
2751       LOG(("CacheIndex::BuildIndex() - Added entry to index. [hash=%s]",
2752            leaf.get()));
2753       entry->Log();
2754     }
2755   }
2756
2757   NS_NOTREACHED("We should never get here");
2758 }
2759
2760 bool
2761 CacheIndex::StartUpdatingIndexIfNeeded(bool aSwitchingToReadyState)
2762 {
2763   // Start updating process when we are in or we are switching to READY state
2764   // and index needs update, but not during shutdown or when removing all
2765   // entries.
2766   if ((mState == READY || aSwitchingToReadyState) && mIndexNeedsUpdate &&
2767       !mShuttingDown && !mRemovingAll) {
2768     LOG(("CacheIndex::StartUpdatingIndexIfNeeded() - starting update process"));
2769     mIndexNeedsUpdate = false;
2770     StartUpdatingIndex(false);
2771     return true;
2772   }
2773
2774   return false;
2775 }
2776
2777 void
2778 CacheIndex::StartUpdatingIndex(bool aRebuild)
2779 {
2780   LOG(("CacheIndex::StartUpdatingIndex() [rebuild=%d]", aRebuild));
2781
2782   AssertOwnsLock();
2783
2784   nsresult rv;
2785
2786   mIndexStats.Log();
2787
2788   ChangeState(aRebuild ? BUILDING : UPDATING);
2789   mDontMarkIndexClean = false;
2790
2791   if (mShuttingDown || mRemovingAll) {
2792     FinishUpdate(false);
2793     return;
2794   }
2795
2796   if (IsUpdatePending()) {
2797     LOG(("CacheIndex::StartUpdatingIndex() - Update is already pending"));
2798     return;
2799   }
2800
2801   uint32_t elapsed = (TimeStamp::NowLoRes() - mStartTime).ToMilliseconds();
2802   if (elapsed < kUpdateIndexStartDelay) {
2803     LOG(("CacheIndex::StartUpdatingIndex() - %u ms elapsed since startup, "
2804          "scheduling timer to fire in %u ms.", elapsed,
2805          kUpdateIndexStartDelay - elapsed));
2806     rv = ScheduleUpdateTimer(kUpdateIndexStartDelay - elapsed);
2807     if (NS_SUCCEEDED(rv)) {
2808       return;
2809     }
2810
2811     LOG(("CacheIndex::StartUpdatingIndex() - ScheduleUpdateTimer() failed. "
2812          "Starting update immediately."));
2813   } else {
2814     LOG(("CacheIndex::StartUpdatingIndex() - %u ms elapsed since startup, "
2815          "starting update now.", elapsed));
2816   }
2817
2818   nsRefPtr<CacheIOThread> ioThread = CacheFileIOManager::IOThread();
2819   MOZ_ASSERT(ioThread);
2820
2821   // We need to dispatch an event even if we are on IO thread since we need to
2822   // update the index with the correct priority.
2823   mUpdateEventPending = true;
2824   rv = ioThread->Dispatch(this, CacheIOThread::INDEX);
2825   if (NS_FAILED(rv)) {
2826     mUpdateEventPending = false;
2827     NS_WARNING("CacheIndex::StartUpdatingIndex() - Can't dispatch event");
2828     LOG(("CacheIndex::StartUpdatingIndex() - Can't dispatch event" ));
2829     FinishUpdate(false);
2830   }
2831 }
2832
2833 void
2834 CacheIndex::UpdateIndex()
2835 {
2836   LOG(("CacheIndex::UpdateIndex()"));
2837
2838   AssertOwnsLock();
2839
2840   MOZ_ASSERT(mPendingUpdates.Count() == 0);
2841
2842   nsresult rv;
2843
2844   if (!mDirEnumerator) {
2845     {
2846       // Do not do IO under the lock.
2847       CacheIndexAutoUnlock unlock(this);
2848       rv = SetupDirectoryEnumerator();
2849     }
2850     if (mState == SHUTDOWN) {
2851       // The index was shut down while we released the lock. FinishUpdate() was
2852       // already called from Shutdown(), so just simply return here.
2853       return;
2854     }
2855
2856     if (NS_FAILED(rv)) {
2857       FinishUpdate(false);
2858       return;
2859     }
2860   }
2861
2862   while (true) {
2863     if (CacheIOThread::YieldAndRerun()) {
2864       LOG(("CacheIndex::UpdateIndex() - Breaking loop for higher level "
2865            "events."));
2866       mUpdateEventPending = true;
2867       return;
2868     }
2869
2870     nsCOMPtr<nsIFile> file;
2871     {
2872       // Do not do IO under the lock.
2873       CacheIndexAutoUnlock unlock(this);
2874       rv = mDirEnumerator->GetNextFile(getter_AddRefs(file));
2875     }
2876     if (mState == SHUTDOWN) {
2877       return;
2878     }
2879     if (!file) {
2880       FinishUpdate(NS_SUCCEEDED(rv));
2881       return;
2882     }
2883
2884     nsAutoCString leaf;
2885     rv = file->GetNativeLeafName(leaf);
2886     if (NS_FAILED(rv)) {
2887       LOG(("CacheIndex::UpdateIndex() - GetNativeLeafName() failed! Skipping "
2888            "file."));
2889       mDontMarkIndexClean = true;
2890       continue;
2891     }
2892
2893     SHA1Sum::Hash hash;
2894     rv = CacheFileIOManager::StrToHash(leaf, &hash);
2895     if (NS_FAILED(rv)) {
2896       LOG(("CacheIndex::UpdateIndex() - Filename is not a hash, removing file. "
2897            "[name=%s]", leaf.get()));
2898       file->Remove(false);
2899       continue;
2900     }
2901
2902     CacheIndexEntry *entry = mIndex.GetEntry(hash);
2903     if (entry && entry->IsRemoved()) {
2904       if (entry->IsFresh()) {
2905         LOG(("CacheIndex::UpdateIndex() - Found file that should not exist. "
2906              "[name=%s]", leaf.get()));
2907         entry->Log();
2908       }
2909       entry = nullptr;
2910     }
2911
2912 #ifdef DEBUG
2913     nsRefPtr<CacheFileHandle> handle;
2914     CacheFileIOManager::gInstance->mHandles.GetHandle(&hash, false,
2915                                                       getter_AddRefs(handle));
2916 #endif
2917
2918     if (entry && entry->IsFresh()) {
2919       // the entry is up to date
2920       LOG(("CacheIndex::UpdateIndex() - Skipping file because the entry is up "
2921            " to date. [name=%s]", leaf.get()));
2922       entry->Log();
2923       // there must be an active CacheFile if the entry is not initialized
2924       MOZ_ASSERT(entry->IsInitialized() || handle);
2925       continue;
2926     }
2927
2928     MOZ_ASSERT(!handle);
2929
2930     if (entry) {
2931       PRTime lastModifiedTime;
2932       {
2933         // Do not do IO under the lock.
2934         CacheIndexAutoUnlock unlock(this);
2935         rv = file->GetLastModifiedTime(&lastModifiedTime);
2936       }
2937       if (mState == SHUTDOWN) {
2938         return;
2939       }
2940       if (NS_FAILED(rv)) {
2941         LOG(("CacheIndex::UpdateIndex() - Cannot get lastModifiedTime. "
2942              "[name=%s]", leaf.get()));
2943         // Assume the file is newer than index
2944       } else {
2945         if (mIndexTimeStamp > (lastModifiedTime / PR_MSEC_PER_SEC)) {
2946           LOG(("CacheIndex::UpdateIndex() - Skipping file because of last "
2947                "modified time. [name=%s, indexTimeStamp=%u, "
2948                "lastModifiedTime=%u]", leaf.get(), mIndexTimeStamp,
2949                lastModifiedTime / PR_MSEC_PER_SEC));
2950
2951           CacheIndexEntryAutoManage entryMng(&hash, this);
2952           entry->MarkFresh();
2953           continue;
2954         }
2955       }
2956     }
2957
2958     nsRefPtr<CacheFileMetadata> meta = new CacheFileMetadata();
2959     int64_t size = 0;
2960
2961     {
2962       // Do not do IO under the lock.
2963       CacheIndexAutoUnlock unlock(this);
2964       rv = meta->SyncReadMetadata(file);
2965
2966       if (NS_SUCCEEDED(rv)) {
2967         rv = file->GetFileSize(&size);
2968         if (NS_FAILED(rv)) {
2969           LOG(("CacheIndex::UpdateIndex() - Cannot get filesize of file that "
2970                "was successfully parsed. [name=%s]", leaf.get()));
2971         }
2972       }
2973     }
2974     if (mState == SHUTDOWN) {
2975       return;
2976     }
2977
2978     // Nobody could add the entry while the lock was released since we modify
2979     // the index only on IO thread and this loop is executed on IO thread too.
2980     entry = mIndex.GetEntry(hash);
2981     MOZ_ASSERT(!entry || !entry->IsFresh());
2982
2983     CacheIndexEntryAutoManage entryMng(&hash, this);
2984
2985     if (NS_FAILED(rv)) {
2986       LOG(("CacheIndex::UpdateIndex() - CacheFileMetadata::SyncReadMetadata() "
2987            "failed, removing file. [name=%s]", leaf.get()));
2988       file->Remove(false);
2989       if (entry) {
2990         entry->MarkRemoved();
2991         entry->MarkFresh();
2992         entry->MarkDirty();
2993       }
2994     } else {
2995       entry = mIndex.PutEntry(hash);
2996       InitEntryFromDiskData(entry, meta, size);
2997       LOG(("CacheIndex::UpdateIndex() - Added/updated entry to/in index. "
2998            "[hash=%s]", leaf.get()));
2999       entry->Log();
3000     }
3001   }
3002
3003   NS_NOTREACHED("We should never get here");
3004 }
3005
3006 void
3007 CacheIndex::FinishUpdate(bool aSucceeded)
3008 {
3009   LOG(("CacheIndex::FinishUpdate() [succeeded=%d]", aSucceeded));
3010
3011   MOZ_ASSERT(mState == UPDATING || mState == BUILDING ||
3012              (!aSucceeded && mState == SHUTDOWN));
3013
3014   AssertOwnsLock();
3015
3016   if (mDirEnumerator) {
3017     if (NS_IsMainThread()) {
3018       LOG(("CacheIndex::FinishUpdate() - posting of PreShutdownInternal failed?"
3019            " Cannot safely release mDirEnumerator, leaking it!"));
3020       NS_WARNING(("CacheIndex::FinishUpdate() - Leaking mDirEnumerator!"));
3021       // This can happen only in case dispatching event to IO thread failed in
3022       // CacheIndex::PreShutdown().
3023       mDirEnumerator.forget(); // Leak it since dir enumerator is not threadsafe
3024     } else {
3025       mDirEnumerator->Close();
3026       mDirEnumerator = nullptr;
3027     }
3028   }
3029
3030   if (!aSucceeded) {
3031     mDontMarkIndexClean = true;
3032   }
3033
3034   if (mState == SHUTDOWN) {
3035     return;
3036   }
3037
3038   if (mState == UPDATING && aSucceeded) {
3039     // If we've iterated over all entries successfully then all entries that
3040     // really exist on the disk are now marked as fresh. All non-fresh entries
3041     // don't exist anymore and must be removed from the index.
3042     mIndex.EnumerateEntries(&CacheIndex::RemoveNonFreshEntries, this);
3043   }
3044
3045   // Make sure we won't start update. If the build or update failed, there is no
3046   // reason to believe that it will succeed next time.
3047   mIndexNeedsUpdate = false;
3048
3049   ChangeState(READY);
3050   mLastDumpTime = TimeStamp::NowLoRes(); // Do not dump new index immediately
3051 }
3052
3053 // static
3054 PLDHashOperator
3055 CacheIndex::RemoveNonFreshEntries(CacheIndexEntry *aEntry, void* aClosure)
3056 {
3057   if (aEntry->IsFresh()) {
3058     return PL_DHASH_NEXT;
3059   }
3060
3061   LOG(("CacheFile::RemoveNonFreshEntries() - Removing entry. "
3062        "[hash=%08x%08x%08x%08x%08x]", LOGSHA1(aEntry->Hash())));
3063
3064   CacheIndex *index = static_cast<CacheIndex *>(aClosure);
3065
3066   CacheIndexEntryAutoManage emng(aEntry->Hash(), index);
3067   emng.DoNotSearchInIndex();
3068
3069   return PL_DHASH_REMOVE;
3070 }
3071
3072 #ifdef PR_LOGGING
3073 // static
3074 char const *
3075 CacheIndex::StateString(EState aState)
3076 {
3077   switch (aState) {
3078     case INITIAL:  return "INITIAL";
3079     case READING:  return "READING";
3080     case WRITING:  return "WRITING";
3081     case BUILDING: return "BUILDING";
3082     case UPDATING: return "UPDATING";
3083     case READY:    return "READY";
3084     case SHUTDOWN: return "SHUTDOWN";
3085   }
3086
3087   MOZ_ASSERT(false, "Unexpected state!");
3088   return "?";
3089 }
3090 #endif
3091
3092 void
3093 CacheIndex::ChangeState(EState aNewState)
3094 {
3095   LOG(("CacheIndex::ChangeState() changing state %s -> %s", StateString(mState),
3096        StateString(aNewState)));
3097
3098   // All pending updates should be processed before changing state
3099   MOZ_ASSERT(mPendingUpdates.Count() == 0);
3100
3101   // PreShutdownInternal() should change the state to READY from every state. It
3102   // may go through different states, but once we are in READY state the only
3103   // possible transition is to SHUTDOWN state.
3104   MOZ_ASSERT(!mShuttingDown || mState != READY || aNewState == SHUTDOWN);
3105
3106   // Start updating process when switching to READY state if needed
3107   if (aNewState == READY && StartUpdatingIndexIfNeeded(true)) {
3108     return;
3109   }
3110
3111   // Try to evict entries over limit everytime we're leaving state READING,
3112   // BUILDING or UPDATING, but not during shutdown or when removing all
3113   // entries.
3114   if (!mShuttingDown && !mRemovingAll && aNewState != SHUTDOWN &&
3115       (mState == READING || mState == BUILDING || mState == UPDATING))  {
3116     CacheFileIOManager::EvictIfOverLimit();
3117   }
3118
3119   mState = aNewState;
3120
3121   if (mState != SHUTDOWN) {
3122     CacheFileIOManager::CacheIndexStateChanged();
3123   }
3124
3125   if (mState == READY && mDiskConsumptionObservers.Length()) {
3126     for (uint32_t i = 0; i < mDiskConsumptionObservers.Length(); ++i) {
3127       DiskConsumptionObserver* o = mDiskConsumptionObservers[i];
3128       // Safe to call under the lock.  We always post to the main thread.
3129       o->OnDiskConsumption(mIndexStats.Size() << 10);
3130     }
3131
3132     mDiskConsumptionObservers.Clear();
3133   }
3134 }
3135
3136 void
3137 CacheIndex::AllocBuffer()
3138 {
3139   switch (mState) {
3140     case WRITING:
3141       mRWBufSize = sizeof(CacheIndexHeader) + sizeof(CacheHash::Hash32_t) +
3142                    mProcessEntries * sizeof(CacheIndexRecord);
3143       if (mRWBufSize > kMaxBufSize) {
3144         mRWBufSize = kMaxBufSize;
3145       }
3146       break;
3147     case READING:
3148       mRWBufSize = kMaxBufSize;
3149       break;
3150     default:
3151       MOZ_ASSERT(false, "Unexpected state!");
3152   }
3153
3154   mRWBuf = static_cast<char *>(moz_xmalloc(mRWBufSize));
3155 }
3156
3157 void
3158 CacheIndex::ReleaseBuffer()
3159 {
3160   if (!mRWBuf) {
3161     return;
3162   }
3163
3164   free(mRWBuf);
3165   mRWBuf = nullptr;
3166   mRWBufSize = 0;
3167   mRWBufPos = 0;
3168 }
3169
3170 namespace { // anon
3171
3172 class FrecencyComparator
3173 {
3174 public:
3175   bool Equals(CacheIndexRecord* a, CacheIndexRecord* b) const {
3176     return a->mFrecency == b->mFrecency;
3177   }
3178   bool LessThan(CacheIndexRecord* a, CacheIndexRecord* b) const {
3179     // Place entries with frecency 0 at the end of the array.
3180     if (a->mFrecency == 0) {
3181       return false;
3182     }
3183     if (b->mFrecency == 0) {
3184       return true;
3185     }
3186     return a->mFrecency < b->mFrecency;
3187   }
3188 };
3189
3190 class ExpirationComparator
3191 {
3192 public:
3193   bool Equals(CacheIndexRecord* a, CacheIndexRecord* b) const {
3194     return a->mExpirationTime == b->mExpirationTime;
3195   }
3196   bool LessThan(CacheIndexRecord* a, CacheIndexRecord* b) const {
3197     return a->mExpirationTime < b->mExpirationTime;
3198   }
3199 };
3200
3201 } // anon
3202
3203 void
3204 CacheIndex::InsertRecordToFrecencyArray(CacheIndexRecord *aRecord)
3205 {
3206   LOG(("CacheIndex::InsertRecordToFrecencyArray() [record=%p, hash=%08x%08x%08x"
3207        "%08x%08x]", aRecord, LOGSHA1(aRecord->mHash)));
3208
3209   MOZ_ASSERT(!mFrecencyArray.Contains(aRecord));
3210   mFrecencyArray.InsertElementSorted(aRecord, FrecencyComparator());
3211 }
3212
3213 void
3214 CacheIndex::InsertRecordToExpirationArray(CacheIndexRecord *aRecord)
3215 {
3216   LOG(("CacheIndex::InsertRecordToExpirationArray() [record=%p, hash=%08x%08x"
3217        "%08x%08x%08x]", aRecord, LOGSHA1(aRecord->mHash)));
3218
3219   MOZ_ASSERT(!mExpirationArray.Contains(aRecord));
3220   mExpirationArray.InsertElementSorted(aRecord, ExpirationComparator());
3221 }
3222
3223 void
3224 CacheIndex::RemoveRecordFromFrecencyArray(CacheIndexRecord *aRecord)
3225 {
3226   LOG(("CacheIndex::RemoveRecordFromFrecencyArray() [record=%p]", aRecord));
3227
3228   DebugOnly<bool> removed;
3229   removed = mFrecencyArray.RemoveElement(aRecord);
3230   MOZ_ASSERT(removed);
3231 }
3232
3233 void
3234 CacheIndex::RemoveRecordFromExpirationArray(CacheIndexRecord *aRecord)
3235 {
3236   LOG(("CacheIndex::RemoveRecordFromExpirationArray() [record=%p]", aRecord));
3237
3238   DebugOnly<bool> removed;
3239   removed = mExpirationArray.RemoveElement(aRecord);
3240   MOZ_ASSERT(removed);
3241 }
3242
3243 void
3244 CacheIndex::AddRecordToIterators(CacheIndexRecord *aRecord)
3245 {
3246   AssertOwnsLock();
3247
3248   for (uint32_t i = 0; i < mIterators.Length(); ++i) {
3249     // Add a new record only when iterator is supposed to be updated.
3250     if (mIterators[i]->ShouldBeNewAdded()) {
3251       mIterators[i]->AddRecord(aRecord);
3252     }
3253   }
3254 }
3255
3256 void
3257 CacheIndex::RemoveRecordFromIterators(CacheIndexRecord *aRecord)
3258 {
3259   AssertOwnsLock();
3260
3261   for (uint32_t i = 0; i < mIterators.Length(); ++i) {
3262     // Remove the record from iterator always, it makes no sence to return
3263     // non-existing entries. Also the pointer to the record is no longer valid
3264     // once the entry is removed from index.
3265     mIterators[i]->RemoveRecord(aRecord);
3266   }
3267 }
3268
3269 void
3270 CacheIndex::ReplaceRecordInIterators(CacheIndexRecord *aOldRecord,
3271                                      CacheIndexRecord *aNewRecord)
3272 {
3273   AssertOwnsLock();
3274
3275   for (uint32_t i = 0; i < mIterators.Length(); ++i) {
3276     // We have to replace the record always since the pointer is no longer
3277     // valid after this point. NOTE: Replacing the record doesn't mean that
3278     // a new entry was added, it just means that the data in the entry was
3279     // changed (e.g. a file size) and we had to track this change in
3280     // mPendingUpdates since mIndex was read-only.
3281     mIterators[i]->ReplaceRecord(aOldRecord, aNewRecord);
3282   }
3283 }
3284
3285 nsresult
3286 CacheIndex::Run()
3287 {
3288   LOG(("CacheIndex::Run()"));
3289
3290   CacheIndexAutoLock lock(this);
3291
3292   if (!IsIndexUsable()) {
3293     return NS_ERROR_NOT_AVAILABLE;
3294   }
3295
3296   if (mState == READY && mShuttingDown) {
3297     return NS_OK;
3298   }
3299
3300   mUpdateEventPending = false;
3301
3302   switch (mState) {
3303     case BUILDING:
3304       BuildIndex();
3305       break;
3306     case UPDATING:
3307       UpdateIndex();
3308       break;
3309     default:
3310       LOG(("CacheIndex::Run() - Update/Build was canceled"));
3311   }
3312
3313   return NS_OK;
3314 }
3315
3316 nsresult
3317 CacheIndex::OnFileOpenedInternal(FileOpenHelper *aOpener,
3318                                  CacheFileHandle *aHandle, nsresult aResult)
3319 {
3320   LOG(("CacheIndex::OnFileOpenedInternal() [opener=%p, handle=%p, "
3321        "result=0x%08x]", aOpener, aHandle, aResult));
3322
3323   nsresult rv;
3324
3325   AssertOwnsLock();
3326
3327   if (!IsIndexUsable()) {
3328     return NS_ERROR_NOT_AVAILABLE;
3329   }
3330
3331   if (mState == READY && mShuttingDown) {
3332     return NS_OK;
3333   }
3334
3335   switch (mState) {
3336     case WRITING:
3337       MOZ_ASSERT(aOpener == mIndexFileOpener);
3338       mIndexFileOpener = nullptr;
3339
3340       if (NS_FAILED(aResult)) {
3341         LOG(("CacheIndex::OnFileOpenedInternal() - Can't open index file for "
3342              "writing [rv=0x%08x]", aResult));
3343         FinishWrite(false);
3344       } else {
3345         mIndexHandle = aHandle;
3346         WriteRecords();
3347       }
3348       break;
3349     case READING:
3350       if (aOpener == mIndexFileOpener) {
3351         mIndexFileOpener = nullptr;
3352
3353         if (NS_SUCCEEDED(aResult)) {
3354           if (aHandle->FileSize() == 0) {
3355             FinishRead(false);
3356             CacheFileIOManager::DoomFile(aHandle, nullptr);
3357             break;
3358           } else {
3359             mIndexHandle = aHandle;
3360           }
3361         } else {
3362           FinishRead(false);
3363           break;
3364         }
3365       } else if (aOpener == mJournalFileOpener) {
3366         mJournalFileOpener = nullptr;
3367         mJournalHandle = aHandle;
3368       } else if (aOpener == mTmpFileOpener) {
3369         mTmpFileOpener = nullptr;
3370         mTmpHandle = aHandle;
3371       } else {
3372         MOZ_ASSERT(false, "Unexpected state!");
3373       }
3374
3375       if (mIndexFileOpener || mJournalFileOpener || mTmpFileOpener) {
3376         // Some opener still didn't finish
3377         break;
3378       }
3379
3380       // We fail and cancel all other openers when we opening index file fails.
3381       MOZ_ASSERT(mIndexHandle);
3382
3383       if (mTmpHandle) {
3384         CacheFileIOManager::DoomFile(mTmpHandle, nullptr);
3385         mTmpHandle = nullptr;
3386
3387         if (mJournalHandle) { // this shouldn't normally happen
3388           LOG(("CacheIndex::OnFileOpenedInternal() - Unexpected state, all "
3389                "files [%s, %s, %s] should never exist. Removing whole index.",
3390                kIndexName, kJournalName, kTempIndexName));
3391           FinishRead(false);
3392           break;
3393         }
3394       }
3395
3396       if (mJournalHandle) {
3397         // Rename journal to make sure we update index on next start in case
3398         // firefox crashes
3399         rv = CacheFileIOManager::RenameFile(
3400           mJournalHandle, NS_LITERAL_CSTRING(kTempIndexName), this);
3401         if (NS_FAILED(rv)) {
3402           LOG(("CacheIndex::OnFileOpenedInternal() - CacheFileIOManager::"
3403                "RenameFile() failed synchronously [rv=0x%08x]", rv));
3404           FinishRead(false);
3405           break;
3406         }
3407       } else {
3408         StartReadingIndex();
3409       }
3410
3411       break;
3412     default:
3413       MOZ_ASSERT(false, "Unexpected state!");
3414   }
3415
3416   return NS_OK;
3417 }
3418
3419 nsresult
3420 CacheIndex::OnFileOpened(CacheFileHandle *aHandle, nsresult aResult)
3421 {
3422   MOZ_CRASH("CacheIndex::OnFileOpened should not be called!");
3423   return NS_ERROR_UNEXPECTED;
3424 }
3425
3426 nsresult
3427 CacheIndex::OnDataWritten(CacheFileHandle *aHandle, const char *aBuf,
3428                           nsresult aResult)
3429 {
3430   LOG(("CacheIndex::OnDataWritten() [handle=%p, result=0x%08x]", aHandle,
3431        aResult));
3432
3433   nsresult rv;
3434
3435   CacheIndexAutoLock lock(this);
3436
3437   if (!IsIndexUsable()) {
3438     return NS_ERROR_NOT_AVAILABLE;
3439   }
3440
3441   if (mState == READY && mShuttingDown) {
3442     return NS_OK;
3443   }
3444
3445   switch (mState) {
3446     case WRITING:
3447       if (mIndexHandle != aHandle) {
3448         LOG(("CacheIndex::OnDataWritten() - ignoring notification since it "
3449              "belongs to previously canceled operation [state=%d]", mState));
3450         break;
3451       }
3452
3453       if (NS_FAILED(aResult)) {
3454         FinishWrite(false);
3455       } else {
3456         if (mSkipEntries == mProcessEntries) {
3457           rv = CacheFileIOManager::RenameFile(mIndexHandle,
3458                                               NS_LITERAL_CSTRING(kIndexName),
3459                                               this);
3460           if (NS_FAILED(rv)) {
3461             LOG(("CacheIndex::OnDataWritten() - CacheFileIOManager::"
3462                  "RenameFile() failed synchronously [rv=0x%08x]", rv));
3463             FinishWrite(false);
3464           }
3465         } else {
3466           WriteRecords();
3467         }
3468       }
3469       break;
3470     default:
3471       // Writing was canceled.
3472       LOG(("CacheIndex::OnDataWritten() - ignoring notification since the "
3473            "operation was previously canceled [state=%d]", mState));
3474   }
3475
3476   return NS_OK;
3477 }
3478
3479 nsresult
3480 CacheIndex::OnDataRead(CacheFileHandle *aHandle, char *aBuf, nsresult aResult)
3481 {
3482   LOG(("CacheIndex::OnDataRead() [handle=%p, result=0x%08x]", aHandle,
3483        aResult));
3484
3485   CacheIndexAutoLock lock(this);
3486
3487   if (!IsIndexUsable()) {
3488     return NS_ERROR_NOT_AVAILABLE;
3489   }
3490
3491   switch (mState) {
3492     case READING:
3493       MOZ_ASSERT(mIndexHandle == aHandle || mJournalHandle == aHandle);
3494
3495       if (NS_FAILED(aResult)) {
3496         FinishRead(false);
3497       } else {
3498         if (!mIndexOnDiskIsValid) {
3499           ParseRecords();
3500         } else {
3501           ParseJournal();
3502         }
3503       }
3504       break;
3505     default:
3506       // Reading was canceled.
3507       LOG(("CacheIndex::OnDataRead() - ignoring notification since the "
3508            "operation was previously canceled [state=%d]", mState));
3509   }
3510
3511   return NS_OK;
3512 }
3513
3514 nsresult
3515 CacheIndex::OnFileDoomed(CacheFileHandle *aHandle, nsresult aResult)
3516 {
3517   MOZ_CRASH("CacheIndex::OnFileDoomed should not be called!");
3518   return NS_ERROR_UNEXPECTED;
3519 }
3520
3521 nsresult
3522 CacheIndex::OnEOFSet(CacheFileHandle *aHandle, nsresult aResult)
3523 {
3524   MOZ_CRASH("CacheIndex::OnEOFSet should not be called!");
3525   return NS_ERROR_UNEXPECTED;
3526 }
3527
3528 nsresult
3529 CacheIndex::OnFileRenamed(CacheFileHandle *aHandle, nsresult aResult)
3530 {
3531   LOG(("CacheIndex::OnFileRenamed() [handle=%p, result=0x%08x]", aHandle,
3532        aResult));
3533
3534   CacheIndexAutoLock lock(this);
3535
3536   if (!IsIndexUsable()) {
3537     return NS_ERROR_NOT_AVAILABLE;
3538   }
3539
3540   if (mState == READY && mShuttingDown) {
3541     return NS_OK;
3542   }
3543
3544   switch (mState) {
3545     case WRITING:
3546       // This is a result of renaming the new index written to tmpfile to index
3547       // file. This is the last step when writing the index and the whole
3548       // writing process is successful iff renaming was successful.
3549
3550       if (mIndexHandle != aHandle) {
3551         LOG(("CacheIndex::OnFileRenamed() - ignoring notification since it "
3552              "belongs to previously canceled operation [state=%d]", mState));
3553         break;
3554       }
3555
3556       FinishWrite(NS_SUCCEEDED(aResult));
3557       break;
3558     case READING:
3559       // This is a result of renaming journal file to tmpfile. It is renamed
3560       // before we start reading index and journal file and it should normally
3561       // succeed. If it fails give up reading of index.
3562
3563       if (mJournalHandle != aHandle) {
3564         LOG(("CacheIndex::OnFileRenamed() - ignoring notification since it "
3565              "belongs to previously canceled operation [state=%d]", mState));
3566         break;
3567       }
3568
3569       if (NS_FAILED(aResult)) {
3570         FinishRead(false);
3571       } else {
3572         StartReadingIndex();
3573       }
3574       break;
3575     default:
3576       // Reading/writing was canceled.
3577       LOG(("CacheIndex::OnFileRenamed() - ignoring notification since the "
3578            "operation was previously canceled [state=%d]", mState));
3579   }
3580
3581   return NS_OK;
3582 }
3583
3584 // Memory reporting
3585
3586 size_t
3587 CacheIndex::SizeOfExcludingThisInternal(mozilla::MallocSizeOf mallocSizeOf) const
3588 {
3589   CacheIndexAutoLock lock(const_cast<CacheIndex*>(this));
3590
3591   size_t n = 0;
3592   nsCOMPtr<nsISizeOf> sizeOf;
3593
3594   // mIndexHandle and mJournalHandle are reported via SizeOfHandlesRunnable
3595   // in CacheFileIOManager::SizeOfExcludingThisInternal as part of special
3596   // handles array.
3597
3598   sizeOf = do_QueryInterface(mCacheDirectory);
3599   if (sizeOf) {
3600     n += sizeOf->SizeOfIncludingThis(mallocSizeOf);
3601   }
3602
3603   sizeOf = do_QueryInterface(mUpdateTimer);
3604   if (sizeOf) {
3605     n += sizeOf->SizeOfIncludingThis(mallocSizeOf);
3606   }
3607
3608   n += mallocSizeOf(mRWBuf);
3609   n += mallocSizeOf(mRWHash);
3610
3611   n += mIndex.SizeOfExcludingThis(mallocSizeOf);
3612   n += mPendingUpdates.SizeOfExcludingThis(mallocSizeOf);
3613   n += mTmpJournal.SizeOfExcludingThis(mallocSizeOf);
3614
3615   // mFrecencyArray and mExpirationArray items are reported by
3616   // mIndex/mPendingUpdates
3617   n += mFrecencyArray.SizeOfExcludingThis(mallocSizeOf);
3618   n += mExpirationArray.SizeOfExcludingThis(mallocSizeOf);
3619   n += mDiskConsumptionObservers.SizeOfExcludingThis(mallocSizeOf);
3620
3621   return n;
3622 }
3623
3624 // static
3625 size_t
3626 CacheIndex::SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf)
3627 {
3628   if (!gInstance)
3629     return 0;
3630
3631   return gInstance->SizeOfExcludingThisInternal(mallocSizeOf);
3632 }
3633
3634 // static
3635 size_t
3636 CacheIndex::SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf)
3637 {
3638   return mallocSizeOf(gInstance) + SizeOfExcludingThis(mallocSizeOf);
3639 }
3640
3641 } // net
3642 } // mozilla