netwerk/cache2/CacheStorageService.cpp

   1 /* This Source Code Form is subject to the terms of the Mozilla Public
   2  * License, v. 2.0. If a copy of the MPL was not distributed with this
   3  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   4
   5 #include "CacheLog.h"
   6 #include "CacheStorageService.h"
   7 #include "CacheFileIOManager.h"
   8 #include "CacheObserver.h"
   9 #include "CacheIndex.h"
  10 #include "CacheIndexIterator.h"
  11 #include "CacheStorage.h"
  12 #include "AppCacheStorage.h"
  13 #include "CacheEntry.h"
  14 #include "CacheFileUtils.h"
  15
  16 #include "OldWrappers.h"
  17 #include "nsCacheService.h"
  18 #include "nsDeleteDir.h"
  19
  20 #include "nsICacheStorageVisitor.h"
  21 #include "nsIObserverService.h"
  22 #include "nsIFile.h"
  23 #include "nsIURI.h"
  24 #include "nsCOMPtr.h"
  25 #include "nsAutoPtr.h"
  26 #include "nsNetCID.h"
  27 #include "nsNetUtil.h"
  28 #include "nsServiceManagerUtils.h"
  29 #include "nsWeakReference.h"
  30 #include "mozilla/TimeStamp.h"
  31 #include "mozilla/DebugOnly.h"
  32 #include "mozilla/VisualEventTracer.h"
  33 #include "mozilla/Services.h"
  34
  35 namespace mozilla {
  36 namespace net {
  37
  38 namespace {
  39
  40 void AppendMemoryStorageID(nsAutoCString &key)
  41 {
  42   key.Append('/');
  43   key.Append('M');
  44 }
  45
  46 }
  47
  48 // Not defining as static or class member of CacheStorageService since
  49 // it would otherwise need to include CacheEntry.h and that then would
  50 // need to be exported to make nsNetModule.cpp compilable.
  51 typedef nsClassHashtable<nsCStringHashKey, CacheEntryTable>
  52         GlobalEntryTables;
  53
  54 /**
  55  * Keeps tables of entries.  There is one entries table for each distinct load
  56  * context type.  The distinction is based on following load context info states:
  57  * <isPrivate|isAnon|appId|inBrowser> which builds a mapping key.
  58  *
  59  * Thread-safe to access, protected by the service mutex.
  60  */
  61 static GlobalEntryTables* sGlobalEntryTables;
  62
  63 CacheMemoryConsumer::CacheMemoryConsumer(uint32_t aFlags)
  64 : mReportedMemoryConsumption(0)
  65 , mFlags(aFlags)
  66 {
  67 }
  68
  69 void
  70 CacheMemoryConsumer::DoMemoryReport(uint32_t aCurrentSize)
  71 {
  72   if (!(mFlags & DONT_REPORT) && CacheStorageService::Self()) {
  73     CacheStorageService::Self()->OnMemoryConsumptionChange(this, aCurrentSize);
  74   }
  75 }
  76
  77 CacheStorageService::MemoryPool::MemoryPool(EType aType)
  78 : mType(aType)
  79 , mMemorySize(0)
  80 {
  81 }
  82
  83 CacheStorageService::MemoryPool::~MemoryPool()
  84 {
  85   if (mMemorySize != 0) {
  86     NS_ERROR("Network cache reported memory consumption is not at 0, probably leaking?");
  87   }
  88 }
  89
  90 uint32_t const
  91 CacheStorageService::MemoryPool::Limit() const
  92 {
  93   switch (mType) {
  94   case DISK:
  95     return CacheObserver::MetadataMemoryLimit();
  96   case MEMORY:
  97     return CacheObserver::MemoryCacheCapacity();
  98   }
  99
 100   MOZ_CRASH("Bad pool type");
 101   return 0;
 102 }
 103
 104 NS_IMPL_ISUPPORTS(CacheStorageService,
 105                   nsICacheStorageService,
 106                   nsIMemoryReporter,
 107                   nsITimerCallback)
 108
 109 CacheStorageService* CacheStorageService::sSelf = nullptr;
 110
 111 CacheStorageService::CacheStorageService()
 112 : mLock("CacheStorageService.mLock")
 113 , mForcedValidEntriesLock("CacheStorageService.mForcedValidEntriesLock")
 114 , mShutdown(false)
 115 , mDiskPool(MemoryPool::DISK)
 116 , mMemoryPool(MemoryPool::MEMORY)
 117 {
 118   CacheFileIOManager::Init();
 119
 120   MOZ_ASSERT(!sSelf);
 121
 122   sSelf = this;
 123   sGlobalEntryTables = new GlobalEntryTables();
 124
 125   RegisterStrongMemoryReporter(this);
 126 }
 127
 128 CacheStorageService::~CacheStorageService()
 129 {
 130   LOG(("CacheStorageService::~CacheStorageService"));
 131   sSelf = nullptr;
 132 }
 133
 134 void CacheStorageService::Shutdown()
 135 {
 136   if (mShutdown)
 137     return;
 138
 139   LOG(("CacheStorageService::Shutdown - start"));
 140
 141   mShutdown = true;
 142
 143   nsCOMPtr<nsIRunnable> event =
 144     NS_NewRunnableMethod(this, &CacheStorageService::ShutdownBackground);
 145   Dispatch(event);
 146
 147   mozilla::MutexAutoLock lock(mLock);
 148   sGlobalEntryTables->Clear();
 149   delete sGlobalEntryTables;
 150   sGlobalEntryTables = nullptr;
 151
 152   LOG(("CacheStorageService::Shutdown - done"));
 153 }
 154
 155 void CacheStorageService::ShutdownBackground()
 156 {
 157   MOZ_ASSERT(IsOnManagementThread());
 158
 159   // Cancel purge timer to avoid leaking.
 160   if (mPurgeTimer) {
 161     mPurgeTimer->Cancel();
 162   }
 163
 164   Pool(false).mFrecencyArray.Clear();
 165   Pool(false).mExpirationArray.Clear();
 166   Pool(true).mFrecencyArray.Clear();
 167   Pool(true).mExpirationArray.Clear();
 168 }
 169
 170 // Internal management methods
 171
 172 namespace { // anon
 173
 174 // WalkCacheRunnable
 175 // Base class for particular storage entries visiting
 176 class WalkCacheRunnable : public nsRunnable
 177                         , public CacheStorageService::EntryInfoCallback
 178 {
 179 protected:
 180   WalkCacheRunnable(nsICacheStorageVisitor* aVisitor,
 181                     bool aVisitEntries)
 182     : mService(CacheStorageService::Self())
 183     , mCallback(aVisitor)
 184     , mSize(0)
 185     , mNotifyStorage(true)
 186     , mVisitEntries(aVisitEntries)
 187   {
 188     MOZ_ASSERT(NS_IsMainThread());
 189   }
 190
 191   virtual ~WalkCacheRunnable()
 192   {
 193     if (mCallback) {
 194       ProxyReleaseMainThread(mCallback);
 195     }
 196   }
 197
 198   nsRefPtr<CacheStorageService> mService;
 199   nsCOMPtr<nsICacheStorageVisitor> mCallback;
 200
 201   uint64_t mSize;
 202
 203   bool mNotifyStorage : 1;
 204   bool mVisitEntries : 1;
 205 };
 206
 207 // WalkMemoryCacheRunnable
 208 // Responsible to visit memory storage and walk
 209 // all entries on it asynchronously.
 210 class WalkMemoryCacheRunnable : public WalkCacheRunnable
 211 {
 212 public:
 213   WalkMemoryCacheRunnable(nsILoadContextInfo *aLoadInfo,
 214                           bool aVisitEntries,
 215                           nsICacheStorageVisitor* aVisitor)
 216     : WalkCacheRunnable(aVisitor, aVisitEntries)
 217   {
 218     CacheFileUtils::AppendKeyPrefix(aLoadInfo, mContextKey);
 219     MOZ_ASSERT(NS_IsMainThread());
 220   }
 221
 222   nsresult Walk()
 223   {
 224     return mService->Dispatch(this);
 225   }
 226
 227 private:
 228   NS_IMETHODIMP Run()
 229   {
 230     if (CacheStorageService::IsOnManagementThread()) {
 231       LOG(("WalkMemoryCacheRunnable::Run - collecting [this=%p]", this));
 232       // First, walk, count and grab all entries from the storage
 233
 234       mozilla::MutexAutoLock lock(CacheStorageService::Self()->Lock());
 235
 236       if (!CacheStorageService::IsRunning())
 237         return NS_ERROR_NOT_INITIALIZED;
 238
 239       CacheEntryTable* entries;
 240       if (sGlobalEntryTables->Get(mContextKey, &entries))
 241         entries->EnumerateRead(&WalkMemoryCacheRunnable::WalkStorage, this);
 242
 243       // Next, we dispatch to the main thread
 244     } else if (NS_IsMainThread()) {
 245       LOG(("WalkMemoryCacheRunnable::Run - notifying [this=%p]", this));
 246
 247       if (mNotifyStorage) {
 248         LOG(("  storage"));
 249
 250         // Second, notify overall storage info
 251         mCallback->OnCacheStorageInfo(mEntryArray.Length(), mSize,
 252                                       CacheObserver::MemoryCacheCapacity(), nullptr);
 253         if (!mVisitEntries)
 254           return NS_OK; // done
 255
 256         mNotifyStorage = false;
 257
 258       } else {
 259         LOG(("  entry [left=%d]", mEntryArray.Length()));
 260
 261         // Third, notify each entry until depleted
 262         if (!mEntryArray.Length()) {
 263           mCallback->OnCacheEntryVisitCompleted();
 264           return NS_OK; // done
 265         }
 266
 267         // Grab the next entry
 268         nsRefPtr<CacheEntry> entry = mEntryArray[0];
 269         mEntryArray.RemoveElementAt(0);
 270
 271         // Invokes this->OnEntryInfo, that calls the callback with all
 272         // information of the entry.
 273         CacheStorageService::GetCacheEntryInfo(entry, this);
 274       }
 275     } else {
 276       MOZ_CRASH("Bad thread");
 277       return NS_ERROR_FAILURE;
 278     }
 279
 280     NS_DispatchToMainThread(this);
 281     return NS_OK;
 282   }
 283
 284   virtual ~WalkMemoryCacheRunnable()
 285   {
 286     if (mCallback)
 287       ProxyReleaseMainThread(mCallback);
 288   }
 289
 290   static PLDHashOperator
 291   WalkStorage(const nsACString& aKey,
 292               CacheEntry* aEntry,
 293               void* aClosure)
 294   {
 295     WalkMemoryCacheRunnable* walker =
 296       static_cast<WalkMemoryCacheRunnable*>(aClosure);
 297
 298     // Ignore disk entries
 299     if (aEntry->IsUsingDisk())
 300       return PL_DHASH_NEXT;
 301
 302     walker->mSize += aEntry->GetMetadataMemoryConsumption();
 303
 304     int64_t size;
 305     if (NS_SUCCEEDED(aEntry->GetDataSize(&size)))
 306       walker->mSize += size;
 307
 308     walker->mEntryArray.AppendElement(aEntry);
 309     return PL_DHASH_NEXT;
 310   }
 311
 312   virtual void OnEntryInfo(const nsACString & aURISpec, const nsACString & aIdEnhance,
 313                            int64_t aDataSize, int32_t aFetchCount,
 314                            uint32_t aLastModifiedTime, uint32_t aExpirationTime)
 315   {
 316     nsCOMPtr<nsIURI> uri;
 317     nsresult rv = NS_NewURI(getter_AddRefs(uri), aURISpec);
 318     if (NS_FAILED(rv))
 319       return;
 320
 321     mCallback->OnCacheEntryInfo(uri, aIdEnhance, aDataSize, aFetchCount,
 322                                 aLastModifiedTime, aExpirationTime);
 323   }
 324
 325 private:
 326   nsCString mContextKey;
 327   nsTArray<nsRefPtr<CacheEntry> > mEntryArray;
 328 };
 329
 330 // WalkDiskCacheRunnable
 331 // Using the cache index information to get the list of files per context.
 332 class WalkDiskCacheRunnable : public WalkCacheRunnable
 333 {
 334 public:
 335   WalkDiskCacheRunnable(nsILoadContextInfo *aLoadInfo,
 336                         bool aVisitEntries,
 337                         nsICacheStorageVisitor* aVisitor)
 338     : WalkCacheRunnable(aVisitor, aVisitEntries)
 339     , mLoadInfo(aLoadInfo)
 340     , mPass(COLLECT_STATS)
 341   {
 342   }
 343
 344   nsresult Walk()
 345   {
 346     // TODO, bug 998693
 347     // Initial index build should be forced here so that about:cache soon
 348     // after startup gives some meaningfull results.
 349
 350     // Dispatch to the INDEX level in hope that very recent cache entries
 351     // information gets to the index list before we grab the index iterator
 352     // for the first time.  This tries to avoid miss of entries that has
 353     // been created right before the visit is required.
 354     nsRefPtr<CacheIOThread> thread = CacheFileIOManager::IOThread();
 355     NS_ENSURE_TRUE(thread, NS_ERROR_NOT_INITIALIZED);
 356
 357     return thread->Dispatch(this, CacheIOThread::INDEX);
 358   }
 359
 360 private:
 361   // Invokes OnCacheEntryInfo callback for each single found entry.
 362   // There is one instance of this class per one entry.
 363   class OnCacheEntryInfoRunnable : public nsRunnable
 364   {
 365   public:
 366     explicit OnCacheEntryInfoRunnable(WalkDiskCacheRunnable* aWalker)
 367       : mWalker(aWalker)
 368     {
 369     }
 370
 371     NS_IMETHODIMP Run()
 372     {
 373       MOZ_ASSERT(NS_IsMainThread());
 374
 375       nsCOMPtr<nsIURI> uri;
 376       nsresult rv = NS_NewURI(getter_AddRefs(uri), mURISpec);
 377       if (NS_FAILED(rv))
 378         return NS_OK;
 379
 380       mWalker->mCallback->OnCacheEntryInfo(
 381         uri, mIdEnhance, mDataSize, mFetchCount,
 382         mLastModifiedTime, mExpirationTime);
 383       return NS_OK;
 384     }
 385
 386     nsRefPtr<WalkDiskCacheRunnable> mWalker;
 387
 388     nsCString mURISpec;
 389     nsCString mIdEnhance;
 390     int64_t mDataSize;
 391     int32_t mFetchCount;
 392     uint32_t mLastModifiedTime;
 393     uint32_t mExpirationTime;
 394   };
 395
 396   NS_IMETHODIMP Run()
 397   {
 398     // The main loop
 399     nsresult rv;
 400
 401     if (CacheStorageService::IsOnManagementThread()) {
 402       switch (mPass) {
 403       case COLLECT_STATS:
 404         // Get quickly the cache stats.
 405         uint32_t size;
 406         rv = CacheIndex::GetCacheStats(mLoadInfo, &size, &mCount);
 407         if (NS_FAILED(rv)) {
 408           if (mVisitEntries) {
 409             // both onStorageInfo and onCompleted are expected
 410             NS_DispatchToMainThread(this);
 411           }
 412           return NS_DispatchToMainThread(this);
 413         }
 414
 415         mSize = size << 10;
 416
 417         // Invoke onCacheStorageInfo with valid information.
 418         NS_DispatchToMainThread(this);
 419
 420         if (!mVisitEntries) {
 421           return NS_OK; // done
 422         }
 423
 424         mPass = ITERATE_METADATA;
 425         // no break
 426
 427       case ITERATE_METADATA:
 428         // Now grab the context iterator.
 429         if (!mIter) {
 430           rv = CacheIndex::GetIterator(mLoadInfo, true, getter_AddRefs(mIter));
 431           if (NS_FAILED(rv)) {
 432             // Invoke onCacheEntryVisitCompleted now
 433             return NS_DispatchToMainThread(this);
 434           }
 435         }
 436
 437         while (true) {
 438           if (CacheIOThread::YieldAndRerun())
 439             return NS_OK;
 440
 441           SHA1Sum::Hash hash;
 442           rv = mIter->GetNextHash(&hash);
 443           if (NS_FAILED(rv))
 444             break; // done (or error?)
 445
 446           // This synchronously invokes onCacheEntryInfo on this class where we
 447           // redispatch to the main thread for the consumer callback.
 448           CacheFileIOManager::GetEntryInfo(&hash, this);
 449         }
 450
 451         // Invoke onCacheEntryVisitCompleted on the main thread
 452         NS_DispatchToMainThread(this);
 453       }
 454     } else if (NS_IsMainThread()) {
 455       if (mNotifyStorage) {
 456         nsCOMPtr<nsIFile> dir;
 457         CacheFileIOManager::GetCacheDirectory(getter_AddRefs(dir));
 458         mCallback->OnCacheStorageInfo(mCount, mSize, CacheObserver::DiskCacheCapacity(), dir);
 459         mNotifyStorage = false;
 460       } else {
 461         mCallback->OnCacheEntryVisitCompleted();
 462       }
 463     } else {
 464       MOZ_CRASH("Bad thread");
 465       return NS_ERROR_FAILURE;
 466     }
 467
 468     return NS_OK;
 469   }
 470
 471   virtual void OnEntryInfo(const nsACString & aURISpec, const nsACString & aIdEnhance,
 472                            int64_t aDataSize, int32_t aFetchCount,
 473                            uint32_t aLastModifiedTime, uint32_t aExpirationTime)
 474   {
 475     // Called directly from CacheFileIOManager::GetEntryInfo.
 476
 477     // Invoke onCacheEntryInfo on the main thread for this entry.
 478     nsRefPtr<OnCacheEntryInfoRunnable> info = new OnCacheEntryInfoRunnable(this);
 479     info->mURISpec = aURISpec;
 480     info->mIdEnhance = aIdEnhance;
 481     info->mDataSize = aDataSize;
 482     info->mFetchCount = aFetchCount;
 483     info->mLastModifiedTime = aLastModifiedTime;
 484     info->mExpirationTime = aExpirationTime;
 485
 486     NS_DispatchToMainThread(info);
 487   }
 488
 489   nsRefPtr<nsILoadContextInfo> mLoadInfo;
 490   enum {
 491     // First, we collect stats for the load context.
 492     COLLECT_STATS,
 493
 494     // Second, if demanded, we iterate over the entries gethered
 495     // from the iterator and call CacheFileIOManager::GetEntryInfo
 496     // for each found entry.
 497     ITERATE_METADATA,
 498   } mPass;
 499
 500   nsRefPtr<CacheIndexIterator> mIter;
 501   uint32_t mCount;
 502 };
 503
 504 PLDHashOperator CollectPrivateContexts(const nsACString& aKey,
 505                                        CacheEntryTable* aTable,
 506                                        void* aClosure)
 507 {
 508   nsCOMPtr<nsILoadContextInfo> info = CacheFileUtils::ParseKey(aKey);
 509   if (info && info->IsPrivate()) {
 510     nsTArray<nsCString>* keys = static_cast<nsTArray<nsCString>*>(aClosure);
 511     keys->AppendElement(aKey);
 512   }
 513
 514   return PL_DHASH_NEXT;
 515 }
 516
 517 PLDHashOperator CollectContexts(const nsACString& aKey,
 518                                        CacheEntryTable* aTable,
 519                                        void* aClosure)
 520 {
 521   nsTArray<nsCString>* keys = static_cast<nsTArray<nsCString>*>(aClosure);
 522   keys->AppendElement(aKey);
 523
 524   return PL_DHASH_NEXT;
 525 }
 526
 527 } // anon
 528
 529 void CacheStorageService::DropPrivateBrowsingEntries()
 530 {
 531   mozilla::MutexAutoLock lock(mLock);
 532
 533   if (mShutdown)
 534     return;
 535
 536   nsTArray<nsCString> keys;
 537   sGlobalEntryTables->EnumerateRead(&CollectPrivateContexts, &keys);
 538
 539   for (uint32_t i = 0; i < keys.Length(); ++i)
 540     DoomStorageEntries(keys[i], nullptr, true, nullptr);
 541 }
 542
 543 namespace { // anon
 544
 545 class CleaupCacheDirectoriesRunnable : public nsRunnable
 546 {
 547 public:
 548   NS_DECL_NSIRUNNABLE
 549   static bool Post(uint32_t aVersion, uint32_t aActive);
 550
 551 private:
 552   CleaupCacheDirectoriesRunnable(uint32_t aVersion, uint32_t aActive)
 553     : mVersion(aVersion), mActive(aActive)
 554   {
 555     nsCacheService::GetDiskCacheDirectory(getter_AddRefs(mCache1Dir));
 556     CacheFileIOManager::GetCacheDirectory(getter_AddRefs(mCache2Dir));
 557 #if defined(MOZ_WIDGET_ANDROID)
 558     CacheFileIOManager::GetProfilelessCacheDirectory(getter_AddRefs(mCache2Profileless));
 559 #endif
 560   }
 561
 562   virtual ~CleaupCacheDirectoriesRunnable() {}
 563   uint32_t mVersion, mActive;
 564   nsCOMPtr<nsIFile> mCache1Dir, mCache2Dir;
 565 #if defined(MOZ_WIDGET_ANDROID)
 566   nsCOMPtr<nsIFile> mCache2Profileless;
 567 #endif
 568 };
 569
 570 // static
 571 bool CleaupCacheDirectoriesRunnable::Post(uint32_t aVersion, uint32_t aActive)
 572 {
 573   // CleaupCacheDirectories is called regardless what cache version is set up to use.
 574   // To obtain the cache1 directory we must unfortunatelly instantiate the old cache
 575   // service despite it may not be used at all...  This also initialize nsDeleteDir.
 576   nsCOMPtr<nsICacheService> service = do_GetService(NS_CACHESERVICE_CONTRACTID);
 577   if (!service)
 578     return false;
 579
 580   nsCOMPtr<nsIEventTarget> thread;
 581   service->GetCacheIOTarget(getter_AddRefs(thread));
 582   if (!thread)
 583     return false;
 584
 585   nsRefPtr<CleaupCacheDirectoriesRunnable> r =
 586     new CleaupCacheDirectoriesRunnable(aVersion, aActive);
 587   thread->Dispatch(r, NS_DISPATCH_NORMAL);
 588   return true;
 589 }
 590
 591 NS_IMETHODIMP CleaupCacheDirectoriesRunnable::Run()
 592 {
 593   MOZ_ASSERT(!NS_IsMainThread());
 594
 595   if (mCache1Dir) {
 596     nsDeleteDir::RemoveOldTrashes(mCache1Dir);
 597   }
 598   if (mCache2Dir) {
 599     nsDeleteDir::RemoveOldTrashes(mCache2Dir);
 600   }
 601 #if defined(MOZ_WIDGET_ANDROID)
 602   if (mCache2Profileless) {
 603     nsDeleteDir::RemoveOldTrashes(mCache2Profileless);
 604     // Always delete the profileless cache on Android
 605     nsDeleteDir::DeleteDir(mCache2Profileless, true, 30000);
 606   }
 607 #endif
 608
 609   // Delete the non-active version cache data right now
 610   if (mVersion == mActive) {
 611     return NS_OK;
 612   }
 613
 614   switch (mVersion) {
 615   case 0:
 616     if (mCache1Dir) {
 617       nsDeleteDir::DeleteDir(mCache1Dir, true, 30000);
 618     }
 619     break;
 620   case 1:
 621     if (mCache2Dir) {
 622       nsDeleteDir::DeleteDir(mCache2Dir, true, 30000);
 623     }
 624     break;
 625   }
 626
 627   return NS_OK;
 628 }
 629
 630 } // anon
 631
 632 // static
 633 void CacheStorageService::CleaupCacheDirectories(uint32_t aVersion, uint32_t aActive)
 634 {
 635   // Make sure we schedule just once in case CleaupCacheDirectories gets called
 636   // multiple times from some reason.
 637   static bool runOnce = CleaupCacheDirectoriesRunnable::Post(aVersion, aActive);
 638   if (!runOnce) {
 639     NS_WARNING("Could not start cache trashes cleanup");
 640   }
 641 }
 642
 643 // Helper methods
 644
 645 // static
 646 bool CacheStorageService::IsOnManagementThread()
 647 {
 648   nsRefPtr<CacheStorageService> service = Self();
 649   if (!service)
 650     return false;
 651
 652   nsCOMPtr<nsIEventTarget> target = service->Thread();
 653   if (!target)
 654     return false;
 655
 656   bool currentThread;
 657   nsresult rv = target->IsOnCurrentThread(&currentThread);
 658   return NS_SUCCEEDED(rv) && currentThread;
 659 }
 660
 661 already_AddRefed<nsIEventTarget> CacheStorageService::Thread() const
 662 {
 663   return CacheFileIOManager::IOTarget();
 664 }
 665
 666 nsresult CacheStorageService::Dispatch(nsIRunnable* aEvent)
 667 {
 668   nsRefPtr<CacheIOThread> cacheIOThread = CacheFileIOManager::IOThread();
 669   if (!cacheIOThread)
 670     return NS_ERROR_NOT_AVAILABLE;
 671
 672   return cacheIOThread->Dispatch(aEvent, CacheIOThread::MANAGEMENT);
 673 }
 674
 675 // nsICacheStorageService
 676
 677 NS_IMETHODIMP CacheStorageService::MemoryCacheStorage(nsILoadContextInfo *aLoadContextInfo,
 678                                                       nsICacheStorage * *_retval)
 679 {
 680   NS_ENSURE_ARG(aLoadContextInfo);
 681   NS_ENSURE_ARG(_retval);
 682
 683   nsCOMPtr<nsICacheStorage> storage;
 684   if (CacheObserver::UseNewCache()) {
 685     storage = new CacheStorage(aLoadContextInfo, false, false);
 686   }
 687   else {
 688     storage = new _OldStorage(aLoadContextInfo, false, false, false, nullptr);
 689   }
 690
 691   storage.forget(_retval);
 692   return NS_OK;
 693 }
 694
 695 NS_IMETHODIMP CacheStorageService::DiskCacheStorage(nsILoadContextInfo *aLoadContextInfo,
 696                                                     bool aLookupAppCache,
 697                                                     nsICacheStorage * *_retval)
 698 {
 699   NS_ENSURE_ARG(aLoadContextInfo);
 700   NS_ENSURE_ARG(_retval);
 701
 702   // TODO save some heap granularity - cache commonly used storages.
 703
 704   // When disk cache is disabled, still provide a storage, but just keep stuff
 705   // in memory.
 706   bool useDisk = CacheObserver::UseDiskCache();
 707
 708   nsCOMPtr<nsICacheStorage> storage;
 709   if (CacheObserver::UseNewCache()) {
 710     storage = new CacheStorage(aLoadContextInfo, useDisk, aLookupAppCache);
 711   }
 712   else {
 713     storage = new _OldStorage(aLoadContextInfo, useDisk, aLookupAppCache, false, nullptr);
 714   }
 715
 716   storage.forget(_retval);
 717   return NS_OK;
 718 }
 719
 720 NS_IMETHODIMP CacheStorageService::AppCacheStorage(nsILoadContextInfo *aLoadContextInfo,
 721                                                    nsIApplicationCache *aApplicationCache,
 722                                                    nsICacheStorage * *_retval)
 723 {
 724   NS_ENSURE_ARG(aLoadContextInfo);
 725   NS_ENSURE_ARG(_retval);
 726
 727   nsCOMPtr<nsICacheStorage> storage;
 728   if (CacheObserver::UseNewCache()) {
 729     // Using classification since cl believes we want to instantiate this method
 730     // having the same name as the desired class...
 731     storage = new mozilla::net::AppCacheStorage(aLoadContextInfo, aApplicationCache);
 732   }
 733   else {
 734     storage = new _OldStorage(aLoadContextInfo, true, false, true, aApplicationCache);
 735   }
 736
 737   storage.forget(_retval);
 738   return NS_OK;
 739 }
 740
 741 NS_IMETHODIMP CacheStorageService::Clear()
 742 {
 743   nsresult rv;
 744
 745   if (CacheObserver::UseNewCache()) {
 746     {
 747       mozilla::MutexAutoLock lock(mLock);
 748
 749       NS_ENSURE_TRUE(!mShutdown, NS_ERROR_NOT_INITIALIZED);
 750
 751       nsTArray<nsCString> keys;
 752       sGlobalEntryTables->EnumerateRead(&CollectContexts, &keys);
 753
 754       for (uint32_t i = 0; i < keys.Length(); ++i)
 755         DoomStorageEntries(keys[i], nullptr, true, nullptr);
 756     }
 757
 758     rv = CacheFileIOManager::EvictAll();
 759     NS_ENSURE_SUCCESS(rv, rv);
 760   } else {
 761     nsCOMPtr<nsICacheService> serv =
 762         do_GetService(NS_CACHESERVICE_CONTRACTID, &rv);
 763     NS_ENSURE_SUCCESS(rv, rv);
 764
 765     rv = serv->EvictEntries(nsICache::STORE_ANYWHERE);
 766     NS_ENSURE_SUCCESS(rv, rv);
 767   }
 768
 769   return NS_OK;
 770 }
 771
 772 NS_IMETHODIMP CacheStorageService::PurgeFromMemory(uint32_t aWhat)
 773 {
 774   uint32_t what;
 775
 776   switch (aWhat) {
 777   case PURGE_DISK_DATA_ONLY:
 778     what = CacheEntry::PURGE_DATA_ONLY_DISK_BACKED;
 779     break;
 780
 781   case PURGE_DISK_ALL:
 782     what = CacheEntry::PURGE_WHOLE_ONLY_DISK_BACKED;
 783     break;
 784
 785   case PURGE_EVERYTHING:
 786     what = CacheEntry::PURGE_WHOLE;
 787     break;
 788
 789   default:
 790     return NS_ERROR_INVALID_ARG;
 791   }
 792
 793   nsCOMPtr<nsIRunnable> event =
 794     new PurgeFromMemoryRunnable(this, what);
 795
 796   return Dispatch(event);
 797 }
 798
 799 NS_IMETHODIMP CacheStorageService::AsyncGetDiskConsumption(
 800   nsICacheStorageConsumptionObserver* aObserver)
 801 {
 802   NS_ENSURE_ARG(aObserver);
 803
 804   nsresult rv;
 805
 806   if (CacheObserver::UseNewCache()) {
 807     rv = CacheIndex::AsyncGetDiskConsumption(aObserver);
 808     NS_ENSURE_SUCCESS(rv, rv);
 809   } else {
 810     rv = _OldGetDiskConsumption::Get(aObserver);
 811     NS_ENSURE_SUCCESS(rv, rv);
 812   }
 813
 814   return NS_OK;
 815 }
 816
 817 NS_IMETHODIMP CacheStorageService::GetIoTarget(nsIEventTarget** aEventTarget)
 818 {
 819   NS_ENSURE_ARG(aEventTarget);
 820
 821   if (CacheObserver::UseNewCache()) {
 822     nsCOMPtr<nsIEventTarget> ioTarget = CacheFileIOManager::IOTarget();
 823     ioTarget.forget(aEventTarget);
 824   }
 825   else {
 826     nsresult rv;
 827
 828     nsCOMPtr<nsICacheService> serv =
 829         do_GetService(NS_CACHESERVICE_CONTRACTID, &rv);
 830     NS_ENSURE_SUCCESS(rv, rv);
 831
 832     rv = serv->GetCacheIOTarget(aEventTarget);
 833     NS_ENSURE_SUCCESS(rv, rv);
 834   }
 835
 836   return NS_OK;
 837 }
 838
 839 // Methods used by CacheEntry for management of in-memory structures.
 840
 841 namespace { // anon
 842
 843 class FrecencyComparator
 844 {
 845 public:
 846   bool Equals(CacheEntry* a, CacheEntry* b) const {
 847     return a->GetFrecency() == b->GetFrecency();
 848   }
 849   bool LessThan(CacheEntry* a, CacheEntry* b) const {
 850     return a->GetFrecency() < b->GetFrecency();
 851   }
 852 };
 853
 854 class ExpirationComparator
 855 {
 856 public:
 857   bool Equals(CacheEntry* a, CacheEntry* b) const {
 858     return a->GetExpirationTime() == b->GetExpirationTime();
 859   }
 860   bool LessThan(CacheEntry* a, CacheEntry* b) const {
 861     return a->GetExpirationTime() < b->GetExpirationTime();
 862   }
 863 };
 864
 865 } // anon
 866
 867 void
 868 CacheStorageService::RegisterEntry(CacheEntry* aEntry)
 869 {
 870   MOZ_ASSERT(IsOnManagementThread());
 871
 872   if (mShutdown || !aEntry->CanRegister())
 873     return;
 874
 875   TelemetryRecordEntryCreation(aEntry);
 876
 877   LOG(("CacheStorageService::RegisterEntry [entry=%p]", aEntry));
 878
 879   MemoryPool& pool = Pool(aEntry->IsUsingDisk());
 880   pool.mFrecencyArray.InsertElementSorted(aEntry, FrecencyComparator());
 881   pool.mExpirationArray.InsertElementSorted(aEntry, ExpirationComparator());
 882
 883   aEntry->SetRegistered(true);
 884 }
 885
 886 void
 887 CacheStorageService::UnregisterEntry(CacheEntry* aEntry)
 888 {
 889   MOZ_ASSERT(IsOnManagementThread());
 890
 891   if (!aEntry->IsRegistered())
 892     return;
 893
 894   TelemetryRecordEntryRemoval(aEntry);
 895
 896   LOG(("CacheStorageService::UnregisterEntry [entry=%p]", aEntry));
 897
 898   MemoryPool& pool = Pool(aEntry->IsUsingDisk());
 899   mozilla::DebugOnly<bool> removedFrecency = pool.mFrecencyArray.RemoveElement(aEntry);
 900   mozilla::DebugOnly<bool> removedExpiration = pool.mExpirationArray.RemoveElement(aEntry);
 901
 902   MOZ_ASSERT(mShutdown || (removedFrecency && removedExpiration));
 903
 904   // Note: aEntry->CanRegister() since now returns false
 905   aEntry->SetRegistered(false);
 906 }
 907
 908 static bool
 909 AddExactEntry(CacheEntryTable* aEntries,
 910               nsCString const& aKey,
 911               CacheEntry* aEntry,
 912               bool aOverwrite)
 913 {
 914   nsRefPtr<CacheEntry> existingEntry;
 915   if (!aOverwrite && aEntries->Get(aKey, getter_AddRefs(existingEntry))) {
 916     bool equals = existingEntry == aEntry;
 917     LOG(("AddExactEntry [entry=%p equals=%d]", aEntry, equals));
 918     return equals; // Already there...
 919   }
 920
 921   LOG(("AddExactEntry [entry=%p put]", aEntry));
 922   aEntries->Put(aKey, aEntry);
 923   return true;
 924 }
 925
 926 static bool
 927 RemoveExactEntry(CacheEntryTable* aEntries,
 928                  nsCString const& aKey,
 929                  CacheEntry* aEntry,
 930                  bool aOverwrite)
 931 {
 932   nsRefPtr<CacheEntry> existingEntry;
 933   if (!aEntries->Get(aKey, getter_AddRefs(existingEntry))) {
 934     LOG(("RemoveExactEntry [entry=%p already gone]", aEntry));
 935     return false; // Already removed...
 936   }
 937
 938   if (!aOverwrite && existingEntry != aEntry) {
 939     LOG(("RemoveExactEntry [entry=%p already replaced]", aEntry));
 940     return false; // Already replaced...
 941   }
 942
 943   LOG(("RemoveExactEntry [entry=%p removed]", aEntry));
 944   aEntries->Remove(aKey);
 945   return true;
 946 }
 947
 948 bool
 949 CacheStorageService::RemoveEntry(CacheEntry* aEntry, bool aOnlyUnreferenced)
 950 {
 951   LOG(("CacheStorageService::RemoveEntry [entry=%p]", aEntry));
 952
 953   nsAutoCString entryKey;
 954   nsresult rv = aEntry->HashingKey(entryKey);
 955   if (NS_FAILED(rv)) {
 956     NS_ERROR("aEntry->HashingKey() failed?");
 957     return false;
 958   }
 959
 960   mozilla::MutexAutoLock lock(mLock);
 961
 962   if (mShutdown) {
 963     LOG(("  after shutdown"));
 964     return false;
 965   }
 966
 967   if (aOnlyUnreferenced) {
 968     if (aEntry->IsReferenced()) {
 969       LOG(("  still referenced, not removing"));
 970       return false;
 971     }
 972
 973     if (!aEntry->IsUsingDisk() && IsForcedValidEntry(entryKey)) {
 974       LOG(("  forced valid, not removing"));
 975       return false;
 976     }
 977   }
 978
 979   CacheEntryTable* entries;
 980   if (sGlobalEntryTables->Get(aEntry->GetStorageID(), &entries))
 981     RemoveExactEntry(entries, entryKey, aEntry, false /* don't overwrite */);
 982
 983   nsAutoCString memoryStorageID(aEntry->GetStorageID());
 984   AppendMemoryStorageID(memoryStorageID);
 985
 986   if (sGlobalEntryTables->Get(memoryStorageID, &entries))
 987     RemoveExactEntry(entries, entryKey, aEntry, false /* don't overwrite */);
 988
 989   return true;
 990 }
 991
 992 void
 993 CacheStorageService::RecordMemoryOnlyEntry(CacheEntry* aEntry,
 994                                            bool aOnlyInMemory,
 995                                            bool aOverwrite)
 996 {
 997   LOG(("CacheStorageService::RecordMemoryOnlyEntry [entry=%p, memory=%d, overwrite=%d]",
 998     aEntry, aOnlyInMemory, aOverwrite));
 999   // This method is responsible to put this entry to a special record hashtable
1000   // that contains only entries that are stored in memory.
1001   // Keep in mind that every entry, regardless of whether is in-memory-only or not
1002   // is always recorded in the storage master hash table, the one identified by
1003   // CacheEntry.StorageID().
1004
1005   mLock.AssertCurrentThreadOwns();
1006
1007   if (mShutdown) {
1008     LOG(("  after shutdown"));
1009     return;
1010   }
1011
1012   nsresult rv;
1013
1014   nsAutoCString entryKey;
1015   rv = aEntry->HashingKey(entryKey);
1016   if (NS_FAILED(rv)) {
1017     NS_ERROR("aEntry->HashingKey() failed?");
1018     return;
1019   }
1020
1021   CacheEntryTable* entries = nullptr;
1022   nsAutoCString memoryStorageID(aEntry->GetStorageID());
1023   AppendMemoryStorageID(memoryStorageID);
1024
1025   if (!sGlobalEntryTables->Get(memoryStorageID, &entries)) {
1026     if (!aOnlyInMemory) {
1027       LOG(("  not recorded as memory only"));
1028       return;
1029     }
1030
1031     entries = new CacheEntryTable(CacheEntryTable::MEMORY_ONLY);
1032     sGlobalEntryTables->Put(memoryStorageID, entries);
1033     LOG(("  new memory-only storage table for %s", memoryStorageID.get()));
1034   }
1035
1036   if (aOnlyInMemory) {
1037     AddExactEntry(entries, entryKey, aEntry, aOverwrite);
1038   }
1039   else {
1040     RemoveExactEntry(entries, entryKey, aEntry, aOverwrite);
1041   }
1042 }
1043
1044 // Checks if a cache entry is forced valid (will be loaded directly from cache
1045 // without further validation) - see nsICacheEntry.idl for further details
1046 bool CacheStorageService::IsForcedValidEntry(nsACString &aCacheEntryKey)
1047 {
1048   mozilla::MutexAutoLock lock(mForcedValidEntriesLock);
1049
1050   TimeStamp validUntil;
1051
1052   if (!mForcedValidEntries.Get(aCacheEntryKey, &validUntil)) {
1053     return false;
1054   }
1055
1056   if (validUntil.IsNull()) {
1057     return false;
1058   }
1059
1060   // Entry timeout not reached yet
1061   if (TimeStamp::NowLoRes() <= validUntil) {
1062     return true;
1063   }
1064
1065   // Entry timeout has been reached
1066   mForcedValidEntries.Remove(aCacheEntryKey);
1067   return false;
1068 }
1069
1070 // Allows a cache entry to be loaded directly from cache without further
1071 // validation - see nsICacheEntry.idl for further details
1072 void CacheStorageService::ForceEntryValidFor(nsACString &aCacheEntryKey,
1073                                              uint32_t aSecondsToTheFuture)
1074 {
1075   mozilla::MutexAutoLock lock(mForcedValidEntriesLock);
1076
1077   TimeStamp now = TimeStamp::NowLoRes();
1078   ForcedValidEntriesPrune(now);
1079
1080   // This will be the timeout
1081   TimeStamp validUntil = now + TimeDuration::FromSeconds(aSecondsToTheFuture);
1082
1083   mForcedValidEntries.Put(aCacheEntryKey, validUntil);
1084 }
1085
1086 namespace { // anon
1087
1088 PLDHashOperator PruneForcedValidEntries(
1089   const nsACString& aKey, TimeStamp& aTimeStamp, void* aClosure)
1090 {
1091   TimeStamp* now = static_cast<TimeStamp*>(aClosure);
1092   if (aTimeStamp < *now) {
1093     return PL_DHASH_REMOVE;
1094   }
1095
1096   return PL_DHASH_NEXT;
1097 }
1098
1099 } // anon
1100
1101 // Cleans out the old entries in mForcedValidEntries
1102 void CacheStorageService::ForcedValidEntriesPrune(TimeStamp &now)
1103 {
1104   static TimeDuration const oneMinute = TimeDuration::FromSeconds(60);
1105   static TimeStamp dontPruneUntil = now + oneMinute;
1106   if (now < dontPruneUntil)
1107     return;
1108
1109   mForcedValidEntries.Enumerate(PruneForcedValidEntries, &now);
1110   dontPruneUntil = now + oneMinute;
1111 }
1112
1113 void
1114 CacheStorageService::OnMemoryConsumptionChange(CacheMemoryConsumer* aConsumer,
1115                                                uint32_t aCurrentMemoryConsumption)
1116 {
1117   LOG(("CacheStorageService::OnMemoryConsumptionChange [consumer=%p, size=%u]",
1118     aConsumer, aCurrentMemoryConsumption));
1119
1120   uint32_t savedMemorySize = aConsumer->mReportedMemoryConsumption;
1121   if (savedMemorySize == aCurrentMemoryConsumption)
1122     return;
1123
1124   // Exchange saved size with current one.
1125   aConsumer->mReportedMemoryConsumption = aCurrentMemoryConsumption;
1126
1127   bool usingDisk = !(aConsumer->mFlags & CacheMemoryConsumer::MEMORY_ONLY);
1128   bool overLimit = Pool(usingDisk).OnMemoryConsumptionChange(
1129     savedMemorySize, aCurrentMemoryConsumption);
1130
1131   if (!overLimit)
1132     return;
1133
1134   // It's likely the timer has already been set when we get here,
1135   // check outside the lock to save resources.
1136   if (mPurgeTimer)
1137     return;
1138
1139   // We don't know if this is called under the service lock or not,
1140   // hence rather dispatch.
1141   nsRefPtr<nsIEventTarget> cacheIOTarget = Thread();
1142   if (!cacheIOTarget)
1143     return;
1144
1145   // Dispatch as a priority task, we want to set the purge timer
1146   // ASAP to prevent vain redispatch of this event.
1147   nsCOMPtr<nsIRunnable> event =
1148     NS_NewRunnableMethod(this, &CacheStorageService::SchedulePurgeOverMemoryLimit);
1149   cacheIOTarget->Dispatch(event, nsIEventTarget::DISPATCH_NORMAL);
1150 }
1151
1152 bool
1153 CacheStorageService::MemoryPool::OnMemoryConsumptionChange(uint32_t aSavedMemorySize,
1154                                                            uint32_t aCurrentMemoryConsumption)
1155 {
1156   mMemorySize -= aSavedMemorySize;
1157   mMemorySize += aCurrentMemoryConsumption;
1158
1159   LOG(("  mMemorySize=%u (+%u,-%u)", uint32_t(mMemorySize), aCurrentMemoryConsumption, aSavedMemorySize));
1160
1161   // Bypass purging when memory has not grew up significantly
1162   if (aCurrentMemoryConsumption <= aSavedMemorySize)
1163     return false;
1164
1165   return mMemorySize > Limit();
1166 }
1167
1168 void
1169 CacheStorageService::SchedulePurgeOverMemoryLimit()
1170 {
1171   mozilla::MutexAutoLock lock(mLock);
1172
1173   if (mPurgeTimer)
1174     return;
1175
1176   mPurgeTimer = do_CreateInstance(NS_TIMER_CONTRACTID);
1177   if (mPurgeTimer)
1178     mPurgeTimer->InitWithCallback(this, 1000, nsITimer::TYPE_ONE_SHOT);
1179 }
1180
1181 NS_IMETHODIMP
1182 CacheStorageService::Notify(nsITimer* aTimer)
1183 {
1184   if (aTimer == mPurgeTimer) {
1185     mPurgeTimer = nullptr;
1186
1187     nsCOMPtr<nsIRunnable> event =
1188       NS_NewRunnableMethod(this, &CacheStorageService::PurgeOverMemoryLimit);
1189     Dispatch(event);
1190   }
1191
1192   return NS_OK;
1193 }
1194
1195 void
1196 CacheStorageService::PurgeOverMemoryLimit()
1197 {
1198   MOZ_ASSERT(IsOnManagementThread());
1199
1200   LOG(("CacheStorageService::PurgeOverMemoryLimit"));
1201
1202   Pool(true).PurgeOverMemoryLimit();
1203   Pool(false).PurgeOverMemoryLimit();
1204 }
1205
1206 void
1207 CacheStorageService::MemoryPool::PurgeOverMemoryLimit()
1208 {
1209 #ifdef PR_LOGGING
1210   TimeStamp start(TimeStamp::Now());
1211 #endif
1212
1213   uint32_t const memoryLimit = Limit();
1214   if (mMemorySize > memoryLimit) {
1215     LOG(("  memory data consumption over the limit, abandon expired entries"));
1216     PurgeExpired();
1217   }
1218
1219   bool frecencyNeedsSort = true;
1220
1221   // No longer makes sense since:
1222   // Memory entries are never purged partially, only as a whole when the memory
1223   // cache limit is overreached.
1224   // Disk entries throw the data away ASAP so that only metadata are kept.
1225   // TODO when this concept of two separate pools is found working, the code should
1226   // clean up.
1227 #if 0
1228   if (mMemorySize > memoryLimit) {
1229     LOG(("  memory data consumption over the limit, abandon disk backed data"));
1230     PurgeByFrecency(frecencyNeedsSort, CacheEntry::PURGE_DATA_ONLY_DISK_BACKED);
1231   }
1232
1233   if (mMemorySize > memoryLimit) {
1234     LOG(("  metadata consumtion over the limit, abandon disk backed entries"));
1235     PurgeByFrecency(frecencyNeedsSort, CacheEntry::PURGE_WHOLE_ONLY_DISK_BACKED);
1236   }
1237 #endif
1238
1239   if (mMemorySize > memoryLimit) {
1240     LOG(("  memory data consumption over the limit, abandon any entry"));
1241     PurgeByFrecency(frecencyNeedsSort, CacheEntry::PURGE_WHOLE);
1242   }
1243
1244   LOG(("  purging took %1.2fms", (TimeStamp::Now() - start).ToMilliseconds()));
1245 }
1246
1247 void
1248 CacheStorageService::MemoryPool::PurgeExpired()
1249 {
1250   MOZ_ASSERT(IsOnManagementThread());
1251
1252   mExpirationArray.Sort(ExpirationComparator());
1253   uint32_t now = NowInSeconds();
1254
1255   uint32_t const memoryLimit = Limit();
1256
1257   for (uint32_t i = 0; mMemorySize > memoryLimit && i < mExpirationArray.Length();) {
1258     if (CacheIOThread::YieldAndRerun())
1259       return;
1260
1261     nsRefPtr<CacheEntry> entry = mExpirationArray[i];
1262
1263     uint32_t expirationTime = entry->GetExpirationTime();
1264     if (expirationTime > 0 && expirationTime <= now) {
1265       LOG(("  dooming expired entry=%p, exptime=%u (now=%u)",
1266         entry.get(), entry->GetExpirationTime(), now));
1267
1268       entry->PurgeAndDoom();
1269       continue;
1270     }
1271
1272     // not purged, move to the next one
1273     ++i;
1274   }
1275 }
1276
1277 void
1278 CacheStorageService::MemoryPool::PurgeByFrecency(bool &aFrecencyNeedsSort, uint32_t aWhat)
1279 {
1280   MOZ_ASSERT(IsOnManagementThread());
1281
1282   if (aFrecencyNeedsSort) {
1283     mFrecencyArray.Sort(FrecencyComparator());
1284     aFrecencyNeedsSort = false;
1285   }
1286
1287   uint32_t const memoryLimit = Limit();
1288
1289   for (uint32_t i = 0; mMemorySize > memoryLimit && i < mFrecencyArray.Length();) {
1290     if (CacheIOThread::YieldAndRerun())
1291       return;
1292
1293     nsRefPtr<CacheEntry> entry = mFrecencyArray[i];
1294
1295     if (entry->Purge(aWhat)) {
1296       LOG(("  abandoned (%d), entry=%p, frecency=%1.10f",
1297         aWhat, entry.get(), entry->GetFrecency()));
1298       continue;
1299     }
1300
1301     // not purged, move to the next one
1302     ++i;
1303   }
1304 }
1305
1306 void
1307 CacheStorageService::MemoryPool::PurgeAll(uint32_t aWhat)
1308 {
1309   LOG(("CacheStorageService::MemoryPool::PurgeAll aWhat=%d", aWhat));
1310   MOZ_ASSERT(IsOnManagementThread());
1311
1312   for (uint32_t i = 0; i < mFrecencyArray.Length();) {
1313     if (CacheIOThread::YieldAndRerun())
1314       return;
1315
1316     nsRefPtr<CacheEntry> entry = mFrecencyArray[i];
1317
1318     if (entry->Purge(aWhat)) {
1319       LOG(("  abandoned entry=%p", entry.get()));
1320       continue;
1321     }
1322
1323     // not purged, move to the next one
1324     ++i;
1325   }
1326 }
1327
1328 // Methods exposed to and used by CacheStorage.
1329
1330 nsresult
1331 CacheStorageService::AddStorageEntry(CacheStorage const* aStorage,
1332                                      nsIURI* aURI,
1333                                      const nsACString & aIdExtension,
1334                                      bool aCreateIfNotExist,
1335                                      bool aReplace,
1336                                      CacheEntryHandle** aResult)
1337 {
1338   NS_ENSURE_FALSE(mShutdown, NS_ERROR_NOT_INITIALIZED);
1339
1340   NS_ENSURE_ARG(aStorage);
1341
1342   nsAutoCString contextKey;
1343   CacheFileUtils::AppendKeyPrefix(aStorage->LoadInfo(), contextKey);
1344
1345   return AddStorageEntry(contextKey, aURI, aIdExtension,
1346                          aStorage->WriteToDisk(), aCreateIfNotExist, aReplace,
1347                          aResult);
1348 }
1349
1350 nsresult
1351 CacheStorageService::AddStorageEntry(nsCSubstring const& aContextKey,
1352                                      nsIURI* aURI,
1353                                      const nsACString & aIdExtension,
1354                                      bool aWriteToDisk,
1355                                      bool aCreateIfNotExist,
1356                                      bool aReplace,
1357                                      CacheEntryHandle** aResult)
1358 {
1359   NS_ENSURE_ARG(aURI);
1360
1361   nsresult rv;
1362
1363   nsAutoCString entryKey;
1364   rv = CacheEntry::HashingKey(EmptyCString(), aIdExtension, aURI, entryKey);
1365   NS_ENSURE_SUCCESS(rv, rv);
1366
1367   LOG(("CacheStorageService::AddStorageEntry [entryKey=%s, contextKey=%s]",
1368     entryKey.get(), aContextKey.BeginReading()));
1369
1370   nsRefPtr<CacheEntry> entry;
1371   nsRefPtr<CacheEntryHandle> handle;
1372
1373   {
1374     mozilla::MutexAutoLock lock(mLock);
1375
1376     NS_ENSURE_FALSE(mShutdown, NS_ERROR_NOT_INITIALIZED);
1377
1378     // Ensure storage table
1379     CacheEntryTable* entries;
1380     if (!sGlobalEntryTables->Get(aContextKey, &entries)) {
1381       entries = new CacheEntryTable(CacheEntryTable::ALL_ENTRIES);
1382       sGlobalEntryTables->Put(aContextKey, entries);
1383       LOG(("  new storage entries table for context '%s'", aContextKey.BeginReading()));
1384     }
1385
1386     bool entryExists = entries->Get(entryKey, getter_AddRefs(entry));
1387
1388     if (entryExists && !aReplace) {
1389       // check whether the file is already doomed or we want to turn this entry
1390       // to a memory-only.
1391       if (MOZ_UNLIKELY(entry->IsFileDoomed())) {
1392         LOG(("  file already doomed, replacing the entry"));
1393         aReplace = true;
1394       } else if (MOZ_UNLIKELY(!aWriteToDisk) && MOZ_LIKELY(entry->IsUsingDisk())) {
1395         LOG(("  entry is persistnet but we want mem-only, replacing it"));
1396         aReplace = true;
1397       }
1398     }
1399
1400     // If truncate is demanded, delete and doom the current entry
1401     if (entryExists && aReplace) {
1402       entries->Remove(entryKey);
1403
1404       LOG(("  dooming entry %p for %s because of OPEN_TRUNCATE", entry.get(), entryKey.get()));
1405       // On purpose called under the lock to prevent races of doom and open on I/O thread
1406       // No need to remove from both memory-only and all-entries tables.  The new entry
1407       // will overwrite the shadow entry in its ctor.
1408       entry->DoomAlreadyRemoved();
1409
1410       entry = nullptr;
1411       entryExists = false;
1412     }
1413
1414     // Ensure entry for the particular URL, if not read/only
1415     if (!entryExists && (aCreateIfNotExist || aReplace)) {
1416       // Entry is not in the hashtable or has just been truncated...
1417       entry = new CacheEntry(aContextKey, aURI, aIdExtension, aWriteToDisk);
1418       entries->Put(entryKey, entry);
1419       LOG(("  new entry %p for %s", entry.get(), entryKey.get()));
1420     }
1421
1422     if (entry) {
1423       // Here, if this entry was not for a long time referenced by any consumer,
1424       // gets again first 'handles count' reference.
1425       handle = entry->NewHandle();
1426     }
1427   }
1428
1429   handle.forget(aResult);
1430   return NS_OK;
1431 }
1432
1433 nsresult
1434 CacheStorageService::CheckStorageEntry(CacheStorage const* aStorage,
1435                                        nsIURI* aURI,
1436                                        const nsACString & aIdExtension,
1437                                        bool* aResult)
1438 {
1439   nsresult rv;
1440
1441   nsAutoCString contextKey;
1442   CacheFileUtils::AppendKeyPrefix(aStorage->LoadInfo(), contextKey);
1443
1444   if (!aStorage->WriteToDisk()) {
1445     AppendMemoryStorageID(contextKey);
1446   }
1447
1448 #ifdef PR_LOGGING
1449   nsAutoCString uriSpec;
1450   aURI->GetAsciiSpec(uriSpec);
1451   LOG(("CacheStorageService::CheckStorageEntry [uri=%s, eid=%s, contextKey=%s]",
1452     uriSpec.get(), aIdExtension.BeginReading(), contextKey.get()));
1453 #endif
1454
1455   {
1456     mozilla::MutexAutoLock lock(mLock);
1457
1458     NS_ENSURE_FALSE(mShutdown, NS_ERROR_NOT_INITIALIZED);
1459
1460     nsAutoCString entryKey;
1461     rv = CacheEntry::HashingKey(EmptyCString(), aIdExtension, aURI, entryKey);
1462     NS_ENSURE_SUCCESS(rv, rv);
1463
1464     CacheEntryTable* entries;
1465     if ((*aResult = sGlobalEntryTables->Get(contextKey, &entries)) &&
1466         entries->GetWeak(entryKey, aResult)) {
1467       LOG(("  found in hash tables"));
1468       return NS_OK;
1469     }
1470   }
1471
1472   if (!aStorage->WriteToDisk()) {
1473     // Memory entry, nothing more to do.
1474     LOG(("  not found in hash tables"));
1475     return NS_OK;
1476   }
1477
1478   // Disk entry, not found in the hashtable, check the index.
1479   nsAutoCString fileKey;
1480   rv = CacheEntry::HashingKey(contextKey, aIdExtension, aURI, fileKey);
1481
1482   CacheIndex::EntryStatus status;
1483   rv = CacheIndex::HasEntry(fileKey, &status);
1484   if (NS_FAILED(rv) || status == CacheIndex::DO_NOT_KNOW) {
1485     LOG(("  index doesn't know, rv=0x%08x", rv));
1486     return NS_ERROR_NOT_AVAILABLE;
1487   }
1488
1489   *aResult = status == CacheIndex::EXISTS;
1490   LOG(("  %sfound in index", *aResult ? "" : "not "));
1491   return NS_OK;
1492 }
1493
1494 namespace { // anon
1495
1496 class CacheEntryDoomByKeyCallback : public CacheFileIOListener
1497                                   , public nsIRunnable
1498 {
1499 public:
1500   NS_DECL_THREADSAFE_ISUPPORTS
1501   NS_DECL_NSIRUNNABLE
1502
1503   explicit CacheEntryDoomByKeyCallback(nsICacheEntryDoomCallback* aCallback)
1504     : mCallback(aCallback) { }
1505
1506 private:
1507   virtual ~CacheEntryDoomByKeyCallback();
1508
1509   NS_IMETHOD OnFileOpened(CacheFileHandle *aHandle, nsresult aResult) { return NS_OK; }
1510   NS_IMETHOD OnDataWritten(CacheFileHandle *aHandle, const char *aBuf, nsresult aResult) { return NS_OK; }
1511   NS_IMETHOD OnDataRead(CacheFileHandle *aHandle, char *aBuf, nsresult aResult) { return NS_OK; }
1512   NS_IMETHOD OnFileDoomed(CacheFileHandle *aHandle, nsresult aResult);
1513   NS_IMETHOD OnEOFSet(CacheFileHandle *aHandle, nsresult aResult) { return NS_OK; }
1514   NS_IMETHOD OnFileRenamed(CacheFileHandle *aHandle, nsresult aResult) { return NS_OK; }
1515
1516   nsCOMPtr<nsICacheEntryDoomCallback> mCallback;
1517   nsresult mResult;
1518 };
1519
1520 CacheEntryDoomByKeyCallback::~CacheEntryDoomByKeyCallback()
1521 {
1522   if (mCallback)
1523     ProxyReleaseMainThread(mCallback);
1524 }
1525
1526 NS_IMETHODIMP CacheEntryDoomByKeyCallback::OnFileDoomed(CacheFileHandle *aHandle,
1527                                                         nsresult aResult)
1528 {
1529   if (!mCallback)
1530     return NS_OK;
1531
1532   mResult = aResult;
1533   if (NS_IsMainThread()) {
1534     Run();
1535   } else {
1536     NS_DispatchToMainThread(this);
1537   }
1538
1539   return NS_OK;
1540 }
1541
1542 NS_IMETHODIMP CacheEntryDoomByKeyCallback::Run()
1543 {
1544   mCallback->OnCacheEntryDoomed(mResult);
1545   return NS_OK;
1546 }
1547
1548 NS_IMPL_ISUPPORTS(CacheEntryDoomByKeyCallback, CacheFileIOListener, nsIRunnable);
1549
1550 } // anon
1551
1552 nsresult
1553 CacheStorageService::DoomStorageEntry(CacheStorage const* aStorage,
1554                                       nsIURI *aURI,
1555                                       const nsACString & aIdExtension,
1556                                       nsICacheEntryDoomCallback* aCallback)
1557 {
1558   LOG(("CacheStorageService::DoomStorageEntry"));
1559
1560   NS_ENSURE_ARG(aStorage);
1561   NS_ENSURE_ARG(aURI);
1562
1563   nsAutoCString contextKey;
1564   CacheFileUtils::AppendKeyPrefix(aStorage->LoadInfo(), contextKey);
1565
1566   nsAutoCString entryKey;
1567   nsresult rv = CacheEntry::HashingKey(EmptyCString(), aIdExtension, aURI, entryKey);
1568   NS_ENSURE_SUCCESS(rv, rv);
1569
1570   nsRefPtr<CacheEntry> entry;
1571   {
1572     mozilla::MutexAutoLock lock(mLock);
1573
1574     NS_ENSURE_FALSE(mShutdown, NS_ERROR_NOT_INITIALIZED);
1575
1576     CacheEntryTable* entries;
1577     if (sGlobalEntryTables->Get(contextKey, &entries)) {
1578       if (entries->Get(entryKey, getter_AddRefs(entry))) {
1579         if (aStorage->WriteToDisk() || !entry->IsUsingDisk()) {
1580           // When evicting from disk storage, purge
1581           // When evicting from memory storage and the entry is memory-only, purge
1582           LOG(("  purging entry %p for %s [storage use disk=%d, entry use disk=%d]",
1583             entry.get(), entryKey.get(), aStorage->WriteToDisk(), entry->IsUsingDisk()));
1584           entries->Remove(entryKey);
1585         }
1586         else {
1587           // Otherwise, leave it
1588           LOG(("  leaving entry %p for %s [storage use disk=%d, entry use disk=%d]",
1589             entry.get(), entryKey.get(), aStorage->WriteToDisk(), entry->IsUsingDisk()));
1590           entry = nullptr;
1591         }
1592       }
1593     }
1594   }
1595
1596   if (entry) {
1597     LOG(("  dooming entry %p for %s", entry.get(), entryKey.get()));
1598     return entry->AsyncDoom(aCallback);
1599   }
1600
1601   LOG(("  no entry loaded for %s", entryKey.get()));
1602
1603   if (aStorage->WriteToDisk()) {
1604     nsAutoCString contextKey;
1605     CacheFileUtils::AppendKeyPrefix(aStorage->LoadInfo(), contextKey);
1606
1607     rv = CacheEntry::HashingKey(contextKey, aIdExtension, aURI, entryKey);
1608     NS_ENSURE_SUCCESS(rv, rv);
1609
1610     LOG(("  dooming file only for %s", entryKey.get()));
1611
1612     nsRefPtr<CacheEntryDoomByKeyCallback> callback(
1613       new CacheEntryDoomByKeyCallback(aCallback));
1614     rv = CacheFileIOManager::DoomFileByKey(entryKey, callback);
1615     NS_ENSURE_SUCCESS(rv, rv);
1616
1617     return NS_OK;
1618   }
1619
1620   class Callback : public nsRunnable
1621   {
1622   public:
1623     Callback(nsICacheEntryDoomCallback* aCallback) : mCallback(aCallback) { }
1624     NS_IMETHODIMP Run()
1625     {
1626       mCallback->OnCacheEntryDoomed(NS_ERROR_NOT_AVAILABLE);
1627       return NS_OK;
1628     }
1629     nsCOMPtr<nsICacheEntryDoomCallback> mCallback;
1630   };
1631
1632   if (aCallback) {
1633     nsRefPtr<nsRunnable> callback = new Callback(aCallback);
1634     return NS_DispatchToMainThread(callback);
1635   }
1636
1637   return NS_OK;
1638 }
1639
1640 nsresult
1641 CacheStorageService::DoomStorageEntries(CacheStorage const* aStorage,
1642                                         nsICacheEntryDoomCallback* aCallback)
1643 {
1644   LOG(("CacheStorageService::DoomStorageEntries"));
1645
1646   NS_ENSURE_FALSE(mShutdown, NS_ERROR_NOT_INITIALIZED);
1647   NS_ENSURE_ARG(aStorage);
1648
1649   nsAutoCString contextKey;
1650   CacheFileUtils::AppendKeyPrefix(aStorage->LoadInfo(), contextKey);
1651
1652   mozilla::MutexAutoLock lock(mLock);
1653
1654   return DoomStorageEntries(contextKey, aStorage->LoadInfo(),
1655                             aStorage->WriteToDisk(), aCallback);
1656 }
1657
1658 nsresult
1659 CacheStorageService::DoomStorageEntries(nsCSubstring const& aContextKey,
1660                                         nsILoadContextInfo* aContext,
1661                                         bool aDiskStorage,
1662                                         nsICacheEntryDoomCallback* aCallback)
1663 {
1664   mLock.AssertCurrentThreadOwns();
1665
1666   NS_ENSURE_TRUE(!mShutdown, NS_ERROR_NOT_INITIALIZED);
1667
1668   nsAutoCString memoryStorageID(aContextKey);
1669   AppendMemoryStorageID(memoryStorageID);
1670
1671   if (aDiskStorage) {
1672     LOG(("  dooming disk+memory storage of %s", aContextKey.BeginReading()));
1673
1674     // Just remove all entries, CacheFileIOManager will take care of the files.
1675     sGlobalEntryTables->Remove(aContextKey);
1676     sGlobalEntryTables->Remove(memoryStorageID);
1677
1678     if (aContext && !aContext->IsPrivate()) {
1679       LOG(("  dooming disk entries"));
1680       CacheFileIOManager::EvictByContext(aContext);
1681     }
1682   } else {
1683     LOG(("  dooming memory-only storage of %s", aContextKey.BeginReading()));
1684
1685     class MemoryEntriesRemoval {
1686     public:
1687       static PLDHashOperator EvictEntry(const nsACString& aKey,
1688                                         CacheEntry* aEntry,
1689                                         void* aClosure)
1690       {
1691         CacheEntryTable* entries = static_cast<CacheEntryTable*>(aClosure);
1692         nsCString key(aKey);
1693         RemoveExactEntry(entries, key, aEntry, false);
1694         return PL_DHASH_NEXT;
1695       }
1696     };
1697
1698     // Remove the memory entries table from the global tables.
1699     // Since we store memory entries also in the disk entries table
1700     // we need to remove the memory entries from the disk table one
1701     // by one manually.
1702     nsAutoPtr<CacheEntryTable> memoryEntries;
1703     sGlobalEntryTables->RemoveAndForget(memoryStorageID, memoryEntries);
1704
1705     CacheEntryTable* entries;
1706     sGlobalEntryTables->Get(aContextKey, &entries);
1707     if (memoryEntries && entries)
1708       memoryEntries->EnumerateRead(&MemoryEntriesRemoval::EvictEntry, entries);
1709   }
1710
1711   // An artificial callback.  This is a candidate for removal tho.  In the new
1712   // cache any 'doom' or 'evict' function ensures that the entry or entries
1713   // being doomed is/are not accessible after the function returns.  So there is
1714   // probably no need for a callback - has no meaning.  But for compatibility
1715   // with the old cache that is still in the tree we keep the API similar to be
1716   // able to make tests as well as other consumers work for now.
1717   class Callback : public nsRunnable
1718   {
1719   public:
1720     explicit Callback(nsICacheEntryDoomCallback* aCallback) : mCallback(aCallback) { }
1721     NS_IMETHODIMP Run()
1722     {
1723       mCallback->OnCacheEntryDoomed(NS_OK);
1724       return NS_OK;
1725     }
1726     nsCOMPtr<nsICacheEntryDoomCallback> mCallback;
1727   };
1728
1729   if (aCallback) {
1730     nsRefPtr<nsRunnable> callback = new Callback(aCallback);
1731     return NS_DispatchToMainThread(callback);
1732   }
1733
1734   return NS_OK;
1735 }
1736
1737 nsresult
1738 CacheStorageService::WalkStorageEntries(CacheStorage const* aStorage,
1739                                         bool aVisitEntries,
1740                                         nsICacheStorageVisitor* aVisitor)
1741 {
1742   LOG(("CacheStorageService::WalkStorageEntries [cb=%p, visitentries=%d]", aVisitor, aVisitEntries));
1743   NS_ENSURE_FALSE(mShutdown, NS_ERROR_NOT_INITIALIZED);
1744
1745   NS_ENSURE_ARG(aStorage);
1746
1747   if (aStorage->WriteToDisk()) {
1748     nsRefPtr<WalkDiskCacheRunnable> event =
1749       new WalkDiskCacheRunnable(aStorage->LoadInfo(), aVisitEntries, aVisitor);
1750     return event->Walk();
1751   }
1752
1753   nsRefPtr<WalkMemoryCacheRunnable> event =
1754     new WalkMemoryCacheRunnable(aStorage->LoadInfo(), aVisitEntries, aVisitor);
1755   return event->Walk();
1756 }
1757
1758 void
1759 CacheStorageService::CacheFileDoomed(nsILoadContextInfo* aLoadContextInfo,
1760                                      const nsACString & aIdExtension,
1761                                      const nsACString & aURISpec)
1762 {
1763   nsAutoCString contextKey;
1764   CacheFileUtils::AppendKeyPrefix(aLoadContextInfo, contextKey);
1765
1766   nsAutoCString entryKey;
1767   CacheEntry::HashingKey(EmptyCString(), aIdExtension, aURISpec, entryKey);
1768
1769   mozilla::MutexAutoLock lock(mLock);
1770
1771   if (mShutdown)
1772     return;
1773
1774   CacheEntryTable* entries;
1775   if (!sGlobalEntryTables->Get(contextKey, &entries))
1776     return;
1777
1778   nsRefPtr<CacheEntry> entry;
1779   if (!entries->Get(entryKey, getter_AddRefs(entry)))
1780     return;
1781
1782   if (!entry->IsFileDoomed())
1783     return;
1784
1785   if (entry->IsReferenced())
1786     return;
1787
1788   // Need to remove under the lock to avoid possible race leading
1789   // to duplication of the entry per its key.
1790   RemoveExactEntry(entries, entryKey, entry, false);
1791   entry->DoomAlreadyRemoved();
1792 }
1793
1794 bool
1795 CacheStorageService::GetCacheEntryInfo(nsILoadContextInfo* aLoadContextInfo,
1796                                        const nsACString & aIdExtension,
1797                                        const nsACString & aURISpec,
1798                                        EntryInfoCallback *aCallback)
1799 {
1800   nsAutoCString contextKey;
1801   CacheFileUtils::AppendKeyPrefix(aLoadContextInfo, contextKey);
1802
1803   nsAutoCString entryKey;
1804   CacheEntry::HashingKey(EmptyCString(), aIdExtension, aURISpec, entryKey);
1805
1806   nsRefPtr<CacheEntry> entry;
1807   {
1808     mozilla::MutexAutoLock lock(mLock);
1809
1810     if (mShutdown) {
1811       return false;
1812     }
1813
1814     CacheEntryTable* entries;
1815     if (!sGlobalEntryTables->Get(contextKey, &entries)) {
1816       return false;
1817     }
1818
1819     if (!entries->Get(entryKey, getter_AddRefs(entry))) {
1820       return false;
1821     }
1822   }
1823
1824   GetCacheEntryInfo(entry, aCallback);
1825   return true;
1826 }
1827
1828 // static
1829 void
1830 CacheStorageService::GetCacheEntryInfo(CacheEntry* aEntry,
1831                                        EntryInfoCallback *aCallback)
1832 {
1833   nsIURI* uri = aEntry->GetURI();
1834   nsAutoCString uriSpec;
1835   if (uri) {
1836     uri->GetAsciiSpec(uriSpec);
1837   }
1838
1839   nsCString const enhanceId = aEntry->GetEnhanceID();
1840   uint32_t dataSize;
1841   if (NS_FAILED(aEntry->GetStorageDataSize(&dataSize))) {
1842     dataSize = 0;
1843   }
1844   int32_t fetchCount;
1845   if (NS_FAILED(aEntry->GetFetchCount(&fetchCount))) {
1846     fetchCount = 0;
1847   }
1848   uint32_t lastModified;
1849   if (NS_FAILED(aEntry->GetLastModified(&lastModified))) {
1850     lastModified = 0;
1851   }
1852   uint32_t expirationTime;
1853   if (NS_FAILED(aEntry->GetExpirationTime(&expirationTime))) {
1854     expirationTime = 0;
1855   }
1856
1857   aCallback->OnEntryInfo(uriSpec, enhanceId, dataSize,
1858                          fetchCount, lastModified, expirationTime);
1859 }
1860
1861 // Telementry collection
1862
1863 namespace { // anon
1864
1865 bool TelemetryEntryKey(CacheEntry const* entry, nsAutoCString& key)
1866 {
1867   nsAutoCString entryKey;
1868   nsresult rv = entry->HashingKey(entryKey);
1869   if (NS_FAILED(rv))
1870     return false;
1871
1872   if (entry->GetStorageID().IsEmpty()) {
1873     // Hopefully this will be const-copied, saves some memory
1874     key = entryKey;
1875   } else {
1876     key.Assign(entry->GetStorageID());
1877     key.Append(':');
1878     key.Append(entryKey);
1879   }
1880
1881   return true;
1882 }
1883
1884 PLDHashOperator PrunePurgeTimeStamps(
1885   const nsACString& aKey, TimeStamp& aTimeStamp, void* aClosure)
1886 {
1887   TimeStamp* now = static_cast<TimeStamp*>(aClosure);
1888   static TimeDuration const fifteenMinutes = TimeDuration::FromSeconds(900);
1889
1890   if (*now - aTimeStamp > fifteenMinutes) {
1891     // We are not interested in resurrection of entries after 15 minutes
1892     // of time.  This is also the limit for the telemetry.
1893     return PL_DHASH_REMOVE;
1894   }
1895
1896   return PL_DHASH_NEXT;
1897 }
1898
1899 } // anon
1900
1901 void
1902 CacheStorageService::TelemetryPrune(TimeStamp &now)
1903 {
1904   static TimeDuration const oneMinute = TimeDuration::FromSeconds(60);
1905   static TimeStamp dontPruneUntil = now + oneMinute;
1906   if (now < dontPruneUntil)
1907     return;
1908
1909   mPurgeTimeStamps.Enumerate(PrunePurgeTimeStamps, &now);
1910   dontPruneUntil = now + oneMinute;
1911 }
1912
1913 void
1914 CacheStorageService::TelemetryRecordEntryCreation(CacheEntry const* entry)
1915 {
1916   MOZ_ASSERT(CacheStorageService::IsOnManagementThread());
1917
1918   nsAutoCString key;
1919   if (!TelemetryEntryKey(entry, key))
1920     return;
1921
1922   TimeStamp now = TimeStamp::NowLoRes();
1923   TelemetryPrune(now);
1924
1925   // When an entry is craeted (registered actually) we check if there is
1926   // a timestamp marked when this very same cache entry has been removed
1927   // (deregistered) because of over-memory-limit purging.  If there is such
1928   // a timestamp found accumulate telemetry on how long the entry was away.
1929   TimeStamp timeStamp;
1930   if (!mPurgeTimeStamps.Get(key, &timeStamp))
1931     return;
1932
1933   mPurgeTimeStamps.Remove(key);
1934
1935   Telemetry::AccumulateTimeDelta(Telemetry::HTTP_CACHE_ENTRY_RELOAD_TIME,
1936                                  timeStamp, TimeStamp::NowLoRes());
1937 }
1938
1939 void
1940 CacheStorageService::TelemetryRecordEntryRemoval(CacheEntry const* entry)
1941 {
1942   MOZ_ASSERT(CacheStorageService::IsOnManagementThread());
1943
1944   // Doomed entries must not be considered, we are only interested in purged
1945   // entries.  Note that the mIsDoomed flag is always set before deregistration
1946   // happens.
1947   if (entry->IsDoomed())
1948     return;
1949
1950   nsAutoCString key;
1951   if (!TelemetryEntryKey(entry, key))
1952     return;
1953
1954   // When an entry is removed (deregistered actually) we put a timestamp for this
1955   // entry to the hashtable so that when the entry is created (registered) again
1956   // we know how long it was away.  Also accumulate number of AsyncOpen calls on
1957   // the entry, this tells us how efficiently the pool actually works.
1958
1959   TimeStamp now = TimeStamp::NowLoRes();
1960   TelemetryPrune(now);
1961   mPurgeTimeStamps.Put(key, now);
1962
1963   Telemetry::Accumulate(Telemetry::HTTP_CACHE_ENTRY_REUSE_COUNT, entry->UseCount());
1964   Telemetry::AccumulateTimeDelta(Telemetry::HTTP_CACHE_ENTRY_ALIVE_TIME,
1965                                  entry->LoadStart(), TimeStamp::NowLoRes());
1966 }
1967
1968 // nsIMemoryReporter
1969
1970 size_t
1971 CacheStorageService::SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const
1972 {
1973   CacheStorageService::Self()->Lock().AssertCurrentThreadOwns();
1974
1975   size_t n = 0;
1976   // The elemets are referenced by sGlobalEntryTables and are reported from there
1977   n += Pool(true).mFrecencyArray.SizeOfExcludingThis(mallocSizeOf);
1978   n += Pool(true).mExpirationArray.SizeOfExcludingThis(mallocSizeOf);
1979   n += Pool(false).mFrecencyArray.SizeOfExcludingThis(mallocSizeOf);
1980   n += Pool(false).mExpirationArray.SizeOfExcludingThis(mallocSizeOf);
1981   // Entries reported manually in CacheStorageService::CollectReports callback
1982   if (sGlobalEntryTables) {
1983     n += sGlobalEntryTables->SizeOfIncludingThis(nullptr, mallocSizeOf);
1984   }
1985
1986   return n;
1987 }
1988
1989 size_t
1990 CacheStorageService::SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const
1991 {
1992   return mallocSizeOf(this) + SizeOfExcludingThis(mallocSizeOf);
1993 }
1994
1995 namespace { // anon
1996
1997 class ReportStorageMemoryData
1998 {
1999 public:
2000   nsIMemoryReporterCallback *mHandleReport;
2001   nsISupports *mData;
2002 };
2003
2004 size_t CollectEntryMemory(nsACString const & aKey,
2005                           nsRefPtr<mozilla::net::CacheEntry> const & aEntry,
2006                           mozilla::MallocSizeOf mallocSizeOf,
2007                           void * aClosure)
2008 {
2009   CacheStorageService::Self()->Lock().AssertCurrentThreadOwns();
2010
2011   CacheEntryTable* aTable = static_cast<CacheEntryTable*>(aClosure);
2012
2013   size_t n = 0;
2014   n += aKey.SizeOfExcludingThisIfUnshared(mallocSizeOf);
2015
2016   // Bypass memory-only entries, those will be reported when iterating
2017   // the memory only table. Memory-only entries are stored in both ALL_ENTRIES
2018   // and MEMORY_ONLY hashtables.
2019   if (aTable->Type() == CacheEntryTable::MEMORY_ONLY || aEntry->IsUsingDisk())
2020     n += aEntry->SizeOfIncludingThis(mallocSizeOf);
2021
2022   return n;
2023 }
2024
2025 PLDHashOperator ReportStorageMemory(const nsACString& aKey,
2026                                     CacheEntryTable* aTable,
2027                                     void* aClosure)
2028 {
2029   CacheStorageService::Self()->Lock().AssertCurrentThreadOwns();
2030
2031   size_t size = aTable->SizeOfIncludingThis(&CollectEntryMemory,
2032                                             CacheStorageService::MallocSizeOf,
2033                                             aTable);
2034
2035   ReportStorageMemoryData& data =
2036     *static_cast<ReportStorageMemoryData*>(aClosure);
2037   // These key names are not privacy-sensitive.
2038   data.mHandleReport->Callback(
2039     EmptyCString(),
2040     nsPrintfCString("explicit/network/cache2/%s-storage(%s)",
2041       aTable->Type() == CacheEntryTable::MEMORY_ONLY ? "memory" : "disk",
2042       aKey.BeginReading()),
2043     nsIMemoryReporter::KIND_HEAP,
2044     nsIMemoryReporter::UNITS_BYTES,
2045     size,
2046     NS_LITERAL_CSTRING("Memory used by the cache storage."),
2047     data.mData);
2048
2049   return PL_DHASH_NEXT;
2050 }
2051
2052 } // anon
2053
2054 NS_IMETHODIMP
2055 CacheStorageService::CollectReports(nsIMemoryReporterCallback* aHandleReport,
2056                                     nsISupports* aData, bool aAnonymize)
2057 {
2058   nsresult rv;
2059
2060   rv = MOZ_COLLECT_REPORT(
2061     "explicit/network/cache2/io", KIND_HEAP, UNITS_BYTES,
2062     CacheFileIOManager::SizeOfIncludingThis(MallocSizeOf),
2063     "Memory used by the cache IO manager.");
2064   if (NS_WARN_IF(NS_FAILED(rv)))
2065     return rv;
2066
2067   rv = MOZ_COLLECT_REPORT(
2068     "explicit/network/cache2/index", KIND_HEAP, UNITS_BYTES,
2069     CacheIndex::SizeOfIncludingThis(MallocSizeOf),
2070     "Memory used by the cache index.");
2071   if (NS_WARN_IF(NS_FAILED(rv)))
2072     return rv;
2073
2074   MutexAutoLock lock(mLock);
2075
2076   // Report the service instance, this doesn't report entries, done lower
2077   rv = MOZ_COLLECT_REPORT(
2078     "explicit/network/cache2/service", KIND_HEAP, UNITS_BYTES,
2079     SizeOfIncludingThis(MallocSizeOf),
2080     "Memory used by the cache storage service.");
2081   if (NS_WARN_IF(NS_FAILED(rv)))
2082     return rv;
2083
2084   // Report all entries, each storage separately (by the context key)
2085   //
2086   // References are:
2087   // sGlobalEntryTables to N CacheEntryTable
2088   // CacheEntryTable to N CacheEntry
2089   // CacheEntry to 1 CacheFile
2090   // CacheFile to
2091   //   N CacheFileChunk (keeping the actual data)
2092   //   1 CacheFileMetadata (keeping http headers etc.)
2093   //   1 CacheFileOutputStream
2094   //   N CacheFileInputStream
2095   ReportStorageMemoryData data;
2096   data.mHandleReport = aHandleReport;
2097   data.mData = aData;
2098   sGlobalEntryTables->EnumerateRead(&ReportStorageMemory, &data);
2099
2100   return NS_OK;
2101 }
2102
2103 } // net
2104 } // mozilla