netwerk/cache2/CacheStorageService.cpp

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "CacheLog.h"
   8 #include "CacheStorageService.h"
   9 #include "CacheFileIOManager.h"
  10 #include "CacheObserver.h"
  11 #include "CacheIndex.h"
  12 #include "CacheIndexIterator.h"
  13 #include "CacheStorage.h"
  14 #include "CacheEntry.h"
  15 #include "CacheFileUtils.h"
  16
  17 #include "nsICacheStorageVisitor.h"
  18 #include "nsIObserverService.h"
  19 #include "nsIFile.h"
  20 #include "nsIURI.h"
  21 #include "nsINetworkPredictor.h"
  22 #include "nsCOMPtr.h"
  23 #include "nsContentUtils.h"
  24 #include "nsNetCID.h"
  25 #include "nsNetUtil.h"
  26 #include "nsServiceManagerUtils.h"
  27 #include "nsXULAppAPI.h"
  28 #include "mozilla/AtomicBitfields.h"
  29 #include "mozilla/TimeStamp.h"
  30 #include "mozilla/DebugOnly.h"
  31 #include "mozilla/Services.h"
  32 #include "mozilla/StoragePrincipalHelper.h"
  33 #include "mozilla/IntegerPrintfMacros.h"
  34 #include "mozilla/Telemetry.h"
  35 #include "mozilla/StaticPrefs_network.h"
  36
  37 namespace mozilla::net {
  38
  39 namespace {
  40
  41 void AppendMemoryStorageTag(nsAutoCString& key) {
  42   // Using DEL as the very last ascii-7 character we can use in the list of
  43   // attributes
  44   key.Append('\x7f');
  45   key.Append(',');
  46 }
  47
  48 }  // namespace
  49
  50 // Not defining as static or class member of CacheStorageService since
  51 // it would otherwise need to include CacheEntry.h and that then would
  52 // need to be exported to make nsNetModule.cpp compilable.
  53 using GlobalEntryTables = nsClassHashtable<nsCStringHashKey, CacheEntryTable>;
  54
  55 /**
  56  * Keeps tables of entries.  There is one entries table for each distinct load
  57  * context type.  The distinction is based on following load context info
  58  * states: <isPrivate|isAnon|inIsolatedMozBrowser> which builds a mapping
  59  * key.
  60  *
  61  * Thread-safe to access, protected by the service mutex.
  62  */
  63 static GlobalEntryTables* sGlobalEntryTables;
  64
  65 CacheMemoryConsumer::CacheMemoryConsumer(uint32_t aFlags) {
  66   StoreFlags(aFlags);
  67 }
  68
  69 void CacheMemoryConsumer::DoMemoryReport(uint32_t aCurrentSize) {
  70   if (!(LoadFlags() & DONT_REPORT) && CacheStorageService::Self()) {
  71     CacheStorageService::Self()->OnMemoryConsumptionChange(this, aCurrentSize);
  72   }
  73 }
  74
  75 CacheStorageService::MemoryPool::MemoryPool(EType aType) : mType(aType) {}
  76
  77 CacheStorageService::MemoryPool::~MemoryPool() {
  78   if (mMemorySize != 0) {
  79     NS_ERROR(
  80         "Network cache reported memory consumption is not at 0, probably "
  81         "leaking?");
  82   }
  83 }
  84
  85 uint32_t CacheStorageService::MemoryPool::Limit() const {
  86   uint32_t limit = 0;
  87
  88   switch (mType) {
  89     case DISK:
  90       limit = CacheObserver::MetadataMemoryLimit();
  91       break;
  92     case MEMORY:
  93       limit = CacheObserver::MemoryCacheCapacity();
  94       break;
  95     default:
  96       MOZ_CRASH("Bad pool type");
  97   }
  98
  99   static const uint32_t kMaxLimit = 0x3FFFFF;
 100   if (limit > kMaxLimit) {
 101     LOG(("  a memory limit (%u) is unexpectedly high, clipping to %u", limit,
 102          kMaxLimit));
 103     limit = kMaxLimit;
 104   }
 105
 106   return limit << 10;
 107 }
 108
 109 NS_IMPL_ISUPPORTS(CacheStorageService, nsICacheStorageService,
 110                   nsIMemoryReporter, nsITimerCallback, nsICacheTesting,
 111                   nsINamed)
 112
 113 CacheStorageService* CacheStorageService::sSelf = nullptr;
 114
 115 CacheStorageService::CacheStorageService() {
 116   CacheFileIOManager::Init();
 117
 118   MOZ_ASSERT(XRE_IsParentProcess());
 119   MOZ_ASSERT(!sSelf);
 120
 121   sSelf = this;
 122   sGlobalEntryTables = new GlobalEntryTables();
 123
 124   RegisterStrongMemoryReporter(this);
 125 }
 126
 127 CacheStorageService::~CacheStorageService() {
 128   LOG(("CacheStorageService::~CacheStorageService"));
 129   sSelf = nullptr;
 130 }
 131
 132 void CacheStorageService::Shutdown() {
 133   mozilla::MutexAutoLock lock(mLock);
 134
 135   if (mShutdown) return;
 136
 137   LOG(("CacheStorageService::Shutdown - start"));
 138
 139   mShutdown = true;
 140
 141   nsCOMPtr<nsIRunnable> event =
 142       NewRunnableMethod("net::CacheStorageService::ShutdownBackground", this,
 143                         &CacheStorageService::ShutdownBackground);
 144   Dispatch(event);
 145
 146 #ifdef NS_FREE_PERMANENT_DATA
 147   sGlobalEntryTables->Clear();
 148   delete sGlobalEntryTables;
 149 #endif
 150   sGlobalEntryTables = nullptr;
 151
 152   LOG(("CacheStorageService::Shutdown - done"));
 153 }
 154
 155 void CacheStorageService::ShutdownBackground() {
 156   LOG(("CacheStorageService::ShutdownBackground - start"));
 157
 158   MOZ_ASSERT(IsOnManagementThread());
 159
 160   {
 161     mozilla::MutexAutoLock lock(mLock);
 162
 163     // Cancel purge timer to avoid leaking.
 164     if (mPurgeTimer) {
 165       LOG(("  freeing the timer"));
 166       mPurgeTimer->Cancel();
 167     }
 168   }
 169
 170 #ifdef NS_FREE_PERMANENT_DATA
 171   Pool(false).mFrecencyArray.Clear();
 172   Pool(false).mExpirationArray.Clear();
 173   Pool(true).mFrecencyArray.Clear();
 174   Pool(true).mExpirationArray.Clear();
 175 #endif
 176
 177   LOG(("CacheStorageService::ShutdownBackground - done"));
 178 }
 179
 180 // Internal management methods
 181
 182 namespace {
 183
 184 // WalkCacheRunnable
 185 // Base class for particular storage entries visiting
 186 class WalkCacheRunnable : public Runnable,
 187                           public CacheStorageService::EntryInfoCallback {
 188  protected:
 189   WalkCacheRunnable(nsICacheStorageVisitor* aVisitor, bool aVisitEntries)
 190       : Runnable("net::WalkCacheRunnable"),
 191         mService(CacheStorageService::Self()),
 192         mCallback(aVisitor) {
 193     MOZ_ASSERT(NS_IsMainThread());
 194     StoreNotifyStorage(true);
 195     StoreVisitEntries(aVisitEntries);
 196   }
 197
 198   virtual ~WalkCacheRunnable() {
 199     if (mCallback) {
 200       ProxyReleaseMainThread("WalkCacheRunnable::mCallback", mCallback);
 201     }
 202   }
 203
 204   RefPtr<CacheStorageService> mService;
 205   nsCOMPtr<nsICacheStorageVisitor> mCallback;
 206
 207   uint64_t mSize{0};
 208
 209   // clang-format off
 210   MOZ_ATOMIC_BITFIELDS(mAtomicBitfields, 8, (
 211     (bool, NotifyStorage, 1),
 212     (bool, VisitEntries, 1)
 213   ))
 214   // clang-format on
 215
 216   Atomic<bool> mCancel{false};
 217 };
 218
 219 // WalkMemoryCacheRunnable
 220 // Responsible to visit memory storage and walk
 221 // all entries on it asynchronously.
 222 class WalkMemoryCacheRunnable : public WalkCacheRunnable {
 223  public:
 224   WalkMemoryCacheRunnable(nsILoadContextInfo* aLoadInfo, bool aVisitEntries,
 225                           nsICacheStorageVisitor* aVisitor)
 226       : WalkCacheRunnable(aVisitor, aVisitEntries) {
 227     CacheFileUtils::AppendKeyPrefix(aLoadInfo, mContextKey);
 228     MOZ_ASSERT(NS_IsMainThread());
 229   }
 230
 231   nsresult Walk() { return mService->Dispatch(this); }
 232
 233  private:
 234   NS_IMETHOD Run() override {
 235     if (CacheStorageService::IsOnManagementThread()) {
 236       LOG(("WalkMemoryCacheRunnable::Run - collecting [this=%p]", this));
 237       // First, walk, count and grab all entries from the storage
 238
 239       mozilla::MutexAutoLock lock(CacheStorageService::Self()->Lock());
 240
 241       if (!CacheStorageService::IsRunning()) return NS_ERROR_NOT_INITIALIZED;
 242
 243       for (const auto& entries : sGlobalEntryTables->Values()) {
 244         if (entries->Type() != CacheEntryTable::MEMORY_ONLY) {
 245           continue;
 246         }
 247
 248         for (CacheEntry* entry : entries->Values()) {
 249           MOZ_ASSERT(!entry->IsUsingDisk());
 250
 251           mSize += entry->GetMetadataMemoryConsumption();
 252
 253           int64_t size;
 254           if (NS_SUCCEEDED(entry->GetDataSize(&size))) {
 255             mSize += size;
 256           }
 257           mEntryArray.AppendElement(entry);
 258         }
 259       }
 260
 261       // Next, we dispatch to the main thread
 262     } else if (NS_IsMainThread()) {
 263       LOG(("WalkMemoryCacheRunnable::Run - notifying [this=%p]", this));
 264
 265       if (LoadNotifyStorage()) {
 266         LOG(("  storage"));
 267
 268         uint64_t capacity = CacheObserver::MemoryCacheCapacity();
 269         capacity <<= 10;  // kilobytes to bytes
 270
 271         // Second, notify overall storage info
 272         mCallback->OnCacheStorageInfo(mEntryArray.Length(), mSize, capacity,
 273                                       nullptr);
 274         if (!LoadVisitEntries()) return NS_OK;  // done
 275
 276         StoreNotifyStorage(false);
 277
 278       } else {
 279         LOG(("  entry [left=%zu, canceled=%d]", mEntryArray.Length(),
 280              (bool)mCancel));
 281
 282         // Third, notify each entry until depleted or canceled
 283         if (!mEntryArray.Length() || mCancel) {
 284           mCallback->OnCacheEntryVisitCompleted();
 285           return NS_OK;  // done
 286         }
 287
 288         // Grab the next entry
 289         RefPtr<CacheEntry> entry = mEntryArray[0];
 290         mEntryArray.RemoveElementAt(0);
 291
 292         // Invokes this->OnEntryInfo, that calls the callback with all
 293         // information of the entry.
 294         CacheStorageService::GetCacheEntryInfo(entry, this);
 295       }
 296     } else {
 297       MOZ_CRASH("Bad thread");
 298       return NS_ERROR_FAILURE;
 299     }
 300
 301     NS_DispatchToMainThread(this);
 302     return NS_OK;
 303   }
 304
 305   virtual ~WalkMemoryCacheRunnable() {
 306     if (mCallback) {
 307       ProxyReleaseMainThread("WalkMemoryCacheRunnable::mCallback", mCallback);
 308     }
 309   }
 310
 311   virtual void OnEntryInfo(const nsACString& aURISpec,
 312                            const nsACString& aIdEnhance, int64_t aDataSize,
 313                            int64_t aAltDataSize, uint32_t aFetchCount,
 314                            uint32_t aLastModifiedTime, uint32_t aExpirationTime,
 315                            bool aPinned, nsILoadContextInfo* aInfo) override {
 316     nsresult rv;
 317
 318     nsCOMPtr<nsIURI> uri;
 319     rv = NS_NewURI(getter_AddRefs(uri), aURISpec);
 320     if (NS_FAILED(rv)) {
 321       return;
 322     }
 323
 324     rv = mCallback->OnCacheEntryInfo(uri, aIdEnhance, aDataSize, aAltDataSize,
 325                                      aFetchCount, aLastModifiedTime,
 326                                      aExpirationTime, aPinned, aInfo);
 327     if (NS_FAILED(rv)) {
 328       LOG(("  callback failed, canceling the walk"));
 329       mCancel = true;
 330     }
 331   }
 332
 333  private:
 334   nsCString mContextKey;
 335   nsTArray<RefPtr<CacheEntry>> mEntryArray;
 336 };
 337
 338 // WalkDiskCacheRunnable
 339 // Using the cache index information to get the list of files per context.
 340 class WalkDiskCacheRunnable : public WalkCacheRunnable {
 341  public:
 342   WalkDiskCacheRunnable(nsILoadContextInfo* aLoadInfo, bool aVisitEntries,
 343                         nsICacheStorageVisitor* aVisitor)
 344       : WalkCacheRunnable(aVisitor, aVisitEntries),
 345         mLoadInfo(aLoadInfo),
 346         mPass(COLLECT_STATS),
 347         mCount(0) {}
 348
 349   nsresult Walk() {
 350     // TODO, bug 998693
 351     // Initial index build should be forced here so that about:cache soon
 352     // after startup gives some meaningfull results.
 353
 354     // Dispatch to the INDEX level in hope that very recent cache entries
 355     // information gets to the index list before we grab the index iterator
 356     // for the first time.  This tries to avoid miss of entries that has
 357     // been created right before the visit is required.
 358     RefPtr<CacheIOThread> thread = CacheFileIOManager::IOThread();
 359     NS_ENSURE_TRUE(thread, NS_ERROR_NOT_INITIALIZED);
 360
 361     return thread->Dispatch(this, CacheIOThread::INDEX);
 362   }
 363
 364  private:
 365   // Invokes OnCacheEntryInfo callback for each single found entry.
 366   // There is one instance of this class per one entry.
 367   class OnCacheEntryInfoRunnable : public Runnable {
 368    public:
 369     explicit OnCacheEntryInfoRunnable(WalkDiskCacheRunnable* aWalker)
 370         : Runnable("net::WalkDiskCacheRunnable::OnCacheEntryInfoRunnable"),
 371           mWalker(aWalker) {}
 372
 373     NS_IMETHOD Run() override {
 374       MOZ_ASSERT(NS_IsMainThread());
 375
 376       nsresult rv;
 377
 378       nsCOMPtr<nsIURI> uri;
 379       rv = NS_NewURI(getter_AddRefs(uri), mURISpec);
 380       if (NS_FAILED(rv)) {
 381         return NS_OK;
 382       }
 383
 384       rv = mWalker->mCallback->OnCacheEntryInfo(
 385           uri, mIdEnhance, mDataSize, mAltDataSize, mFetchCount,
 386           mLastModifiedTime, mExpirationTime, mPinned, mInfo);
 387       if (NS_FAILED(rv)) {
 388         mWalker->mCancel = true;
 389       }
 390
 391       return NS_OK;
 392     }
 393
 394     RefPtr<WalkDiskCacheRunnable> mWalker;
 395
 396     nsCString mURISpec;
 397     nsCString mIdEnhance;
 398     int64_t mDataSize{0};
 399     int64_t mAltDataSize{0};
 400     uint32_t mFetchCount{0};
 401     uint32_t mLastModifiedTime{0};
 402     uint32_t mExpirationTime{0};
 403     bool mPinned{false};
 404     nsCOMPtr<nsILoadContextInfo> mInfo;
 405   };
 406
 407   NS_IMETHOD Run() override {
 408     // The main loop
 409     nsresult rv;
 410
 411     if (CacheStorageService::IsOnManagementThread()) {
 412       switch (mPass) {
 413         case COLLECT_STATS:
 414           // Get quickly the cache stats.
 415           uint32_t size;
 416           rv = CacheIndex::GetCacheStats(mLoadInfo, &size, &mCount);
 417           if (NS_FAILED(rv)) {
 418             if (LoadVisitEntries()) {
 419               // both onStorageInfo and onCompleted are expected
 420               NS_DispatchToMainThread(this);
 421             }
 422             return NS_DispatchToMainThread(this);
 423           }
 424
 425           mSize = static_cast<uint64_t>(size) << 10;
 426
 427           // Invoke onCacheStorageInfo with valid information.
 428           NS_DispatchToMainThread(this);
 429
 430           if (!LoadVisitEntries()) {
 431             return NS_OK;  // done
 432           }
 433
 434           mPass = ITERATE_METADATA;
 435           [[fallthrough]];
 436
 437         case ITERATE_METADATA:
 438           // Now grab the context iterator.
 439           if (!mIter) {
 440             rv =
 441                 CacheIndex::GetIterator(mLoadInfo, true, getter_AddRefs(mIter));
 442             if (NS_FAILED(rv)) {
 443               // Invoke onCacheEntryVisitCompleted now
 444               return NS_DispatchToMainThread(this);
 445             }
 446           }
 447
 448           while (!mCancel && !CacheObserver::ShuttingDown()) {
 449             if (CacheIOThread::YieldAndRerun()) return NS_OK;
 450
 451             SHA1Sum::Hash hash;
 452             rv = mIter->GetNextHash(&hash);
 453             if (NS_FAILED(rv)) break;  // done (or error?)
 454
 455             // This synchronously invokes OnEntryInfo on this class where we
 456             // redispatch to the main thread for the consumer callback.
 457             CacheFileIOManager::GetEntryInfo(&hash, this);
 458           }
 459
 460           // Invoke onCacheEntryVisitCompleted on the main thread
 461           NS_DispatchToMainThread(this);
 462       }
 463     } else if (NS_IsMainThread()) {
 464       if (LoadNotifyStorage()) {
 465         nsCOMPtr<nsIFile> dir;
 466         CacheFileIOManager::GetCacheDirectory(getter_AddRefs(dir));
 467         uint64_t capacity = CacheObserver::DiskCacheCapacity();
 468         capacity <<= 10;  // kilobytes to bytes
 469         mCallback->OnCacheStorageInfo(mCount, mSize, capacity, dir);
 470         StoreNotifyStorage(false);
 471       } else {
 472         mCallback->OnCacheEntryVisitCompleted();
 473       }
 474     } else {
 475       MOZ_CRASH("Bad thread");
 476       return NS_ERROR_FAILURE;
 477     }
 478
 479     return NS_OK;
 480   }
 481
 482   virtual void OnEntryInfo(const nsACString& aURISpec,
 483                            const nsACString& aIdEnhance, int64_t aDataSize,
 484                            int64_t aAltDataSize, uint32_t aFetchCount,
 485                            uint32_t aLastModifiedTime, uint32_t aExpirationTime,
 486                            bool aPinned, nsILoadContextInfo* aInfo) override {
 487     // Called directly from CacheFileIOManager::GetEntryInfo.
 488
 489     // Invoke onCacheEntryInfo on the main thread for this entry.
 490     RefPtr<OnCacheEntryInfoRunnable> info = new OnCacheEntryInfoRunnable(this);
 491     info->mURISpec = aURISpec;
 492     info->mIdEnhance = aIdEnhance;
 493     info->mDataSize = aDataSize;
 494     info->mAltDataSize = aAltDataSize;
 495     info->mFetchCount = aFetchCount;
 496     info->mLastModifiedTime = aLastModifiedTime;
 497     info->mExpirationTime = aExpirationTime;
 498     info->mPinned = aPinned;
 499     info->mInfo = aInfo;
 500
 501     NS_DispatchToMainThread(info);
 502   }
 503
 504   RefPtr<nsILoadContextInfo> mLoadInfo;
 505   enum {
 506     // First, we collect stats for the load context.
 507     COLLECT_STATS,
 508
 509     // Second, if demanded, we iterate over the entries gethered
 510     // from the iterator and call CacheFileIOManager::GetEntryInfo
 511     // for each found entry.
 512     ITERATE_METADATA,
 513   } mPass;
 514
 515   RefPtr<CacheIndexIterator> mIter;
 516   uint32_t mCount;
 517 };
 518
 519 }  // namespace
 520
 521 void CacheStorageService::DropPrivateBrowsingEntries() {
 522   mozilla::MutexAutoLock lock(mLock);
 523
 524   if (mShutdown) return;
 525
 526   nsTArray<nsCString> keys;
 527   for (const nsACString& key : sGlobalEntryTables->Keys()) {
 528     nsCOMPtr<nsILoadContextInfo> info = CacheFileUtils::ParseKey(key);
 529     if (info && info->IsPrivate()) {
 530       keys.AppendElement(key);
 531     }
 532   }
 533
 534   for (uint32_t i = 0; i < keys.Length(); ++i) {
 535     DoomStorageEntries(keys[i], nullptr, true, false, nullptr);
 536   }
 537 }
 538
 539 // Helper methods
 540
 541 // static
 542 bool CacheStorageService::IsOnManagementThread() {
 543   RefPtr<CacheStorageService> service = Self();
 544   if (!service) return false;
 545
 546   nsCOMPtr<nsIEventTarget> target = service->Thread();
 547   if (!target) return false;
 548
 549   bool currentThread;
 550   nsresult rv = target->IsOnCurrentThread(&currentThread);
 551   return NS_SUCCEEDED(rv) && currentThread;
 552 }
 553
 554 already_AddRefed<nsIEventTarget> CacheStorageService::Thread() const {
 555   return CacheFileIOManager::IOTarget();
 556 }
 557
 558 nsresult CacheStorageService::Dispatch(nsIRunnable* aEvent) {
 559   RefPtr<CacheIOThread> cacheIOThread = CacheFileIOManager::IOThread();
 560   if (!cacheIOThread) return NS_ERROR_NOT_AVAILABLE;
 561
 562   return cacheIOThread->Dispatch(aEvent, CacheIOThread::MANAGEMENT);
 563 }
 564
 565 namespace CacheStorageEvictHelper {
 566
 567 nsresult ClearStorage(bool const aPrivate, bool const aAnonymous,
 568                       OriginAttributes& aOa) {
 569   nsresult rv;
 570
 571   aOa.SyncAttributesWithPrivateBrowsing(aPrivate);
 572   RefPtr<LoadContextInfo> info = GetLoadContextInfo(aAnonymous, aOa);
 573
 574   nsCOMPtr<nsICacheStorage> storage;
 575   RefPtr<CacheStorageService> service = CacheStorageService::Self();
 576   NS_ENSURE_TRUE(service, NS_ERROR_FAILURE);
 577
 578   // Clear disk storage
 579   rv = service->DiskCacheStorage(info, getter_AddRefs(storage));
 580   NS_ENSURE_SUCCESS(rv, rv);
 581   rv = storage->AsyncEvictStorage(nullptr);
 582   NS_ENSURE_SUCCESS(rv, rv);
 583
 584   // Clear memory storage
 585   rv = service->MemoryCacheStorage(info, getter_AddRefs(storage));
 586   NS_ENSURE_SUCCESS(rv, rv);
 587   rv = storage->AsyncEvictStorage(nullptr);
 588   NS_ENSURE_SUCCESS(rv, rv);
 589
 590   return NS_OK;
 591 }
 592
 593 nsresult Run(OriginAttributes& aOa) {
 594   nsresult rv;
 595
 596   // Clear all [private X anonymous] combinations
 597   rv = ClearStorage(false, false, aOa);
 598   NS_ENSURE_SUCCESS(rv, rv);
 599   rv = ClearStorage(false, true, aOa);
 600   NS_ENSURE_SUCCESS(rv, rv);
 601   rv = ClearStorage(true, false, aOa);
 602   NS_ENSURE_SUCCESS(rv, rv);
 603   rv = ClearStorage(true, true, aOa);
 604   NS_ENSURE_SUCCESS(rv, rv);
 605
 606   return NS_OK;
 607 }
 608
 609 }  // namespace CacheStorageEvictHelper
 610
 611 // nsICacheStorageService
 612
 613 NS_IMETHODIMP CacheStorageService::MemoryCacheStorage(
 614     nsILoadContextInfo* aLoadContextInfo, nsICacheStorage** _retval) {
 615   NS_ENSURE_ARG(_retval);
 616
 617   nsCOMPtr<nsICacheStorage> storage =
 618       new CacheStorage(aLoadContextInfo, false, false, false);
 619   storage.forget(_retval);
 620   return NS_OK;
 621 }
 622
 623 NS_IMETHODIMP CacheStorageService::DiskCacheStorage(
 624     nsILoadContextInfo* aLoadContextInfo, nsICacheStorage** _retval) {
 625   NS_ENSURE_ARG(_retval);
 626
 627   // TODO save some heap granularity - cache commonly used storages.
 628
 629   // When disk cache is disabled, still provide a storage, but just keep stuff
 630   // in memory.
 631   bool useDisk = CacheObserver::UseDiskCache();
 632
 633   nsCOMPtr<nsICacheStorage> storage = new CacheStorage(
 634       aLoadContextInfo, useDisk, false /* size limit */, false /* don't pin */);
 635   storage.forget(_retval);
 636   return NS_OK;
 637 }
 638
 639 NS_IMETHODIMP CacheStorageService::PinningCacheStorage(
 640     nsILoadContextInfo* aLoadContextInfo, nsICacheStorage** _retval) {
 641   NS_ENSURE_ARG(aLoadContextInfo);
 642   NS_ENSURE_ARG(_retval);
 643
 644   // When disk cache is disabled don't pretend we cache.
 645   if (!CacheObserver::UseDiskCache()) {
 646     return NS_ERROR_NOT_AVAILABLE;
 647   }
 648
 649   nsCOMPtr<nsICacheStorage> storage =
 650       new CacheStorage(aLoadContextInfo, true /* use disk */,
 651                        true /* ignore size checks */, true /* pin */);
 652   storage.forget(_retval);
 653   return NS_OK;
 654 }
 655
 656 NS_IMETHODIMP CacheStorageService::Clear() {
 657   nsresult rv;
 658
 659   // Tell the index to block notification to AsyncGetDiskConsumption.
 660   // Will be allowed again from CacheFileContextEvictor::EvictEntries()
 661   // when all the context have been removed from disk.
 662   CacheIndex::OnAsyncEviction(true);
 663
 664   mozilla::MutexAutoLock lock(mLock);
 665
 666   {
 667     mozilla::MutexAutoLock forcedValidEntriesLock(mForcedValidEntriesLock);
 668     mForcedValidEntries.Clear();
 669   }
 670
 671   NS_ENSURE_TRUE(!mShutdown, NS_ERROR_NOT_INITIALIZED);
 672
 673   const auto keys = ToTArray<nsTArray<nsCString>>(sGlobalEntryTables->Keys());
 674   for (const auto& key : keys) {
 675     DoomStorageEntries(key, nullptr, true, false, nullptr);
 676   }
 677
 678   // Passing null as a load info means to evict all contexts.
 679   // EvictByContext() respects the entry pinning.  EvictAll() does not.
 680   rv = CacheFileIOManager::EvictByContext(nullptr, false, u""_ns);
 681   NS_ENSURE_SUCCESS(rv, rv);
 682
 683   return NS_OK;
 684 }
 685
 686 NS_IMETHODIMP CacheStorageService::ClearOrigin(nsIPrincipal* aPrincipal) {
 687   nsresult rv;
 688
 689   if (NS_WARN_IF(!aPrincipal)) {
 690     return NS_ERROR_FAILURE;
 691   }
 692
 693   nsAutoString origin;
 694   rv = nsContentUtils::GetWebExposedOriginSerialization(aPrincipal, origin);
 695   NS_ENSURE_SUCCESS(rv, rv);
 696
 697   rv = ClearOriginInternal(origin, aPrincipal->OriginAttributesRef(), true);
 698   NS_ENSURE_SUCCESS(rv, rv);
 699
 700   rv = ClearOriginInternal(origin, aPrincipal->OriginAttributesRef(), false);
 701   NS_ENSURE_SUCCESS(rv, rv);
 702
 703   return NS_OK;
 704 }
 705
 706 NS_IMETHODIMP CacheStorageService::ClearOriginAttributes(
 707     const nsAString& aOriginAttributes) {
 708   nsresult rv;
 709
 710   if (NS_WARN_IF(aOriginAttributes.IsEmpty())) {
 711     return NS_ERROR_FAILURE;
 712   }
 713
 714   OriginAttributes oa;
 715   if (!oa.Init(aOriginAttributes)) {
 716     NS_ERROR("Could not parse the argument for OriginAttributes");
 717     return NS_ERROR_FAILURE;
 718   }
 719
 720   rv = CacheStorageEvictHelper::Run(oa);
 721   NS_ENSURE_SUCCESS(rv, rv);
 722
 723   return NS_OK;
 724 }
 725
 726 static bool RemoveExactEntry(CacheEntryTable* aEntries, nsACString const& aKey,
 727                              CacheEntry* aEntry, bool aOverwrite) {
 728   RefPtr<CacheEntry> existingEntry;
 729   if (!aEntries->Get(aKey, getter_AddRefs(existingEntry))) {
 730     LOG(("RemoveExactEntry [entry=%p already gone]", aEntry));
 731     return false;  // Already removed...
 732   }
 733
 734   if (!aOverwrite && existingEntry != aEntry) {
 735     LOG(("RemoveExactEntry [entry=%p already replaced]", aEntry));
 736     return false;  // Already replaced...
 737   }
 738
 739   LOG(("RemoveExactEntry [entry=%p removed]", aEntry));
 740   aEntries->Remove(aKey);
 741   return true;
 742 }
 743
 744 NS_IMETHODIMP CacheStorageService::ClearBaseDomain(
 745     const nsAString& aBaseDomain) {
 746   if (sGlobalEntryTables) {
 747     mozilla::MutexAutoLock lock(mLock);
 748
 749     if (mShutdown) return NS_ERROR_NOT_AVAILABLE;
 750
 751     nsCString cBaseDomain = NS_ConvertUTF16toUTF8(aBaseDomain);
 752
 753     nsTArray<nsCString> keys;
 754     for (const auto& globalEntry : *sGlobalEntryTables) {
 755       // Match by partitionKey base domain. This should cover most cache entries
 756       // because we statically partition the cache. Most first party cache
 757       // entries will also have a partitionKey set where the partitionKey base
 758       // domain will match the entry URI base domain.
 759       const nsACString& key = globalEntry.GetKey();
 760       nsCOMPtr<nsILoadContextInfo> info =
 761           CacheFileUtils::ParseKey(globalEntry.GetKey());
 762
 763       if (info &&
 764           StoragePrincipalHelper::PartitionKeyHasBaseDomain(
 765               info->OriginAttributesPtr()->mPartitionKey, aBaseDomain)) {
 766         keys.AppendElement(key);
 767         continue;
 768       }
 769
 770       // If we didn't get a partitionKey match, try to match by entry URI. This
 771       // requires us to iterate over all entries.
 772       CacheEntryTable* table = globalEntry.GetWeak();
 773       MOZ_ASSERT(table);
 774
 775       nsTArray<RefPtr<CacheEntry>> entriesToDelete;
 776
 777       for (CacheEntry* entry : table->Values()) {
 778         nsCOMPtr<nsIURI> uri;
 779         nsresult rv = NS_NewURI(getter_AddRefs(uri), entry->GetURI());
 780         if (NS_WARN_IF(NS_FAILED(rv))) {
 781           continue;
 782         }
 783
 784         nsAutoCString host;
 785         rv = uri->GetHost(host);
 786         // Some entries may not have valid hosts. We can skip them.
 787         if (NS_FAILED(rv) || host.IsEmpty()) {
 788           continue;
 789         }
 790
 791         bool hasRootDomain = false;
 792         rv = HasRootDomain(host, cBaseDomain, &hasRootDomain);
 793         if (NS_WARN_IF(NS_FAILED(rv))) {
 794           continue;
 795         }
 796         if (hasRootDomain) {
 797           entriesToDelete.AppendElement(entry);
 798         }
 799       }
 800
 801       // Clear individual matched entries.
 802       for (RefPtr<CacheEntry>& entry : entriesToDelete) {
 803         nsAutoCString entryKey;
 804         nsresult rv = entry->HashingKey(entryKey);
 805         if (NS_FAILED(rv)) {
 806           NS_ERROR("aEntry->HashingKey() failed?");
 807           return rv;
 808         }
 809
 810         RemoveExactEntry(table, entryKey, entry, false /* don't overwrite */);
 811       }
 812     }
 813
 814     // Clear matched keys.
 815     for (uint32_t i = 0; i < keys.Length(); ++i) {
 816       DoomStorageEntries(keys[i], nullptr, true, false, nullptr);
 817     }
 818   }
 819
 820   return CacheFileIOManager::EvictByContext(nullptr, false /* pinned */, u""_ns,
 821                                             aBaseDomain);
 822 }
 823
 824 nsresult CacheStorageService::ClearOriginInternal(
 825     const nsAString& aOrigin, const OriginAttributes& aOriginAttributes,
 826     bool aAnonymous) {
 827   nsresult rv;
 828
 829   RefPtr<LoadContextInfo> info =
 830       GetLoadContextInfo(aAnonymous, aOriginAttributes);
 831   if (NS_WARN_IF(!info)) {
 832     return NS_ERROR_FAILURE;
 833   }
 834
 835   mozilla::MutexAutoLock lock(mLock);
 836
 837   if (sGlobalEntryTables) {
 838     for (const auto& globalEntry : *sGlobalEntryTables) {
 839       bool matches = false;
 840       rv = CacheFileUtils::KeyMatchesLoadContextInfo(globalEntry.GetKey(), info,
 841                                                      &matches);
 842       NS_ENSURE_SUCCESS(rv, rv);
 843       if (!matches) {
 844         continue;
 845       }
 846
 847       CacheEntryTable* table = globalEntry.GetWeak();
 848       MOZ_ASSERT(table);
 849
 850       nsTArray<RefPtr<CacheEntry>> entriesToDelete;
 851
 852       for (CacheEntry* entry : table->Values()) {
 853         nsCOMPtr<nsIURI> uri;
 854         rv = NS_NewURI(getter_AddRefs(uri), entry->GetURI());
 855         NS_ENSURE_SUCCESS(rv, rv);
 856
 857         nsAutoString origin;
 858         rv = nsContentUtils::GetWebExposedOriginSerialization(uri, origin);
 859         NS_ENSURE_SUCCESS(rv, rv);
 860
 861         if (origin != aOrigin) {
 862           continue;
 863         }
 864
 865         entriesToDelete.AppendElement(entry);
 866       }
 867
 868       for (RefPtr<CacheEntry>& entry : entriesToDelete) {
 869         nsAutoCString entryKey;
 870         rv = entry->HashingKey(entryKey);
 871         if (NS_FAILED(rv)) {
 872           NS_ERROR("aEntry->HashingKey() failed?");
 873           return rv;
 874         }
 875
 876         RemoveExactEntry(table, entryKey, entry, false /* don't overwrite */);
 877       }
 878     }
 879   }
 880
 881   rv = CacheFileIOManager::EvictByContext(info, false /* pinned */, aOrigin);
 882   NS_ENSURE_SUCCESS(rv, rv);
 883
 884   return NS_OK;
 885 }
 886
 887 NS_IMETHODIMP CacheStorageService::PurgeFromMemory(uint32_t aWhat) {
 888   uint32_t what;
 889
 890   switch (aWhat) {
 891     case PURGE_DISK_DATA_ONLY:
 892       what = CacheEntry::PURGE_DATA_ONLY_DISK_BACKED;
 893       break;
 894
 895     case PURGE_DISK_ALL:
 896       what = CacheEntry::PURGE_WHOLE_ONLY_DISK_BACKED;
 897       break;
 898
 899     case PURGE_EVERYTHING:
 900       what = CacheEntry::PURGE_WHOLE;
 901       break;
 902
 903     default:
 904       return NS_ERROR_INVALID_ARG;
 905   }
 906
 907   nsCOMPtr<nsIRunnable> event = new PurgeFromMemoryRunnable(this, what);
 908
 909   return Dispatch(event);
 910 }
 911
 912 NS_IMETHODIMP CacheStorageService::PurgeFromMemoryRunnable::Run() {
 913   if (NS_IsMainThread()) {
 914     nsCOMPtr<nsIObserverService> observerService =
 915         mozilla::services::GetObserverService();
 916     if (observerService) {
 917       observerService->NotifyObservers(
 918           nullptr, "cacheservice:purge-memory-pools", nullptr);
 919     }
 920
 921     return NS_OK;
 922   }
 923
 924   if (mService) {
 925     // TODO not all flags apply to both pools
 926     mService->Pool(true).PurgeAll(mWhat);
 927     mService->Pool(false).PurgeAll(mWhat);
 928     mService = nullptr;
 929   }
 930
 931   NS_DispatchToMainThread(this);
 932   return NS_OK;
 933 }
 934
 935 NS_IMETHODIMP CacheStorageService::AsyncGetDiskConsumption(
 936     nsICacheStorageConsumptionObserver* aObserver) {
 937   NS_ENSURE_ARG(aObserver);
 938
 939   nsresult rv;
 940
 941   rv = CacheIndex::AsyncGetDiskConsumption(aObserver);
 942   NS_ENSURE_SUCCESS(rv, rv);
 943
 944   return NS_OK;
 945 }
 946
 947 NS_IMETHODIMP CacheStorageService::GetIoTarget(nsIEventTarget** aEventTarget) {
 948   NS_ENSURE_ARG(aEventTarget);
 949
 950   nsCOMPtr<nsIEventTarget> ioTarget = CacheFileIOManager::IOTarget();
 951   ioTarget.forget(aEventTarget);
 952
 953   return NS_OK;
 954 }
 955
 956 NS_IMETHODIMP CacheStorageService::AsyncVisitAllStorages(
 957     nsICacheStorageVisitor* aVisitor, bool aVisitEntries) {
 958   LOG(("CacheStorageService::AsyncVisitAllStorages [cb=%p]", aVisitor));
 959   NS_ENSURE_FALSE(mShutdown, NS_ERROR_NOT_INITIALIZED);
 960
 961   // Walking the disk cache also walks the memory cache.
 962   RefPtr<WalkDiskCacheRunnable> event =
 963       new WalkDiskCacheRunnable(nullptr, aVisitEntries, aVisitor);
 964   return event->Walk();
 965 }
 966
 967 // Methods used by CacheEntry for management of in-memory structures.
 968
 969 namespace {
 970
 971 class FrecencyComparator {
 972  public:
 973   bool Equals(CacheEntry* a, CacheEntry* b) const {
 974     return a->GetFrecency() == b->GetFrecency();
 975   }
 976   bool LessThan(CacheEntry* a, CacheEntry* b) const {
 977     // We deliberately want to keep the '0' frecency entries at the tail of the
 978     // aray, because these are new entries and would just slow down purging of
 979     // the pools based on frecency.
 980     if (a->GetFrecency() == 0.0 && b->GetFrecency() > 0.0) {
 981       return false;
 982     }
 983     if (a->GetFrecency() > 0.0 && b->GetFrecency() == 0.0) {
 984       return true;
 985     }
 986
 987     return a->GetFrecency() < b->GetFrecency();
 988   }
 989 };
 990
 991 class ExpirationComparator {
 992  public:
 993   bool Equals(CacheEntry* a, CacheEntry* b) const {
 994     return a->GetExpirationTime() == b->GetExpirationTime();
 995   }
 996   bool LessThan(CacheEntry* a, CacheEntry* b) const {
 997     return a->GetExpirationTime() < b->GetExpirationTime();
 998   }
 999 };
1000
1001 }  // namespace
1002
1003 void CacheStorageService::RegisterEntry(CacheEntry* aEntry) {
1004   MOZ_ASSERT(IsOnManagementThread());
1005
1006   if (mShutdown || !aEntry->CanRegister()) return;
1007
1008   TelemetryRecordEntryCreation(aEntry);
1009
1010   LOG(("CacheStorageService::RegisterEntry [entry=%p]", aEntry));
1011
1012   MemoryPool& pool = Pool(aEntry->IsUsingDisk());
1013   pool.mFrecencyArray.AppendElement(aEntry);
1014   pool.mExpirationArray.AppendElement(aEntry);
1015
1016   aEntry->SetRegistered(true);
1017 }
1018
1019 void CacheStorageService::UnregisterEntry(CacheEntry* aEntry) {
1020   MOZ_ASSERT(IsOnManagementThread());
1021
1022   if (!aEntry->IsRegistered()) return;
1023
1024   TelemetryRecordEntryRemoval(aEntry);
1025
1026   LOG(("CacheStorageService::UnregisterEntry [entry=%p]", aEntry));
1027
1028   MemoryPool& pool = Pool(aEntry->IsUsingDisk());
1029   mozilla::DebugOnly<bool> removedFrecency =
1030       pool.mFrecencyArray.RemoveElement(aEntry);
1031   mozilla::DebugOnly<bool> removedExpiration =
1032       pool.mExpirationArray.RemoveElement(aEntry);
1033
1034   MOZ_ASSERT(mShutdown || (removedFrecency && removedExpiration));
1035
1036   // Note: aEntry->CanRegister() since now returns false
1037   aEntry->SetRegistered(false);
1038 }
1039
1040 static bool AddExactEntry(CacheEntryTable* aEntries, nsACString const& aKey,
1041                           CacheEntry* aEntry, bool aOverwrite) {
1042   RefPtr<CacheEntry> existingEntry;
1043   if (!aOverwrite && aEntries->Get(aKey, getter_AddRefs(existingEntry))) {
1044     bool equals = existingEntry == aEntry;
1045     LOG(("AddExactEntry [entry=%p equals=%d]", aEntry, equals));
1046     return equals;  // Already there...
1047   }
1048
1049   LOG(("AddExactEntry [entry=%p put]", aEntry));
1050   aEntries->InsertOrUpdate(aKey, RefPtr{aEntry});
1051   return true;
1052 }
1053
1054 bool CacheStorageService::RemoveEntry(CacheEntry* aEntry,
1055                                       bool aOnlyUnreferenced) {
1056   LOG(("CacheStorageService::RemoveEntry [entry=%p]", aEntry));
1057
1058   nsAutoCString entryKey;
1059   nsresult rv = aEntry->HashingKey(entryKey);
1060   if (NS_FAILED(rv)) {
1061     NS_ERROR("aEntry->HashingKey() failed?");
1062     return false;
1063   }
1064
1065   mozilla::MutexAutoLock lock(mLock);
1066
1067   if (mShutdown) {
1068     LOG(("  after shutdown"));
1069     return false;
1070   }
1071
1072   if (aOnlyUnreferenced) {
1073     if (aEntry->IsReferenced()) {
1074       LOG(("  still referenced, not removing"));
1075       return false;
1076     }
1077
1078     if (!aEntry->IsUsingDisk() &&
1079         IsForcedValidEntry(aEntry->GetStorageID(), entryKey)) {
1080       LOG(("  forced valid, not removing"));
1081       return false;
1082     }
1083   }
1084
1085   CacheEntryTable* entries;
1086   if (sGlobalEntryTables->Get(aEntry->GetStorageID(), &entries)) {
1087     RemoveExactEntry(entries, entryKey, aEntry, false /* don't overwrite */);
1088   }
1089
1090   nsAutoCString memoryStorageID(aEntry->GetStorageID());
1091   AppendMemoryStorageTag(memoryStorageID);
1092
1093   if (sGlobalEntryTables->Get(memoryStorageID, &entries)) {
1094     RemoveExactEntry(entries, entryKey, aEntry, false /* don't overwrite */);
1095   }
1096
1097   return true;
1098 }
1099
1100 void CacheStorageService::RecordMemoryOnlyEntry(CacheEntry* aEntry,
1101                                                 bool aOnlyInMemory,
1102                                                 bool aOverwrite) {
1103   LOG(
1104       ("CacheStorageService::RecordMemoryOnlyEntry [entry=%p, memory=%d, "
1105        "overwrite=%d]",
1106        aEntry, aOnlyInMemory, aOverwrite));
1107   // This method is responsible to put this entry to a special record hashtable
1108   // that contains only entries that are stored in memory.
1109   // Keep in mind that every entry, regardless of whether is in-memory-only or
1110   // not is always recorded in the storage master hash table, the one identified
1111   // by CacheEntry.StorageID().
1112
1113   mLock.AssertCurrentThreadOwns();
1114
1115   if (mShutdown) {
1116     LOG(("  after shutdown"));
1117     return;
1118   }
1119
1120   nsresult rv;
1121
1122   nsAutoCString entryKey;
1123   rv = aEntry->HashingKey(entryKey);
1124   if (NS_FAILED(rv)) {
1125     NS_ERROR("aEntry->HashingKey() failed?");
1126     return;
1127   }
1128
1129   CacheEntryTable* entries = nullptr;
1130   nsAutoCString memoryStorageID(aEntry->GetStorageID());
1131   AppendMemoryStorageTag(memoryStorageID);
1132
1133   if (!sGlobalEntryTables->Get(memoryStorageID, &entries)) {
1134     if (!aOnlyInMemory) {
1135       LOG(("  not recorded as memory only"));
1136       return;
1137     }
1138
1139     entries = sGlobalEntryTables
1140                   ->InsertOrUpdate(
1141                       memoryStorageID,
1142                       MakeUnique<CacheEntryTable>(CacheEntryTable::MEMORY_ONLY))
1143                   .get();
1144     LOG(("  new memory-only storage table for %s", memoryStorageID.get()));
1145   }
1146
1147   if (aOnlyInMemory) {
1148     AddExactEntry(entries, entryKey, aEntry, aOverwrite);
1149   } else {
1150     RemoveExactEntry(entries, entryKey, aEntry, aOverwrite);
1151   }
1152 }
1153
1154 // Checks if a cache entry is forced valid (will be loaded directly from cache
1155 // without further validation) - see nsICacheEntry.idl for further details
1156 bool CacheStorageService::IsForcedValidEntry(nsACString const& aContextKey,
1157                                              nsACString const& aEntryKey) {
1158   return IsForcedValidEntry(aContextKey + aEntryKey);
1159 }
1160
1161 bool CacheStorageService::IsForcedValidEntry(
1162     nsACString const& aContextEntryKey) {
1163   mozilla::MutexAutoLock lock(mForcedValidEntriesLock);
1164
1165   ForcedValidData data;
1166
1167   if (!mForcedValidEntries.Get(aContextEntryKey, &data)) {
1168     return false;
1169   }
1170
1171   if (data.validUntil.IsNull()) {
1172     MOZ_ASSERT_UNREACHABLE("the timeStamp should never be null");
1173     return false;
1174   }
1175
1176   // Entry timeout not reached yet
1177   if (TimeStamp::NowLoRes() <= data.validUntil) {
1178     return true;
1179   }
1180
1181   // Entry timeout has been reached
1182   mForcedValidEntries.Remove(aContextEntryKey);
1183
1184   if (!data.viewed) {
1185     Telemetry::AccumulateCategorical(
1186         Telemetry::LABELS_PREDICTOR_PREFETCH_USE_STATUS::WaitedTooLong);
1187   }
1188   return false;
1189 }
1190
1191 void CacheStorageService::MarkForcedValidEntryUse(nsACString const& aContextKey,
1192                                                   nsACString const& aEntryKey) {
1193   mozilla::MutexAutoLock lock(mForcedValidEntriesLock);
1194
1195   ForcedValidData data;
1196
1197   if (!mForcedValidEntries.Get(aContextKey + aEntryKey, &data)) {
1198     return;
1199   }
1200
1201   data.viewed = true;
1202   mForcedValidEntries.InsertOrUpdate(aContextKey + aEntryKey, data);
1203 }
1204
1205 // Allows a cache entry to be loaded directly from cache without further
1206 // validation - see nsICacheEntry.idl for further details
1207 void CacheStorageService::ForceEntryValidFor(nsACString const& aContextKey,
1208                                              nsACString const& aEntryKey,
1209                                              uint32_t aSecondsToTheFuture) {
1210   mozilla::MutexAutoLock lock(mForcedValidEntriesLock);
1211
1212   TimeStamp now = TimeStamp::NowLoRes();
1213   ForcedValidEntriesPrune(now);
1214
1215   ForcedValidData data;
1216   data.validUntil = now + TimeDuration::FromSeconds(aSecondsToTheFuture);
1217   data.viewed = false;
1218
1219   mForcedValidEntries.InsertOrUpdate(aContextKey + aEntryKey, data);
1220 }
1221
1222 void CacheStorageService::RemoveEntryForceValid(nsACString const& aContextKey,
1223                                                 nsACString const& aEntryKey) {
1224   mozilla::MutexAutoLock lock(mForcedValidEntriesLock);
1225
1226   LOG(("CacheStorageService::RemoveEntryForceValid context='%s' entryKey=%s",
1227        aContextKey.BeginReading(), aEntryKey.BeginReading()));
1228   ForcedValidData data;
1229   bool ok = mForcedValidEntries.Get(aContextKey + aEntryKey, &data);
1230   if (ok && !data.viewed) {
1231     Telemetry::AccumulateCategorical(
1232         Telemetry::LABELS_PREDICTOR_PREFETCH_USE_STATUS::WaitedTooLong);
1233   }
1234   mForcedValidEntries.Remove(aContextKey + aEntryKey);
1235 }
1236
1237 // Cleans out the old entries in mForcedValidEntries
1238 void CacheStorageService::ForcedValidEntriesPrune(TimeStamp& now) {
1239   static TimeDuration const oneMinute = TimeDuration::FromSeconds(60);
1240   static TimeStamp dontPruneUntil = now + oneMinute;
1241   if (now < dontPruneUntil) return;
1242
1243   for (auto iter = mForcedValidEntries.Iter(); !iter.Done(); iter.Next()) {
1244     if (iter.Data().validUntil < now) {
1245       if (!iter.Data().viewed) {
1246         Telemetry::AccumulateCategorical(
1247             Telemetry::LABELS_PREDICTOR_PREFETCH_USE_STATUS::WaitedTooLong);
1248       }
1249       iter.Remove();
1250     }
1251   }
1252   dontPruneUntil = now + oneMinute;
1253 }
1254
1255 void CacheStorageService::OnMemoryConsumptionChange(
1256     CacheMemoryConsumer* aConsumer, uint32_t aCurrentMemoryConsumption) {
1257   LOG(("CacheStorageService::OnMemoryConsumptionChange [consumer=%p, size=%u]",
1258        aConsumer, aCurrentMemoryConsumption));
1259
1260   uint32_t savedMemorySize = aConsumer->LoadReportedMemoryConsumption();
1261   if (savedMemorySize == aCurrentMemoryConsumption) return;
1262
1263   // Exchange saved size with current one.
1264   aConsumer->StoreReportedMemoryConsumption(aCurrentMemoryConsumption);
1265
1266   bool usingDisk = !(aConsumer->LoadFlags() & CacheMemoryConsumer::MEMORY_ONLY);
1267   bool overLimit = Pool(usingDisk).OnMemoryConsumptionChange(
1268       savedMemorySize, aCurrentMemoryConsumption);
1269
1270   if (!overLimit) return;
1271
1272     // It's likely the timer has already been set when we get here,
1273     // check outside the lock to save resources.
1274 #ifdef MOZ_TSAN
1275   if (mPurgeTimerActive) {
1276 #else
1277   if (mPurgeTimer) {
1278 #endif
1279     return;
1280   }
1281
1282   // We don't know if this is called under the service lock or not,
1283   // hence rather dispatch.
1284   RefPtr<nsIEventTarget> cacheIOTarget = Thread();
1285   if (!cacheIOTarget) return;
1286
1287   // Dispatch as a priority task, we want to set the purge timer
1288   // ASAP to prevent vain redispatch of this event.
1289   nsCOMPtr<nsIRunnable> event = NewRunnableMethod(
1290       "net::CacheStorageService::SchedulePurgeOverMemoryLimit", this,
1291       &CacheStorageService::SchedulePurgeOverMemoryLimit);
1292   cacheIOTarget->Dispatch(event, nsIEventTarget::DISPATCH_NORMAL);
1293 }
1294
1295 bool CacheStorageService::MemoryPool::OnMemoryConsumptionChange(
1296     uint32_t aSavedMemorySize, uint32_t aCurrentMemoryConsumption) {
1297   mMemorySize -= aSavedMemorySize;
1298   mMemorySize += aCurrentMemoryConsumption;
1299
1300   LOG(("  mMemorySize=%u (+%u,-%u)", uint32_t(mMemorySize),
1301        aCurrentMemoryConsumption, aSavedMemorySize));
1302
1303   // Bypass purging when memory has not grew up significantly
1304   if (aCurrentMemoryConsumption <= aSavedMemorySize) return false;
1305
1306   return mMemorySize > Limit();
1307 }
1308
1309 void CacheStorageService::SchedulePurgeOverMemoryLimit() {
1310   LOG(("CacheStorageService::SchedulePurgeOverMemoryLimit"));
1311
1312   mozilla::MutexAutoLock lock(mLock);
1313
1314   if (mShutdown) {
1315     LOG(("  past shutdown"));
1316     return;
1317   }
1318
1319   if (mPurgeTimer) {
1320     LOG(("  timer already up"));
1321     return;
1322   }
1323
1324   mPurgeTimer = NS_NewTimer();
1325   if (mPurgeTimer) {
1326 #ifdef MOZ_TSAN
1327     mPurgeTimerActive = true;
1328 #endif
1329     nsresult rv;
1330     rv = mPurgeTimer->InitWithCallback(this, 1000, nsITimer::TYPE_ONE_SHOT);
1331     LOG(("  timer init rv=0x%08" PRIx32, static_cast<uint32_t>(rv)));
1332   }
1333 }
1334
1335 NS_IMETHODIMP
1336 CacheStorageService::Notify(nsITimer* aTimer) {
1337   LOG(("CacheStorageService::Notify"));
1338
1339   mozilla::MutexAutoLock lock(mLock);
1340
1341   if (aTimer == mPurgeTimer) {
1342 #ifdef MOZ_TSAN
1343     mPurgeTimerActive = false;
1344 #endif
1345     mPurgeTimer = nullptr;
1346
1347     if (!mShutdown) {
1348       nsCOMPtr<nsIRunnable> event =
1349           NewRunnableMethod("net::CacheStorageService::PurgeOverMemoryLimit",
1350                             this, &CacheStorageService::PurgeOverMemoryLimit);
1351       Dispatch(event);
1352     }
1353   }
1354
1355   return NS_OK;
1356 }
1357
1358 NS_IMETHODIMP
1359 CacheStorageService::GetName(nsACString& aName) {
1360   aName.AssignLiteral("CacheStorageService");
1361   return NS_OK;
1362 }
1363
1364 void CacheStorageService::PurgeOverMemoryLimit() {
1365   MOZ_ASSERT(IsOnManagementThread());
1366
1367   LOG(("CacheStorageService::PurgeOverMemoryLimit"));
1368
1369   static TimeDuration const kFourSeconds = TimeDuration::FromSeconds(4);
1370   TimeStamp now = TimeStamp::NowLoRes();
1371
1372   if (!mLastPurgeTime.IsNull() && now - mLastPurgeTime < kFourSeconds) {
1373     LOG(("  bypassed, too soon"));
1374     return;
1375   }
1376
1377   mLastPurgeTime = now;
1378
1379   Pool(true).PurgeOverMemoryLimit();
1380   Pool(false).PurgeOverMemoryLimit();
1381 }
1382
1383 void CacheStorageService::MemoryPool::PurgeOverMemoryLimit() {
1384   TimeStamp start(TimeStamp::Now());
1385
1386   uint32_t const memoryLimit = Limit();
1387   if (mMemorySize > memoryLimit) {
1388     LOG(("  memory data consumption over the limit, abandon expired entries"));
1389     PurgeExpired();
1390   }
1391
1392   // No longer makes sense since:
1393   // Memory entries are never purged partially, only as a whole when the memory
1394   // cache limit is overreached.
1395   // Disk entries throw the data away ASAP so that only metadata are kept.
1396   // TODO when this concept of two separate pools is found working, the code
1397   // should clean up.
1398 #if 0
1399   if (mMemorySize > memoryLimit) {
1400     LOG(("  memory data consumption over the limit, abandon disk backed data"));
1401     PurgeByFrecency(CacheEntry::PURGE_DATA_ONLY_DISK_BACKED);
1402   }
1403
1404   if (mMemorySize > memoryLimit) {
1405     LOG(("  metadata consumtion over the limit, abandon disk backed entries"));
1406     PurgeByFrecency(CacheEntry::PURGE_WHOLE_ONLY_DISK_BACKED);
1407   }
1408 #endif
1409
1410   if (mMemorySize > memoryLimit) {
1411     LOG(("  memory data consumption over the limit, abandon any entry"));
1412     PurgeByFrecency(CacheEntry::PURGE_WHOLE);
1413   }
1414
1415   LOG(("  purging took %1.2fms", (TimeStamp::Now() - start).ToMilliseconds()));
1416 }
1417
1418 void CacheStorageService::MemoryPool::PurgeExpired() {
1419   MOZ_ASSERT(IsOnManagementThread());
1420
1421   mExpirationArray.Sort(ExpirationComparator());
1422   uint32_t now = NowInSeconds();
1423
1424   uint32_t const memoryLimit = Limit();
1425
1426   for (uint32_t i = 0;
1427        mMemorySize > memoryLimit && i < mExpirationArray.Length();) {
1428     if (CacheIOThread::YieldAndRerun()) return;
1429
1430     RefPtr<CacheEntry> entry = mExpirationArray[i];
1431
1432     uint32_t expirationTime = entry->GetExpirationTime();
1433     if (expirationTime > 0 && expirationTime <= now &&
1434         entry->Purge(CacheEntry::PURGE_WHOLE)) {
1435       LOG(("  purged expired, entry=%p, exptime=%u (now=%u)", entry.get(),
1436            entry->GetExpirationTime(), now));
1437       continue;
1438     }
1439
1440     // not purged, move to the next one
1441     ++i;
1442   }
1443 }
1444
1445 void CacheStorageService::MemoryPool::PurgeByFrecency(uint32_t aWhat) {
1446   MOZ_ASSERT(IsOnManagementThread());
1447
1448   // Pretend the limit is 10% lower so that we get rid of more entries at one
1449   // shot and save the sorting below.
1450   uint32_t const memoryLimit = Limit() * 0.9;
1451
1452   // Let's do our best and try to shorten the array to at least this size so
1453   // that it doesn't overgrow.  We will ignore higher priority events and keep
1454   // looping to try to purge while the array is larget than this size.
1455   static size_t const kFrecencyArrayLengthLimit = 2000;
1456
1457   LOG(("MemoryPool::PurgeByFrecency, len=%zu", mFrecencyArray.Length()));
1458
1459   mFrecencyArray.Sort(FrecencyComparator());
1460
1461   for (uint32_t i = 0;
1462        mMemorySize > memoryLimit && i < mFrecencyArray.Length();) {
1463     if (mFrecencyArray.Length() <= kFrecencyArrayLengthLimit &&
1464         CacheIOThread::YieldAndRerun()) {
1465       LOG(("MemoryPool::PurgeByFrecency interrupted"));
1466       return;
1467     }
1468
1469     RefPtr<CacheEntry> entry = mFrecencyArray[i];
1470     if (entry->Purge(aWhat)) {
1471       LOG(("  abandoned (%d), entry=%p, frecency=%1.10f", aWhat, entry.get(),
1472            entry->GetFrecency()));
1473       continue;
1474     }
1475
1476     // not purged, move to the next one
1477     ++i;
1478   }
1479
1480   LOG(("MemoryPool::PurgeByFrecency done"));
1481 }
1482
1483 void CacheStorageService::MemoryPool::PurgeAll(uint32_t aWhat) {
1484   LOG(("CacheStorageService::MemoryPool::PurgeAll aWhat=%d", aWhat));
1485   MOZ_ASSERT(IsOnManagementThread());
1486
1487   for (uint32_t i = 0; i < mFrecencyArray.Length();) {
1488     if (CacheIOThread::YieldAndRerun()) return;
1489
1490     RefPtr<CacheEntry> entry = mFrecencyArray[i];
1491
1492     if (entry->Purge(aWhat)) {
1493       LOG(("  abandoned entry=%p", entry.get()));
1494       continue;
1495     }
1496
1497     // not purged, move to the next one
1498     ++i;
1499   }
1500 }
1501
1502 // Methods exposed to and used by CacheStorage.
1503
1504 nsresult CacheStorageService::AddStorageEntry(CacheStorage const* aStorage,
1505                                               const nsACString& aURI,
1506                                               const nsACString& aIdExtension,
1507                                               uint32_t aFlags,
1508                                               CacheEntryHandle** aResult) {
1509   NS_ENSURE_FALSE(mShutdown, NS_ERROR_NOT_INITIALIZED);
1510
1511   NS_ENSURE_ARG(aStorage);
1512
1513   nsAutoCString contextKey;
1514   CacheFileUtils::AppendKeyPrefix(aStorage->LoadInfo(), contextKey);
1515
1516   return AddStorageEntry(contextKey, aURI, aIdExtension,
1517                          aStorage->WriteToDisk(), aStorage->SkipSizeCheck(),
1518                          aStorage->Pinning(), aFlags, aResult);
1519 }
1520
1521 nsresult CacheStorageService::AddStorageEntry(
1522     const nsACString& aContextKey, const nsACString& aURI,
1523     const nsACString& aIdExtension, bool aWriteToDisk, bool aSkipSizeCheck,
1524     bool aPin, uint32_t aFlags, CacheEntryHandle** aResult) {
1525   nsresult rv;
1526
1527   nsAutoCString entryKey;
1528   rv = CacheEntry::HashingKey(""_ns, aIdExtension, aURI, entryKey);
1529   NS_ENSURE_SUCCESS(rv, rv);
1530
1531   LOG(("CacheStorageService::AddStorageEntry [entryKey=%s, contextKey=%s]",
1532        entryKey.get(), aContextKey.BeginReading()));
1533
1534   RefPtr<CacheEntry> entry;
1535   RefPtr<CacheEntryHandle> handle;
1536
1537   {
1538     mozilla::MutexAutoLock lock(mLock);
1539
1540     NS_ENSURE_FALSE(mShutdown, NS_ERROR_NOT_INITIALIZED);
1541
1542     // Ensure storage table
1543     CacheEntryTable* const entries =
1544         sGlobalEntryTables
1545             ->LookupOrInsertWith(
1546                 aContextKey,
1547                 [&aContextKey] {
1548                   LOG(("  new storage entries table for context '%s'",
1549                        aContextKey.BeginReading()));
1550                   return MakeUnique<CacheEntryTable>(
1551                       CacheEntryTable::ALL_ENTRIES);
1552                 })
1553             .get();
1554
1555     bool entryExists = entries->Get(entryKey, getter_AddRefs(entry));
1556     if (!entryExists && (aFlags & nsICacheStorage::OPEN_READONLY) &&
1557         (aFlags & nsICacheStorage::OPEN_SECRETLY) &&
1558         StaticPrefs::network_cache_bug1708673()) {
1559       return NS_ERROR_CACHE_KEY_NOT_FOUND;
1560     }
1561
1562     bool replace = aFlags & nsICacheStorage::OPEN_TRUNCATE;
1563
1564     if (entryExists && !replace) {
1565       // check whether we want to turn this entry to a memory-only.
1566       if (MOZ_UNLIKELY(!aWriteToDisk) && MOZ_LIKELY(entry->IsUsingDisk())) {
1567         LOG(("  entry is persistent but we want mem-only, replacing it"));
1568         replace = true;
1569       }
1570     }
1571
1572     // If truncate is demanded, delete and doom the current entry
1573     if (entryExists && replace) {
1574       entries->Remove(entryKey);
1575
1576       LOG(("  dooming entry %p for %s because of OPEN_TRUNCATE", entry.get(),
1577            entryKey.get()));
1578       // On purpose called under the lock to prevent races of doom and open on
1579       // I/O thread No need to remove from both memory-only and all-entries
1580       // tables.  The new entry will overwrite the shadow entry in its ctor.
1581       entry->DoomAlreadyRemoved();
1582
1583       entry = nullptr;
1584       entryExists = false;
1585
1586       // Would only lead to deleting force-valid timestamp again.  We don't need
1587       // the replace information anymore after this point anyway.
1588       replace = false;
1589     }
1590
1591     // Ensure entry for the particular URL
1592     if (!entryExists) {
1593       // When replacing with a new entry, always remove the current force-valid
1594       // timestamp, this is the only place to do it.
1595       if (replace) {
1596         RemoveEntryForceValid(aContextKey, entryKey);
1597       }
1598
1599       // Entry is not in the hashtable or has just been truncated...
1600       entry = new CacheEntry(aContextKey, aURI, aIdExtension, aWriteToDisk,
1601                              aSkipSizeCheck, aPin);
1602       entries->InsertOrUpdate(entryKey, RefPtr{entry});
1603       LOG(("  new entry %p for %s", entry.get(), entryKey.get()));
1604     }
1605
1606     if (entry) {
1607       // Here, if this entry was not for a long time referenced by any consumer,
1608       // gets again first 'handles count' reference.
1609       handle = entry->NewHandle();
1610     }
1611   }
1612
1613   handle.forget(aResult);
1614   return NS_OK;
1615 }
1616
1617 nsresult CacheStorageService::CheckStorageEntry(CacheStorage const* aStorage,
1618                                                 const nsACString& aURI,
1619                                                 const nsACString& aIdExtension,
1620                                                 bool* aResult) {
1621   nsresult rv;
1622
1623   nsAutoCString contextKey;
1624   CacheFileUtils::AppendKeyPrefix(aStorage->LoadInfo(), contextKey);
1625
1626   if (!aStorage->WriteToDisk()) {
1627     AppendMemoryStorageTag(contextKey);
1628   }
1629
1630   LOG(("CacheStorageService::CheckStorageEntry [uri=%s, eid=%s, contextKey=%s]",
1631        aURI.BeginReading(), aIdExtension.BeginReading(), contextKey.get()));
1632
1633   {
1634     mozilla::MutexAutoLock lock(mLock);
1635
1636     NS_ENSURE_FALSE(mShutdown, NS_ERROR_NOT_INITIALIZED);
1637
1638     nsAutoCString entryKey;
1639     rv = CacheEntry::HashingKey(""_ns, aIdExtension, aURI, entryKey);
1640     NS_ENSURE_SUCCESS(rv, rv);
1641
1642     CacheEntryTable* entries;
1643     if ((*aResult = sGlobalEntryTables->Get(contextKey, &entries)) &&
1644         entries->GetWeak(entryKey, aResult)) {
1645       LOG(("  found in hash tables"));
1646       return NS_OK;
1647     }
1648   }
1649
1650   if (!aStorage->WriteToDisk()) {
1651     // Memory entry, nothing more to do.
1652     LOG(("  not found in hash tables"));
1653     return NS_OK;
1654   }
1655
1656   // Disk entry, not found in the hashtable, check the index.
1657   nsAutoCString fileKey;
1658   rv = CacheEntry::HashingKey(contextKey, aIdExtension, aURI, fileKey);
1659
1660   CacheIndex::EntryStatus status;
1661   rv = CacheIndex::HasEntry(fileKey, &status);
1662   if (NS_FAILED(rv) || status == CacheIndex::DO_NOT_KNOW) {
1663     LOG(("  index doesn't know, rv=0x%08" PRIx32, static_cast<uint32_t>(rv)));
1664     return NS_ERROR_NOT_AVAILABLE;
1665   }
1666
1667   *aResult = status == CacheIndex::EXISTS;
1668   LOG(("  %sfound in index", *aResult ? "" : "not "));
1669   return NS_OK;
1670 }
1671
1672 nsresult CacheStorageService::GetCacheIndexEntryAttrs(
1673     CacheStorage const* aStorage, const nsACString& aURI,
1674     const nsACString& aIdExtension, bool* aHasAltData, uint32_t* aFileSizeKb) {
1675   nsresult rv;
1676
1677   nsAutoCString contextKey;
1678   CacheFileUtils::AppendKeyPrefix(aStorage->LoadInfo(), contextKey);
1679
1680   LOG(
1681       ("CacheStorageService::GetCacheIndexEntryAttrs [uri=%s, eid=%s, "
1682        "contextKey=%s]",
1683        aURI.BeginReading(), aIdExtension.BeginReading(), contextKey.get()));
1684
1685   nsAutoCString fileKey;
1686   rv = CacheEntry::HashingKey(contextKey, aIdExtension, aURI, fileKey);
1687   if (NS_FAILED(rv)) {
1688     return rv;
1689   }
1690
1691   *aHasAltData = false;
1692   *aFileSizeKb = 0;
1693   auto closure = [&aHasAltData, &aFileSizeKb](const CacheIndexEntry* entry) {
1694     *aHasAltData = entry->GetHasAltData();
1695     *aFileSizeKb = entry->GetFileSize();
1696   };
1697
1698   CacheIndex::EntryStatus status;
1699   rv = CacheIndex::HasEntry(fileKey, &status, closure);
1700   if (NS_FAILED(rv)) {
1701     return rv;
1702   }
1703
1704   if (status != CacheIndex::EXISTS) {
1705     return NS_ERROR_CACHE_KEY_NOT_FOUND;
1706   }
1707
1708   return NS_OK;
1709 }
1710
1711 namespace {
1712
1713 class CacheEntryDoomByKeyCallback : public CacheFileIOListener,
1714                                     public nsIRunnable {
1715  public:
1716   NS_DECL_THREADSAFE_ISUPPORTS
1717   NS_DECL_NSIRUNNABLE
1718
1719   explicit CacheEntryDoomByKeyCallback(nsICacheEntryDoomCallback* aCallback)
1720       : mCallback(aCallback), mResult(NS_ERROR_NOT_INITIALIZED) {}
1721
1722  private:
1723   virtual ~CacheEntryDoomByKeyCallback();
1724
1725   NS_IMETHOD OnFileOpened(CacheFileHandle* aHandle, nsresult aResult) override {
1726     return NS_OK;
1727   }
1728   NS_IMETHOD OnDataWritten(CacheFileHandle* aHandle, const char* aBuf,
1729                            nsresult aResult) override {
1730     return NS_OK;
1731   }
1732   NS_IMETHOD OnDataRead(CacheFileHandle* aHandle, char* aBuf,
1733                         nsresult aResult) override {
1734     return NS_OK;
1735   }
1736   NS_IMETHOD OnFileDoomed(CacheFileHandle* aHandle, nsresult aResult) override;
1737   NS_IMETHOD OnEOFSet(CacheFileHandle* aHandle, nsresult aResult) override {
1738     return NS_OK;
1739   }
1740   NS_IMETHOD OnFileRenamed(CacheFileHandle* aHandle,
1741                            nsresult aResult) override {
1742     return NS_OK;
1743   }
1744
1745   nsCOMPtr<nsICacheEntryDoomCallback> mCallback;
1746   nsresult mResult;
1747 };
1748
1749 CacheEntryDoomByKeyCallback::~CacheEntryDoomByKeyCallback() {
1750   if (mCallback) {
1751     ProxyReleaseMainThread("CacheEntryDoomByKeyCallback::mCallback", mCallback);
1752   }
1753 }
1754
1755 NS_IMETHODIMP CacheEntryDoomByKeyCallback::OnFileDoomed(
1756     CacheFileHandle* aHandle, nsresult aResult) {
1757   if (!mCallback) return NS_OK;
1758
1759   mResult = aResult;
1760   if (NS_IsMainThread()) {
1761     Run();
1762   } else {
1763     NS_DispatchToMainThread(this);
1764   }
1765
1766   return NS_OK;
1767 }
1768
1769 NS_IMETHODIMP CacheEntryDoomByKeyCallback::Run() {
1770   mCallback->OnCacheEntryDoomed(mResult);
1771   return NS_OK;
1772 }
1773
1774 NS_IMPL_ISUPPORTS(CacheEntryDoomByKeyCallback, CacheFileIOListener,
1775                   nsIRunnable);
1776
1777 }  // namespace
1778
1779 nsresult CacheStorageService::DoomStorageEntry(
1780     CacheStorage const* aStorage, const nsACString& aURI,
1781     const nsACString& aIdExtension, nsICacheEntryDoomCallback* aCallback) {
1782   LOG(("CacheStorageService::DoomStorageEntry"));
1783
1784   NS_ENSURE_ARG(aStorage);
1785
1786   nsAutoCString contextKey;
1787   CacheFileUtils::AppendKeyPrefix(aStorage->LoadInfo(), contextKey);
1788
1789   nsAutoCString entryKey;
1790   nsresult rv = CacheEntry::HashingKey(""_ns, aIdExtension, aURI, entryKey);
1791   NS_ENSURE_SUCCESS(rv, rv);
1792
1793   RefPtr<CacheEntry> entry;
1794   {
1795     mozilla::MutexAutoLock lock(mLock);
1796
1797     NS_ENSURE_FALSE(mShutdown, NS_ERROR_NOT_INITIALIZED);
1798
1799     CacheEntryTable* entries;
1800     if (sGlobalEntryTables->Get(contextKey, &entries)) {
1801       if (entries->Get(entryKey, getter_AddRefs(entry))) {
1802         if (aStorage->WriteToDisk() || !entry->IsUsingDisk()) {
1803           // When evicting from disk storage, purge
1804           // When evicting from memory storage and the entry is memory-only,
1805           // purge
1806           LOG(
1807               ("  purging entry %p for %s [storage use disk=%d, entry use "
1808                "disk=%d]",
1809                entry.get(), entryKey.get(), aStorage->WriteToDisk(),
1810                entry->IsUsingDisk()));
1811           entries->Remove(entryKey);
1812         } else {
1813           // Otherwise, leave it
1814           LOG(
1815               ("  leaving entry %p for %s [storage use disk=%d, entry use "
1816                "disk=%d]",
1817                entry.get(), entryKey.get(), aStorage->WriteToDisk(),
1818                entry->IsUsingDisk()));
1819           entry = nullptr;
1820         }
1821       }
1822     }
1823
1824     if (!entry) {
1825       RemoveEntryForceValid(contextKey, entryKey);
1826     }
1827   }
1828
1829   if (entry) {
1830     LOG(("  dooming entry %p for %s", entry.get(), entryKey.get()));
1831     return entry->AsyncDoom(aCallback);
1832   }
1833
1834   LOG(("  no entry loaded for %s", entryKey.get()));
1835
1836   if (aStorage->WriteToDisk()) {
1837     nsAutoCString contextKey;
1838     CacheFileUtils::AppendKeyPrefix(aStorage->LoadInfo(), contextKey);
1839
1840     rv = CacheEntry::HashingKey(contextKey, aIdExtension, aURI, entryKey);
1841     NS_ENSURE_SUCCESS(rv, rv);
1842
1843     LOG(("  dooming file only for %s", entryKey.get()));
1844
1845     RefPtr<CacheEntryDoomByKeyCallback> callback(
1846         new CacheEntryDoomByKeyCallback(aCallback));
1847     rv = CacheFileIOManager::DoomFileByKey(entryKey, callback);
1848     NS_ENSURE_SUCCESS(rv, rv);
1849
1850     return NS_OK;
1851   }
1852
1853   class Callback : public Runnable {
1854    public:
1855     explicit Callback(nsICacheEntryDoomCallback* aCallback)
1856         : mozilla::Runnable("Callback"), mCallback(aCallback) {}
1857     NS_IMETHOD Run() override {
1858       mCallback->OnCacheEntryDoomed(NS_ERROR_NOT_AVAILABLE);
1859       return NS_OK;
1860     }
1861     nsCOMPtr<nsICacheEntryDoomCallback> mCallback;
1862   };
1863
1864   if (aCallback) {
1865     RefPtr<Runnable> callback = new Callback(aCallback);
1866     return NS_DispatchToMainThread(callback);
1867   }
1868
1869   return NS_OK;
1870 }
1871
1872 nsresult CacheStorageService::DoomStorageEntries(
1873     CacheStorage const* aStorage, nsICacheEntryDoomCallback* aCallback) {
1874   LOG(("CacheStorageService::DoomStorageEntries"));
1875
1876   NS_ENSURE_FALSE(mShutdown, NS_ERROR_NOT_INITIALIZED);
1877   NS_ENSURE_ARG(aStorage);
1878
1879   nsAutoCString contextKey;
1880   CacheFileUtils::AppendKeyPrefix(aStorage->LoadInfo(), contextKey);
1881
1882   mozilla::MutexAutoLock lock(mLock);
1883
1884   return DoomStorageEntries(contextKey, aStorage->LoadInfo(),
1885                             aStorage->WriteToDisk(), aStorage->Pinning(),
1886                             aCallback);
1887 }
1888
1889 nsresult CacheStorageService::DoomStorageEntries(
1890     const nsACString& aContextKey, nsILoadContextInfo* aContext,
1891     bool aDiskStorage, bool aPinned, nsICacheEntryDoomCallback* aCallback) {
1892   LOG(("CacheStorageService::DoomStorageEntries [context=%s]",
1893        aContextKey.BeginReading()));
1894
1895   mLock.AssertCurrentThreadOwns();
1896
1897   NS_ENSURE_TRUE(!mShutdown, NS_ERROR_NOT_INITIALIZED);
1898
1899   nsAutoCString memoryStorageID(aContextKey);
1900   AppendMemoryStorageTag(memoryStorageID);
1901
1902   if (aDiskStorage) {
1903     LOG(("  dooming disk+memory storage of %s", aContextKey.BeginReading()));
1904
1905     // Walk one by one and remove entries according their pin status
1906     CacheEntryTable *diskEntries, *memoryEntries;
1907     if (sGlobalEntryTables->Get(aContextKey, &diskEntries)) {
1908       sGlobalEntryTables->Get(memoryStorageID, &memoryEntries);
1909
1910       for (auto iter = diskEntries->Iter(); !iter.Done(); iter.Next()) {
1911         auto entry = iter.Data();
1912         if (entry->DeferOrBypassRemovalOnPinStatus(aPinned)) {
1913           continue;
1914         }
1915
1916         if (memoryEntries) {
1917           RemoveExactEntry(memoryEntries, iter.Key(), entry, false);
1918         }
1919         iter.Remove();
1920       }
1921     }
1922
1923     if (aContext && !aContext->IsPrivate()) {
1924       LOG(("  dooming disk entries"));
1925       CacheFileIOManager::EvictByContext(aContext, aPinned, u""_ns);
1926     }
1927   } else {
1928     LOG(("  dooming memory-only storage of %s", aContextKey.BeginReading()));
1929
1930     // Remove the memory entries table from the global tables.
1931     // Since we store memory entries also in the disk entries table
1932     // we need to remove the memory entries from the disk table one
1933     // by one manually.
1934     mozilla::UniquePtr<CacheEntryTable> memoryEntries;
1935     sGlobalEntryTables->Remove(memoryStorageID, &memoryEntries);
1936
1937     CacheEntryTable* diskEntries;
1938     if (memoryEntries && sGlobalEntryTables->Get(aContextKey, &diskEntries)) {
1939       for (const auto& memoryEntry : *memoryEntries) {
1940         const auto& entry = memoryEntry.GetData();
1941         RemoveExactEntry(diskEntries, memoryEntry.GetKey(), entry, false);
1942       }
1943     }
1944   }
1945
1946   {
1947     mozilla::MutexAutoLock lock(mForcedValidEntriesLock);
1948
1949     if (aContext) {
1950       for (auto iter = mForcedValidEntries.Iter(); !iter.Done(); iter.Next()) {
1951         bool matches;
1952         DebugOnly<nsresult> rv = CacheFileUtils::KeyMatchesLoadContextInfo(
1953             iter.Key(), aContext, &matches);
1954         MOZ_ASSERT(NS_SUCCEEDED(rv));
1955
1956         if (matches) {
1957           iter.Remove();
1958         }
1959       }
1960     } else {
1961       mForcedValidEntries.Clear();
1962     }
1963   }
1964
1965   // An artificial callback.  This is a candidate for removal tho.  In the new
1966   // cache any 'doom' or 'evict' function ensures that the entry or entries
1967   // being doomed is/are not accessible after the function returns.  So there is
1968   // probably no need for a callback - has no meaning.  But for compatibility
1969   // with the old cache that is still in the tree we keep the API similar to be
1970   // able to make tests as well as other consumers work for now.
1971   class Callback : public Runnable {
1972    public:
1973     explicit Callback(nsICacheEntryDoomCallback* aCallback)
1974         : mozilla::Runnable("Callback"), mCallback(aCallback) {}
1975     NS_IMETHOD Run() override {
1976       mCallback->OnCacheEntryDoomed(NS_OK);
1977       return NS_OK;
1978     }
1979     nsCOMPtr<nsICacheEntryDoomCallback> mCallback;
1980   };
1981
1982   if (aCallback) {
1983     RefPtr<Runnable> callback = new Callback(aCallback);
1984     return NS_DispatchToMainThread(callback);
1985   }
1986
1987   return NS_OK;
1988 }
1989
1990 nsresult CacheStorageService::WalkStorageEntries(
1991     CacheStorage const* aStorage, bool aVisitEntries,
1992     nsICacheStorageVisitor* aVisitor) {
1993   LOG(("CacheStorageService::WalkStorageEntries [cb=%p, visitentries=%d]",
1994        aVisitor, aVisitEntries));
1995   NS_ENSURE_FALSE(mShutdown, NS_ERROR_NOT_INITIALIZED);
1996
1997   NS_ENSURE_ARG(aStorage);
1998
1999   if (aStorage->WriteToDisk()) {
2000     RefPtr<WalkDiskCacheRunnable> event = new WalkDiskCacheRunnable(
2001         aStorage->LoadInfo(), aVisitEntries, aVisitor);
2002     return event->Walk();
2003   }
2004
2005   RefPtr<WalkMemoryCacheRunnable> event = new WalkMemoryCacheRunnable(
2006       aStorage->LoadInfo(), aVisitEntries, aVisitor);
2007   return event->Walk();
2008 }
2009
2010 void CacheStorageService::CacheFileDoomed(nsILoadContextInfo* aLoadContextInfo,
2011                                           const nsACString& aIdExtension,
2012                                           const nsACString& aURISpec) {
2013   nsAutoCString contextKey;
2014   CacheFileUtils::AppendKeyPrefix(aLoadContextInfo, contextKey);
2015
2016   nsAutoCString entryKey;
2017   CacheEntry::HashingKey(""_ns, aIdExtension, aURISpec, entryKey);
2018
2019   mozilla::MutexAutoLock lock(mLock);
2020
2021   if (mShutdown) {
2022     return;
2023   }
2024
2025   CacheEntryTable* entries;
2026   RefPtr<CacheEntry> entry;
2027
2028   if (sGlobalEntryTables->Get(contextKey, &entries) &&
2029       entries->Get(entryKey, getter_AddRefs(entry))) {
2030     if (entry->IsFileDoomed()) {
2031       // Need to remove under the lock to avoid possible race leading
2032       // to duplication of the entry per its key.
2033       RemoveExactEntry(entries, entryKey, entry, false);
2034       entry->DoomAlreadyRemoved();
2035     }
2036
2037     // Entry found, but it's not the entry that has been found doomed
2038     // by the lower eviction layer.  Just leave everything unchanged.
2039     return;
2040   }
2041
2042   RemoveEntryForceValid(contextKey, entryKey);
2043 }
2044
2045 bool CacheStorageService::GetCacheEntryInfo(
2046     nsILoadContextInfo* aLoadContextInfo, const nsACString& aIdExtension,
2047     const nsACString& aURISpec, EntryInfoCallback* aCallback) {
2048   nsAutoCString contextKey;
2049   CacheFileUtils::AppendKeyPrefix(aLoadContextInfo, contextKey);
2050
2051   nsAutoCString entryKey;
2052   CacheEntry::HashingKey(""_ns, aIdExtension, aURISpec, entryKey);
2053
2054   RefPtr<CacheEntry> entry;
2055   {
2056     mozilla::MutexAutoLock lock(mLock);
2057
2058     if (mShutdown) {
2059       return false;
2060     }
2061
2062     CacheEntryTable* entries;
2063     if (!sGlobalEntryTables->Get(contextKey, &entries)) {
2064       return false;
2065     }
2066
2067     if (!entries->Get(entryKey, getter_AddRefs(entry))) {
2068       return false;
2069     }
2070   }
2071
2072   GetCacheEntryInfo(entry, aCallback);
2073   return true;
2074 }
2075
2076 // static
2077 void CacheStorageService::GetCacheEntryInfo(CacheEntry* aEntry,
2078                                             EntryInfoCallback* aCallback) {
2079   nsCString const uriSpec = aEntry->GetURI();
2080   nsCString const enhanceId = aEntry->GetEnhanceID();
2081
2082   nsAutoCString entryKey;
2083   aEntry->HashingKeyWithStorage(entryKey);
2084
2085   nsCOMPtr<nsILoadContextInfo> info = CacheFileUtils::ParseKey(entryKey);
2086
2087   uint32_t dataSize;
2088   if (NS_FAILED(aEntry->GetStorageDataSize(&dataSize))) {
2089     dataSize = 0;
2090   }
2091   int64_t altDataSize;
2092   if (NS_FAILED(aEntry->GetAltDataSize(&altDataSize))) {
2093     altDataSize = 0;
2094   }
2095   uint32_t fetchCount;
2096   if (NS_FAILED(aEntry->GetFetchCount(&fetchCount))) {
2097     fetchCount = 0;
2098   }
2099   uint32_t lastModified;
2100   if (NS_FAILED(aEntry->GetLastModified(&lastModified))) {
2101     lastModified = 0;
2102   }
2103   uint32_t expirationTime;
2104   if (NS_FAILED(aEntry->GetExpirationTime(&expirationTime))) {
2105     expirationTime = 0;
2106   }
2107
2108   aCallback->OnEntryInfo(uriSpec, enhanceId, dataSize, altDataSize, fetchCount,
2109                          lastModified, expirationTime, aEntry->IsPinned(),
2110                          info);
2111 }
2112
2113 // static
2114 uint32_t CacheStorageService::CacheQueueSize(bool highPriority) {
2115   RefPtr<CacheIOThread> thread = CacheFileIOManager::IOThread();
2116   // The thread will be null at shutdown.
2117   if (!thread) {
2118     return 0;
2119   }
2120   return thread->QueueSize(highPriority);
2121 }
2122
2123 // Telemetry collection
2124
2125 namespace {
2126
2127 bool TelemetryEntryKey(CacheEntry const* entry, nsAutoCString& key) {
2128   nsAutoCString entryKey;
2129   nsresult rv = entry->HashingKey(entryKey);
2130   if (NS_FAILED(rv)) return false;
2131
2132   if (entry->GetStorageID().IsEmpty()) {
2133     // Hopefully this will be const-copied, saves some memory
2134     key = entryKey;
2135   } else {
2136     key.Assign(entry->GetStorageID());
2137     key.Append(':');
2138     key.Append(entryKey);
2139   }
2140
2141   return true;
2142 }
2143
2144 }  // namespace
2145
2146 void CacheStorageService::TelemetryPrune(TimeStamp& now) {
2147   static TimeDuration const oneMinute = TimeDuration::FromSeconds(60);
2148   static TimeStamp dontPruneUntil = now + oneMinute;
2149   if (now < dontPruneUntil) return;
2150
2151   static TimeDuration const fifteenMinutes = TimeDuration::FromSeconds(900);
2152   for (auto iter = mPurgeTimeStamps.Iter(); !iter.Done(); iter.Next()) {
2153     if (now - iter.Data() > fifteenMinutes) {
2154       // We are not interested in resurrection of entries after 15 minutes
2155       // of time.  This is also the limit for the telemetry.
2156       iter.Remove();
2157     }
2158   }
2159   dontPruneUntil = now + oneMinute;
2160 }
2161
2162 void CacheStorageService::TelemetryRecordEntryCreation(
2163     CacheEntry const* entry) {
2164   MOZ_ASSERT(CacheStorageService::IsOnManagementThread());
2165
2166   nsAutoCString key;
2167   if (!TelemetryEntryKey(entry, key)) return;
2168
2169   TimeStamp now = TimeStamp::NowLoRes();
2170   TelemetryPrune(now);
2171
2172   // When an entry is craeted (registered actually) we check if there is
2173   // a timestamp marked when this very same cache entry has been removed
2174   // (deregistered) because of over-memory-limit purging.  If there is such
2175   // a timestamp found accumulate telemetry on how long the entry was away.
2176   TimeStamp timeStamp;
2177   if (!mPurgeTimeStamps.Get(key, &timeStamp)) return;
2178
2179   mPurgeTimeStamps.Remove(key);
2180
2181   Telemetry::AccumulateTimeDelta(Telemetry::HTTP_CACHE_ENTRY_RELOAD_TIME,
2182                                  timeStamp, TimeStamp::NowLoRes());
2183 }
2184
2185 void CacheStorageService::TelemetryRecordEntryRemoval(CacheEntry* entry) {
2186   MOZ_ASSERT(CacheStorageService::IsOnManagementThread());
2187
2188   // Doomed entries must not be considered, we are only interested in purged
2189   // entries.  Note that the mIsDoomed flag is always set before deregistration
2190   // happens.
2191   if (entry->IsDoomed()) return;
2192
2193   nsAutoCString key;
2194   if (!TelemetryEntryKey(entry, key)) return;
2195
2196   // When an entry is removed (deregistered actually) we put a timestamp for
2197   // this entry to the hashtable so that when the entry is created (registered)
2198   // again we know how long it was away.  Also accumulate number of AsyncOpen
2199   // calls on the entry, this tells us how efficiently the pool actually works.
2200
2201   TimeStamp now = TimeStamp::NowLoRes();
2202   TelemetryPrune(now);
2203   mPurgeTimeStamps.InsertOrUpdate(key, now);
2204
2205   Telemetry::Accumulate(Telemetry::HTTP_CACHE_ENTRY_REUSE_COUNT,
2206                         entry->UseCount());
2207   Telemetry::AccumulateTimeDelta(Telemetry::HTTP_CACHE_ENTRY_ALIVE_TIME,
2208                                  entry->LoadStart(), TimeStamp::NowLoRes());
2209 }
2210
2211 // nsIMemoryReporter
2212
2213 size_t CacheStorageService::SizeOfExcludingThis(
2214     mozilla::MallocSizeOf mallocSizeOf) const {
2215   CacheStorageService::Self()->Lock().AssertCurrentThreadOwns();
2216
2217   size_t n = 0;
2218   // The elemets are referenced by sGlobalEntryTables and are reported from
2219   // there
2220   n += Pool(true).mFrecencyArray.ShallowSizeOfExcludingThis(mallocSizeOf);
2221   n += Pool(true).mExpirationArray.ShallowSizeOfExcludingThis(mallocSizeOf);
2222   n += Pool(false).mFrecencyArray.ShallowSizeOfExcludingThis(mallocSizeOf);
2223   n += Pool(false).mExpirationArray.ShallowSizeOfExcludingThis(mallocSizeOf);
2224   // Entries reported manually in CacheStorageService::CollectReports callback
2225   if (sGlobalEntryTables) {
2226     n += sGlobalEntryTables->ShallowSizeOfIncludingThis(mallocSizeOf);
2227   }
2228   n += mPurgeTimeStamps.SizeOfExcludingThis(mallocSizeOf);
2229
2230   return n;
2231 }
2232
2233 size_t CacheStorageService::SizeOfIncludingThis(
2234     mozilla::MallocSizeOf mallocSizeOf) const {
2235   return mallocSizeOf(this) + SizeOfExcludingThis(mallocSizeOf);
2236 }
2237
2238 NS_IMETHODIMP
2239 CacheStorageService::CollectReports(nsIHandleReportCallback* aHandleReport,
2240                                     nsISupports* aData, bool aAnonymize) {
2241   MutexAutoLock lock(mLock);
2242   MOZ_COLLECT_REPORT("explicit/network/cache2/io", KIND_HEAP, UNITS_BYTES,
2243                      CacheFileIOManager::SizeOfIncludingThis(MallocSizeOf),
2244                      "Memory used by the cache IO manager.");
2245
2246   MOZ_COLLECT_REPORT("explicit/network/cache2/index", KIND_HEAP, UNITS_BYTES,
2247                      CacheIndex::SizeOfIncludingThis(MallocSizeOf),
2248                      "Memory used by the cache index.");
2249
2250   // Report the service instance, this doesn't report entries, done lower
2251   MOZ_COLLECT_REPORT("explicit/network/cache2/service", KIND_HEAP, UNITS_BYTES,
2252                      SizeOfIncludingThis(MallocSizeOf),
2253                      "Memory used by the cache storage service.");
2254
2255   // Report all entries, each storage separately (by the context key)
2256   //
2257   // References are:
2258   // sGlobalEntryTables to N CacheEntryTable
2259   // CacheEntryTable to N CacheEntry
2260   // CacheEntry to 1 CacheFile
2261   // CacheFile to
2262   //   N CacheFileChunk (keeping the actual data)
2263   //   1 CacheFileMetadata (keeping http headers etc.)
2264   //   1 CacheFileOutputStream
2265   //   N CacheFileInputStream
2266   if (sGlobalEntryTables) {
2267     for (const auto& globalEntry : *sGlobalEntryTables) {
2268       CacheStorageService::Self()->Lock().AssertCurrentThreadOwns();
2269
2270       CacheEntryTable* table = globalEntry.GetWeak();
2271
2272       size_t size = 0;
2273       mozilla::MallocSizeOf mallocSizeOf = CacheStorageService::MallocSizeOf;
2274
2275       size += table->ShallowSizeOfIncludingThis(mallocSizeOf);
2276       for (const auto& tableEntry : *table) {
2277         size += tableEntry.GetKey().SizeOfExcludingThisIfUnshared(mallocSizeOf);
2278
2279         // Bypass memory-only entries, those will be reported when iterating the
2280         // memory only table. Memory-only entries are stored in both ALL_ENTRIES
2281         // and MEMORY_ONLY hashtables.
2282         RefPtr<mozilla::net::CacheEntry> const& entry = tableEntry.GetData();
2283         if (table->Type() == CacheEntryTable::MEMORY_ONLY ||
2284             entry->IsUsingDisk()) {
2285           size += entry->SizeOfIncludingThis(mallocSizeOf);
2286         }
2287       }
2288
2289       aHandleReport->Callback(
2290           ""_ns,
2291           nsPrintfCString(
2292               "explicit/network/cache2/%s-storage(%s)",
2293               table->Type() == CacheEntryTable::MEMORY_ONLY ? "memory" : "disk",
2294               aAnonymize ? "<anonymized>"
2295                          : globalEntry.GetKey().BeginReading()),
2296           nsIMemoryReporter::KIND_HEAP, nsIMemoryReporter::UNITS_BYTES, size,
2297           "Memory used by the cache storage."_ns, aData);
2298     }
2299   }
2300
2301   return NS_OK;
2302 }
2303
2304 // nsICacheTesting
2305
2306 NS_IMETHODIMP
2307 CacheStorageService::IOThreadSuspender::Run() {
2308   MonitorAutoLock mon(mMon);
2309   while (!mSignaled) {
2310     mon.Wait();
2311   }
2312   return NS_OK;
2313 }
2314
2315 void CacheStorageService::IOThreadSuspender::Notify() {
2316   MonitorAutoLock mon(mMon);
2317   mSignaled = true;
2318   mon.Notify();
2319 }
2320
2321 NS_IMETHODIMP
2322 CacheStorageService::SuspendCacheIOThread(uint32_t aLevel) {
2323   RefPtr<CacheIOThread> thread = CacheFileIOManager::IOThread();
2324   if (!thread) {
2325     return NS_ERROR_NOT_AVAILABLE;
2326   }
2327
2328   MOZ_ASSERT(!mActiveIOSuspender);
2329   mActiveIOSuspender = new IOThreadSuspender();
2330   return thread->Dispatch(mActiveIOSuspender, aLevel);
2331 }
2332
2333 NS_IMETHODIMP
2334 CacheStorageService::ResumeCacheIOThread() {
2335   MOZ_ASSERT(mActiveIOSuspender);
2336
2337   RefPtr<IOThreadSuspender> suspender;
2338   suspender.swap(mActiveIOSuspender);
2339   suspender->Notify();
2340   return NS_OK;
2341 }
2342
2343 NS_IMETHODIMP
2344 CacheStorageService::Flush(nsIObserver* aObserver) {
2345   RefPtr<CacheIOThread> thread = CacheFileIOManager::IOThread();
2346   if (!thread) {
2347     return NS_ERROR_NOT_AVAILABLE;
2348   }
2349
2350   nsCOMPtr<nsIObserverService> observerService =
2351       mozilla::services::GetObserverService();
2352   if (!observerService) {
2353     return NS_ERROR_NOT_AVAILABLE;
2354   }
2355
2356   // Adding as weak, the consumer is responsible to keep the reference
2357   // until notified.
2358   observerService->AddObserver(aObserver, "cacheservice:purge-memory-pools",
2359                                false);
2360
2361   // This runnable will do the purging and when done, notifies the above
2362   // observer. We dispatch it to the CLOSE level, so all data writes scheduled
2363   // up to this time will be done before this purging happens.
2364   RefPtr<CacheStorageService::PurgeFromMemoryRunnable> r =
2365       new CacheStorageService::PurgeFromMemoryRunnable(this,
2366                                                        CacheEntry::PURGE_WHOLE);
2367
2368   return thread->Dispatch(r, CacheIOThread::WRITE);
2369 }
2370
2371 }  // namespace mozilla::net