netwerk/cache2/CacheFileIOManager.cpp

   1 /* This Source Code Form is subject to the terms of the Mozilla Public
   2  * License, v. 2.0. If a copy of the MPL was not distributed with this
   3  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   4
   5 #include "CacheLog.h"
   6 #include "CacheFileIOManager.h"
   7
   8 #include "../cache/nsCacheUtils.h"
   9 #include "CacheHashUtils.h"
  10 #include "CacheStorageService.h"
  11 #include "CacheIndex.h"
  12 #include "CacheFileUtils.h"
  13 #include "nsThreadUtils.h"
  14 #include "CacheFile.h"
  15 #include "CacheObserver.h"
  16 #include "nsIFile.h"
  17 #include "CacheFileContextEvictor.h"
  18 #include "nsITimer.h"
  19 #include "nsIDirectoryEnumerator.h"
  20 #include "nsIObserverService.h"
  21 #include "nsISizeOf.h"
  22 #include "mozilla/net/MozURL.h"
  23 #include "mozilla/Telemetry.h"
  24 #include "mozilla/DebugOnly.h"
  25 #include "mozilla/Services.h"
  26 #include "nsDirectoryServiceUtils.h"
  27 #include "nsAppDirectoryServiceDefs.h"
  28 #include "private/pprio.h"
  29 #include "mozilla/IntegerPrintfMacros.h"
  30 #include "mozilla/Preferences.h"
  31 #include "nsNetUtil.h"
  32
  33 // include files for ftruncate (or equivalent)
  34 #if defined(XP_UNIX)
  35 #  include <unistd.h>
  36 #elif defined(XP_WIN)
  37 #  include <windows.h>
  38 #  undef CreateFile
  39 #  undef CREATE_NEW
  40 #else
  41 // XXX add necessary include file for ftruncate (or equivalent)
  42 #endif
  43
  44 namespace mozilla {
  45 namespace net {
  46
  47 #define kOpenHandlesLimit 128
  48 #define kMetadataWriteDelay 5000
  49 #define kRemoveTrashStartDelay 60000    // in milliseconds
  50 #define kSmartSizeUpdateInterval 60000  // in milliseconds
  51
  52 #ifdef ANDROID
  53 const uint32_t kMaxCacheSizeKB = 512 * 1024;  // 512 MB
  54 #else
  55 const uint32_t kMaxCacheSizeKB = 1024 * 1024;  // 1 GB
  56 #endif
  57 const uint32_t kMaxClearOnShutdownCacheSizeKB = 150 * 1024;  // 150 MB
  58
  59 bool CacheFileHandle::DispatchRelease() {
  60   if (CacheFileIOManager::IsOnIOThreadOrCeased()) {
  61     return false;
  62   }
  63
  64   nsCOMPtr<nsIEventTarget> ioTarget = CacheFileIOManager::IOTarget();
  65   if (!ioTarget) {
  66     return false;
  67   }
  68
  69   nsresult rv = ioTarget->Dispatch(
  70       NewNonOwningRunnableMethod("net::CacheFileHandle::Release", this,
  71                                  &CacheFileHandle::Release),
  72       nsIEventTarget::DISPATCH_NORMAL);
  73   if (NS_FAILED(rv)) {
  74     return false;
  75   }
  76
  77   return true;
  78 }
  79
  80 NS_IMPL_ADDREF(CacheFileHandle)
  81 NS_IMETHODIMP_(MozExternalRefCountType)
  82 CacheFileHandle::Release() {
  83   nsrefcnt count = mRefCnt - 1;
  84   if (DispatchRelease()) {
  85     // Redispatched to the IO thread.
  86     return count;
  87   }
  88
  89   MOZ_ASSERT(CacheFileIOManager::IsOnIOThreadOrCeased());
  90
  91   LOG(("CacheFileHandle::Release() [this=%p, refcnt=%" PRIuPTR "]", this,
  92        mRefCnt.get()));
  93   MOZ_ASSERT(0 != mRefCnt, "dup release");
  94   count = --mRefCnt;
  95   NS_LOG_RELEASE(this, count, "CacheFileHandle");
  96
  97   if (0 == count) {
  98     mRefCnt = 1;
  99     delete (this);
 100     return 0;
 101   }
 102
 103   return count;
 104 }
 105
 106 NS_INTERFACE_MAP_BEGIN(CacheFileHandle)
 107   NS_INTERFACE_MAP_ENTRY(nsISupports)
 108 NS_INTERFACE_MAP_END
 109
 110 CacheFileHandle::CacheFileHandle(const SHA1Sum::Hash* aHash, bool aPriority,
 111                                  PinningStatus aPinning)
 112     : mHash(aHash),
 113       mIsDoomed(false),
 114       mClosed(false),
 115       mPriority(aPriority),
 116       mSpecialFile(false),
 117       mInvalid(false),
 118       mFileExists(false),
 119       mDoomWhenFoundPinned(false),
 120       mDoomWhenFoundNonPinned(false),
 121       mKilled(false),
 122       mPinning(aPinning),
 123       mFileSize(-1),
 124       mFD(nullptr) {
 125   // If we initialize mDoomed in the initialization list, that initialization is
 126   // not guaranteeded to be atomic.  Whereas this assignment here is guaranteed
 127   // to be atomic.  TSan will see this (atomic) assignment and be satisfied
 128   // that cross-thread accesses to mIsDoomed are properly synchronized.
 129   mIsDoomed = false;
 130   LOG((
 131       "CacheFileHandle::CacheFileHandle() [this=%p, hash=%08x%08x%08x%08x%08x]",
 132       this, LOGSHA1(aHash)));
 133 }
 134
 135 CacheFileHandle::CacheFileHandle(const nsACString& aKey, bool aPriority,
 136                                  PinningStatus aPinning)
 137     : mHash(nullptr),
 138       mIsDoomed(false),
 139       mClosed(false),
 140       mPriority(aPriority),
 141       mSpecialFile(true),
 142       mInvalid(false),
 143       mFileExists(false),
 144       mDoomWhenFoundPinned(false),
 145       mDoomWhenFoundNonPinned(false),
 146       mKilled(false),
 147       mPinning(aPinning),
 148       mFileSize(-1),
 149       mFD(nullptr),
 150       mKey(aKey) {
 151   // See comment above about the initialization of mIsDoomed.
 152   mIsDoomed = false;
 153   LOG(("CacheFileHandle::CacheFileHandle() [this=%p, key=%s]", this,
 154        PromiseFlatCString(aKey).get()));
 155 }
 156
 157 CacheFileHandle::~CacheFileHandle() {
 158   LOG(("CacheFileHandle::~CacheFileHandle() [this=%p]", this));
 159
 160   MOZ_ASSERT(CacheFileIOManager::IsOnIOThreadOrCeased());
 161
 162   RefPtr<CacheFileIOManager> ioMan = CacheFileIOManager::gInstance;
 163   if (!IsClosed() && ioMan) {
 164     ioMan->CloseHandleInternal(this);
 165   }
 166 }
 167
 168 void CacheFileHandle::Log() {
 169   nsAutoCString leafName;
 170   if (mFile) {
 171     mFile->GetNativeLeafName(leafName);
 172   }
 173
 174   if (mSpecialFile) {
 175     LOG(
 176         ("CacheFileHandle::Log() - special file [this=%p, "
 177          "isDoomed=%d, priority=%d, closed=%d, invalid=%d, "
 178          "pinning=%" PRIu32 ", fileExists=%d, fileSize=%" PRId64
 179          ", leafName=%s, key=%s]",
 180          this, bool(mIsDoomed), bool(mPriority), bool(mClosed), bool(mInvalid),
 181          static_cast<uint32_t>(mPinning), bool(mFileExists), int64_t(mFileSize),
 182          leafName.get(), mKey.get()));
 183   } else {
 184     LOG(
 185         ("CacheFileHandle::Log() - entry file [this=%p, "
 186          "hash=%08x%08x%08x%08x%08x, "
 187          "isDoomed=%d, priority=%d, closed=%d, invalid=%d, "
 188          "pinning=%" PRIu32 ", fileExists=%d, fileSize=%" PRId64
 189          ", leafName=%s, key=%s]",
 190          this, LOGSHA1(mHash), bool(mIsDoomed), bool(mPriority), bool(mClosed),
 191          bool(mInvalid), static_cast<uint32_t>(mPinning), bool(mFileExists),
 192          int64_t(mFileSize), leafName.get(), mKey.get()));
 193   }
 194 }
 195
 196 uint32_t CacheFileHandle::FileSizeInK() const {
 197   MOZ_ASSERT(mFileSize != -1);
 198   uint64_t size64 = mFileSize;
 199
 200   size64 += 0x3FF;
 201   size64 >>= 10;
 202
 203   uint32_t size;
 204   if (size64 >> 32) {
 205     NS_WARNING(
 206         "CacheFileHandle::FileSizeInK() - FileSize is too large, "
 207         "truncating to PR_UINT32_MAX");
 208     size = PR_UINT32_MAX;
 209   } else {
 210     size = static_cast<uint32_t>(size64);
 211   }
 212
 213   return size;
 214 }
 215
 216 bool CacheFileHandle::SetPinned(bool aPinned) {
 217   LOG(("CacheFileHandle::SetPinned [this=%p, pinned=%d]", this, aPinned));
 218
 219   MOZ_ASSERT(CacheFileIOManager::IsOnIOThreadOrCeased());
 220
 221   mPinning = aPinned ? PinningStatus::PINNED : PinningStatus::NON_PINNED;
 222
 223   if ((MOZ_UNLIKELY(mDoomWhenFoundPinned) && aPinned) ||
 224       (MOZ_UNLIKELY(mDoomWhenFoundNonPinned) && !aPinned)) {
 225     LOG(("  dooming, when: pinned=%d, non-pinned=%d, found: pinned=%d",
 226          bool(mDoomWhenFoundPinned), bool(mDoomWhenFoundNonPinned), aPinned));
 227
 228     mDoomWhenFoundPinned = false;
 229     mDoomWhenFoundNonPinned = false;
 230
 231     return false;
 232   }
 233
 234   return true;
 235 }
 236
 237 // Memory reporting
 238
 239 size_t CacheFileHandle::SizeOfExcludingThis(
 240     mozilla::MallocSizeOf mallocSizeOf) const {
 241   size_t n = 0;
 242   nsCOMPtr<nsISizeOf> sizeOf;
 243
 244   sizeOf = do_QueryInterface(mFile);
 245   if (sizeOf) {
 246     n += sizeOf->SizeOfIncludingThis(mallocSizeOf);
 247   }
 248
 249   n += mallocSizeOf(mFD);
 250   n += mKey.SizeOfExcludingThisIfUnshared(mallocSizeOf);
 251   return n;
 252 }
 253
 254 size_t CacheFileHandle::SizeOfIncludingThis(
 255     mozilla::MallocSizeOf mallocSizeOf) const {
 256   return mallocSizeOf(this) + SizeOfExcludingThis(mallocSizeOf);
 257 }
 258
 259 /******************************************************************************
 260  *  CacheFileHandles::HandleHashKey
 261  *****************************************************************************/
 262
 263 void CacheFileHandles::HandleHashKey::AddHandle(CacheFileHandle* aHandle) {
 264   MOZ_ASSERT(CacheFileIOManager::IsOnIOThreadOrCeased());
 265
 266   mHandles.InsertElementAt(0, aHandle);
 267 }
 268
 269 void CacheFileHandles::HandleHashKey::RemoveHandle(CacheFileHandle* aHandle) {
 270   MOZ_ASSERT(CacheFileIOManager::IsOnIOThreadOrCeased());
 271
 272   DebugOnly<bool> found;
 273   found = mHandles.RemoveElement(aHandle);
 274   MOZ_ASSERT(found);
 275 }
 276
 277 already_AddRefed<CacheFileHandle>
 278 CacheFileHandles::HandleHashKey::GetNewestHandle() {
 279   MOZ_ASSERT(CacheFileIOManager::IsOnIOThreadOrCeased());
 280
 281   RefPtr<CacheFileHandle> handle;
 282   if (mHandles.Length()) {
 283     handle = mHandles[0];
 284   }
 285
 286   return handle.forget();
 287 }
 288
 289 void CacheFileHandles::HandleHashKey::GetHandles(
 290     nsTArray<RefPtr<CacheFileHandle> >& aResult) {
 291   MOZ_ASSERT(CacheFileIOManager::IsOnIOThreadOrCeased());
 292
 293   for (uint32_t i = 0; i < mHandles.Length(); ++i) {
 294     CacheFileHandle* handle = mHandles[i];
 295     aResult.AppendElement(handle);
 296   }
 297 }
 298
 299 #ifdef DEBUG
 300
 301 void CacheFileHandles::HandleHashKey::AssertHandlesState() {
 302   for (uint32_t i = 0; i < mHandles.Length(); ++i) {
 303     CacheFileHandle* handle = mHandles[i];
 304     MOZ_ASSERT(handle->IsDoomed());
 305   }
 306 }
 307
 308 #endif
 309
 310 size_t CacheFileHandles::HandleHashKey::SizeOfExcludingThis(
 311     mozilla::MallocSizeOf mallocSizeOf) const {
 312   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
 313
 314   size_t n = 0;
 315   n += mallocSizeOf(mHash.get());
 316   for (uint32_t i = 0; i < mHandles.Length(); ++i) {
 317     n += mHandles[i]->SizeOfIncludingThis(mallocSizeOf);
 318   }
 319
 320   return n;
 321 }
 322
 323 /******************************************************************************
 324  *  CacheFileHandles
 325  *****************************************************************************/
 326
 327 CacheFileHandles::CacheFileHandles() {
 328   LOG(("CacheFileHandles::CacheFileHandles() [this=%p]", this));
 329   MOZ_COUNT_CTOR(CacheFileHandles);
 330 }
 331
 332 CacheFileHandles::~CacheFileHandles() {
 333   LOG(("CacheFileHandles::~CacheFileHandles() [this=%p]", this));
 334   MOZ_COUNT_DTOR(CacheFileHandles);
 335 }
 336
 337 nsresult CacheFileHandles::GetHandle(const SHA1Sum::Hash* aHash,
 338                                      CacheFileHandle** _retval) {
 339   MOZ_ASSERT(CacheFileIOManager::IsOnIOThreadOrCeased());
 340   MOZ_ASSERT(aHash);
 341
 342 #ifdef DEBUG_HANDLES
 343   LOG(("CacheFileHandles::GetHandle() [hash=%08x%08x%08x%08x%08x]",
 344        LOGSHA1(aHash)));
 345 #endif
 346
 347   // find hash entry for key
 348   HandleHashKey* entry = mTable.GetEntry(*aHash);
 349   if (!entry) {
 350     LOG(
 351         ("CacheFileHandles::GetHandle() hash=%08x%08x%08x%08x%08x "
 352          "no handle entries found",
 353          LOGSHA1(aHash)));
 354     return NS_ERROR_NOT_AVAILABLE;
 355   }
 356
 357 #ifdef DEBUG_HANDLES
 358   Log(entry);
 359 #endif
 360
 361   // Check if the entry is doomed
 362   RefPtr<CacheFileHandle> handle = entry->GetNewestHandle();
 363   if (!handle) {
 364     LOG(
 365         ("CacheFileHandles::GetHandle() hash=%08x%08x%08x%08x%08x "
 366          "no handle found %p, entry %p",
 367          LOGSHA1(aHash), handle.get(), entry));
 368     return NS_ERROR_NOT_AVAILABLE;
 369   }
 370
 371   if (handle->IsDoomed()) {
 372     LOG(
 373         ("CacheFileHandles::GetHandle() hash=%08x%08x%08x%08x%08x "
 374          "found doomed handle %p, entry %p",
 375          LOGSHA1(aHash), handle.get(), entry));
 376     return NS_ERROR_NOT_AVAILABLE;
 377   }
 378
 379   LOG(
 380       ("CacheFileHandles::GetHandle() hash=%08x%08x%08x%08x%08x "
 381        "found handle %p, entry %p",
 382        LOGSHA1(aHash), handle.get(), entry));
 383
 384   handle.forget(_retval);
 385   return NS_OK;
 386 }
 387
 388 already_AddRefed<CacheFileHandle> CacheFileHandles::NewHandle(
 389     const SHA1Sum::Hash* aHash, bool aPriority,
 390     CacheFileHandle::PinningStatus aPinning) {
 391   MOZ_ASSERT(CacheFileIOManager::IsOnIOThreadOrCeased());
 392   MOZ_ASSERT(aHash);
 393
 394 #ifdef DEBUG_HANDLES
 395   LOG(("CacheFileHandles::NewHandle() [hash=%08x%08x%08x%08x%08x]",
 396        LOGSHA1(aHash)));
 397 #endif
 398
 399   // find hash entry for key
 400   HandleHashKey* entry = mTable.PutEntry(*aHash);
 401
 402 #ifdef DEBUG_HANDLES
 403   Log(entry);
 404 #endif
 405
 406 #ifdef DEBUG
 407   entry->AssertHandlesState();
 408 #endif
 409
 410   RefPtr<CacheFileHandle> handle =
 411       new CacheFileHandle(entry->Hash(), aPriority, aPinning);
 412   entry->AddHandle(handle);
 413
 414   LOG(
 415       ("CacheFileHandles::NewHandle() hash=%08x%08x%08x%08x%08x "
 416        "created new handle %p, entry=%p",
 417        LOGSHA1(aHash), handle.get(), entry));
 418   return handle.forget();
 419 }
 420
 421 void CacheFileHandles::RemoveHandle(CacheFileHandle* aHandle) {
 422   MOZ_ASSERT(CacheFileIOManager::IsOnIOThreadOrCeased());
 423   MOZ_ASSERT(aHandle);
 424
 425   if (!aHandle) {
 426     return;
 427   }
 428
 429 #ifdef DEBUG_HANDLES
 430   LOG((
 431       "CacheFileHandles::RemoveHandle() [handle=%p, hash=%08x%08x%08x%08x%08x]",
 432       aHandle, LOGSHA1(aHandle->Hash())));
 433 #endif
 434
 435   // find hash entry for key
 436   HandleHashKey* entry = mTable.GetEntry(*aHandle->Hash());
 437   if (!entry) {
 438     MOZ_ASSERT(CacheFileIOManager::IsShutdown(),
 439                "Should find entry when removing a handle before shutdown");
 440
 441     LOG(
 442         ("CacheFileHandles::RemoveHandle() hash=%08x%08x%08x%08x%08x "
 443          "no entries found",
 444          LOGSHA1(aHandle->Hash())));
 445     return;
 446   }
 447
 448 #ifdef DEBUG_HANDLES
 449   Log(entry);
 450 #endif
 451
 452   LOG(
 453       ("CacheFileHandles::RemoveHandle() hash=%08x%08x%08x%08x%08x "
 454        "removing handle %p",
 455        LOGSHA1(entry->Hash()), aHandle));
 456   entry->RemoveHandle(aHandle);
 457
 458   if (entry->IsEmpty()) {
 459     LOG(
 460         ("CacheFileHandles::RemoveHandle() hash=%08x%08x%08x%08x%08x "
 461          "list is empty, removing entry %p",
 462          LOGSHA1(entry->Hash()), entry));
 463     mTable.RemoveEntry(entry);
 464   }
 465 }
 466
 467 void CacheFileHandles::GetAllHandles(
 468     nsTArray<RefPtr<CacheFileHandle> >* _retval) {
 469   MOZ_ASSERT(CacheFileIOManager::IsOnIOThreadOrCeased());
 470   for (auto iter = mTable.Iter(); !iter.Done(); iter.Next()) {
 471     iter.Get()->GetHandles(*_retval);
 472   }
 473 }
 474
 475 void CacheFileHandles::GetActiveHandles(
 476     nsTArray<RefPtr<CacheFileHandle> >* _retval) {
 477   MOZ_ASSERT(CacheFileIOManager::IsOnIOThreadOrCeased());
 478   for (auto iter = mTable.Iter(); !iter.Done(); iter.Next()) {
 479     RefPtr<CacheFileHandle> handle = iter.Get()->GetNewestHandle();
 480     MOZ_ASSERT(handle);
 481
 482     if (!handle->IsDoomed()) {
 483       _retval->AppendElement(handle);
 484     }
 485   }
 486 }
 487
 488 void CacheFileHandles::ClearAll() {
 489   MOZ_ASSERT(CacheFileIOManager::IsOnIOThreadOrCeased());
 490   mTable.Clear();
 491 }
 492
 493 uint32_t CacheFileHandles::HandleCount() { return mTable.Count(); }
 494
 495 #ifdef DEBUG_HANDLES
 496 void CacheFileHandles::Log(CacheFileHandlesEntry* entry) {
 497   LOG(("CacheFileHandles::Log() BEGIN [entry=%p]", entry));
 498
 499   nsTArray<RefPtr<CacheFileHandle> > array;
 500   aEntry->GetHandles(array);
 501
 502   for (uint32_t i = 0; i < array.Length(); ++i) {
 503     CacheFileHandle* handle = array[i];
 504     handle->Log();
 505   }
 506
 507   LOG(("CacheFileHandles::Log() END [entry=%p]", entry));
 508 }
 509 #endif
 510
 511 // Memory reporting
 512
 513 size_t CacheFileHandles::SizeOfExcludingThis(
 514     mozilla::MallocSizeOf mallocSizeOf) const {
 515   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
 516
 517   return mTable.SizeOfExcludingThis(mallocSizeOf);
 518 }
 519
 520 // Events
 521
 522 class ShutdownEvent : public Runnable {
 523  public:
 524   ShutdownEvent()
 525       : Runnable("net::ShutdownEvent"),
 526         mMonitor("ShutdownEvent.mMonitor"),
 527         mNotified(false) {}
 528
 529  protected:
 530   ~ShutdownEvent() = default;
 531
 532  public:
 533   NS_IMETHOD Run() override {
 534     MonitorAutoLock mon(mMonitor);
 535
 536     CacheFileIOManager::gInstance->ShutdownInternal();
 537
 538     mNotified = true;
 539     mon.Notify();
 540
 541     return NS_OK;
 542   }
 543
 544   void PostAndWait() {
 545     MonitorAutoLock mon(mMonitor);
 546
 547     nsresult rv = CacheFileIOManager::gInstance->mIOThread->Dispatch(
 548         this,
 549         CacheIOThread::WRITE);  // When writes and closing of handles is done
 550     MOZ_ASSERT(NS_SUCCEEDED(rv));
 551
 552     // If we failed to post the even there's no reason to go into the loop
 553     // because mNotified will never be set to true.
 554     if (NS_FAILED(rv)) {
 555       NS_WARNING("Posting ShutdownEvent task failed");
 556       return;
 557     }
 558
 559     TimeDuration waitTime = TimeDuration::FromSeconds(1);
 560     while (!mNotified) {
 561       mon.Wait(waitTime);
 562       if (!mNotified) {
 563         // If there is any IO blocking on the IO thread, this will
 564         // try to cancel it.  Returns no later than after two seconds.
 565         MonitorAutoUnlock unmon(mMonitor);  // Prevent delays
 566         CacheFileIOManager::gInstance->mIOThread->CancelBlockingIO();
 567       }
 568     }
 569   }
 570
 571  protected:
 572   mozilla::Monitor mMonitor;
 573   bool mNotified;
 574 };
 575
 576 // Class responsible for reporting IO performance stats
 577 class IOPerfReportEvent {
 578  public:
 579   explicit IOPerfReportEvent(CacheFileUtils::CachePerfStats::EDataType aType)
 580       : mType(aType), mEventCounter(0) {}
 581
 582   void Start(CacheIOThread* aIOThread) {
 583     mStartTime = TimeStamp::Now();
 584     mEventCounter = aIOThread->EventCounter();
 585   }
 586
 587   void Report(CacheIOThread* aIOThread) {
 588     if (mStartTime.IsNull()) {
 589       return;
 590     }
 591
 592     // Single IO operations can take less than 1ms. So we use microseconds to
 593     // keep a good resolution of data.
 594     uint32_t duration = (TimeStamp::Now() - mStartTime).ToMicroseconds();
 595
 596     // This is a simple prefiltering of values that might differ a lot from the
 597     // average value. Do not add the value to the filtered stats when the event
 598     // had to wait in a long queue.
 599     uint32_t eventCounter = aIOThread->EventCounter();
 600     bool shortOnly = eventCounter - mEventCounter < 5 ? false : true;
 601
 602     CacheFileUtils::CachePerfStats::AddValue(mType, duration, shortOnly);
 603   }
 604
 605  protected:
 606   CacheFileUtils::CachePerfStats::EDataType mType;
 607   TimeStamp mStartTime;
 608   uint32_t mEventCounter;
 609 };
 610
 611 class OpenFileEvent : public Runnable, public IOPerfReportEvent {
 612  public:
 613   OpenFileEvent(const nsACString& aKey, uint32_t aFlags,
 614                 CacheFileIOListener* aCallback)
 615       : Runnable("net::OpenFileEvent"),
 616         IOPerfReportEvent(CacheFileUtils::CachePerfStats::IO_OPEN),
 617         mFlags(aFlags),
 618         mCallback(aCallback),
 619         mKey(aKey) {
 620     mIOMan = CacheFileIOManager::gInstance;
 621     if (!(mFlags & CacheFileIOManager::SPECIAL_FILE)) {
 622       Start(mIOMan->mIOThread);
 623     }
 624   }
 625
 626  protected:
 627   ~OpenFileEvent() = default;
 628
 629  public:
 630   NS_IMETHOD Run() override {
 631     nsresult rv = NS_OK;
 632
 633     if (!(mFlags & CacheFileIOManager::SPECIAL_FILE)) {
 634       SHA1Sum sum;
 635       sum.update(mKey.BeginReading(), mKey.Length());
 636       sum.finish(mHash);
 637     }
 638
 639     if (!mIOMan) {
 640       rv = NS_ERROR_NOT_INITIALIZED;
 641     } else {
 642       if (mFlags & CacheFileIOManager::SPECIAL_FILE) {
 643         rv = mIOMan->OpenSpecialFileInternal(mKey, mFlags,
 644                                              getter_AddRefs(mHandle));
 645       } else {
 646         rv = mIOMan->OpenFileInternal(&mHash, mKey, mFlags,
 647                                       getter_AddRefs(mHandle));
 648         if (NS_SUCCEEDED(rv)) {
 649           Report(mIOMan->mIOThread);
 650         }
 651       }
 652       mIOMan = nullptr;
 653       if (mHandle) {
 654         if (mHandle->Key().IsEmpty()) {
 655           mHandle->Key() = mKey;
 656         }
 657       }
 658     }
 659
 660     mCallback->OnFileOpened(mHandle, rv);
 661     return NS_OK;
 662   }
 663
 664  protected:
 665   SHA1Sum::Hash mHash;
 666   uint32_t mFlags;
 667   nsCOMPtr<CacheFileIOListener> mCallback;
 668   RefPtr<CacheFileIOManager> mIOMan;
 669   RefPtr<CacheFileHandle> mHandle;
 670   nsCString mKey;
 671 };
 672
 673 class ReadEvent : public Runnable, public IOPerfReportEvent {
 674  public:
 675   ReadEvent(CacheFileHandle* aHandle, int64_t aOffset, char* aBuf,
 676             int32_t aCount, CacheFileIOListener* aCallback)
 677       : Runnable("net::ReadEvent"),
 678         IOPerfReportEvent(CacheFileUtils::CachePerfStats::IO_READ),
 679         mHandle(aHandle),
 680         mOffset(aOffset),
 681         mBuf(aBuf),
 682         mCount(aCount),
 683         mCallback(aCallback) {
 684     if (!mHandle->IsSpecialFile()) {
 685       Start(CacheFileIOManager::gInstance->mIOThread);
 686     }
 687   }
 688
 689  protected:
 690   ~ReadEvent() = default;
 691
 692  public:
 693   NS_IMETHOD Run() override {
 694     nsresult rv;
 695
 696     if (mHandle->IsClosed() || (mCallback && mCallback->IsKilled())) {
 697       rv = NS_ERROR_NOT_INITIALIZED;
 698     } else {
 699       rv = CacheFileIOManager::gInstance->ReadInternal(mHandle, mOffset, mBuf,
 700                                                        mCount);
 701       if (NS_SUCCEEDED(rv)) {
 702         Report(CacheFileIOManager::gInstance->mIOThread);
 703       }
 704     }
 705
 706     mCallback->OnDataRead(mHandle, mBuf, rv);
 707     return NS_OK;
 708   }
 709
 710  protected:
 711   RefPtr<CacheFileHandle> mHandle;
 712   int64_t mOffset;
 713   char* mBuf;
 714   int32_t mCount;
 715   nsCOMPtr<CacheFileIOListener> mCallback;
 716 };
 717
 718 class WriteEvent : public Runnable, public IOPerfReportEvent {
 719  public:
 720   WriteEvent(CacheFileHandle* aHandle, int64_t aOffset, const char* aBuf,
 721              int32_t aCount, bool aValidate, bool aTruncate,
 722              CacheFileIOListener* aCallback)
 723       : Runnable("net::WriteEvent"),
 724         IOPerfReportEvent(CacheFileUtils::CachePerfStats::IO_WRITE),
 725         mHandle(aHandle),
 726         mOffset(aOffset),
 727         mBuf(aBuf),
 728         mCount(aCount),
 729         mValidate(aValidate),
 730         mTruncate(aTruncate),
 731         mCallback(aCallback) {
 732     if (!mHandle->IsSpecialFile()) {
 733       Start(CacheFileIOManager::gInstance->mIOThread);
 734     }
 735   }
 736
 737  protected:
 738   ~WriteEvent() {
 739     if (!mCallback && mBuf) {
 740       free(const_cast<char*>(mBuf));
 741     }
 742   }
 743
 744  public:
 745   NS_IMETHOD Run() override {
 746     nsresult rv;
 747
 748     if (mHandle->IsClosed() || (mCallback && mCallback->IsKilled())) {
 749       // We usually get here only after the internal shutdown
 750       // (i.e. mShuttingDown == true).  Pretend write has succeeded
 751       // to avoid any past-shutdown file dooming.
 752       rv = (CacheObserver::IsPastShutdownIOLag() ||
 753             CacheFileIOManager::gInstance->mShuttingDown)
 754                ? NS_OK
 755                : NS_ERROR_NOT_INITIALIZED;
 756     } else {
 757       rv = CacheFileIOManager::gInstance->WriteInternal(
 758           mHandle, mOffset, mBuf, mCount, mValidate, mTruncate);
 759       if (NS_SUCCEEDED(rv)) {
 760         Report(CacheFileIOManager::gInstance->mIOThread);
 761       }
 762       if (NS_FAILED(rv) && !mCallback) {
 763         // No listener is going to handle the error, doom the file
 764         CacheFileIOManager::gInstance->DoomFileInternal(mHandle);
 765       }
 766     }
 767     if (mCallback) {
 768       mCallback->OnDataWritten(mHandle, mBuf, rv);
 769     } else {
 770       free(const_cast<char*>(mBuf));
 771       mBuf = nullptr;
 772     }
 773
 774     return NS_OK;
 775   }
 776
 777  protected:
 778   RefPtr<CacheFileHandle> mHandle;
 779   int64_t mOffset;
 780   const char* mBuf;
 781   int32_t mCount;
 782   bool mValidate : 1;
 783   bool mTruncate : 1;
 784   nsCOMPtr<CacheFileIOListener> mCallback;
 785 };
 786
 787 class DoomFileEvent : public Runnable {
 788  public:
 789   DoomFileEvent(CacheFileHandle* aHandle, CacheFileIOListener* aCallback)
 790       : Runnable("net::DoomFileEvent"),
 791         mCallback(aCallback),
 792         mHandle(aHandle) {}
 793
 794  protected:
 795   ~DoomFileEvent() = default;
 796
 797  public:
 798   NS_IMETHOD Run() override {
 799     nsresult rv;
 800
 801     if (mHandle->IsClosed()) {
 802       rv = NS_ERROR_NOT_INITIALIZED;
 803     } else {
 804       rv = CacheFileIOManager::gInstance->DoomFileInternal(mHandle);
 805     }
 806
 807     if (mCallback) {
 808       mCallback->OnFileDoomed(mHandle, rv);
 809     }
 810
 811     return NS_OK;
 812   }
 813
 814  protected:
 815   nsCOMPtr<CacheFileIOListener> mCallback;
 816   nsCOMPtr<nsIEventTarget> mTarget;
 817   RefPtr<CacheFileHandle> mHandle;
 818 };
 819
 820 class DoomFileByKeyEvent : public Runnable {
 821  public:
 822   DoomFileByKeyEvent(const nsACString& aKey, CacheFileIOListener* aCallback)
 823       : Runnable("net::DoomFileByKeyEvent"), mCallback(aCallback) {
 824     SHA1Sum sum;
 825     sum.update(aKey.BeginReading(), aKey.Length());
 826     sum.finish(mHash);
 827
 828     mIOMan = CacheFileIOManager::gInstance;
 829   }
 830
 831  protected:
 832   ~DoomFileByKeyEvent() = default;
 833
 834  public:
 835   NS_IMETHOD Run() override {
 836     nsresult rv;
 837
 838     if (!mIOMan) {
 839       rv = NS_ERROR_NOT_INITIALIZED;
 840     } else {
 841       rv = mIOMan->DoomFileByKeyInternal(&mHash);
 842       mIOMan = nullptr;
 843     }
 844
 845     if (mCallback) {
 846       mCallback->OnFileDoomed(nullptr, rv);
 847     }
 848
 849     return NS_OK;
 850   }
 851
 852  protected:
 853   SHA1Sum::Hash mHash;
 854   nsCOMPtr<CacheFileIOListener> mCallback;
 855   RefPtr<CacheFileIOManager> mIOMan;
 856 };
 857
 858 class ReleaseNSPRHandleEvent : public Runnable {
 859  public:
 860   explicit ReleaseNSPRHandleEvent(CacheFileHandle* aHandle)
 861       : Runnable("net::ReleaseNSPRHandleEvent"), mHandle(aHandle) {}
 862
 863  protected:
 864   ~ReleaseNSPRHandleEvent() = default;
 865
 866  public:
 867   NS_IMETHOD Run() override {
 868     if (!mHandle->IsClosed()) {
 869       CacheFileIOManager::gInstance->MaybeReleaseNSPRHandleInternal(mHandle);
 870     }
 871
 872     return NS_OK;
 873   }
 874
 875  protected:
 876   RefPtr<CacheFileHandle> mHandle;
 877 };
 878
 879 class TruncateSeekSetEOFEvent : public Runnable {
 880  public:
 881   TruncateSeekSetEOFEvent(CacheFileHandle* aHandle, int64_t aTruncatePos,
 882                           int64_t aEOFPos, CacheFileIOListener* aCallback)
 883       : Runnable("net::TruncateSeekSetEOFEvent"),
 884         mHandle(aHandle),
 885         mTruncatePos(aTruncatePos),
 886         mEOFPos(aEOFPos),
 887         mCallback(aCallback) {}
 888
 889  protected:
 890   ~TruncateSeekSetEOFEvent() = default;
 891
 892  public:
 893   NS_IMETHOD Run() override {
 894     nsresult rv;
 895
 896     if (mHandle->IsClosed() || (mCallback && mCallback->IsKilled())) {
 897       rv = NS_ERROR_NOT_INITIALIZED;
 898     } else {
 899       rv = CacheFileIOManager::gInstance->TruncateSeekSetEOFInternal(
 900           mHandle, mTruncatePos, mEOFPos);
 901     }
 902
 903     if (mCallback) {
 904       mCallback->OnEOFSet(mHandle, rv);
 905     }
 906
 907     return NS_OK;
 908   }
 909
 910  protected:
 911   RefPtr<CacheFileHandle> mHandle;
 912   int64_t mTruncatePos;
 913   int64_t mEOFPos;
 914   nsCOMPtr<CacheFileIOListener> mCallback;
 915 };
 916
 917 class RenameFileEvent : public Runnable {
 918  public:
 919   RenameFileEvent(CacheFileHandle* aHandle, const nsACString& aNewName,
 920                   CacheFileIOListener* aCallback)
 921       : Runnable("net::RenameFileEvent"),
 922         mHandle(aHandle),
 923         mNewName(aNewName),
 924         mCallback(aCallback) {}
 925
 926  protected:
 927   ~RenameFileEvent() = default;
 928
 929  public:
 930   NS_IMETHOD Run() override {
 931     nsresult rv;
 932
 933     if (mHandle->IsClosed()) {
 934       rv = NS_ERROR_NOT_INITIALIZED;
 935     } else {
 936       rv = CacheFileIOManager::gInstance->RenameFileInternal(mHandle, mNewName);
 937     }
 938
 939     if (mCallback) {
 940       mCallback->OnFileRenamed(mHandle, rv);
 941     }
 942
 943     return NS_OK;
 944   }
 945
 946  protected:
 947   RefPtr<CacheFileHandle> mHandle;
 948   nsCString mNewName;
 949   nsCOMPtr<CacheFileIOListener> mCallback;
 950 };
 951
 952 class InitIndexEntryEvent : public Runnable {
 953  public:
 954   InitIndexEntryEvent(CacheFileHandle* aHandle,
 955                       OriginAttrsHash aOriginAttrsHash, bool aAnonymous,
 956                       bool aPinning)
 957       : Runnable("net::InitIndexEntryEvent"),
 958         mHandle(aHandle),
 959         mOriginAttrsHash(aOriginAttrsHash),
 960         mAnonymous(aAnonymous),
 961         mPinning(aPinning) {}
 962
 963  protected:
 964   ~InitIndexEntryEvent() = default;
 965
 966  public:
 967   NS_IMETHOD Run() override {
 968     if (mHandle->IsClosed() || mHandle->IsDoomed()) {
 969       return NS_OK;
 970     }
 971
 972     CacheIndex::InitEntry(mHandle->Hash(), mOriginAttrsHash, mAnonymous,
 973                           mPinning);
 974
 975     // We cannot set the filesize before we init the entry. If we're opening
 976     // an existing entry file, frecency will be set after parsing the entry
 977     // file, but we must set the filesize here since nobody is going to set it
 978     // if there is no write to the file.
 979     uint32_t sizeInK = mHandle->FileSizeInK();
 980     CacheIndex::UpdateEntry(mHandle->Hash(), nullptr, nullptr, nullptr, nullptr,
 981                             nullptr, &sizeInK);
 982
 983     return NS_OK;
 984   }
 985
 986  protected:
 987   RefPtr<CacheFileHandle> mHandle;
 988   OriginAttrsHash mOriginAttrsHash;
 989   bool mAnonymous;
 990   bool mPinning;
 991 };
 992
 993 class UpdateIndexEntryEvent : public Runnable {
 994  public:
 995   UpdateIndexEntryEvent(CacheFileHandle* aHandle, const uint32_t* aFrecency,
 996                         const bool* aHasAltData, const uint16_t* aOnStartTime,
 997                         const uint16_t* aOnStopTime,
 998                         const uint8_t* aContentType)
 999       : Runnable("net::UpdateIndexEntryEvent"),
1000         mHandle(aHandle),
1001         mHasFrecency(false),
1002         mHasHasAltData(false),
1003         mHasOnStartTime(false),
1004         mHasOnStopTime(false),
1005         mHasContentType(false),
1006         mFrecency(0),
1007         mHasAltData(false),
1008         mOnStartTime(0),
1009         mOnStopTime(0),
1010         mContentType(nsICacheEntry::CONTENT_TYPE_UNKNOWN) {
1011     if (aFrecency) {
1012       mHasFrecency = true;
1013       mFrecency = *aFrecency;
1014     }
1015     if (aHasAltData) {
1016       mHasHasAltData = true;
1017       mHasAltData = *aHasAltData;
1018     }
1019     if (aOnStartTime) {
1020       mHasOnStartTime = true;
1021       mOnStartTime = *aOnStartTime;
1022     }
1023     if (aOnStopTime) {
1024       mHasOnStopTime = true;
1025       mOnStopTime = *aOnStopTime;
1026     }
1027     if (aContentType) {
1028       mHasContentType = true;
1029       mContentType = *aContentType;
1030     }
1031   }
1032
1033  protected:
1034   ~UpdateIndexEntryEvent() = default;
1035
1036  public:
1037   NS_IMETHOD Run() override {
1038     if (mHandle->IsClosed() || mHandle->IsDoomed()) {
1039       return NS_OK;
1040     }
1041
1042     CacheIndex::UpdateEntry(mHandle->Hash(),
1043                             mHasFrecency ? &mFrecency : nullptr,
1044                             mHasHasAltData ? &mHasAltData : nullptr,
1045                             mHasOnStartTime ? &mOnStartTime : nullptr,
1046                             mHasOnStopTime ? &mOnStopTime : nullptr,
1047                             mHasContentType ? &mContentType : nullptr, nullptr);
1048     return NS_OK;
1049   }
1050
1051  protected:
1052   RefPtr<CacheFileHandle> mHandle;
1053
1054   bool mHasFrecency;
1055   bool mHasHasAltData;
1056   bool mHasOnStartTime;
1057   bool mHasOnStopTime;
1058   bool mHasContentType;
1059
1060   uint32_t mFrecency;
1061   bool mHasAltData;
1062   uint16_t mOnStartTime;
1063   uint16_t mOnStopTime;
1064   uint8_t mContentType;
1065 };
1066
1067 class MetadataWriteScheduleEvent : public Runnable {
1068  public:
1069   enum EMode { SCHEDULE, UNSCHEDULE, SHUTDOWN } mMode;
1070
1071   RefPtr<CacheFile> mFile;
1072   RefPtr<CacheFileIOManager> mIOMan;
1073
1074   MetadataWriteScheduleEvent(CacheFileIOManager* aManager, CacheFile* aFile,
1075                              EMode aMode)
1076       : Runnable("net::MetadataWriteScheduleEvent"),
1077         mMode(aMode),
1078         mFile(aFile),
1079         mIOMan(aManager) {}
1080
1081   virtual ~MetadataWriteScheduleEvent() = default;
1082
1083   NS_IMETHOD Run() override {
1084     RefPtr<CacheFileIOManager> ioMan = CacheFileIOManager::gInstance;
1085     if (!ioMan) {
1086       NS_WARNING(
1087           "CacheFileIOManager already gone in "
1088           "MetadataWriteScheduleEvent::Run()");
1089       return NS_OK;
1090     }
1091
1092     switch (mMode) {
1093       case SCHEDULE:
1094         ioMan->ScheduleMetadataWriteInternal(mFile);
1095         break;
1096       case UNSCHEDULE:
1097         ioMan->UnscheduleMetadataWriteInternal(mFile);
1098         break;
1099       case SHUTDOWN:
1100         ioMan->ShutdownMetadataWriteSchedulingInternal();
1101         break;
1102     }
1103     return NS_OK;
1104   }
1105 };
1106
1107 StaticRefPtr<CacheFileIOManager> CacheFileIOManager::gInstance;
1108
1109 NS_IMPL_ISUPPORTS(CacheFileIOManager, nsITimerCallback, nsINamed)
1110
1111 CacheFileIOManager::CacheFileIOManager()
1112     : mShuttingDown(false),
1113       mTreeCreated(false),
1114       mTreeCreationFailed(false),
1115       mOverLimitEvicting(false),
1116       mCacheSizeOnHardLimit(false),
1117       mRemovingTrashDirs(false) {
1118   LOG(("CacheFileIOManager::CacheFileIOManager [this=%p]", this));
1119   MOZ_ASSERT(!gInstance, "multiple CacheFileIOManager instances!");
1120 }
1121
1122 CacheFileIOManager::~CacheFileIOManager() {
1123   LOG(("CacheFileIOManager::~CacheFileIOManager [this=%p]", this));
1124 }
1125
1126 // static
1127 nsresult CacheFileIOManager::Init() {
1128   LOG(("CacheFileIOManager::Init()"));
1129
1130   MOZ_ASSERT(NS_IsMainThread());
1131
1132   if (gInstance) {
1133     return NS_ERROR_ALREADY_INITIALIZED;
1134   }
1135
1136   RefPtr<CacheFileIOManager> ioMan = new CacheFileIOManager();
1137
1138   nsresult rv = ioMan->InitInternal();
1139   NS_ENSURE_SUCCESS(rv, rv);
1140
1141   gInstance = std::move(ioMan);
1142   return NS_OK;
1143 }
1144
1145 nsresult CacheFileIOManager::InitInternal() {
1146   nsresult rv;
1147
1148   mIOThread = new CacheIOThread();
1149
1150   rv = mIOThread->Init();
1151   MOZ_ASSERT(NS_SUCCEEDED(rv), "Can't create background thread");
1152   NS_ENSURE_SUCCESS(rv, rv);
1153
1154   mStartTime = TimeStamp::NowLoRes();
1155
1156   return NS_OK;
1157 }
1158
1159 // static
1160 nsresult CacheFileIOManager::Shutdown() {
1161   LOG(("CacheFileIOManager::Shutdown() [gInstance=%p]", gInstance.get()));
1162
1163   MOZ_ASSERT(NS_IsMainThread());
1164
1165   if (!gInstance) {
1166     return NS_ERROR_NOT_INITIALIZED;
1167   }
1168
1169   Telemetry::AutoTimer<Telemetry::NETWORK_DISK_CACHE_SHUTDOWN_V2> shutdownTimer;
1170
1171   CacheIndex::PreShutdown();
1172
1173   ShutdownMetadataWriteScheduling();
1174
1175   RefPtr<ShutdownEvent> ev = new ShutdownEvent();
1176   ev->PostAndWait();
1177
1178   MOZ_ASSERT(gInstance->mHandles.HandleCount() == 0);
1179   MOZ_ASSERT(gInstance->mHandlesByLastUsed.Length() == 0);
1180
1181   if (gInstance->mIOThread) {
1182     gInstance->mIOThread->Shutdown();
1183   }
1184
1185   CacheIndex::Shutdown();
1186
1187   if (CacheObserver::ClearCacheOnShutdown()) {
1188     Telemetry::AutoTimer<Telemetry::NETWORK_DISK_CACHE2_SHUTDOWN_CLEAR_PRIVATE>
1189         totalTimer;
1190     gInstance->SyncRemoveAllCacheFiles();
1191   }
1192
1193   gInstance = nullptr;
1194
1195   return NS_OK;
1196 }
1197
1198 void CacheFileIOManager::ShutdownInternal() {
1199   LOG(("CacheFileIOManager::ShutdownInternal() [this=%p]", this));
1200
1201   MOZ_ASSERT(mIOThread->IsCurrentThread());
1202
1203   // No new handles can be created after this flag is set
1204   mShuttingDown = true;
1205
1206   if (mTrashTimer) {
1207     mTrashTimer->Cancel();
1208     mTrashTimer = nullptr;
1209   }
1210
1211   // close all handles and delete all associated files
1212   nsTArray<RefPtr<CacheFileHandle> > handles;
1213   mHandles.GetAllHandles(&handles);
1214   handles.AppendElements(mSpecialHandles);
1215
1216   for (uint32_t i = 0; i < handles.Length(); i++) {
1217     CacheFileHandle* h = handles[i];
1218     h->mClosed = true;
1219
1220     h->Log();
1221
1222     // Close completely written files.
1223     MaybeReleaseNSPRHandleInternal(h);
1224     // Don't bother removing invalid and/or doomed files to improve
1225     // shutdown perfomrance.
1226     // Doomed files are already in the doomed directory from which
1227     // we never reuse files and delete the dir on next session startup.
1228     // Invalid files don't have metadata and thus won't load anyway
1229     // (hashes won't match).
1230
1231     if (!h->IsSpecialFile() && !h->mIsDoomed && !h->mFileExists) {
1232       CacheIndex::RemoveEntry(h->Hash());
1233     }
1234
1235     // Remove the handle from mHandles/mSpecialHandles
1236     if (h->IsSpecialFile()) {
1237       mSpecialHandles.RemoveElement(h);
1238     } else {
1239       mHandles.RemoveHandle(h);
1240     }
1241
1242     // Pointer to the hash is no longer valid once the last handle with the
1243     // given hash is released. Null out the pointer so that we crash if there
1244     // is a bug in this code and we dereference the pointer after this point.
1245     if (!h->IsSpecialFile()) {
1246       h->mHash = nullptr;
1247     }
1248   }
1249
1250   // Assert the table is empty. When we are here, no new handles can be added
1251   // and handles will no longer remove them self from this table and we don't
1252   // want to keep invalid handles here. Also, there is no lookup after this
1253   // point to happen.
1254   MOZ_ASSERT(mHandles.HandleCount() == 0);
1255
1256   // Release trash directory enumerator
1257   if (mTrashDirEnumerator) {
1258     mTrashDirEnumerator->Close();
1259     mTrashDirEnumerator = nullptr;
1260   }
1261
1262   if (mContextEvictor) {
1263     mContextEvictor->Shutdown();
1264     mContextEvictor = nullptr;
1265   }
1266 }
1267
1268 // static
1269 nsresult CacheFileIOManager::OnProfile() {
1270   LOG(("CacheFileIOManager::OnProfile() [gInstance=%p]", gInstance.get()));
1271
1272   RefPtr<CacheFileIOManager> ioMan = gInstance;
1273   if (!ioMan) {
1274     // CacheFileIOManager::Init() failed, probably could not create the IO
1275     // thread, just go with it...
1276     return NS_ERROR_NOT_INITIALIZED;
1277   }
1278
1279   nsresult rv;
1280
1281   nsCOMPtr<nsIFile> directory;
1282
1283   CacheObserver::ParentDirOverride(getter_AddRefs(directory));
1284
1285 #if defined(MOZ_WIDGET_ANDROID)
1286   nsCOMPtr<nsIFile> profilelessDirectory;
1287   char* cachePath = getenv("CACHE_DIRECTORY");
1288   if (!directory && cachePath && *cachePath) {
1289     rv = NS_NewNativeLocalFile(nsDependentCString(cachePath), true,
1290                                getter_AddRefs(directory));
1291     if (NS_SUCCEEDED(rv)) {
1292       // Save this directory as the profileless path.
1293       rv = directory->Clone(getter_AddRefs(profilelessDirectory));
1294       NS_ENSURE_SUCCESS(rv, rv);
1295
1296       // Add profile leaf name to the directory name to distinguish
1297       // multiple profiles Fennec supports.
1298       nsCOMPtr<nsIFile> profD;
1299       rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR,
1300                                   getter_AddRefs(profD));
1301
1302       nsAutoCString leafName;
1303       if (NS_SUCCEEDED(rv)) {
1304         rv = profD->GetNativeLeafName(leafName);
1305       }
1306       if (NS_SUCCEEDED(rv)) {
1307         rv = directory->AppendNative(leafName);
1308       }
1309       if (NS_FAILED(rv)) {
1310         directory = nullptr;
1311       }
1312     }
1313   }
1314 #endif
1315
1316   if (!directory) {
1317     rv = NS_GetSpecialDirectory(NS_APP_CACHE_PARENT_DIR,
1318                                 getter_AddRefs(directory));
1319   }
1320
1321   if (!directory) {
1322     rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_LOCAL_50_DIR,
1323                                 getter_AddRefs(directory));
1324   }
1325
1326   if (directory) {
1327     rv = directory->Append(u"cache2"_ns);
1328     NS_ENSURE_SUCCESS(rv, rv);
1329   }
1330
1331   // All functions return a clone.
1332   ioMan->mCacheDirectory.swap(directory);
1333
1334 #if defined(MOZ_WIDGET_ANDROID)
1335   if (profilelessDirectory) {
1336     rv = profilelessDirectory->Append(u"cache2"_ns);
1337     NS_ENSURE_SUCCESS(rv, rv);
1338   }
1339
1340   ioMan->mCacheProfilelessDirectory.swap(profilelessDirectory);
1341 #endif
1342
1343   if (ioMan->mCacheDirectory) {
1344     CacheIndex::Init(ioMan->mCacheDirectory);
1345   }
1346
1347   return NS_OK;
1348 }
1349
1350 // static
1351 already_AddRefed<nsIEventTarget> CacheFileIOManager::IOTarget() {
1352   nsCOMPtr<nsIEventTarget> target;
1353   if (gInstance && gInstance->mIOThread) {
1354     target = gInstance->mIOThread->Target();
1355   }
1356
1357   return target.forget();
1358 }
1359
1360 // static
1361 already_AddRefed<CacheIOThread> CacheFileIOManager::IOThread() {
1362   RefPtr<CacheIOThread> thread;
1363   if (gInstance) {
1364     thread = gInstance->mIOThread;
1365   }
1366
1367   return thread.forget();
1368 }
1369
1370 // static
1371 bool CacheFileIOManager::IsOnIOThread() {
1372   RefPtr<CacheFileIOManager> ioMan = gInstance;
1373   if (ioMan && ioMan->mIOThread) {
1374     return ioMan->mIOThread->IsCurrentThread();
1375   }
1376
1377   return false;
1378 }
1379
1380 // static
1381 bool CacheFileIOManager::IsOnIOThreadOrCeased() {
1382   RefPtr<CacheFileIOManager> ioMan = gInstance;
1383   if (ioMan && ioMan->mIOThread) {
1384     return ioMan->mIOThread->IsCurrentThread();
1385   }
1386
1387   // Ceased...
1388   return true;
1389 }
1390
1391 // static
1392 bool CacheFileIOManager::IsShutdown() {
1393   if (!gInstance) {
1394     return true;
1395   }
1396   return gInstance->mShuttingDown;
1397 }
1398
1399 // static
1400 nsresult CacheFileIOManager::ScheduleMetadataWrite(CacheFile* aFile) {
1401   RefPtr<CacheFileIOManager> ioMan = gInstance;
1402   NS_ENSURE_TRUE(ioMan, NS_ERROR_NOT_INITIALIZED);
1403
1404   NS_ENSURE_TRUE(!ioMan->mShuttingDown, NS_ERROR_NOT_INITIALIZED);
1405
1406   RefPtr<MetadataWriteScheduleEvent> event = new MetadataWriteScheduleEvent(
1407       ioMan, aFile, MetadataWriteScheduleEvent::SCHEDULE);
1408   nsCOMPtr<nsIEventTarget> target = ioMan->IOTarget();
1409   NS_ENSURE_TRUE(target, NS_ERROR_UNEXPECTED);
1410   return target->Dispatch(event, nsIEventTarget::DISPATCH_NORMAL);
1411 }
1412
1413 nsresult CacheFileIOManager::ScheduleMetadataWriteInternal(CacheFile* aFile) {
1414   MOZ_ASSERT(IsOnIOThreadOrCeased());
1415
1416   nsresult rv;
1417
1418   if (!mMetadataWritesTimer) {
1419     rv = NS_NewTimerWithCallback(getter_AddRefs(mMetadataWritesTimer), this,
1420                                  kMetadataWriteDelay, nsITimer::TYPE_ONE_SHOT);
1421     NS_ENSURE_SUCCESS(rv, rv);
1422   }
1423
1424   if (mScheduledMetadataWrites.IndexOf(aFile) !=
1425       mScheduledMetadataWrites.NoIndex) {
1426     return NS_OK;
1427   }
1428
1429   mScheduledMetadataWrites.AppendElement(aFile);
1430
1431   return NS_OK;
1432 }
1433
1434 // static
1435 nsresult CacheFileIOManager::UnscheduleMetadataWrite(CacheFile* aFile) {
1436   RefPtr<CacheFileIOManager> ioMan = gInstance;
1437   NS_ENSURE_TRUE(ioMan, NS_ERROR_NOT_INITIALIZED);
1438
1439   NS_ENSURE_TRUE(!ioMan->mShuttingDown, NS_ERROR_NOT_INITIALIZED);
1440
1441   RefPtr<MetadataWriteScheduleEvent> event = new MetadataWriteScheduleEvent(
1442       ioMan, aFile, MetadataWriteScheduleEvent::UNSCHEDULE);
1443   nsCOMPtr<nsIEventTarget> target = ioMan->IOTarget();
1444   NS_ENSURE_TRUE(target, NS_ERROR_UNEXPECTED);
1445   return target->Dispatch(event, nsIEventTarget::DISPATCH_NORMAL);
1446 }
1447
1448 void CacheFileIOManager::UnscheduleMetadataWriteInternal(CacheFile* aFile) {
1449   MOZ_ASSERT(IsOnIOThreadOrCeased());
1450
1451   mScheduledMetadataWrites.RemoveElement(aFile);
1452
1453   if (mScheduledMetadataWrites.Length() == 0 && mMetadataWritesTimer) {
1454     mMetadataWritesTimer->Cancel();
1455     mMetadataWritesTimer = nullptr;
1456   }
1457 }
1458
1459 // static
1460 nsresult CacheFileIOManager::ShutdownMetadataWriteScheduling() {
1461   RefPtr<CacheFileIOManager> ioMan = gInstance;
1462   NS_ENSURE_TRUE(ioMan, NS_ERROR_NOT_INITIALIZED);
1463
1464   RefPtr<MetadataWriteScheduleEvent> event = new MetadataWriteScheduleEvent(
1465       ioMan, nullptr, MetadataWriteScheduleEvent::SHUTDOWN);
1466   nsCOMPtr<nsIEventTarget> target = ioMan->IOTarget();
1467   NS_ENSURE_TRUE(target, NS_ERROR_UNEXPECTED);
1468   return target->Dispatch(event, nsIEventTarget::DISPATCH_NORMAL);
1469 }
1470
1471 void CacheFileIOManager::ShutdownMetadataWriteSchedulingInternal() {
1472   MOZ_ASSERT(IsOnIOThreadOrCeased());
1473
1474   nsTArray<RefPtr<CacheFile> > files = std::move(mScheduledMetadataWrites);
1475   for (uint32_t i = 0; i < files.Length(); ++i) {
1476     CacheFile* file = files[i];
1477     file->WriteMetadataIfNeeded();
1478   }
1479
1480   if (mMetadataWritesTimer) {
1481     mMetadataWritesTimer->Cancel();
1482     mMetadataWritesTimer = nullptr;
1483   }
1484 }
1485
1486 NS_IMETHODIMP
1487 CacheFileIOManager::Notify(nsITimer* aTimer) {
1488   MOZ_ASSERT(IsOnIOThreadOrCeased());
1489   MOZ_ASSERT(mMetadataWritesTimer == aTimer);
1490
1491   mMetadataWritesTimer = nullptr;
1492
1493   nsTArray<RefPtr<CacheFile> > files = std::move(mScheduledMetadataWrites);
1494   for (uint32_t i = 0; i < files.Length(); ++i) {
1495     CacheFile* file = files[i];
1496     file->WriteMetadataIfNeeded();
1497   }
1498
1499   return NS_OK;
1500 }
1501
1502 NS_IMETHODIMP
1503 CacheFileIOManager::GetName(nsACString& aName) {
1504   aName.AssignLiteral("CacheFileIOManager");
1505   return NS_OK;
1506 }
1507
1508 // static
1509 nsresult CacheFileIOManager::OpenFile(const nsACString& aKey, uint32_t aFlags,
1510                                       CacheFileIOListener* aCallback) {
1511   LOG(("CacheFileIOManager::OpenFile() [key=%s, flags=%d, listener=%p]",
1512        PromiseFlatCString(aKey).get(), aFlags, aCallback));
1513
1514   nsresult rv;
1515   RefPtr<CacheFileIOManager> ioMan = gInstance;
1516
1517   if (!ioMan) {
1518     return NS_ERROR_NOT_INITIALIZED;
1519   }
1520
1521   bool priority = aFlags & CacheFileIOManager::PRIORITY;
1522   RefPtr<OpenFileEvent> ev = new OpenFileEvent(aKey, aFlags, aCallback);
1523   rv = ioMan->mIOThread->Dispatch(
1524       ev, priority ? CacheIOThread::OPEN_PRIORITY : CacheIOThread::OPEN);
1525   NS_ENSURE_SUCCESS(rv, rv);
1526
1527   return NS_OK;
1528 }
1529
1530 nsresult CacheFileIOManager::OpenFileInternal(const SHA1Sum::Hash* aHash,
1531                                               const nsACString& aKey,
1532                                               uint32_t aFlags,
1533                                               CacheFileHandle** _retval) {
1534   LOG(
1535       ("CacheFileIOManager::OpenFileInternal() [hash=%08x%08x%08x%08x%08x, "
1536        "key=%s, flags=%d]",
1537        LOGSHA1(aHash), PromiseFlatCString(aKey).get(), aFlags));
1538
1539   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
1540
1541   nsresult rv;
1542
1543   if (mShuttingDown) {
1544     return NS_ERROR_NOT_INITIALIZED;
1545   }
1546
1547   CacheIOThread::Cancelable cancelable(
1548       true /* never called for special handles */);
1549
1550   if (!mTreeCreated) {
1551     rv = CreateCacheTree();
1552     if (NS_FAILED(rv)) return rv;
1553   }
1554
1555   CacheFileHandle::PinningStatus pinning =
1556       aFlags & PINNED ? CacheFileHandle::PinningStatus::PINNED
1557                       : CacheFileHandle::PinningStatus::NON_PINNED;
1558
1559   nsCOMPtr<nsIFile> file;
1560   rv = GetFile(aHash, getter_AddRefs(file));
1561   NS_ENSURE_SUCCESS(rv, rv);
1562
1563   RefPtr<CacheFileHandle> handle;
1564   mHandles.GetHandle(aHash, getter_AddRefs(handle));
1565
1566   if ((aFlags & (OPEN | CREATE | CREATE_NEW)) == CREATE_NEW) {
1567     if (handle) {
1568       rv = DoomFileInternal(handle);
1569       NS_ENSURE_SUCCESS(rv, rv);
1570       handle = nullptr;
1571     }
1572
1573     handle = mHandles.NewHandle(aHash, aFlags & PRIORITY, pinning);
1574
1575     bool exists;
1576     rv = file->Exists(&exists);
1577     NS_ENSURE_SUCCESS(rv, rv);
1578
1579     if (exists) {
1580       CacheIndex::RemoveEntry(aHash);
1581
1582       LOG(
1583           ("CacheFileIOManager::OpenFileInternal() - Removing old file from "
1584            "disk"));
1585       rv = file->Remove(false);
1586       if (NS_FAILED(rv)) {
1587         NS_WARNING("Cannot remove old entry from the disk");
1588         LOG(
1589             ("CacheFileIOManager::OpenFileInternal() - Removing old file failed"
1590              ". [rv=0x%08" PRIx32 "]",
1591              static_cast<uint32_t>(rv)));
1592       }
1593     }
1594
1595     CacheIndex::AddEntry(aHash);
1596     handle->mFile.swap(file);
1597     handle->mFileSize = 0;
1598   }
1599
1600   if (handle) {
1601     handle.swap(*_retval);
1602     return NS_OK;
1603   }
1604
1605   bool exists, evictedAsPinned = false, evictedAsNonPinned = false;
1606   rv = file->Exists(&exists);
1607   NS_ENSURE_SUCCESS(rv, rv);
1608
1609   if (exists && mContextEvictor) {
1610     if (mContextEvictor->ContextsCount() == 0) {
1611       mContextEvictor = nullptr;
1612     } else {
1613       mContextEvictor->WasEvicted(aKey, file, &evictedAsPinned,
1614                                   &evictedAsNonPinned);
1615     }
1616   }
1617
1618   if (!exists && (aFlags & (OPEN | CREATE | CREATE_NEW)) == OPEN) {
1619     return NS_ERROR_NOT_AVAILABLE;
1620   }
1621
1622   if (exists) {
1623     // For existing files we determine the pinning status later, after the
1624     // metadata gets parsed.
1625     pinning = CacheFileHandle::PinningStatus::UNKNOWN;
1626   }
1627
1628   handle = mHandles.NewHandle(aHash, aFlags & PRIORITY, pinning);
1629   if (exists) {
1630     // If this file has been found evicted through the context file evictor
1631     // above for any of pinned or non-pinned state, these calls ensure we doom
1632     // the handle ASAP we know the real pinning state after metadta has been
1633     // parsed.  DoomFileInternal on the |handle| doesn't doom right now, since
1634     // the pinning state is unknown and we pass down a pinning restriction.
1635     if (evictedAsPinned) {
1636       rv = DoomFileInternal(handle, DOOM_WHEN_PINNED);
1637       MOZ_ASSERT(!handle->IsDoomed() && NS_SUCCEEDED(rv));
1638     }
1639     if (evictedAsNonPinned) {
1640       rv = DoomFileInternal(handle, DOOM_WHEN_NON_PINNED);
1641       MOZ_ASSERT(!handle->IsDoomed() && NS_SUCCEEDED(rv));
1642     }
1643
1644     int64_t fileSize = -1;
1645     rv = file->GetFileSize(&fileSize);
1646     NS_ENSURE_SUCCESS(rv, rv);
1647
1648     handle->mFileSize = fileSize;
1649     handle->mFileExists = true;
1650
1651     CacheIndex::EnsureEntryExists(aHash);
1652   } else {
1653     handle->mFileSize = 0;
1654
1655     CacheIndex::AddEntry(aHash);
1656   }
1657
1658   handle->mFile.swap(file);
1659   handle.swap(*_retval);
1660   return NS_OK;
1661 }
1662
1663 nsresult CacheFileIOManager::OpenSpecialFileInternal(
1664     const nsACString& aKey, uint32_t aFlags, CacheFileHandle** _retval) {
1665   LOG(("CacheFileIOManager::OpenSpecialFileInternal() [key=%s, flags=%d]",
1666        PromiseFlatCString(aKey).get(), aFlags));
1667
1668   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
1669
1670   nsresult rv;
1671
1672   if (mShuttingDown) {
1673     return NS_ERROR_NOT_INITIALIZED;
1674   }
1675
1676   if (!mTreeCreated) {
1677     rv = CreateCacheTree();
1678     if (NS_FAILED(rv)) return rv;
1679   }
1680
1681   nsCOMPtr<nsIFile> file;
1682   rv = GetSpecialFile(aKey, getter_AddRefs(file));
1683   NS_ENSURE_SUCCESS(rv, rv);
1684
1685   RefPtr<CacheFileHandle> handle;
1686   for (uint32_t i = 0; i < mSpecialHandles.Length(); i++) {
1687     if (!mSpecialHandles[i]->IsDoomed() && mSpecialHandles[i]->Key() == aKey) {
1688       handle = mSpecialHandles[i];
1689       break;
1690     }
1691   }
1692
1693   if ((aFlags & (OPEN | CREATE | CREATE_NEW)) == CREATE_NEW) {
1694     if (handle) {
1695       rv = DoomFileInternal(handle);
1696       NS_ENSURE_SUCCESS(rv, rv);
1697       handle = nullptr;
1698     }
1699
1700     handle = new CacheFileHandle(aKey, aFlags & PRIORITY,
1701                                  CacheFileHandle::PinningStatus::NON_PINNED);
1702     mSpecialHandles.AppendElement(handle);
1703
1704     bool exists;
1705     rv = file->Exists(&exists);
1706     NS_ENSURE_SUCCESS(rv, rv);
1707
1708     if (exists) {
1709       LOG(
1710           ("CacheFileIOManager::OpenSpecialFileInternal() - Removing file from "
1711            "disk"));
1712       rv = file->Remove(false);
1713       if (NS_FAILED(rv)) {
1714         NS_WARNING("Cannot remove old entry from the disk");
1715         LOG(
1716             ("CacheFileIOManager::OpenSpecialFileInternal() - Removing file "
1717              "failed. [rv=0x%08" PRIx32 "]",
1718              static_cast<uint32_t>(rv)));
1719       }
1720     }
1721
1722     handle->mFile.swap(file);
1723     handle->mFileSize = 0;
1724   }
1725
1726   if (handle) {
1727     handle.swap(*_retval);
1728     return NS_OK;
1729   }
1730
1731   bool exists;
1732   rv = file->Exists(&exists);
1733   NS_ENSURE_SUCCESS(rv, rv);
1734
1735   if (!exists && (aFlags & (OPEN | CREATE | CREATE_NEW)) == OPEN) {
1736     return NS_ERROR_NOT_AVAILABLE;
1737   }
1738
1739   handle = new CacheFileHandle(aKey, aFlags & PRIORITY,
1740                                CacheFileHandle::PinningStatus::NON_PINNED);
1741   mSpecialHandles.AppendElement(handle);
1742
1743   if (exists) {
1744     int64_t fileSize = -1;
1745     rv = file->GetFileSize(&fileSize);
1746     NS_ENSURE_SUCCESS(rv, rv);
1747
1748     handle->mFileSize = fileSize;
1749     handle->mFileExists = true;
1750   } else {
1751     handle->mFileSize = 0;
1752   }
1753
1754   handle->mFile.swap(file);
1755   handle.swap(*_retval);
1756   return NS_OK;
1757 }
1758
1759 void CacheFileIOManager::CloseHandleInternal(CacheFileHandle* aHandle) {
1760   nsresult rv;
1761   LOG(("CacheFileIOManager::CloseHandleInternal() [handle=%p]", aHandle));
1762
1763   MOZ_ASSERT(!aHandle->IsClosed());
1764
1765   aHandle->Log();
1766
1767   MOZ_ASSERT(CacheFileIOManager::IsOnIOThreadOrCeased());
1768
1769   CacheIOThread::Cancelable cancelable(!aHandle->IsSpecialFile());
1770
1771   // Maybe close file handle (can be legally bypassed after shutdown)
1772   rv = MaybeReleaseNSPRHandleInternal(aHandle);
1773
1774   // Delete the file if the entry was doomed or invalid and
1775   // filedesc properly closed
1776   if ((aHandle->mIsDoomed || aHandle->mInvalid) && aHandle->mFileExists &&
1777       NS_SUCCEEDED(rv)) {
1778     LOG(
1779         ("CacheFileIOManager::CloseHandleInternal() - Removing file from "
1780          "disk"));
1781
1782     rv = aHandle->mFile->Remove(false);
1783     if (NS_SUCCEEDED(rv)) {
1784       aHandle->mFileExists = false;
1785     } else {
1786       LOG(("  failed to remove the file [rv=0x%08" PRIx32 "]",
1787            static_cast<uint32_t>(rv)));
1788     }
1789   }
1790
1791   if (!aHandle->IsSpecialFile() && !aHandle->mIsDoomed &&
1792       (aHandle->mInvalid || !aHandle->mFileExists)) {
1793     CacheIndex::RemoveEntry(aHandle->Hash());
1794   }
1795
1796   // Don't remove handles after shutdown
1797   if (!mShuttingDown) {
1798     if (aHandle->IsSpecialFile()) {
1799       mSpecialHandles.RemoveElement(aHandle);
1800     } else {
1801       mHandles.RemoveHandle(aHandle);
1802     }
1803   }
1804 }
1805
1806 // static
1807 nsresult CacheFileIOManager::Read(CacheFileHandle* aHandle, int64_t aOffset,
1808                                   char* aBuf, int32_t aCount,
1809                                   CacheFileIOListener* aCallback) {
1810   LOG(("CacheFileIOManager::Read() [handle=%p, offset=%" PRId64 ", count=%d, "
1811        "listener=%p]",
1812        aHandle, aOffset, aCount, aCallback));
1813
1814   if (CacheObserver::ShuttingDown()) {
1815     LOG(("  no reads after shutdown"));
1816     return NS_ERROR_NOT_INITIALIZED;
1817   }
1818
1819   nsresult rv;
1820   RefPtr<CacheFileIOManager> ioMan = gInstance;
1821
1822   if (aHandle->IsClosed() || !ioMan) {
1823     return NS_ERROR_NOT_INITIALIZED;
1824   }
1825
1826   RefPtr<ReadEvent> ev =
1827       new ReadEvent(aHandle, aOffset, aBuf, aCount, aCallback);
1828   rv = ioMan->mIOThread->Dispatch(ev, aHandle->IsPriority()
1829                                           ? CacheIOThread::READ_PRIORITY
1830                                           : CacheIOThread::READ);
1831   NS_ENSURE_SUCCESS(rv, rv);
1832
1833   return NS_OK;
1834 }
1835
1836 nsresult CacheFileIOManager::ReadInternal(CacheFileHandle* aHandle,
1837                                           int64_t aOffset, char* aBuf,
1838                                           int32_t aCount) {
1839   LOG(("CacheFileIOManager::ReadInternal() [handle=%p, offset=%" PRId64
1840        ", count=%d]",
1841        aHandle, aOffset, aCount));
1842
1843   nsresult rv;
1844
1845   if (CacheObserver::ShuttingDown()) {
1846     LOG(("  no reads after shutdown"));
1847     return NS_ERROR_NOT_INITIALIZED;
1848   }
1849
1850   if (!aHandle->mFileExists) {
1851     NS_WARNING("Trying to read from non-existent file");
1852     return NS_ERROR_NOT_AVAILABLE;
1853   }
1854
1855   CacheIOThread::Cancelable cancelable(!aHandle->IsSpecialFile());
1856
1857   if (!aHandle->mFD) {
1858     rv = OpenNSPRHandle(aHandle);
1859     NS_ENSURE_SUCCESS(rv, rv);
1860   } else {
1861     NSPRHandleUsed(aHandle);
1862   }
1863
1864   // Check again, OpenNSPRHandle could figure out the file was gone.
1865   if (!aHandle->mFileExists) {
1866     NS_WARNING("Trying to read from non-existent file");
1867     return NS_ERROR_NOT_AVAILABLE;
1868   }
1869
1870   int64_t offset = PR_Seek64(aHandle->mFD, aOffset, PR_SEEK_SET);
1871   if (offset == -1) {
1872     return NS_ERROR_FAILURE;
1873   }
1874
1875   int32_t bytesRead = PR_Read(aHandle->mFD, aBuf, aCount);
1876   if (bytesRead != aCount) {
1877     return NS_ERROR_FAILURE;
1878   }
1879
1880   return NS_OK;
1881 }
1882
1883 // static
1884 nsresult CacheFileIOManager::Write(CacheFileHandle* aHandle, int64_t aOffset,
1885                                    const char* aBuf, int32_t aCount,
1886                                    bool aValidate, bool aTruncate,
1887                                    CacheFileIOListener* aCallback) {
1888   LOG(("CacheFileIOManager::Write() [handle=%p, offset=%" PRId64 ", count=%d, "
1889        "validate=%d, truncate=%d, listener=%p]",
1890        aHandle, aOffset, aCount, aValidate, aTruncate, aCallback));
1891
1892   nsresult rv;
1893   RefPtr<CacheFileIOManager> ioMan = gInstance;
1894
1895   if (aHandle->IsClosed() || (aCallback && aCallback->IsKilled()) || !ioMan) {
1896     if (!aCallback) {
1897       // When no callback is provided, CacheFileIOManager is responsible for
1898       // releasing the buffer. We must release it even in case of failure.
1899       free(const_cast<char*>(aBuf));
1900     }
1901     return NS_ERROR_NOT_INITIALIZED;
1902   }
1903
1904   RefPtr<WriteEvent> ev = new WriteEvent(aHandle, aOffset, aBuf, aCount,
1905                                          aValidate, aTruncate, aCallback);
1906   rv = ioMan->mIOThread->Dispatch(ev, aHandle->mPriority
1907                                           ? CacheIOThread::WRITE_PRIORITY
1908                                           : CacheIOThread::WRITE);
1909   NS_ENSURE_SUCCESS(rv, rv);
1910
1911   return NS_OK;
1912 }
1913
1914 static nsresult TruncFile(PRFileDesc* aFD, int64_t aEOF) {
1915 #if defined(XP_UNIX)
1916   if (ftruncate(PR_FileDesc2NativeHandle(aFD), aEOF) != 0) {
1917     NS_ERROR("ftruncate failed");
1918     return NS_ERROR_FAILURE;
1919   }
1920 #elif defined(XP_WIN)
1921   int64_t cnt = PR_Seek64(aFD, aEOF, PR_SEEK_SET);
1922   if (cnt == -1) {
1923     return NS_ERROR_FAILURE;
1924   }
1925   if (!SetEndOfFile((HANDLE)PR_FileDesc2NativeHandle(aFD))) {
1926     NS_ERROR("SetEndOfFile failed");
1927     return NS_ERROR_FAILURE;
1928   }
1929 #else
1930   MOZ_ASSERT(false, "Not implemented!");
1931   return NS_ERROR_NOT_IMPLEMENTED;
1932 #endif
1933
1934   return NS_OK;
1935 }
1936
1937 nsresult CacheFileIOManager::WriteInternal(CacheFileHandle* aHandle,
1938                                            int64_t aOffset, const char* aBuf,
1939                                            int32_t aCount, bool aValidate,
1940                                            bool aTruncate) {
1941   LOG(("CacheFileIOManager::WriteInternal() [handle=%p, offset=%" PRId64
1942        ", count=%d, "
1943        "validate=%d, truncate=%d]",
1944        aHandle, aOffset, aCount, aValidate, aTruncate));
1945
1946   nsresult rv;
1947
1948   if (aHandle->mKilled) {
1949     LOG(("  handle already killed, nothing written"));
1950     return NS_OK;
1951   }
1952
1953   if (CacheObserver::ShuttingDown() && (!aValidate || !aHandle->mFD)) {
1954     aHandle->mKilled = true;
1955     LOG(("  killing the handle, nothing written"));
1956     return NS_OK;
1957   }
1958
1959   if (CacheObserver::IsPastShutdownIOLag()) {
1960     LOG(("  past the shutdown I/O lag, nothing written"));
1961     // Pretend the write has succeeded, otherwise upper layers will doom
1962     // the file and we end up with I/O anyway.
1963     return NS_OK;
1964   }
1965
1966   CacheIOThread::Cancelable cancelable(!aHandle->IsSpecialFile());
1967
1968   if (!aHandle->mFileExists) {
1969     rv = CreateFile(aHandle);
1970     NS_ENSURE_SUCCESS(rv, rv);
1971   }
1972
1973   if (!aHandle->mFD) {
1974     rv = OpenNSPRHandle(aHandle);
1975     NS_ENSURE_SUCCESS(rv, rv);
1976   } else {
1977     NSPRHandleUsed(aHandle);
1978   }
1979
1980   // Check again, OpenNSPRHandle could figure out the file was gone.
1981   if (!aHandle->mFileExists) {
1982     return NS_ERROR_NOT_AVAILABLE;
1983   }
1984
1985   // When this operation would increase cache size, check whether the cache size
1986   // reached the hard limit and whether it would cause critical low disk space.
1987   if (aHandle->mFileSize < aOffset + aCount) {
1988     if (mOverLimitEvicting && mCacheSizeOnHardLimit) {
1989       LOG(
1990           ("CacheFileIOManager::WriteInternal() - failing because cache size "
1991            "reached hard limit!"));
1992       return NS_ERROR_FILE_DISK_FULL;
1993     }
1994
1995     int64_t freeSpace;
1996     rv = mCacheDirectory->GetDiskSpaceAvailable(&freeSpace);
1997     if (NS_WARN_IF(NS_FAILED(rv))) {
1998       freeSpace = -1;
1999       LOG(
2000           ("CacheFileIOManager::WriteInternal() - GetDiskSpaceAvailable() "
2001            "failed! [rv=0x%08" PRIx32 "]",
2002            static_cast<uint32_t>(rv)));
2003     } else {
2004       freeSpace >>= 10;  // bytes to kilobytes
2005       uint32_t limit = CacheObserver::DiskFreeSpaceHardLimit();
2006       if (freeSpace - aOffset - aCount + aHandle->mFileSize < limit) {
2007         LOG(
2008             ("CacheFileIOManager::WriteInternal() - Low free space, refusing "
2009              "to write! [freeSpace=%" PRId64 "kB, limit=%ukB]",
2010              freeSpace, limit));
2011         return NS_ERROR_FILE_DISK_FULL;
2012       }
2013     }
2014   }
2015
2016   // Write invalidates the entry by default
2017   aHandle->mInvalid = true;
2018
2019   int64_t offset = PR_Seek64(aHandle->mFD, aOffset, PR_SEEK_SET);
2020   if (offset == -1) {
2021     return NS_ERROR_FAILURE;
2022   }
2023
2024   int32_t bytesWritten = PR_Write(aHandle->mFD, aBuf, aCount);
2025
2026   if (bytesWritten != -1) {
2027     uint32_t oldSizeInK = aHandle->FileSizeInK();
2028     int64_t writeEnd = aOffset + bytesWritten;
2029
2030     if (aTruncate) {
2031       rv = TruncFile(aHandle->mFD, writeEnd);
2032       NS_ENSURE_SUCCESS(rv, rv);
2033
2034       aHandle->mFileSize = writeEnd;
2035     } else {
2036       if (aHandle->mFileSize < writeEnd) {
2037         aHandle->mFileSize = writeEnd;
2038       }
2039     }
2040
2041     uint32_t newSizeInK = aHandle->FileSizeInK();
2042
2043     if (oldSizeInK != newSizeInK && !aHandle->IsDoomed() &&
2044         !aHandle->IsSpecialFile()) {
2045       CacheIndex::UpdateEntry(aHandle->Hash(), nullptr, nullptr, nullptr,
2046                               nullptr, nullptr, &newSizeInK);
2047
2048       if (oldSizeInK < newSizeInK) {
2049         EvictIfOverLimitInternal();
2050       }
2051     }
2052
2053     CacheIndex::UpdateTotalBytesWritten(bytesWritten);
2054   }
2055
2056   if (bytesWritten != aCount) {
2057     return NS_ERROR_FAILURE;
2058   }
2059
2060   // Write was successful and this write validates the entry (i.e. metadata)
2061   if (aValidate) {
2062     aHandle->mInvalid = false;
2063   }
2064
2065   return NS_OK;
2066 }
2067
2068 // static
2069 nsresult CacheFileIOManager::DoomFile(CacheFileHandle* aHandle,
2070                                       CacheFileIOListener* aCallback) {
2071   LOG(("CacheFileIOManager::DoomFile() [handle=%p, listener=%p]", aHandle,
2072        aCallback));
2073
2074   nsresult rv;
2075   RefPtr<CacheFileIOManager> ioMan = gInstance;
2076
2077   if (aHandle->IsClosed() || !ioMan) {
2078     return NS_ERROR_NOT_INITIALIZED;
2079   }
2080
2081   RefPtr<DoomFileEvent> ev = new DoomFileEvent(aHandle, aCallback);
2082   rv = ioMan->mIOThread->Dispatch(ev, aHandle->IsPriority()
2083                                           ? CacheIOThread::OPEN_PRIORITY
2084                                           : CacheIOThread::OPEN);
2085   NS_ENSURE_SUCCESS(rv, rv);
2086
2087   return NS_OK;
2088 }
2089
2090 nsresult CacheFileIOManager::DoomFileInternal(
2091     CacheFileHandle* aHandle, PinningDoomRestriction aPinningDoomRestriction) {
2092   LOG(("CacheFileIOManager::DoomFileInternal() [handle=%p]", aHandle));
2093   aHandle->Log();
2094
2095   MOZ_ASSERT(CacheFileIOManager::IsOnIOThreadOrCeased());
2096
2097   nsresult rv;
2098
2099   if (aHandle->IsDoomed()) {
2100     return NS_OK;
2101   }
2102
2103   CacheIOThread::Cancelable cancelable(!aHandle->IsSpecialFile());
2104
2105   if (aPinningDoomRestriction > NO_RESTRICTION) {
2106     switch (aHandle->mPinning) {
2107       case CacheFileHandle::PinningStatus::NON_PINNED:
2108         if (MOZ_LIKELY(aPinningDoomRestriction != DOOM_WHEN_NON_PINNED)) {
2109           LOG(("  not dooming, it's a non-pinned handle"));
2110           return NS_OK;
2111         }
2112         // Doom now
2113         break;
2114
2115       case CacheFileHandle::PinningStatus::PINNED:
2116         if (MOZ_UNLIKELY(aPinningDoomRestriction != DOOM_WHEN_PINNED)) {
2117           LOG(("  not dooming, it's a pinned handle"));
2118           return NS_OK;
2119         }
2120         // Doom now
2121         break;
2122
2123       case CacheFileHandle::PinningStatus::UNKNOWN:
2124         if (MOZ_LIKELY(aPinningDoomRestriction == DOOM_WHEN_NON_PINNED)) {
2125           LOG(("  doom when non-pinned set"));
2126           aHandle->mDoomWhenFoundNonPinned = true;
2127         } else if (MOZ_UNLIKELY(aPinningDoomRestriction == DOOM_WHEN_PINNED)) {
2128           LOG(("  doom when pinned set"));
2129           aHandle->mDoomWhenFoundPinned = true;
2130         }
2131
2132         LOG(("  pinning status not known, deferring doom decision"));
2133         return NS_OK;
2134     }
2135   }
2136
2137   if (aHandle->mFileExists) {
2138     // we need to move the current file to the doomed directory
2139     rv = MaybeReleaseNSPRHandleInternal(aHandle, true);
2140     NS_ENSURE_SUCCESS(rv, rv);
2141
2142     // find unused filename
2143     nsCOMPtr<nsIFile> file;
2144     rv = GetDoomedFile(getter_AddRefs(file));
2145     NS_ENSURE_SUCCESS(rv, rv);
2146
2147     nsCOMPtr<nsIFile> parentDir;
2148     rv = file->GetParent(getter_AddRefs(parentDir));
2149     NS_ENSURE_SUCCESS(rv, rv);
2150
2151     nsAutoCString leafName;
2152     rv = file->GetNativeLeafName(leafName);
2153     NS_ENSURE_SUCCESS(rv, rv);
2154
2155     rv = aHandle->mFile->MoveToNative(parentDir, leafName);
2156     if (NS_ERROR_FILE_NOT_FOUND == rv ||
2157         NS_ERROR_FILE_TARGET_DOES_NOT_EXIST == rv) {
2158       LOG(("  file already removed under our hands"));
2159       aHandle->mFileExists = false;
2160       rv = NS_OK;
2161     } else {
2162       NS_ENSURE_SUCCESS(rv, rv);
2163       aHandle->mFile.swap(file);
2164     }
2165   }
2166
2167   if (!aHandle->IsSpecialFile()) {
2168     CacheIndex::RemoveEntry(aHandle->Hash());
2169   }
2170
2171   aHandle->mIsDoomed = true;
2172
2173   if (!aHandle->IsSpecialFile()) {
2174     RefPtr<CacheStorageService> storageService = CacheStorageService::Self();
2175     if (storageService) {
2176       nsAutoCString idExtension, url;
2177       nsCOMPtr<nsILoadContextInfo> info =
2178           CacheFileUtils::ParseKey(aHandle->Key(), &idExtension, &url);
2179       MOZ_ASSERT(info);
2180       if (info) {
2181         storageService->CacheFileDoomed(info, idExtension, url);
2182       }
2183     }
2184   }
2185
2186   return NS_OK;
2187 }
2188
2189 // static
2190 nsresult CacheFileIOManager::DoomFileByKey(const nsACString& aKey,
2191                                            CacheFileIOListener* aCallback) {
2192   LOG(("CacheFileIOManager::DoomFileByKey() [key=%s, listener=%p]",
2193        PromiseFlatCString(aKey).get(), aCallback));
2194
2195   nsresult rv;
2196   RefPtr<CacheFileIOManager> ioMan = gInstance;
2197
2198   if (!ioMan) {
2199     return NS_ERROR_NOT_INITIALIZED;
2200   }
2201
2202   RefPtr<DoomFileByKeyEvent> ev = new DoomFileByKeyEvent(aKey, aCallback);
2203   rv = ioMan->mIOThread->DispatchAfterPendingOpens(ev);
2204   NS_ENSURE_SUCCESS(rv, rv);
2205
2206   return NS_OK;
2207 }
2208
2209 nsresult CacheFileIOManager::DoomFileByKeyInternal(const SHA1Sum::Hash* aHash) {
2210   LOG((
2211       "CacheFileIOManager::DoomFileByKeyInternal() [hash=%08x%08x%08x%08x%08x]",
2212       LOGSHA1(aHash)));
2213
2214   MOZ_ASSERT(CacheFileIOManager::IsOnIOThreadOrCeased());
2215
2216   nsresult rv;
2217
2218   if (mShuttingDown) {
2219     return NS_ERROR_NOT_INITIALIZED;
2220   }
2221
2222   if (!mCacheDirectory) {
2223     return NS_ERROR_FILE_INVALID_PATH;
2224   }
2225
2226   // Find active handle
2227   RefPtr<CacheFileHandle> handle;
2228   mHandles.GetHandle(aHash, getter_AddRefs(handle));
2229
2230   if (handle) {
2231     handle->Log();
2232
2233     return DoomFileInternal(handle);
2234   }
2235
2236   CacheIOThread::Cancelable cancelable(true);
2237
2238   // There is no handle for this file, delete the file if exists
2239   nsCOMPtr<nsIFile> file;
2240   rv = GetFile(aHash, getter_AddRefs(file));
2241   NS_ENSURE_SUCCESS(rv, rv);
2242
2243   bool exists;
2244   rv = file->Exists(&exists);
2245   NS_ENSURE_SUCCESS(rv, rv);
2246
2247   if (!exists) {
2248     return NS_ERROR_NOT_AVAILABLE;
2249   }
2250
2251   LOG(
2252       ("CacheFileIOManager::DoomFileByKeyInternal() - Removing file from "
2253        "disk"));
2254   rv = file->Remove(false);
2255   if (NS_FAILED(rv)) {
2256     NS_WARNING("Cannot remove old entry from the disk");
2257     LOG(
2258         ("CacheFileIOManager::DoomFileByKeyInternal() - Removing file failed. "
2259          "[rv=0x%08" PRIx32 "]",
2260          static_cast<uint32_t>(rv)));
2261   }
2262
2263   CacheIndex::RemoveEntry(aHash);
2264
2265   return NS_OK;
2266 }
2267
2268 // static
2269 nsresult CacheFileIOManager::ReleaseNSPRHandle(CacheFileHandle* aHandle) {
2270   LOG(("CacheFileIOManager::ReleaseNSPRHandle() [handle=%p]", aHandle));
2271
2272   nsresult rv;
2273   RefPtr<CacheFileIOManager> ioMan = gInstance;
2274
2275   if (aHandle->IsClosed() || !ioMan) {
2276     return NS_ERROR_NOT_INITIALIZED;
2277   }
2278
2279   RefPtr<ReleaseNSPRHandleEvent> ev = new ReleaseNSPRHandleEvent(aHandle);
2280   rv = ioMan->mIOThread->Dispatch(ev, aHandle->mPriority
2281                                           ? CacheIOThread::WRITE_PRIORITY
2282                                           : CacheIOThread::WRITE);
2283   NS_ENSURE_SUCCESS(rv, rv);
2284
2285   return NS_OK;
2286 }
2287
2288 nsresult CacheFileIOManager::MaybeReleaseNSPRHandleInternal(
2289     CacheFileHandle* aHandle, bool aIgnoreShutdownLag) {
2290   LOG(
2291       ("CacheFileIOManager::MaybeReleaseNSPRHandleInternal() [handle=%p, "
2292        "ignore shutdown=%d]",
2293        aHandle, aIgnoreShutdownLag));
2294
2295   MOZ_ASSERT(CacheFileIOManager::IsOnIOThreadOrCeased());
2296
2297   if (aHandle->mFD) {
2298     DebugOnly<bool> found;
2299     found = mHandlesByLastUsed.RemoveElement(aHandle);
2300     MOZ_ASSERT(found);
2301   }
2302
2303   PRFileDesc* fd = aHandle->mFD;
2304   aHandle->mFD = nullptr;
2305
2306   // Leak invalid (w/o metadata) and doomed handles immediately after shutdown.
2307   // Leak other handles when past the shutdown time maximum lag.
2308   if (
2309 #ifndef DEBUG
2310       ((aHandle->mInvalid || aHandle->mIsDoomed) &&
2311        MOZ_UNLIKELY(CacheObserver::ShuttingDown())) ||
2312 #endif
2313       MOZ_UNLIKELY(!aIgnoreShutdownLag &&
2314                    CacheObserver::IsPastShutdownIOLag())) {
2315     // Don't bother closing this file.  Return a failure code from here will
2316     // cause any following IO operation on the file (mainly removal) to be
2317     // bypassed, which is what we want.
2318     // For mInvalid == true the entry will never be used, since it doesn't
2319     // have correct metadata, thus we don't need to worry about removing it.
2320     // For mIsDoomed == true the file is already in the doomed sub-dir and
2321     // will be removed on next session start.
2322     LOG(("  past the shutdown I/O lag, leaking file handle"));
2323     return NS_ERROR_ILLEGAL_DURING_SHUTDOWN;
2324   }
2325
2326   if (!fd) {
2327     // The filedesc has already been closed before, just let go.
2328     return NS_OK;
2329   }
2330
2331   CacheIOThread::Cancelable cancelable(!aHandle->IsSpecialFile());
2332
2333   PRStatus status = PR_Close(fd);
2334   if (status != PR_SUCCESS) {
2335     LOG(
2336         ("CacheFileIOManager::MaybeReleaseNSPRHandleInternal() "
2337          "failed to close [handle=%p, status=%u]",
2338          aHandle, status));
2339     return NS_ERROR_FAILURE;
2340   }
2341
2342   LOG(("CacheFileIOManager::MaybeReleaseNSPRHandleInternal() DONE"));
2343
2344   return NS_OK;
2345 }
2346
2347 // static
2348 nsresult CacheFileIOManager::TruncateSeekSetEOF(
2349     CacheFileHandle* aHandle, int64_t aTruncatePos, int64_t aEOFPos,
2350     CacheFileIOListener* aCallback) {
2351   LOG(
2352       ("CacheFileIOManager::TruncateSeekSetEOF() [handle=%p, "
2353        "truncatePos=%" PRId64 ", "
2354        "EOFPos=%" PRId64 ", listener=%p]",
2355        aHandle, aTruncatePos, aEOFPos, aCallback));
2356
2357   nsresult rv;
2358   RefPtr<CacheFileIOManager> ioMan = gInstance;
2359
2360   if (aHandle->IsClosed() || (aCallback && aCallback->IsKilled()) || !ioMan) {
2361     return NS_ERROR_NOT_INITIALIZED;
2362   }
2363
2364   RefPtr<TruncateSeekSetEOFEvent> ev =
2365       new TruncateSeekSetEOFEvent(aHandle, aTruncatePos, aEOFPos, aCallback);
2366   rv = ioMan->mIOThread->Dispatch(ev, aHandle->mPriority
2367                                           ? CacheIOThread::WRITE_PRIORITY
2368                                           : CacheIOThread::WRITE);
2369   NS_ENSURE_SUCCESS(rv, rv);
2370
2371   return NS_OK;
2372 }
2373
2374 // static
2375 void CacheFileIOManager::GetCacheDirectory(nsIFile** result) {
2376   *result = nullptr;
2377
2378   RefPtr<CacheFileIOManager> ioMan = gInstance;
2379   if (!ioMan || !ioMan->mCacheDirectory) {
2380     return;
2381   }
2382
2383   ioMan->mCacheDirectory->Clone(result);
2384 }
2385
2386 #if defined(MOZ_WIDGET_ANDROID)
2387
2388 // static
2389 void CacheFileIOManager::GetProfilelessCacheDirectory(nsIFile** result) {
2390   *result = nullptr;
2391
2392   RefPtr<CacheFileIOManager> ioMan = gInstance;
2393   if (!ioMan || !ioMan->mCacheProfilelessDirectory) {
2394     return;
2395   }
2396
2397   ioMan->mCacheProfilelessDirectory->Clone(result);
2398 }
2399
2400 #endif
2401
2402 // static
2403 nsresult CacheFileIOManager::GetEntryInfo(
2404     const SHA1Sum::Hash* aHash,
2405     CacheStorageService::EntryInfoCallback* aCallback) {
2406   MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
2407
2408   nsresult rv;
2409
2410   RefPtr<CacheFileIOManager> ioMan = gInstance;
2411   if (!ioMan) {
2412     return NS_ERROR_NOT_INITIALIZED;
2413   }
2414
2415   nsAutoCString enhanceId;
2416   nsAutoCString uriSpec;
2417
2418   RefPtr<CacheFileHandle> handle;
2419   ioMan->mHandles.GetHandle(aHash, getter_AddRefs(handle));
2420   if (handle) {
2421     RefPtr<nsILoadContextInfo> info =
2422         CacheFileUtils::ParseKey(handle->Key(), &enhanceId, &uriSpec);
2423
2424     MOZ_ASSERT(info);
2425     if (!info) {
2426       return NS_OK;  // ignore
2427     }
2428
2429     RefPtr<CacheStorageService> service = CacheStorageService::Self();
2430     if (!service) {
2431       return NS_ERROR_NOT_INITIALIZED;
2432     }
2433
2434     // Invokes OnCacheEntryInfo when an existing entry is found
2435     if (service->GetCacheEntryInfo(info, enhanceId, uriSpec, aCallback)) {
2436       return NS_OK;
2437     }
2438
2439     // When we are here, there is no existing entry and we need
2440     // to synchrnously load metadata from a disk file.
2441   }
2442
2443   // Locate the actual file
2444   nsCOMPtr<nsIFile> file;
2445   ioMan->GetFile(aHash, getter_AddRefs(file));
2446
2447   // Read metadata from the file synchronously
2448   RefPtr<CacheFileMetadata> metadata = new CacheFileMetadata();
2449   rv = metadata->SyncReadMetadata(file);
2450   if (NS_FAILED(rv)) {
2451     return NS_OK;
2452   }
2453
2454   // Now get the context + enhance id + URL from the key.
2455   RefPtr<nsILoadContextInfo> info =
2456       CacheFileUtils::ParseKey(metadata->GetKey(), &enhanceId, &uriSpec);
2457   MOZ_ASSERT(info);
2458   if (!info) {
2459     return NS_OK;
2460   }
2461
2462   // Pick all data to pass to the callback.
2463   int64_t dataSize = metadata->Offset();
2464   uint32_t fetchCount = metadata->GetFetchCount();
2465   uint32_t expirationTime = metadata->GetExpirationTime();
2466   uint32_t lastModified = metadata->GetLastModified();
2467
2468   // Call directly on the callback.
2469   aCallback->OnEntryInfo(uriSpec, enhanceId, dataSize, fetchCount, lastModified,
2470                          expirationTime, metadata->Pinned(), info);
2471
2472   return NS_OK;
2473 }
2474
2475 nsresult CacheFileIOManager::TruncateSeekSetEOFInternal(
2476     CacheFileHandle* aHandle, int64_t aTruncatePos, int64_t aEOFPos) {
2477   LOG(
2478       ("CacheFileIOManager::TruncateSeekSetEOFInternal() [handle=%p, "
2479        "truncatePos=%" PRId64 ", EOFPos=%" PRId64 "]",
2480        aHandle, aTruncatePos, aEOFPos));
2481
2482   nsresult rv;
2483
2484   if (aHandle->mKilled) {
2485     LOG(("  handle already killed, file not truncated"));
2486     return NS_OK;
2487   }
2488
2489   if (CacheObserver::ShuttingDown() && !aHandle->mFD) {
2490     aHandle->mKilled = true;
2491     LOG(("  killing the handle, file not truncated"));
2492     return NS_OK;
2493   }
2494
2495   CacheIOThread::Cancelable cancelable(!aHandle->IsSpecialFile());
2496
2497   if (!aHandle->mFileExists) {
2498     rv = CreateFile(aHandle);
2499     NS_ENSURE_SUCCESS(rv, rv);
2500   }
2501
2502   if (!aHandle->mFD) {
2503     rv = OpenNSPRHandle(aHandle);
2504     NS_ENSURE_SUCCESS(rv, rv);
2505   } else {
2506     NSPRHandleUsed(aHandle);
2507   }
2508
2509   // Check again, OpenNSPRHandle could figure out the file was gone.
2510   if (!aHandle->mFileExists) {
2511     return NS_ERROR_NOT_AVAILABLE;
2512   }
2513
2514   // When this operation would increase cache size, check whether the cache size
2515   // reached the hard limit and whether it would cause critical low disk space.
2516   if (aHandle->mFileSize < aEOFPos) {
2517     if (mOverLimitEvicting && mCacheSizeOnHardLimit) {
2518       LOG(
2519           ("CacheFileIOManager::TruncateSeekSetEOFInternal() - failing because "
2520            "cache size reached hard limit!"));
2521       return NS_ERROR_FILE_DISK_FULL;
2522     }
2523
2524     int64_t freeSpace;
2525     rv = mCacheDirectory->GetDiskSpaceAvailable(&freeSpace);
2526     if (NS_WARN_IF(NS_FAILED(rv))) {
2527       freeSpace = -1;
2528       LOG(
2529           ("CacheFileIOManager::TruncateSeekSetEOFInternal() - "
2530            "GetDiskSpaceAvailable() failed! [rv=0x%08" PRIx32 "]",
2531            static_cast<uint32_t>(rv)));
2532     } else {
2533       freeSpace >>= 10;  // bytes to kilobytes
2534       uint32_t limit = CacheObserver::DiskFreeSpaceHardLimit();
2535       if (freeSpace - aEOFPos + aHandle->mFileSize < limit) {
2536         LOG(
2537             ("CacheFileIOManager::TruncateSeekSetEOFInternal() - Low free space"
2538              ", refusing to write! [freeSpace=%" PRId64 "kB, limit=%ukB]",
2539              freeSpace, limit));
2540         return NS_ERROR_FILE_DISK_FULL;
2541       }
2542     }
2543   }
2544
2545   // This operation always invalidates the entry
2546   aHandle->mInvalid = true;
2547
2548   rv = TruncFile(aHandle->mFD, aTruncatePos);
2549   NS_ENSURE_SUCCESS(rv, rv);
2550
2551   if (aTruncatePos != aEOFPos) {
2552     rv = TruncFile(aHandle->mFD, aEOFPos);
2553     NS_ENSURE_SUCCESS(rv, rv);
2554   }
2555
2556   uint32_t oldSizeInK = aHandle->FileSizeInK();
2557   aHandle->mFileSize = aEOFPos;
2558   uint32_t newSizeInK = aHandle->FileSizeInK();
2559
2560   if (oldSizeInK != newSizeInK && !aHandle->IsDoomed() &&
2561       !aHandle->IsSpecialFile()) {
2562     CacheIndex::UpdateEntry(aHandle->Hash(), nullptr, nullptr, nullptr, nullptr,
2563                             nullptr, &newSizeInK);
2564
2565     if (oldSizeInK < newSizeInK) {
2566       EvictIfOverLimitInternal();
2567     }
2568   }
2569
2570   return NS_OK;
2571 }
2572
2573 // static
2574 nsresult CacheFileIOManager::RenameFile(CacheFileHandle* aHandle,
2575                                         const nsACString& aNewName,
2576                                         CacheFileIOListener* aCallback) {
2577   LOG(("CacheFileIOManager::RenameFile() [handle=%p, newName=%s, listener=%p]",
2578        aHandle, PromiseFlatCString(aNewName).get(), aCallback));
2579
2580   nsresult rv;
2581   RefPtr<CacheFileIOManager> ioMan = gInstance;
2582
2583   if (aHandle->IsClosed() || !ioMan) {
2584     return NS_ERROR_NOT_INITIALIZED;
2585   }
2586
2587   if (!aHandle->IsSpecialFile()) {
2588     return NS_ERROR_UNEXPECTED;
2589   }
2590
2591   RefPtr<RenameFileEvent> ev =
2592       new RenameFileEvent(aHandle, aNewName, aCallback);
2593   rv = ioMan->mIOThread->Dispatch(ev, aHandle->mPriority
2594                                           ? CacheIOThread::WRITE_PRIORITY
2595                                           : CacheIOThread::WRITE);
2596   NS_ENSURE_SUCCESS(rv, rv);
2597
2598   return NS_OK;
2599 }
2600
2601 nsresult CacheFileIOManager::RenameFileInternal(CacheFileHandle* aHandle,
2602                                                 const nsACString& aNewName) {
2603   LOG(("CacheFileIOManager::RenameFileInternal() [handle=%p, newName=%s]",
2604        aHandle, PromiseFlatCString(aNewName).get()));
2605
2606   nsresult rv;
2607
2608   MOZ_ASSERT(aHandle->IsSpecialFile());
2609
2610   if (aHandle->IsDoomed()) {
2611     return NS_ERROR_NOT_AVAILABLE;
2612   }
2613
2614   // Doom old handle if it exists and is not doomed
2615   for (uint32_t i = 0; i < mSpecialHandles.Length(); i++) {
2616     if (!mSpecialHandles[i]->IsDoomed() &&
2617         mSpecialHandles[i]->Key() == aNewName) {
2618       MOZ_ASSERT(aHandle != mSpecialHandles[i]);
2619       rv = DoomFileInternal(mSpecialHandles[i]);
2620       NS_ENSURE_SUCCESS(rv, rv);
2621       break;
2622     }
2623   }
2624
2625   nsCOMPtr<nsIFile> file;
2626   rv = GetSpecialFile(aNewName, getter_AddRefs(file));
2627   NS_ENSURE_SUCCESS(rv, rv);
2628
2629   bool exists;
2630   rv = file->Exists(&exists);
2631   NS_ENSURE_SUCCESS(rv, rv);
2632
2633   if (exists) {
2634     LOG(
2635         ("CacheFileIOManager::RenameFileInternal() - Removing old file from "
2636          "disk"));
2637     rv = file->Remove(false);
2638     if (NS_FAILED(rv)) {
2639       NS_WARNING("Cannot remove file from the disk");
2640       LOG(
2641           ("CacheFileIOManager::RenameFileInternal() - Removing old file failed"
2642            ". [rv=0x%08" PRIx32 "]",
2643            static_cast<uint32_t>(rv)));
2644     }
2645   }
2646
2647   if (!aHandle->FileExists()) {
2648     aHandle->mKey = aNewName;
2649     return NS_OK;
2650   }
2651
2652   rv = MaybeReleaseNSPRHandleInternal(aHandle, true);
2653   NS_ENSURE_SUCCESS(rv, rv);
2654
2655   rv = aHandle->mFile->MoveToNative(nullptr, aNewName);
2656   NS_ENSURE_SUCCESS(rv, rv);
2657
2658   aHandle->mKey = aNewName;
2659   return NS_OK;
2660 }
2661
2662 // static
2663 nsresult CacheFileIOManager::EvictIfOverLimit() {
2664   LOG(("CacheFileIOManager::EvictIfOverLimit()"));
2665
2666   nsresult rv;
2667   RefPtr<CacheFileIOManager> ioMan = gInstance;
2668
2669   if (!ioMan) {
2670     return NS_ERROR_NOT_INITIALIZED;
2671   }
2672
2673   nsCOMPtr<nsIRunnable> ev;
2674   ev = NewRunnableMethod("net::CacheFileIOManager::EvictIfOverLimitInternal",
2675                          ioMan, &CacheFileIOManager::EvictIfOverLimitInternal);
2676
2677   rv = ioMan->mIOThread->Dispatch(ev, CacheIOThread::EVICT);
2678   NS_ENSURE_SUCCESS(rv, rv);
2679
2680   return NS_OK;
2681 }
2682
2683 nsresult CacheFileIOManager::EvictIfOverLimitInternal() {
2684   LOG(("CacheFileIOManager::EvictIfOverLimitInternal()"));
2685
2686   nsresult rv;
2687
2688   MOZ_ASSERT(mIOThread->IsCurrentThread());
2689
2690   if (mShuttingDown) {
2691     return NS_ERROR_NOT_INITIALIZED;
2692   }
2693
2694   if (mOverLimitEvicting) {
2695     LOG(
2696         ("CacheFileIOManager::EvictIfOverLimitInternal() - Eviction already "
2697          "running."));
2698     return NS_OK;
2699   }
2700
2701   CacheIOThread::Cancelable cancelable(true);
2702
2703   int64_t freeSpace;
2704   rv = mCacheDirectory->GetDiskSpaceAvailable(&freeSpace);
2705   if (NS_WARN_IF(NS_FAILED(rv))) {
2706     freeSpace = -1;
2707
2708     // Do not change smart size.
2709     LOG(
2710         ("CacheFileIOManager::EvictIfOverLimitInternal() - "
2711          "GetDiskSpaceAvailable() failed! [rv=0x%08" PRIx32 "]",
2712          static_cast<uint32_t>(rv)));
2713   } else {
2714     freeSpace >>= 10;  // bytes to kilobytes
2715     UpdateSmartCacheSize(freeSpace);
2716   }
2717
2718   uint32_t cacheUsage;
2719   rv = CacheIndex::GetCacheSize(&cacheUsage);
2720   NS_ENSURE_SUCCESS(rv, rv);
2721
2722   uint32_t cacheLimit = CacheObserver::DiskCacheCapacity();
2723   uint32_t freeSpaceLimit = CacheObserver::DiskFreeSpaceSoftLimit();
2724
2725   if (cacheUsage <= cacheLimit &&
2726       (freeSpace == -1 || freeSpace >= freeSpaceLimit)) {
2727     LOG(
2728         ("CacheFileIOManager::EvictIfOverLimitInternal() - Cache size and free "
2729          "space in limits. [cacheSize=%ukB, cacheSizeLimit=%ukB, "
2730          "freeSpace=%" PRId64 "kB, freeSpaceLimit=%ukB]",
2731          cacheUsage, cacheLimit, freeSpace, freeSpaceLimit));
2732     return NS_OK;
2733   }
2734
2735   LOG(
2736       ("CacheFileIOManager::EvictIfOverLimitInternal() - Cache size exceeded "
2737        "limit. Starting overlimit eviction. [cacheSize=%ukB, limit=%ukB]",
2738        cacheUsage, cacheLimit));
2739
2740   nsCOMPtr<nsIRunnable> ev;
2741   ev = NewRunnableMethod("net::CacheFileIOManager::OverLimitEvictionInternal",
2742                          this, &CacheFileIOManager::OverLimitEvictionInternal);
2743
2744   rv = mIOThread->Dispatch(ev, CacheIOThread::EVICT);
2745   NS_ENSURE_SUCCESS(rv, rv);
2746
2747   mOverLimitEvicting = true;
2748   return NS_OK;
2749 }
2750
2751 nsresult CacheFileIOManager::OverLimitEvictionInternal() {
2752   LOG(("CacheFileIOManager::OverLimitEvictionInternal()"));
2753
2754   nsresult rv;
2755
2756   MOZ_ASSERT(mIOThread->IsCurrentThread());
2757
2758   // mOverLimitEvicting is accessed only on IO thread, so we can set it to false
2759   // here and set it to true again once we dispatch another event that will
2760   // continue with the eviction. The reason why we do so is that we can fail
2761   // early anywhere in this method and the variable will contain a correct
2762   // value. Otherwise we would need to set it to false on every failing place.
2763   mOverLimitEvicting = false;
2764
2765   if (mShuttingDown) {
2766     return NS_ERROR_NOT_INITIALIZED;
2767   }
2768
2769   while (true) {
2770     int64_t freeSpace;
2771     rv = mCacheDirectory->GetDiskSpaceAvailable(&freeSpace);
2772     if (NS_WARN_IF(NS_FAILED(rv))) {
2773       freeSpace = -1;
2774
2775       // Do not change smart size.
2776       LOG(
2777           ("CacheFileIOManager::EvictIfOverLimitInternal() - "
2778            "GetDiskSpaceAvailable() failed! [rv=0x%08" PRIx32 "]",
2779            static_cast<uint32_t>(rv)));
2780     } else {
2781       freeSpace >>= 10;  // bytes to kilobytes
2782       UpdateSmartCacheSize(freeSpace);
2783     }
2784
2785     uint32_t cacheUsage;
2786     rv = CacheIndex::GetCacheSize(&cacheUsage);
2787     NS_ENSURE_SUCCESS(rv, rv);
2788
2789     uint32_t cacheLimit = CacheObserver::DiskCacheCapacity();
2790     uint32_t freeSpaceLimit = CacheObserver::DiskFreeSpaceSoftLimit();
2791
2792     if (cacheUsage > cacheLimit) {
2793       LOG(
2794           ("CacheFileIOManager::OverLimitEvictionInternal() - Cache size over "
2795            "limit. [cacheSize=%ukB, limit=%ukB]",
2796            cacheUsage, cacheLimit));
2797
2798       // We allow cache size to go over the specified limit. Eviction should
2799       // keep the size within the limit in a long run, but in case the eviction
2800       // is too slow, the cache could go way over the limit. To prevent this we
2801       // set flag mCacheSizeOnHardLimit when the size reaches 105% of the limit
2802       // and WriteInternal() and TruncateSeekSetEOFInternal() fail to cache
2803       // additional data.
2804       if ((cacheUsage - cacheLimit) > (cacheLimit / 20)) {
2805         LOG(
2806             ("CacheFileIOManager::OverLimitEvictionInternal() - Cache size "
2807              "reached hard limit."));
2808         mCacheSizeOnHardLimit = true;
2809       } else {
2810         mCacheSizeOnHardLimit = false;
2811       }
2812     } else if (freeSpace != -1 && freeSpace < freeSpaceLimit) {
2813       LOG(
2814           ("CacheFileIOManager::OverLimitEvictionInternal() - Free space under "
2815            "limit. [freeSpace=%" PRId64 "kB, freeSpaceLimit=%ukB]",
2816            freeSpace, freeSpaceLimit));
2817     } else {
2818       LOG(
2819           ("CacheFileIOManager::OverLimitEvictionInternal() - Cache size and "
2820            "free space in limits. [cacheSize=%ukB, cacheSizeLimit=%ukB, "
2821            "freeSpace=%" PRId64 "kB, freeSpaceLimit=%ukB]",
2822            cacheUsage, cacheLimit, freeSpace, freeSpaceLimit));
2823
2824       mCacheSizeOnHardLimit = false;
2825       return NS_OK;
2826     }
2827
2828     if (CacheIOThread::YieldAndRerun()) {
2829       LOG(
2830           ("CacheFileIOManager::OverLimitEvictionInternal() - Breaking loop "
2831            "for higher level events."));
2832       mOverLimitEvicting = true;
2833       return NS_OK;
2834     }
2835
2836     SHA1Sum::Hash hash;
2837     uint32_t cnt;
2838     static uint32_t consecutiveFailures = 0;
2839     rv = CacheIndex::GetEntryForEviction(false, &hash, &cnt);
2840     NS_ENSURE_SUCCESS(rv, rv);
2841
2842     rv = DoomFileByKeyInternal(&hash);
2843     if (NS_SUCCEEDED(rv)) {
2844       consecutiveFailures = 0;
2845     } else if (rv == NS_ERROR_NOT_AVAILABLE) {
2846       LOG(
2847           ("CacheFileIOManager::OverLimitEvictionInternal() - "
2848            "DoomFileByKeyInternal() failed. [rv=0x%08" PRIx32 "]",
2849            static_cast<uint32_t>(rv)));
2850       // TODO index is outdated, start update
2851
2852       // Make sure index won't return the same entry again
2853       CacheIndex::RemoveEntry(&hash);
2854       consecutiveFailures = 0;
2855     } else {
2856       // This shouldn't normally happen, but the eviction must not fail
2857       // completely if we ever encounter this problem.
2858       NS_WARNING(
2859           "CacheFileIOManager::OverLimitEvictionInternal() - Unexpected "
2860           "failure of DoomFileByKeyInternal()");
2861
2862       LOG(
2863           ("CacheFileIOManager::OverLimitEvictionInternal() - "
2864            "DoomFileByKeyInternal() failed. [rv=0x%08" PRIx32 "]",
2865            static_cast<uint32_t>(rv)));
2866
2867       // Normally, CacheIndex::UpdateEntry() is called only to update newly
2868       // created/opened entries which are always fresh and UpdateEntry() expects
2869       // and checks this flag. The way we use UpdateEntry() here is a kind of
2870       // hack and we must make sure the flag is set by calling
2871       // EnsureEntryExists().
2872       rv = CacheIndex::EnsureEntryExists(&hash);
2873       NS_ENSURE_SUCCESS(rv, rv);
2874
2875       // Move the entry at the end of both lists to make sure we won't end up
2876       // failing on one entry forever.
2877       uint32_t frecency = 0;
2878       rv = CacheIndex::UpdateEntry(&hash, &frecency, nullptr, nullptr, nullptr,
2879                                    nullptr, nullptr);
2880       NS_ENSURE_SUCCESS(rv, rv);
2881
2882       consecutiveFailures++;
2883       if (consecutiveFailures >= cnt) {
2884         // This doesn't necessarily mean that we've tried to doom every entry
2885         // but we've reached a sane number of tries. It is likely that another
2886         // eviction will start soon. And as said earlier, this normally doesn't
2887         // happen at all.
2888         return NS_OK;
2889       }
2890     }
2891   }
2892
2893   MOZ_ASSERT_UNREACHABLE("We should never get here");
2894   return NS_OK;
2895 }
2896
2897 // static
2898 nsresult CacheFileIOManager::EvictAll() {
2899   LOG(("CacheFileIOManager::EvictAll()"));
2900
2901   nsresult rv;
2902   RefPtr<CacheFileIOManager> ioMan = gInstance;
2903
2904   if (!ioMan) {
2905     return NS_ERROR_NOT_INITIALIZED;
2906   }
2907
2908   nsCOMPtr<nsIRunnable> ev;
2909   ev = NewRunnableMethod("net::CacheFileIOManager::EvictAllInternal", ioMan,
2910                          &CacheFileIOManager::EvictAllInternal);
2911
2912   rv = ioMan->mIOThread->DispatchAfterPendingOpens(ev);
2913   if (NS_WARN_IF(NS_FAILED(rv))) {
2914     return rv;
2915   }
2916
2917   return NS_OK;
2918 }
2919
2920 namespace {
2921
2922 class EvictionNotifierRunnable : public Runnable {
2923  public:
2924   EvictionNotifierRunnable() : Runnable("EvictionNotifierRunnable") {}
2925   NS_DECL_NSIRUNNABLE
2926 };
2927
2928 NS_IMETHODIMP
2929 EvictionNotifierRunnable::Run() {
2930   nsCOMPtr<nsIObserverService> obsSvc = mozilla::services::GetObserverService();
2931   if (obsSvc) {
2932     obsSvc->NotifyObservers(nullptr, "cacheservice:empty-cache", nullptr);
2933   }
2934   return NS_OK;
2935 }
2936
2937 }  // namespace
2938
2939 nsresult CacheFileIOManager::EvictAllInternal() {
2940   LOG(("CacheFileIOManager::EvictAllInternal()"));
2941
2942   nsresult rv;
2943
2944   MOZ_ASSERT(mIOThread->IsCurrentThread());
2945
2946   RefPtr<EvictionNotifierRunnable> r = new EvictionNotifierRunnable();
2947
2948   if (!mCacheDirectory) {
2949     // This is a kind of hack. Somebody called EvictAll() without a profile.
2950     // This happens in xpcshell tests that use cache without profile. We need
2951     // to notify observers in this case since the tests are waiting for it.
2952     NS_DispatchToMainThread(r);
2953     return NS_ERROR_FILE_INVALID_PATH;
2954   }
2955
2956   if (mShuttingDown) {
2957     return NS_ERROR_NOT_INITIALIZED;
2958   }
2959
2960   if (!mTreeCreated) {
2961     rv = CreateCacheTree();
2962     if (NS_FAILED(rv)) {
2963       return rv;
2964     }
2965   }
2966
2967   // Doom all active handles
2968   nsTArray<RefPtr<CacheFileHandle> > handles;
2969   mHandles.GetActiveHandles(&handles);
2970
2971   for (uint32_t i = 0; i < handles.Length(); ++i) {
2972     rv = DoomFileInternal(handles[i]);
2973     if (NS_WARN_IF(NS_FAILED(rv))) {
2974       LOG(
2975           ("CacheFileIOManager::EvictAllInternal() - Cannot doom handle "
2976            "[handle=%p]",
2977            handles[i].get()));
2978     }
2979   }
2980
2981   nsCOMPtr<nsIFile> file;
2982   rv = mCacheDirectory->Clone(getter_AddRefs(file));
2983   if (NS_WARN_IF(NS_FAILED(rv))) {
2984     return rv;
2985   }
2986
2987   rv = file->AppendNative(nsLiteralCString(ENTRIES_DIR));
2988   if (NS_WARN_IF(NS_FAILED(rv))) {
2989     return rv;
2990   }
2991
2992   // Trash current entries directory
2993   rv = TrashDirectory(file);
2994   if (NS_WARN_IF(NS_FAILED(rv))) {
2995     return rv;
2996   }
2997
2998   // Files are now inaccessible in entries directory, notify observers.
2999   NS_DispatchToMainThread(r);
3000
3001   // Create a new empty entries directory
3002   rv = CheckAndCreateDir(mCacheDirectory, ENTRIES_DIR, false);
3003   if (NS_WARN_IF(NS_FAILED(rv))) {
3004     return rv;
3005   }
3006
3007   CacheIndex::RemoveAll();
3008
3009   return NS_OK;
3010 }
3011
3012 // static
3013 nsresult CacheFileIOManager::EvictByContext(
3014     nsILoadContextInfo* aLoadContextInfo, bool aPinned,
3015     const nsAString& aOrigin) {
3016   LOG(("CacheFileIOManager::EvictByContext() [loadContextInfo=%p]",
3017        aLoadContextInfo));
3018
3019   nsresult rv;
3020   RefPtr<CacheFileIOManager> ioMan = gInstance;
3021
3022   if (!ioMan) {
3023     return NS_ERROR_NOT_INITIALIZED;
3024   }
3025
3026   nsCOMPtr<nsIRunnable> ev;
3027   ev = NewRunnableMethod<nsCOMPtr<nsILoadContextInfo>, bool, nsString>(
3028       "net::CacheFileIOManager::EvictByContextInternal", ioMan,
3029       &CacheFileIOManager::EvictByContextInternal, aLoadContextInfo, aPinned,
3030       aOrigin);
3031
3032   rv = ioMan->mIOThread->DispatchAfterPendingOpens(ev);
3033   if (NS_WARN_IF(NS_FAILED(rv))) {
3034     return rv;
3035   }
3036
3037   return NS_OK;
3038 }
3039
3040 nsresult CacheFileIOManager::EvictByContextInternal(
3041     nsILoadContextInfo* aLoadContextInfo, bool aPinned,
3042     const nsAString& aOrigin) {
3043   LOG(
3044       ("CacheFileIOManager::EvictByContextInternal() [loadContextInfo=%p, "
3045        "pinned=%d]",
3046        aLoadContextInfo, aPinned));
3047
3048   nsresult rv;
3049
3050   if (aLoadContextInfo) {
3051     nsAutoCString suffix;
3052     aLoadContextInfo->OriginAttributesPtr()->CreateSuffix(suffix);
3053     LOG(("  anonymous=%u, suffix=%s]", aLoadContextInfo->IsAnonymous(),
3054          suffix.get()));
3055
3056     MOZ_ASSERT(mIOThread->IsCurrentThread());
3057
3058     MOZ_ASSERT(!aLoadContextInfo->IsPrivate());
3059     if (aLoadContextInfo->IsPrivate()) {
3060       return NS_ERROR_INVALID_ARG;
3061     }
3062   }
3063
3064   if (!mCacheDirectory) {
3065     // This is a kind of hack. Somebody called EvictAll() without a profile.
3066     // This happens in xpcshell tests that use cache without profile. We need
3067     // to notify observers in this case since the tests are waiting for it.
3068     // Also notify for aPinned == true, those are interested as well.
3069     if (!aLoadContextInfo) {
3070       RefPtr<EvictionNotifierRunnable> r = new EvictionNotifierRunnable();
3071       NS_DispatchToMainThread(r);
3072     }
3073     return NS_ERROR_FILE_INVALID_PATH;
3074   }
3075
3076   if (mShuttingDown) {
3077     return NS_ERROR_NOT_INITIALIZED;
3078   }
3079
3080   if (!mTreeCreated) {
3081     rv = CreateCacheTree();
3082     if (NS_FAILED(rv)) {
3083       return rv;
3084     }
3085   }
3086
3087   NS_ConvertUTF16toUTF8 origin(aOrigin);
3088
3089   // Doom all active handles that matches the load context
3090   nsTArray<RefPtr<CacheFileHandle> > handles;
3091   mHandles.GetActiveHandles(&handles);
3092
3093   for (uint32_t i = 0; i < handles.Length(); ++i) {
3094     CacheFileHandle* handle = handles[i];
3095
3096     nsAutoCString uriSpec;
3097     RefPtr<nsILoadContextInfo> info =
3098         CacheFileUtils::ParseKey(handle->Key(), nullptr, &uriSpec);
3099     if (!info) {
3100       LOG(
3101           ("CacheFileIOManager::EvictByContextInternal() - Cannot parse key in "
3102            "handle! [handle=%p, key=%s]",
3103            handle, handle->Key().get()));
3104       MOZ_CRASH("Unexpected error!");
3105     }
3106
3107     if (aLoadContextInfo && !info->Equals(aLoadContextInfo)) {
3108       continue;
3109     }
3110
3111     if (!origin.IsEmpty()) {
3112       RefPtr<MozURL> url;
3113       rv = MozURL::Init(getter_AddRefs(url), uriSpec);
3114       if (NS_FAILED(rv)) {
3115         continue;
3116       }
3117
3118       nsAutoCString urlOrigin;
3119       url->Origin(urlOrigin);
3120
3121       if (!urlOrigin.Equals(origin)) {
3122         continue;
3123       }
3124     }
3125
3126     // handle will be doomed only when pinning status is known and equal or
3127     // doom decision will be deferred until pinning status is determined.
3128     rv = DoomFileInternal(handle,
3129                           aPinned ? CacheFileIOManager::DOOM_WHEN_PINNED
3130                                   : CacheFileIOManager::DOOM_WHEN_NON_PINNED);
3131     if (NS_WARN_IF(NS_FAILED(rv))) {
3132       LOG(
3133           ("CacheFileIOManager::EvictByContextInternal() - Cannot doom handle"
3134            " [handle=%p]",
3135            handle));
3136     }
3137   }
3138
3139   if (!aLoadContextInfo) {
3140     RefPtr<EvictionNotifierRunnable> r = new EvictionNotifierRunnable();
3141     NS_DispatchToMainThread(r);
3142   }
3143
3144   if (!mContextEvictor) {
3145     mContextEvictor = new CacheFileContextEvictor();
3146     mContextEvictor->Init(mCacheDirectory);
3147   }
3148
3149   mContextEvictor->AddContext(aLoadContextInfo, aPinned, aOrigin);
3150
3151   return NS_OK;
3152 }
3153
3154 // static
3155 nsresult CacheFileIOManager::CacheIndexStateChanged() {
3156   LOG(("CacheFileIOManager::CacheIndexStateChanged()"));
3157
3158   nsresult rv;
3159
3160   // CacheFileIOManager lives longer than CacheIndex so gInstance must be
3161   // non-null here.
3162   MOZ_ASSERT(gInstance);
3163
3164   // We have to re-distatch even if we are on IO thread to prevent reentering
3165   // the lock in CacheIndex
3166   nsCOMPtr<nsIRunnable> ev = NewRunnableMethod(
3167       "net::CacheFileIOManager::CacheIndexStateChangedInternal",
3168       gInstance.get(), &CacheFileIOManager::CacheIndexStateChangedInternal);
3169
3170   nsCOMPtr<nsIEventTarget> ioTarget = IOTarget();
3171   MOZ_ASSERT(ioTarget);
3172
3173   rv = ioTarget->Dispatch(ev, nsIEventTarget::DISPATCH_NORMAL);
3174   if (NS_WARN_IF(NS_FAILED(rv))) {
3175     return rv;
3176   }
3177
3178   return NS_OK;
3179 }
3180
3181 void CacheFileIOManager::CacheIndexStateChangedInternal() {
3182   if (mShuttingDown) {
3183     // ignore notification during shutdown
3184     return;
3185   }
3186
3187   if (!mContextEvictor) {
3188     return;
3189   }
3190
3191   mContextEvictor->CacheIndexStateChanged();
3192 }
3193
3194 nsresult CacheFileIOManager::TrashDirectory(nsIFile* aFile) {
3195   LOG(("CacheFileIOManager::TrashDirectory() [file=%s]",
3196        aFile->HumanReadablePath().get()));
3197
3198   nsresult rv;
3199
3200   MOZ_ASSERT(mIOThread->IsCurrentThread());
3201   MOZ_ASSERT(mCacheDirectory);
3202
3203   // When the directory is empty, it is cheaper to remove it directly instead of
3204   // using the trash mechanism.
3205   bool isEmpty;
3206   rv = IsEmptyDirectory(aFile, &isEmpty);
3207   NS_ENSURE_SUCCESS(rv, rv);
3208
3209   if (isEmpty) {
3210     rv = aFile->Remove(false);
3211     LOG(
3212         ("CacheFileIOManager::TrashDirectory() - Directory removed "
3213          "[rv=0x%08" PRIx32 "]",
3214          static_cast<uint32_t>(rv)));
3215     return rv;
3216   }
3217
3218 #ifdef DEBUG
3219   nsCOMPtr<nsIFile> dirCheck;
3220   rv = aFile->GetParent(getter_AddRefs(dirCheck));
3221   NS_ENSURE_SUCCESS(rv, rv);
3222
3223   bool equals = false;
3224   rv = dirCheck->Equals(mCacheDirectory, &equals);
3225   NS_ENSURE_SUCCESS(rv, rv);
3226
3227   MOZ_ASSERT(equals);
3228 #endif
3229
3230   nsCOMPtr<nsIFile> dir, trash;
3231   nsAutoCString leaf;
3232
3233   rv = aFile->Clone(getter_AddRefs(dir));
3234   NS_ENSURE_SUCCESS(rv, rv);
3235
3236   rv = aFile->Clone(getter_AddRefs(trash));
3237   NS_ENSURE_SUCCESS(rv, rv);
3238
3239   const int32_t kMaxTries = 16;
3240   srand(static_cast<unsigned>(PR_Now()));
3241   for (int32_t triesCount = 0;; ++triesCount) {
3242     leaf = TRASH_DIR;
3243     leaf.AppendInt(rand());
3244     rv = trash->SetNativeLeafName(leaf);
3245     NS_ENSURE_SUCCESS(rv, rv);
3246
3247     bool exists;
3248     if (NS_SUCCEEDED(trash->Exists(&exists)) && !exists) {
3249       break;
3250     }
3251
3252     LOG(
3253         ("CacheFileIOManager::TrashDirectory() - Trash directory already "
3254          "exists [leaf=%s]",
3255          leaf.get()));
3256
3257     if (triesCount == kMaxTries) {
3258       LOG(
3259           ("CacheFileIOManager::TrashDirectory() - Could not find unused trash "
3260            "directory in %d tries.",
3261            kMaxTries));
3262       return NS_ERROR_FAILURE;
3263     }
3264   }
3265
3266   LOG(("CacheFileIOManager::TrashDirectory() - Renaming directory [leaf=%s]",
3267        leaf.get()));
3268
3269   rv = dir->MoveToNative(nullptr, leaf);
3270   NS_ENSURE_SUCCESS(rv, rv);
3271
3272   StartRemovingTrash();
3273   return NS_OK;
3274 }
3275
3276 // static
3277 void CacheFileIOManager::OnTrashTimer(nsITimer* aTimer, void* aClosure) {
3278   LOG(("CacheFileIOManager::OnTrashTimer() [timer=%p, closure=%p]", aTimer,
3279        aClosure));
3280
3281   RefPtr<CacheFileIOManager> ioMan = gInstance;
3282
3283   if (!ioMan) {
3284     return;
3285   }
3286
3287   ioMan->mTrashTimer = nullptr;
3288   ioMan->StartRemovingTrash();
3289 }
3290
3291 nsresult CacheFileIOManager::StartRemovingTrash() {
3292   LOG(("CacheFileIOManager::StartRemovingTrash()"));
3293
3294   nsresult rv;
3295
3296   MOZ_ASSERT(mIOThread->IsCurrentThread());
3297
3298   if (mShuttingDown) {
3299     return NS_ERROR_NOT_INITIALIZED;
3300   }
3301
3302   if (!mCacheDirectory) {
3303     return NS_ERROR_FILE_INVALID_PATH;
3304   }
3305
3306   if (mTrashTimer) {
3307     LOG(("CacheFileIOManager::StartRemovingTrash() - Trash timer exists."));
3308     return NS_OK;
3309   }
3310
3311   if (mRemovingTrashDirs) {
3312     LOG(
3313         ("CacheFileIOManager::StartRemovingTrash() - Trash removing in "
3314          "progress."));
3315     return NS_OK;
3316   }
3317
3318   uint32_t elapsed = (TimeStamp::NowLoRes() - mStartTime).ToMilliseconds();
3319   if (elapsed < kRemoveTrashStartDelay) {
3320     nsCOMPtr<nsIEventTarget> ioTarget = IOTarget();
3321     MOZ_ASSERT(ioTarget);
3322
3323     return NS_NewTimerWithFuncCallback(
3324         getter_AddRefs(mTrashTimer), CacheFileIOManager::OnTrashTimer, nullptr,
3325         kRemoveTrashStartDelay - elapsed, nsITimer::TYPE_ONE_SHOT,
3326         "net::CacheFileIOManager::StartRemovingTrash", ioTarget);
3327   }
3328
3329   nsCOMPtr<nsIRunnable> ev;
3330   ev = NewRunnableMethod("net::CacheFileIOManager::RemoveTrashInternal", this,
3331                          &CacheFileIOManager::RemoveTrashInternal);
3332
3333   rv = mIOThread->Dispatch(ev, CacheIOThread::EVICT);
3334   NS_ENSURE_SUCCESS(rv, rv);
3335
3336   mRemovingTrashDirs = true;
3337   return NS_OK;
3338 }
3339
3340 nsresult CacheFileIOManager::RemoveTrashInternal() {
3341   LOG(("CacheFileIOManager::RemoveTrashInternal()"));
3342
3343   nsresult rv;
3344
3345   MOZ_ASSERT(mIOThread->IsCurrentThread());
3346
3347   if (mShuttingDown) {
3348     return NS_ERROR_NOT_INITIALIZED;
3349   }
3350
3351   CacheIOThread::Cancelable cancelable(true);
3352
3353   MOZ_ASSERT(!mTrashTimer);
3354   MOZ_ASSERT(mRemovingTrashDirs);
3355
3356   if (!mTreeCreated) {
3357     rv = CreateCacheTree();
3358     if (NS_FAILED(rv)) {
3359       return rv;
3360     }
3361   }
3362
3363   // mRemovingTrashDirs is accessed only on IO thread, so we can drop the flag
3364   // here and set it again once we dispatch a continuation event. By doing so,
3365   // we don't have to drop the flag on any possible early return.
3366   mRemovingTrashDirs = false;
3367
3368   while (true) {
3369     if (CacheIOThread::YieldAndRerun()) {
3370       LOG(
3371           ("CacheFileIOManager::RemoveTrashInternal() - Breaking loop for "
3372            "higher level events."));
3373       mRemovingTrashDirs = true;
3374       return NS_OK;
3375     }
3376
3377     // Find some trash directory
3378     if (!mTrashDir) {
3379       MOZ_ASSERT(!mTrashDirEnumerator);
3380
3381       rv = FindTrashDirToRemove();
3382       if (rv == NS_ERROR_NOT_AVAILABLE) {
3383         LOG(
3384             ("CacheFileIOManager::RemoveTrashInternal() - No trash directory "
3385              "found."));
3386         return NS_OK;
3387       }
3388       NS_ENSURE_SUCCESS(rv, rv);
3389
3390       rv = mTrashDir->GetDirectoryEntries(getter_AddRefs(mTrashDirEnumerator));
3391       NS_ENSURE_SUCCESS(rv, rv);
3392
3393       continue;  // check elapsed time
3394     }
3395
3396     // We null out mTrashDirEnumerator once we remove all files in the
3397     // directory, so remove the trash directory if we don't have enumerator.
3398     if (!mTrashDirEnumerator) {
3399       rv = mTrashDir->Remove(false);
3400       if (NS_FAILED(rv)) {
3401         // There is no reason why removing an empty directory should fail, but
3402         // if it does, we should continue and try to remove all other trash
3403         // directories.
3404         nsAutoCString leafName;
3405         mTrashDir->GetNativeLeafName(leafName);
3406         mFailedTrashDirs.AppendElement(leafName);
3407         LOG(
3408             ("CacheFileIOManager::RemoveTrashInternal() - Cannot remove "
3409              "trashdir. [name=%s]",
3410              leafName.get()));
3411       }
3412
3413       mTrashDir = nullptr;
3414       continue;  // check elapsed time
3415     }
3416
3417     nsCOMPtr<nsIFile> file;
3418     rv = mTrashDirEnumerator->GetNextFile(getter_AddRefs(file));
3419     if (!file) {
3420       mTrashDirEnumerator->Close();
3421       mTrashDirEnumerator = nullptr;
3422       continue;  // check elapsed time
3423     }
3424     bool isDir = false;
3425     file->IsDirectory(&isDir);
3426     if (isDir) {
3427       NS_WARNING(
3428           "Found a directory in a trash directory! It will be removed "
3429           "recursively, but this can block IO thread for a while!");
3430       if (LOG_ENABLED()) {
3431         LOG(
3432             ("CacheFileIOManager::RemoveTrashInternal() - Found a directory in "
3433              "a trash "
3434              "directory! It will be removed recursively, but this can block IO "
3435              "thread for a while! [file=%s]",
3436              file->HumanReadablePath().get()));
3437       }
3438     }
3439     file->Remove(isDir);
3440   }
3441
3442   MOZ_ASSERT_UNREACHABLE("We should never get here");
3443   return NS_OK;
3444 }
3445
3446 nsresult CacheFileIOManager::FindTrashDirToRemove() {
3447   LOG(("CacheFileIOManager::FindTrashDirToRemove()"));
3448
3449   nsresult rv;
3450
3451   // We call this method on the main thread during shutdown when user wants to
3452   // remove all cache files.
3453   MOZ_ASSERT(mIOThread->IsCurrentThread() || mShuttingDown);
3454
3455   nsCOMPtr<nsIDirectoryEnumerator> iter;
3456   rv = mCacheDirectory->GetDirectoryEntries(getter_AddRefs(iter));
3457   NS_ENSURE_SUCCESS(rv, rv);
3458
3459   nsCOMPtr<nsIFile> file;
3460   while (NS_SUCCEEDED(iter->GetNextFile(getter_AddRefs(file))) && file) {
3461     bool isDir = false;
3462     file->IsDirectory(&isDir);
3463     if (!isDir) {
3464       continue;
3465     }
3466
3467     nsAutoCString leafName;
3468     rv = file->GetNativeLeafName(leafName);
3469     if (NS_FAILED(rv)) {
3470       continue;
3471     }
3472
3473     if (leafName.Length() < strlen(TRASH_DIR)) {
3474       continue;
3475     }
3476
3477     if (!StringBeginsWith(leafName, nsLiteralCString(TRASH_DIR))) {
3478       continue;
3479     }
3480
3481     if (mFailedTrashDirs.Contains(leafName)) {
3482       continue;
3483     }
3484
3485     LOG(("CacheFileIOManager::FindTrashDirToRemove() - Returning directory %s",
3486          leafName.get()));
3487
3488     mTrashDir = file;
3489     return NS_OK;
3490   }
3491
3492   // When we're here we've tried to delete all trash directories. Clear
3493   // mFailedTrashDirs so we will try to delete them again when we start removing
3494   // trash directories next time.
3495   mFailedTrashDirs.Clear();
3496   return NS_ERROR_NOT_AVAILABLE;
3497 }
3498
3499 // static
3500 nsresult CacheFileIOManager::InitIndexEntry(CacheFileHandle* aHandle,
3501                                             OriginAttrsHash aOriginAttrsHash,
3502                                             bool aAnonymous, bool aPinning) {
3503   LOG(
3504       ("CacheFileIOManager::InitIndexEntry() [handle=%p, "
3505        "originAttrsHash=%" PRIx64 ", "
3506        "anonymous=%d, pinning=%d]",
3507        aHandle, aOriginAttrsHash, aAnonymous, aPinning));
3508
3509   nsresult rv;
3510   RefPtr<CacheFileIOManager> ioMan = gInstance;
3511
3512   if (aHandle->IsClosed() || !ioMan) {
3513     return NS_ERROR_NOT_INITIALIZED;
3514   }
3515
3516   if (aHandle->IsSpecialFile()) {
3517     return NS_ERROR_UNEXPECTED;
3518   }
3519
3520   RefPtr<InitIndexEntryEvent> ev =
3521       new InitIndexEntryEvent(aHandle, aOriginAttrsHash, aAnonymous, aPinning);
3522   rv = ioMan->mIOThread->Dispatch(ev, aHandle->mPriority
3523                                           ? CacheIOThread::WRITE_PRIORITY
3524                                           : CacheIOThread::WRITE);
3525   NS_ENSURE_SUCCESS(rv, rv);
3526
3527   return NS_OK;
3528 }
3529
3530 // static
3531 nsresult CacheFileIOManager::UpdateIndexEntry(CacheFileHandle* aHandle,
3532                                               const uint32_t* aFrecency,
3533                                               const bool* aHasAltData,
3534                                               const uint16_t* aOnStartTime,
3535                                               const uint16_t* aOnStopTime,
3536                                               const uint8_t* aContentType) {
3537   LOG(
3538       ("CacheFileIOManager::UpdateIndexEntry() [handle=%p, frecency=%s, "
3539        "hasAltData=%s, onStartTime=%s, onStopTime=%s, contentType=%s]",
3540        aHandle, aFrecency ? nsPrintfCString("%u", *aFrecency).get() : "",
3541        aHasAltData ? (*aHasAltData ? "true" : "false") : "",
3542        aOnStartTime ? nsPrintfCString("%u", *aOnStartTime).get() : "",
3543        aOnStopTime ? nsPrintfCString("%u", *aOnStopTime).get() : "",
3544        aContentType ? nsPrintfCString("%u", *aContentType).get() : ""));
3545
3546   nsresult rv;
3547   RefPtr<CacheFileIOManager> ioMan = gInstance;
3548
3549   if (aHandle->IsClosed() || !ioMan) {
3550     return NS_ERROR_NOT_INITIALIZED;
3551   }
3552
3553   if (aHandle->IsSpecialFile()) {
3554     return NS_ERROR_UNEXPECTED;
3555   }
3556
3557   RefPtr<UpdateIndexEntryEvent> ev = new UpdateIndexEntryEvent(
3558       aHandle, aFrecency, aHasAltData, aOnStartTime, aOnStopTime, aContentType);
3559   rv = ioMan->mIOThread->Dispatch(ev, aHandle->mPriority
3560                                           ? CacheIOThread::WRITE_PRIORITY
3561                                           : CacheIOThread::WRITE);
3562   NS_ENSURE_SUCCESS(rv, rv);
3563
3564   return NS_OK;
3565 }
3566
3567 nsresult CacheFileIOManager::CreateFile(CacheFileHandle* aHandle) {
3568   MOZ_ASSERT(!aHandle->mFD);
3569   MOZ_ASSERT(aHandle->mFile);
3570
3571   nsresult rv;
3572
3573   if (aHandle->IsDoomed()) {
3574     nsCOMPtr<nsIFile> file;
3575
3576     rv = GetDoomedFile(getter_AddRefs(file));
3577     NS_ENSURE_SUCCESS(rv, rv);
3578
3579     aHandle->mFile.swap(file);
3580   } else {
3581     bool exists;
3582     if (NS_SUCCEEDED(aHandle->mFile->Exists(&exists)) && exists) {
3583       NS_WARNING("Found a file that should not exist!");
3584     }
3585   }
3586
3587   rv = OpenNSPRHandle(aHandle, true);
3588   NS_ENSURE_SUCCESS(rv, rv);
3589
3590   aHandle->mFileSize = 0;
3591   return NS_OK;
3592 }
3593
3594 // static
3595 void CacheFileIOManager::HashToStr(const SHA1Sum::Hash* aHash,
3596                                    nsACString& _retval) {
3597   _retval.Truncate();
3598   const char hexChars[] = {'0', '1', '2', '3', '4', '5', '6', '7',
3599                            '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'};
3600   for (uint32_t i = 0; i < sizeof(SHA1Sum::Hash); i++) {
3601     _retval.Append(hexChars[(*aHash)[i] >> 4]);
3602     _retval.Append(hexChars[(*aHash)[i] & 0xF]);
3603   }
3604 }
3605
3606 // static
3607 nsresult CacheFileIOManager::StrToHash(const nsACString& aHash,
3608                                        SHA1Sum::Hash* _retval) {
3609   if (aHash.Length() != 2 * sizeof(SHA1Sum::Hash)) {
3610     return NS_ERROR_INVALID_ARG;
3611   }
3612
3613   for (uint32_t i = 0; i < aHash.Length(); i++) {
3614     uint8_t value;
3615
3616     if (aHash[i] >= '0' && aHash[i] <= '9') {
3617       value = aHash[i] - '0';
3618     } else if (aHash[i] >= 'A' && aHash[i] <= 'F') {
3619       value = aHash[i] - 'A' + 10;
3620     } else if (aHash[i] >= 'a' && aHash[i] <= 'f') {
3621       value = aHash[i] - 'a' + 10;
3622     } else {
3623       return NS_ERROR_INVALID_ARG;
3624     }
3625
3626     if (i % 2 == 0) {
3627       (reinterpret_cast<uint8_t*>(_retval))[i / 2] = value << 4;
3628     } else {
3629       (reinterpret_cast<uint8_t*>(_retval))[i / 2] += value;
3630     }
3631   }
3632
3633   return NS_OK;
3634 }
3635
3636 nsresult CacheFileIOManager::GetFile(const SHA1Sum::Hash* aHash,
3637                                      nsIFile** _retval) {
3638   nsresult rv;
3639   nsCOMPtr<nsIFile> file;
3640   rv = mCacheDirectory->Clone(getter_AddRefs(file));
3641   NS_ENSURE_SUCCESS(rv, rv);
3642
3643   rv = file->AppendNative(nsLiteralCString(ENTRIES_DIR));
3644   NS_ENSURE_SUCCESS(rv, rv);
3645
3646   nsAutoCString leafName;
3647   HashToStr(aHash, leafName);
3648
3649   rv = file->AppendNative(leafName);
3650   NS_ENSURE_SUCCESS(rv, rv);
3651
3652   file.swap(*_retval);
3653   return NS_OK;
3654 }
3655
3656 nsresult CacheFileIOManager::GetSpecialFile(const nsACString& aKey,
3657                                             nsIFile** _retval) {
3658   nsresult rv;
3659   nsCOMPtr<nsIFile> file;
3660   rv = mCacheDirectory->Clone(getter_AddRefs(file));
3661   NS_ENSURE_SUCCESS(rv, rv);
3662
3663   rv = file->AppendNative(aKey);
3664   NS_ENSURE_SUCCESS(rv, rv);
3665
3666   file.swap(*_retval);
3667   return NS_OK;
3668 }
3669
3670 nsresult CacheFileIOManager::GetDoomedFile(nsIFile** _retval) {
3671   nsresult rv;
3672   nsCOMPtr<nsIFile> file;
3673   rv = mCacheDirectory->Clone(getter_AddRefs(file));
3674   NS_ENSURE_SUCCESS(rv, rv);
3675
3676   rv = file->AppendNative(nsLiteralCString(DOOMED_DIR));
3677   NS_ENSURE_SUCCESS(rv, rv);
3678
3679   rv = file->AppendNative("dummyleaf"_ns);
3680   NS_ENSURE_SUCCESS(rv, rv);
3681
3682   const int32_t kMaxTries = 64;
3683   srand(static_cast<unsigned>(PR_Now()));
3684   nsAutoCString leafName;
3685   for (int32_t triesCount = 0;; ++triesCount) {
3686     leafName.AppendInt(rand());
3687     rv = file->SetNativeLeafName(leafName);
3688     NS_ENSURE_SUCCESS(rv, rv);
3689
3690     bool exists;
3691     if (NS_SUCCEEDED(file->Exists(&exists)) && !exists) {
3692       break;
3693     }
3694
3695     if (triesCount == kMaxTries) {
3696       LOG(
3697           ("CacheFileIOManager::GetDoomedFile() - Could not find unused file "
3698            "name in %d tries.",
3699            kMaxTries));
3700       return NS_ERROR_FAILURE;
3701     }
3702
3703     leafName.Truncate();
3704   }
3705
3706   file.swap(*_retval);
3707   return NS_OK;
3708 }
3709
3710 nsresult CacheFileIOManager::IsEmptyDirectory(nsIFile* aFile, bool* _retval) {
3711   MOZ_ASSERT(mIOThread->IsCurrentThread());
3712
3713   nsresult rv;
3714
3715   nsCOMPtr<nsIDirectoryEnumerator> enumerator;
3716   rv = aFile->GetDirectoryEntries(getter_AddRefs(enumerator));
3717   NS_ENSURE_SUCCESS(rv, rv);
3718
3719   bool hasMoreElements = false;
3720   rv = enumerator->HasMoreElements(&hasMoreElements);
3721   NS_ENSURE_SUCCESS(rv, rv);
3722
3723   *_retval = !hasMoreElements;
3724   return NS_OK;
3725 }
3726
3727 nsresult CacheFileIOManager::CheckAndCreateDir(nsIFile* aFile, const char* aDir,
3728                                                bool aEnsureEmptyDir) {
3729   nsresult rv;
3730
3731   nsCOMPtr<nsIFile> file;
3732   if (!aDir) {
3733     file = aFile;
3734   } else {
3735     nsAutoCString dir(aDir);
3736     rv = aFile->Clone(getter_AddRefs(file));
3737     NS_ENSURE_SUCCESS(rv, rv);
3738     rv = file->AppendNative(dir);
3739     NS_ENSURE_SUCCESS(rv, rv);
3740   }
3741
3742   bool exists = false;
3743   rv = file->Exists(&exists);
3744   if (NS_SUCCEEDED(rv) && exists) {
3745     bool isDirectory = false;
3746     rv = file->IsDirectory(&isDirectory);
3747     if (NS_FAILED(rv) || !isDirectory) {
3748       // Try to remove the file
3749       rv = file->Remove(false);
3750       if (NS_SUCCEEDED(rv)) {
3751         exists = false;
3752       }
3753     }
3754     NS_ENSURE_SUCCESS(rv, rv);
3755   }
3756
3757   if (aEnsureEmptyDir && NS_SUCCEEDED(rv) && exists) {
3758     bool isEmpty;
3759     rv = IsEmptyDirectory(file, &isEmpty);
3760     NS_ENSURE_SUCCESS(rv, rv);
3761
3762     if (!isEmpty) {
3763       // Don't check the result, if this fails, it's OK.  We do this
3764       // only for the doomed directory that doesn't need to be deleted
3765       // for the cost of completely disabling the whole browser.
3766       TrashDirectory(file);
3767     }
3768   }
3769
3770   if (NS_SUCCEEDED(rv) && !exists) {
3771     rv = file->Create(nsIFile::DIRECTORY_TYPE, 0700);
3772   }
3773   if (NS_FAILED(rv)) {
3774     NS_WARNING("Cannot create directory");
3775     return NS_ERROR_FAILURE;
3776   }
3777
3778   return NS_OK;
3779 }
3780
3781 nsresult CacheFileIOManager::CreateCacheTree() {
3782   MOZ_ASSERT(mIOThread->IsCurrentThread());
3783   MOZ_ASSERT(!mTreeCreated);
3784
3785   if (!mCacheDirectory || mTreeCreationFailed) {
3786     return NS_ERROR_FILE_INVALID_PATH;
3787   }
3788
3789   nsresult rv;
3790
3791   // Set the flag here and clear it again below when the tree is created
3792   // successfully.
3793   mTreeCreationFailed = true;
3794
3795   // ensure parent directory exists
3796   nsCOMPtr<nsIFile> parentDir;
3797   rv = mCacheDirectory->GetParent(getter_AddRefs(parentDir));
3798   NS_ENSURE_SUCCESS(rv, rv);
3799   rv = CheckAndCreateDir(parentDir, nullptr, false);
3800   NS_ENSURE_SUCCESS(rv, rv);
3801
3802   // ensure cache directory exists
3803   rv = CheckAndCreateDir(mCacheDirectory, nullptr, false);
3804   NS_ENSURE_SUCCESS(rv, rv);
3805
3806   // ensure entries directory exists
3807   rv = CheckAndCreateDir(mCacheDirectory, ENTRIES_DIR, false);
3808   NS_ENSURE_SUCCESS(rv, rv);
3809
3810   // ensure doomed directory exists
3811   rv = CheckAndCreateDir(mCacheDirectory, DOOMED_DIR, true);
3812   NS_ENSURE_SUCCESS(rv, rv);
3813
3814   mTreeCreated = true;
3815   mTreeCreationFailed = false;
3816
3817   if (!mContextEvictor) {
3818     RefPtr<CacheFileContextEvictor> contextEvictor;
3819     contextEvictor = new CacheFileContextEvictor();
3820
3821     // Init() method will try to load unfinished contexts from the disk. Store
3822     // the evictor as a member only when there is some unfinished job.
3823     contextEvictor->Init(mCacheDirectory);
3824     if (contextEvictor->ContextsCount()) {
3825       contextEvictor.swap(mContextEvictor);
3826     }
3827   }
3828
3829   StartRemovingTrash();
3830
3831   return NS_OK;
3832 }
3833
3834 nsresult CacheFileIOManager::OpenNSPRHandle(CacheFileHandle* aHandle,
3835                                             bool aCreate) {
3836   LOG(("CacheFileIOManager::OpenNSPRHandle BEGIN, handle=%p", aHandle));
3837
3838   MOZ_ASSERT(CacheFileIOManager::IsOnIOThreadOrCeased());
3839   MOZ_ASSERT(!aHandle->mFD);
3840   MOZ_ASSERT(mHandlesByLastUsed.IndexOf(aHandle) == mHandlesByLastUsed.NoIndex);
3841   MOZ_ASSERT(mHandlesByLastUsed.Length() <= kOpenHandlesLimit);
3842   MOZ_ASSERT((aCreate && !aHandle->mFileExists) ||
3843              (!aCreate && aHandle->mFileExists));
3844
3845   nsresult rv;
3846
3847   if (mHandlesByLastUsed.Length() == kOpenHandlesLimit) {
3848     // close handle that hasn't been used for the longest time
3849     rv = MaybeReleaseNSPRHandleInternal(mHandlesByLastUsed[0], true);
3850     NS_ENSURE_SUCCESS(rv, rv);
3851   }
3852
3853   if (aCreate) {
3854     rv = aHandle->mFile->OpenNSPRFileDesc(
3855         PR_RDWR | PR_CREATE_FILE | PR_TRUNCATE, 0600, &aHandle->mFD);
3856     if (rv == NS_ERROR_FILE_ALREADY_EXISTS ||   // error from nsLocalFileWin
3857         rv == NS_ERROR_FILE_NO_DEVICE_SPACE) {  // error from nsLocalFileUnix
3858       LOG(
3859           ("CacheFileIOManager::OpenNSPRHandle() - Cannot create a new file, we"
3860            " might reached a limit on FAT32. Will evict a single entry and try "
3861            "again. [hash=%08x%08x%08x%08x%08x]",
3862            LOGSHA1(aHandle->Hash())));
3863
3864       SHA1Sum::Hash hash;
3865       uint32_t cnt;
3866
3867       rv = CacheIndex::GetEntryForEviction(true, &hash, &cnt);
3868       if (NS_SUCCEEDED(rv)) {
3869         rv = DoomFileByKeyInternal(&hash);
3870       }
3871       if (NS_SUCCEEDED(rv)) {
3872         rv = aHandle->mFile->OpenNSPRFileDesc(
3873             PR_RDWR | PR_CREATE_FILE | PR_TRUNCATE, 0600, &aHandle->mFD);
3874         LOG(
3875             ("CacheFileIOManager::OpenNSPRHandle() - Successfully evicted entry"
3876              " with hash %08x%08x%08x%08x%08x. %s to create the new file.",
3877              LOGSHA1(&hash), NS_SUCCEEDED(rv) ? "Succeeded" : "Failed"));
3878
3879         // Report the full size only once per session
3880         static bool sSizeReported = false;
3881         if (!sSizeReported) {
3882           uint32_t cacheUsage;
3883           if (NS_SUCCEEDED(CacheIndex::GetCacheSize(&cacheUsage))) {
3884             cacheUsage >>= 10;
3885             Telemetry::Accumulate(Telemetry::NETWORK_CACHE_SIZE_FULL_FAT,
3886                                   cacheUsage);
3887             sSizeReported = true;
3888           }
3889         }
3890       } else {
3891         LOG(
3892             ("CacheFileIOManager::OpenNSPRHandle() - Couldn't evict an existing"
3893              " entry."));
3894         rv = NS_ERROR_FILE_NO_DEVICE_SPACE;
3895       }
3896     }
3897     if (NS_FAILED(rv)) {
3898       LOG(
3899           ("CacheFileIOManager::OpenNSPRHandle() Create failed with "
3900            "0x%08" PRIx32,
3901            static_cast<uint32_t>(rv)));
3902     }
3903     NS_ENSURE_SUCCESS(rv, rv);
3904
3905     aHandle->mFileExists = true;
3906   } else {
3907     rv = aHandle->mFile->OpenNSPRFileDesc(PR_RDWR, 0600, &aHandle->mFD);
3908     if (NS_ERROR_FILE_NOT_FOUND == rv) {
3909       LOG(("  file doesn't exists"));
3910       aHandle->mFileExists = false;
3911       return DoomFileInternal(aHandle);
3912     }
3913     if (NS_FAILED(rv)) {
3914       LOG(("CacheFileIOManager::OpenNSPRHandle() Open failed with 0x%08" PRIx32,
3915            static_cast<uint32_t>(rv)));
3916     }
3917     NS_ENSURE_SUCCESS(rv, rv);
3918   }
3919
3920   mHandlesByLastUsed.AppendElement(aHandle);
3921
3922   LOG(("CacheFileIOManager::OpenNSPRHandle END, handle=%p", aHandle));
3923
3924   return NS_OK;
3925 }
3926
3927 void CacheFileIOManager::NSPRHandleUsed(CacheFileHandle* aHandle) {
3928   MOZ_ASSERT(CacheFileIOManager::IsOnIOThreadOrCeased());
3929   MOZ_ASSERT(aHandle->mFD);
3930
3931   DebugOnly<bool> found;
3932   found = mHandlesByLastUsed.RemoveElement(aHandle);
3933   MOZ_ASSERT(found);
3934
3935   mHandlesByLastUsed.AppendElement(aHandle);
3936 }
3937
3938 nsresult CacheFileIOManager::SyncRemoveDir(nsIFile* aFile, const char* aDir) {
3939   nsresult rv;
3940   nsCOMPtr<nsIFile> file;
3941
3942   if (!aDir) {
3943     file = aFile;
3944   } else {
3945     rv = aFile->Clone(getter_AddRefs(file));
3946     if (NS_WARN_IF(NS_FAILED(rv))) {
3947       return rv;
3948     }
3949
3950     rv = file->AppendNative(nsDependentCString(aDir));
3951     if (NS_WARN_IF(NS_FAILED(rv))) {
3952       return rv;
3953     }
3954   }
3955
3956   if (LOG_ENABLED()) {
3957     LOG(("CacheFileIOManager::SyncRemoveDir() - Removing directory %s",
3958          file->HumanReadablePath().get()));
3959   }
3960
3961   rv = file->Remove(true);
3962   if (NS_WARN_IF(NS_FAILED(rv))) {
3963     LOG(
3964         ("CacheFileIOManager::SyncRemoveDir() - Removing failed! "
3965          "[rv=0x%08" PRIx32 "]",
3966          static_cast<uint32_t>(rv)));
3967   }
3968
3969   return rv;
3970 }
3971
3972 void CacheFileIOManager::SyncRemoveAllCacheFiles() {
3973   LOG(("CacheFileIOManager::SyncRemoveAllCacheFiles()"));
3974
3975   nsresult rv;
3976
3977   SyncRemoveDir(mCacheDirectory, ENTRIES_DIR);
3978   SyncRemoveDir(mCacheDirectory, DOOMED_DIR);
3979
3980   // Clear any intermediate state of trash dir enumeration.
3981   mFailedTrashDirs.Clear();
3982   mTrashDir = nullptr;
3983
3984   while (true) {
3985     // FindTrashDirToRemove() fills mTrashDir if there is any trash directory.
3986     rv = FindTrashDirToRemove();
3987     if (rv == NS_ERROR_NOT_AVAILABLE) {
3988       LOG(
3989           ("CacheFileIOManager::SyncRemoveAllCacheFiles() - No trash directory "
3990            "found."));
3991       break;
3992     }
3993     if (NS_WARN_IF(NS_FAILED(rv))) {
3994       LOG(
3995           ("CacheFileIOManager::SyncRemoveAllCacheFiles() - "
3996            "FindTrashDirToRemove() returned an unexpected error. "
3997            "[rv=0x%08" PRIx32 "]",
3998            static_cast<uint32_t>(rv)));
3999       break;
4000     }
4001
4002     rv = SyncRemoveDir(mTrashDir, nullptr);
4003     if (NS_FAILED(rv)) {
4004       nsAutoCString leafName;
4005       mTrashDir->GetNativeLeafName(leafName);
4006       mFailedTrashDirs.AppendElement(leafName);
4007     }
4008   }
4009 }
4010
4011 // Returns default ("smart") size (in KB) of cache, given available disk space
4012 // (also in KB)
4013 static uint32_t SmartCacheSize(const int64_t availKB) {
4014   uint32_t maxSize;
4015
4016   if (CacheObserver::ClearCacheOnShutdown()) {
4017     maxSize = kMaxClearOnShutdownCacheSizeKB;
4018   } else {
4019     maxSize = kMaxCacheSizeKB;
4020   }
4021
4022   if (availKB > 25 * 1024 * 1024) {
4023     return maxSize;  // skip computing if we're over 25 GB
4024   }
4025
4026   // Grow/shrink in 10 MB units, deliberately, so that in the common case we
4027   // don't shrink cache and evict items every time we startup (it's important
4028   // that we don't slow down startup benchmarks).
4029   uint32_t sz10MBs = 0;
4030   uint32_t avail10MBs = availKB / (1024 * 10);
4031
4032   // 2.5% of space above 7GB
4033   if (avail10MBs > 700) {
4034     sz10MBs += static_cast<uint32_t>((avail10MBs - 700) * .025);
4035     avail10MBs = 700;
4036   }
4037   // 7.5% of space between 500 MB -> 7 GB
4038   if (avail10MBs > 50) {
4039     sz10MBs += static_cast<uint32_t>((avail10MBs - 50) * .075);
4040     avail10MBs = 50;
4041   }
4042
4043 #ifdef ANDROID
4044   // On Android, smaller/older devices may have very little storage and
4045   // device owners may be sensitive to storage footprint: Use a smaller
4046   // percentage of available space and a smaller minimum.
4047
4048   // 16% of space up to 500 MB (10 MB min)
4049   sz10MBs += std::max<uint32_t>(1, static_cast<uint32_t>(avail10MBs * .16));
4050 #else
4051   // 30% of space up to 500 MB (50 MB min)
4052   sz10MBs += std::max<uint32_t>(5, static_cast<uint32_t>(avail10MBs * .3));
4053 #endif
4054
4055   return std::min<uint32_t>(maxSize, sz10MBs * 10 * 1024);
4056 }
4057
4058 nsresult CacheFileIOManager::UpdateSmartCacheSize(int64_t aFreeSpace) {
4059   MOZ_ASSERT(mIOThread->IsCurrentThread());
4060
4061   nsresult rv;
4062
4063   if (!CacheObserver::SmartCacheSizeEnabled()) {
4064     return NS_ERROR_NOT_AVAILABLE;
4065   }
4066
4067   // Wait at least kSmartSizeUpdateInterval before recomputing smart size.
4068   static const TimeDuration kUpdateLimit =
4069       TimeDuration::FromMilliseconds(kSmartSizeUpdateInterval);
4070   if (!mLastSmartSizeTime.IsNull() &&
4071       (TimeStamp::NowLoRes() - mLastSmartSizeTime) < kUpdateLimit) {
4072     return NS_OK;
4073   }
4074
4075   // Do not compute smart size when cache size is not reliable.
4076   bool isUpToDate = false;
4077   CacheIndex::IsUpToDate(&isUpToDate);
4078   if (!isUpToDate) {
4079     return NS_ERROR_NOT_AVAILABLE;
4080   }
4081
4082   uint32_t cacheUsage;
4083   rv = CacheIndex::GetCacheSize(&cacheUsage);
4084   if (NS_WARN_IF(NS_FAILED(rv))) {
4085     LOG(
4086         ("CacheFileIOManager::UpdateSmartCacheSize() - Cannot get cacheUsage! "
4087          "[rv=0x%08" PRIx32 "]",
4088          static_cast<uint32_t>(rv)));
4089     return rv;
4090   }
4091
4092   mLastSmartSizeTime = TimeStamp::NowLoRes();
4093
4094   uint32_t smartSize = SmartCacheSize(aFreeSpace + cacheUsage);
4095
4096   if (smartSize == CacheObserver::DiskCacheCapacity()) {
4097     // Smart size has not changed.
4098     return NS_OK;
4099   }
4100
4101   CacheObserver::SetSmartDiskCacheCapacity(smartSize);
4102
4103   return NS_OK;
4104 }
4105
4106 // Memory reporting
4107
4108 namespace {
4109
4110 // A helper class that dispatches and waits for an event that gets result of
4111 // CacheFileIOManager->mHandles.SizeOfExcludingThis() on the I/O thread
4112 // to safely get handles memory report.
4113 // We must do this, since the handle list is only accessed and managed w/o
4114 // locking on the I/O thread.  That is by design.
4115 class SizeOfHandlesRunnable : public Runnable {
4116  public:
4117   SizeOfHandlesRunnable(mozilla::MallocSizeOf mallocSizeOf,
4118                         CacheFileHandles const& handles,
4119                         nsTArray<CacheFileHandle*> const& specialHandles)
4120       : Runnable("net::SizeOfHandlesRunnable"),
4121         mMonitor("SizeOfHandlesRunnable.mMonitor"),
4122         mMonitorNotified(false),
4123         mMallocSizeOf(mallocSizeOf),
4124         mHandles(handles),
4125         mSpecialHandles(specialHandles),
4126         mSize(0) {}
4127
4128   size_t Get(CacheIOThread* thread) {
4129     nsCOMPtr<nsIEventTarget> target = thread->Target();
4130     if (!target) {
4131       NS_ERROR("If we have the I/O thread we also must have the I/O target");
4132       return 0;
4133     }
4134
4135     mozilla::MonitorAutoLock mon(mMonitor);
4136     mMonitorNotified = false;
4137     nsresult rv = target->Dispatch(this, nsIEventTarget::DISPATCH_NORMAL);
4138     if (NS_FAILED(rv)) {
4139       NS_ERROR("Dispatch failed, cannot do memory report of CacheFileHandles");
4140       return 0;
4141     }
4142
4143     while (!mMonitorNotified) {
4144       mon.Wait();
4145     }
4146     return mSize;
4147   }
4148
4149   NS_IMETHOD Run() override {
4150     mozilla::MonitorAutoLock mon(mMonitor);
4151     // Excluding this since the object itself is a member of CacheFileIOManager
4152     // reported in CacheFileIOManager::SizeOfIncludingThis as part of |this|.
4153     mSize = mHandles.SizeOfExcludingThis(mMallocSizeOf);
4154     for (uint32_t i = 0; i < mSpecialHandles.Length(); ++i) {
4155       mSize += mSpecialHandles[i]->SizeOfIncludingThis(mMallocSizeOf);
4156     }
4157
4158     mMonitorNotified = true;
4159     mon.Notify();
4160     return NS_OK;
4161   }
4162
4163  private:
4164   mozilla::Monitor mMonitor;
4165   bool mMonitorNotified;
4166   mozilla::MallocSizeOf mMallocSizeOf;
4167   CacheFileHandles const& mHandles;
4168   nsTArray<CacheFileHandle*> const& mSpecialHandles;
4169   size_t mSize;
4170 };
4171
4172 }  // namespace
4173
4174 size_t CacheFileIOManager::SizeOfExcludingThisInternal(
4175     mozilla::MallocSizeOf mallocSizeOf) const {
4176   size_t n = 0;
4177   nsCOMPtr<nsISizeOf> sizeOf;
4178
4179   if (mIOThread) {
4180     n += mIOThread->SizeOfIncludingThis(mallocSizeOf);
4181
4182     // mHandles and mSpecialHandles must be accessed only on the I/O thread,
4183     // must sync dispatch.
4184     RefPtr<SizeOfHandlesRunnable> sizeOfHandlesRunnable =
4185         new SizeOfHandlesRunnable(mallocSizeOf, mHandles, mSpecialHandles);
4186     n += sizeOfHandlesRunnable->Get(mIOThread);
4187   }
4188
4189   // mHandlesByLastUsed just refers handles reported by mHandles.
4190
4191   sizeOf = do_QueryInterface(mCacheDirectory);
4192   if (sizeOf) n += sizeOf->SizeOfIncludingThis(mallocSizeOf);
4193
4194   sizeOf = do_QueryInterface(mMetadataWritesTimer);
4195   if (sizeOf) n += sizeOf->SizeOfIncludingThis(mallocSizeOf);
4196
4197   sizeOf = do_QueryInterface(mTrashTimer);
4198   if (sizeOf) n += sizeOf->SizeOfIncludingThis(mallocSizeOf);
4199
4200   sizeOf = do_QueryInterface(mTrashDir);
4201   if (sizeOf) n += sizeOf->SizeOfIncludingThis(mallocSizeOf);
4202
4203   for (uint32_t i = 0; i < mFailedTrashDirs.Length(); ++i) {
4204     n += mFailedTrashDirs[i].SizeOfExcludingThisIfUnshared(mallocSizeOf);
4205   }
4206
4207   return n;
4208 }
4209
4210 // static
4211 size_t CacheFileIOManager::SizeOfExcludingThis(
4212     mozilla::MallocSizeOf mallocSizeOf) {
4213   if (!gInstance) return 0;
4214
4215   return gInstance->SizeOfExcludingThisInternal(mallocSizeOf);
4216 }
4217
4218 // static
4219 size_t CacheFileIOManager::SizeOfIncludingThis(
4220     mozilla::MallocSizeOf mallocSizeOf) {
4221   return mallocSizeOf(gInstance) + SizeOfExcludingThis(mallocSizeOf);
4222 }
4223
4224 }  // namespace net
4225 }  // namespace mozilla