netwerk/cache2/CacheFileIOManager.h

   1 /* This Source Code Form is subject to the terms of the Mozilla Public
   2  * License, v. 2.0. If a copy of the MPL was not distributed with this
   3  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   4
   5 #ifndef CacheFileIOManager__h__
   6 #define CacheFileIOManager__h__
   7
   8 #include "CacheIOThread.h"
   9 #include "CacheStorageService.h"
  10 #include "CacheHashUtils.h"
  11 #include "nsIEventTarget.h"
  12 #include "nsINamed.h"
  13 #include "nsITimer.h"
  14 #include "nsCOMPtr.h"
  15 #include "mozilla/Atomics.h"
  16 #include "mozilla/SHA1.h"
  17 #include "mozilla/StaticPtr.h"
  18 #include "mozilla/TimeStamp.h"
  19 #include "nsTArray.h"
  20 #include "nsString.h"
  21 #include "nsTHashtable.h"
  22 #include "prio.h"
  23
  24 // #define DEBUG_HANDLES 1
  25
  26 class nsIFile;
  27 class nsITimer;
  28 class nsIDirectoryEnumerator;
  29 class nsILoadContextInfo;
  30
  31 namespace mozilla {
  32 namespace net {
  33
  34 class CacheFile;
  35 class CacheFileIOListener;
  36
  37 #ifdef DEBUG_HANDLES
  38 class CacheFileHandlesEntry;
  39 #endif
  40
  41 #define ENTRIES_DIR "entries"
  42 #define DOOMED_DIR "doomed"
  43 #define TRASH_DIR "trash"
  44
  45 class CacheFileHandle final : public nsISupports {
  46  public:
  47   enum class PinningStatus : uint32_t { UNKNOWN, NON_PINNED, PINNED };
  48
  49   NS_DECL_THREADSAFE_ISUPPORTS
  50   bool DispatchRelease();
  51
  52   CacheFileHandle(const SHA1Sum::Hash* aHash, bool aPriority,
  53                   PinningStatus aPinning);
  54   CacheFileHandle(const nsACString& aKey, bool aPriority,
  55                   PinningStatus aPinning);
  56   void Log();
  57   bool IsDoomed() const { return mIsDoomed; }
  58   const SHA1Sum::Hash* Hash() const { return mHash; }
  59   int64_t FileSize() const { return mFileSize; }
  60   uint32_t FileSizeInK() const;
  61   bool IsPriority() const { return mPriority; }
  62   bool FileExists() const { return mFileExists; }
  63   bool IsClosed() const { return mClosed; }
  64   bool IsSpecialFile() const { return mSpecialFile; }
  65   nsCString& Key() { return mKey; }
  66
  67   // Returns false when this handle has been doomed based on the pinning state
  68   // update.
  69   bool SetPinned(bool aPinned);
  70   void SetInvalid() { mInvalid = true; }
  71
  72   // Memory reporting
  73   size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
  74   size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
  75
  76  private:
  77   friend class CacheFileIOManager;
  78   friend class CacheFileHandles;
  79   friend class ReleaseNSPRHandleEvent;
  80
  81   virtual ~CacheFileHandle();
  82
  83   const SHA1Sum::Hash* mHash;
  84   mozilla::Atomic<bool, ReleaseAcquire> mIsDoomed;
  85   mozilla::Atomic<bool, ReleaseAcquire> mClosed;
  86
  87   // mPriority and mSpecialFile are plain "bool", not "bool:1", so as to
  88   // avoid bitfield races with the byte containing mInvalid et al.  See
  89   // bug 1278502.
  90   bool const mPriority;
  91   bool const mSpecialFile;
  92
  93   mozilla::Atomic<bool, Relaxed> mInvalid;
  94
  95   // These bit flags are all accessed only on the IO thread
  96   bool mFileExists : 1;  // This means that the file should exists,
  97                          // but it can be still deleted by OS/user
  98                          // and then a subsequent OpenNSPRFileDesc()
  99                          // will fail.
 100
 101   // Both initially false.  Can be raised to true only when this handle is to be
 102   // doomed during the period when the pinning status is unknown.  After the
 103   // pinning status determination we check these flags and possibly doom. These
 104   // flags are only accessed on the IO thread.
 105   bool mDoomWhenFoundPinned : 1;
 106   bool mDoomWhenFoundNonPinned : 1;
 107   // Set when after shutdown AND:
 108   // - when writing: writing data (not metadata) OR the physical file handle is
 109   //   not currently open
 110   // - when truncating: the physical file handle is not currently open
 111   // When set it prevents any further writes or truncates on such handles to
 112   // happen immediately after shutdown and gives a chance to write metadata of
 113   // already open files quickly as possible (only that renders them actually
 114   // usable by the cache.)
 115   bool mKilled : 1;
 116   // For existing files this is always pre-set to UNKNOWN.  The status is
 117   // udpated accordingly after the matadata has been parsed. For new files the
 118   // flag is set according to which storage kind is opening the cache entry and
 119   // remains so for the handle's lifetime. The status can only change from
 120   // UNKNOWN (if set so initially) to one of PINNED or NON_PINNED and it stays
 121   // unchanged afterwards. This status is only accessed on the IO thread.
 122   PinningStatus mPinning;
 123
 124   nsCOMPtr<nsIFile> mFile;
 125
 126   // file size is atomic because it is used on main thread by
 127   // nsHttpChannel::ReportNetVSCacheTelemetry()
 128   Atomic<int64_t, Relaxed> mFileSize;
 129   PRFileDesc* mFD;  // if null then the file doesn't exists on the disk
 130   nsCString mKey;
 131 };
 132
 133 class CacheFileHandles {
 134  public:
 135   CacheFileHandles();
 136   ~CacheFileHandles();
 137
 138   nsresult GetHandle(const SHA1Sum::Hash* aHash, CacheFileHandle** _retval);
 139   already_AddRefed<CacheFileHandle> NewHandle(const SHA1Sum::Hash*,
 140                                               bool aPriority,
 141                                               CacheFileHandle::PinningStatus);
 142   void RemoveHandle(CacheFileHandle* aHandle);
 143   void GetAllHandles(nsTArray<RefPtr<CacheFileHandle> >* _retval);
 144   void GetActiveHandles(nsTArray<RefPtr<CacheFileHandle> >* _retval);
 145   void ClearAll();
 146   uint32_t HandleCount();
 147
 148 #ifdef DEBUG_HANDLES
 149   void Log(CacheFileHandlesEntry* entry);
 150 #endif
 151
 152   // Memory reporting
 153   size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
 154   size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
 155
 156   class HandleHashKey : public PLDHashEntryHdr {
 157    public:
 158     using KeyType = const SHA1Sum::Hash&;
 159     using KeyTypePointer = const SHA1Sum::Hash*;
 160
 161     explicit HandleHashKey(KeyTypePointer aKey) {
 162       MOZ_COUNT_CTOR(HandleHashKey);
 163       mHash = MakeUnique<uint8_t[]>(SHA1Sum::kHashSize);
 164       memcpy(mHash.get(), aKey, sizeof(SHA1Sum::Hash));
 165     }
 166     HandleHashKey(const HandleHashKey& aOther) {
 167       MOZ_ASSERT_UNREACHABLE("HandleHashKey copy constructor is forbidden!");
 168     }
 169     MOZ_COUNTED_DTOR(HandleHashKey)
 170
 171     bool KeyEquals(KeyTypePointer aKey) const {
 172       return memcmp(mHash.get(), aKey, sizeof(SHA1Sum::Hash)) == 0;
 173     }
 174     static KeyTypePointer KeyToPointer(KeyType aKey) { return &aKey; }
 175     static PLDHashNumber HashKey(KeyTypePointer aKey) {
 176       return (reinterpret_cast<const uint32_t*>(aKey))[0];
 177     }
 178
 179     void AddHandle(CacheFileHandle* aHandle);
 180     void RemoveHandle(CacheFileHandle* aHandle);
 181     already_AddRefed<CacheFileHandle> GetNewestHandle();
 182     void GetHandles(nsTArray<RefPtr<CacheFileHandle> >& aResult);
 183
 184     SHA1Sum::Hash* Hash() const {
 185       return reinterpret_cast<SHA1Sum::Hash*>(mHash.get());
 186     }
 187     bool IsEmpty() const { return mHandles.Length() == 0; }
 188
 189     enum { ALLOW_MEMMOVE = true };
 190
 191 #ifdef DEBUG
 192     void AssertHandlesState();
 193 #endif
 194
 195     // Memory reporting
 196     size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
 197     size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
 198
 199    private:
 200     // We can't make this UniquePtr<SHA1Sum::Hash>, because you can't have
 201     // UniquePtrs with known bounds.  So we settle for this representation
 202     // and using appropriate casts when we need to access it as a
 203     // SHA1Sum::Hash.
 204     UniquePtr<uint8_t[]> mHash;
 205     // Use weak pointers since the hash table access is on a single thread
 206     // only and CacheFileHandle removes itself from this table in its dtor
 207     // that may only be called on the same thread as we work with the hashtable
 208     // since we dispatch its Release() to this thread.
 209     nsTArray<CacheFileHandle*> mHandles;
 210   };
 211
 212  private:
 213   nsTHashtable<HandleHashKey> mTable;
 214 };
 215
 216 ////////////////////////////////////////////////////////////////////////////////
 217
 218 class OpenFileEvent;
 219 class ReadEvent;
 220 class WriteEvent;
 221 class MetadataWriteScheduleEvent;
 222 class CacheFileContextEvictor;
 223
 224 #define CACHEFILEIOLISTENER_IID                      \
 225   { /* dcaf2ddc-17cf-4242-bca1-8c86936375a5 */       \
 226     0xdcaf2ddc, 0x17cf, 0x4242, {                    \
 227       0xbc, 0xa1, 0x8c, 0x86, 0x93, 0x63, 0x75, 0xa5 \
 228     }                                                \
 229   }
 230
 231 class CacheFileIOListener : public nsISupports {
 232  public:
 233   NS_DECLARE_STATIC_IID_ACCESSOR(CACHEFILEIOLISTENER_IID)
 234
 235   NS_IMETHOD OnFileOpened(CacheFileHandle* aHandle, nsresult aResult) = 0;
 236   NS_IMETHOD OnDataWritten(CacheFileHandle* aHandle, const char* aBuf,
 237                            nsresult aResult) = 0;
 238   NS_IMETHOD OnDataRead(CacheFileHandle* aHandle, char* aBuf,
 239                         nsresult aResult) = 0;
 240   NS_IMETHOD OnFileDoomed(CacheFileHandle* aHandle, nsresult aResult) = 0;
 241   NS_IMETHOD OnEOFSet(CacheFileHandle* aHandle, nsresult aResult) = 0;
 242   NS_IMETHOD OnFileRenamed(CacheFileHandle* aHandle, nsresult aResult) = 0;
 243
 244   virtual bool IsKilled() { return false; }
 245 };
 246
 247 NS_DEFINE_STATIC_IID_ACCESSOR(CacheFileIOListener, CACHEFILEIOLISTENER_IID)
 248
 249 class CacheFileIOManager final : public nsITimerCallback, public nsINamed {
 250  public:
 251   NS_DECL_THREADSAFE_ISUPPORTS
 252   NS_DECL_NSITIMERCALLBACK
 253   NS_DECL_NSINAMED
 254
 255   enum {
 256     OPEN = 0U,
 257     CREATE = 1U,
 258     CREATE_NEW = 2U,
 259     PRIORITY = 4U,
 260     SPECIAL_FILE = 8U,
 261     PINNED = 16U
 262   };
 263
 264   CacheFileIOManager();
 265
 266   static nsresult Init();
 267   static nsresult Shutdown();
 268   static nsresult OnProfile();
 269   static nsresult OnDelayedStartupFinished();
 270   static nsresult OnIdleDaily();
 271   static already_AddRefed<nsIEventTarget> IOTarget();
 272   static already_AddRefed<CacheIOThread> IOThread();
 273   static bool IsOnIOThread();
 274   static bool IsOnIOThreadOrCeased();
 275   static bool IsShutdown();
 276
 277   // Make aFile's WriteMetadataIfNeeded be called automatically after
 278   // a short interval.
 279   static nsresult ScheduleMetadataWrite(CacheFile* aFile);
 280   // Remove aFile from the scheduling registry array.
 281   // WriteMetadataIfNeeded will not be automatically called.
 282   static nsresult UnscheduleMetadataWrite(CacheFile* aFile);
 283   // Shuts the scheduling off and flushes all pending metadata writes.
 284   static nsresult ShutdownMetadataWriteScheduling();
 285
 286   static nsresult OpenFile(const nsACString& aKey, uint32_t aFlags,
 287                            CacheFileIOListener* aCallback);
 288   static nsresult Read(CacheFileHandle* aHandle, int64_t aOffset, char* aBuf,
 289                        int32_t aCount, CacheFileIOListener* aCallback);
 290   // This function must be called with a callback. The caller is responsible for
 291   // releasing |aBuf|.
 292   static nsresult Write(CacheFileHandle* aHandle, int64_t aOffset,
 293                         const char* aBuf, int32_t aCount, bool aValidate,
 294                         bool aTruncate, CacheFileIOListener* aCallback);
 295   // Similar to the above, but without the callback. Note that |aBuf| will be
 296   // released by CacheFileIOManager.
 297   static nsresult WriteWithoutCallback(CacheFileHandle* aHandle,
 298                                        int64_t aOffset, char* aBuf,
 299                                        int32_t aCount, bool aValidate,
 300                                        bool aTruncate);
 301   // PinningDoomRestriction:
 302   // NO_RESTRICTION
 303   //    no restriction is checked, the file is simply always doomed
 304   // DOOM_WHEN_(NON)_PINNED, we branch based on the pinning status of the
 305   // handle:
 306   //   UNKNOWN: the handle is marked to be doomed when later found (non)pinned
 307   //   PINNED/NON_PINNED: doom only when the restriction matches the pin status
 308   //      and the handle has not yet been required to doom during the UNKNOWN
 309   //      period
 310   enum PinningDoomRestriction {
 311     NO_RESTRICTION,
 312     DOOM_WHEN_NON_PINNED,
 313     DOOM_WHEN_PINNED
 314   };
 315   static nsresult DoomFile(CacheFileHandle* aHandle,
 316                            CacheFileIOListener* aCallback);
 317   static nsresult DoomFileByKey(const nsACString& aKey,
 318                                 CacheFileIOListener* aCallback);
 319   static nsresult ReleaseNSPRHandle(CacheFileHandle* aHandle);
 320   static nsresult TruncateSeekSetEOF(CacheFileHandle* aHandle,
 321                                      int64_t aTruncatePos, int64_t aEOFPos,
 322                                      CacheFileIOListener* aCallback);
 323   static nsresult RenameFile(CacheFileHandle* aHandle,
 324                              const nsACString& aNewName,
 325                              CacheFileIOListener* aCallback);
 326   static nsresult EvictIfOverLimit();
 327   static nsresult EvictAll();
 328   static nsresult EvictByContext(nsILoadContextInfo* aLoadContextInfo,
 329                                  bool aPinned, const nsAString& aOrigin,
 330                                  const nsAString& aBaseDomain = u""_ns);
 331
 332   static nsresult InitIndexEntry(CacheFileHandle* aHandle,
 333                                  OriginAttrsHash aOriginAttrsHash,
 334                                  bool aAnonymous, bool aPinning);
 335   static nsresult UpdateIndexEntry(CacheFileHandle* aHandle,
 336                                    const uint32_t* aFrecency,
 337                                    const bool* aHasAltData,
 338                                    const uint16_t* aOnStartTime,
 339                                    const uint16_t* aOnStopTime,
 340                                    const uint8_t* aContentType);
 341
 342   static nsresult UpdateIndexEntry();
 343
 344   enum EEnumerateMode { ENTRIES, DOOMED };
 345
 346   static void GetCacheDirectory(nsIFile** result);
 347 #if defined(MOZ_WIDGET_ANDROID)
 348   static void GetProfilelessCacheDirectory(nsIFile** result);
 349 #endif
 350
 351   // Calls synchronously OnEntryInfo for an entry with the given hash.
 352   // Tries to find an existing entry in the service hashtables first, if not
 353   // found, loads synchronously from disk file.
 354   // Callable on the IO thread only.
 355   static nsresult GetEntryInfo(
 356       const SHA1Sum::Hash* aHash,
 357       CacheStorageService::EntryInfoCallback* aCallback);
 358
 359   // Memory reporting
 360   static size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf);
 361   static size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf);
 362
 363  private:
 364   friend class CacheFileHandle;
 365   friend class CacheFileChunk;
 366   friend class CacheFile;
 367   friend class ShutdownEvent;
 368   friend class OpenFileEvent;
 369   friend class CloseHandleEvent;
 370   friend class ReadEvent;
 371   friend class WriteEvent;
 372   friend class DoomFileEvent;
 373   friend class DoomFileByKeyEvent;
 374   friend class ReleaseNSPRHandleEvent;
 375   friend class TruncateSeekSetEOFEvent;
 376   friend class RenameFileEvent;
 377   friend class CacheIndex;
 378   friend class MetadataWriteScheduleEvent;
 379   friend class CacheFileContextEvictor;
 380
 381   virtual ~CacheFileIOManager();
 382
 383   nsresult InitInternal();
 384   void ShutdownInternal();
 385
 386   nsresult OpenFileInternal(const SHA1Sum::Hash* aHash, const nsACString& aKey,
 387                             uint32_t aFlags, CacheFileHandle** _retval);
 388   nsresult OpenSpecialFileInternal(const nsACString& aKey, uint32_t aFlags,
 389                                    CacheFileHandle** _retval);
 390   void CloseHandleInternal(CacheFileHandle* aHandle);
 391   nsresult ReadInternal(CacheFileHandle* aHandle, int64_t aOffset, char* aBuf,
 392                         int32_t aCount);
 393   nsresult WriteInternal(CacheFileHandle* aHandle, int64_t aOffset,
 394                          const char* aBuf, int32_t aCount, bool aValidate,
 395                          bool aTruncate);
 396   nsresult DoomFileInternal(
 397       CacheFileHandle* aHandle,
 398       PinningDoomRestriction aPinningDoomRestriction = NO_RESTRICTION);
 399   nsresult DoomFileByKeyInternal(const SHA1Sum::Hash* aHash);
 400   nsresult MaybeReleaseNSPRHandleInternal(CacheFileHandle* aHandle,
 401                                           bool aIgnoreShutdownLag = false);
 402   nsresult TruncateSeekSetEOFInternal(CacheFileHandle* aHandle,
 403                                       int64_t aTruncatePos, int64_t aEOFPos);
 404   nsresult RenameFileInternal(CacheFileHandle* aHandle,
 405                               const nsACString& aNewName);
 406   nsresult EvictIfOverLimitInternal();
 407   nsresult OverLimitEvictionInternal();
 408   nsresult EvictAllInternal();
 409   nsresult EvictByContextInternal(nsILoadContextInfo* aLoadContextInfo,
 410                                   bool aPinned, const nsAString& aOrigin,
 411                                   const nsAString& aBaseDomain = u""_ns);
 412
 413   nsresult TrashDirectory(nsIFile* aFile);
 414   static void OnTrashTimer(nsITimer* aTimer, void* aClosure);
 415   nsresult StartRemovingTrash();
 416   nsresult RemoveTrashInternal();
 417   nsresult FindTrashDirToRemove();
 418
 419   nsresult CreateFile(CacheFileHandle* aHandle);
 420   static void HashToStr(const SHA1Sum::Hash* aHash, nsACString& _retval);
 421   static nsresult StrToHash(const nsACString& aHash, SHA1Sum::Hash* _retval);
 422   nsresult GetFile(const SHA1Sum::Hash* aHash, nsIFile** _retval);
 423   nsresult GetSpecialFile(const nsACString& aKey, nsIFile** _retval);
 424   nsresult GetDoomedFile(nsIFile** _retval);
 425   nsresult IsEmptyDirectory(nsIFile* aFile, bool* _retval);
 426   nsresult CheckAndCreateDir(nsIFile* aFile, const char* aDir,
 427                              bool aEnsureEmptyDir);
 428   nsresult CreateCacheTree();
 429   nsresult OpenNSPRHandle(CacheFileHandle* aHandle, bool aCreate = false);
 430   void NSPRHandleUsed(CacheFileHandle* aHandle);
 431
 432   // Removing all cache files during shutdown
 433   nsresult SyncRemoveDir(nsIFile* aFile, const char* aDir);
 434   void SyncRemoveAllCacheFiles();
 435
 436   nsresult ScheduleMetadataWriteInternal(CacheFile* aFile);
 437   void UnscheduleMetadataWriteInternal(CacheFile* aFile);
 438   void ShutdownMetadataWriteSchedulingInternal();
 439
 440   static nsresult CacheIndexStateChanged();
 441   void CacheIndexStateChangedInternal();
 442
 443   // Dispatches a purgeHTTP background task to delete the cache directoy
 444   // indicated by aCacheDirName.
 445   // When this feature is enabled, a task will be dispatched at shutdown
 446   // or after browser startup (to cleanup potential left-over directories)
 447   nsresult DispatchPurgeTask(const nsCString& aCacheDirName,
 448                              const nsCString& aSecondsToWait,
 449                              const nsCString& aPurgeExtension);
 450
 451   // Smart size calculation. UpdateSmartCacheSize() must be called on IO thread.
 452   // It is called in EvictIfOverLimitInternal() just before we decide whether to
 453   // start overlimit eviction or not and also in OverLimitEvictionInternal()
 454   // before we start an eviction loop.
 455   nsresult UpdateSmartCacheSize(int64_t aFreeSpace);
 456
 457   // Memory reporting (private part)
 458   size_t SizeOfExcludingThisInternal(mozilla::MallocSizeOf mallocSizeOf) const;
 459
 460   static StaticRefPtr<CacheFileIOManager> gInstance;
 461
 462   TimeStamp mStartTime;
 463   // Set true on the IO thread, CLOSE level as part of the internal shutdown
 464   // procedure.
 465   bool mShuttingDown{false};
 466   RefPtr<CacheIOThread> mIOThread;
 467   nsCOMPtr<nsIFile> mCacheDirectory;
 468 #if defined(MOZ_WIDGET_ANDROID)
 469   // On Android we add the active profile directory name between the path
 470   // and the 'cache2' leaf name.  However, to delete any leftover data from
 471   // times before we were doing it, we still need to access the directory
 472   // w/o the profile name in the path.  Here it is stored.
 473   nsCOMPtr<nsIFile> mCacheProfilelessDirectory;
 474 #endif
 475   bool mTreeCreated{false};
 476   bool mTreeCreationFailed{false};
 477   CacheFileHandles mHandles;
 478   nsTArray<CacheFileHandle*> mHandlesByLastUsed;
 479   nsTArray<CacheFileHandle*> mSpecialHandles;
 480   nsTArray<RefPtr<CacheFile> > mScheduledMetadataWrites;
 481   nsCOMPtr<nsITimer> mMetadataWritesTimer;
 482   bool mOverLimitEvicting{false};
 483   // When overlimit eviction is too slow and cache size reaches 105% of the
 484   // limit, this flag is set and no other content is cached to prevent
 485   // uncontrolled cache growing.
 486   bool mCacheSizeOnHardLimit{false};
 487   bool mRemovingTrashDirs{false};
 488   nsCOMPtr<nsITimer> mTrashTimer;
 489   nsCOMPtr<nsIFile> mTrashDir;
 490   nsCOMPtr<nsIDirectoryEnumerator> mTrashDirEnumerator;
 491   nsTArray<nsCString> mFailedTrashDirs;
 492   RefPtr<CacheFileContextEvictor> mContextEvictor;
 493   TimeStamp mLastSmartSizeTime;
 494 };
 495
 496 }  // namespace net
 497 }  // namespace mozilla
 498
 499 #endif