netwerk/cache2/CacheFileIOManager.h

   1 /* This Source Code Form is subject to the terms of the Mozilla Public
   2  * License, v. 2.0. If a copy of the MPL was not distributed with this
   3  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   4
   5 #ifndef CacheFileIOManager__h__
   6 #define CacheFileIOManager__h__
   7
   8 #include "CacheIOThread.h"
   9 #include "CacheStorageService.h"
  10 #include "CacheHashUtils.h"
  11 #include "nsIEventTarget.h"
  12 #include "nsINamed.h"
  13 #include "nsITimer.h"
  14 #include "nsCOMPtr.h"
  15 #include "mozilla/Atomics.h"
  16 #include "mozilla/SHA1.h"
  17 #include "mozilla/StaticPtr.h"
  18 #include "mozilla/TimeStamp.h"
  19 #include "nsTArray.h"
  20 #include "nsString.h"
  21 #include "nsTHashtable.h"
  22 #include "prio.h"
  23
  24 // #define DEBUG_HANDLES 1
  25
  26 class nsIFile;
  27 class nsITimer;
  28 class nsIDirectoryEnumerator;
  29 class nsILoadContextInfo;
  30
  31 namespace mozilla {
  32 namespace net {
  33
  34 class CacheFile;
  35 class CacheFileIOListener;
  36
  37 #ifdef DEBUG_HANDLES
  38 class CacheFileHandlesEntry;
  39 #endif
  40
  41 #define ENTRIES_DIR "entries"
  42 #define DOOMED_DIR "doomed"
  43 #define TRASH_DIR "trash"
  44
  45 class CacheFileHandle final : public nsISupports {
  46  public:
  47   enum class PinningStatus : uint32_t { UNKNOWN, NON_PINNED, PINNED };
  48
  49   NS_DECL_THREADSAFE_ISUPPORTS
  50   bool DispatchRelease();
  51
  52   CacheFileHandle(const SHA1Sum::Hash* aHash, bool aPriority,
  53                   PinningStatus aPinning);
  54   CacheFileHandle(const nsACString& aKey, bool aPriority,
  55                   PinningStatus aPinning);
  56   void Log();
  57   bool IsDoomed() const { return mIsDoomed; }
  58   const SHA1Sum::Hash* Hash() const { return mHash; }
  59   int64_t FileSize() const { return mFileSize; }
  60   uint32_t FileSizeInK() const;
  61   bool IsPriority() const { return mPriority; }
  62   bool FileExists() const { return mFileExists; }
  63   bool IsClosed() const { return mClosed; }
  64   bool IsSpecialFile() const { return mSpecialFile; }
  65   nsCString& Key() { return mKey; }
  66
  67   // Returns false when this handle has been doomed based on the pinning state
  68   // update.
  69   bool SetPinned(bool aPinned);
  70   void SetInvalid() { mInvalid = true; }
  71
  72   // Memory reporting
  73   size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
  74   size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
  75
  76  private:
  77   friend class CacheFileIOManager;
  78   friend class CacheFileHandles;
  79   friend class ReleaseNSPRHandleEvent;
  80
  81   virtual ~CacheFileHandle();
  82
  83   const SHA1Sum::Hash* mHash;
  84   mozilla::Atomic<bool, ReleaseAcquire> mIsDoomed;
  85   mozilla::Atomic<bool, ReleaseAcquire> mClosed;
  86
  87   // mPriority and mSpecialFile are plain "bool", not "bool:1", so as to
  88   // avoid bitfield races with the byte containing mInvalid et al.  See
  89   // bug 1278502.
  90   bool const mPriority;
  91   bool const mSpecialFile;
  92
  93   mozilla::Atomic<bool, Relaxed> mInvalid;
  94
  95   // These bit flags are all accessed only on the IO thread
  96   bool mFileExists : 1;  // This means that the file should exists,
  97                          // but it can be still deleted by OS/user
  98                          // and then a subsequent OpenNSPRFileDesc()
  99                          // will fail.
 100
 101   // Both initially false.  Can be raised to true only when this handle is to be
 102   // doomed during the period when the pinning status is unknown.  After the
 103   // pinning status determination we check these flags and possibly doom. These
 104   // flags are only accessed on the IO thread.
 105   bool mDoomWhenFoundPinned : 1;
 106   bool mDoomWhenFoundNonPinned : 1;
 107   // Set when after shutdown AND:
 108   // - when writing: writing data (not metadata) OR the physical file handle is
 109   //   not currently open
 110   // - when truncating: the physical file handle is not currently open
 111   // When set it prevents any further writes or truncates on such handles to
 112   // happen immediately after shutdown and gives a chance to write metadata of
 113   // already open files quickly as possible (only that renders them actually
 114   // usable by the cache.)
 115   bool mKilled : 1;
 116   // For existing files this is always pre-set to UNKNOWN.  The status is
 117   // udpated accordingly after the matadata has been parsed. For new files the
 118   // flag is set according to which storage kind is opening the cache entry and
 119   // remains so for the handle's lifetime. The status can only change from
 120   // UNKNOWN (if set so initially) to one of PINNED or NON_PINNED and it stays
 121   // unchanged afterwards. This status is only accessed on the IO thread.
 122   PinningStatus mPinning;
 123
 124   nsCOMPtr<nsIFile> mFile;
 125
 126   // file size is atomic because it is used on main thread by
 127   // nsHttpChannel::ReportNetVSCacheTelemetry()
 128   Atomic<int64_t, Relaxed> mFileSize;
 129   PRFileDesc* mFD;  // if null then the file doesn't exists on the disk
 130   nsCString mKey;
 131 };
 132
 133 class CacheFileHandles {
 134  public:
 135   CacheFileHandles();
 136   ~CacheFileHandles();
 137
 138   nsresult GetHandle(const SHA1Sum::Hash* aHash, CacheFileHandle** _retval);
 139   already_AddRefed<CacheFileHandle> NewHandle(const SHA1Sum::Hash*,
 140                                               bool aPriority,
 141                                               CacheFileHandle::PinningStatus);
 142   void RemoveHandle(CacheFileHandle* aHandle);
 143   void GetAllHandles(nsTArray<RefPtr<CacheFileHandle> >* _retval);
 144   void GetActiveHandles(nsTArray<RefPtr<CacheFileHandle> >* _retval);
 145   void ClearAll();
 146   uint32_t HandleCount();
 147
 148 #ifdef DEBUG_HANDLES
 149   void Log(CacheFileHandlesEntry* entry);
 150 #endif
 151
 152   // Memory reporting
 153   size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
 154   size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
 155
 156   class HandleHashKey : public PLDHashEntryHdr {
 157    public:
 158     using KeyType = const SHA1Sum::Hash&;
 159     using KeyTypePointer = const SHA1Sum::Hash*;
 160
 161     explicit HandleHashKey(KeyTypePointer aKey) {
 162       MOZ_COUNT_CTOR(HandleHashKey);
 163       mHash = MakeUnique<uint8_t[]>(SHA1Sum::kHashSize);
 164       memcpy(mHash.get(), aKey, sizeof(SHA1Sum::Hash));
 165     }
 166     HandleHashKey(const HandleHashKey& aOther) {
 167       MOZ_ASSERT_UNREACHABLE("HandleHashKey copy constructor is forbidden!");
 168     }
 169     MOZ_COUNTED_DTOR(HandleHashKey)
 170
 171     bool KeyEquals(KeyTypePointer aKey) const {
 172       return memcmp(mHash.get(), aKey, sizeof(SHA1Sum::Hash)) == 0;
 173     }
 174     static KeyTypePointer KeyToPointer(KeyType aKey) { return &aKey; }
 175     static PLDHashNumber HashKey(KeyTypePointer aKey) {
 176       return (reinterpret_cast<const uint32_t*>(aKey))[0];
 177     }
 178
 179     void AddHandle(CacheFileHandle* aHandle);
 180     void RemoveHandle(CacheFileHandle* aHandle);
 181     already_AddRefed<CacheFileHandle> GetNewestHandle();
 182     void GetHandles(nsTArray<RefPtr<CacheFileHandle> >& aResult);
 183
 184     SHA1Sum::Hash* Hash() const {
 185       return reinterpret_cast<SHA1Sum::Hash*>(mHash.get());
 186     }
 187     bool IsEmpty() const { return mHandles.Length() == 0; }
 188
 189     enum { ALLOW_MEMMOVE = true };
 190
 191 #ifdef DEBUG
 192     void AssertHandlesState();
 193 #endif
 194
 195     // Memory reporting
 196     size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
 197     size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
 198
 199    private:
 200     // We can't make this UniquePtr<SHA1Sum::Hash>, because you can't have
 201     // UniquePtrs with known bounds.  So we settle for this representation
 202     // and using appropriate casts when we need to access it as a
 203     // SHA1Sum::Hash.
 204     UniquePtr<uint8_t[]> mHash;
 205     // Use weak pointers since the hash table access is on a single thread
 206     // only and CacheFileHandle removes itself from this table in its dtor
 207     // that may only be called on the same thread as we work with the hashtable
 208     // since we dispatch its Release() to this thread.
 209     nsTArray<CacheFileHandle*> mHandles;
 210   };
 211
 212  private:
 213   nsTHashtable<HandleHashKey> mTable;
 214 };
 215
 216 ////////////////////////////////////////////////////////////////////////////////
 217
 218 class OpenFileEvent;
 219 class ReadEvent;
 220 class WriteEvent;
 221 class MetadataWriteScheduleEvent;
 222 class CacheFileContextEvictor;
 223
 224 #define CACHEFILEIOLISTENER_IID                      \
 225   { /* dcaf2ddc-17cf-4242-bca1-8c86936375a5 */       \
 226     0xdcaf2ddc, 0x17cf, 0x4242, {                    \
 227       0xbc, 0xa1, 0x8c, 0x86, 0x93, 0x63, 0x75, 0xa5 \
 228     }                                                \
 229   }
 230
 231 class CacheFileIOListener : public nsISupports {
 232  public:
 233   NS_DECLARE_STATIC_IID_ACCESSOR(CACHEFILEIOLISTENER_IID)
 234
 235   NS_IMETHOD OnFileOpened(CacheFileHandle* aHandle, nsresult aResult) = 0;
 236   NS_IMETHOD OnDataWritten(CacheFileHandle* aHandle, const char* aBuf,
 237                            nsresult aResult) = 0;
 238   NS_IMETHOD OnDataRead(CacheFileHandle* aHandle, char* aBuf,
 239                         nsresult aResult) = 0;
 240   NS_IMETHOD OnFileDoomed(CacheFileHandle* aHandle, nsresult aResult) = 0;
 241   NS_IMETHOD OnEOFSet(CacheFileHandle* aHandle, nsresult aResult) = 0;
 242   NS_IMETHOD OnFileRenamed(CacheFileHandle* aHandle, nsresult aResult) = 0;
 243
 244   virtual bool IsKilled() { return false; }
 245 };
 246
 247 NS_DEFINE_STATIC_IID_ACCESSOR(CacheFileIOListener, CACHEFILEIOLISTENER_IID)
 248
 249 class CacheFileIOManager final : public nsITimerCallback, public nsINamed {
 250  public:
 251   NS_DECL_THREADSAFE_ISUPPORTS
 252   NS_DECL_NSITIMERCALLBACK
 253   NS_DECL_NSINAMED
 254
 255   enum {
 256     OPEN = 0U,
 257     CREATE = 1U,
 258     CREATE_NEW = 2U,
 259     PRIORITY = 4U,
 260     SPECIAL_FILE = 8U,
 261     PINNED = 16U
 262   };
 263
 264   CacheFileIOManager();
 265
 266   static nsresult Init();
 267   static nsresult Shutdown();
 268   static nsresult OnProfile();
 269   static nsresult OnDelayedStartupFinished();
 270   static nsresult OnIdleDaily();
 271   static already_AddRefed<nsIEventTarget> IOTarget();
 272   static already_AddRefed<CacheIOThread> IOThread();
 273   static bool IsOnIOThread();
 274   static bool IsOnIOThreadOrCeased();
 275   static bool IsShutdown();
 276
 277   // Make aFile's WriteMetadataIfNeeded be called automatically after
 278   // a short interval.
 279   static nsresult ScheduleMetadataWrite(CacheFile* aFile);
 280   // Remove aFile from the scheduling registry array.
 281   // WriteMetadataIfNeeded will not be automatically called.
 282   static nsresult UnscheduleMetadataWrite(CacheFile* aFile);
 283   // Shuts the scheduling off and flushes all pending metadata writes.
 284   static nsresult ShutdownMetadataWriteScheduling();
 285
 286   static nsresult OpenFile(const nsACString& aKey, uint32_t aFlags,
 287                            CacheFileIOListener* aCallback);
 288   static nsresult Read(CacheFileHandle* aHandle, int64_t aOffset, char* aBuf,
 289                        int32_t aCount, CacheFileIOListener* aCallback);
 290   static nsresult Write(CacheFileHandle* aHandle, int64_t aOffset,
 291                         const char* aBuf, int32_t aCount, bool aValidate,
 292                         bool aTruncate, CacheFileIOListener* aCallback);
 293   // PinningDoomRestriction:
 294   // NO_RESTRICTION
 295   //    no restriction is checked, the file is simply always doomed
 296   // DOOM_WHEN_(NON)_PINNED, we branch based on the pinning status of the
 297   // handle:
 298   //   UNKNOWN: the handle is marked to be doomed when later found (non)pinned
 299   //   PINNED/NON_PINNED: doom only when the restriction matches the pin status
 300   //      and the handle has not yet been required to doom during the UNKNOWN
 301   //      period
 302   enum PinningDoomRestriction {
 303     NO_RESTRICTION,
 304     DOOM_WHEN_NON_PINNED,
 305     DOOM_WHEN_PINNED
 306   };
 307   static nsresult DoomFile(CacheFileHandle* aHandle,
 308                            CacheFileIOListener* aCallback);
 309   static nsresult DoomFileByKey(const nsACString& aKey,
 310                                 CacheFileIOListener* aCallback);
 311   static nsresult ReleaseNSPRHandle(CacheFileHandle* aHandle);
 312   static nsresult TruncateSeekSetEOF(CacheFileHandle* aHandle,
 313                                      int64_t aTruncatePos, int64_t aEOFPos,
 314                                      CacheFileIOListener* aCallback);
 315   static nsresult RenameFile(CacheFileHandle* aHandle,
 316                              const nsACString& aNewName,
 317                              CacheFileIOListener* aCallback);
 318   static nsresult EvictIfOverLimit();
 319   static nsresult EvictAll();
 320   static nsresult EvictByContext(nsILoadContextInfo* aLoadContextInfo,
 321                                  bool aPinned, const nsAString& aOrigin,
 322                                  const nsAString& aBaseDomain = u""_ns);
 323
 324   static nsresult InitIndexEntry(CacheFileHandle* aHandle,
 325                                  OriginAttrsHash aOriginAttrsHash,
 326                                  bool aAnonymous, bool aPinning);
 327   static nsresult UpdateIndexEntry(CacheFileHandle* aHandle,
 328                                    const uint32_t* aFrecency,
 329                                    const bool* aHasAltData,
 330                                    const uint16_t* aOnStartTime,
 331                                    const uint16_t* aOnStopTime,
 332                                    const uint8_t* aContentType);
 333
 334   static nsresult UpdateIndexEntry();
 335
 336   enum EEnumerateMode { ENTRIES, DOOMED };
 337
 338   static void GetCacheDirectory(nsIFile** result);
 339 #if defined(MOZ_WIDGET_ANDROID)
 340   static void GetProfilelessCacheDirectory(nsIFile** result);
 341 #endif
 342
 343   // Calls synchronously OnEntryInfo for an entry with the given hash.
 344   // Tries to find an existing entry in the service hashtables first, if not
 345   // found, loads synchronously from disk file.
 346   // Callable on the IO thread only.
 347   static nsresult GetEntryInfo(
 348       const SHA1Sum::Hash* aHash,
 349       CacheStorageService::EntryInfoCallback* aCallback);
 350
 351   // Memory reporting
 352   static size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf);
 353   static size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf);
 354
 355  private:
 356   friend class CacheFileHandle;
 357   friend class CacheFileChunk;
 358   friend class CacheFile;
 359   friend class ShutdownEvent;
 360   friend class OpenFileEvent;
 361   friend class CloseHandleEvent;
 362   friend class ReadEvent;
 363   friend class WriteEvent;
 364   friend class DoomFileEvent;
 365   friend class DoomFileByKeyEvent;
 366   friend class ReleaseNSPRHandleEvent;
 367   friend class TruncateSeekSetEOFEvent;
 368   friend class RenameFileEvent;
 369   friend class CacheIndex;
 370   friend class MetadataWriteScheduleEvent;
 371   friend class CacheFileContextEvictor;
 372
 373   virtual ~CacheFileIOManager();
 374
 375   nsresult InitInternal();
 376   void ShutdownInternal();
 377
 378   nsresult OpenFileInternal(const SHA1Sum::Hash* aHash, const nsACString& aKey,
 379                             uint32_t aFlags, CacheFileHandle** _retval);
 380   nsresult OpenSpecialFileInternal(const nsACString& aKey, uint32_t aFlags,
 381                                    CacheFileHandle** _retval);
 382   void CloseHandleInternal(CacheFileHandle* aHandle);
 383   nsresult ReadInternal(CacheFileHandle* aHandle, int64_t aOffset, char* aBuf,
 384                         int32_t aCount);
 385   nsresult WriteInternal(CacheFileHandle* aHandle, int64_t aOffset,
 386                          const char* aBuf, int32_t aCount, bool aValidate,
 387                          bool aTruncate);
 388   nsresult DoomFileInternal(
 389       CacheFileHandle* aHandle,
 390       PinningDoomRestriction aPinningDoomRestriction = NO_RESTRICTION);
 391   nsresult DoomFileByKeyInternal(const SHA1Sum::Hash* aHash);
 392   nsresult MaybeReleaseNSPRHandleInternal(CacheFileHandle* aHandle,
 393                                           bool aIgnoreShutdownLag = false);
 394   nsresult TruncateSeekSetEOFInternal(CacheFileHandle* aHandle,
 395                                       int64_t aTruncatePos, int64_t aEOFPos);
 396   nsresult RenameFileInternal(CacheFileHandle* aHandle,
 397                               const nsACString& aNewName);
 398   nsresult EvictIfOverLimitInternal();
 399   nsresult OverLimitEvictionInternal();
 400   nsresult EvictAllInternal();
 401   nsresult EvictByContextInternal(nsILoadContextInfo* aLoadContextInfo,
 402                                   bool aPinned, const nsAString& aOrigin,
 403                                   const nsAString& aBaseDomain = u""_ns);
 404
 405   nsresult TrashDirectory(nsIFile* aFile);
 406   static void OnTrashTimer(nsITimer* aTimer, void* aClosure);
 407   nsresult StartRemovingTrash();
 408   nsresult RemoveTrashInternal();
 409   nsresult FindTrashDirToRemove();
 410
 411   nsresult CreateFile(CacheFileHandle* aHandle);
 412   static void HashToStr(const SHA1Sum::Hash* aHash, nsACString& _retval);
 413   static nsresult StrToHash(const nsACString& aHash, SHA1Sum::Hash* _retval);
 414   nsresult GetFile(const SHA1Sum::Hash* aHash, nsIFile** _retval);
 415   nsresult GetSpecialFile(const nsACString& aKey, nsIFile** _retval);
 416   nsresult GetDoomedFile(nsIFile** _retval);
 417   nsresult IsEmptyDirectory(nsIFile* aFile, bool* _retval);
 418   nsresult CheckAndCreateDir(nsIFile* aFile, const char* aDir,
 419                              bool aEnsureEmptyDir);
 420   nsresult CreateCacheTree();
 421   nsresult OpenNSPRHandle(CacheFileHandle* aHandle, bool aCreate = false);
 422   void NSPRHandleUsed(CacheFileHandle* aHandle);
 423
 424   // Removing all cache files during shutdown
 425   nsresult SyncRemoveDir(nsIFile* aFile, const char* aDir);
 426   void SyncRemoveAllCacheFiles();
 427
 428   nsresult ScheduleMetadataWriteInternal(CacheFile* aFile);
 429   void UnscheduleMetadataWriteInternal(CacheFile* aFile);
 430   void ShutdownMetadataWriteSchedulingInternal();
 431
 432   static nsresult CacheIndexStateChanged();
 433   void CacheIndexStateChangedInternal();
 434
 435   // Dispatches a purgeHTTP background task to delete the cache directoy
 436   // indicated by aCacheDirName.
 437   // When this feature is enabled, a task will be dispatched at shutdown
 438   // or after browser startup (to cleanup potential left-over directories)
 439   nsresult DispatchPurgeTask(const nsCString& aCacheDirName,
 440                              const nsCString& aSecondsToWait,
 441                              const nsCString& aPurgeExtension);
 442
 443   // Smart size calculation. UpdateSmartCacheSize() must be called on IO thread.
 444   // It is called in EvictIfOverLimitInternal() just before we decide whether to
 445   // start overlimit eviction or not and also in OverLimitEvictionInternal()
 446   // before we start an eviction loop.
 447   nsresult UpdateSmartCacheSize(int64_t aFreeSpace);
 448
 449   // Memory reporting (private part)
 450   size_t SizeOfExcludingThisInternal(mozilla::MallocSizeOf mallocSizeOf) const;
 451
 452   static StaticRefPtr<CacheFileIOManager> gInstance;
 453
 454   TimeStamp mStartTime;
 455   // Set true on the IO thread, CLOSE level as part of the internal shutdown
 456   // procedure.
 457   bool mShuttingDown{false};
 458   RefPtr<CacheIOThread> mIOThread;
 459   nsCOMPtr<nsIFile> mCacheDirectory;
 460 #if defined(MOZ_WIDGET_ANDROID)
 461   // On Android we add the active profile directory name between the path
 462   // and the 'cache2' leaf name.  However, to delete any leftover data from
 463   // times before we were doing it, we still need to access the directory
 464   // w/o the profile name in the path.  Here it is stored.
 465   nsCOMPtr<nsIFile> mCacheProfilelessDirectory;
 466 #endif
 467   bool mTreeCreated{false};
 468   bool mTreeCreationFailed{false};
 469   CacheFileHandles mHandles;
 470   nsTArray<CacheFileHandle*> mHandlesByLastUsed;
 471   nsTArray<CacheFileHandle*> mSpecialHandles;
 472   nsTArray<RefPtr<CacheFile> > mScheduledMetadataWrites;
 473   nsCOMPtr<nsITimer> mMetadataWritesTimer;
 474   bool mOverLimitEvicting{false};
 475   // When overlimit eviction is too slow and cache size reaches 105% of the
 476   // limit, this flag is set and no other content is cached to prevent
 477   // uncontrolled cache growing.
 478   bool mCacheSizeOnHardLimit{false};
 479   bool mRemovingTrashDirs{false};
 480   nsCOMPtr<nsITimer> mTrashTimer;
 481   nsCOMPtr<nsIFile> mTrashDir;
 482   nsCOMPtr<nsIDirectoryEnumerator> mTrashDirEnumerator;
 483   nsTArray<nsCString> mFailedTrashDirs;
 484   RefPtr<CacheFileContextEvictor> mContextEvictor;
 485   TimeStamp mLastSmartSizeTime;
 486 };
 487
 488 }  // namespace net
 489 }  // namespace mozilla
 490
 491 #endif