Merge mozilla-central to autoland on a CLOSED TREE
[gecko.git] / dom / cache / QuotaClient.cpp
blob565450964f80dee8f3defd95af0ec4482c208b26
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #include "QuotaClientImpl.h"
9 #include "DBAction.h"
10 #include "FileUtilsImpl.h"
11 #include "mozilla/ResultExtensions.h"
12 #include "mozilla/Telemetry.h"
13 #include "mozilla/Unused.h"
14 #include "mozilla/dom/cache/DBSchema.h"
15 #include "mozilla/dom/cache/Manager.h"
16 #include "mozilla/dom/quota/QuotaCommon.h"
17 #include "mozilla/dom/quota/QuotaManager.h"
18 #include "mozilla/dom/quota/UsageInfo.h"
19 #include "mozilla/ipc/BackgroundParent.h"
20 #include "nsIFile.h"
21 #include "nsThreadUtils.h"
23 namespace mozilla::dom::cache {
25 using mozilla::dom::quota::AssertIsOnIOThread;
26 using mozilla::dom::quota::Client;
27 using mozilla::dom::quota::CloneFileAndAppend;
28 using mozilla::dom::quota::DatabaseUsageType;
29 using mozilla::dom::quota::GetDirEntryKind;
30 using mozilla::dom::quota::nsIFileKind;
31 using mozilla::dom::quota::OriginMetadata;
32 using mozilla::dom::quota::PERSISTENCE_TYPE_DEFAULT;
33 using mozilla::dom::quota::PersistenceType;
34 using mozilla::dom::quota::QuotaManager;
35 using mozilla::dom::quota::UsageInfo;
36 using mozilla::ipc::AssertIsOnBackgroundThread;
38 namespace {
40 template <typename StepFunc>
41 Result<UsageInfo, nsresult> ReduceUsageInfo(nsIFile& aDir,
42 const Atomic<bool>& aCanceled,
43 const StepFunc& aStepFunc) {
44 QM_TRY_RETURN(quota::ReduceEachFileAtomicCancelable(
45 aDir, aCanceled, UsageInfo{},
46 [&aStepFunc](UsageInfo usageInfo, const nsCOMPtr<nsIFile>& bodyDir)
47 -> Result<UsageInfo, nsresult> {
48 QM_TRY(OkIf(!QuotaManager::IsShuttingDown()), Err(NS_ERROR_ABORT));
50 QM_TRY_INSPECT(const auto& stepUsageInfo, aStepFunc(bodyDir));
52 return usageInfo + stepUsageInfo;
53 }));
56 Result<UsageInfo, nsresult> GetBodyUsage(nsIFile& aMorgueDir,
57 const Atomic<bool>& aCanceled) {
58 AssertIsOnIOThread();
60 QM_TRY_RETURN(ReduceUsageInfo(
61 aMorgueDir, aCanceled,
62 [](const nsCOMPtr<nsIFile>& bodyDir) -> Result<UsageInfo, nsresult> {
63 QM_TRY_INSPECT(const auto& dirEntryKind, GetDirEntryKind(*bodyDir));
65 if (dirEntryKind != nsIFileKind::ExistsAsDirectory) {
66 if (dirEntryKind == nsIFileKind::ExistsAsFile) {
67 const DebugOnly<nsresult> result =
68 RemoveNsIFile(Nothing(), *bodyDir, /* aTrackQuota */ false);
69 // Try to remove the unexpected files, and keep moving on even if it
70 // fails because it might be created by virus or the operation
71 // system
72 MOZ_ASSERT(NS_SUCCEEDED(result));
75 return UsageInfo{};
78 UsageInfo usageInfo;
79 const auto getUsage =
80 [&usageInfo](nsIFile& bodyFile,
81 const nsACString& leafName) -> Result<bool, nsresult> {
82 Unused << leafName;
84 QM_TRY_INSPECT(const int64_t& fileSize,
85 MOZ_TO_RESULT_INVOKE_MEMBER(bodyFile, GetFileSize));
86 MOZ_DIAGNOSTIC_ASSERT(fileSize >= 0);
87 // FIXME: Separate file usage and database usage in OriginInfo so that
88 // the workaround for treating body file size as database usage can be
89 // removed.
91 // This is needed because we want to remove the mutex lock for padding
92 // files. The lock is needed because the padding file is accessed on
93 // the QM IO thread while getting origin usage and is accessed on the
94 // Cache IO thread in normal Cache operations. Using the cached usage
95 // in QM while getting origin usage can remove the access on the QM IO
96 // thread and thus we can remove the mutex lock. However, QM only
97 // separates usage types in initialization, and the separation is gone
98 // after that. So, before extending the separation of usage types in
99 // QM, this is a workaround to avoid the file usage mismatching in our
100 // tests. Note that file usage hasn't been exposed to users yet.
101 usageInfo += DatabaseUsageType(Some(fileSize));
103 return false;
106 // QM_OR_ELSE_WARN_IF is not used here since we just want to log
107 // NS_ERROR_FILE_FS_CORRUPTED result and not spam the reports (even a
108 // warning in the reports is not desired).
109 QM_TRY(QM_OR_ELSE_LOG_VERBOSE_IF(
110 // Expression.
111 MOZ_TO_RESULT(BodyTraverseFiles(Nothing(), *bodyDir, getUsage,
112 /* aCanRemoveFiles */ true,
113 /* aTrackQuota */ false)),
114 // Predicate.
115 IsSpecificError<NS_ERROR_FILE_FS_CORRUPTED>,
116 // Fallback. We treat NS_ERROR_FILE_FS_CORRUPTED as if the
117 // directory did not exist at all.
118 ErrToDefaultOk<>));
119 return usageInfo;
120 }));
123 Result<int64_t, nsresult> GetPaddingSizeFromDB(
124 nsIFile& aDir, nsIFile& aDBFile, const OriginMetadata& aOriginMetadata) {
125 CacheDirectoryMetadata directoryMetadata(aOriginMetadata);
126 // directoryMetadata.mDirectoryLockId must be -1 (which is default for new
127 // CacheDirectoryMetadata) because this method should only be called from
128 // QuotaClient::InitOrigin when the temporary storage hasn't been initialized
129 // yet. At that time, the in-memory objects (e.g. OriginInfo) are only being
130 // created so it doesn't make sense to tunnel quota information to QuotaVFS
131 // to get corresponding QuotaObject instance for the SQLite file).
132 MOZ_DIAGNOSTIC_ASSERT(directoryMetadata.mDirectoryLockId == -1);
134 #ifdef DEBUG
136 QM_TRY_INSPECT(const bool& exists,
137 MOZ_TO_RESULT_INVOKE_MEMBER(aDBFile, Exists));
138 MOZ_ASSERT(exists);
140 #endif
142 QM_TRY_INSPECT(const auto& conn,
143 OpenDBConnection(directoryMetadata, aDBFile));
145 // Make sure that the database has the latest schema before we try to read
146 // from it. We have to do this because GetPaddingSizeFromDB is called
147 // by InitOrigin. And it means that SetupAction::RunSyncWithDBOnTarget hasn't
148 // checked the schema for the given origin yet).
149 QM_TRY(MOZ_TO_RESULT(db::CreateOrMigrateSchema(*conn)));
151 QM_TRY_RETURN(DirectoryPaddingRestore(aDir, *conn,
152 /* aMustRestore */ false));
155 } // namespace
157 const nsLiteralString kCachesSQLiteFilename = u"caches.sqlite"_ns;
158 const nsLiteralString kMorgueDirectoryFilename = u"morgue"_ns;
160 CacheQuotaClient::CacheQuotaClient() {
161 AssertIsOnBackgroundThread();
162 MOZ_DIAGNOSTIC_ASSERT(!sInstance);
163 sInstance = this;
166 // static
167 CacheQuotaClient* CacheQuotaClient::Get() {
168 MOZ_DIAGNOSTIC_ASSERT(sInstance);
169 return sInstance;
172 CacheQuotaClient::Type CacheQuotaClient::GetType() { return DOMCACHE; }
174 Result<UsageInfo, nsresult> CacheQuotaClient::InitOrigin(
175 PersistenceType aPersistenceType, const OriginMetadata& aOriginMetadata,
176 const AtomicBool& aCanceled) {
177 AssertIsOnIOThread();
178 MOZ_ASSERT(aOriginMetadata.mPersistenceType == aPersistenceType);
180 QuotaManager* const qm = QuotaManager::Get();
181 MOZ_DIAGNOSTIC_ASSERT(qm);
183 QM_TRY_INSPECT(const auto& dir, qm->GetOriginDirectory(aOriginMetadata));
185 QM_TRY(MOZ_TO_RESULT(
186 dir->Append(NS_LITERAL_STRING_FROM_CSTRING(DOMCACHE_DIRECTORY_NAME))));
188 QM_TRY_INSPECT(
189 const auto& cachesSQLiteFile,
190 ([dir]() -> Result<nsCOMPtr<nsIFile>, nsresult> {
191 QM_TRY_INSPECT(const auto& cachesSQLite,
192 CloneFileAndAppend(*dir, kCachesSQLiteFilename));
194 // IsDirectory is used to check if caches.sqlite exists or not. Another
195 // benefit of this is that we can test the failed cases by creating a
196 // directory named "caches.sqlite".
197 QM_TRY_INSPECT(const auto& dirEntryKind,
198 GetDirEntryKind(*cachesSQLite));
199 if (dirEntryKind == nsIFileKind::DoesNotExist) {
200 // We only ensure padding files and morgue directory get removed like
201 // WipeDatabase in DBAction.cpp. The -wal journal file will be
202 // automatically deleted by sqlite when the new database is created.
203 // XXX Ideally, we would delete the -wal journal file as well (here
204 // and also in WipeDatabase).
205 // XXX We should have something like WipeDatabaseNoQuota for this.
206 // XXX Long term, we might even think about removing entire origin
207 // directory because missing caches.sqlite while other files exist can
208 // be interpreted as database corruption.
209 QM_TRY(MOZ_TO_RESULT(mozilla::dom::cache::DirectoryPaddingDeleteFile(
210 *dir, DirPaddingFile::TMP_FILE)));
212 QM_TRY(MOZ_TO_RESULT(mozilla::dom::cache::DirectoryPaddingDeleteFile(
213 *dir, DirPaddingFile::FILE)));
215 QM_TRY_INSPECT(const auto& morgueDir,
216 CloneFileAndAppend(*dir, kMorgueDirectoryFilename));
218 QM_TRY(MOZ_TO_RESULT(mozilla::dom::cache::RemoveNsIFileRecursively(
219 Nothing(), *morgueDir,
220 /* aTrackQuota */ false)));
222 return nsCOMPtr<nsIFile>{nullptr};
225 QM_TRY(OkIf(dirEntryKind == nsIFileKind::ExistsAsFile),
226 Err(NS_ERROR_FAILURE));
228 return cachesSQLite;
229 }()));
231 // If the caches.sqlite doesn't exist, then padding files and morgue directory
232 // should have been removed if they existed. We ignore the rest of known files
233 // because we assume that they will be removed when a new database is created.
234 // XXX Ensure the -wel file is removed if the caches.sqlite doesn't exist.
235 QM_TRY(OkIf(!!cachesSQLiteFile), UsageInfo{});
237 QM_TRY_INSPECT(
238 const auto& paddingSize,
239 ([dir, cachesSQLiteFile,
240 &aOriginMetadata]() -> Result<int64_t, nsresult> {
241 if (!DirectoryPaddingFileExists(*dir, DirPaddingFile::TMP_FILE)) {
242 QM_WARNONLY_TRY_UNWRAP(const auto maybePaddingSize,
243 DirectoryPaddingGet(*dir));
244 if (maybePaddingSize) {
245 return maybePaddingSize.ref();
249 // If the temporary file still exists or failing to get the padding size
250 // from the padding file, then we need to get the padding size from the
251 // database and restore the padding file.
252 QM_TRY_RETURN(
253 GetPaddingSizeFromDB(*dir, *cachesSQLiteFile, aOriginMetadata));
254 }()));
256 QM_TRY_INSPECT(
257 const auto& innerUsageInfo,
258 ReduceUsageInfo(
259 *dir, aCanceled,
260 [&aCanceled](
261 const nsCOMPtr<nsIFile>& file) -> Result<UsageInfo, nsresult> {
262 QM_TRY_INSPECT(const auto& leafName,
263 MOZ_TO_RESULT_INVOKE_MEMBER_TYPED(nsAutoString, file,
264 GetLeafName));
266 QM_TRY_INSPECT(const auto& dirEntryKind, GetDirEntryKind(*file));
268 switch (dirEntryKind) {
269 case nsIFileKind::ExistsAsDirectory:
270 if (leafName.EqualsLiteral("morgue")) {
271 QM_TRY_RETURN(GetBodyUsage(*file, aCanceled));
272 } else {
273 NS_WARNING("Unknown Cache directory found!");
276 break;
278 case nsIFileKind::ExistsAsFile:
279 // Ignore transient sqlite files and marker files
280 if (leafName.EqualsLiteral("caches.sqlite-journal") ||
281 leafName.EqualsLiteral("caches.sqlite-shm") ||
282 StringBeginsWith(leafName, u"caches.sqlite-mj"_ns) ||
283 leafName.EqualsLiteral("context_open.marker")) {
284 break;
287 if (leafName.Equals(kCachesSQLiteFilename) ||
288 leafName.EqualsLiteral("caches.sqlite-wal")) {
289 QM_TRY_INSPECT(
290 const int64_t& fileSize,
291 MOZ_TO_RESULT_INVOKE_MEMBER(file, GetFileSize));
292 MOZ_DIAGNOSTIC_ASSERT(fileSize >= 0);
294 return UsageInfo{DatabaseUsageType(Some(fileSize))};
297 // Ignore directory padding file
298 if (leafName.EqualsLiteral(PADDING_FILE_NAME) ||
299 leafName.EqualsLiteral(PADDING_TMP_FILE_NAME)) {
300 break;
303 NS_WARNING("Unknown Cache file found!");
305 break;
307 case nsIFileKind::DoesNotExist:
308 // Ignore files that got removed externally while iterating.
309 break;
312 return UsageInfo{};
313 }));
315 // FIXME: Separate file usage and database usage in OriginInfo so that the
316 // workaround for treating padding file size as database usage can be removed.
317 return UsageInfo{DatabaseUsageType(Some(paddingSize))} + innerUsageInfo;
320 nsresult CacheQuotaClient::InitOriginWithoutTracking(
321 PersistenceType aPersistenceType, const OriginMetadata& aOriginMetadata,
322 const AtomicBool& aCanceled) {
323 AssertIsOnIOThread();
325 // This is called when a storage/permanent/${origin}/cache directory exists.
326 // Even though this shouldn't happen with a "good" profile, we shouldn't
327 // return an error here, since that would cause origin initialization to fail.
328 // We just warn and otherwise ignore that.
329 UNKNOWN_FILE_WARNING(NS_LITERAL_STRING_FROM_CSTRING(DOMCACHE_DIRECTORY_NAME));
330 return NS_OK;
333 Result<UsageInfo, nsresult> CacheQuotaClient::GetUsageForOrigin(
334 PersistenceType aPersistenceType, const OriginMetadata& aOriginMetadata,
335 const AtomicBool& aCanceled) {
336 AssertIsOnIOThread();
338 // We can't open the database at this point, since it can be already used by
339 // the Cache IO thread. Use the cached value instead.
341 QuotaManager* quotaManager = QuotaManager::Get();
342 MOZ_ASSERT(quotaManager);
344 return quotaManager->GetUsageForClient(PERSISTENCE_TYPE_DEFAULT,
345 aOriginMetadata, Client::DOMCACHE);
348 void CacheQuotaClient::OnOriginClearCompleted(PersistenceType aPersistenceType,
349 const nsACString& aOrigin) {
350 // Nothing to do here.
353 void CacheQuotaClient::OnRepositoryClearCompleted(
354 PersistenceType aPersistenceType) {
355 AssertIsOnIOThread();
357 // Nothing to do here.
360 void CacheQuotaClient::ReleaseIOThreadObjects() {
361 // Nothing to do here as the Context handles cleaning everything up
362 // automatically.
365 void CacheQuotaClient::AbortOperationsForLocks(
366 const DirectoryLockIdTable& aDirectoryLockIds) {
367 AssertIsOnBackgroundThread();
369 Manager::Abort(aDirectoryLockIds);
372 void CacheQuotaClient::AbortOperationsForProcess(
373 ContentParentId aContentParentId) {
374 // The Cache and Context can be shared by multiple client processes. They
375 // are not exclusively owned by a single process.
377 // As far as I can tell this is used by QuotaManager to abort operations
378 // when a particular process goes away. We definitely don't want this
379 // since we are shared. Also, the Cache actor code already properly
380 // handles asynchronous actor destruction when the child process dies.
382 // Therefore, do nothing here.
385 void CacheQuotaClient::AbortAllOperations() {
386 AssertIsOnBackgroundThread();
388 Manager::AbortAll();
391 void CacheQuotaClient::StartIdleMaintenance() {}
393 void CacheQuotaClient::StopIdleMaintenance() {}
395 void CacheQuotaClient::InitiateShutdown() {
396 AssertIsOnBackgroundThread();
398 Manager::InitiateShutdown();
401 bool CacheQuotaClient::IsShutdownCompleted() const {
402 AssertIsOnBackgroundThread();
404 return Manager::IsShutdownAllComplete();
407 void CacheQuotaClient::ForceKillActors() {
408 // Currently we don't implement killing actors (are there any to kill here?).
411 nsCString CacheQuotaClient::GetShutdownStatus() const {
412 AssertIsOnBackgroundThread();
414 return Manager::GetShutdownStatus();
417 void CacheQuotaClient::FinalizeShutdown() {
418 // Nothing to do here.
421 nsresult CacheQuotaClient::UpgradeStorageFrom2_0To2_1(nsIFile* aDirectory) {
422 AssertIsOnIOThread();
423 MOZ_DIAGNOSTIC_ASSERT(aDirectory);
425 QM_TRY(MOZ_TO_RESULT(DirectoryPaddingInit(*aDirectory)));
427 return NS_OK;
430 nsresult CacheQuotaClient::RestorePaddingFileInternal(
431 nsIFile* aBaseDir, mozIStorageConnection* aConn) {
432 MOZ_ASSERT(!NS_IsMainThread());
433 MOZ_DIAGNOSTIC_ASSERT(aBaseDir);
434 MOZ_DIAGNOSTIC_ASSERT(aConn);
436 QM_TRY_INSPECT(const int64_t& dummyPaddingSize,
437 DirectoryPaddingRestore(*aBaseDir, *aConn,
438 /* aMustRestore */ true));
439 Unused << dummyPaddingSize;
441 return NS_OK;
444 nsresult CacheQuotaClient::WipePaddingFileInternal(
445 const CacheDirectoryMetadata& aDirectoryMetadata, nsIFile* aBaseDir) {
446 MOZ_ASSERT(!NS_IsMainThread());
447 MOZ_DIAGNOSTIC_ASSERT(aBaseDir);
449 MOZ_ASSERT(DirectoryPaddingFileExists(*aBaseDir, DirPaddingFile::FILE));
451 QM_TRY_INSPECT(
452 const int64_t& paddingSize, ([&aBaseDir]() -> Result<int64_t, nsresult> {
453 const bool temporaryPaddingFileExist =
454 DirectoryPaddingFileExists(*aBaseDir, DirPaddingFile::TMP_FILE);
456 Maybe<int64_t> directoryPaddingGetResult;
457 if (!temporaryPaddingFileExist) {
458 QM_TRY_UNWRAP(directoryPaddingGetResult,
459 ([&aBaseDir]() -> Result<Maybe<int64_t>, nsresult> {
460 QM_TRY_RETURN(
461 DirectoryPaddingGet(*aBaseDir).map(Some<int64_t>),
462 Maybe<int64_t>{});
463 }()));
466 if (temporaryPaddingFileExist || !directoryPaddingGetResult) {
467 // XXXtt: Maybe have a method in the QuotaManager to clean the usage
468 // under the quota client and the origin. There is nothing we can do
469 // to recover the file.
470 NS_WARNING("Cannnot read padding size from file!");
471 return 0;
474 return *directoryPaddingGetResult;
475 }()));
477 if (paddingSize > 0) {
478 DecreaseUsageForDirectoryMetadata(aDirectoryMetadata, paddingSize);
481 QM_TRY(MOZ_TO_RESULT(
482 DirectoryPaddingDeleteFile(*aBaseDir, DirPaddingFile::FILE)));
484 // Remove temporary file if we have one.
485 QM_TRY(MOZ_TO_RESULT(
486 DirectoryPaddingDeleteFile(*aBaseDir, DirPaddingFile::TMP_FILE)));
488 QM_TRY(MOZ_TO_RESULT(DirectoryPaddingInit(*aBaseDir)));
490 return NS_OK;
493 CacheQuotaClient::~CacheQuotaClient() {
494 AssertIsOnBackgroundThread();
495 MOZ_DIAGNOSTIC_ASSERT(sInstance == this);
497 sInstance = nullptr;
500 // static
501 CacheQuotaClient* CacheQuotaClient::sInstance = nullptr;
503 // static
504 already_AddRefed<quota::Client> CreateQuotaClient() {
505 AssertIsOnBackgroundThread();
507 RefPtr<CacheQuotaClient> ref = new CacheQuotaClient();
508 return ref.forget();
511 // static
512 nsresult RestorePaddingFile(nsIFile* aBaseDir, mozIStorageConnection* aConn) {
513 MOZ_ASSERT(!NS_IsMainThread());
514 MOZ_DIAGNOSTIC_ASSERT(aBaseDir);
515 MOZ_DIAGNOSTIC_ASSERT(aConn);
517 RefPtr<CacheQuotaClient> cacheQuotaClient = CacheQuotaClient::Get();
518 MOZ_DIAGNOSTIC_ASSERT(cacheQuotaClient);
520 QM_TRY(MOZ_TO_RESULT(
521 cacheQuotaClient->RestorePaddingFileInternal(aBaseDir, aConn)));
523 return NS_OK;
526 // static
527 nsresult WipePaddingFile(const CacheDirectoryMetadata& aDirectoryMetadata,
528 nsIFile* aBaseDir) {
529 MOZ_ASSERT(!NS_IsMainThread());
530 MOZ_DIAGNOSTIC_ASSERT(aBaseDir);
532 RefPtr<CacheQuotaClient> cacheQuotaClient = CacheQuotaClient::Get();
533 MOZ_DIAGNOSTIC_ASSERT(cacheQuotaClient);
535 QM_TRY(MOZ_TO_RESULT(
536 cacheQuotaClient->WipePaddingFileInternal(aDirectoryMetadata, aBaseDir)));
538 return NS_OK;
541 } // namespace mozilla::dom::cache