dom/storage/StorageDBThread.cpp

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "StorageDBThread.h"
   8
   9 #include "StorageCommon.h"
  10 #include "StorageDBUpdater.h"
  11 #include "StorageUtils.h"
  12 #include "LocalStorageCache.h"
  13 #include "LocalStorageManager.h"
  14
  15 #include "nsComponentManagerUtils.h"
  16 #include "nsDirectoryServiceUtils.h"
  17 #include "nsAppDirectoryServiceDefs.h"
  18 #include "nsThreadUtils.h"
  19 #include "nsProxyRelease.h"
  20 #include "mozStorageCID.h"
  21 #include "mozStorageHelper.h"
  22 #include "mozIStorageService.h"
  23 #include "mozIStorageBindingParams.h"
  24 #include "mozIStorageValueArray.h"
  25 #include "mozIStorageFunction.h"
  26 #include "mozilla/BasePrincipal.h"
  27 #include "mozilla/ipc/BackgroundParent.h"
  28 #include "nsIObserverService.h"
  29 #include "nsThread.h"
  30 #include "nsThreadManager.h"
  31 #include "nsVariant.h"
  32 #include "mozilla/EventQueue.h"
  33 #include "mozilla/IOInterposer.h"
  34 #include "mozilla/OriginAttributes.h"
  35 #include "mozilla/ThreadEventQueue.h"
  36 #include "mozilla/Services.h"
  37 #include "mozilla/Tokenizer.h"
  38 #include "GeckoProfiler.h"
  39
  40 // How long we collect write oprerations
  41 // before they are flushed to the database
  42 // In milliseconds.
  43 #define FLUSHING_INTERVAL_MS 5000
  44
  45 // Write Ahead Log's maximum size is 512KB
  46 #define MAX_WAL_SIZE_BYTES 512 * 1024
  47
  48 // Current version of the database schema
  49 #define CURRENT_SCHEMA_VERSION 2
  50
  51 namespace mozilla::dom {
  52
  53 using namespace StorageUtils;
  54
  55 namespace {  // anon
  56
  57 StorageDBThread* sStorageThread[kPrivateBrowsingIdCount] = {nullptr, nullptr};
  58
  59 // False until we shut the storage thread down.
  60 bool sStorageThreadDown[kPrivateBrowsingIdCount] = {false, false};
  61
  62 }  // namespace
  63
  64 // XXX Fix me!
  65 #if 0
  66 StorageDBBridge::StorageDBBridge()
  67 {
  68 }
  69 #endif
  70
  71 class StorageDBThread::InitHelper final : public Runnable {
  72   nsCOMPtr<nsIEventTarget> mOwningThread;
  73   mozilla::Mutex mMutex MOZ_UNANNOTATED;
  74   mozilla::CondVar mCondVar;
  75   nsString mProfilePath;
  76   nsresult mMainThreadResultCode;
  77   bool mWaiting;
  78
  79  public:
  80   InitHelper()
  81       : Runnable("dom::StorageDBThread::InitHelper"),
  82         mOwningThread(GetCurrentEventTarget()),
  83         mMutex("InitHelper::mMutex"),
  84         mCondVar(mMutex, "InitHelper::mCondVar"),
  85         mMainThreadResultCode(NS_OK),
  86         mWaiting(true) {}
  87
  88   // Because of the `sync Preload` IPC, we need to be able to synchronously
  89   // initialize, which includes consulting and initializing
  90   // some main-thread-only APIs. Bug 1386441 discusses improving this situation.
  91   nsresult SyncDispatchAndReturnProfilePath(nsAString& aProfilePath);
  92
  93  private:
  94   ~InitHelper() override = default;
  95
  96   nsresult RunOnMainThread();
  97
  98   NS_DECL_NSIRUNNABLE
  99 };
 100
 101 class StorageDBThread::NoteBackgroundThreadRunnable final : public Runnable {
 102   // Expected to be only 0 or 1.
 103   const uint32_t mPrivateBrowsingId;
 104   nsCOMPtr<nsIEventTarget> mOwningThread;
 105
 106  public:
 107   explicit NoteBackgroundThreadRunnable(const uint32_t aPrivateBrowsingId)
 108       : Runnable("dom::StorageDBThread::NoteBackgroundThreadRunnable"),
 109         mPrivateBrowsingId(aPrivateBrowsingId),
 110         mOwningThread(GetCurrentEventTarget()) {
 111     MOZ_RELEASE_ASSERT(aPrivateBrowsingId < kPrivateBrowsingIdCount);
 112   }
 113
 114  private:
 115   ~NoteBackgroundThreadRunnable() override = default;
 116
 117   NS_DECL_NSIRUNNABLE
 118 };
 119
 120 StorageDBThread::StorageDBThread(const uint32_t aPrivateBrowsingId)
 121     : mThread(nullptr),
 122       mThreadObserver(new ThreadObserver()),
 123       mStopIOThread(false),
 124       mWALModeEnabled(false),
 125       mDBReady(false),
 126       mStatus(NS_OK),
 127       mWorkerStatements(mWorkerConnection),
 128       mReaderStatements(mReaderConnection),
 129       mFlushImmediately(false),
 130       mPrivateBrowsingId(aPrivateBrowsingId),
 131       mPriorityCounter(0) {
 132   MOZ_RELEASE_ASSERT(aPrivateBrowsingId < kPrivateBrowsingIdCount);
 133 }
 134
 135 // static
 136 StorageDBThread* StorageDBThread::Get(const uint32_t aPrivateBrowsingId) {
 137   ::mozilla::ipc::AssertIsOnBackgroundThread();
 138   MOZ_RELEASE_ASSERT(aPrivateBrowsingId < kPrivateBrowsingIdCount);
 139
 140   return sStorageThread[aPrivateBrowsingId];
 141 }
 142
 143 // static
 144 StorageDBThread* StorageDBThread::GetOrCreate(
 145     const nsString& aProfilePath, const uint32_t aPrivateBrowsingId) {
 146   ::mozilla::ipc::AssertIsOnBackgroundThread();
 147   MOZ_RELEASE_ASSERT(aPrivateBrowsingId < kPrivateBrowsingIdCount);
 148
 149   StorageDBThread*& storageThread = sStorageThread[aPrivateBrowsingId];
 150   if (storageThread || sStorageThreadDown[aPrivateBrowsingId]) {
 151     // When sStorageThreadDown is at true, sStorageThread is null.
 152     // Checking sStorageThreadDown flag here prevents reinitialization of
 153     // the storage thread after shutdown.
 154     return storageThread;
 155   }
 156
 157   auto newStorageThread = MakeUnique<StorageDBThread>(aPrivateBrowsingId);
 158
 159   nsresult rv = newStorageThread->Init(aProfilePath);
 160   if (NS_WARN_IF(NS_FAILED(rv))) {
 161     return nullptr;
 162   }
 163
 164   storageThread = newStorageThread.release();
 165
 166   return storageThread;
 167 }
 168
 169 // static
 170 nsresult StorageDBThread::GetProfilePath(nsString& aProfilePath) {
 171   MOZ_ASSERT(XRE_IsParentProcess());
 172   MOZ_ASSERT(NS_IsMainThread());
 173
 174   // Need to determine location on the main thread, since
 175   // NS_GetSpecialDirectory accesses the atom table that can
 176   // only be accessed on the main thread.
 177   nsCOMPtr<nsIFile> profileDir;
 178   nsresult rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR,
 179                                        getter_AddRefs(profileDir));
 180   if (NS_WARN_IF(NS_FAILED(rv))) {
 181     return rv;
 182   }
 183
 184   rv = profileDir->GetPath(aProfilePath);
 185   if (NS_WARN_IF(NS_FAILED(rv))) {
 186     return rv;
 187   }
 188
 189   // This service has to be started on the main thread currently.
 190   nsCOMPtr<mozIStorageService> ss =
 191       do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID, &rv);
 192   if (NS_WARN_IF(NS_FAILED(rv))) {
 193     return rv;
 194   }
 195
 196   return NS_OK;
 197 }
 198
 199 nsresult StorageDBThread::Init(const nsString& aProfilePath) {
 200   ::mozilla::ipc::AssertIsOnBackgroundThread();
 201
 202   if (mPrivateBrowsingId == 0) {
 203     nsresult rv;
 204
 205     nsString profilePath;
 206     if (aProfilePath.IsEmpty()) {
 207       RefPtr<InitHelper> helper = new InitHelper();
 208
 209       rv = helper->SyncDispatchAndReturnProfilePath(profilePath);
 210       if (NS_WARN_IF(NS_FAILED(rv))) {
 211         return rv;
 212       }
 213     } else {
 214       profilePath = aProfilePath;
 215     }
 216
 217     mDatabaseFile = do_CreateInstance(NS_LOCAL_FILE_CONTRACTID, &rv);
 218     if (NS_WARN_IF(NS_FAILED(rv))) {
 219       return rv;
 220     }
 221
 222     rv = mDatabaseFile->InitWithPath(profilePath);
 223     if (NS_WARN_IF(NS_FAILED(rv))) {
 224       return rv;
 225     }
 226
 227     rv = mDatabaseFile->Append(u"webappsstore.sqlite"_ns);
 228     NS_ENSURE_SUCCESS(rv, rv);
 229   }
 230
 231   // Need to keep the lock to avoid setting mThread later then
 232   // the thread body executes.
 233   MonitorAutoLock monitor(mThreadObserver->GetMonitor());
 234
 235   mThread = PR_CreateThread(PR_USER_THREAD, &StorageDBThread::ThreadFunc, this,
 236                             PR_PRIORITY_LOW, PR_GLOBAL_THREAD,
 237                             PR_JOINABLE_THREAD, 262144);
 238   if (!mThread) {
 239     return NS_ERROR_OUT_OF_MEMORY;
 240   }
 241
 242   RefPtr<NoteBackgroundThreadRunnable> runnable =
 243       new NoteBackgroundThreadRunnable(mPrivateBrowsingId);
 244   MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(runnable));
 245
 246   return NS_OK;
 247 }
 248
 249 nsresult StorageDBThread::Shutdown() {
 250   ::mozilla::ipc::AssertIsOnBackgroundThread();
 251
 252   if (!mThread) {
 253     return NS_ERROR_NOT_INITIALIZED;
 254   }
 255
 256   Telemetry::AutoTimer<Telemetry::LOCALDOMSTORAGE_SHUTDOWN_DATABASE_MS> timer;
 257
 258   {
 259     MonitorAutoLock monitor(mThreadObserver->GetMonitor());
 260
 261     // After we stop, no other operations can be accepted
 262     mFlushImmediately = true;
 263     mStopIOThread = true;
 264     monitor.Notify();
 265   }
 266
 267   PR_JoinThread(mThread);
 268   mThread = nullptr;
 269
 270   return mStatus;
 271 }
 272
 273 void StorageDBThread::SyncPreload(LocalStorageCacheBridge* aCache,
 274                                   bool aForceSync) {
 275   AUTO_PROFILER_LABEL("StorageDBThread::SyncPreload", OTHER);
 276   if (!aForceSync && aCache->LoadedCount()) {
 277     // Preload already started for this cache, just wait for it to finish.
 278     // LoadWait will exit after LoadDone on the cache has been called.
 279     SetHigherPriority();
 280     aCache->LoadWait();
 281     SetDefaultPriority();
 282     return;
 283   }
 284
 285   // Bypass sync load when an update is pending in the queue to write, we would
 286   // get incosistent data in the cache.  Also don't allow sync main-thread
 287   // preload when DB open and init is still pending on the background thread.
 288   if (mDBReady && mWALModeEnabled) {
 289     bool pendingTasks;
 290     {
 291       MonitorAutoLock monitor(mThreadObserver->GetMonitor());
 292       pendingTasks = mPendingTasks.IsOriginUpdatePending(
 293                          aCache->OriginSuffix(), aCache->OriginNoSuffix()) ||
 294                      mPendingTasks.IsOriginClearPending(
 295                          aCache->OriginSuffix(), aCache->OriginNoSuffix());
 296     }
 297
 298     if (!pendingTasks) {
 299       // WAL is enabled, thus do the load synchronously on the main thread.
 300       DBOperation preload(DBOperation::opPreload, aCache);
 301       preload.PerformAndFinalize(this);
 302       return;
 303     }
 304   }
 305
 306   // Need to go asynchronously since WAL is not allowed or scheduled updates
 307   // need to be flushed first.
 308   // Schedule preload for this cache as the first operation.
 309   nsresult rv =
 310       InsertDBOp(MakeUnique<DBOperation>(DBOperation::opPreloadUrgent, aCache));
 311
 312   // LoadWait exits after LoadDone of the cache has been called.
 313   if (NS_SUCCEEDED(rv)) {
 314     aCache->LoadWait();
 315   }
 316 }
 317
 318 void StorageDBThread::AsyncFlush() {
 319   MonitorAutoLock monitor(mThreadObserver->GetMonitor());
 320   mFlushImmediately = true;
 321   monitor.Notify();
 322 }
 323
 324 bool StorageDBThread::ShouldPreloadOrigin(const nsACString& aOrigin) {
 325   MonitorAutoLock monitor(mThreadObserver->GetMonitor());
 326   return mOriginsHavingData.Contains(aOrigin);
 327 }
 328
 329 void StorageDBThread::GetOriginsHavingData(nsTArray<nsCString>* aOrigins) {
 330   MonitorAutoLock monitor(mThreadObserver->GetMonitor());
 331   AppendToArray(*aOrigins, mOriginsHavingData);
 332 }
 333
 334 nsresult StorageDBThread::InsertDBOp(
 335     UniquePtr<StorageDBThread::DBOperation> aOperation) {
 336   MonitorAutoLock monitor(mThreadObserver->GetMonitor());
 337
 338   if (NS_FAILED(mStatus)) {
 339     MonitorAutoUnlock unlock(mThreadObserver->GetMonitor());
 340     aOperation->Finalize(mStatus);
 341     return mStatus;
 342   }
 343
 344   if (mStopIOThread) {
 345     // Thread use after shutdown demanded.
 346     MOZ_ASSERT(false);
 347     return NS_ERROR_NOT_INITIALIZED;
 348   }
 349
 350   switch (aOperation->Type()) {
 351     case DBOperation::opPreload:
 352     case DBOperation::opPreloadUrgent:
 353       if (mPendingTasks.IsOriginUpdatePending(aOperation->OriginSuffix(),
 354                                               aOperation->OriginNoSuffix())) {
 355         // If there is a pending update operation for the scope first do the
 356         // flush before we preload the cache.  This may happen in an extremely
 357         // rare case when a child process throws away its cache before flush on
 358         // the parent has finished.  If we would preloaded the cache as a
 359         // priority operation before the pending flush, we would have got an
 360         // inconsistent cache content.
 361         mFlushImmediately = true;
 362       } else if (mPendingTasks.IsOriginClearPending(
 363                      aOperation->OriginSuffix(),
 364                      aOperation->OriginNoSuffix())) {
 365         // The scope is scheduled to be cleared, so just quickly load as empty.
 366         // We need to do this to prevent load of the DB data before the scope
 367         // has actually been cleared from the database.  Preloads are processed
 368         // immediately before update and clear operations on the database that
 369         // are flushed periodically in batches.
 370         MonitorAutoUnlock unlock(mThreadObserver->GetMonitor());
 371         aOperation->Finalize(NS_OK);
 372         return NS_OK;
 373       }
 374       [[fallthrough]];
 375
 376     case DBOperation::opGetUsage:
 377       if (aOperation->Type() == DBOperation::opPreloadUrgent) {
 378         SetHigherPriority();  // Dropped back after urgent preload execution
 379         mPreloads.InsertElementAt(0, aOperation.release());
 380       } else {
 381         mPreloads.AppendElement(aOperation.release());
 382       }
 383
 384       // Immediately start executing this.
 385       monitor.Notify();
 386       break;
 387
 388     default:
 389       // Update operations are first collected, coalesced and then flushed
 390       // after a short time.
 391       mPendingTasks.Add(std::move(aOperation));
 392
 393       ScheduleFlush();
 394       break;
 395   }
 396
 397   return NS_OK;
 398 }
 399
 400 void StorageDBThread::SetHigherPriority() {
 401   ++mPriorityCounter;
 402   PR_SetThreadPriority(mThread, PR_PRIORITY_URGENT);
 403 }
 404
 405 void StorageDBThread::SetDefaultPriority() {
 406   if (--mPriorityCounter <= 0) {
 407     PR_SetThreadPriority(mThread, PR_PRIORITY_LOW);
 408   }
 409 }
 410
 411 void StorageDBThread::ThreadFunc(void* aArg) {
 412   {
 413     auto queue = MakeRefPtr<ThreadEventQueue>(MakeUnique<EventQueue>());
 414     Unused << nsThreadManager::get().CreateCurrentThread(
 415         queue, nsThread::NOT_MAIN_THREAD);
 416   }
 417
 418   AUTO_PROFILER_REGISTER_THREAD("localStorage DB");
 419   NS_SetCurrentThreadName("localStorage DB");
 420   mozilla::IOInterposer::RegisterCurrentThread();
 421
 422   StorageDBThread* thread = static_cast<StorageDBThread*>(aArg);
 423   thread->ThreadFunc();
 424   mozilla::IOInterposer::UnregisterCurrentThread();
 425 }
 426
 427 void StorageDBThread::ThreadFunc() {
 428   nsresult rv = InitDatabase();
 429
 430   MonitorAutoLock lockMonitor(mThreadObserver->GetMonitor());
 431
 432   if (NS_FAILED(rv)) {
 433     mStatus = rv;
 434     mStopIOThread = true;
 435     return;
 436   }
 437
 438   // Create an nsIThread for the current PRThread, so we can observe runnables
 439   // dispatched to it.
 440   nsCOMPtr<nsIThread> thread = NS_GetCurrentThread();
 441   nsCOMPtr<nsIThreadInternal> threadInternal = do_QueryInterface(thread);
 442   MOZ_ASSERT(threadInternal);  // Should always succeed.
 443   threadInternal->SetObserver(mThreadObserver);
 444
 445   while (MOZ_LIKELY(!mStopIOThread || mPreloads.Length() ||
 446                     mPendingTasks.HasTasks() ||
 447                     mThreadObserver->HasPendingEvents())) {
 448     // Process xpcom events first.
 449     while (MOZ_UNLIKELY(mThreadObserver->HasPendingEvents())) {
 450       mThreadObserver->ClearPendingEvents();
 451       MonitorAutoUnlock unlock(mThreadObserver->GetMonitor());
 452       bool processedEvent;
 453       do {
 454         rv = thread->ProcessNextEvent(false, &processedEvent);
 455       } while (NS_SUCCEEDED(rv) && processedEvent);
 456     }
 457
 458     TimeDuration timeUntilFlush = TimeUntilFlush();
 459     if (MOZ_UNLIKELY(timeUntilFlush.IsZero())) {
 460       // Flush time is up or flush has been forced, do it now.
 461       UnscheduleFlush();
 462       if (mPendingTasks.Prepare()) {
 463         {
 464           MonitorAutoUnlock unlockMonitor(mThreadObserver->GetMonitor());
 465           rv = mPendingTasks.Execute(this);
 466         }
 467
 468         if (!mPendingTasks.Finalize(rv)) {
 469           mStatus = rv;
 470           NS_WARNING("localStorage DB access broken");
 471         }
 472       }
 473       NotifyFlushCompletion();
 474     } else if (MOZ_LIKELY(mPreloads.Length())) {
 475       UniquePtr<DBOperation> op(mPreloads[0]);
 476       mPreloads.RemoveElementAt(0);
 477       {
 478         MonitorAutoUnlock unlockMonitor(mThreadObserver->GetMonitor());
 479         op->PerformAndFinalize(this);
 480       }
 481
 482       if (op->Type() == DBOperation::opPreloadUrgent) {
 483         SetDefaultPriority();  // urgent preload unscheduled
 484       }
 485     } else if (MOZ_UNLIKELY(!mStopIOThread)) {
 486       AUTO_PROFILER_LABEL("StorageDBThread::ThreadFunc::Wait", IDLE);
 487       lockMonitor.Wait(timeUntilFlush);
 488     }
 489   }  // thread loop
 490
 491   mStatus = ShutdownDatabase();
 492
 493   if (threadInternal) {
 494     threadInternal->SetObserver(nullptr);
 495   }
 496 }
 497
 498 NS_IMPL_ISUPPORTS(StorageDBThread::ThreadObserver, nsIThreadObserver)
 499
 500 NS_IMETHODIMP
 501 StorageDBThread::ThreadObserver::OnDispatchedEvent() {
 502   MonitorAutoLock lock(mMonitor);
 503   mHasPendingEvents = true;
 504   lock.Notify();
 505   return NS_OK;
 506 }
 507
 508 NS_IMETHODIMP
 509 StorageDBThread::ThreadObserver::OnProcessNextEvent(nsIThreadInternal* aThread,
 510                                                     bool mayWait) {
 511   return NS_OK;
 512 }
 513
 514 NS_IMETHODIMP
 515 StorageDBThread::ThreadObserver::AfterProcessNextEvent(
 516     nsIThreadInternal* aThread, bool eventWasProcessed) {
 517   return NS_OK;
 518 }
 519
 520 nsresult StorageDBThread::OpenDatabaseConnection() {
 521   nsresult rv;
 522
 523   MOZ_ASSERT(!NS_IsMainThread());
 524
 525   nsCOMPtr<mozIStorageService> service =
 526       do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID, &rv);
 527   NS_ENSURE_SUCCESS(rv, rv);
 528
 529   if (mPrivateBrowsingId == 0) {
 530     MOZ_ASSERT(mDatabaseFile);
 531
 532     rv = service->OpenUnsharedDatabase(mDatabaseFile,
 533                                        mozIStorageService::CONNECTION_DEFAULT,
 534                                        getter_AddRefs(mWorkerConnection));
 535     if (rv == NS_ERROR_FILE_CORRUPTED) {
 536       // delete the db and try opening again
 537       rv = mDatabaseFile->Remove(false);
 538       NS_ENSURE_SUCCESS(rv, rv);
 539       rv = service->OpenUnsharedDatabase(mDatabaseFile,
 540                                          mozIStorageService::CONNECTION_DEFAULT,
 541                                          getter_AddRefs(mWorkerConnection));
 542     }
 543   } else {
 544     MOZ_ASSERT(mPrivateBrowsingId == 1);
 545
 546     rv = service->OpenSpecialDatabase(kMozStorageMemoryStorageKey,
 547                                       "lsprivatedb"_ns,
 548                                       mozIStorageService::CONNECTION_DEFAULT,
 549                                       getter_AddRefs(mWorkerConnection));
 550   }
 551   NS_ENSURE_SUCCESS(rv, rv);
 552
 553   return NS_OK;
 554 }
 555
 556 nsresult StorageDBThread::OpenAndUpdateDatabase() {
 557   nsresult rv;
 558
 559   // Here we are on the worker thread. This opens the worker connection.
 560   MOZ_ASSERT(!NS_IsMainThread());
 561
 562   rv = OpenDatabaseConnection();
 563   NS_ENSURE_SUCCESS(rv, rv);
 564
 565   // SQLite doesn't support WAL journals for in-memory databases.
 566   if (mPrivateBrowsingId == 0) {
 567     rv = TryJournalMode();
 568     NS_ENSURE_SUCCESS(rv, rv);
 569   }
 570
 571   return NS_OK;
 572 }
 573
 574 nsresult StorageDBThread::InitDatabase() {
 575   nsresult rv;
 576
 577   // Here we are on the worker thread. This opens the worker connection.
 578   MOZ_ASSERT(!NS_IsMainThread());
 579
 580   rv = OpenAndUpdateDatabase();
 581   NS_ENSURE_SUCCESS(rv, rv);
 582
 583   rv = StorageDBUpdater::Update(mWorkerConnection);
 584   if (NS_FAILED(rv)) {
 585     if (mPrivateBrowsingId == 0) {
 586       // Update has failed, rather throw the database away and try
 587       // opening and setting it up again.
 588       rv = mWorkerConnection->Close();
 589       mWorkerConnection = nullptr;
 590       NS_ENSURE_SUCCESS(rv, rv);
 591
 592       rv = mDatabaseFile->Remove(false);
 593       NS_ENSURE_SUCCESS(rv, rv);
 594
 595       rv = OpenAndUpdateDatabase();
 596     }
 597     NS_ENSURE_SUCCESS(rv, rv);
 598   }
 599
 600   // Create a read-only clone
 601   (void)mWorkerConnection->Clone(true, getter_AddRefs(mReaderConnection));
 602   NS_ENSURE_TRUE(mReaderConnection, NS_ERROR_FAILURE);
 603
 604   // Database open and all initiation operation are done.  Switching this flag
 605   // to true allow main thread to read directly from the database.  If we would
 606   // allow this sooner, we would have opened a window where main thread read
 607   // might operate on a totally broken and incosistent database.
 608   mDBReady = true;
 609
 610   // List scopes having any stored data
 611   nsCOMPtr<mozIStorageStatement> stmt;
 612   // Note: result of this select must match StorageManager::CreateOrigin()
 613   rv = mWorkerConnection->CreateStatement(
 614       nsLiteralCString("SELECT DISTINCT originAttributes || ':' || originKey "
 615                        "FROM webappsstore2"),
 616       getter_AddRefs(stmt));
 617   NS_ENSURE_SUCCESS(rv, rv);
 618   mozStorageStatementScoper scope(stmt);
 619
 620   bool exists;
 621   while (NS_SUCCEEDED(rv = stmt->ExecuteStep(&exists)) && exists) {
 622     nsAutoCString foundOrigin;
 623     rv = stmt->GetUTF8String(0, foundOrigin);
 624     NS_ENSURE_SUCCESS(rv, rv);
 625
 626     MonitorAutoLock monitor(mThreadObserver->GetMonitor());
 627     mOriginsHavingData.Insert(foundOrigin);
 628   }
 629
 630   return NS_OK;
 631 }
 632
 633 nsresult StorageDBThread::SetJournalMode(bool aIsWal) {
 634   nsresult rv;
 635
 636   nsAutoCString stmtString(MOZ_STORAGE_UNIQUIFY_QUERY_STR
 637                            "PRAGMA journal_mode = ");
 638   if (aIsWal) {
 639     stmtString.AppendLiteral("wal");
 640   } else {
 641     stmtString.AppendLiteral("truncate");
 642   }
 643
 644   nsCOMPtr<mozIStorageStatement> stmt;
 645   rv = mWorkerConnection->CreateStatement(stmtString, getter_AddRefs(stmt));
 646   NS_ENSURE_SUCCESS(rv, rv);
 647   mozStorageStatementScoper scope(stmt);
 648
 649   bool hasResult = false;
 650   rv = stmt->ExecuteStep(&hasResult);
 651   NS_ENSURE_SUCCESS(rv, rv);
 652   if (!hasResult) {
 653     return NS_ERROR_FAILURE;
 654   }
 655
 656   nsAutoCString journalMode;
 657   rv = stmt->GetUTF8String(0, journalMode);
 658   NS_ENSURE_SUCCESS(rv, rv);
 659   if ((aIsWal && !journalMode.EqualsLiteral("wal")) ||
 660       (!aIsWal && !journalMode.EqualsLiteral("truncate"))) {
 661     return NS_ERROR_FAILURE;
 662   }
 663
 664   return NS_OK;
 665 }
 666
 667 nsresult StorageDBThread::TryJournalMode() {
 668   nsresult rv;
 669
 670   rv = SetJournalMode(true);
 671   if (NS_FAILED(rv)) {
 672     mWALModeEnabled = false;
 673
 674     rv = SetJournalMode(false);
 675     NS_ENSURE_SUCCESS(rv, rv);
 676   } else {
 677     mWALModeEnabled = true;
 678
 679     rv = ConfigureWALBehavior();
 680     NS_ENSURE_SUCCESS(rv, rv);
 681   }
 682
 683   return NS_OK;
 684 }
 685
 686 nsresult StorageDBThread::ConfigureWALBehavior() {
 687   // Get the DB's page size
 688   nsCOMPtr<mozIStorageStatement> stmt;
 689   nsresult rv = mWorkerConnection->CreateStatement(
 690       nsLiteralCString(MOZ_STORAGE_UNIQUIFY_QUERY_STR "PRAGMA page_size"),
 691       getter_AddRefs(stmt));
 692   NS_ENSURE_SUCCESS(rv, rv);
 693
 694   bool hasResult = false;
 695   rv = stmt->ExecuteStep(&hasResult);
 696   NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && hasResult, NS_ERROR_FAILURE);
 697
 698   int32_t pageSize = 0;
 699   rv = stmt->GetInt32(0, &pageSize);
 700   NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && pageSize > 0, NS_ERROR_UNEXPECTED);
 701
 702   // Set the threshold for auto-checkpointing the WAL.
 703   // We don't want giant logs slowing down reads & shutdown.
 704   int32_t thresholdInPages =
 705       static_cast<int32_t>(MAX_WAL_SIZE_BYTES / pageSize);
 706   nsAutoCString thresholdPragma("PRAGMA wal_autocheckpoint = ");
 707   thresholdPragma.AppendInt(thresholdInPages);
 708   rv = mWorkerConnection->ExecuteSimpleSQL(thresholdPragma);
 709   NS_ENSURE_SUCCESS(rv, rv);
 710
 711   // Set the maximum WAL log size to reduce footprint on mobile (large empty
 712   // WAL files will be truncated)
 713   nsAutoCString journalSizePragma("PRAGMA journal_size_limit = ");
 714   // bug 600307: mak recommends setting this to 3 times the auto-checkpoint
 715   // threshold
 716   journalSizePragma.AppendInt(MAX_WAL_SIZE_BYTES * 3);
 717   rv = mWorkerConnection->ExecuteSimpleSQL(journalSizePragma);
 718   NS_ENSURE_SUCCESS(rv, rv);
 719
 720   return NS_OK;
 721 }
 722
 723 nsresult StorageDBThread::ShutdownDatabase() {
 724   // Has to be called on the worker thread.
 725   MOZ_ASSERT(!NS_IsMainThread());
 726
 727   nsresult rv = mStatus;
 728
 729   mDBReady = false;
 730
 731   // Finalize the cached statements.
 732   mReaderStatements.FinalizeStatements();
 733   mWorkerStatements.FinalizeStatements();
 734
 735   if (mReaderConnection) {
 736     // No need to sync access to mReaderConnection since the main thread
 737     // is right now joining this thread, unable to execute any events.
 738     mReaderConnection->Close();
 739     mReaderConnection = nullptr;
 740   }
 741
 742   if (mWorkerConnection) {
 743     rv = mWorkerConnection->Close();
 744     mWorkerConnection = nullptr;
 745   }
 746
 747   return rv;
 748 }
 749
 750 void StorageDBThread::ScheduleFlush() {
 751   if (mDirtyEpoch) {
 752     return;  // Already scheduled
 753   }
 754
 755   // Must be non-zero to indicate we are scheduled
 756   mDirtyEpoch = TimeStamp::Now();
 757
 758   // Wake the monitor from indefinite sleep...
 759   (mThreadObserver->GetMonitor()).Notify();
 760 }
 761
 762 void StorageDBThread::UnscheduleFlush() {
 763   // We are just about to do the flush, drop flags
 764   mFlushImmediately = false;
 765   mDirtyEpoch = TimeStamp();
 766 }
 767
 768 TimeDuration StorageDBThread::TimeUntilFlush() {
 769   if (mFlushImmediately) {
 770     return 0;  // Do it now regardless the timeout.
 771   }
 772
 773   if (!mDirtyEpoch) {
 774     return TimeDuration::Forever();  // No pending task...
 775   }
 776
 777   TimeStamp now = TimeStamp::Now();
 778   TimeDuration age = now - mDirtyEpoch;
 779   static const TimeDuration kMaxAge =
 780       TimeDuration::FromMilliseconds(FLUSHING_INTERVAL_MS);
 781   if (age > kMaxAge) {
 782     return 0;  // It is time.
 783   }
 784
 785   return kMaxAge - age;  // Time left. This is used to sleep the monitor.
 786 }
 787
 788 void StorageDBThread::NotifyFlushCompletion() {
 789 #ifdef DOM_STORAGE_TESTS
 790   if (!NS_IsMainThread()) {
 791     RefPtr<nsRunnableMethod<StorageDBThread, void, false>> event =
 792         NewNonOwningRunnableMethod(
 793             "dom::StorageDBThread::NotifyFlushCompletion", this,
 794             &StorageDBThread::NotifyFlushCompletion);
 795     NS_DispatchToMainThread(event);
 796     return;
 797   }
 798
 799   nsCOMPtr<nsIObserverService> obs = mozilla::services::GetObserverService();
 800   if (obs) {
 801     obs->NotifyObservers(nullptr, "domstorage-test-flushed", nullptr);
 802   }
 803 #endif
 804 }
 805
 806 // Helper SQL function classes
 807
 808 namespace {
 809
 810 class OriginAttrsPatternMatchSQLFunction final : public mozIStorageFunction {
 811   NS_DECL_ISUPPORTS
 812   NS_DECL_MOZISTORAGEFUNCTION
 813
 814   explicit OriginAttrsPatternMatchSQLFunction(
 815       OriginAttributesPattern const& aPattern)
 816       : mPattern(aPattern) {}
 817
 818  private:
 819   OriginAttrsPatternMatchSQLFunction() = delete;
 820   ~OriginAttrsPatternMatchSQLFunction() = default;
 821
 822   OriginAttributesPattern mPattern;
 823 };
 824
 825 NS_IMPL_ISUPPORTS(OriginAttrsPatternMatchSQLFunction, mozIStorageFunction)
 826
 827 NS_IMETHODIMP
 828 OriginAttrsPatternMatchSQLFunction::OnFunctionCall(
 829     mozIStorageValueArray* aFunctionArguments, nsIVariant** aResult) {
 830   nsresult rv;
 831
 832   nsAutoCString suffix;
 833   rv = aFunctionArguments->GetUTF8String(0, suffix);
 834   NS_ENSURE_SUCCESS(rv, rv);
 835
 836   OriginAttributes oa;
 837   bool success = oa.PopulateFromSuffix(suffix);
 838   NS_ENSURE_TRUE(success, NS_ERROR_FAILURE);
 839   bool result = mPattern.Matches(oa);
 840
 841   RefPtr<nsVariant> outVar(new nsVariant());
 842   rv = outVar->SetAsBool(result);
 843   NS_ENSURE_SUCCESS(rv, rv);
 844
 845   outVar.forget(aResult);
 846   return NS_OK;
 847 }
 848
 849 }  // namespace
 850
 851 // StorageDBThread::DBOperation
 852
 853 StorageDBThread::DBOperation::DBOperation(const OperationType aType,
 854                                           LocalStorageCacheBridge* aCache,
 855                                           const nsAString& aKey,
 856                                           const nsAString& aValue)
 857     : mType(aType), mCache(aCache), mKey(aKey), mValue(aValue) {
 858   MOZ_ASSERT(mType == opPreload || mType == opPreloadUrgent ||
 859              mType == opAddItem || mType == opUpdateItem ||
 860              mType == opRemoveItem || mType == opClear || mType == opClearAll);
 861   MOZ_COUNT_CTOR(StorageDBThread::DBOperation);
 862 }
 863
 864 StorageDBThread::DBOperation::DBOperation(const OperationType aType,
 865                                           StorageUsageBridge* aUsage)
 866     : mType(aType), mUsage(aUsage) {
 867   MOZ_ASSERT(mType == opGetUsage);
 868   MOZ_COUNT_CTOR(StorageDBThread::DBOperation);
 869 }
 870
 871 StorageDBThread::DBOperation::DBOperation(const OperationType aType,
 872                                           const nsACString& aOriginNoSuffix)
 873     : mType(aType), mCache(nullptr), mOrigin(aOriginNoSuffix) {
 874   MOZ_ASSERT(mType == opClearMatchingOrigin);
 875   MOZ_COUNT_CTOR(StorageDBThread::DBOperation);
 876 }
 877
 878 StorageDBThread::DBOperation::DBOperation(
 879     const OperationType aType, const OriginAttributesPattern& aOriginNoSuffix)
 880     : mType(aType), mCache(nullptr), mOriginPattern(aOriginNoSuffix) {
 881   MOZ_ASSERT(mType == opClearMatchingOriginAttributes);
 882   MOZ_COUNT_CTOR(StorageDBThread::DBOperation);
 883 }
 884
 885 StorageDBThread::DBOperation::~DBOperation() {
 886   MOZ_COUNT_DTOR(StorageDBThread::DBOperation);
 887 }
 888
 889 const nsCString StorageDBThread::DBOperation::OriginNoSuffix() const {
 890   if (mCache) {
 891     return mCache->OriginNoSuffix();
 892   }
 893
 894   return ""_ns;
 895 }
 896
 897 const nsCString StorageDBThread::DBOperation::OriginSuffix() const {
 898   if (mCache) {
 899     return mCache->OriginSuffix();
 900   }
 901
 902   return ""_ns;
 903 }
 904
 905 const nsCString StorageDBThread::DBOperation::Origin() const {
 906   if (mCache) {
 907     return mCache->Origin();
 908   }
 909
 910   return mOrigin;
 911 }
 912
 913 const nsCString StorageDBThread::DBOperation::Target() const {
 914   switch (mType) {
 915     case opAddItem:
 916     case opUpdateItem:
 917     case opRemoveItem:
 918       return Origin() + "|"_ns + NS_ConvertUTF16toUTF8(mKey);
 919
 920     default:
 921       return Origin();
 922   }
 923 }
 924
 925 void StorageDBThread::DBOperation::PerformAndFinalize(
 926     StorageDBThread* aThread) {
 927   Finalize(Perform(aThread));
 928 }
 929
 930 nsresult StorageDBThread::DBOperation::Perform(StorageDBThread* aThread) {
 931   nsresult rv;
 932
 933   switch (mType) {
 934     case opPreload:
 935     case opPreloadUrgent: {
 936       // Already loaded?
 937       if (mCache->Loaded()) {
 938         break;
 939       }
 940
 941       StatementCache* statements;
 942       if (MOZ_UNLIKELY(::mozilla::ipc::IsOnBackgroundThread())) {
 943         statements = &aThread->mReaderStatements;
 944       } else {
 945         statements = &aThread->mWorkerStatements;
 946       }
 947
 948       // OFFSET is an optimization when we have to do a sync load
 949       // and cache has already loaded some parts asynchronously.
 950       // It skips keys we have already loaded.
 951       nsCOMPtr<mozIStorageStatement> stmt = statements->GetCachedStatement(
 952           "SELECT key, value FROM webappsstore2 "
 953           "WHERE originAttributes = :originAttributes AND originKey = "
 954           ":originKey "
 955           "ORDER BY key LIMIT -1 OFFSET :offset");
 956       NS_ENSURE_STATE(stmt);
 957       mozStorageStatementScoper scope(stmt);
 958
 959       rv = stmt->BindUTF8StringByName("originAttributes"_ns,
 960                                       mCache->OriginSuffix());
 961       NS_ENSURE_SUCCESS(rv, rv);
 962
 963       rv = stmt->BindUTF8StringByName("originKey"_ns, mCache->OriginNoSuffix());
 964       NS_ENSURE_SUCCESS(rv, rv);
 965
 966       rv = stmt->BindInt32ByName("offset"_ns,
 967                                  static_cast<int32_t>(mCache->LoadedCount()));
 968       NS_ENSURE_SUCCESS(rv, rv);
 969
 970       bool exists;
 971       while (NS_SUCCEEDED(rv = stmt->ExecuteStep(&exists)) && exists) {
 972         nsAutoString key;
 973         rv = stmt->GetString(0, key);
 974         NS_ENSURE_SUCCESS(rv, rv);
 975
 976         nsAutoString value;
 977         rv = stmt->GetString(1, value);
 978         NS_ENSURE_SUCCESS(rv, rv);
 979
 980         if (!mCache->LoadItem(key, value)) {
 981           break;
 982         }
 983       }
 984       // The loop condition's call to ExecuteStep() may have terminated because
 985       // !NS_SUCCEEDED(), we need an early return to cover that case.  This also
 986       // covers success cases as well, but that's inductively safe.
 987       NS_ENSURE_SUCCESS(rv, rv);
 988       break;
 989     }
 990
 991     case opGetUsage: {
 992       // Bug 1676410 fixed a regression caused by bug 1165214. However, it
 993       // turns out that 100% correct checking of the eTLD+1 usage is not
 994       // possible to recover easily, see bug 1683299.
 995 #if 0
 996       // This is how it should be done, but due to other problems like lack
 997       // of usage synchronization between content processes, we temporarily
 998       // disabled the matching using "%".
 999
1000       nsCOMPtr<mozIStorageStatement> stmt =
1001           aThread->mWorkerStatements.GetCachedStatement(
1002               "SELECT SUM(LENGTH(key) + LENGTH(value)) FROM webappsstore2 "
1003               "WHERE (originAttributes || ':' || originKey) LIKE :usageOrigin "
1004               "ESCAPE '\\'");
1005       NS_ENSURE_STATE(stmt);
1006
1007       mozStorageStatementScoper scope(stmt);
1008
1009       // The database schema is built around cleverly reversing domain names
1010       // (the "originKey") so that we can efficiently group usage by eTLD+1.
1011       // "foo.example.org" has an eTLD+1 of ".example.org". They reverse to
1012       // "gro.elpmaxe.oof" and "gro.elpmaxe." respectively, noting that the
1013       // reversed eTLD+1 is a prefix of its reversed sub-domain. To this end,
1014       // we can calculate all of the usage for an eTLD+1 by summing up all the
1015       // rows which have the reversed eTLD+1 as a prefix. In SQL we can
1016       // accomplish this using LIKE which provides for case-insensitive
1017       // matching with "_" as a single-character wildcard match and "%" any
1018       // sequence of zero or more characters. So by suffixing the reversed
1019       // eTLD+1 and using "%" we get our case-insensitive (domain names are
1020       // case-insensitive) matching. Note that although legal domain names
1021       // don't include "_" or "%", file origins can include them, so we need
1022       // to escape our OriginScope for correctness.
1023       nsAutoCString originScopeEscaped;
1024       rv = stmt->EscapeUTF8StringForLIKE(mUsage->OriginScope(), '\\',
1025                                          originScopeEscaped);
1026       NS_ENSURE_SUCCESS(rv, rv);
1027
1028       rv = stmt->BindUTF8StringByName("usageOrigin"_ns,
1029                                       originScopeEscaped + "%"_ns);
1030       NS_ENSURE_SUCCESS(rv, rv);
1031 #else
1032       // This is the code before bug 1676410 and bug 1676973. The returned
1033       // usage will be zero in most of the cases, but due to lack of usage
1034       // synchronization between content processes we have to live with this
1035       // semi-broken behaviour because it causes less harm than the matching
1036       // using "%".
1037
1038       nsCOMPtr<mozIStorageStatement> stmt =
1039           aThread->mWorkerStatements.GetCachedStatement(
1040               "SELECT SUM(LENGTH(key) + LENGTH(value)) FROM webappsstore2 "
1041               "WHERE (originAttributes || ':' || originKey) LIKE :usageOrigin");
1042       NS_ENSURE_STATE(stmt);
1043
1044       mozStorageStatementScoper scope(stmt);
1045
1046       rv = stmt->BindUTF8StringByName("usageOrigin"_ns, mUsage->OriginScope());
1047       NS_ENSURE_SUCCESS(rv, rv);
1048 #endif
1049
1050       bool exists;
1051       rv = stmt->ExecuteStep(&exists);
1052       NS_ENSURE_SUCCESS(rv, rv);
1053
1054       int64_t usage = 0;
1055       if (exists) {
1056         rv = stmt->GetInt64(0, &usage);
1057         NS_ENSURE_SUCCESS(rv, rv);
1058       }
1059
1060       mUsage->LoadUsage(usage);
1061       break;
1062     }
1063
1064     case opAddItem:
1065     case opUpdateItem: {
1066       MOZ_ASSERT(!NS_IsMainThread());
1067
1068       nsCOMPtr<mozIStorageStatement> stmt =
1069           aThread->mWorkerStatements.GetCachedStatement(
1070               "INSERT OR REPLACE INTO webappsstore2 (originAttributes, "
1071               "originKey, scope, key, value) "
1072               "VALUES (:originAttributes, :originKey, :scope, :key, :value) ");
1073       NS_ENSURE_STATE(stmt);
1074
1075       mozStorageStatementScoper scope(stmt);
1076
1077       rv = stmt->BindUTF8StringByName("originAttributes"_ns,
1078                                       mCache->OriginSuffix());
1079       NS_ENSURE_SUCCESS(rv, rv);
1080       rv = stmt->BindUTF8StringByName("originKey"_ns, mCache->OriginNoSuffix());
1081       NS_ENSURE_SUCCESS(rv, rv);
1082       // Filling the 'scope' column just for downgrade compatibility reasons
1083       rv = stmt->BindUTF8StringByName(
1084           "scope"_ns,
1085           Scheme0Scope(mCache->OriginSuffix(), mCache->OriginNoSuffix()));
1086       NS_ENSURE_SUCCESS(rv, rv);
1087       rv = stmt->BindStringByName("key"_ns, mKey);
1088       NS_ENSURE_SUCCESS(rv, rv);
1089       rv = stmt->BindStringByName("value"_ns, mValue);
1090       NS_ENSURE_SUCCESS(rv, rv);
1091
1092       rv = stmt->Execute();
1093       NS_ENSURE_SUCCESS(rv, rv);
1094
1095       MonitorAutoLock monitor(aThread->mThreadObserver->GetMonitor());
1096       aThread->mOriginsHavingData.Insert(Origin());
1097       break;
1098     }
1099
1100     case opRemoveItem: {
1101       MOZ_ASSERT(!NS_IsMainThread());
1102
1103       nsCOMPtr<mozIStorageStatement> stmt =
1104           aThread->mWorkerStatements.GetCachedStatement(
1105               "DELETE FROM webappsstore2 "
1106               "WHERE originAttributes = :originAttributes AND originKey = "
1107               ":originKey "
1108               "AND key = :key ");
1109       NS_ENSURE_STATE(stmt);
1110       mozStorageStatementScoper scope(stmt);
1111
1112       rv = stmt->BindUTF8StringByName("originAttributes"_ns,
1113                                       mCache->OriginSuffix());
1114       NS_ENSURE_SUCCESS(rv, rv);
1115       rv = stmt->BindUTF8StringByName("originKey"_ns, mCache->OriginNoSuffix());
1116       NS_ENSURE_SUCCESS(rv, rv);
1117       rv = stmt->BindStringByName("key"_ns, mKey);
1118       NS_ENSURE_SUCCESS(rv, rv);
1119
1120       rv = stmt->Execute();
1121       NS_ENSURE_SUCCESS(rv, rv);
1122
1123       break;
1124     }
1125
1126     case opClear: {
1127       MOZ_ASSERT(!NS_IsMainThread());
1128
1129       nsCOMPtr<mozIStorageStatement> stmt =
1130           aThread->mWorkerStatements.GetCachedStatement(
1131               "DELETE FROM webappsstore2 "
1132               "WHERE originAttributes = :originAttributes AND originKey = "
1133               ":originKey");
1134       NS_ENSURE_STATE(stmt);
1135       mozStorageStatementScoper scope(stmt);
1136
1137       rv = stmt->BindUTF8StringByName("originAttributes"_ns,
1138                                       mCache->OriginSuffix());
1139       NS_ENSURE_SUCCESS(rv, rv);
1140       rv = stmt->BindUTF8StringByName("originKey"_ns, mCache->OriginNoSuffix());
1141       NS_ENSURE_SUCCESS(rv, rv);
1142
1143       rv = stmt->Execute();
1144       NS_ENSURE_SUCCESS(rv, rv);
1145
1146       MonitorAutoLock monitor(aThread->mThreadObserver->GetMonitor());
1147       aThread->mOriginsHavingData.Remove(Origin());
1148       break;
1149     }
1150
1151     case opClearAll: {
1152       MOZ_ASSERT(!NS_IsMainThread());
1153
1154       nsCOMPtr<mozIStorageStatement> stmt =
1155           aThread->mWorkerStatements.GetCachedStatement(
1156               "DELETE FROM webappsstore2");
1157       NS_ENSURE_STATE(stmt);
1158       mozStorageStatementScoper scope(stmt);
1159
1160       rv = stmt->Execute();
1161       NS_ENSURE_SUCCESS(rv, rv);
1162
1163       MonitorAutoLock monitor(aThread->mThreadObserver->GetMonitor());
1164       aThread->mOriginsHavingData.Clear();
1165       break;
1166     }
1167
1168     case opClearMatchingOrigin: {
1169       MOZ_ASSERT(!NS_IsMainThread());
1170
1171       nsCOMPtr<mozIStorageStatement> stmt =
1172           aThread->mWorkerStatements.GetCachedStatement(
1173               "DELETE FROM webappsstore2"
1174               " WHERE originKey GLOB :scope");
1175       NS_ENSURE_STATE(stmt);
1176       mozStorageStatementScoper scope(stmt);
1177
1178       rv = stmt->BindUTF8StringByName("scope"_ns, mOrigin + "*"_ns);
1179       NS_ENSURE_SUCCESS(rv, rv);
1180
1181       rv = stmt->Execute();
1182       NS_ENSURE_SUCCESS(rv, rv);
1183
1184       // No need to selectively clear mOriginsHavingData here.  That hashtable
1185       // only prevents preload for scopes with no data.  Leaving a false record
1186       // in it has a negligible effect on performance.
1187       break;
1188     }
1189
1190     case opClearMatchingOriginAttributes: {
1191       MOZ_ASSERT(!NS_IsMainThread());
1192
1193       // Register the ORIGIN_ATTRS_PATTERN_MATCH function, initialized with the
1194       // pattern
1195       nsCOMPtr<mozIStorageFunction> patternMatchFunction(
1196           new OriginAttrsPatternMatchSQLFunction(mOriginPattern));
1197
1198       rv = aThread->mWorkerConnection->CreateFunction(
1199           "ORIGIN_ATTRS_PATTERN_MATCH"_ns, 1, patternMatchFunction);
1200       NS_ENSURE_SUCCESS(rv, rv);
1201
1202       nsCOMPtr<mozIStorageStatement> stmt =
1203           aThread->mWorkerStatements.GetCachedStatement(
1204               "DELETE FROM webappsstore2"
1205               " WHERE ORIGIN_ATTRS_PATTERN_MATCH(originAttributes)");
1206
1207       if (stmt) {
1208         mozStorageStatementScoper scope(stmt);
1209         rv = stmt->Execute();
1210       } else {
1211         rv = NS_ERROR_UNEXPECTED;
1212       }
1213
1214       // Always remove the function
1215       aThread->mWorkerConnection->RemoveFunction(
1216           "ORIGIN_ATTRS_PATTERN_MATCH"_ns);
1217
1218       NS_ENSURE_SUCCESS(rv, rv);
1219
1220       // No need to selectively clear mOriginsHavingData here.  That hashtable
1221       // only prevents preload for scopes with no data.  Leaving a false record
1222       // in it has a negligible effect on performance.
1223       break;
1224     }
1225
1226     default:
1227       NS_ERROR("Unknown task type");
1228       break;
1229   }
1230
1231   return NS_OK;
1232 }
1233
1234 void StorageDBThread::DBOperation::Finalize(nsresult aRv) {
1235   switch (mType) {
1236     case opPreloadUrgent:
1237     case opPreload:
1238       if (NS_FAILED(aRv)) {
1239         // When we are here, something failed when loading from the database.
1240         // Notify that the storage is loaded to prevent deadlock of the main
1241         // thread, even though it is actually empty or incomplete.
1242         NS_WARNING("Failed to preload localStorage");
1243       }
1244
1245       mCache->LoadDone(aRv);
1246       break;
1247
1248     case opGetUsage:
1249       if (NS_FAILED(aRv)) {
1250         mUsage->LoadUsage(0);
1251       }
1252
1253       break;
1254
1255     default:
1256       if (NS_FAILED(aRv)) {
1257         NS_WARNING(
1258             "localStorage update/clear operation failed,"
1259             " data may not persist or clean up");
1260       }
1261
1262       break;
1263   }
1264 }
1265
1266 // StorageDBThread::PendingOperations
1267
1268 StorageDBThread::PendingOperations::PendingOperations()
1269     : mFlushFailureCount(0) {}
1270
1271 bool StorageDBThread::PendingOperations::HasTasks() const {
1272   return !!mUpdates.Count() || !!mClears.Count();
1273 }
1274
1275 namespace {
1276
1277 bool OriginPatternMatches(const nsACString& aOriginSuffix,
1278                           const OriginAttributesPattern& aPattern) {
1279   OriginAttributes oa;
1280   DebugOnly<bool> rv = oa.PopulateFromSuffix(aOriginSuffix);
1281   MOZ_ASSERT(rv);
1282   return aPattern.Matches(oa);
1283 }
1284
1285 }  // namespace
1286
1287 bool StorageDBThread::PendingOperations::CheckForCoalesceOpportunity(
1288     DBOperation* aNewOp, DBOperation::OperationType aPendingType,
1289     DBOperation::OperationType aNewType) {
1290   if (aNewOp->Type() != aNewType) {
1291     return false;
1292   }
1293
1294   StorageDBThread::DBOperation* pendingTask;
1295   if (!mUpdates.Get(aNewOp->Target(), &pendingTask)) {
1296     return false;
1297   }
1298
1299   if (pendingTask->Type() != aPendingType) {
1300     return false;
1301   }
1302
1303   return true;
1304 }
1305
1306 void StorageDBThread::PendingOperations::Add(
1307     UniquePtr<StorageDBThread::DBOperation> aOperation) {
1308   // Optimize: when a key to remove has never been written to disk
1309   // just bypass this operation.  A key is new when an operation scheduled
1310   // to write it to the database is of type opAddItem.
1311   if (CheckForCoalesceOpportunity(aOperation.get(), DBOperation::opAddItem,
1312                                   DBOperation::opRemoveItem)) {
1313     mUpdates.Remove(aOperation->Target());
1314     return;
1315   }
1316
1317   // Optimize: when changing a key that is new and has never been
1318   // written to disk, keep type of the operation to store it at opAddItem.
1319   // This allows optimization to just forget adding a new key when
1320   // it is removed from the storage before flush.
1321   if (CheckForCoalesceOpportunity(aOperation.get(), DBOperation::opAddItem,
1322                                   DBOperation::opUpdateItem)) {
1323     aOperation->mType = DBOperation::opAddItem;
1324   }
1325
1326   // Optimize: to prevent lose of remove operation on a key when doing
1327   // remove/set/remove on a previously existing key we have to change
1328   // opAddItem to opUpdateItem on the new operation when there is opRemoveItem
1329   // pending for the key.
1330   if (CheckForCoalesceOpportunity(aOperation.get(), DBOperation::opRemoveItem,
1331                                   DBOperation::opAddItem)) {
1332     aOperation->mType = DBOperation::opUpdateItem;
1333   }
1334
1335   switch (aOperation->Type()) {
1336       // Operations on single keys
1337
1338     case DBOperation::opAddItem:
1339     case DBOperation::opUpdateItem:
1340     case DBOperation::opRemoveItem:
1341       // Override any existing operation for the target (=scope+key).
1342       mUpdates.InsertOrUpdate(aOperation->Target(), std::move(aOperation));
1343       break;
1344
1345       // Clear operations
1346
1347     case DBOperation::opClear:
1348     case DBOperation::opClearMatchingOrigin:
1349     case DBOperation::opClearMatchingOriginAttributes:
1350       // Drop all update (insert/remove) operations for equivavelent or matching
1351       // scope.  We do this as an optimization as well as a must based on the
1352       // logic, if we would not delete the update tasks, changes would have been
1353       // stored to the database after clear operations have been executed.
1354       for (auto iter = mUpdates.Iter(); !iter.Done(); iter.Next()) {
1355         const auto& pendingTask = iter.Data();
1356
1357         if (aOperation->Type() == DBOperation::opClear &&
1358             (pendingTask->OriginNoSuffix() != aOperation->OriginNoSuffix() ||
1359              pendingTask->OriginSuffix() != aOperation->OriginSuffix())) {
1360           continue;
1361         }
1362
1363         if (aOperation->Type() == DBOperation::opClearMatchingOrigin &&
1364             !StringBeginsWith(pendingTask->OriginNoSuffix(),
1365                               aOperation->Origin())) {
1366           continue;
1367         }
1368
1369         if (aOperation->Type() ==
1370                 DBOperation::opClearMatchingOriginAttributes &&
1371             !OriginPatternMatches(pendingTask->OriginSuffix(),
1372                                   aOperation->OriginPattern())) {
1373           continue;
1374         }
1375
1376         iter.Remove();
1377       }
1378
1379       mClears.InsertOrUpdate(aOperation->Target(), std::move(aOperation));
1380       break;
1381
1382     case DBOperation::opClearAll:
1383       // Drop simply everything, this is a super-operation.
1384       mUpdates.Clear();
1385       mClears.Clear();
1386       mClears.InsertOrUpdate(aOperation->Target(), std::move(aOperation));
1387       break;
1388
1389     default:
1390       MOZ_ASSERT(false);
1391       break;
1392   }
1393 }
1394
1395 bool StorageDBThread::PendingOperations::Prepare() {
1396   // Called under the lock
1397
1398   // First collect clear operations and then updates, we can
1399   // do this since whenever a clear operation for a scope is
1400   // scheduled, we drop all updates matching that scope. So,
1401   // all scope-related update operations we have here now were
1402   // scheduled after the clear operations.
1403   for (auto iter = mClears.Iter(); !iter.Done(); iter.Next()) {
1404     mExecList.AppendElement(std::move(iter.Data()));
1405   }
1406   mClears.Clear();
1407
1408   for (auto iter = mUpdates.Iter(); !iter.Done(); iter.Next()) {
1409     mExecList.AppendElement(std::move(iter.Data()));
1410   }
1411   mUpdates.Clear();
1412
1413   return !!mExecList.Length();
1414 }
1415
1416 nsresult StorageDBThread::PendingOperations::Execute(StorageDBThread* aThread) {
1417   // Called outside the lock
1418
1419   mozStorageTransaction transaction(aThread->mWorkerConnection, false);
1420
1421   nsresult rv = transaction.Start();
1422   if (NS_FAILED(rv)) {
1423     return rv;
1424   }
1425
1426   for (uint32_t i = 0; i < mExecList.Length(); ++i) {
1427     const auto& task = mExecList[i];
1428     rv = task->Perform(aThread);
1429     if (NS_FAILED(rv)) {
1430       return rv;
1431     }
1432   }
1433
1434   rv = transaction.Commit();
1435   if (NS_FAILED(rv)) {
1436     return rv;
1437   }
1438
1439   return NS_OK;
1440 }
1441
1442 bool StorageDBThread::PendingOperations::Finalize(nsresult aRv) {
1443   // Called under the lock
1444
1445   // The list is kept on a failure to retry it
1446   if (NS_FAILED(aRv)) {
1447     // XXX Followup: we may try to reopen the database and flush these
1448     // pending tasks, however testing showed that even though I/O is actually
1449     // broken some amount of operations is left in sqlite+system buffers and
1450     // seems like successfully flushed to disk.
1451     // Tested by removing a flash card and disconnecting from network while
1452     // using a network drive on Windows system.
1453     NS_WARNING("Flush operation on localStorage database failed");
1454
1455     ++mFlushFailureCount;
1456
1457     return mFlushFailureCount >= 5;
1458   }
1459
1460   mFlushFailureCount = 0;
1461   mExecList.Clear();
1462   return true;
1463 }
1464
1465 namespace {
1466
1467 bool FindPendingClearForOrigin(
1468     const nsACString& aOriginSuffix, const nsACString& aOriginNoSuffix,
1469     StorageDBThread::DBOperation* aPendingOperation) {
1470   if (aPendingOperation->Type() == StorageDBThread::DBOperation::opClearAll) {
1471     return true;
1472   }
1473
1474   if (aPendingOperation->Type() == StorageDBThread::DBOperation::opClear &&
1475       aOriginNoSuffix == aPendingOperation->OriginNoSuffix() &&
1476       aOriginSuffix == aPendingOperation->OriginSuffix()) {
1477     return true;
1478   }
1479
1480   if (aPendingOperation->Type() ==
1481           StorageDBThread::DBOperation::opClearMatchingOrigin &&
1482       StringBeginsWith(aOriginNoSuffix, aPendingOperation->Origin())) {
1483     return true;
1484   }
1485
1486   if (aPendingOperation->Type() ==
1487           StorageDBThread::DBOperation::opClearMatchingOriginAttributes &&
1488       OriginPatternMatches(aOriginSuffix, aPendingOperation->OriginPattern())) {
1489     return true;
1490   }
1491
1492   return false;
1493 }
1494
1495 }  // namespace
1496
1497 bool StorageDBThread::PendingOperations::IsOriginClearPending(
1498     const nsACString& aOriginSuffix, const nsACString& aOriginNoSuffix) const {
1499   // Called under the lock
1500
1501   for (const auto& clear : mClears.Values()) {
1502     if (FindPendingClearForOrigin(aOriginSuffix, aOriginNoSuffix,
1503                                   clear.get())) {
1504       return true;
1505     }
1506   }
1507
1508   for (uint32_t i = 0; i < mExecList.Length(); ++i) {
1509     if (FindPendingClearForOrigin(aOriginSuffix, aOriginNoSuffix,
1510                                   mExecList[i].get())) {
1511       return true;
1512     }
1513   }
1514
1515   return false;
1516 }
1517
1518 namespace {
1519
1520 bool FindPendingUpdateForOrigin(
1521     const nsACString& aOriginSuffix, const nsACString& aOriginNoSuffix,
1522     StorageDBThread::DBOperation* aPendingOperation) {
1523   if ((aPendingOperation->Type() == StorageDBThread::DBOperation::opAddItem ||
1524        aPendingOperation->Type() ==
1525            StorageDBThread::DBOperation::opUpdateItem ||
1526        aPendingOperation->Type() ==
1527            StorageDBThread::DBOperation::opRemoveItem) &&
1528       aOriginNoSuffix == aPendingOperation->OriginNoSuffix() &&
1529       aOriginSuffix == aPendingOperation->OriginSuffix()) {
1530     return true;
1531   }
1532
1533   return false;
1534 }
1535
1536 }  // namespace
1537
1538 bool StorageDBThread::PendingOperations::IsOriginUpdatePending(
1539     const nsACString& aOriginSuffix, const nsACString& aOriginNoSuffix) const {
1540   // Called under the lock
1541
1542   for (const auto& update : mUpdates.Values()) {
1543     if (FindPendingUpdateForOrigin(aOriginSuffix, aOriginNoSuffix,
1544                                    update.get())) {
1545       return true;
1546     }
1547   }
1548
1549   for (uint32_t i = 0; i < mExecList.Length(); ++i) {
1550     if (FindPendingUpdateForOrigin(aOriginSuffix, aOriginNoSuffix,
1551                                    mExecList[i].get())) {
1552       return true;
1553     }
1554   }
1555
1556   return false;
1557 }
1558
1559 nsresult StorageDBThread::InitHelper::SyncDispatchAndReturnProfilePath(
1560     nsAString& aProfilePath) {
1561   ::mozilla::ipc::AssertIsOnBackgroundThread();
1562
1563   MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(this));
1564
1565   mozilla::MutexAutoLock autolock(mMutex);
1566   while (mWaiting) {
1567     mCondVar.Wait();
1568   }
1569
1570   if (NS_WARN_IF(NS_FAILED(mMainThreadResultCode))) {
1571     return mMainThreadResultCode;
1572   }
1573
1574   aProfilePath = mProfilePath;
1575   return NS_OK;
1576 }
1577
1578 NS_IMETHODIMP
1579 StorageDBThread::InitHelper::Run() {
1580   MOZ_ASSERT(NS_IsMainThread());
1581
1582   nsresult rv = GetProfilePath(mProfilePath);
1583   if (NS_WARN_IF(NS_FAILED(rv))) {
1584     mMainThreadResultCode = rv;
1585   }
1586
1587   mozilla::MutexAutoLock lock(mMutex);
1588   MOZ_ASSERT(mWaiting);
1589
1590   mWaiting = false;
1591   mCondVar.Notify();
1592
1593   return NS_OK;
1594 }
1595
1596 NS_IMETHODIMP
1597 StorageDBThread::NoteBackgroundThreadRunnable::Run() {
1598   MOZ_ASSERT(NS_IsMainThread());
1599
1600   StorageObserver* observer = StorageObserver::Self();
1601   MOZ_ASSERT(observer);
1602
1603   observer->NoteBackgroundThread(mPrivateBrowsingId, mOwningThread);
1604
1605   return NS_OK;
1606 }
1607
1608 NS_IMETHODIMP
1609 StorageDBThread::ShutdownRunnable::Run() {
1610   if (NS_IsMainThread()) {
1611     mDone = true;
1612
1613     return NS_OK;
1614   }
1615
1616   ::mozilla::ipc::AssertIsOnBackgroundThread();
1617   MOZ_RELEASE_ASSERT(mPrivateBrowsingId < kPrivateBrowsingIdCount);
1618
1619   StorageDBThread*& storageThread = sStorageThread[mPrivateBrowsingId];
1620   if (storageThread) {
1621     sStorageThreadDown[mPrivateBrowsingId] = true;
1622
1623     storageThread->Shutdown();
1624
1625     delete storageThread;
1626     storageThread = nullptr;
1627   }
1628
1629   MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(this));
1630
1631   return NS_OK;
1632 }
1633
1634 }  // namespace mozilla::dom