dom/storage/StorageDBThread.cpp

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "StorageDBThread.h"
   8
   9 #include "StorageCommon.h"
  10 #include "StorageDBUpdater.h"
  11 #include "StorageUtils.h"
  12 #include "LocalStorageCache.h"
  13 #include "LocalStorageManager.h"
  14
  15 #include "nsComponentManagerUtils.h"
  16 #include "nsDirectoryServiceUtils.h"
  17 #include "nsAppDirectoryServiceDefs.h"
  18 #include "nsThreadUtils.h"
  19 #include "nsProxyRelease.h"
  20 #include "mozStorageCID.h"
  21 #include "mozStorageHelper.h"
  22 #include "mozIStorageService.h"
  23 #include "mozIStorageBindingParams.h"
  24 #include "mozIStorageValueArray.h"
  25 #include "mozIStorageFunction.h"
  26 #include "mozilla/BasePrincipal.h"
  27 #include "mozilla/ipc/BackgroundParent.h"
  28 #include "nsIObserverService.h"
  29 #include "nsThread.h"
  30 #include "nsThreadManager.h"
  31 #include "nsVariant.h"
  32 #include "mozilla/EventQueue.h"
  33 #include "mozilla/IOInterposer.h"
  34 #include "mozilla/OriginAttributes.h"
  35 #include "mozilla/ThreadEventQueue.h"
  36 #include "mozilla/Services.h"
  37 #include "mozilla/Tokenizer.h"
  38 #include "GeckoProfiler.h"
  39
  40 // How long we collect write oprerations
  41 // before they are flushed to the database
  42 // In milliseconds.
  43 #define FLUSHING_INTERVAL_MS 5000
  44
  45 // Write Ahead Log's maximum size is 512KB
  46 #define MAX_WAL_SIZE_BYTES 512 * 1024
  47
  48 // Current version of the database schema
  49 #define CURRENT_SCHEMA_VERSION 2
  50
  51 namespace mozilla::dom {
  52
  53 using namespace StorageUtils;
  54
  55 namespace {  // anon
  56
  57 StorageDBThread* sStorageThread[kPrivateBrowsingIdCount] = {nullptr, nullptr};
  58
  59 // False until we shut the storage thread down.
  60 bool sStorageThreadDown[kPrivateBrowsingIdCount] = {false, false};
  61
  62 }  // namespace
  63
  64 // XXX Fix me!
  65 #if 0
  66 StorageDBBridge::StorageDBBridge()
  67 {
  68 }
  69 #endif
  70
  71 class StorageDBThread::InitHelper final : public Runnable {
  72   nsCOMPtr<nsIEventTarget> mOwningThread;
  73   mozilla::Mutex mMutex MOZ_UNANNOTATED;
  74   mozilla::CondVar mCondVar;
  75   nsString mProfilePath;
  76   nsresult mMainThreadResultCode;
  77   bool mWaiting;
  78
  79  public:
  80   InitHelper()
  81       : Runnable("dom::StorageDBThread::InitHelper"),
  82         mOwningThread(GetCurrentSerialEventTarget()),
  83         mMutex("InitHelper::mMutex"),
  84         mCondVar(mMutex, "InitHelper::mCondVar"),
  85         mMainThreadResultCode(NS_OK),
  86         mWaiting(true) {}
  87
  88   // Because of the `sync Preload` IPC, we need to be able to synchronously
  89   // initialize, which includes consulting and initializing
  90   // some main-thread-only APIs. Bug 1386441 discusses improving this situation.
  91   nsresult SyncDispatchAndReturnProfilePath(nsAString& aProfilePath);
  92
  93  private:
  94   ~InitHelper() override = default;
  95
  96   nsresult RunOnMainThread();
  97
  98   NS_DECL_NSIRUNNABLE
  99 };
 100
 101 class StorageDBThread::NoteBackgroundThreadRunnable final : public Runnable {
 102   // Expected to be only 0 or 1.
 103   const uint32_t mPrivateBrowsingId;
 104   nsCOMPtr<nsIEventTarget> mOwningThread;
 105
 106  public:
 107   explicit NoteBackgroundThreadRunnable(const uint32_t aPrivateBrowsingId)
 108       : Runnable("dom::StorageDBThread::NoteBackgroundThreadRunnable"),
 109         mPrivateBrowsingId(aPrivateBrowsingId),
 110         mOwningThread(GetCurrentSerialEventTarget()) {
 111     MOZ_RELEASE_ASSERT(aPrivateBrowsingId < kPrivateBrowsingIdCount);
 112   }
 113
 114  private:
 115   ~NoteBackgroundThreadRunnable() override = default;
 116
 117   NS_DECL_NSIRUNNABLE
 118 };
 119
 120 StorageDBThread::StorageDBThread(const uint32_t aPrivateBrowsingId)
 121     : mThread(nullptr),
 122       mThreadObserver(new ThreadObserver()),
 123       mStopIOThread(false),
 124       mWALModeEnabled(false),
 125       mDBReady(false),
 126       mStatus(NS_OK),
 127       mWorkerStatements(mWorkerConnection),
 128       mReaderStatements(mReaderConnection),
 129       mFlushImmediately(false),
 130       mPrivateBrowsingId(aPrivateBrowsingId),
 131       mPriorityCounter(0) {
 132   MOZ_RELEASE_ASSERT(aPrivateBrowsingId < kPrivateBrowsingIdCount);
 133 }
 134
 135 // static
 136 StorageDBThread* StorageDBThread::Get(const uint32_t aPrivateBrowsingId) {
 137   ::mozilla::ipc::AssertIsOnBackgroundThread();
 138   MOZ_RELEASE_ASSERT(aPrivateBrowsingId < kPrivateBrowsingIdCount);
 139
 140   return sStorageThread[aPrivateBrowsingId];
 141 }
 142
 143 // static
 144 StorageDBThread* StorageDBThread::GetOrCreate(
 145     const nsString& aProfilePath, const uint32_t aPrivateBrowsingId) {
 146   ::mozilla::ipc::AssertIsOnBackgroundThread();
 147   MOZ_RELEASE_ASSERT(aPrivateBrowsingId < kPrivateBrowsingIdCount);
 148
 149   StorageDBThread*& storageThread = sStorageThread[aPrivateBrowsingId];
 150   if (storageThread || sStorageThreadDown[aPrivateBrowsingId]) {
 151     // When sStorageThreadDown is at true, sStorageThread is null.
 152     // Checking sStorageThreadDown flag here prevents reinitialization of
 153     // the storage thread after shutdown.
 154     return storageThread;
 155   }
 156
 157   auto newStorageThread = MakeUnique<StorageDBThread>(aPrivateBrowsingId);
 158
 159   nsresult rv = newStorageThread->Init(aProfilePath);
 160   if (NS_WARN_IF(NS_FAILED(rv))) {
 161     return nullptr;
 162   }
 163
 164   storageThread = newStorageThread.release();
 165
 166   return storageThread;
 167 }
 168
 169 // static
 170 nsresult StorageDBThread::GetProfilePath(nsString& aProfilePath) {
 171   MOZ_ASSERT(XRE_IsParentProcess());
 172   MOZ_ASSERT(NS_IsMainThread());
 173
 174   // Need to determine location on the main thread, since
 175   // NS_GetSpecialDirectory accesses the atom table that can
 176   // only be accessed on the main thread.
 177   nsCOMPtr<nsIFile> profileDir;
 178   nsresult rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR,
 179                                        getter_AddRefs(profileDir));
 180   if (NS_WARN_IF(NS_FAILED(rv))) {
 181     return rv;
 182   }
 183
 184   rv = profileDir->GetPath(aProfilePath);
 185   if (NS_WARN_IF(NS_FAILED(rv))) {
 186     return rv;
 187   }
 188
 189   // This service has to be started on the main thread currently.
 190   nsCOMPtr<mozIStorageService> ss =
 191       do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID, &rv);
 192   if (NS_WARN_IF(NS_FAILED(rv))) {
 193     return rv;
 194   }
 195
 196   return NS_OK;
 197 }
 198
 199 nsresult StorageDBThread::Init(const nsString& aProfilePath) {
 200   ::mozilla::ipc::AssertIsOnBackgroundThread();
 201
 202   if (mPrivateBrowsingId == 0) {
 203     nsresult rv;
 204
 205     nsString profilePath;
 206     if (aProfilePath.IsEmpty()) {
 207       RefPtr<InitHelper> helper = new InitHelper();
 208
 209       rv = helper->SyncDispatchAndReturnProfilePath(profilePath);
 210       if (NS_WARN_IF(NS_FAILED(rv))) {
 211         return rv;
 212       }
 213     } else {
 214       profilePath = aProfilePath;
 215     }
 216
 217     mDatabaseFile = do_CreateInstance(NS_LOCAL_FILE_CONTRACTID, &rv);
 218     if (NS_WARN_IF(NS_FAILED(rv))) {
 219       return rv;
 220     }
 221
 222     rv = mDatabaseFile->InitWithPath(profilePath);
 223     if (NS_WARN_IF(NS_FAILED(rv))) {
 224       return rv;
 225     }
 226
 227     rv = mDatabaseFile->Append(u"webappsstore.sqlite"_ns);
 228     NS_ENSURE_SUCCESS(rv, rv);
 229   }
 230
 231   // Need to keep the lock to avoid setting mThread later then
 232   // the thread body executes.
 233   MonitorAutoLock monitor(mThreadObserver->GetMonitor());
 234
 235   mThread = PR_CreateThread(PR_USER_THREAD, &StorageDBThread::ThreadFunc, this,
 236                             PR_PRIORITY_LOW, PR_GLOBAL_THREAD,
 237                             PR_JOINABLE_THREAD, 262144);
 238   if (!mThread) {
 239     return NS_ERROR_OUT_OF_MEMORY;
 240   }
 241
 242   RefPtr<NoteBackgroundThreadRunnable> runnable =
 243       new NoteBackgroundThreadRunnable(mPrivateBrowsingId);
 244   MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(runnable));
 245
 246   return NS_OK;
 247 }
 248
 249 nsresult StorageDBThread::Shutdown() {
 250   ::mozilla::ipc::AssertIsOnBackgroundThread();
 251
 252   if (!mThread) {
 253     return NS_ERROR_NOT_INITIALIZED;
 254   }
 255
 256   Telemetry::AutoTimer<Telemetry::LOCALDOMSTORAGE_SHUTDOWN_DATABASE_MS> timer;
 257
 258   {
 259     MonitorAutoLock monitor(mThreadObserver->GetMonitor());
 260
 261     // After we stop, no other operations can be accepted
 262     mFlushImmediately = true;
 263     mStopIOThread = true;
 264     monitor.Notify();
 265   }
 266
 267   PR_JoinThread(mThread);
 268   mThread = nullptr;
 269
 270   return mStatus;
 271 }
 272
 273 void StorageDBThread::SyncPreload(LocalStorageCacheBridge* aCache,
 274                                   bool aForceSync) {
 275   AUTO_PROFILER_LABEL("StorageDBThread::SyncPreload", OTHER);
 276   if (!aForceSync && aCache->LoadedCount()) {
 277     // Preload already started for this cache, just wait for it to finish.
 278     // LoadWait will exit after LoadDone on the cache has been called.
 279     SetHigherPriority();
 280     aCache->LoadWait();
 281     SetDefaultPriority();
 282     return;
 283   }
 284
 285   // Bypass sync load when an update is pending in the queue to write, we would
 286   // get incosistent data in the cache.  Also don't allow sync main-thread
 287   // preload when DB open and init is still pending on the background thread.
 288   if (mDBReady && mWALModeEnabled) {
 289     bool pendingTasks;
 290     {
 291       MonitorAutoLock monitor(mThreadObserver->GetMonitor());
 292       pendingTasks = mPendingTasks.IsOriginUpdatePending(
 293                          aCache->OriginSuffix(), aCache->OriginNoSuffix()) ||
 294                      mPendingTasks.IsOriginClearPending(
 295                          aCache->OriginSuffix(), aCache->OriginNoSuffix());
 296     }
 297
 298     if (!pendingTasks) {
 299       // WAL is enabled, thus do the load synchronously on the main thread.
 300       DBOperation preload(DBOperation::opPreload, aCache);
 301       preload.PerformAndFinalize(this);
 302       return;
 303     }
 304   }
 305
 306   // Need to go asynchronously since WAL is not allowed or scheduled updates
 307   // need to be flushed first.
 308   // Schedule preload for this cache as the first operation.
 309   nsresult rv =
 310       InsertDBOp(MakeUnique<DBOperation>(DBOperation::opPreloadUrgent, aCache));
 311
 312   // LoadWait exits after LoadDone of the cache has been called.
 313   if (NS_SUCCEEDED(rv)) {
 314     aCache->LoadWait();
 315   }
 316 }
 317
 318 void StorageDBThread::AsyncFlush() {
 319   MonitorAutoLock monitor(mThreadObserver->GetMonitor());
 320   mFlushImmediately = true;
 321   monitor.Notify();
 322 }
 323
 324 bool StorageDBThread::ShouldPreloadOrigin(const nsACString& aOrigin) {
 325   MonitorAutoLock monitor(mThreadObserver->GetMonitor());
 326   return mOriginsHavingData.Contains(aOrigin);
 327 }
 328
 329 void StorageDBThread::GetOriginsHavingData(nsTArray<nsCString>* aOrigins) {
 330   MonitorAutoLock monitor(mThreadObserver->GetMonitor());
 331   AppendToArray(*aOrigins, mOriginsHavingData);
 332 }
 333
 334 nsresult StorageDBThread::InsertDBOp(
 335     UniquePtr<StorageDBThread::DBOperation> aOperation) {
 336   MonitorAutoLock monitor(mThreadObserver->GetMonitor());
 337
 338   if (NS_FAILED(mStatus)) {
 339     MonitorAutoUnlock unlock(mThreadObserver->GetMonitor());
 340     aOperation->Finalize(mStatus);
 341     return mStatus;
 342   }
 343
 344   if (mStopIOThread) {
 345     // Thread use after shutdown demanded.
 346     MOZ_ASSERT(false);
 347     return NS_ERROR_NOT_INITIALIZED;
 348   }
 349
 350   switch (aOperation->Type()) {
 351     case DBOperation::opPreload:
 352     case DBOperation::opPreloadUrgent:
 353       if (mPendingTasks.IsOriginUpdatePending(aOperation->OriginSuffix(),
 354                                               aOperation->OriginNoSuffix())) {
 355         // If there is a pending update operation for the scope first do the
 356         // flush before we preload the cache.  This may happen in an extremely
 357         // rare case when a child process throws away its cache before flush on
 358         // the parent has finished.  If we would preloaded the cache as a
 359         // priority operation before the pending flush, we would have got an
 360         // inconsistent cache content.
 361         mFlushImmediately = true;
 362       } else if (mPendingTasks.IsOriginClearPending(
 363                      aOperation->OriginSuffix(),
 364                      aOperation->OriginNoSuffix())) {
 365         // The scope is scheduled to be cleared, so just quickly load as empty.
 366         // We need to do this to prevent load of the DB data before the scope
 367         // has actually been cleared from the database.  Preloads are processed
 368         // immediately before update and clear operations on the database that
 369         // are flushed periodically in batches.
 370         MonitorAutoUnlock unlock(mThreadObserver->GetMonitor());
 371         aOperation->Finalize(NS_OK);
 372         return NS_OK;
 373       }
 374       [[fallthrough]];
 375
 376     case DBOperation::opGetUsage:
 377       if (aOperation->Type() == DBOperation::opPreloadUrgent) {
 378         SetHigherPriority();  // Dropped back after urgent preload execution
 379         mPreloads.InsertElementAt(0, aOperation.release());
 380       } else {
 381         mPreloads.AppendElement(aOperation.release());
 382       }
 383
 384       // Immediately start executing this.
 385       monitor.Notify();
 386       break;
 387
 388     default:
 389       // Update operations are first collected, coalesced and then flushed
 390       // after a short time.
 391       mPendingTasks.Add(std::move(aOperation));
 392
 393       ScheduleFlush();
 394       break;
 395   }
 396
 397   return NS_OK;
 398 }
 399
 400 void StorageDBThread::SetHigherPriority() {
 401   ++mPriorityCounter;
 402   PR_SetThreadPriority(mThread, PR_PRIORITY_URGENT);
 403 }
 404
 405 void StorageDBThread::SetDefaultPriority() {
 406   if (--mPriorityCounter <= 0) {
 407     PR_SetThreadPriority(mThread, PR_PRIORITY_LOW);
 408   }
 409 }
 410
 411 void StorageDBThread::ThreadFunc(void* aArg) {
 412   {
 413     auto queue = MakeRefPtr<ThreadEventQueue>(MakeUnique<EventQueue>());
 414     Unused << nsThreadManager::get().CreateCurrentThread(queue);
 415   }
 416
 417   AUTO_PROFILER_REGISTER_THREAD("localStorage DB");
 418   NS_SetCurrentThreadName("localStorage DB");
 419   mozilla::IOInterposer::RegisterCurrentThread();
 420
 421   StorageDBThread* thread = static_cast<StorageDBThread*>(aArg);
 422   thread->ThreadFunc();
 423   mozilla::IOInterposer::UnregisterCurrentThread();
 424 }
 425
 426 void StorageDBThread::ThreadFunc() {
 427   nsresult rv = InitDatabase();
 428
 429   MonitorAutoLock lockMonitor(mThreadObserver->GetMonitor());
 430
 431   if (NS_FAILED(rv)) {
 432     mStatus = rv;
 433     mStopIOThread = true;
 434     return;
 435   }
 436
 437   // Create an nsIThread for the current PRThread, so we can observe runnables
 438   // dispatched to it.
 439   nsCOMPtr<nsIThread> thread = NS_GetCurrentThread();
 440   nsCOMPtr<nsIThreadInternal> threadInternal = do_QueryInterface(thread);
 441   MOZ_ASSERT(threadInternal);  // Should always succeed.
 442   threadInternal->SetObserver(mThreadObserver);
 443
 444   while (MOZ_LIKELY(!mStopIOThread || mPreloads.Length() ||
 445                     mPendingTasks.HasTasks() ||
 446                     mThreadObserver->HasPendingEvents())) {
 447     // Process xpcom events first.
 448     while (MOZ_UNLIKELY(mThreadObserver->HasPendingEvents())) {
 449       mThreadObserver->ClearPendingEvents();
 450       MonitorAutoUnlock unlock(mThreadObserver->GetMonitor());
 451       bool processedEvent;
 452       do {
 453         rv = thread->ProcessNextEvent(false, &processedEvent);
 454       } while (NS_SUCCEEDED(rv) && processedEvent);
 455     }
 456
 457     TimeDuration timeUntilFlush = TimeUntilFlush();
 458     if (MOZ_UNLIKELY(timeUntilFlush.IsZero())) {
 459       // Flush time is up or flush has been forced, do it now.
 460       UnscheduleFlush();
 461       if (mPendingTasks.Prepare()) {
 462         {
 463           MonitorAutoUnlock unlockMonitor(mThreadObserver->GetMonitor());
 464           rv = mPendingTasks.Execute(this);
 465         }
 466
 467         if (!mPendingTasks.Finalize(rv)) {
 468           mStatus = rv;
 469           NS_WARNING("localStorage DB access broken");
 470         }
 471       }
 472       NotifyFlushCompletion();
 473     } else if (MOZ_LIKELY(mPreloads.Length())) {
 474       UniquePtr<DBOperation> op(mPreloads[0]);
 475       mPreloads.RemoveElementAt(0);
 476       {
 477         MonitorAutoUnlock unlockMonitor(mThreadObserver->GetMonitor());
 478         op->PerformAndFinalize(this);
 479       }
 480
 481       if (op->Type() == DBOperation::opPreloadUrgent) {
 482         SetDefaultPriority();  // urgent preload unscheduled
 483       }
 484     } else if (MOZ_UNLIKELY(!mStopIOThread)) {
 485       AUTO_PROFILER_LABEL("StorageDBThread::ThreadFunc::Wait", IDLE);
 486       lockMonitor.Wait(timeUntilFlush);
 487     }
 488   }  // thread loop
 489
 490   mStatus = ShutdownDatabase();
 491
 492   if (threadInternal) {
 493     threadInternal->SetObserver(nullptr);
 494   }
 495 }
 496
 497 NS_IMPL_ISUPPORTS(StorageDBThread::ThreadObserver, nsIThreadObserver)
 498
 499 NS_IMETHODIMP
 500 StorageDBThread::ThreadObserver::OnDispatchedEvent() {
 501   MonitorAutoLock lock(mMonitor);
 502   mHasPendingEvents = true;
 503   lock.Notify();
 504   return NS_OK;
 505 }
 506
 507 NS_IMETHODIMP
 508 StorageDBThread::ThreadObserver::OnProcessNextEvent(nsIThreadInternal* aThread,
 509                                                     bool mayWait) {
 510   return NS_OK;
 511 }
 512
 513 NS_IMETHODIMP
 514 StorageDBThread::ThreadObserver::AfterProcessNextEvent(
 515     nsIThreadInternal* aThread, bool eventWasProcessed) {
 516   return NS_OK;
 517 }
 518
 519 nsresult StorageDBThread::OpenDatabaseConnection() {
 520   nsresult rv;
 521
 522   MOZ_ASSERT(!NS_IsMainThread());
 523
 524   nsCOMPtr<mozIStorageService> service =
 525       do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID, &rv);
 526   NS_ENSURE_SUCCESS(rv, rv);
 527
 528   if (mPrivateBrowsingId == 0) {
 529     MOZ_ASSERT(mDatabaseFile);
 530
 531     rv = service->OpenUnsharedDatabase(mDatabaseFile,
 532                                        mozIStorageService::CONNECTION_DEFAULT,
 533                                        getter_AddRefs(mWorkerConnection));
 534     if (rv == NS_ERROR_FILE_CORRUPTED) {
 535       // delete the db and try opening again
 536       rv = mDatabaseFile->Remove(false);
 537       NS_ENSURE_SUCCESS(rv, rv);
 538       rv = service->OpenUnsharedDatabase(mDatabaseFile,
 539                                          mozIStorageService::CONNECTION_DEFAULT,
 540                                          getter_AddRefs(mWorkerConnection));
 541     }
 542   } else {
 543     MOZ_ASSERT(mPrivateBrowsingId == 1);
 544
 545     rv = service->OpenSpecialDatabase(kMozStorageMemoryStorageKey,
 546                                       "lsprivatedb"_ns,
 547                                       mozIStorageService::CONNECTION_DEFAULT,
 548                                       getter_AddRefs(mWorkerConnection));
 549   }
 550   NS_ENSURE_SUCCESS(rv, rv);
 551
 552   return NS_OK;
 553 }
 554
 555 nsresult StorageDBThread::OpenAndUpdateDatabase() {
 556   nsresult rv;
 557
 558   // Here we are on the worker thread. This opens the worker connection.
 559   MOZ_ASSERT(!NS_IsMainThread());
 560
 561   rv = OpenDatabaseConnection();
 562   NS_ENSURE_SUCCESS(rv, rv);
 563
 564   // SQLite doesn't support WAL journals for in-memory databases.
 565   if (mPrivateBrowsingId == 0) {
 566     rv = TryJournalMode();
 567     NS_ENSURE_SUCCESS(rv, rv);
 568   }
 569
 570   return NS_OK;
 571 }
 572
 573 nsresult StorageDBThread::InitDatabase() {
 574   nsresult rv;
 575
 576   // Here we are on the worker thread. This opens the worker connection.
 577   MOZ_ASSERT(!NS_IsMainThread());
 578
 579   rv = OpenAndUpdateDatabase();
 580   NS_ENSURE_SUCCESS(rv, rv);
 581
 582   rv = StorageDBUpdater::Update(mWorkerConnection);
 583   if (NS_FAILED(rv)) {
 584     if (mPrivateBrowsingId == 0) {
 585       // Update has failed, rather throw the database away and try
 586       // opening and setting it up again.
 587       rv = mWorkerConnection->Close();
 588       mWorkerConnection = nullptr;
 589       NS_ENSURE_SUCCESS(rv, rv);
 590
 591       rv = mDatabaseFile->Remove(false);
 592       NS_ENSURE_SUCCESS(rv, rv);
 593
 594       rv = OpenAndUpdateDatabase();
 595     }
 596     NS_ENSURE_SUCCESS(rv, rv);
 597   }
 598
 599   // Create a read-only clone
 600   (void)mWorkerConnection->Clone(true, getter_AddRefs(mReaderConnection));
 601   NS_ENSURE_TRUE(mReaderConnection, NS_ERROR_FAILURE);
 602
 603   // Database open and all initiation operation are done.  Switching this flag
 604   // to true allow main thread to read directly from the database.  If we would
 605   // allow this sooner, we would have opened a window where main thread read
 606   // might operate on a totally broken and incosistent database.
 607   mDBReady = true;
 608
 609   // List scopes having any stored data
 610   nsCOMPtr<mozIStorageStatement> stmt;
 611   // Note: result of this select must match StorageManager::CreateOrigin()
 612   rv = mWorkerConnection->CreateStatement(
 613       nsLiteralCString("SELECT DISTINCT originAttributes || ':' || originKey "
 614                        "FROM webappsstore2"),
 615       getter_AddRefs(stmt));
 616   NS_ENSURE_SUCCESS(rv, rv);
 617   mozStorageStatementScoper scope(stmt);
 618
 619   bool exists;
 620   while (NS_SUCCEEDED(rv = stmt->ExecuteStep(&exists)) && exists) {
 621     nsAutoCString foundOrigin;
 622     rv = stmt->GetUTF8String(0, foundOrigin);
 623     NS_ENSURE_SUCCESS(rv, rv);
 624
 625     MonitorAutoLock monitor(mThreadObserver->GetMonitor());
 626     mOriginsHavingData.Insert(foundOrigin);
 627   }
 628
 629   return NS_OK;
 630 }
 631
 632 nsresult StorageDBThread::SetJournalMode(bool aIsWal) {
 633   nsresult rv;
 634
 635   nsAutoCString stmtString(MOZ_STORAGE_UNIQUIFY_QUERY_STR
 636                            "PRAGMA journal_mode = ");
 637   if (aIsWal) {
 638     stmtString.AppendLiteral("wal");
 639   } else {
 640     stmtString.AppendLiteral("truncate");
 641   }
 642
 643   nsCOMPtr<mozIStorageStatement> stmt;
 644   rv = mWorkerConnection->CreateStatement(stmtString, getter_AddRefs(stmt));
 645   NS_ENSURE_SUCCESS(rv, rv);
 646   mozStorageStatementScoper scope(stmt);
 647
 648   bool hasResult = false;
 649   rv = stmt->ExecuteStep(&hasResult);
 650   NS_ENSURE_SUCCESS(rv, rv);
 651   if (!hasResult) {
 652     return NS_ERROR_FAILURE;
 653   }
 654
 655   nsAutoCString journalMode;
 656   rv = stmt->GetUTF8String(0, journalMode);
 657   NS_ENSURE_SUCCESS(rv, rv);
 658   if ((aIsWal && !journalMode.EqualsLiteral("wal")) ||
 659       (!aIsWal && !journalMode.EqualsLiteral("truncate"))) {
 660     return NS_ERROR_FAILURE;
 661   }
 662
 663   return NS_OK;
 664 }
 665
 666 nsresult StorageDBThread::TryJournalMode() {
 667   nsresult rv;
 668
 669   rv = SetJournalMode(true);
 670   if (NS_FAILED(rv)) {
 671     mWALModeEnabled = false;
 672
 673     rv = SetJournalMode(false);
 674     NS_ENSURE_SUCCESS(rv, rv);
 675   } else {
 676     mWALModeEnabled = true;
 677
 678     rv = ConfigureWALBehavior();
 679     NS_ENSURE_SUCCESS(rv, rv);
 680   }
 681
 682   return NS_OK;
 683 }
 684
 685 nsresult StorageDBThread::ConfigureWALBehavior() {
 686   // Get the DB's page size
 687   nsCOMPtr<mozIStorageStatement> stmt;
 688   nsresult rv = mWorkerConnection->CreateStatement(
 689       nsLiteralCString(MOZ_STORAGE_UNIQUIFY_QUERY_STR "PRAGMA page_size"),
 690       getter_AddRefs(stmt));
 691   NS_ENSURE_SUCCESS(rv, rv);
 692
 693   bool hasResult = false;
 694   rv = stmt->ExecuteStep(&hasResult);
 695   NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && hasResult, NS_ERROR_FAILURE);
 696
 697   int32_t pageSize = 0;
 698   rv = stmt->GetInt32(0, &pageSize);
 699   NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && pageSize > 0, NS_ERROR_UNEXPECTED);
 700
 701   // Set the threshold for auto-checkpointing the WAL.
 702   // We don't want giant logs slowing down reads & shutdown.
 703   // Note there is a default journal_size_limit set by mozStorage.
 704   int32_t thresholdInPages =
 705       static_cast<int32_t>(MAX_WAL_SIZE_BYTES / pageSize);
 706   nsAutoCString thresholdPragma("PRAGMA wal_autocheckpoint = ");
 707   thresholdPragma.AppendInt(thresholdInPages);
 708   rv = mWorkerConnection->ExecuteSimpleSQL(thresholdPragma);
 709   NS_ENSURE_SUCCESS(rv, rv);
 710
 711   return NS_OK;
 712 }
 713
 714 nsresult StorageDBThread::ShutdownDatabase() {
 715   // Has to be called on the worker thread.
 716   MOZ_ASSERT(!NS_IsMainThread());
 717
 718   nsresult rv = mStatus;
 719
 720   mDBReady = false;
 721
 722   // Finalize the cached statements.
 723   mReaderStatements.FinalizeStatements();
 724   mWorkerStatements.FinalizeStatements();
 725
 726   if (mReaderConnection) {
 727     // No need to sync access to mReaderConnection since the main thread
 728     // is right now joining this thread, unable to execute any events.
 729     mReaderConnection->Close();
 730     mReaderConnection = nullptr;
 731   }
 732
 733   if (mWorkerConnection) {
 734     rv = mWorkerConnection->Close();
 735     mWorkerConnection = nullptr;
 736   }
 737
 738   return rv;
 739 }
 740
 741 void StorageDBThread::ScheduleFlush() {
 742   if (mDirtyEpoch) {
 743     return;  // Already scheduled
 744   }
 745
 746   // Must be non-zero to indicate we are scheduled
 747   mDirtyEpoch = TimeStamp::Now();
 748
 749   // Wake the monitor from indefinite sleep...
 750   (mThreadObserver->GetMonitor()).Notify();
 751 }
 752
 753 void StorageDBThread::UnscheduleFlush() {
 754   // We are just about to do the flush, drop flags
 755   mFlushImmediately = false;
 756   mDirtyEpoch = TimeStamp();
 757 }
 758
 759 TimeDuration StorageDBThread::TimeUntilFlush() {
 760   if (mFlushImmediately) {
 761     return 0;  // Do it now regardless the timeout.
 762   }
 763
 764   if (!mDirtyEpoch) {
 765     return TimeDuration::Forever();  // No pending task...
 766   }
 767
 768   TimeStamp now = TimeStamp::Now();
 769   TimeDuration age = now - mDirtyEpoch;
 770   static const TimeDuration kMaxAge =
 771       TimeDuration::FromMilliseconds(FLUSHING_INTERVAL_MS);
 772   if (age > kMaxAge) {
 773     return 0;  // It is time.
 774   }
 775
 776   return kMaxAge - age;  // Time left. This is used to sleep the monitor.
 777 }
 778
 779 void StorageDBThread::NotifyFlushCompletion() {
 780 #ifdef DOM_STORAGE_TESTS
 781   if (!NS_IsMainThread()) {
 782     RefPtr<nsRunnableMethod<StorageDBThread, void, false>> event =
 783         NewNonOwningRunnableMethod(
 784             "dom::StorageDBThread::NotifyFlushCompletion", this,
 785             &StorageDBThread::NotifyFlushCompletion);
 786     NS_DispatchToMainThread(event);
 787     return;
 788   }
 789
 790   nsCOMPtr<nsIObserverService> obs = mozilla::services::GetObserverService();
 791   if (obs) {
 792     obs->NotifyObservers(nullptr, "domstorage-test-flushed", nullptr);
 793   }
 794 #endif
 795 }
 796
 797 // Helper SQL function classes
 798
 799 namespace {
 800
 801 class OriginAttrsPatternMatchSQLFunction final : public mozIStorageFunction {
 802   NS_DECL_ISUPPORTS
 803   NS_DECL_MOZISTORAGEFUNCTION
 804
 805   explicit OriginAttrsPatternMatchSQLFunction(
 806       OriginAttributesPattern const& aPattern)
 807       : mPattern(aPattern) {}
 808
 809  private:
 810   OriginAttrsPatternMatchSQLFunction() = delete;
 811   ~OriginAttrsPatternMatchSQLFunction() = default;
 812
 813   OriginAttributesPattern mPattern;
 814 };
 815
 816 NS_IMPL_ISUPPORTS(OriginAttrsPatternMatchSQLFunction, mozIStorageFunction)
 817
 818 NS_IMETHODIMP
 819 OriginAttrsPatternMatchSQLFunction::OnFunctionCall(
 820     mozIStorageValueArray* aFunctionArguments, nsIVariant** aResult) {
 821   nsresult rv;
 822
 823   nsAutoCString suffix;
 824   rv = aFunctionArguments->GetUTF8String(0, suffix);
 825   NS_ENSURE_SUCCESS(rv, rv);
 826
 827   OriginAttributes oa;
 828   bool success = oa.PopulateFromSuffix(suffix);
 829   NS_ENSURE_TRUE(success, NS_ERROR_FAILURE);
 830   bool result = mPattern.Matches(oa);
 831
 832   RefPtr<nsVariant> outVar(new nsVariant());
 833   rv = outVar->SetAsBool(result);
 834   NS_ENSURE_SUCCESS(rv, rv);
 835
 836   outVar.forget(aResult);
 837   return NS_OK;
 838 }
 839
 840 }  // namespace
 841
 842 // StorageDBThread::DBOperation
 843
 844 StorageDBThread::DBOperation::DBOperation(const OperationType aType,
 845                                           LocalStorageCacheBridge* aCache,
 846                                           const nsAString& aKey,
 847                                           const nsAString& aValue)
 848     : mType(aType), mCache(aCache), mKey(aKey), mValue(aValue) {
 849   MOZ_ASSERT(mType == opPreload || mType == opPreloadUrgent ||
 850              mType == opAddItem || mType == opUpdateItem ||
 851              mType == opRemoveItem || mType == opClear || mType == opClearAll);
 852   MOZ_COUNT_CTOR(StorageDBThread::DBOperation);
 853 }
 854
 855 StorageDBThread::DBOperation::DBOperation(const OperationType aType,
 856                                           StorageUsageBridge* aUsage)
 857     : mType(aType), mUsage(aUsage) {
 858   MOZ_ASSERT(mType == opGetUsage);
 859   MOZ_COUNT_CTOR(StorageDBThread::DBOperation);
 860 }
 861
 862 StorageDBThread::DBOperation::DBOperation(const OperationType aType,
 863                                           const nsACString& aOriginNoSuffix)
 864     : mType(aType), mCache(nullptr), mOrigin(aOriginNoSuffix) {
 865   MOZ_ASSERT(mType == opClearMatchingOrigin);
 866   MOZ_COUNT_CTOR(StorageDBThread::DBOperation);
 867 }
 868
 869 StorageDBThread::DBOperation::DBOperation(
 870     const OperationType aType, const OriginAttributesPattern& aOriginNoSuffix)
 871     : mType(aType), mCache(nullptr), mOriginPattern(aOriginNoSuffix) {
 872   MOZ_ASSERT(mType == opClearMatchingOriginAttributes);
 873   MOZ_COUNT_CTOR(StorageDBThread::DBOperation);
 874 }
 875
 876 StorageDBThread::DBOperation::~DBOperation() {
 877   MOZ_COUNT_DTOR(StorageDBThread::DBOperation);
 878 }
 879
 880 const nsCString StorageDBThread::DBOperation::OriginNoSuffix() const {
 881   if (mCache) {
 882     return mCache->OriginNoSuffix();
 883   }
 884
 885   return ""_ns;
 886 }
 887
 888 const nsCString StorageDBThread::DBOperation::OriginSuffix() const {
 889   if (mCache) {
 890     return mCache->OriginSuffix();
 891   }
 892
 893   return ""_ns;
 894 }
 895
 896 const nsCString StorageDBThread::DBOperation::Origin() const {
 897   if (mCache) {
 898     return mCache->Origin();
 899   }
 900
 901   return mOrigin;
 902 }
 903
 904 const nsCString StorageDBThread::DBOperation::Target() const {
 905   switch (mType) {
 906     case opAddItem:
 907     case opUpdateItem:
 908     case opRemoveItem:
 909       return Origin() + "|"_ns + NS_ConvertUTF16toUTF8(mKey);
 910
 911     default:
 912       return Origin();
 913   }
 914 }
 915
 916 void StorageDBThread::DBOperation::PerformAndFinalize(
 917     StorageDBThread* aThread) {
 918   Finalize(Perform(aThread));
 919 }
 920
 921 nsresult StorageDBThread::DBOperation::Perform(StorageDBThread* aThread) {
 922   nsresult rv;
 923
 924   switch (mType) {
 925     case opPreload:
 926     case opPreloadUrgent: {
 927       // Already loaded?
 928       if (mCache->Loaded()) {
 929         break;
 930       }
 931
 932       StatementCache* statements;
 933       if (MOZ_UNLIKELY(::mozilla::ipc::IsOnBackgroundThread())) {
 934         statements = &aThread->mReaderStatements;
 935       } else {
 936         statements = &aThread->mWorkerStatements;
 937       }
 938
 939       // OFFSET is an optimization when we have to do a sync load
 940       // and cache has already loaded some parts asynchronously.
 941       // It skips keys we have already loaded.
 942       nsCOMPtr<mozIStorageStatement> stmt = statements->GetCachedStatement(
 943           "SELECT key, value FROM webappsstore2 "
 944           "WHERE originAttributes = :originAttributes AND originKey = "
 945           ":originKey "
 946           "ORDER BY key LIMIT -1 OFFSET :offset");
 947       NS_ENSURE_STATE(stmt);
 948       mozStorageStatementScoper scope(stmt);
 949
 950       rv = stmt->BindUTF8StringByName("originAttributes"_ns,
 951                                       mCache->OriginSuffix());
 952       NS_ENSURE_SUCCESS(rv, rv);
 953
 954       rv = stmt->BindUTF8StringByName("originKey"_ns, mCache->OriginNoSuffix());
 955       NS_ENSURE_SUCCESS(rv, rv);
 956
 957       rv = stmt->BindInt32ByName("offset"_ns,
 958                                  static_cast<int32_t>(mCache->LoadedCount()));
 959       NS_ENSURE_SUCCESS(rv, rv);
 960
 961       bool exists;
 962       while (NS_SUCCEEDED(rv = stmt->ExecuteStep(&exists)) && exists) {
 963         nsAutoString key;
 964         rv = stmt->GetString(0, key);
 965         NS_ENSURE_SUCCESS(rv, rv);
 966
 967         nsAutoString value;
 968         rv = stmt->GetString(1, value);
 969         NS_ENSURE_SUCCESS(rv, rv);
 970
 971         if (!mCache->LoadItem(key, value)) {
 972           break;
 973         }
 974       }
 975       // The loop condition's call to ExecuteStep() may have terminated because
 976       // !NS_SUCCEEDED(), we need an early return to cover that case.  This also
 977       // covers success cases as well, but that's inductively safe.
 978       NS_ENSURE_SUCCESS(rv, rv);
 979       break;
 980     }
 981
 982     case opGetUsage: {
 983       // Bug 1676410 fixed a regression caused by bug 1165214. However, it
 984       // turns out that 100% correct checking of the eTLD+1 usage is not
 985       // possible to recover easily, see bug 1683299.
 986 #if 0
 987       // This is how it should be done, but due to other problems like lack
 988       // of usage synchronization between content processes, we temporarily
 989       // disabled the matching using "%".
 990
 991       nsCOMPtr<mozIStorageStatement> stmt =
 992           aThread->mWorkerStatements.GetCachedStatement(
 993               "SELECT SUM(LENGTH(key) + LENGTH(value)) FROM webappsstore2 "
 994               "WHERE (originAttributes || ':' || originKey) LIKE :usageOrigin "
 995               "ESCAPE '\\'");
 996       NS_ENSURE_STATE(stmt);
 997
 998       mozStorageStatementScoper scope(stmt);
 999
1000       // The database schema is built around cleverly reversing domain names
1001       // (the "originKey") so that we can efficiently group usage by eTLD+1.
1002       // "foo.example.org" has an eTLD+1 of ".example.org". They reverse to
1003       // "gro.elpmaxe.oof" and "gro.elpmaxe." respectively, noting that the
1004       // reversed eTLD+1 is a prefix of its reversed sub-domain. To this end,
1005       // we can calculate all of the usage for an eTLD+1 by summing up all the
1006       // rows which have the reversed eTLD+1 as a prefix. In SQL we can
1007       // accomplish this using LIKE which provides for case-insensitive
1008       // matching with "_" as a single-character wildcard match and "%" any
1009       // sequence of zero or more characters. So by suffixing the reversed
1010       // eTLD+1 and using "%" we get our case-insensitive (domain names are
1011       // case-insensitive) matching. Note that although legal domain names
1012       // don't include "_" or "%", file origins can include them, so we need
1013       // to escape our OriginScope for correctness.
1014       nsAutoCString originScopeEscaped;
1015       rv = stmt->EscapeUTF8StringForLIKE(mUsage->OriginScope(), '\\',
1016                                          originScopeEscaped);
1017       NS_ENSURE_SUCCESS(rv, rv);
1018
1019       rv = stmt->BindUTF8StringByName("usageOrigin"_ns,
1020                                       originScopeEscaped + "%"_ns);
1021       NS_ENSURE_SUCCESS(rv, rv);
1022 #else
1023       // This is the code before bug 1676410 and bug 1676973. The returned
1024       // usage will be zero in most of the cases, but due to lack of usage
1025       // synchronization between content processes we have to live with this
1026       // semi-broken behaviour because it causes less harm than the matching
1027       // using "%".
1028
1029       nsCOMPtr<mozIStorageStatement> stmt =
1030           aThread->mWorkerStatements.GetCachedStatement(
1031               "SELECT SUM(LENGTH(key) + LENGTH(value)) FROM webappsstore2 "
1032               "WHERE (originAttributes || ':' || originKey) LIKE :usageOrigin");
1033       NS_ENSURE_STATE(stmt);
1034
1035       mozStorageStatementScoper scope(stmt);
1036
1037       rv = stmt->BindUTF8StringByName("usageOrigin"_ns, mUsage->OriginScope());
1038       NS_ENSURE_SUCCESS(rv, rv);
1039 #endif
1040
1041       bool exists;
1042       rv = stmt->ExecuteStep(&exists);
1043       NS_ENSURE_SUCCESS(rv, rv);
1044
1045       int64_t usage = 0;
1046       if (exists) {
1047         rv = stmt->GetInt64(0, &usage);
1048         NS_ENSURE_SUCCESS(rv, rv);
1049       }
1050
1051       mUsage->LoadUsage(usage);
1052       break;
1053     }
1054
1055     case opAddItem:
1056     case opUpdateItem: {
1057       MOZ_ASSERT(!NS_IsMainThread());
1058
1059       nsCOMPtr<mozIStorageStatement> stmt =
1060           aThread->mWorkerStatements.GetCachedStatement(
1061               "INSERT OR REPLACE INTO webappsstore2 (originAttributes, "
1062               "originKey, scope, key, value) "
1063               "VALUES (:originAttributes, :originKey, :scope, :key, :value) ");
1064       NS_ENSURE_STATE(stmt);
1065
1066       mozStorageStatementScoper scope(stmt);
1067
1068       rv = stmt->BindUTF8StringByName("originAttributes"_ns,
1069                                       mCache->OriginSuffix());
1070       NS_ENSURE_SUCCESS(rv, rv);
1071       rv = stmt->BindUTF8StringByName("originKey"_ns, mCache->OriginNoSuffix());
1072       NS_ENSURE_SUCCESS(rv, rv);
1073       // Filling the 'scope' column just for downgrade compatibility reasons
1074       rv = stmt->BindUTF8StringByName(
1075           "scope"_ns,
1076           Scheme0Scope(mCache->OriginSuffix(), mCache->OriginNoSuffix()));
1077       NS_ENSURE_SUCCESS(rv, rv);
1078       rv = stmt->BindStringByName("key"_ns, mKey);
1079       NS_ENSURE_SUCCESS(rv, rv);
1080       rv = stmt->BindStringByName("value"_ns, mValue);
1081       NS_ENSURE_SUCCESS(rv, rv);
1082
1083       rv = stmt->Execute();
1084       NS_ENSURE_SUCCESS(rv, rv);
1085
1086       MonitorAutoLock monitor(aThread->mThreadObserver->GetMonitor());
1087       aThread->mOriginsHavingData.Insert(Origin());
1088       break;
1089     }
1090
1091     case opRemoveItem: {
1092       MOZ_ASSERT(!NS_IsMainThread());
1093
1094       nsCOMPtr<mozIStorageStatement> stmt =
1095           aThread->mWorkerStatements.GetCachedStatement(
1096               "DELETE FROM webappsstore2 "
1097               "WHERE originAttributes = :originAttributes AND originKey = "
1098               ":originKey "
1099               "AND key = :key ");
1100       NS_ENSURE_STATE(stmt);
1101       mozStorageStatementScoper scope(stmt);
1102
1103       rv = stmt->BindUTF8StringByName("originAttributes"_ns,
1104                                       mCache->OriginSuffix());
1105       NS_ENSURE_SUCCESS(rv, rv);
1106       rv = stmt->BindUTF8StringByName("originKey"_ns, mCache->OriginNoSuffix());
1107       NS_ENSURE_SUCCESS(rv, rv);
1108       rv = stmt->BindStringByName("key"_ns, mKey);
1109       NS_ENSURE_SUCCESS(rv, rv);
1110
1111       rv = stmt->Execute();
1112       NS_ENSURE_SUCCESS(rv, rv);
1113
1114       break;
1115     }
1116
1117     case opClear: {
1118       MOZ_ASSERT(!NS_IsMainThread());
1119
1120       nsCOMPtr<mozIStorageStatement> stmt =
1121           aThread->mWorkerStatements.GetCachedStatement(
1122               "DELETE FROM webappsstore2 "
1123               "WHERE originAttributes = :originAttributes AND originKey = "
1124               ":originKey");
1125       NS_ENSURE_STATE(stmt);
1126       mozStorageStatementScoper scope(stmt);
1127
1128       rv = stmt->BindUTF8StringByName("originAttributes"_ns,
1129                                       mCache->OriginSuffix());
1130       NS_ENSURE_SUCCESS(rv, rv);
1131       rv = stmt->BindUTF8StringByName("originKey"_ns, mCache->OriginNoSuffix());
1132       NS_ENSURE_SUCCESS(rv, rv);
1133
1134       rv = stmt->Execute();
1135       NS_ENSURE_SUCCESS(rv, rv);
1136
1137       MonitorAutoLock monitor(aThread->mThreadObserver->GetMonitor());
1138       aThread->mOriginsHavingData.Remove(Origin());
1139       break;
1140     }
1141
1142     case opClearAll: {
1143       MOZ_ASSERT(!NS_IsMainThread());
1144
1145       nsCOMPtr<mozIStorageStatement> stmt =
1146           aThread->mWorkerStatements.GetCachedStatement(
1147               "DELETE FROM webappsstore2");
1148       NS_ENSURE_STATE(stmt);
1149       mozStorageStatementScoper scope(stmt);
1150
1151       rv = stmt->Execute();
1152       NS_ENSURE_SUCCESS(rv, rv);
1153
1154       MonitorAutoLock monitor(aThread->mThreadObserver->GetMonitor());
1155       aThread->mOriginsHavingData.Clear();
1156       break;
1157     }
1158
1159     case opClearMatchingOrigin: {
1160       MOZ_ASSERT(!NS_IsMainThread());
1161
1162       nsCOMPtr<mozIStorageStatement> stmt =
1163           aThread->mWorkerStatements.GetCachedStatement(
1164               "DELETE FROM webappsstore2"
1165               " WHERE originKey GLOB :scope");
1166       NS_ENSURE_STATE(stmt);
1167       mozStorageStatementScoper scope(stmt);
1168
1169       rv = stmt->BindUTF8StringByName("scope"_ns, mOrigin + "*"_ns);
1170       NS_ENSURE_SUCCESS(rv, rv);
1171
1172       rv = stmt->Execute();
1173       NS_ENSURE_SUCCESS(rv, rv);
1174
1175       // No need to selectively clear mOriginsHavingData here.  That hashtable
1176       // only prevents preload for scopes with no data.  Leaving a false record
1177       // in it has a negligible effect on performance.
1178       break;
1179     }
1180
1181     case opClearMatchingOriginAttributes: {
1182       MOZ_ASSERT(!NS_IsMainThread());
1183
1184       // Register the ORIGIN_ATTRS_PATTERN_MATCH function, initialized with the
1185       // pattern
1186       nsCOMPtr<mozIStorageFunction> patternMatchFunction(
1187           new OriginAttrsPatternMatchSQLFunction(mOriginPattern));
1188
1189       rv = aThread->mWorkerConnection->CreateFunction(
1190           "ORIGIN_ATTRS_PATTERN_MATCH"_ns, 1, patternMatchFunction);
1191       NS_ENSURE_SUCCESS(rv, rv);
1192
1193       nsCOMPtr<mozIStorageStatement> stmt =
1194           aThread->mWorkerStatements.GetCachedStatement(
1195               "DELETE FROM webappsstore2"
1196               " WHERE ORIGIN_ATTRS_PATTERN_MATCH(originAttributes)");
1197
1198       if (stmt) {
1199         mozStorageStatementScoper scope(stmt);
1200         rv = stmt->Execute();
1201       } else {
1202         rv = NS_ERROR_UNEXPECTED;
1203       }
1204
1205       // Always remove the function
1206       aThread->mWorkerConnection->RemoveFunction(
1207           "ORIGIN_ATTRS_PATTERN_MATCH"_ns);
1208
1209       NS_ENSURE_SUCCESS(rv, rv);
1210
1211       // No need to selectively clear mOriginsHavingData here.  That hashtable
1212       // only prevents preload for scopes with no data.  Leaving a false record
1213       // in it has a negligible effect on performance.
1214       break;
1215     }
1216
1217     default:
1218       NS_ERROR("Unknown task type");
1219       break;
1220   }
1221
1222   return NS_OK;
1223 }
1224
1225 void StorageDBThread::DBOperation::Finalize(nsresult aRv) {
1226   switch (mType) {
1227     case opPreloadUrgent:
1228     case opPreload:
1229       if (NS_FAILED(aRv)) {
1230         // When we are here, something failed when loading from the database.
1231         // Notify that the storage is loaded to prevent deadlock of the main
1232         // thread, even though it is actually empty or incomplete.
1233         NS_WARNING("Failed to preload localStorage");
1234       }
1235
1236       mCache->LoadDone(aRv);
1237       break;
1238
1239     case opGetUsage:
1240       if (NS_FAILED(aRv)) {
1241         mUsage->LoadUsage(0);
1242       }
1243
1244       break;
1245
1246     default:
1247       if (NS_FAILED(aRv)) {
1248         NS_WARNING(
1249             "localStorage update/clear operation failed,"
1250             " data may not persist or clean up");
1251       }
1252
1253       break;
1254   }
1255 }
1256
1257 // StorageDBThread::PendingOperations
1258
1259 StorageDBThread::PendingOperations::PendingOperations()
1260     : mFlushFailureCount(0) {}
1261
1262 bool StorageDBThread::PendingOperations::HasTasks() const {
1263   return !!mUpdates.Count() || !!mClears.Count();
1264 }
1265
1266 namespace {
1267
1268 bool OriginPatternMatches(const nsACString& aOriginSuffix,
1269                           const OriginAttributesPattern& aPattern) {
1270   OriginAttributes oa;
1271   DebugOnly<bool> rv = oa.PopulateFromSuffix(aOriginSuffix);
1272   MOZ_ASSERT(rv);
1273   return aPattern.Matches(oa);
1274 }
1275
1276 }  // namespace
1277
1278 bool StorageDBThread::PendingOperations::CheckForCoalesceOpportunity(
1279     DBOperation* aNewOp, DBOperation::OperationType aPendingType,
1280     DBOperation::OperationType aNewType) {
1281   if (aNewOp->Type() != aNewType) {
1282     return false;
1283   }
1284
1285   StorageDBThread::DBOperation* pendingTask;
1286   if (!mUpdates.Get(aNewOp->Target(), &pendingTask)) {
1287     return false;
1288   }
1289
1290   if (pendingTask->Type() != aPendingType) {
1291     return false;
1292   }
1293
1294   return true;
1295 }
1296
1297 void StorageDBThread::PendingOperations::Add(
1298     UniquePtr<StorageDBThread::DBOperation> aOperation) {
1299   // Optimize: when a key to remove has never been written to disk
1300   // just bypass this operation.  A key is new when an operation scheduled
1301   // to write it to the database is of type opAddItem.
1302   if (CheckForCoalesceOpportunity(aOperation.get(), DBOperation::opAddItem,
1303                                   DBOperation::opRemoveItem)) {
1304     mUpdates.Remove(aOperation->Target());
1305     return;
1306   }
1307
1308   // Optimize: when changing a key that is new and has never been
1309   // written to disk, keep type of the operation to store it at opAddItem.
1310   // This allows optimization to just forget adding a new key when
1311   // it is removed from the storage before flush.
1312   if (CheckForCoalesceOpportunity(aOperation.get(), DBOperation::opAddItem,
1313                                   DBOperation::opUpdateItem)) {
1314     aOperation->mType = DBOperation::opAddItem;
1315   }
1316
1317   // Optimize: to prevent lose of remove operation on a key when doing
1318   // remove/set/remove on a previously existing key we have to change
1319   // opAddItem to opUpdateItem on the new operation when there is opRemoveItem
1320   // pending for the key.
1321   if (CheckForCoalesceOpportunity(aOperation.get(), DBOperation::opRemoveItem,
1322                                   DBOperation::opAddItem)) {
1323     aOperation->mType = DBOperation::opUpdateItem;
1324   }
1325
1326   switch (aOperation->Type()) {
1327       // Operations on single keys
1328
1329     case DBOperation::opAddItem:
1330     case DBOperation::opUpdateItem:
1331     case DBOperation::opRemoveItem:
1332       // Override any existing operation for the target (=scope+key).
1333       mUpdates.InsertOrUpdate(aOperation->Target(), std::move(aOperation));
1334       break;
1335
1336       // Clear operations
1337
1338     case DBOperation::opClear:
1339     case DBOperation::opClearMatchingOrigin:
1340     case DBOperation::opClearMatchingOriginAttributes:
1341       // Drop all update (insert/remove) operations for equivavelent or matching
1342       // scope.  We do this as an optimization as well as a must based on the
1343       // logic, if we would not delete the update tasks, changes would have been
1344       // stored to the database after clear operations have been executed.
1345       for (auto iter = mUpdates.Iter(); !iter.Done(); iter.Next()) {
1346         const auto& pendingTask = iter.Data();
1347
1348         if (aOperation->Type() == DBOperation::opClear &&
1349             (pendingTask->OriginNoSuffix() != aOperation->OriginNoSuffix() ||
1350              pendingTask->OriginSuffix() != aOperation->OriginSuffix())) {
1351           continue;
1352         }
1353
1354         if (aOperation->Type() == DBOperation::opClearMatchingOrigin &&
1355             !StringBeginsWith(pendingTask->OriginNoSuffix(),
1356                               aOperation->Origin())) {
1357           continue;
1358         }
1359
1360         if (aOperation->Type() ==
1361                 DBOperation::opClearMatchingOriginAttributes &&
1362             !OriginPatternMatches(pendingTask->OriginSuffix(),
1363                                   aOperation->OriginPattern())) {
1364           continue;
1365         }
1366
1367         iter.Remove();
1368       }
1369
1370       mClears.InsertOrUpdate(aOperation->Target(), std::move(aOperation));
1371       break;
1372
1373     case DBOperation::opClearAll:
1374       // Drop simply everything, this is a super-operation.
1375       mUpdates.Clear();
1376       mClears.Clear();
1377       mClears.InsertOrUpdate(aOperation->Target(), std::move(aOperation));
1378       break;
1379
1380     default:
1381       MOZ_ASSERT(false);
1382       break;
1383   }
1384 }
1385
1386 bool StorageDBThread::PendingOperations::Prepare() {
1387   // Called under the lock
1388
1389   // First collect clear operations and then updates, we can
1390   // do this since whenever a clear operation for a scope is
1391   // scheduled, we drop all updates matching that scope. So,
1392   // all scope-related update operations we have here now were
1393   // scheduled after the clear operations.
1394   for (auto iter = mClears.Iter(); !iter.Done(); iter.Next()) {
1395     mExecList.AppendElement(std::move(iter.Data()));
1396   }
1397   mClears.Clear();
1398
1399   for (auto iter = mUpdates.Iter(); !iter.Done(); iter.Next()) {
1400     mExecList.AppendElement(std::move(iter.Data()));
1401   }
1402   mUpdates.Clear();
1403
1404   return !!mExecList.Length();
1405 }
1406
1407 nsresult StorageDBThread::PendingOperations::Execute(StorageDBThread* aThread) {
1408   // Called outside the lock
1409
1410   mozStorageTransaction transaction(aThread->mWorkerConnection, false);
1411
1412   nsresult rv = transaction.Start();
1413   if (NS_FAILED(rv)) {
1414     return rv;
1415   }
1416
1417   for (uint32_t i = 0; i < mExecList.Length(); ++i) {
1418     const auto& task = mExecList[i];
1419     rv = task->Perform(aThread);
1420     if (NS_FAILED(rv)) {
1421       return rv;
1422     }
1423   }
1424
1425   rv = transaction.Commit();
1426   if (NS_FAILED(rv)) {
1427     return rv;
1428   }
1429
1430   return NS_OK;
1431 }
1432
1433 bool StorageDBThread::PendingOperations::Finalize(nsresult aRv) {
1434   // Called under the lock
1435
1436   // The list is kept on a failure to retry it
1437   if (NS_FAILED(aRv)) {
1438     // XXX Followup: we may try to reopen the database and flush these
1439     // pending tasks, however testing showed that even though I/O is actually
1440     // broken some amount of operations is left in sqlite+system buffers and
1441     // seems like successfully flushed to disk.
1442     // Tested by removing a flash card and disconnecting from network while
1443     // using a network drive on Windows system.
1444     NS_WARNING("Flush operation on localStorage database failed");
1445
1446     ++mFlushFailureCount;
1447
1448     return mFlushFailureCount >= 5;
1449   }
1450
1451   mFlushFailureCount = 0;
1452   mExecList.Clear();
1453   return true;
1454 }
1455
1456 namespace {
1457
1458 bool FindPendingClearForOrigin(
1459     const nsACString& aOriginSuffix, const nsACString& aOriginNoSuffix,
1460     StorageDBThread::DBOperation* aPendingOperation) {
1461   if (aPendingOperation->Type() == StorageDBThread::DBOperation::opClearAll) {
1462     return true;
1463   }
1464
1465   if (aPendingOperation->Type() == StorageDBThread::DBOperation::opClear &&
1466       aOriginNoSuffix == aPendingOperation->OriginNoSuffix() &&
1467       aOriginSuffix == aPendingOperation->OriginSuffix()) {
1468     return true;
1469   }
1470
1471   if (aPendingOperation->Type() ==
1472           StorageDBThread::DBOperation::opClearMatchingOrigin &&
1473       StringBeginsWith(aOriginNoSuffix, aPendingOperation->Origin())) {
1474     return true;
1475   }
1476
1477   if (aPendingOperation->Type() ==
1478           StorageDBThread::DBOperation::opClearMatchingOriginAttributes &&
1479       OriginPatternMatches(aOriginSuffix, aPendingOperation->OriginPattern())) {
1480     return true;
1481   }
1482
1483   return false;
1484 }
1485
1486 }  // namespace
1487
1488 bool StorageDBThread::PendingOperations::IsOriginClearPending(
1489     const nsACString& aOriginSuffix, const nsACString& aOriginNoSuffix) const {
1490   // Called under the lock
1491
1492   for (const auto& clear : mClears.Values()) {
1493     if (FindPendingClearForOrigin(aOriginSuffix, aOriginNoSuffix,
1494                                   clear.get())) {
1495       return true;
1496     }
1497   }
1498
1499   for (uint32_t i = 0; i < mExecList.Length(); ++i) {
1500     if (FindPendingClearForOrigin(aOriginSuffix, aOriginNoSuffix,
1501                                   mExecList[i].get())) {
1502       return true;
1503     }
1504   }
1505
1506   return false;
1507 }
1508
1509 namespace {
1510
1511 bool FindPendingUpdateForOrigin(
1512     const nsACString& aOriginSuffix, const nsACString& aOriginNoSuffix,
1513     StorageDBThread::DBOperation* aPendingOperation) {
1514   if ((aPendingOperation->Type() == StorageDBThread::DBOperation::opAddItem ||
1515        aPendingOperation->Type() ==
1516            StorageDBThread::DBOperation::opUpdateItem ||
1517        aPendingOperation->Type() ==
1518            StorageDBThread::DBOperation::opRemoveItem) &&
1519       aOriginNoSuffix == aPendingOperation->OriginNoSuffix() &&
1520       aOriginSuffix == aPendingOperation->OriginSuffix()) {
1521     return true;
1522   }
1523
1524   return false;
1525 }
1526
1527 }  // namespace
1528
1529 bool StorageDBThread::PendingOperations::IsOriginUpdatePending(
1530     const nsACString& aOriginSuffix, const nsACString& aOriginNoSuffix) const {
1531   // Called under the lock
1532
1533   for (const auto& update : mUpdates.Values()) {
1534     if (FindPendingUpdateForOrigin(aOriginSuffix, aOriginNoSuffix,
1535                                    update.get())) {
1536       return true;
1537     }
1538   }
1539
1540   for (uint32_t i = 0; i < mExecList.Length(); ++i) {
1541     if (FindPendingUpdateForOrigin(aOriginSuffix, aOriginNoSuffix,
1542                                    mExecList[i].get())) {
1543       return true;
1544     }
1545   }
1546
1547   return false;
1548 }
1549
1550 nsresult StorageDBThread::InitHelper::SyncDispatchAndReturnProfilePath(
1551     nsAString& aProfilePath) {
1552   ::mozilla::ipc::AssertIsOnBackgroundThread();
1553
1554   MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(this));
1555
1556   mozilla::MutexAutoLock autolock(mMutex);
1557   while (mWaiting) {
1558     mCondVar.Wait();
1559   }
1560
1561   if (NS_WARN_IF(NS_FAILED(mMainThreadResultCode))) {
1562     return mMainThreadResultCode;
1563   }
1564
1565   aProfilePath = mProfilePath;
1566   return NS_OK;
1567 }
1568
1569 NS_IMETHODIMP
1570 StorageDBThread::InitHelper::Run() {
1571   MOZ_ASSERT(NS_IsMainThread());
1572
1573   nsresult rv = GetProfilePath(mProfilePath);
1574   if (NS_WARN_IF(NS_FAILED(rv))) {
1575     mMainThreadResultCode = rv;
1576   }
1577
1578   mozilla::MutexAutoLock lock(mMutex);
1579   MOZ_ASSERT(mWaiting);
1580
1581   mWaiting = false;
1582   mCondVar.Notify();
1583
1584   return NS_OK;
1585 }
1586
1587 NS_IMETHODIMP
1588 StorageDBThread::NoteBackgroundThreadRunnable::Run() {
1589   MOZ_ASSERT(NS_IsMainThread());
1590
1591   StorageObserver* observer = StorageObserver::Self();
1592   MOZ_ASSERT(observer);
1593
1594   observer->NoteBackgroundThread(mPrivateBrowsingId, mOwningThread);
1595
1596   return NS_OK;
1597 }
1598
1599 NS_IMETHODIMP
1600 StorageDBThread::ShutdownRunnable::Run() {
1601   if (NS_IsMainThread()) {
1602     mDone = true;
1603
1604     return NS_OK;
1605   }
1606
1607   ::mozilla::ipc::AssertIsOnBackgroundThread();
1608   MOZ_RELEASE_ASSERT(mPrivateBrowsingId < kPrivateBrowsingIdCount);
1609
1610   StorageDBThread*& storageThread = sStorageThread[mPrivateBrowsingId];
1611   if (storageThread) {
1612     sStorageThreadDown[mPrivateBrowsingId] = true;
1613
1614     storageThread->Shutdown();
1615
1616     delete storageThread;
1617     storageThread = nullptr;
1618   }
1619
1620   MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(this));
1621
1622   return NS_OK;
1623 }
1624
1625 }  // namespace mozilla::dom