dom/storage/StorageDBThread.cpp

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "StorageDBThread.h"
   8 #include "StorageDBUpdater.h"
   9 #include "StorageUtils.h"
  10 #include "LocalStorageCache.h"
  11 #include "LocalStorageManager.h"
  12
  13 #include "nsIEffectiveTLDService.h"
  14 #include "nsDirectoryServiceUtils.h"
  15 #include "nsAppDirectoryServiceDefs.h"
  16 #include "nsThreadUtils.h"
  17 #include "nsProxyRelease.h"
  18 #include "mozStorageCID.h"
  19 #include "mozStorageHelper.h"
  20 #include "mozIStorageService.h"
  21 #include "mozIStorageBindingParamsArray.h"
  22 #include "mozIStorageBindingParams.h"
  23 #include "mozIStorageValueArray.h"
  24 #include "mozIStorageFunction.h"
  25 #include "mozilla/BasePrincipal.h"
  26 #include "mozilla/ipc/BackgroundParent.h"
  27 #include "nsIObserverService.h"
  28 #include "nsThread.h"
  29 #include "nsThreadManager.h"
  30 #include "nsVariant.h"
  31 #include "mozilla/EventQueue.h"
  32 #include "mozilla/IOInterposer.h"
  33 #include "mozilla/ThreadEventQueue.h"
  34 #include "mozilla/Services.h"
  35 #include "mozilla/Tokenizer.h"
  36 #include "GeckoProfiler.h"
  37
  38 // How long we collect write oprerations
  39 // before they are flushed to the database
  40 // In milliseconds.
  41 #define FLUSHING_INTERVAL_MS 5000
  42
  43 // Write Ahead Log's maximum size is 512KB
  44 #define MAX_WAL_SIZE_BYTES 512 * 1024
  45
  46 // Current version of the database schema
  47 #define CURRENT_SCHEMA_VERSION 2
  48
  49 namespace mozilla {
  50 namespace dom {
  51
  52 using namespace StorageUtils;
  53
  54 namespace {  // anon
  55
  56 StorageDBThread* sStorageThread = nullptr;
  57
  58 // False until we shut the storage thread down.
  59 bool sStorageThreadDown = false;
  60
  61 }  // namespace
  62
  63 // XXX Fix me!
  64 #if 0
  65 StorageDBBridge::StorageDBBridge()
  66 {
  67 }
  68 #endif
  69
  70 class StorageDBThread::InitHelper final : public Runnable {
  71   nsCOMPtr<nsIEventTarget> mOwningThread;
  72   mozilla::Mutex mMutex;
  73   mozilla::CondVar mCondVar;
  74   nsString mProfilePath;
  75   nsresult mMainThreadResultCode;
  76   bool mWaiting;
  77
  78  public:
  79   InitHelper()
  80       : Runnable("dom::StorageDBThread::InitHelper"),
  81         mOwningThread(GetCurrentThreadEventTarget()),
  82         mMutex("InitHelper::mMutex"),
  83         mCondVar(mMutex, "InitHelper::mCondVar"),
  84         mMainThreadResultCode(NS_OK),
  85         mWaiting(true) {}
  86
  87   // Because of the `sync Preload` IPC, we need to be able to synchronously
  88   // initialize, which includes consulting and initializing
  89   // some main-thread-only APIs. Bug 1386441 discusses improving this situation.
  90   nsresult SyncDispatchAndReturnProfilePath(nsAString& aProfilePath);
  91
  92  private:
  93   ~InitHelper() override = default;
  94
  95   nsresult RunOnMainThread();
  96
  97   NS_DECL_NSIRUNNABLE
  98 };
  99
 100 class StorageDBThread::NoteBackgroundThreadRunnable final : public Runnable {
 101   nsCOMPtr<nsIEventTarget> mOwningThread;
 102
 103  public:
 104   NoteBackgroundThreadRunnable()
 105       : Runnable("dom::StorageDBThread::NoteBackgroundThreadRunnable"),
 106         mOwningThread(GetCurrentThreadEventTarget()) {}
 107
 108  private:
 109   ~NoteBackgroundThreadRunnable() override = default;
 110
 111   NS_DECL_NSIRUNNABLE
 112 };
 113
 114 StorageDBThread::StorageDBThread()
 115     : mThread(nullptr),
 116       mThreadObserver(new ThreadObserver()),
 117       mStopIOThread(false),
 118       mWALModeEnabled(false),
 119       mDBReady(false),
 120       mStatus(NS_OK),
 121       mWorkerStatements(mWorkerConnection),
 122       mReaderStatements(mReaderConnection),
 123       mFlushImmediately(false),
 124       mPriorityCounter(0) {}
 125
 126 // static
 127 StorageDBThread* StorageDBThread::Get() {
 128   AssertIsOnBackgroundThread();
 129
 130   return sStorageThread;
 131 }
 132
 133 // static
 134 StorageDBThread* StorageDBThread::GetOrCreate(const nsString& aProfilePath) {
 135   AssertIsOnBackgroundThread();
 136
 137   if (sStorageThread || sStorageThreadDown) {
 138     // When sStorageThreadDown is at true, sStorageThread is null.
 139     // Checking sStorageThreadDown flag here prevents reinitialization of
 140     // the storage thread after shutdown.
 141     return sStorageThread;
 142   }
 143
 144   nsAutoPtr<StorageDBThread> storageThread(new StorageDBThread());
 145
 146   nsresult rv = storageThread->Init(aProfilePath);
 147   if (NS_WARN_IF(NS_FAILED(rv))) {
 148     return nullptr;
 149   }
 150
 151   sStorageThread = storageThread.forget();
 152
 153   return sStorageThread;
 154 }
 155
 156 // static
 157 nsresult StorageDBThread::GetProfilePath(nsString& aProfilePath) {
 158   MOZ_ASSERT(XRE_IsParentProcess());
 159   MOZ_ASSERT(NS_IsMainThread());
 160
 161   // Need to determine location on the main thread, since
 162   // NS_GetSpecialDirectory accesses the atom table that can
 163   // only be accessed on the main thread.
 164   nsCOMPtr<nsIFile> profileDir;
 165   nsresult rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR,
 166                                        getter_AddRefs(profileDir));
 167   if (NS_WARN_IF(NS_FAILED(rv))) {
 168     return rv;
 169   }
 170
 171   rv = profileDir->GetPath(aProfilePath);
 172   if (NS_WARN_IF(NS_FAILED(rv))) {
 173     return rv;
 174   }
 175
 176   // This service has to be started on the main thread currently.
 177   nsCOMPtr<mozIStorageService> ss =
 178       do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID, &rv);
 179   if (NS_WARN_IF(NS_FAILED(rv))) {
 180     return rv;
 181   }
 182
 183   return NS_OK;
 184 }
 185
 186 nsresult StorageDBThread::Init(const nsString& aProfilePath) {
 187   AssertIsOnBackgroundThread();
 188
 189   nsresult rv;
 190
 191   nsString profilePath;
 192   if (aProfilePath.IsEmpty()) {
 193     RefPtr<InitHelper> helper = new InitHelper();
 194
 195     rv = helper->SyncDispatchAndReturnProfilePath(profilePath);
 196     if (NS_WARN_IF(NS_FAILED(rv))) {
 197       return rv;
 198     }
 199   } else {
 200     profilePath = aProfilePath;
 201   }
 202
 203   mDatabaseFile = do_CreateInstance(NS_LOCAL_FILE_CONTRACTID, &rv);
 204   if (NS_WARN_IF(NS_FAILED(rv))) {
 205     return rv;
 206   }
 207
 208   rv = mDatabaseFile->InitWithPath(profilePath);
 209   if (NS_WARN_IF(NS_FAILED(rv))) {
 210     return rv;
 211   }
 212
 213   rv = mDatabaseFile->Append(NS_LITERAL_STRING("webappsstore.sqlite"));
 214   NS_ENSURE_SUCCESS(rv, rv);
 215
 216   // Need to keep the lock to avoid setting mThread later then
 217   // the thread body executes.
 218   MonitorAutoLock monitor(mThreadObserver->GetMonitor());
 219
 220   mThread = PR_CreateThread(PR_USER_THREAD, &StorageDBThread::ThreadFunc, this,
 221                             PR_PRIORITY_LOW, PR_GLOBAL_THREAD,
 222                             PR_JOINABLE_THREAD, 262144);
 223   if (!mThread) {
 224     return NS_ERROR_OUT_OF_MEMORY;
 225   }
 226
 227   RefPtr<NoteBackgroundThreadRunnable> runnable =
 228       new NoteBackgroundThreadRunnable();
 229   MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(runnable));
 230
 231   return NS_OK;
 232 }
 233
 234 nsresult StorageDBThread::Shutdown() {
 235   AssertIsOnBackgroundThread();
 236
 237   sStorageThreadDown = true;
 238
 239   if (!mThread) {
 240     return NS_ERROR_NOT_INITIALIZED;
 241   }
 242
 243   Telemetry::AutoTimer<Telemetry::LOCALDOMSTORAGE_SHUTDOWN_DATABASE_MS> timer;
 244
 245   {
 246     MonitorAutoLock monitor(mThreadObserver->GetMonitor());
 247
 248     // After we stop, no other operations can be accepted
 249     mFlushImmediately = true;
 250     mStopIOThread = true;
 251     monitor.Notify();
 252   }
 253
 254   PR_JoinThread(mThread);
 255   mThread = nullptr;
 256
 257   return mStatus;
 258 }
 259
 260 void StorageDBThread::SyncPreload(LocalStorageCacheBridge* aCache,
 261                                   bool aForceSync) {
 262   AUTO_PROFILER_LABEL("StorageDBThread::SyncPreload", OTHER);
 263   if (!aForceSync && aCache->LoadedCount()) {
 264     // Preload already started for this cache, just wait for it to finish.
 265     // LoadWait will exit after LoadDone on the cache has been called.
 266     SetHigherPriority();
 267     aCache->LoadWait();
 268     SetDefaultPriority();
 269     return;
 270   }
 271
 272   // Bypass sync load when an update is pending in the queue to write, we would
 273   // get incosistent data in the cache.  Also don't allow sync main-thread
 274   // preload when DB open and init is still pending on the background thread.
 275   if (mDBReady && mWALModeEnabled) {
 276     bool pendingTasks;
 277     {
 278       MonitorAutoLock monitor(mThreadObserver->GetMonitor());
 279       pendingTasks = mPendingTasks.IsOriginUpdatePending(
 280                          aCache->OriginSuffix(), aCache->OriginNoSuffix()) ||
 281                      mPendingTasks.IsOriginClearPending(
 282                          aCache->OriginSuffix(), aCache->OriginNoSuffix());
 283     }
 284
 285     if (!pendingTasks) {
 286       // WAL is enabled, thus do the load synchronously on the main thread.
 287       DBOperation preload(DBOperation::opPreload, aCache);
 288       preload.PerformAndFinalize(this);
 289       return;
 290     }
 291   }
 292
 293   // Need to go asynchronously since WAL is not allowed or scheduled updates
 294   // need to be flushed first.
 295   // Schedule preload for this cache as the first operation.
 296   nsresult rv =
 297       InsertDBOp(new DBOperation(DBOperation::opPreloadUrgent, aCache));
 298
 299   // LoadWait exits after LoadDone of the cache has been called.
 300   if (NS_SUCCEEDED(rv)) {
 301     aCache->LoadWait();
 302   }
 303 }
 304
 305 void StorageDBThread::AsyncFlush() {
 306   MonitorAutoLock monitor(mThreadObserver->GetMonitor());
 307   mFlushImmediately = true;
 308   monitor.Notify();
 309 }
 310
 311 bool StorageDBThread::ShouldPreloadOrigin(const nsACString& aOrigin) {
 312   MonitorAutoLock monitor(mThreadObserver->GetMonitor());
 313   return mOriginsHavingData.Contains(aOrigin);
 314 }
 315
 316 void StorageDBThread::GetOriginsHavingData(
 317     InfallibleTArray<nsCString>* aOrigins) {
 318   MonitorAutoLock monitor(mThreadObserver->GetMonitor());
 319   for (auto iter = mOriginsHavingData.Iter(); !iter.Done(); iter.Next()) {
 320     aOrigins->AppendElement(iter.Get()->GetKey());
 321   }
 322 }
 323
 324 nsresult StorageDBThread::InsertDBOp(StorageDBThread::DBOperation* aOperation) {
 325   MonitorAutoLock monitor(mThreadObserver->GetMonitor());
 326
 327   // Sentinel to don't forget to delete the operation when we exit early.
 328   nsAutoPtr<StorageDBThread::DBOperation> opScope(aOperation);
 329
 330   if (NS_FAILED(mStatus)) {
 331     MonitorAutoUnlock unlock(mThreadObserver->GetMonitor());
 332     aOperation->Finalize(mStatus);
 333     return mStatus;
 334   }
 335
 336   if (mStopIOThread) {
 337     // Thread use after shutdown demanded.
 338     MOZ_ASSERT(false);
 339     return NS_ERROR_NOT_INITIALIZED;
 340   }
 341
 342   switch (aOperation->Type()) {
 343     case DBOperation::opPreload:
 344     case DBOperation::opPreloadUrgent:
 345       if (mPendingTasks.IsOriginUpdatePending(aOperation->OriginSuffix(),
 346                                               aOperation->OriginNoSuffix())) {
 347         // If there is a pending update operation for the scope first do the
 348         // flush before we preload the cache.  This may happen in an extremely
 349         // rare case when a child process throws away its cache before flush on
 350         // the parent has finished.  If we would preloaded the cache as a
 351         // priority operation before the pending flush, we would have got an
 352         // inconsistent cache content.
 353         mFlushImmediately = true;
 354       } else if (mPendingTasks.IsOriginClearPending(
 355                      aOperation->OriginSuffix(),
 356                      aOperation->OriginNoSuffix())) {
 357         // The scope is scheduled to be cleared, so just quickly load as empty.
 358         // We need to do this to prevent load of the DB data before the scope
 359         // has actually been cleared from the database.  Preloads are processed
 360         // immediately before update and clear operations on the database that
 361         // are flushed periodically in batches.
 362         MonitorAutoUnlock unlock(mThreadObserver->GetMonitor());
 363         aOperation->Finalize(NS_OK);
 364         return NS_OK;
 365       }
 366       MOZ_FALLTHROUGH;
 367
 368     case DBOperation::opGetUsage:
 369       if (aOperation->Type() == DBOperation::opPreloadUrgent) {
 370         SetHigherPriority();  // Dropped back after urgent preload execution
 371         mPreloads.InsertElementAt(0, aOperation);
 372       } else {
 373         mPreloads.AppendElement(aOperation);
 374       }
 375
 376       // DB operation adopted, don't delete it.
 377       opScope.forget();
 378
 379       // Immediately start executing this.
 380       monitor.Notify();
 381       break;
 382
 383     default:
 384       // Update operations are first collected, coalesced and then flushed
 385       // after a short time.
 386       mPendingTasks.Add(aOperation);
 387
 388       // DB operation adopted, don't delete it.
 389       opScope.forget();
 390
 391       ScheduleFlush();
 392       break;
 393   }
 394
 395   return NS_OK;
 396 }
 397
 398 void StorageDBThread::SetHigherPriority() {
 399   ++mPriorityCounter;
 400   PR_SetThreadPriority(mThread, PR_PRIORITY_URGENT);
 401 }
 402
 403 void StorageDBThread::SetDefaultPriority() {
 404   if (--mPriorityCounter <= 0) {
 405     PR_SetThreadPriority(mThread, PR_PRIORITY_LOW);
 406   }
 407 }
 408
 409 void StorageDBThread::ThreadFunc(void* aArg) {
 410   {
 411     auto queue =
 412         MakeRefPtr<ThreadEventQueue<EventQueue>>(MakeUnique<EventQueue>());
 413     Unused << nsThreadManager::get().CreateCurrentThread(
 414         queue, nsThread::NOT_MAIN_THREAD);
 415   }
 416
 417   AUTO_PROFILER_REGISTER_THREAD("localStorage DB");
 418   NS_SetCurrentThreadName("localStorage DB");
 419   mozilla::IOInterposer::RegisterCurrentThread();
 420
 421   StorageDBThread* thread = static_cast<StorageDBThread*>(aArg);
 422   thread->ThreadFunc();
 423   mozilla::IOInterposer::UnregisterCurrentThread();
 424 }
 425
 426 void StorageDBThread::ThreadFunc() {
 427   nsresult rv = InitDatabase();
 428
 429   MonitorAutoLock lockMonitor(mThreadObserver->GetMonitor());
 430
 431   if (NS_FAILED(rv)) {
 432     mStatus = rv;
 433     mStopIOThread = true;
 434     return;
 435   }
 436
 437   // Create an nsIThread for the current PRThread, so we can observe runnables
 438   // dispatched to it.
 439   nsCOMPtr<nsIThread> thread = NS_GetCurrentThread();
 440   nsCOMPtr<nsIThreadInternal> threadInternal = do_QueryInterface(thread);
 441   MOZ_ASSERT(threadInternal);  // Should always succeed.
 442   threadInternal->SetObserver(mThreadObserver);
 443
 444   while (MOZ_LIKELY(!mStopIOThread || mPreloads.Length() ||
 445                     mPendingTasks.HasTasks() ||
 446                     mThreadObserver->HasPendingEvents())) {
 447     // Process xpcom events first.
 448     while (MOZ_UNLIKELY(mThreadObserver->HasPendingEvents())) {
 449       mThreadObserver->ClearPendingEvents();
 450       MonitorAutoUnlock unlock(mThreadObserver->GetMonitor());
 451       bool processedEvent;
 452       do {
 453         rv = thread->ProcessNextEvent(false, &processedEvent);
 454       } while (NS_SUCCEEDED(rv) && processedEvent);
 455     }
 456
 457     TimeDuration timeUntilFlush = TimeUntilFlush();
 458     if (MOZ_UNLIKELY(timeUntilFlush.IsZero())) {
 459       // Flush time is up or flush has been forced, do it now.
 460       UnscheduleFlush();
 461       if (mPendingTasks.Prepare()) {
 462         {
 463           MonitorAutoUnlock unlockMonitor(mThreadObserver->GetMonitor());
 464           rv = mPendingTasks.Execute(this);
 465         }
 466
 467         if (!mPendingTasks.Finalize(rv)) {
 468           mStatus = rv;
 469           NS_WARNING("localStorage DB access broken");
 470         }
 471       }
 472       NotifyFlushCompletion();
 473     } else if (MOZ_LIKELY(mPreloads.Length())) {
 474       nsAutoPtr<DBOperation> op(mPreloads[0]);
 475       mPreloads.RemoveElementAt(0);
 476       {
 477         MonitorAutoUnlock unlockMonitor(mThreadObserver->GetMonitor());
 478         op->PerformAndFinalize(this);
 479       }
 480
 481       if (op->Type() == DBOperation::opPreloadUrgent) {
 482         SetDefaultPriority();  // urgent preload unscheduled
 483       }
 484     } else if (MOZ_UNLIKELY(!mStopIOThread)) {
 485       AUTO_PROFILER_LABEL("StorageDBThread::ThreadFunc::Wait", IDLE);
 486       AUTO_PROFILER_THREAD_SLEEP;
 487       lockMonitor.Wait(timeUntilFlush);
 488     }
 489   }  // thread loop
 490
 491   mStatus = ShutdownDatabase();
 492
 493   if (threadInternal) {
 494     threadInternal->SetObserver(nullptr);
 495   }
 496 }
 497
 498 NS_IMPL_ISUPPORTS(StorageDBThread::ThreadObserver, nsIThreadObserver)
 499
 500 NS_IMETHODIMP
 501 StorageDBThread::ThreadObserver::OnDispatchedEvent() {
 502   MonitorAutoLock lock(mMonitor);
 503   mHasPendingEvents = true;
 504   lock.Notify();
 505   return NS_OK;
 506 }
 507
 508 NS_IMETHODIMP
 509 StorageDBThread::ThreadObserver::OnProcessNextEvent(nsIThreadInternal* aThread,
 510                                                     bool mayWait) {
 511   return NS_OK;
 512 }
 513
 514 NS_IMETHODIMP
 515 StorageDBThread::ThreadObserver::AfterProcessNextEvent(
 516     nsIThreadInternal* aThread, bool eventWasProcessed) {
 517   return NS_OK;
 518 }
 519
 520 nsresult StorageDBThread::OpenDatabaseConnection() {
 521   nsresult rv;
 522
 523   MOZ_ASSERT(!NS_IsMainThread());
 524
 525   nsCOMPtr<mozIStorageService> service =
 526       do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID, &rv);
 527   NS_ENSURE_SUCCESS(rv, rv);
 528
 529   rv = service->OpenUnsharedDatabase(mDatabaseFile,
 530                                      getter_AddRefs(mWorkerConnection));
 531   if (rv == NS_ERROR_FILE_CORRUPTED) {
 532     // delete the db and try opening again
 533     rv = mDatabaseFile->Remove(false);
 534     NS_ENSURE_SUCCESS(rv, rv);
 535     rv = service->OpenUnsharedDatabase(mDatabaseFile,
 536                                        getter_AddRefs(mWorkerConnection));
 537   }
 538   NS_ENSURE_SUCCESS(rv, rv);
 539
 540   return NS_OK;
 541 }
 542
 543 nsresult StorageDBThread::OpenAndUpdateDatabase() {
 544   nsresult rv;
 545
 546   // Here we are on the worker thread. This opens the worker connection.
 547   MOZ_ASSERT(!NS_IsMainThread());
 548
 549   rv = OpenDatabaseConnection();
 550   NS_ENSURE_SUCCESS(rv, rv);
 551
 552   rv = TryJournalMode();
 553   NS_ENSURE_SUCCESS(rv, rv);
 554
 555   return NS_OK;
 556 }
 557
 558 nsresult StorageDBThread::InitDatabase() {
 559   nsresult rv;
 560
 561   // Here we are on the worker thread. This opens the worker connection.
 562   MOZ_ASSERT(!NS_IsMainThread());
 563
 564   rv = OpenAndUpdateDatabase();
 565   NS_ENSURE_SUCCESS(rv, rv);
 566
 567   rv = StorageDBUpdater::Update(mWorkerConnection);
 568   if (NS_FAILED(rv)) {
 569     // Update has failed, rather throw the database away and try
 570     // opening and setting it up again.
 571     rv = mWorkerConnection->Close();
 572     mWorkerConnection = nullptr;
 573     NS_ENSURE_SUCCESS(rv, rv);
 574
 575     rv = mDatabaseFile->Remove(false);
 576     NS_ENSURE_SUCCESS(rv, rv);
 577
 578     rv = OpenAndUpdateDatabase();
 579     NS_ENSURE_SUCCESS(rv, rv);
 580   }
 581
 582   // Create a read-only clone
 583   (void)mWorkerConnection->Clone(true, getter_AddRefs(mReaderConnection));
 584   NS_ENSURE_TRUE(mReaderConnection, NS_ERROR_FAILURE);
 585
 586   // Database open and all initiation operation are done.  Switching this flag
 587   // to true allow main thread to read directly from the database.  If we would
 588   // allow this sooner, we would have opened a window where main thread read
 589   // might operate on a totally broken and incosistent database.
 590   mDBReady = true;
 591
 592   // List scopes having any stored data
 593   nsCOMPtr<mozIStorageStatement> stmt;
 594   // Note: result of this select must match StorageManager::CreateOrigin()
 595   rv = mWorkerConnection->CreateStatement(
 596       NS_LITERAL_CSTRING("SELECT DISTINCT originAttributes || ':' || originKey "
 597                          "FROM webappsstore2"),
 598       getter_AddRefs(stmt));
 599   NS_ENSURE_SUCCESS(rv, rv);
 600   mozStorageStatementScoper scope(stmt);
 601
 602   bool exists;
 603   while (NS_SUCCEEDED(rv = stmt->ExecuteStep(&exists)) && exists) {
 604     nsAutoCString foundOrigin;
 605     rv = stmt->GetUTF8String(0, foundOrigin);
 606     NS_ENSURE_SUCCESS(rv, rv);
 607
 608     MonitorAutoLock monitor(mThreadObserver->GetMonitor());
 609     mOriginsHavingData.PutEntry(foundOrigin);
 610   }
 611
 612   return NS_OK;
 613 }
 614
 615 nsresult StorageDBThread::SetJournalMode(bool aIsWal) {
 616   nsresult rv;
 617
 618   nsAutoCString stmtString(MOZ_STORAGE_UNIQUIFY_QUERY_STR
 619                            "PRAGMA journal_mode = ");
 620   if (aIsWal) {
 621     stmtString.AppendLiteral("wal");
 622   } else {
 623     stmtString.AppendLiteral("truncate");
 624   }
 625
 626   nsCOMPtr<mozIStorageStatement> stmt;
 627   rv = mWorkerConnection->CreateStatement(stmtString, getter_AddRefs(stmt));
 628   NS_ENSURE_SUCCESS(rv, rv);
 629   mozStorageStatementScoper scope(stmt);
 630
 631   bool hasResult = false;
 632   rv = stmt->ExecuteStep(&hasResult);
 633   NS_ENSURE_SUCCESS(rv, rv);
 634   if (!hasResult) {
 635     return NS_ERROR_FAILURE;
 636   }
 637
 638   nsAutoCString journalMode;
 639   rv = stmt->GetUTF8String(0, journalMode);
 640   NS_ENSURE_SUCCESS(rv, rv);
 641   if ((aIsWal && !journalMode.EqualsLiteral("wal")) ||
 642       (!aIsWal && !journalMode.EqualsLiteral("truncate"))) {
 643     return NS_ERROR_FAILURE;
 644   }
 645
 646   return NS_OK;
 647 }
 648
 649 nsresult StorageDBThread::TryJournalMode() {
 650   nsresult rv;
 651
 652   rv = SetJournalMode(true);
 653   if (NS_FAILED(rv)) {
 654     mWALModeEnabled = false;
 655
 656     rv = SetJournalMode(false);
 657     NS_ENSURE_SUCCESS(rv, rv);
 658   } else {
 659     mWALModeEnabled = true;
 660
 661     rv = ConfigureWALBehavior();
 662     NS_ENSURE_SUCCESS(rv, rv);
 663   }
 664
 665   return NS_OK;
 666 }
 667
 668 nsresult StorageDBThread::ConfigureWALBehavior() {
 669   // Get the DB's page size
 670   nsCOMPtr<mozIStorageStatement> stmt;
 671   nsresult rv = mWorkerConnection->CreateStatement(
 672       NS_LITERAL_CSTRING(MOZ_STORAGE_UNIQUIFY_QUERY_STR "PRAGMA page_size"),
 673       getter_AddRefs(stmt));
 674   NS_ENSURE_SUCCESS(rv, rv);
 675
 676   bool hasResult = false;
 677   rv = stmt->ExecuteStep(&hasResult);
 678   NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && hasResult, NS_ERROR_FAILURE);
 679
 680   int32_t pageSize = 0;
 681   rv = stmt->GetInt32(0, &pageSize);
 682   NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && pageSize > 0, NS_ERROR_UNEXPECTED);
 683
 684   // Set the threshold for auto-checkpointing the WAL.
 685   // We don't want giant logs slowing down reads & shutdown.
 686   int32_t thresholdInPages =
 687       static_cast<int32_t>(MAX_WAL_SIZE_BYTES / pageSize);
 688   nsAutoCString thresholdPragma("PRAGMA wal_autocheckpoint = ");
 689   thresholdPragma.AppendInt(thresholdInPages);
 690   rv = mWorkerConnection->ExecuteSimpleSQL(thresholdPragma);
 691   NS_ENSURE_SUCCESS(rv, rv);
 692
 693   // Set the maximum WAL log size to reduce footprint on mobile (large empty
 694   // WAL files will be truncated)
 695   nsAutoCString journalSizePragma("PRAGMA journal_size_limit = ");
 696   // bug 600307: mak recommends setting this to 3 times the auto-checkpoint
 697   // threshold
 698   journalSizePragma.AppendInt(MAX_WAL_SIZE_BYTES * 3);
 699   rv = mWorkerConnection->ExecuteSimpleSQL(journalSizePragma);
 700   NS_ENSURE_SUCCESS(rv, rv);
 701
 702   return NS_OK;
 703 }
 704
 705 nsresult StorageDBThread::ShutdownDatabase() {
 706   // Has to be called on the worker thread.
 707   MOZ_ASSERT(!NS_IsMainThread());
 708
 709   nsresult rv = mStatus;
 710
 711   mDBReady = false;
 712
 713   // Finalize the cached statements.
 714   mReaderStatements.FinalizeStatements();
 715   mWorkerStatements.FinalizeStatements();
 716
 717   if (mReaderConnection) {
 718     // No need to sync access to mReaderConnection since the main thread
 719     // is right now joining this thread, unable to execute any events.
 720     mReaderConnection->Close();
 721     mReaderConnection = nullptr;
 722   }
 723
 724   if (mWorkerConnection) {
 725     rv = mWorkerConnection->Close();
 726     mWorkerConnection = nullptr;
 727   }
 728
 729   return rv;
 730 }
 731
 732 void StorageDBThread::ScheduleFlush() {
 733   if (mDirtyEpoch) {
 734     return;  // Already scheduled
 735   }
 736
 737   // Must be non-zero to indicate we are scheduled
 738   mDirtyEpoch = TimeStamp::Now();
 739
 740   // Wake the monitor from indefinite sleep...
 741   (mThreadObserver->GetMonitor()).Notify();
 742 }
 743
 744 void StorageDBThread::UnscheduleFlush() {
 745   // We are just about to do the flush, drop flags
 746   mFlushImmediately = false;
 747   mDirtyEpoch = TimeStamp();
 748 }
 749
 750 TimeDuration StorageDBThread::TimeUntilFlush() {
 751   if (mFlushImmediately) {
 752     return 0;  // Do it now regardless the timeout.
 753   }
 754
 755   if (!mDirtyEpoch) {
 756     return TimeDuration::Forever();  // No pending task...
 757   }
 758
 759   TimeStamp now = TimeStamp::Now();
 760   TimeDuration age = now - mDirtyEpoch;
 761   static const TimeDuration kMaxAge =
 762       TimeDuration::FromMilliseconds(FLUSHING_INTERVAL_MS);
 763   if (age > kMaxAge) {
 764     return 0;  // It is time.
 765   }
 766
 767   return kMaxAge - age;  // Time left. This is used to sleep the monitor.
 768 }
 769
 770 void StorageDBThread::NotifyFlushCompletion() {
 771 #ifdef DOM_STORAGE_TESTS
 772   if (!NS_IsMainThread()) {
 773     RefPtr<nsRunnableMethod<StorageDBThread, void, false>> event =
 774         NewNonOwningRunnableMethod(
 775             "dom::StorageDBThread::NotifyFlushCompletion", this,
 776             &StorageDBThread::NotifyFlushCompletion);
 777     NS_DispatchToMainThread(event);
 778     return;
 779   }
 780
 781   nsCOMPtr<nsIObserverService> obs = mozilla::services::GetObserverService();
 782   if (obs) {
 783     obs->NotifyObservers(nullptr, "domstorage-test-flushed", nullptr);
 784   }
 785 #endif
 786 }
 787
 788 // Helper SQL function classes
 789
 790 namespace {
 791
 792 class OriginAttrsPatternMatchSQLFunction final : public mozIStorageFunction {
 793   NS_DECL_ISUPPORTS
 794   NS_DECL_MOZISTORAGEFUNCTION
 795
 796   explicit OriginAttrsPatternMatchSQLFunction(
 797       OriginAttributesPattern const& aPattern)
 798       : mPattern(aPattern) {}
 799
 800  private:
 801   OriginAttrsPatternMatchSQLFunction() = delete;
 802   ~OriginAttrsPatternMatchSQLFunction() {}
 803
 804   OriginAttributesPattern mPattern;
 805 };
 806
 807 NS_IMPL_ISUPPORTS(OriginAttrsPatternMatchSQLFunction, mozIStorageFunction)
 808
 809 NS_IMETHODIMP
 810 OriginAttrsPatternMatchSQLFunction::OnFunctionCall(
 811     mozIStorageValueArray* aFunctionArguments, nsIVariant** aResult) {
 812   nsresult rv;
 813
 814   nsAutoCString suffix;
 815   rv = aFunctionArguments->GetUTF8String(0, suffix);
 816   NS_ENSURE_SUCCESS(rv, rv);
 817
 818   OriginAttributes oa;
 819   bool success = oa.PopulateFromSuffix(suffix);
 820   NS_ENSURE_TRUE(success, NS_ERROR_FAILURE);
 821   bool result = mPattern.Matches(oa);
 822
 823   RefPtr<nsVariant> outVar(new nsVariant());
 824   rv = outVar->SetAsBool(result);
 825   NS_ENSURE_SUCCESS(rv, rv);
 826
 827   outVar.forget(aResult);
 828   return NS_OK;
 829 }
 830
 831 }  // namespace
 832
 833 // StorageDBThread::DBOperation
 834
 835 StorageDBThread::DBOperation::DBOperation(const OperationType aType,
 836                                           LocalStorageCacheBridge* aCache,
 837                                           const nsAString& aKey,
 838                                           const nsAString& aValue)
 839     : mType(aType), mCache(aCache), mKey(aKey), mValue(aValue) {
 840   MOZ_ASSERT(mType == opPreload || mType == opPreloadUrgent ||
 841              mType == opAddItem || mType == opUpdateItem ||
 842              mType == opRemoveItem || mType == opClear || mType == opClearAll);
 843   MOZ_COUNT_CTOR(StorageDBThread::DBOperation);
 844 }
 845
 846 StorageDBThread::DBOperation::DBOperation(const OperationType aType,
 847                                           StorageUsageBridge* aUsage)
 848     : mType(aType), mUsage(aUsage) {
 849   MOZ_ASSERT(mType == opGetUsage);
 850   MOZ_COUNT_CTOR(StorageDBThread::DBOperation);
 851 }
 852
 853 StorageDBThread::DBOperation::DBOperation(const OperationType aType,
 854                                           const nsACString& aOriginNoSuffix)
 855     : mType(aType), mCache(nullptr), mOrigin(aOriginNoSuffix) {
 856   MOZ_ASSERT(mType == opClearMatchingOrigin);
 857   MOZ_COUNT_CTOR(StorageDBThread::DBOperation);
 858 }
 859
 860 StorageDBThread::DBOperation::DBOperation(
 861     const OperationType aType, const OriginAttributesPattern& aOriginNoSuffix)
 862     : mType(aType), mCache(nullptr), mOriginPattern(aOriginNoSuffix) {
 863   MOZ_ASSERT(mType == opClearMatchingOriginAttributes);
 864   MOZ_COUNT_CTOR(StorageDBThread::DBOperation);
 865 }
 866
 867 StorageDBThread::DBOperation::~DBOperation() {
 868   MOZ_COUNT_DTOR(StorageDBThread::DBOperation);
 869 }
 870
 871 const nsCString StorageDBThread::DBOperation::OriginNoSuffix() const {
 872   if (mCache) {
 873     return mCache->OriginNoSuffix();
 874   }
 875
 876   return EmptyCString();
 877 }
 878
 879 const nsCString StorageDBThread::DBOperation::OriginSuffix() const {
 880   if (mCache) {
 881     return mCache->OriginSuffix();
 882   }
 883
 884   return EmptyCString();
 885 }
 886
 887 const nsCString StorageDBThread::DBOperation::Origin() const {
 888   if (mCache) {
 889     return mCache->Origin();
 890   }
 891
 892   return mOrigin;
 893 }
 894
 895 const nsCString StorageDBThread::DBOperation::Target() const {
 896   switch (mType) {
 897     case opAddItem:
 898     case opUpdateItem:
 899     case opRemoveItem:
 900       return Origin() + NS_LITERAL_CSTRING("|") + NS_ConvertUTF16toUTF8(mKey);
 901
 902     default:
 903       return Origin();
 904   }
 905 }
 906
 907 void StorageDBThread::DBOperation::PerformAndFinalize(
 908     StorageDBThread* aThread) {
 909   Finalize(Perform(aThread));
 910 }
 911
 912 nsresult StorageDBThread::DBOperation::Perform(StorageDBThread* aThread) {
 913   nsresult rv;
 914
 915   switch (mType) {
 916     case opPreload:
 917     case opPreloadUrgent: {
 918       // Already loaded?
 919       if (mCache->Loaded()) {
 920         break;
 921       }
 922
 923       StatementCache* statements;
 924       if (MOZ_UNLIKELY(IsOnBackgroundThread())) {
 925         statements = &aThread->mReaderStatements;
 926       } else {
 927         statements = &aThread->mWorkerStatements;
 928       }
 929
 930       // OFFSET is an optimization when we have to do a sync load
 931       // and cache has already loaded some parts asynchronously.
 932       // It skips keys we have already loaded.
 933       nsCOMPtr<mozIStorageStatement> stmt = statements->GetCachedStatement(
 934           "SELECT key, value FROM webappsstore2 "
 935           "WHERE originAttributes = :originAttributes AND originKey = "
 936           ":originKey "
 937           "ORDER BY key LIMIT -1 OFFSET :offset");
 938       NS_ENSURE_STATE(stmt);
 939       mozStorageStatementScoper scope(stmt);
 940
 941       rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("originAttributes"),
 942                                       mCache->OriginSuffix());
 943       NS_ENSURE_SUCCESS(rv, rv);
 944
 945       rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("originKey"),
 946                                       mCache->OriginNoSuffix());
 947       NS_ENSURE_SUCCESS(rv, rv);
 948
 949       rv = stmt->BindInt32ByName(NS_LITERAL_CSTRING("offset"),
 950                                  static_cast<int32_t>(mCache->LoadedCount()));
 951       NS_ENSURE_SUCCESS(rv, rv);
 952
 953       bool exists;
 954       while (NS_SUCCEEDED(rv = stmt->ExecuteStep(&exists)) && exists) {
 955         nsAutoString key;
 956         rv = stmt->GetString(0, key);
 957         NS_ENSURE_SUCCESS(rv, rv);
 958
 959         nsAutoString value;
 960         rv = stmt->GetString(1, value);
 961         NS_ENSURE_SUCCESS(rv, rv);
 962
 963         if (!mCache->LoadItem(key, value)) {
 964           break;
 965         }
 966       }
 967       // The loop condition's call to ExecuteStep() may have terminated because
 968       // !NS_SUCCEEDED(), we need an early return to cover that case.  This also
 969       // covers success cases as well, but that's inductively safe.
 970       NS_ENSURE_SUCCESS(rv, rv);
 971       break;
 972     }
 973
 974     case opGetUsage: {
 975       nsCOMPtr<mozIStorageStatement> stmt =
 976           aThread->mWorkerStatements.GetCachedStatement(
 977               "SELECT SUM(LENGTH(key) + LENGTH(value)) FROM webappsstore2 "
 978               "WHERE (originAttributes || ':' || originKey) LIKE :usageOrigin");
 979       NS_ENSURE_STATE(stmt);
 980
 981       mozStorageStatementScoper scope(stmt);
 982
 983       rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("usageOrigin"),
 984                                       mUsage->OriginScope());
 985       NS_ENSURE_SUCCESS(rv, rv);
 986
 987       bool exists;
 988       rv = stmt->ExecuteStep(&exists);
 989       NS_ENSURE_SUCCESS(rv, rv);
 990
 991       int64_t usage = 0;
 992       if (exists) {
 993         rv = stmt->GetInt64(0, &usage);
 994         NS_ENSURE_SUCCESS(rv, rv);
 995       }
 996
 997       mUsage->LoadUsage(usage);
 998       break;
 999     }
1000
1001     case opAddItem:
1002     case opUpdateItem: {
1003       MOZ_ASSERT(!NS_IsMainThread());
1004
1005       nsCOMPtr<mozIStorageStatement> stmt =
1006           aThread->mWorkerStatements.GetCachedStatement(
1007               "INSERT OR REPLACE INTO webappsstore2 (originAttributes, "
1008               "originKey, scope, key, value) "
1009               "VALUES (:originAttributes, :originKey, :scope, :key, :value) ");
1010       NS_ENSURE_STATE(stmt);
1011
1012       mozStorageStatementScoper scope(stmt);
1013
1014       rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("originAttributes"),
1015                                       mCache->OriginSuffix());
1016       NS_ENSURE_SUCCESS(rv, rv);
1017       rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("originKey"),
1018                                       mCache->OriginNoSuffix());
1019       NS_ENSURE_SUCCESS(rv, rv);
1020       // Filling the 'scope' column just for downgrade compatibility reasons
1021       rv = stmt->BindUTF8StringByName(
1022           NS_LITERAL_CSTRING("scope"),
1023           Scheme0Scope(mCache->OriginSuffix(), mCache->OriginNoSuffix()));
1024       NS_ENSURE_SUCCESS(rv, rv);
1025       rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"), mKey);
1026       NS_ENSURE_SUCCESS(rv, rv);
1027       rv = stmt->BindStringByName(NS_LITERAL_CSTRING("value"), mValue);
1028       NS_ENSURE_SUCCESS(rv, rv);
1029
1030       rv = stmt->Execute();
1031       NS_ENSURE_SUCCESS(rv, rv);
1032
1033       MonitorAutoLock monitor(aThread->mThreadObserver->GetMonitor());
1034       aThread->mOriginsHavingData.PutEntry(Origin());
1035       break;
1036     }
1037
1038     case opRemoveItem: {
1039       MOZ_ASSERT(!NS_IsMainThread());
1040
1041       nsCOMPtr<mozIStorageStatement> stmt =
1042           aThread->mWorkerStatements.GetCachedStatement(
1043               "DELETE FROM webappsstore2 "
1044               "WHERE originAttributes = :originAttributes AND originKey = "
1045               ":originKey "
1046               "AND key = :key ");
1047       NS_ENSURE_STATE(stmt);
1048       mozStorageStatementScoper scope(stmt);
1049
1050       rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("originAttributes"),
1051                                       mCache->OriginSuffix());
1052       NS_ENSURE_SUCCESS(rv, rv);
1053       rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("originKey"),
1054                                       mCache->OriginNoSuffix());
1055       NS_ENSURE_SUCCESS(rv, rv);
1056       rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"), mKey);
1057       NS_ENSURE_SUCCESS(rv, rv);
1058
1059       rv = stmt->Execute();
1060       NS_ENSURE_SUCCESS(rv, rv);
1061
1062       break;
1063     }
1064
1065     case opClear: {
1066       MOZ_ASSERT(!NS_IsMainThread());
1067
1068       nsCOMPtr<mozIStorageStatement> stmt =
1069           aThread->mWorkerStatements.GetCachedStatement(
1070               "DELETE FROM webappsstore2 "
1071               "WHERE originAttributes = :originAttributes AND originKey = "
1072               ":originKey");
1073       NS_ENSURE_STATE(stmt);
1074       mozStorageStatementScoper scope(stmt);
1075
1076       rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("originAttributes"),
1077                                       mCache->OriginSuffix());
1078       NS_ENSURE_SUCCESS(rv, rv);
1079       rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("originKey"),
1080                                       mCache->OriginNoSuffix());
1081       NS_ENSURE_SUCCESS(rv, rv);
1082
1083       rv = stmt->Execute();
1084       NS_ENSURE_SUCCESS(rv, rv);
1085
1086       MonitorAutoLock monitor(aThread->mThreadObserver->GetMonitor());
1087       aThread->mOriginsHavingData.RemoveEntry(Origin());
1088       break;
1089     }
1090
1091     case opClearAll: {
1092       MOZ_ASSERT(!NS_IsMainThread());
1093
1094       nsCOMPtr<mozIStorageStatement> stmt =
1095           aThread->mWorkerStatements.GetCachedStatement(
1096               "DELETE FROM webappsstore2");
1097       NS_ENSURE_STATE(stmt);
1098       mozStorageStatementScoper scope(stmt);
1099
1100       rv = stmt->Execute();
1101       NS_ENSURE_SUCCESS(rv, rv);
1102
1103       MonitorAutoLock monitor(aThread->mThreadObserver->GetMonitor());
1104       aThread->mOriginsHavingData.Clear();
1105       break;
1106     }
1107
1108     case opClearMatchingOrigin: {
1109       MOZ_ASSERT(!NS_IsMainThread());
1110
1111       nsCOMPtr<mozIStorageStatement> stmt =
1112           aThread->mWorkerStatements.GetCachedStatement(
1113               "DELETE FROM webappsstore2"
1114               " WHERE originKey GLOB :scope");
1115       NS_ENSURE_STATE(stmt);
1116       mozStorageStatementScoper scope(stmt);
1117
1118       rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("scope"),
1119                                       mOrigin + NS_LITERAL_CSTRING("*"));
1120       NS_ENSURE_SUCCESS(rv, rv);
1121
1122       rv = stmt->Execute();
1123       NS_ENSURE_SUCCESS(rv, rv);
1124
1125       // No need to selectively clear mOriginsHavingData here.  That hashtable
1126       // only prevents preload for scopes with no data.  Leaving a false record
1127       // in it has a negligible effect on performance.
1128       break;
1129     }
1130
1131     case opClearMatchingOriginAttributes: {
1132       MOZ_ASSERT(!NS_IsMainThread());
1133
1134       // Register the ORIGIN_ATTRS_PATTERN_MATCH function, initialized with the
1135       // pattern
1136       nsCOMPtr<mozIStorageFunction> patternMatchFunction(
1137           new OriginAttrsPatternMatchSQLFunction(mOriginPattern));
1138
1139       rv = aThread->mWorkerConnection->CreateFunction(
1140           NS_LITERAL_CSTRING("ORIGIN_ATTRS_PATTERN_MATCH"), 1,
1141           patternMatchFunction);
1142       NS_ENSURE_SUCCESS(rv, rv);
1143
1144       nsCOMPtr<mozIStorageStatement> stmt =
1145           aThread->mWorkerStatements.GetCachedStatement(
1146               "DELETE FROM webappsstore2"
1147               " WHERE ORIGIN_ATTRS_PATTERN_MATCH(originAttributes)");
1148
1149       if (stmt) {
1150         mozStorageStatementScoper scope(stmt);
1151         rv = stmt->Execute();
1152       } else {
1153         rv = NS_ERROR_UNEXPECTED;
1154       }
1155
1156       // Always remove the function
1157       aThread->mWorkerConnection->RemoveFunction(
1158           NS_LITERAL_CSTRING("ORIGIN_ATTRS_PATTERN_MATCH"));
1159
1160       NS_ENSURE_SUCCESS(rv, rv);
1161
1162       // No need to selectively clear mOriginsHavingData here.  That hashtable
1163       // only prevents preload for scopes with no data.  Leaving a false record
1164       // in it has a negligible effect on performance.
1165       break;
1166     }
1167
1168     default:
1169       NS_ERROR("Unknown task type");
1170       break;
1171   }
1172
1173   return NS_OK;
1174 }
1175
1176 void StorageDBThread::DBOperation::Finalize(nsresult aRv) {
1177   switch (mType) {
1178     case opPreloadUrgent:
1179     case opPreload:
1180       if (NS_FAILED(aRv)) {
1181         // When we are here, something failed when loading from the database.
1182         // Notify that the storage is loaded to prevent deadlock of the main
1183         // thread, even though it is actually empty or incomplete.
1184         NS_WARNING("Failed to preload localStorage");
1185       }
1186
1187       mCache->LoadDone(aRv);
1188       break;
1189
1190     case opGetUsage:
1191       if (NS_FAILED(aRv)) {
1192         mUsage->LoadUsage(0);
1193       }
1194
1195       break;
1196
1197     default:
1198       if (NS_FAILED(aRv)) {
1199         NS_WARNING(
1200             "localStorage update/clear operation failed,"
1201             " data may not persist or clean up");
1202       }
1203
1204       break;
1205   }
1206 }
1207
1208 // StorageDBThread::PendingOperations
1209
1210 StorageDBThread::PendingOperations::PendingOperations()
1211     : mFlushFailureCount(0) {}
1212
1213 bool StorageDBThread::PendingOperations::HasTasks() const {
1214   return !!mUpdates.Count() || !!mClears.Count();
1215 }
1216
1217 namespace {
1218
1219 bool OriginPatternMatches(const nsACString& aOriginSuffix,
1220                           const OriginAttributesPattern& aPattern) {
1221   OriginAttributes oa;
1222   DebugOnly<bool> rv = oa.PopulateFromSuffix(aOriginSuffix);
1223   MOZ_ASSERT(rv);
1224   return aPattern.Matches(oa);
1225 }
1226
1227 }  // namespace
1228
1229 bool StorageDBThread::PendingOperations::CheckForCoalesceOpportunity(
1230     DBOperation* aNewOp, DBOperation::OperationType aPendingType,
1231     DBOperation::OperationType aNewType) {
1232   if (aNewOp->Type() != aNewType) {
1233     return false;
1234   }
1235
1236   StorageDBThread::DBOperation* pendingTask;
1237   if (!mUpdates.Get(aNewOp->Target(), &pendingTask)) {
1238     return false;
1239   }
1240
1241   if (pendingTask->Type() != aPendingType) {
1242     return false;
1243   }
1244
1245   return true;
1246 }
1247
1248 void StorageDBThread::PendingOperations::Add(
1249     StorageDBThread::DBOperation* aOperation) {
1250   // Optimize: when a key to remove has never been written to disk
1251   // just bypass this operation.  A key is new when an operation scheduled
1252   // to write it to the database is of type opAddItem.
1253   if (CheckForCoalesceOpportunity(aOperation, DBOperation::opAddItem,
1254                                   DBOperation::opRemoveItem)) {
1255     mUpdates.Remove(aOperation->Target());
1256     delete aOperation;
1257     return;
1258   }
1259
1260   // Optimize: when changing a key that is new and has never been
1261   // written to disk, keep type of the operation to store it at opAddItem.
1262   // This allows optimization to just forget adding a new key when
1263   // it is removed from the storage before flush.
1264   if (CheckForCoalesceOpportunity(aOperation, DBOperation::opAddItem,
1265                                   DBOperation::opUpdateItem)) {
1266     aOperation->mType = DBOperation::opAddItem;
1267   }
1268
1269   // Optimize: to prevent lose of remove operation on a key when doing
1270   // remove/set/remove on a previously existing key we have to change
1271   // opAddItem to opUpdateItem on the new operation when there is opRemoveItem
1272   // pending for the key.
1273   if (CheckForCoalesceOpportunity(aOperation, DBOperation::opRemoveItem,
1274                                   DBOperation::opAddItem)) {
1275     aOperation->mType = DBOperation::opUpdateItem;
1276   }
1277
1278   switch (aOperation->Type()) {
1279       // Operations on single keys
1280
1281     case DBOperation::opAddItem:
1282     case DBOperation::opUpdateItem:
1283     case DBOperation::opRemoveItem:
1284       // Override any existing operation for the target (=scope+key).
1285       mUpdates.Put(aOperation->Target(), aOperation);
1286       break;
1287
1288       // Clear operations
1289
1290     case DBOperation::opClear:
1291     case DBOperation::opClearMatchingOrigin:
1292     case DBOperation::opClearMatchingOriginAttributes:
1293       // Drop all update (insert/remove) operations for equivavelent or matching
1294       // scope.  We do this as an optimization as well as a must based on the
1295       // logic, if we would not delete the update tasks, changes would have been
1296       // stored to the database after clear operations have been executed.
1297       for (auto iter = mUpdates.Iter(); !iter.Done(); iter.Next()) {
1298         nsAutoPtr<DBOperation>& pendingTask = iter.Data();
1299
1300         if (aOperation->Type() == DBOperation::opClear &&
1301             (pendingTask->OriginNoSuffix() != aOperation->OriginNoSuffix() ||
1302              pendingTask->OriginSuffix() != aOperation->OriginSuffix())) {
1303           continue;
1304         }
1305
1306         if (aOperation->Type() == DBOperation::opClearMatchingOrigin &&
1307             !StringBeginsWith(pendingTask->OriginNoSuffix(),
1308                               aOperation->Origin())) {
1309           continue;
1310         }
1311
1312         if (aOperation->Type() ==
1313                 DBOperation::opClearMatchingOriginAttributes &&
1314             !OriginPatternMatches(pendingTask->OriginSuffix(),
1315                                   aOperation->OriginPattern())) {
1316           continue;
1317         }
1318
1319         iter.Remove();
1320       }
1321
1322       mClears.Put(aOperation->Target(), aOperation);
1323       break;
1324
1325     case DBOperation::opClearAll:
1326       // Drop simply everything, this is a super-operation.
1327       mUpdates.Clear();
1328       mClears.Clear();
1329       mClears.Put(aOperation->Target(), aOperation);
1330       break;
1331
1332     default:
1333       MOZ_ASSERT(false);
1334       break;
1335   }
1336 }
1337
1338 bool StorageDBThread::PendingOperations::Prepare() {
1339   // Called under the lock
1340
1341   // First collect clear operations and then updates, we can
1342   // do this since whenever a clear operation for a scope is
1343   // scheduled, we drop all updates matching that scope. So,
1344   // all scope-related update operations we have here now were
1345   // scheduled after the clear operations.
1346   for (auto iter = mClears.Iter(); !iter.Done(); iter.Next()) {
1347     mExecList.AppendElement(iter.Data().forget());
1348   }
1349   mClears.Clear();
1350
1351   for (auto iter = mUpdates.Iter(); !iter.Done(); iter.Next()) {
1352     mExecList.AppendElement(iter.Data().forget());
1353   }
1354   mUpdates.Clear();
1355
1356   return !!mExecList.Length();
1357 }
1358
1359 nsresult StorageDBThread::PendingOperations::Execute(StorageDBThread* aThread) {
1360   // Called outside the lock
1361
1362   mozStorageTransaction transaction(aThread->mWorkerConnection, false);
1363
1364   nsresult rv;
1365
1366   for (uint32_t i = 0; i < mExecList.Length(); ++i) {
1367     StorageDBThread::DBOperation* task = mExecList[i];
1368     rv = task->Perform(aThread);
1369     if (NS_FAILED(rv)) {
1370       return rv;
1371     }
1372   }
1373
1374   rv = transaction.Commit();
1375   if (NS_FAILED(rv)) {
1376     return rv;
1377   }
1378
1379   return NS_OK;
1380 }
1381
1382 bool StorageDBThread::PendingOperations::Finalize(nsresult aRv) {
1383   // Called under the lock
1384
1385   // The list is kept on a failure to retry it
1386   if (NS_FAILED(aRv)) {
1387     // XXX Followup: we may try to reopen the database and flush these
1388     // pending tasks, however testing showed that even though I/O is actually
1389     // broken some amount of operations is left in sqlite+system buffers and
1390     // seems like successfully flushed to disk.
1391     // Tested by removing a flash card and disconnecting from network while
1392     // using a network drive on Windows system.
1393     NS_WARNING("Flush operation on localStorage database failed");
1394
1395     ++mFlushFailureCount;
1396
1397     return mFlushFailureCount >= 5;
1398   }
1399
1400   mFlushFailureCount = 0;
1401   mExecList.Clear();
1402   return true;
1403 }
1404
1405 namespace {
1406
1407 bool FindPendingClearForOrigin(
1408     const nsACString& aOriginSuffix, const nsACString& aOriginNoSuffix,
1409     StorageDBThread::DBOperation* aPendingOperation) {
1410   if (aPendingOperation->Type() == StorageDBThread::DBOperation::opClearAll) {
1411     return true;
1412   }
1413
1414   if (aPendingOperation->Type() == StorageDBThread::DBOperation::opClear &&
1415       aOriginNoSuffix == aPendingOperation->OriginNoSuffix() &&
1416       aOriginSuffix == aPendingOperation->OriginSuffix()) {
1417     return true;
1418   }
1419
1420   if (aPendingOperation->Type() ==
1421           StorageDBThread::DBOperation::opClearMatchingOrigin &&
1422       StringBeginsWith(aOriginNoSuffix, aPendingOperation->Origin())) {
1423     return true;
1424   }
1425
1426   if (aPendingOperation->Type() ==
1427           StorageDBThread::DBOperation::opClearMatchingOriginAttributes &&
1428       OriginPatternMatches(aOriginSuffix, aPendingOperation->OriginPattern())) {
1429     return true;
1430   }
1431
1432   return false;
1433 }
1434
1435 }  // namespace
1436
1437 bool StorageDBThread::PendingOperations::IsOriginClearPending(
1438     const nsACString& aOriginSuffix, const nsACString& aOriginNoSuffix) const {
1439   // Called under the lock
1440
1441   for (auto iter = mClears.ConstIter(); !iter.Done(); iter.Next()) {
1442     if (FindPendingClearForOrigin(aOriginSuffix, aOriginNoSuffix,
1443                                   iter.UserData())) {
1444       return true;
1445     }
1446   }
1447
1448   for (uint32_t i = 0; i < mExecList.Length(); ++i) {
1449     if (FindPendingClearForOrigin(aOriginSuffix, aOriginNoSuffix,
1450                                   mExecList[i])) {
1451       return true;
1452     }
1453   }
1454
1455   return false;
1456 }
1457
1458 namespace {
1459
1460 bool FindPendingUpdateForOrigin(
1461     const nsACString& aOriginSuffix, const nsACString& aOriginNoSuffix,
1462     StorageDBThread::DBOperation* aPendingOperation) {
1463   if ((aPendingOperation->Type() == StorageDBThread::DBOperation::opAddItem ||
1464        aPendingOperation->Type() ==
1465            StorageDBThread::DBOperation::opUpdateItem ||
1466        aPendingOperation->Type() ==
1467            StorageDBThread::DBOperation::opRemoveItem) &&
1468       aOriginNoSuffix == aPendingOperation->OriginNoSuffix() &&
1469       aOriginSuffix == aPendingOperation->OriginSuffix()) {
1470     return true;
1471   }
1472
1473   return false;
1474 }
1475
1476 }  // namespace
1477
1478 bool StorageDBThread::PendingOperations::IsOriginUpdatePending(
1479     const nsACString& aOriginSuffix, const nsACString& aOriginNoSuffix) const {
1480   // Called under the lock
1481
1482   for (auto iter = mUpdates.ConstIter(); !iter.Done(); iter.Next()) {
1483     if (FindPendingUpdateForOrigin(aOriginSuffix, aOriginNoSuffix,
1484                                    iter.UserData())) {
1485       return true;
1486     }
1487   }
1488
1489   for (uint32_t i = 0; i < mExecList.Length(); ++i) {
1490     if (FindPendingUpdateForOrigin(aOriginSuffix, aOriginNoSuffix,
1491                                    mExecList[i])) {
1492       return true;
1493     }
1494   }
1495
1496   return false;
1497 }
1498
1499 nsresult StorageDBThread::InitHelper::SyncDispatchAndReturnProfilePath(
1500     nsAString& aProfilePath) {
1501   AssertIsOnBackgroundThread();
1502
1503   MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(this));
1504
1505   mozilla::MutexAutoLock autolock(mMutex);
1506   while (mWaiting) {
1507     mCondVar.Wait();
1508   }
1509
1510   if (NS_WARN_IF(NS_FAILED(mMainThreadResultCode))) {
1511     return mMainThreadResultCode;
1512   }
1513
1514   aProfilePath = mProfilePath;
1515   return NS_OK;
1516 }
1517
1518 NS_IMETHODIMP
1519 StorageDBThread::InitHelper::Run() {
1520   MOZ_ASSERT(NS_IsMainThread());
1521
1522   nsresult rv = GetProfilePath(mProfilePath);
1523   if (NS_WARN_IF(NS_FAILED(rv))) {
1524     mMainThreadResultCode = rv;
1525   }
1526
1527   mozilla::MutexAutoLock lock(mMutex);
1528   MOZ_ASSERT(mWaiting);
1529
1530   mWaiting = false;
1531   mCondVar.Notify();
1532
1533   return NS_OK;
1534 }
1535
1536 NS_IMETHODIMP
1537 StorageDBThread::NoteBackgroundThreadRunnable::Run() {
1538   MOZ_ASSERT(NS_IsMainThread());
1539
1540   StorageObserver* observer = StorageObserver::Self();
1541   MOZ_ASSERT(observer);
1542
1543   observer->NoteBackgroundThread(mOwningThread);
1544
1545   return NS_OK;
1546 }
1547
1548 NS_IMETHODIMP
1549 StorageDBThread::ShutdownRunnable::Run() {
1550   if (NS_IsMainThread()) {
1551     mDone = true;
1552
1553     return NS_OK;
1554   }
1555
1556   AssertIsOnBackgroundThread();
1557
1558   if (sStorageThread) {
1559     sStorageThread->Shutdown();
1560
1561     delete sStorageThread;
1562     sStorageThread = nullptr;
1563   }
1564
1565   MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(this));
1566
1567   return NS_OK;
1568 }
1569
1570 }  // namespace dom
1571 }  // namespace mozilla