1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 var EXPORTED_SYMBOLS = ["Sqlite"];
9 // The maximum time to wait before considering a transaction stuck and rejecting
10 // it. (Note that the minimum amount of time we wait is 20% less than this, see
11 // the `_getTimeoutPromise` method on `ConnectionData` for details).
12 const TRANSACTIONS_QUEUE_TIMEOUT_MS = 300000; // 5 minutes
14 const { XPCOMUtils } = ChromeUtils.import(
15 "resource://gre/modules/XPCOMUtils.jsm"
17 const { setTimeout } = ChromeUtils.import("resource://gre/modules/Timer.jsm");
19 XPCOMUtils.defineLazyModuleGetters(this, {
20 AsyncShutdown: "resource://gre/modules/AsyncShutdown.jsm",
21 Services: "resource://gre/modules/Services.jsm",
22 OS: "resource://gre/modules/osfile.jsm",
23 Log: "resource://gre/modules/Log.jsm",
24 FileUtils: "resource://gre/modules/FileUtils.jsm",
25 PromiseUtils: "resource://gre/modules/PromiseUtils.jsm",
28 XPCOMUtils.defineLazyServiceGetter(
30 "FinalizationWitnessService",
31 "@mozilla.org/toolkit/finalizationwitness;1",
32 "nsIFinalizationWitnessService"
35 // Regular expression used by isInvalidBoundLikeQuery
36 var likeSqlRegex = /\bLIKE\b\s(?![@:?])/i;
38 // Counts the number of created connections per database basename(). This is
39 // used for logging to distinguish connection instances.
40 var connectionCounters = new Map();
42 // Tracks identifiers of wrapped connections, that are Storage connections
43 // opened through mozStorage and then wrapped by Sqlite.jsm to use its syntactic
44 // sugar API. Since these connections have an unknown origin, we use this set
45 // to differentiate their behavior.
46 var wrappedConnections = new Set();
49 * Once `true`, reject any attempt to open or close a database.
54 // Tests should fail if a connection auto closes. The exception is
55 // when finalization itself is tested, in which case this flag
56 // should be set to false.
57 failTestsOnAutoClose: true,
61 * Helper function to check whether LIKE is implemented using proper bindings.
64 * (string) The SQL query to be verified.
65 * @return boolean value telling us whether query was correct or not
67 function isInvalidBoundLikeQuery(sql) {
68 return likeSqlRegex.test(sql);
71 // Displays a script error message
72 function logScriptError(message) {
73 let consoleMessage = Cc["@mozilla.org/scripterror;1"].createInstance(
76 let stack = new Error();
83 Ci.nsIScriptError.errorFlag,
84 "component javascript"
86 Services.console.logMessage(consoleMessage);
88 // This `Promise.reject` will cause tests to fail. The debugging
89 // flag can be used to suppress this for tests that explicitly
91 if (Debugging.failTestsOnAutoClose) {
92 Promise.reject(new Error(message));
97 * Gets connection identifier from its database file name.
100 * A database file string name.
101 * @return the connection identifier.
103 function getIdentifierByFileName(fileName) {
104 let number = connectionCounters.get(fileName) || 0;
105 connectionCounters.set(fileName, number + 1);
106 return fileName + "#" + number;
110 * Barriers used to ensure that Sqlite.jsm is shutdown after all
113 XPCOMUtils.defineLazyGetter(this, "Barriers", () => {
116 * Public barrier that clients may use to add blockers to the
117 * shutdown of Sqlite.jsm. Triggered by profile-before-change.
118 * Once all blockers of this barrier are lifted, we close the
119 * ability to open new connections.
121 shutdown: new AsyncShutdown.Barrier(
122 "Sqlite.jsm: wait until all clients have completed their task"
126 * Private barrier blocked by connections that are still open.
127 * Triggered after Barriers.shutdown is lifted and `isClosed` is
130 connections: new AsyncShutdown.Barrier(
131 "Sqlite.jsm: wait until all connections are closed"
136 * Observer for the event which is broadcasted when the finalization
137 * witness `_witness` of `OpenedConnection` is garbage collected.
139 * The observer is passed the connection identifier of the database
140 * connection that is being finalized.
142 let finalizationObserver = function(subject, topic, identifier) {
143 let connectionData = ConnectionData.byId.get(identifier);
145 if (connectionData === undefined) {
147 "Error: Attempt to finalize unknown Sqlite connection: " +
154 ConnectionData.byId.delete(identifier);
156 "Warning: Sqlite connection '" +
158 "' was not properly closed. Auto-close triggered by garbage collection.\n"
160 connectionData.close();
162 Services.obs.addObserver(finalizationObserver, "sqlite-finalization-witness");
165 * Ensure that Sqlite.jsm:
166 * - informs its clients before shutting down;
167 * - lets clients open connections during shutdown, if necessary;
168 * - waits for all connections to be closed before shutdown.
170 AsyncShutdown.profileBeforeChange.addBlocker(
171 "Sqlite.jsm shutdown blocker",
173 await Barriers.shutdown.wait();
174 // At this stage, all clients have had a chance to open (and close)
175 // their databases. Some previous close operations may still be pending,
176 // so we need to wait until they are complete before proceeding.
178 // Prevent any new opening.
181 // Now, wait until all databases are closed
182 await Barriers.connections.wait();
184 // Everything closed, no finalization events to catch
185 Services.obs.removeObserver(
186 finalizationObserver,
187 "sqlite-finalization-witness"
193 // We are waiting for the connections to close. The interesting
194 // status is therefore the list of connections still pending.
196 description: "Waiting for connections to close",
197 state: Barriers.connections.state,
201 // We are still in the first stage: waiting for the barrier
202 // to be lifted. The interesting status is therefore that of
205 description: "Waiting for the barrier to be lifted",
206 state: Barriers.shutdown.state,
215 * Connection data with methods necessary for closing the connection.
217 * To support auto-closing in the event of garbage collection, this
218 * data structure contains all the connection data of an opened
219 * connection and all of the methods needed for sucessfully closing
222 * By putting this information in its own separate object, it is
223 * possible to store an additional reference to it without preventing
224 * a garbage collection of a finalization witness in
225 * OpenedConnection. When the witness detects a garbage collection,
226 * this object can be used to close the connection.
228 * This object contains more methods than just `close`. When
229 * OpenedConnection needs to use the methods in this object, it will
230 * dispatch its method calls here.
232 function ConnectionData(connection, identifier, options = {}) {
233 this._log = Log.repository.getLoggerWithMessagePrefix(
237 this._log.info("Opened");
239 this._dbConn = connection;
241 // This is a unique identifier for the connection, generated through
242 // getIdentifierByFileName. It may be used for logging or as a key in Maps.
243 this._identifier = identifier;
247 this._cachedStatements = new Map();
248 this._anonymousStatements = new Map();
249 this._anonymousCounter = 0;
251 // A map from statement index to mozIStoragePendingStatement, to allow for
252 // canceling prior to finalizing the mozIStorageStatements.
253 this._pendingStatements = new Map();
255 // Increments for each executed statement for the life of the connection.
256 this._statementCounter = 0;
258 // Increments whenever we request a unique operation id.
259 this._operationsCounter = 0;
261 if ("defaultTransactionType" in options) {
262 this.defaultTransactionType = options.defaultTransactionType;
264 this.defaultTransactionType = convertStorageTransactionType(
265 this._dbConn.defaultTransactionType
268 this._hasInProgressTransaction = false;
269 // Manages a chain of transactions promises, so that new transactions
270 // always happen in queue to the previous ones. It never rejects.
271 this._transactionQueue = Promise.resolve();
273 this._idleShrinkMS = options.shrinkMemoryOnConnectionIdleMS;
274 if (this._idleShrinkMS) {
275 this._idleShrinkTimer = Cc["@mozilla.org/timer;1"].createInstance(
278 // We wait for the first statement execute to start the timer because
279 // shrinking now would not do anything.
282 // Deferred whose promise is resolved when the connection closing procedure
284 this._deferredClose = PromiseUtils.defer();
285 this._closeRequested = false;
287 // An AsyncShutdown barrier used to make sure that we wait until clients
288 // are done before shutting down the connection.
289 this._barrier = new AsyncShutdown.Barrier(
290 `${this._identifier}: waiting for clients`
293 Barriers.connections.client.addBlocker(
294 this._identifier + ": waiting for shutdown",
295 this._deferredClose.promise,
297 identifier: this._identifier,
298 isCloseRequested: this._closeRequested,
299 hasDbConn: !!this._dbConn,
300 hasInProgressTransaction: this._hasInProgressTransaction,
301 pendingStatements: this._pendingStatements.size,
302 statementCounter: this._statementCounter,
306 // We avoid creating a timer every transaction that exists solely as a safety
307 // check (e.g. one that never should fire) by reusing it if it's sufficiently
308 // close to when the previous promise was created (see bug 1442353 and
309 // `_getTimeoutPromise` for more info).
310 this._timeoutPromise = null;
311 // The last timestamp when we should consider using `this._timeoutPromise`.
312 this._timeoutPromiseExpires = 0;
316 * Map of connection identifiers to ConnectionData objects
318 * The connection identifier is a human-readable name of the
319 * database. Used by finalization witnesses to be able to close opened
320 * connections on garbage collection.
322 * Key: _identifier of ConnectionData
323 * Value: ConnectionData object
325 ConnectionData.byId = new Map();
327 ConnectionData.prototype = Object.freeze({
329 * Run a task, ensuring that its execution will not be interrupted by shutdown.
331 * As the operations of this module are asynchronous, a sequence of operations,
332 * or even an individual operation, can still be pending when the process shuts
333 * down. If any of this operations is a write, this can cause data loss, simply
334 * because the write has not been completed (or even started) by shutdown.
336 * To avoid this risk, clients are encouraged to use `executeBeforeShutdown` for
337 * any write operation, as follows:
339 * myConnection.executeBeforeShutdown("Bookmarks: Removing a bookmark",
340 * async function(db) {
341 * // The connection will not be closed and shutdown will not proceed
342 * // until this task has completed.
344 * // `db` exposes the same API as `myConnection` but provides additional
345 * // logging support to help debug hard-to-catch shutdown timeouts.
347 * await db.execute(...);
350 * @param {string} name A human-readable name for the ongoing operation, used
351 * for logging and debugging purposes.
352 * @param {function(db)} task A function that takes as argument a Sqlite.jsm
353 * db and returns a Promise.
355 executeBeforeShutdown(parent, name, task) {
357 throw new TypeError("Expected a human-readable name as first argument");
359 if (typeof task != "function") {
360 throw new TypeError("Expected a function as second argument");
362 if (this._closeRequested) {
366 }: cannot execute operation ${name}, the connection is already closing`
370 // Status, used for AsyncShutdown crash reports.
372 // The latest command started by `task`, either as a
373 // sql string, or as one of "<not started>" or "<closing>".
374 command: "<not started>",
376 // `true` if `command` was started but not completed yet.
380 // An object with the same API as `this` but with
381 // additional logging. To keep logging simple, we
382 // assume that `task` is not running several queries
384 let loggedDb = Object.create(parent, {
386 value: async (sql, ...rest) => {
387 status.isPending = true;
388 status.command = sql;
390 return await this.execute(sql, ...rest);
392 status.isPending = false;
398 status.isPending = false;
399 status.command = "<close>";
401 return await this.close();
403 status.isPending = false;
408 value: async (sql, ...rest) => {
409 status.isPending = false;
410 status.command = sql;
412 return await this.executeCached(sql, ...rest);
414 status.isPending = false;
420 let promiseResult = task(loggedDb);
423 typeof promiseResult != "object" ||
424 !("then" in promiseResult)
426 throw new TypeError("Expected a Promise");
428 let key = `${this._identifier}: ${name} (${this._getOperationId()})`;
429 let promiseComplete = promiseResult.catch(() => {});
430 this._barrier.client.addBlocker(key, promiseComplete, {
431 fetchState: () => status,
434 return (async () => {
436 return await promiseResult;
438 this._barrier.client.removeBlocker(key, promiseComplete);
443 this._closeRequested = true;
446 return this._deferredClose.promise;
449 this._log.debug("Request to close connection.");
450 this._clearIdleShrinkTimer();
452 return this._barrier.wait().then(() => {
456 return this._finalize();
460 clone(readOnly = false) {
463 this._log.debug("Request to clone connection.");
466 connection: this._dbConn,
469 if (this._idleShrinkMS) {
470 options.shrinkMemoryOnConnectionIdleMS = this._idleShrinkMS;
473 return cloneStorageConnection(options);
476 return this._operationsCounter++;
479 this._log.debug("Finalizing connection.");
480 // Cancel any pending statements.
481 for (let [, /* k */ statement] of this._pendingStatements) {
484 this._pendingStatements.clear();
486 // We no longer need to track these.
487 this._statementCounter = 0;
489 // Next we finalize all active statements.
490 for (let [, /* k */ statement] of this._anonymousStatements) {
491 statement.finalize();
493 this._anonymousStatements.clear();
495 for (let [, /* k */ statement] of this._cachedStatements) {
496 statement.finalize();
498 this._cachedStatements.clear();
500 // This guards against operations performed between the call to this
501 // function and asyncClose() finishing. See also bug 726990.
504 // We must always close the connection at the Sqlite.jsm-level, not
505 // necessarily at the mozStorage-level.
506 let markAsClosed = () => {
507 this._log.info("Closed");
508 // Now that the connection is closed, no need to keep
509 // a blocker for Barriers.connections.
510 Barriers.connections.client.removeBlocker(this._deferredClose.promise);
511 this._deferredClose.resolve();
513 if (wrappedConnections.has(this._identifier)) {
514 wrappedConnections.delete(this._identifier);
518 this._log.debug("Calling asyncClose().");
519 this._dbConn.asyncClose(markAsClosed);
522 return this._deferredClose.promise;
525 executeCached(sql, params = null, onRow = null) {
529 throw new Error("sql argument is empty.");
532 let statement = this._cachedStatements.get(sql);
534 statement = this._dbConn.createAsyncStatement(sql);
535 this._cachedStatements.set(sql, statement);
538 this._clearIdleShrinkTimer();
540 return new Promise((resolve, reject) => {
542 this._executeStatement(sql, statement, params, onRow).then(
544 this._startIdleShrinkTimer();
548 this._startIdleShrinkTimer();
553 this._startIdleShrinkTimer();
559 execute(sql, params = null, onRow = null) {
560 if (typeof sql != "string") {
561 throw new Error("Must define SQL to execute as a string: " + sql);
566 let statement = this._dbConn.createAsyncStatement(sql);
567 let index = this._anonymousCounter++;
569 this._anonymousStatements.set(index, statement);
570 this._clearIdleShrinkTimer();
572 let onFinished = () => {
573 this._anonymousStatements.delete(index);
574 statement.finalize();
575 this._startIdleShrinkTimer();
578 return new Promise((resolve, reject) => {
580 this._executeStatement(sql, statement, params, onRow).then(
597 get transactionInProgress() {
598 return this._open && this._hasInProgressTransaction;
601 executeTransaction(func, type) {
602 if (type == OpenedConnection.prototype.TRANSACTION_DEFAULT) {
603 type = this.defaultTransactionType;
604 } else if (!OpenedConnection.TRANSACTION_TYPES.includes(type)) {
605 throw new Error("Unknown transaction type: " + type);
609 this._log.debug("Beginning transaction");
611 let promise = this._transactionQueue.then(() => {
612 if (this._closeRequested) {
613 throw new Error("Transaction canceled due to a closed connection.");
616 let transactionPromise = (async () => {
617 // At this point we should never have an in progress transaction, since
618 // they are enqueued.
619 if (this._hasInProgressTransaction) {
621 "Unexpected transaction in progress when trying to start a new one."
624 this._hasInProgressTransaction = true;
626 // We catch errors in statement execution to detect nested transactions.
628 await this.execute("BEGIN " + type + " TRANSACTION");
630 // Unfortunately, if we are wrapping an existing connection, a
631 // transaction could have been started by a client of the same
632 // connection that doesn't use Sqlite.jsm (e.g. C++ consumer).
633 // The best we can do is proceed without a transaction and hope
634 // things won't break.
635 if (wrappedConnections.has(this._identifier)) {
637 "A new transaction could not be started cause the wrapped connection had one in progress",
640 // Unmark the in progress transaction, since it's managed by
641 // some other non-Sqlite.jsm client. See the comment above.
642 this._hasInProgressTransaction = false;
645 "A transaction was already in progress, likely a nested transaction",
654 result = await func();
656 // It's possible that the exception has been caused by trying to
657 // close the connection in the middle of a transaction.
658 if (this._closeRequested) {
660 "Connection closed while performing a transaction",
664 this._log.warn("Error during transaction. Rolling back", ex);
665 // If we began a transaction, we must rollback it.
666 if (this._hasInProgressTransaction) {
668 await this.execute("ROLLBACK TRANSACTION");
670 this._log.warn("Could not roll back transaction", inner);
674 // Rethrow the exception.
678 // See comment above about connection being closed during transaction.
679 if (this._closeRequested) {
681 "Connection closed before committing the transaction."
684 "Connection closed before committing the transaction."
688 // If we began a transaction, we must commit it.
689 if (this._hasInProgressTransaction) {
691 await this.execute("COMMIT TRANSACTION");
693 this._log.warn("Error committing transaction", ex);
700 this._hasInProgressTransaction = false;
704 // If a transaction yields on a never resolved promise, or is mistakenly
705 // nested, it could hang the transactions queue forever. Thus we timeout
706 // the execution after a meaningful amount of time, to ensure in any case
707 // we'll proceed after a while.
708 let timeoutPromise = this._getTimeoutPromise();
709 return Promise.race([transactionPromise, timeoutPromise]);
711 // Atomically update the queue before anyone else has a chance to enqueue
712 // further transactions.
713 this._transactionQueue = promise.catch(ex => {
717 // Make sure that we do not shutdown the connection during a transaction.
718 this._barrier.client.addBlocker(
719 `Transaction (${this._getOperationId()})`,
720 this._transactionQueue
726 this._log.info("Shrinking memory usage.");
727 let onShrunk = this._clearIdleShrinkTimer.bind(this);
728 return this.execute("PRAGMA shrink_memory").then(onShrunk, onShrunk);
731 discardCachedStatements() {
733 for (let [, /* k */ statement] of this._cachedStatements) {
735 statement.finalize();
737 this._cachedStatements.clear();
738 this._log.debug("Discarded " + count + " cached statements.");
743 this._log.info("Trying to interrupt.");
745 this._dbConn.interrupt();
749 * Helper method to bind parameters of various kinds through
752 _bindParameters(statement, params) {
757 function bindParam(obj, key, val) {
759 val && typeof val == "object" && val.constructor.name == "Uint8Array";
760 let args = [key, val];
762 args.push(val.length);
764 let methodName = `bind${isBlob ? "Blob" : ""}By${
765 typeof key == "number" ? "Index" : "Name"
767 obj[methodName](...args);
770 if (Array.isArray(params)) {
771 // It's an array of separate params.
772 if (params.length && typeof params[0] == "object" && params[0] !== null) {
773 let paramsArray = statement.newBindingParamsArray();
774 for (let p of params) {
775 let bindings = paramsArray.newBindingParams();
776 for (let [key, value] of Object.entries(p)) {
777 bindParam(bindings, key, value);
779 paramsArray.addParams(bindings);
782 statement.bindParameters(paramsArray);
787 for (let i = 0; i < params.length; i++) {
788 bindParam(statement, i, params[i]);
794 if (params && typeof params == "object") {
795 for (let k in params) {
796 bindParam(statement, k, params[k]);
802 "Invalid type for bound parameters. Expected Array or " +
808 _executeStatement(sql, statement, params, onRow) {
809 if (statement.state != statement.MOZ_STORAGE_STATEMENT_READY) {
810 throw new Error("Statement is not ready for execution.");
813 if (onRow && typeof onRow != "function") {
814 throw new Error("onRow must be a function. Got: " + onRow);
817 this._bindParameters(statement, params);
819 let index = this._statementCounter++;
821 let deferred = PromiseUtils.defer();
822 let userCancelled = false;
825 let handledRow = false;
827 // Don't incur overhead for serializing params unless the messages go
829 if (this._log.level <= Log.Level.Trace) {
830 let msg = "Stmt #" + index + " " + sql;
833 msg += " - " + JSON.stringify(params);
835 this._log.trace(msg);
837 this._log.debug("Stmt #" + index + " starting");
841 let pending = statement.executeAsync({
842 handleResult(resultSet) {
843 // .cancel() may not be immediate and handleResult() could be called
844 // after a .cancel().
846 let row = resultSet.getNextRow();
847 row && !userCancelled;
848 row = resultSet.getNextRow()
859 userCancelled = true;
863 self._log.warn("Exception when calling onRow callback", e);
870 "Error when executing SQL (" + error.result + "): " + error.message
875 handleCompletion(reason) {
876 self._log.debug("Stmt #" + index + " finished.");
877 self._pendingStatements.delete(index);
880 case Ci.mozIStorageStatementCallback.REASON_FINISHED:
881 case Ci.mozIStorageStatementCallback.REASON_CANCELED:
882 // If there is an onRow handler, we always instead resolve to a
883 // boolean indicating whether the onRow handler was called or not.
884 let result = onRow ? handledRow : rows;
885 deferred.resolve(result);
888 case Ci.mozIStorageStatementCallback.REASON_ERROR:
889 let error = new Error(
890 "Error(s) encountered during statement execution: " +
891 errors.map(e => e.message).join(", ")
893 error.errors = errors;
894 deferred.reject(error);
899 new Error("Unknown completion reason code: " + reason)
906 this._pendingStatements.set(index, pending);
907 return deferred.promise;
912 throw new Error("Connection is not open.");
916 _clearIdleShrinkTimer() {
917 if (!this._idleShrinkTimer) {
921 this._idleShrinkTimer.cancel();
924 _startIdleShrinkTimer() {
925 if (!this._idleShrinkTimer) {
929 this._idleShrinkTimer.initWithCallback(
930 this.shrinkMemory.bind(this),
932 this._idleShrinkTimer.TYPE_ONE_SHOT
936 // Returns a promise that will resolve after a time comprised between 80% of
937 // `TRANSACTIONS_QUEUE_TIMEOUT_MS` and `TRANSACTIONS_QUEUE_TIMEOUT_MS`. Use
938 // this promise instead of creating several individual timers to reduce the
939 // overhead due to timers (see bug 1442353).
940 _getTimeoutPromise() {
941 if (this._timeoutPromise && Cu.now() <= this._timeoutPromiseExpires) {
942 return this._timeoutPromise;
944 let timeoutPromise = new Promise((resolve, reject) => {
946 // Clear out this._timeoutPromise if it hasn't changed since we set it.
947 if (this._timeoutPromise == timeoutPromise) {
948 this._timeoutPromise = null;
952 "Transaction timeout, most likely caused by unresolved pending work."
955 }, TRANSACTIONS_QUEUE_TIMEOUT_MS);
957 this._timeoutPromise = timeoutPromise;
958 this._timeoutPromiseExpires =
959 Cu.now() + TRANSACTIONS_QUEUE_TIMEOUT_MS * 0.2;
960 return this._timeoutPromise;
965 * Opens a connection to a SQLite database.
967 * The following parameters can control the connection:
969 * path -- (string) The filesystem path of the database file to open. If the
970 * file does not exist, a new database will be created.
972 * sharedMemoryCache -- (bool) Whether multiple connections to the database
973 * share the same memory cache. Sharing the memory cache likely results
974 * in less memory utilization. However, sharing also requires connections
975 * to obtain a lock, possibly making database access slower. Defaults to
978 * shrinkMemoryOnConnectionIdleMS -- (integer) If defined, the connection
979 * will attempt to minimize its memory usage after this many
980 * milliseconds of connection idle. The connection is idle when no
981 * statements are executing. There is no default value which means no
982 * automatic memory minimization will occur. Please note that this is
983 * *not* a timer on the idle service and this could fire while the
984 * application is active.
986 * readOnly -- (bool) Whether to open the database with SQLITE_OPEN_READONLY
987 * set. If used, writing to the database will fail. Defaults to false.
989 * ignoreLockingMode -- (bool) Whether to ignore locks on the database held
990 * by other connections. If used, implies readOnly. Defaults to false.
991 * USE WITH EXTREME CAUTION. This mode WILL produce incorrect results or
992 * return "false positive" corruption errors if other connections write
993 * to the DB at the same time.
995 * FUTURE options to control:
997 * special named databases
998 * pragma TEMP STORE = MEMORY
1000 * SYNCHRONOUS = full
1003 * (Object) Parameters to control connection and open options.
1005 * @return Promise<OpenedConnection>
1007 function openConnection(options) {
1008 let log = Log.repository.getLogger("Sqlite.ConnectionOpener");
1010 if (!options.path) {
1011 throw new Error("path not specified in connection options.");
1016 "Sqlite.jsm has been shutdown. Cannot open connection to: " + options.path
1020 // Retains absolute paths and normalizes relative as relative to profile.
1021 let path = OS.Path.join(OS.Constants.Path.profileDir, options.path);
1023 let sharedMemoryCache =
1024 "sharedMemoryCache" in options ? options.sharedMemoryCache : true;
1026 let openedOptions = {};
1028 if ("shrinkMemoryOnConnectionIdleMS" in options) {
1029 if (!Number.isInteger(options.shrinkMemoryOnConnectionIdleMS)) {
1031 "shrinkMemoryOnConnectionIdleMS must be an integer. " +
1033 options.shrinkMemoryOnConnectionIdleMS
1037 openedOptions.shrinkMemoryOnConnectionIdleMS =
1038 options.shrinkMemoryOnConnectionIdleMS;
1041 if ("defaultTransactionType" in options) {
1042 let defaultTransactionType = options.defaultTransactionType;
1043 if (!OpenedConnection.TRANSACTION_TYPES.includes(defaultTransactionType)) {
1045 "Unknown default transaction type: " + defaultTransactionType
1049 openedOptions.defaultTransactionType = defaultTransactionType;
1052 let file = FileUtils.File(path);
1053 let identifier = getIdentifierByFileName(OS.Path.basename(path));
1055 log.info("Opening database: " + path + " (" + identifier + ")");
1057 return new Promise((resolve, reject) => {
1058 let dbOptions = Cc["@mozilla.org/hash-property-bag;1"].createInstance(
1059 Ci.nsIWritablePropertyBag
1061 if (!sharedMemoryCache) {
1062 dbOptions.setProperty("shared", false);
1064 if (options.readOnly) {
1065 dbOptions.setProperty("readOnly", true);
1067 if (options.ignoreLockingMode) {
1068 dbOptions.setProperty("ignoreLockingMode", true);
1069 dbOptions.setProperty("readOnly", true);
1072 dbOptions = dbOptions.enumerator.hasMoreElements() ? dbOptions : null;
1074 Services.storage.openAsyncDatabase(
1077 (status, connection) => {
1079 log.warn(`Could not open connection to ${path}: ${status}`);
1080 let error = new Components.Exception(
1081 `Could not open connection to ${path}: ${status}`,
1087 log.info("Connection opened");
1090 new OpenedConnection(
1091 connection.QueryInterface(Ci.mozIStorageAsyncConnection),
1097 log.warn("Could not open database", ex);
1098 connection.asyncClose();
1107 * Creates a clone of an existing and open Storage connection. The clone has
1108 * the same underlying characteristics of the original connection and is
1109 * returned in form of an OpenedConnection handle.
1111 * The following parameters can control the cloned connection:
1113 * connection -- (mozIStorageAsyncConnection) The original Storage connection
1114 * to clone. It's not possible to clone connections to memory databases.
1116 * readOnly -- (boolean) - If true the clone will be read-only. If the
1117 * original connection is already read-only, the clone will be, regardless
1118 * of this option. If the original connection is using the shared cache,
1119 * this parameter will be ignored and the clone will be as privileged as
1120 * the original connection.
1121 * shrinkMemoryOnConnectionIdleMS -- (integer) If defined, the connection
1122 * will attempt to minimize its memory usage after this many
1123 * milliseconds of connection idle. The connection is idle when no
1124 * statements are executing. There is no default value which means no
1125 * automatic memory minimization will occur. Please note that this is
1126 * *not* a timer on the idle service and this could fire while the
1127 * application is active.
1131 * (Object) Parameters to control connection and clone options.
1133 * @return Promise<OpenedConnection>
1135 function cloneStorageConnection(options) {
1136 let log = Log.repository.getLogger("Sqlite.ConnectionCloner");
1138 let source = options && options.connection;
1140 throw new TypeError("connection not specified in clone options.");
1142 if (!(source instanceof Ci.mozIStorageAsyncConnection)) {
1143 throw new TypeError("Connection must be a valid Storage connection.");
1148 "Sqlite.jsm has been shutdown. Cannot clone connection to: " +
1149 source.databaseFile.path
1153 let openedOptions = {};
1155 if ("shrinkMemoryOnConnectionIdleMS" in options) {
1156 if (!Number.isInteger(options.shrinkMemoryOnConnectionIdleMS)) {
1157 throw new TypeError(
1158 "shrinkMemoryOnConnectionIdleMS must be an integer. " +
1160 options.shrinkMemoryOnConnectionIdleMS
1163 openedOptions.shrinkMemoryOnConnectionIdleMS =
1164 options.shrinkMemoryOnConnectionIdleMS;
1167 let path = source.databaseFile.path;
1168 let identifier = getIdentifierByFileName(OS.Path.basename(path));
1170 log.info("Cloning database: " + path + " (" + identifier + ")");
1172 return new Promise((resolve, reject) => {
1173 source.asyncClone(!!options.readOnly, (status, connection) => {
1175 log.warn("Could not clone connection: " + status);
1176 reject(new Error("Could not clone connection: " + status));
1178 log.info("Connection cloned");
1180 let conn = connection.QueryInterface(Ci.mozIStorageAsyncConnection);
1181 resolve(new OpenedConnection(conn, identifier, openedOptions));
1183 log.warn("Could not clone database", ex);
1184 connection.asyncClose();
1192 * Wraps an existing and open Storage connection with Sqlite.jsm API. The
1193 * wrapped connection clone has the same underlying characteristics of the
1194 * original connection and is returned in form of an OpenedConnection handle.
1196 * Clients are responsible for closing both the Sqlite.jsm wrapper and the
1197 * underlying mozStorage connection.
1199 * The following parameters can control the wrapped connection:
1201 * connection -- (mozIStorageAsyncConnection) The original Storage connection
1205 * (Object) Parameters to control connection and wrap options.
1207 * @return Promise<OpenedConnection>
1209 function wrapStorageConnection(options) {
1210 let log = Log.repository.getLogger("Sqlite.ConnectionWrapper");
1212 let connection = options && options.connection;
1213 if (!connection || !(connection instanceof Ci.mozIStorageAsyncConnection)) {
1214 throw new TypeError("connection not specified or invalid.");
1219 "Sqlite.jsm has been shutdown. Cannot wrap connection to: " +
1220 connection.databaseFile.path
1224 let identifier = getIdentifierByFileName(connection.databaseFile.leafName);
1226 log.info("Wrapping database: " + identifier);
1227 return new Promise(resolve => {
1229 let conn = connection.QueryInterface(Ci.mozIStorageAsyncConnection);
1230 let wrapper = new OpenedConnection(conn, identifier);
1231 // We must not handle shutdown of a wrapped connection, since that is
1232 // already handled by the opener.
1233 wrappedConnections.add(identifier);
1236 log.warn("Could not wrap database", ex);
1243 * Handle on an opened SQLite database.
1245 * This is essentially a glorified wrapper around mozIStorageConnection.
1246 * However, it offers some compelling advantages.
1248 * The main functions on this type are `execute` and `executeCached`. These are
1249 * ultimately how all SQL statements are executed. It's worth explaining their
1252 * `execute` is used to execute one-shot SQL statements. These are SQL
1253 * statements that are executed one time and then thrown away. They are useful
1254 * for dynamically generated SQL statements and clients who don't care about
1255 * performance (either their own or wasting resources in the overall
1256 * application). Because of the performance considerations, it is recommended
1257 * to avoid `execute` unless the statement you are executing will only be
1258 * executed once or seldomly.
1260 * `executeCached` is used to execute a statement that will presumably be
1261 * executed multiple times. The statement is parsed once and stuffed away
1262 * inside the connection instance. Subsequent calls to `executeCached` will not
1263 * incur the overhead of creating a new statement object. This should be used
1264 * in preference to `execute` when a specific SQL statement will be executed
1267 * Instances of this type are not meant to be created outside of this file.
1268 * Instead, first open an instance of `UnopenedSqliteConnection` and obtain
1269 * an instance of this type by calling `open`.
1271 * FUTURE IMPROVEMENTS
1273 * Ability to enqueue operations. Currently there can be race conditions,
1274 * especially as far as transactions are concerned. It would be nice to have
1275 * an enqueueOperation(func) API that serially executes passed functions.
1277 * Support for SAVEPOINT (named/nested transactions) might be useful.
1280 * (mozIStorageConnection) Underlying SQLite connection.
1282 * (string) The unique identifier of this database. It may be used for
1283 * logging or as a key in Maps.
1284 * @param options [optional]
1285 * (object) Options to control behavior of connection. See
1288 function OpenedConnection(connection, identifier, options = {}) {
1289 // Store all connection data in a field distinct from the
1290 // witness. This enables us to store an additional reference to this
1291 // field without preventing garbage collection of
1292 // OpenedConnection. On garbage collection, we will still be able to
1293 // close the database using this extra reference.
1294 this._connectionData = new ConnectionData(connection, identifier, options);
1296 // Store the extra reference in a map with connection identifier as
1298 ConnectionData.byId.set(
1299 this._connectionData._identifier,
1300 this._connectionData
1303 // Make a finalization witness. If this object is garbage collected
1304 // before its `forget` method has been called, an event with topic
1305 // "sqlite-finalization-witness" is broadcasted along with the
1306 // connection identifier string of the database.
1307 this._witness = FinalizationWitnessService.make(
1308 "sqlite-finalization-witness",
1309 this._connectionData._identifier
1313 OpenedConnection.TRANSACTION_TYPES = ["DEFERRED", "IMMEDIATE", "EXCLUSIVE"];
1315 // Converts a `mozIStorageAsyncConnection::TRANSACTION_*` constant into the
1316 // corresponding `OpenedConnection.TRANSACTION_TYPES` constant.
1317 function convertStorageTransactionType(type) {
1318 if (!(type in OpenedConnection.TRANSACTION_TYPES)) {
1319 throw new Error("Unknown storage transaction type: " + type);
1321 return OpenedConnection.TRANSACTION_TYPES[type];
1324 OpenedConnection.prototype = Object.freeze({
1325 TRANSACTION_DEFAULT: "DEFAULT",
1326 TRANSACTION_DEFERRED: "DEFERRED",
1327 TRANSACTION_IMMEDIATE: "IMMEDIATE",
1328 TRANSACTION_EXCLUSIVE: "EXCLUSIVE",
1331 * Returns a handle to the underlying `mozIStorageAsyncConnection`. This is
1332 * ⚠️ **extremely unsafe** ⚠️ because `Sqlite.jsm` continues to manage the
1333 * connection's lifecycle, including transactions and shutdown blockers.
1334 * Misusing the raw connection can easily lead to data loss, memory leaks,
1337 * Consumers of the raw connection **must not** close or re-wrap it,
1338 * and should not run statements concurrently with `Sqlite.jsm`.
1340 * It's _much_ safer to open a `mozIStorage{Async}Connection` yourself,
1341 * and access it from JavaScript via `Sqlite.wrapStorageConnection`.
1342 * `unsafeRawConnection` is an escape hatch for cases where you can't
1345 * Please do _not_ add new uses of `unsafeRawConnection` without review
1346 * from a storage peer.
1348 get unsafeRawConnection() {
1349 return this._connectionData._dbConn;
1353 * Returns the maximum number of bound parameters for statements executed
1354 * on this connection.
1358 get variableLimit() {
1359 return this.unsafeRawConnection.variableLimit;
1363 * The integer schema version of the database.
1365 * This is 0 if not schema version has been set.
1367 * @return Promise<int>
1369 getSchemaVersion(schemaName = "main") {
1370 return this.execute(`PRAGMA ${schemaName}.user_version`).then(
1371 function onSuccess(result) {
1372 if (result == null) {
1375 return result[0].getInt32(0);
1380 setSchemaVersion(value, schemaName = "main") {
1381 if (!Number.isInteger(value)) {
1382 // Guarding against accidental SQLi
1383 throw new TypeError("Schema version must be an integer. Got " + value);
1385 this._connectionData.ensureOpen();
1386 return this.execute(`PRAGMA ${schemaName}.user_version = ${value}`);
1390 * Close the database connection.
1392 * This must be performed when you are finished with the database.
1394 * Closing the database connection has the side effect of forcefully
1395 * cancelling all active statements. Therefore, callers should ensure that
1396 * all active statements have completed before closing the connection, if
1399 * The returned promise will be resolved once the connection is closed.
1400 * Successive calls to close() return the same promise.
1402 * IMPROVEMENT: Resolve the promise to a closed connection which can be
1408 // Unless cleanup has already been done by a previous call to
1409 // `close`, delete the database entry from map and tell the
1410 // finalization witness to forget.
1411 if (ConnectionData.byId.has(this._connectionData._identifier)) {
1412 ConnectionData.byId.delete(this._connectionData._identifier);
1413 this._witness.forget();
1415 return this._connectionData.close();
1419 * Clones this connection to a new Sqlite one.
1421 * The following parameters can control the cloned connection:
1424 * (boolean) - If true the clone will be read-only. If the original
1425 * connection is already read-only, the clone will be, regardless of
1426 * this option. If the original connection is using the shared cache,
1427 * this parameter will be ignored and the clone will be as privileged as
1428 * the original connection.
1430 * @return Promise<OpenedConnection>
1432 clone(readOnly = false) {
1433 return this._connectionData.clone(readOnly);
1436 executeBeforeShutdown(name, task) {
1437 return this._connectionData.executeBeforeShutdown(this, name, task);
1441 * Execute a SQL statement and cache the underlying statement object.
1443 * This function executes a SQL statement and also caches the underlying
1444 * derived statement object so subsequent executions are faster and use
1447 * This function optionally binds parameters to the statement as well as
1448 * optionally invokes a callback for every row retrieved.
1450 * By default, no parameters are bound and no callback will be invoked for
1453 * Bound parameters can be defined as an Array of positional arguments or
1454 * an object mapping named parameters to their values. If there are no bound
1455 * parameters, the caller can pass nothing or null for this argument.
1457 * Callers are encouraged to pass objects rather than Arrays for bound
1458 * parameters because they prevent foot guns. With positional arguments, it
1459 * is simple to modify the parameter count or positions without fixing all
1460 * users of the statement. Objects/named parameters are a little safer
1461 * because changes in order alone won't result in bad things happening.
1463 * When `onRow` is not specified, all returned rows are buffered before the
1464 * returned promise is resolved. For INSERT or UPDATE statements, this has
1465 * no effect because no rows are returned from these. However, it has
1466 * implications for SELECT statements.
1468 * If your SELECT statement could return many rows or rows with large amounts
1469 * of data, for performance reasons it is recommended to pass an `onRow`
1470 * handler. Otherwise, the buffering may consume unacceptable amounts of
1473 * If the second parameter of an `onRow` handler is called during execution
1474 * of the `onRow` handler, the execution of the statement is immediately
1475 * cancelled. Subsequent rows will not be processed and no more `onRow`
1476 * invocations will be made. The promise is resolved immediately.
1478 * If an exception is thrown by the `onRow` handler, the exception is logged
1479 * and processing of subsequent rows occurs as if nothing happened. The
1480 * promise is still resolved (not rejected).
1482 * The return value is a promise that will be resolved when the statement
1483 * has completed fully.
1485 * The promise will be rejected with an `Error` instance if the statement
1486 * did not finish execution fully. The `Error` may have an `errors` property.
1487 * If defined, it will be an Array of objects describing individual errors.
1488 * Each object has the properties `result` and `message`. `result` is a
1489 * numeric error code and `message` is a string description of the problem.
1492 * (string) The name of the registered statement to execute.
1493 * @param params optional
1494 * (Array or object) Parameters to bind.
1495 * @param onRow optional
1496 * (function) Callback to receive each row from result.
1498 executeCached(sql, params = null, onRow = null) {
1499 if (isInvalidBoundLikeQuery(sql)) {
1500 throw new Error("Please enter a LIKE clause with bindings");
1502 return this._connectionData.executeCached(sql, params, onRow);
1506 * Execute a one-shot SQL statement.
1508 * If you find yourself feeding the same SQL string in this function, you
1509 * should *not* use this function and instead use `executeCached`.
1511 * See `executeCached` for the meaning of the arguments and extended usage info.
1514 * (string) SQL to execute.
1515 * @param params optional
1516 * (Array or Object) Parameters to bind to the statement.
1517 * @param onRow optional
1518 * (function) Callback to receive result of a single row.
1520 execute(sql, params = null, onRow = null) {
1521 if (isInvalidBoundLikeQuery(sql)) {
1522 throw new Error("Please enter a LIKE clause with bindings");
1524 return this._connectionData.execute(sql, params, onRow);
1528 * The default behavior for transactions run on this connection.
1530 get defaultTransactionType() {
1531 return this._connectionData.defaultTransactionType;
1535 * Whether a transaction is currently in progress.
1537 get transactionInProgress() {
1538 return this._connectionData.transactionInProgress;
1542 * Perform a transaction.
1544 * *****************************************************************************
1545 * YOU SHOULD _NEVER_ NEST executeTransaction CALLS FOR ANY REASON, NOR
1546 * DIRECTLY, NOR THROUGH OTHER PROMISES.
1547 * FOR EXAMPLE, NEVER DO SOMETHING LIKE:
1548 * await executeTransaction(async function () {
1550 * await executeTransaction(async function () { // WRONG!
1553 * await someCodeThatExecuteTransaction(); // WRONG!
1554 * await neverResolvedPromise; // WRONG!
1556 * NESTING CALLS WILL BLOCK ANY FUTURE TRANSACTION UNTIL A TIMEOUT KICKS IN.
1557 * *****************************************************************************
1559 * A transaction is specified by a user-supplied function that is an
1560 * async function. The function receives this connection instance as its argument.
1562 * The supplied function is expected to return promises. These are often
1563 * promises created by calling `execute` and `executeCached`. If the
1564 * generator is exhausted without any errors being thrown, the
1565 * transaction is committed. If an error occurs, the transaction is
1568 * The returned value from this function is a promise that will be resolved
1569 * once the transaction has been committed or rolled back. The promise will
1570 * be resolved to whatever value the supplied function resolves to. If
1571 * the transaction is rolled back, the promise is rejected.
1574 * (function) What to perform as part of the transaction.
1575 * @param type optional
1576 * One of the TRANSACTION_* constants attached to this type.
1578 executeTransaction(func, type = this.TRANSACTION_DEFAULT) {
1579 return this._connectionData.executeTransaction(() => func(this), type);
1583 * Whether a table exists in the database (both persistent and temporary tables).
1586 * (string) Name of the table.
1588 * @return Promise<bool>
1591 return this.execute(
1592 "SELECT name FROM (SELECT * FROM sqlite_master UNION ALL " +
1593 "SELECT * FROM sqlite_temp_master) " +
1594 "WHERE type = 'table' AND name=?",
1596 ).then(function onResult(rows) {
1597 return Promise.resolve(!!rows.length);
1602 * Whether a named index exists (both persistent and temporary tables).
1605 * (string) Name of the index.
1607 * @return Promise<bool>
1610 return this.execute(
1611 "SELECT name FROM (SELECT * FROM sqlite_master UNION ALL " +
1612 "SELECT * FROM sqlite_temp_master) " +
1613 "WHERE type = 'index' AND name=?",
1615 ).then(function onResult(rows) {
1616 return Promise.resolve(!!rows.length);
1621 * Free up as much memory from the underlying database connection as possible.
1626 return this._connectionData.shrinkMemory();
1630 * Discard all cached statements.
1632 * Note that this relies on us being non-interruptible between
1633 * the insertion or retrieval of a statement in the cache and its
1634 * execution: we finalize all statements, which is only safe if
1635 * they will not be executed again.
1637 * @return (integer) the number of statements discarded.
1639 discardCachedStatements() {
1640 return this._connectionData.discardCachedStatements();
1644 * Interrupts pending database operations returning at the first opportunity.
1645 * Statement execution will throw an NS_ERROR_ABORT failure.
1646 * Can only be used on read-only connections.
1649 this._connectionData.interrupt();
1655 cloneStorageConnection,
1656 wrapStorageConnection,
1658 * Shutdown barrier client. May be used by clients to perform last-minute
1659 * cleanup prior to the shutdown of this module.
1661 * See the documentation of AsyncShutdown.Barrier.prototype.client.
1664 return Barriers.shutdown.client;
1666 failTestsOnAutoClose(enabled) {
1667 Debugging.failTestsOnAutoClose = enabled;