1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
5 import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
9 ChromeUtils.defineESModuleGetters(lazy, {
10 AsyncShutdown: "resource://gre/modules/AsyncShutdown.sys.mjs",
11 CommonUtils: "resource://services-common/utils.sys.mjs",
12 Utils: "resource://services-settings/Utils.sys.mjs",
15 XPCOMUtils.defineLazyModuleGetters(lazy, {
16 IDBHelpers: "resource://services-settings/IDBHelpers.jsm",
17 ObjectUtils: "resource://gre/modules/ObjectUtils.jsm",
19 XPCOMUtils.defineLazyGetter(lazy, "console", () => lazy.Utils.log);
22 * Database is a tiny wrapper with the objective
23 * of providing major kinto-offline-client collection API.
24 * (with the objective of getting rid of kinto-offline-client)
26 export class Database {
31 constructor(identifier) {
32 ensureShutdownBlocker();
33 this.identifier = identifier;
36 async list(options = {}) {
37 const { filters = {}, order = "" } = options;
42 (store, rejectTransaction) => {
43 // Fast-path the (very common) no-filters case
44 if (lazy.ObjectUtils.isEmpty(filters)) {
45 const range = IDBKeyRange.only(this.identifier);
46 const request = store.index("cid").getAll(range);
47 request.onsuccess = e => {
48 results = e.target.result;
54 .openCursor(IDBKeyRange.only(this.identifier));
55 const objFilters = transformSubObjectFilters(filters);
56 request.onsuccess = event => {
58 const cursor = event.target.result;
60 const { value } = cursor;
61 if (lazy.Utils.filterObject(objFilters, value)) {
67 rejectTransaction(ex);
74 throw new lazy.IDBHelpers.IndexedDBError(e, "list()", this.identifier);
76 // Remove IDB key field from results.
77 for (const result of results) {
80 return order ? lazy.Utils.sortObjects(order, results) : results;
83 async importChanges(metadata, timestamp, records = [], options = {}) {
84 const { clear = false } = options;
85 const _cid = this.identifier;
88 ["collections", "timestamps", "records"],
89 (stores, rejectTransaction) => {
90 const [storeMetadata, storeTimestamps, storeRecords] = stores;
93 // Our index is over the _cid and id fields. We want to remove
94 // all of the items in the collection for which the object was
95 // created, ie with _cid == this.identifier.
96 // We would like to just tell IndexedDB:
97 // store.index(IDBKeyRange.only(this.identifier)).delete();
98 // to delete all records matching the first part of the 2-part key.
99 // Unfortunately such an API does not exist.
100 // While we could iterate over the index with a cursor, we'd do
101 // a roundtrip to PBackground for each item. Once you have 1000
102 // items, the result is very slow because of all the overhead of
103 // jumping between threads and serializing/deserializing.
104 // So instead, we tell the store to delete everything between
105 // "our" _cid identifier, and what would be the next identifier
106 // (via lexicographical sorting). Unfortunately there does not
107 // seem to be a way to specify bounds for all items that share
108 // the same first part of the key using just that first part, hence
109 // the use of the hypothetical [] for the second part of the end of
112 IDBKeyRange.bound([_cid], [_cid, []], false, true)
116 // Store or erase metadata.
117 if (metadata === null) {
118 storeMetadata.delete(_cid);
119 } else if (metadata) {
120 storeMetadata.put({ cid: _cid, metadata });
122 // Store or erase timestamp.
123 if (timestamp === null) {
124 storeTimestamps.delete(_cid);
125 } else if (timestamp) {
126 storeTimestamps.put({ cid: _cid, value: timestamp });
129 if (!records.length) {
133 // Separate tombstones from creations/updates.
134 const toDelete = records.filter(r => r.deleted);
135 const toInsert = records.filter(r => !r.deleted);
137 `${_cid} ${toDelete.length} to delete, ${toInsert.length} to insert`
139 // Delete local records for each tombstone.
140 lazy.IDBHelpers.bulkOperationHelper(
143 reject: rejectTransaction,
145 // Overwrite all other data.
146 lazy.IDBHelpers.bulkOperationHelper(
149 reject: rejectTransaction,
152 toInsert.map(item => ({ ...item, _cid }))
157 toDelete.map(item => [_cid, item.id])
160 { desc: "importChanges() in " + _cid }
163 throw new lazy.IDBHelpers.IndexedDBError(e, "importChanges()", _cid);
167 async getLastModified() {
173 store.get(this.identifier).onsuccess = e => (entry = e.target.result);
178 throw new lazy.IDBHelpers.IndexedDBError(
187 // Some distributions where released with a modified dump that did not
188 // contain timestamps for last_modified. Work around this here, and return
189 // the timestamp as zero, so that the entries should get updated.
190 if (isNaN(entry.value)) {
191 lazy.console.warn(`Local timestamp is NaN for ${this.identifier}`);
197 async getMetadata() {
203 store.get(this.identifier).onsuccess = e => (entry = e.target.result);
208 throw new lazy.IDBHelpers.IndexedDBError(
214 return entry ? entry.metadata : null;
217 async getAttachment(attachmentId) {
223 store.get([this.identifier, attachmentId]).onsuccess = e => {
224 entry = e.target.result;
230 throw new lazy.IDBHelpers.IndexedDBError(
236 return entry ? entry.attachment : null;
239 async saveAttachment(attachmentId, attachment) {
245 store.put({ cid: this.identifier, attachmentId, attachment });
247 store.delete([this.identifier, attachmentId]);
250 { desc: "saveAttachment(" + attachmentId + ") in " + this.identifier }
253 throw new lazy.IDBHelpers.IndexedDBError(
262 * Delete all attachments which don't match any record.
264 * Attachments are linked to records, except when a fixed `attachmentId` is used.
265 * A record can be updated or deleted, potentially by deleting a record and restoring an updated version
266 * of the record with the same ID. Potentially leaving orphaned attachments in the database.
267 * Since we run the pruning logic after syncing, any attachment without a
268 * matching record can be discarded as they will be unreachable forever.
270 * @param {Array<String>} excludeIds List of attachments IDs to exclude from pruning.
272 async pruneAttachments(excludeIds) {
273 const _cid = this.identifier;
274 let deletedCount = 0;
277 ["attachments", "records"],
278 async (stores, rejectTransaction) => {
279 const [attachmentsStore, recordsStore] = stores;
281 // List all stored attachments.
282 // All keys ≥ [_cid, ..] && < [_cid, []]. See comment in `importChanges()`
283 const rangeAllKeys = IDBKeyRange.bound(
289 const allAttachments = await new Promise((resolve, reject) => {
290 const request = attachmentsStore.getAll(rangeAllKeys);
291 request.onsuccess = e => resolve(e.target.result);
292 request.onerror = e => reject(e);
294 if (!allAttachments.length) {
296 `${this.identifier} No attachments in IDB cache. Nothing to do.`
301 // List all stored records.
302 const allRecords = await new Promise((resolve, reject) => {
303 const rangeAllIndexed = IDBKeyRange.only(_cid);
304 const request = recordsStore.index("cid").getAll(rangeAllIndexed);
305 request.onsuccess = e => resolve(e.target.result);
306 request.onerror = e => reject(e);
309 console.error("allRecords", allRecords);
311 // Compare known records IDs to those stored along the attachments.
312 const currentRecordsIDs = new Set(allRecords.map(r => r.id));
313 const attachmentsToDelete = allAttachments.reduce((acc, entry) => {
314 // Skip excluded attachments.
315 if (excludeIds.includes(entry.attachmentId)) {
318 // Delete attachment if associated record does not exist.
319 if (!currentRecordsIDs.has(entry.attachment.record.id)) {
320 acc.push([_cid, entry.attachmentId]);
325 // Perform a bulk delete of all obsolete attachments.
327 `${this.identifier} Bulk delete ${attachmentsToDelete.length} obsolete attachments`
329 lazy.IDBHelpers.bulkOperationHelper(
332 reject: rejectTransaction,
337 deletedCount = attachmentsToDelete.length;
339 { desc: "pruneAttachments() in " + this.identifier }
342 throw new lazy.IDBHelpers.IndexedDBError(
344 "pruneAttachments()",
353 await this.importChanges(null, null, [], { clear: true });
355 throw new lazy.IDBHelpers.IndexedDBError(e, "clear()", this.identifier);
360 * Methods used by unit tests.
363 async create(record) {
364 if (!("id" in record)) {
365 record = { ...record, id: lazy.CommonUtils.generateUUID() };
371 store.add({ ...record, _cid: this.identifier });
373 { desc: "create() in " + this.identifier }
376 throw new lazy.IDBHelpers.IndexedDBError(e, "create()", this.identifier);
381 async update(record) {
386 store.put({ ...record, _cid: this.identifier });
388 { desc: "update() in " + this.identifier }
391 throw new lazy.IDBHelpers.IndexedDBError(e, "update()", this.identifier);
395 async delete(recordId) {
400 store.delete([this.identifier, recordId]); // [_cid, id]
402 { desc: "delete() in " + this.identifier }
405 throw new lazy.IDBHelpers.IndexedDBError(e, "delete()", this.identifier);
411 let gDBPromise = null;
414 * This function attempts to ensure `gDB` points to a valid database value.
415 * If gDB is already a database, it will do no-op (but this may take a
417 * If opening the database fails, it will throw an IndexedDBError.
419 async function openIDB() {
420 // We can be called multiple times in a race; always ensure that when
421 // we complete, `gDB` is no longer null, but avoid doing the actual
422 // IndexedDB work more than once.
424 // Open and initialize/upgrade if needed.
425 gDBPromise = lazy.IDBHelpers.openIDB();
427 let db = await gDBPromise;
433 const gPendingReadOnlyTransactions = new Set();
434 const gPendingWriteOperations = new Set();
436 * Helper to wrap some IDBObjectStore operations into a promise.
438 * @param {IDBDatabase} db
439 * @param {String|String[]} storeNames - either a string or an array of strings.
440 * @param {function} callback
441 * @param {Object} options
442 * @param {String} options.mode
443 * @param {String} options.desc for shutdown tracking.
445 async function executeIDB(storeNames, callback, options = {}) {
447 // Check if we're shutting down. Services.startup.shuttingDown will
448 // be true sooner, but is never true in xpcshell tests, so we check
449 // both that and a bool we set ourselves when `profile-before-change`
451 if (gShutdownStarted || Services.startup.shuttingDown) {
452 throw new lazy.IDBHelpers.ShutdownError(
453 "The application is shutting down",
459 // Even if we have a db, wait a tick to avoid making IndexedDB sad.
460 // We should be able to remove this once bug 1626935 is fixed.
461 await Promise.resolve();
464 // Check for shutdown again as we've await'd something...
465 if (!gDB && (gShutdownStarted || Services.startup.shuttingDown)) {
466 throw new lazy.IDBHelpers.ShutdownError(
467 "The application is shutting down",
472 // Start the actual transaction:
473 const { mode = "readwrite", desc = "" } = options;
474 let { promise, transaction } = lazy.IDBHelpers.executeIDB(
482 // We track all readonly transactions and abort them at shutdown.
483 // We track all readwrite ones and await their completion at shutdown
484 // (to avoid dataloss when writes fail).
485 // We use a `.finally()` clause for this; it'll run the function irrespective
486 // of whether the promise resolves or rejects, and the promise it returns
487 // will resolve/reject with the same value.
489 if (mode == "readonly") {
490 gPendingReadOnlyTransactions.add(transaction);
491 finishedFn = () => gPendingReadOnlyTransactions.delete(transaction);
493 let obj = { promise, desc };
494 gPendingWriteOperations.add(obj);
495 finishedFn = () => gPendingWriteOperations.delete(obj);
497 return promise.finally(finishedFn);
500 async function destroyIDB() {
502 if (gShutdownStarted || Services.startup.shuttingDown) {
503 throw new lazy.IDBHelpers.ShutdownError(
504 "The application is shutting down",
509 // This will return immediately; the actual close will happen once
510 // there are no more running transactions.
512 const allTransactions = new Set([
513 ...gPendingWriteOperations,
514 ...gPendingReadOnlyTransactions,
516 for (let transaction of Array.from(allTransactions)) {
520 // Ignore errors to abort transactions, we'll destroy everything.
526 return lazy.IDBHelpers.destroyIDB();
529 function makeNestedObjectFromArr(arr, val, nestedFiltersObj) {
530 const last = arr.length - 1;
531 return arr.reduce((acc, cv, i) => {
533 return (acc[cv] = val);
534 } else if (Object.prototype.hasOwnProperty.call(acc, cv)) {
537 return (acc[cv] = {});
538 }, nestedFiltersObj);
541 function transformSubObjectFilters(filtersObj) {
542 const transformedFilters = {};
543 for (const [key, val] of Object.entries(filtersObj)) {
544 const keysArr = key.split(".");
545 makeNestedObjectFromArr(keysArr, val, transformedFilters);
547 return transformedFilters;
550 // We need to expose this wrapper function so we can test
551 // shutdown handling.
552 Database._executeIDB = executeIDB;
554 let gShutdownStarted = false;
555 // Test-only helper to be able to test shutdown multiple times:
556 Database._cancelShutdown = () => {
557 gShutdownStarted = false;
560 let gShutdownBlocker = false;
561 Database._shutdownHandler = () => {
562 gShutdownStarted = true;
563 const NS_ERROR_DOM_INDEXEDDB_NOT_ALLOWED_ERR = 0x80660006;
564 // Duplicate the list (to avoid it being modified) and then
565 // abort all read-only transactions.
566 for (let transaction of Array.from(gPendingReadOnlyTransactions)) {
570 // Ensure we don't throw/break, because either way we're in shutdown.
572 // In particular, `transaction.abort` can throw if the transaction
573 // is complete, ie if we manage to get called in between the
574 // transaction completing, and our completion handler being called
575 // to remove the item from the set. We don't care about that.
576 if (ex.result != NS_ERROR_DOM_INDEXEDDB_NOT_ALLOWED_ERR) {
577 // Report any other errors:
583 // This will return immediately; the actual close will happen once
584 // there are no more running transactions.
589 return Promise.allSettled(
590 Array.from(gPendingWriteOperations).map(op => op.promise)
594 function ensureShutdownBlocker() {
595 if (gShutdownBlocker) {
598 gShutdownBlocker = true;
599 lazy.AsyncShutdown.profileBeforeChange.addBlocker(
600 "RemoteSettingsClient - finish IDB access.",
601 Database._shutdownHandler,
604 return Array.from(gPendingWriteOperations).map(op => op.desc);