1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 ChromeUtils.defineESModuleGetters(lazy, {
8 AsyncShutdown: "resource://gre/modules/AsyncShutdown.sys.mjs",
9 CommonUtils: "resource://services-common/utils.sys.mjs",
10 IDBHelpers: "resource://services-settings/IDBHelpers.sys.mjs",
11 ObjectUtils: "resource://gre/modules/ObjectUtils.sys.mjs",
12 Utils: "resource://services-settings/Utils.sys.mjs",
15 ChromeUtils.defineLazyGetter(lazy, "console", () => lazy.Utils.log);
18 * Database is a tiny wrapper with the objective
19 * of providing major kinto-offline-client collection API.
20 * (with the objective of getting rid of kinto-offline-client)
22 export class Database {
27 constructor(identifier) {
28 ensureShutdownBlocker();
29 this.identifier = identifier;
32 async list(options = {}) {
33 const { filters = {}, order = "" } = options;
38 (store, rejectTransaction) => {
39 // Fast-path the (very common) no-filters case
40 if (lazy.ObjectUtils.isEmpty(filters)) {
41 const range = IDBKeyRange.only(this.identifier);
42 const request = store.index("cid").getAll(range);
43 request.onsuccess = e => {
44 results = e.target.result;
50 .openCursor(IDBKeyRange.only(this.identifier));
51 const objFilters = transformSubObjectFilters(filters);
52 request.onsuccess = event => {
54 const cursor = event.target.result;
56 const { value } = cursor;
57 if (lazy.Utils.filterObject(objFilters, value)) {
63 rejectTransaction(ex);
70 throw new lazy.IDBHelpers.IndexedDBError(e, "list()", this.identifier);
72 // Remove IDB key field from results.
73 for (const result of results) {
76 return order ? lazy.Utils.sortObjects(order, results) : results;
79 async importChanges(metadata, timestamp, records = [], options = {}) {
80 const { clear = false } = options;
81 const _cid = this.identifier;
84 ["collections", "timestamps", "records"],
85 (stores, rejectTransaction) => {
86 const [storeMetadata, storeTimestamps, storeRecords] = stores;
89 // Our index is over the _cid and id fields. We want to remove
90 // all of the items in the collection for which the object was
91 // created, ie with _cid == this.identifier.
92 // We would like to just tell IndexedDB:
93 // store.index(IDBKeyRange.only(this.identifier)).delete();
94 // to delete all records matching the first part of the 2-part key.
95 // Unfortunately such an API does not exist.
96 // While we could iterate over the index with a cursor, we'd do
97 // a roundtrip to PBackground for each item. Once you have 1000
98 // items, the result is very slow because of all the overhead of
99 // jumping between threads and serializing/deserializing.
100 // So instead, we tell the store to delete everything between
101 // "our" _cid identifier, and what would be the next identifier
102 // (via lexicographical sorting). Unfortunately there does not
103 // seem to be a way to specify bounds for all items that share
104 // the same first part of the key using just that first part, hence
105 // the use of the hypothetical [] for the second part of the end of
108 IDBKeyRange.bound([_cid], [_cid, []], false, true)
112 // Store or erase metadata.
113 if (metadata === null) {
114 storeMetadata.delete(_cid);
115 } else if (metadata) {
116 storeMetadata.put({ cid: _cid, metadata });
118 // Store or erase timestamp.
119 if (timestamp === null) {
120 storeTimestamps.delete(_cid);
121 } else if (timestamp) {
122 storeTimestamps.put({ cid: _cid, value: timestamp });
125 if (!records.length) {
129 // Separate tombstones from creations/updates.
130 const toDelete = records.filter(r => r.deleted);
131 const toInsert = records.filter(r => !r.deleted);
133 `${_cid} ${toDelete.length} to delete, ${toInsert.length} to insert`
135 // Delete local records for each tombstone.
136 lazy.IDBHelpers.bulkOperationHelper(
139 reject: rejectTransaction,
141 // Overwrite all other data.
142 lazy.IDBHelpers.bulkOperationHelper(
145 reject: rejectTransaction,
148 toInsert.map(item => ({ ...item, _cid }))
153 toDelete.map(item => [_cid, item.id])
156 { desc: "importChanges() in " + _cid }
159 throw new lazy.IDBHelpers.IndexedDBError(e, "importChanges()", _cid);
163 async getLastModified() {
169 store.get(this.identifier).onsuccess = e => (entry = e.target.result);
174 throw new lazy.IDBHelpers.IndexedDBError(
183 // Some distributions where released with a modified dump that did not
184 // contain timestamps for last_modified. Work around this here, and return
185 // the timestamp as zero, so that the entries should get updated.
186 if (isNaN(entry.value)) {
187 lazy.console.warn(`Local timestamp is NaN for ${this.identifier}`);
193 async getMetadata() {
199 store.get(this.identifier).onsuccess = e => (entry = e.target.result);
204 throw new lazy.IDBHelpers.IndexedDBError(
210 return entry ? entry.metadata : null;
213 async getAttachment(attachmentId) {
219 store.get([this.identifier, attachmentId]).onsuccess = e => {
220 entry = e.target.result;
226 throw new lazy.IDBHelpers.IndexedDBError(
232 return entry ? entry.attachment : null;
235 async saveAttachment(attachmentId, attachment) {
241 store.put({ cid: this.identifier, attachmentId, attachment });
243 store.delete([this.identifier, attachmentId]);
246 { desc: "saveAttachment(" + attachmentId + ") in " + this.identifier }
249 throw new lazy.IDBHelpers.IndexedDBError(
258 * Delete all attachments which don't match any record.
260 * Attachments are linked to records, except when a fixed `attachmentId` is used.
261 * A record can be updated or deleted, potentially by deleting a record and restoring an updated version
262 * of the record with the same ID. Potentially leaving orphaned attachments in the database.
263 * Since we run the pruning logic after syncing, any attachment without a
264 * matching record can be discarded as they will be unreachable forever.
266 * @param {Array<String>} excludeIds List of attachments IDs to exclude from pruning.
268 async pruneAttachments(excludeIds) {
269 const _cid = this.identifier;
270 let deletedCount = 0;
273 ["attachments", "records"],
274 async (stores, rejectTransaction) => {
275 const [attachmentsStore, recordsStore] = stores;
277 // List all stored attachments.
278 // All keys ≥ [_cid, ..] && < [_cid, []]. See comment in `importChanges()`
279 const rangeAllKeys = IDBKeyRange.bound(
285 const allAttachments = await new Promise((resolve, reject) => {
286 const request = attachmentsStore.getAll(rangeAllKeys);
287 request.onsuccess = e => resolve(e.target.result);
288 request.onerror = e => reject(e);
290 if (!allAttachments.length) {
292 `${this.identifier} No attachments in IDB cache. Nothing to do.`
297 // List all stored records.
298 const allRecords = await new Promise((resolve, reject) => {
299 const rangeAllIndexed = IDBKeyRange.only(_cid);
300 const request = recordsStore.index("cid").getAll(rangeAllIndexed);
301 request.onsuccess = e => resolve(e.target.result);
302 request.onerror = e => reject(e);
305 // Compare known records IDs to those stored along the attachments.
306 const currentRecordsIDs = new Set(allRecords.map(r => r.id));
307 const attachmentsToDelete = allAttachments.reduce((acc, entry) => {
308 // Skip excluded attachments.
309 if (excludeIds.includes(entry.attachmentId)) {
312 // Delete attachment if associated record does not exist.
313 if (!currentRecordsIDs.has(entry.attachment.record.id)) {
314 acc.push([_cid, entry.attachmentId]);
319 // Perform a bulk delete of all obsolete attachments.
321 `${this.identifier} Bulk delete ${attachmentsToDelete.length} obsolete attachments`
323 lazy.IDBHelpers.bulkOperationHelper(
326 reject: rejectTransaction,
331 deletedCount = attachmentsToDelete.length;
333 { desc: "pruneAttachments() in " + this.identifier }
336 throw new lazy.IDBHelpers.IndexedDBError(
338 "pruneAttachments()",
347 await this.importChanges(null, null, [], { clear: true });
349 throw new lazy.IDBHelpers.IndexedDBError(e, "clear()", this.identifier);
354 * Methods used by unit tests.
357 async create(record) {
358 if (!("id" in record)) {
359 record = { ...record, id: lazy.CommonUtils.generateUUID() };
365 store.add({ ...record, _cid: this.identifier });
367 { desc: "create() in " + this.identifier }
370 throw new lazy.IDBHelpers.IndexedDBError(e, "create()", this.identifier);
375 async update(record) {
380 store.put({ ...record, _cid: this.identifier });
382 { desc: "update() in " + this.identifier }
385 throw new lazy.IDBHelpers.IndexedDBError(e, "update()", this.identifier);
389 async delete(recordId) {
394 store.delete([this.identifier, recordId]); // [_cid, id]
396 { desc: "delete() in " + this.identifier }
399 throw new lazy.IDBHelpers.IndexedDBError(e, "delete()", this.identifier);
405 let gDBPromise = null;
408 * This function attempts to ensure `gDB` points to a valid database value.
409 * If gDB is already a database, it will do no-op (but this may take a
411 * If opening the database fails, it will throw an IndexedDBError.
413 async function openIDB() {
414 // We can be called multiple times in a race; always ensure that when
415 // we complete, `gDB` is no longer null, but avoid doing the actual
416 // IndexedDB work more than once.
418 // Open and initialize/upgrade if needed.
419 gDBPromise = lazy.IDBHelpers.openIDB();
421 let db = await gDBPromise;
427 const gPendingReadOnlyTransactions = new Set();
428 const gPendingWriteOperations = new Set();
430 * Helper to wrap some IDBObjectStore operations into a promise.
432 * @param {IDBDatabase} db
433 * @param {String|String[]} storeNames - either a string or an array of strings.
434 * @param {function} callback
435 * @param {Object} options
436 * @param {String} options.mode
437 * @param {String} options.desc for shutdown tracking.
439 async function executeIDB(storeNames, callback, options = {}) {
441 // Check if we're shutting down. Services.startup.shuttingDown will
442 // be true sooner, but is never true in xpcshell tests, so we check
443 // both that and a bool we set ourselves when `profile-before-change`
445 if (gShutdownStarted || Services.startup.shuttingDown) {
446 throw new lazy.IDBHelpers.ShutdownError(
447 "The application is shutting down",
453 // Even if we have a db, wait a tick to avoid making IndexedDB sad.
454 // We should be able to remove this once bug 1626935 is fixed.
455 await Promise.resolve();
458 // Check for shutdown again as we've await'd something...
459 if (!gDB && (gShutdownStarted || Services.startup.shuttingDown)) {
460 throw new lazy.IDBHelpers.ShutdownError(
461 "The application is shutting down",
466 // Start the actual transaction:
467 const { mode = "readwrite", desc = "" } = options;
468 let { promise, transaction } = lazy.IDBHelpers.executeIDB(
476 // We track all readonly transactions and abort them at shutdown.
477 // We track all readwrite ones and await their completion at shutdown
478 // (to avoid dataloss when writes fail).
479 // We use a `.finally()` clause for this; it'll run the function irrespective
480 // of whether the promise resolves or rejects, and the promise it returns
481 // will resolve/reject with the same value.
483 if (mode == "readonly") {
484 gPendingReadOnlyTransactions.add(transaction);
485 finishedFn = () => gPendingReadOnlyTransactions.delete(transaction);
487 let obj = { promise, desc };
488 gPendingWriteOperations.add(obj);
489 finishedFn = () => gPendingWriteOperations.delete(obj);
491 return promise.finally(finishedFn);
494 async function destroyIDB() {
496 if (gShutdownStarted || Services.startup.shuttingDown) {
497 throw new lazy.IDBHelpers.ShutdownError(
498 "The application is shutting down",
503 // This will return immediately; the actual close will happen once
504 // there are no more running transactions.
506 const allTransactions = new Set([
507 ...gPendingWriteOperations,
508 ...gPendingReadOnlyTransactions,
510 for (let transaction of Array.from(allTransactions)) {
514 // Ignore errors to abort transactions, we'll destroy everything.
520 return lazy.IDBHelpers.destroyIDB();
523 function makeNestedObjectFromArr(arr, val, nestedFiltersObj) {
524 const last = arr.length - 1;
525 return arr.reduce((acc, cv, i) => {
527 return (acc[cv] = val);
528 } else if (Object.prototype.hasOwnProperty.call(acc, cv)) {
531 return (acc[cv] = {});
532 }, nestedFiltersObj);
535 function transformSubObjectFilters(filtersObj) {
536 const transformedFilters = {};
537 for (const [key, val] of Object.entries(filtersObj)) {
538 const keysArr = key.split(".");
539 makeNestedObjectFromArr(keysArr, val, transformedFilters);
541 return transformedFilters;
544 // We need to expose this wrapper function so we can test
545 // shutdown handling.
546 Database._executeIDB = executeIDB;
548 let gShutdownStarted = false;
549 // Test-only helper to be able to test shutdown multiple times:
550 Database._cancelShutdown = () => {
551 gShutdownStarted = false;
554 let gShutdownBlocker = false;
555 Database._shutdownHandler = () => {
556 gShutdownStarted = true;
557 const NS_ERROR_DOM_INDEXEDDB_NOT_ALLOWED_ERR = 0x80660006;
558 // Duplicate the list (to avoid it being modified) and then
559 // abort all read-only transactions.
560 for (let transaction of Array.from(gPendingReadOnlyTransactions)) {
564 // Ensure we don't throw/break, because either way we're in shutdown.
566 // In particular, `transaction.abort` can throw if the transaction
567 // is complete, ie if we manage to get called in between the
568 // transaction completing, and our completion handler being called
569 // to remove the item from the set. We don't care about that.
570 if (ex.result != NS_ERROR_DOM_INDEXEDDB_NOT_ALLOWED_ERR) {
571 // Report any other errors:
577 // This will return immediately; the actual close will happen once
578 // there are no more running transactions.
583 return Promise.allSettled(
584 Array.from(gPendingWriteOperations).map(op => op.promise)
588 function ensureShutdownBlocker() {
589 if (gShutdownBlocker) {
592 gShutdownBlocker = true;
593 lazy.AsyncShutdown.profileBeforeChange.addBlocker(
594 "RemoteSettingsClient - finish IDB access.",
595 Database._shutdownHandler,
598 return Array.from(gPendingWriteOperations).map(op => op.desc);