1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
5 import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
6 import { IndexedDB } from "resource://gre/modules/IndexedDB.sys.mjs";
10 ChromeUtils.defineESModuleGetters(lazy, {
11 ExtensionStorage: "resource://gre/modules/ExtensionStorage.sys.mjs",
12 ExtensionUtils: "resource://gre/modules/ExtensionUtils.sys.mjs",
13 getTrimmedString: "resource://gre/modules/ExtensionTelemetry.sys.mjs",
16 // The userContextID reserved for the extension storage (its purpose is ensuring that the IndexedDB
17 // storage used by the browser.storage.local API is not directly accessible from the extension code,
18 // it is defined and reserved as "userContextIdInternal.webextStorageLocal" in ContextualIdentityService.sys.mjs).
19 const WEBEXT_STORAGE_USER_CONTEXT_ID = -1 >>> 0;
21 const IDB_NAME = "webExtensions-storage-local";
22 const IDB_DATA_STORENAME = "storage-local-data";
23 const IDB_VERSION = 1;
24 const IDB_MIGRATE_RESULT_HISTOGRAM =
25 "WEBEXT_STORAGE_LOCAL_IDB_MIGRATE_RESULT_COUNT";
27 // Whether or not the installed extensions should be migrated to the storage.local IndexedDB backend.
28 const BACKEND_ENABLED_PREF =
29 "extensions.webextensions.ExtensionStorageIDB.enabled";
30 const IDB_MIGRATED_PREF_BRANCH =
31 "extensions.webextensions.ExtensionStorageIDB.migrated";
33 class DataMigrationAbortedError extends Error {
35 return "DataMigrationAbortedError";
39 var ErrorsTelemetry = {
43 if (this.initialized) {
46 this.initialized = true;
48 // Ensure that these telemetry events category is enabled.
49 Services.telemetry.setEventRecordingEnabled("extensions.data", true);
51 this.resultHistogram = Services.telemetry.getHistogramById(
52 IDB_MIGRATE_RESULT_HISTOGRAM
57 * Get the DOMException error name for a given error object.
59 * @param {Error | undefined} error
60 * The Error object to convert into a string, or undefined if there was no error.
62 * @returns {string | undefined}
63 * The DOMException error name (sliced to a maximum of 80 chars),
64 * "OtherError" if the error object is not a DOMException instance,
65 * or `undefined` if there wasn't an error.
73 DOMException.isInstance(error) ||
74 error instanceof DataMigrationAbortedError
76 if (error.name.length > 80) {
77 return lazy.getTrimmedString(error.name);
87 * Record telemetry related to a data migration result.
89 * @param {object} telemetryData
90 * @param {string} telemetryData.backend
91 * The backend selected ("JSONFile" or "IndexedDB").
92 * @param {boolean} [telemetryData.dataMigrated]
93 * Old extension data has been migrated successfully.
94 * @param {string} telemetryData.extensionId
95 * The id of the extension migrated.
96 * @param {Error | undefined} telemetryData.error
97 * The error raised during the data migration, if any.
98 * @param {boolean} [telemetryData.hasJSONFile]
99 * The extension has an existing JSONFile to migrate.
100 * @param {boolean} [telemetryData.hasOldData]
101 * The extension's JSONFile wasn't empty.
102 * @param {string} telemetryData.histogramCategory
103 * The histogram category for the result ("success" or "failure").
105 recordDataMigrationResult(telemetryData) {
118 this.resultHistogram.add(histogramCategory);
120 const extra = { backend };
122 if (dataMigrated != null) {
123 extra.data_migrated = dataMigrated ? "y" : "n";
126 if (hasJSONFile != null) {
127 extra.has_jsonfile = hasJSONFile ? "y" : "n";
130 if (hasOldData != null) {
131 extra.has_olddata = hasOldData ? "y" : "n";
135 extra.error_name = this.getErrorName(error);
138 let addon_id = lazy.getTrimmedString(extensionId);
139 Services.telemetry.recordEvent(
146 Glean.extensionsData.migrateResult.record({
148 backend: extra.backend,
149 data_migrated: extra.data_migrated,
150 has_jsonfile: extra.has_jsonfile,
151 has_olddata: extra.has_olddata,
152 error_name: extra.error_name,
155 // Report any telemetry error on the browser console, but
156 // we treat it as a non-fatal error and we don't re-throw
163 * Record telemetry related to the unexpected errors raised while executing
164 * a storage.local API call.
166 * @param {object} options
167 * @param {string} options.extensionId
168 * The id of the extension migrated.
169 * @param {string} options.storageMethod
170 * The storage.local API method being run.
171 * @param {Error} options.error
172 * The unexpected error raised during the API call.
174 recordStorageLocalError({ extensionId, storageMethod, error }) {
176 let addon_id = lazy.getTrimmedString(extensionId);
177 let error_name = this.getErrorName(error);
179 Services.telemetry.recordEvent(
186 Glean.extensionsData.storageLocalError.record({
188 method: storageMethod,
194 class ExtensionStorageLocalIDB extends IndexedDB {
195 onupgradeneeded(event) {
196 if (event.oldVersion < 1) {
197 this.createObjectStore(IDB_DATA_STORENAME);
201 static openForPrincipal(storagePrincipal) {
202 // The db is opened using an extension principal isolated in a reserved user context id.
203 return super.openForPrincipal(storagePrincipal, IDB_NAME, IDB_VERSION);
207 const cursor = await this.objectStore(
215 * Asynchronously sets the values of the given storage items.
217 * @param {object} items
218 * The storage items to set. For each property in the object,
219 * the storage value for that property is set to its value in
220 * said object. Any values which are StructuredCloneHolder
221 * instances are deserialized before being stored.
222 * @param {object} options
223 * @param {callback} [options.serialize]
224 * Set to a function which will be used to serialize the values into
225 * a StructuredCloneHolder object (if appropriate) and being sent
226 * across the processes (it is also used to detect data cloning errors
227 * and raise an appropriate error to the caller).
229 * @returns {Promise<null|object>}
230 * Return a promise which resolves to the computed "changes" object
233 async set(items, { serialize } = {}) {
237 // Explicitly create a transaction, so that we can explicitly abort it
238 // as soon as one of the put requests fails.
239 const transaction = this.transaction(IDB_DATA_STORENAME, "readwrite");
240 const objectStore = transaction.objectStore(IDB_DATA_STORENAME);
241 const transactionCompleted = transaction.promiseComplete();
244 serialize = (name, anonymizedName, value) => value;
247 for (let key of Object.keys(items)) {
249 let oldValue = await objectStore.get(key);
251 await objectStore.put(items[key], key);
255 oldValue && serialize(`old/${key}`, `old/<anonymized>`, oldValue),
256 newValue: serialize(`new/${key}`, `new/<anonymized>`, items[key]),
260 transactionCompleted.catch(err => {
261 // We ignore this rejection because we are explicitly aborting the transaction,
262 // the transaction.error will be null, and we throw the original error below.
270 await transactionCompleted;
272 return changed ? changes : null;
276 * Asynchronously retrieves the values for the given storage items.
278 * @param {Array<string>|object|null} [keysOrItems]
279 * The storage items to get. If an array, the value of each key
280 * in the array is returned. If null, the values of all items
281 * are returned. If an object, the value for each key in the
282 * object is returned, or that key's value if the item is not
284 * @returns {Promise<object>}
285 * An object which has a property for each requested key,
286 * containing that key's value as stored in the IndexedDB
289 async get(keysOrItems) {
293 if (typeof keysOrItems === "string") {
294 keys = [keysOrItems];
295 } else if (Array.isArray(keysOrItems)) {
297 } else if (keysOrItems && typeof keysOrItems === "object") {
298 keys = Object.keys(keysOrItems);
299 defaultValues = keysOrItems;
304 // Retrieve all the stored data using a cursor when browser.storage.local.get()
305 // has been called with no keys.
307 const cursor = await this.objectStore(
311 while (!cursor.done) {
312 result[cursor.key] = cursor.value;
313 await cursor.continue();
316 const objectStore = this.objectStore(IDB_DATA_STORENAME);
317 for (let key of keys) {
318 const storedValue = await objectStore.get(key);
319 if (storedValue === undefined) {
320 if (defaultValues && defaultValues[key] !== undefined) {
321 result[key] = defaultValues[key];
324 result[key] = storedValue;
333 * Asynchronously removes the given storage items.
335 * @param {string|Array<string>} keys
336 * A string key of a list of storage items keys to remove.
337 * @returns {Promise<object>}
338 * Returns an object which contains applied changes.
341 // Ensure that keys is an array of strings.
342 keys = [].concat(keys);
344 if (keys.length === 0) {
345 // Early exit if there is nothing to remove.
352 const objectStore = this.objectStore(IDB_DATA_STORENAME, "readwrite");
356 for (let key of keys) {
358 objectStore.getKey(key).then(async foundKey => {
359 if (foundKey === key) {
361 changes[key] = { oldValue: await objectStore.get(key) };
362 return objectStore.delete(key);
368 await Promise.all(promises);
370 return changed ? changes : null;
374 * Asynchronously clears all storage entries.
376 * @returns {Promise<object>}
377 * Returns an object which contains applied changes.
383 const objectStore = this.objectStore(IDB_DATA_STORENAME, "readwrite");
385 const cursor = await objectStore.openCursor();
386 while (!cursor.done) {
387 changes[cursor.key] = { oldValue: cursor.value };
389 await cursor.continue();
392 await objectStore.clear();
394 return changed ? changes : null;
399 * Migrate the data stored in the JSONFile backend to the IDB Backend.
401 * Returns a promise which is resolved once the data migration has been
402 * completed and the new IDB backend can be enabled.
403 * Rejects if the data has been read successfully from the JSONFile backend
404 * but it failed to be saved in the new IDB backend.
406 * This method is called only from the main process (where the file
409 * @param {Extension} extension
410 * The extension to migrate to the new IDB backend.
411 * @param {nsIPrincipal} storagePrincipal
412 * The "internally reserved" extension storagePrincipal to be used to create
413 * the ExtensionStorageLocalIDB instance.
415 async function migrateJSONFileData(extension, storagePrincipal) {
417 let oldStorageExists;
422 let dataMigrateCompleted = false;
423 let hasOldData = false;
425 function abortIfShuttingDown() {
426 if (extension.hasShutdown || Services.startup.shuttingDown) {
427 throw new DataMigrationAbortedError("extension or app is shutting down");
431 if (ExtensionStorageIDB.isMigratedExtension(extension)) {
436 abortIfShuttingDown();
437 idbConn = await ExtensionStorageIDB.open(
439 extension.hasPermission("unlimitedStorage")
441 abortIfShuttingDown();
443 hasEmptyIDB = await idbConn.isEmpty();
446 // If the IDB backend is enabled and there is data already stored in the IDB backend,
447 // there is no "going back": any data that has not been migrated will be still on disk
448 // but it is not going to be migrated anymore, it could be eventually used to allow
449 // a user to manually retrieve the old data file).
450 ExtensionStorageIDB.setMigratedExtensionPref(extension, true);
454 extension.logWarning(
455 `storage.local data migration cancelled, unable to open IDB connection: ${err.message}::${err.stack}`
458 ErrorsTelemetry.recordDataMigrationResult({
460 extensionId: extension.id,
462 histogramCategory: "failure",
469 abortIfShuttingDown();
471 oldStoragePath = lazy.ExtensionStorage.getStorageFile(extension.id);
472 oldStorageExists = await IOUtils.exists(oldStoragePath).catch(fileErr => {
473 // If we can't access the oldStoragePath here, then extension is also going to be unable to
474 // access it, and so we log the error but we don't stop the extension from switching to
475 // the IndexedDB backend.
476 extension.logWarning(
477 `Unable to access extension storage.local data file: ${fileErr.message}::${fileErr.stack}`
482 // Migrate any data stored in the JSONFile backend (if any), and remove the old data file
483 // if the migration has been completed successfully.
484 if (oldStorageExists) {
485 // Do not load the old JSON file content if shutting down is already in progress.
486 abortIfShuttingDown();
488 Services.console.logStringMessage(
489 `Migrating storage.local data for ${extension.policy.debugName}...`
492 jsonFile = await lazy.ExtensionStorage.getFile(extension.id);
494 abortIfShuttingDown();
497 for (let [key, value] of jsonFile.data.entries()) {
502 await idbConn.set(data);
503 Services.console.logStringMessage(
504 `storage.local data successfully migrated to IDB Backend for ${extension.policy.debugName}.`
508 dataMigrateCompleted = true;
510 extension.logWarning(
511 `Error on migrating storage.local data file: ${err.message}::${err.stack}`
514 if (oldStorageExists && !dataMigrateCompleted) {
515 ErrorsTelemetry.recordDataMigrationResult({
517 dataMigrated: dataMigrateCompleted,
518 extensionId: extension.id,
520 hasJSONFile: oldStorageExists,
522 histogramCategory: "failure",
525 // If the data failed to be stored into the IndexedDB backend, then we clear the IndexedDB
526 // backend to allow the extension to retry the migration on its next startup, and reject
527 // the data migration promise explicitly (which would prevent the new backend
528 // from being enabled for this session).
529 await new Promise(resolve => {
530 let req = Services.qms.clearStoragesForPrincipal(storagePrincipal);
531 req.callback = resolve;
537 // This error is not preventing the extension from switching to the IndexedDB backend,
538 // but we may still want to know that it has been triggered and include it into the
539 // telemetry data collected for the extension.
542 // Clear the jsonFilePromise cached by the ExtensionStorage.
543 await lazy.ExtensionStorage.clearCachedFile(extension.id).catch(err => {
544 extension.logWarning(err.message);
548 // If the IDB backend has been enabled, rename the old storage.local data file, but
549 // do not prevent the extension from switching to the IndexedDB backend if it fails.
550 if (oldStorageExists && dataMigrateCompleted) {
552 // Only migrate the file when it actually exists (e.g. the file name is not going to exist
553 // when it is corrupted, because JSONFile internally rename it to `.corrupt`.
554 if (await IOUtils.exists(oldStoragePath)) {
555 const uniquePath = await IOUtils.createUniqueFile(
556 PathUtils.parent(oldStoragePath),
557 `${PathUtils.filename(oldStoragePath)}.migrated`
559 await IOUtils.move(oldStoragePath, uniquePath);
563 extension.logWarning(err.message);
567 ExtensionStorageIDB.setMigratedExtensionPref(extension, true);
569 ErrorsTelemetry.recordDataMigrationResult({
570 backend: "IndexedDB",
571 dataMigrated: dataMigrateCompleted,
572 extensionId: extension.id,
573 error: nonFatalError,
574 hasJSONFile: oldStorageExists,
576 histogramCategory: "success",
581 * This ExtensionStorage class implements a backend for the storage.local API which
582 * uses IndexedDB to store the data.
584 export var ExtensionStorageIDB = {
585 BACKEND_ENABLED_PREF,
586 IDB_MIGRATED_PREF_BRANCH,
587 IDB_MIGRATE_RESULT_HISTOGRAM,
589 // Map<extension-id, Set<Function>>
590 listeners: new Map(),
592 // Keep track if the IDB backend has been selected or not for a running extension
593 // (the selected backend should never change while the extension is running, even if the
594 // related preference has been changed in the meantime):
596 // WeakMap<extension -> Promise<boolean>
597 selectedBackendPromises: new WeakMap(),
600 XPCOMUtils.defineLazyPreferenceGetter(
603 BACKEND_ENABLED_PREF,
608 isMigratedExtension(extension) {
609 return Services.prefs.getBoolPref(
610 `${IDB_MIGRATED_PREF_BRANCH}.${extension.id}`,
615 setMigratedExtensionPref(extension, val) {
616 Services.prefs.setBoolPref(
617 `${IDB_MIGRATED_PREF_BRANCH}.${extension.id}`,
622 clearMigratedExtensionPref(extensionId) {
623 Services.prefs.clearUserPref(`${IDB_MIGRATED_PREF_BRANCH}.${extensionId}`);
626 getStoragePrincipal(extension) {
627 return extension.createPrincipal(extension.baseURI, {
628 userContextId: WEBEXT_STORAGE_USER_CONTEXT_ID,
633 * Select the preferred backend and return a promise which is resolved once the
634 * selected backend is ready to be used (e.g. if the extension is switching from
635 * the old JSONFile storage to the new IDB backend, any previously stored data will
636 * be migrated to the backend before the promise is resolved).
638 * This method is called from both the main and child (content or extension) processes:
639 * - an extension child context will call this method lazily, when the browser.storage.local
640 * is being used for the first time, and it will result into asking the main process
641 * to call the same method in the main process
642 * - on the main process side, it will check if the new IDB backend can be used (and if it can,
643 * it will migrate any existing data into the new backend, which needs to happen in the
644 * main process where the file can directly be accessed)
646 * The result will be cached while the extension is still running, and so an extension
647 * child context is going to ask the main process only once per child process, and on the
648 * main process side the backend selection and data migration will happen only once.
650 * @param {import("ExtensionPageChild.sys.mjs").ExtensionBaseContextChild} context
651 * The extension context that is selecting the storage backend.
653 * @returns {Promise<object>}
654 * Returns a promise which resolves to an object which provides a
655 * `backendEnabled` boolean property, and if it is true the extension should use
656 * the IDB backend and the object also includes a `storagePrincipal` property
657 * of type nsIPrincipal, otherwise `backendEnabled` will be false when the
658 * extension should use the old JSONFile backend (e.g. because the IDB backend has
659 * not been enabled from the preference).
661 selectBackend(context) {
662 const { extension } = context;
664 if (!this.selectedBackendPromises.has(extension)) {
667 if (context.childManager) {
668 return context.childManager
669 .callParentAsyncFunction("storage.local.IDBBackend.selectBackend", [])
670 .then(parentResult => {
673 if (!parentResult.backendEnabled) {
674 result = { backendEnabled: false };
678 // In the child process, we need to deserialize the storagePrincipal
679 // from the StructuredCloneHolder used to send it across the processes.
680 storagePrincipal: parentResult.storagePrincipal.deserialize(
687 // Cache the result once we know that it has been resolved. The promise returned by
688 // context.childManager.callParentAsyncFunction will be dead when context.cloneScope
689 // is destroyed. To keep a promise alive in the cache, we wrap the result in an
690 // independent promise.
691 this.selectedBackendPromises.set(
693 Promise.resolve(result)
700 // If migrating to the IDB backend is not enabled by the preference, then we
701 // don't need to migrate any data and the new backend is not enabled.
702 if (!this.isBackendEnabled) {
703 promise = Promise.resolve({ backendEnabled: false });
705 // In the main process, lazily create a storagePrincipal isolated in a
706 // reserved user context id (its purpose is ensuring that the IndexedDB storage used
707 // by the browser.storage.local API is not directly accessible from the extension code).
708 const storagePrincipal = this.getStoragePrincipal(extension);
710 // Serialize the nsIPrincipal object into a StructuredCloneHolder related to the privileged
711 // js global, ready to be sent to the child processes.
712 const serializedPrincipal = new StructuredCloneHolder(
713 "ExtensionStorageIDB/selectBackend/serializedPrincipal",
719 promise = migrateJSONFileData(extension, storagePrincipal)
721 extension.setSharedData("storageIDBBackend", true);
722 extension.setSharedData("storageIDBPrincipal", storagePrincipal);
723 Services.ppmm.sharedData.flush();
725 backendEnabled: true,
726 storagePrincipal: serializedPrincipal,
730 // If the data migration promise is rejected, the old data has been read
731 // successfully from the old JSONFile backend but it failed to be saved
732 // into the IndexedDB backend (which is likely unrelated to the kind of
733 // data stored and more likely a general issue with the IndexedDB backend)
734 // In this case we keep the JSONFile backend enabled for this session
735 // and we will retry to migrate to the IDB Backend the next time the
736 // extension is being started.
737 // TODO Bug 1465129: This should be a very unlikely scenario, some telemetry
738 // data about it may be useful.
739 extension.logWarning(
740 "JSONFile backend is being kept enabled by an unexpected " +
741 `IDBBackend failure: ${err.message}::${err.stack}`
743 extension.setSharedData("storageIDBBackend", false);
744 Services.ppmm.sharedData.flush();
746 return { backendEnabled: false };
750 this.selectedBackendPromises.set(extension, promise);
753 return this.selectedBackendPromises.get(extension);
756 persist(storagePrincipal) {
757 return new Promise((resolve, reject) => {
758 const request = Services.qms.persist(storagePrincipal);
759 request.callback = () => {
760 if (request.resultCode === Cr.NS_OK) {
765 `Failed to persist storage for principal: ${storagePrincipal.originNoSuffix}`
774 * Open a connection to the IDB storage.local db for a given extension.
777 * @param {nsIPrincipal} storagePrincipal
778 * The "internally reserved" extension storagePrincipal to be used to create
779 * the ExtensionStorageLocalIDB instance.
780 * @param {boolean} persisted
781 * A boolean which indicates if the storage should be set into persistent mode.
783 * @returns {Promise<ExtensionStorageLocalIDB>}
784 * Return a promise which resolves to the opened IDB connection.
786 open(storagePrincipal, persisted) {
787 if (!storagePrincipal) {
788 return Promise.reject(new Error("Unexpected empty principal"));
790 let setPersistentMode = persisted
791 ? this.persist(storagePrincipal)
793 return setPersistentMode.then(() =>
794 ExtensionStorageLocalIDB.openForPrincipal(storagePrincipal)
799 * Ensure that an error originated from the ExtensionStorageIDB methods is normalized
800 * into an ExtensionError (e.g. DataCloneError and QuotaExceededError instances raised
801 * from the internal IndexedDB operations have to be converted into an ExtensionError
802 * to be accessible to the extension code).
804 * @param {object} params
805 * @param {Error|ExtensionError|DOMException} params.error
806 * The error object to normalize.
807 * @param {string} params.extensionId
808 * The id of the extension that was executing the storage.local method.
809 * @param {string} params.storageMethod
810 * The storage method being executed when the error has been thrown
811 * (used to keep track of the unexpected error incidence in telemetry).
813 * @returns {ExtensionError}
814 * Return an ExtensionError error instance.
816 normalizeStorageError({ error, extensionId, storageMethod }) {
817 const { ExtensionError } = lazy.ExtensionUtils;
819 if (error instanceof ExtensionError) {
820 // @ts-ignore (will go away after `lazy` is properly typed)
826 if (DOMException.isInstance(error)) {
827 switch (error.name) {
828 case "DataCloneError":
829 errorMessage = String(error);
831 case "QuotaExceededError":
832 errorMessage = `${error.name}: storage.local API call exceeded its quota limitations.`;
838 Cu.reportError(error);
840 errorMessage = "An unexpected error occurred";
842 ErrorsTelemetry.recordStorageLocalError({
849 return new ExtensionError(errorMessage);
852 addOnChangedListener(extensionId, listener) {
853 let listeners = this.listeners.get(extensionId) || new Set();
854 listeners.add(listener);
855 this.listeners.set(extensionId, listeners);
858 removeOnChangedListener(extensionId, listener) {
859 let listeners = this.listeners.get(extensionId);
860 listeners.delete(listener);
863 notifyListeners(extensionId, changes) {
864 let listeners = this.listeners.get(extensionId);
866 for (let listener of listeners) {
872 hasListeners(extensionId) {
873 let listeners = this.listeners.get(extensionId);
874 return listeners && listeners.size > 0;
878 ExtensionStorageIDB.init();