1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
5 const { ExtensionParent } = ChromeUtils.import(
6 "resource://gre/modules/ExtensionParent.jsm"
9 const { ExtensionUtils } = ChromeUtils.import(
10 "resource://gre/modules/ExtensionUtils.jsm"
13 import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
17 XPCOMUtils.defineLazyModuleGetters(lazy, {
18 Schemas: "resource://gre/modules/Schemas.jsm",
19 PromiseUtils: "resource://gre/modules/PromiseUtils.jsm",
22 ChromeUtils.defineESModuleGetters(lazy, {
23 ExtensionDNR: "resource://gre/modules/ExtensionDNR.sys.mjs",
24 ExtensionDNRLimits: "resource://gre/modules/ExtensionDNRLimits.sys.mjs",
27 const { DefaultMap, ExtensionError } = ExtensionUtils;
28 const { StartupCache } = ExtensionParent;
30 // DNR Rules store subdirectory/file names and file extensions.
32 // NOTE: each extension's stored rules are stored in a per-extension file
33 // and stored rules filename is derived from the extension uuid assigned
36 // TODO(Bug 1803365): consider introducing a startupCache file.
37 const RULES_STORE_DIRNAME = "extension-dnr";
38 const RULES_STORE_FILEEXT = ".json.lz4";
41 * Internal representation of the enabled static rulesets (used in StoreData
42 * and Store methods type signatures).
44 * @typedef {object} EnabledStaticRuleset
46 * @property {number} idx
47 * Represent the position of the static ruleset in the manifest
48 * `declarative_net_request.rule_resources` array.
49 * @property {Array<Rule>} rules
50 * Represent the array of the DNR rules associated with the static
54 // Class defining the format of the data stored into the per-extension files
55 // managed by RulesetsStore.
57 // StoreData instances are saved in the profile extension-dir subdirectory as
58 // lz4-compressed JSON files, only the ruleset_id is stored on disk for the
59 // enabled static rulesets (while the actual rules would need to be loaded back
60 // from the related rules JSON files part of the extension assets).
62 // NOTE: Update schema version upgrade handling code in `RulesetsStore.#readData`
63 // along with bumps to the schema version here.
67 * @param {object} params
68 * @param {number} [params.schemaVersion=StoreData.VERSION]
70 * @param {string} [params.extVersion]
72 * @param {Map<string, EnabledStaticRuleset>} [params.staticRulesets=new Map()]
73 * map of the enabled static rulesets by ruleset_id, as resolved by
74 * `Store.prototype.#getManifestStaticRulesets`.
75 * NOTE: This map is converted in an array of the ruleset_id strings when the StoreData
76 * instance is being stored on disk (see `toJSON` method) and then converted back to a Map
77 * by `Store.prototype.#getManifestStaticRulesets` when the data is loaded back from disk.
78 * @param {Array<Rule>} [params.dynamicRuleset=[]]
79 * array of dynamic rules stored by the extension.
87 this.schemaVersion = schemaVersion || this.constructor.VERSION;
88 this.extVersion = extVersion ?? null;
89 this.setStaticRulesets(staticRulesets);
90 this.setDynamicRuleset(dynamicRuleset);
94 return !this.staticRulesets.size && !this.dynamicRuleset.length;
97 setStaticRulesets(updatedStaticRulesets = new Map()) {
98 this.staticRulesets = updatedStaticRulesets;
101 setDynamicRuleset(updatedDynamicRuleset = []) {
102 this.dynamicRuleset = updatedDynamicRuleset;
105 // This method is used to convert the data in the format stored on disk
109 schemaVersion: this.schemaVersion,
110 extVersion: this.extVersion,
111 // Only store the array of the enabled ruleset_id in the set of data
112 // persisted in a JSON form.
113 staticRulesets: this.staticRulesets
114 ? Array.from(this.staticRulesets.entries(), ([id, _ruleset]) => id)
116 dynamicRuleset: this.dynamicRuleset,
127 get hasPendingTasks() {
128 return !!this.#runningTask || !!this.#tasks.length;
137 const lastTask = this.#tasks[this.#tasks.length - 1];
138 return lastTask?.deferred.promise;
140 const drainedQueuePromise = this.queueTask(() => {});
142 return drainedQueuePromise;
145 queueTask(callback) {
147 throw new Error("Unexpected queueTask call on closed queue");
149 const deferred = lazy.PromiseUtils.defer();
150 this.#tasks.push({ callback, deferred });
151 // Run the queued task right away if there isn't one already running.
152 if (!this.#runningTask) {
155 return deferred.promise;
158 async #runNextTask() {
159 if (!this.#tasks.length) {
160 this.#runningTask = null;
164 this.#runningTask = this.#tasks.shift();
165 const { callback, deferred } = this.#runningTask;
167 let result = callback();
168 if (result instanceof Promise) {
169 result = await result;
171 deferred.resolve(result);
173 deferred.reject(err);
181 * Class managing the rulesets persisted across browser sessions.
183 * The data gets stored in two per-extension files:
185 * - `ProfD/extension-dnr/EXT_UUID.json.lz4` is a lz4-compressed JSON file that is expected to include
186 * the ruleset ids for the enabled static rulesets and the dynamic rules.
188 * All browser data stored is expected to be persisted across browser updates, but the enabled static ruleset
189 * ids are expected to be reset and reinitialized from the extension manifest.json properties when the
190 * add-on is being updated (either downgraded or upgraded).
192 * In case of unexpected data schema downgrades (which may be hit if the user explicit pass --allow-downgrade
193 * while using an older browser version than the one used when the data has been stored), the entire stored
194 * data is reset and re-initialized from scratch based on the manifest.json file.
196 class RulesetsStore {
198 // Map<extensionUUID, StoreData>
199 this._data = new Map();
200 // Map<extensionUUID, Promise<StoreData>>
201 this._dataPromises = new Map();
202 // Map<extensionUUID, Promise<void>>
203 this._savePromises = new Map();
204 // Map<extensionUUID, Queue>
205 this._dataUpdateQueues = new DefaultMap(() => new Queue());
206 // Promise to await on to ensure the store parent directory exist
207 // (the parent directory is shared by all extensions and so we only need one).
208 this._ensureStoreDirectoryPromise = null;
212 * Remove store file for the given extension UUId from disk (used to remove all
213 * data on addon uninstall).
215 * @param {string} extensionUUID
216 * @returns {Promise<void>}
218 async clearOnUninstall(extensionUUID) {
219 const storeFile = this.#getStoreFilePath(extensionUUID);
221 // TODO(Bug 1803363): consider collect telemetry on DNR store file removal errors.
222 // TODO: consider catch and report unexpected errors
223 await IOUtils.remove(storeFile, { ignoreAbsent: true });
227 * Load (or initialize) the store file data for the given extension and
228 * return an Array of the dynamic rules.
230 * @param {Extension} extension
232 * @returns {Promise<Array<Rule>>}
233 * Resolve to a reference to the dynamic rules array.
234 * NOTE: the caller should never mutate the content of this array,
235 * updates to the dynamic rules should always go through
236 * the `updateDynamicRules` method.
238 async getDynamicRules(extension) {
239 let data = await this.#getDataPromise(extension);
240 return data.dynamicRuleset;
244 * Load (or initialize) the store file data for the given extension and
245 * return a Map of the enabled static rulesets and their related rules.
247 * - if the extension manifest doesn't have any static rulesets declared in the
248 * manifest, returns null
250 * - if the extension version from the stored data doesn't match the current
251 * extension versions, the static rules are being reloaded from the manifest.
253 * @param {Extension} extension
255 * @returns {Promise<Map<ruleset_id, EnabledStaticRuleset>>}
256 * Resolves to a reference to the static rulesets map.
257 * NOTE: the caller should never mutate the content of this map,
258 * updates to the enabled static rulesets should always go through
259 * the `updateEnabledStaticRulesets` method.
261 async getEnabledStaticRulesets(extension) {
262 let data = await this.#getDataPromise(extension);
263 return data.staticRulesets;
266 async getAvailableStaticRuleCount(extension) {
267 const { GUARANTEED_MINIMUM_STATIC_RULES } = lazy.ExtensionDNRLimits;
269 const ruleResources =
270 extension.manifest.declarative_net_request?.rule_resources;
271 // TODO: return maximum rules count when no static rules is listed in the manifest?
272 if (!Array.isArray(ruleResources)) {
273 return GUARANTEED_MINIMUM_STATIC_RULES;
276 const enabledRulesets = await this.getEnabledStaticRulesets(extension);
277 const enabledRulesCount = Array.from(enabledRulesets.values()).reduce(
278 (acc, ruleset) => acc + ruleset.rules.length,
282 return GUARANTEED_MINIMUM_STATIC_RULES - enabledRulesCount;
286 * Initialize the DNR store for the given extension, it does also queue the task to make
287 * sure that extension DNR API calls triggered while the initialization may still be
288 * in progress will be executed sequentially.
290 * @param {Extension} extension
292 * @returns {Promise<void>} A promise resolved when the async initialization has been
295 async initExtension(extension) {
296 const ensureExtensionRunning = () => {
297 if (extension.hasShutdown) {
299 `DNR store initialization abort, extension is already shutting down: ${extension.id}`
304 // Make sure we wait for pending save promise to have been
305 // completed and old data unloaded (this may be hit if an
306 // extension updates or reloads while there are still
307 // rules updates being processed and then stored on disk).
308 ensureExtensionRunning();
309 if (this._savePromises.has(extension.uuid)) {
311 `Unexpected pending save task while reading DNR data after an install/update of extension "${extension.id}"`
313 // await pending saving data to be saved and unloaded.
314 await this.#unloadData(extension.uuid);
315 // Make sure the extension is still running after awaiting on
316 // unloadData to be completed.
317 ensureExtensionRunning();
320 return this._dataUpdateQueues.get(extension.uuid).queueTask(() => {
321 return this.#initExtension(extension);
326 * Update the dynamic rules, queue changes to prevent races between calls
327 * that may be triggered while an update is still in process.
329 * @param {Extension} extension
330 * @param {object} params
331 * @param {Array<string>} [params.removeRuleIds=[]]
332 * @param {Array<Rule>} [params.addRules=[]]
334 * @returns {Promise<void>} A promise resolved when the dynamic rules async update has
337 async updateDynamicRules(extension, { removeRuleIds, addRules }) {
338 return this._dataUpdateQueues.get(extension.uuid).queueTask(() => {
339 return this.#updateDynamicRules(extension, {
347 * Update the enabled rulesets, queue changes to prevent races between calls
348 * that may be triggered while an update is still in process.
350 * @param {Extension} extension
351 * @param {object} params
352 * @param {Array<string>} [params.disableRulesetIds=[]]
353 * @param {Array<string>} [params.enableRulesetIds=[]]
355 * @returns {Promise<void>} A promise resolved when the enabled static rulesets async
356 * update has been completed.
358 async updateEnabledStaticRulesets(
360 { disableRulesetIds, enableRulesetIds }
362 return this._dataUpdateQueues.get(extension.uuid).queueTask(() => {
363 return this.#updateEnabledStaticRulesets(extension, {
371 * Update DNR RulesetManager rules to match the current DNR rules enabled in the DNRStore.
373 * @param {Extension} extension
374 * @param {object} [params]
375 * @param {boolean} [params.updateStaticRulesets=true]
376 * @param {boolean} [params.updateDynamicRuleset=true]
378 updateRulesetManager(
380 { updateStaticRulesets = true, updateDynamicRuleset = true } = {}
382 if (!updateStaticRulesets && !updateDynamicRuleset) {
387 !this._dataPromises.has(extension.uuid) ||
388 !this._data.has(extension.uuid)
391 `Unexpected call to updateRulesetManager before DNR store was fully initialized for extension "${extension.id}"`
394 const data = this._data.get(extension.uuid);
395 const ruleManager = lazy.ExtensionDNR.getRuleManager(extension);
397 if (updateStaticRulesets) {
398 let staticRulesetsMap = data.staticRulesets;
399 // Convert into array and ensure order match the order of the rulesets in
400 // the extension manifest.
401 const enabledStaticRules = [];
402 // Order the static rulesets by index of rule_resources in manifest.json.
403 const orderedRulesets = Array.from(staticRulesetsMap.entries()).sort(
404 ([_idA, rsA], [_idB, rsB]) => rsA.idx - rsB.idx
406 for (const [rulesetId, ruleset] of orderedRulesets) {
407 enabledStaticRules.push({ id: rulesetId, rules: ruleset.rules });
409 ruleManager.setEnabledStaticRulesets(enabledStaticRules);
412 if (updateDynamicRuleset) {
413 ruleManager.setDynamicRules(data.dynamicRuleset);
418 * Return the store file path for the given the extension's uuid.
420 * @param {string} extensionUUID
421 * @returns {{ storeFile: string}}
422 * An object including the full paths to the storeFile for the extension.
424 getFilePaths(extensionUUID) {
426 storeFile: this.#getStoreFilePath(extensionUUID),
431 * Save the data for the given extension on disk.
433 * @param {Extension} extension
435 async save(extension) {
436 const { uuid, id } = extension;
437 let savePromise = this._savePromises.get(uuid);
440 savePromise = this.#saveNow(uuid);
441 this._savePromises.set(uuid, savePromise);
442 IOUtils.profileBeforeChange.addBlocker(
443 `Flush WebExtension DNR RulesetsStore: ${id}`,
452 * Register an onClose shutdown handler to cleanup the data from memory when
453 * the extension is shutting down.
455 * @param {Extension} extension
458 unloadOnShutdown(extension) {
459 if (extension.hasShutdown) {
461 `DNR store registering an extension shutdown handler too late, the extension is already shutting down: ${extension.id}`
465 const extensionUUID = extension.uuid;
466 extension.callOnClose({
467 close: async () => this.#unloadData(extensionUUID),
472 * Return a branch new StoreData instance given an extension.
474 * @param {Extension} extension
475 * @returns {StoreData}
477 #getDefaults(extension) {
478 return new StoreData({
479 extVersion: extension.version,
484 * Return the path to the store file given the extension's uuid.
486 * @param {string} extensionUUID
487 * @returns {string} Full path to the store file for the extension.
489 #getStoreFilePath(extensionUUID) {
490 return PathUtils.join(
491 Services.dirsvc.get("ProfD", Ci.nsIFile).path,
493 `${extensionUUID}${RULES_STORE_FILEEXT}`
497 #ensureStoreDirectory(extensionUUID) {
498 // Currently all extensions share the same directory, so we can re-use this promise across all
499 // `#ensureStoreDirectory` calls.
500 if (this._ensureStoreDirectoryPromise === null) {
501 const file = this.#getStoreFilePath(extensionUUID);
502 this._ensureStoreDirectoryPromise = IOUtils.makeDirectory(
503 PathUtils.parent(file),
505 ignoreExisting: true,
506 createAncestors: true,
510 return this._ensureStoreDirectoryPromise;
513 async #getDataPromise(extension) {
514 let dataPromise = this._dataPromises.get(extension.uuid);
516 if (extension.hasShutdown) {
518 `DNR store data loading aborted, the extension is already shutting down: ${extension.id}`
522 this.unloadOnShutdown(extension);
523 dataPromise = this.#readData(extension);
524 this._dataPromises.set(extension.uuid, dataPromise);
530 * Reads the store file for the given extensions and all rules
531 * for the enabled static ruleset ids listed in the store file.
533 * @param {Extension} extension
534 * @param {Array<string>} [enabledRulesetIds]
535 * An optional array of enabled ruleset ids to be loaded
536 * (used to load a specific group of static rulesets,
537 * either when the list of static rules needs to be recreated based
538 * on the enabled rulesets, or when the extension is
539 * changing the enabled rulesets using the `updateEnabledRulesets`
541 * @returns {Promise<Map<ruleset_id, EnabledStaticRuleset>>}
542 * map of the enabled static rulesets by ruleset_id.
544 async #getManifestStaticRulesets(
547 enabledRulesetIds = null,
548 availableStaticRuleCount = lazy.ExtensionDNRLimits
549 .GUARANTEED_MINIMUM_STATIC_RULES,
550 isUpdateEnabledRulesets = false,
553 // Map<ruleset_id, EnabledStaticRuleset>}
554 const rulesets = new Map();
556 const ruleResources =
557 extension.manifest.declarative_net_request?.rule_resources;
558 if (!Array.isArray(ruleResources)) {
563 MAX_NUMBER_OF_ENABLED_STATIC_RULESETS,
564 // Warnings on MAX_NUMBER_OF_STATIC_RULESETS are already
565 // reported (see ExtensionDNR.validateManifestEntry, called
566 // from the DNR API onManifestEntry callback).
567 } = lazy.ExtensionDNRLimits;
569 for (let [idx, { id, enabled, path }] of ruleResources.entries()) {
570 // Retrieve the file path from the normalized path.
571 path = Services.io.newURI(path).filePath;
573 // If passed enabledRulesetIds is used to determine if the enabled
574 // rules in the manifest should be overridden from the list of
575 // enabled static rulesets stored on disk.
576 if (Array.isArray(enabledRulesetIds)) {
577 enabled = enabledRulesetIds.includes(id);
580 // Duplicated ruleset ids are validated as part of the JSONSchema validation,
581 // here we log a warning to signal that we are ignoring it if when the validation
582 // error isn't strict (e.g. for non temporarily installed, which shouldn't normally
583 // hit in the long run because we can also validate it before signing the extension).
584 if (rulesets.has(id)) {
586 `Disabled static ruleset with duplicated ruleset_id "${id}"`
591 if (enabled && rulesets.size >= MAX_NUMBER_OF_ENABLED_STATIC_RULESETS) {
592 // This is technically reported from the manifest validation, as a warning
593 // on extension installed non temporarily, and so checked and logged here
594 // in case we are hitting it while loading the enabled rulesets.
596 `Ignoring enabled static ruleset exceeding the MAX_NUMBER_OF_ENABLED_STATIC_RULESETS limit (${MAX_NUMBER_OF_ENABLED_STATIC_RULESETS}): ruleset_id "${id}" (extension: "${extension.id}")`
603 (await extension.readJSON(path).catch(err => {
606 extension.packagingError(
607 `Reading declarative_net_request static rules file ${path}: ${err.message}`
611 // Skip rulesets that are not enabled or can't be enabled (e.g. if we got error on loading or
612 // parsing the rules JSON file).
617 if (!Array.isArray(rawRules)) {
618 extension.packagingError(
619 `Reading declarative_net_request static rules file ${path}: rules file must contain an Array of rules`
624 // TODO(Bug 1803369): consider to only report the errors and warnings about invalid static rules for
625 // temporarily installed extensions (chrome only shows them for unpacked extensions).
626 const logRuleValidationError = err => extension.packagingWarning(err);
628 const validatedRules = this.#getValidatedRules(extension, id, rawRules, {
629 logRuleValidationError,
632 // NOTE: this is currently only accounting for valid rules because
633 // only the valid rules will be actually be loaded. Reconsider if
634 // we should instead also account for the rules that have been
635 // ignored as invalid.
636 if (availableStaticRuleCount - validatedRules.length < 0) {
637 if (isUpdateEnabledRulesets) {
638 throw new ExtensionError(
639 "updateEnabledRulesets request is exceeding the available static rule count"
643 // TODO(Bug 1803363): consider collect telemetry.
645 `Ignoring static ruleset exceeding the available static rule count: ruleset_id "${id}" (extension: "${extension.id}")`
647 // TODO: currently ignoring the current ruleset but would load the one that follows if it
648 // fits in the available rule count when loading the rule on extension startup,
649 // should it stop loading additional rules instead?
652 availableStaticRuleCount -= validatedRules.length;
654 rulesets.set(id, { idx, rules: validatedRules });
661 * Returns an array of validated and normalized Rule instances given an array
662 * of raw rules data (e.g. in form of plain objects read from the static rules
663 * JSON files or the dynamicRuleset property from the extension DNR store data).
665 * @param {Extension} extension
666 * @param {string} rulesetId
667 * @param {Array<object>} rawRules
668 * @param {object} options
669 * @param {Function} [options.logRuleValidationError]
670 * an optional callback to call for logging the
671 * validation errors, defaults to use Cu.reportError
672 * (but getManifestStaticRulesets overrides it to use
673 * extensions.packagingWarning instead).
675 * @returns {Array<Rule>}
681 { logRuleValidationError = err => Cu.reportError(err) } = {}
683 const ruleValidator = new lazy.ExtensionDNR.RuleValidator([]);
684 // Normalize rules read from JSON.
685 const validationContext = {
686 url: extension.baseURI.spec,
687 principal: extension.principal,
688 logError: logRuleValidationError,
690 manifestVersion: extension.manifestVersion,
693 // TODO(Bug 1803369): consider to also include the rule id if one was available.
694 const getInvalidRuleMessage = (ruleIndex, msg) =>
695 `Invalid rule at index ${ruleIndex} from ruleset "${rulesetId}", ${msg}`;
697 for (const [rawIndex, rawRule] of rawRules.entries()) {
699 const normalizedRule = lazy.Schemas.normalize(
701 "declarativeNetRequest.Rule",
704 if (normalizedRule.value) {
705 ruleValidator.addRules([normalizedRule.value]);
707 logRuleValidationError(
708 getInvalidRuleMessage(
710 normalizedRule.error ?? "Unexpected undefined rule"
715 logRuleValidationError(
716 getInvalidRuleMessage(rawIndex, "An unexpected error occurred")
721 // TODO(Bug 1803369): consider including an index in the invalid rules warnings.
722 if (ruleValidator.getFailures().length) {
723 logRuleValidationError(
724 `Invalid rules found in ruleset "${rulesetId}": ${ruleValidator
731 return ruleValidator.getValidatedRules();
734 #hasInstallOrUpdateStartupReason(extension) {
735 switch (extension.startupReason) {
736 case "ADDON_INSTALL":
737 case "ADDON_UPGRADE":
738 case "ADDON_DOWNGRADE":
746 * Load and add the DNR stored rules to the RuleManager instance for the given
749 * @param {Extension} extension
750 * @returns {Promise<void>}
752 async #initExtension(extension) {
753 // - on new installs the stored rules should be recreated from scratch
754 // (and any stale previously stored data to be ignored)
755 // - on upgrades/downgrades:
756 // - the dynamic rules are expected to be preserved
757 // - the static rules are expected to be refreshed from the new
758 // manifest data (also the enabled rulesets are expected to be
759 // reset to the state described in the manifest)
761 // TODO(Bug 1803369): consider also setting to true if the extension is installed temporarily.
762 if (this.#hasInstallOrUpdateStartupReason(extension)) {
763 // Reset the stored static rules on addon updates.
764 await StartupCache.delete(extension, "dnr", "hasEnabledStaticRules");
767 const hasEnabledStaticRules = await StartupCache.get(
769 ["dnr", "hasEnabledStaticRules"],
771 const staticRulesets = await this.getEnabledStaticRulesets(extension);
773 return staticRulesets.size;
776 const hasDynamicRules = await StartupCache.get(
778 ["dnr", "hasDynamicRules"],
780 const dynamicRuleset = await this.getDynamicRules(extension);
782 return dynamicRuleset.length;
786 if (hasEnabledStaticRules || hasDynamicRules) {
787 await this.#getDataPromise(extension);
788 this.updateRulesetManager(extension, {
789 updateStaticRulesets: hasEnabledStaticRules,
790 updateDynamicRuleset: hasDynamicRules,
796 * Read the stored data for the given extension, either from:
797 * - store file (if available and not detected as a data schema downgrade)
798 * - manifest file and packaged ruleset JSON files (if there was no valid stored data found)
800 * This private method is only called from #getDataPromise, which caches the return value
803 * @param {Extension} extension
805 * @returns {Promise<StoreData>}
807 async #readData(extension) {
808 // Try to load the data stored in the json file.
809 let result = await this.#readStoreData(extension);
811 // Reset the stored data if a data schema version downgrade has been
812 // detected (this should only be hit on downgrades if the user have
813 // also explicitly passed --allow-downgrade CLI option).
814 if (result && result.version > StoreData.VERSION) {
816 `Unsupport DNR store schema version downgrade: resetting stored data for ${extension.id}`
821 // Use defaults and extension manifest if no data stored was found
822 // (or it got reset due to an unsupported profile downgrade being detected).
824 // We don't have any data stored, load the static rules from the manifest.
825 result = this.#getDefaults(extension);
826 // Initialize the staticRules data from the manifest.
827 result.setStaticRulesets(
828 await this.#getManifestStaticRulesets(extension)
832 // TODO: handle DNR store schema changes here when the StoreData.VERSION is being bumped.
833 // if (result && result.version < StoreData.VERSION) {
834 // result = this.upgradeStoreDataSchema(result);
837 // The extension has already shutting down and we may already got past
838 // the unloadData cleanup (given that there is still a promise in
839 // the _dataPromises Map).
840 if (extension.hasShutdown && !this._dataPromises.has(extension.uuid)) {
842 `DNR store data loading aborted, the extension is already shutting down: ${extension.id}`
846 this._data.set(extension.uuid, result);
852 * Reads the store file for the given extensions and all rules
853 * for the enabled static ruleset ids listed in the store file.
855 * @param {Extension} extension
857 * @returns {Promise<StoreData|void>}
859 async #readStoreData(extension) {
860 // TODO(Bug 1803363): record into Glean telemetry DNR RulesetsStore store load time.
861 let file = this.#getStoreFilePath(extension.uuid);
863 let isCorrupted = false;
864 let storeFileFound = false;
866 data = await IOUtils.readJSON(file, { decompress: true });
867 storeFileFound = true;
869 if (!(DOMException.isInstance(e) && e.name === "NotFoundError")) {
872 storeFileFound = true;
874 // TODO(Bug 1803363) record store read errors in telemetry scalar.
877 // Reset data read from disk if its type isn't the expected one.
880 !Array.isArray(data.staticRulesets) ||
881 // DNR data stored in 109 would not have any dynamicRuleset
882 // property and so don't consider the data corrupted if
883 // there isn't any dynamicRuleset property at all.
884 ("dynamicRuleset" in data && !Array.isArray(data.dynamicRuleset));
886 if (isCorrupted && storeFileFound) {
887 // Wipe the corrupted data and backup the corrupted file.
890 let uniquePath = await IOUtils.createUniqueFile(
891 PathUtils.parent(file),
892 PathUtils.filename(file) + ".corrupt",
896 `Detected corrupted DNR store data for ${extension.id}, renaming store data file to ${uniquePath}`
898 await IOUtils.move(file, uniquePath);
908 const resetStaticRulesets =
909 // Reset the static rulesets on install or updating the extension.
911 // NOTE: this method is called only once and its return value cached in
912 // memory for the entire lifetime of the extension and so we don't need
913 // to store any flag to avoid resetting the static rulesets more than
914 // once for the same Extension instance.
915 this.#hasInstallOrUpdateStartupReason(extension) ||
916 // Ignore the stored enabled ruleset ids if the current extension version
917 // mismatches the version the store data was generated from.
918 data.extVersion !== extension.version;
920 if (resetStaticRulesets) {
921 data.staticRulesets = undefined;
922 data.extVersion = extension.version;
925 // If the data is being loaded for a new addon install, make sure to clear
926 // any potential stale dynamic rules stored on disk.
928 // NOTE: this is expected to only be hit if there was a failure to cleanup
929 // state data upon uninstall (e.g. in case the machine shutdowns or
930 // Firefox crashes before we got to update the data stored on disk).
931 if (extension.startupReason === "ADDON_INSTALL") {
932 data.dynamicRuleset = [];
935 // In the JSON stored data we only store the enabled rulestore_id and
936 // the actual rules have to be loaded.
937 data.staticRulesets = await this.#getManifestStaticRulesets(
939 // Only load the rules from rulesets that are enabled in the stored DNR data,
940 // if the array (eventually empty) of the enabled static rules isn't in the
941 // stored data, then load all the ones enabled in the manifest.
942 { enabledRulesetIds: data.staticRulesets }
945 if (data.dynamicRuleset?.length) {
946 // Make sure all dynamic rules loaded from disk as validated and normalized
947 // (in case they may have been tempered, but also for when we are loading
948 // data stored by a different Firefox version from the one that stored the
949 // data on disk, e.g. in case validation or normalization logic may have been
950 // different in the two Firefox version).
951 const validatedDynamicRules = this.#getValidatedRules(
953 "_dynamic" /* rulesetId */,
958 MAX_NUMBER_OF_DYNAMIC_AND_SESSION_RULES,
959 } = lazy.ExtensionDNRLimits;
962 validatedDynamicRules.length > MAX_NUMBER_OF_DYNAMIC_AND_SESSION_RULES
965 `Ignoring dynamic rules exceeding rule count limits while loading DNR store data for ${extension.id}`
967 data.dynamicRuleset = validatedDynamicRules.slice(
969 MAX_NUMBER_OF_DYNAMIC_AND_SESSION_RULES
973 return new StoreData(data);
977 * Save the data for the given extension on disk.
979 * @param {string} extensionUUID
980 * @returns {Promise<void>}
982 async #saveNow(extensionUUID) {
984 if (!this._dataPromises.has(extensionUUID)) {
986 `Unexpected uninitialized DNR store on saving data for extension uuid "${extensionUUID}"`
989 const storeFile = this.#getStoreFilePath(extensionUUID);
991 const data = this._data.get(extensionUUID);
993 await IOUtils.remove(storeFile, { ignoreAbsent: true });
996 await this.#ensureStoreDirectory(extensionUUID);
997 await IOUtils.writeJSON(storeFile, data, {
998 tmpPath: `${storeFile}.tmp`,
1001 // TODO(Bug 1803363): report jsonData lengths into a telemetry scalar.
1002 // TODO(Bug 1803363): report jsonData time to write into a telemetry scalar.
1004 Cu.reportError(err);
1007 this._savePromises.delete(extensionUUID);
1012 * Unload data for the given extension UUID from memory (e.g. when the extension is disabled or uninstalled),
1013 * waits for a pending save promise to be settled if any.
1015 * NOTE: this method clear the data cached in memory and close the update queue
1016 * and so it should only be called from the extension shutdown handler and
1017 * by the initExtension method before pushing into the update queue for the
1018 * for the extension the initExtension task.
1020 * @param {string} extensionUUID
1021 * @returns {Promise<void>}
1023 async #unloadData(extensionUUID) {
1024 // Wait for the update tasks to have been executed, then
1025 // wait for the data to have been saved and finally unload
1026 // the data cached in memory.
1027 const dataUpdateQueue = this._dataUpdateQueues.has(extensionUUID)
1028 ? this._dataUpdateQueues.get(extensionUUID)
1031 if (dataUpdateQueue) {
1033 await dataUpdateQueue.close();
1035 // Unexpected error on closing the update queue.
1036 Cu.reportError(err);
1038 this._dataUpdateQueues.delete(extensionUUID);
1041 const savePromise = this._savePromises.get(extensionUUID);
1044 this._savePromises.delete(extensionUUID);
1047 this._dataPromises.delete(extensionUUID);
1048 this._data.delete(extensionUUID);
1052 * Internal implementation for updating the dynamic ruleset and enforcing
1053 * dynamic rules count limits.
1055 * Callers ensure that there is never a concurrent call of #updateDynamicRules
1056 * for a given extension, so we can safely modify ruleManager.dynamicRules
1057 * from inside this method, even asynchronously.
1059 * @param {Extension} extension
1060 * @param {object} params
1061 * @param {Array<string>} [params.removeRuleIds=[]]
1062 * @param {Array<Rule>} [params.addRules=[]]
1064 async #updateDynamicRules(extension, { removeRuleIds, addRules }) {
1065 const ruleManager = lazy.ExtensionDNR.getRuleManager(extension);
1066 const ruleValidator = new lazy.ExtensionDNR.RuleValidator(
1067 ruleManager.getDynamicRules()
1069 if (removeRuleIds) {
1070 ruleValidator.removeRuleIds(removeRuleIds);
1073 ruleValidator.addRules(addRules);
1075 let failures = ruleValidator.getFailures();
1076 if (failures.length) {
1077 throw new ExtensionError(failures[0].message);
1080 const { MAX_NUMBER_OF_DYNAMIC_AND_SESSION_RULES } = lazy.ExtensionDNRLimits;
1081 const validatedRules = ruleValidator.getValidatedRules();
1083 if (validatedRules.length > MAX_NUMBER_OF_DYNAMIC_AND_SESSION_RULES) {
1084 throw new ExtensionError(
1085 `updateDynamicRules request is exceeding MAX_NUMBER_OF_DYNAMIC_AND_SESSION_RULES limit (${MAX_NUMBER_OF_DYNAMIC_AND_SESSION_RULES})`
1089 this._data.get(extension.uuid).setDynamicRuleset(validatedRules);
1090 await this.save(extension);
1091 // updateRulesetManager calls ruleManager.setDynamicRules using the
1092 // validated rules assigned above to this._data.
1093 this.updateRulesetManager(extension, {
1094 updateDynamicRuleset: true,
1095 updateStaticRulesets: false,
1100 * Internal implementation for updating the enabled rulesets and enforcing
1101 * static rulesets and rules count limits.
1103 * @param {Extension} extension
1104 * @param {object} params
1105 * @param {Array<string>} [params.disableRulesetIds=[]]
1106 * @param {Array<string>} [params.enableRulesetIds=[]]
1108 async #updateEnabledStaticRulesets(
1110 { disableRulesetIds, enableRulesetIds }
1112 const ruleResources =
1113 extension.manifest.declarative_net_request?.rule_resources;
1114 if (!Array.isArray(ruleResources)) {
1118 const enabledRulesets = await this.getEnabledStaticRulesets(extension);
1119 const updatedEnabledRulesets = new Map();
1120 let disableIds = new Set(disableRulesetIds);
1121 let enableIds = new Set(enableRulesetIds);
1123 // valiate the ruleset ids for existence (which will also reject calls
1124 // including the reserved _session and _dynamic, because static rulesets
1125 // id are validated as part of the manifest validation and they are not
1126 // allowed to start with '_').
1127 const existingIds = new Set(ruleResources.map(rs => rs.id));
1128 const errorOnInvalidRulesetIds = rsIdSet => {
1129 for (const rsId of rsIdSet) {
1130 if (!existingIds.has(rsId)) {
1131 throw new ExtensionError(`Invalid ruleset id: "${rsId}"`);
1135 errorOnInvalidRulesetIds(disableIds);
1136 errorOnInvalidRulesetIds(enableIds);
1138 // Copy into the updatedEnabledRulesets Map any ruleset that is not
1139 // requested to be disabled or is enabled back in the same request.
1140 for (const [rulesetId, ruleset] of enabledRulesets) {
1141 if (!disableIds.has(rulesetId) || enableIds.has(rulesetId)) {
1142 updatedEnabledRulesets.set(rulesetId, ruleset);
1143 enableIds.delete(rulesetId);
1148 MAX_NUMBER_OF_ENABLED_STATIC_RULESETS,
1149 GUARANTEED_MINIMUM_STATIC_RULES,
1150 } = lazy.ExtensionDNRLimits;
1152 const maxNewRulesetsCount =
1153 MAX_NUMBER_OF_ENABLED_STATIC_RULESETS - updatedEnabledRulesets.size;
1155 if (enableIds.size > maxNewRulesetsCount) {
1156 // Log an error for the developer.
1157 throw new ExtensionError(
1158 `updatedEnabledRulesets request is exceeding MAX_NUMBER_OF_ENABLED_STATIC_RULESETS`
1162 const availableStaticRuleCount =
1163 GUARANTEED_MINIMUM_STATIC_RULES -
1164 Array.from(updatedEnabledRulesets.values()).reduce(
1165 (acc, ruleset) => acc + ruleset.rules.length,
1169 const newRulesets = await this.#getManifestStaticRulesets(extension, {
1170 enabledRulesetIds: Array.from(enableIds),
1171 availableStaticRuleCount,
1172 isUpdateEnabledRulesets: true,
1175 for (const [rulesetId, ruleset] of newRulesets.entries()) {
1176 updatedEnabledRulesets.set(rulesetId, ruleset);
1179 this._data.get(extension.uuid).setStaticRulesets(updatedEnabledRulesets);
1180 await this.save(extension);
1181 this.updateRulesetManager(extension, {
1182 updateDynamicRuleset: false,
1183 updateStaticRulesets: true,
1188 const store = new RulesetsStore();
1190 const requireTestOnlyCallers = () => {
1191 if (!Services.env.exists("XPCSHELL_TEST_PROFILE_DIR")) {
1192 throw new Error("This should only be called from XPCShell tests");
1196 export const ExtensionDNRStore = {
1197 async clearOnUninstall(extensionUUID) {
1198 return store.clearOnUninstall(extensionUUID);
1200 async initExtension(extension) {
1201 await store.initExtension(extension);
1203 async updateDynamicRules(extension, updateRuleOptions) {
1204 await store.updateDynamicRules(extension, updateRuleOptions);
1206 async updateEnabledStaticRulesets(extension, updateRulesetOptions) {
1207 await store.updateEnabledStaticRulesets(extension, updateRulesetOptions);
1209 // Test-only helpers
1210 _getStoreForTesting() {
1211 requireTestOnlyCallers();