1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
5 import { AppConstants } from "resource://gre/modules/AppConstants.sys.mjs";
7 import { Downloader } from "resource://services-settings/Attachments.sys.mjs";
11 ChromeUtils.defineESModuleGetters(lazy, {
12 ClientEnvironmentBase:
13 "resource://gre/modules/components-utils/ClientEnvironment.sys.mjs",
14 Database: "resource://services-settings/Database.sys.mjs",
15 IDBHelpers: "resource://services-settings/IDBHelpers.sys.mjs",
16 KintoHttpClient: "resource://services-common/kinto-http-client.sys.mjs",
17 ObjectUtils: "resource://gre/modules/ObjectUtils.sys.mjs",
19 "resource://services-settings/RemoteSettingsWorker.sys.mjs",
20 SharedUtils: "resource://services-settings/SharedUtils.sys.mjs",
21 UptakeTelemetry: "resource://services-common/uptake-telemetry.sys.mjs",
22 Utils: "resource://services-settings/Utils.sys.mjs",
25 const TELEMETRY_COMPONENT = "remotesettings";
27 ChromeUtils.defineLazyGetter(lazy, "console", () => lazy.Utils.log);
30 * cacheProxy returns an object Proxy that will memoize properties of the target.
31 * @param {Object} target the object to wrap.
34 function cacheProxy(target) {
35 const cache = new Map();
36 return new Proxy(target, {
37 get(target, prop, receiver) {
38 if (!cache.has(prop)) {
39 cache.set(prop, target[prop]);
41 return cache.get(prop);
47 * Minimalist event emitter.
49 * Note: we don't use `toolkit/modules/EventEmitter` because **we want** to throw
50 * an error when a listener fails to execute.
54 this._listeners = new Map();
55 for (const event of events) {
56 this._listeners.set(event, []);
61 * Event emitter: will execute the registered listeners in the order and
64 * @param {string} event the event name
65 * @param {Object} payload the event payload to call the listeners with
67 async emit(event, payload) {
68 const callbacks = this._listeners.get(event);
70 for (const cb of callbacks) {
83 return this._listeners.has(event) && !!this._listeners.get(event).length;
87 if (!this._listeners.has(event)) {
88 throw new Error(`Unknown event type ${event}`);
90 this._listeners.get(event).push(callback);
93 off(event, callback) {
94 if (!this._listeners.has(event)) {
95 throw new Error(`Unknown event type ${event}`);
97 const callbacks = this._listeners.get(event);
98 const i = callbacks.indexOf(callback);
100 throw new Error(`Unknown callback`);
102 callbacks.splice(i, 1);
107 class APIError extends Error {}
109 class NetworkError extends APIError {
111 super(`Network error: ${e}`, { cause: e });
112 this.name = "NetworkError";
116 class NetworkOfflineError extends APIError {
118 super("Network is offline");
119 this.name = "NetworkOfflineError";
123 class ServerContentParseError extends APIError {
125 super(`Cannot parse server content: ${e}`, { cause: e });
126 this.name = "ServerContentParseError";
130 class BackendError extends APIError {
132 super(`Backend error: ${e}`, { cause: e });
133 this.name = "BackendError";
137 class BackoffError extends APIError {
139 super(`Server backoff: ${e}`, { cause: e });
140 this.name = "BackoffError";
144 class TimeoutError extends APIError {
146 super(`API timeout: ${e}`, { cause: e });
147 this.name = "TimeoutError";
151 class StorageError extends Error {
153 super(`Storage error: ${e}`, { cause: e });
154 this.name = "StorageError";
158 class InvalidSignatureError extends Error {
159 constructor(cid, x5u) {
160 let message = `Invalid content signature (${cid})`;
162 const chain = x5u.split("/").pop();
163 message += ` using '${chain}'`;
166 this.name = "InvalidSignatureError";
170 class MissingSignatureError extends InvalidSignatureError {
173 this.message = `Missing signature (${cid})`;
174 this.name = "MissingSignatureError";
178 class CorruptedDataError extends InvalidSignatureError {
181 this.message = `Corrupted local data (${cid})`;
182 this.name = "CorruptedDataError";
186 class UnknownCollectionError extends Error {
188 super(`Unknown Collection "${cid}"`);
189 this.name = "UnknownCollectionError";
193 class AttachmentDownloader extends Downloader {
194 constructor(client) {
195 super(client.bucketName, client.collectionName);
196 this._client = client;
201 get: async attachmentId => {
202 return this._client.db.getAttachment(attachmentId);
204 set: async (attachmentId, attachment) => {
205 return this._client.db.saveAttachment(attachmentId, attachment);
207 delete: async attachmentId => {
208 return this._client.db.saveAttachment(attachmentId, null);
210 prune: async excludeIds => {
211 return this._client.db.pruneAttachments(excludeIds);
214 Object.defineProperty(this, "cacheImpl", { value: cacheImpl });
219 * Download attachment and report Telemetry on failure.
221 * @see Downloader.download
223 async download(record, options) {
225 // Explicitly await here to ensure we catch a network error.
226 return await super.download(record, options);
228 // Report download error.
229 let status = lazy.UptakeTelemetry.STATUS.DOWNLOAD_ERROR;
230 if (lazy.Utils.isOffline) {
231 status = lazy.UptakeTelemetry.STATUS.NETWORK_OFFLINE_ERROR;
232 } else if (/NetworkError/.test(err.message)) {
233 status = lazy.UptakeTelemetry.STATUS.NETWORK_ERROR;
235 // If the file failed to be downloaded, report it as such in Telemetry.
236 await lazy.UptakeTelemetry.report(TELEMETRY_COMPONENT, status, {
237 source: this._client.identifier,
244 * Delete all downloaded records attachments.
246 * Note: the list of attachments to be deleted is based on the
247 * current list of records.
250 let allRecords = await this._client.db.list();
253 .filter(r => !!r.attachment)
255 Promise.all([this.deleteDownloaded(r), this.deleteFromDisk(r)])
261 export class RemoteSettingsClient extends EventEmitter {
262 static get APIError() {
265 static get NetworkError() {
268 static get NetworkOfflineError() {
269 return NetworkOfflineError;
271 static get ServerContentParseError() {
272 return ServerContentParseError;
274 static get BackendError() {
277 static get BackoffError() {
280 static get TimeoutError() {
283 static get StorageError() {
286 static get InvalidSignatureError() {
287 return InvalidSignatureError;
289 static get MissingSignatureError() {
290 return MissingSignatureError;
292 static get CorruptedDataError() {
293 return CorruptedDataError;
295 static get UnknownCollectionError() {
296 return UnknownCollectionError;
302 bucketName = AppConstants.REMOTE_SETTINGS_DEFAULT_BUCKET,
306 keepAttachmentsIds = [],
310 // Remote Settings cannot be used in child processes (no access to disk,
311 // easily killed, isolated observer notifications etc.).
312 // Since our goal here is to prevent consumers to instantiate while developing their
313 // feature, throwing in Nightly only is enough, and prevents unexpected crashes
314 // in release or beta.
316 !AppConstants.RELEASE_OR_BETA &&
317 Services.appinfo.processType !== Services.appinfo.PROCESS_TYPE_DEFAULT
320 "Cannot instantiate Remote Settings client in child processes."
324 super(["sync"]); // emitted events
326 this.collectionName = collectionName;
327 // Client is constructed with the raw bucket name (eg. "main", "security-state", "blocklist")
328 // The `bucketName` will contain the `-preview` suffix if the preview mode is enabled.
329 this.bucketName = lazy.Utils.actualBucketName(bucketName);
330 this.signerName = signerName;
331 this.filterFunc = filterFunc;
332 this.localFields = localFields;
333 this.keepAttachmentsIds = keepAttachmentsIds;
334 this._lastCheckTimePref = lastCheckTimePref;
335 this._verifier = null;
336 this._syncRunning = false;
338 // This attribute allows signature verification to be disabled, when running tests
339 // or when pulling data from a dev server.
340 this.verifySignature = AppConstants.REMOTE_SETTINGS_VERIFY_SIGNATURE;
342 ChromeUtils.defineLazyGetter(
345 () => new lazy.Database(this.identifier)
348 ChromeUtils.defineLazyGetter(
351 () => new AttachmentDownloader(this)
356 * Internal method to refresh the client bucket name after the preview mode
359 * See `RemoteSettings.enabledPreviewMode()`.
361 refreshBucketName() {
362 this.bucketName = lazy.Utils.actualBucketName(this.bucketName);
363 this.db.identifier = this.identifier;
367 return `${this.bucketName}/${this.collectionName}`;
370 get lastCheckTimePref() {
372 this._lastCheckTimePref ||
373 `services.settings.${this.bucketName}.${this.collectionName}.last_check`
378 const api = new lazy.KintoHttpClient(lazy.Utils.SERVER_URL, {
379 fetchFunc: lazy.Utils.fetch, // Use fetch() wrapper.
381 return api.bucket(this.bucketName).collection(this.collectionName);
385 * Retrieve the collection timestamp for the last synchronization.
386 * This is an opaque and comparable value assigned automatically by
390 * The timestamp in milliseconds, returns -1 if retrieving
391 * the timestamp from the kinto collection fails.
393 async getLastModified() {
396 timestamp = await this.db.getLastModified();
399 `Error retrieving the getLastModified timestamp from ${this.identifier} RemoteSettingsClient`,
410 * @param {Object} options The options object.
411 * @param {Object} options.filters Filter the results (default: `{}`).
412 * @param {String} options.order The order to apply (eg. `"-last_modified"`).
413 * @param {boolean} options.dumpFallback Fallback to dump data if read of local DB fails (default: `true`).
414 * @param {boolean} options.emptyListFallback Fallback to empty list if no dump data and read of local DB fails (default: `true`).
415 * @param {boolean} options.loadDumpIfNewer Use dump data if it is newer than local data (default: `true`).
416 * @param {boolean} options.forceSync Always synchronize from server before returning results (default: `false`).
417 * @param {boolean} options.syncIfEmpty Synchronize from server if local data is empty (default: `true`).
418 * @param {boolean} options.verifySignature Verify the signature of the local data (default: `false`).
421 async get(options = {}) {
424 order = "", // not sorted by default.
426 emptyListFallback = true,
428 loadDumpIfNewer = true,
431 let { verifySignature = false } = options;
433 const hasParallelCall = !!this._importingPromise;
436 let lastModified = forceSync ? null : await this.db.getLastModified();
437 let hasLocalData = lastModified !== null;
440 if (!this._importingPromise) {
441 this._importingPromise = (async () => {
442 await this.sync({ sendEvents: false, trigger: "forced" });
443 return true; // No need to re-verify signature after sync.
446 } else if (syncIfEmpty && !hasLocalData) {
447 // .get() was called before we had the chance to synchronize the local database.
448 // We'll try to avoid returning an empty list.
449 if (!this._importingPromise) {
450 // Prevent parallel loading when .get() is called multiple times.
451 this._importingPromise = (async () => {
452 const importedFromDump = lazy.Utils.LOAD_DUMPS
453 ? await this._importJSONDump()
455 if (importedFromDump < 0) {
456 // There is no JSON dump to load, force a synchronization from the server.
457 // We don't want the "sync" event to be sent, since some consumers use `.get()`
458 // in "sync" callbacks. See Bug 1761953
460 `${this.identifier} Local DB is empty, pull data from server`
462 await this.sync({ loadDump: false, sendEvents: false });
464 // Return `true` to indicate we don't need to `verifySignature`,
465 // since a trusted dump was loaded or a signature verification
466 // happened during synchronization.
470 lazy.console.debug(`${this.identifier} Awaiting existing import.`);
472 } else if (hasLocalData && loadDumpIfNewer) {
473 // Check whether the local data is older than the packaged dump.
474 // If it is, load the packaged dump (which overwrites the local data).
475 let lastModifiedDump = await lazy.Utils.getLocalDumpLastModified(
479 if (lastModified < lastModifiedDump) {
481 `${this.identifier} Local DB is stale (${lastModified}), using dump instead (${lastModifiedDump})`
483 if (!this._importingPromise) {
484 // As part of importing, any existing data is wiped.
485 this._importingPromise = (async () => {
486 const importedFromDump = await this._importJSONDump();
487 // Return `true` to skip signature verification if a dump was found.
488 // The dump can be missing if a collection is listed in the timestamps summary file,
489 // because its dump is present in the source tree, but the dump was not
490 // included in the `package-manifest.in` file. (eg. android, thunderbird)
491 return importedFromDump >= 0;
494 lazy.console.debug(`${this.identifier} Awaiting existing import.`);
499 if (this._importingPromise) {
501 if (await this._importingPromise) {
502 // No need to verify signature, because either we've just loaded a trusted
503 // dump (here or in a parallel call), or it was verified during sync.
504 verifySignature = false;
507 if (!hasParallelCall) {
508 // Sync or load dump failed. Throw.
511 // Report error, but continue because there could have been data
512 // loaded from a parallel call.
515 // then delete this promise again, as now we should have local data:
516 delete this._importingPromise;
520 // Read from the local DB.
521 data = await this.db.list({ filters, order });
523 // If the local DB cannot be read (for unknown reasons, Bug 1649393)
524 // or sync failed, we fallback to the packaged data, and filter/sort in memory.
529 let { data } = await lazy.SharedUtils.loadJSONDump(
534 lazy.console.info(`${this.identifier} falling back to JSON dump`);
535 } else if (emptyListFallback) {
537 `${this.identifier} no dump fallback, return empty list`
541 // Obtaining the records failed, there is no dump, and we don't fallback
542 // to an empty list. Throw the original error.
545 if (!lazy.ObjectUtils.isEmpty(filters)) {
546 data = data.filter(r => lazy.Utils.filterObject(filters, r));
549 data = lazy.Utils.sortObjects(order, data);
551 // No need to verify signature on JSON dumps.
552 // If local DB cannot be read, then we don't even try to do anything,
553 // we return results early.
554 return this._filterEntries(data);
558 `${this.identifier} ${data.length} records before filtering.`
561 if (verifySignature) {
563 `${this.identifier} verify signature of local data on read`
565 const allData = lazy.ObjectUtils.isEmpty(filters)
567 : await this.db.list();
568 const localRecords = allData.map(r => this._cleanLocalFields(r));
569 const timestamp = await this.db.getLastModified();
570 let metadata = await this.db.getMetadata();
571 if (syncIfEmpty && lazy.ObjectUtils.isEmpty(metadata)) {
572 // No sync occured yet, may have records from dump but no metadata.
573 // We don't want the "sync" event to be sent, since some consumers use `.get()`
574 // in "sync" callbacks. See Bug 1761953
575 await this.sync({ loadDump: false, sendEvents: false });
576 metadata = await this.db.getMetadata();
578 // Will throw MissingSignatureError if no metadata and `syncIfEmpty` is false.
579 await this._validateCollectionSignature(
586 // Filter the records based on `this.filterFunc` results.
587 const final = await this._filterEntries(data);
589 `${this.identifier} ${final.length} records after filtering.`
595 * Synchronize the local database with the remote server.
597 * @param {Object} options See #maybeSync() options.
599 async sync(options) {
600 // We want to know which timestamp we are expected to obtain in order to leverage
601 // cache busting. We don't provide ETag because we don't want a 304.
602 const { changes } = await lazy.Utils.fetchLatestChanges(
603 lazy.Utils.SERVER_URL,
606 collection: this.collectionName,
607 bucket: this.bucketName,
611 if (changes.length === 0) {
612 throw new RemoteSettingsClient.UnknownCollectionError(this.identifier);
614 // According to API, there will be one only (fail if not).
615 const [{ last_modified: expectedTimestamp }] = changes;
617 return this.maybeSync(expectedTimestamp, { ...options, trigger: "forced" });
621 * Synchronize the local database with the remote server, **only if necessary**.
623 * @param {int} expectedTimestamp the lastModified date (on the server) for the remote collection.
624 * This will be compared to the local timestamp, and will be used for
625 * cache busting if local data is out of date.
626 * @param {Object} options additional advanced options.
627 * @param {bool} options.loadDump load initial dump from disk on first sync (default: true if server is prod)
628 * @param {bool} options.sendEvents send `"sync"` events (default: `true`)
629 * @param {string} options.trigger label to identify what triggered this sync (eg. ``"timer"``, default: `"manual"`)
630 * @return {Promise} which rejects on sync or process failure.
632 async maybeSync(expectedTimestamp, options = {}) {
633 // Should the clients try to load JSON dump? (mainly disabled in tests)
635 loadDump = lazy.Utils.LOAD_DUMPS,
640 // Make sure we don't run several synchronizations in parallel, mainly
641 // in order to avoid race conditions in "sync" events listeners.
642 if (this._syncRunning) {
643 lazy.console.warn(`${this.identifier} sync already running`);
647 // Prevent network requests and IndexedDB calls to be initiated
649 if (Services.startup.shuttingDown) {
650 lazy.console.warn(`${this.identifier} sync interrupted by shutdown`);
654 this._syncRunning = true;
656 let importedFromDump = [];
657 const startedAt = new Date();
658 let reportStatus = null;
659 let thrownError = null;
661 // If network is offline, we can't synchronize.
662 if (lazy.Utils.isOffline) {
663 throw new RemoteSettingsClient.NetworkOfflineError();
666 // Read last timestamp and local data before sync.
667 let collectionLastModified = await this.db.getLastModified();
668 const allData = await this.db.list();
669 // Local data can contain local fields, strip them.
670 let localRecords = allData.map(r => this._cleanLocalFields(r));
671 const localMetadata = await this.db.getMetadata();
673 // If there is no data currently in the collection, attempt to import
674 // initial data from the application defaults.
675 // This allows to avoid synchronizing the whole collection content on
677 if (!collectionLastModified && loadDump) {
679 const imported = await this._importJSONDump();
680 // The worker only returns an integer. List the imported records to build the sync event.
683 `${this.identifier} ${imported} records loaded from JSON dump`
685 importedFromDump = await this.db.list();
686 // Local data is the data loaded from dump. We will need this later
687 // to compute the sync result.
688 localRecords = importedFromDump;
690 collectionLastModified = await this.db.getLastModified();
698 // Is local timestamp up to date with the server?
699 if (expectedTimestamp == collectionLastModified) {
700 lazy.console.debug(`${this.identifier} local data is up-to-date`);
701 reportStatus = lazy.UptakeTelemetry.STATUS.UP_TO_DATE;
703 // If the data is up-to-date but don't have metadata (records loaded from dump),
704 // we fetch them and validate the signature immediately.
705 if (this.verifySignature && lazy.ObjectUtils.isEmpty(localMetadata)) {
706 lazy.console.debug(`${this.identifier} pull collection metadata`);
707 const metadata = await this.httpClient().getData({
708 query: { _expected: expectedTimestamp },
710 await this.db.importChanges(metadata);
711 // We don't bother validating the signature if the dump was just loaded. We do
712 // if the dump was loaded at some other point (eg. from .get()).
713 if (this.verifySignature && !importedFromDump.length) {
715 `${this.identifier} verify signature of local data`
717 await this._validateCollectionSignature(
719 collectionLastModified,
725 // Since the data is up-to-date, if we didn't load any dump then we're done here.
726 if (!importedFromDump.length) {
729 // Otherwise we want to continue with sending the sync event to notify about the created records.
731 current: importedFromDump,
732 created: importedFromDump,
737 // Local data is either outdated or tampered.
738 // In both cases we will fetch changes from server,
739 // and make sure we overwrite local data.
740 syncResult = await this._importChanges(
742 collectionLastModified,
746 if (sendEvents && this.hasListeners("sync")) {
747 // If we have listeners for the "sync" event, then compute the lists of changes.
748 // The records imported from the dump should be considered as "created" for the
750 const importedById = importedFromDump.reduce((acc, r) => {
754 // Deleted records should not appear as created.
755 syncResult.deleted.forEach(r => importedById.delete(r.id));
756 // Records from dump that were updated should appear in their newest form.
757 syncResult.updated.forEach(u => {
758 if (importedById.has(u.old.id)) {
759 importedById.set(u.old.id, u.new);
762 syncResult.created = syncResult.created.concat(
763 Array.from(importedById.values())
767 // When triggered from the daily timer, and if the sync was successful, and once
768 // all sync listeners have been executed successfully, we prune potential
769 // obsolete attachments that may have been left in the local cache.
770 if (trigger == "timer") {
771 const deleted = await this.attachments.prune(
772 this.keepAttachmentsIds
776 `${this.identifier} Pruned ${deleted} obsolete attachments`
782 if (e instanceof InvalidSignatureError) {
783 // Signature verification failed during synchronization.
785 e instanceof CorruptedDataError
786 ? lazy.UptakeTelemetry.STATUS.CORRUPTION_ERROR
787 : lazy.UptakeTelemetry.STATUS.SIGNATURE_ERROR;
788 // If sync fails with a signature error, it's likely that our
789 // local data has been modified in some way.
790 // We will attempt to fix this by retrieving the whole
791 // remote collection.
794 `${this.identifier} Signature verified failed. Retry from scratch`
796 syncResult = await this._importChanges(
798 collectionLastModified,
804 // If the signature fails again, or if an error occured during wiping out the
805 // local data, then we report it as a *signature retry* error.
806 reportStatus = lazy.UptakeTelemetry.STATUS.SIGNATURE_RETRY_ERROR;
810 // The sync has thrown for other reason than signature verification.
811 // Obtain a more precise error than original one.
812 const adjustedError = this._adjustedError(e);
813 // Default status for errors at this step is SYNC_ERROR.
814 reportStatus = this._telemetryFromError(adjustedError, {
815 default: lazy.UptakeTelemetry.STATUS.SYNC_ERROR,
821 // Filter the synchronization results using `filterFunc` (ie. JEXL).
822 const filteredSyncResult = await this._filterSyncResult(syncResult);
823 // If every changed entry is filtered, we don't even fire the event.
824 if (filteredSyncResult) {
826 await this.emit("sync", { data: filteredSyncResult });
828 reportStatus = lazy.UptakeTelemetry.STATUS.APPLY_ERROR;
833 `All changes are filtered by JEXL expressions for ${this.identifier}`
839 // Obtain a more precise error than original one.
840 const adjustedError = this._adjustedError(e);
841 // If browser is shutting down, then we can report a specific status.
842 // (eg. IndexedDB will abort transactions)
843 if (Services.startup.shuttingDown) {
844 reportStatus = lazy.UptakeTelemetry.STATUS.SHUTDOWN_ERROR;
846 // If no Telemetry status was determined yet (ie. outside sync step),
847 // then introspect error, default status at this step is UNKNOWN.
848 else if (reportStatus == null) {
849 reportStatus = this._telemetryFromError(adjustedError, {
850 default: lazy.UptakeTelemetry.STATUS.UNKNOWN_ERROR,
855 const durationMilliseconds = new Date() - startedAt;
856 // No error was reported, this is a success!
857 if (reportStatus === null) {
858 reportStatus = lazy.UptakeTelemetry.STATUS.SUCCESS;
860 // Report success/error status to Telemetry.
862 source: this.identifier,
864 duration: durationMilliseconds,
866 // In Bug 1617133, we will try to break down specific errors into
867 // more precise statuses by reporting the JavaScript error name
868 // ("TypeError", etc.) to Telemetry on Nightly.
869 const channel = lazy.UptakeTelemetry.Policy.getChannel();
871 thrownError !== null &&
872 channel == "nightly" &&
874 lazy.UptakeTelemetry.STATUS.SYNC_ERROR,
875 lazy.UptakeTelemetry.STATUS.CUSTOM_1_ERROR, // IndexedDB.
876 lazy.UptakeTelemetry.STATUS.UNKNOWN_ERROR,
877 lazy.UptakeTelemetry.STATUS.SHUTDOWN_ERROR,
878 ].includes(reportStatus)
880 // List of possible error names for IndexedDB:
881 // https://searchfox.org/mozilla-central/rev/49ed791/dom/base/DOMException.cpp#28-53
882 reportArgs = { ...reportArgs, errorName: thrownError.name };
885 await lazy.UptakeTelemetry.report(
891 lazy.console.debug(`${this.identifier} sync status is ${reportStatus}`);
892 this._syncRunning = false;
897 * Return a more precise error instance, based on the specified
898 * error and its message.
899 * @param {Error} e the original error
903 if (/unparseable/.test(e.message)) {
904 return new RemoteSettingsClient.ServerContentParseError(e);
906 if (/NetworkError/.test(e.message)) {
907 return new RemoteSettingsClient.NetworkError(e);
909 if (/Timeout/.test(e.message)) {
910 return new RemoteSettingsClient.TimeoutError(e);
912 if (/HTTP 5??/.test(e.message)) {
913 return new RemoteSettingsClient.BackendError(e);
915 if (/Backoff/.test(e.message)) {
916 return new RemoteSettingsClient.BackoffError(e);
919 // Errors from kinto.js IDB adapter.
920 e instanceof lazy.IDBHelpers.IndexedDBError ||
921 // Other IndexedDB errors (eg. RemoteSettingsWorker).
922 /IndexedDB/.test(e.message)
924 return new RemoteSettingsClient.StorageError(e);
930 * Determine the Telemetry uptake status based on the specified
933 _telemetryFromError(e, options = { default: null }) {
934 let reportStatus = options.default;
936 if (e instanceof RemoteSettingsClient.NetworkOfflineError) {
937 reportStatus = lazy.UptakeTelemetry.STATUS.NETWORK_OFFLINE_ERROR;
938 } else if (e instanceof lazy.IDBHelpers.ShutdownError) {
939 reportStatus = lazy.UptakeTelemetry.STATUS.SHUTDOWN_ERROR;
940 } else if (e instanceof RemoteSettingsClient.ServerContentParseError) {
941 reportStatus = lazy.UptakeTelemetry.STATUS.PARSE_ERROR;
942 } else if (e instanceof RemoteSettingsClient.NetworkError) {
943 reportStatus = lazy.UptakeTelemetry.STATUS.NETWORK_ERROR;
944 } else if (e instanceof RemoteSettingsClient.TimeoutError) {
945 reportStatus = lazy.UptakeTelemetry.STATUS.TIMEOUT_ERROR;
946 } else if (e instanceof RemoteSettingsClient.BackendError) {
947 reportStatus = lazy.UptakeTelemetry.STATUS.SERVER_ERROR;
948 } else if (e instanceof RemoteSettingsClient.BackoffError) {
949 reportStatus = lazy.UptakeTelemetry.STATUS.BACKOFF;
950 } else if (e instanceof RemoteSettingsClient.StorageError) {
951 reportStatus = lazy.UptakeTelemetry.STATUS.CUSTOM_1_ERROR;
958 * Import the JSON files from services/settings/dump into the local DB.
960 async _importJSONDump() {
961 lazy.console.info(`${this.identifier} try to restore dump`);
962 const result = await lazy.RemoteSettingsWorker.importJSONDump(
967 lazy.console.debug(`${this.identifier} no dump available`);
970 `${this.identifier} imported ${result} records from dump`
977 * Fetch the signature info from the collection metadata and verifies that the
978 * local set of records has the same.
980 * @param {Array<Object>} records The list of records to validate.
981 * @param {int} timestamp The timestamp associated with the list of remote records.
982 * @param {Object} metadata The collection metadata, that contains the signature payload.
985 async _validateCollectionSignature(records, timestamp, metadata) {
986 if (!metadata?.signature) {
987 throw new MissingSignatureError(this.identifier);
990 if (!this._verifier) {
992 "@mozilla.org/security/contentsignatureverifier;1"
993 ].createInstance(Ci.nsIContentSignatureVerifier);
996 // This is a content-signature field from an autograph response.
998 signature: { x5u, signature },
1000 const certChain = await (await lazy.Utils.fetch(x5u)).text();
1001 // Merge remote records with local ones and serialize as canonical JSON.
1002 const serialized = await lazy.RemoteSettingsWorker.canonicalStringify(
1007 lazy.console.debug(`${this.identifier} verify signature using ${x5u}`);
1009 !(await this._verifier.asyncVerifyContentSignature(
1011 "p384ecdsa=" + signature,
1014 lazy.Utils.CERT_CHAIN_ROOT_IDENTIFIER
1017 throw new InvalidSignatureError(this.identifier, x5u);
1022 * This method is in charge of fetching data from server, applying the diff-based
1023 * changes to the local DB, validating the signature, and computing a synchronization
1024 * result with the list of creation, updates, and deletions.
1026 * @param {Array<Object>} localRecords Current list of records in local DB.
1027 * @param {int} localTimestamp Current timestamp in local DB.
1028 * @param {Object} localMetadata Current metadata in local DB.
1029 * @param {int} expectedTimestamp Cache busting of collection metadata
1030 * @param {Object} options
1031 * @param {bool} options.retry Whether this method is called in the
1034 * @returns {Promise<Object>} the computed sync result.
1036 async _importChanges(
1043 const { retry = false } = options;
1044 const since = retry || !localTimestamp ? undefined : `"${localTimestamp}"`;
1046 // Fetch collection metadata and list of changes from server.
1048 `${this.identifier} Fetch changes from server (expected=${expectedTimestamp}, since=${since})`
1050 const { metadata, remoteTimestamp, remoteRecords } =
1051 await this._fetchChangeset(expectedTimestamp, since);
1053 // We build a sync result, based on remote changes.
1054 const syncResult = {
1055 current: localRecords,
1060 // If data wasn't changed, return empty sync result.
1061 // This can happen when we update the signature but not the data.
1063 `${this.identifier} local timestamp: ${localTimestamp}, remote: ${remoteTimestamp}`
1065 if (localTimestamp && remoteTimestamp < localTimestamp) {
1069 await this.db.importChanges(metadata, remoteTimestamp, remoteRecords, {
1073 // Read the new local data, after updating.
1074 const newLocal = await this.db.list();
1075 const newRecords = newLocal.map(r => this._cleanLocalFields(r));
1076 // And verify the signature on what is now stored.
1077 if (this.verifySignature) {
1079 await this._validateCollectionSignature(
1086 `${this.identifier} Signature failed ${retry ? "again" : ""} ${e}`
1088 if (!(e instanceof InvalidSignatureError)) {
1089 // If it failed for any other kind of error (eg. shutdown)
1090 // then give up quickly.
1094 // In order to distinguish signature errors that happen
1095 // during sync, from hijacks of local DBs, we will verify
1096 // the signature on the data that we had before syncing.
1097 let localTrustworthy = false;
1098 lazy.console.debug(`${this.identifier} verify data before sync`);
1100 await this._validateCollectionSignature(
1105 localTrustworthy = true;
1107 if (!(sigerr instanceof InvalidSignatureError)) {
1108 // If it fails for other reason, keep original error and give up.
1111 lazy.console.debug(`${this.identifier} previous data was invalid`);
1114 if (!localTrustworthy && !retry) {
1115 // Signature failed, clear local DB because it contains
1116 // bad data (local + remote changes).
1117 lazy.console.debug(`${this.identifier} clear local data`);
1118 await this.db.clear();
1119 // Local data was tampered, throw and it will retry from empty DB.
1120 lazy.console.error(`${this.identifier} local data was corrupted`);
1121 throw new CorruptedDataError(this.identifier);
1123 // We retried already, we will restore the previous local data
1124 // before throwing eventually.
1125 if (localTrustworthy) {
1126 await this.db.importChanges(
1131 clear: true, // clear before importing.
1135 // Restore the dump if available (no-op if no dump)
1136 const imported = await this._importJSONDump();
1137 // _importJSONDump() only clears DB if dump is available,
1138 // therefore do it here!
1140 await this.db.clear();
1147 lazy.console.warn(`${this.identifier} has signature disabled`);
1150 if (this.hasListeners("sync")) {
1151 // If we have some listeners for the "sync" event,
1152 // Compute the changes, comparing records before and after.
1153 syncResult.current = newRecords;
1154 const oldById = new Map(localRecords.map(e => [e.id, e]));
1155 for (const r of newRecords) {
1156 const old = oldById.get(r.id);
1158 oldById.delete(r.id);
1159 if (r.last_modified != old.last_modified) {
1160 syncResult.updated.push({ old, new: r });
1163 syncResult.created.push(r);
1166 syncResult.deleted = syncResult.deleted.concat(
1167 Array.from(oldById.values())
1170 `${this.identifier} ${syncResult.created.length} created. ${syncResult.updated.length} updated. ${syncResult.deleted.length} deleted.`
1178 * Fetch information from changeset endpoint.
1180 * @param expectedTimestamp cache busting value
1181 * @param since timestamp of last sync (optional)
1183 async _fetchChangeset(expectedTimestamp, since) {
1184 const client = this.httpClient();
1187 timestamp: remoteTimestamp,
1188 changes: remoteRecords,
1189 } = await client.execute(
1191 path: `/buckets/${this.bucketName}/collections/${this.collectionName}/changeset`,
1195 _expected: expectedTimestamp,
1208 * Use the filter func to filter the lists of changes obtained from synchronization,
1209 * and return them along with the filtered list of local records.
1211 * If the filtered lists of changes are all empty, we return null (and thus don't
1212 * bother listing local DB).
1214 * @param {Object} syncResult Synchronization result without filtering.
1216 * @returns {Promise<Object>} the filtered list of local records, plus the filtered
1217 * list of created, updated and deleted records.
1219 async _filterSyncResult(syncResult) {
1220 // Handle the obtained records (ie. apply locally through events).
1221 // Build the event data list. It should be filtered (ie. by application target)
1224 created: allCreated,
1225 updated: allUpdated,
1226 deleted: allDeleted,
1228 const [created, deleted, updatedFiltered] = await Promise.all(
1229 [allCreated, allDeleted, allUpdated.map(e => e.new)].map(
1230 this._filterEntries.bind(this)
1233 // For updates, keep entries whose updated form matches the target.
1234 const updatedFilteredIds = new Set(updatedFiltered.map(e => e.id));
1235 const updated = allUpdated.filter(({ new: { id } }) =>
1236 updatedFilteredIds.has(id)
1239 if (!created.length && !updated.length && !deleted.length) {
1242 // Read local collection of records (also filtered).
1243 const current = await this._filterEntries(allData);
1244 return { created, updated, deleted, current };
1248 * Filter entries for which calls to `this.filterFunc` returns null.
1250 * @param {Array<Objet>} data
1251 * @returns {Array<Object>}
1253 async _filterEntries(data) {
1254 if (!this.filterFunc) {
1257 const environment = cacheProxy(lazy.ClientEnvironmentBase);
1258 const dataPromises = data.map(e => this.filterFunc(e, environment));
1259 const results = await Promise.all(dataPromises);
1260 return results.filter(Boolean);
1264 * Remove the fields from the specified record
1265 * that are not present on server.
1267 * @param {Object} record
1269 _cleanLocalFields(record) {
1270 const keys = ["_status"].concat(this.localFields);
1271 const result = { ...record };
1272 for (const key of keys) {