1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 var EXPORTED_SYMBOLS = ["RemoteSettingsClient"];
9 const { XPCOMUtils } = ChromeUtils.import(
10 "resource://gre/modules/XPCOMUtils.jsm"
12 const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
14 XPCOMUtils.defineLazyModuleGetters(this, {
15 AppConstants: "resource://gre/modules/AppConstants.jsm",
16 ClientEnvironmentBase:
17 "resource://gre/modules/components-utils/ClientEnvironment.jsm",
18 Database: "resource://services-settings/Database.jsm",
19 Downloader: "resource://services-settings/Attachments.jsm",
20 IDBHelpers: "resource://services-settings/IDBHelpers.jsm",
21 KintoHttpClient: "resource://services-common/kinto-http-client.js",
22 ObjectUtils: "resource://gre/modules/ObjectUtils.jsm",
23 PerformanceCounters: "resource://gre/modules/PerformanceCounters.jsm",
24 RemoteSettingsWorker: "resource://services-settings/RemoteSettingsWorker.jsm",
25 SharedUtils: "resource://services-settings/SharedUtils.jsm",
26 UptakeTelemetry: "resource://services-common/uptake-telemetry.js",
27 Utils: "resource://services-settings/Utils.jsm",
30 const TELEMETRY_COMPONENT = "remotesettings";
32 XPCOMUtils.defineLazyGetter(this, "console", () => Utils.log);
34 XPCOMUtils.defineLazyPreferenceGetter(
37 "services.settings.enablePerformanceCounters",
41 XPCOMUtils.defineLazyPreferenceGetter(
44 "services.settings.load_dump",
49 * cacheProxy returns an object Proxy that will memoize properties of the target.
50 * @param {Object} target the object to wrap.
53 function cacheProxy(target) {
54 const cache = new Map();
55 return new Proxy(target, {
56 get(target, prop, receiver) {
57 if (!cache.has(prop)) {
58 cache.set(prop, target[prop]);
60 return cache.get(prop);
66 * Minimalist event emitter.
68 * Note: we don't use `toolkit/modules/EventEmitter` because **we want** to throw
69 * an error when a listener fails to execute.
73 this._listeners = new Map();
74 for (const event of events) {
75 this._listeners.set(event, []);
80 * Event emitter: will execute the registered listeners in the order and
83 * @param {string} event the event name
84 * @param {Object} payload the event payload to call the listeners with
86 async emit(event, payload) {
87 const callbacks = this._listeners.get(event);
89 for (const cb of callbacks) {
101 hasListeners(event) {
102 return this._listeners.has(event) && this._listeners.get(event).length > 0;
105 on(event, callback) {
106 if (!this._listeners.has(event)) {
107 throw new Error(`Unknown event type ${event}`);
109 this._listeners.get(event).push(callback);
112 off(event, callback) {
113 if (!this._listeners.has(event)) {
114 throw new Error(`Unknown event type ${event}`);
116 const callbacks = this._listeners.get(event);
117 const i = callbacks.indexOf(callback);
119 throw new Error(`Unknown callback`);
121 callbacks.splice(i, 1);
126 class NetworkOfflineError extends Error {
128 super("Network is offline");
129 this.name = "NetworkOfflineError";
133 class InvalidSignatureError extends Error {
135 super(`Invalid content signature (${cid})`);
136 this.name = "InvalidSignatureError";
140 class MissingSignatureError extends InvalidSignatureError {
143 this.message = `Missing signature (${cid})`;
144 this.name = "MissingSignatureError";
148 class CorruptedDataError extends InvalidSignatureError {
151 this.message = `Corrupted local data (${cid})`;
152 this.name = "CorruptedDataError";
156 class UnknownCollectionError extends Error {
158 super(`Unknown Collection "${cid}"`);
159 this.name = "UnknownCollectionError";
163 class AttachmentDownloader extends Downloader {
164 constructor(client) {
165 super(client.bucketName, client.collectionName);
166 this._client = client;
171 get: async attachmentId => {
172 return this._client.db.getAttachment(attachmentId);
174 set: async (attachmentId, attachment) => {
175 return this._client.db.saveAttachment(attachmentId, attachment);
177 delete: async attachmentId => {
178 return this._client.db.saveAttachment(attachmentId, null);
181 Object.defineProperty(this, "cacheImpl", { value: cacheImpl });
186 * Download attachment and report Telemetry on failure.
188 * @see Downloader.download
190 async download(record, options) {
192 // Explicitly await here to ensure we catch a network error.
193 return await super.download(record, options);
195 // Report download error.
196 let status = UptakeTelemetry.STATUS.DOWNLOAD_ERROR;
197 if (Utils.isOffline) {
198 status = UptakeTelemetry.STATUS.NETWORK_OFFLINE_ERROR;
199 } else if (/NetworkError/.test(err.message)) {
200 status = UptakeTelemetry.STATUS.NETWORK_ERROR;
202 // If the file failed to be downloaded, report it as such in Telemetry.
203 await UptakeTelemetry.report(TELEMETRY_COMPONENT, status, {
204 source: this._client.identifier,
211 * Delete all downloaded records attachments.
213 * Note: the list of attachments to be deleted is based on the
214 * current list of records.
217 let allRecords = await this._client.db.list();
219 allRecords.filter(r => !!r.attachment).map(r => this.delete(r))
224 class RemoteSettingsClient extends EventEmitter {
225 static get NetworkOfflineError() {
226 return NetworkOfflineError;
228 static get InvalidSignatureError() {
229 return InvalidSignatureError;
231 static get MissingSignatureError() {
232 return MissingSignatureError;
234 static get CorruptedDataError() {
235 return CorruptedDataError;
237 static get UnknownCollectionError() {
238 return UnknownCollectionError;
252 super(["sync"]); // emitted events
254 this.collectionName = collectionName;
255 this.bucketName = bucketName;
256 this.signerName = signerName;
257 this.filterFunc = filterFunc;
258 this.localFields = localFields;
259 this._lastCheckTimePref = lastCheckTimePref;
260 this._verifier = null;
261 this._syncRunning = false;
263 // This attribute allows signature verification to be disabled, when running tests
264 // or when pulling data from a dev server.
265 this.verifySignature = AppConstants.REMOTE_SETTINGS_VERIFY_SIGNATURE;
268 // TODO bug 1702759: Remove bucketNamePref.
269 // The bucket preference value can be changed (eg. `main` to `main-preview`) in order
270 // to preview the changes to be approved in a real client.
271 this.bucketNamePref = bucketNamePref;
272 XPCOMUtils.defineLazyPreferenceGetter(
278 this.db.identifier = this.identifier;
283 XPCOMUtils.defineLazyGetter(
286 () => new Database(this.identifier)
289 XPCOMUtils.defineLazyGetter(
292 () => new AttachmentDownloader(this)
297 return `${this.bucketName}/${this.collectionName}`;
300 get lastCheckTimePref() {
302 this._lastCheckTimePref ||
303 `services.settings.${this.bucketName}.${this.collectionName}.last_check`
308 const api = new KintoHttpClient(Utils.SERVER_URL, {
309 fetchFunc: Utils.fetch, // Use fetch() wrapper.
311 return api.bucket(this.bucketName).collection(this.collectionName);
315 * Retrieve the collection timestamp for the last synchronization.
316 * This is an opaque and comparable value assigned automatically by
320 * The timestamp in milliseconds, returns -1 if retrieving
321 * the timestamp from the kinto collection fails.
323 async getLastModified() {
326 timestamp = await this.db.getLastModified();
329 `Error retrieving the getLastModified timestamp from ${this.identifier} RemoteSettingsClient`,
340 * @param {Object} options The options object.
341 * @param {Object} options.filters Filter the results (default: `{}`).
342 * @param {String} options.order The order to apply (eg. `"-last_modified"`).
343 * @param {boolean} options.dumpFallback Fallback to dump data if read of local DB fails (default: `true`).
344 * @param {boolean} options.loadDumpIfNewer Use dump data if it is newer than local data (default: `false`).
345 * @param {boolean} options.syncIfEmpty Synchronize from server if local data is empty (default: `true`).
346 * @param {boolean} options.verifySignature Verify the signature of the local data (default: `false`).
349 async get(options = {}) {
352 order = "", // not sorted by default.
354 loadDumpIfNewer = false, // TODO bug 1718083: should default to true.
357 let { verifySignature = false } = options;
361 let lastModified = await this.db.getLastModified();
362 let hasLocalData = lastModified !== null;
364 if (syncIfEmpty && !hasLocalData) {
365 // .get() was called before we had the chance to synchronize the local database.
366 // We'll try to avoid returning an empty list.
367 if (!this._importingPromise) {
368 // Prevent parallel loading when .get() is called multiple times.
369 this._importingPromise = (async () => {
370 const importedFromDump = gLoadDump
371 ? await this._importJSONDump()
373 if (importedFromDump < 0) {
374 // There is no JSON dump to load, force a synchronization from the server.
376 `${this.identifier} Local DB is empty, pull data from server`
378 await this.sync({ loadDump: false });
382 console.debug(`${this.identifier} Awaiting existing import.`);
384 } else if (hasLocalData && loadDumpIfNewer) {
385 // Check whether the local data is older than the packaged dump.
386 // If it is, load the packaged dump (which overwrites the local data).
387 let lastModifiedDump = await Utils.getLocalDumpLastModified(
391 if (lastModified < lastModifiedDump) {
393 `${this.identifier} Local DB is stale (${lastModified}), using dump instead (${lastModifiedDump})`
395 if (!this._importingPromise) {
396 // As part of importing, any existing data is wiped.
397 this._importingPromise = this._importJSONDump();
399 console.debug(`${this.identifier} Awaiting existing import.`);
404 if (this._importingPromise) {
406 await this._importingPromise;
407 // No need to verify signature, because either we've just load a trusted
408 // dump (here or in a parallel call), or it was verified during sync.
409 verifySignature = false;
411 // Report error, but continue because there could have been data
412 // loaded from a parrallel call.
415 // then delete this promise again, as now we should have local data:
416 delete this._importingPromise;
420 // Read from the local DB.
421 data = await this.db.list({ filters, order });
423 // If the local DB cannot be read (for unknown reasons, Bug 1649393)
424 // We fallback to the packaged data, and filter/sort in memory.
429 let { data } = await SharedUtils.loadJSONDump(
434 console.info(`${this.identifier} falling back to JSON dump`);
436 console.info(`${this.identifier} no dump fallback, return empty list`);
439 if (!ObjectUtils.isEmpty(filters)) {
440 data = data.filter(r => Utils.filterObject(filters, r));
443 data = Utils.sortObjects(order, data);
445 // No need to verify signature on JSON dumps.
446 // If local DB cannot be read, then we don't even try to do anything,
447 // we return results early.
448 return this._filterEntries(data);
452 `${this.identifier} ${data.length} records before filtering.`
455 if (verifySignature) {
457 `${this.identifier} verify signature of local data on read`
459 const allData = ObjectUtils.isEmpty(filters)
461 : await this.db.list();
462 const localRecords = allData.map(r => this._cleanLocalFields(r));
463 const timestamp = await this.db.getLastModified();
464 let metadata = await this.db.getMetadata();
465 if (syncIfEmpty && ObjectUtils.isEmpty(metadata)) {
466 // No sync occured yet, may have records from dump but no metadata.
467 await this.sync({ loadDump: false });
468 metadata = await this.db.getMetadata();
470 // Will throw MissingSignatureError if no metadata and `syncIfEmpty` is false.
471 await this._validateCollectionSignature(
478 // Filter the records based on `this.filterFunc` results.
479 const final = await this._filterEntries(data);
481 `${this.identifier} ${final.length} records after filtering.`
487 * Synchronize the local database with the remote server.
489 * @param {Object} options See #maybeSync() options.
491 async sync(options) {
492 // We want to know which timestamp we are expected to obtain in order to leverage
493 // cache busting. We don't provide ETag because we don't want a 304.
494 const { changes } = await Utils.fetchLatestChanges(Utils.SERVER_URL, {
496 collection: this.collectionName,
497 bucket: this.bucketName,
500 if (changes.length === 0) {
501 throw new RemoteSettingsClient.UnknownCollectionError(this.identifier);
503 // According to API, there will be one only (fail if not).
504 const [{ last_modified: expectedTimestamp }] = changes;
506 return this.maybeSync(expectedTimestamp, { ...options, trigger: "forced" });
510 * Synchronize the local database with the remote server, **only if necessary**.
512 * @param {int} expectedTimestamp the lastModified date (on the server) for the remote collection.
513 * This will be compared to the local timestamp, and will be used for
514 * cache busting if local data is out of date.
515 * @param {Object} options additional advanced options.
516 * @param {bool} options.loadDump load initial dump from disk on first sync (default: true, unless
517 * `services.settings.load_dump` says otherwise).
518 * @param {string} options.trigger label to identify what triggered this sync (eg. ``"timer"``, default: `"manual"`)
519 * @return {Promise} which rejects on sync or process failure.
521 async maybeSync(expectedTimestamp, options = {}) {
522 // Should the clients try to load JSON dump? (mainly disabled in tests)
523 const { loadDump = gLoadDump, trigger = "manual" } = options;
525 // Make sure we don't run several synchronizations in parallel, mainly
526 // in order to avoid race conditions in "sync" events listeners.
527 if (this._syncRunning) {
528 console.warn(`${this.identifier} sync already running`);
532 // Prevent network requests and IndexedDB calls to be initiated
534 if (Services.startup.shuttingDown) {
535 console.warn(`${this.identifier} sync interrupted by shutdown`);
539 this._syncRunning = true;
541 let importedFromDump = [];
542 const startedAt = new Date();
543 let reportStatus = null;
544 let thrownError = null;
546 // If network is offline, we can't synchronize.
547 if (Utils.isOffline) {
548 throw new RemoteSettingsClient.NetworkOfflineError();
551 // Read last timestamp and local data before sync.
552 let collectionLastModified = await this.db.getLastModified();
553 const allData = await this.db.list();
554 // Local data can contain local fields, strip them.
555 let localRecords = allData.map(r => this._cleanLocalFields(r));
556 const localMetadata = await this.db.getMetadata();
558 // If there is no data currently in the collection, attempt to import
559 // initial data from the application defaults.
560 // This allows to avoid synchronizing the whole collection content on
562 if (!collectionLastModified && loadDump) {
564 const imported = await this._importJSONDump();
565 // The worker only returns an integer. List the imported records to build the sync event.
568 `${this.identifier} ${imported} records loaded from JSON dump`
570 importedFromDump = await this.db.list();
571 // Local data is the data loaded from dump. We will need this later
572 // to compute the sync result.
573 localRecords = importedFromDump;
575 collectionLastModified = await this.db.getLastModified();
583 // Is local timestamp up to date with the server?
584 if (expectedTimestamp == collectionLastModified) {
585 console.debug(`${this.identifier} local data is up-to-date`);
586 reportStatus = UptakeTelemetry.STATUS.UP_TO_DATE;
588 // If the data is up-to-date but don't have metadata (records loaded from dump),
589 // we fetch them and validate the signature immediately.
590 if (this.verifySignature && ObjectUtils.isEmpty(localMetadata)) {
591 console.debug(`${this.identifier} pull collection metadata`);
592 const metadata = await this.httpClient().getData({
593 query: { _expected: expectedTimestamp },
595 await this.db.importChanges(metadata);
596 // We don't bother validating the signature if the dump was just loaded. We do
597 // if the dump was loaded at some other point (eg. from .get()).
598 if (this.verifySignature && importedFromDump.length == 0) {
600 `${this.identifier} verify signature of local data`
602 await this._validateCollectionSignature(
604 collectionLastModified,
610 // Since the data is up-to-date, if we didn't load any dump then we're done here.
611 if (importedFromDump.length == 0) {
614 // Otherwise we want to continue with sending the sync event to notify about the created records.
616 current: importedFromDump,
617 created: importedFromDump,
622 // Local data is either outdated or tampered.
623 // In both cases we will fetch changes from server,
624 // and make sure we overwrite local data.
625 const startSyncDB = Cu.now() * 1000;
626 syncResult = await this._importChanges(
628 collectionLastModified,
632 if (gTimingEnabled) {
633 const endSyncDB = Cu.now() * 1000;
634 PerformanceCounters.storeExecutionTime(
635 `remotesettings/${this.identifier}`,
637 endSyncDB - startSyncDB,
641 if (this.hasListeners("sync")) {
642 // If we have listeners for the "sync" event, then compute the lists of changes.
643 // The records imported from the dump should be considered as "created" for the
645 const importedById = importedFromDump.reduce((acc, r) => {
649 // Deleted records should not appear as created.
650 syncResult.deleted.forEach(r => importedById.delete(r.id));
651 // Records from dump that were updated should appear in their newest form.
652 syncResult.updated.forEach(u => {
653 if (importedById.has(u.old.id)) {
654 importedById.set(u.old.id, u.new);
657 syncResult.created = syncResult.created.concat(
658 Array.from(importedById.values())
663 if (e instanceof InvalidSignatureError) {
664 // Signature verification failed during synchronization.
666 e instanceof CorruptedDataError
667 ? UptakeTelemetry.STATUS.CORRUPTION_ERROR
668 : UptakeTelemetry.STATUS.SIGNATURE_ERROR;
669 // If sync fails with a signature error, it's likely that our
670 // local data has been modified in some way.
671 // We will attempt to fix this by retrieving the whole
672 // remote collection.
675 `${this.identifier} Signature verified failed. Retry from scratch`
677 syncResult = await this._importChanges(
679 collectionLastModified,
685 // If the signature fails again, or if an error occured during wiping out the
686 // local data, then we report it as a *signature retry* error.
687 reportStatus = UptakeTelemetry.STATUS.SIGNATURE_RETRY_ERROR;
691 // The sync has thrown for other reason than signature verification.
692 // Default status for errors at this step is SYNC_ERROR.
693 reportStatus = this._telemetryFromError(e, {
694 default: UptakeTelemetry.STATUS.SYNC_ERROR,
699 // Filter the synchronization results using `filterFunc` (ie. JEXL).
700 const filteredSyncResult = await this._filterSyncResult(syncResult);
701 // If every changed entry is filtered, we don't even fire the event.
702 if (filteredSyncResult) {
704 await this.emit("sync", { data: filteredSyncResult });
706 reportStatus = UptakeTelemetry.STATUS.APPLY_ERROR;
711 `All changes are filtered by JEXL expressions for ${this.identifier}`
716 // If browser is shutting down, then we can report a specific status.
717 // (eg. IndexedDB will abort transactions)
718 if (Services.startup.shuttingDown) {
719 reportStatus = UptakeTelemetry.STATUS.SHUTDOWN_ERROR;
721 // If no Telemetry status was determined yet (ie. outside sync step),
722 // then introspect error, default status at this step is UNKNOWN.
723 else if (reportStatus == null) {
724 reportStatus = this._telemetryFromError(e, {
725 default: UptakeTelemetry.STATUS.UNKNOWN_ERROR,
730 const durationMilliseconds = new Date() - startedAt;
731 // No error was reported, this is a success!
732 if (reportStatus === null) {
733 reportStatus = UptakeTelemetry.STATUS.SUCCESS;
735 // Report success/error status to Telemetry.
737 source: this.identifier,
739 duration: durationMilliseconds,
741 // In Bug 1617133, we will try to break down specific errors into
742 // more precise statuses by reporting the JavaScript error name
743 // ("TypeError", etc.) to Telemetry on Nightly.
744 const channel = UptakeTelemetry.Policy.getChannel();
746 thrownError !== null &&
747 channel == "nightly" &&
749 UptakeTelemetry.STATUS.SYNC_ERROR,
750 UptakeTelemetry.STATUS.CUSTOM_1_ERROR, // IndexedDB.
751 UptakeTelemetry.STATUS.UNKNOWN_ERROR,
752 UptakeTelemetry.STATUS.SHUTDOWN_ERROR,
753 ].includes(reportStatus)
755 // List of possible error names for IndexedDB:
756 // https://searchfox.org/mozilla-central/rev/49ed791/dom/base/DOMException.cpp#28-53
757 reportArgs = { ...reportArgs, errorName: thrownError.name };
760 await UptakeTelemetry.report(
766 console.debug(`${this.identifier} sync status is ${reportStatus}`);
767 this._syncRunning = false;
772 * Determine the Telemetry uptake status based on the specified
775 _telemetryFromError(e, options = { default: null }) {
776 let reportStatus = options.default;
778 if (e instanceof RemoteSettingsClient.NetworkOfflineError) {
779 reportStatus = UptakeTelemetry.STATUS.NETWORK_OFFLINE_ERROR;
780 } else if (e instanceof IDBHelpers.ShutdownError) {
781 reportStatus = UptakeTelemetry.STATUS.SHUTDOWN_ERROR;
782 } else if (/unparseable/.test(e.message)) {
783 reportStatus = UptakeTelemetry.STATUS.PARSE_ERROR;
784 } else if (/NetworkError/.test(e.message)) {
785 reportStatus = UptakeTelemetry.STATUS.NETWORK_ERROR;
786 } else if (/Timeout/.test(e.message)) {
787 reportStatus = UptakeTelemetry.STATUS.TIMEOUT_ERROR;
788 } else if (/HTTP 5??/.test(e.message)) {
789 reportStatus = UptakeTelemetry.STATUS.SERVER_ERROR;
790 } else if (/Backoff/.test(e.message)) {
791 reportStatus = UptakeTelemetry.STATUS.BACKOFF;
793 // Errors from kinto.js IDB adapter.
794 e instanceof IDBHelpers.IndexedDBError ||
795 // Other IndexedDB errors (eg. RemoteSettingsWorker).
796 /IndexedDB/.test(e.message)
798 reportStatus = UptakeTelemetry.STATUS.CUSTOM_1_ERROR;
805 * Import the JSON files from services/settings/dump into the local DB.
807 async _importJSONDump() {
808 console.info(`${this.identifier} try to restore dump`);
810 const start = Cu.now() * 1000;
811 const result = await RemoteSettingsWorker.importJSONDump(
816 `${this.identifier} imported ${result} records from JSON dump`
818 if (gTimingEnabled) {
819 const end = Cu.now() * 1000;
820 PerformanceCounters.storeExecutionTime(
821 `remotesettings/${this.identifier}`,
828 console.debug(`${this.identifier} no dump available`);
830 console.info(`${this.identifier} imported ${result} records from dump`);
836 * Fetch the signature info from the collection metadata and verifies that the
837 * local set of records has the same.
839 * @param {Array<Object>} records The list of records to validate.
840 * @param {int} timestamp The timestamp associated with the list of remote records.
841 * @param {Object} metadata The collection metadata, that contains the signature payload.
844 async _validateCollectionSignature(records, timestamp, metadata) {
845 const start = Cu.now() * 1000;
847 if (!metadata?.signature) {
848 throw new MissingSignatureError(this.identifier);
851 if (!this._verifier) {
853 "@mozilla.org/security/contentsignatureverifier;1"
854 ].createInstance(Ci.nsIContentSignatureVerifier);
857 // This is a content-signature field from an autograph response.
859 signature: { x5u, signature },
861 const certChain = await (await Utils.fetch(x5u)).text();
862 // Merge remote records with local ones and serialize as canonical JSON.
863 const serialized = await RemoteSettingsWorker.canonicalStringify(
868 !(await this._verifier.asyncVerifyContentSignature(
870 "p384ecdsa=" + signature,
875 throw new InvalidSignatureError(this.identifier);
877 if (gTimingEnabled) {
878 const end = Cu.now() * 1000;
879 PerformanceCounters.storeExecutionTime(
880 `remotesettings/${this.identifier}`,
881 "validateCollectionSignature",
889 * This method is in charge of fetching data from server, applying the diff-based
890 * changes to the local DB, validating the signature, and computing a synchronization
891 * result with the list of creation, updates, and deletions.
893 * @param {Array<Object>} localRecords Current list of records in local DB.
894 * @param {int} localTimestamp Current timestamp in local DB.
895 * @param {Object} localMetadata Current metadata in local DB.
896 * @param {int} expectedTimestamp Cache busting of collection metadata
897 * @param {Object} options
898 * @param {bool} options.retry Whether this method is called in the
901 * @returns {Promise<Object>} the computed sync result.
903 async _importChanges(
910 const { retry = false } = options;
911 const since = retry || !localTimestamp ? undefined : `"${localTimestamp}"`;
913 // Fetch collection metadata and list of changes from server.
915 `${this.identifier} Fetch changes from server (expected=${expectedTimestamp}, since=${since})`
921 } = await this._fetchChangeset(expectedTimestamp, since);
923 // We build a sync result, based on remote changes.
925 current: localRecords,
930 // If data wasn't changed, return empty sync result.
931 // This can happen when we update the signature but not the data.
933 `${this.identifier} local timestamp: ${localTimestamp}, remote: ${remoteTimestamp}`
935 if (localTimestamp && remoteTimestamp < localTimestamp) {
939 const start = Cu.now() * 1000;
940 await this.db.importChanges(metadata, remoteTimestamp, remoteRecords, {
943 if (gTimingEnabled) {
944 const end = Cu.now() * 1000;
945 PerformanceCounters.storeExecutionTime(
946 `remotesettings/${this.identifier}`,
953 // Read the new local data, after updating.
954 const newLocal = await this.db.list();
955 const newRecords = newLocal.map(r => this._cleanLocalFields(r));
956 // And verify the signature on what is now stored.
957 if (this.verifySignature) {
959 await this._validateCollectionSignature(
966 `${this.identifier} Signature failed ${retry ? "again" : ""} ${e}`
968 if (!(e instanceof InvalidSignatureError)) {
969 // If it failed for any other kind of error (eg. shutdown)
970 // then give up quickly.
974 // In order to distinguish signature errors that happen
975 // during sync, from hijacks of local DBs, we will verify
976 // the signature on the data that we had before syncing.
977 let localTrustworthy = false;
978 console.debug(`${this.identifier} verify data before sync`);
980 await this._validateCollectionSignature(
985 localTrustworthy = true;
987 if (!(sigerr instanceof InvalidSignatureError)) {
988 // If it fails for other reason, keep original error and give up.
991 console.debug(`${this.identifier} previous data was invalid`);
994 if (!localTrustworthy && !retry) {
995 // Signature failed, clear local DB because it contains
996 // bad data (local + remote changes).
997 console.debug(`${this.identifier} clear local data`);
998 await this.db.clear();
999 // Local data was tampered, throw and it will retry from empty DB.
1000 console.error(`${this.identifier} local data was corrupted`);
1001 throw new CorruptedDataError(this.identifier);
1003 // We retried already, we will restore the previous local data
1004 // before throwing eventually.
1005 if (localTrustworthy) {
1006 await this.db.importChanges(
1011 clear: true, // clear before importing.
1015 // Restore the dump if available (no-op if no dump)
1016 const imported = await this._importJSONDump();
1017 // _importJSONDump() only clears DB if dump is available,
1018 // therefore do it here!
1020 await this.db.clear();
1027 console.warn(`${this.identifier} has signature disabled`);
1030 if (this.hasListeners("sync")) {
1031 // If we have some listeners for the "sync" event,
1032 // Compute the changes, comparing records before and after.
1033 syncResult.current = newRecords;
1034 const oldById = new Map(localRecords.map(e => [e.id, e]));
1035 for (const r of newRecords) {
1036 const old = oldById.get(r.id);
1038 oldById.delete(r.id);
1039 if (r.last_modified != old.last_modified) {
1040 syncResult.updated.push({ old, new: r });
1043 syncResult.created.push(r);
1046 syncResult.deleted = syncResult.deleted.concat(
1047 Array.from(oldById.values())
1050 `${this.identifier} ${syncResult.created.length} created. ${syncResult.updated.length} updated. ${syncResult.deleted.length} deleted.`
1058 * Fetch information from changeset endpoint.
1060 * @param expectedTimestamp cache busting value
1061 * @param since timestamp of last sync (optional)
1063 async _fetchChangeset(expectedTimestamp, since) {
1064 const client = this.httpClient();
1067 timestamp: remoteTimestamp,
1068 changes: remoteRecords,
1069 } = await client.execute(
1071 path: `/buckets/${this.bucketName}/collections/${this.collectionName}/changeset`,
1075 _expected: expectedTimestamp,
1088 * Use the filter func to filter the lists of changes obtained from synchronization,
1089 * and return them along with the filtered list of local records.
1091 * If the filtered lists of changes are all empty, we return null (and thus don't
1092 * bother listing local DB).
1094 * @param {Object} syncResult Synchronization result without filtering.
1096 * @returns {Promise<Object>} the filtered list of local records, plus the filtered
1097 * list of created, updated and deleted records.
1099 async _filterSyncResult(syncResult) {
1100 // Handle the obtained records (ie. apply locally through events).
1101 // Build the event data list. It should be filtered (ie. by application target)
1104 created: allCreated,
1105 updated: allUpdated,
1106 deleted: allDeleted,
1108 const [created, deleted, updatedFiltered] = await Promise.all(
1109 [allCreated, allDeleted, allUpdated.map(e => e.new)].map(
1110 this._filterEntries.bind(this)
1113 // For updates, keep entries whose updated form matches the target.
1114 const updatedFilteredIds = new Set(updatedFiltered.map(e => e.id));
1115 const updated = allUpdated.filter(({ new: { id } }) =>
1116 updatedFilteredIds.has(id)
1119 if (!created.length && !updated.length && !deleted.length) {
1122 // Read local collection of records (also filtered).
1123 const current = await this._filterEntries(allData);
1124 return { created, updated, deleted, current };
1128 * Filter entries for which calls to `this.filterFunc` returns null.
1130 * @param {Array<Objet>} data
1131 * @returns {Array<Object>}
1133 async _filterEntries(data) {
1134 if (!this.filterFunc) {
1137 const start = Cu.now() * 1000;
1138 const environment = cacheProxy(ClientEnvironmentBase);
1139 const dataPromises = data.map(e => this.filterFunc(e, environment));
1140 const results = await Promise.all(dataPromises);
1141 if (gTimingEnabled) {
1142 const end = Cu.now() * 1000;
1143 PerformanceCounters.storeExecutionTime(
1144 `remotesettings/${this.identifier}`,
1150 return results.filter(Boolean);
1154 * Remove the fields from the specified record
1155 * that are not present on server.
1157 * @param {Object} record
1159 _cleanLocalFields(record) {
1160 const keys = ["_status"].concat(this.localFields);
1161 const result = { ...record };
1162 for (const key of keys) {