1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
5 import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
6 import { AppConstants } from "resource://gre/modules/AppConstants.sys.mjs";
10 ChromeUtils.defineESModuleGetters(lazy, {
11 Database: "resource://services-settings/Database.sys.mjs",
13 "resource://gre/modules/components-utils/FilterExpressions.sys.mjs",
15 "resource://services-settings/RemoteSettingsClient.sys.mjs",
16 SyncHistory: "resource://services-settings/SyncHistory.sys.mjs",
17 UptakeTelemetry: "resource://services-common/uptake-telemetry.sys.mjs",
18 Utils: "resource://services-settings/Utils.sys.mjs",
21 XPCOMUtils.defineLazyModuleGetters(lazy, {
22 pushBroadcastService: "resource://gre/modules/PushBroadcastService.jsm",
25 const PREF_SETTINGS_BRANCH = "services.settings.";
26 const PREF_SETTINGS_SERVER_BACKOFF = "server.backoff";
27 const PREF_SETTINGS_LAST_UPDATE = "last_update_seconds";
28 const PREF_SETTINGS_LAST_ETAG = "last_etag";
29 const PREF_SETTINGS_CLOCK_SKEW_SECONDS = "clock_skew_seconds";
30 const PREF_SETTINGS_SYNC_HISTORY_SIZE = "sync_history_size";
31 const PREF_SETTINGS_SYNC_HISTORY_ERROR_THRESHOLD =
32 "sync_history_error_threshold";
34 // Telemetry identifiers.
35 const TELEMETRY_COMPONENT = "remotesettings";
36 const TELEMETRY_SOURCE_POLL = "settings-changes-monitoring";
37 const TELEMETRY_SOURCE_SYNC = "settings-sync";
40 const BROADCAST_ID = "remote-settings/monitor_changes";
42 // Signer to be used when not specified (see Ci.nsIContentSignatureVerifier).
43 const DEFAULT_SIGNER = "remote-settings.content-signature.mozilla.org";
45 XPCOMUtils.defineLazyGetter(lazy, "gPrefs", () => {
46 return Services.prefs.getBranch(PREF_SETTINGS_BRANCH);
48 XPCOMUtils.defineLazyGetter(lazy, "console", () => lazy.Utils.log);
50 XPCOMUtils.defineLazyGetter(lazy, "gSyncHistory", () => {
51 const prefSize = lazy.gPrefs.getIntPref(PREF_SETTINGS_SYNC_HISTORY_SIZE, 100);
52 const size = Math.min(Math.max(prefSize, 1000), 10);
53 return new lazy.SyncHistory(TELEMETRY_SOURCE_SYNC, { size });
56 XPCOMUtils.defineLazyPreferenceGetter(
58 "gPrefBrokenSyncThreshold",
59 PREF_SETTINGS_BRANCH + PREF_SETTINGS_SYNC_HISTORY_ERROR_THRESHOLD,
63 XPCOMUtils.defineLazyPreferenceGetter(
65 "gPrefDestroyBrokenEnabled",
66 PREF_SETTINGS_BRANCH + "destroy_broken_db_enabled",
71 * Default entry filtering function, in charge of excluding remote settings entries
72 * where the JEXL expression evaluates into a falsy value.
73 * @param {Object} entry The Remote Settings entry to be excluded or kept.
74 * @param {ClientEnvironment} environment Information about version, language, platform etc.
75 * @returns {?Object} the entry or null if excluded.
77 export async function jexlFilterFunc(entry, environment) {
78 const { filter_expression } = entry;
79 if (!filter_expression) {
87 result = await lazy.FilterExpressions.eval(filter_expression, context);
91 return result ? entry : null;
94 function remoteSettingsFunction() {
95 const _clients = new Map();
96 let _invalidatePolling = false;
98 // If not explicitly specified, use the default signer.
99 const defaultOptions = {
100 signerName: DEFAULT_SIGNER,
101 filterFunc: jexlFilterFunc,
105 * RemoteSettings constructor.
107 * @param {String} collectionName The remote settings identifier
108 * @param {Object} options Advanced options
109 * @returns {RemoteSettingsClient} An instance of a Remote Settings client.
111 const remoteSettings = function(collectionName, options) {
112 // Get or instantiate a remote settings client.
113 if (!_clients.has(collectionName)) {
114 // Register a new client!
115 const c = new lazy.RemoteSettingsClient(collectionName, {
119 // Store instance for later call.
120 _clients.set(collectionName, c);
121 // Invalidate the polling status, since we want the new collection to
122 // be taken into account.
123 _invalidatePolling = true;
124 lazy.console.debug(`Instantiated new client ${c.identifier}`);
126 return _clients.get(collectionName);
130 * Internal helper to retrieve existing instances of clients or new instances
131 * with default options if possible, or `null` if bucket/collection are unknown.
133 async function _client(bucketName, collectionName) {
134 // Check if a client was registered for this bucket/collection. Potentially
135 // with some specific options like signer, filter function etc.
136 const client = _clients.get(collectionName);
137 if (client && client.bucketName == bucketName) {
140 // There was no client registered for this collection, but it's the main bucket,
141 // therefore we can instantiate a client with the default options.
142 // So if we have a local database or if we ship a JSON dump, then it means that
143 // this client is known but it was not registered yet (eg. calling module not "imported" yet).
146 lazy.Utils.actualBucketName(AppConstants.REMOTE_SETTINGS_DEFAULT_BUCKET)
148 const c = new lazy.RemoteSettingsClient(collectionName, defaultOptions);
149 const [dbExists, localDump] = await Promise.all([
150 lazy.Utils.hasLocalData(c),
151 lazy.Utils.hasLocalDump(bucketName, collectionName),
153 if (dbExists || localDump) {
157 // Else, we cannot return a client instance because we are not able to synchronize data in specific buckets.
158 // Mainly because we cannot guess which `signerName` has to be used for example.
159 // And we don't want to synchronize data for collections in the main bucket that are
160 // completely unknown (ie. no database and no JSON dump).
161 lazy.console.debug(`No known client for ${bucketName}/${collectionName}`);
166 * Helper to introspect the synchronization history and determine whether it is
167 * consistently failing and thus, broken.
168 * @returns {bool} true if broken.
170 async function isSynchronizationBroken() {
171 // The minimum number of errors is customizable, but with a maximum.
172 const threshold = Math.min(lazy.gPrefBrokenSyncThreshold, 20);
173 // Read history of synchronization past statuses.
174 const pastEntries = await lazy.gSyncHistory.list();
175 const lastSuccessIdx = pastEntries.findIndex(
176 e => e.status == lazy.UptakeTelemetry.STATUS.SUCCESS
179 // Only errors since last success.
180 lastSuccessIdx >= threshold ||
181 // Or only errors with a minimum number of history entries.
182 (lastSuccessIdx < 0 && pastEntries.length >= threshold)
187 * Main polling method, called by the ping mechanism.
189 * @param {Object} options
190 . * @param {Object} options.expectedTimestamp (optional) The expected timestamp to be received — used by servers for cache busting.
191 * @param {string} options.trigger (optional) label to identify what triggered this sync (eg. ``"timer"``, default: `"manual"`)
192 * @param {bool} options.full (optional) Ignore last polling status and fetch all changes (default: `false`)
193 * @returns {Promise} or throws error if something goes wrong.
195 remoteSettings.pollChanges = async ({
200 // When running in full mode, we ignore last polling status.
202 lazy.gPrefs.clearUserPref(PREF_SETTINGS_SERVER_BACKOFF);
203 lazy.gPrefs.clearUserPref(PREF_SETTINGS_LAST_UPDATE);
204 lazy.gPrefs.clearUserPref(PREF_SETTINGS_LAST_ETAG);
207 let pollTelemetryArgs = {
208 source: TELEMETRY_SOURCE_POLL,
212 if (lazy.Utils.isOffline) {
213 lazy.console.info("Network is offline. Give up.");
214 await lazy.UptakeTelemetry.report(
216 lazy.UptakeTelemetry.STATUS.NETWORK_OFFLINE_ERROR,
222 const startedAt = new Date();
224 // Check if the server backoff time is elapsed.
225 if (lazy.gPrefs.prefHasUserValue(PREF_SETTINGS_SERVER_BACKOFF)) {
226 const backoffReleaseTime = lazy.gPrefs.getCharPref(
227 PREF_SETTINGS_SERVER_BACKOFF
229 const remainingMilliseconds =
230 parseInt(backoffReleaseTime, 10) - Date.now();
231 if (remainingMilliseconds > 0) {
232 // Backoff time has not elapsed yet.
233 await lazy.UptakeTelemetry.report(
235 lazy.UptakeTelemetry.STATUS.BACKOFF,
239 `Server is asking clients to back off; retry in ${Math.ceil(
240 remainingMilliseconds / 1000
244 lazy.gPrefs.clearUserPref(PREF_SETTINGS_SERVER_BACKOFF);
248 // When triggered from the daily timer, we try to recover a broken
249 // sync state by destroying the local DB completely and retrying from scratch.
251 lazy.gPrefDestroyBrokenEnabled &&
252 trigger == "timer" &&
253 (await isSynchronizationBroken())
255 // We don't want to destroy the local DB if the failures are related to
256 // network or server errors though.
257 const lastStatus = await lazy.gSyncHistory.last();
258 const lastErrorClass =
259 lazy.RemoteSettingsClient[lastStatus?.infos?.errorName] || Error;
260 const isLocalError = !(
261 lastErrorClass.prototype instanceof lazy.RemoteSettingsClient.APIError
265 "Synchronization has failed consistently. Destroy database."
267 // Clear the last ETag to refetch everything.
268 lazy.gPrefs.clearUserPref(PREF_SETTINGS_LAST_ETAG);
269 // Clear the history, to avoid re-destroying several times in a row.
270 await lazy.gSyncHistory.clear().catch(error => console.error(error));
271 // Delete the whole IndexedDB database.
272 await lazy.Database.destroy().catch(error => console.error(error));
275 `Synchronization is broken, but last error is ${lastStatus}`
280 lazy.console.info("Start polling for changes");
281 Services.obs.notifyObservers(
283 "remote-settings:changes-poll-start",
284 JSON.stringify({ expectedTimestamp })
287 // Do we have the latest version already?
288 // Every time we register a new client, we have to fetch the whole list again.
289 const lastEtag = _invalidatePolling
291 : lazy.gPrefs.getCharPref(PREF_SETTINGS_LAST_ETAG, "");
295 pollResult = await lazy.Utils.fetchLatestChanges(lazy.Utils.SERVER_URL, {
300 // Report polling error to Uptake Telemetry.
302 if (/JSON\.parse/.test(e.message)) {
303 reportStatus = lazy.UptakeTelemetry.STATUS.PARSE_ERROR;
304 } else if (/content-type/.test(e.message)) {
305 reportStatus = lazy.UptakeTelemetry.STATUS.CONTENT_ERROR;
306 } else if (/Server/.test(e.message)) {
307 reportStatus = lazy.UptakeTelemetry.STATUS.SERVER_ERROR;
308 // If the server replied with bad request, clear the last ETag
309 // value to unblock the next run of synchronization.
310 lazy.gPrefs.clearUserPref(PREF_SETTINGS_LAST_ETAG);
311 } else if (/Timeout/.test(e.message)) {
312 reportStatus = lazy.UptakeTelemetry.STATUS.TIMEOUT_ERROR;
313 } else if (/NetworkError/.test(e.message)) {
314 reportStatus = lazy.UptakeTelemetry.STATUS.NETWORK_ERROR;
316 reportStatus = lazy.UptakeTelemetry.STATUS.UNKNOWN_ERROR;
318 await lazy.UptakeTelemetry.report(
323 // No need to go further.
324 throw new Error(`Polling for changes failed: ${e.message}.`);
335 // Report age of server data in Telemetry.
336 pollTelemetryArgs = { age: ageSeconds, ...pollTelemetryArgs };
338 // Report polling success to Uptake Telemetry.
341 ? lazy.UptakeTelemetry.STATUS.UP_TO_DATE
342 : lazy.UptakeTelemetry.STATUS.SUCCESS;
343 await lazy.UptakeTelemetry.report(
349 // Check if the server asked the clients to back off (for next poll).
350 if (backoffSeconds) {
352 "Server asks clients to backoff for ${backoffSeconds} seconds"
354 const backoffReleaseTime = Date.now() + backoffSeconds * 1000;
355 lazy.gPrefs.setCharPref(PREF_SETTINGS_SERVER_BACKOFF, backoffReleaseTime);
358 // Record new update time and the difference between local and server time.
359 // Negative clockDifference means local time is behind server time
360 // by the absolute of that value in seconds (positive means it's ahead)
361 const clockDifference = Math.floor((Date.now() - serverTimeMillis) / 1000);
362 lazy.gPrefs.setIntPref(PREF_SETTINGS_CLOCK_SKEW_SECONDS, clockDifference);
363 const checkedServerTimeInSeconds = Math.round(serverTimeMillis / 1000);
364 lazy.gPrefs.setIntPref(
365 PREF_SETTINGS_LAST_UPDATE,
366 checkedServerTimeInSeconds
369 // Iterate through the collections version info and initiate a synchronization
370 // on the related remote settings clients.
372 for (const change of changes) {
373 const { bucket, collection, last_modified } = change;
375 const client = await _client(bucket, collection);
377 // This collection has no associated client (eg. preview, other platform...)
380 // Start synchronization! It will be a no-op if the specified `lastModified` equals
381 // the one in the local database.
383 await client.maybeSync(last_modified, { trigger });
385 // Save last time this client was successfully synced.
386 Services.prefs.setIntPref(
387 client.lastCheckTimePref,
388 checkedServerTimeInSeconds
391 lazy.console.error(e);
394 firstError.details = change;
400 _invalidatePolling = false;
402 // Report total synchronization duration to Telemetry.
403 const durationMilliseconds = new Date() - startedAt;
404 const syncTelemetryArgs = {
405 source: TELEMETRY_SOURCE_SYNC,
406 duration: durationMilliseconds,
407 timestamp: `${currentEtag}`,
412 // Report the global synchronization failure. Individual uptake reports will also have been sent for each collection.
413 const status = lazy.UptakeTelemetry.STATUS.SYNC_ERROR;
414 await lazy.UptakeTelemetry.report(
419 // Keep track of sync failure in history.
420 await lazy.gSyncHistory
421 .store(currentEtag, status, {
423 errorName: firstError.name,
425 .catch(error => console.error(error));
426 // Notify potential observers of the error.
427 Services.obs.notifyObservers(
428 { wrappedJSObject: { error: firstError } },
429 "remote-settings:sync-error"
432 // If synchronization has been consistently failing, send a specific signal.
433 // See https://bugzilla.mozilla.org/show_bug.cgi?id=1729400
434 // and https://bugzilla.mozilla.org/show_bug.cgi?id=1658597
435 if (await isSynchronizationBroken()) {
436 await lazy.UptakeTelemetry.report(
438 lazy.UptakeTelemetry.STATUS.SYNC_BROKEN_ERROR,
442 Services.obs.notifyObservers(
443 { wrappedJSObject: { error: firstError } },
444 "remote-settings:broken-sync-error"
448 // Rethrow the first observed error
452 // Save current Etag for next poll.
453 lazy.gPrefs.setCharPref(PREF_SETTINGS_LAST_ETAG, currentEtag);
455 // Report the global synchronization success.
456 const status = lazy.UptakeTelemetry.STATUS.SUCCESS;
457 await lazy.UptakeTelemetry.report(
462 // Keep track of sync success in history.
463 await lazy.gSyncHistory
464 .store(currentEtag, status)
465 .catch(error => console.error(error));
467 lazy.console.info("Polling for changes done");
468 Services.obs.notifyObservers(null, "remote-settings:changes-poll-end");
472 * Enables or disables preview mode.
474 * When enabled, all existing and future clients will pull data from
475 * the `*-preview` buckets. This allows developers and QA to test their
476 * changes before publishing them for all clients.
478 remoteSettings.enablePreviewMode = enabled => {
479 // Set the flag for future clients.
480 lazy.Utils.enablePreviewMode(enabled);
481 // Enable it on existing clients.
482 for (const client of _clients.values()) {
483 client.refreshBucketName();
488 * Returns an object with polling status information and the list of
489 * known remote settings collections.
491 remoteSettings.inspect = async () => {
492 // Make sure we fetch the latest server info, use a random cache bust value.
493 const randomCacheBust = 99990000 + Math.floor(Math.random() * 9999);
496 currentEtag: serverTimestamp,
497 } = await lazy.Utils.fetchLatestChanges(lazy.Utils.SERVER_URL, {
498 expected: randomCacheBust,
501 const collections = await Promise.all(
502 changes.map(async change => {
503 const { bucket, collection, last_modified: serverTimestamp } = change;
504 const client = await _client(bucket, collection);
508 const localTimestamp = await client.getLastModified();
509 const lastCheck = Services.prefs.getIntPref(
510 client.lastCheckTimePref,
519 signerName: client.signerName,
525 serverURL: lazy.Utils.SERVER_URL,
526 pollingEndpoint: lazy.Utils.SERVER_URL + lazy.Utils.CHANGES_PATH,
528 localTimestamp: lazy.gPrefs.getCharPref(PREF_SETTINGS_LAST_ETAG, null),
529 lastCheck: lazy.gPrefs.getIntPref(PREF_SETTINGS_LAST_UPDATE, 0),
530 mainBucket: lazy.Utils.actualBucketName(
531 AppConstants.REMOTE_SETTINGS_DEFAULT_BUCKET
533 defaultSigner: DEFAULT_SIGNER,
534 previewMode: lazy.Utils.PREVIEW_MODE,
535 collections: collections.filter(c => !!c),
537 [TELEMETRY_SOURCE_SYNC]: await lazy.gSyncHistory.list(),
543 * Delete all local data, of every collection.
545 remoteSettings.clearAll = async () => {
546 const { collections } = await remoteSettings.inspect();
548 collections.map(async ({ collection }) => {
549 const client = RemoteSettings(collection);
550 // Delete all potential attachments.
551 await client.attachments.deleteAll();
552 // Delete local data.
553 await client.db.clear();
554 // Remove status pref.
555 Services.prefs.clearUserPref(client.lastCheckTimePref);
561 * Startup function called from nsBrowserGlue.
563 remoteSettings.init = () => {
564 lazy.console.info("Initialize Remote Settings");
565 // Hook the Push broadcast and RemoteSettings polling.
566 // When we start on a new profile there will be no ETag stored.
567 // Use an arbitrary ETag that is guaranteed not to occur.
568 // This will trigger a broadcast message but that's fine because we
569 // will check the changes on each collection and retrieve only the
570 // changes (e.g. nothing if we have a dump with the same data).
571 const currentVersion = lazy.gPrefs.getStringPref(
572 PREF_SETTINGS_LAST_ETAG,
577 moduleURI: import.meta.url,
578 symbolName: "remoteSettingsBroadcastHandler",
580 lazy.pushBroadcastService.addListener(
587 return remoteSettings;
590 export var RemoteSettings = remoteSettingsFunction();
592 export var remoteSettingsBroadcastHandler = {
593 async receivedBroadcastMessage(version, broadcastID, context) {
594 const { phase } = context;
596 lazy.pushBroadcastService.PHASES.HELLO,
597 lazy.pushBroadcastService.PHASES.REGISTER,
601 `Push notification received (version=${version} phase=${phase})`
604 return RemoteSettings.pollChanges({
605 expectedTimestamp: version,
606 trigger: isStartup ? "startup" : "broadcast",