1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
5 import { AppConstants } from "resource://gre/modules/AppConstants.sys.mjs";
6 import { ServiceRequest } from "resource://gre/modules/ServiceRequest.sys.mjs";
7 import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
11 ChromeUtils.defineESModuleGetters(lazy, {
12 SharedUtils: "resource://services-settings/SharedUtils.sys.mjs",
15 XPCOMUtils.defineLazyServiceGetter(
17 "CaptivePortalService",
18 "@mozilla.org/network/captive-portal-service;1",
19 "nsICaptivePortalService"
21 XPCOMUtils.defineLazyServiceGetter(
23 "gNetworkLinkService",
24 "@mozilla.org/network/network-link-service;1",
25 "nsINetworkLinkService"
28 // Create a new instance of the ConsoleAPI so we can control the maxLogLevel with a pref.
29 // See LOG_LEVELS in Console.sys.mjs. Common examples: "all", "debug", "info",
32 const { ConsoleAPI } = ChromeUtils.importESModule(
33 "resource://gre/modules/Console.sys.mjs"
35 return new ConsoleAPI({
37 maxLogLevelPref: "services.settings.loglevel",
38 prefix: "services.settings",
42 ChromeUtils.defineLazyGetter(lazy, "isRunningTests", () => {
43 if (Services.env.get("MOZ_DISABLE_NONLOCAL_CONNECTIONS") === "1") {
44 // Allow to override the server URL if non-local connections are disabled,
45 // usually true when running tests.
51 // Overriding the server URL is normally disabled on Beta and Release channels,
52 // except under some conditions.
53 ChromeUtils.defineLazyGetter(lazy, "allowServerURLOverride", () => {
54 if (!AppConstants.RELEASE_OR_BETA) {
55 // Always allow to override the server URL on Nightly/DevEdition.
59 if (lazy.isRunningTests) {
63 if (Services.env.get("MOZ_REMOTE_SETTINGS_DEVTOOLS") === "1") {
64 // Allow to override the server URL when using remote settings devtools.
68 if (lazy.gServerURL != AppConstants.REMOTE_SETTINGS_SERVER_URL) {
69 log.warn("Ignoring preference override of remote settings server");
71 "Allow by setting MOZ_REMOTE_SETTINGS_DEVTOOLS=1 in the environment"
78 XPCOMUtils.defineLazyPreferenceGetter(
81 "services.settings.server",
82 AppConstants.REMOTE_SETTINGS_SERVER_URL
85 XPCOMUtils.defineLazyPreferenceGetter(
88 "services.settings.preview_enabled",
92 function _isUndefined(value) {
93 return typeof value === "undefined";
98 return lazy.allowServerURLOverride
100 : AppConstants.REMOTE_SETTINGS_SERVER_URL;
103 CHANGES_PATH: "/buckets/monitor/collections/changes/changeset",
110 get CERT_CHAIN_ROOT_IDENTIFIER() {
111 if (this.SERVER_URL == AppConstants.REMOTE_SETTINGS_SERVER_URL) {
112 return Ci.nsIContentSignatureVerifier.ContentSignatureProdRoot;
114 if (this.SERVER_URL.includes("allizom.")) {
115 return Ci.nsIContentSignatureVerifier.ContentSignatureStageRoot;
117 if (this.SERVER_URL.includes("dev.")) {
118 return Ci.nsIContentSignatureVerifier.ContentSignatureDevRoot;
120 if (Services.env.exists("XPCSHELL_TEST_PROFILE_DIR")) {
121 return Ci.nsIX509CertDB.AppXPCShellRoot;
123 return Ci.nsIContentSignatureVerifier.ContentSignatureLocalRoot;
127 // Load dumps only if pulling data from the production server, or in tests.
129 this.SERVER_URL == AppConstants.REMOTE_SETTINGS_SERVER_URL ||
135 // We want to offer the ability to set preview mode via a preference
136 // for consumers who want to pull from the preview bucket on startup.
137 if (_isUndefined(this._previewModeEnabled) && lazy.allowServerURLOverride) {
138 return lazy.gPreviewEnabled;
140 return !!this._previewModeEnabled;
144 * Internal method to enable pulling data from preview buckets.
147 enablePreviewMode(enabled) {
148 const bool2str = v =>
149 // eslint-disable-next-line no-nested-ternary
150 _isUndefined(v) ? "unset" : v ? "enabled" : "disabled";
152 `Preview mode: ${bool2str(this._previewModeEnabled)} -> ${bool2str(
156 this._previewModeEnabled = enabled;
160 * Returns the actual bucket name to be used. When preview mode is enabled,
161 * this adds the *preview* suffix.
163 * See also `SharedUtils.loadJSONDump()` which strips the preview suffix to identify
164 * the packaged JSON file.
166 * @param bucketName the client bucket
167 * @returns the final client bucket depending whether preview mode is enabled.
169 actualBucketName(bucketName) {
170 let actual = bucketName.replace("-preview", "");
171 if (this.PREVIEW_MODE) {
172 actual += "-preview";
178 * Check if network is down.
180 * Note that if this returns false, it does not guarantee
181 * that network is up.
183 * @return {bool} Whether network is down or not.
188 Services.io.offline ||
189 lazy.CaptivePortalService.state ==
190 lazy.CaptivePortalService.LOCKED_PORTAL ||
191 !lazy.gNetworkLinkService.isLinkUp
194 log.warn("Could not determine network status.", ex);
200 * A wrapper around `ServiceRequest` that behaves like `fetch()`.
202 * Use this in order to leverage the `beConservative` flag, for
203 * example to avoid using HTTP3 to fetch critical data.
205 * @param input a resource
206 * @param init request options
207 * @returns a Response object
209 async fetch(input, init = {}) {
210 return new Promise(function (resolve, reject) {
211 const request = new ServiceRequest();
212 function fallbackOrReject(err) {
214 // At most one recursive Utils.fetch call (bypassProxy=false to true).
216 Services.startup.shuttingDown ||
218 !request.isProxied ||
219 !request.bypassProxyEnabled
224 ServiceRequest.logProxySource(request.channel, "remote-settings");
225 resolve(Utils.fetch(input, { ...init, bypassProxy: true }));
228 request.onerror = () =>
229 fallbackOrReject(new TypeError("NetworkError: Network request failed"));
230 request.ontimeout = () =>
231 fallbackOrReject(new TypeError("Timeout: Network request failed"));
232 request.onabort = () =>
233 fallbackOrReject(new DOMException("Aborted", "AbortError"));
234 request.onload = () => {
235 // Parse raw response headers into `Headers` object.
236 const headers = new Headers();
237 const rawHeaders = request.getAllResponseHeaders();
242 const parts = line.split(": ");
243 const header = parts.shift();
244 const value = parts.join(": ");
245 headers.set(header, value);
248 const responseAttributes = {
249 status: request.status,
250 statusText: request.statusText,
251 url: request.responseURL,
254 resolve(new Response(request.response, responseAttributes));
257 const { method = "GET", headers = {}, bypassProxy = false } = init;
259 request.open(method, input, { bypassProxy });
260 // By default, XMLHttpRequest converts the response based on the
261 // Content-Type header, or UTF-8 otherwise. This may mangle binary
262 // responses. Avoid that by requesting the raw bytes.
263 request.responseType = "arraybuffer";
265 for (const [name, value] of Object.entries(headers)) {
266 request.setRequestHeader(name, value);
274 * Check if local data exist for the specified client.
276 * @param {RemoteSettingsClient} client
277 * @return {bool} Whether it exists or not.
279 async hasLocalData(client) {
280 const timestamp = await client.db.getLastModified();
281 return timestamp !== null;
285 * Check if we ship a JSON dump for the specified bucket and collection.
287 * @param {String} bucket
288 * @param {String} collection
289 * @return {bool} Whether it is present or not.
291 async hasLocalDump(bucket, collection) {
294 `resource://app/defaults/settings/${bucket}/${collection}.json`,
306 * Look up the last modification time of the JSON dump.
308 * @param {String} bucket
309 * @param {String} collection
310 * @return {int} The last modification time of the dump. -1 if non-existent.
312 async getLocalDumpLastModified(bucket, collection) {
313 if (!this._dumpStats) {
314 if (!this._dumpStatsInitPromise) {
315 this._dumpStatsInitPromise = (async () => {
317 let res = await fetch(
318 "resource://app/defaults/settings/last_modified.json"
320 this._dumpStats = await res.json();
322 log.warn(`Failed to load last_modified.json: ${e}`);
323 this._dumpStats = {};
325 delete this._dumpStatsInitPromise;
328 await this._dumpStatsInitPromise;
330 const identifier = `${bucket}/${collection}`;
331 let lastModified = this._dumpStats[identifier];
332 if (lastModified === undefined) {
333 const { timestamp: dumpTimestamp } = await lazy.SharedUtils.loadJSONDump(
337 // Client recognize -1 as missing dump.
338 lastModified = dumpTimestamp ?? -1;
339 this._dumpStats[identifier] = lastModified;
345 * Fetch the list of remote collections and their timestamp.
348 * "timestamp": 1486545678,
351 * "host":"kinto-ota.dev.mozaws.net",
352 * "last_modified":1450717104423,
353 * "bucket":"blocklists",
354 * "collection":"certificates"
361 * @param {String} serverUrl The server URL (eg. `https://server.org/v1`)
362 * @param {int} expectedTimestamp The timestamp that the server is supposed to return.
363 * We obtained it from the Megaphone notification payload,
364 * and we use it only for cache busting (Bug 1497159).
365 * @param {String} lastEtag (optional) The Etag of the latest poll to be matched
366 * by the server (eg. `"123456789"`).
367 * @param {Object} filters
369 async fetchLatestChanges(serverUrl, options = {}) {
370 const { expectedTimestamp, lastEtag = "", filters = {} } = options;
372 let url = serverUrl + Utils.CHANGES_PATH;
375 _expected: expectedTimestamp ?? 0,
377 if (lastEtag != "") {
378 params._since = lastEtag;
383 Object.entries(params)
384 .map(([k, v]) => `${k}=${encodeURIComponent(v)}`)
387 const response = await Utils.fetch(url);
389 if (response.status >= 500) {
390 throw new Error(`Server error ${response.status} ${response.statusText}`);
393 const is404FromCustomServer =
394 response.status == 404 &&
395 Services.prefs.prefHasUserValue("services.settings.server");
397 const ct = response.headers.get("Content-Type");
398 if (!is404FromCustomServer && (!ct || !ct.includes("application/json"))) {
399 throw new Error(`Unexpected content-type "${ct}"`);
404 payload = await response.json();
409 if (!payload.hasOwnProperty("changes")) {
410 // If the server is failing, the JSON response might not contain the
411 // expected data. For example, real server errors (Bug 1259145)
412 // or dummy local server for tests (Bug 1481348)
413 if (!is404FromCustomServer) {
415 `Server error ${url} ${response.status} ${
417 }: ${JSON.stringify(payload)}`
422 const { changes = [], timestamp } = payload;
424 let serverTimeMillis = Date.parse(response.headers.get("Date"));
425 // Since the response is served via a CDN, the Date header value could have been cached.
426 const cacheAgeSeconds = response.headers.has("Age")
427 ? parseInt(response.headers.get("Age"), 10)
429 serverTimeMillis += cacheAgeSeconds * 1000;
431 // Age of data (time between publication and now).
432 const ageSeconds = (serverTimeMillis - timestamp) / 1000;
434 // Check if the server asked the clients to back off.
436 if (response.headers.has("Backoff")) {
437 const value = parseInt(response.headers.get("Backoff"), 10);
439 backoffSeconds = value;
445 currentEtag: `"${timestamp}"`,
453 * Test if a single object matches all given filters.
455 * @param {Object} filters The filters object.
456 * @param {Object} entry The object to filter.
459 filterObject(filters, entry) {
460 return Object.entries(filters).every(([filter, value]) => {
461 if (Array.isArray(value)) {
462 return value.some(candidate => candidate === entry[filter]);
463 } else if (typeof value === "object") {
464 return Utils.filterObject(value, entry[filter]);
465 } else if (!Object.prototype.hasOwnProperty.call(entry, filter)) {
466 console.error(`The property ${filter} does not exist`);
469 return entry[filter] === value;
474 * Sorts records in a list according to a given ordering.
476 * @param {String} order The ordering, eg. `-last_modified`.
477 * @param {Array} list The collection to order.
480 sortObjects(order, list) {
481 const hasDash = order[0] === "-";
482 const field = hasDash ? order.slice(1) : order;
483 const direction = hasDash ? -1 : 1;
484 return list.slice().sort((a, b) => {
485 if (a[field] && _isUndefined(b[field])) {
488 if (b[field] && _isUndefined(a[field])) {
491 if (_isUndefined(a[field]) && _isUndefined(b[field])) {
494 return a[field] > b[field] ? direction : -direction;