1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
5 var EXPORTED_SYMBOLS = ["Downloader"];
7 const { XPCOMUtils } = ChromeUtils.importESModule(
8 "resource://gre/modules/XPCOMUtils.sys.mjs"
11 XPCOMUtils.defineLazyModuleGetters(lazy, {
12 RemoteSettingsWorker: "resource://services-settings/RemoteSettingsWorker.jsm",
13 Utils: "resource://services-settings/Utils.jsm",
15 ChromeUtils.defineModuleGetter(lazy, "OS", "resource://gre/modules/osfile.jsm");
17 class DownloadError extends Error {
18 constructor(url, resp) {
19 super(`Could not download ${url}`);
20 this.name = "DownloadError";
25 class BadContentError extends Error {
27 super(`${path} content does not match server hash`);
28 this.name = "BadContentError";
32 class ServerInfoError extends Error {
34 super(`Server response is invalid ${error}`);
35 this.name = "ServerInfoError";
36 this.original = error;
40 // Helper for the `download` method for commonly used methods, to help with
41 // lazily accessing the record and attachment content.
42 class LazyRecordAndBuffer {
43 constructor(getRecordAndLazyBuffer) {
44 this.getRecordAndLazyBuffer = getRecordAndLazyBuffer;
47 async _ensureRecordAndLazyBuffer() {
48 if (!this.recordAndLazyBufferPromise) {
49 this.recordAndLazyBufferPromise = this.getRecordAndLazyBuffer();
51 return this.recordAndLazyBufferPromise;
55 * @returns {object} The attachment record, if found. null otherwise.
59 return (await this._ensureRecordAndLazyBuffer()).record;
66 * @param {object} requestedRecord An attachment record
67 * @returns {boolean} Whether the requested record matches this record.
69 async isMatchingRequestedRecord(requestedRecord) {
70 const record = await this.getRecord();
73 record.last_modified === requestedRecord.last_modified &&
74 record.attachment.size === requestedRecord.attachment.size &&
75 record.attachment.hash === requestedRecord.attachment.hash
80 * Generate the return value for the "download" method.
82 * @throws {*} if the record or attachment content is unavailable.
83 * @returns {Object} An object with two properties:
84 * buffer: ArrayBuffer with the file content.
85 * record: Record associated with the bytes.
88 const { record, readBuffer } = await this._ensureRecordAndLazyBuffer();
89 if (!this.bufferPromise) {
90 this.bufferPromise = readBuffer();
92 return { record, buffer: await this.bufferPromise };
97 static get DownloadError() {
100 static get BadContentError() {
101 return BadContentError;
103 static get ServerInfoError() {
104 return ServerInfoError;
107 constructor(...folders) {
108 this.folders = ["settings", ...folders];
113 * @returns {Object} An object with async "get", "set" and "delete" methods.
114 * The keys are strings, the values may be any object that
115 * can be stored in IndexedDB (including Blob).
118 throw new Error("This Downloader does not support caching");
122 * Download attachment and return the result together with the record.
123 * If the requested record cannot be downloaded and fallbacks are enabled, the
124 * returned attachment may have a different record than the input record.
126 * @param {Object} record A Remote Settings entry with attachment.
127 * If omitted, the attachmentId option must be set.
128 * @param {Object} options Some download options.
129 * @param {Number} options.retries Number of times download should be retried (default: `3`)
130 * @param {Boolean} options.checkHash Check content integrity (default: `true`)
131 * @param {string} options.attachmentId The attachment identifier to use for
132 * caching and accessing the attachment.
133 * (default: `record.id`)
134 * @param {Boolean} options.fallbackToCache Return the cached attachment when the
135 * input record cannot be fetched.
137 * @param {Boolean} options.fallbackToDump Use the remote settings dump as a
138 * potential source of the attachment.
140 * @throws {Downloader.DownloadError} if the file could not be fetched.
141 * @throws {Downloader.BadContentError} if the downloaded content integrity is not valid.
142 * @throws {Downloader.ServerInfoError} if the server response is not valid.
143 * @throws {NetworkError} if fetching the server infos and fetching the attachment fails.
144 * @returns {Object} An object with two properties:
145 * `buffer` `ArrayBuffer`: the file content.
146 * `record` `Object`: record associated with the attachment.
147 * `_source` `String`: identifies the source of the result. Used for testing.
149 async download(record, options) {
153 attachmentId = record?.id,
154 fallbackToCache = false,
155 fallbackToDump = false,
158 // Check for pre-condition. This should not happen, but it is explicitly
159 // checked to avoid mixing up attachments, which could be dangerous.
161 "download() was called without attachmentId or `record.id`"
165 const dumpInfo = new LazyRecordAndBuffer(() =>
166 this._readAttachmentDump(attachmentId)
168 const cacheInfo = new LazyRecordAndBuffer(() =>
169 this._readAttachmentCache(attachmentId)
172 // Check if an attachment dump has been packaged with the client.
173 // The dump is checked before the cache because dumps are expected to match
174 // the requested record, at least shortly after the release of the client.
175 if (fallbackToDump && record) {
176 if (await dumpInfo.isMatchingRequestedRecord(record)) {
178 return { ...(await dumpInfo.getResult()), _source: "dump_match" };
180 // Failed to read dump: record found but attachment file is missing.
186 // Check if the requested attachment has already been cached.
188 if (await cacheInfo.isMatchingRequestedRecord(record)) {
190 return { ...(await cacheInfo.getResult()), _source: "cache_match" };
192 // Failed to read cache, e.g. IndexedDB unusable.
200 // There is no local version that matches the requested record.
201 // Try to download the attachment specified in record.
202 if (record && record.attachment) {
204 const newBuffer = await this.downloadAsBytes(record, {
208 const blob = new Blob([newBuffer]);
209 // Store in cache but don't wait for it before returning.
211 .set(attachmentId, { record, blob })
212 .catch(e => console.error(e));
213 return { buffer: newBuffer, record, _source: "remote_match" };
215 // No network, corrupted content, etc.
220 // Unable to find an attachment that matches the record. Consider falling
221 // back to local versions, even if their attachment hash do not match the
222 // one from the requested record.
224 // Unable to find a valid attachment, fall back to the cached attachment.
225 const cacheRecord = fallbackToCache && (await cacheInfo.getRecord());
227 const dumpRecord = fallbackToDump && (await dumpInfo.getRecord());
228 if (dumpRecord?.last_modified >= cacheRecord.last_modified) {
229 // The dump can be more recent than the cache when the client (and its
230 // packaged dump) is updated.
232 return { ...(await dumpInfo.getResult()), _source: "dump_fallback" };
234 // Failed to read dump: record found but attachment file is missing.
240 return { ...(await cacheInfo.getResult()), _source: "cache_fallback" };
242 // Failed to read from cache, e.g. IndexedDB unusable.
247 // Unable to find a valid attachment, fall back to the packaged dump.
248 if (fallbackToDump && (await dumpInfo.getRecord())) {
250 return { ...(await dumpInfo.getResult()), _source: "dump_fallback" };
256 if (errorIfAllFails) {
257 throw errorIfAllFails;
260 throw new Downloader.DownloadError(attachmentId);
264 * Delete the record attachment downloaded locally.
265 * No-op if the attachment does not exist.
267 * @param record A Remote Settings entry with attachment.
268 * @param {Object} options Some options.
269 * @param {string} options.attachmentId The attachment identifier to use for
270 * accessing and deleting the attachment.
271 * (default: `record.id`)
273 async deleteDownloaded(record, options) {
274 let { attachmentId = record?.id } = options || {};
276 // Check for pre-condition. This should not happen, but it is explicitly
277 // checked to avoid mixing up attachments, which could be dangerous.
279 "deleteDownloaded() was called without attachmentId or `record.id`"
282 return this.cacheImpl.delete(attachmentId);
286 * @deprecated See https://bugzilla.mozilla.org/show_bug.cgi?id=1634127
288 * Download the record attachment into the local profile directory
289 * and return a file:// URL that points to the local path.
291 * No-op if the file was already downloaded and not corrupted.
293 * @param {Object} record A Remote Settings entry with attachment.
294 * @param {Object} options Some download options.
295 * @param {Number} options.retries Number of times download should be retried (default: `3`)
296 * @throws {Downloader.DownloadError} if the file could not be fetched.
297 * @throws {Downloader.BadContentError} if the downloaded file integrity is not valid.
298 * @throws {Downloader.ServerInfoError} if the server response is not valid.
299 * @throws {NetworkError} if fetching the attachment fails.
300 * @returns {String} the absolute file path to the downloaded attachment.
302 async downloadToDisk(record, options = {}) {
303 const { retries = 3 } = options;
305 attachment: { filename, size, hash },
307 const localFilePath = lazy.OS.Path.join(
308 lazy.OS.Constants.Path.localProfileDir,
312 const localFileUrl = `file://${[
313 ...lazy.OS.Path.split(lazy.OS.Constants.Path.localProfileDir).components,
318 await this._makeDirs();
323 await lazy.RemoteSettingsWorker.checkFileHash(localFileUrl, size, hash)
327 // File does not exist or is corrupted.
328 if (retried > retries) {
329 throw new Downloader.BadContentError(localFilePath);
332 // Download and write on disk.
333 const buffer = await this.downloadAsBytes(record, {
334 checkHash: false, // Hash will be checked on file.
335 retries: 0, // Already in a retry loop.
337 await IOUtils.write(localFilePath, new Uint8Array(buffer), {
338 tmpPath: `${localFilePath}.tmp`,
341 if (retried >= retries) {
350 * Download the record attachment and return its content as bytes.
352 * @param {Object} record A Remote Settings entry with attachment.
353 * @param {Object} options Some download options.
354 * @param {Number} options.retries Number of times download should be retried (default: `3`)
355 * @param {Boolean} options.checkHash Check content integrity (default: `true`)
356 * @throws {Downloader.DownloadError} if the file could not be fetched.
357 * @throws {Downloader.BadContentError} if the downloaded content integrity is not valid.
358 * @returns {ArrayBuffer} the file content.
360 async downloadAsBytes(record, options = {}) {
362 attachment: { location, hash, size },
365 const remoteFileUrl = (await this._baseAttachmentsURL()) + location;
367 const { retries = 3, checkHash = true } = options;
371 const buffer = await this._fetchAttachment(remoteFileUrl);
376 await lazy.RemoteSettingsWorker.checkContentHash(buffer, size, hash)
380 // Content is corrupted.
381 throw new Downloader.BadContentError(location);
383 if (retried >= retries) {
392 * @deprecated See https://bugzilla.mozilla.org/show_bug.cgi?id=1634127
394 * Delete the record attachment downloaded locally.
395 * This is the counterpart of `downloadToDisk()`.
396 * Use `deleteDownloaded()` if `download()` was used to retrieve
399 * No-op if the related file does not exist.
401 * @param record A Remote Settings entry with attachment.
403 async deleteFromDisk(record) {
405 attachment: { filename },
407 const path = lazy.OS.Path.join(
408 lazy.OS.Constants.Path.localProfileDir,
412 await IOUtils.remove(path);
413 await this._rmDirs();
416 async _baseAttachmentsURL() {
417 if (!this._cdnURLs[lazy.Utils.SERVER_URL]) {
418 const resp = await lazy.Utils.fetch(`${lazy.Utils.SERVER_URL}/`);
421 serverInfo = await resp.json();
423 throw new Downloader.ServerInfoError(error);
425 // Server capabilities expose attachments configuration.
428 attachments: { base_url },
431 // Make sure the URL always has a trailing slash.
432 this._cdnURLs[lazy.Utils.SERVER_URL] =
433 base_url + (base_url.endsWith("/") ? "" : "/");
435 return this._cdnURLs[lazy.Utils.SERVER_URL];
438 async _fetchAttachment(url) {
439 const headers = new Headers();
440 headers.set("Accept-Encoding", "gzip");
441 const resp = await lazy.Utils.fetch(url, { headers });
443 throw new Downloader.DownloadError(url, resp);
445 return resp.arrayBuffer();
448 async _readAttachmentCache(attachmentId) {
449 const cached = await this.cacheImpl.get(attachmentId);
451 throw new Downloader.DownloadError(attachmentId);
454 record: cached.record,
456 const buffer = await cached.blob.arrayBuffer();
457 const { size, hash } = cached.record.attachment;
459 await lazy.RemoteSettingsWorker.checkContentHash(buffer, size, hash)
463 // Really unexpected, could indicate corruption in IndexedDB.
464 throw new Downloader.BadContentError(attachmentId);
469 async _readAttachmentDump(attachmentId) {
470 async function fetchResource(resourceUrl) {
472 return await fetch(resourceUrl);
474 throw new Downloader.DownloadError(resourceUrl);
477 const resourceUrlPrefix =
478 Downloader._RESOURCE_BASE_URL + "/" + this.folders.join("/") + "/";
479 const recordUrl = `${resourceUrlPrefix}${attachmentId}.meta.json`;
480 const attachmentUrl = `${resourceUrlPrefix}${attachmentId}`;
481 const record = await (await fetchResource(recordUrl)).json();
485 return (await fetchResource(attachmentUrl)).arrayBuffer();
490 // Separate variable to allow tests to override this.
491 static _RESOURCE_BASE_URL = "resource://app/defaults";
494 const dirPath = lazy.OS.Path.join(
495 lazy.OS.Constants.Path.localProfileDir,
498 await IOUtils.makeDirectory(dirPath, { createAncestors: true });
502 for (let i = this.folders.length; i > 0; i--) {
503 const dirPath = lazy.OS.Path.join(
504 lazy.OS.Constants.Path.localProfileDir,
505 ...this.folders.slice(0, i)
508 await IOUtils.remove(dirPath);
510 // This could fail if there's something in
511 // the folder we're not permitted to remove.