1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
5 var EXPORTED_SYMBOLS = ["Downloader"];
7 const { XPCOMUtils } = ChromeUtils.import(
8 "resource://gre/modules/XPCOMUtils.jsm"
10 XPCOMUtils.defineLazyModuleGetters(this, {
11 RemoteSettingsWorker: "resource://services-settings/RemoteSettingsWorker.jsm",
12 Utils: "resource://services-settings/Utils.jsm",
14 ChromeUtils.defineModuleGetter(this, "OS", "resource://gre/modules/osfile.jsm");
15 XPCOMUtils.defineLazyGlobalGetters(this, ["fetch"]);
17 class DownloadError extends Error {
18 constructor(url, resp) {
19 super(`Could not download ${url}`);
20 this.name = "DownloadError";
25 class BadContentError extends Error {
27 super(`${path} content does not match server hash`);
28 this.name = "BadContentError";
32 // Helper for the `download` method for commonly used methods, to help with
33 // lazily accessing the record and attachment content.
34 class LazyRecordAndBuffer {
35 constructor(getRecordAndLazyBuffer) {
36 this.getRecordAndLazyBuffer = getRecordAndLazyBuffer;
39 async _ensureRecordAndLazyBuffer() {
40 if (!this.recordAndLazyBufferPromise) {
41 this.recordAndLazyBufferPromise = this.getRecordAndLazyBuffer();
43 return this.recordAndLazyBufferPromise;
47 * @returns {object} The attachment record, if found. null otherwise.
51 return (await this._ensureRecordAndLazyBuffer()).record;
58 * @param {object} requestedRecord An attachment record
59 * @returns {boolean} Whether the requested record matches this record.
61 async isMatchingRequestedRecord(requestedRecord) {
62 const record = await this.getRecord();
65 record.last_modified === requestedRecord.last_modified &&
66 record.attachment.size === requestedRecord.attachment.size &&
67 record.attachment.hash === requestedRecord.attachment.hash
72 * Generate the return value for the "download" method.
74 * @throws {*} if the record or attachment content is unavailable.
75 * @returns {Object} An object with two properties:
76 * buffer: ArrayBuffer with the file content.
77 * record: Record associated with the bytes.
80 const { record, readBuffer } = await this._ensureRecordAndLazyBuffer();
81 if (!this.bufferPromise) {
82 this.bufferPromise = readBuffer();
84 return { record, buffer: await this.bufferPromise };
89 static get DownloadError() {
92 static get BadContentError() {
93 return BadContentError;
96 constructor(...folders) {
97 this.folders = ["settings", ...folders];
102 * @returns {Object} An object with async "get", "set" and "delete" methods.
103 * The keys are strings, the values may be any object that
104 * can be stored in IndexedDB (including Blob).
107 throw new Error("This Downloader does not support caching");
111 * Download attachment and return the result together with the record.
112 * If the requested record cannot be downloaded and fallbacks are enabled, the
113 * returned attachment may have a different record than the input record.
115 * @param {Object} record A Remote Settings entry with attachment.
116 * If omitted, the attachmentId option must be set.
117 * @param {Object} options Some download options.
118 * @param {Number} options.retries Number of times download should be retried (default: `3`)
119 * @param {Number} options.checkHash Check content integrity (default: `true`)
120 * @param {string} options.attachmentId The attachment identifier to use for
121 * caching and accessing the attachment.
122 * (default: record.id)
123 * @param {Boolean} options.useCache Whether to use a cache to read and store
124 * the attachment. (default: false)
125 * @param {Boolean} options.fallbackToCache Return the cached attachment when the
126 * input record cannot be fetched.
128 * @param {Boolean} options.fallbackToDump Use the remote settings dump as a
129 * potential source of the attachment.
131 * @throws {Downloader.DownloadError} if the file could not be fetched.
132 * @throws {Downloader.BadContentError} if the downloaded content integrity is not valid.
133 * @returns {Object} An object with two properties:
134 * buffer: ArrayBuffer with the file content.
135 * record: Record associated with the bytes.
136 * _source: identifies the source of the result. Used for testing.
138 async download(record, options) {
142 attachmentId = record?.id,
144 fallbackToCache = false,
145 fallbackToDump = false,
149 // For backwards compatibility.
150 // WARNING: Its return type is different from what's documented.
151 // See downloadToDisk's documentation.
152 return this.downloadToDisk(record, options);
155 if (!this.cacheImpl) {
156 throw new Error("useCache is true but there is no cacheImpl!");
160 // Check for pre-condition. This should not happen, but it is explicitly
161 // checked to avoid mixing up attachments, which could be dangerous.
162 throw new Error("download() was called without attachmentId or recordID");
165 const dumpInfo = new LazyRecordAndBuffer(() =>
166 this._readAttachmentDump(attachmentId)
168 const cacheInfo = new LazyRecordAndBuffer(() =>
169 this._readAttachmentCache(attachmentId)
172 // Check if an attachment dump has been packaged with the client.
173 // The dump is checked before the cache because dumps are expected to match
174 // the requested record, at least shortly after the release of the client.
175 if (fallbackToDump && record) {
176 if (await dumpInfo.isMatchingRequestedRecord(record)) {
178 return { ...(await dumpInfo.getResult()), _source: "dump_match" };
180 // Failed to read dump: record found but attachment file is missing.
186 // Check if the requested attachment has already been cached.
187 if (useCache && record) {
188 if (await cacheInfo.isMatchingRequestedRecord(record)) {
190 return { ...(await cacheInfo.getResult()), _source: "cache_match" };
192 // Failed to read cache, e.g. IndexedDB unusable.
200 // There is no local version that matches the requested record.
201 // Try to download the attachment specified in record.
202 if (record && record.attachment) {
204 const newBuffer = await this.downloadAsBytes(record, {
208 const blob = new Blob([newBuffer]);
210 // Caching is optional, don't wait for the cache before returning.
212 .set(attachmentId, { record, blob })
213 .catch(e => Cu.reportError(e));
215 return { buffer: newBuffer, record, _source: "remote_match" };
217 // No network, corrupted content, etc.
222 // Unable to find an attachment that matches the record. Consider falling
223 // back to local versions, even if their attachment hash do not match the
224 // one from the requested record.
226 // Unable to find a valid attachment, fall back to the cached attachment.
227 const cacheRecord = fallbackToCache && (await cacheInfo.getRecord());
229 const dumpRecord = fallbackToDump && (await dumpInfo.getRecord());
230 if (dumpRecord?.last_modified >= cacheRecord.last_modified) {
231 // The dump can be more recent than the cache when the client (and its
232 // packaged dump) is updated.
234 return { ...(await dumpInfo.getResult()), _source: "dump_fallback" };
236 // Failed to read dump: record found but attachment file is missing.
242 return { ...(await cacheInfo.getResult()), _source: "cache_fallback" };
244 // Failed to read from cache, e.g. IndexedDB unusable.
249 // Unable to find a valid attachment, fall back to the packaged dump.
250 if (fallbackToDump && (await dumpInfo.getRecord())) {
252 return { ...(await dumpInfo.getResult()), _source: "dump_fallback" };
258 if (errorIfAllFails) {
259 throw errorIfAllFails;
262 throw new Downloader.DownloadError(attachmentId);
266 * Download the record attachment into the local profile directory
267 * and return a file:// URL that points to the local path.
269 * No-op if the file was already downloaded and not corrupted.
271 * @param {Object} record A Remote Settings entry with attachment.
272 * @param {Object} options Some download options.
273 * @param {Number} options.retries Number of times download should be retried (default: `3`)
274 * @throws {Downloader.DownloadError} if the file could not be fetched.
275 * @throws {Downloader.BadContentError} if the downloaded file integrity is not valid.
276 * @returns {String} the absolute file path to the downloaded attachment.
278 async downloadToDisk(record, options = {}) {
279 const { retries = 3 } = options;
281 attachment: { filename, size, hash },
283 const localFilePath = OS.Path.join(
284 OS.Constants.Path.localProfileDir,
288 const localFileUrl = `file://${[
289 ...OS.Path.split(OS.Constants.Path.localProfileDir).components,
294 await this._makeDirs();
298 if (await RemoteSettingsWorker.checkFileHash(localFileUrl, size, hash)) {
301 // File does not exist or is corrupted.
302 if (retried > retries) {
303 throw new Downloader.BadContentError(localFilePath);
306 // Download and write on disk.
307 const buffer = await this.downloadAsBytes(record, {
308 checkHash: false, // Hash will be checked on file.
309 retries: 0, // Already in a retry loop.
311 await OS.File.writeAtomic(localFilePath, buffer, {
312 tmpPath: `${localFilePath}.tmp`,
315 if (retried >= retries) {
324 * Download the record attachment and return its content as bytes.
326 * @param {Object} record A Remote Settings entry with attachment.
327 * @param {Object} options Some download options.
328 * @param {Number} options.retries Number of times download should be retried (default: `3`)
329 * @param {Number} options.checkHash Check content integrity (default: `true`)
330 * @throws {Downloader.DownloadError} if the file could not be fetched.
331 * @throws {Downloader.BadContentError} if the downloaded content integrity is not valid.
332 * @returns {ArrayBuffer} the file content.
334 async downloadAsBytes(record, options = {}) {
336 attachment: { location, hash, size },
339 const remoteFileUrl = (await this._baseAttachmentsURL()) + location;
341 const { retries = 3, checkHash = true } = options;
345 const buffer = await this._fetchAttachment(remoteFileUrl);
349 if (await RemoteSettingsWorker.checkContentHash(buffer, size, hash)) {
352 // Content is corrupted.
353 throw new Downloader.BadContentError(location);
355 if (retried >= retries) {
364 * Delete the record attachment downloaded locally.
365 * No-op if the related file does not exist.
367 * @param record A Remote Settings entry with attachment.
369 async delete(record) {
371 attachment: { filename },
373 const path = OS.Path.join(
374 OS.Constants.Path.localProfileDir,
378 await OS.File.remove(path, { ignoreAbsent: true });
379 await this._rmDirs();
382 async deleteCached(attachmentId) {
383 return this.cacheImpl.delete(attachmentId);
386 async _baseAttachmentsURL() {
388 const server = Utils.SERVER_URL;
389 const serverInfo = await (await Utils.fetch(`${server}/`)).json();
390 // Server capabilities expose attachments configuration.
393 attachments: { base_url },
396 // Make sure the URL always has a trailing slash.
397 this._cdnURL = base_url + (base_url.endsWith("/") ? "" : "/");
402 async _fetchAttachment(url) {
403 const headers = new Headers();
404 headers.set("Accept-Encoding", "gzip");
405 const resp = await Utils.fetch(url, { headers });
407 throw new Downloader.DownloadError(url, resp);
409 return resp.arrayBuffer();
412 async _readAttachmentCache(attachmentId) {
413 const cached = await this.cacheImpl.get(attachmentId);
415 throw new Downloader.DownloadError(attachmentId);
418 record: cached.record,
420 const buffer = await cached.blob.arrayBuffer();
421 const { size, hash } = cached.record.attachment;
422 if (await RemoteSettingsWorker.checkContentHash(buffer, size, hash)) {
425 // Really unexpected, could indicate corruption in IndexedDB.
426 throw new Downloader.BadContentError(attachmentId);
431 async _readAttachmentDump(attachmentId) {
432 async function fetchResource(resourceUrl) {
434 return await fetch(resourceUrl);
436 throw new Downloader.DownloadError(resourceUrl);
439 const resourceUrlPrefix =
440 Downloader._RESOURCE_BASE_URL + "/" + this.folders.join("/") + "/";
441 const recordUrl = `${resourceUrlPrefix}${attachmentId}.meta.json`;
442 const attachmentUrl = `${resourceUrlPrefix}${attachmentId}`;
443 const record = await (await fetchResource(recordUrl)).json();
447 return (await fetchResource(attachmentUrl)).arrayBuffer();
452 // Separate variable to allow tests to override this.
453 static _RESOURCE_BASE_URL = "resource://app/defaults";
456 const dirPath = OS.Path.join(
457 OS.Constants.Path.localProfileDir,
460 await OS.File.makeDir(dirPath, { from: OS.Constants.Path.localProfileDir });
464 for (let i = this.folders.length; i > 0; i--) {
465 const dirPath = OS.Path.join(
466 OS.Constants.Path.localProfileDir,
467 ...this.folders.slice(0, i)
470 await OS.File.removeEmptyDir(dirPath, { ignoreAbsent: true });
472 // This could fail if there's something in
473 // the folder we're not permitted to remove.