1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 * Handles serialization of the data and persistence into a file.
8 * This modules handles the raw data stored in JavaScript serializable objects,
9 * and contains no special validation or query logic, that is handled entirely
10 * by "storage.js" instead.
12 * The data can be manipulated only after it has been loaded from disk. The
13 * load process can happen asynchronously, through the "load" method, or
14 * synchronously, through "ensureDataReady". After any modification, the
15 * "saveSoon" method must be called to flush the data to disk asynchronously.
17 * The raw data should be manipulated synchronously, without waiting for the
18 * event loop or for promise resolution, so that the saved file is always
19 * consistent. This synchronous approach also simplifies the query and update
20 * logic. For example, it is possible to find an object and modify it
21 * immediately without caring whether other code modifies it in the meantime.
23 * An asynchronous shutdown observer makes sure that data is always saved before
24 * the browser is closed. The data cannot be modified during shutdown.
26 * The file is stored in JSON format, without indentation, using UTF-8 encoding.
33 ChromeUtils.defineESModuleGetters(lazy, {
34 DeferredTask: "resource://gre/modules/DeferredTask.sys.mjs",
35 FileUtils: "resource://gre/modules/FileUtils.sys.mjs",
36 NetUtil: "resource://gre/modules/NetUtil.sys.mjs",
39 ChromeUtils.defineLazyGetter(lazy, "gTextDecoder", function () {
40 return new TextDecoder();
43 const FileInputStream = Components.Constructor(
44 "@mozilla.org/network/file-input-stream;1",
50 * Delay between a change to the data and the related save operation.
52 const kSaveDelayMs = 1500;
55 * Cleansed basenames of the filenames that telemetry can be recorded for.
56 * Keep synchronized with 'objects' from Events.yaml.
58 const TELEMETRY_BASENAMES = new Set(["logins", "autofillprofiles"]);
63 * Handles serialization of the data and persistence into a file.
65 * @param config An object containing following members:
66 * - path: String containing the file path where data should be saved.
67 * - sanitizedBasename: Sanitized string identifier used for logging,
68 * shutdown debugging, and telemetry. Defaults to
69 * basename of given `path`, sanitized.
70 * - dataPostProcessor: Function triggered when data is just loaded. The
71 * data object will be passed as the first argument
72 * and should be returned no matter it's modified or
73 * not. Its failure leads to the failure of load()
74 * and ensureDataReady().
75 * - saveDelayMs: Number indicating the delay (in milliseconds) between a
76 * change to the data and the related save operation. The
77 * default value will be applied if omitted.
78 * - beforeSave: Promise-returning function triggered just before the
79 * data is written to disk. This can be used to create any
80 * intermediate directories before saving. The file will
81 * not be saved if the promise rejects or the function
82 * throws an exception.
83 * - finalizeAt: An `IOUtils` phase or barrier client that should
84 * automatically finalize the file when triggered. Defaults
85 * to `profileBeforeChange`; exposed as an option for
87 * - compression: A compression algorithm to use when reading and
89 * - backupTo: A string value indicating where writeAtomic should create
90 * a backup before writing to json files. Note that using this
91 * option currently ensures that we automatically restore backed
92 * up json files in load() and ensureDataReady() when original
93 * files are missing or corrupt.
95 export function JSONFile(config) {
96 this.path = config.path;
97 this.sanitizedBasename =
98 config.sanitizedBasename ??
99 PathUtils.filename(this.path)
100 .replace(/\.json(.lz4)?$/, "")
101 .replaceAll(/[^a-zA-Z0-9_.]/g, "");
103 if (typeof config.dataPostProcessor === "function") {
104 this._dataPostProcessor = config.dataPostProcessor;
106 if (typeof config.beforeSave === "function") {
107 this._beforeSave = config.beforeSave;
110 if (config.saveDelayMs === undefined) {
111 config.saveDelayMs = kSaveDelayMs;
113 this._saver = new lazy.DeferredTask(() => this._save(), config.saveDelayMs);
116 if (config.compression) {
117 this._options.decompress = this._options.compress = true;
120 if (config.backupTo) {
121 this._options.backupFile = this._options.backupTo = config.backupTo;
124 this._finalizeAt = config.finalizeAt || IOUtils.profileBeforeChange;
125 this._finalizeInternalBound = this._finalizeInternal.bind(this);
126 this._finalizeAt.addBlocker(
127 `JSON store: writing data for '${this.sanitizedBasename}'`,
128 this._finalizeInternalBound,
129 () => ({ sanitizedBasename: this.sanitizedBasename })
132 Services.telemetry.setEventRecordingEnabled("jsonfile", true);
135 JSONFile.prototype = {
137 * String containing the file path where data should be saved.
142 * Sanitized identifier used for logging, shutdown debugging, and telemetry.
144 sanitizedBasename: "",
147 * True when data has been loaded.
152 * DeferredTask that handles the save operation.
157 * Internal data object.
162 * Internal fields used during finalization.
165 _finalizePromise: null,
166 _finalizeInternalBound: null,
169 * Serializable object containing the data. This is populated directly with
170 * the data loaded from the file, and is saved without modifications.
172 * The raw data should be manipulated synchronously, without waiting for the
173 * event loop or for promise resolution, so that the saved file is always
177 if (!this.dataReady) {
178 throw new Error("Data is not ready.");
184 * Sets the loaded data to a new object. This will overwrite any persisted
185 * data on the next save.
189 this.dataReady = true;
193 * Loads persistent data from the file to memory.
196 * @resolves When the operation finished successfully.
197 * @rejects JavaScript exception when dataPostProcessor fails. It never fails
198 * if there is no dataPostProcessor.
201 if (this.dataReady) {
208 data = await IOUtils.readJSON(this.path, this._options);
210 // If synchronous loading happened in the meantime, exit now.
211 if (this.dataReady) {
215 // If an exception occurs because the file does not exist or it cannot be read,
217 // 1. For consumers of JSONFile.sys.mjs that have configured a `backupTo` path option,
218 // we try to look for and use backed up json files first. If the backup
219 // is also not found or if the backup is unreadable, we then start with an empty file.
220 // 2. If a consumer does not configure a `backupTo` path option, we just start
221 // with an empty file.
223 // In the event that the file exists, but an exception is thrown because it cannot be read,
224 // we store it as a .corrupt file for debugging purposes.
226 let errorNo = ex.winLastError || ex.unixErrno;
227 this._recordTelemetry("load", errorNo ? errorNo.toString() : "");
228 if (!(DOMException.isInstance(ex) && ex.name == "NotFoundError")) {
231 // Move the original file to a backup location, ignoring errors.
233 let uniquePath = await IOUtils.createUniqueFile(
234 PathUtils.parent(this.path),
235 PathUtils.filename(this.path) + ".corrupt",
238 await IOUtils.move(this.path, uniquePath);
239 this._recordTelemetry("load", "invalid_json");
245 if (this._options.backupFile) {
246 // Restore the original file from the backup here so fresh writes to empty
247 // json files don't happen at any time in the future compromising the backup
250 await IOUtils.copy(this._options.backupFile, this.path);
252 if (!(DOMException.isInstance(e) && e.name == "NotFoundError")) {
258 // We still read from the backup file here instead of the original file in case
259 // access to the original file is blocked, e.g. by anti-virus software on the
261 data = await IOUtils.readJSON(
262 this._options.backupFile,
265 // If synchronous loading happened in the meantime, exit now.
266 if (this.dataReady) {
269 this._recordTelemetry("load", "used_backup");
271 if (!(DOMException.isInstance(e3) && e3.name == "NotFoundError")) {
277 // In some rare cases it's possible for data to have been added to
278 // our database between the call to IOUtils.read and when we've been
279 // notified that there was a problem with it. In that case, leave the
280 // synchronously-added data alone.
281 if (this.dataReady) {
286 this._processLoadedData(data);
290 * Loads persistent data from the file to memory, synchronously. An exception
291 * can be thrown only if dataPostProcessor exists and fails.
294 if (this.dataReady) {
301 // This reads the file and automatically detects the UTF-8 encoding.
302 let inputStream = new FileInputStream(
303 new lazy.FileUtils.File(this.path),
304 lazy.FileUtils.MODE_RDONLY,
305 lazy.FileUtils.PERMS_FILE,
309 let bytes = lazy.NetUtil.readInputStream(
311 inputStream.available()
313 data = JSON.parse(lazy.gTextDecoder.decode(bytes));
318 // If an exception occurs because the file does not exist or it cannot be read,
320 // 1. For consumers of JSONFile.sys.mjs that have configured a `backupTo` path option,
321 // we try to look for and use backed up json files first. If the backup
322 // is also not found or if the backup is unreadable, we then start with an empty file.
323 // 2. If a consumer does not configure a `backupTo` path option, we just start
324 // with an empty file.
326 // In the event that the file exists, but an exception is thrown because it cannot be read,
327 // we store it as a .corrupt file for debugging purposes.
330 ex instanceof Components.Exception &&
331 ex.result == Cr.NS_ERROR_FILE_NOT_FOUND
335 // Move the original file to a backup location, ignoring errors.
337 let originalFile = new lazy.FileUtils.File(this.path);
338 let backupFile = originalFile.clone();
339 backupFile.leafName += ".corrupt";
340 backupFile.createUnique(
341 Ci.nsIFile.NORMAL_FILE_TYPE,
342 lazy.FileUtils.PERMS_FILE
344 backupFile.remove(false);
345 originalFile.moveTo(backupFile.parent, backupFile.leafName);
351 if (this._options.backupFile) {
352 // Restore the original file from the backup here so fresh writes to empty
353 // json files don't happen at any time in the future compromising the backup
356 let basename = PathUtils.filename(this.path);
357 let backupFile = new lazy.FileUtils.File(this._options.backupFile);
358 backupFile.copyTo(null, basename);
360 if (e.result != Cr.NS_ERROR_FILE_NOT_FOUND) {
366 // We still read from the backup file here instead of the original file in case
367 // access to the original file is blocked, e.g. by anti-virus software on the
369 // This reads the file and automatically detects the UTF-8 encoding.
370 let inputStream = new FileInputStream(
371 new lazy.FileUtils.File(this._options.backupFile),
372 lazy.FileUtils.MODE_RDONLY,
373 lazy.FileUtils.PERMS_FILE,
377 let bytes = lazy.NetUtil.readInputStream(
379 inputStream.available()
381 data = JSON.parse(lazy.gTextDecoder.decode(bytes));
386 if (e3.result != Cr.NS_ERROR_FILE_NOT_FOUND) {
393 this._processLoadedData(data);
397 * Called when the data changed, this triggers asynchronous serialization.
400 return this._saver.arm();
404 * Saves persistent data from memory to the file.
406 * If an error occurs, the previous file is not deleted.
409 * @resolves When the operation finished successfully.
410 * @rejects JavaScript exception.
413 // Create or overwrite the file.
414 if (this._beforeSave) {
415 await Promise.resolve(this._beforeSave());
419 await IOUtils.writeJSON(
422 Object.assign({ tmpPath: this.path + ".tmp" }, this._options)
425 if (typeof this._data.toJSONSafe == "function") {
426 // If serialization fails, try fallback safe JSON converter.
427 await IOUtils.writeUTF8(
429 this._data.toJSONSafe(),
430 Object.assign({ tmpPath: this.path + ".tmp" }, this._options)
437 * Synchronously work on the data just loaded into memory.
439 _processLoadedData(data) {
440 if (this._finalizePromise) {
441 // It's possible for `load` to race with `finalize`. In that case, don't
442 // process or set the loaded data.
445 this.data = this._dataPostProcessor ? this._dataPostProcessor(data) : data;
448 _recordTelemetry(method, value) {
449 if (!TELEMETRY_BASENAMES.has(this.sanitizedBasename)) {
450 // Avoid recording so we don't log an error in the console.
454 Services.telemetry.recordEvent(
457 this.sanitizedBasename,
463 * Finishes persisting data to disk and resets all state for this file.
466 * @resolves When the object is finalized.
468 _finalizeInternal() {
469 if (this._finalizePromise) {
470 // Finalization already in progress; return the pending promise. This is
471 // possible if `finalize` is called concurrently with shutdown.
472 return this._finalizePromise;
474 this._finalizePromise = (async () => {
475 await this._saver.finalize();
477 this.dataReady = false;
479 return this._finalizePromise;
483 * Ensures that all data is persisted to disk, and prevents future calls to
484 * `saveSoon`. This is called automatically on shutdown, but can also be
485 * called explicitly when the file is no longer needed.
488 if (this._finalizePromise) {
489 throw new Error(`The file ${this.path} has already been finalized`);
491 // Wait for finalization before removing the shutdown blocker.
492 await this._finalizeInternal();
493 this._finalizeAt.removeBlocker(this._finalizeInternalBound);