1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
5 #ifndef MERGED_COMPARTMENT
9 this.EXPORTED_SYMBOLS = ["HealthReporter"];
11 const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
13 const MILLISECONDS_PER_DAY = 24 * 60 * 60 * 1000;
15 Cu.import("resource://gre/modules/Metrics.jsm");
16 Cu.import("resource://services-common/async.js");
18 Cu.import("resource://services-common/bagheeraclient.js");
21 Cu.import("resource://gre/modules/Log.jsm");
22 Cu.import("resource://services-common/utils.js");
23 Cu.import("resource://gre/modules/Promise.jsm");
24 Cu.import("resource://gre/modules/osfile.jsm");
25 Cu.import("resource://gre/modules/Preferences.jsm");
26 Cu.import("resource://gre/modules/Services.jsm");
27 Cu.import("resource://gre/modules/Task.jsm");
28 Cu.import("resource://gre/modules/TelemetryStopwatch.jsm");
29 Cu.import("resource://gre/modules/XPCOMUtils.jsm");
31 XPCOMUtils.defineLazyModuleGetter(this, "UpdateChannel",
32 "resource://gre/modules/UpdateChannel.jsm");
34 // Oldest year to allow in date preferences. This module was implemented in
35 // 2012 and no dates older than that should be encountered.
36 const OLDEST_ALLOWED_YEAR = 2012;
38 const DAYS_IN_PAYLOAD = 180;
40 const DEFAULT_DATABASE_NAME = "healthreport.sqlite";
42 const TELEMETRY_INIT = "HEALTHREPORT_INIT_MS";
43 const TELEMETRY_INIT_FIRSTRUN = "HEALTHREPORT_INIT_FIRSTRUN_MS";
44 const TELEMETRY_DB_OPEN = "HEALTHREPORT_DB_OPEN_MS";
45 const TELEMETRY_DB_OPEN_FIRSTRUN = "HEALTHREPORT_DB_OPEN_FIRSTRUN_MS";
46 const TELEMETRY_GENERATE_PAYLOAD = "HEALTHREPORT_GENERATE_JSON_PAYLOAD_MS";
47 const TELEMETRY_JSON_PAYLOAD_SERIALIZE = "HEALTHREPORT_JSON_PAYLOAD_SERIALIZE_MS";
48 const TELEMETRY_PAYLOAD_SIZE_UNCOMPRESSED = "HEALTHREPORT_PAYLOAD_UNCOMPRESSED_BYTES";
49 const TELEMETRY_PAYLOAD_SIZE_COMPRESSED = "HEALTHREPORT_PAYLOAD_COMPRESSED_BYTES";
50 const TELEMETRY_UPLOAD = "HEALTHREPORT_UPLOAD_MS";
51 const TELEMETRY_COLLECT_CONSTANT = "HEALTHREPORT_COLLECT_CONSTANT_DATA_MS";
52 const TELEMETRY_COLLECT_DAILY = "HEALTHREPORT_COLLECT_DAILY_MS";
53 const TELEMETRY_SHUTDOWN = "HEALTHREPORT_SHUTDOWN_MS";
54 const TELEMETRY_COLLECT_CHECKPOINT = "HEALTHREPORT_POST_COLLECT_CHECKPOINT_MS";
58 * Helper type to assist with management of Health Reporter state.
60 * Instances are not meant to be created outside of a HealthReporter instance.
62 * There are two types of IDs associated with clients.
64 * Since the beginning of FHR, there has existed a per-upload ID: a UUID is
65 * generated at upload time and associated with the state before upload starts.
66 * That same upload includes a request to delete all other upload IDs known by
69 * Per-upload IDs had the unintended side-effect of creating "orphaned"
70 * records/upload IDs on the server. So, a stable client identifer has been
71 * introduced. This client identifier is generated when it's missing and sent
72 * as part of every upload.
74 * There is a high chance we may remove upload IDs in the future.
76 function HealthReporterState(reporter) {
77 this._reporter = reporter;
79 let profD = OS.Constants.Path.profileDir;
81 if (!profD || !profD.length) {
82 throw new Error("Could not obtain profile directory. OS.File not " +
83 "initialized properly?");
86 this._log = reporter._log;
88 this._stateDir = OS.Path.join(profD, "healthreport");
90 // To facilitate testing.
91 let leaf = reporter._stateLeaf || "state.json";
93 this._filename = OS.Path.join(this._stateDir, leaf);
94 this._log.debug("Storing state in " + this._filename);
98 HealthReporterState.prototype = Object.freeze({
100 * Persistent string identifier associated with this client.
103 return this._s.clientID;
107 * The version associated with the client ID.
109 get clientIDVersion() {
110 return this._s.clientIDVersion;
114 return new Date(this._s.lastPingTime);
118 return this._s.remoteIDs[0];
122 return this._s.remoteIDs;
125 get _lastPayloadPath() {
126 return OS.Path.join(this._stateDir, "lastpayload.json");
130 return Task.spawn(function* init() {
131 yield OS.File.makeDir(this._stateDir);
133 let drs = Cc["@mozilla.org/datareporting/service;1"]
134 .getService(Ci.nsISupports)
136 let drsClientID = yield drs.getClientID();
138 let resetObjectState = function () {
140 // The payload version. This is bumped whenever there is a
141 // backwards-incompatible change.
143 // The persistent client identifier.
144 clientID: drsClientID,
145 // Denotes the mechanism used to generate the client identifier.
148 // Upload IDs that might be on the server.
150 // When we last performed an uploaded.
152 // Tracks whether we removed an outdated payload.
153 removedOutdatedLastpayload: false,
158 this._s = yield CommonUtils.readJSON(this._filename);
159 } catch (ex if ex instanceof OS.File.Error &&
160 ex.becauseNoSuchFile) {
161 this._log.warn("Saved state file does not exist.");
164 this._log.error("Exception when reading state from disk: " +
165 CommonUtils.exceptionStr(ex));
168 // Don't save in case it goes away on next run.
171 if (typeof(this._s) != "object") {
172 this._log.warn("Read state is not an object. Resetting state.");
177 if (this._s.v != 1) {
178 this._log.warn("Unknown version in state file: " + this._s.v);
180 // We explicitly don't save here in the hopes an application re-upgrade
181 // comes along and fixes us.
184 this._s.clientID = drsClientID;
186 // Always look for preferences. This ensures that downgrades followed
187 // by reupgrades don't result in excessive data loss.
188 for (let promise of this._migratePrefs()) {
195 this._log.info("Writing state file: " + this._filename);
196 return CommonUtils.writeJSON(this._s, this._filename);
199 addRemoteID: function (id) {
200 this._log.warn("Recording new remote ID: " + id);
201 this._s.remoteIDs.push(id);
205 removeRemoteID: function (id) {
206 return this.removeRemoteIDs(id ? [id] : []);
209 removeRemoteIDs: function (ids) {
210 if (!ids || !ids.length) {
211 this._log.warn("No IDs passed for removal.");
212 return Promise.resolve();
215 this._log.warn("Removing documents from remote ID list: " + ids);
216 let filtered = this._s.remoteIDs.filter((x) => ids.indexOf(x) === -1);
218 if (filtered.length == this._s.remoteIDs.length) {
219 return Promise.resolve();
222 this._s.remoteIDs = filtered;
226 setLastPingDate: function (date) {
227 this._s.lastPingTime = date.getTime();
232 updateLastPingAndRemoveRemoteID: function (date, id) {
233 return this.updateLastPingAndRemoveRemoteIDs(date, id ? [id] : []);
236 updateLastPingAndRemoveRemoteIDs: function (date, ids) {
238 return this.setLastPingDate(date);
241 this._log.info("Recording last ping time and deleted remote document.");
242 this._s.lastPingTime = date.getTime();
243 return this.removeRemoteIDs(ids);
247 * Reset the client ID to something else.
248 * Returns a promise that is resolved when completed.
250 resetClientID: Task.async(function* () {
251 let drs = Cc["@mozilla.org/datareporting/service;1"]
252 .getService(Ci.nsISupports)
254 yield drs.resetClientID();
255 this._s.clientID = yield drs.getClientID();
256 this._log.info("Reset client id to " + this._s.clientID + ".");
261 _migratePrefs: function () {
262 let prefs = this._reporter._prefs;
264 let lastID = prefs.get("lastSubmitID", null);
265 let lastPingDate = CommonUtils.getDatePref(prefs, "lastPingTime",
266 0, this._log, OLDEST_ALLOWED_YEAR);
268 // If we have state from prefs, migrate and save it to a file then clear
270 if (lastID || (lastPingDate && lastPingDate.getTime() > 0)) {
271 this._log.warn("Migrating saved state from preferences.");
274 this._log.info("Migrating last saved ID: " + lastID);
275 this._s.remoteIDs.push(lastID);
278 let ourLast = this.lastPingDate;
280 if (lastPingDate && lastPingDate.getTime() > ourLast.getTime()) {
281 this._log.info("Migrating last ping time: " + lastPingDate);
282 this._s.lastPingTime = lastPingDate.getTime();
286 prefs.reset(["lastSubmitID", "lastPingTime"]);
288 this._log.warn("No prefs data found.");
294 * This is the abstract base class of `HealthReporter`. It exists so that
295 * we can sanely divide work on platforms where control of Firefox Health
296 * Report is outside of Gecko (e.g., Android).
298 function AbstractHealthReporter(branch, policy, sessionRecorder) {
299 if (!branch.endsWith(".")) {
300 throw new Error("Branch must end with a period (.): " + branch);
304 throw new Error("Must provide policy to HealthReporter constructor.");
307 this._log = Log.repository.getLogger("Services.HealthReport.HealthReporter");
308 this._log.info("Initializing health reporter instance against " + branch);
310 this._branch = branch;
311 this._prefs = new Preferences(branch);
313 this._policy = policy;
314 this.sessionRecorder = sessionRecorder;
316 this._dbName = this._prefs.get("dbName") || DEFAULT_DATABASE_NAME;
318 this._storage = null;
319 this._storageInProgress = false;
320 this._providerManager = null;
321 this._providerManagerInProgress = false;
322 this._initializeStarted = false;
323 this._initialized = false;
324 this._initializeHadError = false;
325 this._initializedDeferred = Promise.defer();
326 this._shutdownRequested = false;
327 this._shutdownInitiated = false;
328 this._shutdownComplete = false;
329 this._deferredShutdown = Promise.defer();
330 this._promiseShutdown = this._deferredShutdown.promise;
334 this._lastDailyDate = null;
336 // Yes, this will probably run concurrently with remaining constructor work.
337 let hasFirstRun = this._prefs.get("service.firstRun", false);
338 this._initHistogram = hasFirstRun ? TELEMETRY_INIT : TELEMETRY_INIT_FIRSTRUN;
339 this._dbOpenHistogram = hasFirstRun ? TELEMETRY_DB_OPEN : TELEMETRY_DB_OPEN_FIRSTRUN;
342 AbstractHealthReporter.prototype = Object.freeze({
343 QueryInterface: XPCOMUtils.generateQI([Ci.nsIObserver]),
346 * Whether the service is fully initialized and running.
348 * If this is false, it is not safe to call most functions.
351 return this._initialized;
355 * Initialize the instance.
357 * This must be called once after object construction or the instance is
361 if (this._initializeStarted) {
362 throw new Error("We have already started initialization.");
365 this._initializeStarted = true;
367 return Task.spawn(function*() {
368 TelemetryStopwatch.start(this._initHistogram, this);
371 yield this._state.init();
373 if (!this._state._s.removedOutdatedLastpayload) {
374 yield this._deleteOldLastPayload();
375 this._state._s.removedOutdatedLastpayload = true;
376 // Normally we should save this to a file but it directly conflicts with
377 // the "application re-upgrade" decision in HealthReporterState::init()
378 // which specifically does not save the state to a file.
381 this._log.error("Error deleting last payload: " +
382 CommonUtils.exceptionStr(ex));
385 // As soon as we have could have storage, we need to register cleanup or
386 // else bad things happen on shutdown.
387 Services.obs.addObserver(this, "quit-application", false);
389 // The database needs to be shut down by the end of shutdown
390 // phase profileBeforeChange.
391 Metrics.Storage.shutdown.addBlocker("FHR: Flushing storage shutdown",
393 // Workaround bug 1017706
394 // Apparently, in some cases, quit-application is not triggered
395 // (or is triggered after profile-before-change), so we need to
396 // make sure that `_initiateShutdown()` is triggered at least
398 this._initiateShutdown();
399 return this._promiseShutdown;
402 shutdownInitiated: this._shutdownInitiated,
403 initialized: this._initialized,
404 shutdownRequested: this._shutdownRequested,
405 initializeHadError: this._initializeHadError,
406 providerManagerInProgress: this._providerManagerInProgress,
407 storageInProgress: this._storageInProgress,
408 hasProviderManager: !!this._providerManager,
409 hasStorage: !!this._storage,
410 shutdownComplete: this._shutdownComplete
414 this._storageInProgress = true;
415 TelemetryStopwatch.start(this._dbOpenHistogram, this);
416 let storage = yield Metrics.Storage(this._dbName);
417 TelemetryStopwatch.finish(this._dbOpenHistogram, this);
418 yield this._onStorageCreated();
420 delete this._dbOpenHistogram;
421 this._log.info("Storage initialized.");
422 this._storage = storage;
423 this._storageInProgress = false;
425 if (this._shutdownRequested) {
426 this._initiateShutdown();
430 yield this._initializeProviderManager();
431 yield this._onProviderManagerInitialized();
432 this._initializedDeferred.resolve();
433 return this.onInit();
435 yield this._onInitError(ex);
436 this._initializedDeferred.reject(ex);
441 //----------------------------------------------------
442 // SERVICE CONTROL FUNCTIONS
444 // You shouldn't need to call any of these externally.
445 //----------------------------------------------------
447 _onInitError: function (error) {
448 TelemetryStopwatch.cancel(this._initHistogram, this);
449 TelemetryStopwatch.cancel(this._dbOpenHistogram, this);
450 delete this._initHistogram;
451 delete this._dbOpenHistogram;
453 this._recordError("Error during initialization", error);
454 this._initializeHadError = true;
455 this._initiateShutdown();
456 return Promise.reject(error);
458 // FUTURE consider poisoning prototype's functions so calls fail with a
459 // useful error message.
464 * Removes the outdated lastpaylaod.json and lastpayload.json.tmp files
466 * @return a promise for when all the files have been deleted
468 _deleteOldLastPayload: function () {
469 let paths = [this._state._lastPayloadPath, this._state._lastPayloadPath + ".tmp"];
470 return Task.spawn(function removeAllFiles () {
471 for (let path of paths) {
473 OS.File.remove(path);
475 if (!ex.becauseNoSuchFile) {
476 this._log.error("Exception when removing outdated payload files: " +
477 CommonUtils.exceptionStr(ex));
484 _initializeProviderManager: Task.async(function* _initializeProviderManager() {
485 if (this._collector) {
486 throw new Error("Provider manager has already been initialized.");
489 this._log.info("Initializing provider manager.");
490 this._providerManager = new Metrics.ProviderManager(this._storage);
491 this._providerManager.onProviderError = this._recordError.bind(this);
492 this._providerManager.onProviderInit = this._initProvider.bind(this);
493 this._providerManagerInProgress = true;
495 let catString = this._prefs.get("service.providerCategories") || "";
496 if (catString.length) {
497 for (let category of catString.split(",")) {
498 yield this._providerManager.registerProvidersFromCategoryManager(category);
503 _onProviderManagerInitialized: function () {
504 TelemetryStopwatch.finish(this._initHistogram, this);
505 delete this._initHistogram;
506 this._log.debug("Provider manager initialized.");
507 this._providerManagerInProgress = false;
509 if (this._shutdownRequested) {
510 this._initiateShutdown();
514 this._log.info("HealthReporter started.");
515 this._initialized = true;
516 Services.obs.addObserver(this, "idle-daily", false);
518 // If upload is not enabled, ensure daily collection works. If upload
519 // is enabled, this will be performed as part of upload.
521 // This is important because it ensures about:healthreport contains
522 // longitudinal data even if upload is disabled. Having about:healthreport
523 // provide useful info even if upload is disabled was a core launch
526 // We do not catch changes to the backing pref. So, if the session lasts
527 // many days, we may fail to collect. However, most sessions are short and
528 // this code will likely be refactored as part of splitting up policy to
529 // serve Android. So, meh.
530 if (!this._policy.healthReportUploadEnabled) {
531 this._log.info("Upload not enabled. Scheduling daily collection.");
532 // Since the timer manager is a singleton and there could be multiple
533 // HealthReporter instances, we need to encode a unique identifier in
536 let timerName = this._branch.replace(".", "-", "g") + "lastDailyCollection";
537 let tm = Cc["@mozilla.org/updates/timer-manager;1"]
538 .getService(Ci.nsIUpdateTimerManager);
539 tm.registerTimer(timerName, this.collectMeasurements.bind(this),
542 this._log.error("Error registering collection timer: " +
543 CommonUtils.exceptionStr(ex));
547 // Clean up caches and reduce memory usage.
548 this._storage.compact();
551 // nsIObserver to handle shutdown.
552 observe: function (subject, topic, data) {
554 case "quit-application":
555 Services.obs.removeObserver(this, "quit-application");
556 this._initiateShutdown();
560 this._performDailyMaintenance();
565 _initiateShutdown: function () {
566 // Ensure we only begin the main shutdown sequence once.
567 if (this._shutdownInitiated) {
568 this._log.warn("Shutdown has already been initiated. No-op.");
572 this._log.info("Request to shut down.");
574 this._initialized = false;
575 this._shutdownRequested = true;
577 if (this._initializeHadError) {
578 this._log.warn("Initialization had error. Shutting down immediately.");
580 if (this._providerManagerInProgress) {
581 this._log.warn("Provider manager is in progress of initializing. " +
582 "Waiting to finish.");
586 // If storage is in the process of initializing, we need to wait for it
587 // to finish before continuing. The initialization process will call us
588 // again once storage has initialized.
589 if (this._storageInProgress) {
590 this._log.warn("Storage is in progress of initializing. Waiting to finish.");
595 this._log.warn("Initiating main shutdown procedure.");
597 // Everything from here must only be performed once or else race conditions
600 TelemetryStopwatch.start(TELEMETRY_SHUTDOWN, this);
601 this._shutdownInitiated = true;
603 // We may not have registered the observer yet. If not, this will
606 Services.obs.removeObserver(this, "idle-daily");
609 Task.spawn(function*() {
611 if (this._providerManager) {
612 this._log.info("Shutting down provider manager.");
613 for (let provider of this._providerManager.providers) {
615 yield provider.shutdown();
617 this._log.warn("Error when shutting down provider: " +
618 CommonUtils.exceptionStr(ex));
621 this._log.info("Provider manager shut down.");
622 this._providerManager = null;
623 this._onProviderManagerShutdown();
626 this._log.info("Shutting down storage.");
628 yield this._storage.close();
629 yield this._onStorageClose();
631 this._log.warn("Error when closing storage: " +
632 CommonUtils.exceptionStr(error));
634 this._storage = null;
637 this._log.warn("Shutdown complete.");
638 this._shutdownComplete = true;
640 this._deferredShutdown.resolve();
641 TelemetryStopwatch.finish(TELEMETRY_SHUTDOWN, this);
647 return this._initializedDeferred.promise;
650 _onStorageCreated: function() {
652 // This method provides a hook point for the test suite.
655 _onStorageClose: function() {
657 // This method provides a hook point for the test suite.
660 _onProviderManagerShutdown: function() {
662 // This method provides a hook point for the test suite.
666 * Convenience method to shut down the instance.
668 * This should *not* be called outside of tests.
670 _shutdown: function () {
671 this._initiateShutdown();
672 return this._promiseShutdown;
675 _performDailyMaintenance: function () {
676 this._log.info("Request to perform daily maintenance.");
678 if (!this._initialized) {
682 let now = new Date();
683 let cutoff = new Date(now.getTime() - MILLISECONDS_PER_DAY * (DAYS_IN_PAYLOAD - 1));
685 // The operation is enqueued and put in a transaction by the storage module.
686 this._storage.pruneDataBefore(cutoff);
689 //--------------------
690 // Provider Management
691 //--------------------
694 * Obtain a provider from its name.
696 * This will only return providers that are currently initialized. If
697 * a provider is lazy initialized (like pull-only providers) this
698 * will likely not return anything.
700 getProvider: function (name) {
701 if (!this._providerManager) {
705 return this._providerManager.getProvider(name);
708 _initProvider: function (provider) {
709 provider.healthReporter = this;
713 * Record an exception for reporting in the payload.
715 * A side effect is the exception is logged.
717 * Note that callers need to be extra sensitive about ensuring personal
718 * or otherwise private details do not leak into this. All of the user data
719 * on the stack in FHR code should be limited to data we were collecting with
720 * the intent to submit. So, it is covered under the user's consent to use
724 * (string) Human readable message describing error.
726 * (Error) The error that should be captured.
728 _recordError: function (message, ex) {
729 let recordMessage = message;
730 let logMessage = message;
733 recordMessage += ": " + CommonUtils.exceptionStr(ex);
734 logMessage += ": " + CommonUtils.exceptionStr(ex);
737 // Scrub out potentially identifying information from strings that could
739 let appData = Services.dirsvc.get("UAppData", Ci.nsIFile);
740 let profile = Services.dirsvc.get("ProfD", Ci.nsIFile);
742 let appDataURI = Services.io.newFileURI(appData);
743 let profileURI = Services.io.newFileURI(profile);
745 // Order of operation is important here. We do the URI before the path version
746 // because the path may be a subset of the URI. We also have to check for the case
747 // where UAppData is underneath the profile directory (or vice-versa) so we
748 // don't substitute incomplete strings.
750 function replace(uri, path, thing) {
751 // Try is because .spec can throw on invalid URI.
753 recordMessage = recordMessage.replace(uri.spec, '<' + thing + 'URI>', 'g');
756 recordMessage = recordMessage.replace(path, '<' + thing + 'Path>', 'g');
759 if (appData.path.contains(profile.path)) {
760 replace(appDataURI, appData.path, 'AppData');
761 replace(profileURI, profile.path, 'Profile');
763 replace(profileURI, profile.path, 'Profile');
764 replace(appDataURI, appData.path, 'AppData');
767 this._log.warn(logMessage);
768 this._errors.push(recordMessage);
772 * Collect all measurements for all registered providers.
774 collectMeasurements: function () {
775 if (!this._initialized) {
776 return Promise.reject(new Error("Not initialized."));
779 return Task.spawn(function doCollection() {
780 yield this._providerManager.ensurePullOnlyProvidersRegistered();
783 TelemetryStopwatch.start(TELEMETRY_COLLECT_CONSTANT, this);
784 yield this._providerManager.collectConstantData();
785 TelemetryStopwatch.finish(TELEMETRY_COLLECT_CONSTANT, this);
787 TelemetryStopwatch.cancel(TELEMETRY_COLLECT_CONSTANT, this);
788 this._log.warn("Error collecting constant data: " +
789 CommonUtils.exceptionStr(ex));
792 // Daily data is collected if it hasn't yet been collected this
793 // application session or if it has been more than a day since the
794 // last collection. This means that providers could see many calls to
795 // collectDailyData per calendar day. However, this collection API
796 // makes no guarantees about limits. The alternative would involve
797 // recording state. The simpler implementation prevails for now.
798 if (!this._lastDailyDate ||
799 Date.now() - this._lastDailyDate > MILLISECONDS_PER_DAY) {
802 TelemetryStopwatch.start(TELEMETRY_COLLECT_DAILY, this);
803 this._lastDailyDate = new Date();
804 yield this._providerManager.collectDailyData();
805 TelemetryStopwatch.finish(TELEMETRY_COLLECT_DAILY, this);
807 TelemetryStopwatch.cancel(TELEMETRY_COLLECT_DAILY, this);
808 this._log.warn("Error collecting daily data from providers: " +
809 CommonUtils.exceptionStr(ex));
813 yield this._providerManager.ensurePullOnlyProvidersUnregistered();
815 // Flush gathered data to disk. This will incur an fsync. But, if
816 // there is ever a time we want to persist data to disk, it's
817 // after a massive collection.
819 TelemetryStopwatch.start(TELEMETRY_COLLECT_CHECKPOINT, this);
820 yield this._storage.checkpoint();
821 TelemetryStopwatch.finish(TELEMETRY_COLLECT_CHECKPOINT, this);
823 TelemetryStopwatch.cancel(TELEMETRY_COLLECT_CHECKPOINT, this);
827 throw new Task.Result();
832 * Helper function to perform data collection and obtain the JSON payload.
834 * If you are looking for an up-to-date snapshot of FHR data that pulls in
835 * new data since the last upload, this is how you should obtain it.
838 * (bool) Whether to resolve an object or JSON-encoded string of that
839 * object (the default).
841 * @return Promise<Object | string>
843 collectAndObtainJSONPayload: function (asObject=false) {
844 if (!this._initialized) {
845 return Promise.reject(new Error("Not initialized."));
848 return Task.spawn(function collectAndObtain() {
849 yield this._storage.setAutoCheckpoint(0);
850 yield this._providerManager.ensurePullOnlyProvidersRegistered();
856 yield this.collectMeasurements();
857 payload = yield this.getJSONPayload(asObject);
860 this._collectException("Error collecting and/or retrieving JSON payload",
863 yield this._providerManager.ensurePullOnlyProvidersUnregistered();
864 yield this._storage.setAutoCheckpoint(1);
871 // We hold off throwing to ensure that behavior between finally
872 // and generators and throwing is sane.
873 throw new Task.Result(payload);
879 * Obtain the JSON payload for currently-collected data.
881 * The payload only contains data that has been recorded to FHR. Some
882 * providers may have newer data available. If you want to ensure you
883 * have all available data, call `collectAndObtainJSONPayload`
887 * (bool) Whether to return an object or JSON encoding of that
888 * object (the default).
890 * @return Promise<string|object>
892 getJSONPayload: function (asObject=false) {
893 TelemetryStopwatch.start(TELEMETRY_GENERATE_PAYLOAD, this);
894 let deferred = Promise.defer();
896 Task.spawn(this._getJSONPayload.bind(this, this._now(), asObject)).then(
897 function onResult(result) {
898 TelemetryStopwatch.finish(TELEMETRY_GENERATE_PAYLOAD, this);
899 deferred.resolve(result);
901 function onError(error) {
902 TelemetryStopwatch.cancel(TELEMETRY_GENERATE_PAYLOAD, this);
903 deferred.reject(error);
907 return deferred.promise;
910 _getJSONPayload: function (now, asObject=false) {
911 let pingDateString = this._formatDate(now);
912 this._log.info("Producing JSON payload for " + pingDateString);
914 // May not be present if we are generating as a result of init error.
915 if (this._providerManager) {
916 yield this._providerManager.ensurePullOnlyProvidersRegistered();
921 clientID: this._state.clientID,
922 clientIDVersion: this._state.clientIDVersion,
923 thisPingDate: pingDateString,
924 geckoAppInfo: this.obtainAppInfo(this._log),
925 data: {last: {}, days: {}},
928 let outputDataDays = o.data.days;
930 // Guard here in case we don't track this (e.g., on Android).
931 let lastPingDate = this.lastPingDate;
932 if (lastPingDate && lastPingDate.getTime() > 0) {
933 o.lastPingDate = this._formatDate(lastPingDate);
936 // We can still generate a payload even if we're not initialized.
937 // This is to facilitate error upload on init failure.
938 if (this._initialized) {
939 for (let provider of this._providerManager.providers) {
940 let providerName = provider.name;
942 let providerEntry = {
946 // Measurement name to recorded version.
947 let lastVersions = {};
948 // Day string to mapping of measurement name to recorded version.
949 let dayVersions = {};
951 for (let [measurementKey, measurement] of provider.measurements) {
952 let name = providerName + "." + measurement.name;
953 let version = measurement.version;
957 // The measurement is responsible for returning a serializer which
958 // is aware of the measurement version.
959 serializer = measurement.serializer(measurement.SERIALIZE_JSON);
961 this._recordError("Error obtaining serializer for measurement: " +
968 data = yield measurement.getValues();
970 this._recordError("Error obtaining data for measurement: " + name,
975 if (data.singular.size) {
977 let serialized = serializer.singular(data.singular);
979 // Only replace the existing data if there is no data or if our
980 // version is newer than the old one.
981 if (!(name in o.data.last) || version > lastVersions[name]) {
982 o.data.last[name] = serialized;
983 lastVersions[name] = version;
987 this._recordError("Error serializing singular data: " + name,
993 let dataDays = data.days;
994 for (let i = 0; i < DAYS_IN_PAYLOAD; i++) {
995 let date = new Date(now.getTime() - i * MILLISECONDS_PER_DAY);
996 if (!dataDays.hasDay(date)) {
999 let dateFormatted = this._formatDate(date);
1002 let serialized = serializer.daily(dataDays.getDay(date));
1007 if (!(dateFormatted in outputDataDays)) {
1008 outputDataDays[dateFormatted] = {};
1011 // This needs to be separate because dayVersions is provider
1012 // specific and gets blown away in a loop while outputDataDays
1014 if (!(dateFormatted in dayVersions)) {
1015 dayVersions[dateFormatted] = {};
1018 if (!(name in outputDataDays[dateFormatted]) ||
1019 version > dayVersions[dateFormatted][name]) {
1020 outputDataDays[dateFormatted][name] = serialized;
1021 dayVersions[dateFormatted][name] = version;
1024 this._recordError("Error populating data for day: " + name, ex);
1031 o.notInitialized = 1;
1032 this._log.warn("Not initialized. Sending report with only error info.");
1035 if (this._errors.length) {
1036 o.errors = this._errors.slice(0, 20);
1039 if (this._initialized) {
1040 this._storage.compact();
1044 TelemetryStopwatch.start(TELEMETRY_JSON_PAYLOAD_SERIALIZE, this);
1045 o = JSON.stringify(o);
1046 TelemetryStopwatch.finish(TELEMETRY_JSON_PAYLOAD_SERIALIZE, this);
1049 if (this._providerManager) {
1050 yield this._providerManager.ensurePullOnlyProvidersUnregistered();
1053 throw new Task.Result(o);
1056 _now: function _now() {
1060 // These are stolen from AppInfoProvider.
1063 // From nsIXULAppInfo.
1068 appBuildID: "appBuildID",
1069 platformVersion: "platformVersion",
1070 platformBuildID: "platformBuildID",
1072 // From nsIXULRuntime.
1074 xpcomabi: "XPCOMABI",
1078 * Statically return a bundle of app info data, a subset of that produced by
1079 * AppInfoProvider._populateConstants. This allows us to more usefully handle
1080 * payloads that, due to error, contain no data.
1082 * Returns a very sparse object if Services.appinfo is unavailable.
1084 obtainAppInfo: function () {
1085 let out = {"_v": this.appInfoVersion};
1087 let ai = Services.appinfo;
1088 for (let [k, v] in Iterator(this.appInfoFields)) {
1092 this._log.warn("Could not obtain Services.appinfo: " +
1093 CommonUtils.exceptionStr(ex));
1097 out["updateChannel"] = UpdateChannel.get();
1099 this._log.warn("Could not obtain update channel: " +
1100 CommonUtils.exceptionStr(ex));
1108 * HealthReporter and its abstract superclass coordinate collection and
1109 * submission of health report metrics.
1111 * This is the main type for Firefox Health Report on desktop. It glues all the
1112 * lower-level components (such as collection and submission) together.
1114 * An instance of this type is created as an XPCOM service. See
1115 * DataReportingService.js and
1116 * DataReporting.manifest/HealthReportComponents.manifest.
1118 * It is theoretically possible to have multiple instances of this running
1119 * in the application. For example, this type may one day handle submission
1120 * of telemetry data as well. However, there is some moderate coupling between
1121 * this type and *the* Firefox Health Report (e.g., the policy). This could
1122 * be abstracted if needed.
1124 * Note that `AbstractHealthReporter` exists to allow for Firefox Health Report
1125 * to be more easily implemented on platforms where a separate controlling
1126 * layer is responsible for payload upload and deletion.
1128 * IMPLEMENTATION NOTES
1129 * ====================
1131 * These notes apply to the combination of `HealthReporter` and
1132 * `AbstractHealthReporter`.
1134 * Initialization and shutdown are somewhat complicated and worth explaining
1137 * The complexity is driven by the requirements of SQLite connection management.
1138 * Once you have a SQLite connection, it isn't enough to just let the
1139 * application shut down. If there is an open connection or if there are
1140 * outstanding SQL statements come XPCOM shutdown time, Storage will assert.
1141 * On debug builds you will crash. On release builds you will get a shutdown
1142 * hang. This must be avoided!
1144 * During initialization, the second we create a SQLite connection (via
1145 * Metrics.Storage) we register observers for application shutdown. The
1146 * "quit-application" notification initiates our shutdown procedure. The
1147 * subsequent "profile-do-change" notification ensures it has completed.
1149 * The handler for "profile-do-change" may result in event loop spinning. This
1150 * is because of race conditions between our shutdown code and application
1153 * All of our shutdown routines are async. There is the potential that these
1154 * async functions will not complete before XPCOM shutdown. If they don't
1155 * finish in time, we could get assertions in Storage. Our solution is to
1156 * initiate storage early in the shutdown cycle ("quit-application").
1157 * Hopefully all the async operations have completed by the time we reach
1158 * "profile-do-change." If so, great. If not, we spin the event loop until
1159 * they have completed, avoiding potential race conditions.
1162 * (string) The preferences branch to use for state storage. The value
1163 * must end with a period (.).
1166 * (HealthReportPolicy) Policy driving execution of HealthReporter.
1168 this.HealthReporter = function (branch, policy, sessionRecorder, stateLeaf=null) {
1169 this._stateLeaf = stateLeaf;
1170 this._uploadInProgress = false;
1172 AbstractHealthReporter.call(this, branch, policy, sessionRecorder);
1174 if (!this.serverURI) {
1175 throw new Error("No server URI defined. Did you forget to define the pref?");
1178 if (!this.serverNamespace) {
1179 throw new Error("No server namespace defined. Did you forget a pref?");
1182 this._state = new HealthReporterState(this);
1185 this.HealthReporter.prototype = Object.freeze({
1186 __proto__: AbstractHealthReporter.prototype,
1188 QueryInterface: XPCOMUtils.generateQI([Ci.nsIObserver]),
1190 get lastSubmitID() {
1191 return this._state.lastSubmitID;
1195 * When we last successfully submitted data to the server.
1197 * This is sent as part of the upload. This is redundant with similar data
1198 * in the policy because we like the modules to be loosely coupled and the
1199 * similar data in the policy is only used for forensic purposes.
1201 get lastPingDate() {
1202 return this._state.lastPingDate;
1206 * The base URI of the document server to which to submit data.
1208 * This is typically a Bagheera server instance. It is the URI up to but not
1209 * including the version prefix. e.g. https://data.metrics.mozilla.com/
1212 return this._prefs.get("documentServerURI", null);
1215 set serverURI(value) {
1217 throw new Error("serverURI must have a value.");
1220 if (typeof(value) != "string") {
1221 throw new Error("serverURI must be a string: " + value);
1224 this._prefs.set("documentServerURI", value);
1228 * The namespace on the document server to which we will be submitting data.
1230 get serverNamespace() {
1231 return this._prefs.get("documentServerNamespace", "metrics");
1234 set serverNamespace(value) {
1236 throw new Error("serverNamespace must have a value.");
1239 if (typeof(value) != "string") {
1240 throw new Error("serverNamespace must be a string: " + value);
1243 this._prefs.set("documentServerNamespace", value);
1247 * Whether this instance will upload data to a server.
1249 get willUploadData() {
1250 return this._policy.userNotifiedOfCurrentPolicy &&
1251 this._policy.healthReportUploadEnabled;
1255 * Whether remote data is currently stored.
1259 haveRemoteData: function () {
1260 return !!this._state.lastSubmitID;
1264 * Called to initiate a data upload.
1266 * The passed argument is a `DataSubmissionRequest` from policy.jsm.
1268 requestDataUpload: function (request) {
1269 if (!this._initialized) {
1270 return Promise.reject(new Error("Not initialized."));
1273 return Task.spawn(function doUpload() {
1274 yield this._providerManager.ensurePullOnlyProvidersRegistered();
1276 yield this.collectMeasurements();
1278 yield this._uploadData(request);
1280 this._onSubmitDataRequestFailure(ex);
1283 yield this._providerManager.ensurePullOnlyProvidersUnregistered();
1289 * Request that server data be deleted.
1291 * If deletion is scheduled to occur immediately, a promise will be returned
1292 * that will be fulfilled when the deletion attempt finishes. Otherwise,
1293 * callers should poll haveRemoteData() to determine when remote data is
1296 requestDeleteRemoteData: function (reason) {
1297 if (!this.haveRemoteData()) {
1301 return this._policy.deleteRemoteData(reason);
1305 * Override default handler to incur an upload describing the error.
1307 _onInitError: function (error) {
1308 // Need to capture this before we call the parent else it's always
1310 let inShutdown = this._shutdownRequested;
1314 result = AbstractHealthReporter.prototype._onInitError.call(this, error);
1316 this._log.error("Error when calling _onInitError: " +
1317 CommonUtils.exceptionStr(ex));
1320 // This bypasses a lot of the checks in policy, such as respect for
1321 // backoff. We should arguably not do this. However, reporting
1322 // startup errors is important. And, they should not occur with much
1323 // frequency in the wild. So, it shouldn't be too big of a deal.
1325 this._policy.healthReportUploadEnabled &&
1326 this._policy.ensureUserNotified()) {
1327 // We don't care about what happens to this request. It's best
1330 onNoDataAvailable: function () {},
1331 onSubmissionSuccess: function () {},
1332 onSubmissionFailureSoft: function () {},
1333 onSubmissionFailureHard: function () {},
1334 onUploadInProgress: function () {},
1337 this._uploadData(request);
1343 _onBagheeraResult: function (request, isDelete, date, result) {
1344 this._log.debug("Received Bagheera result.");
1346 return Task.spawn(function onBagheeraResult() {
1347 let hrProvider = this.getProvider("org.mozilla.healthreport");
1349 if (!result.transportSuccess) {
1350 // The built-in provider may not be initialized if this instance failed
1351 // to initialize fully.
1352 if (hrProvider && !isDelete) {
1354 hrProvider.recordEvent("uploadTransportFailure", date);
1356 this._log.error("Error recording upload transport failure: " +
1357 CommonUtils.exceptionStr(ex));
1361 request.onSubmissionFailureSoft("Network transport error.");
1362 throw new Task.Result(false);
1365 if (!result.serverSuccess) {
1366 if (hrProvider && !isDelete) {
1368 hrProvider.recordEvent("uploadServerFailure", date);
1370 this._log.error("Error recording server failure: " +
1371 CommonUtils.exceptionStr(ex));
1375 request.onSubmissionFailureHard("Server failure.");
1376 throw new Task.Result(false);
1379 if (hrProvider && !isDelete) {
1381 hrProvider.recordEvent("uploadSuccess", date);
1383 this._log.error("Error recording upload success: " +
1384 CommonUtils.exceptionStr(ex));
1389 this._log.warn("Marking delete as successful.");
1390 yield this._state.removeRemoteIDs([result.id]);
1392 this._log.warn("Marking upload as successful.");
1393 yield this._state.updateLastPingAndRemoveRemoteIDs(date, result.deleteIDs);
1396 request.onSubmissionSuccess(this._now());
1398 throw new Task.Result(true);
1402 _onSubmitDataRequestFailure: function (error) {
1403 this._log.error("Error processing request to submit data: " +
1404 CommonUtils.exceptionStr(error));
1407 _formatDate: function (date) {
1408 // Why, oh, why doesn't JS have a strftime() equivalent?
1409 return date.toISOString().substr(0, 10);
1412 _uploadData: function (request) {
1413 // Under ideal circumstances, clients should never race to this
1414 // function. However, server logs have observed behavior where
1415 // racing to this function could be a cause. So, this lock was
1417 if (this._uploadInProgress) {
1418 this._log.warn("Upload requested but upload already in progress.");
1419 let provider = this.getProvider("org.mozilla.healthreport");
1420 let promise = provider.recordEvent("uploadAlreadyInProgress");
1421 request.onUploadInProgress("Upload already in progress.");
1425 let id = CommonUtils.generateUUID();
1427 this._log.info("Uploading data to server: " + this.serverURI + " " +
1428 this.serverNamespace + ":" + id);
1429 let client = new BagheeraClient(this.serverURI);
1430 let now = this._now();
1432 return Task.spawn(function doUpload() {
1434 // The test for upload locking monkeypatches getJSONPayload.
1435 // If the next two lines change, be sure to verify the test is
1437 this._uploadInProgress = true;
1438 let payload = yield this.getJSONPayload();
1440 let histogram = Services.telemetry.getHistogramById(TELEMETRY_PAYLOAD_SIZE_UNCOMPRESSED);
1441 histogram.add(payload.length);
1443 let lastID = this.lastSubmitID;
1444 yield this._state.addRemoteID(id);
1446 let hrProvider = this.getProvider("org.mozilla.healthreport");
1448 let event = lastID ? "continuationUploadAttempt"
1449 : "firstDocumentUploadAttempt";
1451 hrProvider.recordEvent(event, now);
1453 this._log.error("Error when recording upload attempt: " +
1454 CommonUtils.exceptionStr(ex));
1458 TelemetryStopwatch.start(TELEMETRY_UPLOAD, this);
1462 deleteIDs: this._state.remoteIDs.filter((x) => { return x != id; }),
1463 telemetryCompressed: TELEMETRY_PAYLOAD_SIZE_COMPRESSED,
1465 result = yield client.uploadJSON(this.serverNamespace, id, payload,
1467 TelemetryStopwatch.finish(TELEMETRY_UPLOAD, this);
1469 TelemetryStopwatch.cancel(TELEMETRY_UPLOAD, this);
1472 hrProvider.recordEvent("uploadClientFailure", now);
1474 this._log.error("Error when recording client failure: " +
1475 CommonUtils.exceptionStr(ex));
1481 yield this._onBagheeraResult(request, false, now, result);
1483 this._uploadInProgress = false;
1489 * Request deletion of remote data.
1492 * (DataSubmissionRequest) Tracks progress of this request.
1494 deleteRemoteData: function (request) {
1495 if (!this._state.lastSubmitID) {
1496 this._log.info("Received request to delete remote data but no data stored.");
1497 request.onNoDataAvailable();
1501 this._log.warn("Deleting remote data.");
1502 let client = new BagheeraClient(this.serverURI);
1504 return Task.spawn(function* doDelete() {
1506 let result = yield client.deleteDocument(this.serverNamespace,
1508 yield this._onBagheeraResult(request, true, this._now(), result);
1510 this._log.error("Error processing request to delete data: " +
1511 CommonUtils.exceptionStr(error));
1513 // If we don't have any remote documents left, nuke the ID.
1514 // This is done for privacy reasons. Why preserve the ID if we
1516 if (!this.haveRemoteData()) {
1517 yield this._state.resetClientID();