dom/media/DriftCompensation.h

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
   6
   7 #ifndef DriftCompensation_h_
   8 #define DriftCompensation_h_
   9
  10 #include "MediaSegment.h"
  11 #include "VideoUtils.h"
  12 #include "mozilla/Atomics.h"
  13 #include "mozilla/Unused.h"
  14
  15 namespace mozilla {
  16
  17 static LazyLogModule gDriftCompensatorLog("DriftCompensator");
  18 #define LOG(type, ...) MOZ_LOG(gDriftCompensatorLog, type, (__VA_ARGS__))
  19
  20 /**
  21  * DriftCompensator can be used to handle drift between audio and video tracks
  22  * from the MediaTrackGraph.
  23  *
  24  * Drift can occur because audio is driven by a MediaTrackGraph running off an
  25  * audio callback, thus it's progressed by the clock of one the audio output
  26  * devices on the user's machine. Video on the other hand is always expressed in
  27  * wall-clock TimeStamps, i.e., it's progressed by the system clock. These
  28  * clocks will, over time, drift apart.
  29  *
  30  * Do not use the DriftCompensator across multiple audio tracks, as it will
  31  * automatically record the start time of the first audio samples, and all
  32  * samples for the same audio track on the same audio clock will have to be
  33  * processed to retain accuracy.
  34  *
  35  * DriftCompensator is designed to be used from two threads:
  36  * - The audio thread for notifications of audio samples.
  37  * - The video thread for compensating drift of video frames to match the audio
  38  *   clock.
  39  */
  40 class DriftCompensator {
  41   const RefPtr<nsIEventTarget> mVideoThread;
  42   const TrackRate mAudioRate;
  43
  44   // Number of audio samples produced. Any thread.
  45   Atomic<TrackTime> mAudioSamples{0};
  46
  47   // Time the first audio samples were added. mVideoThread only.
  48   TimeStamp mAudioStartTime;
  49
  50   void SetAudioStartTime(TimeStamp aTime) {
  51     MOZ_ASSERT(mVideoThread->IsOnCurrentThread());
  52     MOZ_ASSERT(mAudioStartTime.IsNull());
  53     mAudioStartTime = aTime;
  54   }
  55
  56  protected:
  57   virtual ~DriftCompensator() = default;
  58
  59  public:
  60   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(DriftCompensator)
  61
  62   DriftCompensator(RefPtr<nsIEventTarget> aVideoThread, TrackRate aAudioRate)
  63       : mVideoThread(std::move(aVideoThread)), mAudioRate(aAudioRate) {
  64     MOZ_ASSERT(mAudioRate > 0);
  65   }
  66
  67   void NotifyAudioStart(TimeStamp aStart) {
  68     MOZ_ASSERT(mAudioSamples == 0);
  69     LOG(LogLevel::Info, "DriftCompensator %p at rate %d started", this,
  70         mAudioRate);
  71     nsresult rv = mVideoThread->Dispatch(NewRunnableMethod<TimeStamp>(
  72         "DriftCompensator::SetAudioStartTime", this,
  73         &DriftCompensator::SetAudioStartTime, aStart));
  74     MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
  75     Unused << rv;
  76   }
  77
  78   /**
  79    * aSamples is the number of samples fed by an AudioStream.
  80    */
  81   void NotifyAudio(TrackTime aSamples) {
  82     MOZ_ASSERT(aSamples > 0);
  83     mAudioSamples += aSamples;
  84
  85     LOG(LogLevel::Verbose,
  86         "DriftCompensator %p Processed another %" PRId64
  87         " samples; now %.3fs audio",
  88         this, aSamples, static_cast<double>(mAudioSamples) / mAudioRate);
  89   }
  90
  91   /**
  92    * Drift compensates a video TimeStamp based on historical audio data.
  93    */
  94   virtual TimeStamp GetVideoTime(TimeStamp aNow, TimeStamp aTime) {
  95     MOZ_ASSERT(mVideoThread->IsOnCurrentThread());
  96     TrackTime samples = mAudioSamples;
  97
  98     if (samples / mAudioRate < 10) {
  99       // We don't apply compensation for the first 10 seconds because of the
 100       // higher inaccuracy during this time.
 101       LOG(LogLevel::Debug, "DriftCompensator %p %" PRId64 "ms so far; ignoring",
 102           this, samples * 1000 / mAudioRate);
 103       return aTime;
 104     }
 105
 106     if (aNow == mAudioStartTime) {
 107       LOG(LogLevel::Warning,
 108           "DriftCompensator %p video scale 0, assuming no drift", this);
 109       return aTime;
 110     }
 111
 112     double videoScaleUs = (aNow - mAudioStartTime).ToMicroseconds();
 113     double audioScaleUs = FramesToUsecs(samples, mAudioRate).value();
 114     double videoDurationUs = (aTime - mAudioStartTime).ToMicroseconds();
 115
 116     TimeStamp reclocked =
 117         mAudioStartTime + TimeDuration::FromMicroseconds(
 118                               videoDurationUs * audioScaleUs / videoScaleUs);
 119
 120     LOG(LogLevel::Debug,
 121         "DriftCompensator %p GetVideoTime, v-now: %.3fs, a-now: %.3fs; %.3fs "
 122         "-> %.3fs (d %.3fms)",
 123         this, (aNow - mAudioStartTime).ToSeconds(),
 124         TimeDuration::FromMicroseconds(audioScaleUs).ToSeconds(),
 125         (aTime - mAudioStartTime).ToSeconds(),
 126         (reclocked - mAudioStartTime).ToSeconds(),
 127         (reclocked - aTime).ToMilliseconds());
 128
 129     return reclocked;
 130   }
 131 };
 132
 133 #undef LOG
 134
 135 }  // namespace mozilla
 136
 137 #endif /* DriftCompensation_h_ */