dom/media/AudioDriftCorrection.h

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
   2 /* This Source Code Form is subject to the terms of the Mozilla Public
   3  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
   4  * You can obtain one at http://mozilla.org/MPL/2.0/. */
   5
   6 #ifndef MOZILLA_AUDIO_DRIFT_CORRECTION_H_
   7 #define MOZILLA_AUDIO_DRIFT_CORRECTION_H_
   8
   9 #include "DynamicResampler.h"
  10
  11 namespace mozilla {
  12
  13 extern LazyLogModule gMediaTrackGraphLog;
  14
  15 /**
  16  * ClockDrift calculates the diverge of the source clock from the nominal
  17  * (provided) rate compared to the target clock, which is considered the master
  18  * clock. In the case of different sampling rates, it is assumed that resampling
  19  * will take place so the returned correction is estimated after the resampling.
  20  * That means that resampling is taken into account in the calculations but it
  21  * does appear in the correction. The correction must be applied to the top of
  22  * the resampling.
  23  *
  24  * It works by measuring the incoming, the outgoing frames, and the amount of
  25  * buffered data and estimates the correction needed. The correction logic has
  26  * been created with two things in mind. First, not to run out of frames because
  27  * that means the audio will glitch. Second, not to change the correction very
  28  * often because this will result in a change in the resampling ratio. The
  29  * resampler recreates its internal memory when the ratio changes which has a
  30  * performance impact.
  31  *
  32  * The pref `media.clock drift.buffering` can be used to configure the desired
  33  * internal buffering. Right now it is at 50ms. But it can be increased if there
  34  * are audio quality problems.
  35  */
  36 class ClockDrift final {
  37  public:
  38   /**
  39    * Provide the nominal source and the target sample rate.
  40    */
  41   ClockDrift(uint32_t aSourceRate, uint32_t aTargetRate,
  42              uint32_t aDesiredBuffering)
  43       : mSourceRate(aSourceRate),
  44         mTargetRate(aTargetRate),
  45         mDesiredBuffering(aDesiredBuffering) {}
  46
  47   /**
  48    * The correction in the form of a ratio. A correction of 0.98 means that the
  49    * target is 2% slower compared to the source or 1.03 which means that the
  50    * target is 3% faster than the source.
  51    */
  52   float GetCorrection() { return mCorrection; }
  53
  54   /**
  55    * Update the available source frames, target frames, and the current
  56    * buffer, in every iteration. If the conditions are met a new correction is
  57    * calculated. A new correction is calculated in the following cases:
  58    *   1. Every mAdjustmentIntervalMs milliseconds (1000ms).
  59    *   2. Every time we run low on buffered frames (less than 20ms).
  60    * In addition to that, the correction is clamped to 10% to avoid sound
  61    * distortion so the result will be in [0.9, 1.1].
  62    */
  63   void UpdateClock(uint32_t aSourceFrames, uint32_t aTargetFrames,
  64                    uint32_t aBufferedFrames, uint32_t aRemainingFrames) {
  65     if (mSourceClock >= mSourceRate / 10 || mTargetClock >= mTargetRate / 10) {
  66       // Only update the correction if 100ms has passed since last update.
  67       if (aBufferedFrames < mDesiredBuffering * 4 / 10 /*40%*/ ||
  68           aRemainingFrames < mDesiredBuffering * 4 / 10 /*40%*/) {
  69         // We are getting close to the lower or upper bound of the internal
  70         // buffer. Steer clear.
  71         CalculateCorrection(0.9, aBufferedFrames, aRemainingFrames);
  72       } else if ((mTargetClock * 1000 / mTargetRate) >= mAdjustmentIntervalMs ||
  73                  (mSourceClock * 1000 / mSourceRate) >= mAdjustmentIntervalMs) {
  74         // The adjustment interval has passed on one side. Recalculate.
  75         CalculateCorrection(0.6, aBufferedFrames, aRemainingFrames);
  76       }
  77     }
  78     mTargetClock += aTargetFrames;
  79     mSourceClock += aSourceFrames;
  80   }
  81
  82  private:
  83   /**
  84    * aCalculationWeight is a percentage [0, 1] with which the calculated
  85    * correction will be weighted. The existing correction will be weighted with
  86    * 1 - aCalculationWeight. This gives some inertia to the speed at which the
  87    * correction changes, for smoother changes.
  88    */
  89   void CalculateCorrection(float aCalculationWeight, uint32_t aBufferedFrames,
  90                            uint32_t aRemainingFrames) {
  91     // We want to maintain the desired buffer
  92     uint32_t bufferedFramesDiff = aBufferedFrames - mDesiredBuffering;
  93     uint32_t resampledSourceClock =
  94         std::max(1u, mSourceClock + bufferedFramesDiff);
  95     if (mTargetRate != mSourceRate) {
  96       resampledSourceClock *= static_cast<float>(mTargetRate) / mSourceRate;
  97     }
  98
  99     MOZ_LOG(gMediaTrackGraphLog, LogLevel::Verbose,
 100             ("ClockDrift %p Calculated correction %.3f (with weight: %.1f -> "
 101              "%.3f) (buffer: %u, desired: %u, remaining: %u)",
 102              this, static_cast<float>(mTargetClock) / resampledSourceClock,
 103              aCalculationWeight,
 104              (1 - aCalculationWeight) * mCorrection +
 105                  aCalculationWeight * mTargetClock / resampledSourceClock,
 106              aBufferedFrames, mDesiredBuffering, aRemainingFrames));
 107
 108     mCorrection = (1 - aCalculationWeight) * mCorrection +
 109                   aCalculationWeight * mTargetClock / resampledSourceClock;
 110
 111     // Clamp to range [0.9, 1.1] to avoid distortion
 112     mCorrection = std::min(std::max(mCorrection, 0.9f), 1.1f);
 113
 114     // Reset the counters to prepare for the next period.
 115     mTargetClock = 0;
 116     mSourceClock = 0;
 117   }
 118
 119  public:
 120   const uint32_t mSourceRate;
 121   const uint32_t mTargetRate;
 122   const uint32_t mAdjustmentIntervalMs = 1000;
 123   const uint32_t mDesiredBuffering;
 124
 125  private:
 126   float mCorrection = 1.0;
 127
 128   uint32_t mSourceClock = 0;
 129   uint32_t mTargetClock = 0;
 130 };
 131
 132 /**
 133  * Correct the drift between two independent clocks, the source, and the target
 134  * clock. The target clock is the master clock so the correction syncs the drift
 135  * of the source clock to the target. The nominal sampling rates of source and
 136  * target must be provided. If the source and the target operate in different
 137  * sample rate the drift correction will be performed on the top of resampling
 138  * from the source rate to the target rate.
 139  *
 140  * It works with AudioSegment in order to be able to be used from the
 141  * MediaTrackGraph/MediaTrack. The audio buffers are pre-allocated so there is
 142  * no new allocation takes place during operation. The preallocation capacity is
 143  * 100ms for input and 100ms for output. The class consists of ClockDrift and
 144  * AudioResampler check there for more details.
 145  *
 146  * The class is not thread-safe. The construction can happen in any thread but
 147  * the member method must be used in a single thread that can be different than
 148  * the construction thread. Appropriate for being used in the high priority
 149  * audio thread.
 150  */
 151 class AudioDriftCorrection final {
 152   const uint32_t kMinBufferMs = 5;
 153
 154  public:
 155   AudioDriftCorrection(uint32_t aSourceRate, uint32_t aTargetRate,
 156                        uint32_t aBufferMs,
 157                        const PrincipalHandle& aPrincipalHandle)
 158       : mDesiredBuffering(std::max(kMinBufferMs, aBufferMs) * aSourceRate /
 159                           1000),
 160         mTargetRate(aTargetRate),
 161         mClockDrift(aSourceRate, aTargetRate, mDesiredBuffering),
 162         mResampler(aSourceRate, aTargetRate, mDesiredBuffering,
 163                    aPrincipalHandle) {}
 164
 165   /**
 166    * The source audio frames and request the number of target audio frames must
 167    * be provided. The duration of the source and the output is considered as the
 168    * source clock and the target clock. The input is buffered internally so some
 169    * latency exists. The returned AudioSegment must be cleaned up because the
 170    * internal buffer will be reused after 100ms. If the drift correction (and
 171    * possible resampling) is not possible due to lack of input data an empty
 172    * AudioSegment will be returned. Not thread-safe.
 173    */
 174   AudioSegment RequestFrames(const AudioSegment& aInput,
 175                              uint32_t aOutputFrames) {
 176     // Very important to go first since the Dynamic will get the sample format
 177     // from the chunk.
 178     if (aInput.GetDuration()) {
 179       // Always go through the resampler because the clock might shift later.
 180       mResampler.AppendInput(aInput);
 181     }
 182     mClockDrift.UpdateClock(aInput.GetDuration(), aOutputFrames,
 183                             mResampler.InputReadableFrames(),
 184                             mResampler.InputWritableFrames());
 185     TrackRate receivingRate = mTargetRate * mClockDrift.GetCorrection();
 186     // Update resampler's rate if there is a new correction.
 187     mResampler.UpdateOutRate(receivingRate);
 188     // If it does not have enough frames the result will be an empty segment.
 189     AudioSegment output = mResampler.Resample(aOutputFrames);
 190     if (output.IsEmpty()) {
 191       NS_WARNING("Got nothing from the resampler");
 192       output.AppendNullData(aOutputFrames);
 193     }
 194     return output;
 195   }
 196
 197   // Only accessible from the same thread that is driving RequestFrames().
 198   uint32_t CurrentBuffering() const { return mResampler.InputReadableFrames(); }
 199
 200   const uint32_t mDesiredBuffering;
 201   const uint32_t mTargetRate;
 202
 203  private:
 204   ClockDrift mClockDrift;
 205   AudioResampler mResampler;
 206 };
 207
 208 };     // namespace mozilla
 209 #endif /* MOZILLA_AUDIO_DRIFT_CORRECTION_H_ */