dom/media/mediasink/AudioDecoderInputTrack.h

   1 /* This Source Code Form is subject to the terms of the Mozilla Public
   2  * License, v. 2.0. If a copy of the MPL was not distributed with this
   3  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   4
   5 #ifndef AudioDecoderInputTrack_h
   6 #define AudioDecoderInputTrack_h
   7
   8 #include "AudioSegment.h"
   9 #include "MediaEventSource.h"
  10 #include "MediaTimer.h"
  11 #include "MediaTrackGraph.h"
  12 #include "MediaSegment.h"
  13 #include "TimeUnits.h"
  14 #include "mozilla/SPSCQueue.h"
  15 #include "mozilla/StateMirroring.h"
  16 #include "nsISerialEventTarget.h"
  17
  18 namespace mozilla {
  19
  20 class AudioData;
  21 class AudioInfo;
  22 class RLBoxSoundTouch;
  23
  24 /**
  25  * AudioDecoderInputTrack is used as a source for the audio decoder data, which
  26  * supports adjusting playback rate and preserve pitch.
  27  * The owner of this track would be responsible to push audio data via
  28  * `AppendData()` into a SPSC queue, which is a thread-safe queue between the
  29  * decoder thread (producer) and the graph thread (consumer). MediaTrackGraph
  30  * requires data via `ProcessInput()`, then AudioDecoderInputTrack would convert
  31  * (based on sample rate and playback rate) and append the amount of needed
  32  * audio frames onto the output segment that would be used by MediaTrackGraph.
  33  */
  34 class AudioDecoderInputTrack final : public ProcessedMediaTrack {
  35  public:
  36   static AudioDecoderInputTrack* Create(MediaTrackGraph* aGraph,
  37                                         nsISerialEventTarget* aDecoderThread,
  38                                         const AudioInfo& aInfo,
  39                                         float aPlaybackRate, float aVolume,
  40                                         bool aPreservesPitch);
  41
  42   // SPSCData suppports filling different supported type variants, and is used
  43   // to achieve a thread-safe information exchange between the decoder thread
  44   // and the graph thread.
  45   struct SPSCData final {
  46     struct Empty {};
  47     struct ClearFutureData {};
  48     struct DecodedData {
  49       DecodedData()
  50           : mStartTime(media::TimeUnit::Invalid()),
  51             mEndTime(media::TimeUnit::Invalid()) {}
  52       DecodedData(DecodedData&& aDecodedData)
  53           : mSegment(std::move(aDecodedData.mSegment)) {
  54         mStartTime = aDecodedData.mStartTime;
  55         mEndTime = aDecodedData.mEndTime;
  56         aDecodedData.Clear();
  57       }
  58       DecodedData(media::TimeUnit aStartTime, media::TimeUnit aEndTime)
  59           : mStartTime(aStartTime), mEndTime(aEndTime) {}
  60       DecodedData(const DecodedData&) = delete;
  61       DecodedData& operator=(const DecodedData&) = delete;
  62       void Clear() {
  63         mSegment.Clear();
  64         mStartTime = media::TimeUnit::Invalid();
  65         mEndTime = media::TimeUnit::Invalid();
  66       }
  67       AudioSegment mSegment;
  68       media::TimeUnit mStartTime;
  69       media::TimeUnit mEndTime;
  70     };
  71     struct EOS {};
  72
  73     SPSCData() : mData(Empty()){};
  74     explicit SPSCData(ClearFutureData&& aArg) : mData(std::move(aArg)){};
  75     explicit SPSCData(DecodedData&& aArg) : mData(std::move(aArg)){};
  76     explicit SPSCData(EOS&& aArg) : mData(std::move(aArg)){};
  77
  78     bool HasData() const { return !mData.is<Empty>(); }
  79     bool IsClearFutureData() const { return mData.is<ClearFutureData>(); }
  80     bool IsDecodedData() const { return mData.is<DecodedData>(); }
  81     bool IsEOS() const { return mData.is<EOS>(); }
  82
  83     DecodedData* AsDecodedData() {
  84       return IsDecodedData() ? &mData.as<DecodedData>() : nullptr;
  85     }
  86
  87     Variant<Empty, ClearFutureData, DecodedData, EOS> mData;
  88   };
  89
  90   // Decoder thread API
  91   void AppendData(AudioData* aAudio, const PrincipalHandle& aPrincipalHandle);
  92   void AppendData(nsTArray<RefPtr<AudioData>>& aAudioArray,
  93                   const PrincipalHandle& aPrincipalHandle);
  94   void NotifyEndOfStream();
  95   void ClearFutureData();
  96   void SetVolume(float aVolume);
  97   void SetPlaybackRate(float aPlaybackRate);
  98   void SetPreservesPitch(bool aPreservesPitch);
  99   // After calling this, the track are not expected to receive any new data.
 100   void Close();
 101   bool HasBatchedData() const;
 102
 103   MediaEventSource<int64_t>& OnOutput() { return mOnOutput; }
 104   MediaEventSource<void>& OnEnd() { return mOnEnd; }
 105
 106   // Graph Thread API
 107   void DestroyImpl() override;
 108   void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override;
 109   uint32_t NumberOfChannels() const override;
 110
 111   // The functions below are only used for testing.
 112   TrackTime WrittenFrames() const {
 113     AssertOnGraphThread();
 114     return mWrittenFrames;
 115   }
 116   float Volume() const {
 117     AssertOnGraphThread();
 118     return mVolume;
 119   }
 120   float PlaybackRate() const {
 121     AssertOnGraphThread();
 122     return mPlaybackRate;
 123   }
 124
 125  protected:
 126   ~AudioDecoderInputTrack();
 127
 128  private:
 129   AudioDecoderInputTrack(nsISerialEventTarget* aDecoderThread,
 130                          TrackRate aGraphRate, const AudioInfo& aInfo,
 131                          float aPlaybackRate, float aVolume,
 132                          bool aPreservesPitch);
 133
 134   // Return false if the converted segment contains zero duration.
 135   bool ConvertAudioDataToSegment(AudioData* aAudio, AudioSegment& aSegment,
 136                                  const PrincipalHandle& aPrincipalHandle);
 137
 138   void HandleSPSCData(SPSCData& aData);
 139
 140   // These methods would return the total frames that we consumed from
 141   // `mBufferedData`.
 142   TrackTime AppendBufferedDataToOutput(TrackTime aExpectedDuration);
 143   TrackTime FillDataToTimeStretcher(TrackTime aExpectedDuration);
 144   TrackTime AppendTimeStretchedDataToSegment(TrackTime aExpectedDuration,
 145                                              AudioSegment& aOutput);
 146   TrackTime AppendUnstretchedDataToSegment(TrackTime aExpectedDuration,
 147                                            AudioSegment& aOutput);
 148
 149   // Return the total frames that we retrieve from the time stretcher.
 150   TrackTime DrainStretchedDataIfNeeded(TrackTime aExpectedDuration,
 151                                        AudioSegment& aOutput);
 152   TrackTime GetDataFromTimeStretcher(TrackTime aExpectedDuration,
 153                                      AudioSegment& aOutput);
 154   void NotifyInTheEndOfProcessInput(TrackTime aFillDuration);
 155
 156   bool HasSentAllData() const;
 157
 158   bool ShouldBatchData() const;
 159   void BatchData(AudioData* aAudio, const PrincipalHandle& aPrincipalHandle);
 160   void DispatchPushBatchedDataIfNeeded();
 161   void PushBatchedDataIfNeeded();
 162   void PushDataToSPSCQueue(SPSCData& data);
 163
 164   void SetVolumeImpl(float aVolume);
 165   void SetPlaybackRateImpl(float aPlaybackRate);
 166   void SetPreservesPitchImpl(bool aPreservesPitch);
 167
 168   void EnsureTimeStretcher();
 169   void SetTempoAndRateForTimeStretcher();
 170   uint32_t GetChannelCountForTimeStretcher() const;
 171
 172   inline void AssertOnDecoderThread() const {
 173     MOZ_ASSERT(mDecoderThread->IsOnCurrentThread());
 174   }
 175
 176   const RefPtr<nsISerialEventTarget> mDecoderThread;
 177
 178   // Notify the amount of audio frames which have been sent to the track.
 179   MediaEventProducer<int64_t> mOnOutput;
 180   // Notify when the track is ended.
 181   MediaEventProducer<void> mOnEnd;
 182
 183   // These variables are ONLY used in the decoder thread.
 184   nsAutoRef<SpeexResamplerState> mResampler;
 185   uint32_t mResamplerChannelCount;
 186   const uint32_t mInitialInputChannels;
 187   TrackRate mInputSampleRate;
 188   DelayedScheduler mDelayedScheduler;
 189   bool mShutdownSPSCQueue = false;
 190
 191   // These attributes are ONLY used in the graph thread.
 192   bool mReceivedEOS = false;
 193   TrackTime mWrittenFrames = 0;
 194   float mPlaybackRate;
 195   float mVolume;
 196   bool mPreservesPitch;
 197
 198   // A thread-safe queue shared by the decoder thread and the graph thread.
 199   // The decoder thread is the producer side, and the graph thread is the
 200   // consumer side. This queue should NEVER get full. In order to achieve that,
 201   // we would batch input samples when SPSC queue doesn't have many available
 202   // capacity.
 203   // In addition, as the media track isn't guaranteed to be destroyed on the
 204   // graph thread (it could be destroyed on the main thread as well) so we might
 205   // not clear all data in SPSC queue when the track's `DestroyImpl()` gets
 206   // called. We leave to destroy the queue later when the track gets destroyed.
 207   SPSCQueue<SPSCData> mSPSCQueue{40};
 208
 209   // When the graph requires the less amount of audio frames than the amount of
 210   // frames an audio data has, then the remaining part of frames would be stored
 211   // and used in next iteration.
 212   // This is ONLY used in the graph thread.
 213   AudioSegment mBufferedData;
 214
 215   // In order to prevent SPSC queue from being full, we want to batch multiple
 216   // data into one to control the density of SPSC queue, the length of batched
 217   // data would be dynamically adjusted by queue's available capacity.
 218   // This is ONLY used in the decoder thread.
 219   SPSCData::DecodedData mBatchedData;
 220
 221   // True if we've sent all data to the graph, then the track will be marked as
 222   // ended in the next iteration.
 223   bool mSentAllData = false;
 224
 225   // This is used to adjust the playback rate and pitch.
 226   RLBoxSoundTouch* mTimeStretcher = nullptr;
 227
 228   // Buffers that would be used for the time stretching.
 229   AutoTArray<AudioDataValue, 2> mInterleavedBuffer;
 230 };
 231
 232 }  // namespace mozilla
 233
 234 #endif  // AudioDecoderInputTrack_h