dom/media/mediasink/AudioDecoderInputTrack.h

   1 /* This Source Code Form is subject to the terms of the Mozilla Public
   2  * License, v. 2.0. If a copy of the MPL was not distributed with this
   3  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   4
   5 #ifndef AudioDecoderInputTrack_h
   6 #define AudioDecoderInputTrack_h
   7
   8 #include "AudioSegment.h"
   9 #include "MediaEventSource.h"
  10 #include "MediaTimer.h"
  11 #include "MediaTrackGraph.h"
  12 #include "MediaSegment.h"
  13 #include "TimeUnits.h"
  14 #include "mozilla/SPSCQueue.h"
  15 #include "mozilla/StateMirroring.h"
  16 #include "mozilla/TimeStamp.h"
  17 #include "nsISerialEventTarget.h"
  18
  19 namespace mozilla {
  20
  21 class AudioData;
  22 class AudioInfo;
  23 class RLBoxSoundTouch;
  24
  25 /**
  26  * AudioDecoderInputTrack is used as a source for the audio decoder data, which
  27  * supports adjusting playback rate and preserve pitch.
  28  * The owner of this track would be responsible to push audio data via
  29  * `AppendData()` into a SPSC queue, which is a thread-safe queue between the
  30  * decoder thread (producer) and the graph thread (consumer). MediaTrackGraph
  31  * requires data via `ProcessInput()`, then AudioDecoderInputTrack would convert
  32  * (based on sample rate and playback rate) and append the amount of needed
  33  * audio frames onto the output segment that would be used by MediaTrackGraph.
  34  */
  35 class AudioDecoderInputTrack final : public ProcessedMediaTrack {
  36  public:
  37   static AudioDecoderInputTrack* Create(MediaTrackGraph* aGraph,
  38                                         nsISerialEventTarget* aDecoderThread,
  39                                         const AudioInfo& aInfo,
  40                                         float aPlaybackRate, float aVolume,
  41                                         bool aPreservesPitch);
  42
  43   // SPSCData suppports filling different supported type variants, and is used
  44   // to achieve a thread-safe information exchange between the decoder thread
  45   // and the graph thread.
  46   struct SPSCData final {
  47     struct Empty {};
  48     struct ClearFutureData {};
  49     struct DecodedData {
  50       DecodedData()
  51           : mStartTime(media::TimeUnit::Invalid()),
  52             mEndTime(media::TimeUnit::Invalid()) {}
  53       DecodedData(DecodedData&& aDecodedData)
  54           : mSegment(std::move(aDecodedData.mSegment)) {
  55         mStartTime = aDecodedData.mStartTime;
  56         mEndTime = aDecodedData.mEndTime;
  57         aDecodedData.Clear();
  58       }
  59       DecodedData(media::TimeUnit aStartTime, media::TimeUnit aEndTime)
  60           : mStartTime(aStartTime), mEndTime(aEndTime) {}
  61       DecodedData(const DecodedData&) = delete;
  62       DecodedData& operator=(const DecodedData&) = delete;
  63       void Clear() {
  64         mSegment.Clear();
  65         mStartTime = media::TimeUnit::Invalid();
  66         mEndTime = media::TimeUnit::Invalid();
  67       }
  68       AudioSegment mSegment;
  69       media::TimeUnit mStartTime;
  70       media::TimeUnit mEndTime;
  71     };
  72     struct EOS {};
  73
  74     SPSCData() : mData(Empty()) {};
  75     explicit SPSCData(ClearFutureData&& aArg) : mData(std::move(aArg)) {};
  76     explicit SPSCData(DecodedData&& aArg) : mData(std::move(aArg)) {};
  77     explicit SPSCData(EOS&& aArg) : mData(std::move(aArg)) {};
  78
  79     bool HasData() const { return !mData.is<Empty>(); }
  80     bool IsClearFutureData() const { return mData.is<ClearFutureData>(); }
  81     bool IsDecodedData() const { return mData.is<DecodedData>(); }
  82     bool IsEOS() const { return mData.is<EOS>(); }
  83
  84     DecodedData* AsDecodedData() {
  85       return IsDecodedData() ? &mData.as<DecodedData>() : nullptr;
  86     }
  87
  88     Variant<Empty, ClearFutureData, DecodedData, EOS> mData;
  89   };
  90
  91   // Decoder thread API
  92   void AppendData(AudioData* aAudio, const PrincipalHandle& aPrincipalHandle);
  93   void AppendData(nsTArray<RefPtr<AudioData>>& aAudioArray,
  94                   const PrincipalHandle& aPrincipalHandle);
  95   void NotifyEndOfStream();
  96   void ClearFutureData();
  97   void SetVolume(float aVolume);
  98   void SetPlaybackRate(float aPlaybackRate);
  99   void SetPreservesPitch(bool aPreservesPitch);
 100   // After calling this, the track are not expected to receive any new data.
 101   void Close();
 102   bool HasBatchedData() const;
 103
 104   MediaEventSource<int64_t>& OnOutput() { return mOnOutput; }
 105   MediaEventSource<void>& OnEnd() { return mOnEnd; }
 106
 107   // Graph Thread API
 108   void DestroyImpl() override;
 109   void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override;
 110   uint32_t NumberOfChannels() const override;
 111
 112   // The functions below are only used for testing.
 113   TrackTime WrittenFrames() const {
 114     AssertOnGraphThread();
 115     return mWrittenFrames;
 116   }
 117   float Volume() const {
 118     AssertOnGraphThread();
 119     return mVolume;
 120   }
 121   float PlaybackRate() const {
 122     AssertOnGraphThread();
 123     return mPlaybackRate;
 124   }
 125
 126  protected:
 127   ~AudioDecoderInputTrack();
 128
 129  private:
 130   AudioDecoderInputTrack(nsISerialEventTarget* aDecoderThread,
 131                          TrackRate aGraphRate, const AudioInfo& aInfo,
 132                          float aPlaybackRate, float aVolume,
 133                          bool aPreservesPitch);
 134
 135   // Return false if the converted segment contains zero duration.
 136   bool ConvertAudioDataToSegment(AudioData* aAudio, AudioSegment& aSegment,
 137                                  const PrincipalHandle& aPrincipalHandle);
 138
 139   void HandleSPSCData(SPSCData& aData);
 140
 141   // These methods would return the total frames that we consumed from
 142   // `mBufferedData`.
 143   TrackTime AppendBufferedDataToOutput(TrackTime aExpectedDuration);
 144   TrackTime FillDataToTimeStretcher(TrackTime aExpectedDuration);
 145   TrackTime AppendTimeStretchedDataToSegment(TrackTime aExpectedDuration,
 146                                              AudioSegment& aOutput);
 147   TrackTime AppendUnstretchedDataToSegment(TrackTime aExpectedDuration,
 148                                            AudioSegment& aOutput);
 149
 150   // Return the total frames that we retrieve from the time stretcher.
 151   TrackTime DrainStretchedDataIfNeeded(TrackTime aExpectedDuration,
 152                                        AudioSegment& aOutput);
 153   TrackTime GetDataFromTimeStretcher(TrackTime aExpectedDuration,
 154                                      AudioSegment& aOutput);
 155   void NotifyInTheEndOfProcessInput(TrackTime aFillDuration);
 156
 157   bool HasSentAllData() const;
 158
 159   bool ShouldBatchData() const;
 160   void BatchData(AudioData* aAudio, const PrincipalHandle& aPrincipalHandle);
 161   void DispatchPushBatchedDataIfNeeded();
 162   void PushBatchedDataIfNeeded();
 163   void PushDataToSPSCQueue(SPSCData& data);
 164
 165   void SetVolumeImpl(float aVolume);
 166   void SetPlaybackRateImpl(float aPlaybackRate);
 167   void SetPreservesPitchImpl(bool aPreservesPitch);
 168
 169   void EnsureTimeStretcher();
 170   void SetTempoAndRateForTimeStretcher();
 171   uint32_t GetChannelCountForTimeStretcher() const;
 172
 173   inline void AssertOnDecoderThread() const {
 174     MOZ_ASSERT(mDecoderThread->IsOnCurrentThread());
 175   }
 176
 177   const RefPtr<nsISerialEventTarget> mDecoderThread;
 178
 179   // Notify the amount of audio frames which have been sent to the track.
 180   MediaEventProducer<int64_t> mOnOutput;
 181   // Notify when the track is ended.
 182   MediaEventProducer<void> mOnEnd;
 183
 184   // These variables are ONLY used in the decoder thread.
 185   nsAutoRef<SpeexResamplerState> mResampler;
 186   uint32_t mResamplerChannelCount;
 187   const uint32_t mInitialInputChannels;
 188   TrackRate mInputSampleRate;
 189   DelayedScheduler<TimeStamp> mDelayedScheduler;
 190   bool mShutdownSPSCQueue = false;
 191
 192   // These attributes are ONLY used in the graph thread.
 193   bool mReceivedEOS = false;
 194   TrackTime mWrittenFrames = 0;
 195   float mPlaybackRate;
 196   float mVolume;
 197   bool mPreservesPitch;
 198
 199   // A thread-safe queue shared by the decoder thread and the graph thread.
 200   // The decoder thread is the producer side, and the graph thread is the
 201   // consumer side. This queue should NEVER get full. In order to achieve that,
 202   // we would batch input samples when SPSC queue doesn't have many available
 203   // capacity.
 204   // In addition, as the media track isn't guaranteed to be destroyed on the
 205   // graph thread (it could be destroyed on the main thread as well) so we might
 206   // not clear all data in SPSC queue when the track's `DestroyImpl()` gets
 207   // called. We leave to destroy the queue later when the track gets destroyed.
 208   SPSCQueue<SPSCData> mSPSCQueue{40};
 209
 210   // When the graph requires the less amount of audio frames than the amount of
 211   // frames an audio data has, then the remaining part of frames would be stored
 212   // and used in next iteration.
 213   // This is ONLY used in the graph thread.
 214   AudioSegment mBufferedData;
 215
 216   // In order to prevent SPSC queue from being full, we want to batch multiple
 217   // data into one to control the density of SPSC queue, the length of batched
 218   // data would be dynamically adjusted by queue's available capacity.
 219   // This is ONLY used in the decoder thread.
 220   SPSCData::DecodedData mBatchedData;
 221
 222   // True if we've sent all data to the graph, then the track will be marked as
 223   // ended in the next iteration.
 224   bool mSentAllData = false;
 225
 226   // This is used to adjust the playback rate and pitch.
 227   RLBoxSoundTouch* mTimeStretcher = nullptr;
 228
 229   // Buffers that would be used for the time stretching.
 230   AutoTArray<AudioDataValue, 2> mInterleavedBuffer;
 231 };
 232
 233 }  // namespace mozilla
 234
 235 #endif  // AudioDecoderInputTrack_h