dom/media/webaudio/AudioNodeTrack.h

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
   2 /* This Source Code Form is subject to the terms of the Mozilla Public
   3  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
   4  * You can obtain one at http://mozilla.org/MPL/2.0/. */
   5
   6 #ifndef MOZILLA_AUDIONODETRACK_H_
   7 #define MOZILLA_AUDIONODETRACK_H_
   8
   9 #include "MediaTrackGraph.h"
  10 #include "mozilla/dom/AudioNodeBinding.h"
  11 #include "AlignedTArray.h"
  12 #include "AudioBlock.h"
  13 #include "AudioSegment.h"
  14
  15 namespace WebCore {
  16 class Reverb;
  17 }  // namespace WebCore
  18
  19 namespace mozilla {
  20
  21 namespace dom {
  22 struct ThreeDPoint;
  23 struct AudioParamEvent;
  24 class AudioContext;
  25 }  // namespace dom
  26
  27 class AbstractThread;
  28 class ThreadSharedFloatArrayBufferList;
  29 class AudioNodeEngine;
  30
  31 typedef AlignedAutoTArray<float, GUESS_AUDIO_CHANNELS * WEBAUDIO_BLOCK_SIZE, 16>
  32     DownmixBufferType;
  33
  34 /**
  35  * An AudioNodeTrack produces one audio track with ID AUDIO_TRACK.
  36  * The start time of the AudioTrack is aligned to the start time of the
  37  * AudioContext's destination node track, plus some multiple of BLOCK_SIZE
  38  * samples.
  39  *
  40  * An AudioNodeTrack has an AudioNodeEngine plugged into it that does the
  41  * actual audio processing. AudioNodeTrack contains the glue code that
  42  * integrates audio processing with the MediaTrackGraph.
  43  */
  44 class AudioNodeTrack : public ProcessedMediaTrack {
  45   typedef dom::ChannelCountMode ChannelCountMode;
  46   typedef dom::ChannelInterpretation ChannelInterpretation;
  47
  48  public:
  49   typedef mozilla::dom::AudioContext AudioContext;
  50
  51   enum { AUDIO_TRACK = 1 };
  52
  53   typedef AutoTArray<AudioBlock, 1> OutputChunks;
  54
  55   // Flags re main thread updates and track output.
  56   typedef unsigned Flags;
  57   enum : Flags {
  58     NO_TRACK_FLAGS = 0U,
  59     NEED_MAIN_THREAD_ENDED = 1U << 0,
  60     NEED_MAIN_THREAD_CURRENT_TIME = 1U << 1,
  61     // Internal AudioNodeTracks can only pass their output to another
  62     // AudioNode, whereas external AudioNodeTracks can pass their output
  63     // to other ProcessedMediaTracks or hardware audio output.
  64     EXTERNAL_OUTPUT = 1U << 2,
  65   };
  66   /**
  67    * Create a track that will process audio for an AudioNode.
  68    * Takes ownership of aEngine.
  69    * aGraph is required and equals the graph of aCtx in most cases. An exception
  70    * is AudioDestinationNode where the context's graph hasn't been set up yet.
  71    */
  72   static already_AddRefed<AudioNodeTrack> Create(AudioContext* aCtx,
  73                                                  AudioNodeEngine* aEngine,
  74                                                  Flags aKind,
  75                                                  MediaTrackGraph* aGraph);
  76
  77  protected:
  78   /**
  79    * Transfers ownership of aEngine to the new AudioNodeTrack.
  80    */
  81   AudioNodeTrack(AudioNodeEngine* aEngine, Flags aFlags, TrackRate aSampleRate);
  82
  83   ~AudioNodeTrack();
  84
  85  public:
  86   // Control API
  87   /**
  88    * Sets a parameter that's a time relative to some track's played time.
  89    * This time is converted to a time relative to this track when it's set.
  90    */
  91   void SetTrackTimeParameter(uint32_t aIndex, AudioContext* aContext,
  92                              double aTrackTime);
  93   void SetDoubleParameter(uint32_t aIndex, double aValue);
  94   void SetInt32Parameter(uint32_t aIndex, int32_t aValue);
  95   void SetThreeDPointParameter(uint32_t aIndex, const dom::ThreeDPoint& aValue);
  96   void SetBuffer(AudioChunk&& aBuffer);
  97   void SetReverb(WebCore::Reverb* aReverb, uint32_t aImpulseChannelCount);
  98   // This sends a single event to the timeline on the MTG thread side.
  99   void SendTimelineEvent(uint32_t aIndex, const dom::AudioParamEvent& aEvent);
 100   // This consumes the contents of aData.  aData will be emptied after this
 101   // returns.
 102   void SetRawArrayData(nsTArray<float>&& aData);
 103   void SetChannelMixingParameters(uint32_t aNumberOfChannels,
 104                                   ChannelCountMode aChannelCountMoe,
 105                                   ChannelInterpretation aChannelInterpretation);
 106   void SetPassThrough(bool aPassThrough);
 107   void SendRunnable(already_AddRefed<nsIRunnable> aRunnable);
 108   ChannelInterpretation GetChannelInterpretation() {
 109     return mChannelInterpretation;
 110   }
 111
 112   void SetAudioParamHelperTrack() {
 113     MOZ_ASSERT(!mAudioParamTrack, "Can only do this once");
 114     mAudioParamTrack = true;
 115   }
 116   // The value for channelCount on an AudioNode, but on the audio thread side.
 117   uint32_t NumberOfChannels() const override;
 118
 119   /*
 120    * Resume track after updating its concept of current time by aAdvance.
 121    * Main thread.  Used only from AudioDestinationNode when resuming a track
 122    * suspended to save running the MediaTrackGraph when there are no other
 123    * nodes in the AudioContext.
 124    */
 125   void AdvanceAndResume(TrackTime aAdvance);
 126
 127   AudioNodeTrack* AsAudioNodeTrack() override { return this; }
 128   void AddInput(MediaInputPort* aPort) override;
 129   void RemoveInput(MediaInputPort* aPort) override;
 130
 131   // Graph thread only
 132   void SetTrackTimeParameterImpl(uint32_t aIndex, MediaTrack* aRelativeToTrack,
 133                                  double aTrackTime);
 134   void SetChannelMixingParametersImpl(
 135       uint32_t aNumberOfChannels, ChannelCountMode aChannelCountMoe,
 136       ChannelInterpretation aChannelInterpretation);
 137   void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override;
 138   /**
 139    * Produce the next block of output, before input is provided.
 140    * ProcessInput() will be called later, and it then should not change
 141    * the output.  This is used only for DelayNodeEngine in a feedback loop.
 142    */
 143   void ProduceOutputBeforeInput(GraphTime aFrom);
 144   bool IsAudioParamTrack() const { return mAudioParamTrack; }
 145
 146   const OutputChunks& LastChunks() const { return mLastChunks; }
 147   bool MainThreadNeedsUpdates() const override {
 148     return ((mFlags & NEED_MAIN_THREAD_ENDED) && mEnded) ||
 149            (mFlags & NEED_MAIN_THREAD_CURRENT_TIME);
 150   }
 151
 152   // Any thread
 153   AudioNodeEngine* Engine() { return mEngine.get(); }
 154
 155   size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const override;
 156   size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const override;
 157
 158   void SizeOfAudioNodesIncludingThis(MallocSizeOf aMallocSizeOf,
 159                                      AudioNodeSizes& aUsage) const;
 160
 161   /*
 162    * SetActive() is called when either an active input is added or the engine
 163    * for a source node transitions from inactive to active.  This is not
 164    * called from engines for processing nodes because they only become active
 165    * when there are active input tracks, in which case this track is already
 166    * active.
 167    */
 168   void SetActive();
 169   /*
 170    * ScheduleCheckForInactive() is called during track processing when the
 171    * engine transitions from active to inactive, or the track finishes.  It
 172    * schedules a call to CheckForInactive() after track processing.
 173    */
 174   void ScheduleCheckForInactive();
 175
 176  protected:
 177   void OnGraphThreadDone() override;
 178   void DestroyImpl() override;
 179
 180   /*
 181    * CheckForInactive() is called when the engine transitions from active to
 182    * inactive, or an active input is removed, or the track finishes.  If the
 183    * track is now inactive, then mInputChunks will be cleared and mLastChunks
 184    * will be set to null.  ProcessBlock() will not be called on the engine
 185    * again until SetActive() is called.
 186    */
 187   void CheckForInactive();
 188
 189   void AdvanceOutputSegment();
 190   void FinishOutput();
 191   void AccumulateInputChunk(uint32_t aInputIndex, const AudioBlock& aChunk,
 192                             AudioBlock* aBlock,
 193                             DownmixBufferType* aDownmixBuffer);
 194   void UpMixDownMixChunk(const AudioBlock* aChunk, uint32_t aOutputChannelCount,
 195                          nsTArray<const float*>& aOutputChannels,
 196                          DownmixBufferType& aDownmixBuffer);
 197
 198   uint32_t ComputedNumberOfChannels(uint32_t aInputChannelCount);
 199   void ObtainInputBlock(AudioBlock& aTmpChunk, uint32_t aPortIndex);
 200   void IncrementActiveInputCount();
 201   void DecrementActiveInputCount();
 202
 203   // The engine that will generate output for this node.
 204   const UniquePtr<AudioNodeEngine> mEngine;
 205   // The mixed input blocks are kept from iteration to iteration to avoid
 206   // reallocating channel data arrays and any buffers for mixing.
 207   OutputChunks mInputChunks;
 208   // The last block produced by this node.
 209   OutputChunks mLastChunks;
 210   // Whether this is an internal or external track
 211   const Flags mFlags;
 212   // The number of input tracks that may provide non-silent input.
 213   uint32_t mActiveInputCount = 0;
 214   // The number of input channels that this track requires. 0 means don't care.
 215   uint32_t mNumberOfInputChannels;
 216   // The mixing modes
 217   ChannelCountMode mChannelCountMode;
 218   ChannelInterpretation mChannelInterpretation;
 219   // Tracks are considered active if the track has not finished and either
 220   // the engine is active or there are active input tracks.
 221   bool mIsActive;
 222   // Whether the track should be marked as ended as soon
 223   // as the current time range has been computed block by block.
 224   bool mMarkAsEndedAfterThisBlock;
 225   // Whether the track is an AudioParamHelper track.
 226   bool mAudioParamTrack;
 227   // Whether the track just passes its input through.
 228   bool mPassThrough;
 229 };
 230
 231 }  // namespace mozilla
 232
 233 #endif /* MOZILLA_AUDIONODETRACK_H_ */