dom/media/webaudio/AudioNodeEngine.h

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6 #ifndef MOZILLA_AUDIONODEENGINE_H_
   7 #define MOZILLA_AUDIONODEENGINE_H_
   8
   9 #include "AudioSegment.h"
  10 #include "mozilla/dom/AudioNode.h"
  11 #include "mozilla/MemoryReporting.h"
  12 #include "mozilla/Mutex.h"
  13
  14 namespace WebCore {
  15 class Reverb;
  16 }  // namespace WebCore
  17
  18 namespace mozilla {
  19
  20 namespace dom {
  21 struct ThreeDPoint;
  22 class AudioParamTimeline;
  23 class DelayNodeEngine;
  24 struct AudioTimelineEvent;
  25 }  // namespace dom
  26
  27 class AbstractThread;
  28 class AudioBlock;
  29 class AudioNodeTrack;
  30
  31 /**
  32  * This class holds onto a set of immutable channel buffers. The storage
  33  * for the buffers must be malloced, but the buffer pointers and the malloc
  34  * pointers can be different (e.g. if the buffers are contained inside
  35  * some malloced object).
  36  */
  37 class ThreadSharedFloatArrayBufferList final : public ThreadSharedObject {
  38  public:
  39   /**
  40    * Construct with null channel data pointers.
  41    */
  42   explicit ThreadSharedFloatArrayBufferList(uint32_t aCount) {
  43     mContents.SetLength(aCount);
  44   }
  45   /**
  46    * Create with buffers suitable for transfer to
  47    * JS::NewArrayBufferWithContents().  The buffer contents are uninitialized
  48    * and so should be set using GetDataForWrite().
  49    */
  50   static already_AddRefed<ThreadSharedFloatArrayBufferList> Create(
  51       uint32_t aChannelCount, size_t aLength, const mozilla::fallible_t&);
  52
  53   ThreadSharedFloatArrayBufferList* AsThreadSharedFloatArrayBufferList()
  54       override {
  55     return this;
  56   };
  57
  58   struct Storage final {
  59     Storage() : mDataToFree(nullptr), mFree(nullptr), mSampleData(nullptr) {}
  60     ~Storage() {
  61       if (mFree) {
  62         mFree(mDataToFree);
  63       } else {
  64         MOZ_ASSERT(!mDataToFree);
  65       }
  66     }
  67     size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const {
  68       // NB: mSampleData might not be owned, if it is it just points to
  69       //     mDataToFree.
  70       return aMallocSizeOf(mDataToFree);
  71     }
  72     void* mDataToFree;
  73     void (*mFree)(void*);
  74     float* mSampleData;
  75   };
  76
  77   /**
  78    * This can be called on any thread.
  79    */
  80   uint32_t GetChannels() const { return mContents.Length(); }
  81   /**
  82    * This can be called on any thread.
  83    */
  84   const float* GetData(uint32_t aIndex) const {
  85     return mContents[aIndex].mSampleData;
  86   }
  87   /**
  88    * This can be called on any thread, but only when the calling thread is the
  89    * only owner.
  90    */
  91   float* GetDataForWrite(uint32_t aIndex) {
  92     MOZ_ASSERT(!IsShared());
  93     return mContents[aIndex].mSampleData;
  94   }
  95
  96   /**
  97    * Call this only during initialization, before the object is handed to
  98    * any other thread.
  99    */
 100   void SetData(uint32_t aIndex, void* aDataToFree, void (*aFreeFunc)(void*),
 101                float* aData) {
 102     Storage* s = &mContents[aIndex];
 103     if (s->mFree) {
 104       s->mFree(s->mDataToFree);
 105     } else {
 106       MOZ_ASSERT(!s->mDataToFree);
 107     }
 108
 109     s->mDataToFree = aDataToFree;
 110     s->mFree = aFreeFunc;
 111     s->mSampleData = aData;
 112   }
 113
 114   /**
 115    * Put this object into an error state where there are no channels.
 116    */
 117   void Clear() { mContents.Clear(); }
 118
 119   size_t SizeOfExcludingThis(
 120       mozilla::MallocSizeOf aMallocSizeOf) const override {
 121     size_t amount = ThreadSharedObject::SizeOfExcludingThis(aMallocSizeOf);
 122     amount += mContents.ShallowSizeOfExcludingThis(aMallocSizeOf);
 123     for (size_t i = 0; i < mContents.Length(); i++) {
 124       amount += mContents[i].SizeOfExcludingThis(aMallocSizeOf);
 125     }
 126
 127     return amount;
 128   }
 129
 130   size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const override {
 131     return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
 132   }
 133
 134  private:
 135   AutoTArray<Storage, 2> mContents;
 136 };
 137
 138 /**
 139  * aChunk must have been allocated by AllocateAudioBlock.
 140  */
 141 void WriteZeroesToAudioBlock(AudioBlock* aChunk, uint32_t aStart,
 142                              uint32_t aLength);
 143
 144 /**
 145  * Copy with scale. aScale == 1.0f should be optimized.
 146  */
 147 void AudioBufferCopyWithScale(const float* aInput, float aScale, float* aOutput,
 148                               uint32_t aSize);
 149
 150 /**
 151  * Pointwise multiply-add operation. aScale == 1.0f should be optimized.
 152  */
 153 void AudioBufferAddWithScale(const float* aInput, float aScale, float* aOutput,
 154                              uint32_t aSize);
 155
 156 /**
 157  * Pointwise multiply-add operation. aScale == 1.0f should be optimized.
 158  */
 159 void AudioBlockAddChannelWithScale(const float aInput[WEBAUDIO_BLOCK_SIZE],
 160                                    float aScale,
 161                                    float aOutput[WEBAUDIO_BLOCK_SIZE]);
 162
 163 /**
 164  * Pointwise copy-scaled operation. aScale == 1.0f should be optimized.
 165  *
 166  * Buffer size is implicitly assumed to be WEBAUDIO_BLOCK_SIZE.
 167  */
 168 void AudioBlockCopyChannelWithScale(const float* aInput, float aScale,
 169                                     float* aOutput);
 170
 171 /**
 172  * Vector copy-scaled operation.
 173  */
 174 void AudioBlockCopyChannelWithScale(const float aInput[WEBAUDIO_BLOCK_SIZE],
 175                                     const float aScale[WEBAUDIO_BLOCK_SIZE],
 176                                     float aOutput[WEBAUDIO_BLOCK_SIZE]);
 177
 178 /**
 179  * Vector complex multiplication on arbitrary sized buffers.
 180  */
 181 void BufferComplexMultiply(const float* aInput, const float* aScale,
 182                            float* aOutput, uint32_t aSize);
 183
 184 /**
 185  * Vector maximum element magnitude ( max(abs(aInput)) ).
 186  */
 187 float AudioBufferPeakValue(const float* aInput, uint32_t aSize);
 188
 189 /**
 190  * In place gain. aScale == 1.0f should be optimized.
 191  */
 192 void AudioBlockInPlaceScale(float aBlock[WEBAUDIO_BLOCK_SIZE], float aScale);
 193
 194 /**
 195  * In place gain. aScale == 1.0f should be optimized.
 196  */
 197 void AudioBufferInPlaceScale(float* aBlock, float aScale, uint32_t aSize);
 198
 199 /**
 200  * a-rate in place gain.
 201  */
 202 void AudioBlockInPlaceScale(float aBlock[WEBAUDIO_BLOCK_SIZE],
 203                             float aScale[WEBAUDIO_BLOCK_SIZE]);
 204 /**
 205  * a-rate in place gain.
 206  */
 207 void AudioBufferInPlaceScale(float* aBlock, float* aScale, uint32_t aSize);
 208
 209 /**
 210  * Upmix a mono input to a stereo output, scaling the two output channels by two
 211  * different gain value.
 212  * This algorithm is specified in the WebAudio spec.
 213  */
 214 void AudioBlockPanMonoToStereo(const float aInput[WEBAUDIO_BLOCK_SIZE],
 215                                float aGainL, float aGainR,
 216                                float aOutputL[WEBAUDIO_BLOCK_SIZE],
 217                                float aOutputR[WEBAUDIO_BLOCK_SIZE]);
 218
 219 void AudioBlockPanMonoToStereo(const float aInput[WEBAUDIO_BLOCK_SIZE],
 220                                float aGainL[WEBAUDIO_BLOCK_SIZE],
 221                                float aGainR[WEBAUDIO_BLOCK_SIZE],
 222                                float aOutputL[WEBAUDIO_BLOCK_SIZE],
 223                                float aOutputR[WEBAUDIO_BLOCK_SIZE]);
 224 /**
 225  * Pan a stereo source according to right and left gain, and the position
 226  * (whether the listener is on the left of the source or not).
 227  * This algorithm is specified in the WebAudio spec.
 228  */
 229 void AudioBlockPanStereoToStereo(const float aInputL[WEBAUDIO_BLOCK_SIZE],
 230                                  const float aInputR[WEBAUDIO_BLOCK_SIZE],
 231                                  float aGainL, float aGainR, bool aIsOnTheLeft,
 232                                  float aOutputL[WEBAUDIO_BLOCK_SIZE],
 233                                  float aOutputR[WEBAUDIO_BLOCK_SIZE]);
 234 void AudioBlockPanStereoToStereo(const float aInputL[WEBAUDIO_BLOCK_SIZE],
 235                                  const float aInputR[WEBAUDIO_BLOCK_SIZE],
 236                                  const float aGainL[WEBAUDIO_BLOCK_SIZE],
 237                                  const float aGainR[WEBAUDIO_BLOCK_SIZE],
 238                                  const bool aIsOnTheLeft[WEBAUDIO_BLOCK_SIZE],
 239                                  float aOutputL[WEBAUDIO_BLOCK_SIZE],
 240                                  float aOutputR[WEBAUDIO_BLOCK_SIZE]);
 241
 242 /**
 243  * Replace NaN by zeros in aSamples.
 244  */
 245 void NaNToZeroInPlace(float* aSamples, size_t aCount);
 246
 247 /**
 248  * Return the sum of squares of all of the samples in the input.
 249  */
 250 float AudioBufferSumOfSquares(const float* aInput, uint32_t aLength);
 251
 252 /**
 253  * All methods of this class and its subclasses are called on the
 254  * MediaTrackGraph thread.
 255  */
 256 class AudioNodeEngine {
 257  public:
 258   // This should be compatible with AudioNodeTrack::OutputChunks.
 259   typedef AutoTArray<AudioBlock, 1> OutputChunks;
 260
 261   explicit AudioNodeEngine(dom::AudioNode* aNode);
 262
 263   virtual ~AudioNodeEngine() {
 264     MOZ_ASSERT(!mNode, "The node reference must be already cleared");
 265     MOZ_COUNT_DTOR(AudioNodeEngine);
 266   }
 267
 268   virtual dom::DelayNodeEngine* AsDelayNodeEngine() { return nullptr; }
 269
 270   virtual void SetTrackTimeParameter(uint32_t aIndex, TrackTime aParam) {
 271     NS_ERROR("Invalid SetTrackTimeParameter index");
 272   }
 273   virtual void SetDoubleParameter(uint32_t aIndex, double aParam) {
 274     NS_ERROR("Invalid SetDoubleParameter index");
 275   }
 276   virtual void SetInt32Parameter(uint32_t aIndex, int32_t aParam) {
 277     NS_ERROR("Invalid SetInt32Parameter index");
 278   }
 279   virtual void RecvTimelineEvent(uint32_t aIndex,
 280                                  dom::AudioTimelineEvent& aValue) {
 281     NS_ERROR("Invalid RecvTimelineEvent index");
 282   }
 283   virtual void SetBuffer(AudioChunk&& aBuffer) {
 284     NS_ERROR("SetBuffer called on engine that doesn't support it");
 285   }
 286   // This consumes the contents of aData.  aData will be emptied after this
 287   // returns.
 288   virtual void SetRawArrayData(nsTArray<float>&& aData) {
 289     NS_ERROR("SetRawArrayData called on an engine that doesn't support it");
 290   }
 291
 292   virtual void SetReverb(WebCore::Reverb* aBuffer,
 293                          uint32_t aImpulseChannelCount) {
 294     NS_ERROR("SetReverb called on engine that doesn't support it");
 295   }
 296
 297   /**
 298    * Produce the next block of audio samples, given input samples aInput
 299    * (the mixed data for input 0).
 300    * aInput is guaranteed to have float sample format (if it has samples at all)
 301    * and to have been resampled to the sampling rate for the track, and to have
 302    * exactly WEBAUDIO_BLOCK_SIZE samples.
 303    * *aFinished is set to false by the caller. The callee must not set this to
 304    * true unless silent output is produced. If set to true, we'll finish the
 305    * track, consider this input inactive on any downstream nodes, and not
 306    * call this again.
 307    */
 308   virtual void ProcessBlock(AudioNodeTrack* aTrack, GraphTime aFrom,
 309                             const AudioBlock& aInput, AudioBlock* aOutput,
 310                             bool* aFinished);
 311   /**
 312    * Produce the next block of audio samples, before input is provided.
 313    * ProcessBlock() will be called later, and it then should not change
 314    * aOutput.  This is used only for DelayNodeEngine in a feedback loop.
 315    */
 316   virtual void ProduceBlockBeforeInput(AudioNodeTrack* aTrack, GraphTime aFrom,
 317                                        AudioBlock* aOutput) {
 318     MOZ_ASSERT_UNREACHABLE("ProduceBlockBeforeInput called on wrong engine");
 319   }
 320
 321   /**
 322    * Produce the next block of audio samples, given input samples in the aInput
 323    * array.  There is one input sample per port in aInput, in order.
 324    * This is the multi-input/output version of ProcessBlock.  Only one kind
 325    * of ProcessBlock is called on each node.  ProcessBlocksOnPorts() is called
 326    * instead of ProcessBlock() if either the number of inputs or the number of
 327    * outputs is greater than 1.
 328    *
 329    * The numbers of AudioBlocks in aInput and aOutput are always guaranteed to
 330    * match the numbers of inputs and outputs for the node.
 331    */
 332   virtual void ProcessBlocksOnPorts(AudioNodeTrack* aTrack, GraphTime aFrom,
 333                                     Span<const AudioBlock> aInput,
 334                                     Span<AudioBlock> aOutput, bool* aFinished);
 335
 336   // IsActive() returns true if the engine needs to continue processing an
 337   // unfinished track even when it has silent or no input connections.  This
 338   // includes tail-times and when sources have been scheduled to start.  If
 339   // returning false, then the track can be suspended.
 340   virtual bool IsActive() const { return false; }
 341
 342   // Called on graph thread when the engine will not be used again.
 343   virtual void OnGraphThreadDone() {}
 344
 345   bool HasNode() const {
 346     MOZ_ASSERT(NS_IsMainThread());
 347     return !!mNode;
 348   }
 349
 350   dom::AudioNode* NodeMainThread() const {
 351     MOZ_ASSERT(NS_IsMainThread());
 352     return mNode;
 353   }
 354
 355   void ClearNode() {
 356     MOZ_ASSERT(NS_IsMainThread());
 357     MOZ_ASSERT(mNode != nullptr);
 358     mNode = nullptr;
 359   }
 360
 361   uint16_t InputCount() const { return mInputCount; }
 362   uint16_t OutputCount() const { return mOutputCount; }
 363
 364   virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const {
 365     // NB: |mNode| is tracked separately so it is excluded here.
 366     return 0;
 367   }
 368
 369   virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const {
 370     return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
 371   }
 372
 373   void SizeOfIncludingThis(MallocSizeOf aMallocSizeOf,
 374                            AudioNodeSizes& aUsage) const {
 375     aUsage.mEngine = SizeOfIncludingThis(aMallocSizeOf);
 376     aUsage.mNodeType = mNodeType;
 377   }
 378
 379  private:
 380   // This is cleared from AudioNode::DestroyMediaTrack()
 381   dom::AudioNode* MOZ_NON_OWNING_REF mNode;  // main thread only
 382   const char* const mNodeType;
 383   const uint16_t mInputCount;
 384   const uint16_t mOutputCount;
 385
 386  protected:
 387   const RefPtr<AbstractThread> mAbstractMainThread;
 388 };
 389
 390 }  // namespace mozilla
 391
 392 #endif /* MOZILLA_AUDIONODEENGINE_H_ */