From c8964509ccb0ef139cff842be073a666af8cc171 Mon Sep 17 00:00:00 2001 From: Paul Adenot Date: Thu, 21 Mar 2013 17:45:53 +0100 Subject: [PATCH] Bug 849918 - Initial support for PannerNode's 3D positional audio (equalpower panning model). r=ehsan,roc --- content/media/AudioNodeEngine.cpp | 44 ++++++ content/media/AudioNodeEngine.h | 29 ++++ content/media/webaudio/AudioListener.h | 1 + content/media/webaudio/Makefile.in | 1 + content/media/webaudio/PannerNode.cpp | 237 ++++++++++++++++++++++++++++++++- content/media/webaudio/ThreeDPoint.cpp | 40 ++++++ content/media/webaudio/ThreeDPoint.h | 42 ++++++ 7 files changed, 392 insertions(+), 2 deletions(-) create mode 100644 content/media/webaudio/ThreeDPoint.cpp diff --git a/content/media/AudioNodeEngine.cpp b/content/media/AudioNodeEngine.cpp index 2a2b1d176abb..844ca2f1cf60 100644 --- a/content/media/AudioNodeEngine.cpp +++ b/content/media/AudioNodeEngine.cpp @@ -78,4 +78,48 @@ AudioBlockCopyChannelWithScale(const float aInput[WEBAUDIO_BLOCK_SIZE], } } +void +AudioBlockInPlaceScale(float aBlock[WEBAUDIO_BLOCK_SIZE], + uint32_t aChannelCount, + float aScale) +{ + if (aScale == 1.0f) { + return; + } + for (uint32_t i = 0; i < WEBAUDIO_BLOCK_SIZE * aChannelCount; ++i) { + *aBlock++ *= aScale; + } +} + +void +AudioBlockPanMonoToStereo(const float aInput[WEBAUDIO_BLOCK_SIZE], + float aGainL, float aGainR, + float aOutputL[WEBAUDIO_BLOCK_SIZE], + float aOutputR[WEBAUDIO_BLOCK_SIZE]) +{ + AudioBlockCopyChannelWithScale(aInput, aGainL, aOutputL); + AudioBlockCopyChannelWithScale(aInput, aGainR, aOutputR); +} + +void +AudioBlockPanStereoToStereo(const float aInputL[WEBAUDIO_BLOCK_SIZE], + const float aInputR[WEBAUDIO_BLOCK_SIZE], + float aGainL, float aGainR, bool aIsOnTheLeft, + float aOutputL[WEBAUDIO_BLOCK_SIZE], + float aOutputR[WEBAUDIO_BLOCK_SIZE]) +{ + uint32_t i; + + if (aIsOnTheLeft) { + for (i = 0; i < WEBAUDIO_BLOCK_SIZE; ++i) { + *aOutputL++ = *aInputL++ + *aInputR * aGainL; + *aOutputR++ = *aInputR++ * aGainR; + } + } else { + for (i = 0; i < WEBAUDIO_BLOCK_SIZE; ++i) { + *aOutputL++ = *aInputL * aGainL; + *aOutputR++ = *aInputR++ + *aInputL++ * aGainR; + } + } +} } diff --git a/content/media/AudioNodeEngine.h b/content/media/AudioNodeEngine.h index 23beb73bf3b1..8219f464f61f 100644 --- a/content/media/AudioNodeEngine.h +++ b/content/media/AudioNodeEngine.h @@ -112,6 +112,35 @@ void AudioBlockCopyChannelWithScale(const float aInput[WEBAUDIO_BLOCK_SIZE], float aOutput[WEBAUDIO_BLOCK_SIZE]); /** + * In place gain. aScale == 1.0f should be optimized. + */ +void AudioBlockInPlaceScale(float aBlock[WEBAUDIO_BLOCK_SIZE], + uint32_t aChannelCount, + float aScale); + +/** + * Upmix a mono input to a stereo output, scaling the two output channels by two + * different gain value. + * This algorithm is specified in the WebAudio spec. + */ +void +AudioBlockPanMonoToStereo(const float aInput[WEBAUDIO_BLOCK_SIZE], + float aGainL, float aGainR, + float aOutputL[WEBAUDIO_BLOCK_SIZE], + float aOutputR[WEBAUDIO_BLOCK_SIZE]); +/** + * Pan a stereo source according to right and left gain, and the position + * (whether the listener is on the left of the source or not). + * This algorithm is specified in the WebAudio spec. + */ +void +AudioBlockPanStereoToStereo(const float aInputL[WEBAUDIO_BLOCK_SIZE], + const float aInputR[WEBAUDIO_BLOCK_SIZE], + float aGainL, float aGainR, bool aIsOnTheLeft, + float aOutputL[WEBAUDIO_BLOCK_SIZE], + float aOutputR[WEBAUDIO_BLOCK_SIZE]); + +/** * All methods of this class and its subclasses are called on the * MediaStreamGraph thread. */ diff --git a/content/media/webaudio/AudioListener.h b/content/media/webaudio/AudioListener.h index 634bf2b113c7..cf0c1323cc3c 100644 --- a/content/media/webaudio/AudioListener.h +++ b/content/media/webaudio/AudioListener.h @@ -121,6 +121,7 @@ private: void SendThreeDPointParameterToStream(uint32_t aIndex, const ThreeDPoint& aValue); private: + friend class PannerNode; nsRefPtr mContext; ThreeDPoint mPosition; ThreeDPoint mOrientation; diff --git a/content/media/webaudio/Makefile.in b/content/media/webaudio/Makefile.in index 781fb6c03e01..c799e1882c88 100644 --- a/content/media/webaudio/Makefile.in +++ b/content/media/webaudio/Makefile.in @@ -31,6 +31,7 @@ CPPSRCS := \ GainNode.cpp \ MediaBufferDecoder.cpp \ PannerNode.cpp \ + ThreeDPoint.cpp \ $(NULL) EXPORTS_NAMESPACES := mozilla/dom diff --git a/content/media/webaudio/PannerNode.cpp b/content/media/webaudio/PannerNode.cpp index fa6efb28e7a1..469b10d2b564 100644 --- a/content/media/webaudio/PannerNode.cpp +++ b/content/media/webaudio/PannerNode.cpp @@ -12,13 +12,17 @@ namespace mozilla { namespace dom { +using namespace std; + class PannerNodeEngine : public AudioNodeEngine { public: PannerNodeEngine() // Please keep these default values consistent with PannerNode::PannerNode below. : mPanningModel(PanningModelTypeValues::HRTF) + , mPanningModelFunction(&PannerNodeEngine::HRTFPanningFunction) , mDistanceModel(DistanceModelTypeValues::Inverse) + , mDistanceModelFunction(&PannerNodeEngine::InverseGainFunction) , mPosition() , mOrientation(1., 0., 0.) , mVelocity() @@ -40,9 +44,31 @@ public: switch (aIndex) { case PannerNode::PANNING_MODEL: mPanningModel = PanningModelType(aParam); + switch (mPanningModel) { + case PanningModelTypeValues::Equalpower: + mPanningModelFunction = &PannerNodeEngine::EqualPowerPanningFunction; + break; + case PanningModelTypeValues::HRTF: + mPanningModelFunction = &PannerNodeEngine::HRTFPanningFunction; + break; + case PanningModelTypeValues::Soundfield: + mPanningModelFunction = &PannerNodeEngine::SoundfieldPanningFunction; + break; + } break; case PannerNode::DISTANCE_MODEL: mDistanceModel = DistanceModelType(aParam); + switch (mDistanceModel) { + case DistanceModelTypeValues::Inverse: + mDistanceModelFunction = &PannerNodeEngine::InverseGainFunction; + break; + case DistanceModelTypeValues::Linear: + mDistanceModelFunction = &PannerNodeEngine::LinearGainFunction; + break; + case DistanceModelTypeValues::Exponential: + mDistanceModelFunction = &PannerNodeEngine::ExponentialGainFunction; + break; + } break; default: NS_ERROR("Bad PannerNodeEngine Int32Parameter"); @@ -83,12 +109,35 @@ public: AudioChunk* aOutput, bool *aFinished) MOZ_OVERRIDE { - // TODO: actually do 3D positioning computations here - *aOutput = aInput; + if (aInput.IsNull()) { + *aOutput = aInput; + return; + } + (this->*mPanningModelFunction)(aInput, aOutput); } + void ComputeAzimuthAndElevation(float& aAzimuth, float& aElevation); + void DistanceGain(AudioChunk* aChunk, float aGain); + + void GainMonoToStereo(const AudioChunk& aInput, AudioChunk* aOutput, + float aGainL, float aGainR); + void GainStereoToStereo(const AudioChunk& aInput, AudioChunk* aOutput, + float aGainL, float aGainR, double aAzimuth); + + void EqualPowerPanningFunction(const AudioChunk& aInput, AudioChunk* aOutput); + void HRTFPanningFunction(const AudioChunk& aInput, AudioChunk* aOutput); + void SoundfieldPanningFunction(const AudioChunk& aInput, AudioChunk* aOutput); + + float LinearGainFunction(float aDistance); + float InverseGainFunction(float aDistance); + float ExponentialGainFunction(float aDistance); + PanningModelType mPanningModel; + typedef void (PannerNodeEngine::*PanningModelFunction)(const AudioChunk& aInput, AudioChunk* aOutput); + PanningModelFunction mPanningModelFunction; DistanceModelType mDistanceModel; + typedef float (PannerNodeEngine::*DistanceModelFunction)(float aDistance); + DistanceModelFunction mDistanceModelFunction; ThreeDPoint mPosition; ThreeDPoint mOrientation; ThreeDPoint mVelocity; @@ -138,6 +187,190 @@ PannerNode::WrapObject(JSContext* aCx, JSObject* aScope) return PannerNodeBinding::Wrap(aCx, aScope, this); } +// Those three functions are described in the spec. +float +PannerNodeEngine::LinearGainFunction(float aDistance) +{ + return 1 - mRolloffFactor * (aDistance - mRefDistance) / (mMaxDistance - mRefDistance); +} + +float +PannerNodeEngine::InverseGainFunction(float aDistance) +{ + return mRefDistance / (mRefDistance + mRolloffFactor * (aDistance - mRefDistance)); +} + +float +PannerNodeEngine::ExponentialGainFunction(float aDistance) +{ + return pow(aDistance / mRefDistance, -mRolloffFactor); +} + +void +PannerNodeEngine::SoundfieldPanningFunction(const AudioChunk& aInput, + AudioChunk* aOutput) +{ + // not implemented: noop + *aOutput = aInput; +} + +void +PannerNodeEngine::HRTFPanningFunction(const AudioChunk& aInput, + AudioChunk* aOutput) +{ + // not implemented: noop + *aOutput = aInput; +} + +void +PannerNodeEngine::EqualPowerPanningFunction(const AudioChunk& aInput, + AudioChunk* aOutput) +{ + float azimuth, elevation, gainL, gainR, normalizedAzimuth, distance, distanceGain; + int inputChannels = aInput.mChannelData.Length(); + ThreeDPoint distanceVec; + + // If both the listener are in the same spot, and no cone gain is specified, + // this node is noop. + if (mListenerPosition == mPosition && + mConeInnerAngle == 360 && + mConeOuterAngle == 360) { + *aOutput = aInput; + } + + // The output of this node is always stereo, no matter what the inputs are. + AllocateAudioBlock(2, aOutput); + + ComputeAzimuthAndElevation(azimuth, elevation); + + // The following algorithm is described in the spec. + // Clamp azimuth in the [-90, 90] range. + azimuth = min(180.f, max(-180.f, azimuth)); + + // Wrap around + if (azimuth < -90.f) { + azimuth = -180.f - azimuth; + } else if (azimuth > 90) { + azimuth = 180.f - azimuth; + } + + // Normalize the value in the [0, 1] range. + if (inputChannels == 1) { + normalizedAzimuth = (azimuth + 90.f) / 180.f; + } else { + if (azimuth <= 0) { + normalizedAzimuth = (azimuth + 90.f) / 90.f; + } else { + normalizedAzimuth = azimuth / 90.f; + } + } + + // Compute how much the distance contributes to the gain reduction. + distanceVec = mPosition - mListenerPosition; + distance = sqrt(distanceVec.DotProduct(distanceVec)); + distanceGain = (this->*mDistanceModelFunction)(distance); + + // Actually compute the left and right gain. + gainL = cos(0.5 * M_PI * normalizedAzimuth); + gainR = sin(0.5 * M_PI * normalizedAzimuth); + + // Compute the output. + if (inputChannels == 1) { + GainMonoToStereo(aInput, aOutput, gainL, gainR); + } else { + GainStereoToStereo(aInput, aOutput, gainL, gainR, azimuth); + } + + DistanceGain(aOutput, distanceGain); +} + +void +PannerNodeEngine::GainMonoToStereo(const AudioChunk& aInput, AudioChunk* aOutput, + float aGainL, float aGainR) +{ + float* outputL = static_cast(const_cast(aOutput->mChannelData[0])); + float* outputR = static_cast(const_cast(aOutput->mChannelData[1])); + const float* input = static_cast(const_cast(aInput.mChannelData[0])); + + AudioBlockPanMonoToStereo(input, aGainL, aGainR, outputL, outputR); +} + +void +PannerNodeEngine::GainStereoToStereo(const AudioChunk& aInput, AudioChunk* aOutput, + float aGainL, float aGainR, double aAzimuth) +{ + float* outputL = static_cast(const_cast(aOutput->mChannelData[0])); + float* outputR = static_cast(const_cast(aOutput->mChannelData[1])); + const float* inputL = static_cast(const_cast(aInput.mChannelData[0])); + const float* inputR = static_cast(const_cast(aInput.mChannelData[1])); + + AudioBlockPanStereoToStereo(inputL, inputR, aGainL, aGainR, aAzimuth <= 0, outputL, outputR); +} + +void +PannerNodeEngine::DistanceGain(AudioChunk* aChunk, float aGain) +{ + float* samples = static_cast(const_cast(*aChunk->mChannelData.Elements())); + uint32_t channelCount = aChunk->mChannelData.Length(); + + AudioBlockInPlaceScale(samples, channelCount, aGain); +} + +// This algorithm is specicied in the webaudio spec. +void +PannerNodeEngine::ComputeAzimuthAndElevation(float& aAzimuth, float& aElevation) +{ + ThreeDPoint sourceListener = mPosition - mListenerPosition; + + if (sourceListener.IsZero()) { + aAzimuth = 0.0; + aElevation = 0.0; + return; + } + + sourceListener.Normalize(); + + // Project the source-listener vector on the x-z plane. + ThreeDPoint& listenerFront = mListenerOrientation; + ThreeDPoint listenerRightNorm = listenerFront.CrossProduct(mListenerUpVector); + listenerRightNorm.Normalize(); + + ThreeDPoint listenerFrontNorm(listenerFront); + listenerFrontNorm.Normalize(); + + ThreeDPoint up = listenerRightNorm.CrossProduct(listenerFrontNorm); + + double upProjection = sourceListener.DotProduct(up); + + ThreeDPoint projectedSource = sourceListener - up * upProjection; + projectedSource.Normalize(); + + // Actually compute the angle, and convert to degrees + double projection = projectedSource.DotProduct(listenerRightNorm); + aAzimuth = 180 * acos(projection) / M_PI; + + // Compute whether the source is in front or behind the listener. + double frontBack = projectedSource.DotProduct(listenerFrontNorm); + if (frontBack < 0) { + aAzimuth = 360 - aAzimuth; + } + // Rotate the azimuth so it is relative to the listener front vector instead + // of the right vector. + if ((aAzimuth >= 0) && (aAzimuth <= 270)) { + aAzimuth = 90 - aAzimuth; + } else { + aAzimuth = 450 - aAzimuth; + } + + aElevation = 90 - 180 * acos(sourceListener.DotProduct(up)) / M_PI; + + if (aElevation > 90) { + aElevation = 180 - aElevation; + } else if (aElevation < -90) { + aElevation = -180 - aElevation; + } +} + } } diff --git a/content/media/webaudio/ThreeDPoint.cpp b/content/media/webaudio/ThreeDPoint.cpp new file mode 100644 index 000000000000..f4187e7b37f9 --- /dev/null +++ b/content/media/webaudio/ThreeDPoint.cpp @@ -0,0 +1,40 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Other similar methods can be added if needed. + */ + +#include "ThreeDPoint.h" + +namespace mozilla { + +namespace dom { + +ThreeDPoint operator-(const ThreeDPoint& lhs, const ThreeDPoint& rhs) +{ + return ThreeDPoint(lhs.x - rhs.x, lhs.y - rhs.y, lhs.z - rhs.z); +} + +ThreeDPoint operator*(const ThreeDPoint& lhs, const ThreeDPoint& rhs) +{ + return ThreeDPoint(lhs.x * rhs.x, lhs.y * rhs.y, lhs.z * rhs.z); +} + +ThreeDPoint operator*(const ThreeDPoint& lhs, const double rhs) +{ + return ThreeDPoint(lhs.x * rhs, lhs.y * rhs, lhs.z * rhs); +} + +bool operator==(const ThreeDPoint& lhs, const ThreeDPoint& rhs) +{ + return lhs.x == rhs.x && + lhs.y == rhs.y && + lhs.z == rhs.z; +} + +} +} diff --git a/content/media/webaudio/ThreeDPoint.h b/content/media/webaudio/ThreeDPoint.h index 8a3e460a01da..2c7fb88555d2 100644 --- a/content/media/webaudio/ThreeDPoint.h +++ b/content/media/webaudio/ThreeDPoint.h @@ -7,6 +7,9 @@ #ifndef ThreeDPoint_h_ #define ThreeDPoint_h_ +#include +#include + namespace mozilla { namespace dom { @@ -25,9 +28,48 @@ struct ThreeDPoint { { } + double Magnitude() const + { + return sqrt(x * x + y * y + z * z); + } + + void Normalize() + { + double invDistance = 1 / Magnitude(); + x *= invDistance; + y *= invDistance; + z *= invDistance; + } + + ThreeDPoint CrossProduct(const ThreeDPoint& rhs) const + { + return ThreeDPoint(y * rhs.z - z * rhs.y, + z * rhs.x - x * rhs.z, + x * rhs.y - y * rhs.x); + } + + double DotProduct(const ThreeDPoint& rhs) + { + return x * rhs.x + y * rhs.y + z * rhs.z; + } + + double Distance(const ThreeDPoint& rhs) + { + return sqrt(hypot(rhs.x, x) + hypot(rhs.y, y) + hypot(rhs.z, z)); + } + + bool IsZero() const + { + return x == 0 && y == 0 && z == 0; + } double x, y, z; }; +ThreeDPoint operator-(const ThreeDPoint& lhs, const ThreeDPoint& rhs); +ThreeDPoint operator*(const ThreeDPoint& lhs, const ThreeDPoint& rhs); +ThreeDPoint operator*(const ThreeDPoint& lhs, const double rhs); +bool operator==(const ThreeDPoint& lhs, const ThreeDPoint& rhs); + } } -- 2.11.4.GIT