From 27695e2b24dea23d0db451fb0f9ae1c1af6f416e Mon Sep 17 00:00:00 2001 From: Chris Robinson Date: Sun, 12 Feb 2017 21:03:30 -0800 Subject: [PATCH] Add NEON-enhanced resamplers --- Alc/mixer.c | 16 ++++ Alc/mixer_defs.h | 22 ++++- Alc/mixer_neon.c | 260 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 294 insertions(+), 4 deletions(-) diff --git a/Alc/mixer.c b/Alc/mixer.c index 0ee1edd6..5442954e 100644 --- a/Alc/mixer.c +++ b/Alc/mixer.c @@ -113,6 +113,10 @@ static inline ResamplerFunc SelectResampler(enum Resampler resampler) case PointResampler: return Resample_point32_C; case LinearResampler: +#ifdef HAVE_NEON + if((CPUCapFlags&CPU_CAP_NEON)) + return Resample_lerp32_Neon; +#endif #ifdef HAVE_SSE4_1 if((CPUCapFlags&CPU_CAP_SSE4_1)) return Resample_lerp32_SSE41; @@ -123,6 +127,10 @@ static inline ResamplerFunc SelectResampler(enum Resampler resampler) #endif return Resample_lerp32_C; case FIR4Resampler: +#ifdef HAVE_NEON + if((CPUCapFlags&CPU_CAP_NEON)) + return Resample_fir4_32_Neon; +#endif #ifdef HAVE_SSE4_1 if((CPUCapFlags&CPU_CAP_SSE4_1)) return Resample_fir4_32_SSE41; @@ -133,6 +141,10 @@ static inline ResamplerFunc SelectResampler(enum Resampler resampler) #endif return Resample_fir4_32_C; case FIR8Resampler: +#ifdef HAVE_NEON + if((CPUCapFlags&CPU_CAP_NEON)) + return Resample_fir8_32_Neon; +#endif #ifdef HAVE_SSE4_1 if((CPUCapFlags&CPU_CAP_SSE4_1)) return Resample_fir8_32_SSE41; @@ -143,6 +155,10 @@ static inline ResamplerFunc SelectResampler(enum Resampler resampler) #endif return Resample_fir8_32_C; case BSincResampler: +#ifdef HAVE_NEON + if((CPUCapFlags&CPU_CAP_NEON)) + return Resample_bsinc32_Neon; +#endif #ifdef HAVE_SSE if((CPUCapFlags&CPU_CAP_SSE)) return Resample_bsinc32_SSE; diff --git a/Alc/mixer_defs.h b/Alc/mixer_defs.h index 64da7680..4bafc839 100644 --- a/Alc/mixer_defs.h +++ b/Alc/mixer_defs.h @@ -67,10 +67,6 @@ inline void InitiatePositionArrays(ALuint frac, ALint increment, ALuint *restric } } -const ALfloat *Resample_bsinc32_SSE(const BsincState *state, const ALfloat *restrict src, - ALuint frac, ALint increment, ALfloat *restrict dst, - ALsizei dstlen); - const ALfloat *Resample_lerp32_SSE2(const BsincState *state, const ALfloat *restrict src, ALuint frac, ALint increment, ALfloat *restrict dst, ALsizei numsamples); @@ -92,6 +88,10 @@ const ALfloat *Resample_fir8_32_SSE41(const BsincState *state, const ALfloat *re ALuint frac, ALint increment, ALfloat *restrict dst, ALsizei numsamples); +const ALfloat *Resample_bsinc32_SSE(const BsincState *state, const ALfloat *restrict src, + ALuint frac, ALint increment, ALfloat *restrict dst, + ALsizei dstlen); + /* Neon mixers */ void MixHrtf_Neon(ALfloat *restrict LeftOut, ALfloat *restrict RightOut, const ALfloat *data, ALsizei Counter, ALsizei Offset, ALsizei OutPos, @@ -108,4 +108,18 @@ void MixRow_Neon(ALfloat *OutBuffer, const ALfloat *Gains, const ALfloat (*restrict data)[BUFFERSIZE], ALsizei InChans, ALsizei InPos, ALsizei BufferSize); +/* Neon resamplers */ +const ALfloat *Resample_lerp32_Neon(const BsincState *state, const ALfloat *restrict src, + ALuint frac, ALint increment, ALfloat *restrict dst, + ALsizei numsamples); +const ALfloat *Resample_fir4_32_Neon(const BsincState *state, const ALfloat *restrict src, + ALuint frac, ALint increment, ALfloat *restrict dst, + ALsizei numsamples); +const ALfloat *Resample_fir8_32_Neon(const BsincState *state, const ALfloat *restrict src, + ALuint frac, ALint increment, ALfloat *restrict dst, + ALsizei numsamples); +const ALfloat *Resample_bsinc32_Neon(const BsincState *state, const ALfloat *restrict src, + ALuint frac, ALint increment, ALfloat *restrict dst, + ALsizei dstlen); + #endif /* MIXER_DEFS_H */ diff --git a/Alc/mixer_neon.c b/Alc/mixer_neon.c index 0fbcea67..75ad14eb 100644 --- a/Alc/mixer_neon.c +++ b/Alc/mixer_neon.c @@ -7,6 +7,266 @@ #include "alMain.h" #include "alu.h" #include "hrtf.h" +#include "mixer_defs.h" + + +#ifdef __GNUC__ +#define ASSUME_ALIGNED(ptr, ...) __builtin_assume_aligned((ptr), __VA_ARGS__) +#else +#define ASSUME_ALIGNED(ptr, ...) (ptr) +#endif + +const ALfloat *Resample_lerp32_Neon(const BsincState* UNUSED(state), const ALfloat *restrict src, + ALuint frac, ALint increment, ALfloat *restrict dst, + ALsizei numsamples) +{ + const int32x4_t increment4 = vdupq_n_s32(increment*4); + const float32x4_t fracOne4 = vdupq_n_f32(1.0f/FRACTIONONE); + const uint32x4_t fracMask4 = vdupq_n_u32(FRACTIONMASK); + alignas(16) ALint pos_[4]; + alignas(16) ALuint frac_[4]; + int32x4_t pos4; + uint32x4_t frac4; + ALsizei i; + + InitiatePositionArrays(frac, increment, frac_, pos_, 4); + + frac4 = vld1q_u32(frac_); + pos4 = vld1q_s32(pos_); + + for(i = 0;numsamples-i > 3;i += 4) + { + const float32x4_t val1 = (float32x4_t){src[pos_[0]], src[pos_[1]], src[pos_[2]], src[pos_[3]]}; + const float32x4_t val2 = (float32x4_t){src[pos_[0]+1], src[pos_[1]+1], src[pos_[2]+1], src[pos_[3]+1]}; + + /* val1 + (val2-val1)*mu */ + const float32x4_t r0 = vsubq_f32(val2, val1); + const float32x4_t mu = vmulq_f32(vcvtq_f32_u32(frac4), fracOne4); + const float32x4_t out = vmlaq_f32(val1, mu, r0); + + vst1q_f32(&dst[i], out); + + frac4 = vaddq_u32(frac4, (uint32x4_t)increment4); + pos4 = vaddq_s32(pos4, (int32x4_t)vshrq_n_u32(frac4, FRACTIONBITS)); + frac4 = vandq_u32(frac4, fracMask4); + + vst1q_s32(pos_, pos4); + } + + if(i < numsamples) + { + /* NOTE: These four elements represent the position *after* the last + * four samples, so the lowest element is the next position to + * resample. + */ + ALint pos = pos_[0]; + frac = vgetq_lane_u32(frac4, 0); + do { + dst[i] = lerp(src[pos], src[pos+1], frac * (1.0f/FRACTIONONE)); + + frac += increment; + pos += frac>>FRACTIONBITS; + frac &= FRACTIONMASK; + } while(++i < numsamples); + } + return dst; +} + +const ALfloat *Resample_fir4_32_Neon(const BsincState* UNUSED(state), const ALfloat *restrict src, + ALuint frac, ALint increment, ALfloat *restrict dst, + ALsizei numsamples) +{ + const int32x4_t increment4 = vdupq_n_s32(increment*4); + const uint32x4_t fracMask4 = vdupq_n_u32(FRACTIONMASK); + alignas(16) ALint pos_[4]; + alignas(16) ALuint frac_[4]; + int32x4_t pos4; + uint32x4_t frac4; + ALsizei i; + + InitiatePositionArrays(frac, increment, frac_, pos_, 4); + + frac4 = vld1q_u32(frac_); + pos4 = vld1q_s32(pos_); + + --src; + for(i = 0;numsamples-i > 3;i += 4) + { + const float32x4_t val0 = vld1q_f32(&src[pos_[0]]); + const float32x4_t val1 = vld1q_f32(&src[pos_[1]]); + const float32x4_t val2 = vld1q_f32(&src[pos_[2]]); + const float32x4_t val3 = vld1q_f32(&src[pos_[3]]); + float32x4_t k0 = vld1q_f32(ResampleCoeffs.FIR4[frac_[0]]); + float32x4_t k1 = vld1q_f32(ResampleCoeffs.FIR4[frac_[1]]); + float32x4_t k2 = vld1q_f32(ResampleCoeffs.FIR4[frac_[2]]); + float32x4_t k3 = vld1q_f32(ResampleCoeffs.FIR4[frac_[3]]); + float32x4_t out; + + k0 = vmulq_f32(k0, val0); + k1 = vmulq_f32(k1, val1); + k2 = vmulq_f32(k2, val2); + k3 = vmulq_f32(k3, val3); + k0 = vcombine_f32(vpadd_f32(vget_low_f32(k0), vget_high_f32(k0)), + vpadd_f32(vget_low_f32(k1), vget_high_f32(k1))); + k2 = vcombine_f32(vpadd_f32(vget_low_f32(k2), vget_high_f32(k2)), + vpadd_f32(vget_low_f32(k3), vget_high_f32(k3))); + out = vcombine_f32(vpadd_f32(vget_low_f32(k0), vget_high_f32(k0)), + vpadd_f32(vget_low_f32(k2), vget_high_f32(k2))); + + vst1q_f32(&dst[i], out); + + frac4 = vaddq_u32(frac4, (uint32x4_t)increment4); + pos4 = vaddq_s32(pos4, (int32x4_t)vshrq_n_u32(frac4, FRACTIONBITS)); + frac4 = vandq_u32(frac4, fracMask4); + + vst1q_s32(pos_, pos4); + vst1q_u32(frac_, frac4); + } + + if(i < numsamples) + { + /* NOTE: These four elements represent the position *after* the last + * four samples, so the lowest element is the next position to + * resample. + */ + ALint pos = pos_[0]; + frac = frac_[0]; + do { + dst[i] = resample_fir4(src[pos], src[pos+1], src[pos+2], src[pos+3], frac); + + frac += increment; + pos += frac>>FRACTIONBITS; + frac &= FRACTIONMASK; + } while(++i < numsamples); + } + return dst; +} + +const ALfloat *Resample_fir8_32_Neon(const BsincState* UNUSED(state), const ALfloat *restrict src, + ALuint frac, ALint increment, ALfloat *restrict dst, + ALsizei numsamples) +{ + const int32x4_t increment4 = vdupq_n_s32(increment*4); + const uint32x4_t fracMask4 = vdupq_n_u32(FRACTIONMASK); + alignas(16) ALint pos_[4]; + alignas(16) ALuint frac_[4]; + int32x4_t pos4; + uint32x4_t frac4; + ALsizei i, j; + + InitiatePositionArrays(frac, increment, frac_, pos_, 4); + + frac4 = vld1q_u32(frac_); + pos4 = vld1q_s32(pos_); + + src -= 3; + for(i = 0;numsamples-i > 3;i += 4) + { + float32x4_t out[2]; + for(j = 0;j < 8;j+=4) + { + const float32x4_t val0 = vld1q_f32(&src[pos_[0]+j]); + const float32x4_t val1 = vld1q_f32(&src[pos_[1]+j]); + const float32x4_t val2 = vld1q_f32(&src[pos_[2]+j]); + const float32x4_t val3 = vld1q_f32(&src[pos_[3]+j]); + float32x4_t k0 = vld1q_f32(&ResampleCoeffs.FIR4[frac_[0]][j]); + float32x4_t k1 = vld1q_f32(&ResampleCoeffs.FIR4[frac_[1]][j]); + float32x4_t k2 = vld1q_f32(&ResampleCoeffs.FIR4[frac_[2]][j]); + float32x4_t k3 = vld1q_f32(&ResampleCoeffs.FIR4[frac_[3]][j]); + + k0 = vmulq_f32(k0, val0); + k1 = vmulq_f32(k1, val1); + k2 = vmulq_f32(k2, val2); + k3 = vmulq_f32(k3, val3); + k0 = vcombine_f32(vpadd_f32(vget_low_f32(k0), vget_high_f32(k0)), + vpadd_f32(vget_low_f32(k1), vget_high_f32(k1))); + k2 = vcombine_f32(vpadd_f32(vget_low_f32(k2), vget_high_f32(k2)), + vpadd_f32(vget_low_f32(k3), vget_high_f32(k3))); + out[j>>2] = vcombine_f32(vpadd_f32(vget_low_f32(k0), vget_high_f32(k0)), + vpadd_f32(vget_low_f32(k2), vget_high_f32(k2))); + } + + out[0] = vaddq_f32(out[0], out[1]); + vst1q_f32(&dst[i], out[0]); + + frac4 = vaddq_u32(frac4, (uint32x4_t)increment4); + pos4 = vaddq_s32(pos4, (int32x4_t)vshrq_n_u32(frac4, FRACTIONBITS)); + frac4 = vandq_u32(frac4, fracMask4); + + vst1q_s32(pos_, pos4); + vst1q_u32(frac_, frac4); + } + + if(i < numsamples) + { + /* NOTE: These four elements represent the position *after* the last + * four samples, so the lowest element is the next position to + * resample. + */ + ALint pos = pos_[0]; + frac = frac_[0]; + do { + dst[i] = resample_fir8(src[pos ], src[pos+1], src[pos+2], src[pos+3], + src[pos+4], src[pos+5], src[pos+6], src[pos+7], frac); + + frac += increment; + pos += frac>>FRACTIONBITS; + frac &= FRACTIONMASK; + } while(++i < numsamples); + } + return dst; +} + +const ALfloat *Resample_bsinc32_Neon(const BsincState *state, const ALfloat *restrict src, + ALuint frac, ALint increment, ALfloat *restrict dst, + ALsizei dstlen) +{ + const float32x4_t sf4 = vdupq_n_f32(state->sf); + const ALsizei m = state->m; + const ALfloat *fil, *scd, *phd, *spd; + ALsizei pi, i, j; + float32x4_t r4; + ALfloat pf; + + src += state->l; + for(i = 0;i < dstlen;i++) + { + // Calculate the phase index and factor. +#define FRAC_PHASE_BITDIFF (FRACTIONBITS-BSINC_PHASE_BITS) + pi = frac >> FRAC_PHASE_BITDIFF; + pf = (frac & ((1<coeffs[pi].filter, 16); + scd = ASSUME_ALIGNED(state->coeffs[pi].scDelta, 16); + phd = ASSUME_ALIGNED(state->coeffs[pi].phDelta, 16); + spd = ASSUME_ALIGNED(state->coeffs[pi].spDelta, 16); + + // Apply the scale and phase interpolated filter. + r4 = vdupq_n_f32(0.0f); + { + const float32x4_t pf4 = vdupq_n_f32(pf); + for(j = 0;j < m;j+=4) + { + float32x4_t f4 = vmlaq_f32(vld1q_f32(&fil[j]), sf4, vld1q_f32(&scd[j])); + f4 = vmlaq_f32(f4, + pf4, vmlaq_f32(vld1q_f32(&phd[j]), + sf4, vld1q_f32(&spd[j]) + ) + ); + r4 = vmlaq_f32(r4, f4, vld1q_f32(&src[j])); + } + } + r4 = vaddq_f32(r4, vcombine_f32(vrev64_f32(vget_high_f32(r4)), + vrev64_f32(vget_low_f32(r4)))); + dst[i] = vget_lane_f32(vadd_f32(vget_low_f32(r4), vget_high_f32(r4)), 0); + + frac += increment; + src += frac>>FRACTIONBITS; + frac &= FRACTIONMASK; + } + return dst; +} static inline void ApplyCoeffsStep(ALsizei Offset, ALfloat (*restrict Values)[2], -- 2.11.4.GIT