Alc/mixer_sse.c

   1 #include "config.h"
   2
   3 #ifdef IN_IDE_PARSER
   4 /* KDevelop's parser won't recognize these defines that get added by the -msse
   5  * switch used to compile this source. Without them, xmmintrin.h fails to
   6  * declare anything. */
   7 #define __MMX__
   8 #define __SSE__
   9 #endif
  10 #include <xmmintrin.h>
  11
  12 #include "AL/al.h"
  13 #include "AL/alc.h"
  14 #include "alMain.h"
  15 #include "alu.h"
  16
  17 #include "alSource.h"
  18 #include "alAuxEffectSlot.h"
  19 #include "mixer_defs.h"
  20
  21
  22 static inline void ApplyCoeffsStep(ALuint Offset, ALfloat (*restrict Values)[2],
  23                                    const ALuint IrSize,
  24                                    ALfloat (*restrict Coeffs)[2],
  25                                    const ALfloat (*restrict CoeffStep)[2],
  26                                    ALfloat left, ALfloat right)
  27 {
  28     const __m128 lrlr = _mm_setr_ps(left, right, left, right);
  29     __m128 coeffs, deltas, imp0, imp1;
  30     __m128 vals = _mm_setzero_ps();
  31     ALuint i;
  32
  33     if((Offset&1))
  34     {
  35         const ALuint o0 = Offset&HRIR_MASK;
  36         const ALuint o1 = (Offset+IrSize-1)&HRIR_MASK;
  37
  38         coeffs = _mm_load_ps(&Coeffs[0][0]);
  39         deltas = _mm_load_ps(&CoeffStep[0][0]);
  40         vals = _mm_loadl_pi(vals, (__m64*)&Values[o0][0]);
  41         imp0 = _mm_mul_ps(lrlr, coeffs);
  42         coeffs = _mm_add_ps(coeffs, deltas);
  43         vals = _mm_add_ps(imp0, vals);
  44         _mm_store_ps(&Coeffs[0][0], coeffs);
  45         _mm_storel_pi((__m64*)&Values[o0][0], vals);
  46         for(i = 1;i < IrSize-1;i += 2)
  47         {
  48             const ALuint o2 = (Offset+i)&HRIR_MASK;
  49
  50             coeffs = _mm_load_ps(&Coeffs[i+1][0]);
  51             deltas = _mm_load_ps(&CoeffStep[i+1][0]);
  52             vals = _mm_load_ps(&Values[o2][0]);
  53             imp1 = _mm_mul_ps(lrlr, coeffs);
  54             coeffs = _mm_add_ps(coeffs, deltas);
  55             imp0 = _mm_shuffle_ps(imp0, imp1, _MM_SHUFFLE(1, 0, 3, 2));
  56             vals = _mm_add_ps(imp0, vals);
  57             _mm_store_ps(&Coeffs[i+1][0], coeffs);
  58             _mm_store_ps(&Values[o2][0], vals);
  59             imp0 = imp1;
  60         }
  61         vals = _mm_loadl_pi(vals, (__m64*)&Values[o1][0]);
  62         imp0 = _mm_movehl_ps(imp0, imp0);
  63         vals = _mm_add_ps(imp0, vals);
  64         _mm_storel_pi((__m64*)&Values[o1][0], vals);
  65     }
  66     else
  67     {
  68         for(i = 0;i < IrSize;i += 2)
  69         {
  70             const ALuint o = (Offset + i)&HRIR_MASK;
  71
  72             coeffs = _mm_load_ps(&Coeffs[i][0]);
  73             deltas = _mm_load_ps(&CoeffStep[i][0]);
  74             vals = _mm_load_ps(&Values[o][0]);
  75             imp0 = _mm_mul_ps(lrlr, coeffs);
  76             coeffs = _mm_add_ps(coeffs, deltas);
  77             vals = _mm_add_ps(imp0, vals);
  78             _mm_store_ps(&Coeffs[i][0], coeffs);
  79             _mm_store_ps(&Values[o][0], vals);
  80         }
  81     }
  82 }
  83
  84 static inline void ApplyCoeffs(ALuint Offset, ALfloat (*restrict Values)[2],
  85                                const ALuint IrSize,
  86                                ALfloat (*restrict Coeffs)[2],
  87                                ALfloat left, ALfloat right)
  88 {
  89     const __m128 lrlr = _mm_setr_ps(left, right, left, right);
  90     __m128 vals = _mm_setzero_ps();
  91     __m128 coeffs;
  92     ALuint i;
  93
  94     if((Offset&1))
  95     {
  96         const ALuint o0 = Offset&HRIR_MASK;
  97         const ALuint o1 = (Offset+IrSize-1)&HRIR_MASK;
  98         __m128 imp0, imp1;
  99
 100         coeffs = _mm_load_ps(&Coeffs[0][0]);
 101         vals = _mm_loadl_pi(vals, (__m64*)&Values[o0][0]);
 102         imp0 = _mm_mul_ps(lrlr, coeffs);
 103         vals = _mm_add_ps(imp0, vals);
 104         _mm_storel_pi((__m64*)&Values[o0][0], vals);
 105         for(i = 1;i < IrSize-1;i += 2)
 106         {
 107             const ALuint o2 = (Offset+i)&HRIR_MASK;
 108
 109             coeffs = _mm_load_ps(&Coeffs[i+1][0]);
 110             vals = _mm_load_ps(&Values[o2][0]);
 111             imp1 = _mm_mul_ps(lrlr, coeffs);
 112             imp0 = _mm_shuffle_ps(imp0, imp1, _MM_SHUFFLE(1, 0, 3, 2));
 113             vals = _mm_add_ps(imp0, vals);
 114             _mm_store_ps(&Values[o2][0], vals);
 115             imp0 = imp1;
 116         }
 117         vals = _mm_loadl_pi(vals, (__m64*)&Values[o1][0]);
 118         imp0 = _mm_movehl_ps(imp0, imp0);
 119         vals = _mm_add_ps(imp0, vals);
 120         _mm_storel_pi((__m64*)&Values[o1][0], vals);
 121     }
 122     else
 123     {
 124         for(i = 0;i < IrSize;i += 2)
 125         {
 126             const ALuint o = (Offset + i)&HRIR_MASK;
 127
 128             coeffs = _mm_load_ps(&Coeffs[i][0]);
 129             vals = _mm_load_ps(&Values[o][0]);
 130             vals = _mm_add_ps(vals, _mm_mul_ps(lrlr, coeffs));
 131             _mm_store_ps(&Values[o][0], vals);
 132         }
 133     }
 134 }
 135
 136 #define SUFFIX SSE
 137 #include "mixer_inc.c"
 138 #undef SUFFIX
 139
 140
 141 void MixDirect_SSE(ALfloat (*restrict OutBuffer)[BUFFERSIZE], const ALfloat *data,
 142                    MixGains *Gains, ALuint Counter, ALuint OutPos, ALuint BufferSize)
 143 {
 144     ALfloat DrySend, Step;
 145     __m128 gain, step;
 146     ALuint c;
 147
 148     for(c = 0;c < MaxChannels;c++)
 149     {
 150         ALuint pos = 0;
 151         DrySend = Gains->Current[c];
 152         Step = Gains->Step[c];
 153         if(Step != 1.0f && Counter > 0)
 154         {
 155             /* Mix with applying gain steps in aligned multiples of 4. */
 156             if(BufferSize-pos > 3 && Counter-pos > 3)
 157             {
 158                 gain = _mm_setr_ps(
 159                     DrySend,
 160                     DrySend * Step,
 161                     DrySend * Step * Step,
 162                     DrySend * Step * Step * Step
 163                 );
 164                 step = _mm_set1_ps(Step * Step * Step * Step);
 165                 do {
 166                     const __m128 val4 = _mm_load_ps(&data[pos]);
 167                     __m128 dry4 = _mm_load_ps(&OutBuffer[c][OutPos+pos]);
 168                     dry4 = _mm_add_ps(dry4, _mm_mul_ps(val4, gain));
 169                     gain = _mm_mul_ps(gain, step);
 170                     _mm_store_ps(&OutBuffer[c][OutPos+pos], dry4);
 171                     pos += 4;
 172                 } while(BufferSize-pos > 3 && Counter-pos > 3);
 173                 DrySend = _mm_cvtss_f32(gain);
 174             }
 175             /* Mix with applying left over gain steps that aren't aligned multiples of 4. */
 176             for(;pos < BufferSize && pos < Counter;pos++)
 177             {
 178                 OutBuffer[c][OutPos+pos] += data[pos]*DrySend;
 179                 DrySend *= Step;
 180             }
 181             if(pos == Counter)
 182                 DrySend = Gains->Target[c];
 183             Gains->Current[c] = DrySend;
 184             /* Mix until pos is aligned with 4 or the mix is done. */
 185             for(;pos < BufferSize && (pos&3) != 0;pos++)
 186                 OutBuffer[c][OutPos+pos] += data[pos]*DrySend;
 187         }
 188
 189         if(!(DrySend > GAIN_SILENCE_THRESHOLD))
 190             continue;
 191         gain = _mm_set1_ps(DrySend);
 192         for(;BufferSize-pos > 3;pos += 4)
 193         {
 194             const __m128 val4 = _mm_load_ps(&data[pos]);
 195             __m128 dry4 = _mm_load_ps(&OutBuffer[c][OutPos+pos]);
 196             dry4 = _mm_add_ps(dry4, _mm_mul_ps(val4, gain));
 197             _mm_store_ps(&OutBuffer[c][OutPos+pos], dry4);
 198         }
 199         for(;pos < BufferSize;pos++)
 200             OutBuffer[c][OutPos+pos] += data[pos]*DrySend;
 201     }
 202 }
 203
 204
 205 void MixSend_SSE(ALfloat (*restrict OutBuffer)[BUFFERSIZE], const ALfloat *data,
 206                  MixGainMono *Gain, ALuint Counter, ALuint OutPos, ALuint BufferSize)
 207 {
 208     ALfloat WetGain, Step;
 209     __m128 gain, step;
 210
 211     {
 212         ALuint pos = 0;
 213         WetGain = Gain->Current;
 214         Step = Gain->Step;
 215         if(Step != 1.0f && Counter > 0)
 216         {
 217             if(BufferSize-pos > 3 && Counter-pos > 3)
 218             {
 219                 gain = _mm_setr_ps(
 220                     WetGain,
 221                     WetGain * Step,
 222                     WetGain * Step * Step,
 223                     WetGain * Step * Step * Step
 224                 );
 225                 step = _mm_set1_ps(Step * Step * Step * Step);
 226                 do {
 227                     const __m128 val4 = _mm_load_ps(&data[pos]);
 228                     __m128 dry4 = _mm_load_ps(&OutBuffer[0][OutPos+pos]);
 229                     dry4 = _mm_add_ps(dry4, _mm_mul_ps(val4, gain));
 230                     gain = _mm_mul_ps(gain, step);
 231                     _mm_store_ps(&OutBuffer[0][OutPos+pos], dry4);
 232                     pos += 4;
 233                 } while(BufferSize-pos > 3 && Counter-pos > 3);
 234                 WetGain = _mm_cvtss_f32(gain);
 235             }
 236             for(;pos < BufferSize && pos < Counter;pos++)
 237             {
 238                 OutBuffer[0][OutPos+pos] += data[pos]*WetGain;
 239                 WetGain *= Step;
 240             }
 241             if(pos == Counter)
 242                 WetGain = Gain->Target;
 243             Gain->Current = WetGain;
 244             for(;pos < BufferSize && (pos&3) != 0;pos++)
 245                 OutBuffer[0][OutPos+pos] += data[pos]*WetGain;
 246         }
 247
 248         if(!(WetGain > GAIN_SILENCE_THRESHOLD))
 249             return;
 250         gain = _mm_set1_ps(WetGain);
 251         for(;BufferSize-pos > 3;pos += 4)
 252         {
 253             const __m128 val4 = _mm_load_ps(&data[pos]);
 254             __m128 wet4 = _mm_load_ps(&OutBuffer[0][OutPos+pos]);
 255             wet4 = _mm_add_ps(wet4, _mm_mul_ps(val4, gain));
 256             _mm_store_ps(&OutBuffer[0][OutPos+pos], wet4);
 257         }
 258         for(;pos < BufferSize;pos++)
 259             OutBuffer[0][OutPos+pos] += data[pos] * WetGain;
 260     }
 261 }