Alc/mixer_sse.c

   1 #include "config.h"
   2
   3 #ifdef IN_IDE_PARSER
   4 /* KDevelop's parser won't recognize these defines that get added by the -msse
   5  * switch used to compile this source. Without them, xmmintrin.h fails to
   6  * declare anything. */
   7 #define __MMX__
   8 #define __SSE__
   9 #endif
  10 #include <xmmintrin.h>
  11
  12 #include "AL/al.h"
  13 #include "AL/alc.h"
  14 #include "alMain.h"
  15 #include "alu.h"
  16
  17 #include "alSource.h"
  18 #include "alAuxEffectSlot.h"
  19 #include "mixer_defs.h"
  20
  21
  22 static inline void ApplyCoeffsStep(ALuint Offset, ALfloat (*restrict Values)[2],
  23                                    const ALuint IrSize,
  24                                    ALfloat (*restrict Coeffs)[2],
  25                                    const ALfloat (*restrict CoeffStep)[2],
  26                                    ALfloat left, ALfloat right)
  27 {
  28     const __m128 lrlr = _mm_set_ps(left, right, left, right);
  29     __m128 coeffs, deltas, imp0, imp1;
  30     __m128 vals = _mm_setzero_ps();
  31     ALuint i;
  32
  33     if((Offset&1))
  34     {
  35         const ALuint o0 = Offset&HRIR_MASK;
  36         const ALuint o1 = (Offset+IrSize-1)&HRIR_MASK;
  37
  38         coeffs = _mm_load_ps(&Coeffs[0][0]);
  39         deltas = _mm_load_ps(&CoeffStep[0][0]);
  40         vals = _mm_loadl_pi(vals, (__m64*)&Values[o0][0]);
  41         imp0 = _mm_mul_ps(lrlr, coeffs);
  42         coeffs = _mm_add_ps(coeffs, deltas);
  43         vals = _mm_add_ps(imp0, vals);
  44         _mm_store_ps(&Coeffs[0][0], coeffs);
  45         _mm_storel_pi((__m64*)&Values[o0][0], vals);
  46         for(i = 1;i < IrSize-1;i += 2)
  47         {
  48             const ALuint o2 = (Offset+i)&HRIR_MASK;
  49
  50             coeffs = _mm_load_ps(&Coeffs[i+1][0]);
  51             deltas = _mm_load_ps(&CoeffStep[i+1][0]);
  52             vals = _mm_load_ps(&Values[o2][0]);
  53             imp1 = _mm_mul_ps(lrlr, coeffs);
  54             coeffs = _mm_add_ps(coeffs, deltas);
  55             imp0 = _mm_shuffle_ps(imp0, imp1, _MM_SHUFFLE(1, 0, 3, 2));
  56             vals = _mm_add_ps(imp0, vals);
  57             _mm_store_ps(&Coeffs[i+1][0], coeffs);
  58             _mm_store_ps(&Values[o2][0], vals);
  59             imp0 = imp1;
  60         }
  61         vals = _mm_loadl_pi(vals, (__m64*)&Values[o1][0]);
  62         imp0 = _mm_movehl_ps(imp0, imp0);
  63         vals = _mm_add_ps(imp0, vals);
  64         _mm_storel_pi((__m64*)&Values[o1][0], vals);
  65     }
  66     else
  67     {
  68         for(i = 0;i < IrSize;i += 2)
  69         {
  70             const ALuint o = (Offset + i)&HRIR_MASK;
  71
  72             coeffs = _mm_load_ps(&Coeffs[i][0]);
  73             deltas = _mm_load_ps(&CoeffStep[i][0]);
  74             vals = _mm_load_ps(&Values[o][0]);
  75             imp0 = _mm_mul_ps(lrlr, coeffs);
  76             coeffs = _mm_add_ps(coeffs, deltas);
  77             vals = _mm_add_ps(imp0, vals);
  78             _mm_store_ps(&Coeffs[i][0], coeffs);
  79             _mm_store_ps(&Values[o][0], vals);
  80         }
  81     }
  82 }
  83
  84 static inline void ApplyCoeffs(ALuint Offset, ALfloat (*restrict Values)[2],
  85                                const ALuint IrSize,
  86                                ALfloat (*restrict Coeffs)[2],
  87                                ALfloat left, ALfloat right)
  88 {
  89     const __m128 lrlr = _mm_set_ps(left, right, left, right);
  90     __m128 vals = _mm_setzero_ps();
  91     __m128 coeffs;
  92     ALuint i;
  93
  94     if((Offset&1))
  95     {
  96         const ALuint o0 = Offset&HRIR_MASK;
  97         const ALuint o1 = (Offset+IrSize-1)&HRIR_MASK;
  98         __m128 imp0, imp1;
  99
 100         coeffs = _mm_load_ps(&Coeffs[0][0]);
 101         vals = _mm_loadl_pi(vals, (__m64*)&Values[o0][0]);
 102         imp0 = _mm_mul_ps(lrlr, coeffs);
 103         vals = _mm_add_ps(imp0, vals);
 104         _mm_storel_pi((__m64*)&Values[o0][0], vals);
 105         for(i = 1;i < IrSize-1;i += 2)
 106         {
 107             const ALuint o2 = (Offset+i)&HRIR_MASK;
 108
 109             coeffs = _mm_load_ps(&Coeffs[i+1][0]);
 110             vals = _mm_load_ps(&Values[o2][0]);
 111             imp1 = _mm_mul_ps(lrlr, coeffs);
 112             imp0 = _mm_shuffle_ps(imp0, imp1, _MM_SHUFFLE(1, 0, 3, 2));
 113             vals = _mm_add_ps(imp0, vals);
 114             _mm_store_ps(&Values[o2][0], vals);
 115             imp0 = imp1;
 116         }
 117         vals = _mm_loadl_pi(vals, (__m64*)&Values[o1][0]);
 118         imp0 = _mm_movehl_ps(imp0, imp0);
 119         vals = _mm_add_ps(imp0, vals);
 120         _mm_storel_pi((__m64*)&Values[o1][0], vals);
 121     }
 122     else
 123     {
 124         for(i = 0;i < IrSize;i += 2)
 125         {
 126             const ALuint o = (Offset + i)&HRIR_MASK;
 127
 128             coeffs = _mm_load_ps(&Coeffs[i][0]);
 129             vals = _mm_load_ps(&Values[o][0]);
 130             vals = _mm_add_ps(vals, _mm_mul_ps(lrlr, coeffs));
 131             _mm_store_ps(&Values[o][0], vals);
 132         }
 133     }
 134 }
 135
 136 #define SUFFIX SSE
 137 #include "mixer_inc.c"
 138 #undef SUFFIX
 139
 140
 141 void MixDirect_SSE(DirectParams *params, const ALfloat *restrict data, ALuint srcchan,
 142   ALuint OutPos, ALuint BufferSize)
 143 {
 144     ALfloat (*restrict OutBuffer)[BUFFERSIZE] = params->OutBuffer;
 145     ALuint Counter = maxu(params->Counter, OutPos) - OutPos;
 146     ALfloat DrySend, Step;
 147     __m128 gain, step;
 148     ALuint c;
 149
 150     for(c = 0;c < MaxChannels;c++)
 151     {
 152         ALuint pos = 0;
 153         DrySend = params->Mix.Gains.Current[srcchan][c];
 154         Step = params->Mix.Gains.Step[srcchan][c];
 155         if(Step != 1.0f && Counter > 0)
 156         {
 157             /* Mix with applying gain steps in aligned multiples of 4. */
 158             if(BufferSize-pos > 3 && Counter-pos > 3)
 159             {
 160                 gain = _mm_set_ps(
 161                     DrySend,
 162                     DrySend * Step,
 163                     DrySend * Step * Step,
 164                     DrySend * Step * Step * Step
 165                 );
 166                 step = _mm_set1_ps(Step * Step * Step * Step);
 167                 do {
 168                     const __m128 val4 = _mm_load_ps(&data[pos]);
 169                     __m128 dry4 = _mm_load_ps(&OutBuffer[c][OutPos+pos]);
 170                     dry4 = _mm_add_ps(dry4, _mm_mul_ps(val4, gain));
 171                     gain = _mm_mul_ps(gain, step);
 172                     _mm_store_ps(&OutBuffer[c][OutPos+pos], dry4);
 173                     pos += 4;
 174                 } while(BufferSize-pos > 3 && Counter-pos > 3);
 175                 DrySend = _mm_cvtss_f32(_mm_shuffle_ps(gain, gain, _MM_SHUFFLE(3, 3, 3, 3)));
 176             }
 177             /* Mix with applying left over gain steps that aren't aligned multiples of 4. */
 178             for(;pos < BufferSize && pos < Counter;pos++)
 179             {
 180                 OutBuffer[c][OutPos+pos] += data[pos]*DrySend;
 181                 DrySend *= Step;
 182             }
 183             if(pos == Counter)
 184                 DrySend = params->Mix.Gains.Target[srcchan][c];
 185             params->Mix.Gains.Current[srcchan][c] = DrySend;
 186             /* Mix until pos is aligned with 4 or the mix is done. */
 187             for(;pos < BufferSize && (pos&3) != 0;pos++)
 188                 OutBuffer[c][OutPos+pos] += data[pos]*DrySend;
 189         }
 190
 191         if(!(DrySend > GAIN_SILENCE_THRESHOLD))
 192             continue;
 193         gain = _mm_set1_ps(DrySend);
 194         for(;BufferSize-pos > 3;pos += 4)
 195         {
 196             const __m128 val4 = _mm_load_ps(&data[pos]);
 197             __m128 dry4 = _mm_load_ps(&OutBuffer[c][OutPos+pos]);
 198             dry4 = _mm_add_ps(dry4, _mm_mul_ps(val4, gain));
 199             _mm_store_ps(&OutBuffer[c][OutPos+pos], dry4);
 200         }
 201         for(;pos < BufferSize;pos++)
 202             OutBuffer[c][OutPos+pos] += data[pos]*DrySend;
 203     }
 204 }
 205
 206
 207 void MixSend_SSE(SendParams *params, const ALfloat *restrict data,
 208   ALuint OutPos, ALuint BufferSize)
 209 {
 210     ALfloat (*restrict OutBuffer)[BUFFERSIZE] = params->OutBuffer;
 211     ALuint Counter = maxu(params->Counter, OutPos) - OutPos;
 212     ALfloat WetGain, Step;
 213     __m128 gain, step;
 214
 215     {
 216         ALuint pos = 0;
 217         WetGain = params->Gain.Current;
 218         Step = params->Gain.Step;
 219         if(Step != 1.0f && Counter > 0)
 220         {
 221             if(BufferSize-pos > 3 && Counter-pos > 3)
 222             {
 223                 gain = _mm_set_ps(
 224                     WetGain,
 225                     WetGain * Step,
 226                     WetGain * Step * Step,
 227                     WetGain * Step * Step * Step
 228                 );
 229                 step = _mm_set1_ps(Step * Step * Step * Step);
 230                 do {
 231                     const __m128 val4 = _mm_load_ps(&data[pos]);
 232                     __m128 dry4 = _mm_load_ps(&OutBuffer[0][OutPos+pos]);
 233                     dry4 = _mm_add_ps(dry4, _mm_mul_ps(val4, gain));
 234                     gain = _mm_mul_ps(gain, step);
 235                     _mm_store_ps(&OutBuffer[0][OutPos+pos], dry4);
 236                     pos += 4;
 237                 } while(BufferSize-pos > 3 && Counter-pos > 3);
 238                 WetGain = _mm_cvtss_f32(_mm_shuffle_ps(gain, gain, _MM_SHUFFLE(3, 3, 3, 3)));
 239             }
 240             for(;pos < BufferSize && pos < Counter;pos++)
 241             {
 242                 OutBuffer[0][OutPos+pos] += data[pos]*WetGain;
 243                 WetGain *= Step;
 244             }
 245             if(pos == Counter)
 246                 WetGain = params->Gain.Target;
 247             params->Gain.Current = WetGain;
 248             for(;pos < BufferSize && (pos&3) != 0;pos++)
 249                 OutBuffer[0][OutPos+pos] += data[pos]*WetGain;
 250         }
 251
 252         if(!(WetGain > GAIN_SILENCE_THRESHOLD))
 253             return;
 254         gain = _mm_set1_ps(WetGain);
 255         for(;BufferSize-pos > 3;pos += 4)
 256         {
 257             const __m128 val4 = _mm_load_ps(&data[pos]);
 258             __m128 wet4 = _mm_load_ps(&OutBuffer[0][OutPos+pos]);
 259             wet4 = _mm_add_ps(wet4, _mm_mul_ps(val4, gain));
 260             _mm_store_ps(&OutBuffer[0][OutPos+pos], wet4);
 261         }
 262         for(;pos < BufferSize;pos++)
 263             OutBuffer[0][OutPos+pos] += data[pos] * WetGain;
 264     }
 265 }