dom/media/DynamicResampler.cpp

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
   2 /* This Source Code Form is subject to the terms of the Mozilla Public
   3  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
   4  * You can obtain one at http://mozilla.org/MPL/2.0/. */
   5
   6 #include "DynamicResampler.h"
   7
   8 namespace mozilla {
   9
  10 DynamicResampler::DynamicResampler(uint32_t aInRate, uint32_t aOutRate,
  11                                    uint32_t aPreBufferFrames)
  12     : mInRate(aInRate), mPreBufferFrames(aPreBufferFrames), mOutRate(aOutRate) {
  13   MOZ_ASSERT(aInRate);
  14   MOZ_ASSERT(aOutRate);
  15   UpdateResampler(mOutRate, STEREO);
  16 }
  17
  18 DynamicResampler::~DynamicResampler() {
  19   if (mResampler) {
  20     speex_resampler_destroy(mResampler);
  21   }
  22 }
  23
  24 void DynamicResampler::SetSampleFormat(AudioSampleFormat aFormat) {
  25   MOZ_ASSERT(mSampleFormat == AUDIO_FORMAT_SILENCE);
  26   MOZ_ASSERT(aFormat == AUDIO_FORMAT_S16 || aFormat == AUDIO_FORMAT_FLOAT32);
  27
  28   mSampleFormat = aFormat;
  29   for (AudioRingBuffer& b : mInternalInBuffer) {
  30     b.SetSampleFormat(mSampleFormat);
  31   }
  32   if (mPreBufferFrames) {
  33     AppendInputSilence(mPreBufferFrames);
  34   }
  35 }
  36
  37 bool DynamicResampler::Resample(float* aOutBuffer, uint32_t* aOutFrames,
  38                                 uint32_t aChannelIndex) {
  39   MOZ_ASSERT(mSampleFormat == AUDIO_FORMAT_FLOAT32);
  40   return ResampleInternal(aOutBuffer, aOutFrames, aChannelIndex);
  41 }
  42
  43 bool DynamicResampler::Resample(int16_t* aOutBuffer, uint32_t* aOutFrames,
  44                                 uint32_t aChannelIndex) {
  45   MOZ_ASSERT(mSampleFormat == AUDIO_FORMAT_S16);
  46   return ResampleInternal(aOutBuffer, aOutFrames, aChannelIndex);
  47 }
  48
  49 void DynamicResampler::ResampleInternal(const float* aInBuffer,
  50                                         uint32_t* aInFrames, float* aOutBuffer,
  51                                         uint32_t* aOutFrames,
  52                                         uint32_t aChannelIndex) {
  53   MOZ_ASSERT(mResampler);
  54   MOZ_ASSERT(mChannels);
  55   MOZ_ASSERT(mInRate);
  56   MOZ_ASSERT(mOutRate);
  57
  58   MOZ_ASSERT(aInBuffer);
  59   MOZ_ASSERT(aInFrames);
  60   MOZ_ASSERT(*aInFrames > 0);
  61   MOZ_ASSERT(aOutBuffer);
  62   MOZ_ASSERT(aOutFrames);
  63   MOZ_ASSERT(*aOutFrames > 0);
  64
  65   MOZ_ASSERT(aChannelIndex <= mChannels);
  66
  67 #ifdef DEBUG
  68   int rv =
  69 #endif
  70       speex_resampler_process_float(mResampler, aChannelIndex, aInBuffer,
  71                                     aInFrames, aOutBuffer, aOutFrames);
  72   MOZ_ASSERT(rv == RESAMPLER_ERR_SUCCESS);
  73 }
  74
  75 void DynamicResampler::ResampleInternal(const int16_t* aInBuffer,
  76                                         uint32_t* aInFrames,
  77                                         int16_t* aOutBuffer,
  78                                         uint32_t* aOutFrames,
  79                                         uint32_t aChannelIndex) {
  80   MOZ_ASSERT(mResampler);
  81   MOZ_ASSERT(mChannels);
  82   MOZ_ASSERT(mInRate);
  83   MOZ_ASSERT(mOutRate);
  84
  85   MOZ_ASSERT(aInBuffer);
  86   MOZ_ASSERT(aInFrames);
  87   MOZ_ASSERT(*aInFrames > 0);
  88   MOZ_ASSERT(aOutBuffer);
  89   MOZ_ASSERT(aOutFrames);
  90   MOZ_ASSERT(*aOutFrames > 0);
  91
  92   MOZ_ASSERT(aChannelIndex <= mChannels);
  93
  94 #ifdef DEBUG
  95   int rv =
  96 #endif
  97       speex_resampler_process_int(mResampler, aChannelIndex, aInBuffer,
  98                                   aInFrames, aOutBuffer, aOutFrames);
  99   MOZ_ASSERT(rv == RESAMPLER_ERR_SUCCESS);
 100 }
 101
 102 void DynamicResampler::UpdateResampler(uint32_t aOutRate, uint32_t aChannels) {
 103   MOZ_ASSERT(aOutRate);
 104   MOZ_ASSERT(aChannels);
 105
 106   if (mChannels != aChannels) {
 107     if (mResampler) {
 108       speex_resampler_destroy(mResampler);
 109     }
 110     mResampler = speex_resampler_init(aChannels, mInRate, aOutRate,
 111                                       SPEEX_RESAMPLER_QUALITY_MIN, nullptr);
 112     MOZ_ASSERT(mResampler);
 113     mChannels = aChannels;
 114     mOutRate = aOutRate;
 115     // Between mono and stereo changes, keep always allocated 2 channels to
 116     // avoid reallocations in the most common case.
 117     if ((mChannels == STEREO || mChannels == 1) &&
 118         mInternalInBuffer.Length() == STEREO) {
 119       // Don't worry if format is not set it will write silence then.
 120       if ((mSampleFormat == AUDIO_FORMAT_S16 ||
 121            mSampleFormat == AUDIO_FORMAT_FLOAT32) &&
 122           mChannels == STEREO) {
 123         // The mono channel is always up to date. When we are going from mono
 124         // to stereo upmix the mono to stereo channel
 125         uint32_t bufferedDuration = mInternalInBuffer[0].AvailableRead();
 126         mInternalInBuffer[1].Clear();
 127         if (bufferedDuration) {
 128           mInternalInBuffer[1].Write(mInternalInBuffer[0], bufferedDuration);
 129         }
 130       }
 131       // Maintain stereo size
 132       mInputTail.SetLength(STEREO);
 133       WarmUpResampler(false);
 134       return;
 135     }
 136     // upmix or downmix, for now just clear but it has to be updated
 137     // because allocates and this is executed in audio thread.
 138     mInternalInBuffer.Clear();
 139     for (uint32_t i = 0; i < mChannels; ++i) {
 140       // Pre-allocate something big, twice the pre-buffer, or at least 100ms.
 141       AudioRingBuffer* b = mInternalInBuffer.AppendElement(
 142           sizeof(float) * std::max(2 * mPreBufferFrames, mInRate / 10));
 143       if (mSampleFormat != AUDIO_FORMAT_SILENCE) {
 144         // In ctor this update is not needed
 145         b->SetSampleFormat(mSampleFormat);
 146       }
 147     }
 148     mInputTail.SetLength(mChannels);
 149     return;
 150   }
 151
 152   if (mOutRate != aOutRate) {
 153     // If the rates was the same the resampler was not being used so warm up.
 154     if (mOutRate == mInRate) {
 155       WarmUpResampler(true);
 156     }
 157
 158 #ifdef DEBUG
 159     int rv =
 160 #endif
 161         speex_resampler_set_rate(mResampler, mInRate, aOutRate);
 162     MOZ_ASSERT(rv == RESAMPLER_ERR_SUCCESS);
 163     mOutRate = aOutRate;
 164   }
 165 }
 166
 167 void DynamicResampler::WarmUpResampler(bool aSkipLatency) {
 168   MOZ_ASSERT(mInputTail.Length());
 169   for (uint32_t i = 0; i < mChannels; ++i) {
 170     if (!mInputTail[i].Length()) {
 171       continue;
 172     }
 173     uint32_t inFrames = mInputTail[i].Length();
 174     uint32_t outFrames = 5 * TailBuffer::MAXSIZE;  // something big
 175     if (mSampleFormat == AUDIO_FORMAT_S16) {
 176       short outBuffer[5 * TailBuffer::MAXSIZE] = {};
 177       ResampleInternal(mInputTail[i].Buffer<short>(), &inFrames, outBuffer,
 178                        &outFrames, i);
 179       MOZ_ASSERT(inFrames == (uint32_t)mInputTail[i].Length());
 180     } else {
 181       float outBuffer[100] = {};
 182       ResampleInternal(mInputTail[i].Buffer<float>(), &inFrames, outBuffer,
 183                        &outFrames, i);
 184       MOZ_ASSERT(inFrames == (uint32_t)mInputTail[i].Length());
 185     }
 186   }
 187   if (aSkipLatency) {
 188     int inputLatency = speex_resampler_get_input_latency(mResampler);
 189     MOZ_ASSERT(inputLatency > 0);
 190     uint32_t ratioNum, ratioDen;
 191     speex_resampler_get_ratio(mResampler, &ratioNum, &ratioDen);
 192     // Ratio at this point is one so only skip the input latency. No special
 193     // calculations are needed.
 194     speex_resampler_set_skip_frac_num(mResampler, inputLatency * ratioDen);
 195   }
 196 }
 197
 198 void DynamicResampler::AppendInput(const nsTArray<const float*>& aInBuffer,
 199                                    uint32_t aInFrames) {
 200   MOZ_ASSERT(mSampleFormat == AUDIO_FORMAT_FLOAT32);
 201   AppendInputInternal(aInBuffer, aInFrames);
 202 }
 203 void DynamicResampler::AppendInput(const nsTArray<const int16_t*>& aInBuffer,
 204                                    uint32_t aInFrames) {
 205   MOZ_ASSERT(mSampleFormat == AUDIO_FORMAT_S16);
 206   AppendInputInternal(aInBuffer, aInFrames);
 207 }
 208
 209 bool DynamicResampler::EnoughInFrames(uint32_t aOutFrames,
 210                                       uint32_t aChannelIndex) const {
 211   if (mInRate == mOutRate) {
 212     return InFramesBuffered(aChannelIndex) >= aOutFrames;
 213   }
 214   if (!(mOutRate % mInRate) && !(aOutFrames % mOutRate / mInRate)) {
 215     return InFramesBuffered(aChannelIndex) >= aOutFrames / (mOutRate / mInRate);
 216   }
 217   if (!(mInRate % mOutRate) && !(aOutFrames % mOutRate / mInRate)) {
 218     return InFramesBuffered(aChannelIndex) >= aOutFrames * mInRate / mOutRate;
 219   }
 220   return InFramesBuffered(aChannelIndex) > aOutFrames * mInRate / mOutRate;
 221 }
 222
 223 bool DynamicResampler::CanResample(uint32_t aOutFrames) const {
 224   for (uint32_t i = 0; i < mChannels; ++i) {
 225     if (!EnoughInFrames(aOutFrames, i)) {
 226       return false;
 227     }
 228   }
 229   return true;
 230 }
 231
 232 void DynamicResampler::AppendInputSilence(const uint32_t aInFrames) {
 233   MOZ_ASSERT(aInFrames);
 234   MOZ_ASSERT(mChannels);
 235   MOZ_ASSERT(mInternalInBuffer.Length() >= (uint32_t)mChannels);
 236   for (uint32_t i = 0; i < mChannels; ++i) {
 237     mInternalInBuffer[i].WriteSilence(aInFrames);
 238   }
 239 }
 240
 241 uint32_t DynamicResampler::InFramesBuffered(uint32_t aChannelIndex) const {
 242   MOZ_ASSERT(mChannels);
 243   MOZ_ASSERT(aChannelIndex <= mChannels);
 244   MOZ_ASSERT(aChannelIndex <= mInternalInBuffer.Length());
 245   return mInternalInBuffer[aChannelIndex].AvailableRead();
 246 }
 247
 248 uint32_t DynamicResampler::InFramesLeftToBuffer(uint32_t aChannelIndex) const {
 249   MOZ_ASSERT(mChannels);
 250   MOZ_ASSERT(aChannelIndex <= mChannels);
 251   MOZ_ASSERT(aChannelIndex <= mInternalInBuffer.Length());
 252   return mInternalInBuffer[aChannelIndex].AvailableWrite();
 253 }
 254
 255 AudioChunkList::AudioChunkList(uint32_t aTotalDuration, uint32_t aChannels,
 256                                const PrincipalHandle& aPrincipalHandle)
 257     : mPrincipalHandle(aPrincipalHandle) {
 258   uint32_t numOfChunks = aTotalDuration / mChunkCapacity;
 259   if (aTotalDuration % mChunkCapacity) {
 260     ++numOfChunks;
 261   }
 262   CreateChunks(numOfChunks, aChannels);
 263 }
 264
 265 void AudioChunkList::CreateChunks(uint32_t aNumOfChunks, uint32_t aChannels) {
 266   MOZ_ASSERT(!mChunks.Length());
 267   MOZ_ASSERT(aNumOfChunks);
 268   MOZ_ASSERT(aChannels);
 269   mChunks.AppendElements(aNumOfChunks);
 270
 271   for (AudioChunk& chunk : mChunks) {
 272     AutoTArray<nsTArray<float>, STEREO> buffer;
 273     buffer.AppendElements(aChannels);
 274
 275     AutoTArray<const float*, STEREO> bufferPtrs;
 276     bufferPtrs.AppendElements(aChannels);
 277
 278     for (uint32_t i = 0; i < aChannels; ++i) {
 279       float* ptr = buffer[i].AppendElements(mChunkCapacity);
 280       bufferPtrs[i] = ptr;
 281     }
 282
 283     chunk.mBuffer = new mozilla::SharedChannelArrayBuffer(std::move(buffer));
 284     chunk.mChannelData.AppendElements(aChannels);
 285     for (uint32_t i = 0; i < aChannels; ++i) {
 286       chunk.mChannelData[i] = bufferPtrs[i];
 287     }
 288   }
 289 }
 290
 291 void AudioChunkList::UpdateToMonoOrStereo(uint32_t aChannels) {
 292   MOZ_ASSERT(mChunks.Length());
 293   MOZ_ASSERT(mSampleFormat == AUDIO_FORMAT_S16 ||
 294              mSampleFormat == AUDIO_FORMAT_FLOAT32);
 295   MOZ_ASSERT(aChannels == 1 || aChannels == 2);
 296
 297   for (AudioChunk& chunk : mChunks) {
 298     MOZ_ASSERT(chunk.ChannelCount() != (uint32_t)aChannels);
 299     MOZ_ASSERT(chunk.ChannelCount() == 1 || chunk.ChannelCount() == 2);
 300     chunk.mChannelData.SetLengthAndRetainStorage(aChannels);
 301     if (mSampleFormat == AUDIO_FORMAT_S16) {
 302       SharedChannelArrayBuffer<short>* channelArray =
 303           static_cast<SharedChannelArrayBuffer<short>*>(chunk.mBuffer.get());
 304       channelArray->mBuffers.SetLengthAndRetainStorage(aChannels);
 305       if (aChannels == 2) {
 306         // This an indirect allocation, unfortunately.
 307         channelArray->mBuffers[1].SetLength(mChunkCapacity);
 308         chunk.mChannelData[1] = channelArray->mBuffers[1].Elements();
 309       }
 310     } else {
 311       SharedChannelArrayBuffer<float>* channelArray =
 312           static_cast<SharedChannelArrayBuffer<float>*>(chunk.mBuffer.get());
 313       channelArray->mBuffers.SetLengthAndRetainStorage(aChannels);
 314       if (aChannels == 2) {
 315         // This an indirect allocation, unfortunately.
 316         channelArray->mBuffers[1].SetLength(mChunkCapacity);
 317         chunk.mChannelData[1] = channelArray->mBuffers[1].Elements();
 318       }
 319     }
 320   }
 321 }
 322
 323 void AudioChunkList::SetSampleFormat(AudioSampleFormat aFormat) {
 324   MOZ_ASSERT(mSampleFormat == AUDIO_FORMAT_SILENCE);
 325   MOZ_ASSERT(aFormat == AUDIO_FORMAT_S16 || aFormat == AUDIO_FORMAT_FLOAT32);
 326   mSampleFormat = aFormat;
 327   if (mSampleFormat == AUDIO_FORMAT_S16) {
 328     mChunkCapacity = 2 * mChunkCapacity;
 329   }
 330 }
 331
 332 AudioChunk& AudioChunkList::GetNext() {
 333   AudioChunk& chunk = mChunks[mIndex];
 334   MOZ_ASSERT(!chunk.mChannelData.IsEmpty());
 335   MOZ_ASSERT(chunk.mBuffer);
 336   MOZ_ASSERT(!chunk.mBuffer->IsShared());
 337   MOZ_ASSERT(mSampleFormat == AUDIO_FORMAT_S16 ||
 338              mSampleFormat == AUDIO_FORMAT_FLOAT32);
 339   chunk.mDuration = 0;
 340   chunk.mVolume = 1.0f;
 341   chunk.mPrincipalHandle = mPrincipalHandle;
 342   chunk.mBufferFormat = mSampleFormat;
 343   IncrementIndex();
 344   return chunk;
 345 }
 346
 347 void AudioChunkList::Update(uint32_t aChannels) {
 348   MOZ_ASSERT(mChunks.Length());
 349   if (mChunks[0].ChannelCount() == aChannels) {
 350     return;
 351   }
 352
 353   // Special handling between mono and stereo to avoid reallocations.
 354   if (aChannels <= 2 && mChunks[0].ChannelCount() <= 2) {
 355     UpdateToMonoOrStereo(aChannels);
 356     return;
 357   }
 358
 359   uint32_t numOfChunks = mChunks.Length();
 360   mChunks.ClearAndRetainStorage();
 361   CreateChunks(numOfChunks, aChannels);
 362 }
 363
 364 AudioResampler::AudioResampler(uint32_t aInRate, uint32_t aOutRate,
 365                                uint32_t aPreBufferFrames,
 366                                const PrincipalHandle& aPrincipalHandle)
 367     : mResampler(aInRate, aOutRate, aPreBufferFrames),
 368       mOutputChunks(aOutRate / 10, STEREO, aPrincipalHandle) {}
 369
 370 void AudioResampler::AppendInput(const AudioSegment& aInSegment) {
 371   MOZ_ASSERT(aInSegment.GetDuration());
 372   for (AudioSegment::ConstChunkIterator iter(aInSegment); !iter.IsEnded();
 373        iter.Next()) {
 374     const AudioChunk& chunk = *iter;
 375     if (!mIsSampleFormatSet) {
 376       // We don't know the format yet and all buffers are empty.
 377       if (chunk.mBufferFormat == AUDIO_FORMAT_SILENCE) {
 378         // Only silence has been received and the format is unkown. Igonre it,
 379         // if Resampler() is called it will return silence too.
 380         continue;
 381       }
 382       // First no silence data, set the format once for lifetime and let it
 383       // continue the rest of the flow. We will not get in here again.
 384       mOutputChunks.SetSampleFormat(chunk.mBufferFormat);
 385       mResampler.SetSampleFormat(chunk.mBufferFormat);
 386       mIsSampleFormatSet = true;
 387     }
 388     MOZ_ASSERT(mIsSampleFormatSet);
 389     if (chunk.IsNull()) {
 390       mResampler.AppendInputSilence(chunk.GetDuration());
 391       continue;
 392     }
 393     // Make sure the channel is up to date. An AudioSegment can contain chunks
 394     // with different channel count.
 395     UpdateChannels(chunk.mChannelData.Length());
 396     if (chunk.mBufferFormat == AUDIO_FORMAT_FLOAT32) {
 397       mResampler.AppendInput(chunk.ChannelData<float>(), chunk.GetDuration());
 398     } else {
 399       mResampler.AppendInput(chunk.ChannelData<int16_t>(), chunk.GetDuration());
 400     }
 401   }
 402 }
 403
 404 AudioSegment AudioResampler::Resample(uint32_t aOutFrames) {
 405   AudioSegment segment;
 406
 407   // We don't know what to do yet and we only have received silence if any just
 408   // return what they want and leave
 409   if (!mIsSampleFormatSet) {
 410     segment.AppendNullData(aOutFrames);
 411     return segment;
 412   }
 413
 414   // Not enough input frames abort. We check for the requested frames plus one.
 415   // This is to make sure that the individual resample iteration that will
 416   // follow up, will have enough frames even if one of them consume an extra
 417   // frame.
 418   if (!mResampler.CanResample(aOutFrames + 1)) {
 419     return segment;
 420   }
 421
 422   uint32_t totalFrames = aOutFrames;
 423   while (totalFrames) {
 424     MOZ_ASSERT(totalFrames > 0);
 425     AudioChunk& chunk = mOutputChunks.GetNext();
 426     uint32_t outFrames = std::min(totalFrames, mOutputChunks.ChunkCapacity());
 427     totalFrames -= outFrames;
 428
 429     for (uint32_t i = 0; i < chunk.ChannelCount(); ++i) {
 430       uint32_t outFramesUsed = outFrames;
 431       if (chunk.mBufferFormat == AUDIO_FORMAT_FLOAT32) {
 432 #ifdef DEBUG
 433         bool rv =
 434 #endif
 435             mResampler.Resample(chunk.ChannelDataForWrite<float>(i),
 436                                 &outFramesUsed, i);
 437         MOZ_ASSERT(rv);
 438       } else {
 439 #ifdef DEBUG
 440         bool rv =
 441 #endif
 442             mResampler.Resample(chunk.ChannelDataForWrite<int16_t>(i),
 443                                 &outFramesUsed, i);
 444         MOZ_ASSERT(rv);
 445       }
 446       MOZ_ASSERT(outFramesUsed == outFrames);
 447       chunk.mDuration = outFrames;
 448     }
 449
 450     // Create a copy in order to consume that copy and not the pre-allocated
 451     // chunk
 452     segment.AppendAndConsumeChunk(AudioChunk(chunk));
 453   }
 454
 455   return segment;
 456 }
 457
 458 void AudioResampler::Update(uint32_t aOutRate, uint32_t aChannels) {
 459   mResampler.UpdateResampler(aOutRate, aChannels);
 460   mOutputChunks.Update(aChannels);
 461 }
 462
 463 uint32_t AudioResampler::InputReadableFrames() const {
 464   if (!mIsSampleFormatSet) {
 465     return mResampler.mPreBufferFrames;
 466   }
 467   return mResampler.InFramesBuffered(0);
 468 }
 469
 470 uint32_t AudioResampler::InputWritableFrames() const {
 471   if (!mIsSampleFormatSet) {
 472     return mResampler.mPreBufferFrames;
 473   }
 474   return mResampler.InFramesLeftToBuffer(0);
 475 }
 476
 477 }  // namespace mozilla