dom/media/AudioConverter.cpp

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "AudioConverter.h"
   8 #include <speex/speex_resampler.h>
   9 #include <string.h>
  10 #include <cmath>
  11
  12 /*
  13  *  Parts derived from MythTV AudioConvert Class
  14  *  Created by Jean-Yves Avenard.
  15  *
  16  *  Copyright (C) Bubblestuff Pty Ltd 2013
  17  *  Copyright (C) foobum@gmail.com 2010
  18  */
  19
  20 namespace mozilla {
  21
  22 AudioConverter::AudioConverter(const AudioConfig& aIn, const AudioConfig& aOut)
  23     : mIn(aIn), mOut(aOut), mResampler(nullptr) {
  24   MOZ_DIAGNOSTIC_ASSERT(CanConvert(aIn, aOut),
  25                         "The conversion is not supported");
  26   mIn.Layout().MappingTable(mOut.Layout(), &mChannelOrderMap);
  27   if (aIn.Rate() != aOut.Rate()) {
  28     RecreateResampler();
  29   }
  30 }
  31
  32 AudioConverter::~AudioConverter() {
  33   if (mResampler) {
  34     speex_resampler_destroy(mResampler);
  35     mResampler = nullptr;
  36   }
  37 }
  38
  39 bool AudioConverter::CanConvert(const AudioConfig& aIn,
  40                                 const AudioConfig& aOut) {
  41   if (aIn.Format() != aOut.Format() ||
  42       aIn.Interleaved() != aOut.Interleaved()) {
  43     NS_WARNING("No format conversion is supported at this stage");
  44     return false;
  45   }
  46   if (aIn.Channels() != aOut.Channels() && aOut.Channels() > 2) {
  47     NS_WARNING(
  48         "Only down/upmixing to mono or stereo is supported at this stage");
  49     return false;
  50   }
  51   if (!aOut.Interleaved()) {
  52     NS_WARNING("planar audio format not supported");
  53     return false;
  54   }
  55   return true;
  56 }
  57
  58 bool AudioConverter::CanWorkInPlace() const {
  59   bool needDownmix = mIn.Channels() > mOut.Channels();
  60   bool needUpmix = mIn.Channels() < mOut.Channels();
  61   bool canDownmixInPlace =
  62       mIn.Channels() * AudioConfig::SampleSize(mIn.Format()) >=
  63       mOut.Channels() * AudioConfig::SampleSize(mOut.Format());
  64   bool needResample = mIn.Rate() != mOut.Rate();
  65   bool canResampleInPlace = mIn.Rate() >= mOut.Rate();
  66   // We should be able to work in place if 1s of audio input takes less space
  67   // than 1s of audio output. However, as we downmix before resampling we can't
  68   // perform any upsampling in place (e.g. if incoming rate >= outgoing rate)
  69   return !needUpmix && (!needDownmix || canDownmixInPlace) &&
  70          (!needResample || canResampleInPlace);
  71 }
  72
  73 size_t AudioConverter::ProcessInternal(void* aOut, const void* aIn,
  74                                        size_t aFrames) {
  75   if (!aFrames) {
  76     return 0;
  77   }
  78   if (mIn.Channels() > mOut.Channels()) {
  79     return DownmixAudio(aOut, aIn, aFrames);
  80   } else if (mIn.Channels() < mOut.Channels()) {
  81     return UpmixAudio(aOut, aIn, aFrames);
  82   } else if (mIn.Layout() != mOut.Layout() && CanReorderAudio()) {
  83     ReOrderInterleavedChannels(aOut, aIn, aFrames);
  84   } else if (aIn != aOut) {
  85     memmove(aOut, aIn, FramesOutToBytes(aFrames));
  86   }
  87   return aFrames;
  88 }
  89
  90 // Reorder interleaved channels.
  91 // Can work in place (e.g aOut == aIn).
  92 template <class AudioDataType>
  93 void _ReOrderInterleavedChannels(AudioDataType* aOut, const AudioDataType* aIn,
  94                                  uint32_t aFrames, uint32_t aChannels,
  95                                  const uint8_t* aChannelOrderMap) {
  96   MOZ_DIAGNOSTIC_ASSERT(aChannels <= AudioConfig::ChannelLayout::MAX_CHANNELS);
  97   AudioDataType val[AudioConfig::ChannelLayout::MAX_CHANNELS];
  98   for (uint32_t i = 0; i < aFrames; i++) {
  99     for (uint32_t j = 0; j < aChannels; j++) {
 100       val[j] = aIn[aChannelOrderMap[j]];
 101     }
 102     for (uint32_t j = 0; j < aChannels; j++) {
 103       aOut[j] = val[j];
 104     }
 105     aOut += aChannels;
 106     aIn += aChannels;
 107   }
 108 }
 109
 110 void AudioConverter::ReOrderInterleavedChannels(void* aOut, const void* aIn,
 111                                                 size_t aFrames) const {
 112   MOZ_DIAGNOSTIC_ASSERT(mIn.Channels() == mOut.Channels());
 113   MOZ_DIAGNOSTIC_ASSERT(CanReorderAudio());
 114
 115   if (mChannelOrderMap.IsEmpty() || mOut.Channels() == 1 ||
 116       mOut.Layout() == mIn.Layout()) {
 117     // If channel count is 1, planar and non-planar formats are the same or
 118     // there's nothing to reorder, or if we don't know how to re-order.
 119     if (aOut != aIn) {
 120       memmove(aOut, aIn, FramesOutToBytes(aFrames));
 121     }
 122     return;
 123   }
 124
 125   uint32_t bits = AudioConfig::FormatToBits(mOut.Format());
 126   switch (bits) {
 127     case 8:
 128       _ReOrderInterleavedChannels((uint8_t*)aOut, (const uint8_t*)aIn, aFrames,
 129                                   mIn.Channels(), mChannelOrderMap.Elements());
 130       break;
 131     case 16:
 132       _ReOrderInterleavedChannels((int16_t*)aOut, (const int16_t*)aIn, aFrames,
 133                                   mIn.Channels(), mChannelOrderMap.Elements());
 134       break;
 135     default:
 136       MOZ_DIAGNOSTIC_ASSERT(AudioConfig::SampleSize(mOut.Format()) == 4);
 137       _ReOrderInterleavedChannels((int32_t*)aOut, (const int32_t*)aIn, aFrames,
 138                                   mIn.Channels(), mChannelOrderMap.Elements());
 139       break;
 140   }
 141 }
 142
 143 static inline int16_t clipTo15(int32_t aX) {
 144   return aX < -32768 ? -32768 : aX <= 32767 ? aX : 32767;
 145 }
 146
 147 template <typename TYPE>
 148 static void dumbUpDownMix(TYPE* aOut, int32_t aOutChannels, const TYPE* aIn,
 149                           int32_t aInChannels, int32_t aFrames) {
 150   if (aIn == aOut) {
 151     return;
 152   }
 153   int32_t commonChannels = std::min(aInChannels, aOutChannels);
 154
 155   for (int32_t i = 0; i < aFrames; i++) {
 156     for (int32_t j = 0; j < commonChannels; j++) {
 157       aOut[i * aOutChannels + j] = aIn[i * aInChannels + j];
 158     }
 159     for (int32_t j = 0; j < aInChannels - aOutChannels; j++) {
 160       aOut[i * aOutChannels + j] = 0;
 161     }
 162   }
 163 }
 164
 165 size_t AudioConverter::DownmixAudio(void* aOut, const void* aIn,
 166                                     size_t aFrames) const {
 167   MOZ_DIAGNOSTIC_ASSERT(mIn.Format() == AudioConfig::FORMAT_S16 ||
 168                         mIn.Format() == AudioConfig::FORMAT_FLT);
 169   MOZ_DIAGNOSTIC_ASSERT(mIn.Channels() >= mOut.Channels());
 170   MOZ_DIAGNOSTIC_ASSERT(mOut.Layout() == AudioConfig::ChannelLayout(2) ||
 171                         mOut.Layout() == AudioConfig::ChannelLayout(1));
 172
 173   uint32_t inChannels = mIn.Channels();
 174   uint32_t outChannels = mOut.Channels();
 175
 176   if (inChannels == outChannels) {
 177     if (aOut != aIn) {
 178       memmove(aOut, aIn, FramesOutToBytes(aFrames));
 179     }
 180     return aFrames;
 181   }
 182
 183   if (!mIn.Layout().IsValid() || !mOut.Layout().IsValid()) {
 184     // Dumb copy dropping extra channels.
 185     if (mIn.Format() == AudioConfig::FORMAT_FLT) {
 186       dumbUpDownMix(static_cast<float*>(aOut), outChannels,
 187                     static_cast<const float*>(aIn), inChannels, aFrames);
 188     } else if (mIn.Format() == AudioConfig::FORMAT_S16) {
 189       dumbUpDownMix(static_cast<int16_t*>(aOut), outChannels,
 190                     static_cast<const int16_t*>(aIn), inChannels, aFrames);
 191     } else {
 192       MOZ_DIAGNOSTIC_ASSERT(false, "Unsupported data type");
 193     }
 194     return aFrames;
 195   }
 196
 197   MOZ_ASSERT(
 198       mIn.Layout() == AudioConfig::ChannelLayout::SMPTEDefault(mIn.Layout()),
 199       "Can only downmix input data in SMPTE layout");
 200   if (inChannels > 2) {
 201     if (mIn.Format() == AudioConfig::FORMAT_FLT) {
 202       // Downmix matrix. Per-row normalization 1 for rows 3,4 and 2 for rows
 203       // 5-8.
 204       static const float dmatrix[6][8][2] = {
 205           /*3*/ {{0.5858f, 0}, {0, 0.5858f}, {0.4142f, 0.4142f}},
 206           /*4*/
 207           {{0.4226f, 0}, {0, 0.4226f}, {0.366f, 0.2114f}, {0.2114f, 0.366f}},
 208           /*5*/
 209           {{0.6510f, 0},
 210            {0, 0.6510f},
 211            {0.4600f, 0.4600f},
 212            {0.5636f, 0.3254f},
 213            {0.3254f, 0.5636f}},
 214           /*6*/
 215           {{0.5290f, 0},
 216            {0, 0.5290f},
 217            {0.3741f, 0.3741f},
 218            {0.3741f, 0.3741f},
 219            {0.4582f, 0.2645f},
 220            {0.2645f, 0.4582f}},
 221           /*7*/
 222           {{0.4553f, 0},
 223            {0, 0.4553f},
 224            {0.3220f, 0.3220f},
 225            {0.3220f, 0.3220f},
 226            {0.2788f, 0.2788f},
 227            {0.3943f, 0.2277f},
 228            {0.2277f, 0.3943f}},
 229           /*8*/
 230           {{0.3886f, 0},
 231            {0, 0.3886f},
 232            {0.2748f, 0.2748f},
 233            {0.2748f, 0.2748f},
 234            {0.3366f, 0.1943f},
 235            {0.1943f, 0.3366f},
 236            {0.3366f, 0.1943f},
 237            {0.1943f, 0.3366f}},
 238       };
 239       // Re-write the buffer with downmixed data
 240       const float* in = static_cast<const float*>(aIn);
 241       float* out = static_cast<float*>(aOut);
 242       for (uint32_t i = 0; i < aFrames; i++) {
 243         float sampL = 0.0;
 244         float sampR = 0.0;
 245         for (uint32_t j = 0; j < inChannels; j++) {
 246           sampL += in[i * inChannels + j] * dmatrix[inChannels - 3][j][0];
 247           sampR += in[i * inChannels + j] * dmatrix[inChannels - 3][j][1];
 248         }
 249         if (outChannels == 2) {
 250           *out++ = sampL;
 251           *out++ = sampR;
 252         } else {
 253           *out++ = (sampL + sampR) * 0.5;
 254         }
 255       }
 256     } else if (mIn.Format() == AudioConfig::FORMAT_S16) {
 257       // Downmix matrix. Per-row normalization 1 for rows 3,4 and 2 for rows
 258       // 5-8. Coefficients in Q14.
 259       static const int16_t dmatrix[6][8][2] = {
 260           /*3*/ {{9598, 0}, {0, 9598}, {6786, 6786}},
 261           /*4*/ {{6925, 0}, {0, 6925}, {5997, 3462}, {3462, 5997}},
 262           /*5*/
 263           {{10663, 0}, {0, 10663}, {7540, 7540}, {9234, 5331}, {5331, 9234}},
 264           /*6*/
 265           {{8668, 0},
 266            {0, 8668},
 267            {6129, 6129},
 268            {6129, 6129},
 269            {7507, 4335},
 270            {4335, 7507}},
 271           /*7*/
 272           {{7459, 0},
 273            {0, 7459},
 274            {5275, 5275},
 275            {5275, 5275},
 276            {4568, 4568},
 277            {6460, 3731},
 278            {3731, 6460}},
 279           /*8*/
 280           {{6368, 0},
 281            {0, 6368},
 282            {4502, 4502},
 283            {4502, 4502},
 284            {5514, 3184},
 285            {3184, 5514},
 286            {5514, 3184},
 287            {3184, 5514}}};
 288       // Re-write the buffer with downmixed data
 289       const int16_t* in = static_cast<const int16_t*>(aIn);
 290       int16_t* out = static_cast<int16_t*>(aOut);
 291       for (uint32_t i = 0; i < aFrames; i++) {
 292         int32_t sampL = 0;
 293         int32_t sampR = 0;
 294         for (uint32_t j = 0; j < inChannels; j++) {
 295           sampL += in[i * inChannels + j] * dmatrix[inChannels - 3][j][0];
 296           sampR += in[i * inChannels + j] * dmatrix[inChannels - 3][j][1];
 297         }
 298         sampL = clipTo15((sampL + 8192) >> 14);
 299         sampR = clipTo15((sampR + 8192) >> 14);
 300         if (outChannels == 2) {
 301           *out++ = sampL;
 302           *out++ = sampR;
 303         } else {
 304           *out++ = (sampL + sampR) * 0.5;
 305         }
 306       }
 307     } else {
 308       MOZ_DIAGNOSTIC_ASSERT(false, "Unsupported data type");
 309     }
 310     return aFrames;
 311   }
 312
 313   MOZ_DIAGNOSTIC_ASSERT(inChannels == 2 && outChannels == 1);
 314   if (mIn.Format() == AudioConfig::FORMAT_FLT) {
 315     const float* in = static_cast<const float*>(aIn);
 316     float* out = static_cast<float*>(aOut);
 317     for (size_t fIdx = 0; fIdx < aFrames; ++fIdx) {
 318       float sample = 0.0;
 319       // The sample of the buffer would be interleaved.
 320       sample = (in[fIdx * inChannels] + in[fIdx * inChannels + 1]) * 0.5;
 321       *out++ = sample;
 322     }
 323   } else if (mIn.Format() == AudioConfig::FORMAT_S16) {
 324     const int16_t* in = static_cast<const int16_t*>(aIn);
 325     int16_t* out = static_cast<int16_t*>(aOut);
 326     for (size_t fIdx = 0; fIdx < aFrames; ++fIdx) {
 327       int32_t sample = 0.0;
 328       // The sample of the buffer would be interleaved.
 329       sample = (in[fIdx * inChannels] + in[fIdx * inChannels + 1]) * 0.5;
 330       *out++ = sample;
 331     }
 332   } else {
 333     MOZ_DIAGNOSTIC_ASSERT(false, "Unsupported data type");
 334   }
 335   return aFrames;
 336 }
 337
 338 size_t AudioConverter::ResampleAudio(void* aOut, const void* aIn,
 339                                      size_t aFrames) {
 340   if (!mResampler) {
 341     return 0;
 342   }
 343   uint32_t outframes = ResampleRecipientFrames(aFrames);
 344   uint32_t inframes = aFrames;
 345
 346   int error;
 347   if (mOut.Format() == AudioConfig::FORMAT_FLT) {
 348     const float* in = reinterpret_cast<const float*>(aIn);
 349     float* out = reinterpret_cast<float*>(aOut);
 350     error = speex_resampler_process_interleaved_float(mResampler, in, &inframes,
 351                                                       out, &outframes);
 352   } else if (mOut.Format() == AudioConfig::FORMAT_S16) {
 353     const int16_t* in = reinterpret_cast<const int16_t*>(aIn);
 354     int16_t* out = reinterpret_cast<int16_t*>(aOut);
 355     error = speex_resampler_process_interleaved_int(mResampler, in, &inframes,
 356                                                     out, &outframes);
 357   } else {
 358     MOZ_DIAGNOSTIC_ASSERT(false, "Unsupported data type");
 359     error = RESAMPLER_ERR_ALLOC_FAILED;
 360   }
 361   MOZ_ASSERT(error == RESAMPLER_ERR_SUCCESS);
 362   if (error != RESAMPLER_ERR_SUCCESS) {
 363     speex_resampler_destroy(mResampler);
 364     mResampler = nullptr;
 365     return 0;
 366   }
 367   MOZ_ASSERT(inframes == aFrames, "Some frames will be dropped");
 368   return outframes;
 369 }
 370
 371 void AudioConverter::RecreateResampler() {
 372   if (mResampler) {
 373     speex_resampler_destroy(mResampler);
 374   }
 375   int error;
 376   mResampler = speex_resampler_init(mOut.Channels(), mIn.Rate(), mOut.Rate(),
 377                                     SPEEX_RESAMPLER_QUALITY_DEFAULT, &error);
 378
 379   if (error == RESAMPLER_ERR_SUCCESS) {
 380     speex_resampler_skip_zeros(mResampler);
 381   } else {
 382     NS_WARNING("Failed to initialize resampler.");
 383     mResampler = nullptr;
 384   }
 385 }
 386
 387 size_t AudioConverter::DrainResampler(void* aOut) {
 388   if (!mResampler) {
 389     return 0;
 390   }
 391   int frames = speex_resampler_get_input_latency(mResampler);
 392   AlignedByteBuffer buffer(FramesOutToBytes(frames));
 393   if (!buffer) {
 394     // OOM
 395     return 0;
 396   }
 397   frames = ResampleAudio(aOut, buffer.Data(), frames);
 398   // Tore down the resampler as it's easier than handling follow-up.
 399   RecreateResampler();
 400   return frames;
 401 }
 402
 403 size_t AudioConverter::UpmixAudio(void* aOut, const void* aIn,
 404                                   size_t aFrames) const {
 405   MOZ_ASSERT(mIn.Format() == AudioConfig::FORMAT_S16 ||
 406              mIn.Format() == AudioConfig::FORMAT_FLT);
 407   MOZ_ASSERT(mIn.Channels() < mOut.Channels());
 408   MOZ_ASSERT(mIn.Channels() == 1, "Can only upmix mono for now");
 409   MOZ_ASSERT(mOut.Channels() == 2, "Can only upmix to stereo for now");
 410
 411   if (!mIn.Layout().IsValid() || !mOut.Layout().IsValid() ||
 412       mOut.Channels() != 2) {
 413     // Dumb copy the channels and insert silence for the extra channels.
 414     if (mIn.Format() == AudioConfig::FORMAT_FLT) {
 415       dumbUpDownMix(static_cast<float*>(aOut), mOut.Channels(),
 416                     static_cast<const float*>(aIn), mIn.Channels(), aFrames);
 417     } else if (mIn.Format() == AudioConfig::FORMAT_S16) {
 418       dumbUpDownMix(static_cast<int16_t*>(aOut), mOut.Channels(),
 419                     static_cast<const int16_t*>(aIn), mIn.Channels(), aFrames);
 420     } else {
 421       MOZ_DIAGNOSTIC_ASSERT(false, "Unsupported data type");
 422     }
 423     return aFrames;
 424   }
 425
 426   // Upmix mono to stereo.
 427   // This is a very dumb mono to stereo upmixing, power levels are preserved
 428   // following the calculation: left = right = -3dB*mono.
 429   if (mIn.Format() == AudioConfig::FORMAT_FLT) {
 430     const float m3db = std::sqrt(0.5);  // -3dB = sqrt(1/2)
 431     const float* in = static_cast<const float*>(aIn);
 432     float* out = static_cast<float*>(aOut);
 433     for (size_t fIdx = 0; fIdx < aFrames; ++fIdx) {
 434       float sample = in[fIdx] * m3db;
 435       // The samples of the buffer would be interleaved.
 436       *out++ = sample;
 437       *out++ = sample;
 438     }
 439   } else if (mIn.Format() == AudioConfig::FORMAT_S16) {
 440     const int16_t* in = static_cast<const int16_t*>(aIn);
 441     int16_t* out = static_cast<int16_t*>(aOut);
 442     for (size_t fIdx = 0; fIdx < aFrames; ++fIdx) {
 443       int16_t sample =
 444           ((int32_t)in[fIdx] * 11585) >> 14;  // close enough to i*sqrt(0.5)
 445       // The samples of the buffer would be interleaved.
 446       *out++ = sample;
 447       *out++ = sample;
 448     }
 449   } else {
 450     MOZ_DIAGNOSTIC_ASSERT(false, "Unsupported data type");
 451   }
 452
 453   return aFrames;
 454 }
 455
 456 size_t AudioConverter::ResampleRecipientFrames(size_t aFrames) const {
 457   if (!aFrames && mIn.Rate() != mOut.Rate()) {
 458     if (!mResampler) {
 459       return 0;
 460     }
 461     // We drain by pushing in get_input_latency() samples of 0
 462     aFrames = speex_resampler_get_input_latency(mResampler);
 463   }
 464   return (uint64_t)aFrames * mOut.Rate() / mIn.Rate() + 1;
 465 }
 466
 467 size_t AudioConverter::FramesOutToSamples(size_t aFrames) const {
 468   return aFrames * mOut.Channels();
 469 }
 470
 471 size_t AudioConverter::SamplesInToFrames(size_t aSamples) const {
 472   return aSamples / mIn.Channels();
 473 }
 474
 475 size_t AudioConverter::FramesOutToBytes(size_t aFrames) const {
 476   return FramesOutToSamples(aFrames) * AudioConfig::SampleSize(mOut.Format());
 477 }
 478 }  // namespace mozilla