dom/media/AudioCaptureStream.cpp

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
   2 /* This Source Code Form is subject to the terms of the Mozilla Public
   3  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
   4  * You can obtain one at http://mozilla.org/MPL/2.0/. */
   5
   6 #include "MediaStreamGraphImpl.h"
   7 #include "MediaStreamListener.h"
   8 #include "mozilla/MathAlgorithms.h"
   9 #include "mozilla/Unused.h"
  10
  11 #include "AudioSegment.h"
  12 #include "mozilla/Logging.h"
  13 #include "mozilla/Attributes.h"
  14 #include "AudioCaptureStream.h"
  15 #include "ImageContainer.h"
  16 #include "AudioNodeEngine.h"
  17 #include "AudioNodeStream.h"
  18 #include "AudioNodeExternalInputStream.h"
  19 #include "webaudio/MediaStreamAudioDestinationNode.h"
  20 #include <algorithm>
  21 #include "DOMMediaStream.h"
  22
  23 using namespace mozilla::layers;
  24 using namespace mozilla::dom;
  25 using namespace mozilla::gfx;
  26
  27 namespace mozilla
  28 {
  29
  30 // We are mixing to mono until PeerConnection can accept stereo
  31 static const uint32_t MONO = 1;
  32
  33 AudioCaptureStream::AudioCaptureStream(TrackID aTrackId)
  34   : ProcessedMediaStream()
  35   , mTrackId(aTrackId)
  36   , mStarted(false)
  37   , mTrackCreated(false)
  38 {
  39   MOZ_ASSERT(NS_IsMainThread());
  40   MOZ_COUNT_CTOR(AudioCaptureStream);
  41   mMixer.AddCallback(this);
  42 }
  43
  44 AudioCaptureStream::~AudioCaptureStream()
  45 {
  46   MOZ_COUNT_DTOR(AudioCaptureStream);
  47   mMixer.RemoveCallback(this);
  48 }
  49
  50 void
  51 AudioCaptureStream::Start()
  52 {
  53   class Message : public ControlMessage {
  54   public:
  55     explicit Message(AudioCaptureStream* aStream)
  56       : ControlMessage(aStream), mStream(aStream) {}
  57
  58     virtual void Run()
  59     {
  60       mStream->mStarted = true;
  61     }
  62
  63   protected:
  64     AudioCaptureStream* mStream;
  65   };
  66   GraphImpl()->AppendMessage(MakeUnique<Message>(this));
  67 }
  68
  69 void
  70 AudioCaptureStream::ProcessInput(GraphTime aFrom, GraphTime aTo,
  71                                  uint32_t aFlags)
  72 {
  73   if (!mStarted) {
  74     return;
  75   }
  76
  77   uint32_t inputCount = mInputs.Length();
  78   StreamTracks::Track* track = EnsureTrack(mTrackId);
  79   // Notify the DOM everything is in order.
  80   if (!mTrackCreated) {
  81     for (uint32_t i = 0; i < mListeners.Length(); i++) {
  82       MediaStreamListener* l = mListeners[i];
  83       AudioSegment tmp;
  84       l->NotifyQueuedTrackChanges(
  85         Graph(), mTrackId, 0, TrackEventCommand::TRACK_EVENT_CREATED, tmp);
  86       l->NotifyFinishedTrackCreation(Graph());
  87     }
  88     mTrackCreated = true;
  89   }
  90
  91   if (IsFinishedOnGraphThread()) {
  92     return;
  93   }
  94
  95   // If the captured stream is connected back to a object on the page (be it an
  96   // HTMLMediaElement with a stream as source, or an AudioContext), a cycle
  97   // situation occur. This can work if it's an AudioContext with at least one
  98   // DelayNode, but the MSG will mute the whole cycle otherwise.
  99   if (InMutedCycle() || inputCount == 0) {
 100     track->Get<AudioSegment>()->AppendNullData(aTo - aFrom);
 101   } else {
 102     // We mix down all the tracks of all inputs, to a stereo track. Everything
 103     // is {up,down}-mixed to stereo.
 104     mMixer.StartMixing();
 105     AudioSegment output;
 106     for (uint32_t i = 0; i < inputCount; i++) {
 107       MediaStream* s = mInputs[i]->GetSource();
 108       StreamTracks::TrackIter track(s->GetStreamTracks(), MediaSegment::AUDIO);
 109       if (track.IsEnded()) {
 110         // No tracks for this input. Still we append data to trigger the mixer.
 111         AudioSegment toMix;
 112         toMix.AppendNullData(aTo - aFrom);
 113         toMix.Mix(mMixer, MONO, Graph()->GraphRate());
 114       }
 115       for (; !track.IsEnded(); track.Next()) {
 116         AudioSegment* inputSegment = track->Get<AudioSegment>();
 117         StreamTime inputStart = s->GraphTimeToStreamTimeWithBlocking(aFrom);
 118         StreamTime inputEnd = s->GraphTimeToStreamTimeWithBlocking(aTo);
 119         AudioSegment toMix;
 120         if (track->IsEnded() && inputSegment->GetDuration() <= inputStart) {
 121           toMix.AppendNullData(aTo - aFrom);
 122         } else {
 123           toMix.AppendSlice(*inputSegment, inputStart, inputEnd);
 124           // Care for streams blocked in the [aTo, aFrom] range.
 125           if (inputEnd - inputStart < aTo - aFrom) {
 126             toMix.AppendNullData((aTo - aFrom) - (inputEnd - inputStart));
 127           }
 128         }
 129         toMix.Mix(mMixer, MONO, Graph()->GraphRate());
 130       }
 131     }
 132     // This calls MixerCallback below
 133     mMixer.FinishMixing();
 134   }
 135
 136   // Regardless of the status of the input tracks, we go foward.
 137   mTracks.AdvanceKnownTracksTime(GraphTimeToStreamTimeWithBlocking((aTo)));
 138 }
 139
 140 void
 141 AudioCaptureStream::MixerCallback(AudioDataValue* aMixedBuffer,
 142                                   AudioSampleFormat aFormat, uint32_t aChannels,
 143                                   uint32_t aFrames, uint32_t aSampleRate)
 144 {
 145   AutoTArray<nsTArray<AudioDataValue>, MONO> output;
 146   AutoTArray<const AudioDataValue*, MONO> bufferPtrs;
 147   output.SetLength(MONO);
 148   bufferPtrs.SetLength(MONO);
 149
 150   uint32_t written = 0;
 151   // We need to copy here, because the mixer will reuse the storage, we should
 152   // not hold onto it. Buffers are in planar format.
 153   for (uint32_t channel = 0; channel < aChannels; channel++) {
 154     AudioDataValue* out = output[channel].AppendElements(aFrames);
 155     PodCopy(out, aMixedBuffer + written, aFrames);
 156     bufferPtrs[channel] = out;
 157     written += aFrames;
 158   }
 159   AudioChunk chunk;
 160   chunk.mBuffer = new mozilla::SharedChannelArrayBuffer<AudioDataValue>(&output);
 161   chunk.mDuration = aFrames;
 162   chunk.mBufferFormat = aFormat;
 163   chunk.mChannelData.SetLength(MONO);
 164   for (uint32_t channel = 0; channel < aChannels; channel++) {
 165     chunk.mChannelData[channel] = bufferPtrs[channel];
 166   }
 167
 168   // Now we have mixed data, simply append it to out track.
 169   EnsureTrack(mTrackId)->Get<AudioSegment>()->AppendAndConsumeChunk(&chunk);
 170 }
 171 }