dom/media/encoder/Muxer.cpp

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
   2 /* This Source Code Form is subject to the terms of the Mozilla Public
   3  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
   4  * You can obtain one at http://mozilla.org/MPL/2.0/. */
   5
   6 #include "Muxer.h"
   7
   8 #include "ContainerWriter.h"
   9
  10 namespace mozilla {
  11
  12 LazyLogModule gMuxerLog("Muxer");
  13 #define LOG(type, ...) MOZ_LOG(gMuxerLog, type, (__VA_ARGS__))
  14
  15 Muxer::Muxer(UniquePtr<ContainerWriter> aWriter,
  16              MediaQueue<EncodedFrame>& aEncodedAudioQueue,
  17              MediaQueue<EncodedFrame>& aEncodedVideoQueue)
  18     : mEncodedAudioQueue(aEncodedAudioQueue),
  19       mEncodedVideoQueue(aEncodedVideoQueue),
  20       mWriter(std::move(aWriter)) {}
  21
  22 void Muxer::Disconnect() {
  23   mAudioPushListener.DisconnectIfExists();
  24   mAudioFinishListener.DisconnectIfExists();
  25   mVideoPushListener.DisconnectIfExists();
  26   mVideoFinishListener.DisconnectIfExists();
  27 }
  28
  29 bool Muxer::IsFinished() { return mWriter->IsWritingComplete(); }
  30
  31 nsresult Muxer::SetMetadata(
  32     const nsTArray<RefPtr<TrackMetadataBase>>& aMetadata) {
  33   MOZ_DIAGNOSTIC_ASSERT(!mMetadataSet);
  34   MOZ_DIAGNOSTIC_ASSERT(!mHasAudio);
  35   MOZ_DIAGNOSTIC_ASSERT(!mHasVideo);
  36   nsresult rv = mWriter->SetMetadata(aMetadata);
  37   if (NS_FAILED(rv)) {
  38     LOG(LogLevel::Error, "%p Setting metadata failed, tracks=%zu", this,
  39         aMetadata.Length());
  40     return rv;
  41   }
  42
  43   for (const auto& track : aMetadata) {
  44     switch (track->GetKind()) {
  45       case TrackMetadataBase::METADATA_OPUS:
  46       case TrackMetadataBase::METADATA_VORBIS:
  47       case TrackMetadataBase::METADATA_AAC:
  48       case TrackMetadataBase::METADATA_AMR:
  49       case TrackMetadataBase::METADATA_EVRC:
  50         MOZ_ASSERT(!mHasAudio, "Only one audio track supported");
  51         mHasAudio = true;
  52         break;
  53       case TrackMetadataBase::METADATA_VP8:
  54         MOZ_ASSERT(!mHasVideo, "Only one video track supported");
  55         mHasVideo = true;
  56         break;
  57       default:
  58         MOZ_CRASH("Unknown codec metadata");
  59     };
  60   }
  61   mMetadataSet = true;
  62   MOZ_ASSERT(mHasAudio || mHasVideo);
  63   LOG(LogLevel::Info, "%p Metadata set; audio=%d, video=%d", this, mHasAudio,
  64       mHasVideo);
  65   return NS_OK;
  66 }
  67
  68 nsresult Muxer::GetData(nsTArray<nsTArray<uint8_t>>* aOutputBuffers) {
  69   MOZ_ASSERT(mHasAudio || mHasVideo);
  70
  71   nsresult rv;
  72   if (!mMetadataEncoded) {
  73     rv = mWriter->GetContainerData(aOutputBuffers, ContainerWriter::GET_HEADER);
  74     if (NS_FAILED(rv)) {
  75       LOG(LogLevel::Error, "%p Failed getting metadata from writer", this);
  76       return rv;
  77     }
  78     mMetadataEncoded = true;
  79   }
  80
  81   if (mEncodedAudioQueue.GetSize() == 0 && !mEncodedAudioQueue.IsFinished() &&
  82       mEncodedVideoQueue.GetSize() == 0 && !mEncodedVideoQueue.IsFinished()) {
  83     // Nothing to mux.
  84     return NS_OK;
  85   }
  86
  87   rv = Mux();
  88   if (NS_FAILED(rv)) {
  89     LOG(LogLevel::Error, "%p Failed muxing data into writer", this);
  90     return rv;
  91   }
  92
  93   MOZ_ASSERT_IF(
  94       mEncodedAudioQueue.IsFinished() && mEncodedVideoQueue.IsFinished(),
  95       mEncodedAudioQueue.AtEndOfStream());
  96   MOZ_ASSERT_IF(
  97       mEncodedAudioQueue.IsFinished() && mEncodedVideoQueue.IsFinished(),
  98       mEncodedVideoQueue.AtEndOfStream());
  99   uint32_t flags =
 100       mEncodedAudioQueue.AtEndOfStream() && mEncodedVideoQueue.AtEndOfStream()
 101           ? ContainerWriter::FLUSH_NEEDED
 102           : 0;
 103
 104   if (mEncodedAudioQueue.AtEndOfStream() &&
 105       mEncodedVideoQueue.AtEndOfStream()) {
 106     LOG(LogLevel::Info, "%p All data written", this);
 107   }
 108
 109   return mWriter->GetContainerData(aOutputBuffers, flags);
 110 }
 111
 112 nsresult Muxer::Mux() {
 113   MOZ_ASSERT(mMetadataSet);
 114   MOZ_ASSERT(mHasAudio || mHasVideo);
 115
 116   nsTArray<RefPtr<EncodedFrame>> frames;
 117   // The times at which we expect our next video and audio frames. These are
 118   // based on the time + duration (GetEndTime()) of the last seen frames.
 119   // Assumes that the encoders write the correct duration for frames.;
 120   media::TimeUnit expectedNextVideoTime;
 121   media::TimeUnit expectedNextAudioTime;
 122   // Interleave frames until we're out of audio or video
 123   while (mEncodedVideoQueue.GetSize() > 0 && mEncodedAudioQueue.GetSize() > 0) {
 124     RefPtr<EncodedFrame> videoFrame = mEncodedVideoQueue.PeekFront();
 125     RefPtr<EncodedFrame> audioFrame = mEncodedAudioQueue.PeekFront();
 126     // For any expected time our frames should occur at or after that time.
 127     MOZ_ASSERT(videoFrame->mTime >= expectedNextVideoTime);
 128     MOZ_ASSERT(audioFrame->mTime >= expectedNextAudioTime);
 129     if (videoFrame->mTime <= audioFrame->mTime) {
 130       expectedNextVideoTime = videoFrame->GetEndTime();
 131       RefPtr<EncodedFrame> frame = mEncodedVideoQueue.PopFront();
 132       frames.AppendElement(std::move(frame));
 133     } else {
 134       expectedNextAudioTime = audioFrame->GetEndTime();
 135       RefPtr<EncodedFrame> frame = mEncodedAudioQueue.PopFront();
 136       frames.AppendElement(std::move(frame));
 137     }
 138   }
 139
 140   // If we're out of audio we still may be able to add more video...
 141   if (mEncodedAudioQueue.GetSize() == 0) {
 142     while (mEncodedVideoQueue.GetSize() > 0) {
 143       if (!mEncodedAudioQueue.AtEndOfStream() &&
 144           mEncodedVideoQueue.PeekFront()->mTime > expectedNextAudioTime) {
 145         // Audio encoding is not complete and since the video frame comes
 146         // after our next audio frame we cannot safely add it.
 147         break;
 148       }
 149       frames.AppendElement(mEncodedVideoQueue.PopFront());
 150     }
 151   }
 152
 153   // If we're out of video we still may be able to add more audio...
 154   if (mEncodedVideoQueue.GetSize() == 0) {
 155     while (mEncodedAudioQueue.GetSize() > 0) {
 156       if (!mEncodedVideoQueue.AtEndOfStream() &&
 157           mEncodedAudioQueue.PeekFront()->mTime > expectedNextVideoTime) {
 158         // Video encoding is not complete and since the audio frame comes
 159         // after our next video frame we cannot safely add it.
 160         break;
 161       }
 162       frames.AppendElement(mEncodedAudioQueue.PopFront());
 163     }
 164   }
 165
 166   LOG(LogLevel::Debug,
 167       "%p Muxed data, remaining-audio=%zu, remaining-video=%zu", this,
 168       mEncodedAudioQueue.GetSize(), mEncodedVideoQueue.GetSize());
 169
 170   // If encoding is complete for both encoders we should signal end of stream,
 171   // otherwise we keep going.
 172   uint32_t flags =
 173       mEncodedVideoQueue.AtEndOfStream() && mEncodedAudioQueue.AtEndOfStream()
 174           ? ContainerWriter::END_OF_STREAM
 175           : 0;
 176   nsresult rv = mWriter->WriteEncodedTrack(frames, flags);
 177   if (NS_FAILED(rv)) {
 178     LOG(LogLevel::Error, "Error! Failed to write muxed data to the container");
 179   }
 180   return rv;
 181 }
 182
 183 }  // namespace mozilla
 184
 185 #undef LOG