dom/media/TrackUnionStream.cpp

   1 /* This Source Code Form is subject to the terms of the Mozilla Public
   2  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
   3  * You can obtain one at http://mozilla.org/MPL/2.0/. */
   4
   5 #include "MediaStreamGraphImpl.h"
   6 #include "MediaStreamListener.h"
   7 #include "mozilla/MathAlgorithms.h"
   8 #include "mozilla/Unused.h"
   9
  10 #include "AudioSegment.h"
  11 #include "VideoSegment.h"
  12 #include "nsContentUtils.h"
  13 #include "nsIAppShell.h"
  14 #include "nsIObserver.h"
  15 #include "nsPrintfCString.h"
  16 #include "nsServiceManagerUtils.h"
  17 #include "nsWidgetsCID.h"
  18 #include "prerror.h"
  19 #include "mozilla/Logging.h"
  20 #include "mozilla/Attributes.h"
  21 #include "TrackUnionStream.h"
  22 #include "ImageContainer.h"
  23 #include "AudioChannelService.h"
  24 #include "AudioNodeEngine.h"
  25 #include "AudioNodeStream.h"
  26 #include "AudioNodeExternalInputStream.h"
  27 #include "webaudio/MediaStreamAudioDestinationNode.h"
  28 #include <algorithm>
  29 #include "DOMMediaStream.h"
  30 #include "GeckoProfiler.h"
  31
  32 using namespace mozilla::layers;
  33 using namespace mozilla::dom;
  34 using namespace mozilla::gfx;
  35
  36 namespace mozilla {
  37
  38 #ifdef STREAM_LOG
  39 #  undef STREAM_LOG
  40 #endif
  41
  42 LazyLogModule gTrackUnionStreamLog("TrackUnionStream");
  43 #define STREAM_LOG(type, msg) MOZ_LOG(gTrackUnionStreamLog, type, msg)
  44
  45 TrackUnionStream::TrackUnionStream()
  46     : ProcessedMediaStream(), mNextAvailableTrackID(1) {}
  47
  48 void TrackUnionStream::RemoveInput(MediaInputPort* aPort) {
  49   STREAM_LOG(LogLevel::Debug,
  50              ("TrackUnionStream %p removing input %p", this, aPort));
  51   for (int32_t i = mTrackMap.Length() - 1; i >= 0; --i) {
  52     if (mTrackMap[i].mInputPort == aPort) {
  53       STREAM_LOG(LogLevel::Debug,
  54                  ("TrackUnionStream %p removing trackmap entry %d", this, i));
  55       nsTArray<RefPtr<DirectMediaStreamTrackListener>> listeners(
  56           mTrackMap[i].mOwnedDirectListeners);
  57       for (auto listener : listeners) {
  58         // Remove listeners while the entry still exists.
  59         RemoveDirectTrackListenerImpl(listener, mTrackMap[i].mOutputTrackID);
  60       }
  61       EndTrack(i);
  62       mTrackMap.RemoveElementAt(i);
  63     }
  64   }
  65   ProcessedMediaStream::RemoveInput(aPort);
  66 }
  67 void TrackUnionStream::ProcessInput(GraphTime aFrom, GraphTime aTo,
  68                                     uint32_t aFlags) {
  69   TRACE_AUDIO_CALLBACK_COMMENT("TrackUnionStream %p", this);
  70   if (IsFinishedOnGraphThread()) {
  71     return;
  72   }
  73   AutoTArray<bool, 8> mappedTracksFinished;
  74   AutoTArray<bool, 8> mappedTracksWithMatchingInputTracks;
  75   for (uint32_t i = 0; i < mTrackMap.Length(); ++i) {
  76     mappedTracksFinished.AppendElement(true);
  77     mappedTracksWithMatchingInputTracks.AppendElement(false);
  78   }
  79
  80   AutoTArray<MediaInputPort*, 32> inputs(mInputs);
  81   inputs.AppendElements(mSuspendedInputs);
  82
  83   bool allFinished = !inputs.IsEmpty();
  84   for (uint32_t i = 0; i < inputs.Length(); ++i) {
  85     MediaStream* stream = inputs[i]->GetSource();
  86     if (!stream->IsFinishedOnGraphThread()) {
  87       // XXX we really should check whether 'stream' has finished within time
  88       // aTo, not just that it's finishing when all its queued data eventually
  89       // runs out.
  90       allFinished = false;
  91     }
  92     for (StreamTracks::TrackIter tracks(stream->GetStreamTracks());
  93          !tracks.IsEnded(); tracks.Next()) {
  94       bool found = false;
  95       for (uint32_t j = 0; j < mTrackMap.Length(); ++j) {
  96         TrackMapEntry* map = &mTrackMap[j];
  97         if (map->mInputPort == inputs[i] &&
  98             map->mInputTrackID == tracks->GetID()) {
  99           bool trackFinished = false;
 100           StreamTracks::Track* outputTrack =
 101               mTracks.FindTrack(map->mOutputTrackID);
 102           found = true;
 103           if (!outputTrack || outputTrack->IsEnded() ||
 104               !inputs[i]->PassTrackThrough(tracks->GetID())) {
 105             trackFinished = true;
 106           } else {
 107             CopyTrackData(tracks.get(), j, aFrom, aTo, &trackFinished);
 108           }
 109           mappedTracksFinished[j] = trackFinished;
 110           mappedTracksWithMatchingInputTracks[j] = true;
 111           break;
 112         }
 113       }
 114       if (!found && inputs[i]->AllowCreationOf(tracks->GetID())) {
 115         bool trackFinished = false;
 116         uint32_t mapIndex = AddTrack(inputs[i], tracks.get(), aFrom);
 117         CopyTrackData(tracks.get(), mapIndex, aFrom, aTo, &trackFinished);
 118         mappedTracksFinished.AppendElement(trackFinished);
 119         mappedTracksWithMatchingInputTracks.AppendElement(true);
 120       }
 121     }
 122   }
 123   for (int32_t i = mTrackMap.Length() - 1; i >= 0; --i) {
 124     if (mappedTracksFinished[i]) {
 125       EndTrack(i);
 126     } else {
 127       allFinished = false;
 128     }
 129     if (!mappedTracksWithMatchingInputTracks[i]) {
 130       for (auto listener : mTrackMap[i].mOwnedDirectListeners) {
 131         // Remove listeners while the entry still exists.
 132         RemoveDirectTrackListenerImpl(listener, mTrackMap[i].mOutputTrackID);
 133       }
 134       mTrackMap.RemoveElementAt(i);
 135     }
 136   }
 137   if (allFinished && mAutofinish && (aFlags & ALLOW_FINISH)) {
 138     // All streams have finished and won't add any more tracks, and
 139     // all our tracks have actually finished and been removed from our map,
 140     // so we're finished now.
 141     FinishOnGraphThread();
 142   }
 143 }
 144
 145 uint32_t TrackUnionStream::AddTrack(MediaInputPort* aPort,
 146                                     StreamTracks::Track* aTrack,
 147                                     GraphTime aFrom) {
 148   STREAM_LOG(LogLevel::Verbose,
 149              ("TrackUnionStream %p adding track %d for "
 150               "input stream %p track %d, desired id %d",
 151               this, aTrack->GetID(), aPort->GetSource(), aTrack->GetID(),
 152               aPort->GetDestinationTrackId()));
 153
 154   TrackID id;
 155   if (IsTrackIDExplicit(id = aPort->GetDestinationTrackId())) {
 156     MOZ_ASSERT(id >= mNextAvailableTrackID && !mUsedTracks.ContainsSorted(id),
 157                "Desired destination id taken. Only provide a destination ID "
 158                "if you can assure its availability, or we may not be able "
 159                "to bind to the correct DOM-side track.");
 160 #ifdef DEBUG
 161     AutoTArray<MediaInputPort*, 32> inputs(mInputs);
 162     inputs.AppendElements(mSuspendedInputs);
 163     for (size_t i = 0; inputs[i] != aPort; ++i) {
 164       MOZ_ASSERT(inputs[i]->GetSourceTrackId() != TRACK_ANY,
 165                  "You are adding a MediaInputPort with a track mapping "
 166                  "while there already exist generic MediaInputPorts for this "
 167                  "destination stream. This can lead to TrackID collisions!");
 168     }
 169 #endif
 170     mUsedTracks.InsertElementSorted(id);
 171   } else if ((id = aTrack->GetID()) && id > mNextAvailableTrackID &&
 172              !mUsedTracks.ContainsSorted(id)) {
 173     // Input id available. Mark it used in mUsedTracks.
 174     mUsedTracks.InsertElementSorted(id);
 175   } else {
 176     // No desired destination id and Input id taken, allocate a new one.
 177     id = mNextAvailableTrackID;
 178
 179     // Update mNextAvailableTrackID and prune any mUsedTracks members it now
 180     // covers.
 181     while (1) {
 182       if (!mUsedTracks.RemoveElementSorted(++mNextAvailableTrackID)) {
 183         // Not in use. We're done.
 184         break;
 185       }
 186     }
 187   }
 188
 189   // Round up the track start time so the track, if anything, starts a
 190   // little later than the true time. This means we'll have enough
 191   // samples in our input stream to go just beyond the destination time.
 192   StreamTime outputStart = GraphTimeToStreamTimeWithBlocking(aFrom);
 193
 194   nsAutoPtr<MediaSegment> segment;
 195   segment = aTrack->GetSegment()->CreateEmptyClone();
 196   segment->AppendNullData(outputStart);
 197   StreamTracks::Track* track =
 198       &mTracks.AddTrack(id, outputStart, segment.forget());
 199   STREAM_LOG(LogLevel::Debug, ("TrackUnionStream %p added track %d for input "
 200                                "stream %p track %d, start ticks %lld",
 201                                this, track->GetID(), aPort->GetSource(),
 202                                aTrack->GetID(), (long long)outputStart));
 203
 204   TrackMapEntry* map = mTrackMap.AppendElement();
 205   map->mEndOfConsumedInputTicks = 0;
 206   map->mEndOfLastInputIntervalInInputStream = -1;
 207   map->mEndOfLastInputIntervalInOutputStream = -1;
 208   map->mInputPort = aPort;
 209   map->mInputTrackID = aTrack->GetID();
 210   map->mOutputTrackID = track->GetID();
 211   map->mSegment = aTrack->GetSegment()->CreateEmptyClone();
 212
 213   for (int32_t i = mPendingDirectTrackListeners.Length() - 1; i >= 0; --i) {
 214     TrackBound<DirectMediaStreamTrackListener>& bound =
 215         mPendingDirectTrackListeners[i];
 216     if (bound.mTrackID != map->mOutputTrackID) {
 217       continue;
 218     }
 219     MediaStream* source = map->mInputPort->GetSource();
 220     map->mOwnedDirectListeners.AppendElement(bound.mListener);
 221     DisabledTrackMode currentMode = GetDisabledTrackMode(bound.mTrackID);
 222     if (currentMode != DisabledTrackMode::ENABLED) {
 223       bound.mListener->IncreaseDisabled(currentMode);
 224     }
 225     STREAM_LOG(LogLevel::Debug, ("TrackUnionStream %p adding direct listener "
 226                                  "%p for track %d. Forwarding to input "
 227                                  "stream %p track %d.",
 228                                  this, bound.mListener.get(), bound.mTrackID,
 229                                  source, map->mInputTrackID));
 230     source->AddDirectTrackListenerImpl(bound.mListener.forget(),
 231                                        map->mInputTrackID);
 232     mPendingDirectTrackListeners.RemoveElementAt(i);
 233   }
 234
 235   return mTrackMap.Length() - 1;
 236 }
 237
 238 void TrackUnionStream::EndTrack(uint32_t aIndex) {
 239   StreamTracks::Track* outputTrack =
 240       mTracks.FindTrack(mTrackMap[aIndex].mOutputTrackID);
 241   if (!outputTrack || outputTrack->IsEnded()) return;
 242   STREAM_LOG(LogLevel::Debug, ("TrackUnionStream %p ending track %d", this,
 243                                outputTrack->GetID()));
 244   outputTrack->SetEnded();
 245 }
 246
 247 void TrackUnionStream::CopyTrackData(StreamTracks::Track* aInputTrack,
 248                                      uint32_t aMapIndex, GraphTime aFrom,
 249                                      GraphTime aTo,
 250                                      bool* aOutputTrackFinished) {
 251   TrackMapEntry* map = &mTrackMap[aMapIndex];
 252   TRACE_AUDIO_CALLBACK_COMMENT(
 253       "Input stream %p track %i -> TrackUnionStream %p track %i",
 254       map->mInputPort->GetSource(), map->mInputTrackID, this,
 255       map->mOutputTrackID);
 256   StreamTracks::Track* outputTrack = mTracks.FindTrack(map->mOutputTrackID);
 257   MOZ_ASSERT(outputTrack && !outputTrack->IsEnded(),
 258              "Can't copy to ended track");
 259
 260   MediaSegment* segment = map->mSegment;
 261   MediaStream* source = map->mInputPort->GetSource();
 262
 263   GraphTime next;
 264   *aOutputTrackFinished = false;
 265   for (GraphTime t = aFrom; t < aTo; t = next) {
 266     MediaInputPort::InputInterval interval =
 267         map->mInputPort->GetNextInputInterval(t);
 268     interval.mEnd = std::min(interval.mEnd, aTo);
 269     StreamTime inputEnd =
 270         source->GraphTimeToStreamTimeWithBlocking(interval.mEnd);
 271
 272     if (aInputTrack->IsEnded() && aInputTrack->GetEnd() <= inputEnd) {
 273       *aOutputTrackFinished = true;
 274       break;
 275     }
 276
 277     if (interval.mStart >= interval.mEnd) {
 278       break;
 279     }
 280     StreamTime ticks = interval.mEnd - interval.mStart;
 281     next = interval.mEnd;
 282
 283     StreamTime outputStart = outputTrack->GetEnd();
 284
 285     if (interval.mInputIsBlocked) {
 286       segment->AppendNullData(ticks);
 287       STREAM_LOG(
 288           LogLevel::Verbose,
 289           ("TrackUnionStream %p appending %lld ticks of null data to track %d",
 290            this, (long long)ticks, outputTrack->GetID()));
 291     } else if (InMutedCycle()) {
 292       segment->AppendNullData(ticks);
 293     } else {
 294       if (source->IsSuspended()) {
 295         segment->AppendNullData(aTo - aFrom);
 296       } else {
 297         MOZ_ASSERT(outputTrack->GetEnd() ==
 298                        GraphTimeToStreamTimeWithBlocking(interval.mStart),
 299                    "Samples missing");
 300         StreamTime inputStart =
 301             source->GraphTimeToStreamTimeWithBlocking(interval.mStart);
 302         segment->AppendSlice(*aInputTrack->GetSegment(), inputStart, inputEnd);
 303       }
 304     }
 305     ApplyTrackDisabling(outputTrack->GetID(), segment);
 306     for (TrackBound<MediaStreamTrackListener>& b : mTrackListeners) {
 307       if (b.mTrackID != outputTrack->GetID()) {
 308         continue;
 309       }
 310       b.mListener->NotifyQueuedChanges(Graph(), outputStart, *segment);
 311     }
 312     outputTrack->GetSegment()->AppendFrom(segment);
 313   }
 314 }
 315
 316 void TrackUnionStream::SetTrackEnabledImpl(TrackID aTrackID,
 317                                            DisabledTrackMode aMode) {
 318   bool enabled = aMode == DisabledTrackMode::ENABLED;
 319   for (TrackMapEntry& entry : mTrackMap) {
 320     if (entry.mOutputTrackID == aTrackID) {
 321       STREAM_LOG(LogLevel::Info,
 322                  ("TrackUnionStream %p track %d was explicitly %s", this,
 323                   aTrackID, enabled ? "enabled" : "disabled"));
 324       for (DirectMediaStreamTrackListener* listener :
 325            entry.mOwnedDirectListeners) {
 326         DisabledTrackMode oldMode = GetDisabledTrackMode(aTrackID);
 327         bool oldEnabled = oldMode == DisabledTrackMode::ENABLED;
 328         if (!oldEnabled && enabled) {
 329           STREAM_LOG(LogLevel::Debug, ("TrackUnionStream %p track %d setting "
 330                                        "direct listener enabled",
 331                                        this, aTrackID));
 332           listener->DecreaseDisabled(oldMode);
 333         } else if (oldEnabled && !enabled) {
 334           STREAM_LOG(LogLevel::Debug, ("TrackUnionStream %p track %d setting "
 335                                        "direct listener disabled",
 336                                        this, aTrackID));
 337           listener->IncreaseDisabled(aMode);
 338         }
 339       }
 340     }
 341   }
 342   MediaStream::SetTrackEnabledImpl(aTrackID, aMode);
 343 }
 344
 345 MediaStream* TrackUnionStream::GetInputStreamFor(TrackID aTrackID) {
 346   for (TrackMapEntry& entry : mTrackMap) {
 347     if (entry.mOutputTrackID == aTrackID && entry.mInputPort) {
 348       return entry.mInputPort->GetSource();
 349     }
 350   }
 351
 352   return nullptr;
 353 }
 354
 355 TrackID TrackUnionStream::GetInputTrackIDFor(TrackID aTrackID) {
 356   for (TrackMapEntry& entry : mTrackMap) {
 357     if (entry.mOutputTrackID == aTrackID) {
 358       return entry.mInputTrackID;
 359     }
 360   }
 361
 362   return TRACK_NONE;
 363 }
 364
 365 void TrackUnionStream::AddDirectTrackListenerImpl(
 366     already_AddRefed<DirectMediaStreamTrackListener> aListener,
 367     TrackID aTrackID) {
 368   RefPtr<DirectMediaStreamTrackListener> listener = aListener;
 369
 370   for (TrackMapEntry& entry : mTrackMap) {
 371     if (entry.mOutputTrackID == aTrackID) {
 372       MediaStream* source = entry.mInputPort->GetSource();
 373       STREAM_LOG(LogLevel::Debug,
 374                  ("TrackUnionStream %p adding direct listener "
 375                   "%p for track %d. Forwarding to input "
 376                   "stream %p track %d.",
 377                   this, listener.get(), aTrackID, source, entry.mInputTrackID));
 378       entry.mOwnedDirectListeners.AppendElement(listener);
 379       DisabledTrackMode currentMode = GetDisabledTrackMode(aTrackID);
 380       if (currentMode != DisabledTrackMode::ENABLED) {
 381         listener->IncreaseDisabled(currentMode);
 382       }
 383       source->AddDirectTrackListenerImpl(listener.forget(),
 384                                          entry.mInputTrackID);
 385       return;
 386     }
 387   }
 388
 389   TrackBound<DirectMediaStreamTrackListener>* bound =
 390       mPendingDirectTrackListeners.AppendElement();
 391   bound->mListener = listener.forget();
 392   bound->mTrackID = aTrackID;
 393 }
 394
 395 void TrackUnionStream::RemoveDirectTrackListenerImpl(
 396     DirectMediaStreamTrackListener* aListener, TrackID aTrackID) {
 397   for (TrackMapEntry& entry : mTrackMap) {
 398     // OutputTrackID is unique to this stream so we only need to do this once.
 399     if (entry.mOutputTrackID != aTrackID) {
 400       continue;
 401     }
 402     for (size_t i = 0; i < entry.mOwnedDirectListeners.Length(); ++i) {
 403       if (entry.mOwnedDirectListeners[i] == aListener) {
 404         STREAM_LOG(LogLevel::Debug,
 405                    ("TrackUnionStream %p removing direct "
 406                     "listener %p for track %d, forwarding "
 407                     "to input stream %p track %d",
 408                     this, aListener, aTrackID, entry.mInputPort->GetSource(),
 409                     entry.mInputTrackID));
 410         DisabledTrackMode currentMode = GetDisabledTrackMode(aTrackID);
 411         if (currentMode != DisabledTrackMode::ENABLED) {
 412           // Reset the listener's state.
 413           aListener->DecreaseDisabled(currentMode);
 414         }
 415         entry.mOwnedDirectListeners.RemoveElementAt(i);
 416         break;
 417       }
 418     }
 419     // Forward to the input
 420     MediaStream* source = entry.mInputPort->GetSource();
 421     source->RemoveDirectTrackListenerImpl(aListener, entry.mInputTrackID);
 422     return;
 423   }
 424
 425   for (size_t i = 0; i < mPendingDirectTrackListeners.Length(); ++i) {
 426     TrackBound<DirectMediaStreamTrackListener>& bound =
 427         mPendingDirectTrackListeners[i];
 428     if (bound.mListener == aListener && bound.mTrackID == aTrackID) {
 429       mPendingDirectTrackListeners.RemoveElementAt(i);
 430       return;
 431     }
 432   }
 433 }
 434
 435 void TrackUnionStream::RemoveAllDirectListenersImpl() {
 436   for (TrackMapEntry& entry : mTrackMap) {
 437     nsTArray<RefPtr<DirectMediaStreamTrackListener>> listeners(
 438         entry.mOwnedDirectListeners);
 439     for (const auto& listener : listeners) {
 440       RemoveDirectTrackListenerImpl(listener, entry.mOutputTrackID);
 441     }
 442     MOZ_DIAGNOSTIC_ASSERT(entry.mOwnedDirectListeners.IsEmpty());
 443   }
 444
 445   nsTArray<TrackBound<DirectMediaStreamTrackListener>> boundListeners(
 446       mPendingDirectTrackListeners);
 447   for (const auto& binding : boundListeners) {
 448     RemoveDirectTrackListenerImpl(binding.mListener, binding.mTrackID);
 449   }
 450   MOZ_DIAGNOSTIC_ASSERT(mPendingDirectTrackListeners.IsEmpty());
 451 }
 452
 453 }  // namespace mozilla