1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
3 * You can obtain one at http://mozilla.org/MPL/2.0/. */
5 #include "MediaStreamGraphImpl.h"
6 #include "MediaStreamListener.h"
7 #include "mozilla/MathAlgorithms.h"
8 #include "mozilla/Unused.h"
10 #include "AudioSegment.h"
11 #include "VideoSegment.h"
12 #include "nsContentUtils.h"
13 #include "nsIAppShell.h"
14 #include "nsIObserver.h"
15 #include "nsPrintfCString.h"
16 #include "nsServiceManagerUtils.h"
17 #include "nsWidgetsCID.h"
19 #include "mozilla/Logging.h"
20 #include "mozilla/Attributes.h"
21 #include "TrackUnionStream.h"
22 #include "ImageContainer.h"
23 #include "AudioChannelService.h"
24 #include "AudioNodeEngine.h"
25 #include "AudioNodeStream.h"
26 #include "AudioNodeExternalInputStream.h"
27 #include "webaudio/MediaStreamAudioDestinationNode.h"
29 #include "DOMMediaStream.h"
30 #include "GeckoProfiler.h"
32 using namespace mozilla::layers
;
33 using namespace mozilla::dom
;
34 using namespace mozilla::gfx
;
42 LazyLogModule
gTrackUnionStreamLog("TrackUnionStream");
43 #define STREAM_LOG(type, msg) MOZ_LOG(gTrackUnionStreamLog, type, msg)
45 TrackUnionStream::TrackUnionStream()
46 : ProcessedMediaStream(), mNextAvailableTrackID(1) {}
48 void TrackUnionStream::RemoveInput(MediaInputPort
* aPort
) {
49 STREAM_LOG(LogLevel::Debug
,
50 ("TrackUnionStream %p removing input %p", this, aPort
));
51 for (int32_t i
= mTrackMap
.Length() - 1; i
>= 0; --i
) {
52 if (mTrackMap
[i
].mInputPort
== aPort
) {
53 STREAM_LOG(LogLevel::Debug
,
54 ("TrackUnionStream %p removing trackmap entry %d", this, i
));
55 nsTArray
<RefPtr
<DirectMediaStreamTrackListener
>> listeners(
56 mTrackMap
[i
].mOwnedDirectListeners
);
57 for (auto listener
: listeners
) {
58 // Remove listeners while the entry still exists.
59 RemoveDirectTrackListenerImpl(listener
, mTrackMap
[i
].mOutputTrackID
);
62 mTrackMap
.RemoveElementAt(i
);
65 ProcessedMediaStream::RemoveInput(aPort
);
67 void TrackUnionStream::ProcessInput(GraphTime aFrom
, GraphTime aTo
,
69 TRACE_AUDIO_CALLBACK_COMMENT("TrackUnionStream %p", this);
70 if (IsFinishedOnGraphThread()) {
73 AutoTArray
<bool, 8> mappedTracksFinished
;
74 AutoTArray
<bool, 8> mappedTracksWithMatchingInputTracks
;
75 for (uint32_t i
= 0; i
< mTrackMap
.Length(); ++i
) {
76 mappedTracksFinished
.AppendElement(true);
77 mappedTracksWithMatchingInputTracks
.AppendElement(false);
80 AutoTArray
<MediaInputPort
*, 32> inputs(mInputs
);
81 inputs
.AppendElements(mSuspendedInputs
);
83 bool allFinished
= !inputs
.IsEmpty();
84 for (uint32_t i
= 0; i
< inputs
.Length(); ++i
) {
85 MediaStream
* stream
= inputs
[i
]->GetSource();
86 if (!stream
->IsFinishedOnGraphThread()) {
87 // XXX we really should check whether 'stream' has finished within time
88 // aTo, not just that it's finishing when all its queued data eventually
92 for (StreamTracks::TrackIter
tracks(stream
->GetStreamTracks());
93 !tracks
.IsEnded(); tracks
.Next()) {
95 for (uint32_t j
= 0; j
< mTrackMap
.Length(); ++j
) {
96 TrackMapEntry
* map
= &mTrackMap
[j
];
97 if (map
->mInputPort
== inputs
[i
] &&
98 map
->mInputTrackID
== tracks
->GetID()) {
99 bool trackFinished
= false;
100 StreamTracks::Track
* outputTrack
=
101 mTracks
.FindTrack(map
->mOutputTrackID
);
103 if (!outputTrack
|| outputTrack
->IsEnded() ||
104 !inputs
[i
]->PassTrackThrough(tracks
->GetID())) {
105 trackFinished
= true;
107 CopyTrackData(tracks
.get(), j
, aFrom
, aTo
, &trackFinished
);
109 mappedTracksFinished
[j
] = trackFinished
;
110 mappedTracksWithMatchingInputTracks
[j
] = true;
114 if (!found
&& inputs
[i
]->AllowCreationOf(tracks
->GetID())) {
115 bool trackFinished
= false;
116 uint32_t mapIndex
= AddTrack(inputs
[i
], tracks
.get(), aFrom
);
117 CopyTrackData(tracks
.get(), mapIndex
, aFrom
, aTo
, &trackFinished
);
118 mappedTracksFinished
.AppendElement(trackFinished
);
119 mappedTracksWithMatchingInputTracks
.AppendElement(true);
123 for (int32_t i
= mTrackMap
.Length() - 1; i
>= 0; --i
) {
124 if (mappedTracksFinished
[i
]) {
129 if (!mappedTracksWithMatchingInputTracks
[i
]) {
130 for (auto listener
: mTrackMap
[i
].mOwnedDirectListeners
) {
131 // Remove listeners while the entry still exists.
132 RemoveDirectTrackListenerImpl(listener
, mTrackMap
[i
].mOutputTrackID
);
134 mTrackMap
.RemoveElementAt(i
);
137 if (allFinished
&& mAutofinish
&& (aFlags
& ALLOW_FINISH
)) {
138 // All streams have finished and won't add any more tracks, and
139 // all our tracks have actually finished and been removed from our map,
140 // so we're finished now.
141 FinishOnGraphThread();
145 uint32_t TrackUnionStream::AddTrack(MediaInputPort
* aPort
,
146 StreamTracks::Track
* aTrack
,
148 STREAM_LOG(LogLevel::Verbose
,
149 ("TrackUnionStream %p adding track %d for "
150 "input stream %p track %d, desired id %d",
151 this, aTrack
->GetID(), aPort
->GetSource(), aTrack
->GetID(),
152 aPort
->GetDestinationTrackId()));
155 if (IsTrackIDExplicit(id
= aPort
->GetDestinationTrackId())) {
156 MOZ_ASSERT(id
>= mNextAvailableTrackID
&& !mUsedTracks
.ContainsSorted(id
),
157 "Desired destination id taken. Only provide a destination ID "
158 "if you can assure its availability, or we may not be able "
159 "to bind to the correct DOM-side track.");
161 AutoTArray
<MediaInputPort
*, 32> inputs(mInputs
);
162 inputs
.AppendElements(mSuspendedInputs
);
163 for (size_t i
= 0; inputs
[i
] != aPort
; ++i
) {
164 MOZ_ASSERT(inputs
[i
]->GetSourceTrackId() != TRACK_ANY
,
165 "You are adding a MediaInputPort with a track mapping "
166 "while there already exist generic MediaInputPorts for this "
167 "destination stream. This can lead to TrackID collisions!");
170 mUsedTracks
.InsertElementSorted(id
);
171 } else if ((id
= aTrack
->GetID()) && id
> mNextAvailableTrackID
&&
172 !mUsedTracks
.ContainsSorted(id
)) {
173 // Input id available. Mark it used in mUsedTracks.
174 mUsedTracks
.InsertElementSorted(id
);
176 // No desired destination id and Input id taken, allocate a new one.
177 id
= mNextAvailableTrackID
;
179 // Update mNextAvailableTrackID and prune any mUsedTracks members it now
182 if (!mUsedTracks
.RemoveElementSorted(++mNextAvailableTrackID
)) {
183 // Not in use. We're done.
189 // Round up the track start time so the track, if anything, starts a
190 // little later than the true time. This means we'll have enough
191 // samples in our input stream to go just beyond the destination time.
192 StreamTime outputStart
= GraphTimeToStreamTimeWithBlocking(aFrom
);
194 nsAutoPtr
<MediaSegment
> segment
;
195 segment
= aTrack
->GetSegment()->CreateEmptyClone();
196 segment
->AppendNullData(outputStart
);
197 StreamTracks::Track
* track
=
198 &mTracks
.AddTrack(id
, outputStart
, segment
.forget());
199 STREAM_LOG(LogLevel::Debug
, ("TrackUnionStream %p added track %d for input "
200 "stream %p track %d, start ticks %lld",
201 this, track
->GetID(), aPort
->GetSource(),
202 aTrack
->GetID(), (long long)outputStart
));
204 TrackMapEntry
* map
= mTrackMap
.AppendElement();
205 map
->mEndOfConsumedInputTicks
= 0;
206 map
->mEndOfLastInputIntervalInInputStream
= -1;
207 map
->mEndOfLastInputIntervalInOutputStream
= -1;
208 map
->mInputPort
= aPort
;
209 map
->mInputTrackID
= aTrack
->GetID();
210 map
->mOutputTrackID
= track
->GetID();
211 map
->mSegment
= aTrack
->GetSegment()->CreateEmptyClone();
213 for (int32_t i
= mPendingDirectTrackListeners
.Length() - 1; i
>= 0; --i
) {
214 TrackBound
<DirectMediaStreamTrackListener
>& bound
=
215 mPendingDirectTrackListeners
[i
];
216 if (bound
.mTrackID
!= map
->mOutputTrackID
) {
219 MediaStream
* source
= map
->mInputPort
->GetSource();
220 map
->mOwnedDirectListeners
.AppendElement(bound
.mListener
);
221 DisabledTrackMode currentMode
= GetDisabledTrackMode(bound
.mTrackID
);
222 if (currentMode
!= DisabledTrackMode::ENABLED
) {
223 bound
.mListener
->IncreaseDisabled(currentMode
);
225 STREAM_LOG(LogLevel::Debug
, ("TrackUnionStream %p adding direct listener "
226 "%p for track %d. Forwarding to input "
227 "stream %p track %d.",
228 this, bound
.mListener
.get(), bound
.mTrackID
,
229 source
, map
->mInputTrackID
));
230 source
->AddDirectTrackListenerImpl(bound
.mListener
.forget(),
232 mPendingDirectTrackListeners
.RemoveElementAt(i
);
235 return mTrackMap
.Length() - 1;
238 void TrackUnionStream::EndTrack(uint32_t aIndex
) {
239 StreamTracks::Track
* outputTrack
=
240 mTracks
.FindTrack(mTrackMap
[aIndex
].mOutputTrackID
);
241 if (!outputTrack
|| outputTrack
->IsEnded()) return;
242 STREAM_LOG(LogLevel::Debug
, ("TrackUnionStream %p ending track %d", this,
243 outputTrack
->GetID()));
244 outputTrack
->SetEnded();
247 void TrackUnionStream::CopyTrackData(StreamTracks::Track
* aInputTrack
,
248 uint32_t aMapIndex
, GraphTime aFrom
,
250 bool* aOutputTrackFinished
) {
251 TrackMapEntry
* map
= &mTrackMap
[aMapIndex
];
252 TRACE_AUDIO_CALLBACK_COMMENT(
253 "Input stream %p track %i -> TrackUnionStream %p track %i",
254 map
->mInputPort
->GetSource(), map
->mInputTrackID
, this,
255 map
->mOutputTrackID
);
256 StreamTracks::Track
* outputTrack
= mTracks
.FindTrack(map
->mOutputTrackID
);
257 MOZ_ASSERT(outputTrack
&& !outputTrack
->IsEnded(),
258 "Can't copy to ended track");
260 MediaSegment
* segment
= map
->mSegment
;
261 MediaStream
* source
= map
->mInputPort
->GetSource();
264 *aOutputTrackFinished
= false;
265 for (GraphTime t
= aFrom
; t
< aTo
; t
= next
) {
266 MediaInputPort::InputInterval interval
=
267 map
->mInputPort
->GetNextInputInterval(t
);
268 interval
.mEnd
= std::min(interval
.mEnd
, aTo
);
269 StreamTime inputEnd
=
270 source
->GraphTimeToStreamTimeWithBlocking(interval
.mEnd
);
272 if (aInputTrack
->IsEnded() && aInputTrack
->GetEnd() <= inputEnd
) {
273 *aOutputTrackFinished
= true;
277 if (interval
.mStart
>= interval
.mEnd
) {
280 StreamTime ticks
= interval
.mEnd
- interval
.mStart
;
281 next
= interval
.mEnd
;
283 StreamTime outputStart
= outputTrack
->GetEnd();
285 if (interval
.mInputIsBlocked
) {
286 segment
->AppendNullData(ticks
);
289 ("TrackUnionStream %p appending %lld ticks of null data to track %d",
290 this, (long long)ticks
, outputTrack
->GetID()));
291 } else if (InMutedCycle()) {
292 segment
->AppendNullData(ticks
);
294 if (source
->IsSuspended()) {
295 segment
->AppendNullData(aTo
- aFrom
);
297 MOZ_ASSERT(outputTrack
->GetEnd() ==
298 GraphTimeToStreamTimeWithBlocking(interval
.mStart
),
300 StreamTime inputStart
=
301 source
->GraphTimeToStreamTimeWithBlocking(interval
.mStart
);
302 segment
->AppendSlice(*aInputTrack
->GetSegment(), inputStart
, inputEnd
);
305 ApplyTrackDisabling(outputTrack
->GetID(), segment
);
306 for (TrackBound
<MediaStreamTrackListener
>& b
: mTrackListeners
) {
307 if (b
.mTrackID
!= outputTrack
->GetID()) {
310 b
.mListener
->NotifyQueuedChanges(Graph(), outputStart
, *segment
);
312 outputTrack
->GetSegment()->AppendFrom(segment
);
316 void TrackUnionStream::SetTrackEnabledImpl(TrackID aTrackID
,
317 DisabledTrackMode aMode
) {
318 bool enabled
= aMode
== DisabledTrackMode::ENABLED
;
319 for (TrackMapEntry
& entry
: mTrackMap
) {
320 if (entry
.mOutputTrackID
== aTrackID
) {
321 STREAM_LOG(LogLevel::Info
,
322 ("TrackUnionStream %p track %d was explicitly %s", this,
323 aTrackID
, enabled
? "enabled" : "disabled"));
324 for (DirectMediaStreamTrackListener
* listener
:
325 entry
.mOwnedDirectListeners
) {
326 DisabledTrackMode oldMode
= GetDisabledTrackMode(aTrackID
);
327 bool oldEnabled
= oldMode
== DisabledTrackMode::ENABLED
;
328 if (!oldEnabled
&& enabled
) {
329 STREAM_LOG(LogLevel::Debug
, ("TrackUnionStream %p track %d setting "
330 "direct listener enabled",
332 listener
->DecreaseDisabled(oldMode
);
333 } else if (oldEnabled
&& !enabled
) {
334 STREAM_LOG(LogLevel::Debug
, ("TrackUnionStream %p track %d setting "
335 "direct listener disabled",
337 listener
->IncreaseDisabled(aMode
);
342 MediaStream::SetTrackEnabledImpl(aTrackID
, aMode
);
345 MediaStream
* TrackUnionStream::GetInputStreamFor(TrackID aTrackID
) {
346 for (TrackMapEntry
& entry
: mTrackMap
) {
347 if (entry
.mOutputTrackID
== aTrackID
&& entry
.mInputPort
) {
348 return entry
.mInputPort
->GetSource();
355 TrackID
TrackUnionStream::GetInputTrackIDFor(TrackID aTrackID
) {
356 for (TrackMapEntry
& entry
: mTrackMap
) {
357 if (entry
.mOutputTrackID
== aTrackID
) {
358 return entry
.mInputTrackID
;
365 void TrackUnionStream::AddDirectTrackListenerImpl(
366 already_AddRefed
<DirectMediaStreamTrackListener
> aListener
,
368 RefPtr
<DirectMediaStreamTrackListener
> listener
= aListener
;
370 for (TrackMapEntry
& entry
: mTrackMap
) {
371 if (entry
.mOutputTrackID
== aTrackID
) {
372 MediaStream
* source
= entry
.mInputPort
->GetSource();
373 STREAM_LOG(LogLevel::Debug
,
374 ("TrackUnionStream %p adding direct listener "
375 "%p for track %d. Forwarding to input "
376 "stream %p track %d.",
377 this, listener
.get(), aTrackID
, source
, entry
.mInputTrackID
));
378 entry
.mOwnedDirectListeners
.AppendElement(listener
);
379 DisabledTrackMode currentMode
= GetDisabledTrackMode(aTrackID
);
380 if (currentMode
!= DisabledTrackMode::ENABLED
) {
381 listener
->IncreaseDisabled(currentMode
);
383 source
->AddDirectTrackListenerImpl(listener
.forget(),
384 entry
.mInputTrackID
);
389 TrackBound
<DirectMediaStreamTrackListener
>* bound
=
390 mPendingDirectTrackListeners
.AppendElement();
391 bound
->mListener
= listener
.forget();
392 bound
->mTrackID
= aTrackID
;
395 void TrackUnionStream::RemoveDirectTrackListenerImpl(
396 DirectMediaStreamTrackListener
* aListener
, TrackID aTrackID
) {
397 for (TrackMapEntry
& entry
: mTrackMap
) {
398 // OutputTrackID is unique to this stream so we only need to do this once.
399 if (entry
.mOutputTrackID
!= aTrackID
) {
402 for (size_t i
= 0; i
< entry
.mOwnedDirectListeners
.Length(); ++i
) {
403 if (entry
.mOwnedDirectListeners
[i
] == aListener
) {
404 STREAM_LOG(LogLevel::Debug
,
405 ("TrackUnionStream %p removing direct "
406 "listener %p for track %d, forwarding "
407 "to input stream %p track %d",
408 this, aListener
, aTrackID
, entry
.mInputPort
->GetSource(),
409 entry
.mInputTrackID
));
410 DisabledTrackMode currentMode
= GetDisabledTrackMode(aTrackID
);
411 if (currentMode
!= DisabledTrackMode::ENABLED
) {
412 // Reset the listener's state.
413 aListener
->DecreaseDisabled(currentMode
);
415 entry
.mOwnedDirectListeners
.RemoveElementAt(i
);
419 // Forward to the input
420 MediaStream
* source
= entry
.mInputPort
->GetSource();
421 source
->RemoveDirectTrackListenerImpl(aListener
, entry
.mInputTrackID
);
425 for (size_t i
= 0; i
< mPendingDirectTrackListeners
.Length(); ++i
) {
426 TrackBound
<DirectMediaStreamTrackListener
>& bound
=
427 mPendingDirectTrackListeners
[i
];
428 if (bound
.mListener
== aListener
&& bound
.mTrackID
== aTrackID
) {
429 mPendingDirectTrackListeners
.RemoveElementAt(i
);
435 void TrackUnionStream::RemoveAllDirectListenersImpl() {
436 for (TrackMapEntry
& entry
: mTrackMap
) {
437 nsTArray
<RefPtr
<DirectMediaStreamTrackListener
>> listeners(
438 entry
.mOwnedDirectListeners
);
439 for (const auto& listener
: listeners
) {
440 RemoveDirectTrackListenerImpl(listener
, entry
.mOutputTrackID
);
442 MOZ_DIAGNOSTIC_ASSERT(entry
.mOwnedDirectListeners
.IsEmpty());
445 nsTArray
<TrackBound
<DirectMediaStreamTrackListener
>> boundListeners(
446 mPendingDirectTrackListeners
);
447 for (const auto& binding
: boundListeners
) {
448 RemoveDirectTrackListenerImpl(binding
.mListener
, binding
.mTrackID
);
450 MOZ_DIAGNOSTIC_ASSERT(mPendingDirectTrackListeners
.IsEmpty());
453 } // namespace mozilla