1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 * You can obtain one at http://mozilla.org/MPL/2.0/. */
6 #ifndef MOZILLA_TRACKUNIONSTREAM_H_
7 #define MOZILLA_TRACKUNIONSTREAM_H_
9 #include "MediaStreamGraph.h"
15 #define STREAM_LOG(type, msg) PR_LOG(gMediaStreamGraphLog, type, msg)
17 #define STREAM_LOG(type, msg)
21 * See MediaStreamGraph::CreateTrackUnionStream.
22 * This file is only included by MediaStreamGraph.cpp so it's OK to put the
23 * entire implementation in this header file.
25 class TrackUnionStream
: public ProcessedMediaStream
{
27 TrackUnionStream(DOMMediaStream
* aWrapper
) :
28 ProcessedMediaStream(aWrapper
),
29 mFilterCallback(nullptr),
32 virtual void RemoveInput(MediaInputPort
* aPort
) MOZ_OVERRIDE
34 for (int32_t i
= mTrackMap
.Length() - 1; i
>= 0; --i
) {
35 if (mTrackMap
[i
].mInputPort
== aPort
) {
37 mTrackMap
.RemoveElementAt(i
);
40 ProcessedMediaStream::RemoveInput(aPort
);
42 virtual void ProduceOutput(GraphTime aFrom
, GraphTime aTo
, uint32_t aFlags
) MOZ_OVERRIDE
44 if (IsFinishedOnGraphThread()) {
47 nsAutoTArray
<bool,8> mappedTracksFinished
;
48 nsAutoTArray
<bool,8> mappedTracksWithMatchingInputTracks
;
49 for (uint32_t i
= 0; i
< mTrackMap
.Length(); ++i
) {
50 mappedTracksFinished
.AppendElement(true);
51 mappedTracksWithMatchingInputTracks
.AppendElement(false);
53 bool allFinished
= true;
54 bool allHaveCurrentData
= true;
55 for (uint32_t i
= 0; i
< mInputs
.Length(); ++i
) {
56 MediaStream
* stream
= mInputs
[i
]->GetSource();
57 if (!stream
->IsFinishedOnGraphThread()) {
58 // XXX we really should check whether 'stream' has finished within time aTo,
59 // not just that it's finishing when all its queued data eventually runs
63 if (!stream
->HasCurrentData()) {
64 allHaveCurrentData
= false;
66 for (StreamBuffer::TrackIter
tracks(stream
->GetStreamBuffer());
67 !tracks
.IsEnded(); tracks
.Next()) {
69 for (uint32_t j
= 0; j
< mTrackMap
.Length(); ++j
) {
70 TrackMapEntry
* map
= &mTrackMap
[j
];
71 if (map
->mInputPort
== mInputs
[i
] && map
->mInputTrackID
== tracks
->GetID()) {
73 StreamBuffer::Track
* outputTrack
= mBuffer
.FindTrack(map
->mOutputTrackID
);
74 if (!outputTrack
|| outputTrack
->IsEnded()) {
77 CopyTrackData(tracks
.get(), j
, aFrom
, aTo
, &trackFinished
);
79 mappedTracksFinished
[j
] = trackFinished
;
80 mappedTracksWithMatchingInputTracks
[j
] = true;
85 if (!found
&& (!mFilterCallback
|| mFilterCallback(tracks
.get()))) {
86 bool trackFinished
= false;
87 uint32_t mapIndex
= AddTrack(mInputs
[i
], tracks
.get(), aFrom
);
88 CopyTrackData(tracks
.get(), mapIndex
, aFrom
, aTo
, &trackFinished
);
89 mappedTracksFinished
.AppendElement(trackFinished
);
90 mappedTracksWithMatchingInputTracks
.AppendElement(true);
94 for (int32_t i
= mTrackMap
.Length() - 1; i
>= 0; --i
) {
95 if (mappedTracksFinished
[i
]) {
100 if (!mappedTracksWithMatchingInputTracks
[i
]) {
101 mTrackMap
.RemoveElementAt(i
);
104 if (allFinished
&& mAutofinish
&& (aFlags
& ALLOW_FINISH
)) {
105 // All streams have finished and won't add any more tracks, and
106 // all our tracks have actually finished and been removed from our map,
107 // so we're finished now.
108 FinishOnGraphThread();
110 mBuffer
.AdvanceKnownTracksTime(GraphTimeToStreamTime(aTo
));
111 if (allHaveCurrentData
) {
112 // We can make progress if we're not blocked
113 mHasCurrentData
= true;
117 // Consumers may specify a filtering callback to apply to every input track.
118 // Returns true to allow the track to act as an input; false to reject it entirely.
119 typedef bool (*TrackIDFilterCallback
)(StreamBuffer::Track
*);
120 void SetTrackIDFilter(TrackIDFilterCallback aCallback
) {
121 mFilterCallback
= aCallback
;
124 // Forward SetTrackEnabled(output_track_id, enabled) to the Source MediaStream,
125 // translating the output track ID into the correct ID in the source.
126 virtual void ForwardTrackEnabled(TrackID aOutputID
, bool aEnabled
) MOZ_OVERRIDE
{
127 for (int32_t i
= mTrackMap
.Length() - 1; i
>= 0; --i
) {
128 if (mTrackMap
[i
].mOutputTrackID
== aOutputID
) {
129 mTrackMap
[i
].mInputPort
->GetSource()->
130 SetTrackEnabled(mTrackMap
[i
].mInputTrackID
, aEnabled
);
136 TrackIDFilterCallback mFilterCallback
;
138 // Only non-ended tracks are allowed to persist in this map.
139 struct TrackMapEntry
{
140 // mEndOfConsumedInputTicks is the end of the input ticks that we've consumed.
141 // 0 if we haven't consumed any yet.
142 TrackTicks mEndOfConsumedInputTicks
;
143 // mEndOfLastInputIntervalInInputStream is the timestamp for the end of the
144 // previous interval which was unblocked for both the input and output
145 // stream, in the input stream's timeline, or -1 if there wasn't one.
146 StreamTime mEndOfLastInputIntervalInInputStream
;
147 // mEndOfLastInputIntervalInOutputStream is the timestamp for the end of the
148 // previous interval which was unblocked for both the input and output
149 // stream, in the output stream's timeline, or -1 if there wasn't one.
150 StreamTime mEndOfLastInputIntervalInOutputStream
;
151 MediaInputPort
* mInputPort
;
152 // We keep track IDs instead of track pointers because
153 // tracks can be removed without us being notified (e.g.
154 // when a finished track is forgotten.) When we need a Track*,
155 // we call StreamBuffer::FindTrack, which will return null if
156 // the track has been deleted.
157 TrackID mInputTrackID
;
158 TrackID mOutputTrackID
;
159 nsAutoPtr
<MediaSegment
> mSegment
;
162 uint32_t AddTrack(MediaInputPort
* aPort
, StreamBuffer::Track
* aTrack
,
165 // Use the ID of the source track if we can, otherwise allocate a new
167 TrackID id
= std::max(mMaxTrackID
+ 1, aTrack
->GetID());
170 TrackRate rate
= aTrack
->GetRate();
171 // Round up the track start time so the track, if anything, starts a
172 // little later than the true time. This means we'll have enough
173 // samples in our input stream to go just beyond the destination time.
174 TrackTicks outputStart
= TimeToTicksRoundUp(rate
, GraphTimeToStreamTime(aFrom
));
176 nsAutoPtr
<MediaSegment
> segment
;
177 segment
= aTrack
->GetSegment()->CreateEmptyClone();
178 for (uint32_t j
= 0; j
< mListeners
.Length(); ++j
) {
179 MediaStreamListener
* l
= mListeners
[j
];
180 l
->NotifyQueuedTrackChanges(Graph(), id
, rate
, outputStart
,
181 MediaStreamListener::TRACK_EVENT_CREATED
,
184 segment
->AppendNullData(outputStart
);
185 StreamBuffer::Track
* track
=
186 &mBuffer
.AddTrack(id
, rate
, outputStart
, segment
.forget());
187 STREAM_LOG(PR_LOG_DEBUG
, ("TrackUnionStream %p adding track %d for input stream %p track %d, start ticks %lld",
188 this, id
, aPort
->GetSource(), aTrack
->GetID(),
189 (long long)outputStart
));
191 TrackMapEntry
* map
= mTrackMap
.AppendElement();
192 map
->mEndOfConsumedInputTicks
= 0;
193 map
->mEndOfLastInputIntervalInInputStream
= -1;
194 map
->mEndOfLastInputIntervalInOutputStream
= -1;
195 map
->mInputPort
= aPort
;
196 map
->mInputTrackID
= aTrack
->GetID();
197 map
->mOutputTrackID
= track
->GetID();
198 map
->mSegment
= aTrack
->GetSegment()->CreateEmptyClone();
199 return mTrackMap
.Length() - 1;
201 void EndTrack(uint32_t aIndex
)
203 StreamBuffer::Track
* outputTrack
= mBuffer
.FindTrack(mTrackMap
[aIndex
].mOutputTrackID
);
204 if (!outputTrack
|| outputTrack
->IsEnded())
206 for (uint32_t j
= 0; j
< mListeners
.Length(); ++j
) {
207 MediaStreamListener
* l
= mListeners
[j
];
208 TrackTicks offset
= outputTrack
->GetSegment()->GetDuration();
209 nsAutoPtr
<MediaSegment
> segment
;
210 segment
= outputTrack
->GetSegment()->CreateEmptyClone();
211 l
->NotifyQueuedTrackChanges(Graph(), outputTrack
->GetID(),
212 outputTrack
->GetRate(), offset
,
213 MediaStreamListener::TRACK_EVENT_ENDED
,
216 outputTrack
->SetEnded();
218 void CopyTrackData(StreamBuffer::Track
* aInputTrack
,
219 uint32_t aMapIndex
, GraphTime aFrom
, GraphTime aTo
,
220 bool* aOutputTrackFinished
)
222 TrackMapEntry
* map
= &mTrackMap
[aMapIndex
];
223 StreamBuffer::Track
* outputTrack
= mBuffer
.FindTrack(map
->mOutputTrackID
);
224 MOZ_ASSERT(outputTrack
&& !outputTrack
->IsEnded(), "Can't copy to ended track");
226 TrackRate rate
= outputTrack
->GetRate();
227 MediaSegment
* segment
= map
->mSegment
;
228 MediaStream
* source
= map
->mInputPort
->GetSource();
231 *aOutputTrackFinished
= false;
232 for (GraphTime t
= aFrom
; t
< aTo
; t
= next
) {
233 MediaInputPort::InputInterval interval
= map
->mInputPort
->GetNextInputInterval(t
);
234 interval
.mEnd
= std::min(interval
.mEnd
, aTo
);
235 StreamTime inputEnd
= source
->GraphTimeToStreamTime(interval
.mEnd
);
236 TrackTicks inputTrackEndPoint
= TRACK_TICKS_MAX
;
238 if (aInputTrack
->IsEnded() &&
239 aInputTrack
->GetEndTimeRoundDown() <= inputEnd
) {
240 inputTrackEndPoint
= aInputTrack
->GetEnd();
241 *aOutputTrackFinished
= true;
244 if (interval
.mStart
>= interval
.mEnd
)
246 next
= interval
.mEnd
;
248 // Ticks >= startTicks and < endTicks are in the interval
249 StreamTime outputEnd
= GraphTimeToStreamTime(interval
.mEnd
);
250 TrackTicks startTicks
= outputTrack
->GetEnd();
251 StreamTime outputStart
= GraphTimeToStreamTime(interval
.mStart
);
252 NS_ASSERTION(startTicks
== TimeToTicksRoundUp(rate
, outputStart
),
254 TrackTicks endTicks
= TimeToTicksRoundUp(rate
, outputEnd
);
255 TrackTicks ticks
= endTicks
- startTicks
;
256 StreamTime inputStart
= source
->GraphTimeToStreamTime(interval
.mStart
);
258 if (interval
.mInputIsBlocked
) {
259 // Maybe the input track ended?
260 segment
->AppendNullData(ticks
);
261 STREAM_LOG(PR_LOG_DEBUG
+1, ("TrackUnionStream %p appending %lld ticks of null data to track %d",
262 this, (long long)ticks
, outputTrack
->GetID()));
264 // Figuring out which samples to use from the input stream is tricky
265 // because its start time and our start time may differ by a fraction
266 // of a tick. Assuming the input track hasn't ended, we have to ensure
267 // that 'ticks' samples are gathered, even though a tick boundary may
268 // occur between outputStart and outputEnd but not between inputStart
270 // These are the properties we need to ensure:
271 // 1) Exactly 'ticks' ticks of output are produced, i.e.
272 // inputEndTicks - inputStartTicks = ticks.
273 // 2) inputEndTicks <= aInputTrack->GetSegment()->GetDuration().
274 // 3) In any sequence of intervals where neither stream is blocked,
275 // the content of the input track we use is a contiguous sequence of
276 // ticks with no gaps or overlaps.
277 if (map
->mEndOfLastInputIntervalInInputStream
!= inputStart
||
278 map
->mEndOfLastInputIntervalInOutputStream
!= outputStart
) {
279 // Start of a new series of intervals where neither stream is blocked.
280 map
->mEndOfConsumedInputTicks
= TimeToTicksRoundDown(rate
, inputStart
) - 1;
282 TrackTicks inputStartTicks
= map
->mEndOfConsumedInputTicks
;
283 TrackTicks inputEndTicks
= inputStartTicks
+ ticks
;
284 map
->mEndOfConsumedInputTicks
= inputEndTicks
;
285 map
->mEndOfLastInputIntervalInInputStream
= inputEnd
;
286 map
->mEndOfLastInputIntervalInOutputStream
= outputEnd
;
287 // Now we prove that the above properties hold:
288 // Property #1: trivial by construction.
289 // Property #3: trivial by construction. Between every two
290 // intervals where both streams are not blocked, the above if condition
291 // is false and mEndOfConsumedInputTicks advances exactly to match
292 // the ticks that were consumed.
294 // Let originalOutputStart be the value of outputStart and originalInputStart
295 // be the value of inputStart when the body of the "if" block was last
297 // Let allTicks be the sum of the values of 'ticks' computed since then.
298 // The interval [originalInputStart/rate, inputEnd/rate) is the
299 // same length as the interval [originalOutputStart/rate, outputEnd/rate),
300 // so the latter interval can have at most one more integer in it. Thus
301 // TimeToTicksRoundUp(rate, outputEnd) - TimeToTicksRoundUp(rate, originalOutputStart)
302 // <= TimeToTicksRoundDown(rate, inputEnd) - TimeToTicksRoundDown(rate, originalInputStart) + 1
304 // inputEndTicks = TimeToTicksRoundDown(rate, originalInputStart) - 1 + allTicks
305 // = TimeToTicksRoundDown(rate, originalInputStart) - 1 + TimeToTicksRoundUp(rate, outputEnd) - TimeToTicksRoundUp(rate, originalOutputStart)
306 // <= TimeToTicksRoundDown(rate, originalInputStart) - 1 + TimeToTicksRoundDown(rate, inputEnd) - TimeToTicksRoundDown(rate, originalInputStart) + 1
307 // = TimeToTicksRoundDown(rate, inputEnd)
309 // (now using the fact that inputEnd <= track->GetEndTimeRoundDown() for a non-ended track)
310 // <= TicksToTimeRoundDown(rate, aInputTrack->GetSegment()->GetDuration())/rate
311 // <= rate*aInputTrack->GetSegment()->GetDuration()/rate
312 // = aInputTrack->GetSegment()->GetDuration()
315 if (inputStartTicks
< 0) {
316 // Data before the start of the track is just null.
317 // We have to add a small amount of delay to ensure that there is
318 // always a sample available if we see an interval that contains a
319 // tick boundary on the output stream's timeline but does not contain
320 // a tick boundary on the input stream's timeline. 1 tick delay is
321 // necessary and sufficient.
322 segment
->AppendNullData(-inputStartTicks
);
325 if (inputEndTicks
> inputStartTicks
) {
326 segment
->AppendSlice(*aInputTrack
->GetSegment(),
327 std::min(inputTrackEndPoint
, inputStartTicks
),
328 std::min(inputTrackEndPoint
, inputEndTicks
));
330 STREAM_LOG(PR_LOG_DEBUG
+1, ("TrackUnionStream %p appending %lld ticks of input data to track %d",
331 this, (long long)(std::min(inputTrackEndPoint
, inputEndTicks
) - std::min(inputTrackEndPoint
, inputStartTicks
)),
332 outputTrack
->GetID()));
334 ApplyTrackDisabling(outputTrack
->GetID(), segment
);
335 for (uint32_t j
= 0; j
< mListeners
.Length(); ++j
) {
336 MediaStreamListener
* l
= mListeners
[j
];
337 l
->NotifyQueuedTrackChanges(Graph(), outputTrack
->GetID(),
338 outputTrack
->GetRate(), startTicks
, 0,
341 outputTrack
->GetSegment()->AppendFrom(segment
);
345 nsTArray
<TrackMapEntry
> mTrackMap
;
351 #endif /* MOZILLA_MEDIASTREAMGRAPH_H_ */