1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 * You can obtain one at http://mozilla.org/MPL/2.0/. */
6 #ifndef MEDIASTREAMTRACK_H_
7 #define MEDIASTREAMTRACK_H_
9 #include "MediaTrackConstraints.h"
10 #include "PrincipalChangeObserver.h"
11 #include "PrincipalHandle.h"
12 #include "mozilla/DOMEventTargetHelper.h"
13 #include "mozilla/dom/MediaStreamTrackBinding.h"
14 #include "mozilla/dom/MediaTrackSettingsBinding.h"
15 #include "mozilla/media/MediaUtils.h"
16 #include "mozilla/WeakPtr.h"
19 #include "nsIPrincipal.h"
20 #include "PerformanceRecorder.h"
25 class MediaEnginePhotoCallback
;
28 class MediaTrackGraph
;
29 class MediaTrackGraphImpl
;
30 class MediaTrackListener
;
31 class DirectMediaTrackListener
;
32 class PeerConnectionImpl
;
34 class ProcessedMediaTrack
;
35 class RemoteSourceStreamInfo
;
36 class SourceStreamInfo
;
41 class AudioStreamTrack
;
42 class VideoStreamTrack
;
43 enum class CallerType
: uint32_t;
46 * Common interface through which a MediaStreamTrack can communicate with its
47 * producer on the main thread.
49 * Kept alive by a strong ref in all MediaStreamTracks (original and clones)
50 * sharing this source.
52 class MediaStreamTrackSource
: public nsISupports
{
53 NS_DECL_CYCLE_COLLECTING_ISUPPORTS
54 NS_DECL_CYCLE_COLLECTION_CLASS(MediaStreamTrackSource
)
57 class Sink
: public SupportsWeakPtr
{
60 * Must be constant throughout the Sink's lifetime.
62 * Return true to keep the MediaStreamTrackSource where this sink is
64 * Return false to allow the source to stop.
66 * Typically MediaStreamTrack::Sink returns true and other Sinks
67 * (like HTMLMediaElement::StreamCaptureTrackSource) return false.
69 virtual bool KeepsSourceAlive() const = 0;
72 * Return true to ensure that the MediaStreamTrackSource where this Sink is
73 * registered is kept turned on and active.
74 * Return false to allow the source to pause, and any underlying devices to
77 * When the underlying enabled state of the sink changes,
78 * call MediaStreamTrackSource::SinkEnabledStateChanged().
80 * Typically MediaStreamTrack returns the track's enabled state and other
81 * Sinks (like HTMLMediaElement::StreamCaptureTrackSource) return false so
82 * control over device state remains with tracks and their enabled state.
84 virtual bool Enabled() const = 0;
87 * Called when the principal of the MediaStreamTrackSource where this sink
88 * is registered has changed.
90 virtual void PrincipalChanged() = 0;
93 * Called when the muted state of the MediaStreamTrackSource where this sink
94 * is registered has changed.
96 virtual void MutedChanged(bool aNewState
) = 0;
99 * Called when the MediaStreamTrackSource where this sink is registered has
100 * stopped producing data for good, i.e., it has ended.
102 virtual void OverrideEnded() = 0;
105 virtual ~Sink() = default;
108 MediaStreamTrackSource(nsIPrincipal
* aPrincipal
, const nsString
& aLabel
,
109 TrackingId aTrackingId
)
110 : mPrincipal(aPrincipal
),
112 mTrackingId(std::move(aTrackingId
)),
116 * Use to clean up any resources that have to be cleaned before the
117 * destructor is called. It is often too late in the destructor because
118 * of garbage collection having removed the members already.
120 virtual void Destroy() {}
123 * Gets the source's MediaSourceEnum for usage by PeerConnections.
125 virtual MediaSourceEnum
GetMediaSource() const = 0;
128 * Get this TrackSource's principal.
130 nsIPrincipal
* GetPrincipal() const { return mPrincipal
; }
133 * This is used in WebRTC. A peerIdentity constrained MediaStreamTrack cannot
134 * be sent across the network to anything other than a peer with the provided
135 * identity. If this is set, then GetPrincipal() should return an instance of
138 * A track's PeerIdentity is immutable and will not change during the track's
141 virtual const PeerIdentity
* GetPeerIdentity() const { return nullptr; }
144 * MediaStreamTrack::GetLabel (see spec) calls through to here.
146 void GetLabel(nsAString
& aLabel
) { aLabel
.Assign(mLabel
); }
149 * Whether this TrackSource provides video frames with an alpha channel. Only
150 * applies to video sources. Used by HTMLVideoElement.
152 virtual bool HasAlpha() const { return false; }
155 * Forwards a photo request to backends that support it. Other backends return
156 * NS_ERROR_NOT_IMPLEMENTED to indicate that a MediaTrackGraph-based fallback
159 virtual nsresult
TakePhoto(MediaEnginePhotoCallback
*) const {
160 return NS_ERROR_NOT_IMPLEMENTED
;
163 typedef MozPromise
<bool /* aIgnored */, RefPtr
<MediaMgrError
>, true>
164 ApplyConstraintsPromise
;
167 * We provide a fallback solution to ApplyConstraints() here.
168 * Sources that support ApplyConstraints() will have to override it.
170 virtual RefPtr
<ApplyConstraintsPromise
> ApplyConstraints(
171 const dom::MediaTrackConstraints
& aConstraints
, CallerType aCallerType
);
174 * Same for GetSettings (no-op).
176 virtual void GetSettings(dom::MediaTrackSettings
& aResult
){};
179 * Called by the source interface when all registered sinks with
180 * KeepsSourceAlive() == true have unregistered.
182 virtual void Stop() = 0;
185 * Called by the source interface when all registered sinks with
186 * KeepsSourceAlive() == true become disabled.
188 virtual void Disable() = 0;
191 * Called by the source interface when at least one registered sink with
192 * KeepsSourceAlive() == true become enabled.
194 virtual void Enable() = 0;
197 * Called when a Sink's Enabled() state changed. Will iterate through all
198 * sinks and notify the source of the aggregated enabled state.
200 * Note that a Sink with KeepsSourceAlive() == false counts as disabled.
202 void SinkEnabledStateChanged() {
211 * Called by each MediaStreamTrack clone on initialization.
213 void RegisterSink(Sink
* aSink
) {
214 MOZ_ASSERT(NS_IsMainThread());
218 mSinks
.AppendElement(aSink
);
219 while (mSinks
.RemoveElement(nullptr)) {
220 MOZ_ASSERT_UNREACHABLE("Sink was not explicitly removed");
225 * Called by each MediaStreamTrack clone on Stop() if supported by the
226 * source (us) or destruction.
228 void UnregisterSink(Sink
* aSink
) {
229 MOZ_ASSERT(NS_IsMainThread());
230 while (mSinks
.RemoveElement(nullptr)) {
231 MOZ_ASSERT_UNREACHABLE("Sink was not explicitly removed");
233 if (mSinks
.RemoveElement(aSink
) && !IsActive()) {
234 MOZ_ASSERT(!aSink
->KeepsSourceAlive() || !mStopped
,
235 "When the last sink keeping the source alive is removed, "
236 "we should still be live");
241 SinkEnabledStateChanged();
246 virtual ~MediaStreamTrackSource() = default;
249 for (const WeakPtr
<Sink
>& sink
: mSinks
) {
250 if (sink
&& sink
->KeepsSourceAlive()) {
258 for (const WeakPtr
<Sink
>& sink
: mSinks
) {
259 if (sink
&& sink
->KeepsSourceAlive() && sink
->Enabled()) {
267 * Called by a sub class when the principal has changed.
268 * Notifies all sinks.
270 void PrincipalChanged() {
271 MOZ_ASSERT(NS_IsMainThread());
272 for (auto& sink
: mSinks
.Clone()) {
274 DebugOnly
<bool> removed
= mSinks
.RemoveElement(sink
);
275 MOZ_ASSERT(!removed
, "Sink was not explicitly removed");
278 sink
->PrincipalChanged();
283 * Called by a sub class when the source's muted state has changed. Note that
284 * the source is responsible for making the content black/silent during mute.
285 * Notifies all sinks.
287 void MutedChanged(bool aNewState
) {
288 MOZ_ASSERT(NS_IsMainThread());
289 for (auto& sink
: mSinks
.Clone()) {
291 DebugOnly
<bool> removed
= mSinks
.RemoveElement(sink
);
292 MOZ_ASSERT(!removed
, "Sink was not explicitly removed");
295 sink
->MutedChanged(aNewState
);
300 * Called by a sub class when the source has stopped producing data for good,
301 * i.e., it has ended. Notifies all sinks.
303 void OverrideEnded() {
304 MOZ_ASSERT(NS_IsMainThread());
305 for (auto& sink
: mSinks
.Clone()) {
307 DebugOnly
<bool> removed
= mSinks
.RemoveElement(sink
);
308 MOZ_ASSERT(!removed
, "Sink was not explicitly removed");
311 sink
->OverrideEnded();
315 // Principal identifying who may access the contents of this source.
316 RefPtr
<nsIPrincipal
> mPrincipal
;
318 // Currently registered sinks.
319 nsTArray
<WeakPtr
<Sink
>> mSinks
;
322 // The label of the track we are the source of per the MediaStreamTrack spec.
323 const nsString mLabel
;
325 // Set for all video sources; an id for tracking the source of the video
326 // frames for this track.
327 const TrackingId mTrackingId
;
330 // True if all MediaStreamTrack users have unregistered from this source and
331 // Stop() has been called.
336 * Base class that consumers of a MediaStreamTrack can use to get notifications
337 * about state changes in the track.
339 class MediaStreamTrackConsumer
: public SupportsWeakPtr
{
342 * Called when the track's readyState transitions to "ended".
343 * Unlike the "ended" event exposed to script this is called for any reason,
344 * including MediaStreamTrack::Stop().
346 virtual void NotifyEnded(MediaStreamTrack
* aTrack
){};
349 * Called when the track's enabled state changes.
351 virtual void NotifyEnabledChanged(MediaStreamTrack
* aTrack
, bool aEnabled
){};
356 * DOM wrapper for MediaTrackGraph-MediaTracks.
358 * To account for cloning, a MediaStreamTrack wraps two internal (and chained)
361 * - Controlled by the producer of the data in the track. The producer
362 * decides on lifetime of the MediaTrack and the track inside it.
363 * - It can be any type of MediaTrack.
364 * - Contains one track only.
366 * - A ForwardedInputTrack representing this MediaStreamTrack.
367 * - Its data is piped from mInputTrack through mPort.
368 * - Contains one track only.
369 * - When this MediaStreamTrack is enabled/disabled this is reflected in
370 * the chunks in the track in mTrack.
371 * - When this MediaStreamTrack has ended, mTrack gets destroyed.
372 * Note that mInputTrack is unaffected, such that any clones of mTrack
373 * can live on. When all clones are ended, this is signaled to the
374 * producer via our MediaStreamTrackSource. It is then likely to destroy
377 * A graphical representation of how tracks are connected when cloned follows:
384 * MediaStreamTrack B \ (clone of A)
385 * mInputTrack \ mTrack
388 * (*) is a copy of A's mInputTrack
391 class MediaStreamTrack
: public DOMEventTargetHelper
, public SupportsWeakPtr
{
392 // PeerConnection and friends need to know our owning DOMStream and track id.
393 friend class mozilla::PeerConnectionImpl
;
394 friend class mozilla::SourceStreamInfo
;
395 friend class mozilla::RemoteSourceStreamInfo
;
402 nsPIDOMWindowInner
* aWindow
, mozilla::MediaTrack
* aInputTrack
,
403 MediaStreamTrackSource
* aSource
,
404 MediaStreamTrackState aReadyState
= MediaStreamTrackState::Live
,
406 const MediaTrackConstraints
& aConstraints
= MediaTrackConstraints());
408 NS_DECL_ISUPPORTS_INHERITED
409 NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(MediaStreamTrack
,
410 DOMEventTargetHelper
)
412 nsPIDOMWindowInner
* GetParentObject() const { return mWindow
; }
413 JSObject
* WrapObject(JSContext
* aCx
,
414 JS::Handle
<JSObject
*> aGivenProto
) override
;
416 virtual AudioStreamTrack
* AsAudioStreamTrack() { return nullptr; }
417 virtual VideoStreamTrack
* AsVideoStreamTrack() { return nullptr; }
419 virtual const AudioStreamTrack
* AsAudioStreamTrack() const { return nullptr; }
420 virtual const VideoStreamTrack
* AsVideoStreamTrack() const { return nullptr; }
423 virtual void GetKind(nsAString
& aKind
) = 0;
424 void GetId(nsAString
& aID
) const;
425 virtual void GetLabel(nsAString
& aLabel
, CallerType
/* aCallerType */) {
426 GetSource().GetLabel(aLabel
);
428 bool Enabled() const { return mEnabled
; }
429 void SetEnabled(bool aEnabled
);
430 bool Muted() { return mMuted
; }
432 void GetConstraints(dom::MediaTrackConstraints
& aResult
);
433 void GetSettings(dom::MediaTrackSettings
& aResult
, CallerType aCallerType
);
435 already_AddRefed
<Promise
> ApplyConstraints(
436 const dom::MediaTrackConstraints
& aConstraints
, CallerType aCallerType
,
438 already_AddRefed
<MediaStreamTrack
> Clone();
439 MediaStreamTrackState
ReadyState() { return mReadyState
; }
441 IMPL_EVENT_HANDLER(mute
)
442 IMPL_EVENT_HANDLER(unmute
)
443 IMPL_EVENT_HANDLER(ended
)
446 * Convenience (and legacy) method for when ready state is "ended".
448 bool Ended() const { return mReadyState
== MediaStreamTrackState::Ended
; }
451 * Get this track's principal.
453 nsIPrincipal
* GetPrincipal() const { return mPrincipal
; }
456 * Get this track's PeerIdentity.
458 const PeerIdentity
* GetPeerIdentity() const {
459 return GetSource().GetPeerIdentity();
462 ProcessedMediaTrack
* GetTrack() const;
463 MediaTrackGraph
* Graph() const;
464 MediaTrackGraphImpl
* GraphImpl() const;
466 MediaStreamTrackSource
& GetSource() const {
467 MOZ_RELEASE_ASSERT(mSource
,
468 "The track source is only removed on destruction");
472 // Webrtc allows the remote side to name tracks whatever it wants, and we
473 // need to surface this to content.
474 void AssignId(const nsAString
& aID
) { mID
= aID
; }
477 * Add a PrincipalChangeObserver to this track.
479 * Returns true if it was successfully added.
481 * Ownership of the PrincipalChangeObserver remains with the caller, and it's
482 * the caller's responsibility to remove the observer before it dies.
484 bool AddPrincipalChangeObserver(
485 PrincipalChangeObserver
<MediaStreamTrack
>* aObserver
);
488 * Remove an added PrincipalChangeObserver from this track.
490 * Returns true if it was successfully removed.
492 bool RemovePrincipalChangeObserver(
493 PrincipalChangeObserver
<MediaStreamTrack
>* aObserver
);
496 * Add a MediaStreamTrackConsumer to this track.
498 * Adding the same consumer multiple times is prohibited.
500 void AddConsumer(MediaStreamTrackConsumer
* aConsumer
);
503 * Remove an added MediaStreamTrackConsumer from this track.
505 void RemoveConsumer(MediaStreamTrackConsumer
* aConsumer
);
508 * Adds a MediaTrackListener to the MediaTrackGraph representation of
511 virtual void AddListener(MediaTrackListener
* aListener
);
514 * Removes a MediaTrackListener from the MediaTrackGraph representation
517 void RemoveListener(MediaTrackListener
* aListener
);
520 * Attempts to add a direct track listener to this track.
521 * Callers must listen to the NotifyInstalled event to know if installing
522 * the listener succeeded (tracks originating from SourceMediaTracks) or
523 * failed (e.g., WebAudio originated tracks).
525 virtual void AddDirectListener(DirectMediaTrackListener
* aListener
);
526 void RemoveDirectListener(DirectMediaTrackListener
* aListener
);
529 * Sets up a MediaInputPort from the underlying track that this
530 * MediaStreamTrack represents, to aTrack, and returns it.
532 already_AddRefed
<MediaInputPort
> ForwardTrackContentsTo(
533 ProcessedMediaTrack
* aTrack
);
536 virtual ~MediaStreamTrack();
539 * Forces the ready state to a particular value, for instance when we're
540 * cloning an already ended track.
542 void SetReadyState(MediaStreamTrackState aState
);
545 * Notified by the MediaTrackGraph, through our owning MediaStream on the
548 * Note that this sets the track to ended and raises the "ended" event
551 void OverrideEnded();
554 * Called by the MTGListener when this track's PrincipalHandle changes on
555 * the MediaTrackGraph thread. When the PrincipalHandle matches the pending
556 * principal we know that the principal change has propagated to consumers.
558 void NotifyPrincipalHandleChanged(const PrincipalHandle
& aNewPrincipalHandle
);
561 * Called when this track's readyState transitions to "ended".
562 * Notifies all MediaStreamTrackConsumers that this track ended.
567 * Called when this track's enabled state has changed.
568 * Notifies all MediaStreamTrackConsumers.
570 void NotifyEnabledChanged();
573 * Called when mSource's principal has changed.
575 void PrincipalChanged();
578 * Called when mSource's muted state has changed.
580 void MutedChanged(bool aNewState
);
583 * Sets this track's muted state without raising any events.
584 * Only really set by cloning. See MutedChanged for runtime changes.
586 void SetMuted(bool aMuted
) { mMuted
= aMuted
; }
588 virtual void Destroy();
591 * Sets the principal and notifies PrincipalChangeObservers if it changes.
593 void SetPrincipal(nsIPrincipal
* aPrincipal
);
596 * Creates a new MediaStreamTrack with the same kind, input track, input
597 * track ID and source as this MediaStreamTrack.
599 virtual already_AddRefed
<MediaStreamTrack
> CloneInternal() = 0;
601 nsTArray
<PrincipalChangeObserver
<MediaStreamTrack
>*>
602 mPrincipalChangeObservers
;
604 nsTArray
<WeakPtr
<MediaStreamTrackConsumer
>> mConsumers
;
606 // We need this to track our parent object.
607 nsCOMPtr
<nsPIDOMWindowInner
> mWindow
;
609 // The input MediaTrack assigned us by the data producer.
610 // Owned by the producer.
611 const RefPtr
<mozilla::MediaTrack
> mInputTrack
;
612 // The MediaTrack representing this MediaStreamTrack in the MediaTrackGraph.
613 // Set on construction if we're live. Valid until we end. Owned by us.
614 RefPtr
<ProcessedMediaTrack
> mTrack
;
615 // The MediaInputPort connecting mInputTrack to mTrack. Set on construction
616 // if mInputTrack is non-destroyed and we're live. Valid until we end. Owned
618 RefPtr
<MediaInputPort
> mPort
;
619 RefPtr
<MediaStreamTrackSource
> mSource
;
620 const UniquePtr
<TrackSink
> mSink
;
621 nsCOMPtr
<nsIPrincipal
> mPrincipal
;
622 nsCOMPtr
<nsIPrincipal
> mPendingPrincipal
;
623 RefPtr
<MTGListener
> mMTGListener
;
624 // Keep tracking MediaTrackListener and DirectMediaTrackListener,
625 // so we can remove them in |Destory|.
626 nsTArray
<RefPtr
<MediaTrackListener
>> mTrackListeners
;
627 nsTArray
<RefPtr
<DirectMediaTrackListener
>> mDirectTrackListeners
;
629 MediaStreamTrackState mReadyState
;
632 dom::MediaTrackConstraints mConstraints
;
636 } // namespace mozilla
638 #endif /* MEDIASTREAMTRACK_H_ */