1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 * You can obtain one at http://mozilla.org/MPL/2.0/. */
6 #ifndef MEDIASTREAMTRACK_H_
7 #define MEDIASTREAMTRACK_H_
9 #include "MediaTrackConstraints.h"
10 #include "PrincipalChangeObserver.h"
11 #include "PrincipalHandle.h"
12 #include "mozilla/DOMEventTargetHelper.h"
13 #include "mozilla/dom/MediaStreamTrackBinding.h"
14 #include "mozilla/dom/MediaTrackSettingsBinding.h"
15 #include "mozilla/media/MediaUtils.h"
16 #include "mozilla/WeakPtr.h"
19 #include "nsIPrincipal.h"
24 class MediaEnginePhotoCallback
;
27 class MediaTrackGraph
;
28 class MediaTrackGraphImpl
;
29 class MediaTrackListener
;
30 class DirectMediaTrackListener
;
31 class PeerConnectionImpl
;
32 class PeerConnectionMedia
;
34 class ProcessedMediaTrack
;
35 class RemoteSourceStreamInfo
;
36 class SourceStreamInfo
;
41 class AudioStreamTrack
;
42 class VideoStreamTrack
;
43 enum class CallerType
: uint32_t;
46 * Common interface through which a MediaStreamTrack can communicate with its
47 * producer on the main thread.
49 * Kept alive by a strong ref in all MediaStreamTracks (original and clones)
50 * sharing this source.
52 class MediaStreamTrackSource
: public nsISupports
{
53 NS_DECL_CYCLE_COLLECTING_ISUPPORTS
54 NS_DECL_CYCLE_COLLECTION_CLASS(MediaStreamTrackSource
)
57 class Sink
: public SupportsWeakPtr
{
60 * Must be constant throughout the Sink's lifetime.
62 * Return true to keep the MediaStreamTrackSource where this sink is
64 * Return false to allow the source to stop.
66 * Typically MediaStreamTrack::Sink returns true and other Sinks
67 * (like HTMLMediaElement::StreamCaptureTrackSource) return false.
69 virtual bool KeepsSourceAlive() const = 0;
72 * Return true to ensure that the MediaStreamTrackSource where this Sink is
73 * registered is kept turned on and active.
74 * Return false to allow the source to pause, and any underlying devices to
77 * When the underlying enabled state of the sink changes,
78 * call MediaStreamTrackSource::SinkEnabledStateChanged().
80 * Typically MediaStreamTrack returns the track's enabled state and other
81 * Sinks (like HTMLMediaElement::StreamCaptureTrackSource) return false so
82 * control over device state remains with tracks and their enabled state.
84 virtual bool Enabled() const = 0;
87 * Called when the principal of the MediaStreamTrackSource where this sink
88 * is registered has changed.
90 virtual void PrincipalChanged() = 0;
93 * Called when the muted state of the MediaStreamTrackSource where this sink
94 * is registered has changed.
96 virtual void MutedChanged(bool aNewState
) = 0;
99 * Called when the MediaStreamTrackSource where this sink is registered has
100 * stopped producing data for good, i.e., it has ended.
102 virtual void OverrideEnded() = 0;
105 virtual ~Sink() = default;
108 MediaStreamTrackSource(nsIPrincipal
* aPrincipal
, const nsString
& aLabel
)
109 : mPrincipal(aPrincipal
), mLabel(aLabel
), mStopped(false) {}
112 * Use to clean up any resources that have to be cleaned before the
113 * destructor is called. It is often too late in the destructor because
114 * of garbage collection having removed the members already.
116 virtual void Destroy() {}
119 * Gets the source's MediaSourceEnum for usage by PeerConnections.
121 virtual MediaSourceEnum
GetMediaSource() const = 0;
124 * Get this TrackSource's principal.
126 nsIPrincipal
* GetPrincipal() const { return mPrincipal
; }
129 * This is used in WebRTC. A peerIdentity constrained MediaStreamTrack cannot
130 * be sent across the network to anything other than a peer with the provided
131 * identity. If this is set, then GetPrincipal() should return an instance of
134 * A track's PeerIdentity is immutable and will not change during the track's
137 virtual const PeerIdentity
* GetPeerIdentity() const { return nullptr; }
140 * MediaStreamTrack::GetLabel (see spec) calls through to here.
142 void GetLabel(nsAString
& aLabel
) { aLabel
.Assign(mLabel
); }
145 * Whether this TrackSource provides video frames with an alpha channel. Only
146 * applies to video sources. Used by HTMLVideoElement.
148 virtual bool HasAlpha() const { return false; }
151 * Forwards a photo request to backends that support it. Other backends return
152 * NS_ERROR_NOT_IMPLEMENTED to indicate that a MediaTrackGraph-based fallback
155 virtual nsresult
TakePhoto(MediaEnginePhotoCallback
*) const {
156 return NS_ERROR_NOT_IMPLEMENTED
;
159 typedef MozPromise
<bool /* aIgnored */, RefPtr
<MediaMgrError
>, true>
160 ApplyConstraintsPromise
;
163 * We provide a fallback solution to ApplyConstraints() here.
164 * Sources that support ApplyConstraints() will have to override it.
166 virtual RefPtr
<ApplyConstraintsPromise
> ApplyConstraints(
167 const dom::MediaTrackConstraints
& aConstraints
, CallerType aCallerType
);
170 * Same for GetSettings (no-op).
172 virtual void GetSettings(dom::MediaTrackSettings
& aResult
){};
175 * Called by the source interface when all registered sinks with
176 * KeepsSourceAlive() == true have unregistered.
178 virtual void Stop() = 0;
181 * Called by the source interface when all registered sinks with
182 * KeepsSourceAlive() == true become disabled.
184 virtual void Disable() = 0;
187 * Called by the source interface when at least one registered sink with
188 * KeepsSourceAlive() == true become enabled.
190 virtual void Enable() = 0;
193 * Called when a Sink's Enabled() state changed. Will iterate through all
194 * sinks and notify the source of the aggregated enabled state.
196 * Note that a Sink with KeepsSourceAlive() == false counts as disabled.
198 void SinkEnabledStateChanged() {
207 * Called by each MediaStreamTrack clone on initialization.
209 void RegisterSink(Sink
* aSink
) {
210 MOZ_ASSERT(NS_IsMainThread());
214 mSinks
.AppendElement(aSink
);
215 while (mSinks
.RemoveElement(nullptr)) {
216 MOZ_ASSERT_UNREACHABLE("Sink was not explicitly removed");
221 * Called by each MediaStreamTrack clone on Stop() if supported by the
222 * source (us) or destruction.
224 void UnregisterSink(Sink
* aSink
) {
225 MOZ_ASSERT(NS_IsMainThread());
226 while (mSinks
.RemoveElement(nullptr)) {
227 MOZ_ASSERT_UNREACHABLE("Sink was not explicitly removed");
229 if (mSinks
.RemoveElement(aSink
) && !IsActive()) {
230 MOZ_ASSERT(!aSink
->KeepsSourceAlive() || !mStopped
,
231 "When the last sink keeping the source alive is removed, "
232 "we should still be live");
237 SinkEnabledStateChanged();
242 virtual ~MediaStreamTrackSource() = default;
245 for (const WeakPtr
<Sink
>& sink
: mSinks
) {
246 if (sink
&& sink
->KeepsSourceAlive()) {
254 for (const WeakPtr
<Sink
>& sink
: mSinks
) {
255 if (sink
&& sink
->KeepsSourceAlive() && sink
->Enabled()) {
263 * Called by a sub class when the principal has changed.
264 * Notifies all sinks.
266 void PrincipalChanged() {
267 MOZ_ASSERT(NS_IsMainThread());
268 for (auto& sink
: mSinks
.Clone()) {
270 DebugOnly
<bool> removed
= mSinks
.RemoveElement(sink
);
271 MOZ_ASSERT(!removed
, "Sink was not explicitly removed");
274 sink
->PrincipalChanged();
279 * Called by a sub class when the source's muted state has changed. Note that
280 * the source is responsible for making the content black/silent during mute.
281 * Notifies all sinks.
283 void MutedChanged(bool aNewState
) {
284 MOZ_ASSERT(NS_IsMainThread());
285 for (auto& sink
: mSinks
.Clone()) {
287 DebugOnly
<bool> removed
= mSinks
.RemoveElement(sink
);
288 MOZ_ASSERT(!removed
, "Sink was not explicitly removed");
291 sink
->MutedChanged(aNewState
);
296 * Called by a sub class when the source has stopped producing data for good,
297 * i.e., it has ended. Notifies all sinks.
299 void OverrideEnded() {
300 MOZ_ASSERT(NS_IsMainThread());
301 for (auto& sink
: mSinks
.Clone()) {
303 DebugOnly
<bool> removed
= mSinks
.RemoveElement(sink
);
304 MOZ_ASSERT(!removed
, "Sink was not explicitly removed");
307 sink
->OverrideEnded();
311 // Principal identifying who may access the contents of this source.
312 RefPtr
<nsIPrincipal
> mPrincipal
;
314 // Currently registered sinks.
315 nsTArray
<WeakPtr
<Sink
>> mSinks
;
317 // The label of the track we are the source of per the MediaStreamTrack spec.
318 const nsString mLabel
;
320 // True if all MediaStreamTrack users have unregistered from this source and
321 // Stop() has been called.
326 * Basic implementation of MediaStreamTrackSource that doesn't forward Stop().
328 class BasicTrackSource
: public MediaStreamTrackSource
{
330 explicit BasicTrackSource(
331 nsIPrincipal
* aPrincipal
,
332 const MediaSourceEnum aMediaSource
= MediaSourceEnum::Other
)
333 : MediaStreamTrackSource(aPrincipal
, nsString()),
334 mMediaSource(aMediaSource
) {}
336 MediaSourceEnum
GetMediaSource() const override
{ return mMediaSource
; }
338 void Stop() override
{}
339 void Disable() override
{}
340 void Enable() override
{}
343 ~BasicTrackSource() = default;
345 const MediaSourceEnum mMediaSource
;
349 * Base class that consumers of a MediaStreamTrack can use to get notifications
350 * about state changes in the track.
352 class MediaStreamTrackConsumer
: public SupportsWeakPtr
{
355 * Called when the track's readyState transitions to "ended".
356 * Unlike the "ended" event exposed to script this is called for any reason,
357 * including MediaStreamTrack::Stop().
359 virtual void NotifyEnded(MediaStreamTrack
* aTrack
){};
362 * Called when the track's enabled state changes.
364 virtual void NotifyEnabledChanged(MediaStreamTrack
* aTrack
, bool aEnabled
){};
369 * DOM wrapper for MediaTrackGraph-MediaTracks.
371 * To account for cloning, a MediaStreamTrack wraps two internal (and chained)
374 * - Controlled by the producer of the data in the track. The producer
375 * decides on lifetime of the MediaTrack and the track inside it.
376 * - It can be any type of MediaTrack.
377 * - Contains one track only.
379 * - A ForwardedInputTrack representing this MediaStreamTrack.
380 * - Its data is piped from mInputTrack through mPort.
381 * - Contains one track only.
382 * - When this MediaStreamTrack is enabled/disabled this is reflected in
383 * the chunks in the track in mTrack.
384 * - When this MediaStreamTrack has ended, mTrack gets destroyed.
385 * Note that mInputTrack is unaffected, such that any clones of mTrack
386 * can live on. When all clones are ended, this is signaled to the
387 * producer via our MediaStreamTrackSource. It is then likely to destroy
390 * A graphical representation of how tracks are connected when cloned follows:
397 * MediaStreamTrack B \ (clone of A)
398 * mInputTrack \ mTrack
401 * (*) is a copy of A's mInputTrack
404 class MediaStreamTrack
: public DOMEventTargetHelper
, public SupportsWeakPtr
{
405 // PeerConnection and friends need to know our owning DOMStream and track id.
406 friend class mozilla::PeerConnectionImpl
;
407 friend class mozilla::PeerConnectionMedia
;
408 friend class mozilla::SourceStreamInfo
;
409 friend class mozilla::RemoteSourceStreamInfo
;
416 nsPIDOMWindowInner
* aWindow
, mozilla::MediaTrack
* aInputTrack
,
417 MediaStreamTrackSource
* aSource
,
418 MediaStreamTrackState aReadyState
= MediaStreamTrackState::Live
,
420 const MediaTrackConstraints
& aConstraints
= MediaTrackConstraints());
422 NS_DECL_ISUPPORTS_INHERITED
423 NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(MediaStreamTrack
,
424 DOMEventTargetHelper
)
426 nsPIDOMWindowInner
* GetParentObject() const { return mWindow
; }
427 JSObject
* WrapObject(JSContext
* aCx
,
428 JS::Handle
<JSObject
*> aGivenProto
) override
;
430 virtual AudioStreamTrack
* AsAudioStreamTrack() { return nullptr; }
431 virtual VideoStreamTrack
* AsVideoStreamTrack() { return nullptr; }
433 virtual const AudioStreamTrack
* AsAudioStreamTrack() const { return nullptr; }
434 virtual const VideoStreamTrack
* AsVideoStreamTrack() const { return nullptr; }
437 virtual void GetKind(nsAString
& aKind
) = 0;
438 void GetId(nsAString
& aID
) const;
439 virtual void GetLabel(nsAString
& aLabel
, CallerType
/* aCallerType */) {
440 GetSource().GetLabel(aLabel
);
442 bool Enabled() const { return mEnabled
; }
443 void SetEnabled(bool aEnabled
);
444 bool Muted() { return mMuted
; }
446 void GetConstraints(dom::MediaTrackConstraints
& aResult
);
447 void GetSettings(dom::MediaTrackSettings
& aResult
, CallerType aCallerType
);
449 already_AddRefed
<Promise
> ApplyConstraints(
450 const dom::MediaTrackConstraints
& aConstraints
, CallerType aCallerType
,
452 already_AddRefed
<MediaStreamTrack
> Clone();
453 MediaStreamTrackState
ReadyState() { return mReadyState
; }
455 IMPL_EVENT_HANDLER(mute
)
456 IMPL_EVENT_HANDLER(unmute
)
457 IMPL_EVENT_HANDLER(ended
)
460 * Convenience (and legacy) method for when ready state is "ended".
462 bool Ended() const { return mReadyState
== MediaStreamTrackState::Ended
; }
465 * Get this track's principal.
467 nsIPrincipal
* GetPrincipal() const { return mPrincipal
; }
470 * Get this track's PeerIdentity.
472 const PeerIdentity
* GetPeerIdentity() const {
473 return GetSource().GetPeerIdentity();
476 ProcessedMediaTrack
* GetTrack() const;
477 MediaTrackGraph
* Graph() const;
478 MediaTrackGraphImpl
* GraphImpl() const;
480 MediaStreamTrackSource
& GetSource() const {
481 MOZ_RELEASE_ASSERT(mSource
,
482 "The track source is only removed on destruction");
486 // Webrtc allows the remote side to name tracks whatever it wants, and we
487 // need to surface this to content.
488 void AssignId(const nsAString
& aID
) { mID
= aID
; }
491 * Add a PrincipalChangeObserver to this track.
493 * Returns true if it was successfully added.
495 * Ownership of the PrincipalChangeObserver remains with the caller, and it's
496 * the caller's responsibility to remove the observer before it dies.
498 bool AddPrincipalChangeObserver(
499 PrincipalChangeObserver
<MediaStreamTrack
>* aObserver
);
502 * Remove an added PrincipalChangeObserver from this track.
504 * Returns true if it was successfully removed.
506 bool RemovePrincipalChangeObserver(
507 PrincipalChangeObserver
<MediaStreamTrack
>* aObserver
);
510 * Add a MediaStreamTrackConsumer to this track.
512 * Adding the same consumer multiple times is prohibited.
514 void AddConsumer(MediaStreamTrackConsumer
* aConsumer
);
517 * Remove an added MediaStreamTrackConsumer from this track.
519 void RemoveConsumer(MediaStreamTrackConsumer
* aConsumer
);
522 * Adds a MediaTrackListener to the MediaTrackGraph representation of
525 virtual void AddListener(MediaTrackListener
* aListener
);
528 * Removes a MediaTrackListener from the MediaTrackGraph representation
531 void RemoveListener(MediaTrackListener
* aListener
);
534 * Attempts to add a direct track listener to this track.
535 * Callers must listen to the NotifyInstalled event to know if installing
536 * the listener succeeded (tracks originating from SourceMediaTracks) or
537 * failed (e.g., WebAudio originated tracks).
539 virtual void AddDirectListener(DirectMediaTrackListener
* aListener
);
540 void RemoveDirectListener(DirectMediaTrackListener
* aListener
);
543 * Sets up a MediaInputPort from the underlying track that this
544 * MediaStreamTrack represents, to aTrack, and returns it.
546 already_AddRefed
<MediaInputPort
> ForwardTrackContentsTo(
547 ProcessedMediaTrack
* aTrack
);
550 virtual ~MediaStreamTrack();
553 * Forces the ready state to a particular value, for instance when we're
554 * cloning an already ended track.
556 virtual void SetReadyState(MediaStreamTrackState aState
);
559 * Notified by the MediaTrackGraph, through our owning MediaStream on the
562 * Note that this sets the track to ended and raises the "ended" event
565 void OverrideEnded();
568 * Called by the MTGListener when this track's PrincipalHandle changes on
569 * the MediaTrackGraph thread. When the PrincipalHandle matches the pending
570 * principal we know that the principal change has propagated to consumers.
572 void NotifyPrincipalHandleChanged(const PrincipalHandle
& aNewPrincipalHandle
);
575 * Called when this track's readyState transitions to "ended".
576 * Notifies all MediaStreamTrackConsumers that this track ended.
581 * Called when this track's enabled state has changed.
582 * Notifies all MediaStreamTrackConsumers.
584 void NotifyEnabledChanged();
587 * Called when mSource's principal has changed.
589 void PrincipalChanged();
592 * Called when mSource's muted state has changed.
594 void MutedChanged(bool aNewState
);
597 * Sets this track's muted state without raising any events.
598 * Only really set by cloning. See MutedChanged for runtime changes.
600 void SetMuted(bool aMuted
) { mMuted
= aMuted
; }
602 virtual void Destroy();
605 * Sets the principal and notifies PrincipalChangeObservers if it changes.
607 void SetPrincipal(nsIPrincipal
* aPrincipal
);
610 * Creates a new MediaStreamTrack with the same kind, input track, input
611 * track ID and source as this MediaStreamTrack.
613 virtual already_AddRefed
<MediaStreamTrack
> CloneInternal() = 0;
615 nsTArray
<PrincipalChangeObserver
<MediaStreamTrack
>*>
616 mPrincipalChangeObservers
;
618 nsTArray
<WeakPtr
<MediaStreamTrackConsumer
>> mConsumers
;
620 // We need this to track our parent object.
621 nsCOMPtr
<nsPIDOMWindowInner
> mWindow
;
623 // The input MediaTrack assigned us by the data producer.
624 // Owned by the producer.
625 const RefPtr
<mozilla::MediaTrack
> mInputTrack
;
626 // The MediaTrack representing this MediaStreamTrack in the MediaTrackGraph.
627 // Set on construction if we're live. Valid until we end. Owned by us.
628 RefPtr
<ProcessedMediaTrack
> mTrack
;
629 // The MediaInputPort connecting mInputTrack to mTrack. Set on construction
630 // if mInputTrack is non-destroyed and we're live. Valid until we end. Owned
632 RefPtr
<MediaInputPort
> mPort
;
633 RefPtr
<MediaStreamTrackSource
> mSource
;
634 const UniquePtr
<TrackSink
> mSink
;
635 nsCOMPtr
<nsIPrincipal
> mPrincipal
;
636 nsCOMPtr
<nsIPrincipal
> mPendingPrincipal
;
637 RefPtr
<MTGListener
> mMTGListener
;
638 // Keep tracking MediaTrackListener and DirectMediaTrackListener,
639 // so we can remove them in |Destory|.
640 nsTArray
<RefPtr
<MediaTrackListener
>> mTrackListeners
;
641 nsTArray
<RefPtr
<DirectMediaTrackListener
>> mDirectTrackListeners
;
643 MediaStreamTrackState mReadyState
;
646 dom::MediaTrackConstraints mConstraints
;
650 } // namespace mozilla
652 #endif /* MEDIASTREAMTRACK_H_ */