dom/media/MediaStreamTrack.h

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
   2 /* This Source Code Form is subject to the terms of the Mozilla Public
   3  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
   4  * You can obtain one at http://mozilla.org/MPL/2.0/. */
   5
   6 #ifndef MEDIASTREAMTRACK_H_
   7 #define MEDIASTREAMTRACK_H_
   8
   9 #include "MediaTrackConstraints.h"
  10 #include "PrincipalChangeObserver.h"
  11 #include "StreamTracks.h"
  12 #include "mozilla/CORSMode.h"
  13 #include "mozilla/DOMEventTargetHelper.h"
  14 #include "mozilla/dom/MediaStreamTrackBinding.h"
  15 #include "mozilla/dom/MediaTrackSettingsBinding.h"
  16 #include "mozilla/media/MediaUtils.h"
  17 #include "mozilla/WeakPtr.h"
  18 #include "nsError.h"
  19 #include "nsID.h"
  20 #include "nsIPrincipal.h"
  21
  22 namespace mozilla {
  23
  24 class DOMMediaStream;
  25 class MediaEnginePhotoCallback;
  26 class MediaInputPort;
  27 class MediaStream;
  28 class MediaStreamGraph;
  29 class MediaStreamGraphImpl;
  30 class MediaStreamTrackListener;
  31 class DirectMediaStreamTrackListener;
  32 class PeerConnectionImpl;
  33 class PeerConnectionMedia;
  34 class PeerIdentity;
  35 class ProcessedMediaStream;
  36 class RemoteSourceStreamInfo;
  37 class SourceStreamInfo;
  38 class MediaMgrError;
  39
  40 namespace dom {
  41
  42 class AudioStreamTrack;
  43 class VideoStreamTrack;
  44 class TrackSink;
  45 enum class CallerType : uint32_t;
  46
  47 /**
  48  * Common interface through which a MediaStreamTrack can communicate with its
  49  * producer on the main thread.
  50  *
  51  * Kept alive by a strong ref in all MediaStreamTracks (original and clones)
  52  * sharing this source.
  53  */
  54 class MediaStreamTrackSource : public nsISupports {
  55   NS_DECL_CYCLE_COLLECTING_ISUPPORTS
  56   NS_DECL_CYCLE_COLLECTION_CLASS(MediaStreamTrackSource)
  57
  58  public:
  59   class Sink : public SupportsWeakPtr<Sink> {
  60    public:
  61     MOZ_DECLARE_WEAKREFERENCE_TYPENAME(MediaStreamTrackSource::Sink)
  62
  63     /**
  64      * Must be constant throughout the Sink's lifetime.
  65      *
  66      * Return true to keep the MediaStreamTrackSource where this sink is
  67      * registered alive.
  68      * Return false to allow the source to stop.
  69      *
  70      * Typically MediaStreamTrack::Sink returns true and other Sinks
  71      * (like HTMLMediaElement::StreamCaptureTrackSource) return false.
  72      */
  73     virtual bool KeepsSourceAlive() const = 0;
  74
  75     /**
  76      * Return true to ensure that the MediaStreamTrackSource where this Sink is
  77      * registered is kept turned on and active.
  78      * Return false to allow the source to pause, and any underlying devices to
  79      * temporarily stop.
  80      *
  81      * When the underlying enabled state of the sink changes,
  82      * call MediaStreamTrackSource::SinkEnabledStateChanged().
  83      *
  84      * Typically MediaStreamTrack returns the track's enabled state and other
  85      * Sinks (like HTMLMediaElement::StreamCaptureTrackSource) return false so
  86      * control over device state remains with tracks and their enabled state.
  87      */
  88     virtual bool Enabled() const = 0;
  89
  90     /**
  91      * Called when the principal of the MediaStreamTrackSource where this sink
  92      * is registered has changed.
  93      */
  94     virtual void PrincipalChanged() = 0;
  95
  96     /**
  97      * Called when the muted state of the MediaStreamTrackSource where this sink
  98      * is registered has changed.
  99      */
 100     virtual void MutedChanged(bool aNewState) = 0;
 101
 102    protected:
 103     virtual ~Sink() = default;
 104   };
 105
 106   MediaStreamTrackSource(nsIPrincipal* aPrincipal, const nsString& aLabel)
 107       : mPrincipal(aPrincipal), mLabel(aLabel), mStopped(false) {}
 108
 109   /**
 110    * Use to clean up any resources that have to be cleaned before the
 111    * destructor is called. It is often too late in the destructor because
 112    * of garbage collection having removed the members already.
 113    */
 114   virtual void Destroy() {}
 115
 116   /**
 117    * Gets the source's MediaSourceEnum for usage by PeerConnections.
 118    */
 119   virtual MediaSourceEnum GetMediaSource() const = 0;
 120
 121   /**
 122    * Get this TrackSource's principal.
 123    */
 124   nsIPrincipal* GetPrincipal() const { return mPrincipal; }
 125
 126   /**
 127    * Get the source's current CORSMode. If not applicable CORS_NONE is returned.
 128    * The sink will be notified of changes to our CORSMode through
 129    * PrincipalChanged().
 130    */
 131   virtual CORSMode GetCORSMode() const { return CORS_NONE; }
 132
 133   /**
 134    * This is used in WebRTC. A peerIdentity constrained MediaStreamTrack cannot
 135    * be sent across the network to anything other than a peer with the provided
 136    * identity. If this is set, then GetPrincipal() should return an instance of
 137    * NullPrincipal.
 138    *
 139    * A track's PeerIdentity is immutable and will not change during the track's
 140    * lifetime.
 141    */
 142   virtual const PeerIdentity* GetPeerIdentity() const { return nullptr; }
 143
 144   /**
 145    * MediaStreamTrack::GetLabel (see spec) calls through to here.
 146    */
 147   void GetLabel(nsAString& aLabel) { aLabel.Assign(mLabel); }
 148
 149   /**
 150    * Forwards a photo request to backends that support it. Other backends return
 151    * NS_ERROR_NOT_IMPLEMENTED to indicate that a MediaStreamGraph-based fallback
 152    * should be used.
 153    */
 154   virtual nsresult TakePhoto(MediaEnginePhotoCallback*) const {
 155     return NS_ERROR_NOT_IMPLEMENTED;
 156   }
 157
 158   typedef MozPromise<bool /* aIgnored */, RefPtr<MediaMgrError>, true>
 159       ApplyConstraintsPromise;
 160
 161   /**
 162    * We provide a fallback solution to ApplyConstraints() here.
 163    * Sources that support ApplyConstraints() will have to override it.
 164    */
 165   virtual RefPtr<ApplyConstraintsPromise> ApplyConstraints(
 166       const dom::MediaTrackConstraints& aConstraints, CallerType aCallerType);
 167
 168   /**
 169    * Same for GetSettings (no-op).
 170    */
 171   virtual void GetSettings(dom::MediaTrackSettings& aResult){};
 172
 173   /**
 174    * Called by the source interface when all registered sinks with
 175    * KeepsSourceAlive() == true have unregistered.
 176    */
 177   virtual void Stop() = 0;
 178
 179   /**
 180    * Called by the source interface when all registered sinks with
 181    * KeepsSourceAlive() == true become disabled.
 182    */
 183   virtual void Disable() = 0;
 184
 185   /**
 186    * Called by the source interface when at least one registered sink with
 187    * KeepsSourceAlive() == true become enabled.
 188    */
 189   virtual void Enable() = 0;
 190
 191   /**
 192    * Called when a Sink's Enabled() state changed. Will iterate through all
 193    * sinks and notify the source of the aggregated enabled state.
 194    *
 195    * Note that a Sink with KeepsSourceAlive() == false counts as disabled.
 196    */
 197   void SinkEnabledStateChanged() {
 198     if (IsEnabled()) {
 199       Enable();
 200     } else {
 201       Disable();
 202     }
 203   }
 204
 205   /**
 206    * Called by each MediaStreamTrack clone on initialization.
 207    */
 208   void RegisterSink(Sink* aSink) {
 209     MOZ_ASSERT(NS_IsMainThread());
 210     if (mStopped) {
 211       return;
 212     }
 213     mSinks.AppendElement(aSink);
 214     while (mSinks.RemoveElement(nullptr)) {
 215       MOZ_ASSERT_UNREACHABLE("Sink was not explicitly removed");
 216     }
 217   }
 218
 219   /**
 220    * Called by each MediaStreamTrack clone on Stop() if supported by the
 221    * source (us) or destruction.
 222    */
 223   void UnregisterSink(Sink* aSink) {
 224     MOZ_ASSERT(NS_IsMainThread());
 225     while (mSinks.RemoveElement(nullptr)) {
 226       MOZ_ASSERT_UNREACHABLE("Sink was not explicitly removed");
 227     }
 228     if (mSinks.RemoveElement(aSink) && !IsActive()) {
 229       MOZ_ASSERT(!aSink->KeepsSourceAlive() || !mStopped,
 230                  "When the last sink keeping the source alive is removed, "
 231                  "we should still be live");
 232       Stop();
 233       mStopped = true;
 234     }
 235     if (!mStopped) {
 236       SinkEnabledStateChanged();
 237     }
 238   }
 239
 240  protected:
 241   virtual ~MediaStreamTrackSource() {}
 242
 243   bool IsActive() {
 244     for (const WeakPtr<Sink>& sink : mSinks) {
 245       if (sink && sink->KeepsSourceAlive()) {
 246         return true;
 247       }
 248     }
 249     return false;
 250   }
 251
 252   bool IsEnabled() {
 253     for (const WeakPtr<Sink>& sink : mSinks) {
 254       if (sink && sink->KeepsSourceAlive() && sink->Enabled()) {
 255         return true;
 256       }
 257     }
 258     return false;
 259   }
 260
 261   /**
 262    * Called by a sub class when the principal has changed.
 263    * Notifies all sinks.
 264    */
 265   void PrincipalChanged() {
 266     MOZ_ASSERT(NS_IsMainThread());
 267     nsTArray<WeakPtr<Sink>> sinks(mSinks);
 268     for (auto& sink : sinks) {
 269       if (!sink) {
 270         MOZ_ASSERT_UNREACHABLE("Sink was not explicitly removed");
 271         mSinks.RemoveElement(sink);
 272         continue;
 273       }
 274       sink->PrincipalChanged();
 275     }
 276   }
 277
 278   /**
 279    * Called by a sub class when the source's muted state has changed. Note that
 280    * the source is responsible for making the content black/silent during mute.
 281    * Notifies all sinks.
 282    */
 283   void MutedChanged(bool aNewState) {
 284     MOZ_ASSERT(NS_IsMainThread());
 285     nsTArray<WeakPtr<Sink>> sinks(mSinks);
 286     for (auto& sink : sinks) {
 287       if (!sink) {
 288         MOZ_ASSERT_UNREACHABLE("Sink was not explicitly removed");
 289         mSinks.RemoveElement(sink);
 290         continue;
 291       }
 292       sink->MutedChanged(aNewState);
 293     }
 294   }
 295
 296   // Principal identifying who may access the contents of this source.
 297   nsCOMPtr<nsIPrincipal> mPrincipal;
 298
 299   // Currently registered sinks.
 300   nsTArray<WeakPtr<Sink>> mSinks;
 301
 302   // The label of the track we are the source of per the MediaStreamTrack spec.
 303   const nsString mLabel;
 304
 305   // True if all MediaStreamTrack users have unregistered from this source and
 306   // Stop() has been called.
 307   bool mStopped;
 308 };
 309
 310 /**
 311  * Basic implementation of MediaStreamTrackSource that doesn't forward Stop().
 312  */
 313 class BasicTrackSource : public MediaStreamTrackSource {
 314  public:
 315   explicit BasicTrackSource(
 316       nsIPrincipal* aPrincipal,
 317       const MediaSourceEnum aMediaSource = MediaSourceEnum::Other)
 318       : MediaStreamTrackSource(aPrincipal, nsString()),
 319         mMediaSource(aMediaSource) {}
 320
 321   MediaSourceEnum GetMediaSource() const override { return mMediaSource; }
 322
 323   void Stop() override {}
 324   void Disable() override {}
 325   void Enable() override {}
 326
 327  protected:
 328   ~BasicTrackSource() {}
 329
 330   const MediaSourceEnum mMediaSource;
 331 };
 332
 333 /**
 334  * Base class that consumers of a MediaStreamTrack can use to get notifications
 335  * about state changes in the track.
 336  */
 337 class MediaStreamTrackConsumer
 338     : public SupportsWeakPtr<MediaStreamTrackConsumer> {
 339  public:
 340   MOZ_DECLARE_WEAKREFERENCE_TYPENAME(MediaStreamTrackConsumer)
 341
 342   /**
 343    * Called when the track's readyState transitions to "ended".
 344    * Unlike the "ended" event exposed to script this is called for any reason,
 345    * including MediaStreamTrack::Stop().
 346    */
 347   virtual void NotifyEnded(MediaStreamTrack* aTrack){};
 348 };
 349
 350 /**
 351  * Class representing a track in a DOMMediaStream.
 352  */
 353 class MediaStreamTrack : public DOMEventTargetHelper,
 354                          public SupportsWeakPtr<MediaStreamTrack> {
 355   // DOMMediaStream owns MediaStreamTrack instances, and requires access to
 356   // some internal state, e.g., GetInputStream(), GetOwnedStream().
 357   friend class mozilla::DOMMediaStream;
 358
 359   // PeerConnection and friends need to know our owning DOMStream and track id.
 360   friend class mozilla::PeerConnectionImpl;
 361   friend class mozilla::PeerConnectionMedia;
 362   friend class mozilla::SourceStreamInfo;
 363   friend class mozilla::RemoteSourceStreamInfo;
 364
 365   class MSGListener;
 366
 367  public:
 368   /**
 369    * aTrackID is the MediaStreamGraph track ID for the track in the
 370    * MediaStream owned by aStream.
 371    */
 372   MediaStreamTrack(
 373       DOMMediaStream* aStream, TrackID aTrackID, TrackID aInputTrackID,
 374       MediaStreamTrackSource* aSource,
 375       const MediaTrackConstraints& aConstraints = MediaTrackConstraints());
 376
 377   NS_DECL_ISUPPORTS_INHERITED
 378   NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(MediaStreamTrack,
 379                                            DOMEventTargetHelper)
 380
 381   MOZ_DECLARE_WEAKREFERENCE_TYPENAME(MediaStreamTrack)
 382
 383   nsPIDOMWindowInner* GetParentObject() const;
 384   JSObject* WrapObject(JSContext* aCx,
 385                        JS::Handle<JSObject*> aGivenProto) override;
 386
 387   virtual AudioStreamTrack* AsAudioStreamTrack() { return nullptr; }
 388   virtual VideoStreamTrack* AsVideoStreamTrack() { return nullptr; }
 389
 390   virtual const AudioStreamTrack* AsAudioStreamTrack() const { return nullptr; }
 391   virtual const VideoStreamTrack* AsVideoStreamTrack() const { return nullptr; }
 392
 393   // WebIDL
 394   virtual void GetKind(nsAString& aKind) = 0;
 395   void GetId(nsAString& aID) const;
 396   virtual void GetLabel(nsAString& aLabel, CallerType /* aCallerType */) {
 397     GetSource().GetLabel(aLabel);
 398   }
 399   bool Enabled() const { return mEnabled; }
 400   void SetEnabled(bool aEnabled);
 401   bool Muted() { return mMuted; }
 402   void Stop();
 403   void GetConstraints(dom::MediaTrackConstraints& aResult);
 404   void GetSettings(dom::MediaTrackSettings& aResult, CallerType aCallerType);
 405
 406   already_AddRefed<Promise> ApplyConstraints(
 407       const dom::MediaTrackConstraints& aConstraints, CallerType aCallerType,
 408       ErrorResult& aRv);
 409   already_AddRefed<MediaStreamTrack> Clone();
 410   MediaStreamTrackState ReadyState() { return mReadyState; }
 411
 412   IMPL_EVENT_HANDLER(mute)
 413   IMPL_EVENT_HANDLER(unmute)
 414   IMPL_EVENT_HANDLER(ended)
 415
 416   /**
 417    * Convenience (and legacy) method for when ready state is "ended".
 418    */
 419   bool Ended() const { return mReadyState == MediaStreamTrackState::Ended; }
 420
 421   /**
 422    * Forces the ready state to a particular value, for instance when we're
 423    * cloning an already ended track.
 424    */
 425   void SetReadyState(MediaStreamTrackState aState);
 426
 427   /**
 428    * Notified by the MediaStreamGraph, through our owning MediaStream on the
 429    * main thread.
 430    *
 431    * Note that this sets the track to ended and raises the "ended" event
 432    * synchronously.
 433    */
 434   void OverrideEnded();
 435
 436   /**
 437    * Get this track's principal.
 438    */
 439   nsIPrincipal* GetPrincipal() const { return mPrincipal; }
 440
 441   /**
 442    * Called by the MSGListener when this track's PrincipalHandle changes on
 443    * the MediaStreamGraph thread. When the PrincipalHandle matches the pending
 444    * principal we know that the principal change has propagated to consumers.
 445    */
 446   void NotifyPrincipalHandleChanged(const PrincipalHandle& aPrincipalHandle);
 447
 448   /**
 449    * Called when this track's readyState transitions to "ended".
 450    * Notifies all MediaStreamTrackConsumers that this track ended.
 451    */
 452   void NotifyEnded();
 453
 454   /**
 455    * Get this track's CORS mode.
 456    */
 457   CORSMode GetCORSMode() const { return GetSource().GetCORSMode(); }
 458
 459   /**
 460    * Get this track's PeerIdentity.
 461    */
 462   const PeerIdentity* GetPeerIdentity() const {
 463     return GetSource().GetPeerIdentity();
 464   }
 465
 466   MediaStreamGraph* Graph();
 467   MediaStreamGraphImpl* GraphImpl();
 468
 469   MediaStreamTrackSource& GetSource() const {
 470     MOZ_RELEASE_ASSERT(mSource,
 471                        "The track source is only removed on destruction");
 472     return *mSource;
 473   }
 474
 475   // Webrtc allows the remote side to name tracks whatever it wants, and we
 476   // need to surface this to content.
 477   void AssignId(const nsAString& aID) { mID = aID; }
 478
 479   /**
 480    * Called when mSource's principal has changed.
 481    */
 482   void PrincipalChanged();
 483
 484   /**
 485    * Called when mSource's muted state has changed.
 486    */
 487   void MutedChanged(bool aNewState);
 488
 489   /**
 490    * Add a PrincipalChangeObserver to this track.
 491    *
 492    * Returns true if it was successfully added.
 493    *
 494    * Ownership of the PrincipalChangeObserver remains with the caller, and it's
 495    * the caller's responsibility to remove the observer before it dies.
 496    */
 497   bool AddPrincipalChangeObserver(
 498       PrincipalChangeObserver<MediaStreamTrack>* aObserver);
 499
 500   /**
 501    * Remove an added PrincipalChangeObserver from this track.
 502    *
 503    * Returns true if it was successfully removed.
 504    */
 505   bool RemovePrincipalChangeObserver(
 506       PrincipalChangeObserver<MediaStreamTrack>* aObserver);
 507
 508   /**
 509    * Add a MediaStreamTrackConsumer to this track.
 510    *
 511    * Adding the same consumer multiple times is prohibited.
 512    */
 513   void AddConsumer(MediaStreamTrackConsumer* aConsumer);
 514
 515   /**
 516    * Remove an added MediaStreamTrackConsumer from this track.
 517    */
 518   void RemoveConsumer(MediaStreamTrackConsumer* aConsumer);
 519
 520   /**
 521    * Adds a MediaStreamTrackListener to the MediaStreamGraph representation of
 522    * this track.
 523    */
 524   virtual void AddListener(MediaStreamTrackListener* aListener);
 525
 526   /**
 527    * Removes a MediaStreamTrackListener from the MediaStreamGraph representation
 528    * of this track.
 529    */
 530   void RemoveListener(MediaStreamTrackListener* aListener);
 531
 532   /**
 533    * Attempts to add a direct track listener to this track.
 534    * Callers must listen to the NotifyInstalled event to know if installing
 535    * the listener succeeded (tracks originating from SourceMediaStreams) or
 536    * failed (e.g., WebAudio originated tracks).
 537    */
 538   virtual void AddDirectListener(DirectMediaStreamTrackListener* aListener);
 539   void RemoveDirectListener(DirectMediaStreamTrackListener* aListener);
 540
 541   /**
 542    * Sets up a MediaInputPort from the underlying track that this
 543    * MediaStreamTrack represents, to aStream, and returns it.
 544    */
 545   already_AddRefed<MediaInputPort> ForwardTrackContentsTo(
 546       ProcessedMediaStream* aStream, TrackID aDestinationTrackID = TRACK_ANY);
 547
 548   /**
 549    * Returns true if this track is connected to aPort and forwarded to aPort's
 550    * output stream.
 551    */
 552   bool IsForwardedThrough(MediaInputPort* aPort);
 553
 554   void SetMediaStreamSizeListener(DirectMediaStreamTrackListener* aListener);
 555
 556   // Returns the original DOMMediaStream's underlying input stream.
 557   MediaStream* GetInputStream();
 558
 559   TrackID GetInputTrackId() const { return mInputTrackID; }
 560
 561  protected:
 562   virtual ~MediaStreamTrack();
 563
 564   /**
 565    * Sets this track's muted state without raising any events.
 566    */
 567   void SetMuted(bool aMuted) { mMuted = aMuted; }
 568
 569   void Destroy();
 570
 571   // Returns the owning DOMMediaStream's underlying owned stream.
 572   ProcessedMediaStream* GetOwnedStream();
 573
 574   // Returns the original DOMMediaStream. If this track is a clone,
 575   // the original track's owning DOMMediaStream is returned.
 576   DOMMediaStream* GetInputDOMStream();
 577
 578   /**
 579    * Sets the principal and notifies PrincipalChangeObservers if it changes.
 580    */
 581   void SetPrincipal(nsIPrincipal* aPrincipal);
 582
 583   /**
 584    * Creates a new MediaStreamTrack with the same type, input track ID and
 585    * source as this MediaStreamTrack.
 586    * aTrackID is the TrackID the new track will have in its owned stream.
 587    */
 588   virtual already_AddRefed<MediaStreamTrack> CloneInternal(
 589       DOMMediaStream* aOwningStream, TrackID aTrackID) = 0;
 590
 591   nsTArray<PrincipalChangeObserver<MediaStreamTrack>*>
 592       mPrincipalChangeObservers;
 593
 594   nsTArray<WeakPtr<MediaStreamTrackConsumer>> mConsumers;
 595
 596   RefPtr<DOMMediaStream> mOwningStream;
 597   TrackID mTrackID;
 598   TrackID mInputTrackID;
 599   RefPtr<MediaStreamTrackSource> mSource;
 600   const UniquePtr<TrackSink> mSink;
 601   RefPtr<MediaStreamTrack> mOriginalTrack;
 602   nsCOMPtr<nsIPrincipal> mPrincipal;
 603   nsCOMPtr<nsIPrincipal> mPendingPrincipal;
 604   RefPtr<MSGListener> mMSGListener;
 605   // Keep tracking MediaStreamTrackListener and DirectMediaStreamTrackListener,
 606   // so we can remove them in |Destory|.
 607   nsTArray<RefPtr<MediaStreamTrackListener>> mTrackListeners;
 608   nsTArray<RefPtr<DirectMediaStreamTrackListener>> mDirectTrackListeners;
 609   nsString mID;
 610   MediaStreamTrackState mReadyState;
 611   bool mEnabled;
 612   bool mMuted;
 613   dom::MediaTrackConstraints mConstraints;
 614 };
 615
 616 }  // namespace dom
 617 }  // namespace mozilla
 618
 619 #endif /* MEDIASTREAMTRACK_H_ */