dom/media/MediaStreamListener.h

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #ifndef MOZILLA_MEDIASTREAMLISTENER_h_
   8 #define MOZILLA_MEDIASTREAMLISTENER_h_
   9
  10 #include "StreamTracks.h"
  11
  12 #include "MediaStreamGraph.h"
  13
  14 namespace mozilla {
  15
  16 class AudioSegment;
  17 class MediaStream;
  18 class MediaStreamGraph;
  19 class MediaStreamVideoSink;
  20 class VideoSegment;
  21
  22 /**
  23  * This is a base class for media graph thread listener callbacks.
  24  * Override methods to be notified of audio or video data or changes in stream
  25  * state.
  26  *
  27  * This can be used by stream recorders or network connections that receive
  28  * stream input. It could also be used for debugging.
  29  *
  30  * All notification methods are called from the media graph thread. Overriders
  31  * of these methods are responsible for all synchronization. Beware!
  32  * These methods are called without the media graph monitor held, so
  33  * reentry into media graph methods is possible, although very much discouraged!
  34  * You should do something non-blocking and non-reentrant (e.g. dispatch an
  35  * event to some thread) and return.
  36  * The listener is not allowed to add/remove any listeners from the stream.
  37  *
  38  * When a listener is first attached, we guarantee to send a NotifyBlockingChanged
  39  * callback to notify of the initial blocking state. Also, if a listener is
  40  * attached to a stream that has already finished, we'll call NotifyFinished.
  41  */
  42 class MediaStreamListener
  43 {
  44 protected:
  45   // Protected destructor, to discourage deletion outside of Release():
  46   virtual ~MediaStreamListener() {}
  47
  48 public:
  49   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaStreamListener)
  50
  51   /**
  52    * When a SourceMediaStream has pulling enabled, and the MediaStreamGraph
  53    * control loop is ready to pull, this gets called. A NotifyPull implementation
  54    * is allowed to call the SourceMediaStream methods that alter track
  55    * data. It is not allowed to make other MediaStream API calls, including
  56    * calls to add or remove MediaStreamListeners. It is not allowed to block
  57    * for any length of time.
  58    * aDesiredTime is the stream time we would like to get data up to. Data
  59    * beyond this point will not be played until NotifyPull runs again, so there's
  60    * not much point in providing it. Note that if the stream is blocked for
  61    * some reason, then data before aDesiredTime may not be played immediately.
  62    */
  63   virtual void NotifyPull(MediaStreamGraph* aGraph, StreamTime aDesiredTime) {}
  64   virtual RefPtr<SourceMediaStream::NotifyPullPromise> AsyncNotifyPull(
  65     MediaStreamGraph* aGraph,
  66     StreamTime aDesiredTime)
  67   {
  68     NotifyPull(aGraph, aDesiredTime);
  69     return SourceMediaStream::NotifyPullPromise::CreateAndResolve(true,
  70                                                                   __func__);
  71   }
  72
  73   enum Blocking {
  74     BLOCKED,
  75     UNBLOCKED
  76   };
  77   /**
  78    * Notify that the blocking status of the stream changed. The initial state
  79    * is assumed to be BLOCKED.
  80    */
  81   virtual void NotifyBlockingChanged(MediaStreamGraph* aGraph, Blocking aBlocked) {}
  82
  83   /**
  84    * Notify that the stream has data in each track
  85    * for the stream's current time. Once this state becomes true, it will
  86    * always be true since we block stream time from progressing to times where
  87    * there isn't data in each track.
  88    */
  89   virtual void NotifyHasCurrentData(MediaStreamGraph* aGraph) {}
  90
  91   /**
  92    * Notify that the stream output is advancing. aCurrentTime is the graph's
  93    * current time. MediaStream::GraphTimeToStreamTime can be used to get the
  94    * stream time.
  95    */
  96   virtual void NotifyOutput(MediaStreamGraph* aGraph, GraphTime aCurrentTime) {}
  97
  98   /**
  99    * Notify that an event has occurred on the Stream
 100    */
 101   virtual void NotifyEvent(MediaStreamGraph* aGraph, MediaStreamGraphEvent aEvent) {}
 102
 103   /**
 104    * Notify that changes to one of the stream tracks have been queued.
 105    * aTrackEvents can be any combination of TRACK_EVENT_CREATED and
 106    * TRACK_EVENT_ENDED. aQueuedMedia is the data being added to the track
 107    * at aTrackOffset (relative to the start of the stream).
 108    * aInputStream and aInputTrackID will be set if the changes originated
 109    * from an input stream's track. In practice they will only be used for
 110    * ProcessedMediaStreams.
 111    */
 112   virtual void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
 113                                         StreamTime aTrackOffset,
 114                                         TrackEventCommand aTrackEvents,
 115                                         const MediaSegment& aQueuedMedia,
 116                                         MediaStream* aInputStream = nullptr,
 117                                         TrackID aInputTrackID = TRACK_INVALID) {}
 118
 119   /**
 120    * Notify queued audio data. Only audio data need to be queued. The video data
 121    * will be notified by MediaStreamVideoSink::SetCurrentFrame.
 122    */
 123   virtual void NotifyQueuedAudioData(MediaStreamGraph* aGraph, TrackID aID,
 124                                      StreamTime aTrackOffset,
 125                                      const AudioSegment& aQueuedMedia,
 126                                      MediaStream* aInputStream = nullptr,
 127                                      TrackID aInputTrackID = TRACK_INVALID) {}
 128
 129   /**
 130    * Notify that all new tracks this iteration have been created.
 131    * This is to ensure that tracks added atomically to MediaStreamGraph
 132    * are also notified of atomically to MediaStreamListeners.
 133    */
 134   virtual void NotifyFinishedTrackCreation(MediaStreamGraph* aGraph) {}
 135 };
 136
 137 /**
 138  * This is a base class for media graph thread listener callbacks locked to
 139  * specific tracks. Override methods to be notified of audio or video data or
 140  * changes in track state.
 141  *
 142  * All notification methods are called from the media graph thread. Overriders
 143  * of these methods are responsible for all synchronization. Beware!
 144  * These methods are called without the media graph monitor held, so
 145  * reentry into media graph methods is possible, although very much discouraged!
 146  * You should do something non-blocking and non-reentrant (e.g. dispatch an
 147  * event to some thread) and return.
 148  * The listener is not allowed to add/remove any listeners from the parent
 149  * stream.
 150  *
 151  * If a listener is attached to a track that has already ended, we guarantee
 152  * to call NotifyEnded.
 153  */
 154 class MediaStreamTrackListener
 155 {
 156   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaStreamTrackListener)
 157
 158 public:
 159   virtual void NotifyQueuedChanges(MediaStreamGraph* aGraph,
 160                                    StreamTime aTrackOffset,
 161                                    const MediaSegment& aQueuedMedia) {}
 162
 163   virtual void NotifyPrincipalHandleChanged(MediaStreamGraph* aGraph,
 164                                             const PrincipalHandle& aNewPrincipalHandle) {}
 165
 166   virtual void NotifyEnded() {}
 167
 168   virtual void NotifyRemoved() {}
 169
 170 protected:
 171   virtual ~MediaStreamTrackListener() {}
 172 };
 173
 174 /**
 175  * This is a base class for media graph thread listener direct callbacks from
 176  * within AppendToTrack(). It is bound to a certain track and can only be
 177  * installed on audio tracks. Once added to a track on any stream in the graph,
 178  * the graph will try to install it at that track's source of media data.
 179  *
 180  * This works for TrackUnionStreams, which will forward the listener to the
 181  * track's input track if it exists, or wait for it to be created before
 182  * forwarding if it doesn't.
 183  * Once it reaches a SourceMediaStream, it can be successfully installed.
 184  * Other types of streams will fail installation since they are not supported.
 185  *
 186  * Note that this listener and others for the same track will still get
 187  * NotifyQueuedChanges() callbacks from the MSG tread, so you must be careful
 188  * to ignore them if this listener was successfully installed.
 189  */
 190 class DirectMediaStreamTrackListener : public MediaStreamTrackListener
 191 {
 192   friend class SourceMediaStream;
 193   friend class TrackUnionStream;
 194
 195 public:
 196   /*
 197    * This will be called on any DirectMediaStreamTrackListener added to a
 198    * SourceMediaStream when AppendToTrack() is called for the listener's bound
 199    * track, using the thread of the AppendToTrack() caller. The MediaSegment
 200    * will be the RawSegment (unresampled) if available in AppendToTrack().
 201    * If the track is enabled at the source but has been disabled in one of the
 202    * streams in between the source and where it was originally added, aMedia
 203    * will be a disabled version of the one passed to AppendToTrack() as well.
 204    * Note that NotifyQueuedTrackChanges() calls will also still occur.
 205    */
 206   virtual void NotifyRealtimeTrackData(MediaStreamGraph* aGraph,
 207                                        StreamTime aTrackOffset,
 208                                        const MediaSegment& aMedia) {}
 209
 210   /**
 211    * When a direct listener is processed for installation by the
 212    * MediaStreamGraph it will be notified with whether the installation was
 213    * successful or not. The results of this installation are the following:
 214    * TRACK_NOT_FOUND_AT_SOURCE
 215    *    We found the source stream of media data for this track, but the track
 216    *    didn't exist. This should only happen if you try to install the listener
 217    *    directly to a SourceMediaStream that doesn't contain the given TrackID.
 218    * STREAM_NOT_SUPPORTED
 219    *    While looking for the data source of this track, we found a MediaStream
 220    *    that is not a SourceMediaStream or a TrackUnionStream.
 221    * ALREADY_EXIST
 222    *    This DirectMediaStreamTrackListener already exists in the
 223    *    SourceMediaStream.
 224    * SUCCESS
 225    *    Installation was successful and this listener will start receiving
 226    *    NotifyRealtimeData on the next AppendToTrack().
 227    */
 228   enum class InstallationResult {
 229     TRACK_NOT_FOUND_AT_SOURCE,
 230     TRACK_TYPE_NOT_SUPPORTED,
 231     STREAM_NOT_SUPPORTED,
 232     ALREADY_EXISTS,
 233     SUCCESS
 234   };
 235   virtual void NotifyDirectListenerInstalled(InstallationResult aResult) {}
 236   virtual void NotifyDirectListenerUninstalled() {}
 237
 238   virtual MediaStreamVideoSink* AsMediaStreamVideoSink() { return nullptr; }
 239
 240 protected:
 241   virtual ~DirectMediaStreamTrackListener() {}
 242
 243   void MirrorAndDisableSegment(AudioSegment& aFrom, AudioSegment& aTo);
 244   void MirrorAndDisableSegment(VideoSegment& aFrom,
 245                                VideoSegment& aTo,
 246                                DisabledTrackMode aMode);
 247   void NotifyRealtimeTrackDataAndApplyTrackDisabling(MediaStreamGraph* aGraph,
 248                                                      StreamTime aTrackOffset,
 249                                                      MediaSegment& aMedia);
 250
 251   void IncreaseDisabled(DisabledTrackMode aMode);
 252   void DecreaseDisabled(DisabledTrackMode aMode);
 253
 254   // Matches the number of disabled streams to which this listener is attached.
 255   // The number of streams are those between the stream the listener was added
 256   // and the SourceMediaStream that is the input of the data.
 257   Atomic<int32_t> mDisabledFreezeCount;
 258   Atomic<int32_t> mDisabledBlackCount;
 259
 260   nsAutoPtr<MediaSegment> mMedia;
 261 };
 262
 263 } // namespace mozilla
 264
 265 #endif // MOZILLA_MEDIASTREAMLISTENER_h_