1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 * You can obtain one at http://mozilla.org/MPL/2.0/. */
6 #ifndef MediaEncoder_h_
7 #define MediaEncoder_h_
9 #include "ContainerWriter.h"
10 #include "CubebUtils.h"
11 #include "MediaQueue.h"
12 #include "MediaTrackGraph.h"
13 #include "MediaTrackListener.h"
14 #include "mozilla/DebugOnly.h"
15 #include "mozilla/MemoryReporting.h"
16 #include "mozilla/MozPromise.h"
17 #include "mozilla/UniquePtr.h"
18 #include "nsIMemoryReporter.h"
19 #include "TrackEncoder.h"
23 class DriftCompensator
;
30 class AudioStreamTrack
;
32 class MediaStreamTrack
;
33 class MutableBlobStorage
;
34 class VideoStreamTrack
;
37 class DriftCompensator
;
40 * MediaEncoder is the framework of encoding module, it controls and manages
41 * procedures between Muxer, ContainerWriter and TrackEncoder. ContainerWriter
42 * writes the encoded track data into a specific container (e.g. ogg, webm).
43 * AudioTrackEncoder and VideoTrackEncoder are subclasses of TrackEncoder, and
44 * are responsible for encoding raw data coming from MediaStreamTracks.
46 * MediaEncoder solves threading issues by doing message passing to a TaskQueue
47 * (the "encoder thread") as passed in to the constructor. Each
48 * MediaStreamTrack to be recorded is set up with a MediaTrackListener.
49 * Typically there are a non-direct track listeners for audio, direct listeners
50 * for video, and there is always a non-direct listener on each track for
51 * time-keeping. The listeners forward data to their corresponding TrackEncoders
52 * on the encoder thread.
54 * The MediaEncoder listens to events from all TrackEncoders, and in turn
55 * signals events to interested parties. Typically a MediaRecorder::Session.
56 * The MediaEncoder automatically encodes incoming data, muxes it, writes it
57 * into a container and stores the container data into a MutableBlobStorage.
58 * It is timeslice-aware so that it can notify listeners when it's time to
59 * expose a blob due to filling the timeslice.
61 * MediaEncoder is designed to be a passive component, neither does it own or is
62 * in charge of managing threads. Instead this is done by its owner.
64 * For example, usage from MediaRecorder of this component would be:
65 * 1) Create an encoder with a valid MIME type. Note that there are more
66 * configuration options, see the docs on MediaEncoder::CreateEncoder.
67 * => encoder = MediaEncoder::CreateEncoder(aMIMEType);
68 * It then creates track encoders and the appropriate ContainerWriter
69 * according to the MIME type
71 * 2) Connect handlers through MediaEventListeners to the MediaEncoder's
72 * MediaEventSources, StartedEvent(), DataAvailableEvent(), ErrorEvent() and
74 * => listener = encoder->DataAvailableEvent().Connect(mainThread, &OnBlob);
76 * 3) Connect the sources to be recorded. Either through:
77 * => encoder->ConnectAudioNode(node);
79 * => encoder->ConnectMediaStreamTrack(track);
80 * These should not be mixed. When connecting MediaStreamTracks there is
81 * support for at most one of each kind.
83 * 4) MediaEncoder automatically encodes data from the connected tracks, muxes
84 * them and writes it all into a blob, including metadata. When the blob
85 * contains at least `timeslice` worth of data it notifies the
86 * DataAvailableEvent that was connected in step 2.
87 * => void OnBlob(RefPtr<BlobImpl> aBlob) {
88 * => DispatchBlobEvent(Blob::Create(GetOwnerGlobal(), aBlob));
91 * 5) To stop encoding, there are multiple options:
93 * 5.1) Stop() for a graceful stop.
96 * 5.2) Cancel() for an immediate stop, if you don't need the data currently
98 * => encoder->Cancel();
100 * 5.3) When all input tracks end, the MediaEncoder will automatically stop
105 class AudioTrackListener
;
106 class VideoTrackListener
;
107 class EncoderListener
;
111 MozPromise
<RefPtr
<dom::BlobImpl
>, nsresult
, false /* IsExclusive */>;
112 using SizeOfPromise
= MozPromise
<size_t, size_t, true /* IsExclusive */>;
114 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaEncoder
)
117 MediaEncoder(RefPtr
<TaskQueue
> aEncoderThread
,
118 RefPtr
<DriftCompensator
> aDriftCompensator
,
119 UniquePtr
<ContainerWriter
> aWriter
,
120 UniquePtr
<AudioTrackEncoder
> aAudioEncoder
,
121 UniquePtr
<VideoTrackEncoder
> aVideoEncoder
,
122 UniquePtr
<MediaQueue
<EncodedFrame
>> aEncodedAudioQueue
,
123 UniquePtr
<MediaQueue
<EncodedFrame
>> aEncodedVideoQueue
,
124 TrackRate aTrackRate
, const nsAString
& aMIMEType
,
125 uint64_t aMaxMemory
, TimeDuration aTimeslice
);
129 * Called on main thread from MediaRecorder::Pause.
134 * Called on main thread from MediaRecorder::Resume.
139 * Disconnects the input tracks, causing the encoding to stop.
141 void DisconnectTracks();
144 * Connects an AudioNode with the appropriate encoder.
146 void ConnectAudioNode(dom::AudioNode
* aNode
, uint32_t aOutput
);
149 * Connects a MediaStreamTrack with the appropriate encoder.
151 void ConnectMediaStreamTrack(dom::MediaStreamTrack
* aTrack
);
154 * Removes a connected MediaStreamTrack.
156 void RemoveMediaStreamTrack(dom::MediaStreamTrack
* aTrack
);
159 * Creates an encoder with the given MIME type. This must be a valid MIME type
160 * or we will crash hard.
161 * Bitrates are given either explicit, or with 0 for defaults.
162 * aTrackRate is the rate in which data will be fed to the TrackEncoders.
163 * aMaxMemory is the maximum number of bytes of muxed data allowed in memory.
164 * Beyond that the blob is moved to a temporary file.
165 * aTimeslice is the minimum duration of muxed data we gather before
166 * automatically issuing a dataavailable event.
168 static already_AddRefed
<MediaEncoder
> CreateEncoder(
169 RefPtr
<TaskQueue
> aEncoderThread
, const nsAString
& aMimeType
,
170 uint32_t aAudioBitrate
, uint32_t aVideoBitrate
, uint8_t aTrackTypes
,
171 TrackRate aTrackRate
, uint64_t aMaxMemory
, TimeDuration aTimeslice
);
174 * Encodes raw data for all tracks to aOutputBufs. The buffer of container
175 * data is allocated in ContainerWriter::GetContainerData().
177 * On its first call, metadata is also encoded. TrackEncoders must have been
178 * initialized before this is called.
180 nsresult
GetEncodedData(nsTArray
<nsTArray
<uint8_t>>* aOutputBufs
);
183 * Asserts that Shutdown() has been called. Reasons are encoding
184 * complete, encounter an error, or being canceled by its caller.
186 void AssertShutdownCalled() { MOZ_ASSERT(mShutdownPromise
); }
189 * Stops (encoding any data currently buffered) the encoding and shuts down
190 * the encoder using Shutdown().
192 RefPtr
<GenericNonExclusivePromise
> Stop();
195 * Cancels (discarding any data currently buffered) the encoding and shuts
196 * down the encoder using Shutdown().
198 RefPtr
<GenericNonExclusivePromise
> Cancel();
202 static bool IsWebMEncoderEnabled();
205 * Updates internal state when track encoders are all initialized.
207 void UpdateInitialized();
210 * Updates internal state when track encoders are all initialized, and
211 * notifies listeners that this MediaEncoder has been started.
213 void UpdateStarted();
215 MOZ_DEFINE_MALLOC_SIZE_OF(MallocSizeOf
)
217 * Measure the size of the buffer, and heap memory in bytes occupied by
218 * mAudioEncoder and mVideoEncoder.
220 RefPtr
<SizeOfPromise
> SizeOfExcludingThis(
221 mozilla::MallocSizeOf aMallocSizeOf
);
224 * Encode, mux and store into blob storage what has been buffered until now,
225 * then return the blob backed by that storage.
227 RefPtr
<BlobPromise
> RequestData();
229 // Event that gets notified when all track encoders have received data.
230 MediaEventSource
<void>& StartedEvent() { return mStartedEvent
; }
231 // Event that gets notified when there was an error preventing continued
232 // recording somewhere in the MediaEncoder stack.
233 MediaEventSource
<void>& ErrorEvent() { return mErrorEvent
; }
234 // Event that gets notified when the MediaEncoder stack has been shut down.
235 MediaEventSource
<void>& ShutdownEvent() { return mShutdownEvent
; }
236 // Event that gets notified after we have muxed at least mTimeslice worth of
237 // data into the current blob storage.
238 MediaEventSource
<RefPtr
<dom::BlobImpl
>>& DataAvailableEvent() {
239 return mDataAvailableEvent
;
247 * Registers listeners.
249 void RegisterListeners();
252 * Sets mGraphTrack if not already set, using a new stream from aTrack's
255 void EnsureGraphTrackFrom(MediaTrack
* aTrack
);
258 * Shuts down gracefully if there is no remaining live track encoder.
260 void MaybeShutdown();
263 * Waits for TrackEncoders to shut down, then shuts down the MediaEncoder and
264 * cleans up track encoders.
266 RefPtr
<GenericNonExclusivePromise
> Shutdown();
269 * Sets mError to true, notifies listeners of the error if mError changed,
270 * and stops encoding.
275 * Creates a new MutableBlobStorage if one doesn't exist.
277 void MaybeCreateMutableBlobStorage();
280 * Called when an encoded audio frame has been pushed by the audio encoder.
282 void OnEncodedAudioPushed(const RefPtr
<EncodedFrame
>& aFrame
);
285 * Called when an encoded video frame has been pushed by the video encoder.
287 void OnEncodedVideoPushed(const RefPtr
<EncodedFrame
>& aFrame
);
290 * If enough data has been pushed to the muxer, extract it into the current
291 * blob storage. If more than mTimeslice data has been pushed to the muxer
292 * since the last DataAvailableEvent was notified, also gather the blob and
293 * notify MediaRecorder.
295 void MaybeExtractOrGatherBlob();
297 // Extracts encoded and muxed data into the current blob storage, creating one
298 // if it doesn't exist. The returned promise resolves when data has been
299 // stored into the blob.
300 RefPtr
<GenericPromise
> Extract();
302 // Stops gathering data into the current blob and resolves when the current
303 // blob is available. Future data will be stored in a new blob.
304 // Should a previous async GatherBlob() operation still be in progress, we'll
305 // wait for it to finish before starting this one.
306 RefPtr
<BlobPromise
> GatherBlob();
308 RefPtr
<BlobPromise
> GatherBlobImpl();
310 const RefPtr
<nsISerialEventTarget
> mMainThread
;
311 const RefPtr
<TaskQueue
> mEncoderThread
;
312 const RefPtr
<DriftCompensator
> mDriftCompensator
;
314 const UniquePtr
<MediaQueue
<EncodedFrame
>> mEncodedAudioQueue
;
315 const UniquePtr
<MediaQueue
<EncodedFrame
>> mEncodedVideoQueue
;
317 const UniquePtr
<Muxer
> mMuxer
;
318 const UniquePtr
<AudioTrackEncoder
> mAudioEncoder
;
319 const RefPtr
<AudioTrackListener
> mAudioListener
;
320 const UniquePtr
<VideoTrackEncoder
> mVideoEncoder
;
321 const RefPtr
<VideoTrackListener
> mVideoListener
;
322 const RefPtr
<EncoderListener
> mEncoderListener
;
325 const nsString mMimeType
;
327 // Max memory to use for the MutableBlobStorage.
328 const uint64_t mMaxMemory
;
330 // The interval of passing encoded data from MutableBlobStorage to
331 // onDataAvailable handler.
332 const TimeDuration mTimeslice
;
335 MediaEventListener mAudioPushListener
;
336 MediaEventListener mAudioFinishListener
;
337 MediaEventListener mVideoPushListener
;
338 MediaEventListener mVideoFinishListener
;
340 MediaEventProducer
<void> mStartedEvent
;
341 MediaEventProducer
<void> mErrorEvent
;
342 MediaEventProducer
<void> mShutdownEvent
;
343 MediaEventProducer
<RefPtr
<dom::BlobImpl
>> mDataAvailableEvent
;
345 // The AudioNode we are encoding.
346 // Will be null when input is media stream or destination node.
347 RefPtr
<dom::AudioNode
> mAudioNode
;
348 // Pipe-track for allowing a track listener on a non-destination AudioNode.
349 // Will be null when input is media stream or destination node.
350 RefPtr
<AudioNodeTrack
> mPipeTrack
;
351 // Input port that connect mAudioNode to mPipeTrack.
352 // Will be null when input is media stream or destination node.
353 RefPtr
<MediaInputPort
> mInputPort
;
354 // An audio track that we are encoding. Will be null if the input stream
355 // doesn't contain audio on start() or if the input is an AudioNode.
356 RefPtr
<dom::AudioStreamTrack
> mAudioTrack
;
357 // A video track that we are encoding. Will be null if the input stream
358 // doesn't contain video on start() or if the input is an AudioNode.
359 RefPtr
<dom::VideoStreamTrack
> mVideoTrack
;
361 // A stream to keep the MediaTrackGraph alive while we're recording.
362 RefPtr
<SharedDummyTrack
> mGraphTrack
;
364 // A buffer to cache muxed encoded data.
365 RefPtr
<dom::MutableBlobStorage
> mMutableBlobStorage
;
366 // If set, is a promise for the latest GatherBlob() operation. Allows
367 // GatherBlob() operations to be serialized in order to avoid races.
368 RefPtr
<BlobPromise
> mBlobPromise
;
369 // The end time of the muxed data in the last gathered blob. If more than one
370 // track is present, this is the end time of the track that ends the earliest
371 // in the last blob. Encoder thread only.
372 media::TimeUnit mLastBlobTime
;
373 // The end time of the muxed data in the current blob storage. If more than
374 // one track is present, this is the end time of the track that ends the
375 // earliest in the current blob storage. Encoder thread only.
376 media::TimeUnit mLastExtractTime
;
377 // The end time of encoded audio data sent to the muxer. Positive infinity if
378 // there is no audio encoder. Encoder thread only.
379 media::TimeUnit mMuxedAudioEndTime
;
380 // The end time of encoded video data sent to the muxer. Positive infinity if
381 // there is no video encoder. Encoder thread only.
382 media::TimeUnit mMuxedVideoEndTime
;
384 TimeStamp mStartTime
;
389 // Set when shutdown starts.
390 RefPtr
<GenericNonExclusivePromise
> mShutdownPromise
;
391 // Get duration from create encoder, for logging purpose
392 double GetEncodeTimeStamp() {
393 TimeDuration decodeTime
;
394 decodeTime
= TimeStamp::Now() - mStartTime
;
395 return decodeTime
.ToMilliseconds();
399 } // namespace mozilla