1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 #if !defined(MediaInfo_h)
9 # include "mozilla/UniquePtr.h"
10 # include "mozilla/RefPtr.h"
11 # include "mozilla/Variant.h"
12 # include "nsTHashMap.h"
13 # include "nsString.h"
14 # include "nsTArray.h"
15 # include "AudioConfig.h"
16 # include "ImageTypes.h"
17 # include "MediaData.h"
18 # include "TimeUnits.h"
19 # include "mozilla/gfx/Point.h" // for gfx::IntSize
20 # include "mozilla/gfx/Rect.h" // for gfx::IntRect
21 # include "mozilla/gfx/Types.h" // for gfx::ColorDepth
31 MetadataTag(const nsACString
& aKey
, const nsACString
& aValue
)
32 : mKey(aKey
), mValue(aValue
) {}
35 bool operator==(const MetadataTag
& rhs
) const {
36 return mKey
== rhs
.mKey
&& mValue
== rhs
.mValue
;
40 typedef nsTHashMap
<nsCStringHashKey
, nsCString
> MetadataTags
;
42 // Start codec specific data structs. If modifying these remember to also
43 // modify the MediaIPCUtils so that any new members are sent across IPC.
45 // Generic types, we should prefer a specific type when we can.
47 // Generic empty type. Prefer to use a specific type but not populate members
48 // if possible, as that helps with type checking.
49 struct NoCodecSpecificData
{
50 bool operator==(const NoCodecSpecificData
& rhs
) const { return true; }
53 // Generic binary blob type. Prefer not to use this structure. It's here to ease
54 // the transition to codec specific structures in the code.
55 struct AudioCodecSpecificBinaryBlob
{
56 bool operator==(const AudioCodecSpecificBinaryBlob
& rhs
) const {
57 return *mBinaryBlob
== *rhs
.mBinaryBlob
;
60 RefPtr
<MediaByteBuffer
> mBinaryBlob
{new MediaByteBuffer
};
65 // Audio codec specific data types.
67 struct AacCodecSpecificData
{
68 bool operator==(const AacCodecSpecificData
& rhs
) const {
69 return *mEsDescriptorBinaryBlob
== *rhs
.mEsDescriptorBinaryBlob
&&
70 *mDecoderConfigDescriptorBinaryBlob
==
71 *rhs
.mDecoderConfigDescriptorBinaryBlob
;
74 // The bytes of the ES_Descriptor field parsed out of esds box. We store
75 // this as a blob as some decoders want this.
76 RefPtr
<MediaByteBuffer
> mEsDescriptorBinaryBlob
{new MediaByteBuffer
};
78 // The bytes of the DecoderConfigDescriptor field within the parsed
79 // ES_Descriptor. This is a subset of the ES_Descriptor, so it is technically
80 // redundant to store both. However, some decoders expect this binary blob
81 // instead of the whole ES_Descriptor, so both are stored for convenience
82 // and clarity (rather than reparsing the ES_Descriptor).
83 // TODO(bug 1768562): use a Span to track this rather than duplicating data.
84 RefPtr
<MediaByteBuffer
> mDecoderConfigDescriptorBinaryBlob
{
88 struct FlacCodecSpecificData
{
89 bool operator==(const FlacCodecSpecificData
& rhs
) const {
90 return *mStreamInfoBinaryBlob
== *rhs
.mStreamInfoBinaryBlob
;
93 // A binary blob of the data from the METADATA_BLOCK_STREAMINFO block
94 // in the flac header.
95 // See https://xiph.org/flac/format.html#metadata_block_streaminfo
96 // Consumers of this data (ffmpeg) take a blob, so we don't parse the data,
97 // just store the blob. For headerless flac files this will be left empty.
98 RefPtr
<MediaByteBuffer
> mStreamInfoBinaryBlob
{new MediaByteBuffer
};
101 struct Mp3CodecSpecificData
{
102 bool operator==(const Mp3CodecSpecificData
& rhs
) const {
103 return mEncoderDelayFrames
== rhs
.mEncoderDelayFrames
&&
104 mEncoderPaddingFrames
== rhs
.mEncoderPaddingFrames
;
107 // The number of frames that should be skipped from the beginning of the
109 // See https://bugzilla.mozilla.org/show_bug.cgi?id=1566389 for more info.
110 uint32_t mEncoderDelayFrames
{0};
112 // The number of frames that should be skipped from the end of the decoded
114 // See https://bugzilla.mozilla.org/show_bug.cgi?id=1566389 for more info.
115 uint32_t mEncoderPaddingFrames
{0};
118 struct OpusCodecSpecificData
{
119 bool operator==(const OpusCodecSpecificData
& rhs
) const {
120 return mContainerCodecDelayMicroSeconds
==
121 rhs
.mContainerCodecDelayMicroSeconds
&&
122 *mHeadersBinaryBlob
== *rhs
.mHeadersBinaryBlob
;
124 // The codec delay (aka pre-skip) in microseconds.
125 // See https://tools.ietf.org/html/rfc7845#section-4.2 for more info.
126 // This member should store the codec delay parsed from the container file.
127 // In some cases (such as the ogg container), this information is derived
128 // from the same headers stored in the header blob, making storing this
129 // separately redundant. However, other containers store the delay in
130 // addition to the header blob, in which case we can check this container
131 // delay against the header delay to ensure they're consistent.
132 int64_t mContainerCodecDelayMicroSeconds
{-1};
134 // A binary blob of opus header data, specifically the Identification Header.
135 // See https://datatracker.ietf.org/doc/html/rfc7845.html#section-5.1
136 RefPtr
<MediaByteBuffer
> mHeadersBinaryBlob
{new MediaByteBuffer
};
139 struct VorbisCodecSpecificData
{
140 bool operator==(const VorbisCodecSpecificData
& rhs
) const {
141 return *mHeadersBinaryBlob
== *rhs
.mHeadersBinaryBlob
;
144 // A binary blob of headers in the 'extradata' format (the format ffmpeg
145 // expects for packing the extradata field). This is also the format some
146 // containers use for storing the data. Specifically, this format consists of
147 // the page_segments field, followed by the segment_table field, followed by
148 // the three Vorbis header packets, respectively the identification header,
149 // the comments header, and the setup header, in that order.
150 // See also https://xiph.org/vorbis/doc/framing.html and
151 // https://xiph.org/vorbis/doc/Vorbis_I_spec.html#x1-610004.2
152 RefPtr
<MediaByteBuffer
> mHeadersBinaryBlob
{new MediaByteBuffer
};
155 struct WaveCodecSpecificData
{
156 bool operator==(const WaveCodecSpecificData
& rhs
) const { return true; }
157 // Intentionally empty. We don't store any wave specific data, but this
158 // variant is useful for type checking.
161 using AudioCodecSpecificVariant
=
162 mozilla::Variant
<NoCodecSpecificData
, AudioCodecSpecificBinaryBlob
,
163 AacCodecSpecificData
, FlacCodecSpecificData
,
164 Mp3CodecSpecificData
, OpusCodecSpecificData
,
165 VorbisCodecSpecificData
, WaveCodecSpecificData
>;
167 // Returns a binary blob representation of the AudioCodecSpecificVariant. This
168 // does not guarantee that a binary representation exists. Will return an empty
169 // buffer if no representation exists. Prefer `GetAudioCodecSpecificBlob` which
170 // asserts if getting a blob is unexpected for a given codec config.
171 inline already_AddRefed
<MediaByteBuffer
> ForceGetAudioCodecSpecificBlob(
172 const AudioCodecSpecificVariant
& v
) {
174 [](const NoCodecSpecificData
&) {
175 return RefPtr
<MediaByteBuffer
>(new MediaByteBuffer
).forget();
177 [](const AudioCodecSpecificBinaryBlob
& binaryBlob
) {
178 return RefPtr
<MediaByteBuffer
>(binaryBlob
.mBinaryBlob
).forget();
180 [](const AacCodecSpecificData
& aacData
) {
181 // We return the mDecoderConfigDescriptor blob here, as it is more
182 // commonly used by decoders at time of writing than the
183 // ES_Descriptor data. However, consumers of this data should
184 // prefer getting one or the other specifically, rather than
186 return RefPtr
<MediaByteBuffer
>(
187 aacData
.mDecoderConfigDescriptorBinaryBlob
)
190 [](const FlacCodecSpecificData
& flacData
) {
191 return RefPtr
<MediaByteBuffer
>(flacData
.mStreamInfoBinaryBlob
).forget();
193 [](const Mp3CodecSpecificData
&) {
194 return RefPtr
<MediaByteBuffer
>(new MediaByteBuffer
).forget();
196 [](const OpusCodecSpecificData
& opusData
) {
197 return RefPtr
<MediaByteBuffer
>(opusData
.mHeadersBinaryBlob
).forget();
199 [](const VorbisCodecSpecificData
& vorbisData
) {
200 return RefPtr
<MediaByteBuffer
>(vorbisData
.mHeadersBinaryBlob
).forget();
202 [](const WaveCodecSpecificData
&) {
203 return RefPtr
<MediaByteBuffer
>(new MediaByteBuffer
).forget();
207 // Same as `ForceGetAudioCodecSpecificBlob` but with extra asserts to ensure
208 // we're not trying to get a binary blob from codecs where we don't store the
209 // information as a blob or where a blob is ambiguous.
210 inline already_AddRefed
<MediaByteBuffer
> GetAudioCodecSpecificBlob(
211 const AudioCodecSpecificVariant
& v
) {
212 MOZ_ASSERT(!v
.is
<NoCodecSpecificData
>(),
213 "NoCodecSpecificData shouldn't be used as a blob");
214 MOZ_ASSERT(!v
.is
<AacCodecSpecificData
>(),
215 "AacCodecSpecificData has 2 blobs internally, one should "
216 "explicitly be selected");
217 MOZ_ASSERT(!v
.is
<Mp3CodecSpecificData
>(),
218 "Mp3CodecSpecificData shouldn't be used as a blob");
220 return ForceGetAudioCodecSpecificBlob(v
);
223 // End audio codec specific data types.
225 // End codec specific data structs.
229 enum TrackType
{ kUndefinedTrack
, kAudioTrack
, kVideoTrack
, kTextTrack
};
230 TrackInfo(TrackType aType
, const nsAString
& aId
, const nsAString
& aKind
,
231 const nsAString
& aLabel
, const nsAString
& aLanguage
, bool aEnabled
,
236 mLanguage(aLanguage
),
239 mIsRenderedExternally(false),
241 MOZ_COUNT_CTOR(TrackInfo
);
244 // Only used for backward compatibility. Do not use in new code.
245 void Init(const nsAString
& aId
, const nsAString
& aKind
,
246 const nsAString
& aLabel
, const nsAString
& aLanguage
,
251 mLanguage
= aLanguage
;
255 // Fields common with MediaTrack object.
265 media::TimeUnit mDuration
;
266 media::TimeUnit mMediaTime
;
269 CopyableTArray
<MetadataTag
> mTags
;
271 // True if the track is gonna be (decrypted)/decoded and
272 // rendered directly by non-gecko components.
273 bool mIsRenderedExternally
;
275 virtual AudioInfo
* GetAsAudioInfo() { return nullptr; }
276 virtual VideoInfo
* GetAsVideoInfo() { return nullptr; }
277 virtual TextInfo
* GetAsTextInfo() { return nullptr; }
278 virtual const AudioInfo
* GetAsAudioInfo() const { return nullptr; }
279 virtual const VideoInfo
* GetAsVideoInfo() const { return nullptr; }
280 virtual const TextInfo
* GetAsTextInfo() const { return nullptr; }
282 bool IsAudio() const { return !!GetAsAudioInfo(); }
283 bool IsVideo() const { return !!GetAsVideoInfo(); }
284 bool IsText() const { return !!GetAsTextInfo(); }
285 TrackType
GetType() const { return mType
; }
287 bool virtual IsValid() const = 0;
289 virtual UniquePtr
<TrackInfo
> Clone() const = 0;
291 MOZ_COUNTED_DTOR_VIRTUAL(TrackInfo
)
294 TrackInfo(const TrackInfo
& aOther
) {
296 mKind
= aOther
.mKind
;
297 mLabel
= aOther
.mLabel
;
298 mLanguage
= aOther
.mLanguage
;
299 mEnabled
= aOther
.mEnabled
;
300 mTrackId
= aOther
.mTrackId
;
301 mMimeType
= aOther
.mMimeType
;
302 mDuration
= aOther
.mDuration
;
303 mMediaTime
= aOther
.mMediaTime
;
304 mCrypto
= aOther
.mCrypto
;
305 mIsRenderedExternally
= aOther
.mIsRenderedExternally
;
306 mType
= aOther
.mType
;
307 mTags
= aOther
.mTags
.Clone();
308 MOZ_COUNT_CTOR(TrackInfo
);
310 bool IsEqualTo(const TrackInfo
& rhs
) const;
316 // String version of track type.
317 const char* TrackTypeToStr(TrackInfo::TrackType aTrack
);
319 // Stores info relevant to presenting media frames.
320 class VideoInfo
: public TrackInfo
{
328 VideoInfo() : VideoInfo(-1, -1) {}
330 VideoInfo(int32_t aWidth
, int32_t aHeight
)
331 : VideoInfo(gfx::IntSize(aWidth
, aHeight
)) {}
333 explicit VideoInfo(const gfx::IntSize
& aSize
)
334 : TrackInfo(kVideoTrack
, u
"2"_ns
, u
"main"_ns
, u
""_ns
, u
""_ns
, true, 2),
336 mStereoMode(StereoMode::MONO
),
338 mCodecSpecificConfig(new MediaByteBuffer
),
339 mExtraData(new MediaByteBuffer
),
340 mRotation(kDegree_0
) {}
342 VideoInfo(const VideoInfo
& aOther
) = default;
344 bool operator==(const VideoInfo
& rhs
) const;
346 bool IsValid() const override
{
347 return mDisplay
.width
> 0 && mDisplay
.height
> 0;
350 VideoInfo
* GetAsVideoInfo() override
{ return this; }
352 const VideoInfo
* GetAsVideoInfo() const override
{ return this; }
354 UniquePtr
<TrackInfo
> Clone() const override
{
355 return MakeUnique
<VideoInfo
>(*this);
358 void SetAlpha(bool aAlphaPresent
) { mAlphaPresent
= aAlphaPresent
; }
360 bool HasAlpha() const { return mAlphaPresent
; }
362 gfx::IntRect
ImageRect() const {
364 return gfx::IntRect(0, 0, mImage
.width
, mImage
.height
);
369 void SetImageRect(const gfx::IntRect
& aRect
) { mImageRect
= Some(aRect
); }
370 void ResetImageRect() { mImageRect
.reset(); }
372 // Returned the crop rectangle scaled to aWidth/aHeight size relative to
374 // If aWidth and aHeight are identical to the original
375 // mImage.width/mImage.height then the scaling ratio will be 1. This is used
376 // for when the frame size is different from what the container reports. This
377 // is legal in WebM, and we will preserve the ratio of the crop rectangle as
378 // it was reported relative to the picture size reported by the container.
379 gfx::IntRect
ScaledImageRect(int64_t aWidth
, int64_t aHeight
) const {
380 if ((aWidth
== mImage
.width
&& aHeight
== mImage
.height
) || !mImage
.width
||
385 gfx::IntRect imageRect
= ImageRect();
386 int64_t w
= (aWidth
* imageRect
.Width()) / mImage
.width
;
387 int64_t h
= (aHeight
* imageRect
.Height()) / mImage
.height
;
392 imageRect
.x
= (imageRect
.x
* aWidth
) / mImage
.width
;
393 imageRect
.y
= (imageRect
.y
* aHeight
) / mImage
.height
;
394 imageRect
.SetWidth(w
);
395 imageRect
.SetHeight(h
);
399 Rotation
ToSupportedRotation(int32_t aDegree
) const {
408 NS_WARNING_ASSERTION(aDegree
== 0, "Invalid rotation degree, ignored");
413 // Size in pixels at which the video is rendered. This is after it has
414 // been scaled by its aspect ratio.
415 gfx::IntSize mDisplay
;
417 // Indicates the frame layout for single track stereo videos.
418 StereoMode mStereoMode
;
420 // Size of the decoded video's image.
423 RefPtr
<MediaByteBuffer
> mCodecSpecificConfig
;
424 RefPtr
<MediaByteBuffer
> mExtraData
;
426 // Describing how many degrees video frames should be rotated in clock-wise to
430 // Should be 8, 10 or 12. Default value is 8.
431 gfx::ColorDepth mColorDepth
= gfx::ColorDepth::COLOR_8
;
433 // Matrix coefficients (if specified by the video) imply a colorspace.
434 Maybe
<gfx::YUVColorSpace
> mColorSpace
;
436 // Color primaries are assumed to match the colorspace.
438 // Transfer functions get their own member, which may not be strongly
439 // correlated to the colorspace.
440 Maybe
<gfx::TransferFunction
> mTransferFunction
;
442 // True indicates no restriction on Y, U, V values (otherwise 16-235 for 8
444 gfx::ColorRange mColorRange
= gfx::ColorRange::LIMITED
;
446 Maybe
<int32_t> GetFrameRate() const { return mFrameRate
; }
447 void SetFrameRate(int32_t aRate
) { mFrameRate
= Some(aRate
); }
450 friend struct IPC::ParamTraits
<VideoInfo
>;
452 // mImage may be cropped; currently only used with the WebM container.
453 // If unset, no cropping is to occur.
454 Maybe
<gfx::IntRect
> mImageRect
;
456 // Indicates whether or not frames may contain alpha information.
457 bool mAlphaPresent
= false;
459 Maybe
<int32_t> mFrameRate
;
462 class AudioInfo
: public TrackInfo
{
465 : TrackInfo(kAudioTrack
, u
"1"_ns
, u
"main"_ns
, u
""_ns
, u
""_ns
, true, 1),
468 mChannelMap(AudioConfig::ChannelLayout::UNKNOWN_MAP
),
471 mExtendedProfile(0) {}
473 AudioInfo(const AudioInfo
& aOther
) = default;
475 bool operator==(const AudioInfo
& rhs
) const;
477 static const uint32_t MAX_RATE
= 640000;
479 bool IsValid() const override
{
480 return mChannels
> 0 && mRate
> 0 && mRate
<= MAX_RATE
;
483 AudioInfo
* GetAsAudioInfo() override
{ return this; }
485 const AudioInfo
* GetAsAudioInfo() const override
{ return this; }
487 UniquePtr
<TrackInfo
> Clone() const override
{
488 return MakeUnique
<AudioInfo
>(*this);
494 // Number of audio channels.
496 // The AudioConfig::ChannelLayout map. Channels are ordered as per SMPTE
497 // definition. A value of UNKNOWN_MAP indicates unknown layout.
498 // ChannelMap is an unsigned bitmap compatible with Windows' WAVE and FFmpeg
500 AudioConfig::ChannelLayout::ChannelMap mChannelMap
;
508 // Extended codec profile.
509 int8_t mExtendedProfile
;
511 AudioCodecSpecificVariant mCodecSpecificConfig
{NoCodecSpecificData
{}};
514 class EncryptionInfo
{
516 EncryptionInfo() : mEncrypted(false) {}
519 template <typename AInitDatas
>
520 InitData(const nsAString
& aType
, AInitDatas
&& aInitData
)
521 : mType(aType
), mInitData(std::forward
<AInitDatas
>(aInitData
)) {}
523 // Encryption type to be passed to JS. Usually `cenc'.
527 CopyableTArray
<uint8_t> mInitData
;
529 typedef CopyableTArray
<InitData
> InitDatas
;
531 // True if the stream has encryption metadata
532 bool IsEncrypted() const { return mEncrypted
; }
539 template <typename AInitDatas
>
540 void AddInitData(const nsAString
& aType
, AInitDatas
&& aInitData
) {
541 mInitDatas
.AppendElement(
542 InitData(aType
, std::forward
<AInitDatas
>(aInitData
)));
546 void AddInitData(const EncryptionInfo
& aInfo
) {
547 mInitDatas
.AppendElements(aInfo
.mInitDatas
);
548 mEncrypted
= !!mInitDatas
.Length();
551 // One 'InitData' per encrypted buffer.
552 InitDatas mInitDatas
;
560 bool HasVideo() const { return mVideo
.IsValid(); }
566 // Set dummy values so that HasVideo() will return true;
567 // See VideoInfo::IsValid()
568 mVideo
.mDisplay
= gfx::IntSize(1, 1);
571 bool HasAudio() const { return mAudio
.IsValid(); }
577 // Set dummy values so that HasAudio() will return true;
578 // See AudioInfo::IsValid()
579 mAudio
.mChannels
= 2;
580 mAudio
.mRate
= 44100;
583 bool IsEncrypted() const {
584 return (HasAudio() && mAudio
.mCrypto
.IsEncrypted()) ||
585 (HasVideo() && mVideo
.mCrypto
.IsEncrypted());
588 bool HasValidMedia() const { return HasVideo() || HasAudio(); }
590 // TODO: Store VideoInfo and AudioIndo in arrays to support multi-tracks.
594 // If the metadata includes a duration, we store it here.
595 media::NullableTimeUnit mMetadataDuration
;
597 // The Ogg reader tries to kinda-sorta compute the duration by seeking to the
598 // end and determining the timestamp of the last frame. This isn't useful as
599 // a duration until we know the start time, so we need to track it separately.
600 media::NullableTimeUnit mUnadjustedMetadataEndTime
;
602 // True if the media is seekable (i.e. supports random access).
603 bool mMediaSeekable
= true;
605 // True if the media is only seekable within its buffered ranges.
606 bool mMediaSeekableOnlyInBufferedRanges
= false;
608 EncryptionInfo mCrypto
;
610 // The minimum of start times of audio and video tracks.
611 // Use to map the zero time on the media timeline to the first frame.
612 media::TimeUnit mStartTime
;
615 class TrackInfoSharedPtr
{
616 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(TrackInfoSharedPtr
)
618 TrackInfoSharedPtr(const TrackInfo
& aOriginal
, uint32_t aStreamID
)
619 : mInfo(aOriginal
.Clone()),
620 mStreamSourceID(aStreamID
),
621 mMimeType(mInfo
->mMimeType
) {}
623 uint32_t GetID() const { return mStreamSourceID
; }
625 operator const TrackInfo
*() const { return mInfo
.get(); }
627 const TrackInfo
* operator*() const { return mInfo
.get(); }
629 const TrackInfo
* operator->() const {
630 MOZ_ASSERT(mInfo
.get(), "dereferencing a UniquePtr containing nullptr");
634 const AudioInfo
* GetAsAudioInfo() const {
635 return mInfo
? mInfo
->GetAsAudioInfo() : nullptr;
638 const VideoInfo
* GetAsVideoInfo() const {
639 return mInfo
? mInfo
->GetAsVideoInfo() : nullptr;
642 const TextInfo
* GetAsTextInfo() const {
643 return mInfo
? mInfo
->GetAsTextInfo() : nullptr;
647 ~TrackInfoSharedPtr() = default;
648 UniquePtr
<TrackInfo
> mInfo
;
649 // A unique ID, guaranteed to change when changing streams.
650 uint32_t mStreamSourceID
;
653 const nsCString
& mMimeType
;
656 } // namespace mozilla
658 #endif // MediaInfo_h