no bug - Bumping Firefox l10n changesets r=release a=l10n-bump DONTBUILD CLOSED TREE
[gecko.git] / dom / media / MediaData.cpp
blob252e29d281a60e529281008a5620dbea0ce52a0f
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #include "MediaData.h"
9 #include "ImageContainer.h"
10 #include "MediaInfo.h"
11 #include "PerformanceRecorder.h"
12 #include "VideoUtils.h"
13 #include "YCbCrUtils.h"
14 #include "mozilla/gfx/gfxVars.h"
15 #include "mozilla/layers/ImageBridgeChild.h"
16 #include "mozilla/layers/KnowsCompositor.h"
17 #include "mozilla/layers/SharedRGBImage.h"
19 #include <stdint.h>
21 #ifdef XP_WIN
22 # include "mozilla/gfx/DeviceManagerDx.h"
23 # include "mozilla/layers/D3D11ShareHandleImage.h"
24 # include "mozilla/layers/D3D11YCbCrImage.h"
25 #elif XP_MACOSX
26 # include "MacIOSurfaceImage.h"
27 # include "mozilla/gfx/gfxVars.h"
28 #endif
30 namespace mozilla {
32 using namespace mozilla::gfx;
33 using layers::PlanarYCbCrData;
34 using layers::PlanarYCbCrImage;
35 using media::TimeUnit;
37 const char* AudioData::sTypeName = "audio";
38 const char* VideoData::sTypeName = "video";
40 AudioData::AudioData(int64_t aOffset, const media::TimeUnit& aTime,
41 AlignedAudioBuffer&& aData, uint32_t aChannels,
42 uint32_t aRate, uint32_t aChannelMap)
43 // Passing TimeUnit::Zero() here because we can't pass the result of an
44 // arithmetic operation to the CheckedInt ctor. We set the duration in the
45 // ctor body below.
46 : MediaData(sType, aOffset, aTime, TimeUnit::Zero()),
47 mChannels(aChannels),
48 mChannelMap(aChannelMap),
49 mRate(aRate),
50 mOriginalTime(aTime),
51 mAudioData(std::move(aData)),
52 mFrames(mAudioData.Length() / aChannels) {
53 MOZ_RELEASE_ASSERT(aChannels != 0,
54 "Can't create an AudioData with 0 channels.");
55 MOZ_RELEASE_ASSERT(aRate != 0,
56 "Can't create an AudioData with a sample-rate of 0.");
57 mDuration = TimeUnit(mFrames, aRate);
60 Span<AudioDataValue> AudioData::Data() const {
61 return Span{GetAdjustedData(), mFrames * mChannels};
64 void AudioData::SetOriginalStartTime(const media::TimeUnit& aStartTime) {
65 MOZ_ASSERT(mTime == mOriginalTime,
66 "Do not call this if data has been trimmed!");
67 mTime = aStartTime;
68 mOriginalTime = aStartTime;
71 bool AudioData::AdjustForStartTime(const media::TimeUnit& aStartTime) {
72 mOriginalTime -= aStartTime;
73 mTime -= aStartTime;
74 if (mTrimWindow) {
75 *mTrimWindow -= aStartTime;
77 if (mTime.IsNegative()) {
78 NS_WARNING("Negative audio start time after time-adjustment!");
80 return mTime.IsValid() && mOriginalTime.IsValid();
83 bool AudioData::SetTrimWindow(const media::TimeInterval& aTrim) {
84 MOZ_DIAGNOSTIC_ASSERT(aTrim.mStart.IsValid() && aTrim.mEnd.IsValid(),
85 "An overflow occurred on the provided TimeInterval");
86 if (!mAudioData) {
87 // MoveableData got called. Can no longer work on it.
88 return false;
90 if (aTrim.mStart < mOriginalTime || aTrim.mEnd > GetEndTime()) {
91 return false;
94 auto trimBefore = aTrim.mStart - mOriginalTime;
95 auto trimAfter = aTrim.mEnd - mOriginalTime;
96 if (!trimBefore.IsValid() || !trimAfter.IsValid()) {
97 // Overflow.
98 return false;
100 if (!mTrimWindow && trimBefore.IsZero() && trimAfter == mDuration) {
101 // Nothing to change, abort early to prevent rounding errors.
102 return true;
105 size_t frameOffset = trimBefore.ToTicksAtRate(mRate);
106 mTrimWindow = Some(aTrim);
107 mDataOffset = frameOffset * mChannels;
108 MOZ_DIAGNOSTIC_ASSERT(mDataOffset <= mAudioData.Length(),
109 "Data offset outside original buffer");
110 mFrames = (trimAfter - trimBefore).ToTicksAtRate(mRate);
111 MOZ_DIAGNOSTIC_ASSERT(mFrames <= mAudioData.Length() / mChannels,
112 "More frames than found in container");
113 mTime = mOriginalTime + trimBefore;
114 mDuration = TimeUnit(mFrames, mRate);
116 return true;
119 AudioDataValue* AudioData::GetAdjustedData() const {
120 if (!mAudioData) {
121 return nullptr;
123 return mAudioData.Data() + mDataOffset;
126 void AudioData::EnsureAudioBuffer() {
127 if (mAudioBuffer || !mAudioData) {
128 return;
130 const AudioDataValue* srcData = GetAdjustedData();
131 CheckedInt<size_t> bufferSize(sizeof(AudioDataValue));
132 bufferSize *= mFrames;
133 bufferSize *= mChannels;
134 mAudioBuffer = SharedBuffer::Create(bufferSize);
136 AudioDataValue* destData = static_cast<AudioDataValue*>(mAudioBuffer->Data());
137 for (uint32_t i = 0; i < mFrames; ++i) {
138 for (uint32_t j = 0; j < mChannels; ++j) {
139 destData[j * mFrames + i] = srcData[i * mChannels + j];
144 size_t AudioData::SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const {
145 size_t size =
146 aMallocSizeOf(this) + mAudioData.SizeOfExcludingThis(aMallocSizeOf);
147 if (mAudioBuffer) {
148 size += mAudioBuffer->SizeOfIncludingThis(aMallocSizeOf);
150 return size;
153 AlignedAudioBuffer AudioData::MoveableData() {
154 // Trim buffer according to trimming mask.
155 mAudioData.PopFront(mDataOffset);
156 mAudioData.SetLength(mFrames * mChannels);
157 mDataOffset = 0;
158 mFrames = 0;
159 mTrimWindow.reset();
160 return std::move(mAudioData);
163 static bool ValidatePlane(const VideoData::YCbCrBuffer::Plane& aPlane) {
164 return aPlane.mWidth <= PlanarYCbCrImage::MAX_DIMENSION &&
165 aPlane.mHeight <= PlanarYCbCrImage::MAX_DIMENSION &&
166 aPlane.mWidth * aPlane.mHeight < MAX_VIDEO_WIDTH * MAX_VIDEO_HEIGHT &&
167 aPlane.mStride > 0 && aPlane.mWidth <= aPlane.mStride;
170 static bool ValidateBufferAndPicture(const VideoData::YCbCrBuffer& aBuffer,
171 const IntRect& aPicture) {
172 // The following situation should never happen unless there is a bug
173 // in the decoder
174 if (aBuffer.mPlanes[1].mWidth != aBuffer.mPlanes[2].mWidth ||
175 aBuffer.mPlanes[1].mHeight != aBuffer.mPlanes[2].mHeight) {
176 NS_ERROR("C planes with different sizes");
177 return false;
180 // The following situations could be triggered by invalid input
181 if (aPicture.width <= 0 || aPicture.height <= 0) {
182 NS_WARNING("Empty picture rect");
183 return false;
185 if (!ValidatePlane(aBuffer.mPlanes[0]) ||
186 !ValidatePlane(aBuffer.mPlanes[1]) ||
187 !ValidatePlane(aBuffer.mPlanes[2])) {
188 NS_WARNING("Invalid plane size");
189 return false;
192 // Ensure the picture size specified in the headers can be extracted out of
193 // the frame we've been supplied without indexing out of bounds.
194 CheckedUint32 xLimit = aPicture.x + CheckedUint32(aPicture.width);
195 CheckedUint32 yLimit = aPicture.y + CheckedUint32(aPicture.height);
196 if (!xLimit.isValid() || xLimit.value() > aBuffer.mPlanes[0].mStride ||
197 !yLimit.isValid() || yLimit.value() > aBuffer.mPlanes[0].mHeight) {
198 // The specified picture dimensions can't be contained inside the video
199 // frame, we'll stomp memory if we try to copy it. Fail.
200 NS_WARNING("Overflowing picture rect");
201 return false;
203 return true;
206 VideoData::VideoData(int64_t aOffset, const TimeUnit& aTime,
207 const TimeUnit& aDuration, bool aKeyframe,
208 const TimeUnit& aTimecode, IntSize aDisplay,
209 layers::ImageContainer::FrameID aFrameID)
210 : MediaData(Type::VIDEO_DATA, aOffset, aTime, aDuration),
211 mDisplay(aDisplay),
212 mFrameID(aFrameID),
213 mSentToCompositor(false),
214 mNextKeyFrameTime(TimeUnit::Invalid()) {
215 MOZ_ASSERT(!mDuration.IsNegative(), "Frame must have non-negative duration.");
216 mKeyframe = aKeyframe;
217 mTimecode = aTimecode;
220 VideoData::~VideoData() = default;
222 size_t VideoData::SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const {
223 size_t size = aMallocSizeOf(this);
225 // Currently only PLANAR_YCBCR has a well defined function for determining
226 // it's size, so reporting is limited to that type.
227 if (mImage && mImage->GetFormat() == ImageFormat::PLANAR_YCBCR) {
228 const mozilla::layers::PlanarYCbCrImage* img =
229 static_cast<const mozilla::layers::PlanarYCbCrImage*>(mImage.get());
230 size += img->SizeOfIncludingThis(aMallocSizeOf);
233 return size;
236 ColorDepth VideoData::GetColorDepth() const {
237 if (!mImage) {
238 return ColorDepth::COLOR_8;
241 return mImage->GetColorDepth();
244 void VideoData::UpdateDuration(const TimeUnit& aDuration) {
245 MOZ_ASSERT(!aDuration.IsNegative());
246 mDuration = aDuration;
249 void VideoData::UpdateTimestamp(const TimeUnit& aTimestamp) {
250 MOZ_ASSERT(!aTimestamp.IsNegative());
252 auto updatedDuration = GetEndTime() - aTimestamp;
253 MOZ_ASSERT(!updatedDuration.IsNegative());
255 mTime = aTimestamp;
256 mDuration = updatedDuration;
259 bool VideoData::AdjustForStartTime(const media::TimeUnit& aStartTime) {
260 mTime -= aStartTime;
261 if (mTime.IsNegative()) {
262 NS_WARNING("Negative video start time after time-adjustment!");
264 return mTime.IsValid();
267 PlanarYCbCrData ConstructPlanarYCbCrData(const VideoInfo& aInfo,
268 const VideoData::YCbCrBuffer& aBuffer,
269 const IntRect& aPicture) {
270 const VideoData::YCbCrBuffer::Plane& Y = aBuffer.mPlanes[0];
271 const VideoData::YCbCrBuffer::Plane& Cb = aBuffer.mPlanes[1];
272 const VideoData::YCbCrBuffer::Plane& Cr = aBuffer.mPlanes[2];
274 PlanarYCbCrData data;
275 data.mYChannel = Y.mData;
276 data.mYStride = AssertedCast<int32_t>(Y.mStride);
277 data.mYSkip = AssertedCast<int32_t>(Y.mSkip);
278 data.mCbChannel = Cb.mData;
279 data.mCrChannel = Cr.mData;
280 data.mCbCrStride = AssertedCast<int32_t>(Cb.mStride);
281 data.mCbSkip = AssertedCast<int32_t>(Cb.mSkip);
282 data.mCrSkip = AssertedCast<int32_t>(Cr.mSkip);
283 data.mPictureRect = aPicture;
284 data.mStereoMode = aInfo.mStereoMode;
285 data.mYUVColorSpace = aBuffer.mYUVColorSpace;
286 data.mColorPrimaries = aBuffer.mColorPrimaries;
287 data.mColorDepth = aBuffer.mColorDepth;
288 if (aInfo.mTransferFunction) {
289 data.mTransferFunction = *aInfo.mTransferFunction;
291 data.mColorRange = aBuffer.mColorRange;
292 data.mChromaSubsampling = aBuffer.mChromaSubsampling;
293 return data;
296 /* static */
297 bool VideoData::SetVideoDataToImage(PlanarYCbCrImage* aVideoImage,
298 const VideoInfo& aInfo,
299 const YCbCrBuffer& aBuffer,
300 const IntRect& aPicture, bool aCopyData) {
301 if (!aVideoImage) {
302 return false;
305 PlanarYCbCrData data = ConstructPlanarYCbCrData(aInfo, aBuffer, aPicture);
307 if (aCopyData) {
308 return aVideoImage->CopyData(data);
310 return aVideoImage->AdoptData(data);
313 /* static */
314 already_AddRefed<VideoData> VideoData::CreateAndCopyData(
315 const VideoInfo& aInfo, ImageContainer* aContainer, int64_t aOffset,
316 const TimeUnit& aTime, const TimeUnit& aDuration,
317 const YCbCrBuffer& aBuffer, bool aKeyframe, const TimeUnit& aTimecode,
318 const IntRect& aPicture, layers::KnowsCompositor* aAllocator) {
319 if (!aContainer) {
320 // Create a dummy VideoData with no image. This gives us something to
321 // send to media streams if necessary.
322 RefPtr<VideoData> v(new VideoData(aOffset, aTime, aDuration, aKeyframe,
323 aTimecode, aInfo.mDisplay, 0));
324 return v.forget();
327 if (!ValidateBufferAndPicture(aBuffer, aPicture)) {
328 return nullptr;
331 PerformanceRecorder<PlaybackStage> perfRecorder(MediaStage::CopyDecodedVideo,
332 aInfo.mImage.height);
333 RefPtr<VideoData> v(new VideoData(aOffset, aTime, aDuration, aKeyframe,
334 aTimecode, aInfo.mDisplay, 0));
336 // Currently our decoder only knows how to output to ImageFormat::PLANAR_YCBCR
337 // format.
338 #if XP_MACOSX
339 if (aAllocator && aAllocator->GetWebRenderCompositorType() !=
340 layers::WebRenderCompositor::SOFTWARE) {
341 RefPtr<layers::MacIOSurfaceImage> ioImage =
342 new layers::MacIOSurfaceImage(nullptr);
343 PlanarYCbCrData data = ConstructPlanarYCbCrData(aInfo, aBuffer, aPicture);
344 if (ioImage->SetData(aContainer, data)) {
345 v->mImage = ioImage;
346 perfRecorder.Record();
347 return v.forget();
350 #endif
351 if (!v->mImage) {
352 v->mImage = aContainer->CreatePlanarYCbCrImage();
355 if (!v->mImage) {
356 return nullptr;
358 NS_ASSERTION(v->mImage->GetFormat() == ImageFormat::PLANAR_YCBCR,
359 "Wrong format?");
360 PlanarYCbCrImage* videoImage = v->mImage->AsPlanarYCbCrImage();
361 MOZ_ASSERT(videoImage);
363 if (!VideoData::SetVideoDataToImage(videoImage, aInfo, aBuffer, aPicture,
364 true /* aCopyData */)) {
365 return nullptr;
368 perfRecorder.Record();
369 return v.forget();
372 /* static */
373 already_AddRefed<VideoData> VideoData::CreateAndCopyData(
374 const VideoInfo& aInfo, ImageContainer* aContainer, int64_t aOffset,
375 const TimeUnit& aTime, const TimeUnit& aDuration,
376 const YCbCrBuffer& aBuffer, const YCbCrBuffer::Plane& aAlphaPlane,
377 bool aKeyframe, const TimeUnit& aTimecode, const IntRect& aPicture) {
378 if (!aContainer) {
379 // Create a dummy VideoData with no image. This gives us something to
380 // send to media streams if necessary.
381 RefPtr<VideoData> v(new VideoData(aOffset, aTime, aDuration, aKeyframe,
382 aTimecode, aInfo.mDisplay, 0));
383 return v.forget();
386 if (!ValidateBufferAndPicture(aBuffer, aPicture)) {
387 return nullptr;
390 RefPtr<VideoData> v(new VideoData(aOffset, aTime, aDuration, aKeyframe,
391 aTimecode, aInfo.mDisplay, 0));
393 // Convert from YUVA to BGRA format on the software side.
394 RefPtr<layers::SharedRGBImage> videoImage =
395 aContainer->CreateSharedRGBImage();
396 v->mImage = videoImage;
398 if (!v->mImage) {
399 return nullptr;
401 if (!videoImage->Allocate(
402 IntSize(aBuffer.mPlanes[0].mWidth, aBuffer.mPlanes[0].mHeight),
403 SurfaceFormat::B8G8R8A8)) {
404 return nullptr;
407 RefPtr<layers::TextureClient> texture =
408 videoImage->GetTextureClient(/* aKnowsCompositor */ nullptr);
409 if (!texture) {
410 NS_WARNING("Failed to allocate TextureClient");
411 return nullptr;
414 layers::TextureClientAutoLock autoLock(texture,
415 layers::OpenMode::OPEN_WRITE_ONLY);
416 if (!autoLock.Succeeded()) {
417 NS_WARNING("Failed to lock TextureClient");
418 return nullptr;
421 layers::MappedTextureData buffer;
422 if (!texture->BorrowMappedData(buffer)) {
423 NS_WARNING("Failed to borrow mapped data");
424 return nullptr;
427 // The naming convention for libyuv and associated utils is word-order.
428 // The naming convention in the gfx stack is byte-order.
429 ConvertI420AlphaToARGB(aBuffer.mPlanes[0].mData, aBuffer.mPlanes[1].mData,
430 aBuffer.mPlanes[2].mData, aAlphaPlane.mData,
431 AssertedCast<int>(aBuffer.mPlanes[0].mStride),
432 AssertedCast<int>(aBuffer.mPlanes[1].mStride),
433 buffer.data, buffer.stride, buffer.size.width,
434 buffer.size.height);
436 return v.forget();
439 /* static */
440 already_AddRefed<VideoData> VideoData::CreateFromImage(
441 const IntSize& aDisplay, int64_t aOffset, const TimeUnit& aTime,
442 const TimeUnit& aDuration, const RefPtr<Image>& aImage, bool aKeyframe,
443 const TimeUnit& aTimecode) {
444 RefPtr<VideoData> v(new VideoData(aOffset, aTime, aDuration, aKeyframe,
445 aTimecode, aDisplay, 0));
446 v->mImage = aImage;
447 return v.forget();
450 MediaRawData::MediaRawData()
451 : MediaData(Type::RAW_DATA), mCrypto(mCryptoInternal) {}
453 MediaRawData::MediaRawData(const uint8_t* aData, size_t aSize)
454 : MediaData(Type::RAW_DATA),
455 mCrypto(mCryptoInternal),
456 mBuffer(aData, aSize) {}
458 MediaRawData::MediaRawData(const uint8_t* aData, size_t aSize,
459 const uint8_t* aAlphaData, size_t aAlphaSize)
460 : MediaData(Type::RAW_DATA),
461 mCrypto(mCryptoInternal),
462 mBuffer(aData, aSize),
463 mAlphaBuffer(aAlphaData, aAlphaSize) {}
465 MediaRawData::MediaRawData(AlignedByteBuffer&& aData)
466 : MediaData(Type::RAW_DATA),
467 mCrypto(mCryptoInternal),
468 mBuffer(std::move(aData)) {}
470 MediaRawData::MediaRawData(AlignedByteBuffer&& aData,
471 AlignedByteBuffer&& aAlphaData)
472 : MediaData(Type::RAW_DATA),
473 mCrypto(mCryptoInternal),
474 mBuffer(std::move(aData)),
475 mAlphaBuffer(std::move(aAlphaData)) {}
477 already_AddRefed<MediaRawData> MediaRawData::Clone() const {
478 int32_t sampleHeight = 0;
479 if (mTrackInfo && mTrackInfo->GetAsVideoInfo()) {
480 sampleHeight = mTrackInfo->GetAsVideoInfo()->mImage.height;
482 PerformanceRecorder<PlaybackStage> perfRecorder(MediaStage::CopyDemuxedData,
483 sampleHeight);
484 RefPtr<MediaRawData> s = new MediaRawData;
485 s->mTimecode = mTimecode;
486 s->mTime = mTime;
487 s->mDuration = mDuration;
488 s->mOffset = mOffset;
489 s->mKeyframe = mKeyframe;
490 s->mExtraData = mExtraData;
491 s->mCryptoInternal = mCryptoInternal;
492 s->mTrackInfo = mTrackInfo;
493 s->mEOS = mEOS;
494 s->mOriginalPresentationWindow = mOriginalPresentationWindow;
495 if (!s->mBuffer.Append(mBuffer.Data(), mBuffer.Length())) {
496 return nullptr;
498 if (!s->mAlphaBuffer.Append(mAlphaBuffer.Data(), mAlphaBuffer.Length())) {
499 return nullptr;
501 perfRecorder.Record();
502 return s.forget();
505 MediaRawData::~MediaRawData() = default;
507 size_t MediaRawData::SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const {
508 size_t size = aMallocSizeOf(this);
509 size += mBuffer.SizeOfExcludingThis(aMallocSizeOf);
510 return size;
513 UniquePtr<MediaRawDataWriter> MediaRawData::CreateWriter() {
514 UniquePtr<MediaRawDataWriter> p(new MediaRawDataWriter(this));
515 return p;
518 MediaRawDataWriter::MediaRawDataWriter(MediaRawData* aMediaRawData)
519 : mCrypto(aMediaRawData->mCryptoInternal), mTarget(aMediaRawData) {}
521 bool MediaRawDataWriter::SetSize(size_t aSize) {
522 return mTarget->mBuffer.SetLength(aSize);
525 bool MediaRawDataWriter::Prepend(const uint8_t* aData, size_t aSize) {
526 return mTarget->mBuffer.Prepend(aData, aSize);
529 bool MediaRawDataWriter::Append(const uint8_t* aData, size_t aSize) {
530 return mTarget->mBuffer.Append(aData, aSize);
533 bool MediaRawDataWriter::Replace(const uint8_t* aData, size_t aSize) {
534 return mTarget->mBuffer.Replace(aData, aSize);
537 void MediaRawDataWriter::Clear() { mTarget->mBuffer.Clear(); }
539 uint8_t* MediaRawDataWriter::Data() { return mTarget->mBuffer.Data(); }
541 size_t MediaRawDataWriter::Size() { return mTarget->Size(); }
543 void MediaRawDataWriter::PopFront(size_t aSize) {
544 mTarget->mBuffer.PopFront(aSize);
547 const char* CryptoSchemeToString(const CryptoScheme& aScheme) {
548 switch (aScheme) {
549 case CryptoScheme::None:
550 return "None";
551 case CryptoScheme::Cenc:
552 return "Cenc";
553 case CryptoScheme::Cbcs:
554 return "Cbcs";
555 default:
556 MOZ_ASSERT_UNREACHABLE();
557 return "";
561 } // namespace mozilla