content/media/plugins/MediaPluginReader.cpp

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
   5  * You can obtain one at http://mozilla.org/MPL/2.0/. */
   6 #include "MediaPluginReader.h"
   7 #include "mozilla/TimeStamp.h"
   8 #include "mozilla/dom/TimeRanges.h"
   9 #include "MediaResource.h"
  10 #include "VideoUtils.h"
  11 #include "MediaPluginDecoder.h"
  12 #include "MediaPluginHost.h"
  13 #include "MediaDecoderStateMachine.h"
  14 #include "ImageContainer.h"
  15 #include "AbstractMediaDecoder.h"
  16
  17 namespace mozilla {
  18
  19 typedef mozilla::layers::Image Image;
  20
  21 MediaPluginReader::MediaPluginReader(AbstractMediaDecoder *aDecoder,
  22                                      const nsACString& aContentType) :
  23   MediaDecoderReader(aDecoder),
  24   mType(aContentType),
  25   mPlugin(NULL),
  26   mHasAudio(false),
  27   mHasVideo(false),
  28   mVideoSeekTimeUs(-1),
  29   mAudioSeekTimeUs(-1),
  30   mLastVideoFrame(NULL)
  31 {
  32 }
  33
  34 MediaPluginReader::~MediaPluginReader()
  35 {
  36   ResetDecode();
  37 }
  38
  39 nsresult MediaPluginReader::Init(MediaDecoderReader* aCloneDonor)
  40 {
  41   return NS_OK;
  42 }
  43
  44 nsresult MediaPluginReader::ReadMetadata(VideoInfo* aInfo,
  45                                            MetadataTags** aTags)
  46 {
  47   NS_ASSERTION(mDecoder->OnDecodeThread(), "Should be on decode thread.");
  48
  49   if (!mPlugin) {
  50     mPlugin = GetMediaPluginHost()->CreateDecoder(mDecoder->GetResource(), mType);
  51     if (!mPlugin) {
  52       return NS_ERROR_FAILURE;
  53     }
  54   }
  55
  56   // Set the total duration (the max of the audio and video track).
  57   int64_t durationUs;
  58   mPlugin->GetDuration(mPlugin, &durationUs);
  59   if (durationUs) {
  60     ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
  61     mDecoder->SetMediaDuration(durationUs);
  62   }
  63
  64   if (mPlugin->HasVideo(mPlugin)) {
  65     int32_t width, height;
  66     mPlugin->GetVideoParameters(mPlugin, &width, &height);
  67     nsIntRect pictureRect(0, 0, width, height);
  68
  69     // Validate the container-reported frame and pictureRect sizes. This ensures
  70     // that our video frame creation code doesn't overflow.
  71     nsIntSize displaySize(width, height);
  72     nsIntSize frameSize(width, height);
  73     if (!VideoInfo::ValidateVideoRegion(frameSize, pictureRect, displaySize)) {
  74       return NS_ERROR_FAILURE;
  75     }
  76
  77     // Video track's frame sizes will not overflow. Activate the video track.
  78     mHasVideo = mInfo.mHasVideo = true;
  79     mInfo.mDisplay = displaySize;
  80     mPicture = pictureRect;
  81     mInitialFrame = frameSize;
  82     VideoFrameContainer* container = mDecoder->GetVideoFrameContainer();
  83     if (container) {
  84       container->SetCurrentFrame(gfxIntSize(displaySize.width, displaySize.height),
  85                                  nullptr,
  86                                  mozilla::TimeStamp::Now());
  87     }
  88   }
  89
  90   if (mPlugin->HasAudio(mPlugin)) {
  91     int32_t numChannels, sampleRate;
  92     mPlugin->GetAudioParameters(mPlugin, &numChannels, &sampleRate);
  93     mHasAudio = mInfo.mHasAudio = true;
  94     mInfo.mAudioChannels = numChannels;
  95     mInfo.mAudioRate = sampleRate;
  96   }
  97
  98  *aInfo = mInfo;
  99  *aTags = nullptr;
 100   return NS_OK;
 101 }
 102
 103 // Resets all state related to decoding, emptying all buffers etc.
 104 nsresult MediaPluginReader::ResetDecode()
 105 {
 106   if (mLastVideoFrame) {
 107     delete mLastVideoFrame;
 108     mLastVideoFrame = NULL;
 109   }
 110   if (mPlugin) {
 111     GetMediaPluginHost()->DestroyDecoder(mPlugin);
 112     mPlugin = NULL;
 113   }
 114
 115   return NS_OK;
 116 }
 117
 118 bool MediaPluginReader::DecodeVideoFrame(bool &aKeyframeSkip,
 119                                            int64_t aTimeThreshold)
 120 {
 121   // Record number of frames decoded and parsed. Automatically update the
 122   // stats counters using the AutoNotifyDecoded stack-based class.
 123   uint32_t parsed = 0, decoded = 0;
 124   AbstractMediaDecoder::AutoNotifyDecoded autoNotify(mDecoder, parsed, decoded);
 125
 126   // Throw away the currently buffered frame if we are seeking.
 127   if (mLastVideoFrame && mVideoSeekTimeUs != -1) {
 128     delete mLastVideoFrame;
 129     mLastVideoFrame = NULL;
 130   }
 131
 132   ImageBufferCallback bufferCallback(mDecoder->GetImageContainer());
 133   nsRefPtr<Image> currentImage;
 134
 135   // Read next frame
 136   while (true) {
 137     MPAPI::VideoFrame frame;
 138     if (!mPlugin->ReadVideo(mPlugin, &frame, mVideoSeekTimeUs, &bufferCallback)) {
 139       // We reached the end of the video stream. If we have a buffered
 140       // video frame, push it the video queue using the total duration
 141       // of the video as the end time.
 142       if (mLastVideoFrame) {
 143         int64_t durationUs;
 144         mPlugin->GetDuration(mPlugin, &durationUs);
 145         mLastVideoFrame->mEndTime = (durationUs > mLastVideoFrame->mTime)
 146                                   ? durationUs
 147                                   : mLastVideoFrame->mTime;
 148         mVideoQueue.Push(mLastVideoFrame);
 149         mLastVideoFrame = NULL;
 150       }
 151       mVideoQueue.Finish();
 152       return false;
 153     }
 154     mVideoSeekTimeUs = -1;
 155
 156     if (aKeyframeSkip) {
 157       // Disable keyframe skipping for now as
 158       // stagefright doesn't seem to be telling us
 159       // when a frame is a keyframe.
 160 #if 0
 161       if (!frame.mKeyFrame) {
 162         ++parsed;
 163         continue;
 164       }
 165 #endif
 166       aKeyframeSkip = false;
 167     }
 168
 169     if (frame.mSize == 0)
 170       return true;
 171
 172     currentImage = bufferCallback.GetImage();
 173     int64_t pos = mDecoder->GetResource()->Tell();
 174     nsIntRect picture = mPicture;
 175
 176     VideoData *v;
 177     if (currentImage) {
 178       gfxIntSize frameSize = currentImage->GetSize();
 179       if (frameSize.width != mInitialFrame.width ||
 180           frameSize.height != mInitialFrame.height) {
 181         // Frame size is different from what the container reports. This is legal,
 182         // and we will preserve the ratio of the crop rectangle as it
 183         // was reported relative to the picture size reported by the container.
 184         picture.x = (mPicture.x * frameSize.width) / mInitialFrame.width;
 185         picture.y = (mPicture.y * frameSize.height) / mInitialFrame.height;
 186         picture.width = (frameSize.width * mPicture.width) / mInitialFrame.width;
 187         picture.height = (frameSize.height * mPicture.height) / mInitialFrame.height;
 188       }
 189
 190       v = VideoData::CreateFromImage(mInfo,
 191                                      mDecoder->GetImageContainer(),
 192                                      pos,
 193                                      frame.mTimeUs,
 194                                      frame.mTimeUs+1, // We don't know the end time.
 195                                      currentImage,
 196                                      frame.mKeyFrame,
 197                                      -1,
 198                                      picture);
 199     } else {
 200       // Assume YUV
 201       VideoData::YCbCrBuffer b;
 202       b.mPlanes[0].mData = static_cast<uint8_t *>(frame.Y.mData);
 203       b.mPlanes[0].mStride = frame.Y.mStride;
 204       b.mPlanes[0].mHeight = frame.Y.mHeight;
 205       b.mPlanes[0].mWidth = frame.Y.mWidth;
 206       b.mPlanes[0].mOffset = frame.Y.mOffset;
 207       b.mPlanes[0].mSkip = frame.Y.mSkip;
 208
 209       b.mPlanes[1].mData = static_cast<uint8_t *>(frame.Cb.mData);
 210       b.mPlanes[1].mStride = frame.Cb.mStride;
 211       b.mPlanes[1].mHeight = frame.Cb.mHeight;
 212       b.mPlanes[1].mWidth = frame.Cb.mWidth;
 213       b.mPlanes[1].mOffset = frame.Cb.mOffset;
 214       b.mPlanes[1].mSkip = frame.Cb.mSkip;
 215
 216       b.mPlanes[2].mData = static_cast<uint8_t *>(frame.Cr.mData);
 217       b.mPlanes[2].mStride = frame.Cr.mStride;
 218       b.mPlanes[2].mHeight = frame.Cr.mHeight;
 219       b.mPlanes[2].mWidth = frame.Cr.mWidth;
 220       b.mPlanes[2].mOffset = frame.Cr.mOffset;
 221       b.mPlanes[2].mSkip = frame.Cr.mSkip;
 222
 223       if (frame.Y.mWidth != mInitialFrame.width ||
 224           frame.Y.mHeight != mInitialFrame.height) {
 225
 226         // Frame size is different from what the container reports. This is legal,
 227         // and we will preserve the ratio of the crop rectangle as it
 228         // was reported relative to the picture size reported by the container.
 229         picture.x = (mPicture.x * frame.Y.mWidth) / mInitialFrame.width;
 230         picture.y = (mPicture.y * frame.Y.mHeight) / mInitialFrame.height;
 231         picture.width = (frame.Y.mWidth * mPicture.width) / mInitialFrame.width;
 232         picture.height = (frame.Y.mHeight * mPicture.height) / mInitialFrame.height;
 233       }
 234
 235       // This is the approximate byte position in the stream.
 236       v = VideoData::Create(mInfo,
 237                             mDecoder->GetImageContainer(),
 238                             pos,
 239                             frame.mTimeUs,
 240                             frame.mTimeUs+1, // We don't know the end time.
 241                             b,
 242                             frame.mKeyFrame,
 243                             -1,
 244                             picture);
 245     }
 246
 247     if (!v) {
 248       return false;
 249     }
 250     parsed++;
 251     decoded++;
 252     NS_ASSERTION(decoded <= parsed, "Expect to decode fewer frames than parsed in MediaPlugin...");
 253
 254     // Since MPAPI doesn't give us the end time of frames, we keep one frame
 255     // buffered in MediaPluginReader and push it into the queue as soon
 256     // we read the following frame so we can use that frame's start time as
 257     // the end time of the buffered frame.
 258     if (!mLastVideoFrame) {
 259       mLastVideoFrame = v;
 260       continue;
 261     }
 262
 263     mLastVideoFrame->mEndTime = v->mTime;
 264
 265     // We have the start time of the next frame, so we can push the previous
 266     // frame into the queue, except if the end time is below the threshold,
 267     // in which case it wouldn't be displayed anyway.
 268     if (mLastVideoFrame->mEndTime < aTimeThreshold) {
 269       delete mLastVideoFrame;
 270       mLastVideoFrame = NULL;
 271       continue;
 272     }
 273
 274     mVideoQueue.Push(mLastVideoFrame);
 275
 276     // Buffer the current frame we just decoded.
 277     mLastVideoFrame = v;
 278
 279     break;
 280   }
 281
 282   return true;
 283 }
 284
 285 bool MediaPluginReader::DecodeAudioData()
 286 {
 287   NS_ASSERTION(mDecoder->OnDecodeThread(), "Should be on decode thread.");
 288
 289   // This is the approximate byte position in the stream.
 290   int64_t pos = mDecoder->GetResource()->Tell();
 291
 292   // Read next frame
 293   MPAPI::AudioFrame frame;
 294   if (!mPlugin->ReadAudio(mPlugin, &frame, mAudioSeekTimeUs)) {
 295     mAudioQueue.Finish();
 296     return false;
 297   }
 298   mAudioSeekTimeUs = -1;
 299
 300   // Ignore empty buffers which stagefright media read will sporadically return
 301   if (frame.mSize == 0)
 302     return true;
 303
 304   nsAutoArrayPtr<AudioDataValue> buffer(new AudioDataValue[frame.mSize/2] );
 305   memcpy(buffer.get(), frame.mData, frame.mSize);
 306
 307   uint32_t frames = frame.mSize / (2 * frame.mAudioChannels);
 308   CheckedInt64 duration = FramesToUsecs(frames, frame.mAudioSampleRate);
 309   if (!duration.isValid()) {
 310     return false;
 311   }
 312
 313   mAudioQueue.Push(new AudioData(pos,
 314                                  frame.mTimeUs,
 315                                  duration.value(),
 316                                  frames,
 317                                  buffer.forget(),
 318                                  frame.mAudioChannels));
 319   return true;
 320 }
 321
 322 nsresult MediaPluginReader::Seek(int64_t aTarget, int64_t aStartTime, int64_t aEndTime, int64_t aCurrentTime)
 323 {
 324   NS_ASSERTION(mDecoder->OnDecodeThread(), "Should be on decode thread.");
 325
 326   mVideoQueue.Erase();
 327   mAudioQueue.Erase();
 328
 329   mAudioSeekTimeUs = mVideoSeekTimeUs = aTarget;
 330
 331   return DecodeToTarget(aTarget);
 332 }
 333
 334 nsresult MediaPluginReader::GetBuffered(mozilla::dom::TimeRanges* aBuffered, int64_t aStartTime)
 335 {
 336   if (!mPlugin)
 337     return NS_OK;
 338
 339   MediaResource* stream = mDecoder->GetResource();
 340
 341   int64_t durationUs = 0;
 342   mPlugin->GetDuration(mPlugin, &durationUs);
 343
 344   GetEstimatedBufferedTimeRanges(stream, durationUs, aBuffered);
 345
 346   return NS_OK;
 347 }
 348
 349 MediaPluginReader::ImageBufferCallback::ImageBufferCallback(mozilla::layers::ImageContainer *aImageContainer) :
 350   mImageContainer(aImageContainer)
 351 {
 352 }
 353
 354 void *
 355 MediaPluginReader::ImageBufferCallback::operator()(size_t aWidth, size_t aHeight,
 356                                                      MPAPI::ColorFormat aColorFormat)
 357 {
 358   if (!mImageContainer) {
 359     NS_WARNING("No image container to construct an image");
 360     return nullptr;
 361   }
 362
 363   nsRefPtr<Image> rgbImage;
 364   switch(aColorFormat) {
 365     case MPAPI::RGB565:
 366       rgbImage = mozilla::layers::CreateSharedRGBImage(mImageContainer,
 367                                                        nsIntSize(aWidth, aHeight),
 368                                                        gfxASurface::ImageFormatRGB16_565);
 369       mImage = rgbImage;
 370       return rgbImage->AsSharedImage()->GetBuffer();
 371     case MPAPI::YCbCr:
 372     default:
 373       NS_NOTREACHED("Color format not supported");
 374       return nullptr;
 375   }
 376 }
 377
 378 already_AddRefed<Image>
 379 MediaPluginReader::ImageBufferCallback::GetImage()
 380 {
 381   return mImage.forget();
 382 }
 383
 384 } // namespace mozilla