dom/media/platforms/omx/OmxDataDecoder.cpp

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "OmxDataDecoder.h"
   8
   9 #include "OMX_Audio.h"
  10 #include "OMX_Component.h"
  11 #include "OMX_Types.h"
  12 #include "OmxPlatformLayer.h"
  13 #include "mozilla/IntegerPrintfMacros.h"
  14
  15 #ifdef LOG
  16 #  undef LOG
  17 #  undef LOGL
  18 #endif
  19
  20 #define LOG(arg, ...)                                                  \
  21   DDMOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, "::%s: " arg, __func__, \
  22             ##__VA_ARGS__)
  23
  24 #define LOGL(arg, ...)                                                     \
  25   DDMOZ_LOGEX(self.get(), sPDMLog, mozilla::LogLevel::Debug, "::%s: " arg, \
  26               __func__, ##__VA_ARGS__)
  27
  28 #define CHECK_OMX_ERR(err)      \
  29   if (err != OMX_ErrorNone) {   \
  30     NotifyError(err, __func__); \
  31     return;                     \
  32   }
  33
  34 namespace mozilla {
  35
  36 using namespace gfx;
  37
  38 static const char* StateTypeToStr(OMX_STATETYPE aType) {
  39   MOZ_ASSERT(aType == OMX_StateLoaded || aType == OMX_StateIdle ||
  40              aType == OMX_StateExecuting || aType == OMX_StatePause ||
  41              aType == OMX_StateWaitForResources || aType == OMX_StateInvalid);
  42
  43   switch (aType) {
  44     case OMX_StateLoaded:
  45       return "OMX_StateLoaded";
  46     case OMX_StateIdle:
  47       return "OMX_StateIdle";
  48     case OMX_StateExecuting:
  49       return "OMX_StateExecuting";
  50     case OMX_StatePause:
  51       return "OMX_StatePause";
  52     case OMX_StateWaitForResources:
  53       return "OMX_StateWaitForResources";
  54     case OMX_StateInvalid:
  55       return "OMX_StateInvalid";
  56     default:
  57       return "Unknown";
  58   }
  59 }
  60
  61 // A helper class to retrieve AudioData or VideoData.
  62 class MediaDataHelper {
  63  protected:
  64   virtual ~MediaDataHelper() = default;
  65
  66  public:
  67   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaDataHelper)
  68
  69   MediaDataHelper(const TrackInfo* aTrackInfo,
  70                   layers::ImageContainer* aImageContainer,
  71                   OmxPromiseLayer* aOmxLayer);
  72
  73   already_AddRefed<MediaData> GetMediaData(BufferData* aBufferData,
  74                                            bool& aPlatformDepenentData);
  75
  76  protected:
  77   already_AddRefed<AudioData> CreateAudioData(BufferData* aBufferData);
  78
  79   already_AddRefed<VideoData> CreateYUV420VideoData(BufferData* aBufferData);
  80
  81   const TrackInfo* mTrackInfo;
  82
  83   OMX_PARAM_PORTDEFINITIONTYPE mOutputPortDef;
  84
  85   // audio output
  86   MediaQueue<AudioData> mAudioQueue;
  87
  88   AudioCompactor mAudioCompactor;
  89
  90   // video output
  91   RefPtr<layers::ImageContainer> mImageContainer;
  92 };
  93
  94 OmxDataDecoder::OmxDataDecoder(const TrackInfo& aTrackInfo,
  95                                layers::ImageContainer* aImageContainer)
  96     : mOmxTaskQueue(
  97           CreateMediaDecodeTaskQueue("OmxDataDecoder::mOmxTaskQueue")),
  98       mImageContainer(aImageContainer),
  99       mWatchManager(this, mOmxTaskQueue),
 100       mOmxState(OMX_STATETYPE::OMX_StateInvalid, "OmxDataDecoder::mOmxState"),
 101       mTrackInfo(aTrackInfo.Clone()),
 102       mFlushing(false),
 103       mShuttingDown(false),
 104       mCheckingInputExhausted(false),
 105       mPortSettingsChanged(-1, "OmxDataDecoder::mPortSettingsChanged") {
 106   LOG("");
 107   mOmxLayer = new OmxPromiseLayer(mOmxTaskQueue, this, aImageContainer);
 108 }
 109
 110 OmxDataDecoder::~OmxDataDecoder() { LOG(""); }
 111
 112 void OmxDataDecoder::InitializationTask() {
 113   mWatchManager.Watch(mOmxState, &OmxDataDecoder::OmxStateRunner);
 114   mWatchManager.Watch(mPortSettingsChanged,
 115                       &OmxDataDecoder::PortSettingsChanged);
 116 }
 117
 118 void OmxDataDecoder::EndOfStream() {
 119   LOG("");
 120   MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
 121
 122   RefPtr<OmxDataDecoder> self = this;
 123   mOmxLayer->SendCommand(OMX_CommandFlush, OMX_ALL, nullptr)
 124       ->Then(mOmxTaskQueue, __func__,
 125              [self, this](OmxCommandPromise::ResolveOrRejectValue&& aValue) {
 126                mDrainPromise.ResolveIfExists(std::move(mDecodedData), __func__);
 127                mDecodedData = DecodedData();
 128              });
 129 }
 130
 131 RefPtr<MediaDataDecoder::InitPromise> OmxDataDecoder::Init() {
 132   LOG("");
 133
 134   mThread = GetCurrentSerialEventTarget();
 135   RefPtr<OmxDataDecoder> self = this;
 136   return InvokeAsync(mOmxTaskQueue, __func__, [self, this]() {
 137     InitializationTask();
 138
 139     RefPtr<InitPromise> p = mInitPromise.Ensure(__func__);
 140     mOmxLayer->Init(mTrackInfo.get())
 141         ->Then(
 142             mOmxTaskQueue, __func__,
 143             [self, this]() {
 144               // Omx state should be OMX_StateIdle.
 145               mOmxState = mOmxLayer->GetState();
 146               MOZ_ASSERT(mOmxState != OMX_StateIdle);
 147             },
 148             [self, this]() {
 149               RejectInitPromise(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
 150             });
 151     return p;
 152   });
 153 }
 154
 155 RefPtr<MediaDataDecoder::DecodePromise> OmxDataDecoder::Decode(
 156     MediaRawData* aSample) {
 157   LOG("sample %p", aSample);
 158   MOZ_ASSERT(mThread->IsOnCurrentThread());
 159   MOZ_ASSERT(mInitPromise.IsEmpty());
 160
 161   RefPtr<OmxDataDecoder> self = this;
 162   RefPtr<MediaRawData> sample = aSample;
 163   return InvokeAsync(mOmxTaskQueue, __func__, [self, this, sample]() {
 164     RefPtr<DecodePromise> p = mDecodePromise.Ensure(__func__);
 165     mMediaRawDatas.AppendElement(std::move(sample));
 166
 167     // Start to fill/empty buffers.
 168     if (mOmxState == OMX_StateIdle || mOmxState == OMX_StateExecuting) {
 169       FillAndEmptyBuffers();
 170     }
 171     return p;
 172   });
 173 }
 174
 175 RefPtr<MediaDataDecoder::FlushPromise> OmxDataDecoder::Flush() {
 176   LOG("");
 177   MOZ_ASSERT(mThread->IsOnCurrentThread());
 178
 179   mFlushing = true;
 180
 181   return InvokeAsync(mOmxTaskQueue, this, __func__, &OmxDataDecoder::DoFlush);
 182 }
 183
 184 RefPtr<MediaDataDecoder::DecodePromise> OmxDataDecoder::Drain() {
 185   LOG("");
 186   MOZ_ASSERT(mThread->IsOnCurrentThread());
 187
 188   RefPtr<OmxDataDecoder> self = this;
 189   return InvokeAsync(mOmxTaskQueue, __func__, [self]() {
 190     RefPtr<DecodePromise> p = self->mDrainPromise.Ensure(__func__);
 191     self->SendEosBuffer();
 192     return p;
 193   });
 194 }
 195
 196 RefPtr<ShutdownPromise> OmxDataDecoder::Shutdown() {
 197   LOG("");
 198   // mThread may not be set if Init hasn't been called first.
 199   MOZ_ASSERT(!mThread || mThread->IsOnCurrentThread());
 200
 201   mShuttingDown = true;
 202
 203   return InvokeAsync(mOmxTaskQueue, this, __func__,
 204                      &OmxDataDecoder::DoAsyncShutdown);
 205 }
 206
 207 RefPtr<ShutdownPromise> OmxDataDecoder::DoAsyncShutdown() {
 208   LOG("");
 209   MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
 210   MOZ_ASSERT(!mFlushing);
 211
 212   mWatchManager.Unwatch(mOmxState, &OmxDataDecoder::OmxStateRunner);
 213   mWatchManager.Unwatch(mPortSettingsChanged,
 214                         &OmxDataDecoder::PortSettingsChanged);
 215
 216   // Flush to all ports, so all buffers can be returned from component.
 217   RefPtr<OmxDataDecoder> self = this;
 218   mOmxLayer->SendCommand(OMX_CommandFlush, OMX_ALL, nullptr)
 219       ->Then(
 220           mOmxTaskQueue, __func__,
 221           [self]() -> RefPtr<OmxCommandPromise> {
 222             LOGL("DoAsyncShutdown: flush complete");
 223             return self->mOmxLayer->SendCommand(OMX_CommandStateSet,
 224                                                 OMX_StateIdle, nullptr);
 225           },
 226           [self](const OmxCommandFailureHolder& aError) {
 227             self->mOmxLayer->Shutdown();
 228             return OmxCommandPromise::CreateAndReject(aError, __func__);
 229           })
 230       ->Then(
 231           mOmxTaskQueue, __func__,
 232           [self]() -> RefPtr<OmxCommandPromise> {
 233             RefPtr<OmxCommandPromise> p = self->mOmxLayer->SendCommand(
 234                 OMX_CommandStateSet, OMX_StateLoaded, nullptr);
 235
 236             // According to spec 3.1.1.2.2.1:
 237             // OMX_StateLoaded needs to be sent before releasing buffers.
 238             // And state transition from OMX_StateIdle to OMX_StateLoaded
 239             // is completed when all of the buffers have been removed
 240             // from the component.
 241             // Here the buffer promises are not resolved due to displaying
 242             // in layer, it needs to wait before the layer returns the
 243             // buffers.
 244             LOGL("DoAsyncShutdown: releasing buffers...");
 245             self->ReleaseBuffers(OMX_DirInput);
 246             self->ReleaseBuffers(OMX_DirOutput);
 247
 248             return p;
 249           },
 250           [self](const OmxCommandFailureHolder& aError) {
 251             self->mOmxLayer->Shutdown();
 252             return OmxCommandPromise::CreateAndReject(aError, __func__);
 253           })
 254       ->Then(
 255           mOmxTaskQueue, __func__,
 256           [self]() -> RefPtr<ShutdownPromise> {
 257             LOGL(
 258                 "DoAsyncShutdown: OMX_StateLoaded, it is safe to shutdown omx");
 259             self->mOmxLayer->Shutdown();
 260             self->mWatchManager.Shutdown();
 261             self->mOmxLayer = nullptr;
 262             self->mMediaDataHelper = nullptr;
 263             self->mShuttingDown = false;
 264             return ShutdownPromise::CreateAndResolve(true, __func__);
 265           },
 266           [self]() -> RefPtr<ShutdownPromise> {
 267             self->mOmxLayer->Shutdown();
 268             self->mWatchManager.Shutdown();
 269             self->mOmxLayer = nullptr;
 270             self->mMediaDataHelper = nullptr;
 271             return ShutdownPromise::CreateAndReject(false, __func__);
 272           })
 273       ->Then(
 274           mThread, __func__,
 275           [self]() {
 276             self->mOmxTaskQueue->BeginShutdown();
 277             self->mOmxTaskQueue->AwaitShutdownAndIdle();
 278             self->mShutdownPromise.Resolve(true, __func__);
 279           },
 280           [self]() {
 281             self->mOmxTaskQueue->BeginShutdown();
 282             self->mOmxTaskQueue->AwaitShutdownAndIdle();
 283             self->mShutdownPromise.Resolve(true, __func__);
 284           });
 285   return mShutdownPromise.Ensure(__func__);
 286 }
 287
 288 void OmxDataDecoder::FillBufferDone(BufferData* aData) {
 289   MOZ_ASSERT(!aData || aData->mStatus == BufferData::BufferStatus::OMX_CLIENT);
 290
 291   // Don't output sample when flush or shutting down, especially for video
 292   // decoded frame. Because video decoded frame can have a promise in
 293   // BufferData waiting for layer to resolve it via recycle callback, if other
 294   // module doesn't send it to layer, it will cause a unresolved promise and
 295   // waiting for resolve infinitely.
 296   if (mFlushing || mShuttingDown) {
 297     LOG("mFlush or mShuttingDown, drop data");
 298     aData->mStatus = BufferData::BufferStatus::FREE;
 299     return;
 300   }
 301
 302   if (aData->mBuffer->nFlags & OMX_BUFFERFLAG_EOS) {
 303     // Reach eos, it's an empty data so it doesn't need to output.
 304     EndOfStream();
 305     aData->mStatus = BufferData::BufferStatus::FREE;
 306   } else {
 307     Output(aData);
 308     FillAndEmptyBuffers();
 309   }
 310 }
 311
 312 void OmxDataDecoder::Output(BufferData* aData) {
 313   if (!mMediaDataHelper) {
 314     mMediaDataHelper =
 315         new MediaDataHelper(mTrackInfo.get(), mImageContainer, mOmxLayer);
 316   }
 317
 318   bool isPlatformData = false;
 319   RefPtr<MediaData> data =
 320       mMediaDataHelper->GetMediaData(aData, isPlatformData);
 321   if (!data) {
 322     aData->mStatus = BufferData::BufferStatus::FREE;
 323     return;
 324   }
 325
 326   if (isPlatformData) {
 327     // If the MediaData is platform dependnet data, it's mostly a kind of
 328     // limited resource, so we use promise to notify when the resource is free.
 329     aData->mStatus = BufferData::BufferStatus::OMX_CLIENT_OUTPUT;
 330
 331     MOZ_RELEASE_ASSERT(aData->mPromise.IsEmpty());
 332     RefPtr<OmxBufferPromise> p = aData->mPromise.Ensure(__func__);
 333
 334     RefPtr<OmxDataDecoder> self = this;
 335     RefPtr<BufferData> buffer = aData;
 336     p->Then(
 337         mOmxTaskQueue, __func__,
 338         [self, buffer]() {
 339           MOZ_RELEASE_ASSERT(buffer->mStatus ==
 340                              BufferData::BufferStatus::OMX_CLIENT_OUTPUT);
 341           buffer->mStatus = BufferData::BufferStatus::FREE;
 342           self->FillAndEmptyBuffers();
 343         },
 344         [buffer]() {
 345           MOZ_RELEASE_ASSERT(buffer->mStatus ==
 346                              BufferData::BufferStatus::OMX_CLIENT_OUTPUT);
 347           buffer->mStatus = BufferData::BufferStatus::FREE;
 348         });
 349   } else {
 350     aData->mStatus = BufferData::BufferStatus::FREE;
 351   }
 352
 353   mDecodedData.AppendElement(std::move(data));
 354 }
 355
 356 void OmxDataDecoder::FillBufferFailure(OmxBufferFailureHolder aFailureHolder) {
 357   NotifyError(aFailureHolder.mError, __func__);
 358 }
 359
 360 void OmxDataDecoder::EmptyBufferDone(BufferData* aData) {
 361   MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
 362   MOZ_ASSERT(!aData || aData->mStatus == BufferData::BufferStatus::OMX_CLIENT);
 363
 364   // Nothing to do when status of input buffer is OMX_CLIENT.
 365   aData->mStatus = BufferData::BufferStatus::FREE;
 366   FillAndEmptyBuffers();
 367
 368   // There is no way to know if component gets enough raw samples to generate
 369   // output, especially for video decoding. So here it needs to request raw
 370   // samples aggressively.
 371   if (!mCheckingInputExhausted && !mMediaRawDatas.Length()) {
 372     mCheckingInputExhausted = true;
 373
 374     RefPtr<OmxDataDecoder> self = this;
 375     nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction(
 376         "OmxDataDecoder::EmptyBufferDone", [self, this]() {
 377           mCheckingInputExhausted = false;
 378
 379           if (mMediaRawDatas.Length()) {
 380             return;
 381           }
 382
 383           mDecodePromise.ResolveIfExists(std::move(mDecodedData), __func__);
 384           mDecodedData = DecodedData();
 385         });
 386
 387     nsresult rv = mOmxTaskQueue->Dispatch(r.forget());
 388     MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
 389     Unused << rv;
 390   }
 391 }
 392
 393 void OmxDataDecoder::EmptyBufferFailure(OmxBufferFailureHolder aFailureHolder) {
 394   NotifyError(aFailureHolder.mError, __func__);
 395 }
 396
 397 void OmxDataDecoder::NotifyError(OMX_ERRORTYPE aOmxError, const char* aLine,
 398                                  const MediaResult& aError) {
 399   LOG("NotifyError %d (%s) at %s", static_cast<int>(aOmxError),
 400       aError.ErrorName().get(), aLine);
 401   mDecodedData = DecodedData();
 402   mDecodePromise.RejectIfExists(aError, __func__);
 403   mDrainPromise.RejectIfExists(aError, __func__);
 404   mFlushPromise.RejectIfExists(aError, __func__);
 405 }
 406
 407 void OmxDataDecoder::FillAndEmptyBuffers() {
 408   MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
 409   MOZ_ASSERT(mOmxState == OMX_StateExecuting);
 410
 411   // During the port setting changed, it is forbidden to do any buffer
 412   // operation.
 413   if (mPortSettingsChanged != -1 || mShuttingDown || mFlushing) {
 414     return;
 415   }
 416
 417   // Trigger input port.
 418   while (!!mMediaRawDatas.Length()) {
 419     // input buffer must be used by component if there is data available.
 420     RefPtr<BufferData> inbuf = FindAvailableBuffer(OMX_DirInput);
 421     if (!inbuf) {
 422       LOG("no input buffer!");
 423       break;
 424     }
 425
 426     RefPtr<MediaRawData> data = mMediaRawDatas[0];
 427     // Buffer size should large enough for raw data.
 428     MOZ_RELEASE_ASSERT(inbuf->mBuffer->nAllocLen >= data->Size());
 429
 430     memcpy(inbuf->mBuffer->pBuffer, data->Data(), data->Size());
 431     inbuf->mBuffer->nFilledLen = data->Size();
 432     inbuf->mBuffer->nOffset = 0;
 433     inbuf->mBuffer->nFlags = inbuf->mBuffer->nAllocLen > data->Size()
 434                                  ? OMX_BUFFERFLAG_ENDOFFRAME
 435                                  : 0;
 436     inbuf->mBuffer->nTimeStamp = data->mTime.ToMicroseconds();
 437     if (data->Size()) {
 438       inbuf->mRawData = mMediaRawDatas[0];
 439     } else {
 440       LOG("send EOS buffer");
 441       inbuf->mBuffer->nFlags |= OMX_BUFFERFLAG_EOS;
 442     }
 443
 444     LOG("feed sample %p to omx component, len %ld, flag %lX", data.get(),
 445         inbuf->mBuffer->nFilledLen, inbuf->mBuffer->nFlags);
 446     mOmxLayer->EmptyBuffer(inbuf)->Then(mOmxTaskQueue, __func__, this,
 447                                         &OmxDataDecoder::EmptyBufferDone,
 448                                         &OmxDataDecoder::EmptyBufferFailure);
 449     mMediaRawDatas.RemoveElementAt(0);
 450   }
 451
 452   // Trigger output port.
 453   while (true) {
 454     RefPtr<BufferData> outbuf = FindAvailableBuffer(OMX_DirOutput);
 455     if (!outbuf) {
 456       break;
 457     }
 458
 459     mOmxLayer->FillBuffer(outbuf)->Then(mOmxTaskQueue, __func__, this,
 460                                         &OmxDataDecoder::FillBufferDone,
 461                                         &OmxDataDecoder::FillBufferFailure);
 462   }
 463 }
 464
 465 OmxPromiseLayer::BufferData* OmxDataDecoder::FindAvailableBuffer(
 466     OMX_DIRTYPE aType) {
 467   BUFFERLIST* buffers = GetBuffers(aType);
 468
 469   for (uint32_t i = 0; i < buffers->Length(); i++) {
 470     BufferData* buf = buffers->ElementAt(i);
 471     if (buf->mStatus == BufferData::BufferStatus::FREE) {
 472       return buf;
 473     }
 474   }
 475
 476   return nullptr;
 477 }
 478
 479 nsresult OmxDataDecoder::AllocateBuffers(OMX_DIRTYPE aType) {
 480   MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
 481
 482   return mOmxLayer->AllocateOmxBuffer(aType, GetBuffers(aType));
 483 }
 484
 485 nsresult OmxDataDecoder::ReleaseBuffers(OMX_DIRTYPE aType) {
 486   MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
 487
 488   return mOmxLayer->ReleaseOmxBuffer(aType, GetBuffers(aType));
 489 }
 490
 491 nsTArray<RefPtr<OmxPromiseLayer::BufferData>>* OmxDataDecoder::GetBuffers(
 492     OMX_DIRTYPE aType) {
 493   MOZ_ASSERT(aType == OMX_DIRTYPE::OMX_DirInput ||
 494              aType == OMX_DIRTYPE::OMX_DirOutput);
 495
 496   if (aType == OMX_DIRTYPE::OMX_DirInput) {
 497     return &mInPortBuffers;
 498   }
 499   return &mOutPortBuffers;
 500 }
 501
 502 void OmxDataDecoder::ResolveInitPromise(const char* aMethodName) {
 503   MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
 504   LOG("called from %s", aMethodName);
 505   mInitPromise.ResolveIfExists(mTrackInfo->GetType(), aMethodName);
 506 }
 507
 508 void OmxDataDecoder::RejectInitPromise(MediaResult aError,
 509                                        const char* aMethodName) {
 510   MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
 511   mInitPromise.RejectIfExists(aError, aMethodName);
 512 }
 513
 514 void OmxDataDecoder::OmxStateRunner() {
 515   MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
 516   LOG("OMX state: %s", StateTypeToStr(mOmxState));
 517
 518   // TODO: maybe it'd be better to use promise CompletionPromise() to replace
 519   //       this state machine.
 520   if (mOmxState == OMX_StateLoaded) {
 521     ConfigCodec();
 522
 523     // Send OpenMax state command to OMX_StateIdle.
 524     RefPtr<OmxDataDecoder> self = this;
 525     mOmxLayer->SendCommand(OMX_CommandStateSet, OMX_StateIdle, nullptr)
 526         ->Then(
 527             mOmxTaskQueue, __func__,
 528             [self]() {
 529               // Current state should be OMX_StateIdle.
 530               self->mOmxState = self->mOmxLayer->GetState();
 531               MOZ_ASSERT(self->mOmxState == OMX_StateIdle);
 532             },
 533             [self]() {
 534               self->RejectInitPromise(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
 535             });
 536
 537     // Allocate input and output buffers.
 538     OMX_DIRTYPE types[] = {OMX_DIRTYPE::OMX_DirInput,
 539                            OMX_DIRTYPE::OMX_DirOutput};
 540     for (const auto id : types) {
 541       if (NS_FAILED(AllocateBuffers(id))) {
 542         LOG("Failed to allocate buffer on port %d", id);
 543         RejectInitPromise(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
 544         break;
 545       }
 546     }
 547   } else if (mOmxState == OMX_StateIdle) {
 548     RefPtr<OmxDataDecoder> self = this;
 549     mOmxLayer->SendCommand(OMX_CommandStateSet, OMX_StateExecuting, nullptr)
 550         ->Then(
 551             mOmxTaskQueue, __func__,
 552             [self]() {
 553               self->mOmxState = self->mOmxLayer->GetState();
 554               MOZ_ASSERT(self->mOmxState == OMX_StateExecuting);
 555
 556               self->ResolveInitPromise(__func__);
 557             },
 558             [self]() {
 559               self->RejectInitPromise(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
 560             });
 561   } else if (mOmxState == OMX_StateExecuting) {
 562     // Configure codec once it gets OMX_StateExecuting state.
 563     FillCodecConfigDataToOmx();
 564   } else {
 565     MOZ_ASSERT(0);
 566   }
 567 }
 568
 569 void OmxDataDecoder::ConfigCodec() {
 570   OMX_ERRORTYPE err = mOmxLayer->Config();
 571   CHECK_OMX_ERR(err);
 572 }
 573
 574 void OmxDataDecoder::FillCodecConfigDataToOmx() {
 575   // Codec configure data should be the first sample running on Omx TaskQueue.
 576   MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
 577   MOZ_ASSERT(!mMediaRawDatas.Length());
 578   MOZ_ASSERT(mOmxState == OMX_StateIdle || mOmxState == OMX_StateExecuting);
 579
 580   RefPtr<BufferData> inbuf = FindAvailableBuffer(OMX_DirInput);
 581   RefPtr<MediaByteBuffer> csc;
 582   if (mTrackInfo->IsAudio()) {
 583     // It would be nice to instead use more specific information here, but
 584     // we force a byte buffer for now since this handles arbitrary codecs.
 585     // TODO(bug 1768566): implement further type checking for codec data.
 586     csc = ForceGetAudioCodecSpecificBlob(
 587         mTrackInfo->GetAsAudioInfo()->mCodecSpecificConfig);
 588   } else if (mTrackInfo->IsVideo()) {
 589     csc = mTrackInfo->GetAsVideoInfo()->mExtraData;
 590   }
 591
 592   MOZ_RELEASE_ASSERT(csc);
 593
 594   // Some codecs like h264, its codec specific data is at the first packet, not
 595   // in container.
 596   if (csc->Length()) {
 597     // Buffer size should large enough for raw data.
 598     MOZ_RELEASE_ASSERT(inbuf->mBuffer->nAllocLen >= csc->Length());
 599
 600     memcpy(inbuf->mBuffer->pBuffer, csc->Elements(), csc->Length());
 601     inbuf->mBuffer->nFilledLen = csc->Length();
 602     inbuf->mBuffer->nOffset = 0;
 603     inbuf->mBuffer->nFlags =
 604         (OMX_BUFFERFLAG_ENDOFFRAME | OMX_BUFFERFLAG_CODECCONFIG);
 605
 606     LOG("Feed codec configure data to OMX component");
 607     mOmxLayer->EmptyBuffer(inbuf)->Then(mOmxTaskQueue, __func__, this,
 608                                         &OmxDataDecoder::EmptyBufferDone,
 609                                         &OmxDataDecoder::EmptyBufferFailure);
 610   }
 611 }
 612
 613 bool OmxDataDecoder::Event(OMX_EVENTTYPE aEvent, OMX_U32 aData1,
 614                            OMX_U32 aData2) {
 615   MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
 616
 617   if (mOmxLayer->Event(aEvent, aData1, aData2)) {
 618     return true;
 619   }
 620
 621   switch (aEvent) {
 622     case OMX_EventPortSettingsChanged: {
 623       // Don't always disable port. See bug 1235340.
 624       if (aData2 == 0 || aData2 == OMX_IndexParamPortDefinition) {
 625         // According to spec: "To prevent the loss of any input data, the
 626         // component issuing the OMX_EventPortSettingsChanged event on its input
 627         // port should buffer all input port data that arrives between the
 628         // emission of the OMX_EventPortSettingsChanged event and the arrival of
 629         // the command to disable the input port."
 630         //
 631         // So client needs to disable port and reallocate buffers.
 632         MOZ_ASSERT(mPortSettingsChanged == -1);
 633         mPortSettingsChanged = aData1;
 634       }
 635       LOG("Got OMX_EventPortSettingsChanged event");
 636       break;
 637     }
 638     default: {
 639       // Got error during decoding, send msg to MFR skipping to next key frame.
 640       if (aEvent == OMX_EventError && mOmxState == OMX_StateExecuting) {
 641         NotifyError((OMX_ERRORTYPE)aData1, __func__,
 642                     MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, __func__));
 643         return true;
 644       }
 645       LOG("WARNING: got none handle event: %d, aData1: %ld, aData2: %ld",
 646           aEvent, aData1, aData2);
 647       return false;
 648     }
 649   }
 650
 651   return true;
 652 }
 653
 654 bool OmxDataDecoder::BuffersCanBeReleased(OMX_DIRTYPE aType) {
 655   BUFFERLIST* buffers = GetBuffers(aType);
 656   uint32_t len = buffers->Length();
 657   for (uint32_t i = 0; i < len; i++) {
 658     BufferData::BufferStatus buf_status = buffers->ElementAt(i)->mStatus;
 659     if (buf_status == BufferData::BufferStatus::OMX_COMPONENT ||
 660         buf_status == BufferData::BufferStatus::OMX_CLIENT_OUTPUT) {
 661       return false;
 662     }
 663   }
 664   return true;
 665 }
 666
 667 OMX_DIRTYPE
 668 OmxDataDecoder::GetPortDirection(uint32_t aPortIndex) {
 669   OMX_PARAM_PORTDEFINITIONTYPE def;
 670   InitOmxParameter(&def);
 671   def.nPortIndex = mPortSettingsChanged;
 672
 673   OMX_ERRORTYPE err =
 674       mOmxLayer->GetParameter(OMX_IndexParamPortDefinition, &def, sizeof(def));
 675   if (err != OMX_ErrorNone) {
 676     return OMX_DirMax;
 677   }
 678   return def.eDir;
 679 }
 680
 681 RefPtr<OmxPromiseLayer::OmxBufferPromise::AllPromiseType>
 682 OmxDataDecoder::CollectBufferPromises(OMX_DIRTYPE aType) {
 683   MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
 684
 685   nsTArray<RefPtr<OmxBufferPromise>> promises;
 686   OMX_DIRTYPE types[] = {OMX_DIRTYPE::OMX_DirInput, OMX_DIRTYPE::OMX_DirOutput};
 687   for (const auto type : types) {
 688     if ((aType == type) || (aType == OMX_DirMax)) {
 689       // find the buffer which has promise.
 690       BUFFERLIST* buffers = GetBuffers(type);
 691
 692       for (uint32_t i = 0; i < buffers->Length(); i++) {
 693         BufferData* buf = buffers->ElementAt(i);
 694         if (!buf->mPromise.IsEmpty()) {
 695           // OmxBufferPromise is not exclusive, it can be multiple "Then"s, so
 696           // it is safe to call "Ensure" here.
 697           promises.AppendElement(buf->mPromise.Ensure(__func__));
 698         }
 699       }
 700     }
 701   }
 702
 703   LOG("CollectBufferPromises: type %d, total %zu promiese", aType,
 704       promises.Length());
 705   if (promises.Length()) {
 706     return OmxBufferPromise::All(mOmxTaskQueue, promises);
 707   }
 708
 709   return OmxBufferPromise::AllPromiseType::CreateAndResolve(
 710       nsTArray<BufferData*>(), __func__);
 711 }
 712
 713 void OmxDataDecoder::PortSettingsChanged() {
 714   MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
 715
 716   if (mPortSettingsChanged == -1 ||
 717       mOmxState == OMX_STATETYPE::OMX_StateInvalid) {
 718     return;
 719   }
 720
 721   // The PortSettingsChanged algorithm:
 722   //
 723   //   1. disable port.
 724   //   2. wait for port buffers return to client and then release these buffers.
 725   //   3. enable port.
 726   //   4. allocate port buffers.
 727   //
 728
 729   // Disable port. Get port definition if the target port is enable.
 730   OMX_PARAM_PORTDEFINITIONTYPE def;
 731   InitOmxParameter(&def);
 732   def.nPortIndex = mPortSettingsChanged;
 733
 734   OMX_ERRORTYPE err =
 735       mOmxLayer->GetParameter(OMX_IndexParamPortDefinition, &def, sizeof(def));
 736   CHECK_OMX_ERR(err);
 737
 738   RefPtr<OmxDataDecoder> self = this;
 739   if (def.bEnabled) {
 740     // 1. disable port.
 741     LOG("PortSettingsChanged: disable port %lu", def.nPortIndex);
 742     mOmxLayer
 743         ->SendCommand(OMX_CommandPortDisable, mPortSettingsChanged, nullptr)
 744         ->Then(
 745             mOmxTaskQueue, __func__,
 746             [self, def]() -> RefPtr<OmxCommandPromise> {
 747               // 3. enable port.
 748               // Send enable port command.
 749               RefPtr<OmxCommandPromise> p = self->mOmxLayer->SendCommand(
 750                   OMX_CommandPortEnable, self->mPortSettingsChanged, nullptr);
 751
 752               // 4. allocate port buffers.
 753               // Allocate new port buffers.
 754               nsresult rv = self->AllocateBuffers(def.eDir);
 755               if (NS_FAILED(rv)) {
 756                 self->NotifyError(OMX_ErrorUndefined, __func__);
 757               }
 758
 759               return p;
 760             },
 761             [self](const OmxCommandFailureHolder& aError) {
 762               self->NotifyError(OMX_ErrorUndefined, __func__);
 763               return OmxCommandPromise::CreateAndReject(aError, __func__);
 764             })
 765         ->Then(
 766             mOmxTaskQueue, __func__,
 767             [self]() {
 768               LOGL("PortSettingsChanged: port settings changed complete");
 769               // finish port setting changed.
 770               self->mPortSettingsChanged = -1;
 771               self->FillAndEmptyBuffers();
 772             },
 773             [self]() { self->NotifyError(OMX_ErrorUndefined, __func__); });
 774
 775     // 2. wait for port buffers return to client and then release these buffers.
 776     //
 777     // Port buffers will be returned to client soon once OMX_CommandPortDisable
 778     // command is sent. Then releasing these buffers.
 779     CollectBufferPromises(def.eDir)->Then(
 780         mOmxTaskQueue, __func__,
 781         [self, def]() {
 782           MOZ_ASSERT(self->BuffersCanBeReleased(def.eDir));
 783           nsresult rv = self->ReleaseBuffers(def.eDir);
 784           if (NS_FAILED(rv)) {
 785             MOZ_RELEASE_ASSERT(0);
 786             self->NotifyError(OMX_ErrorUndefined, __func__);
 787           }
 788         },
 789         [self]() { self->NotifyError(OMX_ErrorUndefined, __func__); });
 790   }
 791 }
 792
 793 void OmxDataDecoder::SendEosBuffer() {
 794   MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
 795
 796   // There is no 'Drain' API in OpenMax, so it needs to wait for output sample
 797   // with EOS flag. However, MediaRawData doesn't provide EOS information,
 798   // so here it generates an empty BufferData with eos OMX_BUFFERFLAG_EOS in
 799   // queue. This behaviour should be compliant with spec, I think...
 800   RefPtr<MediaRawData> eos_data = new MediaRawData();
 801   mMediaRawDatas.AppendElement(eos_data);
 802   FillAndEmptyBuffers();
 803 }
 804
 805 RefPtr<MediaDataDecoder::FlushPromise> OmxDataDecoder::DoFlush() {
 806   MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
 807
 808   mDecodedData = DecodedData();
 809   mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
 810   mDrainPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
 811
 812   RefPtr<FlushPromise> p = mFlushPromise.Ensure(__func__);
 813
 814   // 1. Call OMX command OMX_CommandFlush in Omx TaskQueue.
 815   // 2. Remove all elements in mMediaRawDatas when flush is completed.
 816   mOmxLayer->SendCommand(OMX_CommandFlush, OMX_ALL, nullptr)
 817       ->Then(mOmxTaskQueue, __func__, this, &OmxDataDecoder::FlushComplete,
 818              &OmxDataDecoder::FlushFailure);
 819
 820   return p;
 821 }
 822
 823 void OmxDataDecoder::FlushComplete(OMX_COMMANDTYPE aCommandType) {
 824   mMediaRawDatas.Clear();
 825   mFlushing = false;
 826
 827   LOG("Flush complete");
 828   mFlushPromise.ResolveIfExists(true, __func__);
 829 }
 830
 831 void OmxDataDecoder::FlushFailure(OmxCommandFailureHolder aFailureHolder) {
 832   mFlushing = false;
 833   mFlushPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
 834 }
 835
 836 MediaDataHelper::MediaDataHelper(const TrackInfo* aTrackInfo,
 837                                  layers::ImageContainer* aImageContainer,
 838                                  OmxPromiseLayer* aOmxLayer)
 839     : mTrackInfo(aTrackInfo),
 840       mAudioCompactor(mAudioQueue),
 841       mImageContainer(aImageContainer) {
 842   InitOmxParameter(&mOutputPortDef);
 843   mOutputPortDef.nPortIndex = aOmxLayer->OutputPortIndex();
 844   aOmxLayer->GetParameter(OMX_IndexParamPortDefinition, &mOutputPortDef,
 845                           sizeof(mOutputPortDef));
 846 }
 847
 848 already_AddRefed<MediaData> MediaDataHelper::GetMediaData(
 849     BufferData* aBufferData, bool& aPlatformDepenentData) {
 850   aPlatformDepenentData = false;
 851   RefPtr<MediaData> data;
 852
 853   if (mTrackInfo->IsAudio()) {
 854     if (!aBufferData->mBuffer->nFilledLen) {
 855       return nullptr;
 856     }
 857     data = CreateAudioData(aBufferData);
 858   } else if (mTrackInfo->IsVideo()) {
 859     data = aBufferData->GetPlatformMediaData();
 860     if (data) {
 861       aPlatformDepenentData = true;
 862     } else {
 863       if (!aBufferData->mBuffer->nFilledLen) {
 864         return nullptr;
 865       }
 866       // Get YUV VideoData, it uses more CPU, in most cases, on software codec.
 867       data = CreateYUV420VideoData(aBufferData);
 868     }
 869
 870     // Update video time code, duration... from the raw data.
 871     VideoData* video(data->As<VideoData>());
 872     if (aBufferData->mRawData) {
 873       video->mTime = aBufferData->mRawData->mTime;
 874       video->mTimecode = aBufferData->mRawData->mTimecode;
 875       video->mOffset = aBufferData->mRawData->mOffset;
 876       video->mDuration = aBufferData->mRawData->mDuration;
 877       video->mKeyframe = aBufferData->mRawData->mKeyframe;
 878     }
 879   }
 880
 881   return data.forget();
 882 }
 883
 884 already_AddRefed<AudioData> MediaDataHelper::CreateAudioData(
 885     BufferData* aBufferData) {
 886   RefPtr<AudioData> audio;
 887   OMX_BUFFERHEADERTYPE* buf = aBufferData->mBuffer;
 888   const AudioInfo* info = mTrackInfo->GetAsAudioInfo();
 889   if (buf->nFilledLen) {
 890     uint64_t offset = 0;
 891     uint32_t frames = buf->nFilledLen / (2 * info->mChannels);
 892     if (aBufferData->mRawData) {
 893       offset = aBufferData->mRawData->mOffset;
 894     }
 895     typedef AudioCompactor::NativeCopy OmxCopy;
 896     mAudioCompactor.Push(
 897         offset, buf->nTimeStamp, info->mRate, frames, info->mChannels,
 898         OmxCopy(buf->pBuffer + buf->nOffset, buf->nFilledLen, info->mChannels));
 899     audio = mAudioQueue.PopFront();
 900   }
 901
 902   return audio.forget();
 903 }
 904
 905 already_AddRefed<VideoData> MediaDataHelper::CreateYUV420VideoData(
 906     BufferData* aBufferData) {
 907   uint8_t* yuv420p_buffer = (uint8_t*)aBufferData->mBuffer->pBuffer;
 908   int32_t stride = mOutputPortDef.format.video.nStride;
 909   int32_t slice_height = mOutputPortDef.format.video.nSliceHeight;
 910   int32_t width = mTrackInfo->GetAsVideoInfo()->mImage.width;
 911   int32_t height = mTrackInfo->GetAsVideoInfo()->mImage.height;
 912
 913   // TODO: convert other formats to YUV420.
 914   if (mOutputPortDef.format.video.eColorFormat !=
 915       OMX_COLOR_FormatYUV420Planar) {
 916     return nullptr;
 917   }
 918
 919   size_t yuv420p_y_size = stride * slice_height;
 920   size_t yuv420p_u_size = ((stride + 1) / 2) * ((slice_height + 1) / 2);
 921   uint8_t* yuv420p_y = yuv420p_buffer;
 922   uint8_t* yuv420p_u = yuv420p_y + yuv420p_y_size;
 923   uint8_t* yuv420p_v = yuv420p_u + yuv420p_u_size;
 924
 925   VideoData::YCbCrBuffer b;
 926   b.mPlanes[0].mData = yuv420p_y;
 927   b.mPlanes[0].mWidth = width;
 928   b.mPlanes[0].mHeight = height;
 929   b.mPlanes[0].mStride = stride;
 930   b.mPlanes[0].mSkip = 0;
 931
 932   b.mPlanes[1].mData = yuv420p_u;
 933   b.mPlanes[1].mWidth = (width + 1) / 2;
 934   b.mPlanes[1].mHeight = (height + 1) / 2;
 935   b.mPlanes[1].mStride = (stride + 1) / 2;
 936   b.mPlanes[1].mSkip = 0;
 937
 938   b.mPlanes[2].mData = yuv420p_v;
 939   b.mPlanes[2].mWidth = (width + 1) / 2;
 940   b.mPlanes[2].mHeight = (height + 1) / 2;
 941   b.mPlanes[2].mStride = (stride + 1) / 2;
 942   b.mPlanes[2].mSkip = 0;
 943
 944   b.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH_AND_HEIGHT;
 945
 946   VideoInfo info(*mTrackInfo->GetAsVideoInfo());
 947
 948   auto maybeColorSpace = info.mColorSpace;
 949   if (!maybeColorSpace) {
 950     maybeColorSpace = Some(DefaultColorSpace({width, height}));
 951   }
 952   b.mYUVColorSpace = *maybeColorSpace;
 953
 954   auto maybeColorPrimaries = info.mColorPrimaries;
 955   if (!maybeColorPrimaries) {
 956     maybeColorPrimaries = Some(gfx::ColorSpace2::BT709);
 957   }
 958   b.mColorPrimaries = *maybeColorPrimaries;
 959
 960   RefPtr<VideoData> data = VideoData::CreateAndCopyData(
 961       info, mImageContainer,
 962       0,                                     // Filled later by caller.
 963       media::TimeUnit::Zero(),               // Filled later by caller.
 964       media::TimeUnit::FromMicroseconds(1),  // We don't know the duration.
 965       b,
 966       0,  // Filled later by caller.
 967       media::TimeUnit::FromMicroseconds(-1), info.ImageRect(), nullptr);
 968
 969   MOZ_LOG(sPDMLog, mozilla::LogLevel::Debug,
 970           ("YUV420 VideoData: disp width %d, height %d, pic width %d, height "
 971            "%d, time %lld",
 972            info.mDisplay.width, info.mDisplay.height, info.mImage.width,
 973            info.mImage.height, aBufferData->mBuffer->nTimeStamp));
 974
 975   return data.forget();
 976 }
 977
 978 }  // namespace mozilla