dom/media/webcodecs/EncoderTemplate.h

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #ifndef mozilla_dom_EncoderTemplate_h
   8 #define mozilla_dom_EncoderTemplate_h
   9
  10 #include <queue>
  11
  12 #include "EncoderAgent.h"
  13 #include "MediaData.h"
  14 #include "WebCodecsUtils.h"
  15 #include "mozilla/DOMEventTargetHelper.h"
  16 #include "mozilla/MozPromise.h"
  17 #include "mozilla/RefPtr.h"
  18 #include "mozilla/Result.h"
  19 #include "mozilla/UniquePtr.h"
  20 #include "mozilla/dom/VideoEncoderBinding.h"
  21 #include "mozilla/dom/AudioEncoderBinding.h"
  22 #include "mozilla/dom/WorkerRef.h"
  23 #include "mozilla/media/MediaUtils.h"
  24 #include "nsStringFwd.h"
  25
  26 namespace mozilla::dom {
  27
  28 class WebCodecsErrorCallback;
  29 class Promise;
  30 enum class CodecState : uint8_t;
  31
  32 using Id = size_t;
  33
  34 template <typename EncoderType>
  35 class EncoderTemplate : public DOMEventTargetHelper {
  36   using Self = EncoderTemplate<EncoderType>;
  37   using ConfigType = typename EncoderType::ConfigType;
  38   using ConfigTypeInternal = typename EncoderType::ConfigTypeInternal;
  39   using OutputConfigType = typename EncoderType::OutputConfigType;
  40   using InputType = typename EncoderType::InputType;
  41   using InputTypeInternal = typename EncoderType::InputTypeInternal;
  42   using OutputType = typename EncoderType::OutputType;
  43   using OutputCallbackType = typename EncoderType::OutputCallbackType;
  44
  45   /* ControlMessage classes */
  46  protected:
  47   class ConfigureMessage;
  48   class EncodeMessage;
  49   class FlushMessage;
  50
  51   class ControlMessage {
  52    public:
  53     NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ControlMessage)
  54     explicit ControlMessage(Id aConfigureId);
  55     virtual void Cancel() = 0;
  56     virtual bool IsProcessing() = 0;
  57
  58     virtual nsCString ToString() const = 0;
  59     virtual RefPtr<ConfigureMessage> AsConfigureMessage() { return nullptr; }
  60     virtual RefPtr<EncodeMessage> AsEncodeMessage() { return nullptr; }
  61     virtual RefPtr<FlushMessage> AsFlushMessage() { return nullptr; }
  62
  63     // For logging purposes
  64     const WebCodecsId mConfigureId;
  65     const WebCodecsId mMessageId;
  66
  67    protected:
  68     virtual ~ControlMessage() = default;
  69   };
  70
  71   class ConfigureMessage final
  72       : public ControlMessage,
  73         public MessageRequestHolder<EncoderAgent::ConfigurePromise> {
  74    public:
  75     ConfigureMessage(Id aConfigureId,
  76                      const RefPtr<ConfigTypeInternal>& aConfig);
  77     virtual void Cancel() override { Disconnect(); }
  78     virtual bool IsProcessing() override { return Exists(); };
  79     virtual RefPtr<ConfigureMessage> AsConfigureMessage() override {
  80       return this;
  81     }
  82     RefPtr<ConfigTypeInternal> Config() { return mConfig; }
  83     nsCString ToString() const override {
  84       nsCString rv;
  85       rv.AppendPrintf("ConfigureMessage(#%zu): %s", this->mMessageId,
  86                       mConfig ? mConfig->ToString().get() : "null cfg");
  87       return rv;
  88     }
  89
  90    private:
  91     const RefPtr<ConfigTypeInternal> mConfig;
  92   };
  93
  94   class EncodeMessage final
  95       : public ControlMessage,
  96         public MessageRequestHolder<EncoderAgent::EncodePromise> {
  97    public:
  98     EncodeMessage(WebCodecsId aConfigureId, RefPtr<InputTypeInternal>&& aData,
  99                   Maybe<VideoEncoderEncodeOptions>&& aOptions = Nothing());
 100     nsCString ToString() const override {
 101       nsCString rv;
 102       bool isKeyFrame = mOptions.isSome() && mOptions.ref().mKeyFrame;
 103       rv.AppendPrintf("EncodeMessage(#%zu,#%zu): %s (%s)", this->mConfigureId,
 104                       this->mMessageId, mData->ToString().get(),
 105                       isKeyFrame ? "kf" : "");
 106       return rv;
 107     }
 108     virtual void Cancel() override { Disconnect(); }
 109     virtual bool IsProcessing() override { return Exists(); };
 110     virtual RefPtr<EncodeMessage> AsEncodeMessage() override { return this; }
 111     RefPtr<InputTypeInternal> mData;
 112     Maybe<VideoEncoderEncodeOptions> mOptions;
 113   };
 114
 115   class FlushMessage final
 116       : public ControlMessage,
 117         public MessageRequestHolder<EncoderAgent::EncodePromise> {
 118    public:
 119     FlushMessage(WebCodecsId aConfigureId, Promise* aPromise);
 120     virtual void Cancel() override { Disconnect(); }
 121     virtual bool IsProcessing() override { return Exists(); };
 122     virtual RefPtr<FlushMessage> AsFlushMessage() override { return this; }
 123     already_AddRefed<Promise> TakePromise() { return mPromise.forget(); }
 124     void RejectPromiseIfAny(const nsresult& aReason);
 125
 126     nsCString ToString() const override {
 127       nsCString rv;
 128       rv.AppendPrintf("FlushMessage(#%zu,#%zu)", this->mConfigureId,
 129                       this->mMessageId);
 130       return rv;
 131     }
 132
 133    private:
 134     RefPtr<Promise> mPromise;
 135   };
 136
 137  protected:
 138   EncoderTemplate(nsIGlobalObject* aGlobalObject,
 139                   RefPtr<WebCodecsErrorCallback>&& aErrorCallback,
 140                   RefPtr<OutputCallbackType>&& aOutputCallback);
 141
 142   virtual ~EncoderTemplate() = default;
 143
 144   /* WebCodecs interfaces */
 145  public:
 146   IMPL_EVENT_HANDLER(dequeue)
 147
 148   void StartBlockingMessageQueue();
 149   void StopBlockingMessageQueue();
 150
 151   MOZ_CAN_RUN_SCRIPT
 152   void OutputEncodedData(const nsTArray<RefPtr<MediaRawData>>&& aData);
 153
 154   CodecState State() const { return mState; };
 155
 156   uint32_t EncodeQueueSize() const { return mEncodeQueueSize; };
 157
 158   MOZ_CAN_RUN_SCRIPT
 159   void Configure(const ConfigType& aConfig, ErrorResult& aRv);
 160
 161   void EncodeAudioData(InputType& aInput, ErrorResult& aRv);
 162   void EncodeVideoFrame(InputType& aInput,
 163                         const VideoEncoderEncodeOptions& aOptions,
 164                         ErrorResult& aRv);
 165
 166   already_AddRefed<Promise> Flush(ErrorResult& aRv);
 167
 168   void Reset(ErrorResult& aRv);
 169
 170   MOZ_CAN_RUN_SCRIPT
 171   void Close(ErrorResult& aRv);
 172
 173   /* Type conversion functions for the Encoder implementation */
 174  protected:
 175   virtual RefPtr<OutputType> EncodedDataToOutputType(
 176       nsIGlobalObject* aGlobalObject, const RefPtr<MediaRawData>& aData) = 0;
 177   virtual OutputConfigType EncoderConfigToDecoderConfig(
 178       nsIGlobalObject* aGlobalObject, const RefPtr<MediaRawData>& aData,
 179       const ConfigTypeInternal& aOutputConfig) const = 0;
 180   template <typename T, typename U>
 181   void CopyExtradataToDescriptionIfNeeded(nsIGlobalObject* aGlobal,
 182                                           const T& aConfigInternal, U& aConfig);
 183   /* Internal member variables and functions */
 184  protected:
 185   // EncoderTemplate can run on either main thread or worker thread.
 186   void AssertIsOnOwningThread() const {
 187     NS_ASSERT_OWNINGTHREAD(EncoderTemplate);
 188   }
 189
 190   Result<Ok, nsresult> ResetInternal(const nsresult& aResult);
 191   MOZ_CAN_RUN_SCRIPT
 192   Result<Ok, nsresult> CloseInternalWithAbort();
 193   MOZ_CAN_RUN_SCRIPT
 194   void CloseInternal(const nsresult& aResult);
 195
 196   MOZ_CAN_RUN_SCRIPT void ReportError(const nsresult& aResult);
 197
 198   MOZ_CAN_RUN_SCRIPT void OutputEncodedVideoData(
 199       const nsTArray<RefPtr<MediaRawData>>&& aData);
 200   MOZ_CAN_RUN_SCRIPT void OutputEncodedAudioData(
 201       const nsTArray<RefPtr<MediaRawData>>&& aData);
 202
 203   void ScheduleDequeueEvent();
 204   nsresult FireEvent(nsAtom* aTypeWithOn, const nsAString& aEventType);
 205
 206   void SchedulePromiseResolveOrReject(already_AddRefed<Promise> aPromise,
 207                                       const nsresult& aResult);
 208
 209   void ProcessControlMessageQueue();
 210   void CancelPendingControlMessages(const nsresult& aResult);
 211
 212   template <typename Func>
 213   void QueueATask(const char* aName, Func&& aSteps);
 214
 215   MessageProcessedResult ProcessConfigureMessage(
 216       RefPtr<ConfigureMessage> aMessage);
 217
 218   MessageProcessedResult ProcessEncodeMessage(RefPtr<EncodeMessage> aMessage);
 219
 220   MessageProcessedResult ProcessFlushMessage(RefPtr<FlushMessage> aMessage);
 221
 222   void Configure(RefPtr<ConfigureMessage> aMessage);
 223   void Reconfigure(RefPtr<ConfigureMessage> aMessage);
 224
 225   // Returns true when mAgent can be created.
 226   bool CreateEncoderAgent(WebCodecsId aId, RefPtr<ConfigTypeInternal> aConfig);
 227   void DestroyEncoderAgentIfAny();
 228
 229   // Constant in practice, only set in ctor.
 230   RefPtr<WebCodecsErrorCallback> mErrorCallback;
 231   RefPtr<OutputCallbackType> mOutputCallback;
 232
 233   CodecState mState;
 234
 235   bool mMessageQueueBlocked;
 236   std::queue<RefPtr<ControlMessage>> mControlMessageQueue;
 237   RefPtr<ControlMessage> mProcessingMessage;
 238
 239   uint32_t mEncodeQueueSize;
 240   bool mDequeueEventScheduled;
 241
 242   // A unique id tracking the ConfigureMessage and will be used as the
 243   // EncoderAgent's Id.
 244   uint32_t mLatestConfigureId;
 245   // Tracking how many encoded data has been enqueued and this number will be
 246   // used as the EncodeMessage's Id.
 247   size_t mEncodeCounter;
 248   // Tracking how many flush request has been enqueued and this number will be
 249   // used as the FlushMessage's Id.
 250   size_t mFlushCounter;
 251
 252   // EncoderAgent will be created the first time "configure" is being
 253   // processed, and will be destroyed when "reset" is called. If another
 254   // "configure" is called, either it's possible to reconfigure the underlying
 255   // encoder without tearing everything down (e.g. a bitrate change), or it's
 256   // not possible, and the current encoder will be destroyed and a new one
 257   // create. In both cases, the encoder is implicitely flushed before the
 258   // configuration change. See CanReconfigure on the
 259   // {Audio,Video}EncoderConfigInternal
 260   RefPtr<EncoderAgent> mAgent;
 261   RefPtr<ConfigTypeInternal> mActiveConfig;
 262   // This is true when a configure call has just been processed, and it's
 263   // necessary to pass the new decoding configuration when the callback is
 264   // called. Read and modified on owner thread only.
 265   bool mOutputNewDecoderConfig = false;
 266
 267   // Used to add a nsIAsyncShutdownBlocker on main thread to block
 268   // xpcom-shutdown before the underlying MediaDataEncoder is created. The
 269   // blocker will be held until the underlying MediaDataEncoder has been shut
 270   // down. This blocker guarantees RemoteEncoderManagerChild's thread, where
 271   // the underlying RemoteMediaDataEncoder is on, outlives the
 272   // RemoteMediaDataEncoder since the thread releasing, which happens on main
 273   // thread when getting a xpcom-shutdown signal, is blocked by the added
 274   // blocker. As a result, RemoteMediaDataEncoder can safely work on worker
 275   // thread with a holding blocker (otherwise, if RemoteEncoderManagerChild
 276   // releases its thread on main thread before RemoteMediaDataEncoder's
 277   // Shutdown() task run on worker thread, RemoteMediaDataEncoder has no
 278   // thread to run).
 279   UniquePtr<media::ShutdownBlockingTicket> mShutdownBlocker;
 280
 281   // Held to make sure the dispatched tasks can be done before worker is going
 282   // away. As long as this worker-ref is held somewhere, the tasks dispatched
 283   // to the worker can be executed (otherwise the tasks would be canceled).
 284   // This ref should be activated as long as the underlying MediaDataEncoder
 285   // is alive, and should keep alive until mShutdownBlocker is dropped, so all
 286   // MediaDataEncoder's tasks and mShutdownBlocker-releasing task can be
 287   // executed.
 288   // TODO: Use StrongWorkerRef instead if this is always used in the same
 289   // thread?
 290   RefPtr<ThreadSafeWorkerRef> mWorkerRef;
 291   uint64_t mPacketsOutput = 0;
 292 };
 293
 294 }  // namespace mozilla::dom
 295
 296 #endif  // mozilla_dom_EncoderTemplate_h