widget/TextRecognition.cpp

   1 /* This Source Code Form is subject to the terms of the Mozilla Public
   2  * License, v. 2.0. If a copy of the MPL was not distributed with this
   3  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   4
   5 #include "TextRecognition.h"
   6 #include "mozilla/dom/Promise.h"
   7 #include "mozilla/dom/ShadowRoot.h"
   8 #include "mozilla/dom/Document.h"
   9 #include "mozilla/dom/ContentChild.h"
  10 #include "nsTextNode.h"
  11 #include "imgIContainer.h"
  12
  13 #ifdef XP_MACOSX
  14 #  include "nsCocoaFeatures.h"
  15 #endif
  16
  17 using namespace mozilla::dom;
  18
  19 namespace mozilla::widget {
  20
  21 auto TextRecognition::FindText(imgIContainer& aImage,
  22                                const nsTArray<nsCString>& aLanguages)
  23     -> RefPtr<NativePromise> {
  24   // TODO: Maybe decode async.
  25   RefPtr<gfx::SourceSurface> surface = aImage.GetFrame(
  26       imgIContainer::FRAME_CURRENT,
  27       imgIContainer::FLAG_SYNC_DECODE | imgIContainer::FLAG_ASYNC_NOTIFY);
  28   if (NS_WARN_IF(!surface)) {
  29     return NativePromise::CreateAndReject("Failed to get surface"_ns, __func__);
  30   }
  31   RefPtr<gfx::DataSourceSurface> dataSurface = surface->GetDataSurface();
  32   if (NS_WARN_IF(!dataSurface)) {
  33     return NativePromise::CreateAndReject("Failed to get data surface"_ns,
  34                                           __func__);
  35   }
  36   return FindText(*dataSurface, aLanguages);
  37 }
  38
  39 auto TextRecognition::FindText(gfx::DataSourceSurface& aSurface,
  40                                const nsTArray<nsCString>& aLanguages)
  41     -> RefPtr<NativePromise> {
  42   if (XRE_IsContentProcess()) {
  43     auto* contentChild = ContentChild::GetSingleton();
  44     auto image = nsContentUtils::SurfaceToIPCImage(aSurface);
  45     if (!image) {
  46       return NativePromise::CreateAndReject("Failed to share data surface"_ns,
  47                                             __func__);
  48     }
  49     auto promise = MakeRefPtr<NativePromise::Private>(__func__);
  50     contentChild->SendFindImageText(std::move(*image), aLanguages)
  51         ->Then(
  52             GetCurrentSerialEventTarget(), __func__,
  53             [promise](TextRecognitionResultOrError&& aResultOrError) {
  54               switch (aResultOrError.type()) {
  55                 case TextRecognitionResultOrError::Type::TTextRecognitionResult:
  56                   promise->Resolve(
  57                       std::move(aResultOrError.get_TextRecognitionResult()),
  58                       __func__);
  59                   break;
  60                 case TextRecognitionResultOrError::Type::TnsCString:
  61                   promise->Reject(std::move(aResultOrError.get_nsCString()),
  62                                   __func__);
  63                   break;
  64                 default:
  65                   MOZ_ASSERT_UNREACHABLE("Unknown result?");
  66                   promise->Reject("Unknown error"_ns, __func__);
  67                   break;
  68               }
  69             },
  70             [promise](mozilla::ipc::ResponseRejectReason) {
  71               promise->Reject("IPC rejection"_ns, __func__);
  72             });
  73     return promise;
  74   }
  75   return DoFindText(aSurface, aLanguages);
  76 }
  77
  78 void TextRecognition::FillShadow(ShadowRoot& aShadow,
  79                                  const TextRecognitionResult& aResult) {
  80   auto& doc = *aShadow.OwnerDoc();
  81   RefPtr<Element> div = doc.CreateHTMLElement(nsGkAtoms::div);
  82   for (const auto& quad : aResult.quads()) {
  83     RefPtr<Element> span = doc.CreateHTMLElement(nsGkAtoms::span);
  84     // TODO: We probably want to position them here and so on. For now, expose
  85     // the data as attributes so that it's easy to play with the returned values
  86     // in JS.
  87     {
  88       nsAutoString points;
  89       for (const auto& point : quad.points()) {
  90         points.AppendFloat(point.x);
  91         points.Append(u',');
  92         points.AppendFloat(point.y);
  93         points.Append(u',');
  94       }
  95       points.Trim(",");
  96       span->SetAttribute(u"data-points"_ns, points, IgnoreErrors());
  97       nsAutoString confidence;
  98       confidence.AppendFloat(quad.confidence());
  99       span->SetAttribute(u"data-confidence"_ns, confidence, IgnoreErrors());
 100     }
 101
 102     {
 103       RefPtr<nsTextNode> text = doc.CreateTextNode(quad.string());
 104       span->AppendChildTo(text, true, IgnoreErrors());
 105     }
 106     div->AppendChildTo(span, true, IgnoreErrors());
 107   }
 108   aShadow.AppendChildTo(div, true, IgnoreErrors());
 109 }
 110
 111 #ifndef XP_MACOSX
 112 auto TextRecognition::DoFindText(gfx::DataSourceSurface&,
 113                                  const nsTArray<nsCString>&)
 114     -> RefPtr<NativePromise> {
 115   MOZ_RELEASE_ASSERT(XRE_IsParentProcess(),
 116                      "This should only run in the parent process");
 117   return NativePromise::CreateAndReject("Text recognition not available"_ns,
 118                                         __func__);
 119 }
 120 #endif
 121
 122 bool TextRecognition::IsSupported() {
 123 #ifdef XP_MACOSX
 124   // Catalina (10.15) or higher is required because of the following API:
 125   // VNRecognizeTextRequest - macOS 10.15+
 126   // https://developer.apple.com/documentation/vision/vnrecognizetextrequest?language=objc
 127   return nsCocoaFeatures::OnCatalinaOrLater();
 128 #else
 129   return false;
 130 #endif
 131 }
 132
 133 }  // namespace mozilla::widget