widget/TextRecognition.cpp

   1 /* This Source Code Form is subject to the terms of the Mozilla Public
   2  * License, v. 2.0. If a copy of the MPL was not distributed with this
   3  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   4
   5 #include "TextRecognition.h"
   6 #include "mozilla/dom/Promise.h"
   7 #include "mozilla/dom/ShadowRoot.h"
   8 #include "mozilla/dom/Document.h"
   9 #include "mozilla/dom/ContentChild.h"
  10 #include "nsTextNode.h"
  11 #include "imgIContainer.h"
  12
  13 using namespace mozilla::dom;
  14
  15 namespace mozilla::widget {
  16
  17 auto TextRecognition::FindText(imgIContainer& aImage,
  18                                const nsTArray<nsCString>& aLanguages)
  19     -> RefPtr<NativePromise> {
  20   // TODO: Maybe decode async.
  21   RefPtr<gfx::SourceSurface> surface = aImage.GetFrame(
  22       imgIContainer::FRAME_CURRENT,
  23       imgIContainer::FLAG_SYNC_DECODE | imgIContainer::FLAG_ASYNC_NOTIFY);
  24   if (NS_WARN_IF(!surface)) {
  25     return NativePromise::CreateAndReject("Failed to get surface"_ns, __func__);
  26   }
  27   RefPtr<gfx::DataSourceSurface> dataSurface = surface->GetDataSurface();
  28   if (NS_WARN_IF(!dataSurface)) {
  29     return NativePromise::CreateAndReject("Failed to get data surface"_ns,
  30                                           __func__);
  31   }
  32   return FindText(*dataSurface, aLanguages);
  33 }
  34
  35 auto TextRecognition::FindText(gfx::DataSourceSurface& aSurface,
  36                                const nsTArray<nsCString>& aLanguages)
  37     -> RefPtr<NativePromise> {
  38   if (!IsSupported()) {
  39     return NativePromise::CreateAndReject("Text recognition not available"_ns,
  40                                           __func__);
  41   }
  42
  43   if (XRE_IsContentProcess()) {
  44     auto* contentChild = ContentChild::GetSingleton();
  45     auto image = nsContentUtils::SurfaceToIPCImage(aSurface);
  46     if (!image) {
  47       return NativePromise::CreateAndReject("Failed to share data surface"_ns,
  48                                             __func__);
  49     }
  50     auto promise = MakeRefPtr<NativePromise::Private>(__func__);
  51     contentChild->SendFindImageText(std::move(*image), aLanguages)
  52         ->Then(
  53             GetCurrentSerialEventTarget(), __func__,
  54             [promise](TextRecognitionResultOrError&& aResultOrError) {
  55               switch (aResultOrError.type()) {
  56                 case TextRecognitionResultOrError::Type::TTextRecognitionResult:
  57                   promise->Resolve(
  58                       std::move(aResultOrError.get_TextRecognitionResult()),
  59                       __func__);
  60                   break;
  61                 case TextRecognitionResultOrError::Type::TnsCString:
  62                   promise->Reject(std::move(aResultOrError.get_nsCString()),
  63                                   __func__);
  64                   break;
  65                 default:
  66                   MOZ_ASSERT_UNREACHABLE("Unknown result?");
  67                   promise->Reject("Unknown error"_ns, __func__);
  68                   break;
  69               }
  70             },
  71             [promise](mozilla::ipc::ResponseRejectReason) {
  72               promise->Reject("IPC rejection"_ns, __func__);
  73             });
  74     return promise;
  75   }
  76   return DoFindText(aSurface, aLanguages);
  77 }
  78
  79 void TextRecognition::FillShadow(ShadowRoot& aShadow,
  80                                  const TextRecognitionResult& aResult) {
  81   auto& doc = *aShadow.OwnerDoc();
  82   RefPtr<Element> div = doc.CreateHTMLElement(nsGkAtoms::div);
  83   for (const auto& quad : aResult.quads()) {
  84     RefPtr<Element> span = doc.CreateHTMLElement(nsGkAtoms::span);
  85     // TODO: We probably want to position them here and so on. For now, expose
  86     // the data as attributes so that it's easy to play with the returned values
  87     // in JS.
  88     {
  89       nsAutoString points;
  90       for (const auto& point : quad.points()) {
  91         points.AppendFloat(point.x);
  92         points.Append(u',');
  93         points.AppendFloat(point.y);
  94         points.Append(u',');
  95       }
  96       points.Trim(",");
  97       span->SetAttribute(u"data-points"_ns, points, IgnoreErrors());
  98       nsAutoString confidence;
  99       confidence.AppendFloat(quad.confidence());
 100       span->SetAttribute(u"data-confidence"_ns, confidence, IgnoreErrors());
 101     }
 102
 103     {
 104       RefPtr<nsTextNode> text = doc.CreateTextNode(quad.string());
 105       span->AppendChildTo(text, true, IgnoreErrors());
 106     }
 107     div->AppendChildTo(span, true, IgnoreErrors());
 108   }
 109   aShadow.AppendChildTo(div, true, IgnoreErrors());
 110 }
 111
 112 #ifndef XP_MACOSX
 113 auto TextRecognition::DoFindText(gfx::DataSourceSurface&,
 114                                  const nsTArray<nsCString>&)
 115     -> RefPtr<NativePromise> {
 116   MOZ_CRASH("DoFindText is not implemented on this platform");
 117 }
 118 #endif
 119
 120 bool TextRecognition::IsSupported() {
 121 #ifdef XP_MACOSX
 122   return true;
 123 #else
 124   return false;
 125 #endif
 126 }
 127
 128 }  // namespace mozilla::widget