widget/TextRecognition.cpp

   1 /* This Source Code Form is subject to the terms of the Mozilla Public
   2  * License, v. 2.0. If a copy of the MPL was not distributed with this
   3  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   4
   5 #include "TextRecognition.h"
   6 #include "mozilla/dom/Promise.h"
   7 #include "mozilla/dom/ShadowRoot.h"
   8 #include "mozilla/dom/Document.h"
   9 #include "mozilla/dom/ContentChild.h"
  10 #include "nsTextNode.h"
  11 #include "imgIContainer.h"
  12
  13 using namespace mozilla::dom;
  14
  15 namespace mozilla::widget {
  16
  17 auto TextRecognition::FindText(imgIContainer& aImage,
  18                                const nsTArray<nsCString>& aLanguages)
  19     -> RefPtr<NativePromise> {
  20   // TODO: Maybe decode async.
  21   RefPtr<gfx::SourceSurface> surface = aImage.GetFrame(
  22       imgIContainer::FRAME_CURRENT,
  23       imgIContainer::FLAG_SYNC_DECODE | imgIContainer::FLAG_ASYNC_NOTIFY);
  24   if (NS_WARN_IF(!surface)) {
  25     return NativePromise::CreateAndReject("Failed to get surface"_ns, __func__);
  26   }
  27   RefPtr<gfx::DataSourceSurface> dataSurface = surface->GetDataSurface();
  28   if (NS_WARN_IF(!dataSurface)) {
  29     return NativePromise::CreateAndReject("Failed to get data surface"_ns,
  30                                           __func__);
  31   }
  32   return FindText(*dataSurface, aLanguages);
  33 }
  34
  35 auto TextRecognition::FindText(gfx::DataSourceSurface& aSurface,
  36                                const nsTArray<nsCString>& aLanguages)
  37     -> RefPtr<NativePromise> {
  38   if (XRE_IsContentProcess()) {
  39     auto* contentChild = ContentChild::GetSingleton();
  40     auto image = nsContentUtils::SurfaceToIPCImage(aSurface);
  41     if (!image) {
  42       return NativePromise::CreateAndReject("Failed to share data surface"_ns,
  43                                             __func__);
  44     }
  45     auto promise = MakeRefPtr<NativePromise::Private>(__func__);
  46     contentChild->SendFindImageText(std::move(*image), aLanguages)
  47         ->Then(
  48             GetCurrentSerialEventTarget(), __func__,
  49             [promise](TextRecognitionResultOrError&& aResultOrError) {
  50               switch (aResultOrError.type()) {
  51                 case TextRecognitionResultOrError::Type::TTextRecognitionResult:
  52                   promise->Resolve(
  53                       std::move(aResultOrError.get_TextRecognitionResult()),
  54                       __func__);
  55                   break;
  56                 case TextRecognitionResultOrError::Type::TnsCString:
  57                   promise->Reject(std::move(aResultOrError.get_nsCString()),
  58                                   __func__);
  59                   break;
  60                 default:
  61                   MOZ_ASSERT_UNREACHABLE("Unknown result?");
  62                   promise->Reject("Unknown error"_ns, __func__);
  63                   break;
  64               }
  65             },
  66             [promise](mozilla::ipc::ResponseRejectReason) {
  67               promise->Reject("IPC rejection"_ns, __func__);
  68             });
  69     return promise;
  70   }
  71   return DoFindText(aSurface, aLanguages);
  72 }
  73
  74 void TextRecognition::FillShadow(ShadowRoot& aShadow,
  75                                  const TextRecognitionResult& aResult) {
  76   auto& doc = *aShadow.OwnerDoc();
  77   RefPtr<Element> div = doc.CreateHTMLElement(nsGkAtoms::div);
  78   for (const auto& quad : aResult.quads()) {
  79     RefPtr<Element> span = doc.CreateHTMLElement(nsGkAtoms::span);
  80     // TODO: We probably want to position them here and so on. For now, expose
  81     // the data as attributes so that it's easy to play with the returned values
  82     // in JS.
  83     {
  84       nsAutoString points;
  85       for (const auto& point : quad.points()) {
  86         points.AppendFloat(point.x);
  87         points.Append(u',');
  88         points.AppendFloat(point.y);
  89         points.Append(u',');
  90       }
  91       points.Trim(",");
  92       span->SetAttribute(u"data-points"_ns, points, IgnoreErrors());
  93       nsAutoString confidence;
  94       confidence.AppendFloat(quad.confidence());
  95       span->SetAttribute(u"data-confidence"_ns, confidence, IgnoreErrors());
  96     }
  97
  98     {
  99       RefPtr<nsTextNode> text = doc.CreateTextNode(quad.string());
 100       span->AppendChildTo(text, true, IgnoreErrors());
 101     }
 102     div->AppendChildTo(span, true, IgnoreErrors());
 103   }
 104   aShadow.AppendChildTo(div, true, IgnoreErrors());
 105 }
 106
 107 #ifndef XP_MACOSX
 108 auto TextRecognition::DoFindText(gfx::DataSourceSurface&,
 109                                  const nsTArray<nsCString>&)
 110     -> RefPtr<NativePromise> {
 111   MOZ_RELEASE_ASSERT(XRE_IsParentProcess(),
 112                      "This should only run in the parent process");
 113   return NativePromise::CreateAndReject("Text recognition not available"_ns,
 114                                         __func__);
 115 }
 116 #endif
 117
 118 bool TextRecognition::IsSupported() {
 119 #ifdef XP_MACOSX
 120   return true;
 121 #else
 122   return false;
 123 #endif
 124 }
 125
 126 }  // namespace mozilla::widget