widget/TextRecognition.cpp

   1 /* This Source Code Form is subject to the terms of the Mozilla Public
   2  * License, v. 2.0. If a copy of the MPL was not distributed with this
   3  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   4
   5 #include "TextRecognition.h"
   6 #include "mozilla/dom/Promise.h"
   7 #include "mozilla/dom/ShadowRoot.h"
   8 #include "mozilla/dom/Document.h"
   9 #include "mozilla/dom/ContentChild.h"
  10 #include "nsTextNode.h"
  11 #include "imgIContainer.h"
  12
  13 using namespace mozilla::dom;
  14
  15 namespace mozilla::widget {
  16
  17 auto TextRecognition::FindText(imgIContainer& aImage) -> RefPtr<NativePromise> {
  18   // TODO: Maybe decode async.
  19   RefPtr<gfx::SourceSurface> surface = aImage.GetFrame(
  20       imgIContainer::FRAME_CURRENT,
  21       imgIContainer::FLAG_SYNC_DECODE | imgIContainer::FLAG_ASYNC_NOTIFY);
  22   if (NS_WARN_IF(!surface)) {
  23     return NativePromise::CreateAndReject("Failed to get surface"_ns, __func__);
  24   }
  25   RefPtr<gfx::DataSourceSurface> dataSurface = surface->GetDataSurface();
  26   if (NS_WARN_IF(!dataSurface)) {
  27     return NativePromise::CreateAndReject("Failed to get data surface"_ns,
  28                                           __func__);
  29   }
  30   return FindText(*dataSurface);
  31 }
  32
  33 auto TextRecognition::FindText(gfx::DataSourceSurface& aSurface)
  34     -> RefPtr<NativePromise> {
  35   if (XRE_IsContentProcess()) {
  36     auto* contentChild = ContentChild::GetSingleton();
  37     auto image = nsContentUtils::SurfaceToIPCImage(aSurface, contentChild);
  38     if (!image) {
  39       return NativePromise::CreateAndReject("Failed to share data surface"_ns,
  40                                             __func__);
  41     }
  42     auto promise = MakeRefPtr<NativePromise::Private>(__func__);
  43     contentChild->SendFindImageText(*image)->Then(
  44         GetCurrentSerialEventTarget(), __func__,
  45         [promise](TextRecognitionResultOrError&& aResultOrError) {
  46           switch (aResultOrError.type()) {
  47             case TextRecognitionResultOrError::Type::TTextRecognitionResult:
  48               promise->Resolve(
  49                   std::move(aResultOrError.get_TextRecognitionResult()),
  50                   __func__);
  51               break;
  52             case TextRecognitionResultOrError::Type::TnsCString:
  53               promise->Reject(std::move(aResultOrError.get_nsCString()),
  54                               __func__);
  55               break;
  56             default:
  57               MOZ_ASSERT_UNREACHABLE("Unknown result?");
  58               promise->Reject("Unknown error"_ns, __func__);
  59               break;
  60           }
  61         },
  62         [promise](mozilla::ipc::ResponseRejectReason) {
  63           promise->Reject("IPC rejection"_ns, __func__);
  64         });
  65     return promise;
  66   }
  67   return DoFindText(aSurface);
  68 }
  69
  70 void TextRecognition::FillShadow(ShadowRoot& aShadow,
  71                                  const TextRecognitionResult& aResult) {
  72   auto& doc = *aShadow.OwnerDoc();
  73   RefPtr<Element> div = doc.CreateHTMLElement(nsGkAtoms::div);
  74   for (const auto& quad : aResult.quads()) {
  75     RefPtr<Element> span = doc.CreateHTMLElement(nsGkAtoms::span);
  76     // TODO: We probably want to position them here and so on. For now, expose
  77     // the data as attributes so that it's easy to play with the returned values
  78     // in JS.
  79     {
  80       nsAutoString points;
  81       for (auto& point : quad.points()) {
  82         points.AppendFloat(point.x);
  83         points.Append(u',');
  84         points.AppendFloat(point.y);
  85         points.Append(u',');
  86       }
  87       points.Trim(",");
  88       span->SetAttribute(u"data-points"_ns, points, IgnoreErrors());
  89       nsAutoString confidence;
  90       confidence.AppendFloat(quad.confidence());
  91       span->SetAttribute(u"data-confidence"_ns, confidence, IgnoreErrors());
  92     }
  93
  94     {
  95       RefPtr<nsTextNode> text = doc.CreateTextNode(quad.string());
  96       span->AppendChildTo(text, true, IgnoreErrors());
  97     }
  98     div->AppendChildTo(span, true, IgnoreErrors());
  99   }
 100   aShadow.AppendChildTo(div, true, IgnoreErrors());
 101 }
 102
 103 auto TextRecognition::DoFindText(gfx::DataSourceSurface&)
 104     -> RefPtr<NativePromise> {
 105   MOZ_RELEASE_ASSERT(XRE_IsParentProcess(),
 106                      "This should only run in the parent process");
 107   return NativePromise::CreateAndReject("Text recognition not available"_ns,
 108                                         __func__);
 109 }
 110
 111 }  // namespace mozilla::widget