1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
5 #include "TextRecognition.h"
6 #include "mozilla/dom/Promise.h"
7 #include "mozilla/dom/ShadowRoot.h"
8 #include "mozilla/dom/Document.h"
9 #include "mozilla/dom/ContentChild.h"
10 #include "nsTextNode.h"
11 #include "imgIContainer.h"
13 using namespace mozilla::dom
;
15 namespace mozilla::widget
{
17 auto TextRecognition::FindText(imgIContainer
& aImage
) -> RefPtr
<NativePromise
> {
18 // TODO: Maybe decode async.
19 RefPtr
<gfx::SourceSurface
> surface
= aImage
.GetFrame(
20 imgIContainer::FRAME_CURRENT
,
21 imgIContainer::FLAG_SYNC_DECODE
| imgIContainer::FLAG_ASYNC_NOTIFY
);
22 if (NS_WARN_IF(!surface
)) {
23 return NativePromise::CreateAndReject("Failed to get surface"_ns
, __func__
);
25 RefPtr
<gfx::DataSourceSurface
> dataSurface
= surface
->GetDataSurface();
26 if (NS_WARN_IF(!dataSurface
)) {
27 return NativePromise::CreateAndReject("Failed to get data surface"_ns
,
30 return FindText(*dataSurface
);
33 auto TextRecognition::FindText(gfx::DataSourceSurface
& aSurface
)
34 -> RefPtr
<NativePromise
> {
35 if (XRE_IsContentProcess()) {
36 auto* contentChild
= ContentChild::GetSingleton();
37 auto image
= nsContentUtils::SurfaceToIPCImage(aSurface
, contentChild
);
39 return NativePromise::CreateAndReject("Failed to share data surface"_ns
,
42 auto promise
= MakeRefPtr
<NativePromise::Private
>(__func__
);
43 contentChild
->SendFindImageText(*image
)->Then(
44 GetCurrentSerialEventTarget(), __func__
,
45 [promise
](TextRecognitionResultOrError
&& aResultOrError
) {
46 switch (aResultOrError
.type()) {
47 case TextRecognitionResultOrError::Type::TTextRecognitionResult
:
49 std::move(aResultOrError
.get_TextRecognitionResult()),
52 case TextRecognitionResultOrError::Type::TnsCString
:
53 promise
->Reject(std::move(aResultOrError
.get_nsCString()),
57 MOZ_ASSERT_UNREACHABLE("Unknown result?");
58 promise
->Reject("Unknown error"_ns
, __func__
);
62 [promise
](mozilla::ipc::ResponseRejectReason
) {
63 promise
->Reject("IPC rejection"_ns
, __func__
);
67 return DoFindText(aSurface
);
70 void TextRecognition::FillShadow(ShadowRoot
& aShadow
,
71 const TextRecognitionResult
& aResult
) {
72 auto& doc
= *aShadow
.OwnerDoc();
73 RefPtr
<Element
> div
= doc
.CreateHTMLElement(nsGkAtoms::div
);
74 for (const auto& quad
: aResult
.quads()) {
75 RefPtr
<Element
> span
= doc
.CreateHTMLElement(nsGkAtoms::span
);
76 // TODO: We probably want to position them here and so on. For now, expose
77 // the data as attributes so that it's easy to play with the returned values
81 for (auto& point
: quad
.points()) {
82 points
.AppendFloat(point
.x
);
84 points
.AppendFloat(point
.y
);
88 span
->SetAttribute(u
"data-points"_ns
, points
, IgnoreErrors());
89 nsAutoString confidence
;
90 confidence
.AppendFloat(quad
.confidence());
91 span
->SetAttribute(u
"data-confidence"_ns
, confidence
, IgnoreErrors());
95 RefPtr
<nsTextNode
> text
= doc
.CreateTextNode(quad
.string());
96 span
->AppendChildTo(text
, true, IgnoreErrors());
98 div
->AppendChildTo(span
, true, IgnoreErrors());
100 aShadow
.AppendChildTo(div
, true, IgnoreErrors());
103 auto TextRecognition::DoFindText(gfx::DataSourceSurface
&)
104 -> RefPtr
<NativePromise
> {
105 MOZ_RELEASE_ASSERT(XRE_IsParentProcess(),
106 "This should only run in the parent process");
107 return NativePromise::CreateAndReject("Text recognition not available"_ns
,
111 } // namespace mozilla::widget