widget/cocoa/TextRecognition.mm

   1 /* This Source Code Form is subject to the terms of the Mozilla Public
   2  * License, v. 2.0. If a copy of the MPL was not distributed with this
   3  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   4
   5 #import <Vision/Vision.h>
   6
   7 #include "mozilla/dom/Promise.h"
   8 #include "mozilla/gfx/2D.h"
   9 #include "mozilla/ErrorResult.h"
  10 #include "ErrorList.h"
  11 #include "nsClipboard.h"
  12 #include "nsCocoaUtils.h"
  13 #include "mozilla/MacStringHelpers.h"
  14 #include "mozilla/ScopeExit.h"
  15 #include "mozilla/widget/TextRecognition.h"
  16 #include "mozilla/dom/PContent.h"
  17
  18 namespace mozilla::widget {
  19
  20 auto TextRecognition::DoFindText(gfx::DataSourceSurface& aSurface,
  21                                  const nsTArray<nsCString>& aLanguages)
  22     -> RefPtr<NativePromise> {
  23   NS_OBJC_BEGIN_TRY_IGNORE_BLOCK
  24
  25   // TODO - Is this the most efficient path? Maybe we can write a new
  26   // CreateCGImageFromXXX that enables more efficient marshalling of the data.
  27   CGImageRef imageRef = NULL;
  28   nsresult rv = nsCocoaUtils::CreateCGImageFromSurface(&aSurface, &imageRef);
  29   if (NS_FAILED(rv) || !imageRef) {
  30     return NativePromise::CreateAndReject("Failed to create CGImage"_ns,
  31                                           __func__);
  32   }
  33
  34   auto promise = MakeRefPtr<NativePromise::Private>(__func__);
  35
  36   NSMutableArray* recognitionLanguages = [[NSMutableArray alloc] init];
  37   for (const auto& locale : aLanguages) {
  38     [recognitionLanguages addObject:nsCocoaUtils::ToNSString(locale)];
  39   }
  40
  41   NS_DispatchBackgroundTask(
  42       NS_NewRunnableFunction(
  43           __func__,
  44           [promise, imageRef, recognitionLanguages] {
  45             auto unrefImage = MakeScopeExit([&] {
  46               ::CGImageRelease(imageRef);
  47               [recognitionLanguages release];
  48             });
  49
  50             dom::TextRecognitionResult result;
  51             dom::TextRecognitionResult* pResult = &result;
  52
  53             // Define the request to use, which also handles the result. It will
  54             // be run below directly in this thread. After creating this
  55             // request.
  56             VNRecognizeTextRequest* textRecognitionRequest =
  57                 [[VNRecognizeTextRequest alloc] initWithCompletionHandler:^(
  58                                                     VNRequest* _Nonnull request,
  59                                                     NSError* _Nullable error) {
  60                   NSArray<VNRecognizedTextObservation*>* observations =
  61                       request.results;
  62
  63                   [observations enumerateObjectsUsingBlock:^(
  64                                     VNRecognizedTextObservation* _Nonnull obj,
  65                                     NSUInteger idx, BOOL* _Nonnull stop) {
  66                     // Requests the n top candidates for a recognized text
  67                     // string.
  68                     VNRecognizedText* recognizedText =
  69                         [obj topCandidates:1].firstObject;
  70
  71                     // https://developer.apple.com/documentation/vision/vnrecognizedtext?language=objc
  72                     auto& quad = *pResult->quads().AppendElement();
  73                     CopyNSStringToXPCOMString(recognizedText.string,
  74                                               quad.string());
  75                     quad.confidence() = recognizedText.confidence;
  76
  77                     auto ToImagePoint = [](CGPoint aPoint) -> ImagePoint {
  78                       return {static_cast<float>(aPoint.x),
  79                               static_cast<float>(aPoint.y)};
  80                     };
  81                     *quad.points().AppendElement() =
  82                         ToImagePoint(obj.bottomLeft);
  83                     *quad.points().AppendElement() = ToImagePoint(obj.topLeft);
  84                     *quad.points().AppendElement() = ToImagePoint(obj.topRight);
  85                     *quad.points().AppendElement() =
  86                         ToImagePoint(obj.bottomRight);
  87                   }];
  88                 }];
  89
  90             textRecognitionRequest.recognitionLevel =
  91                 VNRequestTextRecognitionLevelAccurate;
  92             textRecognitionRequest.recognitionLanguages = recognitionLanguages;
  93             textRecognitionRequest.usesLanguageCorrection = true;
  94
  95             // Send out the request. This blocks execution of this thread with
  96             // an expensive CPU call.
  97             NSError* error = nil;
  98             VNImageRequestHandler* requestHandler =
  99                 [[[VNImageRequestHandler alloc] initWithCGImage:imageRef
 100                                                         options:@{}]
 101                     autorelease];
 102
 103             [requestHandler performRequests:@[ textRecognitionRequest ]
 104                                       error:&error];
 105             if (error != nil) {
 106               promise->Reject(
 107                   nsPrintfCString(
 108                       "Failed to perform text recognition request (%ld)\n",
 109                       error.code),
 110                   __func__);
 111             } else {
 112               promise->Resolve(std::move(result), __func__);
 113             }
 114           }),
 115       NS_DISPATCH_EVENT_MAY_BLOCK);
 116   return promise;
 117
 118   NS_OBJC_END_TRY_IGNORE_BLOCK
 119 }
 120
 121 }  // namespace mozilla::widget