widget/cocoa/TextRecognition.mm

   1 /* This Source Code Form is subject to the terms of the Mozilla Public
   2  * License, v. 2.0. If a copy of the MPL was not distributed with this
   3  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   4
   5 #import <Vision/Vision.h>
   6
   7 #include "mozilla/dom/Promise.h"
   8 #include "mozilla/gfx/2D.h"
   9 #include "mozilla/ErrorResult.h"
  10 #include "ErrorList.h"
  11 #include "nsClipboard.h"
  12 #include "nsCocoaUtils.h"
  13 #include "mozilla/MacStringHelpers.h"
  14 #include "mozilla/ScopeExit.h"
  15 #include "mozilla/widget/TextRecognition.h"
  16 #include "mozilla/dom/PContent.h"
  17
  18 namespace mozilla::widget {
  19
  20 auto TextRecognition::DoFindText(gfx::DataSourceSurface& aSurface) -> RefPtr<NativePromise> {
  21   NS_OBJC_BEGIN_TRY_IGNORE_BLOCK
  22   if (@available(macOS 10.15, *)) {
  23     // TODO - Is this the most efficient path? Maybe we can write a new
  24     // CreateCGImageFromXXX that enables more efficient marshalling of the data.
  25     CGImageRef imageRef = NULL;
  26     nsresult rv = nsCocoaUtils::CreateCGImageFromSurface(&aSurface, &imageRef);
  27     if (NS_FAILED(rv) || !imageRef) {
  28       return NativePromise::CreateAndReject("Failed to create CGImage"_ns, __func__);
  29     }
  30
  31     auto promise = MakeRefPtr<NativePromise::Private>(__func__);
  32
  33     NS_DispatchBackgroundTask(
  34         NS_NewRunnableFunction(
  35             __func__,
  36             [promise, imageRef] {
  37               auto unrefImage = MakeScopeExit([&] { ::CGImageRelease(imageRef); });
  38
  39               dom::TextRecognitionResult result;
  40               dom::TextRecognitionResult* pResult = &result;
  41
  42               // Define the request to use, which also handles the result. It will be run below
  43               // directly in this thread. After creating this request.
  44               VNRecognizeTextRequest* textRecognitionRequest = [[VNRecognizeTextRequest alloc]
  45                   initWithCompletionHandler:^(VNRequest* _Nonnull request,
  46                                               NSError* _Nullable error) {
  47                     NSArray<VNRecognizedTextObservation*>* observations = request.results;
  48
  49                     [observations
  50                         enumerateObjectsUsingBlock:^(VNRecognizedTextObservation* _Nonnull obj,
  51                                                      NSUInteger idx, BOOL* _Nonnull stop) {
  52                           // Requests the n top candidates for a recognized text string.
  53                           VNRecognizedText* recognizedText = [obj topCandidates:1].firstObject;
  54
  55                           // https://developer.apple.com/documentation/vision/vnrecognizedtext?language=objc
  56                           auto& quad = *pResult->quads().AppendElement();
  57                           CopyCocoaStringToXPCOMString(recognizedText.string, quad.string());
  58                           quad.confidence() = recognizedText.confidence;
  59
  60                           auto ToImagePoint = [](CGPoint aPoint) -> ImagePoint {
  61                             return {static_cast<float>(aPoint.x), static_cast<float>(aPoint.y)};
  62                           };
  63                           *quad.points().AppendElement() = ToImagePoint(obj.bottomLeft);
  64                           *quad.points().AppendElement() = ToImagePoint(obj.topLeft);
  65                           *quad.points().AppendElement() = ToImagePoint(obj.topRight);
  66                           *quad.points().AppendElement() = ToImagePoint(obj.bottomRight);
  67                         }];
  68                   }];
  69
  70               // Send out the request. This blocks execution of this thread with an expensive
  71               // CPU call.
  72               NSError* error = nil;
  73               VNImageRequestHandler* requestHandler =
  74                   [[[VNImageRequestHandler alloc] initWithCGImage:imageRef
  75                                                           options:@{}] autorelease];
  76
  77               [requestHandler performRequests:@[ textRecognitionRequest ] error:&error];
  78               if (error != nil) {
  79                 promise->Reject(
  80                     nsPrintfCString("Failed to perform text recognition request (%ld)\n",
  81                                     error.code),
  82                     __func__);
  83               } else {
  84                 promise->Resolve(std::move(result), __func__);
  85               }
  86             }),
  87         NS_DISPATCH_EVENT_MAY_BLOCK);
  88     return promise;
  89   } else {
  90     return NativePromise::CreateAndReject("Text recognition is not available"_ns, __func__);
  91   }
  92
  93   NS_OBJC_END_TRY_IGNORE_BLOCK
  94 }
  95
  96 }  // namespace mozilla::widget