1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this
4 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 #include "mozilla/CheckedInt.h"
7 #include "mozilla/intl/UnicodeProperties.h"
8 #include "mozilla/intl/WordBreaker.h"
9 #include "mozilla/StaticPrefs_layout.h"
10 #include "nsComplexBreaker.h"
12 #include "nsUnicodeProperties.h"
14 #if defined(MOZ_ICU4X) && defined(JS_HAS_INTL_API)
15 # include "ICU4XDataProvider.h"
16 # include "ICU4XWordBreakIteratorUtf16.hpp"
17 # include "ICU4XWordSegmenter.hpp"
18 # include "mozilla/intl/ICU4XGeckoDataProvider.h"
19 # include "mozilla/StaticPrefs_intl.h"
20 # include "nsUnicharUtils.h"
23 using mozilla::intl::Script
;
24 using mozilla::intl::UnicodeProperties
;
25 using mozilla::intl::WordBreaker
;
26 using mozilla::intl::WordRange
;
27 using mozilla::unicode::GetGenCategory
;
29 #define ASCII_IS_ALPHA(c) \
30 ((('a' <= (c)) && ((c) <= 'z')) || (('A' <= (c)) && ((c) <= 'Z')))
31 #define ASCII_IS_DIGIT(c) (('0' <= (c)) && ((c) <= '9'))
32 #define ASCII_IS_SPACE(c) \
33 ((' ' == (c)) || ('\t' == (c)) || ('\r' == (c)) || ('\n' == (c)))
34 #define IS_ALPHABETICAL_SCRIPT(c) ((c) < 0x2E80)
36 // we change the beginning of IS_HAN from 0x4e00 to 0x3400 to relfect
39 ((0x3400 <= (c)) && ((c) <= 0x9fff)) || ((0xf900 <= (c)) && ((c) <= 0xfaff))
40 #define IS_KATAKANA(c) ((0x30A0 <= (c)) && ((c) <= 0x30FF))
41 #define IS_HIRAGANA(c) ((0x3040 <= (c)) && ((c) <= 0x309F))
42 #define IS_HALFWIDTHKATAKANA(c) ((0xFF60 <= (c)) && ((c) <= 0xFF9F))
44 // Return true if aChar belongs to a SEAsian script that is written without
45 // word spaces, so we need to use the "complex breaker" to find possible word
46 // boundaries. (https://en.wikipedia.org/wiki/Scriptio_continua)
47 // (How well this works depends on the level of platform support for finding
48 // possible line breaks - or possible word boundaries - in the particular
49 // script. Thai, at least, works pretty well on the major desktop OSes. If
50 // the script is not supported by the platform, we just won't find any useful
52 static bool IsScriptioContinua(char16_t aChar
) {
53 Script sc
= UnicodeProperties::GetScriptCode(aChar
);
54 return sc
== Script::THAI
|| sc
== Script::MYANMAR
|| sc
== Script::KHMER
||
55 sc
== Script::JAVANESE
|| sc
== Script::BALINESE
||
56 sc
== Script::SUNDANESE
|| sc
== Script::LAO
;
60 WordBreaker::WordBreakClass
WordBreaker::GetClass(char16_t c
) {
63 if (IS_ALPHABETICAL_SCRIPT(c
)) {
65 if (ASCII_IS_SPACE(c
)) {
68 if (ASCII_IS_ALPHA(c
) || ASCII_IS_DIGIT(c
) ||
69 (c
== '_' && !StaticPrefs::layout_word_select_stop_at_underscore())) {
70 return kWbClassAlphaLetter
;
74 if (c
== 0x00A0 /*NBSP*/) {
77 if (GetGenCategory(c
) == nsUGenCategory::kPunctuation
) {
80 if (IsScriptioContinua(c
)) {
81 return kWbClassScriptioContinua
;
83 return kWbClassAlphaLetter
;
86 return kWbClassHanLetter
;
89 return kWbClassKatakanaLetter
;
92 return kWbClassHiraganaLetter
;
94 if (IS_HALFWIDTHKATAKANA(c
)) {
95 return kWbClassHWKatakanaLetter
;
97 if (GetGenCategory(c
) == nsUGenCategory::kPunctuation
) {
100 if (IsScriptioContinua(c
)) {
101 return kWbClassScriptioContinua
;
103 return kWbClassAlphaLetter
;
106 WordRange
WordBreaker::FindWord(const nsAString
& aText
, uint32_t aPos
,
107 const FindWordOptions aOptions
) {
108 const CheckedInt
<uint32_t> len
= aText
.Length();
109 MOZ_RELEASE_ASSERT(len
.isValid());
111 if (aPos
>= len
.value()) {
112 return {len
.value(), len
.value()};
115 WordRange range
{0, len
.value()};
117 #if defined(MOZ_ICU4X) && defined(JS_HAS_INTL_API)
118 if (StaticPrefs::intl_icu4x_segmenter_enabled()) {
120 capi::ICU4XWordSegmenter_create_auto(mozilla::intl::GetDataProvider());
121 MOZ_ASSERT(result
.is_ok
);
122 ICU4XWordSegmenter
segmenter(result
.ok
);
123 ICU4XWordBreakIteratorUtf16 iterator
=
124 segmenter
.segment_utf16(diplomat::span
<const uint16_t>(
125 (const uint16_t*)aText
.BeginReading(), aText
.Length()));
127 uint32_t previousPos
= 0;
129 const int32_t nextPos
= iterator
.next();
131 range
.mBegin
= previousPos
;
132 range
.mEnd
= len
.value();
135 if ((uint32_t)nextPos
> aPos
) {
136 range
.mBegin
= previousPos
;
137 range
.mEnd
= (uint32_t)nextPos
;
141 previousPos
= nextPos
;
144 if (aOptions
!= FindWordOptions::StopAtPunctuation
) {
148 for (uint32_t i
= range
.mBegin
; i
< range
.mEnd
; i
++) {
149 if (mozilla::IsPunctuationForWordSelect(aText
[i
])) {
160 range
.mBegin
= i
+ 1;
169 WordBreakClass c
= GetClass(aText
[aPos
]);
172 for (uint32_t i
= aPos
+ 1; i
< len
.value(); i
++) {
173 if (c
!= GetClass(aText
[i
])) {
180 for (uint32_t i
= aPos
; i
> 0; i
--) {
181 if (c
!= GetClass(aText
[i
- 1])) {
187 if (kWbClassScriptioContinua
== c
) {
188 // we pass the whole text segment to the complex word breaker to find a
190 AutoTArray
<uint8_t, 256> breakBefore
;
191 breakBefore
.SetLength(range
.mEnd
- range
.mBegin
);
192 ComplexBreaker::GetBreaks(aText
.BeginReading() + range
.mBegin
,
193 range
.mEnd
- range
.mBegin
,
194 breakBefore
.Elements());
197 for (uint32_t i
= aPos
+ 1; i
< range
.mEnd
; i
++) {
198 if (breakBefore
[i
- range
.mBegin
]) {
205 for (uint32_t i
= aPos
; i
> range
.mBegin
; i
--) {
206 if (breakBefore
[i
- range
.mBegin
]) {
215 int32_t WordBreaker::Next(const char16_t
* aText
, uint32_t aLen
, uint32_t aPos
) {
219 return NS_WORDBREAKER_NEED_MORE_TEXT
;
222 const WordBreakClass posClass
= GetClass(aText
[aPos
]);
223 uint32_t nextBreakPos
;
224 for (nextBreakPos
= aPos
+ 1; nextBreakPos
< aLen
; ++nextBreakPos
) {
225 if (posClass
!= GetClass(aText
[nextBreakPos
])) {
230 if (kWbClassScriptioContinua
== posClass
) {
231 // We pass the whole text segment to the complex word breaker to find a
233 const char16_t
* segStart
= aText
+ aPos
;
234 const uint32_t segLen
= nextBreakPos
- aPos
+ 1;
235 AutoTArray
<uint8_t, 256> breakBefore
;
236 breakBefore
.SetLength(segLen
);
237 ComplexBreaker::GetBreaks(segStart
, segLen
, breakBefore
.Elements());
239 for (uint32_t i
= aPos
+ 1; i
< nextBreakPos
; ++i
) {
240 if (breakBefore
[i
- aPos
]) {
247 MOZ_ASSERT(nextBreakPos
!= aPos
);