1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this
4 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 #include "mozilla/intl/LineBreaker.h"
8 #include "jisx4051class.h"
9 #include "nsComplexBreaker.h"
11 #include "nsUnicodeProperties.h"
12 #include "mozilla/ArrayUtils.h"
14 using namespace mozilla::unicode
;
15 using namespace mozilla::intl
;
18 already_AddRefed
<LineBreaker
> LineBreaker::Create() {
19 return RefPtr
<LineBreaker
>(new LineBreaker()).forget();
24 Simplification of Pair Table in JIS X 4051
26 1. The Origion Table - in 4.1.3
28 In JIS x 4051. The pair table is defined as below
31 Leading Class of Trailing Char Class
34 1 2 3 4 5 6 7 8 9 10 11 12 13 13 14 14 15 16 17 18 19 20
36 1 X X X X X X X X X X X X X X X X X X X X X E
54 19 X E E E E E X X X X X X X X X X X X E X E E
65 3: Prohibit a line break before
66 4: Punctuation for sentence end (except Full stop, e.g., "!" and "?")
67 5: Middle dot (e.g., U+30FB KATAKANA MIDDLE DOT)
69 7: Non-breakable between same characters
70 8: Prefix (e.g., "$", "NO.")
71 9: Postfix (e.g., "%")
74 12: Japanese characters (except class 11)
79 17: Space for Western language
80 18: Western characters (except class 17)
81 19: Split line note (Warichu) begin quote
82 20: Split line note (Warichu) end quote
84 2. Simplified by remove the class which we do not care
86 However, since we do not care about class 13(Subscript), 14(Ruby),
87 16 (Aphabet), 19(split line note begin quote), and 20(split line note end
88 quote) we can simplify this par table into the following
91 Leading Class of Trailing Char Class
94 1 2 3 4 5 6 7 8 9 10 11 12 15 17 18
96 1 X X X X X X X X X X X X X X X
112 3. Simplified by merged classes
114 After the 2 simplification, the pair table have some duplication
115 a. class 2, 3, 4, 5, 6, are the same- we can merged them
116 b. class 10, 11, 12, 17 are the same- we can merged them
118 We introduce an extra non-breaking pair at [b]/7 to better match
119 the expectations of CSS line-breaking as tested by WPT tests.
120 This added entry is marked as * in the tables below.
123 Leading Class of Trailing Char Class
138 4. We add COMPLEX characters and make it breakable w/ all ther class
139 except after class 1 and before class [a]
142 Leading Class of Trailing Char Class
145 1 [a] 7 8 9 [b]15 18 COMPLEX
157 T : need special handling
160 5. However, we need two special class for some punctuations/parentheses,
161 theirs breaking rules like character class (18), see bug 389056.
162 And also we need character like punctuation that is same behavior with 18,
163 but the characters are not letters of all languages. (e.g., '_')
164 [c]. Based on open parenthesis class (1), but it is not breakable after
165 character class (18) or numeric class (15).
166 [d]. Based on close parenthesis (or punctuation) class (2), but it is not
167 breakable before character class (18) or numeric class (15).
170 Leading Class of Trailing Char Class
173 1 [a] 7 8 9 [b]15 18 COMPLEX [c] [d]
175 1 X X X X X X X X X X X
184 [c] X X X X X X X X X X X
188 6. And Unicode has "NON-BREAK" characters. The lines should be broken around
189 them. But in JIS X 4051, such class is not, therefore, we create [e].
192 Leading Class of Trailing Char Class
195 1 [a] 7 8 9 [b]15 18 COMPLEX [c] [d] [e]
197 1 X X X X X X X X X X X X
206 [c] X X X X X X X X X X X X
208 [e] X X X X X X X X X X X X
211 7. Now we use one bit to encode whether it is breakable, and use 2 bytes
212 for one row, then the bit table will look like:
216 1 0000 1111 1111 1111 = 0x0FFF
217 [a] 0000 1100 0000 0010 = 0x0C02
218 7 0000 1000 0000 0110 = 0x0806
219 8 0000 1000 0100 0010 = 0x0842
220 9 0000 1000 0000 0010 = 0x0802
221 [b] 0000 1100 0000 0110 = 0x0C06
222 15 0000 1110 1101 0010 = 0x0ED2
223 18 0000 1110 1100 0010 = 0x0EC2
224 COMPLEX 0000 1001 0000 0010 = 0x0902
225 [c] 0000 1111 1111 1111 = 0x0FFF
226 [d] 0000 1100 1100 0010 = 0x0CC2
227 [e] 0000 1111 1111 1111 = 0x0FFF
230 #define MAX_CLASSES 12
232 static const uint16_t gPair
[MAX_CLASSES
] = {0x0FFF, 0x0C02, 0x0806, 0x0842,
233 0x0802, 0x0C06, 0x0ED2, 0x0EC2,
234 0x0902, 0x0FFF, 0x0CC2, 0x0FFF};
238 8. And if the character is not enough far from word start, word end and
239 another break point, we should not break in non-CJK languages.
240 I.e., Don't break around 15, 18, [c] and [d], but don't change
241 that if they are related to [b].
244 Leading Class of Trailing Char Class
247 1 [a] 7 8 9 [b]15 18 COMPLEX [c] [d] [e]
249 1 X X X X X X X X X X X X
255 15 X X X X X X X X X X X
256 18 X X X X X X X X X X X
257 COMPLEX X X X T X X X
258 [c] X X X X X X X X X X X X
259 [d] X X X X X X X X X X X
260 [e] X X X X X X X X X X X X
264 1 0000 1111 1111 1111 = 0x0FFF
265 [a] 0000 1110 1100 0010 = 0x0EC2
266 7 0000 1110 1100 0110 = 0x0EC6
267 8 0000 1110 1100 0010 = 0x0EC2
268 9 0000 1110 1100 0010 = 0x0EC2
269 [b] 0000 1100 0000 0110 = 0x0C06
270 15 0000 1111 1101 1111 = 0x0FDF
271 18 0000 1111 1101 1111 = 0x0FDF
272 COMPLEX 0000 1111 1100 0010 = 0x0FC2
273 [c] 0000 1111 1111 1111 = 0x0FFF
274 [d] 0000 1111 1101 1111 = 0x0FDF
275 [e] 0000 1111 1111 1111 = 0x0FFF
278 static const uint16_t gPairConservative
[MAX_CLASSES
] = {
279 0x0FFF, 0x0EC2, 0x0EC6, 0x0EC2, 0x0EC2, 0x0C06,
280 0x0FDF, 0x0FDF, 0x0FC2, 0x0FFF, 0x0FDF, 0x0FFF};
284 9. Now we map the class to number
287 1: [a]- 2, 3, 4, 5, 6
291 5: [b]- 10, 11, 12, 17
301 1: Punctuation that prohibits break before
302 2: Non-breakable between same classes
305 5: Breakable character (Spaces and Most Japanese characters)
308 8: Need special handling characters (E.g., Thai)
309 9: Open parentheses like Character (See bug 389056)
310 A: Close parenthese (or punctuations) like Character (See bug 389056)
311 B: Non breakable (See bug 390920)
315 #define CLASS_NONE INT8_MAX
317 #define CLASS_OPEN 0x00
318 #define CLASS_CLOSE 0x01
319 #define CLASS_NON_BREAKABLE_BETWEEN_SAME_CLASS 0x02
320 #define CLASS_PREFIX 0x03
321 #define CLASS_POSTFFIX 0x04
322 #define CLASS_BREAKABLE 0x05
323 #define CLASS_NUMERIC 0x06
324 #define CLASS_CHARACTER 0x07
325 #define CLASS_COMPLEX 0x08
326 #define CLASS_OPEN_LIKE_CHARACTER 0x09
327 #define CLASS_CLOSE_LIKE_CHARACTER 0x0A
328 #define CLASS_NON_BREAKABLE 0x0B
330 #define U_NULL char16_t(0x0000)
331 #define U_SLASH char16_t('/')
332 #define U_SPACE char16_t(' ')
333 #define U_HYPHEN char16_t('-')
334 #define U_EQUAL char16_t('=')
335 #define U_PERCENT char16_t('%')
336 #define U_AMPERSAND char16_t('&')
337 #define U_SEMICOLON char16_t(';')
338 #define U_BACKSLASH char16_t('\\')
339 #define U_OPEN_SINGLE_QUOTE char16_t(0x2018)
340 #define U_OPEN_DOUBLE_QUOTE char16_t(0x201C)
341 #define U_OPEN_GUILLEMET char16_t(0x00AB)
343 #define NEED_CONTEXTUAL_ANALYSIS(c) \
344 (IS_HYPHEN(c) || (c) == U_SLASH || (c) == U_PERCENT || (c) == U_AMPERSAND || \
345 (c) == U_SEMICOLON || (c) == U_BACKSLASH || (c) == U_OPEN_SINGLE_QUOTE || \
346 (c) == U_OPEN_DOUBLE_QUOTE || (c) == U_OPEN_GUILLEMET)
348 #define IS_ASCII_DIGIT(u) (0x0030 <= (u) && (u) <= 0x0039)
350 static inline int GETCLASSFROMTABLE(const uint32_t* t
, uint16_t l
) {
351 return ((((t
)[(l
>> 3)]) >> ((l
& 0x0007) << 2)) & 0x000f);
354 static inline int IS_HALFWIDTH_IN_JISx4051_CLASS3(char16_t u
) {
355 return ((0xff66 <= (u
)) && ((u
) <= 0xff70));
358 static inline int IS_CJK_CHAR(char32_t u
) {
360 (0x1100 <= (u
) && (u
) <= 0x11ff) || (0x2e80 <= (u
) && (u
) <= 0xd7ff) ||
361 (0xf900 <= (u
) && (u
) <= 0xfaff) || (0xff00 <= (u
) && (u
) <= 0xffef) ||
362 (0x20000 <= (u
) && (u
) <= 0x2fffd));
365 static inline bool IS_NONBREAKABLE_SPACE(char16_t u
) {
366 return u
== 0x00A0 || u
== 0x2007; // NO-BREAK SPACE, FIGURE SPACE
369 static inline bool IS_HYPHEN(char16_t u
) {
370 return (u
== U_HYPHEN
|| u
== 0x058A || // ARMENIAN HYPHEN
371 u
== 0x2010 || // HYPHEN
372 u
== 0x2012 || // FIGURE DASH
373 u
== 0x2013); // EN DASH
376 static int8_t GetClass(uint32_t u
, LineBreaker::Strictness aLevel
,
377 bool aIsChineseOrJapanese
) {
378 // Mapping for Unicode LineBreak.txt classes to the (simplified) set of
379 // character classes used here.
380 // XXX The mappings here were derived by comparing the Unicode LineBreak
381 // values of BMP characters to the classes our existing GetClass returns
382 // for the same codepoints; in cases where characters with the same
383 // LineBreak class mapped to various classes here, I picked what seemed
384 // the most prevalent equivalence.
385 // Some of these are unclear to me, but currently they are ONLY used
386 // for characters not handled by the old code below, so all the JISx405
387 // special cases should already be accounted for.
388 static const int8_t sUnicodeLineBreakToClass
[] = {
389 /* UNKNOWN = 0, [XX] */ CLASS_CHARACTER
,
390 /* AMBIGUOUS = 1, [AI] */ CLASS_CHARACTER
,
391 /* ALPHABETIC = 2, [AL] */ CLASS_CHARACTER
,
392 /* BREAK_BOTH = 3, [B2] */ CLASS_CHARACTER
,
393 /* BREAK_AFTER = 4, [BA] */ CLASS_CHARACTER
,
394 /* BREAK_BEFORE = 5, [BB] */ CLASS_OPEN_LIKE_CHARACTER
,
395 /* MANDATORY_BREAK = 6, [BK] */ CLASS_CHARACTER
,
396 /* CONTINGENT_BREAK = 7, [CB] */ CLASS_CHARACTER
,
397 /* CLOSE_PUNCTUATION = 8, [CL] */ CLASS_CHARACTER
,
398 /* COMBINING_MARK = 9, [CM] */ CLASS_CHARACTER
,
399 /* CARRIAGE_RETURN = 10, [CR] */ CLASS_BREAKABLE
,
400 /* EXCLAMATION = 11, [EX] */ CLASS_CHARACTER
,
401 /* GLUE = 12, [GL] */ CLASS_NON_BREAKABLE
,
402 /* HYPHEN = 13, [HY] */ CLASS_CHARACTER
,
403 /* IDEOGRAPHIC = 14, [ID] */ CLASS_BREAKABLE
,
404 /* INSEPARABLE = 15, [IN] */ CLASS_CLOSE_LIKE_CHARACTER
,
405 /* INFIX_NUMERIC = 16, [IS] */ CLASS_CHARACTER
,
406 /* LINE_FEED = 17, [LF] */ CLASS_BREAKABLE
,
407 /* NONSTARTER = 18, [NS] */ CLASS_CLOSE_LIKE_CHARACTER
,
408 /* NUMERIC = 19, [NU] */ CLASS_NUMERIC
,
409 /* OPEN_PUNCTUATION = 20, [OP] */ CLASS_CHARACTER
,
410 /* POSTFIX_NUMERIC = 21, [PO] */ CLASS_CHARACTER
,
411 /* PREFIX_NUMERIC = 22, [PR] */ CLASS_CHARACTER
,
412 /* QUOTATION = 23, [QU] */ CLASS_CHARACTER
,
413 /* COMPLEX_CONTEXT = 24, [SA] */ CLASS_CHARACTER
,
414 /* SURROGATE = 25, [SG] */ CLASS_CHARACTER
,
415 /* SPACE = 26, [SP] */ CLASS_BREAKABLE
,
416 /* BREAK_SYMBOLS = 27, [SY] */ CLASS_CHARACTER
,
417 /* ZWSPACE = 28, [ZW] */ CLASS_BREAKABLE
,
418 /* NEXT_LINE = 29, [NL] */ CLASS_CHARACTER
,
419 /* WORD_JOINER = 30, [WJ] */ CLASS_NON_BREAKABLE
,
420 /* H2 = 31, [H2] */ CLASS_BREAKABLE
,
421 /* H3 = 32, [H3] */ CLASS_BREAKABLE
,
422 /* JL = 33, [JL] */ CLASS_CHARACTER
,
423 /* JT = 34, [JT] */ CLASS_CHARACTER
,
424 /* JV = 35, [JV] */ CLASS_CHARACTER
,
425 /* CLOSE_PARENTHESIS = 36, [CP] */ CLASS_CLOSE_LIKE_CHARACTER
,
426 /* CONDITIONAL_JAPANESE_STARTER = 37, [CJ] */ CLASS_CLOSE
,
427 /* HEBREW_LETTER = 38, [HL] */ CLASS_CHARACTER
,
428 /* REGIONAL_INDICATOR = 39, [RI] */ CLASS_CHARACTER
,
429 /* E_BASE = 40, [EB] */ CLASS_BREAKABLE
,
430 /* E_MODIFIER = 41, [EM] */ CLASS_CHARACTER
,
431 /* ZWJ = 42, [ZWJ]*/ CLASS_CHARACTER
};
433 static_assert(U_LB_COUNT
== mozilla::ArrayLength(sUnicodeLineBreakToClass
),
434 "Gecko vs ICU LineBreak class mismatch");
436 auto cls
= GetLineBreakClass(u
);
437 MOZ_ASSERT(cls
< mozilla::ArrayLength(sUnicodeLineBreakToClass
));
439 // Overrides based on rules for the different line-break values given in
440 // https://drafts.csswg.org/css-text-3/#line-break-property
442 case LineBreaker::Strictness::Auto
:
443 // For now, just use legacy Gecko behavior.
444 // XXX Possible enhancement - vary strictness according to line width
445 // or other criteria.
447 case LineBreaker::Strictness::Strict
:
448 if (cls
== U_LB_CONDITIONAL_JAPANESE_STARTER
||
449 (u
== 0x3095 || u
== 0x3096 || u
== 0x30f5 || u
== 0x30f6)) {
452 if (cls
== U_LB_INSEPARABLE
) {
453 return CLASS_NON_BREAKABLE_BETWEEN_SAME_CLASS
;
455 if (u
== 0x3005 || u
== 0x303B || u
== 0x309D || u
== 0x309E ||
456 u
== 0x30FD || u
== 0x30FE) {
457 return CLASS_CLOSE_LIKE_CHARACTER
;
459 if (aIsChineseOrJapanese
) {
460 if (cls
== U_LB_POSTFIX_NUMERIC
&& IsEastAsianWidthAFW(u
)) {
461 return CLASS_CLOSE_LIKE_CHARACTER
;
463 if (cls
== U_LB_PREFIX_NUMERIC
&& IsEastAsianWidthAFW(u
)) {
464 return CLASS_OPEN_LIKE_CHARACTER
;
466 if (u
== 0x2010 || u
== 0x2013 || u
== 0x301C || u
== 0x30A0) {
467 return CLASS_CLOSE_LIKE_CHARACTER
;
471 case LineBreaker::Strictness::Normal
:
472 if (cls
== U_LB_CONDITIONAL_JAPANESE_STARTER
) {
473 return CLASS_BREAKABLE
;
475 if (cls
== U_LB_INSEPARABLE
) {
476 return CLASS_NON_BREAKABLE_BETWEEN_SAME_CLASS
;
478 if (u
== 0x3005 || u
== 0x303B || u
== 0x309D || u
== 0x309E ||
479 u
== 0x30FD || u
== 0x30FE) {
480 return CLASS_CLOSE_LIKE_CHARACTER
;
482 if (aIsChineseOrJapanese
) {
483 if (cls
== U_LB_POSTFIX_NUMERIC
&& IsEastAsianWidthAFW(u
)) {
484 return CLASS_CLOSE_LIKE_CHARACTER
;
486 if (cls
== U_LB_PREFIX_NUMERIC
&& IsEastAsianWidthAFW(u
)) {
487 return CLASS_OPEN_LIKE_CHARACTER
;
489 if (u
== 0x2010 || u
== 0x2013 || u
== 0x301C || u
== 0x30A0) {
490 return CLASS_BREAKABLE
;
494 case LineBreaker::Strictness::Loose
:
495 if (cls
== U_LB_CONDITIONAL_JAPANESE_STARTER
) {
496 return CLASS_BREAKABLE
;
498 if (u
== 0x3005 || u
== 0x303B || u
== 0x309D || u
== 0x309E ||
499 u
== 0x30FD || u
== 0x30FE) {
500 return CLASS_BREAKABLE
;
502 if (cls
== U_LB_INSEPARABLE
) {
503 return CLASS_BREAKABLE
;
505 if (aIsChineseOrJapanese
) {
506 if (u
== 0x30FB || u
== 0xFF1A || u
== 0xFF1B || u
== 0xFF65 ||
507 u
== 0x203C || u
== 0x2047 || u
== 0x2048 || u
== 0x2049 ||
508 u
== 0xFF01 || u
== 0xFF1F) {
509 return CLASS_BREAKABLE
;
511 if (cls
== U_LB_POSTFIX_NUMERIC
&& IsEastAsianWidthAFW(u
)) {
512 return CLASS_BREAKABLE
;
514 if (cls
== U_LB_PREFIX_NUMERIC
&& IsEastAsianWidthAFW(u
)) {
515 return CLASS_BREAKABLE
;
517 if (u
== 0x2010 || u
== 0x2013 || u
== 0x301C || u
== 0x30A0) {
518 return CLASS_BREAKABLE
;
522 case LineBreaker::Strictness::Anywhere
:
523 MOZ_ASSERT_UNREACHABLE("should have been handled already");
528 uint16_t h
= u
& 0xFF00;
529 uint16_t l
= u
& 0x00ff;
531 // Handle 3 range table first
533 return GETCLASSFROMTABLE(gLBClass00
, l
);
536 return GETCLASSFROMTABLE(gLBClass17
, l
);
538 if (NS_NeedsPlatformNativeHandling(u
)) {
539 return CLASS_COMPLEX
;
542 return GETCLASSFROMTABLE(gLBClass0E
, l
);
545 return GETCLASSFROMTABLE(gLBClass20
, l
);
548 return GETCLASSFROMTABLE(gLBClass21
, l
);
551 return GETCLASSFROMTABLE(gLBClass30
, l
);
554 if (l
< 0x0060) { // Fullwidth ASCII variant
555 return GETCLASSFROMTABLE(gLBClass00
, (l
+ 0x20));
557 if (l
< 0x00a0) { // Halfwidth Katakana variants
560 return GetClass(0x3002, aLevel
, aIsChineseOrJapanese
);
562 return GetClass(0x300c, aLevel
, aIsChineseOrJapanese
);
564 return GetClass(0x300d, aLevel
, aIsChineseOrJapanese
);
566 return GetClass(0x3001, aLevel
, aIsChineseOrJapanese
);
568 return GetClass(0x30fb, aLevel
, aIsChineseOrJapanese
);
570 return GetClass(0x309b, aLevel
, aIsChineseOrJapanese
);
572 return GetClass(0x309c, aLevel
, aIsChineseOrJapanese
);
574 if (IS_HALFWIDTH_IN_JISx4051_CLASS3(u
)) {
575 return CLASS_CLOSE
; // jis x4051 class 3
577 return CLASS_BREAKABLE
; // jis x4051 class 11
581 return CLASS_CHARACTER
; // Halfwidth Hangul variants
584 static char16_t NarrowFFEx
[16] = {
585 0x00A2, 0x00A3, 0x00AC, 0x00AF, 0x00A6, 0x00A5, 0x20A9, 0x0000,
586 0x2502, 0x2190, 0x2191, 0x2192, 0x2193, 0x25A0, 0x25CB, 0x0000};
587 return GetClass(NarrowFFEx
[l
- 0x00e0], aLevel
, aIsChineseOrJapanese
);
589 } else if (0x3100 == h
) {
590 if (l
<= 0xbf) { // Hangul Compatibility Jamo, Bopomofo, Kanbun
591 // XXX: This is per UAX #14, but UAX #14 may change
592 // the line breaking rules about Kanbun and Bopomofo.
593 return CLASS_BREAKABLE
;
595 if (l
>= 0xf0) { // Katakana small letters for Ainu
598 } else if (0x0300 == h
) {
599 if (0x4F == l
|| (0x5C <= l
&& l
<= 0x62)) {
600 return CLASS_NON_BREAKABLE
;
602 } else if (0x0500 == h
) {
603 // ARMENIAN HYPHEN (for "Breaking Hyphens" of UAX#14)
605 return GETCLASSFROMTABLE(gLBClass00
, uint16_t(U_HYPHEN
));
607 } else if (0x0F00 == h
) {
608 if (0x08 == l
|| 0x0C == l
|| 0x12 == l
) {
609 return CLASS_NON_BREAKABLE
;
611 } else if (0x1800 == h
) {
613 return CLASS_NON_BREAKABLE
;
615 } else if (0x1600 == h
) {
616 if (0x80 == l
) { // U+1680 OGHAM SPACE MARK
617 return CLASS_BREAKABLE
;
619 } else if (u
== 0xfeff) {
620 return CLASS_NON_BREAKABLE
;
624 return sUnicodeLineBreakToClass
[cls
];
627 static bool GetPair(int8_t c1
, int8_t c2
) {
628 NS_ASSERTION(c1
< MAX_CLASSES
, "illegal classes 1");
629 NS_ASSERTION(c2
< MAX_CLASSES
, "illegal classes 2");
631 return (0 == ((gPair
[c1
] >> c2
) & 0x0001));
634 static bool GetPairConservative(int8_t c1
, int8_t c2
) {
635 NS_ASSERTION(c1
< MAX_CLASSES
, "illegal classes 1");
636 NS_ASSERTION(c2
< MAX_CLASSES
, "illegal classes 2");
638 return (0 == ((gPairConservative
[c1
] >> c2
) & 0x0001));
643 ContextState(const char16_t
* aText
, uint32_t aLength
)
644 : mUniText(aText
), mText(nullptr), mLength(aLength
) {
648 ContextState(const uint8_t* aText
, uint32_t aLength
)
649 : mUniText(nullptr), mText(aText
), mLength(aLength
) {
653 uint32_t Length() const { return mLength
; }
654 uint32_t Index() const { return mIndex
; }
656 // This gets a single code unit of the text, without checking for surrogates
657 // (in the case of a 16-bit text buffer). That's OK if we're only checking for
658 // specific characters that are known to be BMP values.
659 char16_t
GetCodeUnitAt(uint32_t aIndex
) const {
660 MOZ_ASSERT(aIndex
< mLength
, "Out of range!");
661 return mUniText
? mUniText
[aIndex
] : char16_t(mText
[aIndex
]);
664 // This gets a 32-bit Unicode character (codepoint), handling surrogate pairs
665 // as necessary. It must ONLY be called for 16-bit text, not 8-bit.
666 char32_t
GetUnicodeCharAt(uint32_t aIndex
) const {
667 MOZ_ASSERT(mUniText
, "Only for 16-bit text!");
668 MOZ_ASSERT(aIndex
< mLength
, "Out of range!");
669 char32_t c
= mUniText
[aIndex
];
670 if (aIndex
+ 1 < mLength
&& NS_IS_SURROGATE_PAIR(c
, mUniText
[aIndex
+ 1])) {
671 c
= SURROGATE_TO_UCS4(c
, mUniText
[aIndex
+ 1]);
676 void AdvanceIndex() { ++mIndex
; }
678 void NotifyBreakBefore() { mLastBreakIndex
= mIndex
; }
680 // A word of western language should not be broken. But even if the word has
681 // only ASCII characters, non-natural context words should be broken, e.g.,
682 // URL and file path. For protecting the natural words, we should use
683 // conservative breaking rules at following conditions:
684 // 1. at near the start of word
685 // 2. at near the end of word
686 // 3. at near the latest broken point
687 // CONSERVATIVE_RANGE_{LETTER,OTHER} define the 'near' in characters,
688 // which varies depending whether we are looking at a letter or a non-letter
689 // character: for non-letters, we use an extended "conservative" range.
691 #define CONSERVATIVE_RANGE_LETTER 2
692 #define CONSERVATIVE_RANGE_OTHER 6
694 bool UseConservativeBreaking(uint32_t aOffset
= 0) const {
695 if (mHasCJKChar
) return false;
696 uint32_t index
= mIndex
+ aOffset
;
698 // If the character at index is a letter (rather than various punctuation
699 // characters, etc) then we want a shorter "conservative" range
700 uint32_t conservativeRangeStart
, conservativeRangeEnd
;
701 if (index
< mLength
&&
702 nsUGenCategory::kLetter
==
703 (mText
? GetGenCategory(mText
[index
])
704 : GetGenCategory(GetUnicodeCharAt(index
)))) {
705 // Primarily for hyphenated word prefixes/suffixes; we add 1 to Start
706 // to get more balanced behavior (if we break off a 2-letter prefix,
707 // that means the break will actually be three letters from start of
708 // word, to include the hyphen; whereas a 2-letter suffix will be
709 // broken only two letters from end of word).
710 conservativeRangeEnd
= CONSERVATIVE_RANGE_LETTER
;
711 conservativeRangeStart
= CONSERVATIVE_RANGE_LETTER
+ 1;
713 conservativeRangeEnd
= conservativeRangeStart
= CONSERVATIVE_RANGE_OTHER
;
716 bool result
= (index
< conservativeRangeStart
||
717 mLength
- index
< conservativeRangeEnd
||
718 index
- mLastBreakIndex
< conservativeRangeStart
);
719 if (result
|| !mHasNonbreakableSpace
) return result
;
721 // This text has no-breakable space, we need to check whether the index
724 // Note that index is always larger than conservativeRange here.
725 for (uint32_t i
= index
; index
- conservativeRangeStart
< i
; --i
) {
726 if (IS_NONBREAKABLE_SPACE(GetCodeUnitAt(i
- 1))) return true;
728 // Note that index is always less than mLength - conservativeRange.
729 for (uint32_t i
= index
+ 1; i
< index
+ conservativeRangeEnd
; ++i
) {
730 if (IS_NONBREAKABLE_SPACE(GetCodeUnitAt(i
))) return true;
735 bool HasPreviousEqualsSign() const { return mHasPreviousEqualsSign
; }
736 void NotifySeenEqualsSign() { mHasPreviousEqualsSign
= true; }
738 bool HasPreviousSlash() const { return mHasPreviousSlash
; }
739 void NotifySeenSlash() { mHasPreviousSlash
= true; }
741 bool HasPreviousBackslash() const { return mHasPreviousBackslash
; }
742 void NotifySeenBackslash() { mHasPreviousBackslash
= true; }
744 uint32_t GetPreviousNonHyphenCharacter() const {
745 return mPreviousNonHyphenCharacter
;
747 void NotifyNonHyphenCharacter(uint32_t ch
) {
748 mPreviousNonHyphenCharacter
= ch
;
755 mPreviousNonHyphenCharacter
= U_NULL
;
757 mHasNonbreakableSpace
= false;
758 mHasPreviousEqualsSign
= false;
759 mHasPreviousSlash
= false;
760 mHasPreviousBackslash
= false;
763 // 8-bit text: we only need to check for
764 for (uint32_t i
= 0; i
< mLength
; ++i
) {
765 if (IS_NONBREAKABLE_SPACE(mText
[i
])) {
766 mHasNonbreakableSpace
= true;
771 // 16-bit text: handle surrogates and check for CJK as well as
772 for (uint32_t i
= 0; i
< mLength
; ++i
) {
773 char32_t u
= GetUnicodeCharAt(i
);
774 if (!mHasNonbreakableSpace
&& IS_NONBREAKABLE_SPACE(u
)) {
775 mHasNonbreakableSpace
= true;
779 } else if (!mHasCJKChar
&& IS_CJK_CHAR(u
)) {
781 if (mHasNonbreakableSpace
) {
786 ++i
; // step over trailing low surrogate
792 const char16_t
* const mUniText
;
793 const uint8_t* const mText
;
796 const uint32_t mLength
; // length of text
797 uint32_t mLastBreakIndex
;
798 char32_t mPreviousNonHyphenCharacter
; // The last character we have seen
799 // which is not U_HYPHEN
800 bool mHasCJKChar
; // if the text has CJK character, this is true.
801 bool mHasNonbreakableSpace
; // if the text has no-breakable space,
803 bool mHasPreviousEqualsSign
; // True if we have seen a U_EQUAL
804 bool mHasPreviousSlash
; // True if we have seen a U_SLASH
805 bool mHasPreviousBackslash
; // True if we have seen a U_BACKSLASH
808 static int8_t ContextualAnalysis(char32_t prev
, char32_t cur
, char32_t next
,
809 ContextState
& aState
,
810 LineBreaker::Strictness aLevel
,
811 bool aIsChineseOrJapanese
) {
812 // Don't return CLASS_OPEN/CLASS_CLOSE if aState.UseJISX4051 is FALSE.
814 if (IS_HYPHEN(cur
)) {
815 // If next character is hyphen, we don't need to break between them.
816 if (IS_HYPHEN(next
)) return CLASS_CHARACTER
;
817 // If prev and next characters are numeric, it may be in Math context.
818 // So, we should not break here.
819 bool prevIsNum
= IS_ASCII_DIGIT(prev
);
820 bool nextIsNum
= IS_ASCII_DIGIT(next
);
821 if (prevIsNum
&& nextIsNum
) return CLASS_NUMERIC
;
822 // If one side is numeric and the other is a character, or if both sides are
823 // characters, the hyphen should be breakable.
824 if (!aState
.UseConservativeBreaking(1)) {
825 char32_t prevOfHyphen
= aState
.GetPreviousNonHyphenCharacter();
826 if (prevOfHyphen
&& next
) {
827 int8_t prevClass
= GetClass(prevOfHyphen
, aLevel
, aIsChineseOrJapanese
);
828 int8_t nextClass
= GetClass(next
, aLevel
, aIsChineseOrJapanese
);
829 bool prevIsNumOrCharOrClose
=
831 (prevClass
== CLASS_CHARACTER
&&
832 !NEED_CONTEXTUAL_ANALYSIS(prevOfHyphen
)) ||
833 prevClass
== CLASS_CLOSE
|| prevClass
== CLASS_CLOSE_LIKE_CHARACTER
;
834 bool nextIsNumOrCharOrOpen
=
836 (nextClass
== CLASS_CHARACTER
&& !NEED_CONTEXTUAL_ANALYSIS(next
)) ||
837 nextClass
== CLASS_OPEN
|| nextClass
== CLASS_OPEN_LIKE_CHARACTER
||
838 next
== U_OPEN_SINGLE_QUOTE
|| next
== U_OPEN_DOUBLE_QUOTE
||
839 next
== U_OPEN_GUILLEMET
;
840 if (prevIsNumOrCharOrClose
&& nextIsNumOrCharOrOpen
) {
846 aState
.NotifyNonHyphenCharacter(cur
);
847 if (cur
== U_SLASH
|| cur
== U_BACKSLASH
) {
848 // If this is immediately after same char, we should not break here.
849 if (prev
== cur
) return CLASS_CHARACTER
;
850 // If this text has two or more (BACK)SLASHs, this may be file path or
851 // URL. Make sure to compute shouldReturn before we notify on this slash.
852 bool shouldReturn
= !aState
.UseConservativeBreaking() &&
853 (cur
== U_SLASH
? aState
.HasPreviousSlash()
854 : aState
.HasPreviousBackslash());
856 if (cur
== U_SLASH
) {
857 aState
.NotifySeenSlash();
859 aState
.NotifySeenBackslash();
862 if (shouldReturn
) return CLASS_OPEN
;
863 } else if (cur
== U_PERCENT
) {
864 // If this is a part of the param of URL, we should break before.
865 if (!aState
.UseConservativeBreaking()) {
866 if (aState
.Index() >= 3 &&
867 aState
.GetCodeUnitAt(aState
.Index() - 3) == U_PERCENT
)
869 if (aState
.Index() + 3 < aState
.Length() &&
870 aState
.GetCodeUnitAt(aState
.Index() + 3) == U_PERCENT
)
873 } else if (cur
== U_AMPERSAND
|| cur
== U_SEMICOLON
) {
874 // If this may be a separator of params of URL, we should break after.
875 if (!aState
.UseConservativeBreaking(1) && aState
.HasPreviousEqualsSign())
877 } else if (cur
== U_OPEN_SINGLE_QUOTE
|| cur
== U_OPEN_DOUBLE_QUOTE
||
878 cur
== U_OPEN_GUILLEMET
) {
879 // for CJK usage, we treat these as openers to allow a break before them,
880 // but otherwise treat them as normal characters because quote mark usage
881 // in various Western languages varies too much; see bug #450088
883 if (!aState
.UseConservativeBreaking() && IS_CJK_CHAR(next
))
886 NS_ERROR("Forgot to handle the current character!");
889 return GetClass(cur
, aLevel
, aIsChineseOrJapanese
);
892 int32_t LineBreaker::WordMove(const char16_t
* aText
, uint32_t aLen
,
893 uint32_t aPos
, int8_t aDirection
) {
894 bool textNeedsJISx4051
= false;
897 for (begin
= aPos
; begin
> 0 && !NS_IsSpace(aText
[begin
- 1]); --begin
) {
898 if (IS_CJK_CHAR(aText
[begin
]) ||
899 NS_NeedsPlatformNativeHandling(aText
[begin
])) {
900 textNeedsJISx4051
= true;
903 for (end
= aPos
+ 1; end
< int32_t(aLen
) && !NS_IsSpace(aText
[end
]); ++end
) {
904 if (IS_CJK_CHAR(aText
[end
]) || NS_NeedsPlatformNativeHandling(aText
[end
])) {
905 textNeedsJISx4051
= true;
910 AutoTArray
<uint8_t, 2000> breakState
;
911 if (!textNeedsJISx4051
|| !breakState
.AppendElements(end
- begin
)) {
912 // No complex text character, do not try to do complex line break.
913 // (This is required for serializers. See Bug #344816.)
914 // Also fall back to this when out of memory.
915 if (aDirection
< 0) {
916 ret
= (begin
== int32_t(aPos
)) ? begin
- 1 : begin
;
921 GetJISx4051Breaks(aText
+ begin
, end
- begin
, WordBreak::Normal
,
922 Strictness::Auto
, false, breakState
.Elements());
927 } while (begin
< ret
&& ret
< end
&& !breakState
[ret
- begin
]);
933 int32_t LineBreaker::Next(const char16_t
* aText
, uint32_t aLen
, uint32_t aPos
) {
934 NS_ASSERTION(aText
, "aText shouldn't be null");
935 NS_ASSERTION(aLen
> aPos
,
936 "Bad position passed to nsJISx4051LineBreaker::Next");
938 int32_t nextPos
= WordMove(aText
, aLen
, aPos
, 1);
939 return nextPos
< int32_t(aLen
) ? nextPos
: NS_LINEBREAKER_NEED_MORE_TEXT
;
942 int32_t LineBreaker::Prev(const char16_t
* aText
, uint32_t aLen
, uint32_t aPos
) {
943 NS_ASSERTION(aText
, "aText shouldn't be null");
944 NS_ASSERTION(aLen
>= aPos
&& aPos
> 0,
945 "Bad position passed to nsJISx4051LineBreaker::Prev");
947 int32_t prevPos
= WordMove(aText
, aLen
, aPos
, -1);
948 return prevPos
> 0 ? prevPos
: NS_LINEBREAKER_NEED_MORE_TEXT
;
951 void LineBreaker::GetJISx4051Breaks(const char16_t
* aChars
, uint32_t aLength
,
952 WordBreak aWordBreak
, Strictness aLevel
,
953 bool aIsChineseOrJapanese
,
954 uint8_t* aBreakBefore
) {
956 int8_t lastClass
= CLASS_NONE
;
957 ContextState
state(aChars
, aLength
);
959 for (cur
= 0; cur
< aLength
; ++cur
, state
.AdvanceIndex()) {
960 char32_t ch
= state
.GetUnicodeCharAt(cur
);
961 uint32_t chLen
= ch
> 0xFFFFu
? 2 : 1;
964 if (NEED_CONTEXTUAL_ANALYSIS(ch
)) {
967 // not using state.GetUnicodeCharAt() here because we're looking back
968 // rather than forward for possible surrogates
969 prev
= aChars
[cur
- 1];
970 if (cur
> 1 && NS_IS_SURROGATE_PAIR(aChars
[cur
- 2], prev
)) {
971 prev
= SURROGATE_TO_UCS4(aChars
[cur
- 2], prev
);
976 if (cur
+ chLen
< aLength
) {
977 next
= state
.GetUnicodeCharAt(cur
+ chLen
);
981 cl
= ContextualAnalysis(prev
, ch
, next
, state
, aLevel
,
982 aIsChineseOrJapanese
);
984 if (ch
== U_EQUAL
) state
.NotifySeenEqualsSign();
985 state
.NotifyNonHyphenCharacter(ch
);
986 cl
= GetClass(ch
, aLevel
, aIsChineseOrJapanese
);
989 // To implement word-break:break-all, we overwrite the line-break class of
990 // alphanumeric characters so they are treated the same as ideographic.
991 // The relevant characters will have been assigned CLASS_CHARACTER, _CLOSE,
992 // or _NUMERIC by GetClass(), but those classes also include others that
993 // we don't want to touch here, so we re-check the Unicode line-break class
994 // to determine which ones to modify.
995 if (aWordBreak
== WordBreak::BreakAll
&&
996 (cl
== CLASS_CHARACTER
|| cl
== CLASS_CLOSE
|| cl
== CLASS_NUMERIC
)) {
997 auto cls
= GetLineBreakClass(ch
);
998 if (cls
== U_LB_ALPHABETIC
|| cls
== U_LB_NUMERIC
||
999 cls
== U_LB_AMBIGUOUS
|| cls
== U_LB_COMPLEX_CONTEXT
||
1000 /* Additional Japanese and Korean LB classes; CSS Text spec doesn't
1001 explicitly mention these, but this appears to give expected
1002 behavior (spec issue?) */
1003 cls
== U_LB_CONDITIONAL_JAPANESE_STARTER
||
1004 (cls
>= U_LB_H2
&& cls
<= U_LB_JV
)) {
1005 cl
= CLASS_BREAKABLE
;
1009 bool allowBreak
= false;
1011 NS_ASSERTION(CLASS_COMPLEX
!= lastClass
|| CLASS_COMPLEX
!= cl
,
1012 "Loop should have prevented adjacent complex chars here");
1013 if (aWordBreak
== WordBreak::Normal
||
1014 aWordBreak
== WordBreak::BreakAll
) {
1015 allowBreak
= (state
.UseConservativeBreaking())
1016 ? GetPairConservative(lastClass
, cl
)
1017 : GetPair(lastClass
, cl
);
1020 aBreakBefore
[cur
] = allowBreak
;
1021 if (allowBreak
) state
.NotifyBreakBefore();
1023 if (CLASS_COMPLEX
== cl
) {
1024 uint32_t end
= cur
+ chLen
;
1026 while (end
< aLength
) {
1027 char32_t c
= state
.GetUnicodeCharAt(end
);
1028 if (CLASS_COMPLEX
!= GetClass(c
, aLevel
, false)) {
1032 if (c
> 0xFFFFU
) { // it was a surrogate pair
1037 if (aWordBreak
== WordBreak::BreakAll
) {
1038 // For break-all, we don't need to run a dictionary-based breaking
1039 // algorithm, we just allow breaks between all grapheme clusters.
1040 ClusterIterator
ci(aChars
+ cur
, end
- cur
);
1041 while (!ci
.AtEnd()) {
1043 aBreakBefore
[ci
- aChars
] = true;
1046 NS_GetComplexLineBreaks(aChars
+ cur
, end
- cur
, aBreakBefore
+ cur
);
1047 // restore breakability at chunk begin, which was always set to false
1048 // by the complex line breaker
1049 aBreakBefore
[cur
] = allowBreak
;
1056 // Supplementary-plane character: mark that we cannot break before the
1057 // trailing low surrogate, and advance past it.
1059 aBreakBefore
[cur
] = false;
1060 state
.AdvanceIndex();
1065 void LineBreaker::GetJISx4051Breaks(const uint8_t* aChars
, uint32_t aLength
,
1066 WordBreak aWordBreak
, Strictness aLevel
,
1067 bool aIsChineseOrJapanese
,
1068 uint8_t* aBreakBefore
) {
1070 int8_t lastClass
= CLASS_NONE
;
1071 ContextState
state(aChars
, aLength
);
1073 for (cur
= 0; cur
< aLength
; ++cur
, state
.AdvanceIndex()) {
1074 char32_t ch
= aChars
[cur
];
1077 if (NEED_CONTEXTUAL_ANALYSIS(ch
)) {
1078 cl
= ContextualAnalysis(cur
> 0 ? aChars
[cur
- 1] : U_NULL
, ch
,
1079 cur
+ 1 < aLength
? aChars
[cur
+ 1] : U_NULL
,
1080 state
, aLevel
, aIsChineseOrJapanese
);
1082 if (ch
== U_EQUAL
) state
.NotifySeenEqualsSign();
1083 state
.NotifyNonHyphenCharacter(ch
);
1084 cl
= GetClass(ch
, aLevel
, aIsChineseOrJapanese
);
1086 if (aWordBreak
== WordBreak::BreakAll
&&
1087 (cl
== CLASS_CHARACTER
|| cl
== CLASS_CLOSE
|| cl
== CLASS_NUMERIC
)) {
1088 auto cls
= GetLineBreakClass(ch
);
1089 // Don't need to check additional Japanese/Korean classes in 8-bit
1090 if (cls
== U_LB_ALPHABETIC
|| cls
== U_LB_NUMERIC
||
1091 cls
== U_LB_COMPLEX_CONTEXT
) {
1092 cl
= CLASS_BREAKABLE
;
1096 bool allowBreak
= false;
1098 if (aWordBreak
== WordBreak::Normal
||
1099 aWordBreak
== WordBreak::BreakAll
) {
1100 allowBreak
= (state
.UseConservativeBreaking())
1101 ? GetPairConservative(lastClass
, cl
)
1102 : GetPair(lastClass
, cl
);
1105 aBreakBefore
[cur
] = allowBreak
;
1106 if (allowBreak
) state
.NotifyBreakBefore();