1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this
4 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 #include "mozilla/intl/LineBreaker.h"
8 #include "jisx4051class.h"
9 #include "nsComplexBreaker.h"
11 #include "nsUnicodeProperties.h"
12 #include "mozilla/ArrayUtils.h"
13 #include "mozilla/intl/Segmenter.h"
14 #include "mozilla/intl/UnicodeProperties.h"
16 using namespace mozilla::unicode
;
17 using namespace mozilla::intl
;
21 Simplification of Pair Table in JIS X 4051
23 1. The Origion Table - in 4.1.3
25 In JIS x 4051. The pair table is defined as below
28 Leading Class of Trailing Char Class
31 1 2 3 4 5 6 7 8 9 10 11 12 13 13 14 14 15 16 17 18 19 20
33 1 X X X X X X X X X X X X X X X X X X X X X E
51 19 X E E E E E X X X X X X X X X X X X E X E E
62 3: Prohibit a line break before
63 4: Punctuation for sentence end (except Full stop, e.g., "!" and "?")
64 5: Middle dot (e.g., U+30FB KATAKANA MIDDLE DOT)
66 7: Non-breakable between same characters
67 8: Prefix (e.g., "$", "NO.")
68 9: Postfix (e.g., "%")
71 12: Japanese characters (except class 11)
76 17: Space for Western language
77 18: Western characters (except class 17)
78 19: Split line note (Warichu) begin quote
79 20: Split line note (Warichu) end quote
81 2. Simplified by remove the class which we do not care
83 However, since we do not care about class 13(Subscript), 14(Ruby),
84 16 (Aphabet), 19(split line note begin quote), and 20(split line note end
85 quote) we can simplify this par table into the following
88 Leading Class of Trailing Char Class
91 1 2 3 4 5 6 7 8 9 10 11 12 15 17 18
93 1 X X X X X X X X X X X X X X X
109 3. Simplified by merged classes
111 After the 2 simplification, the pair table have some duplication
112 a. class 2, 3, 4, 5, 6, are the same- we can merged them
113 b. class 10, 11, 12, 17 are the same- we can merged them
115 We introduce an extra non-breaking pair at [b]/7 to better match
116 the expectations of CSS line-breaking as tested by WPT tests.
117 This added entry is marked as * in the tables below.
120 Leading Class of Trailing Char Class
135 4. We add COMPLEX characters and make it breakable w/ all ther class
136 except after class 1 and before class [a]
139 Leading Class of Trailing Char Class
142 1 [a] 7 8 9 [b]15 18 COMPLEX
154 T : need special handling
157 5. However, we need two special class for some punctuations/parentheses,
158 theirs breaking rules like character class (18), see bug 389056.
159 And also we need character like punctuation that is same behavior with 18,
160 but the characters are not letters of all languages. (e.g., '_')
161 [c]. Based on open parenthesis class (1), but it is not breakable after
162 character class (18) or numeric class (15).
163 [d]. Based on close parenthesis (or punctuation) class (2), but it is not
164 breakable before character class (18) or numeric class (15).
167 Leading Class of Trailing Char Class
170 1 [a] 7 8 9 [b]15 18 COMPLEX [c] [d]
172 1 X X X X X X X X X X X
181 [c] X X X X X X X X X X X
185 6. And Unicode has "NON-BREAK" characters. The lines should be broken around
186 them. But in JIS X 4051, such class is not, therefore, we create [e].
189 Leading Class of Trailing Char Class
192 1 [a] 7 8 9 [b]15 18 COMPLEX [c] [d] [e]
194 1 X X X X X X X X X X X X
203 [c] X X X X X X X X X X X X
205 [e] X X X X X X X X X X X X
208 7. Now we use one bit to encode whether it is breakable, and use 2 bytes
209 for one row, then the bit table will look like:
213 1 0000 1111 1111 1111 = 0x0FFF
214 [a] 0000 1100 0000 0010 = 0x0C02
215 7 0000 1000 0000 0110 = 0x0806
216 8 0000 1000 0100 0010 = 0x0842
217 9 0000 1000 0000 0010 = 0x0802
218 [b] 0000 1100 0000 0110 = 0x0C06
219 15 0000 1110 1101 0010 = 0x0ED2
220 18 0000 1110 1100 0010 = 0x0EC2
221 COMPLEX 0000 1001 0000 0010 = 0x0902
222 [c] 0000 1111 1111 1111 = 0x0FFF
223 [d] 0000 1100 1100 0010 = 0x0CC2
224 [e] 0000 1111 1111 1111 = 0x0FFF
227 #define MAX_CLASSES 12
229 static const uint16_t gPair
[MAX_CLASSES
] = {0x0FFF, 0x0C02, 0x0806, 0x0842,
230 0x0802, 0x0C06, 0x0ED2, 0x0EC2,
231 0x0902, 0x0FFF, 0x0CC2, 0x0FFF};
235 8. And if the character is not enough far from word start, word end and
236 another break point, we should not break in non-CJK languages.
237 I.e., Don't break around 15, 18, [c] and [d], but don't change
238 that if they are related to [b].
241 Leading Class of Trailing Char Class
244 1 [a] 7 8 9 [b]15 18 COMPLEX [c] [d] [e]
246 1 X X X X X X X X X X X X
252 15 X X X X X X X X X X X
253 18 X X X X X X X X X X X
254 COMPLEX X X X T X X X
255 [c] X X X X X X X X X X X X
256 [d] X X X X X X X X X X X
257 [e] X X X X X X X X X X X X
261 1 0000 1111 1111 1111 = 0x0FFF
262 [a] 0000 1110 1100 0010 = 0x0EC2
263 7 0000 1110 1100 0110 = 0x0EC6
264 8 0000 1110 1100 0010 = 0x0EC2
265 9 0000 1110 1100 0010 = 0x0EC2
266 [b] 0000 1100 0000 0110 = 0x0C06
267 15 0000 1111 1101 1111 = 0x0FDF
268 18 0000 1111 1101 1111 = 0x0FDF
269 COMPLEX 0000 1111 1100 0010 = 0x0FC2
270 [c] 0000 1111 1111 1111 = 0x0FFF
271 [d] 0000 1111 1101 1111 = 0x0FDF
272 [e] 0000 1111 1111 1111 = 0x0FFF
275 static const uint16_t gPairConservative
[MAX_CLASSES
] = {
276 0x0FFF, 0x0EC2, 0x0EC6, 0x0EC2, 0x0EC2, 0x0C06,
277 0x0FDF, 0x0FDF, 0x0FC2, 0x0FFF, 0x0FDF, 0x0FFF};
281 9. Now we map the class to number
284 1: [a]- 2, 3, 4, 5, 6
288 5: [b]- 10, 11, 12, 17
298 1: Punctuation that prohibits break before
299 2: Non-breakable between same classes
302 5: Breakable character (Spaces and Most Japanese characters)
305 8: Need special handling characters (E.g., Thai)
306 9: Open parentheses like Character (See bug 389056)
307 A: Close parenthese (or punctuations) like Character (See bug 389056)
308 B: Non breakable (See bug 390920)
312 #define CLASS_NONE INT8_MAX
314 #define CLASS_OPEN 0x00
315 #define CLASS_CLOSE 0x01
316 #define CLASS_NON_BREAKABLE_BETWEEN_SAME_CLASS 0x02
317 #define CLASS_PREFIX 0x03
318 #define CLASS_POSTFFIX 0x04
319 #define CLASS_BREAKABLE 0x05
320 #define CLASS_NUMERIC 0x06
321 #define CLASS_CHARACTER 0x07
322 #define CLASS_COMPLEX 0x08
323 #define CLASS_OPEN_LIKE_CHARACTER 0x09
324 #define CLASS_CLOSE_LIKE_CHARACTER 0x0A
325 #define CLASS_NON_BREAKABLE 0x0B
327 #define U_NULL char16_t(0x0000)
328 #define U_SLASH char16_t('/')
329 #define U_SPACE char16_t(' ')
330 #define U_HYPHEN char16_t('-')
331 #define U_EQUAL char16_t('=')
332 #define U_PERCENT char16_t('%')
333 #define U_AMPERSAND char16_t('&')
334 #define U_SEMICOLON char16_t(';')
335 #define U_BACKSLASH char16_t('\\')
336 #define U_OPEN_SINGLE_QUOTE char16_t(0x2018)
337 #define U_OPEN_DOUBLE_QUOTE char16_t(0x201C)
338 #define U_OPEN_GUILLEMET char16_t(0x00AB)
340 #define NEED_CONTEXTUAL_ANALYSIS(c) \
341 (IS_HYPHEN(c) || (c) == U_SLASH || (c) == U_PERCENT || (c) == U_AMPERSAND || \
342 (c) == U_SEMICOLON || (c) == U_BACKSLASH || (c) == U_OPEN_SINGLE_QUOTE || \
343 (c) == U_OPEN_DOUBLE_QUOTE || (c) == U_OPEN_GUILLEMET)
345 #define IS_ASCII_DIGIT(u) (0x0030 <= (u) && (u) <= 0x0039)
347 static inline int GETCLASSFROMTABLE(const uint32_t* t
, uint16_t l
) {
348 return ((((t
)[(l
>> 3)]) >> ((l
& 0x0007) << 2)) & 0x000f);
351 static inline int IS_HALFWIDTH_IN_JISx4051_CLASS3(char16_t u
) {
352 return ((0xff66 <= (u
)) && ((u
) <= 0xff70));
355 static inline int IS_CJK_CHAR(char32_t u
) {
357 (0x1100 <= (u
) && (u
) <= 0x11ff) || (0x2e80 <= (u
) && (u
) <= 0xd7ff) ||
358 (0xf900 <= (u
) && (u
) <= 0xfaff) || (0xff00 <= (u
) && (u
) <= 0xffef) ||
359 (0x20000 <= (u
) && (u
) <= 0x2fffd));
362 static inline bool IS_NONBREAKABLE_SPACE(char16_t u
) {
363 return u
== 0x00A0 || u
== 0x2007; // NO-BREAK SPACE, FIGURE SPACE
366 static inline bool IS_HYPHEN(char16_t u
) {
367 return (u
== U_HYPHEN
|| u
== 0x2010 || // HYPHEN
368 u
== 0x2012 || // FIGURE DASH
369 u
== 0x2013 || // EN DASH
370 #if ANDROID || XP_WIN
371 /* Bug 1647377: On Android and Windows, we don't have a "platform"
372 * backend that supports Tibetan (nsRuleBreaker.cpp only knows about
373 * Thai, and ScriptBreak doesn't handle Tibetan well either), so
374 * instead we just treat the TSHEG like a hyphen to provide basic
375 * line-breaking possibilities.
377 u
== 0x0F0B || // TIBETAN MARK INTERSYLLABIC TSHEG
379 u
== 0x058A); // ARMENIAN HYPHEN
382 static int8_t GetClass(uint32_t u
, LineBreakRule aLevel
,
383 bool aIsChineseOrJapanese
) {
384 // Mapping for Unicode LineBreak.txt classes to the (simplified) set of
385 // character classes used here.
386 // XXX The mappings here were derived by comparing the Unicode LineBreak
387 // values of BMP characters to the classes our existing GetClass returns
388 // for the same codepoints; in cases where characters with the same
389 // LineBreak class mapped to various classes here, I picked what seemed
390 // the most prevalent equivalence.
391 // Some of these are unclear to me, but currently they are ONLY used
392 // for characters not handled by the old code below, so all the JISx405
393 // special cases should already be accounted for.
394 static const int8_t sUnicodeLineBreakToClass
[] = {
395 /* UNKNOWN = 0, [XX] */ CLASS_CHARACTER
,
396 /* AMBIGUOUS = 1, [AI] */ CLASS_CHARACTER
,
397 /* ALPHABETIC = 2, [AL] */ CLASS_CHARACTER
,
398 /* BREAK_BOTH = 3, [B2] */ CLASS_CHARACTER
,
399 /* BREAK_AFTER = 4, [BA] */ CLASS_BREAKABLE
,
400 /* BREAK_BEFORE = 5, [BB] */ CLASS_OPEN_LIKE_CHARACTER
,
401 /* MANDATORY_BREAK = 6, [BK] */ CLASS_CHARACTER
,
402 /* CONTINGENT_BREAK = 7, [CB] */ CLASS_CHARACTER
,
403 /* CLOSE_PUNCTUATION = 8, [CL] */ CLASS_CLOSE_LIKE_CHARACTER
,
404 /* COMBINING_MARK = 9, [CM] */ CLASS_CHARACTER
,
405 /* CARRIAGE_RETURN = 10, [CR] */ CLASS_BREAKABLE
,
406 /* EXCLAMATION = 11, [EX] */ CLASS_CLOSE_LIKE_CHARACTER
,
407 /* GLUE = 12, [GL] */ CLASS_NON_BREAKABLE
,
408 /* HYPHEN = 13, [HY] */ CLASS_CHARACTER
,
409 /* IDEOGRAPHIC = 14, [ID] */ CLASS_BREAKABLE
,
410 /* INSEPARABLE = 15, [IN] */ CLASS_CLOSE_LIKE_CHARACTER
,
411 /* INFIX_NUMERIC = 16, [IS] */ CLASS_CHARACTER
,
412 /* LINE_FEED = 17, [LF] */ CLASS_BREAKABLE
,
413 /* NONSTARTER = 18, [NS] */ CLASS_CLOSE_LIKE_CHARACTER
,
414 /* NUMERIC = 19, [NU] */ CLASS_NUMERIC
,
415 /* OPEN_PUNCTUATION = 20, [OP] */ CLASS_OPEN_LIKE_CHARACTER
,
416 /* POSTFIX_NUMERIC = 21, [PO] */ CLASS_CLOSE_LIKE_CHARACTER
,
417 /* PREFIX_NUMERIC = 22, [PR] */ CLASS_CHARACTER
,
418 /* QUOTATION = 23, [QU] */ CLASS_CHARACTER
,
419 /* COMPLEX_CONTEXT = 24, [SA] */ CLASS_CHARACTER
,
420 /* SURROGATE = 25, [SG] */ CLASS_CHARACTER
,
421 /* SPACE = 26, [SP] */ CLASS_BREAKABLE
,
422 /* BREAK_SYMBOLS = 27, [SY] */ CLASS_CHARACTER
,
423 /* ZWSPACE = 28, [ZW] */ CLASS_BREAKABLE
,
424 /* NEXT_LINE = 29, [NL] */ CLASS_CHARACTER
,
425 /* WORD_JOINER = 30, [WJ] */ CLASS_NON_BREAKABLE
,
426 /* H2 = 31, [H2] */ CLASS_BREAKABLE
,
427 /* H3 = 32, [H3] */ CLASS_BREAKABLE
,
428 /* JL = 33, [JL] */ CLASS_CHARACTER
,
429 /* JT = 34, [JT] */ CLASS_CHARACTER
,
430 /* JV = 35, [JV] */ CLASS_CHARACTER
,
431 /* CLOSE_PARENTHESIS = 36, [CP] */ CLASS_CLOSE_LIKE_CHARACTER
,
432 /* CONDITIONAL_JAPANESE_STARTER = 37, [CJ] */ CLASS_CLOSE
,
433 /* HEBREW_LETTER = 38, [HL] */ CLASS_CHARACTER
,
434 /* REGIONAL_INDICATOR = 39, [RI] */ CLASS_CHARACTER
,
435 /* E_BASE = 40, [EB] */ CLASS_BREAKABLE
,
436 /* E_MODIFIER = 41, [EM] */ CLASS_CHARACTER
,
437 /* ZWJ = 42, [ZWJ]*/ CLASS_CHARACTER
};
439 static_assert(U_LB_COUNT
== mozilla::ArrayLength(sUnicodeLineBreakToClass
),
440 "Gecko vs ICU LineBreak class mismatch");
442 auto cls
= GetLineBreakClass(u
);
443 MOZ_ASSERT(cls
< mozilla::ArrayLength(sUnicodeLineBreakToClass
));
445 // Overrides based on rules for the different line-break values given in
446 // https://drafts.csswg.org/css-text-3/#line-break-property
448 case LineBreakRule::Auto
:
449 // For now, just use legacy Gecko behavior.
450 // XXX Possible enhancement - vary strictness according to line width
451 // or other criteria.
453 case LineBreakRule::Strict
:
454 if (cls
== U_LB_CONDITIONAL_JAPANESE_STARTER
||
455 (u
== 0x3095 || u
== 0x3096 || u
== 0x30f5 || u
== 0x30f6)) {
458 if (cls
== U_LB_INSEPARABLE
) {
459 return CLASS_NON_BREAKABLE_BETWEEN_SAME_CLASS
;
461 if (u
== 0x3005 || u
== 0x303B || u
== 0x309D || u
== 0x309E ||
462 u
== 0x30FD || u
== 0x30FE) {
463 return CLASS_CLOSE_LIKE_CHARACTER
;
465 if (aIsChineseOrJapanese
) {
466 if (cls
== U_LB_POSTFIX_NUMERIC
&&
467 UnicodeProperties::IsEastAsianWidthAFW(u
)) {
468 return CLASS_CLOSE_LIKE_CHARACTER
;
470 if (cls
== U_LB_PREFIX_NUMERIC
&&
471 UnicodeProperties::IsEastAsianWidthAFW(u
)) {
472 return CLASS_OPEN_LIKE_CHARACTER
;
474 if (u
== 0x2010 || u
== 0x2013 || u
== 0x301C || u
== 0x30A0) {
475 return CLASS_CLOSE_LIKE_CHARACTER
;
479 case LineBreakRule::Normal
:
480 if (cls
== U_LB_CONDITIONAL_JAPANESE_STARTER
) {
481 return CLASS_BREAKABLE
;
483 if (cls
== U_LB_INSEPARABLE
) {
484 return CLASS_NON_BREAKABLE_BETWEEN_SAME_CLASS
;
486 if (u
== 0x3005 || u
== 0x303B || u
== 0x309D || u
== 0x309E ||
487 u
== 0x30FD || u
== 0x30FE) {
488 return CLASS_CLOSE_LIKE_CHARACTER
;
490 if (aIsChineseOrJapanese
) {
491 if (cls
== U_LB_POSTFIX_NUMERIC
&&
492 UnicodeProperties::IsEastAsianWidthAFW(u
)) {
493 return CLASS_CLOSE_LIKE_CHARACTER
;
495 if (cls
== U_LB_PREFIX_NUMERIC
&&
496 UnicodeProperties::IsEastAsianWidthAFW(u
)) {
497 return CLASS_OPEN_LIKE_CHARACTER
;
499 if (u
== 0x2010 || u
== 0x2013 || u
== 0x301C || u
== 0x30A0) {
500 return CLASS_BREAKABLE
;
504 case LineBreakRule::Loose
:
505 if (cls
== U_LB_CONDITIONAL_JAPANESE_STARTER
) {
506 return CLASS_BREAKABLE
;
508 if (u
== 0x3005 || u
== 0x303B || u
== 0x309D || u
== 0x309E ||
509 u
== 0x30FD || u
== 0x30FE) {
510 return CLASS_BREAKABLE
;
512 if (cls
== U_LB_INSEPARABLE
) {
513 return CLASS_BREAKABLE
;
515 if (aIsChineseOrJapanese
) {
516 if (u
== 0x30FB || u
== 0xFF1A || u
== 0xFF1B || u
== 0xFF65 ||
517 u
== 0x203C || u
== 0x2047 || u
== 0x2048 || u
== 0x2049 ||
518 u
== 0xFF01 || u
== 0xFF1F) {
519 return CLASS_BREAKABLE
;
521 if (cls
== U_LB_POSTFIX_NUMERIC
&&
522 UnicodeProperties::IsEastAsianWidthAFW(u
)) {
523 return CLASS_BREAKABLE
;
525 if (cls
== U_LB_PREFIX_NUMERIC
&&
526 UnicodeProperties::IsEastAsianWidthAFW(u
)) {
527 return CLASS_BREAKABLE
;
529 if (u
== 0x2010 || u
== 0x2013 || u
== 0x301C || u
== 0x30A0) {
530 return CLASS_BREAKABLE
;
534 case LineBreakRule::Anywhere
:
535 MOZ_ASSERT_UNREACHABLE("should have been handled already");
540 uint16_t h
= u
& 0xFF00;
541 uint16_t l
= u
& 0x00ff;
543 // Handle 3 range table first
545 return GETCLASSFROMTABLE(gLBClass00
, l
);
548 return GETCLASSFROMTABLE(gLBClass17
, l
);
550 if (NS_NeedsPlatformNativeHandling(u
)) {
551 return CLASS_COMPLEX
;
554 return GETCLASSFROMTABLE(gLBClass0E
, l
);
557 return GETCLASSFROMTABLE(gLBClass20
, l
);
560 return GETCLASSFROMTABLE(gLBClass21
, l
);
563 return GETCLASSFROMTABLE(gLBClass30
, l
);
566 if (l
<= 0x0060) { // Fullwidth ASCII variant
567 // Previously, we treated Fullwidth chars the same as their ASCII
568 // counterparts, but UAX#14 (LineBreak.txt) disagrees with this and
569 // treats many of them as ideograph-like.
570 return sUnicodeLineBreakToClass
[cls
];
572 if (l
< 0x00a0) { // Halfwidth Katakana variants
575 return GetClass(0x3002, aLevel
, aIsChineseOrJapanese
);
577 return GetClass(0x300c, aLevel
, aIsChineseOrJapanese
);
579 return GetClass(0x300d, aLevel
, aIsChineseOrJapanese
);
581 return GetClass(0x3001, aLevel
, aIsChineseOrJapanese
);
583 return GetClass(0x30fb, aLevel
, aIsChineseOrJapanese
);
585 return GetClass(0x309b, aLevel
, aIsChineseOrJapanese
);
587 return GetClass(0x309c, aLevel
, aIsChineseOrJapanese
);
589 if (IS_HALFWIDTH_IN_JISx4051_CLASS3(u
)) {
590 return CLASS_CLOSE
; // jis x4051 class 3
592 return CLASS_BREAKABLE
; // jis x4051 class 11
596 return CLASS_CHARACTER
; // Halfwidth Hangul variants
599 static char16_t NarrowFFEx
[16] = {
600 0x00A2, 0x00A3, 0x00AC, 0x00AF, 0x00A6, 0x00A5, 0x20A9, 0x0000,
601 0x2502, 0x2190, 0x2191, 0x2192, 0x2193, 0x25A0, 0x25CB, 0x0000};
602 return GetClass(NarrowFFEx
[l
- 0x00e0], aLevel
, aIsChineseOrJapanese
);
604 } else if (0x3100 == h
) {
605 if (l
<= 0xbf) { // Hangul Compatibility Jamo, Bopomofo, Kanbun
606 // XXX: This is per UAX #14, but UAX #14 may change
607 // the line breaking rules about Kanbun and Bopomofo.
608 return CLASS_BREAKABLE
;
610 if (l
>= 0xf0) { // Katakana small letters for Ainu
613 } else if (0x0300 == h
) {
614 if (0x4F == l
|| (0x5C <= l
&& l
<= 0x62)) {
615 return CLASS_NON_BREAKABLE
;
617 } else if (0x0500 == h
) {
618 // ARMENIAN HYPHEN (for "Breaking Hyphens" of UAX#14)
620 return GETCLASSFROMTABLE(gLBClass00
, uint16_t(U_HYPHEN
));
622 } else if (0x0F00 == h
) {
623 // We treat Tibetan TSHEG as a hyphen (when not using platform breaker);
624 // other Tibetan chars with LineBreak class=BA will be handled by the
625 // default sUnicodeLineBreakToClass mapping below.
627 return GETCLASSFROMTABLE(gLBClass00
, uint16_t(U_HYPHEN
));
629 } else if (0x1800 == h
) {
631 return CLASS_NON_BREAKABLE
;
633 } else if (0x1600 == h
) {
634 if (0x80 == l
) { // U+1680 OGHAM SPACE MARK
635 return CLASS_BREAKABLE
;
637 } else if (u
== 0xfeff) {
638 return CLASS_NON_BREAKABLE
;
642 return sUnicodeLineBreakToClass
[cls
];
645 static bool GetPair(int8_t c1
, int8_t c2
) {
646 NS_ASSERTION(c1
< MAX_CLASSES
, "illegal classes 1");
647 NS_ASSERTION(c2
< MAX_CLASSES
, "illegal classes 2");
649 return (0 == ((gPair
[c1
] >> c2
) & 0x0001));
652 static bool GetPairConservative(int8_t c1
, int8_t c2
) {
653 NS_ASSERTION(c1
< MAX_CLASSES
, "illegal classes 1");
654 NS_ASSERTION(c2
< MAX_CLASSES
, "illegal classes 2");
656 return (0 == ((gPairConservative
[c1
] >> c2
) & 0x0001));
661 ContextState(const char16_t
* aText
, uint32_t aLength
)
662 : mUniText(aText
), mText(nullptr), mLength(aLength
) {
666 ContextState(const uint8_t* aText
, uint32_t aLength
)
667 : mUniText(nullptr), mText(aText
), mLength(aLength
) {
671 uint32_t Length() const { return mLength
; }
672 uint32_t Index() const { return mIndex
; }
674 // This gets a single code unit of the text, without checking for surrogates
675 // (in the case of a 16-bit text buffer). That's OK if we're only checking for
676 // specific characters that are known to be BMP values.
677 char16_t
GetCodeUnitAt(uint32_t aIndex
) const {
678 MOZ_ASSERT(aIndex
< mLength
, "Out of range!");
679 return mUniText
? mUniText
[aIndex
] : char16_t(mText
[aIndex
]);
682 // This gets a 32-bit Unicode character (codepoint), handling surrogate pairs
683 // as necessary. It must ONLY be called for 16-bit text, not 8-bit.
684 char32_t
GetUnicodeCharAt(uint32_t aIndex
) const {
685 MOZ_ASSERT(mUniText
, "Only for 16-bit text!");
686 MOZ_ASSERT(aIndex
< mLength
, "Out of range!");
687 char32_t c
= mUniText
[aIndex
];
688 if (aIndex
+ 1 < mLength
&& NS_IS_SURROGATE_PAIR(c
, mUniText
[aIndex
+ 1])) {
689 c
= SURROGATE_TO_UCS4(c
, mUniText
[aIndex
+ 1]);
694 void AdvanceIndex() { ++mIndex
; }
696 void NotifyBreakBefore() { mLastBreakIndex
= mIndex
; }
698 // A word of western language should not be broken. But even if the word has
699 // only ASCII characters, non-natural context words should be broken, e.g.,
700 // URL and file path. For protecting the natural words, we should use
701 // conservative breaking rules at following conditions:
702 // 1. at near the start of word
703 // 2. at near the end of word
704 // 3. at near the latest broken point
705 // CONSERVATIVE_RANGE_{LETTER,OTHER} define the 'near' in characters,
706 // which varies depending whether we are looking at a letter or a non-letter
707 // character: for non-letters, we use an extended "conservative" range.
709 #define CONSERVATIVE_RANGE_LETTER 2
710 #define CONSERVATIVE_RANGE_OTHER 6
712 bool UseConservativeBreaking(uint32_t aOffset
= 0) const {
713 if (mHasCJKChar
) return false;
714 uint32_t index
= mIndex
+ aOffset
;
716 // If the character at index is a letter (rather than various punctuation
717 // characters, etc) then we want a shorter "conservative" range
718 uint32_t conservativeRangeStart
, conservativeRangeEnd
;
719 if (index
< mLength
&&
720 nsUGenCategory::kLetter
==
721 (mText
? GetGenCategory(mText
[index
])
722 : GetGenCategory(GetUnicodeCharAt(index
)))) {
723 // Primarily for hyphenated word prefixes/suffixes; we add 1 to Start
724 // to get more balanced behavior (if we break off a 2-letter prefix,
725 // that means the break will actually be three letters from start of
726 // word, to include the hyphen; whereas a 2-letter suffix will be
727 // broken only two letters from end of word).
728 conservativeRangeEnd
= CONSERVATIVE_RANGE_LETTER
;
729 conservativeRangeStart
= CONSERVATIVE_RANGE_LETTER
+ 1;
731 conservativeRangeEnd
= conservativeRangeStart
= CONSERVATIVE_RANGE_OTHER
;
734 bool result
= (index
< conservativeRangeStart
||
735 mLength
- index
< conservativeRangeEnd
||
736 index
- mLastBreakIndex
< conservativeRangeStart
);
737 if (result
|| !mHasNonbreakableSpace
) return result
;
739 // This text has no-breakable space, we need to check whether the index
742 // Note that index is always larger than conservativeRange here.
743 for (uint32_t i
= index
; index
- conservativeRangeStart
< i
; --i
) {
744 if (IS_NONBREAKABLE_SPACE(GetCodeUnitAt(i
- 1))) return true;
746 // Note that index is always less than mLength - conservativeRange.
747 for (uint32_t i
= index
+ 1; i
< index
+ conservativeRangeEnd
; ++i
) {
748 if (IS_NONBREAKABLE_SPACE(GetCodeUnitAt(i
))) return true;
753 bool HasPreviousEqualsSign() const { return mHasPreviousEqualsSign
; }
754 void NotifySeenEqualsSign() { mHasPreviousEqualsSign
= true; }
756 bool HasPreviousSlash() const { return mHasPreviousSlash
; }
757 void NotifySeenSlash() { mHasPreviousSlash
= true; }
759 bool HasPreviousBackslash() const { return mHasPreviousBackslash
; }
760 void NotifySeenBackslash() { mHasPreviousBackslash
= true; }
762 uint32_t GetPreviousNonHyphenCharacter() const {
763 return mPreviousNonHyphenCharacter
;
765 void NotifyNonHyphenCharacter(uint32_t ch
) {
766 mPreviousNonHyphenCharacter
= ch
;
773 mPreviousNonHyphenCharacter
= U_NULL
;
775 mHasNonbreakableSpace
= false;
776 mHasPreviousEqualsSign
= false;
777 mHasPreviousSlash
= false;
778 mHasPreviousBackslash
= false;
781 // 8-bit text: we only need to check for
782 for (uint32_t i
= 0; i
< mLength
; ++i
) {
783 if (IS_NONBREAKABLE_SPACE(mText
[i
])) {
784 mHasNonbreakableSpace
= true;
789 // 16-bit text: handle surrogates and check for CJK as well as
790 for (uint32_t i
= 0; i
< mLength
; ++i
) {
791 char32_t u
= GetUnicodeCharAt(i
);
792 if (!mHasNonbreakableSpace
&& IS_NONBREAKABLE_SPACE(u
)) {
793 mHasNonbreakableSpace
= true;
797 } else if (!mHasCJKChar
&& IS_CJK_CHAR(u
)) {
799 if (mHasNonbreakableSpace
) {
804 ++i
; // step over trailing low surrogate
810 const char16_t
* const mUniText
;
811 const uint8_t* const mText
;
814 const uint32_t mLength
; // length of text
815 uint32_t mLastBreakIndex
;
816 char32_t mPreviousNonHyphenCharacter
; // The last character we have seen
817 // which is not U_HYPHEN
818 bool mHasCJKChar
; // if the text has CJK character, this is true.
819 bool mHasNonbreakableSpace
; // if the text has no-breakable space,
821 bool mHasPreviousEqualsSign
; // True if we have seen a U_EQUAL
822 bool mHasPreviousSlash
; // True if we have seen a U_SLASH
823 bool mHasPreviousBackslash
; // True if we have seen a U_BACKSLASH
826 static int8_t ContextualAnalysis(char32_t prev
, char32_t cur
, char32_t next
,
827 ContextState
& aState
, LineBreakRule aLevel
,
828 bool aIsChineseOrJapanese
) {
829 // Don't return CLASS_OPEN/CLASS_CLOSE if aState.UseJISX4051 is FALSE.
831 if (IS_HYPHEN(cur
)) {
832 // If next character is hyphen, we don't need to break between them.
833 if (IS_HYPHEN(next
)) return CLASS_CHARACTER
;
834 // If prev and next characters are numeric, it may be in Math context.
835 // So, we should not break here.
836 bool prevIsNum
= IS_ASCII_DIGIT(prev
);
837 bool nextIsNum
= IS_ASCII_DIGIT(next
);
838 if (prevIsNum
&& nextIsNum
) return CLASS_NUMERIC
;
839 // If one side is numeric and the other is a character, or if both sides are
840 // characters, the hyphen should be breakable.
841 if (!aState
.UseConservativeBreaking(1)) {
842 char32_t prevOfHyphen
= aState
.GetPreviousNonHyphenCharacter();
843 if (prevOfHyphen
&& next
) {
844 int8_t prevClass
= GetClass(prevOfHyphen
, aLevel
, aIsChineseOrJapanese
);
845 int8_t nextClass
= GetClass(next
, aLevel
, aIsChineseOrJapanese
);
846 bool prevIsNumOrCharOrClose
=
848 (prevClass
== CLASS_CHARACTER
&&
849 !NEED_CONTEXTUAL_ANALYSIS(prevOfHyphen
)) ||
850 prevClass
== CLASS_CLOSE
|| prevClass
== CLASS_CLOSE_LIKE_CHARACTER
;
851 bool nextIsNumOrCharOrOpen
=
853 (nextClass
== CLASS_CHARACTER
&& !NEED_CONTEXTUAL_ANALYSIS(next
)) ||
854 nextClass
== CLASS_OPEN
|| nextClass
== CLASS_OPEN_LIKE_CHARACTER
||
855 next
== U_OPEN_SINGLE_QUOTE
|| next
== U_OPEN_DOUBLE_QUOTE
||
856 next
== U_OPEN_GUILLEMET
;
857 if (prevIsNumOrCharOrClose
&& nextIsNumOrCharOrOpen
) {
863 aState
.NotifyNonHyphenCharacter(cur
);
864 if (cur
== U_SLASH
|| cur
== U_BACKSLASH
) {
865 // If this is immediately after same char, we should not break here.
866 if (prev
== cur
) return CLASS_CHARACTER
;
867 // If this text has two or more (BACK)SLASHs, this may be file path or
868 // URL. Make sure to compute shouldReturn before we notify on this slash.
869 bool shouldReturn
= !aState
.UseConservativeBreaking() &&
870 (cur
== U_SLASH
? aState
.HasPreviousSlash()
871 : aState
.HasPreviousBackslash());
873 if (cur
== U_SLASH
) {
874 aState
.NotifySeenSlash();
876 aState
.NotifySeenBackslash();
879 if (shouldReturn
) return CLASS_OPEN
;
880 } else if (cur
== U_PERCENT
) {
881 // If this is a part of the param of URL, we should break before.
882 if (!aState
.UseConservativeBreaking()) {
883 if (aState
.Index() >= 3 &&
884 aState
.GetCodeUnitAt(aState
.Index() - 3) == U_PERCENT
)
886 if (aState
.Index() + 3 < aState
.Length() &&
887 aState
.GetCodeUnitAt(aState
.Index() + 3) == U_PERCENT
)
890 } else if (cur
== U_AMPERSAND
|| cur
== U_SEMICOLON
) {
891 // If this may be a separator of params of URL, we should break after.
892 if (!aState
.UseConservativeBreaking(1) && aState
.HasPreviousEqualsSign())
894 } else if (cur
== U_OPEN_SINGLE_QUOTE
|| cur
== U_OPEN_DOUBLE_QUOTE
||
895 cur
== U_OPEN_GUILLEMET
) {
896 // for CJK usage, we treat these as openers to allow a break before them,
897 // but otherwise treat them as normal characters because quote mark usage
898 // in various Western languages varies too much; see bug #450088
900 if (!aState
.UseConservativeBreaking() && IS_CJK_CHAR(next
))
903 NS_ERROR("Forgot to handle the current character!");
906 return GetClass(cur
, aLevel
, aIsChineseOrJapanese
);
909 int32_t LineBreaker::Next(const char16_t
* aText
, uint32_t aLen
, uint32_t aPos
) {
913 return NS_LINEBREAKER_NEED_MORE_TEXT
;
916 bool textNeedsComplexLineBreak
= false;
919 for (begin
= aPos
; begin
> 0 && !NS_IsSpace(aText
[begin
- 1]); --begin
) {
920 if (IS_CJK_CHAR(aText
[begin
]) ||
921 NS_NeedsPlatformNativeHandling(aText
[begin
])) {
922 textNeedsComplexLineBreak
= true;
925 for (end
= aPos
+ 1; end
< int32_t(aLen
) && !NS_IsSpace(aText
[end
]); ++end
) {
926 if (IS_CJK_CHAR(aText
[end
]) || NS_NeedsPlatformNativeHandling(aText
[end
])) {
927 textNeedsComplexLineBreak
= true;
932 if (!textNeedsComplexLineBreak
) {
933 // No complex text character, do not try to do complex line break.
934 // (This is required for serializers. See Bug #344816.)
937 AutoTArray
<uint8_t, 2000> breakState
;
938 // XXX(Bug 1631371) Check if this should use a fallible operation as it
939 // pretended earlier.
940 breakState
.AppendElements(end
- begin
);
941 ComputeBreakPositions(aText
+ begin
, end
- begin
, WordBreakRule::Normal
,
942 LineBreakRule::Auto
, false, breakState
.Elements());
947 } while (begin
< ret
&& ret
< end
&& !breakState
[ret
- begin
]);
953 static bool SuppressBreakForKeepAll(uint32_t aPrev
, uint32_t aCh
) {
954 auto affectedByKeepAll
= [](uint8_t aLBClass
) {
956 // Per https://drafts.csswg.org/css-text-3/#valdef-word-break-keep-all:
957 // "implicit soft wrap opportunities between typographic letter units
958 // (or other typographic character units belonging to the NU, AL, AI,
959 // or ID Unicode line breaking classes [UAX14]) are suppressed..."
960 case U_LB_ALPHABETIC
:
963 case U_LB_IDEOGRAPHIC
:
964 // Additional classes that should be treated similarly, but have been
965 // broken out as separate classes in newer Unicode versions:
971 case U_LB_CONDITIONAL_JAPANESE_STARTER
:
977 return affectedByKeepAll(GetLineBreakClass(aPrev
)) &&
978 affectedByKeepAll(GetLineBreakClass(aCh
));
981 void LineBreaker::ComputeBreakPositions(
982 const char16_t
* aChars
, uint32_t aLength
, WordBreakRule aWordBreak
,
983 LineBreakRule aLevel
, bool aIsChineseOrJapanese
, uint8_t* aBreakBefore
) {
985 int8_t lastClass
= CLASS_NONE
;
986 ContextState
state(aChars
, aLength
);
988 for (cur
= 0; cur
< aLength
; ++cur
, state
.AdvanceIndex()) {
989 char32_t ch
= state
.GetUnicodeCharAt(cur
);
990 uint32_t chLen
= ch
> 0xFFFFu
? 2 : 1;
993 auto prev
= [=]() -> char32_t
{
997 char32_t c
= aChars
[cur
- 1];
998 if (cur
> 1 && NS_IS_SURROGATE_PAIR(aChars
[cur
- 2], c
)) {
999 c
= SURROGATE_TO_UCS4(aChars
[cur
- 2], c
);
1004 if (NEED_CONTEXTUAL_ANALYSIS(ch
)) {
1006 if (cur
+ chLen
< aLength
) {
1007 next
= state
.GetUnicodeCharAt(cur
+ chLen
);
1011 cl
= ContextualAnalysis(prev(), ch
, next
, state
, aLevel
,
1012 aIsChineseOrJapanese
);
1014 if (ch
== U_EQUAL
) state
.NotifySeenEqualsSign();
1015 state
.NotifyNonHyphenCharacter(ch
);
1016 cl
= GetClass(ch
, aLevel
, aIsChineseOrJapanese
);
1019 // To implement word-break:break-all, we overwrite the line-break class of
1020 // alphanumeric characters so they are treated the same as ideographic.
1021 // The relevant characters will have been assigned CLASS_CHARACTER, _CLOSE,
1022 // _CLOSE_LIKE_CHARACTER, or _NUMERIC by GetClass(), but those classes also
1023 // include others that we don't want to touch here, so we re-check the
1024 // Unicode line-break class to determine which ones to modify.
1025 if (aWordBreak
== WordBreakRule::BreakAll
&&
1026 (cl
== CLASS_CHARACTER
|| cl
== CLASS_CLOSE
||
1027 cl
== CLASS_CLOSE_LIKE_CHARACTER
|| cl
== CLASS_NUMERIC
)) {
1028 auto cls
= GetLineBreakClass(ch
);
1029 if (cls
== U_LB_ALPHABETIC
|| cls
== U_LB_NUMERIC
||
1030 cls
== U_LB_AMBIGUOUS
|| cls
== U_LB_COMPLEX_CONTEXT
||
1031 /* Additional Japanese and Korean LB classes; CSS Text spec doesn't
1032 explicitly mention these, but this appears to give expected
1033 behavior (spec issue?) */
1034 cls
== U_LB_CONDITIONAL_JAPANESE_STARTER
||
1035 (cls
>= U_LB_H2
&& cls
<= U_LB_JV
)) {
1036 cl
= CLASS_BREAKABLE
;
1040 bool allowBreak
= false;
1042 NS_ASSERTION(CLASS_COMPLEX
!= lastClass
|| CLASS_COMPLEX
!= cl
,
1043 "Loop should have prevented adjacent complex chars here");
1045 (state
.UseConservativeBreaking() ? GetPairConservative(lastClass
, cl
)
1046 : GetPair(lastClass
, cl
));
1047 // Special cases where a normally-allowed break is suppressed:
1049 // word-break:keep-all suppresses breaks between certain line-break
1051 if (aWordBreak
== WordBreakRule::KeepAll
&&
1052 SuppressBreakForKeepAll(prev(), ch
)) {
1055 // We also don't allow a break within a run of U+3000 chars unless
1056 // word-break:break-all is in effect.
1057 if (ch
== 0x3000 && prev() == 0x3000 &&
1058 aWordBreak
!= WordBreakRule::BreakAll
) {
1063 aBreakBefore
[cur
] = allowBreak
;
1064 if (allowBreak
) state
.NotifyBreakBefore();
1066 if (CLASS_COMPLEX
== cl
) {
1067 uint32_t end
= cur
+ chLen
;
1069 while (end
< aLength
) {
1070 char32_t c
= state
.GetUnicodeCharAt(end
);
1071 if (CLASS_COMPLEX
!= GetClass(c
, aLevel
, false)) {
1075 if (c
> 0xFFFFU
) { // it was a surrogate pair
1080 if (aWordBreak
== WordBreakRule::BreakAll
) {
1081 // For break-all, we don't need to run a dictionary-based breaking
1082 // algorithm, we just allow breaks between all grapheme clusters.
1083 GraphemeClusterBreakIteratorUtf16
ci(
1084 Span
<const char16_t
>(aChars
+ cur
, end
- cur
));
1085 while (Maybe
<uint32_t> pos
= ci
.Next()) {
1086 aBreakBefore
[cur
+ *pos
] = true;
1089 ComplexBreaker::GetBreaks(aChars
+ cur
, end
- cur
, aBreakBefore
+ cur
);
1090 // restore breakability at chunk begin, which was always set to false
1091 // by the complex line breaker
1092 aBreakBefore
[cur
] = allowBreak
;
1099 // Supplementary-plane character: mark that we cannot break before the
1100 // trailing low surrogate, and advance past it.
1102 aBreakBefore
[cur
] = false;
1103 state
.AdvanceIndex();
1108 void LineBreaker::ComputeBreakPositions(const uint8_t* aChars
, uint32_t aLength
,
1109 WordBreakRule aWordBreak
,
1110 LineBreakRule aLevel
,
1111 bool aIsChineseOrJapanese
,
1112 uint8_t* aBreakBefore
) {
1114 int8_t lastClass
= CLASS_NONE
;
1115 ContextState
state(aChars
, aLength
);
1117 for (cur
= 0; cur
< aLength
; ++cur
, state
.AdvanceIndex()) {
1118 char32_t ch
= aChars
[cur
];
1121 if (NEED_CONTEXTUAL_ANALYSIS(ch
)) {
1122 cl
= ContextualAnalysis(cur
> 0 ? aChars
[cur
- 1] : U_NULL
, ch
,
1123 cur
+ 1 < aLength
? aChars
[cur
+ 1] : U_NULL
,
1124 state
, aLevel
, aIsChineseOrJapanese
);
1126 if (ch
== U_EQUAL
) state
.NotifySeenEqualsSign();
1127 state
.NotifyNonHyphenCharacter(ch
);
1128 cl
= GetClass(ch
, aLevel
, aIsChineseOrJapanese
);
1130 if (aWordBreak
== WordBreakRule::BreakAll
&&
1131 (cl
== CLASS_CHARACTER
|| cl
== CLASS_CLOSE
||
1132 cl
== CLASS_CLOSE_LIKE_CHARACTER
|| cl
== CLASS_NUMERIC
)) {
1133 auto cls
= GetLineBreakClass(ch
);
1134 // Don't need to check additional Japanese/Korean classes in 8-bit
1135 if (cls
== U_LB_ALPHABETIC
|| cls
== U_LB_NUMERIC
||
1136 cls
== U_LB_COMPLEX_CONTEXT
) {
1137 cl
= CLASS_BREAKABLE
;
1141 bool allowBreak
= false;
1144 (state
.UseConservativeBreaking() ? GetPairConservative(lastClass
, cl
)
1145 : GetPair(lastClass
, cl
)) &&
1146 (aWordBreak
!= WordBreakRule::KeepAll
||
1147 !SuppressBreakForKeepAll(aChars
[cur
- 1], ch
));
1149 aBreakBefore
[cur
] = allowBreak
;
1150 if (allowBreak
) state
.NotifyBreakBefore();