1 #include "git-compat-util.h"
5 /* This code is originally from https://www.cl.cam.ac.uk/~mgk25/ucs/ */
7 static const char utf16_be_bom
[] = {'\xFE', '\xFF'};
8 static const char utf16_le_bom
[] = {'\xFF', '\xFE'};
9 static const char utf32_be_bom
[] = {'\0', '\0', '\xFE', '\xFF'};
10 static const char utf32_le_bom
[] = {'\xFF', '\xFE', '\0', '\0'};
17 size_t display_mode_esc_sequence_len(const char *s
)
24 while (isdigit(*p
) || *p
== ';')
31 /* auxiliary function for binary search in interval table */
32 static int bisearch(ucs_char_t ucs
, const struct interval
*table
, int max
)
37 if (ucs
< table
[0].first
|| ucs
> table
[max
].last
)
40 mid
= min
+ (max
- min
) / 2;
41 if (ucs
> table
[mid
].last
)
43 else if (ucs
< table
[mid
].first
)
52 /* The following two functions define the column width of an ISO 10646
53 * character as follows:
55 * - The null character (U+0000) has a column width of 0.
57 * - Other C0/C1 control characters and DEL will lead to a return
60 * - Non-spacing and enclosing combining characters (general
61 * category code Mn or Me in the Unicode database) have a
64 * - SOFT HYPHEN (U+00AD) has a column width of 1.
66 * - Other format characters (general category code Cf in the Unicode
67 * database) and ZERO WIDTH SPACE (U+200B) have a column width of 0.
69 * - Hangul Jamo medial vowels and final consonants (U+1160-U+11FF)
70 * have a column width of 0.
72 * - Spacing characters in the East Asian Wide (W) or East Asian
73 * Full-width (F) category as defined in Unicode Technical
74 * Report #11 have a column width of 2.
76 * - All remaining characters (including all printable
77 * ISO 8859-1 and WGL4 characters, Unicode control characters,
78 * etc.) have a column width of 1.
80 * This implementation assumes that ucs_char_t characters are encoded
84 static int git_wcwidth(ucs_char_t ch
)
87 * Sorted list of non-overlapping intervals of non-spacing characters,
89 #include "unicode-width.h"
91 /* test for 8-bit control characters */
94 if (ch
< 32 || (ch
>= 0x7f && ch
< 0xa0))
97 /* binary search in table of non-spacing characters */
98 if (bisearch(ch
, zero_width
, ARRAY_SIZE(zero_width
) - 1))
101 /* binary search in table of double width characters */
102 if (bisearch(ch
, double_width
, ARRAY_SIZE(double_width
) - 1))
109 * Pick one ucs character starting from the location *start points at,
110 * and return it, while updating the *start pointer to point at the
111 * end of that character. When remainder_p is not NULL, the location
112 * holds the number of bytes remaining in the string that we are allowed
113 * to pick from. Otherwise we are allowed to pick up to the NUL that
114 * would eventually appear in the string. *remainder_p is also reduced
115 * by the number of bytes we have consumed.
117 * If the string was not a valid UTF-8, *start pointer is set to NULL
118 * and the return value is undefined.
120 static ucs_char_t
pick_one_utf8_char(const char **start
, size_t *remainder_p
)
122 unsigned char *s
= (unsigned char *)*start
;
124 size_t remainder
, incr
;
127 * A caller that assumes NUL terminated text can choose
128 * not to bother with the remainder length. We will
129 * stop at the first NUL.
131 remainder
= (remainder_p
? *remainder_p
: 999);
135 } else if (*s
< 0x80) {
139 } else if ((s
[0] & 0xe0) == 0xc0) {
140 /* 110XXXXx 10xxxxxx */
142 (s
[1] & 0xc0) != 0x80 ||
143 (s
[0] & 0xfe) == 0xc0)
145 ch
= ((s
[0] & 0x1f) << 6) | (s
[1] & 0x3f);
147 } else if ((s
[0] & 0xf0) == 0xe0) {
148 /* 1110XXXX 10Xxxxxx 10xxxxxx */
150 (s
[1] & 0xc0) != 0x80 ||
151 (s
[2] & 0xc0) != 0x80 ||
153 (s
[0] == 0xe0 && (s
[1] & 0xe0) == 0x80) ||
155 (s
[0] == 0xed && (s
[1] & 0xe0) == 0xa0) ||
156 /* U+FFFE or U+FFFF? */
157 (s
[0] == 0xef && s
[1] == 0xbf &&
158 (s
[2] & 0xfe) == 0xbe))
160 ch
= ((s
[0] & 0x0f) << 12) |
161 ((s
[1] & 0x3f) << 6) | (s
[2] & 0x3f);
163 } else if ((s
[0] & 0xf8) == 0xf0) {
164 /* 11110XXX 10XXxxxx 10xxxxxx 10xxxxxx */
166 (s
[1] & 0xc0) != 0x80 ||
167 (s
[2] & 0xc0) != 0x80 ||
168 (s
[3] & 0xc0) != 0x80 ||
170 (s
[0] == 0xf0 && (s
[1] & 0xf0) == 0x80) ||
172 (s
[0] == 0xf4 && s
[1] > 0x8f) || s
[0] > 0xf4)
174 ch
= ((s
[0] & 0x07) << 18) | ((s
[1] & 0x3f) << 12) |
175 ((s
[2] & 0x3f) << 6) | (s
[3] & 0x3f);
185 *remainder_p
= remainder
- incr
;
190 * This function returns the number of columns occupied by the character
191 * pointed to by the variable start. The pointer is updated to point at
192 * the next character. When remainder_p is not NULL, it points at the
193 * location that stores the number of remaining bytes we can use to pick
194 * a character (see pick_one_utf8_char() above).
196 int utf8_width(const char **start
, size_t *remainder_p
)
198 ucs_char_t ch
= pick_one_utf8_char(start
, remainder_p
);
201 return git_wcwidth(ch
);
205 * Returns the total number of columns required by a null-terminated
206 * string, assuming that the string is utf8. Returns strlen() instead
207 * if the string does not look like a valid utf8 string.
209 int utf8_strnwidth(const char *string
, size_t len
, int skip_ansi
)
211 const char *orig
= string
;
214 while (string
&& string
< orig
+ len
) {
219 (skip
= display_mode_esc_sequence_len(string
)) != 0)
222 glyph_width
= utf8_width(&string
, NULL
);
224 width
+= glyph_width
;
228 * TODO: fix the interface of this function and `utf8_strwidth()` to
229 * return `size_t` instead of `int`.
231 return cast_size_t_to_int(string
? width
: len
);
234 int utf8_strwidth(const char *string
)
236 return utf8_strnwidth(string
, strlen(string
), 0);
239 int is_utf8(const char *text
)
242 if (*text
== '\n' || *text
== '\t' || *text
== '\r') {
246 utf8_width(&text
, NULL
);
253 static void strbuf_add_indented_text(struct strbuf
*buf
, const char *text
,
254 int indent
, int indent2
)
259 const char *eol
= strchrnul(text
, '\n');
262 strbuf_addchars(buf
, ' ', indent
);
263 strbuf_add(buf
, text
, eol
- text
);
270 * Wrap the text, if necessary. The variable indent is the indent for the
271 * first line, indent2 is the indent for all other lines.
272 * If indent is negative, assume that already -indent columns have been
273 * consumed (and no extra indent is necessary for the first line).
275 void strbuf_add_wrapped_text(struct strbuf
*buf
,
276 const char *text
, int indent1
, int indent2
, int width
)
278 int indent
, w
, assume_utf8
= 1;
279 const char *bol
, *space
, *start
= text
;
280 size_t orig_len
= buf
->len
;
283 strbuf_add_indented_text(buf
, text
, indent1
, indent2
);
289 w
= indent
= indent1
;
300 while ((skip
= display_mode_esc_sequence_len(text
)))
304 if (!c
|| isspace(c
)) {
305 if (w
<= width
|| !space
) {
306 const char *start
= bol
;
307 if (!c
&& text
== start
)
312 strbuf_addchars(buf
, ' ', indent
);
313 strbuf_add(buf
, start
, text
- start
);
319 else if (c
== '\n') {
321 if (*space
== '\n') {
322 strbuf_addch(buf
, '\n');
325 else if (!isalnum(*space
))
328 strbuf_addch(buf
, ' ');
335 strbuf_addch(buf
, '\n');
336 text
= bol
= space
+ isspace(*space
);
338 w
= indent
= indent2
;
343 w
+= utf8_width(&text
, NULL
);
347 strbuf_setlen(buf
, orig_len
);
357 void strbuf_add_wrapped_bytes(struct strbuf
*buf
, const char *data
, int len
,
358 int indent
, int indent2
, int width
)
360 char *tmp
= xstrndup(data
, len
);
361 strbuf_add_wrapped_text(buf
, tmp
, indent
, indent2
, width
);
365 void strbuf_utf8_replace(struct strbuf
*sb_src
, int pos
, int width
,
368 const char *src
= sb_src
->buf
, *end
= sb_src
->buf
+ sb_src
->len
;
372 strbuf_init(&dst
, sb_src
->len
);
379 while ((n
= display_mode_esc_sequence_len(src
))) {
380 strbuf_add(&dst
, src
, n
);
388 glyph_width
= utf8_width((const char**)&src
, NULL
);
389 if (!src
) /* broken utf-8, do nothing */
393 * In case we see a control character we copy it into the
394 * buffer, but don't add it to the width.
399 if (glyph_width
&& w
>= pos
&& w
< pos
+ width
) {
401 strbuf_addstr(&dst
, subst
);
405 strbuf_add(&dst
, old
, src
- old
);
411 strbuf_swap(sb_src
, &dst
);
413 strbuf_release(&dst
);
417 * Returns true (1) if the src encoding name matches the dst encoding
418 * name directly or one of its alternative names. E.g. UTF-16BE is the
421 static int same_utf_encoding(const char *src
, const char *dst
)
423 if (skip_iprefix(src
, "utf", &src
) && skip_iprefix(dst
, "utf", &dst
)) {
424 skip_prefix(src
, "-", &src
);
425 skip_prefix(dst
, "-", &dst
);
426 return !strcasecmp(src
, dst
);
431 int is_encoding_utf8(const char *name
)
435 if (same_utf_encoding("utf-8", name
))
440 int same_encoding(const char *src
, const char *dst
)
442 static const char utf8
[] = "UTF-8";
448 if (same_utf_encoding(src
, dst
))
450 return !strcasecmp(src
, dst
);
454 * Wrapper for fprintf and returns the total number of columns required
455 * for the printed string, assuming that the string is utf8.
457 int utf8_fprintf(FILE *stream
, const char *format
, ...)
459 struct strbuf buf
= STRBUF_INIT
;
463 va_start(arg
, format
);
464 strbuf_vaddf(&buf
, format
, arg
);
467 columns
= fputs(buf
.buf
, stream
);
468 if (0 <= columns
) /* keep the error from the I/O */
469 columns
= utf8_strwidth(buf
.buf
);
470 strbuf_release(&buf
);
475 * Given a buffer and its encoding, return it re-encoded
476 * with iconv. If the conversion fails, returns NULL.
479 #if defined(OLD_ICONV) || (defined(__sun__) && !defined(_XPG6))
480 typedef const char * iconv_ibp
;
482 typedef char * iconv_ibp
;
484 char *reencode_string_iconv(const char *in
, size_t insz
, iconv_t conv
,
485 size_t bom_len
, size_t *outsz_p
)
487 size_t outsz
, outalloc
;
492 outalloc
= st_add(outsz
, 1 + bom_len
); /* for terminating NUL */
493 out
= xmalloc(outalloc
);
494 outpos
= out
+ bom_len
;
498 size_t cnt
= iconv(conv
, &cp
, &insz
, &outpos
, &outsz
);
500 if (cnt
== (size_t) -1) {
502 if (errno
!= E2BIG
) {
506 /* insz has remaining number of bytes.
507 * since we started outsz the same as insz,
508 * it is likely that insz is not enough for
509 * converting the rest.
511 sofar
= outpos
- out
;
512 outalloc
= st_add3(sofar
, st_mult(insz
, 2), 32);
513 out
= xrealloc(out
, outalloc
);
514 outpos
= out
+ sofar
;
515 outsz
= outalloc
- sofar
- 1;
520 *outsz_p
= outpos
- out
;
527 static const char *fallback_encoding(const char *name
)
530 * Some platforms do not have the variously spelled variants of
531 * UTF-8, so let's fall back to trying the most official
532 * spelling. We do so only as a fallback in case the platform
533 * does understand the user's spelling, but not our official
536 if (is_encoding_utf8(name
))
540 * Even though latin-1 is still seen in e-mail
541 * headers, some platforms only install ISO-8859-1.
543 if (!strcasecmp(name
, "latin-1"))
549 char *reencode_string_len(const char *in
, size_t insz
,
550 const char *out_encoding
, const char *in_encoding
,
555 const char *bom_str
= NULL
;
561 /* UTF-16LE-BOM is the same as UTF-16 for reading */
562 if (same_utf_encoding("UTF-16LE-BOM", in_encoding
))
563 in_encoding
= "UTF-16";
566 * For writing, UTF-16 iconv typically creates "UTF-16BE-BOM"
567 * Some users under Windows want the little endian version
569 * We handle UTF-16 and UTF-32 ourselves only if the platform does not
570 * provide a BOM (which we require), since we want to match the behavior
571 * of the system tools and libc as much as possible.
573 if (same_utf_encoding("UTF-16LE-BOM", out_encoding
)) {
574 bom_str
= utf16_le_bom
;
575 bom_len
= sizeof(utf16_le_bom
);
576 out_encoding
= "UTF-16LE";
577 } else if (same_utf_encoding("UTF-16BE-BOM", out_encoding
)) {
578 bom_str
= utf16_be_bom
;
579 bom_len
= sizeof(utf16_be_bom
);
580 out_encoding
= "UTF-16BE";
581 #ifdef ICONV_OMITS_BOM
582 } else if (same_utf_encoding("UTF-16", out_encoding
)) {
583 bom_str
= utf16_be_bom
;
584 bom_len
= sizeof(utf16_be_bom
);
585 out_encoding
= "UTF-16BE";
586 } else if (same_utf_encoding("UTF-32", out_encoding
)) {
587 bom_str
= utf32_be_bom
;
588 bom_len
= sizeof(utf32_be_bom
);
589 out_encoding
= "UTF-32BE";
593 conv
= iconv_open(out_encoding
, in_encoding
);
594 if (conv
== (iconv_t
) -1) {
595 in_encoding
= fallback_encoding(in_encoding
);
596 out_encoding
= fallback_encoding(out_encoding
);
598 conv
= iconv_open(out_encoding
, in_encoding
);
599 if (conv
== (iconv_t
) -1)
602 out
= reencode_string_iconv(in
, insz
, conv
, bom_len
, outsz
);
604 if (out
&& bom_str
&& bom_len
)
605 memcpy(out
, bom_str
, bom_len
);
610 static int has_bom_prefix(const char *data
, size_t len
,
611 const char *bom
, size_t bom_len
)
613 return data
&& bom
&& (len
>= bom_len
) && !memcmp(data
, bom
, bom_len
);
616 int has_prohibited_utf_bom(const char *enc
, const char *data
, size_t len
)
619 (same_utf_encoding("UTF-16BE", enc
) ||
620 same_utf_encoding("UTF-16LE", enc
)) &&
621 (has_bom_prefix(data
, len
, utf16_be_bom
, sizeof(utf16_be_bom
)) ||
622 has_bom_prefix(data
, len
, utf16_le_bom
, sizeof(utf16_le_bom
)))
624 (same_utf_encoding("UTF-32BE", enc
) ||
625 same_utf_encoding("UTF-32LE", enc
)) &&
626 (has_bom_prefix(data
, len
, utf32_be_bom
, sizeof(utf32_be_bom
)) ||
627 has_bom_prefix(data
, len
, utf32_le_bom
, sizeof(utf32_le_bom
)))
631 int is_missing_required_utf_bom(const char *enc
, const char *data
, size_t len
)
634 (same_utf_encoding(enc
, "UTF-16")) &&
635 !(has_bom_prefix(data
, len
, utf16_be_bom
, sizeof(utf16_be_bom
)) ||
636 has_bom_prefix(data
, len
, utf16_le_bom
, sizeof(utf16_le_bom
)))
638 (same_utf_encoding(enc
, "UTF-32")) &&
639 !(has_bom_prefix(data
, len
, utf32_be_bom
, sizeof(utf32_be_bom
)) ||
640 has_bom_prefix(data
, len
, utf32_le_bom
, sizeof(utf32_le_bom
)))
645 * Returns first character length in bytes for multi-byte `text` according to
648 * - The `text` pointer is updated to point at the next character.
649 * - When `remainder_p` is not NULL, on entry `*remainder_p` is how much bytes
650 * we can consume from text, and on exit `*remainder_p` is reduced by returned
651 * character length. Otherwise `text` is treated as limited by NUL.
653 int mbs_chrlen(const char **text
, size_t *remainder_p
, const char *encoding
)
656 const char *p
= *text
;
657 size_t r
= (remainder_p
? *remainder_p
: SIZE_MAX
);
662 if (is_encoding_utf8(encoding
)) {
663 pick_one_utf8_char(&p
, &r
);
665 chrlen
= p
? (p
- *text
)
666 : 1 /* not valid UTF-8 -> raw byte sequence */;
670 * TODO use iconv to decode one char and obtain its chrlen
671 * for now, let's treat encodings != UTF-8 as one-byte
678 *remainder_p
-= chrlen
;
684 * Pick the next char from the stream, ignoring codepoints an HFS+ would.
685 * Note that this is _not_ complete by any means. It's just enough
686 * to make is_hfs_dotgit() work, and should not be used otherwise.
688 static ucs_char_t
next_hfs_char(const char **in
)
691 ucs_char_t out
= pick_one_utf8_char(in
, NULL
);
693 * check for malformed utf8. Technically this
694 * gets converted to a percent-sequence, but
695 * returning 0 is good enough for is_hfs_dotgit
696 * to realize it cannot be .git
701 /* these code points are ignored completely */
703 case 0x200c: /* ZERO WIDTH NON-JOINER */
704 case 0x200d: /* ZERO WIDTH JOINER */
705 case 0x200e: /* LEFT-TO-RIGHT MARK */
706 case 0x200f: /* RIGHT-TO-LEFT MARK */
707 case 0x202a: /* LEFT-TO-RIGHT EMBEDDING */
708 case 0x202b: /* RIGHT-TO-LEFT EMBEDDING */
709 case 0x202c: /* POP DIRECTIONAL FORMATTING */
710 case 0x202d: /* LEFT-TO-RIGHT OVERRIDE */
711 case 0x202e: /* RIGHT-TO-LEFT OVERRIDE */
712 case 0x206a: /* INHIBIT SYMMETRIC SWAPPING */
713 case 0x206b: /* ACTIVATE SYMMETRIC SWAPPING */
714 case 0x206c: /* INHIBIT ARABIC FORM SHAPING */
715 case 0x206d: /* ACTIVATE ARABIC FORM SHAPING */
716 case 0x206e: /* NATIONAL DIGIT SHAPES */
717 case 0x206f: /* NOMINAL DIGIT SHAPES */
718 case 0xfeff: /* ZERO WIDTH NO-BREAK SPACE */
726 static int is_hfs_dot_generic(const char *path
,
727 const char *needle
, size_t needle_len
)
731 c
= next_hfs_char(&path
);
736 * there's a great deal of other case-folding that occurs
737 * in HFS+, but this is enough to catch our fairly vanilla
738 * hard-coded needles.
740 for (; needle_len
> 0; needle
++, needle_len
--) {
741 c
= next_hfs_char(&path
);
744 * We know our needles contain only ASCII, so we clamp here to
745 * make the results of tolower() sane.
749 if (tolower(c
) != *needle
)
753 c
= next_hfs_char(&path
);
754 if (c
&& !is_dir_sep(c
))
761 * Inline wrapper to make sure the compiler resolves strlen() on literals at
764 static inline int is_hfs_dot_str(const char *path
, const char *needle
)
766 return is_hfs_dot_generic(path
, needle
, strlen(needle
));
769 int is_hfs_dotgit(const char *path
)
771 return is_hfs_dot_str(path
, "git");
774 int is_hfs_dotgitmodules(const char *path
)
776 return is_hfs_dot_str(path
, "gitmodules");
779 int is_hfs_dotgitignore(const char *path
)
781 return is_hfs_dot_str(path
, "gitignore");
784 int is_hfs_dotgitattributes(const char *path
)
786 return is_hfs_dot_str(path
, "gitattributes");
789 int is_hfs_dotmailmap(const char *path
)
791 return is_hfs_dot_str(path
, "mailmap");
794 const char utf8_bom
[] = "\357\273\277";
796 int skip_utf8_bom(char **text
, size_t len
)
798 if (len
< strlen(utf8_bom
) ||
799 memcmp(*text
, utf8_bom
, strlen(utf8_bom
)))
801 *text
+= strlen(utf8_bom
);
805 void strbuf_utf8_align(struct strbuf
*buf
, align_type position
, unsigned int width
,
808 size_t slen
= strlen(s
);
809 int display_len
= utf8_strnwidth(s
, slen
, 0);
810 int utf8_compensation
= slen
- display_len
;
812 if (display_len
>= width
) {
813 strbuf_addstr(buf
, s
);
817 if (position
== ALIGN_LEFT
)
818 strbuf_addf(buf
, "%-*s", width
+ utf8_compensation
, s
);
819 else if (position
== ALIGN_MIDDLE
) {
820 int left
= (width
- display_len
) / 2;
821 strbuf_addf(buf
, "%*s%-*s", left
, "", width
- left
+ utf8_compensation
, s
);
822 } else if (position
== ALIGN_RIGHT
)
823 strbuf_addf(buf
, "%*s", width
+ utf8_compensation
, s
);