1 #include "git-compat-util.h"
5 /* This code is originally from http://www.cl.cam.ac.uk/~mgk25/ucs/ */
7 static const char utf16_be_bom
[] = {'\xFE', '\xFF'};
8 static const char utf16_le_bom
[] = {'\xFF', '\xFE'};
9 static const char utf32_be_bom
[] = {'\0', '\0', '\xFE', '\xFF'};
10 static const char utf32_le_bom
[] = {'\xFF', '\xFE', '\0', '\0'};
17 size_t display_mode_esc_sequence_len(const char *s
)
24 while (isdigit(*p
) || *p
== ';')
31 /* auxiliary function for binary search in interval table */
32 static int bisearch(ucs_char_t ucs
, const struct interval
*table
, int max
)
37 if (ucs
< table
[0].first
|| ucs
> table
[max
].last
)
40 mid
= min
+ (max
- min
) / 2;
41 if (ucs
> table
[mid
].last
)
43 else if (ucs
< table
[mid
].first
)
52 /* The following two functions define the column width of an ISO 10646
53 * character as follows:
55 * - The null character (U+0000) has a column width of 0.
57 * - Other C0/C1 control characters and DEL will lead to a return
60 * - Non-spacing and enclosing combining characters (general
61 * category code Mn or Me in the Unicode database) have a
64 * - SOFT HYPHEN (U+00AD) has a column width of 1.
66 * - Other format characters (general category code Cf in the Unicode
67 * database) and ZERO WIDTH SPACE (U+200B) have a column width of 0.
69 * - Hangul Jamo medial vowels and final consonants (U+1160-U+11FF)
70 * have a column width of 0.
72 * - Spacing characters in the East Asian Wide (W) or East Asian
73 * Full-width (F) category as defined in Unicode Technical
74 * Report #11 have a column width of 2.
76 * - All remaining characters (including all printable
77 * ISO 8859-1 and WGL4 characters, Unicode control characters,
78 * etc.) have a column width of 1.
80 * This implementation assumes that ucs_char_t characters are encoded
84 static int git_wcwidth(ucs_char_t ch
)
87 * Sorted list of non-overlapping intervals of non-spacing characters,
89 #include "unicode-width.h"
91 /* test for 8-bit control characters */
94 if (ch
< 32 || (ch
>= 0x7f && ch
< 0xa0))
97 /* binary search in table of non-spacing characters */
98 if (bisearch(ch
, zero_width
, sizeof(zero_width
)
99 / sizeof(struct interval
) - 1))
102 /* binary search in table of double width characters */
103 if (bisearch(ch
, double_width
, sizeof(double_width
)
104 / sizeof(struct interval
) - 1))
111 * Pick one ucs character starting from the location *start points at,
112 * and return it, while updating the *start pointer to point at the
113 * end of that character. When remainder_p is not NULL, the location
114 * holds the number of bytes remaining in the string that we are allowed
115 * to pick from. Otherwise we are allowed to pick up to the NUL that
116 * would eventually appear in the string. *remainder_p is also reduced
117 * by the number of bytes we have consumed.
119 * If the string was not a valid UTF-8, *start pointer is set to NULL
120 * and the return value is undefined.
122 static ucs_char_t
pick_one_utf8_char(const char **start
, size_t *remainder_p
)
124 unsigned char *s
= (unsigned char *)*start
;
126 size_t remainder
, incr
;
129 * A caller that assumes NUL terminated text can choose
130 * not to bother with the remainder length. We will
131 * stop at the first NUL.
133 remainder
= (remainder_p
? *remainder_p
: 999);
137 } else if (*s
< 0x80) {
141 } else if ((s
[0] & 0xe0) == 0xc0) {
142 /* 110XXXXx 10xxxxxx */
144 (s
[1] & 0xc0) != 0x80 ||
145 (s
[0] & 0xfe) == 0xc0)
147 ch
= ((s
[0] & 0x1f) << 6) | (s
[1] & 0x3f);
149 } else if ((s
[0] & 0xf0) == 0xe0) {
150 /* 1110XXXX 10Xxxxxx 10xxxxxx */
152 (s
[1] & 0xc0) != 0x80 ||
153 (s
[2] & 0xc0) != 0x80 ||
155 (s
[0] == 0xe0 && (s
[1] & 0xe0) == 0x80) ||
157 (s
[0] == 0xed && (s
[1] & 0xe0) == 0xa0) ||
158 /* U+FFFE or U+FFFF? */
159 (s
[0] == 0xef && s
[1] == 0xbf &&
160 (s
[2] & 0xfe) == 0xbe))
162 ch
= ((s
[0] & 0x0f) << 12) |
163 ((s
[1] & 0x3f) << 6) | (s
[2] & 0x3f);
165 } else if ((s
[0] & 0xf8) == 0xf0) {
166 /* 11110XXX 10XXxxxx 10xxxxxx 10xxxxxx */
168 (s
[1] & 0xc0) != 0x80 ||
169 (s
[2] & 0xc0) != 0x80 ||
170 (s
[3] & 0xc0) != 0x80 ||
172 (s
[0] == 0xf0 && (s
[1] & 0xf0) == 0x80) ||
174 (s
[0] == 0xf4 && s
[1] > 0x8f) || s
[0] > 0xf4)
176 ch
= ((s
[0] & 0x07) << 18) | ((s
[1] & 0x3f) << 12) |
177 ((s
[2] & 0x3f) << 6) | (s
[3] & 0x3f);
187 *remainder_p
= remainder
- incr
;
192 * This function returns the number of columns occupied by the character
193 * pointed to by the variable start. The pointer is updated to point at
194 * the next character. When remainder_p is not NULL, it points at the
195 * location that stores the number of remaining bytes we can use to pick
196 * a character (see pick_one_utf8_char() above).
198 int utf8_width(const char **start
, size_t *remainder_p
)
200 ucs_char_t ch
= pick_one_utf8_char(start
, remainder_p
);
203 return git_wcwidth(ch
);
207 * Returns the total number of columns required by a null-terminated
208 * string, assuming that the string is utf8. Returns strlen() instead
209 * if the string does not look like a valid utf8 string.
211 int utf8_strnwidth(const char *string
, int len
, int skip_ansi
)
214 const char *orig
= string
;
217 len
= strlen(string
);
218 while (string
&& string
< orig
+ len
) {
221 (skip
= display_mode_esc_sequence_len(string
)) != 0)
223 width
+= utf8_width(&string
, NULL
);
225 return string
? width
: len
;
228 int utf8_strwidth(const char *string
)
230 return utf8_strnwidth(string
, -1, 0);
233 int is_utf8(const char *text
)
236 if (*text
== '\n' || *text
== '\t' || *text
== '\r') {
240 utf8_width(&text
, NULL
);
247 static void strbuf_add_indented_text(struct strbuf
*buf
, const char *text
,
248 int indent
, int indent2
)
253 const char *eol
= strchrnul(text
, '\n');
256 strbuf_addchars(buf
, ' ', indent
);
257 strbuf_add(buf
, text
, eol
- text
);
264 * Wrap the text, if necessary. The variable indent is the indent for the
265 * first line, indent2 is the indent for all other lines.
266 * If indent is negative, assume that already -indent columns have been
267 * consumed (and no extra indent is necessary for the first line).
269 void strbuf_add_wrapped_text(struct strbuf
*buf
,
270 const char *text
, int indent1
, int indent2
, int width
)
272 int indent
, w
, assume_utf8
= 1;
273 const char *bol
, *space
, *start
= text
;
274 size_t orig_len
= buf
->len
;
277 strbuf_add_indented_text(buf
, text
, indent1
, indent2
);
283 w
= indent
= indent1
;
294 while ((skip
= display_mode_esc_sequence_len(text
)))
298 if (!c
|| isspace(c
)) {
299 if (w
<= width
|| !space
) {
300 const char *start
= bol
;
301 if (!c
&& text
== start
)
306 strbuf_addchars(buf
, ' ', indent
);
307 strbuf_add(buf
, start
, text
- start
);
313 else if (c
== '\n') {
315 if (*space
== '\n') {
316 strbuf_addch(buf
, '\n');
319 else if (!isalnum(*space
))
322 strbuf_addch(buf
, ' ');
329 strbuf_addch(buf
, '\n');
330 text
= bol
= space
+ isspace(*space
);
332 w
= indent
= indent2
;
337 w
+= utf8_width(&text
, NULL
);
341 strbuf_setlen(buf
, orig_len
);
351 void strbuf_add_wrapped_bytes(struct strbuf
*buf
, const char *data
, int len
,
352 int indent
, int indent2
, int width
)
354 char *tmp
= xstrndup(data
, len
);
355 strbuf_add_wrapped_text(buf
, tmp
, indent
, indent2
, width
);
359 void strbuf_utf8_replace(struct strbuf
*sb_src
, int pos
, int width
,
362 struct strbuf sb_dst
= STRBUF_INIT
;
363 char *src
= sb_src
->buf
;
364 char *end
= src
+ sb_src
->len
;
366 int w
= 0, subst_len
= 0;
369 subst_len
= strlen(subst
);
370 strbuf_grow(&sb_dst
, sb_src
->len
+ subst_len
);
377 while ((n
= display_mode_esc_sequence_len(src
))) {
387 n
= utf8_width((const char**)&src
, NULL
);
388 if (!src
) /* broken utf-8, do nothing */
390 if (n
&& w
>= pos
&& w
< pos
+ width
) {
392 memcpy(dst
, subst
, subst_len
);
399 memcpy(dst
, old
, src
- old
);
403 strbuf_setlen(&sb_dst
, dst
- sb_dst
.buf
);
404 strbuf_swap(sb_src
, &sb_dst
);
406 strbuf_release(&sb_dst
);
410 * Returns true (1) if the src encoding name matches the dst encoding
411 * name directly or one of its alternative names. E.g. UTF-16BE is the
414 static int same_utf_encoding(const char *src
, const char *dst
)
416 if (istarts_with(src
, "utf") && istarts_with(dst
, "utf")) {
417 /* src[3] or dst[3] might be '\0' */
418 int i
= (src
[3] == '-' ? 4 : 3);
419 int j
= (dst
[3] == '-' ? 4 : 3);
420 return !strcasecmp(src
+i
, dst
+j
);
425 int is_encoding_utf8(const char *name
)
429 if (same_utf_encoding("utf-8", name
))
434 int same_encoding(const char *src
, const char *dst
)
436 static const char utf8
[] = "UTF-8";
442 if (same_utf_encoding(src
, dst
))
444 return !strcasecmp(src
, dst
);
448 * Wrapper for fprintf and returns the total number of columns required
449 * for the printed string, assuming that the string is utf8.
451 int utf8_fprintf(FILE *stream
, const char *format
, ...)
453 struct strbuf buf
= STRBUF_INIT
;
457 va_start(arg
, format
);
458 strbuf_vaddf(&buf
, format
, arg
);
461 columns
= fputs(buf
.buf
, stream
);
462 if (0 <= columns
) /* keep the error from the I/O */
463 columns
= utf8_strwidth(buf
.buf
);
464 strbuf_release(&buf
);
469 * Given a buffer and its encoding, return it re-encoded
470 * with iconv. If the conversion fails, returns NULL.
473 #if defined(OLD_ICONV) || (defined(__sun__) && !defined(_XPG6))
474 typedef const char * iconv_ibp
;
476 typedef char * iconv_ibp
;
478 char *reencode_string_iconv(const char *in
, size_t insz
, iconv_t conv
,
479 size_t bom_len
, size_t *outsz_p
)
481 size_t outsz
, outalloc
;
486 outalloc
= st_add(outsz
, 1 + bom_len
); /* for terminating NUL */
487 out
= xmalloc(outalloc
);
488 outpos
= out
+ bom_len
;
492 size_t cnt
= iconv(conv
, &cp
, &insz
, &outpos
, &outsz
);
494 if (cnt
== (size_t) -1) {
496 if (errno
!= E2BIG
) {
500 /* insz has remaining number of bytes.
501 * since we started outsz the same as insz,
502 * it is likely that insz is not enough for
503 * converting the rest.
505 sofar
= outpos
- out
;
506 outalloc
= st_add3(sofar
, st_mult(insz
, 2), 32);
507 out
= xrealloc(out
, outalloc
);
508 outpos
= out
+ sofar
;
509 outsz
= outalloc
- sofar
- 1;
514 *outsz_p
= outpos
- out
;
521 static const char *fallback_encoding(const char *name
)
524 * Some platforms do not have the variously spelled variants of
525 * UTF-8, so let's fall back to trying the most official
526 * spelling. We do so only as a fallback in case the platform
527 * does understand the user's spelling, but not our official
530 if (is_encoding_utf8(name
))
534 * Even though latin-1 is still seen in e-mail
535 * headers, some platforms only install ISO-8859-1.
537 if (!strcasecmp(name
, "latin-1"))
543 char *reencode_string_len(const char *in
, size_t insz
,
544 const char *out_encoding
, const char *in_encoding
,
549 const char *bom_str
= NULL
;
555 /* UTF-16LE-BOM is the same as UTF-16 for reading */
556 if (same_utf_encoding("UTF-16LE-BOM", in_encoding
))
557 in_encoding
= "UTF-16";
560 * For writing, UTF-16 iconv typically creates "UTF-16BE-BOM"
561 * Some users under Windows want the little endian version
563 * We handle UTF-16 and UTF-32 ourselves only if the platform does not
564 * provide a BOM (which we require), since we want to match the behavior
565 * of the system tools and libc as much as possible.
567 if (same_utf_encoding("UTF-16LE-BOM", out_encoding
)) {
568 bom_str
= utf16_le_bom
;
569 bom_len
= sizeof(utf16_le_bom
);
570 out_encoding
= "UTF-16LE";
571 } else if (same_utf_encoding("UTF-16BE-BOM", out_encoding
)) {
572 bom_str
= utf16_be_bom
;
573 bom_len
= sizeof(utf16_be_bom
);
574 out_encoding
= "UTF-16BE";
575 #ifdef ICONV_OMITS_BOM
576 } else if (same_utf_encoding("UTF-16", out_encoding
)) {
577 bom_str
= utf16_be_bom
;
578 bom_len
= sizeof(utf16_be_bom
);
579 out_encoding
= "UTF-16BE";
580 } else if (same_utf_encoding("UTF-32", out_encoding
)) {
581 bom_str
= utf32_be_bom
;
582 bom_len
= sizeof(utf32_be_bom
);
583 out_encoding
= "UTF-32BE";
587 conv
= iconv_open(out_encoding
, in_encoding
);
588 if (conv
== (iconv_t
) -1) {
589 in_encoding
= fallback_encoding(in_encoding
);
590 out_encoding
= fallback_encoding(out_encoding
);
592 conv
= iconv_open(out_encoding
, in_encoding
);
593 if (conv
== (iconv_t
) -1)
596 out
= reencode_string_iconv(in
, insz
, conv
, bom_len
, outsz
);
598 if (out
&& bom_str
&& bom_len
)
599 memcpy(out
, bom_str
, bom_len
);
604 static int has_bom_prefix(const char *data
, size_t len
,
605 const char *bom
, size_t bom_len
)
607 return data
&& bom
&& (len
>= bom_len
) && !memcmp(data
, bom
, bom_len
);
610 int has_prohibited_utf_bom(const char *enc
, const char *data
, size_t len
)
613 (same_utf_encoding("UTF-16BE", enc
) ||
614 same_utf_encoding("UTF-16LE", enc
)) &&
615 (has_bom_prefix(data
, len
, utf16_be_bom
, sizeof(utf16_be_bom
)) ||
616 has_bom_prefix(data
, len
, utf16_le_bom
, sizeof(utf16_le_bom
)))
618 (same_utf_encoding("UTF-32BE", enc
) ||
619 same_utf_encoding("UTF-32LE", enc
)) &&
620 (has_bom_prefix(data
, len
, utf32_be_bom
, sizeof(utf32_be_bom
)) ||
621 has_bom_prefix(data
, len
, utf32_le_bom
, sizeof(utf32_le_bom
)))
625 int is_missing_required_utf_bom(const char *enc
, const char *data
, size_t len
)
628 (same_utf_encoding(enc
, "UTF-16")) &&
629 !(has_bom_prefix(data
, len
, utf16_be_bom
, sizeof(utf16_be_bom
)) ||
630 has_bom_prefix(data
, len
, utf16_le_bom
, sizeof(utf16_le_bom
)))
632 (same_utf_encoding(enc
, "UTF-32")) &&
633 !(has_bom_prefix(data
, len
, utf32_be_bom
, sizeof(utf32_be_bom
)) ||
634 has_bom_prefix(data
, len
, utf32_le_bom
, sizeof(utf32_le_bom
)))
639 * Returns first character length in bytes for multi-byte `text` according to
642 * - The `text` pointer is updated to point at the next character.
643 * - When `remainder_p` is not NULL, on entry `*remainder_p` is how much bytes
644 * we can consume from text, and on exit `*remainder_p` is reduced by returned
645 * character length. Otherwise `text` is treated as limited by NUL.
647 int mbs_chrlen(const char **text
, size_t *remainder_p
, const char *encoding
)
650 const char *p
= *text
;
651 size_t r
= (remainder_p
? *remainder_p
: SIZE_MAX
);
656 if (is_encoding_utf8(encoding
)) {
657 pick_one_utf8_char(&p
, &r
);
659 chrlen
= p
? (p
- *text
)
660 : 1 /* not valid UTF-8 -> raw byte sequence */;
664 * TODO use iconv to decode one char and obtain its chrlen
665 * for now, let's treat encodings != UTF-8 as one-byte
672 *remainder_p
-= chrlen
;
678 * Pick the next char from the stream, ignoring codepoints an HFS+ would.
679 * Note that this is _not_ complete by any means. It's just enough
680 * to make is_hfs_dotgit() work, and should not be used otherwise.
682 static ucs_char_t
next_hfs_char(const char **in
)
685 ucs_char_t out
= pick_one_utf8_char(in
, NULL
);
687 * check for malformed utf8. Technically this
688 * gets converted to a percent-sequence, but
689 * returning 0 is good enough for is_hfs_dotgit
690 * to realize it cannot be .git
695 /* these code points are ignored completely */
697 case 0x200c: /* ZERO WIDTH NON-JOINER */
698 case 0x200d: /* ZERO WIDTH JOINER */
699 case 0x200e: /* LEFT-TO-RIGHT MARK */
700 case 0x200f: /* RIGHT-TO-LEFT MARK */
701 case 0x202a: /* LEFT-TO-RIGHT EMBEDDING */
702 case 0x202b: /* RIGHT-TO-LEFT EMBEDDING */
703 case 0x202c: /* POP DIRECTIONAL FORMATTING */
704 case 0x202d: /* LEFT-TO-RIGHT OVERRIDE */
705 case 0x202e: /* RIGHT-TO-LEFT OVERRIDE */
706 case 0x206a: /* INHIBIT SYMMETRIC SWAPPING */
707 case 0x206b: /* ACTIVATE SYMMETRIC SWAPPING */
708 case 0x206c: /* INHIBIT ARABIC FORM SHAPING */
709 case 0x206d: /* ACTIVATE ARABIC FORM SHAPING */
710 case 0x206e: /* NATIONAL DIGIT SHAPES */
711 case 0x206f: /* NOMINAL DIGIT SHAPES */
712 case 0xfeff: /* ZERO WIDTH NO-BREAK SPACE */
720 static int is_hfs_dot_generic(const char *path
,
721 const char *needle
, size_t needle_len
)
725 c
= next_hfs_char(&path
);
730 * there's a great deal of other case-folding that occurs
731 * in HFS+, but this is enough to catch our fairly vanilla
732 * hard-coded needles.
734 for (; needle_len
> 0; needle
++, needle_len
--) {
735 c
= next_hfs_char(&path
);
738 * We know our needles contain only ASCII, so we clamp here to
739 * make the results of tolower() sane.
743 if (tolower(c
) != *needle
)
747 c
= next_hfs_char(&path
);
748 if (c
&& !is_dir_sep(c
))
755 * Inline wrapper to make sure the compiler resolves strlen() on literals at
758 static inline int is_hfs_dot_str(const char *path
, const char *needle
)
760 return is_hfs_dot_generic(path
, needle
, strlen(needle
));
763 int is_hfs_dotgit(const char *path
)
765 return is_hfs_dot_str(path
, "git");
768 int is_hfs_dotgitmodules(const char *path
)
770 return is_hfs_dot_str(path
, "gitmodules");
773 int is_hfs_dotgitignore(const char *path
)
775 return is_hfs_dot_str(path
, "gitignore");
778 int is_hfs_dotgitattributes(const char *path
)
780 return is_hfs_dot_str(path
, "gitattributes");
783 const char utf8_bom
[] = "\357\273\277";
785 int skip_utf8_bom(char **text
, size_t len
)
787 if (len
< strlen(utf8_bom
) ||
788 memcmp(*text
, utf8_bom
, strlen(utf8_bom
)))
790 *text
+= strlen(utf8_bom
);
794 void strbuf_utf8_align(struct strbuf
*buf
, align_type position
, unsigned int width
,
797 int slen
= strlen(s
);
798 int display_len
= utf8_strnwidth(s
, slen
, 0);
799 int utf8_compensation
= slen
- display_len
;
801 if (display_len
>= width
) {
802 strbuf_addstr(buf
, s
);
806 if (position
== ALIGN_LEFT
)
807 strbuf_addf(buf
, "%-*s", width
+ utf8_compensation
, s
);
808 else if (position
== ALIGN_MIDDLE
) {
809 int left
= (width
- display_len
) / 2;
810 strbuf_addf(buf
, "%*s%-*s", left
, "", width
- left
+ utf8_compensation
, s
);
811 } else if (position
== ALIGN_RIGHT
)
812 strbuf_addf(buf
, "%*s", width
+ utf8_compensation
, s
);