1 #include "git-compat-util.h"
5 /* This code is originally from http://www.cl.cam.ac.uk/~mgk25/ucs/ */
7 static const char utf16_be_bom
[] = {'\xFE', '\xFF'};
8 static const char utf16_le_bom
[] = {'\xFF', '\xFE'};
9 static const char utf32_be_bom
[] = {'\0', '\0', '\xFE', '\xFF'};
10 static const char utf32_le_bom
[] = {'\xFF', '\xFE', '\0', '\0'};
17 size_t display_mode_esc_sequence_len(const char *s
)
24 while (isdigit(*p
) || *p
== ';')
31 /* auxiliary function for binary search in interval table */
32 static int bisearch(ucs_char_t ucs
, const struct interval
*table
, int max
)
37 if (ucs
< table
[0].first
|| ucs
> table
[max
].last
)
40 mid
= min
+ (max
- min
) / 2;
41 if (ucs
> table
[mid
].last
)
43 else if (ucs
< table
[mid
].first
)
52 /* The following two functions define the column width of an ISO 10646
53 * character as follows:
55 * - The null character (U+0000) has a column width of 0.
57 * - Other C0/C1 control characters and DEL will lead to a return
60 * - Non-spacing and enclosing combining characters (general
61 * category code Mn or Me in the Unicode database) have a
64 * - SOFT HYPHEN (U+00AD) has a column width of 1.
66 * - Other format characters (general category code Cf in the Unicode
67 * database) and ZERO WIDTH SPACE (U+200B) have a column width of 0.
69 * - Hangul Jamo medial vowels and final consonants (U+1160-U+11FF)
70 * have a column width of 0.
72 * - Spacing characters in the East Asian Wide (W) or East Asian
73 * Full-width (F) category as defined in Unicode Technical
74 * Report #11 have a column width of 2.
76 * - All remaining characters (including all printable
77 * ISO 8859-1 and WGL4 characters, Unicode control characters,
78 * etc.) have a column width of 1.
80 * This implementation assumes that ucs_char_t characters are encoded
84 static int git_wcwidth(ucs_char_t ch
)
87 * Sorted list of non-overlapping intervals of non-spacing characters,
89 #include "unicode-width.h"
91 /* test for 8-bit control characters */
94 if (ch
< 32 || (ch
>= 0x7f && ch
< 0xa0))
97 /* binary search in table of non-spacing characters */
98 if (bisearch(ch
, zero_width
, ARRAY_SIZE(zero_width
) - 1))
101 /* binary search in table of double width characters */
102 if (bisearch(ch
, double_width
, ARRAY_SIZE(double_width
) - 1))
109 * Pick one ucs character starting from the location *start points at,
110 * and return it, while updating the *start pointer to point at the
111 * end of that character. When remainder_p is not NULL, the location
112 * holds the number of bytes remaining in the string that we are allowed
113 * to pick from. Otherwise we are allowed to pick up to the NUL that
114 * would eventually appear in the string. *remainder_p is also reduced
115 * by the number of bytes we have consumed.
117 * If the string was not a valid UTF-8, *start pointer is set to NULL
118 * and the return value is undefined.
120 static ucs_char_t
pick_one_utf8_char(const char **start
, size_t *remainder_p
)
122 unsigned char *s
= (unsigned char *)*start
;
124 size_t remainder
, incr
;
127 * A caller that assumes NUL terminated text can choose
128 * not to bother with the remainder length. We will
129 * stop at the first NUL.
131 remainder
= (remainder_p
? *remainder_p
: 999);
135 } else if (*s
< 0x80) {
139 } else if ((s
[0] & 0xe0) == 0xc0) {
140 /* 110XXXXx 10xxxxxx */
142 (s
[1] & 0xc0) != 0x80 ||
143 (s
[0] & 0xfe) == 0xc0)
145 ch
= ((s
[0] & 0x1f) << 6) | (s
[1] & 0x3f);
147 } else if ((s
[0] & 0xf0) == 0xe0) {
148 /* 1110XXXX 10Xxxxxx 10xxxxxx */
150 (s
[1] & 0xc0) != 0x80 ||
151 (s
[2] & 0xc0) != 0x80 ||
153 (s
[0] == 0xe0 && (s
[1] & 0xe0) == 0x80) ||
155 (s
[0] == 0xed && (s
[1] & 0xe0) == 0xa0) ||
156 /* U+FFFE or U+FFFF? */
157 (s
[0] == 0xef && s
[1] == 0xbf &&
158 (s
[2] & 0xfe) == 0xbe))
160 ch
= ((s
[0] & 0x0f) << 12) |
161 ((s
[1] & 0x3f) << 6) | (s
[2] & 0x3f);
163 } else if ((s
[0] & 0xf8) == 0xf0) {
164 /* 11110XXX 10XXxxxx 10xxxxxx 10xxxxxx */
166 (s
[1] & 0xc0) != 0x80 ||
167 (s
[2] & 0xc0) != 0x80 ||
168 (s
[3] & 0xc0) != 0x80 ||
170 (s
[0] == 0xf0 && (s
[1] & 0xf0) == 0x80) ||
172 (s
[0] == 0xf4 && s
[1] > 0x8f) || s
[0] > 0xf4)
174 ch
= ((s
[0] & 0x07) << 18) | ((s
[1] & 0x3f) << 12) |
175 ((s
[2] & 0x3f) << 6) | (s
[3] & 0x3f);
185 *remainder_p
= remainder
- incr
;
190 * This function returns the number of columns occupied by the character
191 * pointed to by the variable start. The pointer is updated to point at
192 * the next character. When remainder_p is not NULL, it points at the
193 * location that stores the number of remaining bytes we can use to pick
194 * a character (see pick_one_utf8_char() above).
196 int utf8_width(const char **start
, size_t *remainder_p
)
198 ucs_char_t ch
= pick_one_utf8_char(start
, remainder_p
);
201 return git_wcwidth(ch
);
205 * Returns the total number of columns required by a null-terminated
206 * string, assuming that the string is utf8. Returns strlen() instead
207 * if the string does not look like a valid utf8 string.
209 int utf8_strnwidth(const char *string
, int len
, int skip_ansi
)
212 const char *orig
= string
;
215 len
= strlen(string
);
216 while (string
&& string
< orig
+ len
) {
219 (skip
= display_mode_esc_sequence_len(string
)) != 0)
221 width
+= utf8_width(&string
, NULL
);
223 return string
? width
: len
;
226 int utf8_strwidth(const char *string
)
228 return utf8_strnwidth(string
, -1, 0);
231 int is_utf8(const char *text
)
234 if (*text
== '\n' || *text
== '\t' || *text
== '\r') {
238 utf8_width(&text
, NULL
);
245 static void strbuf_add_indented_text(struct strbuf
*buf
, const char *text
,
246 int indent
, int indent2
)
251 const char *eol
= strchrnul(text
, '\n');
254 strbuf_addchars(buf
, ' ', indent
);
255 strbuf_add(buf
, text
, eol
- text
);
262 * Wrap the text, if necessary. The variable indent is the indent for the
263 * first line, indent2 is the indent for all other lines.
264 * If indent is negative, assume that already -indent columns have been
265 * consumed (and no extra indent is necessary for the first line).
267 void strbuf_add_wrapped_text(struct strbuf
*buf
,
268 const char *text
, int indent1
, int indent2
, int width
)
270 int indent
, w
, assume_utf8
= 1;
271 const char *bol
, *space
, *start
= text
;
272 size_t orig_len
= buf
->len
;
275 strbuf_add_indented_text(buf
, text
, indent1
, indent2
);
281 w
= indent
= indent1
;
292 while ((skip
= display_mode_esc_sequence_len(text
)))
296 if (!c
|| isspace(c
)) {
297 if (w
<= width
|| !space
) {
298 const char *start
= bol
;
299 if (!c
&& text
== start
)
304 strbuf_addchars(buf
, ' ', indent
);
305 strbuf_add(buf
, start
, text
- start
);
311 else if (c
== '\n') {
313 if (*space
== '\n') {
314 strbuf_addch(buf
, '\n');
317 else if (!isalnum(*space
))
320 strbuf_addch(buf
, ' ');
327 strbuf_addch(buf
, '\n');
328 text
= bol
= space
+ isspace(*space
);
330 w
= indent
= indent2
;
335 w
+= utf8_width(&text
, NULL
);
339 strbuf_setlen(buf
, orig_len
);
349 void strbuf_add_wrapped_bytes(struct strbuf
*buf
, const char *data
, int len
,
350 int indent
, int indent2
, int width
)
352 char *tmp
= xstrndup(data
, len
);
353 strbuf_add_wrapped_text(buf
, tmp
, indent
, indent2
, width
);
357 void strbuf_utf8_replace(struct strbuf
*sb_src
, int pos
, int width
,
360 struct strbuf sb_dst
= STRBUF_INIT
;
361 char *src
= sb_src
->buf
;
362 char *end
= src
+ sb_src
->len
;
364 int w
= 0, subst_len
= 0;
367 subst_len
= strlen(subst
);
368 strbuf_grow(&sb_dst
, sb_src
->len
+ subst_len
);
375 while ((n
= display_mode_esc_sequence_len(src
))) {
385 n
= utf8_width((const char**)&src
, NULL
);
386 if (!src
) /* broken utf-8, do nothing */
388 if (n
&& w
>= pos
&& w
< pos
+ width
) {
390 memcpy(dst
, subst
, subst_len
);
397 memcpy(dst
, old
, src
- old
);
401 strbuf_setlen(&sb_dst
, dst
- sb_dst
.buf
);
402 strbuf_swap(sb_src
, &sb_dst
);
404 strbuf_release(&sb_dst
);
408 * Returns true (1) if the src encoding name matches the dst encoding
409 * name directly or one of its alternative names. E.g. UTF-16BE is the
412 static int same_utf_encoding(const char *src
, const char *dst
)
414 if (istarts_with(src
, "utf") && istarts_with(dst
, "utf")) {
415 /* src[3] or dst[3] might be '\0' */
416 int i
= (src
[3] == '-' ? 4 : 3);
417 int j
= (dst
[3] == '-' ? 4 : 3);
418 return !strcasecmp(src
+i
, dst
+j
);
423 int is_encoding_utf8(const char *name
)
427 if (same_utf_encoding("utf-8", name
))
432 int same_encoding(const char *src
, const char *dst
)
434 static const char utf8
[] = "UTF-8";
440 if (same_utf_encoding(src
, dst
))
442 return !strcasecmp(src
, dst
);
446 * Wrapper for fprintf and returns the total number of columns required
447 * for the printed string, assuming that the string is utf8.
449 int utf8_fprintf(FILE *stream
, const char *format
, ...)
451 struct strbuf buf
= STRBUF_INIT
;
455 va_start(arg
, format
);
456 strbuf_vaddf(&buf
, format
, arg
);
459 columns
= fputs(buf
.buf
, stream
);
460 if (0 <= columns
) /* keep the error from the I/O */
461 columns
= utf8_strwidth(buf
.buf
);
462 strbuf_release(&buf
);
467 * Given a buffer and its encoding, return it re-encoded
468 * with iconv. If the conversion fails, returns NULL.
471 #if defined(OLD_ICONV) || (defined(__sun__) && !defined(_XPG6))
472 typedef const char * iconv_ibp
;
474 typedef char * iconv_ibp
;
476 char *reencode_string_iconv(const char *in
, size_t insz
, iconv_t conv
,
477 size_t bom_len
, size_t *outsz_p
)
479 size_t outsz
, outalloc
;
484 outalloc
= st_add(outsz
, 1 + bom_len
); /* for terminating NUL */
485 out
= xmalloc(outalloc
);
486 outpos
= out
+ bom_len
;
490 size_t cnt
= iconv(conv
, &cp
, &insz
, &outpos
, &outsz
);
492 if (cnt
== (size_t) -1) {
494 if (errno
!= E2BIG
) {
498 /* insz has remaining number of bytes.
499 * since we started outsz the same as insz,
500 * it is likely that insz is not enough for
501 * converting the rest.
503 sofar
= outpos
- out
;
504 outalloc
= st_add3(sofar
, st_mult(insz
, 2), 32);
505 out
= xrealloc(out
, outalloc
);
506 outpos
= out
+ sofar
;
507 outsz
= outalloc
- sofar
- 1;
512 *outsz_p
= outpos
- out
;
519 static const char *fallback_encoding(const char *name
)
522 * Some platforms do not have the variously spelled variants of
523 * UTF-8, so let's fall back to trying the most official
524 * spelling. We do so only as a fallback in case the platform
525 * does understand the user's spelling, but not our official
528 if (is_encoding_utf8(name
))
532 * Even though latin-1 is still seen in e-mail
533 * headers, some platforms only install ISO-8859-1.
535 if (!strcasecmp(name
, "latin-1"))
541 char *reencode_string_len(const char *in
, size_t insz
,
542 const char *out_encoding
, const char *in_encoding
,
547 const char *bom_str
= NULL
;
553 /* UTF-16LE-BOM is the same as UTF-16 for reading */
554 if (same_utf_encoding("UTF-16LE-BOM", in_encoding
))
555 in_encoding
= "UTF-16";
558 * For writing, UTF-16 iconv typically creates "UTF-16BE-BOM"
559 * Some users under Windows want the little endian version
561 * We handle UTF-16 and UTF-32 ourselves only if the platform does not
562 * provide a BOM (which we require), since we want to match the behavior
563 * of the system tools and libc as much as possible.
565 if (same_utf_encoding("UTF-16LE-BOM", out_encoding
)) {
566 bom_str
= utf16_le_bom
;
567 bom_len
= sizeof(utf16_le_bom
);
568 out_encoding
= "UTF-16LE";
569 } else if (same_utf_encoding("UTF-16BE-BOM", out_encoding
)) {
570 bom_str
= utf16_be_bom
;
571 bom_len
= sizeof(utf16_be_bom
);
572 out_encoding
= "UTF-16BE";
573 #ifdef ICONV_OMITS_BOM
574 } else if (same_utf_encoding("UTF-16", out_encoding
)) {
575 bom_str
= utf16_be_bom
;
576 bom_len
= sizeof(utf16_be_bom
);
577 out_encoding
= "UTF-16BE";
578 } else if (same_utf_encoding("UTF-32", out_encoding
)) {
579 bom_str
= utf32_be_bom
;
580 bom_len
= sizeof(utf32_be_bom
);
581 out_encoding
= "UTF-32BE";
585 conv
= iconv_open(out_encoding
, in_encoding
);
586 if (conv
== (iconv_t
) -1) {
587 in_encoding
= fallback_encoding(in_encoding
);
588 out_encoding
= fallback_encoding(out_encoding
);
590 conv
= iconv_open(out_encoding
, in_encoding
);
591 if (conv
== (iconv_t
) -1)
594 out
= reencode_string_iconv(in
, insz
, conv
, bom_len
, outsz
);
596 if (out
&& bom_str
&& bom_len
)
597 memcpy(out
, bom_str
, bom_len
);
602 static int has_bom_prefix(const char *data
, size_t len
,
603 const char *bom
, size_t bom_len
)
605 return data
&& bom
&& (len
>= bom_len
) && !memcmp(data
, bom
, bom_len
);
608 int has_prohibited_utf_bom(const char *enc
, const char *data
, size_t len
)
611 (same_utf_encoding("UTF-16BE", enc
) ||
612 same_utf_encoding("UTF-16LE", enc
)) &&
613 (has_bom_prefix(data
, len
, utf16_be_bom
, sizeof(utf16_be_bom
)) ||
614 has_bom_prefix(data
, len
, utf16_le_bom
, sizeof(utf16_le_bom
)))
616 (same_utf_encoding("UTF-32BE", enc
) ||
617 same_utf_encoding("UTF-32LE", enc
)) &&
618 (has_bom_prefix(data
, len
, utf32_be_bom
, sizeof(utf32_be_bom
)) ||
619 has_bom_prefix(data
, len
, utf32_le_bom
, sizeof(utf32_le_bom
)))
623 int is_missing_required_utf_bom(const char *enc
, const char *data
, size_t len
)
626 (same_utf_encoding(enc
, "UTF-16")) &&
627 !(has_bom_prefix(data
, len
, utf16_be_bom
, sizeof(utf16_be_bom
)) ||
628 has_bom_prefix(data
, len
, utf16_le_bom
, sizeof(utf16_le_bom
)))
630 (same_utf_encoding(enc
, "UTF-32")) &&
631 !(has_bom_prefix(data
, len
, utf32_be_bom
, sizeof(utf32_be_bom
)) ||
632 has_bom_prefix(data
, len
, utf32_le_bom
, sizeof(utf32_le_bom
)))
637 * Returns first character length in bytes for multi-byte `text` according to
640 * - The `text` pointer is updated to point at the next character.
641 * - When `remainder_p` is not NULL, on entry `*remainder_p` is how much bytes
642 * we can consume from text, and on exit `*remainder_p` is reduced by returned
643 * character length. Otherwise `text` is treated as limited by NUL.
645 int mbs_chrlen(const char **text
, size_t *remainder_p
, const char *encoding
)
648 const char *p
= *text
;
649 size_t r
= (remainder_p
? *remainder_p
: SIZE_MAX
);
654 if (is_encoding_utf8(encoding
)) {
655 pick_one_utf8_char(&p
, &r
);
657 chrlen
= p
? (p
- *text
)
658 : 1 /* not valid UTF-8 -> raw byte sequence */;
662 * TODO use iconv to decode one char and obtain its chrlen
663 * for now, let's treat encodings != UTF-8 as one-byte
670 *remainder_p
-= chrlen
;
676 * Pick the next char from the stream, ignoring codepoints an HFS+ would.
677 * Note that this is _not_ complete by any means. It's just enough
678 * to make is_hfs_dotgit() work, and should not be used otherwise.
680 static ucs_char_t
next_hfs_char(const char **in
)
683 ucs_char_t out
= pick_one_utf8_char(in
, NULL
);
685 * check for malformed utf8. Technically this
686 * gets converted to a percent-sequence, but
687 * returning 0 is good enough for is_hfs_dotgit
688 * to realize it cannot be .git
693 /* these code points are ignored completely */
695 case 0x200c: /* ZERO WIDTH NON-JOINER */
696 case 0x200d: /* ZERO WIDTH JOINER */
697 case 0x200e: /* LEFT-TO-RIGHT MARK */
698 case 0x200f: /* RIGHT-TO-LEFT MARK */
699 case 0x202a: /* LEFT-TO-RIGHT EMBEDDING */
700 case 0x202b: /* RIGHT-TO-LEFT EMBEDDING */
701 case 0x202c: /* POP DIRECTIONAL FORMATTING */
702 case 0x202d: /* LEFT-TO-RIGHT OVERRIDE */
703 case 0x202e: /* RIGHT-TO-LEFT OVERRIDE */
704 case 0x206a: /* INHIBIT SYMMETRIC SWAPPING */
705 case 0x206b: /* ACTIVATE SYMMETRIC SWAPPING */
706 case 0x206c: /* INHIBIT ARABIC FORM SHAPING */
707 case 0x206d: /* ACTIVATE ARABIC FORM SHAPING */
708 case 0x206e: /* NATIONAL DIGIT SHAPES */
709 case 0x206f: /* NOMINAL DIGIT SHAPES */
710 case 0xfeff: /* ZERO WIDTH NO-BREAK SPACE */
718 static int is_hfs_dot_generic(const char *path
,
719 const char *needle
, size_t needle_len
)
723 c
= next_hfs_char(&path
);
728 * there's a great deal of other case-folding that occurs
729 * in HFS+, but this is enough to catch our fairly vanilla
730 * hard-coded needles.
732 for (; needle_len
> 0; needle
++, needle_len
--) {
733 c
= next_hfs_char(&path
);
736 * We know our needles contain only ASCII, so we clamp here to
737 * make the results of tolower() sane.
741 if (tolower(c
) != *needle
)
745 c
= next_hfs_char(&path
);
746 if (c
&& !is_dir_sep(c
))
753 * Inline wrapper to make sure the compiler resolves strlen() on literals at
756 static inline int is_hfs_dot_str(const char *path
, const char *needle
)
758 return is_hfs_dot_generic(path
, needle
, strlen(needle
));
761 int is_hfs_dotgit(const char *path
)
763 return is_hfs_dot_str(path
, "git");
766 int is_hfs_dotgitmodules(const char *path
)
768 return is_hfs_dot_str(path
, "gitmodules");
771 int is_hfs_dotgitignore(const char *path
)
773 return is_hfs_dot_str(path
, "gitignore");
776 int is_hfs_dotgitattributes(const char *path
)
778 return is_hfs_dot_str(path
, "gitattributes");
781 const char utf8_bom
[] = "\357\273\277";
783 int skip_utf8_bom(char **text
, size_t len
)
785 if (len
< strlen(utf8_bom
) ||
786 memcmp(*text
, utf8_bom
, strlen(utf8_bom
)))
788 *text
+= strlen(utf8_bom
);
792 void strbuf_utf8_align(struct strbuf
*buf
, align_type position
, unsigned int width
,
795 int slen
= strlen(s
);
796 int display_len
= utf8_strnwidth(s
, slen
, 0);
797 int utf8_compensation
= slen
- display_len
;
799 if (display_len
>= width
) {
800 strbuf_addstr(buf
, s
);
804 if (position
== ALIGN_LEFT
)
805 strbuf_addf(buf
, "%-*s", width
+ utf8_compensation
, s
);
806 else if (position
== ALIGN_MIDDLE
) {
807 int left
= (width
- display_len
) / 2;
808 strbuf_addf(buf
, "%*s%-*s", left
, "", width
- left
+ utf8_compensation
, s
);
809 } else if (position
== ALIGN_RIGHT
)
810 strbuf_addf(buf
, "%*s", width
+ utf8_compensation
, s
);