utf8.c

   1 /**
   2  * UTF-8 utility functions
   3  *
   4  * (c) 2010-2016 Steve Bennett <steveb@workware.net.au>
   5  *
   6  * See LICENCE for licence details.
   7  */
   8
   9 #include <ctype.h>
  10 #include <stdlib.h>
  11 #include <string.h>
  12 #include <stdio.h>
  13 #include <assert.h>
  14 #include "utf8.h"
  15
  16 /* This one is always implemented */
  17 int utf8_fromunicode(char *p, unsigned uc)
  18 {
  19     if (uc <= 0x7f) {
  20         *p = uc;
  21         return 1;
  22     }
  23     else if (uc <= 0x7ff) {
  24         *p++ = 0xc0 | ((uc & 0x7c0) >> 6);
  25         *p = 0x80 | (uc & 0x3f);
  26         return 2;
  27     }
  28     else if (uc <= 0xffff) {
  29         *p++ = 0xe0 | ((uc & 0xf000) >> 12);
  30         *p++ = 0x80 | ((uc & 0xfc0) >> 6);
  31         *p = 0x80 | (uc & 0x3f);
  32         return 3;
  33     }
  34     /* Note: We silently truncate to 21 bits here: 0x1fffff */
  35     else {
  36         *p++ = 0xf0 | ((uc & 0x1c0000) >> 18);
  37         *p++ = 0x80 | ((uc & 0x3f000) >> 12);
  38         *p++ = 0x80 | ((uc & 0xfc0) >> 6);
  39         *p = 0x80 | (uc & 0x3f);
  40         return 4;
  41     }
  42 }
  43
  44 #if defined(USE_UTF8) && !defined(JIM_BOOTSTRAP)
  45 int utf8_charlen(int c)
  46 {
  47     if ((c & 0x80) == 0) {
  48         return 1;
  49     }
  50     if ((c & 0xe0) == 0xc0) {
  51         return 2;
  52     }
  53     if ((c & 0xf0) == 0xe0) {
  54         return 3;
  55     }
  56     if ((c & 0xf8) == 0xf0) {
  57         return 4;
  58     }
  59     /* Invalid sequence */
  60     return -1;
  61 }
  62
  63 int utf8_strlen(const char *str, int bytelen)
  64 {
  65     int charlen = 0;
  66     if (bytelen < 0) {
  67         bytelen = strlen(str);
  68     }
  69     while (bytelen > 0) {
  70         int c;
  71         int l = utf8_tounicode(str, &c);
  72         charlen++;
  73         str += l;
  74         bytelen -= l;
  75     }
  76     return charlen;
  77 }
  78
  79 int utf8_strwidth(const char *str, int charlen)
  80 {
  81     int width = 0;
  82     while (charlen) {
  83         int c;
  84         int l = utf8_tounicode(str, &c);
  85         width += utf8_width(c);
  86         str += l;
  87         charlen--;
  88     }
  89     return width;
  90 }
  91
  92 int utf8_index(const char *str, int index)
  93 {
  94     const char *s = str;
  95     while (index--) {
  96         int c;
  97         s += utf8_tounicode(s, &c);
  98     }
  99     return s - str;
 100 }
 101
 102 int utf8_prev_len(const char *str, int len)
 103 {
 104     int n = 1;
 105
 106     assert(len > 0);
 107
 108     /* Look up to len chars backward for a start-of-char byte */
 109     while (--len) {
 110         if ((str[-n] & 0x80) == 0) {
 111             /* Start of a 1-byte char */
 112             break;
 113         }
 114         if ((str[-n] & 0xc0) == 0xc0) {
 115             /* Start of a multi-byte char */
 116             break;
 117         }
 118         n++;
 119     }
 120     return n;
 121 }
 122
 123 int utf8_tounicode(const char *str, int *uc)
 124 {
 125     unsigned const char *s = (unsigned const char *)str;
 126
 127     if (s[0] < 0xc0) {
 128         *uc = s[0];
 129         return 1;
 130     }
 131     if (s[0] < 0xe0) {
 132         if ((s[1] & 0xc0) == 0x80) {
 133             *uc = ((s[0] & ~0xc0) << 6) | (s[1] & ~0x80);
 134             return 2;
 135         }
 136     }
 137     else if (s[0] < 0xf0) {
 138         if (((str[1] & 0xc0) == 0x80) && ((str[2] & 0xc0) == 0x80)) {
 139             *uc = ((s[0] & ~0xe0) << 12) | ((s[1] & ~0x80) << 6) | (s[2] & ~0x80);
 140             return 3;
 141         }
 142     }
 143     else if (s[0] < 0xf8) {
 144         if (((str[1] & 0xc0) == 0x80) && ((str[2] & 0xc0) == 0x80) && ((str[3] & 0xc0) == 0x80)) {
 145             *uc = ((s[0] & ~0xf0) << 18) | ((s[1] & ~0x80) << 12) | ((s[2] & ~0x80) << 6) | (s[3] & ~0x80);
 146             return 4;
 147         }
 148     }
 149
 150     /* Invalid sequence, so just return the byte */
 151     *uc = *s;
 152     return 1;
 153 }
 154
 155 struct casemap {
 156     unsigned short code;        /* code point */
 157     unsigned short altcode;     /* alternate case code point */
 158 };
 159
 160 struct utf8range {
 161     unsigned lower;     /* lower inclusive */
 162     unsigned upper;     /* upper exclusive */
 163 };
 164
 165
 166 /* Generated mapping tables */
 167 #include "_unicode_mapping.c"
 168
 169 #define ARRAYSIZE(A) sizeof(A) / sizeof(*(A))
 170
 171 static int cmp_casemap(const void *key, const void *cm)
 172 {
 173     return *(int *)key - (int)((const struct casemap *)cm)->code;
 174 }
 175
 176 static int utf8_map_case(const struct casemap *mapping, int num, int ch)
 177 {
 178     /* We only support 16 bit case mapping */
 179     if (ch <= 0xffff) {
 180         const struct casemap *cm =
 181             bsearch(&ch, mapping, num, sizeof(*mapping), cmp_casemap);
 182
 183         if (cm) {
 184             return cm->altcode;
 185         }
 186     }
 187     return ch;
 188 }
 189
 190 static int cmp_range(const void *key, const void *cm)
 191 {
 192     const struct utf8range *range = (const struct utf8range *)cm;
 193     int ch = *(int *)key;
 194     if (ch < range->lower) {
 195         return -1;
 196     }
 197     if (ch >= range->upper) {
 198         return 1;
 199     }
 200     return 0;
 201 }
 202
 203 static int utf8_in_range(const struct utf8range *range, int num, int ch)
 204 {
 205     const struct utf8range *r =
 206         bsearch(&ch, range, num, sizeof(*range), cmp_range);
 207
 208     if (r) {
 209         return 1;
 210     }
 211     return 0;
 212 }
 213
 214 int utf8_upper(int ch)
 215 {
 216     if (isascii(ch)) {
 217         return toupper(ch);
 218     }
 219     return utf8_map_case(unicode_case_mapping_upper, ARRAYSIZE(unicode_case_mapping_upper), ch);
 220 }
 221
 222 int utf8_lower(int ch)
 223 {
 224     if (isascii(ch)) {
 225         return tolower(ch);
 226     }
 227     return utf8_map_case(unicode_case_mapping_lower, ARRAYSIZE(unicode_case_mapping_lower), ch);
 228 }
 229
 230 int utf8_title(int ch)
 231 {
 232     if (!isascii(ch)) {
 233         int newch = utf8_map_case(unicode_case_mapping_title, ARRAYSIZE(unicode_case_mapping_title), ch);
 234         if (newch != ch) {
 235             return newch ? newch : ch;
 236         }
 237     }
 238     return utf8_upper(ch);
 239 }
 240
 241 int utf8_width(int ch)
 242 {
 243     if (!isascii(ch)) {
 244         if (utf8_in_range(unicode_range_combining, ARRAYSIZE(unicode_range_combining), ch)) {
 245             return 0;
 246         }
 247         if (utf8_in_range(unicode_range_wide, ARRAYSIZE(unicode_range_wide), ch)) {
 248             return 2;
 249         }
 250     }
 251     return 1;
 252 }
 253
 254 #endif /* JIM_BOOTSTRAP */