eglib/src/gunicode.c

   1 /*
   2  * gunicode.c: Some Unicode routines
   3  *
   4  * Author:
   5  *   Miguel de Icaza (miguel@novell.com)
   6  *
   7  * (C) 2006 Novell, Inc.
   8  *
   9  * utf8 validation code came from:
  10  *      libxml2-2.6.26 licensed under the MIT X11 license
  11  *
  12  * Authors credit in libxml's string.c:
  13  *   William Brack <wbrack@mmm.com.hk>
  14  *   daniel@veillard.com
  15  *
  16  * Permission is hereby granted, free of charge, to any person obtaining
  17  * a copy of this software and associated documentation files (the
  18  * "Software"), to deal in the Software without restriction, including
  19  * without limitation the rights to use, copy, modify, merge, publish,
  20  * distribute, sublicense, and/or sell copies of the Software, and to
  21  * permit persons to whom the Software is furnished to do so, subject to
  22  * the following conditions:
  23  *
  24  * The above copyright notice and this permission notice shall be
  25  * included in all copies or substantial portions of the Software.
  26  *
  27  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  28  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  29  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  30  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  31  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  32  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  33  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  34  *
  35  */
  36 #include <config.h>
  37 #include <stdio.h>
  38 #include <glib.h>
  39 #include <unicode-data.h>
  40 #include <errno.h>
  41
  42 #if defined(_MSC_VER) || defined(G_OS_WIN32)
  43 /* FIXME */
  44 #  define CODESET 1
  45 #  include <windows.h>
  46 #  ifdef _MSC_VER
  47        typedef int iconv_t;
  48 #  endif
  49 #else
  50 #    ifdef HAVE_LANGINFO_H
  51 #       include <langinfo.h>
  52 #    endif
  53 #    ifdef HAVE_ICONV_H
  54 #       include <iconv.h>
  55 #    endif
  56 #    ifdef HAVE_LOCALCHARSET_H
  57 #       include <localcharset.h>
  58 #    endif
  59 #endif
  60
  61 static char *my_charset;
  62 static gboolean is_utf8;
  63
  64 /*
  65  * Character set conversion
  66  */
  67 /*
  68 * Index into the table below with the first byte of a UTF-8 sequence to
  69 * get the number of trailing bytes that are supposed to follow it.
  70 * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is
  71 * left as-is for anyone who may want to do such conversion, which was
  72 * allowed in earlier algorithms.
  73 */
  74 const gchar g_trailingBytesForUTF8 [256] = {
  75         0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  76         0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  77         0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  78         0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  79         0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  80         0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  81         1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
  82         2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,0,0
  83 };
  84
  85 /*
  86 * Magic values subtracted from a buffer value during UTF8 conversion.
  87 * This table contains as many values as there might be trailing bytes
  88 * in a UTF-8 sequence.
  89 */
  90 static const gulong offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL,
  91 0x03C82080UL, 0xFA082080UL, 0x82082080UL };
  92
  93 GUnicodeType
  94 g_unichar_type (gunichar c)
  95 {
  96 int i;
  97
  98         guint16 cp = (guint16) c;
  99         for (i = 0; i < unicode_category_ranges_count; i++) {
 100                 if (cp < unicode_category_ranges [i].start)
 101                         continue;
 102                 if (unicode_category_ranges [i].end <= cp)
 103                         continue;
 104                 return unicode_category [i] [cp - unicode_category_ranges [i].start];
 105         }
 106
 107         /*
 108         // 3400-4DB5: OtherLetter
 109         // 4E00-9FC3: OtherLetter
 110         // AC00-D7A3: OtherLetter
 111         // D800-DFFF: OtherSurrogate
 112         // E000-F8FF: OtherPrivateUse
 113         // 20000-2A6D6 OtherLetter
 114         // F0000-FFFFD OtherPrivateUse
 115         // 100000-10FFFD OtherPrivateUse
 116         */
 117         if (0x3400 <= cp && cp < 0x4DB5)
 118                 return G_UNICODE_OTHER_LETTER;
 119         if (0x4E00 <= cp && cp < 0x9FC3)
 120                 return G_UNICODE_OTHER_LETTER;
 121         if (0xAC00<= cp && cp < 0xD7A3)
 122                 return G_UNICODE_OTHER_LETTER;
 123         if (0xD800 <= cp && cp < 0xDFFF)
 124                 return G_UNICODE_SURROGATE;
 125         if (0xE000 <= cp && cp < 0xF8FF)
 126                 return G_UNICODE_PRIVATE_USE;
 127         /* since the argument is UTF-16, we cannot check beyond FFFF */
 128
 129         /* It should match any of above */
 130         return 0;
 131 }
 132
 133 GUnicodeBreakType
 134 g_unichar_break_type (gunichar c)
 135 {
 136         // MOONLIGHT_FIXME
 137         return G_UNICODE_BREAK_UNKNOWN;
 138 }
 139
 140 gunichar
 141 g_unichar_case (gunichar c, gboolean upper)
 142 {
 143         gint8 i, i2;
 144         guint32 cp = (guint32) c, v;
 145
 146         for (i = 0; i < simple_case_map_ranges_count; i++) {
 147                 if (cp < simple_case_map_ranges [i].start)
 148                         return c;
 149                 if (simple_case_map_ranges [i].end <= cp)
 150                         continue;
 151                 if (c < 0x10000) {
 152                         const guint16 *tab = upper ? simple_upper_case_mapping_lowarea [i] : simple_lower_case_mapping_lowarea [i];
 153                         v = tab [cp - simple_case_map_ranges [i].start];
 154                 } else {
 155                         const guint32 *tab;
 156                         i2 = (gint8)(i - (upper ? simple_upper_case_mapping_lowarea_table_count : simple_lower_case_mapping_lowarea_table_count));
 157                         tab = upper ? simple_upper_case_mapping_higharea [i2] : simple_lower_case_mapping_higharea [i2];
 158                         v = tab [cp - simple_case_map_ranges [i].start];
 159                 }
 160                 return v != 0 ? (gunichar) v : c;
 161         }
 162         return c;
 163 }
 164
 165 gunichar
 166 g_unichar_toupper (gunichar c)
 167 {
 168         return g_unichar_case (c, TRUE);
 169 }
 170
 171 gunichar
 172 g_unichar_tolower (gunichar c)
 173 {
 174         return g_unichar_case (c, FALSE);
 175 }
 176
 177 gunichar
 178 g_unichar_totitle (gunichar c)
 179 {
 180         guint8 i;
 181         guint32 cp;
 182
 183         cp = (guint32) c;
 184         for (i = 0; i < simple_titlecase_mapping_count; i++) {
 185                 if (simple_titlecase_mapping [i].codepoint == cp)
 186                         return simple_titlecase_mapping [i].title;
 187                 if (simple_titlecase_mapping [i].codepoint > cp)
 188                         /* it is ordered, hence no more match */
 189                         break;
 190         }
 191         return g_unichar_toupper (c);
 192 }
 193
 194 gboolean
 195 g_unichar_isxdigit (gunichar c)
 196 {
 197         return (g_unichar_xdigit_value (c) != -1);
 198
 199 }
 200
 201 gint
 202 g_unichar_xdigit_value (gunichar c)
 203 {
 204         if (c >= 0x30 && c <= 0x39) /*0-9*/
 205                 return (c - 0x30);
 206         if (c >= 0x41 && c <= 0x46) /*A-F*/
 207                 return (c - 0x37);
 208         if (c >= 0x61 && c <= 0x66) /*a-f*/
 209                 return (c - 0x57);
 210         return -1;
 211 }
 212
 213 gboolean
 214 g_unichar_isspace (gunichar c)
 215 {
 216         GUnicodeType type = g_unichar_type (c);
 217         if (type == G_UNICODE_LINE_SEPARATOR ||
 218             type == G_UNICODE_PARAGRAPH_SEPARATOR ||
 219             type == G_UNICODE_SPACE_SEPARATOR)
 220                 return TRUE;
 221
 222         return FALSE;
 223 }
 224
 225 gchar *
 226 g_convert (const gchar *str, gssize len,
 227            const gchar *to_codeset, const gchar *from_codeset,
 228            gsize *bytes_read, gsize *bytes_written, GError **error)
 229 {
 230         char *result = NULL;
 231 #ifdef G_OS_WIN32
 232 #elif HAVE_ICONV_H
 233         iconv_t convertor;
 234         char *buffer, *output;
 235         const char *strptr = (const char *) str;
 236         size_t str_len = len == -1 ? strlen (str) : len;
 237         size_t buffer_size;
 238         size_t left, out_left;
 239
 240         convertor = iconv_open (to_codeset, from_codeset);
 241         if (convertor == (iconv_t) -1){
 242                 if (bytes_written)
 243                         *bytes_written = 0;
 244                 if (bytes_read)
 245                         *bytes_read = 0;
 246                 return NULL;
 247         }
 248
 249         buffer_size = str_len + 1 + 8;
 250         buffer = g_malloc (buffer_size);
 251         out_left = str_len;
 252         output = buffer;
 253         left = str_len;
 254         while (left > 0){
 255                 int res = iconv (convertor, (char **) &strptr, &left, &output, &out_left);
 256                 if (res == (size_t) -1){
 257                         if (errno == E2BIG){
 258                                 char *n;
 259                                 size_t extra_space = 8 + left;
 260                                 size_t output_used = output - buffer;
 261
 262                                 buffer_size += extra_space;
 263
 264                                 n = g_realloc (buffer, buffer_size);
 265
 266                                 if (n == NULL){
 267                                         if (error != NULL)
 268                                                 *error = g_error_new (NULL, G_CONVERT_ERROR_FAILED, "No memory left");
 269                                         g_free (buffer);
 270                                         result = NULL;
 271                                         goto leave;
 272                                 }
 273                                 buffer = n;
 274                                 out_left += extra_space;
 275                                 output = buffer + output_used;
 276                         } else if (errno == EILSEQ){
 277                                 if (error != NULL)
 278                                         *error = g_error_new (NULL, G_CONVERT_ERROR_ILLEGAL_SEQUENCE, "Invalid multi-byte sequence on input");
 279                                 result = NULL;
 280                                 g_free (buffer);
 281                                 goto leave;
 282                         } else if (errno == EINVAL){
 283                                 if (error != NULL)
 284                                         *error = g_error_new (NULL, G_CONVERT_ERROR_PARTIAL_INPUT, "Partial character sequence");
 285                                 result = NULL;
 286                                 g_free (buffer);
 287                                 goto leave;
 288                         }
 289                 }
 290         }
 291         if (bytes_read != NULL)
 292                 *bytes_read = strptr - str;
 293         if (bytes_written != NULL)
 294                 *bytes_written = output - buffer;
 295         *output = 0;
 296         result = buffer;
 297  leave:
 298         iconv_close (convertor);
 299 #endif
 300         return result;
 301 }
 302
 303 /*
 304  * This is broken, and assumes an UTF8 system, but will do for eglib's first user
 305  */
 306 gchar *
 307 g_filename_from_utf8 (const gchar *utf8string, gssize len, gsize *bytes_read, gsize *bytes_written, GError **error)
 308 {
 309         char *res;
 310
 311         if (len == -1)
 312                 len = strlen (utf8string);
 313
 314         res = g_malloc (len + 1);
 315         g_strlcpy (res, utf8string, len + 1);
 316         return res;
 317 }
 318
 319 gboolean
 320 g_get_charset (G_CONST_RETURN char **charset)
 321 {
 322         if (my_charset == NULL) {
 323 #ifdef G_OS_WIN32
 324                 static char buf [14];
 325                 sprintf (buf, "CP%u", GetACP ());
 326                 my_charset = buf;
 327                 is_utf8 = FALSE;
 328 #else
 329                 /* These shouldn't be heap allocated */
 330 #if HAVE_LOCALCHARSET_H
 331                 my_charset = locale_charset ();
 332 #elif defined(HAVE_LANGINFO_H)
 333                 my_charset = nl_langinfo (CODESET);
 334 #else
 335                 my_charset = "UTF-8";
 336 #endif
 337                 is_utf8 = strcmp (my_charset, "UTF-8") == 0;
 338 #endif
 339         }
 340
 341         if (charset != NULL)
 342                 *charset = my_charset;
 343
 344         return is_utf8;
 345 }
 346
 347 gchar *
 348 g_locale_to_utf8 (const gchar *opsysstring, gssize len, gsize *bytes_read, gsize *bytes_written, GError **error)
 349 {
 350         g_get_charset (NULL);
 351
 352         return g_convert (opsysstring, len, "UTF-8", my_charset, bytes_read, bytes_written, error);
 353 }
 354
 355 gchar *
 356 g_locale_from_utf8 (const gchar *utf8string, gssize len, gsize *bytes_read, gsize *bytes_written, GError **error)
 357 {
 358         g_get_charset (NULL);
 359
 360         return g_convert (utf8string, len, my_charset, "UTF-8", bytes_read, bytes_written, error);
 361 }
 362 /**
 363  * g_utf8_validate
 364  * @utf: Pointer to putative UTF-8 encoded string.
 365  *
 366  * Checks @utf for being valid UTF-8. @utf is assumed to be
 367  * null-terminated. This function is not super-strict, as it will
 368  * allow longer UTF-8 sequences than necessary. Note that Java is
 369  * capable of producing these sequences if provoked. Also note, this
 370  * routine checks for the 4-byte maximum size, but does not check for
 371  * 0x10ffff maximum value.
 372  *
 373  * Return value: true if @utf is valid.
 374  **/
 375 gboolean
 376 g_utf8_validate (const gchar *str, gssize max_len, const gchar **end)
 377 {
 378         gssize byteCount = 0;
 379         gboolean retVal = TRUE;
 380         gboolean lastRet = TRUE;
 381         guchar* ptr = (guchar*) str;
 382         guint length;
 383         guchar a;
 384         guchar* srcPtr;
 385         if (max_len == 0)
 386                 return 0;
 387         else if (max_len < 0)
 388                 byteCount = max_len;
 389         while (*ptr != 0 && byteCount <= max_len) {
 390                 length = g_trailingBytesForUTF8 [*ptr] + 1;
 391                 srcPtr = (guchar*) ptr + length;
 392                 switch (length) {
 393                 default: retVal = FALSE;
 394                 /* Everything else falls through when "TRUE"... */
 395                 case 4: if ((a = (*--srcPtr)) < (guchar) 0x80 || a > (guchar) 0xBF) retVal = FALSE;
 396                                 if ((a == (guchar) 0xBF || a == (guchar) 0xBE) && *(srcPtr-1) == (guchar) 0xBF) {
 397                                 if (*(srcPtr-2) == (guchar) 0x8F || *(srcPtr-2) == (guchar) 0x9F ||
 398                                         *(srcPtr-2) == (guchar) 0xAF || *(srcPtr-2) == (guchar) 0xBF)
 399                                         retVal = FALSE;
 400                                 }
 401                 case 3: if ((a = (*--srcPtr)) < (guchar) 0x80 || a > (guchar) 0xBF) retVal = FALSE;
 402                 case 2: if ((a = (*--srcPtr)) < (guchar) 0x80 || a > (guchar) 0xBF) retVal = FALSE;
 403
 404                 switch (*ptr) {
 405                 /* no fall-through in this inner switch */
 406                 case 0xE0: if (a < (guchar) 0xA0) retVal = FALSE; break;
 407                 case 0xED: if (a > (guchar) 0x9F) retVal = FALSE; break;
 408                 case 0xEF: if (a == (guchar)0xB7 && (*(srcPtr+1) > (guchar) 0x8F && *(srcPtr+1) < 0xB0)) retVal = FALSE;
 409                                    if (a == (guchar)0xBF && (*(srcPtr+1) == (guchar) 0xBE || *(srcPtr+1) == 0xBF)) retVal = FALSE; break;
 410                 case 0xF0: if (a < (guchar) 0x90) retVal = FALSE; break;
 411                 case 0xF4: if (a > (guchar) 0x8F) retVal = FALSE; break;
 412                 default:   if (a < (guchar) 0x80) retVal = FALSE;
 413                 }
 414
 415                 case 1: if (*ptr >= (guchar ) 0x80 && *ptr < (guchar) 0xC2) retVal = FALSE;
 416                 }
 417                 if (*ptr > (guchar) 0xF4)
 418                         retVal = FALSE;
 419                 //If the string is invalid, set the end to the invalid byte.
 420                 if (!retVal && lastRet) {
 421                         if (end != NULL)
 422                                 *end = (gchar*) ptr;
 423                         lastRet = FALSE;
 424                 }
 425                 ptr += length;
 426                 if(max_len > 0)
 427                         byteCount += length;
 428         }
 429         if (retVal && end != NULL)
 430                 *end = (gchar*) ptr;
 431         return retVal;
 432 }
 433 /**
 434  * g_utf8_get_char
 435  * @src: Pointer to UTF-8 encoded character.
 436  *
 437  * Return value: UTF-16 value of @src
 438  **/
 439 gunichar
 440 g_utf8_get_char (const gchar *src)
 441 {
 442         gunichar ch = 0;
 443         guchar* ptr = (guchar*) src;
 444         gushort extraBytesToRead = g_trailingBytesForUTF8 [*ptr];
 445
 446         switch (extraBytesToRead) {
 447         case 5: ch += *ptr++; ch <<= 6; // remember, illegal UTF-8
 448         case 4: ch += *ptr++; ch <<= 6; // remember, illegal UTF-8
 449         case 3: ch += *ptr++; ch <<= 6;
 450         case 2: ch += *ptr++; ch <<= 6;
 451         case 1: ch += *ptr++; ch <<= 6;
 452         case 0: ch += *ptr;
 453         }
 454         ch -= offsetsFromUTF8 [extraBytesToRead];
 455         return ch;
 456 }
 457 glong
 458 g_utf8_strlen (const gchar *str, gssize max)
 459 {
 460         gssize byteCount = 0;
 461         guchar* ptr = (guchar*) str;
 462         glong length = 0;
 463         if (max == 0)
 464                 return 0;
 465         else if (max < 0)
 466                 byteCount = max;
 467         while (*ptr != 0 && byteCount <= max) {
 468                 gssize cLen = g_trailingBytesForUTF8 [*ptr] + 1;
 469                 if (max > 0 && (byteCount + cLen) > max)
 470                         return length;
 471                 ptr += cLen;
 472                 length++;
 473                 if (max > 0)
 474                         byteCount += cLen;
 475         }
 476         return length;
 477 }