src/text/unicode.c

   1 /*****************************************************************************
   2  * unicode.c: Unicode <-> locale functions
   3  *****************************************************************************
   4  * Copyright (C) 2005-2006 the VideoLAN team
   5  * Copyright © 2005-2008 Rémi Denis-Courmont
   6  *
   7  * Authors: Rémi Denis-Courmont <rem # videolan.org>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 2 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this program; if not, write to the Free Software
  21  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  22  *****************************************************************************/
  23
  24 /*****************************************************************************
  25  * Preamble
  26  *****************************************************************************/
  27 #ifdef HAVE_CONFIG_H
  28 # include "config.h"
  29 #endif
  30
  31 #include <vlc_common.h>
  32 #include <vlc_charset.h>
  33
  34 #include <assert.h>
  35
  36 #include <stdio.h>
  37 #include <stdarg.h>
  38 #include <stdlib.h>
  39 #include <sys/types.h>
  40 #ifdef UNDER_CE
  41 #  include <tchar.h>
  42 #endif
  43 #include <errno.h>
  44
  45 #if defined (__APPLE__) || defined (HAVE_MAEMO)
  46 /* Define this if the OS always use UTF-8 internally */
  47 # define ASSUME_UTF8 1
  48 #endif
  49
  50 #if defined (ASSUME_UTF8)
  51 /* Cool */
  52 #elif defined (WIN32) || defined (UNDER_CE)
  53 # define USE_MB2MB 1
  54 #elif defined (HAVE_ICONV)
  55 # define USE_ICONV 1
  56 #else
  57 # error No UTF8 charset conversion implemented on this platform!
  58 #endif
  59
  60 static char *locale_fast (const char *string, bool from)
  61 {
  62     if( string == NULL )
  63         return NULL;
  64
  65 #if defined (USE_ICONV)
  66     vlc_iconv_t hd = vlc_iconv_open (from ? "UTF-8" : "",
  67                                      from ? "" : "UTF-8");
  68     if (hd == (vlc_iconv_t)(-1))
  69         return NULL; /* Uho! */
  70
  71     const char *iptr = string;
  72     size_t inb = strlen (string);
  73     size_t outb = inb * 6 + 1;
  74     char output[outb], *optr = output;
  75
  76     while (vlc_iconv (hd, &iptr, &inb, &optr, &outb) == (size_t)(-1))
  77     {
  78         *optr++ = '?';
  79         outb--;
  80         iptr++;
  81         inb--;
  82         vlc_iconv (hd, NULL, NULL, NULL, NULL); /* reset */
  83     }
  84     *optr = '\0';
  85     vlc_iconv_close (hd);
  86
  87     assert (inb == 0);
  88     assert (*iptr == '\0');
  89     assert (*optr == '\0');
  90     assert (strlen (output) == (size_t)(optr - output));
  91     return strdup (output);
  92 #elif defined (USE_MB2MB)
  93     char *out;
  94     int len;
  95
  96     len = 1 + MultiByteToWideChar (from ? CP_ACP : CP_UTF8,
  97                                    0, string, -1, NULL, 0);
  98     wchar_t *wide = malloc (len * sizeof (wchar_t));
  99     if (wide == NULL)
 100         return NULL;
 101
 102     MultiByteToWideChar (from ? CP_ACP : CP_UTF8, 0, string, -1, wide, len);
 103     len = 1 + WideCharToMultiByte (from ? CP_UTF8 : CP_ACP, 0, wide, -1,
 104                                    NULL, 0, NULL, NULL);
 105     out = malloc (len);
 106     if (out != NULL)
 107         WideCharToMultiByte (from ? CP_UTF8 : CP_ACP, 0, wide, -1, out, len,
 108                              NULL, NULL);
 109     free (wide);
 110     return out;
 111 #else
 112     (void)from;
 113     return (char *)string;
 114 #endif
 115 }
 116
 117
 118 static inline char *locale_dup (const char *string, bool from)
 119 {
 120     assert( string );
 121
 122 #if defined (USE_ICONV)
 123     return locale_fast (string, from);
 124 #elif defined (USE_MB2MB)
 125     return locale_fast (string, from);
 126 #else
 127     (void)from;
 128     return strdup (string);
 129 #endif
 130 }
 131
 132 /**
 133  * Releases (if needed) a localized or uniformized string.
 134  * @param str non-NULL return value from FromLocale() or ToLocale().
 135  */
 136 void LocaleFree (const char *str)
 137 {
 138 #if defined (USE_ICONV)
 139     free ((char *)str);
 140 #elif defined (USE_MB2MB)
 141     free ((char *)str);
 142 #else
 143     (void)str;
 144 #endif
 145 }
 146
 147
 148 /**
 149  * Converts a string from the system locale character encoding to UTF-8.
 150  *
 151  * @param locale nul-terminated string to convert
 152  *
 153  * @return a nul-terminated UTF-8 string, or NULL in case of error.
 154  * To avoid memory leak, you have to pass the result to LocaleFree()
 155  * when it is no longer needed.
 156  */
 157 char *FromLocale (const char *locale)
 158 {
 159     return locale_fast (locale, true);
 160 }
 161
 162 /**
 163  * converts a string from the system locale character encoding to utf-8,
 164  * the result is always allocated on the heap.
 165  *
 166  * @param locale nul-terminated string to convert
 167  *
 168  * @return a nul-terminated utf-8 string, or null in case of error.
 169  * The result must be freed using free() - as with the strdup() function.
 170  */
 171 char *FromLocaleDup (const char *locale)
 172 {
 173     return locale_dup (locale, true);
 174 }
 175
 176
 177 /**
 178  * ToLocale: converts an UTF-8 string to local system encoding.
 179  *
 180  * @param utf8 nul-terminated string to be converted
 181  *
 182  * @return a nul-terminated string, or NULL in case of error.
 183  * To avoid memory leak, you have to pass the result to LocaleFree()
 184  * when it is no longer needed.
 185  */
 186 char *ToLocale (const char *utf8)
 187 {
 188     return locale_fast (utf8, false);
 189 }
 190
 191
 192 /**
 193  * converts a string from UTF-8 to the system locale character encoding,
 194  * the result is always allocated on the heap.
 195  *
 196  * @param utf8 nul-terminated string to convert
 197  *
 198  * @return a nul-terminated string, or null in case of error.
 199  * The result must be freed using free() - as with the strdup() function.
 200  */
 201 char *ToLocaleDup (const char *utf8)
 202 {
 203     return locale_dup (utf8, false);
 204 }
 205
 206 /**
 207  * Formats an UTF-8 string as vasprintf(), then print it to stdout, with
 208  * appropriate conversion to local encoding.
 209  */
 210 static int utf8_vasprintf( char **str, const char *fmt, va_list ap )
 211 {
 212     char *utf8;
 213     int res = vasprintf( &utf8, fmt, ap );
 214     if( res == -1 )
 215         return -1;
 216
 217     *str = ToLocaleDup( utf8 );
 218     free( utf8 );
 219     return res;
 220 }
 221
 222 /**
 223  * Formats an UTF-8 string as vfprintf(), then print it, with
 224  * appropriate conversion to local encoding.
 225  */
 226 int utf8_vfprintf( FILE *stream, const char *fmt, va_list ap )
 227 {
 228     char *str;
 229     int res = utf8_vasprintf( &str, fmt, ap );
 230     if( res == -1 )
 231         return -1;
 232
 233     fputs( str, stream );
 234     free( str );
 235     return res;
 236 }
 237
 238 /**
 239  * Formats an UTF-8 string as fprintf(), then print it, with
 240  * appropriate conversion to local encoding.
 241  */
 242 int utf8_fprintf( FILE *stream, const char *fmt, ... )
 243 {
 244     va_list ap;
 245     int res;
 246
 247     va_start( ap, fmt );
 248     res = utf8_vfprintf( stream, fmt, ap );
 249     va_end( ap );
 250     return res;
 251 }
 252
 253
 254 static char *CheckUTF8( char *str, char rep )
 255 {
 256     uint8_t *ptr = (uint8_t *)str;
 257     assert (str != NULL);
 258
 259     for (;;)
 260     {
 261         uint8_t c = ptr[0];
 262
 263         if (c == '\0')
 264             break;
 265
 266         if (c > 0xF4)
 267             goto error;
 268
 269         int charlen = clz8 (c ^ 0xFF);
 270         switch (charlen)
 271         {
 272             case 0: // 7-bit ASCII character -> OK
 273                 ptr++;
 274                 continue;
 275
 276             case 1: // continuation byte -> error
 277                 goto error;
 278         }
 279
 280         assert (charlen >= 2 && charlen <= 4);
 281
 282         uint32_t cp = c & ~((0xff >> (7 - charlen)) << (7 - charlen));
 283         for (int i = 1; i < charlen; i++)
 284         {
 285             assert (cp < (1 << 26));
 286             c = ptr[i];
 287
 288             if ((c >> 6) != 2) // not a continuation byte
 289                 goto error;
 290
 291             cp = (cp << 6) | (ptr[i] & 0x3f);
 292         }
 293
 294         switch (charlen)
 295         {
 296             case 4:
 297                 if (cp > 0x10FFFF) // beyond Unicode
 298                     goto error;
 299             case 3:
 300                 if (cp >= 0xD800 && cp < 0xC000) // UTF-16 surrogate
 301                     goto error;
 302             case 2:
 303                 if (cp < 128) // ASCII overlong
 304                     goto error;
 305                 if (cp < (1u << (5 * charlen - 3))) // overlong
 306                     goto error;
 307         }
 308         ptr += charlen;
 309         continue;
 310
 311     error:
 312         if (rep == 0)
 313             return NULL;
 314         *ptr++ = rep;
 315         str = NULL;
 316     }
 317
 318     return str;
 319 }
 320
 321 /**
 322  * Replaces invalid/overlong UTF-8 sequences with question marks.
 323  * Note that it is not possible to convert from Latin-1 to UTF-8 on the fly,
 324  * so we don't try that, even though it would be less disruptive.
 325  *
 326  * @return str if it was valid UTF-8, NULL if not.
 327  */
 328 char *EnsureUTF8( char *str )
 329 {
 330     return CheckUTF8( str, '?' );
 331 }
 332
 333
 334 /**
 335  * Checks whether a string is a valid UTF-8 byte sequence.
 336  *
 337  * @param str nul-terminated string to be checked
 338  *
 339  * @return str if it was valid UTF-8, NULL if not.
 340  */
 341 const char *IsUTF8( const char *str )
 342 {
 343     return CheckUTF8( (char *)str, 0 );
 344 }
 345
 346 /**
 347  * Converts a string from the given character encoding to utf-8.
 348  *
 349  * @return a nul-terminated utf-8 string, or null in case of error.
 350  * The result must be freed using free().
 351  */
 352 char *FromCharset(const char *charset, const void *data, size_t data_size)
 353 {
 354     vlc_iconv_t handle = vlc_iconv_open ("UTF-8", charset);
 355     if (handle == (vlc_iconv_t)(-1))
 356         return NULL;
 357
 358     char *out = NULL;
 359     for(unsigned mul = 4; mul < 8; mul++ )
 360     {
 361         size_t in_size = data_size;
 362         const char *in = data;
 363         size_t out_max = mul * data_size;
 364         char *tmp = out = malloc (1 + out_max);
 365         if (!out)
 366             break;
 367
 368         if (vlc_iconv (handle, &in, &in_size, &tmp, &out_max) != (size_t)(-1)) {
 369             *tmp = '\0';
 370             break;
 371         }
 372         free(out);
 373         out = NULL;
 374
 375         if (errno != E2BIG)
 376             break;
 377     }
 378     vlc_iconv_close(handle);
 379     return out;
 380 }
 381