lib/unicodeio.c

   1 /* Unicode character output to streams with locale dependent encoding.
   2
   3    Copyright (C) 2000-2003, 2006, 2008-2020 Free Software Foundation, Inc.
   4
   5    This program is free software: you can redistribute it and/or modify
   6    it under the terms of the GNU General Public License as published by
   7    the Free Software Foundation; either version 3 of the License, or
   8    (at your option) any later version.
   9
  10    This program is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13    GNU General Public License for more details.
  14
  15    You should have received a copy of the GNU General Public License
  16    along with this program.  If not, see <https://www.gnu.org/licenses/>.  */
  17
  18 /* Written by Bruno Haible <haible@clisp.cons.org>.  */
  19
  20 #include <config.h>
  21
  22 /* Specification.  */
  23 #include "unicodeio.h"
  24
  25 #include <stdio.h>
  26 #include <string.h>
  27 #include <errno.h>
  28
  29 #if HAVE_ICONV
  30 # include <iconv.h>
  31 #endif
  32
  33 #include <error.h>
  34
  35 #include "gettext.h"
  36 #define _(msgid) gettext (msgid)
  37 #define N_(msgid) msgid
  38
  39 #include "localcharset.h"
  40 #include "unistr.h"
  41
  42 /* When we pass a Unicode character to iconv(), we must pass it in a
  43    suitable encoding. The standardized Unicode encodings are
  44    UTF-8, UCS-2, UCS-4, UTF-16, UTF-16BE, UTF-16LE, UTF-7.
  45    UCS-2 supports only characters up to \U0000FFFF.
  46    UTF-16 and variants support only characters up to \U0010FFFF.
  47    UTF-7 is way too complex and not supported by glibc-2.1.
  48    UCS-4 specification leaves doubts about endianness and byte order
  49    mark. glibc currently interprets it as big endian without byte order
  50    mark, but this is not backed by an RFC.
  51    So we use UTF-8. It supports characters up to \U7FFFFFFF and is
  52    unambiguously defined.  */
  53
  54 /* Luckily, the encoding's name is platform independent.  */
  55 #define UTF8_NAME "UTF-8"
  56
  57 /* Converts the Unicode character CODE to its multibyte representation
  58    in the current locale and calls the SUCCESS callback on the resulting
  59    byte sequence.  If an error occurs, invokes the FAILURE callback instead,
  60    passing it CODE and an English error string.
  61    Returns whatever the callback returned.
  62    Assumes that the locale doesn't change between two calls.  */
  63 long
  64 unicode_to_mb (unsigned int code,
  65                long (*success) (const char *buf, size_t buflen,
  66                                 void *callback_arg),
  67                long (*failure) (unsigned int code, const char *msg,
  68                                 void *callback_arg),
  69                void *callback_arg)
  70 {
  71   static int initialized;
  72   static int is_utf8;
  73 #if HAVE_ICONV
  74   static iconv_t utf8_to_local;
  75 #endif
  76
  77   char inbuf[6];
  78   int count;
  79
  80   if (!initialized)
  81     {
  82       const char *charset = locale_charset ();
  83
  84       is_utf8 = !strcmp (charset, UTF8_NAME);
  85 #if HAVE_ICONV
  86       if (!is_utf8)
  87         {
  88           utf8_to_local = iconv_open (charset, UTF8_NAME);
  89           if (utf8_to_local == (iconv_t)(-1))
  90             /* For an unknown encoding, assume ASCII.  */
  91             utf8_to_local = iconv_open ("ASCII", UTF8_NAME);
  92         }
  93 #endif
  94       initialized = 1;
  95     }
  96
  97   /* Test whether the utf8_to_local converter is available at all.  */
  98   if (!is_utf8)
  99     {
 100 #if HAVE_ICONV
 101       if (utf8_to_local == (iconv_t)(-1))
 102         return failure (code, N_("iconv function not usable"), callback_arg);
 103 #else
 104       return failure (code, N_("iconv function not available"), callback_arg);
 105 #endif
 106     }
 107
 108   /* Convert the character to UTF-8.  */
 109   count = u8_uctomb ((unsigned char *) inbuf, code, sizeof (inbuf));
 110   if (count < 0)
 111     return failure (code, N_("character out of range"), callback_arg);
 112
 113 #if HAVE_ICONV
 114   if (!is_utf8)
 115     {
 116       char outbuf[25];
 117       const char *inptr;
 118       size_t inbytesleft;
 119       char *outptr;
 120       size_t outbytesleft;
 121       size_t res;
 122
 123       inptr = inbuf;
 124       inbytesleft = count;
 125       outptr = outbuf;
 126       outbytesleft = sizeof (outbuf);
 127
 128       /* Convert the character from UTF-8 to the locale's charset.  */
 129       res = iconv (utf8_to_local,
 130                    (ICONV_CONST char **)&inptr, &inbytesleft,
 131                    &outptr, &outbytesleft);
 132       /* Analyze what iconv() actually did and distinguish replacements
 133          that are OK (no need to invoke the FAILURE callback), such as
 134            - replacing GREEK SMALL LETTER MU with MICRO SIGN, or
 135            - replacing FULLWIDTH COLON with ':', or
 136            - replacing a Unicode TAG character (U+E00xx) with an empty string,
 137          from replacements that are worse than the FAILURE callback, such as
 138            - replacing 'ç' with '?' (NetBSD, Solaris 11) or '*' (musl) or
 139              NUL (IRIX).  */
 140       if (inbytesleft > 0 || res == (size_t)(-1)
 141           /* Irix iconv() inserts a NUL byte if it cannot convert.  */
 142 # if !defined _LIBICONV_VERSION && (defined sgi || defined __sgi)
 143           || (res > 0 && code != 0 && outptr - outbuf == 1 && *outbuf == '\0')
 144 # endif
 145           /* NetBSD iconv() and Solaris 11 iconv() insert a '?' if they cannot
 146              convert.  */
 147 # if !defined _LIBICONV_VERSION && (defined __NetBSD__ || defined __sun)
 148           || (res > 0 && outptr - outbuf == 1 && *outbuf == '?')
 149 # endif
 150           /* musl libc iconv() inserts a '*' if it cannot convert.  */
 151 # if !defined _LIBICONV_VERSION && MUSL_LIBC
 152           || (res > 0 && outptr - outbuf == 1 && *outbuf == '*')
 153 # endif
 154          )
 155         return failure (code, NULL, callback_arg);
 156
 157       /* Avoid glibc-2.1 bug and Solaris 7 bug.  */
 158 # if defined _LIBICONV_VERSION \
 159     || !(((__GLIBC__ - 0 == 2 && __GLIBC_MINOR__ - 0 <= 1) \
 160           && !defined __UCLIBC__) \
 161          || defined __sun)
 162
 163       /* Get back to the initial shift state.  */
 164       res = iconv (utf8_to_local, NULL, NULL, &outptr, &outbytesleft);
 165       if (res == (size_t)(-1))
 166         return failure (code, NULL, callback_arg);
 167 # endif
 168
 169       return success (outbuf, outptr - outbuf, callback_arg);
 170     }
 171 #endif
 172
 173   /* At this point, is_utf8 is true, so no conversion is needed.  */
 174   return success (inbuf, count, callback_arg);
 175 }
 176
 177 /* Simple success callback that outputs the converted string.
 178    The STREAM is passed as callback_arg.  */
 179 long
 180 fwrite_success_callback (const char *buf, size_t buflen, void *callback_arg)
 181 {
 182   FILE *stream = (FILE *) callback_arg;
 183
 184   /* The return value of fwrite can be ignored here, because under normal
 185      conditions (STREAM is an open stream and not wide-character oriented)
 186      when fwrite() returns a value != buflen it also sets STREAM's error
 187      indicator.  */
 188   fwrite (buf, 1, buflen, stream);
 189   return 0;
 190 }
 191
 192 /* Simple failure callback that displays an error and exits.  */
 193 static long
 194 exit_failure_callback (unsigned int code, const char *msg,
 195                        void *callback_arg _GL_UNUSED)
 196 {
 197   if (msg == NULL)
 198     error (1, 0, _("cannot convert U+%04X to local character set"), code);
 199   else
 200     error (1, 0, _("cannot convert U+%04X to local character set: %s"), code,
 201            gettext (msg));
 202   return -1;
 203 }
 204
 205 /* Simple failure callback that displays a fallback representation in plain
 206    ASCII, using the same notation as ISO C99 strings.  */
 207 static long
 208 fallback_failure_callback (unsigned int code,
 209                            const char *msg _GL_UNUSED,
 210                            void *callback_arg)
 211 {
 212   FILE *stream = (FILE *) callback_arg;
 213
 214   if (code < 0x10000)
 215     fprintf (stream, "\\u%04X", code);
 216   else
 217     fprintf (stream, "\\U%08X", code);
 218   return -1;
 219 }
 220
 221 /* Outputs the Unicode character CODE to the output stream STREAM.
 222    Upon failure, exit if exit_on_error is true, otherwise output a fallback
 223    notation.  */
 224 void
 225 print_unicode_char (FILE *stream, unsigned int code, int exit_on_error)
 226 {
 227   unicode_to_mb (code, fwrite_success_callback,
 228                  exit_on_error
 229                  ? exit_failure_callback
 230                  : fallback_failure_callback,
 231                  stream);
 232 }