lib/mbfile.h

   1 /* Multibyte character I/O: macros for multi-byte encodings.
   2    Copyright (C) 2001, 2005, 2009-2017 Free Software Foundation, Inc.
   3
   4    This program is free software: you can redistribute it and/or modify
   5    it under the terms of the GNU General Public License as published by
   6    the Free Software Foundation; either version 3 of the License, or
   7    (at your option) any later version.
   8
   9    This program is distributed in the hope that it will be useful,
  10    but WITHOUT ANY WARRANTY; without even the implied warranty of
  11    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12    GNU General Public License for more details.
  13
  14    You should have received a copy of the GNU General Public License
  15    along with this program.  If not, see <https://www.gnu.org/licenses/>.  */
  16
  17 /* Written by Mitsuru Chinen <mchinen@yamato.ibm.com>
  18    and Bruno Haible <bruno@clisp.org>.  */
  19
  20 /* The macros in this file implement multi-byte character input from a
  21    stream.
  22
  23    mb_file_t
  24      is the type for multibyte character input stream, usable for variable
  25      declarations.
  26
  27    mbf_char_t
  28      is the type for multibyte character or EOF, usable for variable
  29      declarations.
  30
  31    mbf_init (mbf, stream)
  32      initializes the MB_FILE for reading from stream.
  33
  34    mbf_getc (mbc, mbf)
  35      reads the next multibyte character from mbf and stores it in mbc.
  36
  37    mb_iseof (mbc)
  38      returns true if mbc represents the EOF value.
  39
  40    Here are the function prototypes of the macros.
  41
  42    extern void          mbf_init (mb_file_t mbf, FILE *stream);
  43    extern void          mbf_getc (mbf_char_t mbc, mb_file_t mbf);
  44    extern bool          mb_iseof (const mbf_char_t mbc);
  45  */
  46
  47 #ifndef _MBFILE_H
  48 #define _MBFILE_H 1
  49
  50 #include <assert.h>
  51 #include <stdbool.h>
  52 #include <stdio.h>
  53 #include <string.h>
  54
  55 /* Tru64 with Desktop Toolkit C has a bug: <stdio.h> must be included before
  56    <wchar.h>.
  57    BSD/OS 4.1 has a bug: <stdio.h> and <time.h> must be included before
  58    <wchar.h>.  */
  59 #include <stdio.h>
  60 #include <time.h>
  61 #include <wchar.h>
  62
  63 #include "mbchar.h"
  64
  65 #ifndef _GL_INLINE_HEADER_BEGIN
  66  #error "Please include config.h first."
  67 #endif
  68 _GL_INLINE_HEADER_BEGIN
  69 #ifndef MBFILE_INLINE
  70 # define MBFILE_INLINE _GL_INLINE
  71 #endif
  72
  73 struct mbfile_multi {
  74   FILE *fp;
  75   bool eof_seen;
  76   bool have_pushback;
  77   mbstate_t state;
  78   unsigned int bufcount;
  79   char buf[MBCHAR_BUF_SIZE];
  80   struct mbchar pushback;
  81 };
  82
  83 MBFILE_INLINE void
  84 mbfile_multi_getc (struct mbchar *mbc, struct mbfile_multi *mbf)
  85 {
  86   size_t bytes;
  87
  88   /* If EOF has already been seen, don't use getc.  This matters if
  89      mbf->fp is connected to an interactive tty.  */
  90   if (mbf->eof_seen)
  91     goto eof;
  92
  93   /* Return character pushed back, if there is one.  */
  94   if (mbf->have_pushback)
  95     {
  96       mb_copy (mbc, &mbf->pushback);
  97       mbf->have_pushback = false;
  98       return;
  99     }
 100
 101   /* Before using mbrtowc, we need at least one byte.  */
 102   if (mbf->bufcount == 0)
 103     {
 104       int c = getc (mbf->fp);
 105       if (c == EOF)
 106         {
 107           mbf->eof_seen = true;
 108           goto eof;
 109         }
 110       mbf->buf[0] = (unsigned char) c;
 111       mbf->bufcount++;
 112     }
 113
 114   /* Handle most ASCII characters quickly, without calling mbrtowc().  */
 115   if (mbf->bufcount == 1 && mbsinit (&mbf->state) && is_basic (mbf->buf[0]))
 116     {
 117       /* These characters are part of the basic character set.  ISO C 99
 118          guarantees that their wide character code is identical to their
 119          char code.  */
 120       mbc->wc = mbc->buf[0] = mbf->buf[0];
 121       mbc->wc_valid = true;
 122       mbc->ptr = &mbc->buf[0];
 123       mbc->bytes = 1;
 124       mbf->bufcount = 0;
 125       return;
 126     }
 127
 128   /* Use mbrtowc on an increasing number of bytes.  Read only as many bytes
 129      from mbf->fp as needed.  This is needed to give reasonable interactive
 130      behaviour when mbf->fp is connected to an interactive tty.  */
 131   for (;;)
 132     {
 133       /* We don't know whether the 'mbrtowc' function updates the state when
 134          it returns -2, - this is the ISO C 99 and glibc-2.2 behaviour - or
 135          not - amended ANSI C, glibc-2.1 and Solaris 2.7 behaviour.  We
 136          don't have an autoconf test for this, yet.
 137          The new behaviour would allow us to feed the bytes one by one into
 138          mbrtowc.  But the old behaviour forces us to feed all bytes since
 139          the end of the last character into mbrtowc.  Since we want to retry
 140          with more bytes when mbrtowc returns -2, we must backup the state
 141          before calling mbrtowc, because implementations with the new
 142          behaviour will clobber it.  */
 143       mbstate_t backup_state = mbf->state;
 144
 145       bytes = mbrtowc (&mbc->wc, &mbf->buf[0], mbf->bufcount, &mbf->state);
 146
 147       if (bytes == (size_t) -1)
 148         {
 149           /* An invalid multibyte sequence was encountered.  */
 150           /* Return a single byte.  */
 151           bytes = 1;
 152           mbc->wc_valid = false;
 153           break;
 154         }
 155       else if (bytes == (size_t) -2)
 156         {
 157           /* An incomplete multibyte character.  */
 158           mbf->state = backup_state;
 159           if (mbf->bufcount == MBCHAR_BUF_SIZE)
 160             {
 161               /* An overlong incomplete multibyte sequence was encountered.  */
 162               /* Return a single byte.  */
 163               bytes = 1;
 164               mbc->wc_valid = false;
 165               break;
 166             }
 167           else
 168             {
 169               /* Read one more byte and retry mbrtowc.  */
 170               int c = getc (mbf->fp);
 171               if (c == EOF)
 172                 {
 173                   /* An incomplete multibyte character at the end.  */
 174                   mbf->eof_seen = true;
 175                   bytes = mbf->bufcount;
 176                   mbc->wc_valid = false;
 177                   break;
 178                 }
 179               mbf->buf[mbf->bufcount] = (unsigned char) c;
 180               mbf->bufcount++;
 181             }
 182         }
 183       else
 184         {
 185           if (bytes == 0)
 186             {
 187               /* A null wide character was encountered.  */
 188               bytes = 1;
 189               assert (mbf->buf[0] == '\0');
 190               assert (mbc->wc == 0);
 191             }
 192           mbc->wc_valid = true;
 193           break;
 194         }
 195     }
 196
 197   /* Return the multibyte sequence mbf->buf[0..bytes-1].  */
 198   mbc->ptr = &mbc->buf[0];
 199   memcpy (&mbc->buf[0], &mbf->buf[0], bytes);
 200   mbc->bytes = bytes;
 201
 202   mbf->bufcount -= bytes;
 203   if (mbf->bufcount > 0)
 204     {
 205       /* It's not worth calling memmove() for so few bytes.  */
 206       unsigned int count = mbf->bufcount;
 207       char *p = &mbf->buf[0];
 208
 209       do
 210         {
 211           *p = *(p + bytes);
 212           p++;
 213         }
 214       while (--count > 0);
 215     }
 216   return;
 217
 218 eof:
 219   /* An mbchar_t with bytes == 0 is used to indicate EOF.  */
 220   mbc->ptr = NULL;
 221   mbc->bytes = 0;
 222   mbc->wc_valid = false;
 223   return;
 224 }
 225
 226 MBFILE_INLINE void
 227 mbfile_multi_ungetc (const struct mbchar *mbc, struct mbfile_multi *mbf)
 228 {
 229   mb_copy (&mbf->pushback, mbc);
 230   mbf->have_pushback = true;
 231 }
 232
 233 typedef struct mbfile_multi mb_file_t;
 234
 235 typedef mbchar_t mbf_char_t;
 236
 237 #define mbf_init(mbf, stream)                                           \
 238   ((mbf).fp = (stream),                                                 \
 239    (mbf).eof_seen = false,                                              \
 240    (mbf).have_pushback = false,                                         \
 241    memset (&(mbf).state, '\0', sizeof (mbstate_t)),                     \
 242    (mbf).bufcount = 0)
 243
 244 #define mbf_getc(mbc, mbf) mbfile_multi_getc (&(mbc), &(mbf))
 245
 246 #define mbf_ungetc(mbc, mbf) mbfile_multi_ungetc (&(mbc), &(mbf))
 247
 248 #define mb_iseof(mbc) ((mbc).bytes == 0)
 249
 250 _GL_INLINE_HEADER_END
 251
 252 #endif /* _MBFILE_H */