iconv/iconv_charmap.c

   1 /* Convert using charmaps and possibly iconv().
   2    Copyright (C) 2001-2019 Free Software Foundation, Inc.
   3    This file is part of the GNU C Library.
   4    Contributed by Ulrich Drepper <drepper@redhat.com>, 2001.
   5
   6    This program is free software; you can redistribute it and/or modify
   7    it under the terms of the GNU General Public License as published
   8    by the Free Software Foundation; version 2 of the License, or
   9    (at your option) any later version.
  10
  11    This program is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14    GNU General Public License for more details.
  15
  16    You should have received a copy of the GNU General Public License
  17    along with this program; if not, see <https://www.gnu.org/licenses/>.  */
  18
  19 #include <assert.h>
  20 #include <errno.h>
  21 #include <error.h>
  22 #include <fcntl.h>
  23 #include <iconv.h>
  24 #include <libintl.h>
  25 #include <stdio.h>
  26 #include <stdlib.h>
  27 #include <unistd.h>
  28 #include <stdint.h>
  29 #include <sys/mman.h>
  30 #include <sys/stat.h>
  31
  32 #include "iconv_prog.h"
  33
  34
  35 /* Prototypes for a few program-wide used functions.  */
  36 #include <programs/xmalloc.h>
  37
  38
  39 struct convtable
  40 {
  41   int term[256 / 8];
  42   union
  43   {
  44     struct convtable *sub;
  45     struct charseq *out;
  46   } val[256];
  47 };
  48
  49
  50 static inline struct convtable *
  51 allocate_table (void)
  52 {
  53   return (struct convtable *) xcalloc (1, sizeof (struct convtable));
  54 }
  55
  56
  57 static inline int
  58 is_term (struct convtable *tbl, unsigned int idx)
  59 {
  60   return tbl->term[idx / 8] & (1 << (idx % 8));
  61 }
  62
  63
  64 static inline void
  65 clear_term (struct convtable *tbl, unsigned int idx)
  66 {
  67   tbl->term[idx / 8] &= ~(1 << (idx % 8));
  68 }
  69
  70
  71 static inline void
  72 set_term (struct convtable *tbl, unsigned int idx)
  73 {
  74   tbl->term[idx / 8] |= 1 << (idx % 8);
  75 }
  76
  77
  78 /* Generate the conversion table.  */
  79 static struct convtable *use_from_charmap (struct charmap_t *from_charmap,
  80                                            const char *to_code);
  81 static struct convtable *use_to_charmap (const char *from_code,
  82                                          struct charmap_t *to_charmap);
  83 static struct convtable *use_both_charmaps (struct charmap_t *from_charmap,
  84                                             struct charmap_t *to_charmap);
  85
  86 /* Prototypes for the functions doing the actual work.  */
  87 static int process_block (struct convtable *tbl, char *addr, size_t len,
  88                           FILE *output);
  89 static int process_fd (struct convtable *tbl, int fd, FILE *output);
  90 static int process_file (struct convtable *tbl, FILE *input, FILE *output);
  91
  92
  93 int
  94 charmap_conversion (const char *from_code, struct charmap_t *from_charmap,
  95                     const char *to_code, struct charmap_t *to_charmap,
  96                     int argc, int remaining, char *argv[],
  97                     const char *output_file)
  98 {
  99   struct convtable *cvtbl;
 100   int status = EXIT_SUCCESS;
 101
 102   /* We have three different cases to handle:
 103
 104      - both, from_charmap and to_charmap, are available.  This means we
 105        can assume that the symbolic names match and use them to create
 106        the mapping.
 107
 108      - only from_charmap is available.  In this case we can only hope that
 109        the symbolic names used are of the <Uxxxx> form in which case we
 110        can use a UCS4->"to_code" iconv() conversion for the second step.
 111
 112      - only to_charmap is available.  This is similar, only that we would
 113        use iconv() for the "to_code"->UCS4 conversion.
 114
 115        We first create a table which maps input bytes into output bytes.
 116        Once this is done we can handle all three of the cases above
 117        equally.  */
 118   if (from_charmap != NULL)
 119     {
 120       if (to_charmap == NULL)
 121         cvtbl = use_from_charmap (from_charmap, to_code);
 122       else
 123         cvtbl = use_both_charmaps (from_charmap, to_charmap);
 124     }
 125   else
 126     {
 127       assert (to_charmap != NULL);
 128       cvtbl = use_to_charmap (from_code, to_charmap);
 129     }
 130
 131   /* If we couldn't generate a table stop now.  */
 132   if (cvtbl == NULL)
 133     return EXIT_FAILURE;
 134
 135   /* Determine output file.  */
 136   FILE *output;
 137   if (output_file != NULL && strcmp (output_file, "-") != 0)
 138     {
 139       output = fopen (output_file, "w");
 140       if (output == NULL)
 141         error (EXIT_FAILURE, errno, _("cannot open output file"));
 142     }
 143   else
 144     output = stdout;
 145
 146   /* We can now start the conversion.  */
 147   if (remaining == argc)
 148     {
 149       if (process_file (cvtbl, stdin, output) != 0)
 150         status = EXIT_FAILURE;
 151     }
 152   else
 153     do
 154       {
 155         int fd;
 156
 157         if (verbose)
 158           printf ("%s:\n", argv[remaining]);
 159         if (strcmp (argv[remaining], "-") == 0)
 160           fd = 0;
 161         else
 162           {
 163             fd = open (argv[remaining], O_RDONLY);
 164
 165             if (fd == -1)
 166               {
 167                 error (0, errno, _("cannot open input file `%s'"),
 168                        argv[remaining]);
 169                 status = EXIT_FAILURE;
 170                 continue;
 171               }
 172           }
 173
 174 #ifdef _POSIX_MAPPED_FILES
 175         struct stat64 st;
 176         char *addr;
 177         /* We have possibilities for reading the input file.  First try
 178            to mmap() it since this will provide the fastest solution.  */
 179         if (fstat64 (fd, &st) == 0
 180             && ((addr = mmap (NULL, st.st_size, PROT_READ, MAP_PRIVATE,
 181                               fd, 0)) != MAP_FAILED))
 182           {
 183             /* Yes, we can use mmap().  The descriptor is not needed
 184                anymore.  */
 185             if (close (fd) != 0)
 186               error (EXIT_FAILURE, errno,
 187                      _("error while closing input `%s'"), argv[remaining]);
 188
 189             if (process_block (cvtbl, addr, st.st_size, output) < 0)
 190               {
 191                 /* Something went wrong.  */
 192                 status = EXIT_FAILURE;
 193
 194                 /* We don't need the input data anymore.  */
 195                 munmap ((void *) addr, st.st_size);
 196
 197                 /* We cannot go on with producing output since it might
 198                    lead to problem because the last output might leave
 199                    the output stream in an undefined state.  */
 200                 break;
 201               }
 202
 203             /* We don't need the input data anymore.  */
 204             munmap ((void *) addr, st.st_size);
 205           }
 206         else
 207 #endif  /* _POSIX_MAPPED_FILES */
 208           {
 209             /* Read the file in pieces.  */
 210             if (process_fd (cvtbl, fd, output) != 0)
 211               {
 212                 /* Something went wrong.  */
 213                 status = EXIT_FAILURE;
 214
 215                 /* We don't need the input file anymore.  */
 216                 close (fd);
 217
 218                 /* We cannot go on with producing output since it might
 219                    lead to problem because the last output might leave
 220                    the output stream in an undefined state.  */
 221                 break;
 222               }
 223
 224             /* Now close the file.  */
 225             close (fd);
 226           }
 227       }
 228     while (++remaining < argc);
 229
 230   /* All done.  */
 231   return status;
 232 }
 233
 234
 235 /* Add the IN->OUT mapping to TBL.  OUT is potentially stored in the table.
 236    IN is used only here, so it need not be kept live afterwards.  */
 237 static void
 238 add_bytes (struct convtable *tbl, const struct charseq *in, struct charseq *out)
 239 {
 240   int n = 0;
 241   unsigned int byte;
 242
 243   assert (in->nbytes > 0);
 244
 245   byte = ((unsigned char *) in->bytes)[n];
 246   while (n + 1 < in->nbytes)
 247     {
 248       if (is_term (tbl, byte) || tbl->val[byte].sub == NULL)
 249         {
 250           /* Note that we simply ignore a definition for a byte sequence
 251              which is also the prefix for a longer one.  */
 252           clear_term (tbl, byte);
 253           tbl->val[byte].sub =
 254             (struct convtable *) xcalloc (1, sizeof (struct convtable));
 255         }
 256
 257       tbl = tbl->val[byte].sub;
 258
 259       byte = ((unsigned char *) in->bytes)[++n];
 260     }
 261
 262   /* Only add the new sequence if there is none yet and the byte sequence
 263      is not part of an even longer one.  */
 264   if (! is_term (tbl, byte) && tbl->val[byte].sub == NULL)
 265     {
 266       set_term (tbl, byte);
 267       tbl->val[byte].out = out;
 268     }
 269 }
 270
 271 /* Try to convert SEQ from WCHAR_T format using CD.
 272    Returns a malloc'd struct or NULL.  */
 273 static struct charseq *
 274 convert_charseq (iconv_t cd, const struct charseq *seq)
 275 {
 276   struct charseq *result = NULL;
 277
 278   if (seq->ucs4 != UNINITIALIZED_CHAR_VALUE)
 279     {
 280       /* There is a chance.  Try the iconv module.  */
 281       wchar_t inbuf[1] = { seq->ucs4 };
 282       unsigned char outbuf[64];
 283       char *inptr = (char *) inbuf;
 284       size_t inlen = sizeof (inbuf);
 285       char *outptr = (char *) outbuf;
 286       size_t outlen = sizeof (outbuf);
 287
 288       (void) iconv (cd, &inptr, &inlen, &outptr, &outlen);
 289
 290       if (outptr != (char *) outbuf)
 291         {
 292           /* We got some output.  Good, use it.  */
 293           outlen = sizeof (outbuf) - outlen;
 294           assert ((char *) outbuf + outlen == outptr);
 295
 296           result = xmalloc (sizeof (struct charseq) + outlen);
 297           result->name = seq->name;
 298           result->ucs4 = seq->ucs4;
 299           result->nbytes = outlen;
 300           memcpy (result->bytes, outbuf, outlen);
 301         }
 302
 303       /* Clear any possible state left behind.  */
 304       (void) iconv (cd, NULL, NULL, NULL, NULL);
 305     }
 306
 307   return result;
 308 }
 309
 310
 311 static struct convtable *
 312 use_from_charmap (struct charmap_t *from_charmap, const char *to_code)
 313 {
 314   /* We iterate over all entries in the from_charmap and for those which
 315      have a known UCS4 representation we use an iconv() call to determine
 316      the mapping to the to_code charset.  */
 317   struct convtable *rettbl;
 318   iconv_t cd;
 319   void *ptr = NULL;
 320   const void *key;
 321   size_t keylen;
 322   void *data;
 323
 324   cd = iconv_open (to_code, "WCHAR_T");
 325   if (cd == (iconv_t) -1)
 326     /* We cannot do anything.  */
 327     return NULL;
 328
 329   rettbl = allocate_table ();
 330
 331   while (iterate_table (&from_charmap->char_table, &ptr, &key, &keylen, &data)
 332          >= 0)
 333     {
 334       struct charseq *in = data;
 335       struct charseq *newp = convert_charseq (cd, in);
 336       if (newp != NULL)
 337         add_bytes (rettbl, in, newp);
 338     }
 339
 340   iconv_close (cd);
 341
 342   return rettbl;
 343 }
 344
 345
 346 static struct convtable *
 347 use_to_charmap (const char *from_code, struct charmap_t *to_charmap)
 348 {
 349   /* We iterate over all entries in the to_charmap and for those which
 350      have a known UCS4 representation we use an iconv() call to determine
 351      the mapping to the from_code charset.  */
 352   struct convtable *rettbl;
 353   iconv_t cd;
 354   void *ptr = NULL;
 355   const void *key;
 356   size_t keylen;
 357   void *data;
 358
 359   /* Note that the conversion we use here is the reverse direction.  Without
 360      exhaustive search we cannot figure out which input yields the UCS4
 361      character we are looking for.  Therefore we determine it the other
 362      way round.  */
 363   cd = iconv_open (from_code, "WCHAR_T");
 364   if (cd == (iconv_t) -1)
 365     /* We cannot do anything.  */
 366     return NULL;
 367
 368   rettbl = allocate_table ();
 369
 370   while (iterate_table (&to_charmap->char_table, &ptr, &key, &keylen, &data)
 371          >= 0)
 372     {
 373       struct charseq *out = data;
 374       struct charseq *newp = convert_charseq (cd, out);
 375       if (newp != NULL)
 376         {
 377           add_bytes (rettbl, newp, out);
 378           free (newp);
 379         }
 380     }
 381
 382   iconv_close (cd);
 383
 384   return rettbl;
 385 }
 386
 387
 388 static struct convtable *
 389 use_both_charmaps (struct charmap_t *from_charmap,
 390                    struct charmap_t *to_charmap)
 391 {
 392   /* In this case we iterate over all the entries in the from_charmap,
 393      determine the internal name, and find an appropriate entry in the
 394      to_charmap (if it exists).  */
 395   struct convtable *rettbl = allocate_table ();
 396   void *ptr = NULL;
 397   const void *key;
 398   size_t keylen;
 399   void *data;
 400
 401   while (iterate_table (&from_charmap->char_table, &ptr, &key, &keylen, &data)
 402          >= 0)
 403     {
 404       struct charseq *in = (struct charseq *) data;
 405       struct charseq *out = charmap_find_value (to_charmap, key, keylen);
 406
 407       if (out != NULL)
 408         add_bytes (rettbl, in, out);
 409     }
 410
 411   return rettbl;
 412 }
 413
 414
 415 static int
 416 process_block (struct convtable *tbl, char *addr, size_t len, FILE *output)
 417 {
 418   size_t n = 0;
 419
 420   while (n < len)
 421     {
 422       struct convtable *cur = tbl;
 423       unsigned char *curp = (unsigned char *) addr;
 424       unsigned int byte = *curp;
 425       int cnt;
 426       struct charseq *out;
 427
 428       while (! is_term (cur, byte))
 429         if (cur->val[byte].sub == NULL)
 430           {
 431             /* This is an invalid sequence.  Skip the first byte if we are
 432                ignoring errors.  Otherwise punt.  */
 433             if (! omit_invalid)
 434               {
 435                 error (0, 0, _("illegal input sequence at position %Zd"), n);
 436                 return -1;
 437               }
 438
 439             n -= curp - (unsigned char *) addr;
 440
 441             byte = *(curp = (unsigned char *) ++addr);
 442             if (++n >= len)
 443               /* All converted.  */
 444               return 0;
 445
 446             cur = tbl;
 447           }
 448         else
 449           {
 450             cur = cur->val[byte].sub;
 451
 452             if (++n >= len)
 453               {
 454                 error (0, 0, _("\
 455 incomplete character or shift sequence at end of buffer"));
 456                 return -1;
 457               }
 458
 459             byte = *++curp;
 460           }
 461
 462       /* We found a final byte.  Write the output bytes.  */
 463       out = cur->val[byte].out;
 464       for (cnt = 0; cnt < out->nbytes; ++cnt)
 465         fputc_unlocked (out->bytes[cnt], output);
 466
 467       addr = (char *) curp + 1;
 468       ++n;
 469     }
 470
 471   return 0;
 472 }
 473
 474
 475 static int
 476 process_fd (struct convtable *tbl, int fd, FILE *output)
 477 {
 478   /* We have a problem with reading from a descriptor since we must not
 479      provide the iconv() function an incomplete character or shift
 480      sequence at the end of the buffer.  Since we have to deal with
 481      arbitrary encodings we must read the whole text in a buffer and
 482      process it in one step.  */
 483   static char *inbuf = NULL;
 484   static size_t maxlen = 0;
 485   char *inptr = inbuf;
 486   size_t actlen = 0;
 487
 488   while (actlen < maxlen)
 489     {
 490       ssize_t n = read (fd, inptr, maxlen - actlen);
 491
 492       if (n == 0)
 493         /* No more text to read.  */
 494         break;
 495
 496       if (n == -1)
 497         {
 498           /* Error while reading.  */
 499           error (0, errno, _("error while reading the input"));
 500           return -1;
 501         }
 502
 503       inptr += n;
 504       actlen += n;
 505     }
 506
 507   if (actlen == maxlen)
 508     while (1)
 509       {
 510         ssize_t n;
 511         char *new_inbuf;
 512
 513         /* Increase the buffer.  */
 514         new_inbuf = (char *) realloc (inbuf, maxlen + 32768);
 515         if (new_inbuf == NULL)
 516           {
 517             error (0, errno, _("unable to allocate buffer for input"));
 518             return -1;
 519           }
 520         inbuf = new_inbuf;
 521         maxlen += 32768;
 522         inptr = inbuf + actlen;
 523
 524         do
 525           {
 526             n = read (fd, inptr, maxlen - actlen);
 527
 528             if (n == 0)
 529               /* No more text to read.  */
 530               break;
 531
 532             if (n == -1)
 533               {
 534                 /* Error while reading.  */
 535                 error (0, errno, _("error while reading the input"));
 536                 return -1;
 537               }
 538
 539             inptr += n;
 540             actlen += n;
 541           }
 542         while (actlen < maxlen);
 543
 544         if (n == 0)
 545           /* Break again so we leave both loops.  */
 546           break;
 547       }
 548
 549   /* Now we have all the input in the buffer.  Process it in one run.  */
 550   return process_block (tbl, inbuf, actlen, output);
 551 }
 552
 553
 554 static int
 555 process_file (struct convtable *tbl, FILE *input, FILE *output)
 556 {
 557   /* This should be safe since we use this function only for `stdin' and
 558      we haven't read anything so far.  */
 559   return process_fd (tbl, fileno (input), output);
 560 }