iconv/iconv_charmap.c

   1 /* Convert using charmaps and possibly iconv().
   2    Copyright (C) 2001-2014 Free Software Foundation, Inc.
   3    This file is part of the GNU C Library.
   4    Contributed by Ulrich Drepper <drepper@redhat.com>, 2001.
   5
   6    This program is free software; you can redistribute it and/or modify
   7    it under the terms of the GNU General Public License as published
   8    by the Free Software Foundation; version 2 of the License, or
   9    (at your option) any later version.
  10
  11    This program is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14    GNU General Public License for more details.
  15
  16    You should have received a copy of the GNU General Public License
  17    along with this program; if not, see <http://www.gnu.org/licenses/>.  */
  18
  19 #include <assert.h>
  20 #include <errno.h>
  21 #include <error.h>
  22 #include <fcntl.h>
  23 #include <iconv.h>
  24 #include <libintl.h>
  25 #include <stdio.h>
  26 #include <stdlib.h>
  27 #include <unistd.h>
  28 #include <stdint.h>
  29 #include <sys/mman.h>
  30 #include <sys/stat.h>
  31
  32 #include "iconv_prog.h"
  33
  34
  35 /* Prototypes for a few program-wide used functions.  */
  36 #include <programs/xmalloc.h>
  37
  38
  39 struct convtable
  40 {
  41   int term[256 / 8];
  42   union
  43   {
  44     struct convtable *sub;
  45     struct charseq *out;
  46   } val[256];
  47 };
  48
  49
  50 static inline struct convtable *
  51 allocate_table (void)
  52 {
  53   return (struct convtable *) xcalloc (1, sizeof (struct convtable));
  54 }
  55
  56
  57 static inline int
  58 is_term (struct convtable *tbl, unsigned int idx)
  59 {
  60   return tbl->term[idx / 8] & (1 << (idx % 8));
  61 }
  62
  63
  64 static inline void
  65 clear_term (struct convtable *tbl, unsigned int idx)
  66 {
  67   tbl->term[idx / 8] &= ~(1 << (idx % 8));
  68 }
  69
  70
  71 static inline void
  72 set_term (struct convtable *tbl, unsigned int idx)
  73 {
  74   tbl->term[idx / 8] |= 1 << (idx % 8);
  75 }
  76
  77
  78 /* Generate the conversion table.  */
  79 static struct convtable *use_from_charmap (struct charmap_t *from_charmap,
  80                                            const char *to_code);
  81 static struct convtable *use_to_charmap (const char *from_code,
  82                                          struct charmap_t *to_charmap);
  83 static struct convtable *use_both_charmaps (struct charmap_t *from_charmap,
  84                                             struct charmap_t *to_charmap);
  85
  86 /* Prototypes for the functions doing the actual work.  */
  87 static int process_block (struct convtable *tbl, char *addr, size_t len,
  88                           FILE *output);
  89 static int process_fd (struct convtable *tbl, int fd, FILE *output);
  90 static int process_file (struct convtable *tbl, FILE *input, FILE *output);
  91
  92
  93 int
  94 charmap_conversion (const char *from_code, struct charmap_t *from_charmap,
  95                     const char *to_code, struct charmap_t *to_charmap,
  96                     int argc, int remaining, char *argv[],
  97                     const char *output_file)
  98 {
  99   struct convtable *cvtbl;
 100   int status = EXIT_SUCCESS;
 101
 102   /* We have three different cases to handle:
 103
 104      - both, from_charmap and to_charmap, are available.  This means we
 105        can assume that the symbolic names match and use them to create
 106        the mapping.
 107
 108      - only from_charmap is available.  In this case we can only hope that
 109        the symbolic names used are of the <Uxxxx> form in which case we
 110        can use a UCS4->"to_code" iconv() conversion for the second step.
 111
 112      - only to_charmap is available.  This is similar, only that we would
 113        use iconv() for the "to_code"->UCS4 conversion.
 114
 115        We first create a table which maps input bytes into output bytes.
 116        Once this is done we can handle all three of the cases above
 117        equally.  */
 118   if (from_charmap != NULL)
 119     {
 120       if (to_charmap == NULL)
 121         cvtbl = use_from_charmap (from_charmap, to_code);
 122       else
 123         cvtbl = use_both_charmaps (from_charmap, to_charmap);
 124     }
 125   else
 126     {
 127       assert (to_charmap != NULL);
 128       cvtbl = use_to_charmap (from_code, to_charmap);
 129     }
 130
 131   /* If we couldn't generate a table stop now.  */
 132   if (cvtbl == NULL)
 133     return EXIT_FAILURE;
 134
 135   /* Determine output file.  */
 136   FILE *output;
 137   if (output_file != NULL && strcmp (output_file, "-") != 0)
 138     {
 139       output = fopen (output_file, "w");
 140       if (output == NULL)
 141         error (EXIT_FAILURE, errno, _("cannot open output file"));
 142     }
 143   else
 144     output = stdout;
 145
 146   /* We can now start the conversion.  */
 147   if (remaining == argc)
 148     {
 149       if (process_file (cvtbl, stdin, output) != 0)
 150         status = EXIT_FAILURE;
 151     }
 152   else
 153     do
 154       {
 155         int fd;
 156
 157         if (verbose)
 158           printf ("%s:\n", argv[remaining]);
 159         if (strcmp (argv[remaining], "-") == 0)
 160           fd = 0;
 161         else
 162           {
 163             fd = open (argv[remaining], O_RDONLY);
 164
 165             if (fd == -1)
 166               {
 167                 error (0, errno, _("cannot open input file `%s'"),
 168                        argv[remaining]);
 169                 status = EXIT_FAILURE;
 170                 continue;
 171               }
 172           }
 173
 174 #ifdef _POSIX_MAPPED_FILES
 175         struct stat64 st;
 176         char *addr;
 177         /* We have possibilities for reading the input file.  First try
 178            to mmap() it since this will provide the fastest solution.  */
 179         if (fstat64 (fd, &st) == 0
 180             && ((addr = mmap (NULL, st.st_size, PROT_READ, MAP_PRIVATE,
 181                               fd, 0)) != MAP_FAILED))
 182           {
 183             /* Yes, we can use mmap().  The descriptor is not needed
 184                anymore.  */
 185             if (close (fd) != 0)
 186               error (EXIT_FAILURE, errno,
 187                      _("error while closing input `%s'"), argv[remaining]);
 188
 189             if (process_block (cvtbl, addr, st.st_size, output) < 0)
 190               {
 191                 /* Something went wrong.  */
 192                 status = EXIT_FAILURE;
 193
 194                 /* We don't need the input data anymore.  */
 195                 munmap ((void *) addr, st.st_size);
 196
 197                 /* We cannot go on with producing output since it might
 198                    lead to problem because the last output might leave
 199                    the output stream in an undefined state.  */
 200                 break;
 201               }
 202
 203             /* We don't need the input data anymore.  */
 204             munmap ((void *) addr, st.st_size);
 205           }
 206         else
 207 #endif  /* _POSIX_MAPPED_FILES */
 208           {
 209             /* Read the file in pieces.  */
 210             if (process_fd (cvtbl, fd, output) != 0)
 211               {
 212                 /* Something went wrong.  */
 213                 status = EXIT_FAILURE;
 214
 215                 /* We don't need the input file anymore.  */
 216                 close (fd);
 217
 218                 /* We cannot go on with producing output since it might
 219                    lead to problem because the last output might leave
 220                    the output stream in an undefined state.  */
 221                 break;
 222               }
 223
 224             /* Now close the file.  */
 225             close (fd);
 226           }
 227       }
 228     while (++remaining < argc);
 229
 230   /* All done.  */
 231   return status;
 232 }
 233
 234
 235 static void
 236 add_bytes (struct convtable *tbl, struct charseq *in, struct charseq *out)
 237 {
 238   int n = 0;
 239   unsigned int byte;
 240
 241   assert (in->nbytes > 0);
 242
 243   byte = ((unsigned char *) in->bytes)[n];
 244   while (n + 1 < in->nbytes)
 245     {
 246       if (is_term (tbl, byte) || tbl->val[byte].sub == NULL)
 247         {
 248           /* Note that we simply ignore a definition for a byte sequence
 249              which is also the prefix for a longer one.  */
 250           clear_term (tbl, byte);
 251           tbl->val[byte].sub =
 252             (struct convtable *) xcalloc (1, sizeof (struct convtable));
 253         }
 254
 255       tbl = tbl->val[byte].sub;
 256
 257       byte = ((unsigned char *) in->bytes)[++n];
 258     }
 259
 260   /* Only add the new sequence if there is none yet and the byte sequence
 261      is not part of an even longer one.  */
 262   if (! is_term (tbl, byte) && tbl->val[byte].sub == NULL)
 263     {
 264       set_term (tbl, byte);
 265       tbl->val[byte].out = out;
 266     }
 267 }
 268
 269
 270 static struct convtable *
 271 use_from_charmap (struct charmap_t *from_charmap, const char *to_code)
 272 {
 273   /* We iterate over all entries in the from_charmap and for those which
 274      have a known UCS4 representation we use an iconv() call to determine
 275      the mapping to the to_code charset.  */
 276   struct convtable *rettbl;
 277   iconv_t cd;
 278   void *ptr = NULL;
 279   const void *key;
 280   size_t keylen;
 281   void *data;
 282
 283   cd = iconv_open (to_code, "WCHAR_T");
 284   if (cd == (iconv_t) -1)
 285     /* We cannot do anything.  */
 286     return NULL;
 287
 288   rettbl = allocate_table ();
 289
 290   while (iterate_table (&from_charmap->char_table, &ptr, &key, &keylen, &data)
 291          >= 0)
 292     {
 293       struct charseq *in = (struct charseq *) data;
 294
 295       if (in->ucs4 != UNINITIALIZED_CHAR_VALUE)
 296         {
 297           /* There is a chance.  Try the iconv module.  */
 298           wchar_t inbuf[1] = { in->ucs4 };
 299           unsigned char outbuf[64];
 300           char *inptr = (char *) inbuf;
 301           size_t inlen = sizeof (inbuf);
 302           char *outptr = (char *) outbuf;
 303           size_t outlen = sizeof (outbuf);
 304
 305           (void) iconv (cd, &inptr, &inlen, &outptr, &outlen);
 306
 307           if (outptr != (char *) outbuf)
 308             {
 309               /* We got some output.  Good, use it.  */
 310               struct charseq *newp;
 311
 312               outlen = sizeof (outbuf) - outlen;
 313               assert ((char *) outbuf + outlen == outptr);
 314
 315               newp = (struct charseq *) xmalloc (sizeof (struct charseq)
 316                                                  + outlen);
 317               newp->name = in->name;
 318               newp->ucs4 = in->ucs4;
 319               newp->nbytes = outlen;
 320               memcpy (newp->bytes, outbuf, outlen);
 321
 322               add_bytes (rettbl, in, newp);
 323             }
 324
 325           /* Clear any possible state left behind.  */
 326           (void) iconv (cd, NULL, NULL, NULL, NULL);
 327         }
 328     }
 329
 330   iconv_close (cd);
 331
 332   return rettbl;
 333 }
 334
 335
 336 static struct convtable *
 337 use_to_charmap (const char *from_code, struct charmap_t *to_charmap)
 338 {
 339   /* We iterate over all entries in the to_charmap and for those which
 340      have a known UCS4 representation we use an iconv() call to determine
 341      the mapping to the from_code charset.  */
 342   struct convtable *rettbl;
 343   iconv_t cd;
 344   void *ptr = NULL;
 345   const void *key;
 346   size_t keylen;
 347   void *data;
 348
 349   /* Note that the conversion we use here is the reverse direction.  Without
 350      exhaustive search we cannot figure out which input yields the UCS4
 351      character we are looking for.  Therefore we determine it the other
 352      way round.  */
 353   cd = iconv_open (from_code, "WCHAR_T");
 354   if (cd == (iconv_t) -1)
 355     /* We cannot do anything.  */
 356     return NULL;
 357
 358   rettbl = allocate_table ();
 359
 360   while (iterate_table (&to_charmap->char_table, &ptr, &key, &keylen, &data)
 361          >= 0)
 362     {
 363       struct charseq *out = (struct charseq *) data;
 364
 365       if (out->ucs4 != UNINITIALIZED_CHAR_VALUE)
 366         {
 367           /* There is a chance.  Try the iconv module.  */
 368           wchar_t inbuf[1] = { out->ucs4 };
 369           unsigned char outbuf[64];
 370           char *inptr = (char *) inbuf;
 371           size_t inlen = sizeof (inbuf);
 372           char *outptr = (char *) outbuf;
 373           size_t outlen = sizeof (outbuf);
 374
 375           (void) iconv (cd, &inptr, &inlen, &outptr, &outlen);
 376
 377           if (outptr != (char *) outbuf)
 378             {
 379               /* We got some output.  Good, use it.  */
 380               union
 381               {
 382                 struct charseq seq;
 383                 struct
 384                 {
 385                   const char *name;
 386                   uint32_t ucs4;
 387                   int nbytes;
 388                   unsigned char bytes[outlen];
 389                 } mem;
 390               } new;
 391
 392               outlen = sizeof (outbuf) - outlen;
 393               assert ((char *) outbuf + outlen == outptr);
 394
 395               new.mem.name = out->name;
 396               new.mem.ucs4 = out->ucs4;
 397               new.mem.nbytes = outlen;
 398               memcpy (new.mem.bytes, outbuf, outlen);
 399
 400               add_bytes (rettbl, &new.seq, out);
 401             }
 402
 403           /* Clear any possible state left behind.  */
 404           (void) iconv (cd, NULL, NULL, NULL, NULL);
 405         }
 406     }
 407
 408   iconv_close (cd);
 409
 410   return rettbl;
 411 }
 412
 413
 414 static struct convtable *
 415 use_both_charmaps (struct charmap_t *from_charmap,
 416                    struct charmap_t *to_charmap)
 417 {
 418   /* In this case we iterate over all the entries in the from_charmap,
 419      determine the internal name, and find an appropriate entry in the
 420      to_charmap (if it exists).  */
 421   struct convtable *rettbl = allocate_table ();
 422   void *ptr = NULL;
 423   const void *key;
 424   size_t keylen;
 425   void *data;
 426
 427   while (iterate_table (&from_charmap->char_table, &ptr, &key, &keylen, &data)
 428          >= 0)
 429     {
 430       struct charseq *in = (struct charseq *) data;
 431       struct charseq *out = charmap_find_value (to_charmap, key, keylen);
 432
 433       if (out != NULL)
 434         add_bytes (rettbl, in, out);
 435     }
 436
 437   return rettbl;
 438 }
 439
 440
 441 static int
 442 process_block (struct convtable *tbl, char *addr, size_t len, FILE *output)
 443 {
 444   size_t n = 0;
 445
 446   while (n < len)
 447     {
 448       struct convtable *cur = tbl;
 449       unsigned char *curp = (unsigned char *) addr;
 450       unsigned int byte = *curp;
 451       int cnt;
 452       struct charseq *out;
 453
 454       while (! is_term (cur, byte))
 455         if (cur->val[byte].sub == NULL)
 456           {
 457             /* This is an invalid sequence.  Skip the first byte if we are
 458                ignoring errors.  Otherwise punt.  */
 459             if (! omit_invalid)
 460               {
 461                 error (0, 0, _("illegal input sequence at position %Zd"), n);
 462                 return -1;
 463               }
 464
 465             n -= curp - (unsigned char *) addr;
 466
 467             byte = *(curp = (unsigned char *) ++addr);
 468             if (++n >= len)
 469               /* All converted.  */
 470               return 0;
 471
 472             cur = tbl;
 473           }
 474         else
 475           {
 476             cur = cur->val[byte].sub;
 477
 478             if (++n >= len)
 479               {
 480                 error (0, 0, _("\
 481 incomplete character or shift sequence at end of buffer"));
 482                 return -1;
 483               }
 484
 485             byte = *++curp;
 486           }
 487
 488       /* We found a final byte.  Write the output bytes.  */
 489       out = cur->val[byte].out;
 490       for (cnt = 0; cnt < out->nbytes; ++cnt)
 491         fputc_unlocked (out->bytes[cnt], output);
 492
 493       addr = (char *) curp + 1;
 494       ++n;
 495     }
 496
 497   return 0;
 498 }
 499
 500
 501 static int
 502 process_fd (struct convtable *tbl, int fd, FILE *output)
 503 {
 504   /* We have a problem with reading from a descriptor since we must not
 505      provide the iconv() function an incomplete character or shift
 506      sequence at the end of the buffer.  Since we have to deal with
 507      arbitrary encodings we must read the whole text in a buffer and
 508      process it in one step.  */
 509   static char *inbuf = NULL;
 510   static size_t maxlen = 0;
 511   char *inptr = inbuf;
 512   size_t actlen = 0;
 513
 514   while (actlen < maxlen)
 515     {
 516       ssize_t n = read (fd, inptr, maxlen - actlen);
 517
 518       if (n == 0)
 519         /* No more text to read.  */
 520         break;
 521
 522       if (n == -1)
 523         {
 524           /* Error while reading.  */
 525           error (0, errno, _("error while reading the input"));
 526           return -1;
 527         }
 528
 529       inptr += n;
 530       actlen += n;
 531     }
 532
 533   if (actlen == maxlen)
 534     while (1)
 535       {
 536         ssize_t n;
 537         char *new_inbuf;
 538
 539         /* Increase the buffer.  */
 540         new_inbuf = (char *) realloc (inbuf, maxlen + 32768);
 541         if (new_inbuf == NULL)
 542           {
 543             error (0, errno, _("unable to allocate buffer for input"));
 544             return -1;
 545           }
 546         inbuf = new_inbuf;
 547         maxlen += 32768;
 548         inptr = inbuf + actlen;
 549
 550         do
 551           {
 552             n = read (fd, inptr, maxlen - actlen);
 553
 554             if (n == 0)
 555               /* No more text to read.  */
 556               break;
 557
 558             if (n == -1)
 559               {
 560                 /* Error while reading.  */
 561                 error (0, errno, _("error while reading the input"));
 562                 return -1;
 563               }
 564
 565             inptr += n;
 566             actlen += n;
 567           }
 568         while (actlen < maxlen);
 569
 570         if (n == 0)
 571           /* Break again so we leave both loops.  */
 572           break;
 573       }
 574
 575   /* Now we have all the input in the buffer.  Process it in one run.  */
 576   return process_block (tbl, inbuf, actlen, output);
 577 }
 578
 579
 580 static int
 581 process_file (struct convtable *tbl, FILE *input, FILE *output)
 582 {
 583   /* This should be safe since we use this function only for `stdin' and
 584      we haven't read anything so far.  */
 585   return process_fd (tbl, fileno (input), output);
 586 }