iconv/iconv_charmap.c

   1 /* Convert using charmaps and possibly iconv().
   2    Copyright (C) 2001, 2005, 2006, 2008, 2012 Free Software Foundation, Inc.
   3    This file is part of the GNU C Library.
   4    Contributed by Ulrich Drepper <drepper@redhat.com>, 2001.
   5
   6    This program is free software; you can redistribute it and/or modify
   7    it under the terms of the GNU General Public License as published
   8    by the Free Software Foundation; version 2 of the License, or
   9    (at your option) any later version.
  10
  11    This program is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14    GNU General Public License for more details.
  15
  16    You should have received a copy of the GNU General Public License
  17    along with this program; if not, see <http://www.gnu.org/licenses/>.  */
  18
  19 #include <assert.h>
  20 #include <errno.h>
  21 #include <error.h>
  22 #include <fcntl.h>
  23 #include <iconv.h>
  24 #include <libintl.h>
  25 #include <stdio.h>
  26 #include <stdlib.h>
  27 #include <unistd.h>
  28 #include <sys/mman.h>
  29 #include <sys/stat.h>
  30
  31 #include "iconv_prog.h"
  32
  33
  34 /* Prototypes for a few program-wide used functions.  */
  35 extern void *xmalloc (size_t n)
  36   __attribute_malloc__ __attribute_alloc_size (1);
  37 extern void *xcalloc (size_t n, size_t s)
  38   __attribute_malloc__ __attribute_alloc_size (1, 2);
  39
  40
  41 struct convtable
  42 {
  43   int term[256 / 8];
  44   union
  45   {
  46     struct convtable *sub;
  47     struct charseq *out;
  48   } val[256];
  49 };
  50
  51
  52 static inline struct convtable *
  53 allocate_table (void)
  54 {
  55   return (struct convtable *) xcalloc (1, sizeof (struct convtable));
  56 }
  57
  58
  59 static inline int
  60 is_term (struct convtable *tbl, unsigned int idx)
  61 {
  62   return tbl->term[idx / 8] & (1 << (idx % 8));
  63 }
  64
  65
  66 static inline void
  67 clear_term (struct convtable *tbl, unsigned int idx)
  68 {
  69   tbl->term[idx / 8] &= ~(1 << (idx % 8));
  70 }
  71
  72
  73 static inline void
  74 set_term (struct convtable *tbl, unsigned int idx)
  75 {
  76   tbl->term[idx / 8] |= 1 << (idx % 8);
  77 }
  78
  79
  80 /* Generate the conversion table.  */
  81 static struct convtable *use_from_charmap (struct charmap_t *from_charmap,
  82                                            const char *to_code);
  83 static struct convtable *use_to_charmap (const char *from_code,
  84                                          struct charmap_t *to_charmap);
  85 static struct convtable *use_both_charmaps (struct charmap_t *from_charmap,
  86                                             struct charmap_t *to_charmap);
  87
  88 /* Prototypes for the functions doing the actual work.  */
  89 static int process_block (struct convtable *tbl, char *addr, size_t len,
  90                           FILE *output);
  91 static int process_fd (struct convtable *tbl, int fd, FILE *output);
  92 static int process_file (struct convtable *tbl, FILE *input, FILE *output);
  93
  94
  95 int
  96 charmap_conversion (const char *from_code, struct charmap_t *from_charmap,
  97                     const char *to_code, struct charmap_t *to_charmap,
  98                     int argc, int remaining, char *argv[],
  99                     const char *output_file)
 100 {
 101   struct convtable *cvtbl;
 102   int status = EXIT_SUCCESS;
 103
 104   /* We have three different cases to handle:
 105
 106      - both, from_charmap and to_charmap, are available.  This means we
 107        can assume that the symbolic names match and use them to create
 108        the mapping.
 109
 110      - only from_charmap is available.  In this case we can only hope that
 111        the symbolic names used are of the <Uxxxx> form in which case we
 112        can use a UCS4->"to_code" iconv() conversion for the second step.
 113
 114      - only to_charmap is available.  This is similar, only that we would
 115        use iconv() for the "to_code"->UCS4 conversion.
 116
 117        We first create a table which maps input bytes into output bytes.
 118        Once this is done we can handle all three of the cases above
 119        equally.  */
 120   if (from_charmap != NULL)
 121     {
 122       if (to_charmap == NULL)
 123         cvtbl = use_from_charmap (from_charmap, to_code);
 124       else
 125         cvtbl = use_both_charmaps (from_charmap, to_charmap);
 126     }
 127   else
 128     {
 129       assert (to_charmap != NULL);
 130       cvtbl = use_to_charmap (from_code, to_charmap);
 131     }
 132
 133   /* If we couldn't generate a table stop now.  */
 134   if (cvtbl == NULL)
 135     return EXIT_FAILURE;
 136
 137   /* Determine output file.  */
 138   FILE *output;
 139   if (output_file != NULL && strcmp (output_file, "-") != 0)
 140     {
 141       output = fopen (output_file, "w");
 142       if (output == NULL)
 143         error (EXIT_FAILURE, errno, _("cannot open output file"));
 144     }
 145   else
 146     output = stdout;
 147
 148   /* We can now start the conversion.  */
 149   if (remaining == argc)
 150     {
 151       if (process_file (cvtbl, stdin, output) != 0)
 152         status = EXIT_FAILURE;
 153     }
 154   else
 155     do
 156       {
 157         struct stat st;
 158         char *addr;
 159         int fd;
 160
 161         if (verbose)
 162           printf ("%s:\n", argv[remaining]);
 163         if (strcmp (argv[remaining], "-") == 0)
 164           fd = 0;
 165         else
 166           {
 167             fd = open (argv[remaining], O_RDONLY);
 168
 169             if (fd == -1)
 170               {
 171                 error (0, errno, _("cannot open input file `%s'"),
 172                        argv[remaining]);
 173                 status = EXIT_FAILURE;
 174                 continue;
 175               }
 176           }
 177
 178 #ifdef _POSIX_MAPPED_FILES
 179         /* We have possibilities for reading the input file.  First try
 180            to mmap() it since this will provide the fastest solution.  */
 181         if (fstat (fd, &st) == 0
 182             && ((addr = mmap (NULL, st.st_size, PROT_READ, MAP_PRIVATE,
 183                               fd, 0)) != MAP_FAILED))
 184           {
 185             /* Yes, we can use mmap().  The descriptor is not needed
 186                anymore.  */
 187             if (close (fd) != 0)
 188               error (EXIT_FAILURE, errno,
 189                      _("error while closing input `%s'"), argv[remaining]);
 190
 191             if (process_block (cvtbl, addr, st.st_size, output) < 0)
 192               {
 193                 /* Something went wrong.  */
 194                 status = EXIT_FAILURE;
 195
 196                 /* We don't need the input data anymore.  */
 197                 munmap ((void *) addr, st.st_size);
 198
 199                 /* We cannot go on with producing output since it might
 200                    lead to problem because the last output might leave
 201                    the output stream in an undefined state.  */
 202                 break;
 203               }
 204
 205             /* We don't need the input data anymore.  */
 206             munmap ((void *) addr, st.st_size);
 207           }
 208         else
 209 #endif  /* _POSIX_MAPPED_FILES */
 210           {
 211             /* Read the file in pieces.  */
 212             if (process_fd (cvtbl, fd, output) != 0)
 213               {
 214                 /* Something went wrong.  */
 215                 status = EXIT_FAILURE;
 216
 217                 /* We don't need the input file anymore.  */
 218                 close (fd);
 219
 220                 /* We cannot go on with producing output since it might
 221                    lead to problem because the last output might leave
 222                    the output stream in an undefined state.  */
 223                 break;
 224               }
 225
 226             /* Now close the file.  */
 227             close (fd);
 228           }
 229       }
 230     while (++remaining < argc);
 231
 232   /* All done.  */
 233   return status;
 234 }
 235
 236
 237 static void
 238 add_bytes (struct convtable *tbl, struct charseq *in, struct charseq *out)
 239 {
 240   int n = 0;
 241   unsigned int byte;
 242
 243   assert (in->nbytes > 0);
 244
 245   byte = ((unsigned char *) in->bytes)[n];
 246   while (n + 1 < in->nbytes)
 247     {
 248       if (is_term (tbl, byte) || tbl->val[byte].sub == NULL)
 249         {
 250           /* Note that we simply ignore a definition for a byte sequence
 251              which is also the prefix for a longer one.  */
 252           clear_term (tbl, byte);
 253           tbl->val[byte].sub =
 254             (struct convtable *) xcalloc (1, sizeof (struct convtable));
 255         }
 256
 257       tbl = tbl->val[byte].sub;
 258
 259       byte = ((unsigned char *) in->bytes)[++n];
 260     }
 261
 262   /* Only add the new sequence if there is none yet and the byte sequence
 263      is not part of an even longer one.  */
 264   if (! is_term (tbl, byte) && tbl->val[byte].sub == NULL)
 265     {
 266       set_term (tbl, byte);
 267       tbl->val[byte].out = out;
 268     }
 269 }
 270
 271
 272 static struct convtable *
 273 use_from_charmap (struct charmap_t *from_charmap, const char *to_code)
 274 {
 275   /* We iterate over all entries in the from_charmap and for those which
 276      have a known UCS4 representation we use an iconv() call to determine
 277      the mapping to the to_code charset.  */
 278   struct convtable *rettbl;
 279   iconv_t cd;
 280   void *ptr = NULL;
 281   const void *key;
 282   size_t keylen;
 283   void *data;
 284
 285   cd = iconv_open (to_code, "WCHAR_T");
 286   if (cd == (iconv_t) -1)
 287     /* We cannot do anything.  */
 288     return NULL;
 289
 290   rettbl = allocate_table ();
 291
 292   while (iterate_table (&from_charmap->char_table, &ptr, &key, &keylen, &data)
 293          >= 0)
 294     {
 295       struct charseq *in = (struct charseq *) data;
 296
 297       if (in->ucs4 != UNINITIALIZED_CHAR_VALUE)
 298         {
 299           /* There is a chance.  Try the iconv module.  */
 300           wchar_t inbuf[1] = { in->ucs4 };
 301           unsigned char outbuf[64];
 302           char *inptr = (char *) inbuf;
 303           size_t inlen = sizeof (inbuf);
 304           char *outptr = (char *) outbuf;
 305           size_t outlen = sizeof (outbuf);
 306
 307           (void) iconv (cd, &inptr, &inlen, &outptr, &outlen);
 308
 309           if (outptr != (char *) outbuf)
 310             {
 311               /* We got some output.  Good, use it.  */
 312               struct charseq *newp;
 313
 314               outlen = sizeof (outbuf) - outlen;
 315               assert ((char *) outbuf + outlen == outptr);
 316
 317               newp = (struct charseq *) xmalloc (sizeof (struct charseq)
 318                                                  + outlen);
 319               newp->name = in->name;
 320               newp->ucs4 = in->ucs4;
 321               newp->nbytes = outlen;
 322               memcpy (newp->bytes, outbuf, outlen);
 323
 324               add_bytes (rettbl, in, newp);
 325             }
 326
 327           /* Clear any possible state left behind.  */
 328           (void) iconv (cd, NULL, NULL, NULL, NULL);
 329         }
 330     }
 331
 332   iconv_close (cd);
 333
 334   return rettbl;
 335 }
 336
 337
 338 static struct convtable *
 339 use_to_charmap (const char *from_code, struct charmap_t *to_charmap)
 340 {
 341   /* We iterate over all entries in the to_charmap and for those which
 342      have a known UCS4 representation we use an iconv() call to determine
 343      the mapping to the from_code charset.  */
 344   struct convtable *rettbl;
 345   iconv_t cd;
 346   void *ptr = NULL;
 347   const void *key;
 348   size_t keylen;
 349   void *data;
 350
 351   /* Note that the conversion we use here is the reverse direction.  Without
 352      exhaustive search we cannot figure out which input yields the UCS4
 353      character we are looking for.  Therefore we determine it the other
 354      way round.  */
 355   cd = iconv_open (from_code, "WCHAR_T");
 356   if (cd == (iconv_t) -1)
 357     /* We cannot do anything.  */
 358     return NULL;
 359
 360   rettbl = allocate_table ();
 361
 362   while (iterate_table (&to_charmap->char_table, &ptr, &key, &keylen, &data)
 363          >= 0)
 364     {
 365       struct charseq *out = (struct charseq *) data;
 366
 367       if (out->ucs4 != UNINITIALIZED_CHAR_VALUE)
 368         {
 369           /* There is a chance.  Try the iconv module.  */
 370           wchar_t inbuf[1] = { out->ucs4 };
 371           unsigned char outbuf[64];
 372           char *inptr = (char *) inbuf;
 373           size_t inlen = sizeof (inbuf);
 374           char *outptr = (char *) outbuf;
 375           size_t outlen = sizeof (outbuf);
 376
 377           (void) iconv (cd, &inptr, &inlen, &outptr, &outlen);
 378
 379           if (outptr != (char *) outbuf)
 380             {
 381               /* We got some output.  Good, use it.  */
 382               union
 383               {
 384                 struct charseq seq;
 385                 struct
 386                 {
 387                   const char *name;
 388                   uint32_t ucs4;
 389                   int nbytes;
 390                   unsigned char bytes[outlen];
 391                 } mem;
 392               } new;
 393
 394               outlen = sizeof (outbuf) - outlen;
 395               assert ((char *) outbuf + outlen == outptr);
 396
 397               new.mem.name = out->name;
 398               new.mem.ucs4 = out->ucs4;
 399               new.mem.nbytes = outlen;
 400               memcpy (new.mem.bytes, outbuf, outlen);
 401
 402               add_bytes (rettbl, &new.seq, out);
 403             }
 404
 405           /* Clear any possible state left behind.  */
 406           (void) iconv (cd, NULL, NULL, NULL, NULL);
 407         }
 408     }
 409
 410   iconv_close (cd);
 411
 412   return rettbl;
 413 }
 414
 415
 416 static struct convtable *
 417 use_both_charmaps (struct charmap_t *from_charmap,
 418                    struct charmap_t *to_charmap)
 419 {
 420   /* In this case we iterate over all the entries in the from_charmap,
 421      determine the internal name, and find an appropriate entry in the
 422      to_charmap (if it exists).  */
 423   struct convtable *rettbl = allocate_table ();
 424   void *ptr = NULL;
 425   const void *key;
 426   size_t keylen;
 427   void *data;
 428
 429   while (iterate_table (&from_charmap->char_table, &ptr, &key, &keylen, &data)
 430          >= 0)
 431     {
 432       struct charseq *in = (struct charseq *) data;
 433       struct charseq *out = charmap_find_value (to_charmap, key, keylen);
 434
 435       if (out != NULL)
 436         add_bytes (rettbl, in, out);
 437     }
 438
 439   return rettbl;
 440 }
 441
 442
 443 static int
 444 process_block (struct convtable *tbl, char *addr, size_t len, FILE *output)
 445 {
 446   size_t n = 0;
 447
 448   while (n < len)
 449     {
 450       struct convtable *cur = tbl;
 451       unsigned char *curp = (unsigned char *) addr;
 452       unsigned int byte = *curp;
 453       int cnt;
 454       struct charseq *out;
 455
 456       while (! is_term (cur, byte))
 457         if (cur->val[byte].sub == NULL)
 458           {
 459             /* This is a invalid sequence.  Skip the first byte if we are
 460                ignoring errors.  Otherwise punt.  */
 461             if (! omit_invalid)
 462               {
 463                 error (0, 0, _("illegal input sequence at position %Zd"), n);
 464                 return -1;
 465               }
 466
 467             n -= curp - (unsigned char *) addr;
 468
 469             byte = *(curp = (unsigned char *) ++addr);
 470             if (++n >= len)
 471               /* All converted.  */
 472               return 0;
 473
 474             cur = tbl;
 475           }
 476         else
 477           {
 478             cur = cur->val[byte].sub;
 479
 480             if (++n >= len)
 481               {
 482                 error (0, 0, _("\
 483 incomplete character or shift sequence at end of buffer"));
 484                 return -1;
 485               }
 486
 487             byte = *++curp;
 488           }
 489
 490       /* We found a final byte.  Write the output bytes.  */
 491       out = cur->val[byte].out;
 492       for (cnt = 0; cnt < out->nbytes; ++cnt)
 493         fputc_unlocked (out->bytes[cnt], output);
 494
 495       addr = (char *) curp + 1;
 496       ++n;
 497     }
 498
 499   return 0;
 500 }
 501
 502
 503 static int
 504 process_fd (struct convtable *tbl, int fd, FILE *output)
 505 {
 506   /* We have a problem with reading from a descriptor since we must not
 507      provide the iconv() function an incomplete character or shift
 508      sequence at the end of the buffer.  Since we have to deal with
 509      arbitrary encodings we must read the whole text in a buffer and
 510      process it in one step.  */
 511   static char *inbuf = NULL;
 512   static size_t maxlen = 0;
 513   char *inptr = inbuf;
 514   size_t actlen = 0;
 515
 516   while (actlen < maxlen)
 517     {
 518       ssize_t n = read (fd, inptr, maxlen - actlen);
 519
 520       if (n == 0)
 521         /* No more text to read.  */
 522         break;
 523
 524       if (n == -1)
 525         {
 526           /* Error while reading.  */
 527           error (0, errno, _("error while reading the input"));
 528           return -1;
 529         }
 530
 531       inptr += n;
 532       actlen += n;
 533     }
 534
 535   if (actlen == maxlen)
 536     while (1)
 537       {
 538         ssize_t n;
 539         char *new_inbuf;
 540
 541         /* Increase the buffer.  */
 542         new_inbuf = (char *) realloc (inbuf, maxlen + 32768);
 543         if (new_inbuf == NULL)
 544           {
 545             error (0, errno, _("unable to allocate buffer for input"));
 546             return -1;
 547           }
 548         inbuf = new_inbuf;
 549         maxlen += 32768;
 550         inptr = inbuf + actlen;
 551
 552         do
 553           {
 554             n = read (fd, inptr, maxlen - actlen);
 555
 556             if (n == 0)
 557               /* No more text to read.  */
 558               break;
 559
 560             if (n == -1)
 561               {
 562                 /* Error while reading.  */
 563                 error (0, errno, _("error while reading the input"));
 564                 return -1;
 565               }
 566
 567             inptr += n;
 568             actlen += n;
 569           }
 570         while (actlen < maxlen);
 571
 572         if (n == 0)
 573           /* Break again so we leave both loops.  */
 574           break;
 575       }
 576
 577   /* Now we have all the input in the buffer.  Process it in one run.  */
 578   return process_block (tbl, inbuf, actlen, output);
 579 }
 580
 581
 582 static int
 583 process_file (struct convtable *tbl, FILE *input, FILE *output)
 584 {
 585   /* This should be safe since we use this function only for `stdin' and
 586      we haven't read anything so far.  */
 587   return process_fd (tbl, fileno (input), output);
 588 }