iconv/iconv_prog.c

   1 /* Convert text in given files from the specified from-set to the to-set.
   2    Copyright (C) 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
   3    This file is part of the GNU C Library.
   4    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998.
   5
   6    The GNU C Library is free software; you can redistribute it and/or
   7    modify it under the terms of the GNU Lesser General Public
   8    License as published by the Free Software Foundation; either
   9    version 2.1 of the License, or (at your option) any later version.
  10
  11    The GNU C Library is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14    Lesser General Public License for more details.
  15
  16    You should have received a copy of the GNU Lesser General Public
  17    License along with the GNU C Library; if not, write to the Free
  18    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
  19    02111-1307 USA.  */
  20
  21 #include <argp.h>
  22 #include <assert.h>
  23 #include <ctype.h>
  24 #include <errno.h>
  25 #include <error.h>
  26 #include <fcntl.h>
  27 #include <iconv.h>
  28 #include <langinfo.h>
  29 #include <locale.h>
  30 #include <search.h>
  31 #include <stdio.h>
  32 #include <stdlib.h>
  33 #include <string.h>
  34 #include <unistd.h>
  35 #include <libintl.h>
  36 #ifdef _POSIX_MAPPED_FILES
  37 # include <sys/mman.h>
  38 #endif
  39 #include <charmap.h>
  40 #include <gconv_int.h>
  41 #include "iconv_prog.h"
  42 #include "iconvconfig.h"
  43
  44 /* Get libc version number.  */
  45 #include "../version.h"
  46
  47 #define PACKAGE _libc_intl_domainname
  48
  49
  50 /* Defined in gconv_cache.c.  */
  51 extern void *__gconv_cache;
  52
  53 /* Name and version of program.  */
  54 static void print_version (FILE *stream, struct argp_state *state);
  55 void (*argp_program_version_hook) (FILE *, struct argp_state *) = print_version;
  56
  57 #define OPT_VERBOSE     1000
  58 #define OPT_LIST        'l'
  59
  60 /* Definitions of arguments for argp functions.  */
  61 static const struct argp_option options[] =
  62 {
  63   { NULL, 0, NULL, 0, N_("Input/Output format specification:") },
  64   { "from-code", 'f', "NAME", 0, N_("encoding of original text") },
  65   { "to-code", 't', "NAME", 0, N_("encoding for output") },
  66   { NULL, 0, NULL, 0, N_("Information:") },
  67   { "list", 'l', NULL, 0, N_("list all known coded character sets") },
  68   { NULL, 0, NULL, 0, N_("Output control:") },
  69   { NULL, 'c', NULL, 0, N_("omit invalid characters from output") },
  70   { "output", 'o', "FILE", 0, N_("output file") },
  71   { "silent", 's', NULL, 0, N_("suppress warnings") },
  72   { "verbose", OPT_VERBOSE, NULL, 0, N_("print progress information") },
  73   { NULL, 0, NULL, 0, NULL }
  74 };
  75
  76 /* Short description of program.  */
  77 static const char doc[] = N_("\
  78 Convert encoding of given files from one encoding to another.");
  79
  80 /* Strings for arguments in help texts.  */
  81 static const char args_doc[] = N_("[FILE...]");
  82
  83 /* Prototype for option handler.  */
  84 static error_t parse_opt (int key, char *arg, struct argp_state *state);
  85
  86 /* Function to print some extra text in the help message.  */
  87 static char *more_help (int key, const char *text, void *input);
  88
  89 /* Data structure to communicate with argp functions.  */
  90 static struct argp argp =
  91 {
  92   options, parse_opt, args_doc, doc, NULL, more_help
  93 };
  94
  95 /* Code sets to convert from and to respectively.  */
  96 static const char *from_code;
  97 static const char *to_code;
  98
  99 /* File to write output to.  If NULL write to stdout.  */
 100 static const char *output_file;
 101
 102 /* Nonzero if verbose ouput is wanted.  */
 103 int verbose;
 104
 105 /* Nonzero if list of all coded character sets is wanted.  */
 106 static int list;
 107
 108 /* If nonzero omit invalid character from output.  */
 109 int omit_invalid;
 110
 111 /* Prototypes for the functions doing the actual work.  */
 112 static int process_block (iconv_t cd, char *addr, size_t len, FILE *output);
 113 static int process_fd (iconv_t cd, int fd, FILE *output);
 114 static int process_file (iconv_t cd, FILE *input, FILE *output);
 115 static void print_known_names (void) internal_function;
 116
 117
 118 int
 119 main (int argc, char *argv[])
 120 {
 121   int status = EXIT_SUCCESS;
 122   int remaining;
 123   FILE *output;
 124   iconv_t cd;
 125   const char *orig_to_code;
 126   struct charmap_t *from_charmap = NULL;
 127   struct charmap_t *to_charmap = NULL;
 128
 129   /* Set locale via LC_ALL.  */
 130   setlocale (LC_ALL, "");
 131
 132   /* Set the text message domain.  */
 133   textdomain (_libc_intl_domainname);
 134
 135   /* Parse and process arguments.  */
 136   argp_parse (&argp, argc, argv, 0, &remaining, NULL);
 137
 138   /* List all coded character sets if wanted.  */
 139   if (list)
 140     {
 141       print_known_names ();
 142       exit (EXIT_SUCCESS);
 143     }
 144   if (from_code == NULL)
 145     {
 146       /* The Unix standard says that in this case the charset of the current
 147          locale is used.  */
 148       from_code = nl_langinfo (CODESET);
 149       assert (from_code != NULL);
 150     }
 151   if (to_code == NULL)
 152     {
 153       /* The Unix standard says that in this case the charset of the current
 154          locale is used.  */
 155       to_code = nl_langinfo (CODESET);
 156       assert (to_code != NULL);
 157     }
 158
 159   /* If we have to ignore errors make sure we use the appropriate name for
 160      the to-character-set.  */
 161   orig_to_code = to_code;
 162   if (omit_invalid)
 163     {
 164       const char *errhand = strchrnul (to_code, '/');
 165       int nslash = 2;
 166       char *newp;
 167       char *cp;
 168
 169       if (*errhand == '/')
 170         {
 171           --nslash;
 172           errhand = strchrnul (errhand, '/');
 173
 174           if (*errhand == '/')
 175             {
 176               --nslash;
 177               ++errhand;
 178             }
 179         }
 180
 181       newp = (char *) alloca (errhand - to_code + nslash + 6 + 1);
 182       cp = mempcpy (newp, to_code, errhand - to_code);
 183       while (nslash-- > 0)
 184         *cp++ = '/';
 185       memcpy (cp, "IGNORE", sizeof ("IGNORE"));
 186
 187       to_code = newp;
 188     }
 189
 190   /* POSIX 1003.2b introduces a silly thing: the arguments to -t anf -f
 191      can be file names of charmaps.  In this case iconv will have to read
 192      those charmaps and use them to do the conversion.  But there are
 193      holes in the specification.  There is nothing said that if -f is a
 194      charmap filename that -t must be, too.  And vice versa.  There is
 195      also no word about the symbolic names used.  What if they don't
 196      match?  */
 197   if (strchr (from_code, '/') != NULL)
 198     /* The from-name might be a charmap file name.  Try reading the
 199        file.  */
 200     from_charmap = charmap_read (from_code, /*0, 1*/1, 0, 0);
 201
 202   if (strchr (orig_to_code, '/') != NULL)
 203     /* The to-name might be a charmap file name.  Try reading the
 204        file.  */
 205     to_charmap = charmap_read (orig_to_code, /*0, 1,*/1,0, 0);
 206
 207
 208   /* Determine output file.  */
 209   if (output_file != NULL && strcmp (output_file, "-") != 0)
 210     {
 211       output = fopen (output_file, "w");
 212       if (output == NULL)
 213         error (EXIT_FAILURE, errno, _("cannot open output file"));
 214     }
 215   else
 216     output = stdout;
 217
 218   /* At this point we have to handle two cases.  The first one is
 219      where a charmap is used for the from- or to-charset, or both.  We
 220      handle this special since it is very different from the sane way of
 221      doing things.  The other case allows converting using the iconv()
 222      function.  */
 223   if (from_charmap != NULL || to_charmap != NULL)
 224     /* Construct the conversion table and do the conversion.  */
 225     status = charmap_conversion (from_code, from_charmap, to_code, to_charmap,
 226                                  argc, remaining, argv, output);
 227   else
 228     {
 229       /* Let's see whether we have these coded character sets.  */
 230       cd = iconv_open (to_code, from_code);
 231       if (cd == (iconv_t) -1)
 232         {
 233           if (errno == EINVAL)
 234             error (EXIT_FAILURE, 0,
 235                    _("conversion from `%s' to `%s' not supported"),
 236                    from_code, orig_to_code);
 237           else
 238             error (EXIT_FAILURE, errno,
 239                    _("failed to start conversion processing"));
 240         }
 241
 242       /* Now process the remaining files.  Write them to stdout or the file
 243          specified with the `-o' parameter.  If we have no file given as
 244          the parameter process all from stdin.  */
 245       if (remaining == argc)
 246         {
 247           if (process_file (cd, stdin, output) != 0)
 248             status = EXIT_FAILURE;
 249         }
 250       else
 251         do
 252           {
 253 #ifdef _POSIX_MAPPED_FILES
 254             struct stat st;
 255             char *addr;
 256 #endif
 257             int fd;
 258
 259             if (verbose)
 260               printf ("%s:\n", argv[remaining]);
 261             if (strcmp (argv[remaining], "-") == 0)
 262               fd = 0;
 263             else
 264               {
 265                 fd = open (argv[remaining], O_RDONLY);
 266
 267                 if (fd == -1)
 268                   {
 269                     error (0, errno, _("cannot open input file `%s'"),
 270                            argv[remaining]);
 271                     status = EXIT_FAILURE;
 272                     continue;
 273                   }
 274               }
 275
 276 #ifdef _POSIX_MAPPED_FILES
 277             /* We have possibilities for reading the input file.  First try
 278                to mmap() it since this will provide the fastest solution.  */
 279             if (fstat (fd, &st) == 0
 280                 && ((addr = mmap (NULL, st.st_size, PROT_READ, MAP_PRIVATE,
 281                                   fd, 0)) != MAP_FAILED))
 282               {
 283                 /* Yes, we can use mmap().  The descriptor is not needed
 284                    anymore.  */
 285                 if (close (fd) != 0)
 286                   error (EXIT_FAILURE, errno,
 287                          _("error while closing input `%s'"),
 288                          argv[remaining]);
 289
 290                 if (process_block (cd, addr, st.st_size, output) < 0)
 291                   {
 292                     /* Something went wrong.  */
 293                     status = EXIT_FAILURE;
 294
 295                     /* We don't need the input data anymore.  */
 296                     munmap ((void *) addr, st.st_size);
 297
 298                     /* We cannot go on with producing output since it might
 299                        lead to problem because the last output might leave
 300                        the output stream in an undefined state.  */
 301                     break;
 302                   }
 303
 304                 /* We don't need the input data anymore.  */
 305                 munmap ((void *) addr, st.st_size);
 306               }
 307             else
 308 #endif  /* _POSIX_MAPPED_FILES */
 309               {
 310                 /* Read the file in pieces.  */
 311                 if (process_fd (cd, fd, output) != 0)
 312                   {
 313                     /* Something went wrong.  */
 314                     status = EXIT_FAILURE;
 315
 316                     /* We don't need the input file anymore.  */
 317                     close (fd);
 318
 319                     /* We cannot go on with producing output since it might
 320                        lead to problem because the last output might leave
 321                        the output stream in an undefined state.  */
 322                     break;
 323                   }
 324
 325                 /* Now close the file.  */
 326                 close (fd);
 327               }
 328           }
 329         while (++remaining < argc);
 330     }
 331
 332   /* Close the output file now.  */
 333   if (fclose (output))
 334     error (EXIT_FAILURE, errno, _("error while closing output file"));
 335
 336   return status;
 337 }
 338
 339
 340 /* Handle program arguments.  */
 341 static error_t
 342 parse_opt (int key, char *arg, struct argp_state *state)
 343 {
 344   switch (key)
 345     {
 346     case 'f':
 347       from_code = arg;
 348       break;
 349     case 't':
 350       to_code = arg;
 351       break;
 352     case 'o':
 353       output_file = arg;
 354       break;
 355     case 's':
 356       /* Nothing, for now at least.  We are not giving out any information
 357          about missing character or so.  */
 358       break;
 359     case 'c':
 360       /* Omit invalid characters from output.  */
 361       omit_invalid = 1;
 362       break;
 363     case OPT_VERBOSE:
 364       verbose = 1;
 365       break;
 366     case OPT_LIST:
 367       list = 1;
 368       break;
 369     default:
 370       return ARGP_ERR_UNKNOWN;
 371     }
 372   return 0;
 373 }
 374
 375
 376 static char *
 377 more_help (int key, const char *text, void *input)
 378 {
 379   switch (key)
 380     {
 381     case ARGP_KEY_HELP_EXTRA:
 382       /* We print some extra information.  */
 383       return strdup (gettext ("\
 384 Report bugs using the `glibcbug' script to <bugs@gnu.org>.\n"));
 385     default:
 386       break;
 387     }
 388   return (char *) text;
 389 }
 390
 391
 392 /* Print the version information.  */
 393 static void
 394 print_version (FILE *stream, struct argp_state *state)
 395 {
 396   fprintf (stream, "iconv (GNU %s) %s\n", PACKAGE, VERSION);
 397   fprintf (stream, gettext ("\
 398 Copyright (C) %s Free Software Foundation, Inc.\n\
 399 This is free software; see the source for copying conditions.  There is NO\n\
 400 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n\
 401 "), "2001");
 402   fprintf (stream, gettext ("Written by %s.\n"), "Ulrich Drepper");
 403 }
 404
 405
 406 static int
 407 process_block (iconv_t cd, char *addr, size_t len, FILE *output)
 408 {
 409 #define OUTBUF_SIZE     32768
 410   const char *start = addr;
 411   char outbuf[OUTBUF_SIZE];
 412   char *outptr;
 413   size_t outlen;
 414   size_t n;
 415
 416   while (len > 0)
 417     {
 418       outptr = outbuf;
 419       outlen = OUTBUF_SIZE;
 420       n = iconv (cd, &addr, &len, &outptr, &outlen);
 421
 422       if (outptr != outbuf)
 423         {
 424           /* We have something to write out.  */
 425           int errno_save = errno;
 426
 427           if (fwrite (outbuf, 1, outptr - outbuf, output) < outptr - outbuf
 428               || ferror (output))
 429             {
 430               /* Error occurred while printing the result.  */
 431               error (0, 0, _("\
 432 conversion stopped due to problem in writing the output"));
 433               return -1;
 434             }
 435
 436           errno = errno_save;
 437         }
 438
 439       if (n != (size_t) -1)
 440         {
 441           /* All the input test is processed.  For state-dependent
 442              character sets we have to flush the state now.  */
 443           outptr = outbuf;
 444           outlen = OUTBUF_SIZE;
 445           (void) iconv (cd, NULL, NULL, &outptr, &outlen);
 446
 447           if (outptr != outbuf)
 448             {
 449               /* We have something to write out.  */
 450               int errno_save = errno;
 451
 452               if (fwrite (outbuf, 1, outptr - outbuf, output) < outptr - outbuf
 453                   || ferror (output))
 454                 {
 455                   /* Error occurred while printing the result.  */
 456                   error (0, 0, _("\
 457 conversion stopped due to problem in writing the output"));
 458                   return -1;
 459                 }
 460
 461               errno = errno_save;
 462             }
 463
 464           break;
 465         }
 466
 467       if (errno != E2BIG)
 468         {
 469           /* iconv() ran into a problem.  */
 470           switch (errno)
 471             {
 472             case EILSEQ:
 473               error (0, 0, _("illegal input sequence at position %ld"),
 474                      (long) (addr - start));
 475               break;
 476             case EINVAL:
 477               error (0, 0, _("\
 478 incomplete character or shift sequence at end of buffer"));
 479               break;
 480             case EBADF:
 481               error (0, 0, _("internal error (illegal descriptor)"));
 482               break;
 483             default:
 484               error (0, 0, _("unknown iconv() error %d"), errno);
 485               break;
 486             }
 487
 488           return -1;
 489         }
 490     }
 491
 492   return 0;
 493 }
 494
 495
 496 static int
 497 process_fd (iconv_t cd, int fd, FILE *output)
 498 {
 499   /* we have a problem with reading from a desriptor since we must not
 500      provide the iconv() function an incomplete character or shift
 501      sequence at the end of the buffer.  Since we have to deal with
 502      arbitrary encodings we must read the whole text in a buffer and
 503      process it in one step.  */
 504   static char *inbuf = NULL;
 505   static size_t maxlen = 0;
 506   char *inptr = NULL;
 507   size_t actlen = 0;
 508
 509   while (actlen < maxlen)
 510     {
 511       ssize_t n = read (fd, inptr, maxlen - actlen);
 512
 513       if (n == 0)
 514         /* No more text to read.  */
 515         break;
 516
 517       if (n == -1)
 518         {
 519           /* Error while reading.  */
 520           error (0, errno, _("error while reading the input"));
 521           return -1;
 522         }
 523
 524       inptr += n;
 525       actlen += n;
 526     }
 527
 528   if (actlen == maxlen)
 529     while (1)
 530       {
 531         ssize_t n;
 532
 533         /* Increase the buffer.  */
 534         maxlen += 32768;
 535         inbuf = realloc (inbuf, maxlen);
 536         if (inbuf == NULL)
 537           error (0, errno, _("unable to allocate buffer for input"));
 538         inptr = inbuf + actlen;
 539
 540         do
 541           {
 542             n = read (fd, inptr, maxlen - actlen);
 543
 544             if (n == 0)
 545               /* No more text to read.  */
 546               break;
 547
 548             if (n == -1)
 549               {
 550                 /* Error while reading.  */
 551                 error (0, errno, _("error while reading the input"));
 552                 return -1;
 553               }
 554
 555             inptr += n;
 556             actlen += n;
 557           }
 558         while (actlen < maxlen);
 559
 560         if (n == 0)
 561           /* Break again so we leave both loops.  */
 562           break;
 563       }
 564
 565   /* Now we have all the input in the buffer.  Process it in one run.  */
 566   return process_block (cd, inbuf, actlen, output);
 567 }
 568
 569
 570 static int
 571 process_file (iconv_t cd, FILE *input, FILE *output)
 572 {
 573   /* This should be safe since we use this function only for `stdin' and
 574      we haven't read anything so far.  */
 575   return process_fd (cd, fileno (input), output);
 576 }
 577
 578
 579 /* Print all known character sets/encodings.  */
 580 static void *printlist;
 581 static size_t column;
 582 static int not_first;
 583
 584 static void
 585 insert_print_list (const void *nodep, VISIT value, int level)
 586 {
 587   if (value == leaf || value == postorder)
 588     {
 589       const struct gconv_alias *s = *(const struct gconv_alias **) nodep;
 590       tsearch (s->fromname, &printlist, (__compar_fn_t) strverscmp);
 591     }
 592 }
 593
 594 static void
 595 do_print_human  (const void *nodep, VISIT value, int level)
 596 {
 597   if (value == leaf || value == postorder)
 598     {
 599       const char *s = *(const char **) nodep;
 600       size_t len = strlen (s);
 601       size_t cnt;
 602
 603       while (len > 0 && s[len - 1] == '/')
 604         --len;
 605
 606       for (cnt = 0; cnt < len; ++cnt)
 607         if (isalnum (s[cnt]))
 608           break;
 609       if (cnt == len)
 610         return;
 611
 612       if (not_first)
 613         {
 614           putchar (',');
 615           ++column;
 616
 617           if (column > 2 && column + len > 77)
 618             {
 619               fputs ("\n  ", stdout);
 620               column = 2;
 621             }
 622           else
 623             {
 624               putchar (' ');
 625               ++column;
 626             }
 627         }
 628       else
 629         not_first = 1;
 630
 631       fwrite (s, len, 1, stdout);
 632       column += len;
 633     }
 634 }
 635
 636 static void
 637 do_print  (const void *nodep, VISIT value, int level)
 638 {
 639   if (value == leaf || value == postorder)
 640     {
 641       const char *s = *(const char **) nodep;
 642
 643       puts (s);
 644     }
 645 }
 646
 647 static void
 648 internal_function
 649 add_known_names (struct gconv_module *node)
 650 {
 651   if (node->left != NULL)
 652     add_known_names (node->left);
 653   if (node->right != NULL)
 654     add_known_names (node->right);
 655   do
 656     {
 657       if (strcmp (node->from_string, "INTERNAL"))
 658         tsearch (node->from_string, &printlist,
 659                  (__compar_fn_t) strverscmp);
 660       if (strcmp (node->to_string, "INTERNAL") != 0)
 661         tsearch (node->to_string, &printlist, (__compar_fn_t) strverscmp);
 662
 663       node = node->same;
 664     }
 665   while (node != NULL);
 666 }
 667
 668
 669 static void
 670 insert_cache (void)
 671 {
 672   const struct gconvcache_header *header;
 673   const char *strtab;
 674   const struct hash_entry *hashtab;
 675   size_t cnt;
 676
 677   header = (const struct gconvcache_header *) __gconv_cache;
 678   strtab = (char *) __gconv_cache + header->string_offset;
 679   hashtab = (struct hash_entry *) ((char *) __gconv_cache
 680                                    + header->hash_offset);
 681
 682   for (cnt = 0; cnt < header->hash_size; ++cnt)
 683     if (hashtab[cnt].string_offset != 0)
 684       {
 685         const char *str = strtab + hashtab[cnt].string_offset;
 686
 687         if (strcmp (str, "INTERNAL") != 0)
 688           tsearch (str, &printlist, (__compar_fn_t) strverscmp);
 689       }
 690 }
 691
 692
 693 static void
 694 internal_function
 695 print_known_names (void)
 696 {
 697   iconv_t h;
 698
 699   /* We must initialize the internal databases first.  */
 700   h = iconv_open ("L1", "L1");
 701   iconv_close (h);
 702
 703   /* See whether we have a cache.  */
 704   if (__gconv_cache != NULL)
 705     /* Yep, use only this information.  */
 706     insert_cache ();
 707   else
 708     {
 709       /* No, then use the information read from the gconv-modules file.
 710          First add the aliases.  */
 711       twalk (__gconv_alias_db, insert_print_list);
 712
 713       /* Add the from- and to-names from the known modules.  */
 714       if (__gconv_modules_db != NULL)
 715         add_known_names (__gconv_modules_db);
 716     }
 717
 718   fputs (_("\
 719 The following list contain all the coded character sets known.  This does\n\
 720 not necessarily mean that all combinations of these names can be used for\n\
 721 the FROM and TO command line parameters.  One coded character set can be\n\
 722 listed with several different names (aliases).\n\n  "), stdout);
 723
 724   /* Now print the collected names.  */
 725   column = 2;
 726   if (isatty (fileno (stdout)))
 727     {
 728       twalk (printlist, do_print_human);
 729
 730       if (column != 0)
 731         puts ("");
 732     }
 733   else
 734     twalk (printlist, do_print);
 735 }