iconv/iconv_prog.c

   1 /* Convert text in given files from the specified from-set to the to-set.
   2    Copyright (C) 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
   3    This file is part of the GNU C Library.
   4    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998.
   5
   6    The GNU C Library is free software; you can redistribute it and/or
   7    modify it under the terms of the GNU Library General Public License as
   8    published by the Free Software Foundation; either version 2 of the
   9    License, or (at your option) any later version.
  10
  11    The GNU C Library is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14    Library General Public License for more details.
  15
  16    You should have received a copy of the GNU Library General Public
  17    License along with the GNU C Library; see the file COPYING.LIB.  If not,
  18    write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  19    Boston, MA 02111-1307, USA.  */
  20
  21 #include <argp.h>
  22 #include <assert.h>
  23 #include <ctype.h>
  24 #include <errno.h>
  25 #include <error.h>
  26 #include <fcntl.h>
  27 #include <iconv.h>
  28 #include <langinfo.h>
  29 #include <locale.h>
  30 #include <search.h>
  31 #include <stdio.h>
  32 #include <stdlib.h>
  33 #include <string.h>
  34 #include <unistd.h>
  35 #include <libintl.h>
  36 #ifdef _POSIX_MAPPED_FILES
  37 # include <sys/mman.h>
  38 #endif
  39 #include <charmap.h>
  40 #include <gconv_int.h>
  41 #include "iconv_prog.h"
  42
  43 /* Get libc version number.  */
  44 #include "../version.h"
  45
  46 #define PACKAGE _libc_intl_domainname
  47
  48
  49 /* Name and version of program.  */
  50 static void print_version (FILE *stream, struct argp_state *state);
  51 void (*argp_program_version_hook) (FILE *, struct argp_state *) = print_version;
  52
  53 #define OPT_VERBOSE     1000
  54 #define OPT_LIST        'l'
  55
  56 /* Definitions of arguments for argp functions.  */
  57 static const struct argp_option options[] =
  58 {
  59   { NULL, 0, NULL, 0, N_("Input/Output format specification:") },
  60   { "from-code", 'f', "NAME", 0, N_("encoding of original text") },
  61   { "to-code", 't', "NAME", 0, N_("encoding for output") },
  62   { NULL, 0, NULL, 0, N_("Information:") },
  63   { "list", 'l', NULL, 0, N_("list all known coded character sets") },
  64   { NULL, 0, NULL, 0, N_("Output control:") },
  65   { NULL, 'c', NULL, 0, N_("omit invalid characters from output") },
  66   { "output", 'o', "FILE", 0, N_("output file") },
  67   { "silent", 's', NULL, 0, N_("suppress warnings") },
  68   { "verbose", OPT_VERBOSE, NULL, 0, N_("print progress information") },
  69   { NULL, 0, NULL, 0, NULL }
  70 };
  71
  72 /* Short description of program.  */
  73 static const char doc[] = N_("\
  74 Convert encoding of given files from one encoding to another.");
  75
  76 /* Strings for arguments in help texts.  */
  77 static const char args_doc[] = N_("[FILE...]");
  78
  79 /* Prototype for option handler.  */
  80 static error_t parse_opt (int key, char *arg, struct argp_state *state);
  81
  82 /* Function to print some extra text in the help message.  */
  83 static char *more_help (int key, const char *text, void *input);
  84
  85 /* Data structure to communicate with argp functions.  */
  86 static struct argp argp =
  87 {
  88   options, parse_opt, args_doc, doc, NULL, more_help
  89 };
  90
  91 /* Code sets to convert from and to respectively.  */
  92 static const char *from_code;
  93 static const char *to_code;
  94
  95 /* File to write output to.  If NULL write to stdout.  */
  96 static const char *output_file;
  97
  98 /* Nonzero if verbose ouput is wanted.  */
  99 int verbose;
 100
 101 /* Nonzero if list of all coded character sets is wanted.  */
 102 static int list;
 103
 104 /* If nonzero omit invalid character from output.  */
 105 int omit_invalid;
 106
 107 /* Prototypes for the functions doing the actual work.  */
 108 static int process_block (iconv_t cd, char *addr, size_t len, FILE *output);
 109 static int process_fd (iconv_t cd, int fd, FILE *output);
 110 static int process_file (iconv_t cd, FILE *input, FILE *output);
 111 static void print_known_names (void) internal_function;
 112
 113
 114 int
 115 main (int argc, char *argv[])
 116 {
 117   int status = EXIT_SUCCESS;
 118   int remaining;
 119   FILE *output;
 120   iconv_t cd;
 121   const char *orig_to_code;
 122   struct charmap_t *from_charmap = NULL;
 123   struct charmap_t *to_charmap = NULL;
 124
 125   /* Set locale via LC_ALL.  */
 126   setlocale (LC_ALL, "");
 127
 128   /* Set the text message domain.  */
 129   textdomain (_libc_intl_domainname);
 130
 131   /* Parse and process arguments.  */
 132   argp_parse (&argp, argc, argv, 0, &remaining, NULL);
 133
 134   /* List all coded character sets if wanted.  */
 135   if (list)
 136     {
 137       print_known_names ();
 138       exit (EXIT_SUCCESS);
 139     }
 140   if (from_code == NULL)
 141     {
 142       /* The Unix standard says that in this case the charset of the current
 143          locale is used.  */
 144       from_code = nl_langinfo (CODESET);
 145       assert (from_code != NULL);
 146     }
 147   if (to_code == NULL)
 148     {
 149       /* The Unix standard says that in this case the charset of the current
 150          locale is used.  */
 151       to_code = nl_langinfo (CODESET);
 152       assert (to_code != NULL);
 153     }
 154
 155   /* If we have to ignore errors make sure we use the appropriate name for
 156      the to-character-set.  */
 157   orig_to_code = to_code;
 158   if (omit_invalid)
 159     {
 160       const char *errhand = strchrnul (to_code, '/');
 161       int nslash = 2;
 162       char *newp;
 163       char *cp;
 164
 165       if (*errhand == '/')
 166         {
 167           --nslash;
 168           errhand = strchrnul (errhand, '/');
 169
 170           if (*errhand == '/')
 171             {
 172               --nslash;
 173               ++errhand;
 174             }
 175         }
 176
 177       newp = (char *) alloca (errhand - to_code + nslash + 6 + 1);
 178       cp = mempcpy (newp, to_code, errhand - to_code);
 179       while (nslash-- > 0)
 180         *cp++ = '/';
 181       memcpy (cp, "IGNORE", sizeof ("IGNORE"));
 182
 183       to_code = newp;
 184     }
 185
 186   /* POSIX 1003.2b introduces a silly thing: the arguments to -t anf -f
 187      can be file names of charmaps.  In this case iconv will have to read
 188      those charmaps and use them to do the conversion.  But there are
 189      holes in the specification.  There is nothing said that if -f is a
 190      charmap filename that -t must be, too.  And vice versa.  There is
 191      also no word about the symbolic names used.  What if they don't
 192      match?  */
 193   if (strchr (from_code, '/') != NULL)
 194     /* The from-name might be a charmap file name.  Try reading the
 195        file.  */
 196     from_charmap = charmap_read (from_code, /*0, 1*/1, 0, 0);
 197
 198   if (strchr (orig_to_code, '/') != NULL)
 199     /* The to-name might be a charmap file name.  Try reading the
 200        file.  */
 201     to_charmap = charmap_read (orig_to_code, /*0, 1,*/1,0, 0);
 202
 203
 204   /* Determine output file.  */
 205   if (output_file != NULL && strcmp (output_file, "-") != 0)
 206     {
 207       output = fopen (output_file, "w");
 208       if (output == NULL)
 209         error (EXIT_FAILURE, errno, _("cannot open output file"));
 210     }
 211   else
 212     output = stdout;
 213
 214   /* At this point we have to handle two cases.  The first one is
 215      where a charmap is used for the from- or to-charset, or both.  We
 216      handle this special since it is very different from the sane way of
 217      doing things.  The other case allows converting using the iconv()
 218      function.  */
 219   if (from_charmap != NULL || to_charmap != NULL)
 220     /* Construct the conversion table and do the conversion.  */
 221     status = charmap_conversion (from_code, from_charmap, to_code, to_charmap,
 222                                  argc, remaining, argv, output);
 223   else
 224     {
 225       /* Let's see whether we have these coded character sets.  */
 226       cd = iconv_open (to_code, from_code);
 227       if (cd == (iconv_t) -1)
 228         {
 229           if (errno == EINVAL)
 230             error (EXIT_FAILURE, 0,
 231                    _("conversion from `%s' to `%s' not supported"),
 232                    from_code, orig_to_code);
 233           else
 234             error (EXIT_FAILURE, errno,
 235                    _("failed to start conversion processing"));
 236         }
 237
 238       /* Now process the remaining files.  Write them to stdout or the file
 239          specified with the `-o' parameter.  If we have no file given as
 240          the parameter process all from stdin.  */
 241       if (remaining == argc)
 242         {
 243           if (process_file (cd, stdin, output) != 0)
 244             status = EXIT_FAILURE;
 245         }
 246       else
 247         do
 248           {
 249             struct stat st;
 250             char *addr;
 251             int fd;
 252
 253             if (verbose)
 254               printf ("%s:\n", argv[remaining]);
 255             if (strcmp (argv[remaining], "-") == 0)
 256               fd = 0;
 257             else
 258               {
 259                 fd = open (argv[remaining], O_RDONLY);
 260
 261                 if (fd == -1)
 262                   {
 263                     error (0, errno, _("cannot open input file `%s'"),
 264                            argv[remaining]);
 265                     status = EXIT_FAILURE;
 266                     continue;
 267                   }
 268               }
 269
 270 #ifdef _POSIX_MAPPED_FILES
 271             /* We have possibilities for reading the input file.  First try
 272                to mmap() it since this will provide the fastest solution.  */
 273             if (fstat (fd, &st) == 0
 274                 && ((addr = mmap (NULL, st.st_size, PROT_READ, MAP_PRIVATE,
 275                                   fd, 0)) != MAP_FAILED))
 276               {
 277                 /* Yes, we can use mmap().  The descriptor is not needed
 278                    anymore.  */
 279                 if (close (fd) != 0)
 280                   error (EXIT_FAILURE, errno,
 281                          _("error while closing input `%s'"),
 282                          argv[remaining]);
 283
 284                 if (process_block (cd, addr, st.st_size, output) < 0)
 285                   {
 286                     /* Something went wrong.  */
 287                     status = EXIT_FAILURE;
 288
 289                     /* We don't need the input data anymore.  */
 290                     munmap ((void *) addr, st.st_size);
 291
 292                     /* We cannot go on with producing output since it might
 293                        lead to problem because the last output might leave
 294                        the output stream in an undefined state.  */
 295                     break;
 296                   }
 297
 298                 /* We don't need the input data anymore.  */
 299                 munmap ((void *) addr, st.st_size);
 300               }
 301             else
 302 #endif  /* _POSIX_MAPPED_FILES */
 303               {
 304                 /* Read the file in pieces.  */
 305                 if (process_fd (cd, fd, output) != 0)
 306                   {
 307                     /* Something went wrong.  */
 308                     status = EXIT_FAILURE;
 309
 310                     /* We don't need the input file anymore.  */
 311                     close (fd);
 312
 313                     /* We cannot go on with producing output since it might
 314                        lead to problem because the last output might leave
 315                        the output stream in an undefined state.  */
 316                     break;
 317                   }
 318
 319                 /* Now close the file.  */
 320                 close (fd);
 321               }
 322           }
 323         while (++remaining < argc);
 324     }
 325
 326   /* Close the output file now.  */
 327   if (fclose (output))
 328     error (EXIT_FAILURE, errno, _("error while closing output file"));
 329
 330   return status;
 331 }
 332
 333
 334 /* Handle program arguments.  */
 335 static error_t
 336 parse_opt (int key, char *arg, struct argp_state *state)
 337 {
 338   switch (key)
 339     {
 340     case 'f':
 341       from_code = arg;
 342       break;
 343     case 't':
 344       to_code = arg;
 345       break;
 346     case 'o':
 347       output_file = arg;
 348       break;
 349     case 's':
 350       /* Nothing, for now at least.  We are not giving out any information
 351          about missing character or so.  */
 352       break;
 353     case 'c':
 354       /* Omit invalid characters from output.  */
 355       omit_invalid = 1;
 356       break;
 357     case OPT_VERBOSE:
 358       verbose = 1;
 359       break;
 360     case OPT_LIST:
 361       list = 1;
 362       break;
 363     default:
 364       return ARGP_ERR_UNKNOWN;
 365     }
 366   return 0;
 367 }
 368
 369
 370 static char *
 371 more_help (int key, const char *text, void *input)
 372 {
 373   switch (key)
 374     {
 375     case ARGP_KEY_HELP_EXTRA:
 376       /* We print some extra information.  */
 377       return strdup (gettext ("\
 378 Report bugs using the `glibcbug' script to <bugs@gnu.org>.\n"));
 379     default:
 380       break;
 381     }
 382   return (char *) text;
 383 }
 384
 385
 386 /* Print the version information.  */
 387 static void
 388 print_version (FILE *stream, struct argp_state *state)
 389 {
 390   fprintf (stream, "iconv (GNU %s) %s\n", PACKAGE, VERSION);
 391   fprintf (stream, gettext ("\
 392 Copyright (C) %s Free Software Foundation, Inc.\n\
 393 This is free software; see the source for copying conditions.  There is NO\n\
 394 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n\
 395 "), "2001");
 396   fprintf (stream, gettext ("Written by %s.\n"), "Ulrich Drepper");
 397 }
 398
 399
 400 static int
 401 process_block (iconv_t cd, char *addr, size_t len, FILE *output)
 402 {
 403 #define OUTBUF_SIZE     32768
 404   const char *start = addr;
 405   char outbuf[OUTBUF_SIZE];
 406   char *outptr;
 407   size_t outlen;
 408   size_t n;
 409
 410   while (len > 0)
 411     {
 412       outptr = outbuf;
 413       outlen = OUTBUF_SIZE;
 414       n = iconv (cd, &addr, &len, &outptr, &outlen);
 415
 416       if (outptr != outbuf)
 417         {
 418           /* We have something to write out.  */
 419           int errno_save = errno;
 420
 421           if (fwrite (outbuf, 1, outptr - outbuf, output) < outptr - outbuf
 422               || ferror (output))
 423             {
 424               /* Error occurred while printing the result.  */
 425               error (0, 0, _("\
 426 conversion stopped due to problem in writing the output"));
 427               return -1;
 428             }
 429
 430           errno = errno_save;
 431         }
 432
 433       if (n != (size_t) -1)
 434         {
 435           /* All the input test is processed.  For state-dependent
 436              character sets we have to flush the state now.  */
 437           outptr = outbuf;
 438           outlen = OUTBUF_SIZE;
 439           (void) iconv (cd, NULL, NULL, &outptr, &outlen);
 440
 441           if (outptr != outbuf)
 442             {
 443               /* We have something to write out.  */
 444               int errno_save = errno;
 445
 446               if (fwrite (outbuf, 1, outptr - outbuf, output) < outptr - outbuf
 447                   || ferror (output))
 448                 {
 449                   /* Error occurred while printing the result.  */
 450                   error (0, 0, _("\
 451 conversion stopped due to problem in writing the output"));
 452                   return -1;
 453                 }
 454
 455               errno = errno_save;
 456             }
 457
 458           break;
 459         }
 460
 461       if (errno != E2BIG)
 462         {
 463           /* iconv() ran into a problem.  */
 464           switch (errno)
 465             {
 466             case EILSEQ:
 467               error (0, 0, _("illegal input sequence at position %ld"),
 468                      (long) (addr - start));
 469               break;
 470             case EINVAL:
 471               error (0, 0, _("\
 472 incomplete character or shift sequence at end of buffer"));
 473               break;
 474             case EBADF:
 475               error (0, 0, _("internal error (illegal descriptor)"));
 476               break;
 477             default:
 478               error (0, 0, _("unknown iconv() error %d"), errno);
 479               break;
 480             }
 481
 482           return -1;
 483         }
 484     }
 485
 486   return 0;
 487 }
 488
 489
 490 static int
 491 process_fd (iconv_t cd, int fd, FILE *output)
 492 {
 493   /* we have a problem with reading from a desriptor since we must not
 494      provide the iconv() function an incomplete character or shift
 495      sequence at the end of the buffer.  Since we have to deal with
 496      arbitrary encodings we must read the whole text in a buffer and
 497      process it in one step.  */
 498   static char *inbuf = NULL;
 499   static size_t maxlen = 0;
 500   char *inptr = NULL;
 501   size_t actlen = 0;
 502
 503   while (actlen < maxlen)
 504     {
 505       ssize_t n = read (fd, inptr, maxlen - actlen);
 506
 507       if (n == 0)
 508         /* No more text to read.  */
 509         break;
 510
 511       if (n == -1)
 512         {
 513           /* Error while reading.  */
 514           error (0, errno, _("error while reading the input"));
 515           return -1;
 516         }
 517
 518       inptr += n;
 519       actlen += n;
 520     }
 521
 522   if (actlen == maxlen)
 523     while (1)
 524       {
 525         ssize_t n;
 526
 527         /* Increase the buffer.  */
 528         maxlen += 32768;
 529         inbuf = realloc (inbuf, maxlen);
 530         if (inbuf == NULL)
 531           error (0, errno, _("unable to allocate buffer for input"));
 532         inptr = inbuf + actlen;
 533
 534         do
 535           {
 536             n = read (fd, inptr, maxlen - actlen);
 537
 538             if (n == 0)
 539               /* No more text to read.  */
 540               break;
 541
 542             if (n == -1)
 543               {
 544                 /* Error while reading.  */
 545                 error (0, errno, _("error while reading the input"));
 546                 return -1;
 547               }
 548
 549             inptr += n;
 550             actlen += n;
 551           }
 552         while (actlen < maxlen);
 553
 554         if (n == 0)
 555           /* Break again so we leave both loops.  */
 556           break;
 557       }
 558
 559   /* Now we have all the input in the buffer.  Process it in one run.  */
 560   return process_block (cd, inbuf, actlen, output);
 561 }
 562
 563
 564 static int
 565 process_file (iconv_t cd, FILE *input, FILE *output)
 566 {
 567   /* This should be safe since we use this function only for `stdin' and
 568      we haven't read anything so far.  */
 569   return process_fd (cd, fileno (input), output);
 570 }
 571
 572
 573 /* Print all known character sets/encodings.  */
 574 static void *printlist;
 575 static size_t column;
 576 static int not_first;
 577
 578 static void
 579 insert_print_list (const void *nodep, VISIT value, int level)
 580 {
 581   if (value == leaf || value == postorder)
 582     {
 583       const struct gconv_alias *s = *(const struct gconv_alias **) nodep;
 584       tsearch (s->fromname, &printlist, (__compar_fn_t) strverscmp);
 585     }
 586 }
 587
 588 static void
 589 do_print  (const void *nodep, VISIT value, int level)
 590 {
 591   if (value == leaf || value == postorder)
 592     {
 593       const char *s = *(const char **) nodep;
 594       size_t len = strlen (s);
 595       size_t cnt;
 596
 597       while (len > 0 && s[len - 1] == '/')
 598         --len;
 599
 600       for (cnt = 0; cnt < len; ++cnt)
 601         if (isalnum (s[cnt]))
 602           break;
 603       if (cnt == len)
 604         return;
 605
 606       if (not_first)
 607         {
 608           putchar (',');
 609           ++column;
 610
 611           if (column > 2 && column + len > 77)
 612             {
 613               fputs ("\n  ", stdout);
 614               column = 2;
 615             }
 616           else
 617             {
 618               putchar (' ');
 619               ++column;
 620             }
 621         }
 622       else
 623         not_first = 1;
 624
 625       fwrite (s, len, 1, stdout);
 626       column += len;
 627     }
 628 }
 629
 630 static void
 631 internal_function
 632 add_known_names (struct gconv_module *node)
 633 {
 634   if (node->left != NULL)
 635     add_known_names (node->left);
 636   if (node->right != NULL)
 637     add_known_names (node->right);
 638   do
 639     {
 640       if (strcmp (node->from_string, "INTERNAL"))
 641         tsearch (node->from_string, &printlist,
 642                  (__compar_fn_t) strverscmp);
 643       if (strcmp (node->to_string, "INTERNAL"))
 644         tsearch (node->to_string, &printlist, (__compar_fn_t) strverscmp);
 645
 646       node = node->same;
 647     }
 648   while (node != NULL);
 649 }
 650
 651 static void
 652 internal_function
 653 print_known_names (void)
 654 {
 655   iconv_t h;
 656
 657   /* We must initialize the internal databases first.  */
 658   h = iconv_open ("L1", "L1");
 659   iconv_close (h);
 660
 661   /* First add the aliases.  */
 662   twalk (__gconv_alias_db, insert_print_list);
 663
 664   /* Add the from- and to-names from the known modules.  */
 665   add_known_names (__gconv_modules_db);
 666
 667   fputs (_("\
 668 The following list contain all the coded character sets known.  This does\n\
 669 not necessarily mean that all combinations of these names can be used for\n\
 670 the FROM and TO command line parameters.  One coded character set can be\n\
 671 listed with several different names (aliases).\n\n  "), stdout);
 672
 673   /* Now print the collected names.  */
 674   column = 2;
 675   twalk (printlist, do_print);
 676
 677   if (column != 0)
 678     puts ("");
 679 }