iconv/iconv_prog.c

   1 /* Convert text in given files from the specified from-set to the to-set.
   2    Copyright (C) 1998, 1999 Free Software Foundation, Inc.
   3    This file is part of the GNU C Library.
   4    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998.
   5
   6    The GNU C Library is free software; you can redistribute it and/or
   7    modify it under the terms of the GNU Library General Public License as
   8    published by the Free Software Foundation; either version 2 of the
   9    License, or (at your option) any later version.
  10
  11    The GNU C Library is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14    Library General Public License for more details.
  15
  16    You should have received a copy of the GNU Library General Public
  17    License along with the GNU C Library; see the file COPYING.LIB.  If not,
  18    write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  19    Boston, MA 02111-1307, USA.  */
  20
  21 #include <argp.h>
  22 #include <ctype.h>
  23 #include <errno.h>
  24 #include <error.h>
  25 #include <fcntl.h>
  26 #include <iconv.h>
  27 #include <locale.h>
  28 #include <search.h>
  29 #include <stdio.h>
  30 #include <stdlib.h>
  31 #include <string.h>
  32 #include <unistd.h>
  33 #include <libintl.h>
  34 #ifdef _POSIX_MAPPED_FILES
  35 # include <sys/mman.h>
  36 #endif
  37 #include <gconv_int.h>
  38
  39 /* Get libc version number.  */
  40 #include "../version.h"
  41
  42 #define PACKAGE _libc_intl_domainname
  43
  44
  45 /* Name and version of program.  */
  46 static void print_version (FILE *stream, struct argp_state *state);
  47 void (*argp_program_version_hook) (FILE *, struct argp_state *) = print_version;
  48
  49 #define OPT_VERBOSE     1000
  50 #define OPT_LIST        1001
  51
  52 /* Definitions of arguments for argp functions.  */
  53 static const struct argp_option options[] =
  54 {
  55   { NULL, 0, NULL, 0, N_("Input/Output format specification:") },
  56   { "from-code", 'f', "NAME", 0, N_("encoding of original text") },
  57   { "to-code", 't', "NAME", 0, N_("encoding for output") },
  58   { NULL, 0, NULL, 0, N_("Information:") },
  59   { "list", OPT_LIST, NULL, 0, N_("list all known coded character sets") },
  60   { NULL, 0, NULL, 0, N_("Output control:") },
  61   { "output", 'o', "FILE", 0, N_("output file") },
  62   { "verbose", OPT_VERBOSE, NULL, 0, N_("print progress information") },
  63   { NULL, 0, NULL, 0, NULL }
  64 };
  65
  66 /* Short description of program.  */
  67 static const char doc[] = N_("\
  68 Convert encoding of given files from one encoding to another.");
  69
  70 /* Strings for arguments in help texts.  */
  71 static const char args_doc[] = N_("[FILE...]");
  72
  73 /* Prototype for option handler.  */
  74 static error_t parse_opt __P ((int key, char *arg, struct argp_state *state));
  75
  76 /* Function to print some extra text in the help message.  */
  77 static char *more_help __P ((int key, const char *text, void *input));
  78
  79 /* Data structure to communicate with argp functions.  */
  80 static struct argp argp =
  81 {
  82   options, parse_opt, args_doc, doc, NULL, more_help
  83 };
  84
  85 /* Code sets to convert from and to respectively.  */
  86 static const char *from_code;
  87 static const char *to_code;
  88
  89 /* File to write output to.  If NULL write to stdout.  */
  90 static const char *output_file;
  91
  92 /* Nonzero if verbose ouput is wanted.  */
  93 static int verbose;
  94
  95 /* Nonzero if list of all coded character sets is wanted.  */
  96 static int list;
  97
  98 /* Prototypes for the functions doing the actual work.  */
  99 static int process_block (iconv_t cd, const char *addr, size_t len,
 100                           FILE *output);
 101 static int process_fd (iconv_t cd, int fd, FILE *output);
 102 static int process_file (iconv_t cd, FILE *input, FILE *output);
 103 static void print_known_names (void) internal_function;
 104
 105
 106 int
 107 main (int argc, char *argv[])
 108 {
 109   int status = EXIT_SUCCESS;
 110   int remaining;
 111   FILE *output;
 112   iconv_t cd;
 113
 114   /* Set locale via LC_ALL.  */
 115   setlocale (LC_ALL, "");
 116
 117   /* Set the text message domain.  */
 118   textdomain (_libc_intl_domainname);
 119
 120   /* Parse and process arguments.  */
 121   argp_parse (&argp, argc, argv, 0, &remaining, NULL);
 122
 123   /* List all coded character sets if wanted.  */
 124   if (list)
 125     {
 126       print_known_names ();
 127       exit (EXIT_SUCCESS);
 128     }
 129
 130   /* If either the from- or to-code is not specified this is an error
 131      since we do not know what to do.  */
 132   if (from_code == NULL && to_code == NULL)
 133     error (EXIT_FAILURE, 0,
 134            _("neither original nor target encoding specified"));
 135   if (from_code == NULL)
 136     error (EXIT_FAILURE, 0, _("original encoding not specified using `-f'"));
 137   if (to_code == NULL)
 138     error (EXIT_FAILURE, 0, _("target encoding not specified using `-t'"));
 139
 140   /* Let's see whether we have these coded character sets.  */
 141   cd = iconv_open (to_code, from_code);
 142   if (cd == (iconv_t) -1)
 143     {
 144       if (errno == EINVAL)
 145         error (EXIT_FAILURE, 0, _("conversion from `%s' to `%s' not supported"),
 146                from_code, to_code);
 147       else
 148         error (EXIT_FAILURE, errno, _("failed to start conversion processing"));
 149     }
 150
 151   /* Determine output file.  */
 152   if (output_file != NULL)
 153     {
 154       output = fopen (output_file, "w");
 155       if (output == NULL)
 156         error (EXIT_FAILURE, errno, _("cannot open output file"));
 157     }
 158   else
 159     output = stdout;
 160
 161   /* Now process the remaining files.  Write them to stdout or the file
 162      specified with the `-o' parameter.  If we have no file given as
 163      the parameter process all from stdin.  */
 164   if (remaining == argc)
 165     {
 166       if (process_file (cd, stdin, output) != 0)
 167         status = EXIT_FAILURE;
 168     }
 169   else
 170     do
 171       {
 172         struct stat st;
 173         const char *addr;
 174         int fd = open (argv[remaining], O_RDONLY);
 175
 176         if (verbose)
 177           printf ("%s:\n", argv[remaining]);
 178
 179         if (fd == -1)
 180           {
 181             error (0, errno, _("cannot open input file `%s'"),
 182                    argv[remaining]);
 183             status = EXIT_FAILURE;
 184             continue;
 185           }
 186
 187 #ifdef _POSIX_MAPPED_FILES
 188         /* We have possibilities for reading the input file.  First try
 189            to mmap() it since this will provide the fastest solution.  */
 190         if (fstat (fd, &st) == 0
 191             && ((addr = mmap (NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0))
 192                 != MAP_FAILED))
 193           {
 194             /* Yes, we can use mmap().  The descriptor is not needed
 195                anymore.  */
 196             if (close (fd) != 0)
 197               error (EXIT_FAILURE, errno, _("error while closing input `%s'"),
 198                      argv[remaining]);
 199
 200             if (process_block (cd, addr, st.st_size, output) < 0)
 201               {
 202                 /* Something went wrong.  */
 203                 status = EXIT_FAILURE;
 204
 205                 /* We don't need the input data anymore.  */
 206                 munmap ((void *) addr, st.st_size);
 207
 208                 /* We cannot go on with producing output since it might
 209                    lead to problem because the last output might leave
 210                    the output stream in an undefined state.  */
 211                 break;
 212               }
 213
 214             /* We don't need the input data anymore.  */
 215             munmap ((void *) addr, st.st_size);
 216           }
 217         else
 218 #endif  /* _POSIX_MAPPED_FILES */
 219           {
 220             /* Read the file in pieces.  */
 221             if (process_fd (cd, fd, output) != 0)
 222               {
 223                 /* Something went wrong.  */
 224                 status = EXIT_FAILURE;
 225
 226                 /* We don't need the input file anymore.  */
 227                 close (fd);
 228
 229                 /* We cannot go on with producing output since it might
 230                    lead to problem because the last output might leave
 231                    the output stream in an undefined state.  */
 232                 break;
 233               }
 234
 235             /* Now close the file.  */
 236             close (fd);
 237           }
 238       }
 239     while (++remaining < argc);
 240
 241   /* Close the output file now.  */
 242   if (fclose (output))
 243     error (EXIT_FAILURE, errno, _("error while closing output file"));
 244
 245   return status;
 246 }
 247
 248
 249 /* Handle program arguments.  */
 250 static error_t
 251 parse_opt (int key, char *arg, struct argp_state *state)
 252 {
 253   switch (key)
 254     {
 255     case 'f':
 256       from_code = arg;
 257       break;
 258     case 't':
 259       to_code = arg;
 260       break;
 261     case 'o':
 262       output_file = arg;
 263       break;
 264     case OPT_VERBOSE:
 265       verbose = 1;
 266       break;
 267     case OPT_LIST:
 268       list = 1;
 269       break;
 270     default:
 271       return ARGP_ERR_UNKNOWN;
 272     }
 273   return 0;
 274 }
 275
 276
 277 static char *
 278 more_help (int key, const char *text, void *input)
 279 {
 280   switch (key)
 281     {
 282     case ARGP_KEY_HELP_EXTRA:
 283       /* We print some extra information.  */
 284       return strdup (gettext ("\
 285 Report bugs using the `glibcbug' script to <bugs@gnu.org>.\n"));
 286     default:
 287       break;
 288     }
 289   return (char *) text;
 290 }
 291
 292
 293 /* Print the version information.  */
 294 static void
 295 print_version (FILE *stream, struct argp_state *state)
 296 {
 297   fprintf (stream, "iconv (GNU %s) %s\n", PACKAGE, VERSION);
 298   fprintf (stream, gettext ("\
 299 Copyright (C) %s Free Software Foundation, Inc.\n\
 300 This is free software; see the source for copying conditions.  There is NO\n\
 301 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n\
 302 "), "1999");
 303   fprintf (stream, gettext ("Written by %s.\n"), "Ulrich Drepper");
 304 }
 305
 306
 307 static int
 308 process_block (iconv_t cd, const char *addr, size_t len, FILE *output)
 309 {
 310 #define OUTBUF_SIZE     32768
 311   const char *start = addr;
 312   char outbuf[OUTBUF_SIZE];
 313   char *outptr;
 314   size_t outlen;
 315   size_t n;
 316
 317   while (len > 0)
 318     {
 319       outptr = outbuf;
 320       outlen = OUTBUF_SIZE;
 321       n = iconv (cd, &addr, &len, &outptr, &outlen);
 322
 323       if (outptr != outbuf)
 324         {
 325           /* We have something to write out.  */
 326           int errno_save = errno;
 327
 328           if (fwrite (outbuf, 1, outptr - outbuf, output) < outptr - outbuf
 329               || ferror (output))
 330             {
 331               /* Error occurred while printing the result.  */
 332               error (0, 0, _("\
 333 conversion stopped due to problem in writing the output"));
 334               return -1;
 335             }
 336
 337           errno = errno_save;
 338         }
 339
 340       if (n != (size_t) -1)
 341         /* Everything is processed.  */
 342         break;
 343
 344       if (errno != E2BIG)
 345         {
 346           /* iconv() ran into a problem.  */
 347           switch (errno)
 348             {
 349             case EILSEQ:
 350               error (0, 0, _("illegal input sequence at position %ld"),
 351                      addr - start);
 352               break;
 353             case EINVAL:
 354               error (0, 0, _("\
 355 incomplete character or shift sequence at end of buffer"));
 356               break;
 357             case EBADF:
 358               error (0, 0, _("internal error (illegal descriptor)"));
 359               break;
 360             default:
 361               error (0, 0, _("unknown iconv() error %d"), errno);
 362               break;
 363             }
 364
 365           return -1;
 366         }
 367     }
 368
 369   return 0;
 370 }
 371
 372
 373 static int
 374 process_fd (iconv_t cd, int fd, FILE *output)
 375 {
 376   /* we have a problem with reading from a desriptor since we must not
 377      provide the iconv() function an incomplete character or shift
 378      sequence at the end of the buffer.  Since we have to deal with
 379      arbitrary encodings we must read the whole text in a buffer and
 380      process it in one step.  */
 381   static char *inbuf = NULL;
 382   static size_t maxlen = 0;
 383   char *inptr = NULL;
 384   size_t actlen = 0;
 385
 386   while (actlen < maxlen)
 387     {
 388       size_t n = read (fd, inptr, maxlen - actlen);
 389
 390       if (n == 0)
 391         /* No more text to read.  */
 392         break;
 393
 394       if (n == -1)
 395         {
 396           /* Error while reading.  */
 397           error (0, errno, _("error while reading the input"));
 398           return -1;
 399         }
 400
 401       inptr += n;
 402       actlen += n;
 403     }
 404
 405   if (actlen == maxlen)
 406     while (1)
 407       {
 408         size_t n;
 409
 410         /* Increase the buffer.  */
 411         maxlen += 32768;
 412         inbuf = realloc (inbuf, maxlen);
 413         if (inbuf == NULL)
 414           error (0, errno, _("unable to allocate buffer for input"));
 415         inptr = inbuf + actlen;
 416
 417         do
 418           {
 419             n = read (fd, inptr, maxlen - actlen);
 420
 421             if (n == 0)
 422               /* No more text to read.  */
 423               break;
 424
 425             if (n == -1)
 426               {
 427                 /* Error while reading.  */
 428                 error (0, errno, _("error while reading the input"));
 429                 return -1;
 430               }
 431
 432             inptr += n;
 433             actlen += n;
 434           }
 435         while (actlen < maxlen);
 436
 437         if (n == 0)
 438           /* Break again so we leave both loops.  */
 439           break;
 440       }
 441
 442   /* Now we have all the input in the buffer.  Process it in one run.  */
 443   return process_block (cd, inbuf, actlen, output);
 444 }
 445
 446
 447 static int
 448 process_file (iconv_t cd, FILE *input, FILE *output)
 449 {
 450   /* This should be safe since we use this function only for `stdin' and
 451      we haven't read anything so far.  */
 452   return process_fd (cd, fileno (input), output);
 453 }
 454
 455
 456 /* Print all known character sets/encodings.  */
 457 static void *printlist;
 458 static size_t column;
 459 static int not_first;
 460
 461 static void
 462 insert_print_list (const void *nodep, VISIT value, int level)
 463 {
 464   if (value == leaf || value == postorder)
 465     {
 466       const struct gconv_alias *s = *(const struct gconv_alias **) nodep;
 467       tsearch (s->fromname, &printlist, (__compar_fn_t) strverscmp);
 468     }
 469 }
 470
 471 static void
 472 do_print  (const void *nodep, VISIT value, int level)
 473 {
 474   if (value == leaf || value == postorder)
 475     {
 476       const char *s = *(const char **) nodep;
 477       size_t len = strlen (s);
 478       size_t cnt;
 479
 480       while (len > 0 && s[len - 1] == '/')
 481         --len;
 482
 483       for (cnt = 0; cnt < len; ++cnt)
 484         if (isalnum (s[cnt]))
 485           break;
 486       if (cnt == len)
 487         return;
 488
 489       if (not_first)
 490         {
 491           putchar (',');
 492           ++column;
 493
 494           if (column > 2 && column + len > 77)
 495             {
 496               fputs ("\n  ", stdout);
 497               column = 2;
 498             }
 499           else
 500             {
 501               putchar (' ');
 502               ++column;
 503             }
 504         }
 505       else
 506         not_first = 1;
 507
 508       fwrite (s, len, 1, stdout);
 509       column += len;
 510     }
 511 }
 512
 513 static void
 514 internal_function
 515 add_known_names (struct gconv_module *node)
 516 {
 517   if (node->left != NULL)
 518     add_known_names (node->left);
 519   if (node->right != NULL)
 520     add_known_names (node->right);
 521   if (node->same != NULL)
 522     add_known_names (node->same);
 523   do
 524     {
 525       if (node->from_pattern == NULL)
 526         {
 527           if (strcmp (node->from_constpfx, "INTERNAL"))
 528             tsearch (node->from_constpfx, &printlist,
 529                      (__compar_fn_t) strverscmp);
 530           if (strcmp (node->to_string, "INTERNAL"))
 531             tsearch (node->to_string, &printlist, (__compar_fn_t) strverscmp);
 532         }
 533       else
 534         if (strcmp (node->from_pattern, "INTERNAL"))
 535           tsearch (node->from_pattern, &printlist, (__compar_fn_t) strverscmp);
 536
 537       node = node->matching;
 538     }
 539   while (node != NULL);
 540 }
 541
 542 static void
 543 internal_function
 544 print_known_names (void)
 545 {
 546   iconv_t h;
 547
 548   /* We must initialize the internal databases first.  */
 549   h = iconv_open ("L1", "L1");
 550   iconv_close (h);
 551
 552   /* First add the aliases.  */
 553   twalk (__gconv_alias_db, insert_print_list);
 554
 555   /* Add the from- and to-names from the known modules.  */
 556   add_known_names (__gconv_modules_db);
 557
 558   fputs (_("\
 559 The following list contain all the coded character sets known.  This does\n\
 560 not necessarily mean that all combinations of these names can be used for\n\
 561 the FROM and TO command line parameters.  One coded character set can be\n\
 562 listed with several different names (aliases).\n\
 563   Some of the names are no plain strings but instead regular expressions and\n\
 564 they match a variety of names which can be given as parameters to the\n\
 565 program.\n\n  "), stdout);
 566
 567   /* Now print the collected names.  */
 568   column = 2;
 569   twalk (printlist, do_print);
 570
 571   if (column != 0)
 572     puts ("");
 573 }