locale/programs/linereader.c

   1 /* Copyright (C) 1996-2023 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3
   4    This program is free software; you can redistribute it and/or modify
   5    it under the terms of the GNU General Public License as published
   6    by the Free Software Foundation; version 2 of the License, or
   7    (at your option) any later version.
   8
   9    This program is distributed in the hope that it will be useful,
  10    but WITHOUT ANY WARRANTY; without even the implied warranty of
  11    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12    GNU General Public License for more details.
  13
  14    You should have received a copy of the GNU General Public License
  15    along with this program; if not, see <https://www.gnu.org/licenses/>.  */
  16
  17 #ifdef HAVE_CONFIG_H
  18 # include <config.h>
  19 #endif
  20
  21 #include <assert.h>
  22 #include <ctype.h>
  23 #include <errno.h>
  24 #include <libintl.h>
  25 #include <stdarg.h>
  26 #include <stdlib.h>
  27 #include <string.h>
  28 #include <stdint.h>
  29
  30 #include "localedef.h"
  31 #include "charmap.h"
  32 #include "error.h"
  33 #include "linereader.h"
  34 #include "locfile.h"
  35
  36 /* Prototypes for local functions.  */
  37 static struct token *get_toplvl_escape (struct linereader *lr);
  38 static struct token *get_symname (struct linereader *lr);
  39 static struct token *get_ident (struct linereader *lr);
  40 static struct token *get_string (struct linereader *lr,
  41                                  const struct charmap_t *charmap,
  42                                  struct localedef_t *locale,
  43                                  const struct repertoire_t *repertoire,
  44                                  int verbose);
  45 static bool utf8_decode (struct linereader *lr, uint8_t ch1, uint32_t *wch);
  46
  47
  48 struct linereader *
  49 lr_open (const char *fname, kw_hash_fct_t hf)
  50 {
  51   FILE *fp;
  52
  53   if (fname == NULL || strcmp (fname, "-") == 0
  54       || strcmp (fname, "/dev/stdin") == 0)
  55     return lr_create (stdin, "<stdin>", hf);
  56   else
  57     {
  58       fp = fopen (fname, "rm");
  59       if (fp == NULL)
  60         return NULL;
  61       return lr_create (fp, fname, hf);
  62     }
  63 }
  64
  65 struct linereader *
  66 lr_create (FILE *fp, const char *fname, kw_hash_fct_t hf)
  67 {
  68   struct linereader *result;
  69   int n;
  70
  71   result = (struct linereader *) xmalloc (sizeof (*result));
  72
  73   result->fp = fp;
  74   result->fname = xstrdup (fname);
  75   result->buf = NULL;
  76   result->bufsize = 0;
  77   result->lineno = 1;
  78   result->idx = 0;
  79   result->comment_char = '#';
  80   result->escape_char = '\\';
  81   result->translate_strings = 1;
  82   result->return_widestr = 0;
  83
  84   n = getdelim (&result->buf, &result->bufsize, '\n', result->fp);
  85   if (n < 0)
  86     {
  87       int save = errno;
  88       fclose (result->fp);
  89       free ((char *) result->fname);
  90       free (result);
  91       errno = save;
  92       return NULL;
  93     }
  94
  95   if (n > 1 && result->buf[n - 2] == '\\' && result->buf[n - 1] == '\n')
  96     n -= 2;
  97
  98   result->buf[n] = '\0';
  99   result->bufact = n;
 100   result->hash_fct = hf;
 101
 102   return result;
 103 }
 104
 105
 106 int
 107 lr_eof (struct linereader *lr)
 108 {
 109   return lr->bufact = 0;
 110 }
 111
 112
 113 void
 114 lr_ignore_rest (struct linereader *lr, int verbose)
 115 {
 116   if (verbose)
 117     {
 118       while (isspace (lr->buf[lr->idx]) && lr->buf[lr->idx] != '\n'
 119              && lr->buf[lr->idx] != lr->comment_char)
 120         if (lr->buf[lr->idx] == '\0')
 121           {
 122             if (lr_next (lr) < 0)
 123               return;
 124           }
 125         else
 126           ++lr->idx;
 127
 128       if (lr->buf[lr->idx] != '\n' && ! feof (lr->fp)
 129           && lr->buf[lr->idx] != lr->comment_char)
 130         lr_error (lr, _("trailing garbage at end of line"));
 131     }
 132
 133   /* Ignore continued line.  */
 134   while (lr->bufact > 0 && lr->buf[lr->bufact - 1] != '\n')
 135     if (lr_next (lr) < 0)
 136       break;
 137
 138   lr->idx = lr->bufact;
 139 }
 140
 141
 142 void
 143 lr_close (struct linereader *lr)
 144 {
 145   fclose (lr->fp);
 146   free (lr->buf);
 147   free (lr);
 148 }
 149
 150
 151 int
 152 lr_next (struct linereader *lr)
 153 {
 154   int n;
 155
 156   n = getdelim (&lr->buf, &lr->bufsize, '\n', lr->fp);
 157   if (n < 0)
 158     return -1;
 159
 160   ++lr->lineno;
 161
 162   if (n > 1 && lr->buf[n - 2] == lr->escape_char && lr->buf[n - 1] == '\n')
 163     {
 164 #if 0
 165       /* XXX Is this correct?  */
 166       /* An escaped newline character is substituted with a single <SP>.  */
 167       --n;
 168       lr->buf[n - 1] = ' ';
 169 #else
 170       n -= 2;
 171 #endif
 172     }
 173
 174   lr->buf[n] = '\0';
 175   lr->bufact = n;
 176   lr->idx = 0;
 177
 178   return 0;
 179 }
 180
 181
 182 /* Defined in error.c.  */
 183 /* This variable is incremented each time `error' is called.  */
 184 extern unsigned int error_message_count;
 185
 186 /* The calling program should define program_name and set it to the
 187    name of the executing program.  */
 188 extern char *program_name;
 189
 190
 191 struct token *
 192 lr_token (struct linereader *lr, const struct charmap_t *charmap,
 193           struct localedef_t *locale, const struct repertoire_t *repertoire,
 194           int verbose)
 195 {
 196   int ch;
 197
 198   while (1)
 199     {
 200       do
 201         {
 202           ch = lr_getc (lr);
 203
 204           if (ch == EOF)
 205             {
 206               lr->token.tok = tok_eof;
 207               return &lr->token;
 208             };
 209
 210           if (ch == '\n')
 211             {
 212               lr->token.tok = tok_eol;
 213               return &lr->token;
 214             }
 215         }
 216       while (isspace (ch));
 217
 218       if (ch != lr->comment_char)
 219         break;
 220
 221       /* Is there an newline at the end of the buffer?  */
 222       if (lr->buf[lr->bufact - 1] != '\n')
 223         {
 224           /* No.  Some people want this to mean that only the line in
 225              the file not the logical, concatenated line is ignored.
 226              Let's try this.  */
 227           lr->idx = lr->bufact;
 228           continue;
 229         }
 230
 231       /* Ignore rest of line.  */
 232       lr_ignore_rest (lr, 0);
 233       lr->token.tok = tok_eol;
 234       return &lr->token;
 235     }
 236
 237   /* Match escape sequences.  */
 238   if (ch == lr->escape_char)
 239     return get_toplvl_escape (lr);
 240
 241   /* Match ellipsis.  */
 242   if (ch == '.')
 243     {
 244       if (strncmp (&lr->buf[lr->idx], "...(2)....", 10) == 0)
 245         {
 246           int cnt;
 247           for (cnt = 0; cnt < 10; ++cnt)
 248             lr_getc (lr);
 249           lr->token.tok = tok_ellipsis4_2;
 250           return &lr->token;
 251         }
 252       if (strncmp (&lr->buf[lr->idx], "...", 3) == 0)
 253         {
 254           lr_getc (lr);
 255           lr_getc (lr);
 256           lr_getc (lr);
 257           lr->token.tok = tok_ellipsis4;
 258           return &lr->token;
 259         }
 260       if (strncmp (&lr->buf[lr->idx], "..", 2) == 0)
 261         {
 262           lr_getc (lr);
 263           lr_getc (lr);
 264           lr->token.tok = tok_ellipsis3;
 265           return &lr->token;
 266         }
 267       if (strncmp (&lr->buf[lr->idx], ".(2)..", 6) == 0)
 268         {
 269           int cnt;
 270           for (cnt = 0; cnt < 6; ++cnt)
 271             lr_getc (lr);
 272           lr->token.tok = tok_ellipsis2_2;
 273           return &lr->token;
 274         }
 275       if (lr->buf[lr->idx] == '.')
 276         {
 277           lr_getc (lr);
 278           lr->token.tok = tok_ellipsis2;
 279           return &lr->token;
 280         }
 281     }
 282
 283   switch (ch)
 284     {
 285     case '<':
 286       return get_symname (lr);
 287
 288     case '0' ... '9':
 289       lr->token.tok = tok_number;
 290       lr->token.val.num = ch - '0';
 291
 292       while (isdigit (ch = lr_getc (lr)))
 293         {
 294           lr->token.val.num *= 10;
 295           lr->token.val.num += ch - '0';
 296         }
 297       if (isalpha (ch))
 298         lr_error (lr, _("garbage at end of number"));
 299       lr_ungetn (lr, 1);
 300
 301       return &lr->token;
 302
 303     case ';':
 304       lr->token.tok = tok_semicolon;
 305       return &lr->token;
 306
 307     case ',':
 308       lr->token.tok = tok_comma;
 309       return &lr->token;
 310
 311     case '(':
 312       lr->token.tok = tok_open_brace;
 313       return &lr->token;
 314
 315     case ')':
 316       lr->token.tok = tok_close_brace;
 317       return &lr->token;
 318
 319     case '"':
 320       return get_string (lr, charmap, locale, repertoire, verbose);
 321
 322     case '-':
 323       ch = lr_getc (lr);
 324       if (ch == '1')
 325         {
 326           lr->token.tok = tok_minus1;
 327           return &lr->token;
 328         }
 329       lr_ungetn (lr, 2);
 330       break;
 331
 332     case 0x80 ... 0xff:         /* UTF-8 sequence.  */
 333       {
 334         uint32_t wch;
 335         if (!utf8_decode (lr, ch, &wch))
 336           {
 337             lr->token.tok = tok_error;
 338             return &lr->token;
 339           }
 340         lr->token.tok = tok_ucs4;
 341         lr->token.val.ucs4 = wch;
 342         return &lr->token;
 343       }
 344     }
 345
 346   return get_ident (lr);
 347 }
 348
 349
 350 static struct token *
 351 get_toplvl_escape (struct linereader *lr)
 352 {
 353   /* This is supposed to be a numeric value.  We return the
 354      numerical value and the number of bytes.  */
 355   size_t start_idx = lr->idx - 1;
 356   unsigned char *bytes = lr->token.val.charcode.bytes;
 357   size_t nbytes = 0;
 358   int ch;
 359
 360   do
 361     {
 362       unsigned int byte = 0;
 363       unsigned int base = 8;
 364
 365       ch = lr_getc (lr);
 366
 367       if (ch == 'd')
 368         {
 369           base = 10;
 370           ch = lr_getc (lr);
 371         }
 372       else if (ch == 'x')
 373         {
 374           base = 16;
 375           ch = lr_getc (lr);
 376         }
 377
 378       if ((base == 16 && !isxdigit (ch))
 379           || (base != 16 && (ch < '0' || ch >= (int) ('0' + base))))
 380         {
 381         esc_error:
 382           lr->token.val.str.startmb = &lr->buf[start_idx];
 383
 384           while (ch != EOF && !isspace (ch))
 385             ch = lr_getc (lr);
 386           lr->token.val.str.lenmb = lr->idx - start_idx;
 387
 388           lr->token.tok = tok_error;
 389           return &lr->token;
 390         }
 391
 392       if (isdigit (ch))
 393         byte = ch - '0';
 394       else
 395         byte = tolower (ch) - 'a' + 10;
 396
 397       ch = lr_getc (lr);
 398       if ((base == 16 && !isxdigit (ch))
 399           || (base != 16 && (ch < '0' || ch >= (int) ('0' + base))))
 400         goto esc_error;
 401
 402       byte *= base;
 403       if (isdigit (ch))
 404         byte += ch - '0';
 405       else
 406         byte += tolower (ch) - 'a' + 10;
 407
 408       ch = lr_getc (lr);
 409       if (base != 16 && isdigit (ch))
 410         {
 411           byte *= base;
 412           byte += ch - '0';
 413
 414           ch = lr_getc (lr);
 415         }
 416
 417       bytes[nbytes++] = byte;
 418     }
 419   while (ch == lr->escape_char
 420          && nbytes < (int) sizeof (lr->token.val.charcode.bytes));
 421
 422   if (!isspace (ch))
 423     lr_error (lr, _("garbage at end of character code specification"));
 424
 425   lr_ungetn (lr, 1);
 426
 427   lr->token.tok = tok_charcode;
 428   lr->token.val.charcode.nbytes = nbytes;
 429
 430   return &lr->token;
 431 }
 432
 433 /* Multibyte string buffer.  */
 434 struct lr_buffer
 435 {
 436   size_t act;
 437   size_t max;
 438   char *buf;
 439 };
 440
 441 /* Initialize *LRB with a default-sized buffer.  */
 442 static void
 443 lr_buffer_init (struct lr_buffer *lrb)
 444 {
 445  lrb->act = 0;
 446  lrb->max = 56;
 447  lrb->buf = xmalloc (lrb->max);
 448 }
 449
 450 /* Transfers the buffer string from *LRB to LR->token.mbstr.  */
 451 static void
 452 lr_buffer_to_token (struct lr_buffer *lrb, struct linereader *lr)
 453 {
 454   lr->token.val.str.startmb = xrealloc (lrb->buf, lrb->act + 1);
 455   lr->token.val.str.startmb[lrb->act] = '\0';
 456   lr->token.val.str.lenmb = lrb->act;
 457 }
 458
 459 /* Adds CH to *LRB.  */
 460 static void
 461 addc (struct lr_buffer *lrb, char ch)
 462 {
 463   if (lrb->act == lrb->max)
 464     {
 465       lrb->max *= 2;
 466       lrb->buf = xrealloc (lrb->buf, lrb->max);
 467     }
 468   lrb->buf[lrb->act++] = ch;
 469 }
 470
 471 /* Adds L bytes at S to *LRB.  */
 472 static void
 473 adds (struct lr_buffer *lrb, const unsigned char *s, size_t l)
 474 {
 475   if (lrb->max - lrb->act < l)
 476     {
 477       size_t required_size = lrb->act + l;
 478       size_t new_max = 2 * lrb->max;
 479       if (new_max < required_size)
 480         new_max = required_size;
 481       lrb->buf = xrealloc (lrb->buf, new_max);
 482       lrb->max = new_max;
 483     }
 484   memcpy (lrb->buf + lrb->act, s, l);
 485   lrb->act += l;
 486 }
 487
 488 #define ADDWC(ch) \
 489   do                                                                          \
 490     {                                                                         \
 491       if (buf2act == buf2max)                                                 \
 492         {                                                                     \
 493           buf2max *= 2;                                                       \
 494           buf2 = xrealloc (buf2, buf2max * 4);                                \
 495         }                                                                     \
 496       buf2[buf2act++] = (ch);                                                 \
 497     }                                                                         \
 498   while (0)
 499
 500
 501 static struct token *
 502 get_symname (struct linereader *lr)
 503 {
 504   /* Symbol in brackets.  We must distinguish three kinds:
 505      1. reserved words
 506      2. ISO 10646 position values
 507      3. all other.  */
 508   const struct keyword_t *kw;
 509   int ch;
 510   struct lr_buffer lrb;
 511
 512   lr_buffer_init (&lrb);
 513
 514   do
 515     {
 516       ch = lr_getc (lr);
 517       if (ch == lr->escape_char)
 518         {
 519           int c2 = lr_getc (lr);
 520           addc (&lrb, c2);
 521
 522           if (c2 == '\n')
 523             ch = '\n';
 524         }
 525       else
 526         addc (&lrb, ch);
 527     }
 528   while (ch != '>' && ch != '\n');
 529
 530   if (ch == '\n')
 531     lr_error (lr, _("unterminated symbolic name"));
 532
 533   /* Test for ISO 10646 position value.  */
 534   if (lrb.buf[0] == 'U' && (lrb.act == 6 || lrb.act == 10))
 535     {
 536       char *cp = lrb.buf + 1;
 537       while (cp < &lrb.buf[lrb.act - 1] && isxdigit (*cp))
 538         ++cp;
 539
 540       if (cp == &lrb.buf[lrb.act - 1])
 541         {
 542           /* Yes, it is.  */
 543           lr->token.tok = tok_ucs4;
 544           lr->token.val.ucs4 = strtoul (lrb.buf + 1, NULL, 16);
 545
 546           return &lr->token;
 547         }
 548     }
 549
 550   /* It is a symbolic name.  Test for reserved words.  */
 551   kw = lr->hash_fct (lrb.buf, lrb.act - 1);
 552
 553   if (kw != NULL && kw->symname_or_ident == 1)
 554     {
 555       lr->token.tok = kw->token;
 556       free (lrb.buf);
 557     }
 558   else
 559     {
 560       lr->token.tok = tok_bsymbol;
 561       lr_buffer_to_token (&lrb, lr);
 562       --lr->token.val.str.lenmb;  /* Hide the training '>'.  */
 563     }
 564
 565   return &lr->token;
 566 }
 567
 568
 569 static struct token *
 570 get_ident (struct linereader *lr)
 571 {
 572   const struct keyword_t *kw;
 573   int ch;
 574   struct lr_buffer lrb;
 575
 576   lr_buffer_init (&lrb);
 577
 578   addc (&lrb, lr->buf[lr->idx - 1]);
 579
 580   while (!isspace ((ch = lr_getc (lr))) && ch != '"' && ch != ';'
 581          && ch != '<' && ch != ',' && ch != EOF)
 582     {
 583       if (ch == lr->escape_char)
 584         {
 585           ch = lr_getc (lr);
 586           if (ch == '\n' || ch == EOF)
 587             {
 588               lr_error (lr, _("invalid escape sequence"));
 589               break;
 590             }
 591         }
 592       addc (&lrb, ch);
 593     }
 594
 595   lr_ungetc (lr, ch);
 596
 597   kw = lr->hash_fct (lrb.buf, lrb.act);
 598
 599   if (kw != NULL && kw->symname_or_ident == 0)
 600     {
 601       lr->token.tok = kw->token;
 602       free (lrb.buf);
 603     }
 604   else
 605     {
 606       lr->token.tok = tok_ident;
 607       lr_buffer_to_token (&lrb, lr);
 608     }
 609
 610   return &lr->token;
 611 }
 612
 613 /* Process a decoded Unicode codepoint WCH in a string, placing the
 614    multibyte sequence into LRB.  Return false if the character is not
 615    found in CHARMAP/REPERTOIRE.  */
 616 static bool
 617 translate_unicode_codepoint (struct localedef_t *locale,
 618                              const struct charmap_t *charmap,
 619                              const struct repertoire_t *repertoire,
 620                              uint32_t wch, struct lr_buffer *lrb)
 621 {
 622   /* See whether the charmap contains the Uxxxxxxxx names.  */
 623   char utmp[10];
 624   snprintf (utmp, sizeof (utmp), "U%08X", wch);
 625   struct charseq *seq = charmap_find_value (charmap, utmp, 9);
 626
 627   if (seq == NULL)
 628     {
 629       /* No, this isn't the case.  Now determine from
 630          the repertoire the name of the character and
 631          find it in the charmap.  */
 632       if (repertoire != NULL)
 633         {
 634           const char *symbol = repertoire_find_symbol (repertoire, wch);
 635           if (symbol != NULL)
 636             seq = charmap_find_value (charmap, symbol, strlen (symbol));
 637         }
 638
 639       if (seq == NULL)
 640         {
 641 #ifndef NO_TRANSLITERATION
 642           /* Transliterate if possible.  */
 643           if (locale != NULL)
 644             {
 645               if ((locale->avail & CTYPE_LOCALE) == 0)
 646                 {
 647                   /* Load the CTYPE data now.  */
 648                   int old_needed = locale->needed;
 649
 650                   locale->needed = 0;
 651                   locale = load_locale (LC_CTYPE, locale->name,
 652                                         locale->repertoire_name,
 653                                         charmap, locale);
 654                   locale->needed = old_needed;
 655                 }
 656
 657               uint32_t *translit;
 658               if ((locale->avail & CTYPE_LOCALE) != 0
 659                   && ((translit = find_translit (locale, charmap, wch))
 660                       != NULL))
 661                 /* The CTYPE data contains a matching
 662                    transliteration.  */
 663                 {
 664                   for (int i = 0; translit[i] != 0; ++i)
 665                     {
 666                       snprintf (utmp, sizeof (utmp), "U%08X", translit[i]);
 667                       seq = charmap_find_value (charmap, utmp, 9);
 668                       assert (seq != NULL);
 669                       adds (lrb, seq->bytes, seq->nbytes);
 670                     }
 671                   return true;
 672                 }
 673             }
 674 #endif  /* NO_TRANSLITERATION */
 675
 676           /* Not a known name.  */
 677           return false;
 678         }
 679     }
 680
 681   if (seq != NULL)
 682     {
 683       adds (lrb, seq->bytes, seq->nbytes);
 684       return true;
 685     }
 686   else
 687     return false;
 688 }
 689
 690 /* Returns true if ch is not EOF (that is, non-negative) and a valid
 691    UTF-8 trailing byte.  */
 692 static bool
 693 utf8_valid_trailing (int ch)
 694 {
 695   return ch >= 0 && (ch & 0xc0) == 0x80;
 696 }
 697
 698 /* Reports an error for a broken UTF-8 sequence.  CH2 to CH4 may be
 699    EOF.  Always returns false.  */
 700 static bool
 701 utf8_sequence_error (struct linereader *lr, uint8_t ch1, int ch2, int ch3,
 702                      int ch4)
 703 {
 704   char buf[38];
 705
 706   if (ch2 < 0)
 707     snprintf (buf, sizeof (buf), "0x%02x", ch1);
 708   else if (ch3 < 0)
 709     snprintf (buf, sizeof (buf), "0x%02x 0x%02x", ch1, ch2);
 710   else if (ch4 < 0)
 711     snprintf (buf, sizeof (buf), "0x%02x 0x%02x 0x%02x", ch1, ch2, ch3);
 712   else
 713     snprintf (buf, sizeof (buf), "0x%02x 0x%02x 0x%02x 0x%02x",
 714               ch1, ch2, ch3, ch4);
 715
 716   lr_error (lr, _("invalid UTF-8 sequence %s"), buf);
 717   return false;
 718 }
 719
 720 /* Reads a UTF-8 sequence from LR, with the leading byte CH1, and
 721    stores the decoded codepoint in *WCH.  Returns false on failure and
 722    reports an error.  */
 723 static bool
 724 utf8_decode (struct linereader *lr, uint8_t ch1, uint32_t *wch)
 725 {
 726   /* See RFC 3629 section 4 and __gconv_transform_utf8_internal.  */
 727   if (ch1 < 0xc2)
 728     return utf8_sequence_error (lr, ch1, -1, -1, -1);
 729
 730   int ch2 = lr_getc (lr);
 731   if (!utf8_valid_trailing (ch2))
 732     return utf8_sequence_error (lr, ch1, ch2, -1, -1);
 733
 734   if (ch1 <= 0xdf)
 735     {
 736       uint32_t result = ((ch1 & 0x1f)  << 6) | (ch2 & 0x3f);
 737       if (result < 0x80)
 738         return utf8_sequence_error (lr, ch1, ch2, -1, -1);
 739       *wch = result;
 740       return true;
 741     }
 742
 743   int ch3 = lr_getc (lr);
 744   if (!utf8_valid_trailing (ch3) || ch1 < 0xe0)
 745     return utf8_sequence_error (lr, ch1, ch2, ch3, -1);
 746
 747   if (ch1 <= 0xef)
 748     {
 749       uint32_t result = (((ch1 & 0x0f)  << 12)
 750                          | ((ch2 & 0x3f) << 6)
 751                          | (ch3 & 0x3f));
 752       if (result < 0x800)
 753         return utf8_sequence_error (lr, ch1, ch2, ch3, -1);
 754       *wch = result;
 755       return true;
 756     }
 757
 758   int ch4 = lr_getc (lr);
 759   if (!utf8_valid_trailing (ch4) || ch1 < 0xf0 || ch1 > 0xf4)
 760     return utf8_sequence_error (lr, ch1, ch2, ch3, ch4);
 761
 762   uint32_t result = (((ch1 & 0x07)  << 18)
 763                      | ((ch2 & 0x3f) << 12)
 764                      | ((ch3 & 0x3f) << 6)
 765                      | (ch4 & 0x3f));
 766   if (result < 0x10000)
 767     return utf8_sequence_error (lr, ch1, ch2, ch3, ch4);
 768   *wch = result;
 769   return true;
 770 }
 771
 772 static struct token *
 773 get_string (struct linereader *lr, const struct charmap_t *charmap,
 774             struct localedef_t *locale, const struct repertoire_t *repertoire,
 775             int verbose)
 776 {
 777   int return_widestr = lr->return_widestr;
 778   struct lr_buffer lrb;
 779   wchar_t *buf2 = NULL;
 780
 781   lr_buffer_init (&lrb);
 782
 783   /* We know it'll be a string.  */
 784   lr->token.tok = tok_string;
 785
 786   /* If we need not translate the strings (i.e., expand <...> parts)
 787      we can run a simple loop.  */
 788   if (!lr->translate_strings)
 789     {
 790       int ch;
 791
 792       buf2 = NULL;
 793       while ((ch = lr_getc (lr)) != '"' && ch != '\n' && ch != EOF)
 794         {
 795           if (ch >= 0x80)
 796             lr_error (lr, _("illegal 8-bit character in untranslated string"));
 797           addc (&lrb, ch);
 798         }
 799
 800       /* Catch errors with trailing escape character.  */
 801       if (lrb.act > 0 && lrb.buf[lrb.act - 1] == lr->escape_char
 802           && (lrb.act == 1 || lrb.buf[lrb.act - 2] != lr->escape_char))
 803         {
 804           lr_error (lr, _("illegal escape sequence at end of string"));
 805           --lrb.act;
 806         }
 807       else if (ch == '\n' || ch == EOF)
 808         lr_error (lr, _("unterminated string"));
 809
 810       addc (&lrb, '\0');
 811     }
 812   else
 813     {
 814       bool illegal_string = false;
 815       size_t buf2act = 0;
 816       size_t buf2max = 56 * sizeof (uint32_t);
 817       int ch;
 818
 819       /* We have to provide the wide character result as well.  */
 820       if (return_widestr)
 821         buf2 = xmalloc (buf2max);
 822
 823       /* Read until the end of the string (or end of the line or file).  */
 824       while ((ch = lr_getc (lr)) != '"' && ch != '\n' && ch != EOF)
 825         {
 826           size_t startidx;
 827           uint32_t wch;
 828           struct charseq *seq;
 829
 830           if (ch != '<')
 831             {
 832               /* The standards leave it up to the implementation to
 833                  decide what to do with characters which stand for
 834                  themselves.  This implementation treats the input
 835                  file as encoded in UTF-8.  */
 836               if (ch == lr->escape_char)
 837                 {
 838                   ch = lr_getc (lr);
 839                   if (ch >= 0x80)
 840                     {
 841                       lr_error (lr, _("illegal 8-bit escape sequence"));
 842                       illegal_string = true;
 843                       break;
 844                     }
 845                   if (ch == '\n' || ch == EOF)
 846                     break;
 847                   addc (&lrb, ch);
 848                   wch = ch;
 849                 }
 850               else if (ch < 0x80)
 851                 {
 852                   wch = ch;
 853                   addc (&lrb, ch);
 854                 }
 855               else              /* UTF-8 sequence.  */
 856                 {
 857                   if (!utf8_decode (lr, ch, &wch))
 858                     {
 859                       illegal_string = true;
 860                       break;
 861                     }
 862                   if (!translate_unicode_codepoint (locale, charmap,
 863                                                     repertoire, wch, &lrb))
 864                     {
 865                       /* Ignore the rest of the string.  Callers may
 866                          skip this string because it cannot be encoded
 867                          in the output character set.  */
 868                       illegal_string = true;
 869                       continue;
 870                     }
 871                 }
 872
 873               if (return_widestr)
 874                 ADDWC (wch);
 875
 876               continue;
 877             }
 878
 879           /* Now we have to search for the end of the symbolic name, i.e.,
 880              the closing '>'.  */
 881           startidx = lrb.act;
 882           while ((ch = lr_getc (lr)) != '>' && ch != '\n' && ch != EOF)
 883             {
 884               if (ch == lr->escape_char)
 885                 {
 886                   ch = lr_getc (lr);
 887                   if (ch == '\n' || ch == EOF)
 888                     break;
 889                 }
 890               addc (&lrb, ch);
 891             }
 892           if (ch == '\n' || ch == EOF)
 893             /* Not a correct string.  */
 894             break;
 895           if (lrb.act == startidx)
 896             {
 897               /* <> is no correct name.  Ignore it and also signal an
 898                  error.  */
 899               illegal_string = true;
 900               continue;
 901             }
 902
 903           /* It might be a Uxxxx symbol.  */
 904           if (lrb.buf[startidx] == 'U'
 905               && (lrb.act - startidx == 5 || lrb.act - startidx == 9))
 906             {
 907               char *cp = lrb.buf + startidx + 1;
 908               while (cp < &lrb.buf[lrb.act] && isxdigit (*cp))
 909                 ++cp;
 910
 911               if (cp == &lrb.buf[lrb.act])
 912                 {
 913                   /* Yes, it is.  */
 914                   addc (&lrb, '\0');
 915                   wch = strtoul (lrb.buf + startidx + 1, NULL, 16);
 916
 917                   /* Now forget about the name we just added.  */
 918                   lrb.act = startidx;
 919
 920                   if (return_widestr)
 921                     ADDWC (wch);
 922
 923                   if (!translate_unicode_codepoint (locale, charmap,
 924                                                     repertoire, wch, &lrb))
 925                     illegal_string = true;
 926                   continue;
 927                 }
 928             }
 929
 930           /* We now have the symbolic name in lrb.buf[startidx] to
 931              lrb.buf[lrb.act-1].  Now find out the value for this character
 932              in the charmap as well as in the repertoire map (in this
 933              order).  */
 934           seq = charmap_find_value (charmap, &lrb.buf[startidx],
 935                                     lrb.act - startidx);
 936
 937           if (seq == NULL)
 938             {
 939               /* This name is not in the charmap.  */
 940               lr_error (lr, _("symbol `%.*s' not in charmap"),
 941                         (int) (lrb.act - startidx), &lrb.buf[startidx]);
 942               illegal_string = true;
 943             }
 944
 945           if (return_widestr)
 946             {
 947               /* Now the same for the multibyte representation.  */
 948               if (seq != NULL && seq->ucs4 != UNINITIALIZED_CHAR_VALUE)
 949                 wch = seq->ucs4;
 950               else
 951                 {
 952                   wch = repertoire_find_value (repertoire, &lrb.buf[startidx],
 953                                                lrb.act - startidx);
 954                   if (seq != NULL)
 955                     seq->ucs4 = wch;
 956                 }
 957
 958               if (wch == ILLEGAL_CHAR_VALUE)
 959                 {
 960                   /* This name is not in the repertoire map.  */
 961                   lr_error (lr, _("symbol `%.*s' not in repertoire map"),
 962                             (int) (lrb.act - startidx), &lrb.buf[startidx]);
 963                   illegal_string = true;
 964                 }
 965               else
 966                 ADDWC (wch);
 967             }
 968
 969           /* Now forget about the name we just added.  */
 970           lrb.act = startidx;
 971
 972           /* And copy the bytes.  */
 973           if (seq != NULL)
 974             adds (&lrb, seq->bytes, seq->nbytes);
 975         }
 976
 977       if (ch == '\n' || ch == EOF)
 978         {
 979           lr_error (lr, _("unterminated string"));
 980           illegal_string = true;
 981         }
 982
 983       if (illegal_string)
 984         {
 985           free (lrb.buf);
 986           free (buf2);
 987           lr->token.val.str.startmb = NULL;
 988           lr->token.val.str.lenmb = 0;
 989           lr->token.val.str.startwc = NULL;
 990           lr->token.val.str.lenwc = 0;
 991
 992           return &lr->token;
 993         }
 994
 995       addc (&lrb, '\0');
 996
 997       if (return_widestr)
 998         {
 999           ADDWC (0);
1000           lr->token.val.str.startwc = xrealloc (buf2,
1001                                                 buf2act * sizeof (uint32_t));
1002           lr->token.val.str.lenwc = buf2act;
1003         }
1004     }
1005
1006   lr_buffer_to_token (&lrb, lr);
1007
1008   return &lr->token;
1009 }