tools/wmc/mcl.c

   1 /*
   2  * Wine Message Compiler lexical scanner
   3  *
   4  * Copyright 2000 Bertho A. Stultiens (BS)
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
  19  */
  20
  21 #include "config.h"
  22
  23 #include <stdio.h>
  24 #include <stdlib.h>
  25 #include <ctype.h>
  26 #include <assert.h>
  27 #include <string.h>
  28
  29 #include "utils.h"
  30 #include "wmc.h"
  31 #include "lang.h"
  32
  33 #include "mcy.tab.h"
  34
  35 /*
  36  * Keywords are case insensitive. All normal input is treated as
  37  * being in codepage iso-8859-1 for ascii input files (unicode
  38  * page 0) and as equivalent unicode if unicode input is selected.
  39  * All normal input, which is not part of a message text, is
  40  * enforced to be unicode page 0. Otherwise an error will be
  41  * generated. The normal file data should only be ASCII because
  42  * that is the basic definition of the grammar.
  43  *
  44  * Byteorder or unicode input is determined automatically by
  45  * reading the first 8 bytes and checking them against unicode
  46  * page 0 byteorder (hibyte must be 0).
  47  * -- FIXME --
  48  * Alternatively, the input is checked against a special byte
  49  * sequence to identify the file.
  50  * -- FIXME --
  51  *
  52  *
  53  * Keywords:
  54  *      Codepages
  55  *      Facility
  56  *      FacilityNames
  57  *      LanguageNames
  58  *      MessageId
  59  *      MessageIdTypedef
  60  *      Severity
  61  *      SeverityNames
  62  *      SymbolicName
  63  *
  64  * Default added identifiers for classes:
  65  * SeverityNames:
  66  *      Success         = 0x0
  67  *      Informational   = 0x1
  68  *      Warning         = 0x2
  69  *      Error           = 0x3
  70  * FacilityNames:
  71  *      System          = 0x0FF
  72  *      Application     = 0xFFF
  73  *
  74  * The 'Codepages' keyword is a wmc extension.
  75  */
  76
  77 static const WCHAR ustr_application[]   = { 'A', 'p', 'p', 'l', 'i', 'c', 'a', 't', 'i', 'o', 'n', 0 };
  78 static const WCHAR ustr_codepages[]     = { 'C', 'o', 'd', 'e', 'p', 'a', 'g', 'e', 's', 0 };
  79 static const WCHAR ustr_english[]       = { 'E', 'n', 'g', 'l', 'i', 's', 'h', 0 };
  80 static const WCHAR ustr_error[]         = { 'E', 'r', 'r', 'o', 'r', 0 };
  81 static const WCHAR ustr_facility[]      = { 'F', 'a', 'c', 'i', 'l', 'i', 't', 'y', 0 };
  82 static const WCHAR ustr_facilitynames[] = { 'F', 'a', 'c', 'i', 'l', 'i', 't', 'y', 'N', 'a', 'm', 'e', 's', 0 };
  83 static const WCHAR ustr_informational[] = { 'I', 'n', 'f', 'o', 'r', 'm', 'a', 't', 'i', 'o', 'n', 'a', 'l', 0 };
  84 static const WCHAR ustr_language[]      = { 'L', 'a', 'n', 'g', 'u', 'a', 'g', 'e', 0};
  85 static const WCHAR ustr_languagenames[] = { 'L', 'a', 'n', 'g', 'u', 'a', 'g', 'e', 'N', 'a', 'm', 'e', 's', 0};
  86 static const WCHAR ustr_messageid[]     = { 'M', 'e', 's', 's', 'a', 'g', 'e', 'I', 'd', 0 };
  87 static const WCHAR ustr_messageidtypedef[] = { 'M', 'e', 's', 's', 'a', 'g', 'e', 'I', 'd', 'T', 'y', 'p', 'e', 'd', 'e', 'f', 0 };
  88 static const WCHAR ustr_outputbase[]    = { 'O', 'u', 't', 'p', 'u', 't', 'B', 'a', 's', 'e', 0 };
  89 static const WCHAR ustr_severity[]      = { 'S', 'e', 'v', 'e', 'r', 'i', 't', 'y', 0 };
  90 static const WCHAR ustr_severitynames[] = { 'S', 'e', 'v', 'e', 'r', 'i', 't', 'y', 'N', 'a', 'm', 'e', 's', 0 };
  91 static const WCHAR ustr_success[]       = { 'S', 'u', 'c', 'c', 'e', 's', 's', 0 };
  92 static const WCHAR ustr_symbolicname[]  = { 'S', 'y', 'm', 'b', 'o', 'l', 'i', 'c', 'N', 'a', 'm', 'e', 0 };
  93 static const WCHAR ustr_system[]        = { 'S', 'y', 's', 't', 'e', 'm', 0 };
  94 static const WCHAR ustr_warning[]       = { 'W', 'a', 'r', 'n', 'i', 'n', 'g', 0 };
  95 static const WCHAR ustr_msg00001[]      = { 'm', 's', 'g', '0', '0', '0', '0', '1', 0 };
  96 /*
  97  * This table is to beat any form of "expression building" to check for
  98  * correct filename characters. It is also used for ident checks.
  99  * FIXME: use it more consistently.
 100  */
 101
 102 #define CH_SHORTNAME    0x01
 103 #define CH_LONGNAME     0x02
 104 #define CH_IDENT        0x04
 105 #define CH_NUMBER       0x08
 106 /*#define CH_WILDCARD   0x10*/
 107 /*#define CH_DOT        0x20*/
 108 #define CH_PUNCT        0x40
 109 #define CH_INVALID      0x80
 110
 111 static const char char_table[256] = {
 112         0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, /* 0x00 - 0x07 */
 113         0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, /* 0x08 - 0x0F */
 114         0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, /* 0x10 - 0x17 */
 115         0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, /* 0x18 - 0x1F */
 116         0x80, 0x03, 0x80, 0x03, 0x03, 0x03, 0x03, 0x03, /* 0x20 - 0x27 " !"#$%&'" */
 117         0x43, 0x43, 0x10, 0x80, 0x03, 0x03, 0x22, 0x80, /* 0x28 - 0x2F "()*+,-./" */
 118         0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, /* 0x30 - 0x37 "01234567" */
 119         0x0b, 0x0b, 0xc0, 0x80, 0x80, 0x80, 0x80, 0x10, /* 0x38 - 0x3F "89:;<=>?" */
 120         0x03, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, /* 0x40 - 0x47 "@ABCDEFG" */
 121         0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, /* 0x48 - 0x4F "HIJKLMNO" */
 122         0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, /* 0x50 - 0x57 "PQRSTUVW" */
 123         0x07, 0x07, 0x07, 0x80, 0x80, 0x80, 0x80, 0x07, /* 0x58 - 0x5F "XYZ[\]^_" */
 124         0x03, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, /* 0x60 - 0x67 "`abcdefg" */
 125         0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, /* 0x68 - 0x6F "hijklmno" */
 126         0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, /* 0x70 - 0x77 "pqrstuvw" */
 127         0x07, 0x07, 0x07, 0x03, 0x80, 0x03, 0x03, 0x80, /* 0x78 - 0x7F "xyz{|}~ " */
 128         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0x80 - 0x87 */
 129         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0x88 - 0x8F */
 130         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0x90 - 0x97 */
 131         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0x98 - 0x9F */
 132         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xA0 - 0xA7 */
 133         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xA8 - 0xAF */
 134         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xB0 - 0xB7 */
 135         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xB8 - 0xBF */
 136         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xC0 - 0xC7 */
 137         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xC8 - 0xCF */
 138         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xD0 - 0xD7 */
 139         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xD8 - 0xDF */
 140         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xE0 - 0xE7 */
 141         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xE8 - 0xEF */
 142         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xF0 - 0xF7 */
 143         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x80, /* 0xF8 - 0xFF */
 144 };
 145
 146 static int isisochar(int ch)
 147 {
 148         return !(ch & (~0xff));
 149 }
 150
 151 static int codepage;
 152
 153 void set_codepage(int cp)
 154 {
 155         codepage = cp;
 156         if (!is_valid_codepage( cp ))
 157                 xyyerror("Codepage %d not found; cannot process\n", codepage);
 158 }
 159
 160 /*
 161  * Input functions
 162  */
 163 static int nungetstack = 0;
 164 static int allocungetstack = 0;
 165 static char *ungetstack = NULL;
 166 static int ninputbuffer = 0;
 167 static WCHAR *inputbuffer = NULL;
 168 static char *xlatebuffer = NULL;
 169
 170 #define INPUTBUFFER_SIZE        2048    /* Must be larger than 4 and approx. large enough to hold a line */
 171
 172 /*
 173  * Fill the input buffer with *one* line of input.
 174  * The line is '\n' terminated so that scanning
 175  * messages with translation works as expected
 176  * (otherwise we cannot pre-translate because the
 177  * language is first known one line before the
 178  * actual message).
 179  */
 180 static int fill_inputbuffer(void)
 181 {
 182         int n;
 183         static const char err_fatalread[] = "Fatal: reading input failed";
 184         static int endian = -1;
 185
 186         if(!inputbuffer)
 187         {
 188                 inputbuffer = xmalloc(INPUTBUFFER_SIZE*sizeof(WCHAR));
 189                 xlatebuffer = xmalloc(INPUTBUFFER_SIZE);
 190         }
 191
 192 try_again:
 193         if(!unicodein)
 194         {
 195                 char *cptr;
 196                 cptr = fgets(xlatebuffer, INPUTBUFFER_SIZE, yyin);
 197                 if(!cptr && ferror(yyin))
 198                         xyyerror(err_fatalread);
 199                 else if(!cptr)
 200                         return 0;
 201                 n = wmc_mbstowcs(codepage, 0, xlatebuffer, strlen(xlatebuffer)+1, inputbuffer, INPUTBUFFER_SIZE);
 202                 if(n < 0)
 203                         internal_error(__FILE__, __LINE__, "Could not translate to unicode (%d)\n", n);
 204                 if(n <= 1)
 205                         goto try_again; /* Should not happen */
 206                 n--;    /* Strip added conversion '\0' from input length */
 207                 /*
 208                  * FIXME:
 209                  * Detect UTF-8 in the first time we read some bytes by
 210                  * checking the special sequence "FE..." or something like
 211                  * that. I need to check www.unicode.org for details.
 212                  */
 213         }
 214         else
 215         {
 216                 if(endian == -1)
 217                 {
 218                         n = fread(inputbuffer, 1, 8, yyin);
 219                         if(n != 8)
 220                         {
 221                                 if(!n && ferror(yyin))
 222                                         xyyerror(err_fatalread);
 223                                 else
 224                                         xyyerror("Fatal: file too short to determine byteorder (should never happen)\n");
 225                         }
 226                         if(isisochar(inputbuffer[0]) &&
 227                                 isisochar(inputbuffer[1]) &&
 228                                 isisochar(inputbuffer[2]) &&
 229                                 isisochar(inputbuffer[3]))
 230                         {
 231 #ifdef WORDS_BIGENDIAN
 232                                 endian = WMC_BO_BIG;
 233 #else
 234                                 endian = WMC_BO_LITTLE;
 235 #endif
 236                         }
 237                         else if(isisochar(BYTESWAP_WORD(inputbuffer[0])) &&
 238                                 isisochar(BYTESWAP_WORD(inputbuffer[1])) &&
 239                                 isisochar(BYTESWAP_WORD(inputbuffer[2])) &&
 240                                 isisochar(BYTESWAP_WORD(inputbuffer[3])))
 241                         {
 242 #ifdef WORDS_BIGENDIAN
 243                                 endian = WMC_BO_LITTLE;
 244 #else
 245                                 endian = WMC_BO_BIG;
 246 #endif
 247                         }
 248                         else
 249                                 xyyerror("Fatal: cannot determine file's byteorder\n");
 250                         /* FIXME:
 251                          * Determine the file-endian with the leader-bytes
 252                          * "FF FE..."; can't remember the exact sequence.
 253                          */
 254                         n /= 2;
 255 #ifdef WORDS_BIGENDIAN
 256                         if(endian == WMC_BO_LITTLE)
 257 #else
 258                         if(endian == WMC_BO_BIG)
 259 #endif
 260                         {
 261                                 inputbuffer[0] = BYTESWAP_WORD(inputbuffer[0]);
 262                                 inputbuffer[1] = BYTESWAP_WORD(inputbuffer[1]);
 263                                 inputbuffer[2] = BYTESWAP_WORD(inputbuffer[2]);
 264                                 inputbuffer[3] = BYTESWAP_WORD(inputbuffer[3]);
 265                         }
 266
 267                 }
 268                 else
 269                 {
 270                         int i;
 271                         n = 0;
 272                         for(i = 0; i < INPUTBUFFER_SIZE; i++)
 273                         {
 274                                 int t;
 275                                 t = fread(&inputbuffer[i], 2, 1, yyin);
 276                                 if(!t && ferror(yyin))
 277                                         xyyerror(err_fatalread);
 278                                 else if(!t && n)
 279                                         break;
 280                                 n++;
 281 #ifdef WORDS_BIGENDIAN
 282                                 if(endian == WMC_BO_LITTLE)
 283 #else
 284                                 if(endian == WMC_BO_BIG)
 285 #endif
 286                                 {
 287                                         if((inputbuffer[i] = BYTESWAP_WORD(inputbuffer[i])) == '\n')
 288                                                 break;
 289                                 }
 290                                 else
 291                                 {
 292                                         if(inputbuffer[i] == '\n')
 293                                                 break;
 294                                 }
 295                         }
 296                 }
 297
 298         }
 299
 300         if(!n)
 301         {
 302                 mcy_warning("Re-read line (input was or converted to zilch)\n");
 303                 goto try_again; /* Should not happen, but could be due to stdin reading and a signal */
 304         }
 305
 306         ninputbuffer += n;
 307         return 1;
 308 }
 309
 310 static int get_unichar(void)
 311 {
 312         static WCHAR *b = NULL;
 313         char_number++;
 314
 315         if(nungetstack)
 316                 return ungetstack[--nungetstack];
 317
 318         if(!ninputbuffer)
 319         {
 320                 if(!fill_inputbuffer())
 321                         return EOF;
 322                 b = inputbuffer;
 323         }
 324
 325         ninputbuffer--;
 326         return (int)(*b++ & 0xffff);
 327 }
 328
 329 static void unget_unichar(int ch)
 330 {
 331         if(ch == EOF)
 332                 return;
 333
 334         char_number--;
 335
 336         if(nungetstack == allocungetstack)
 337         {
 338                 allocungetstack += 32;
 339                 ungetstack = xrealloc(ungetstack, allocungetstack * sizeof(*ungetstack));
 340         }
 341
 342         ungetstack[nungetstack++] = (WCHAR)ch;
 343 }
 344
 345
 346 /*
 347  * Normal character stack.
 348  * Used for number scanning.
 349  */
 350 static int ncharstack = 0;
 351 static int alloccharstack = 0;
 352 static char *charstack = NULL;
 353
 354 static void empty_char_stack(void)
 355 {
 356         ncharstack = 0;
 357 }
 358
 359 static void push_char(int ch)
 360 {
 361         if(ncharstack == alloccharstack)
 362         {
 363                 alloccharstack += 32;
 364                 charstack = xrealloc(charstack, alloccharstack * sizeof(*charstack));
 365         }
 366         charstack[ncharstack++] = (char)ch;
 367 }
 368
 369 static int tos_char_stack(void)
 370 {
 371         if(!ncharstack)
 372                 return 0;
 373         else
 374                 return (int)(charstack[ncharstack-1] & 0xff);
 375 }
 376
 377 static char *get_char_stack(void)
 378 {
 379         return charstack;
 380 }
 381
 382 /*
 383  * Unicode character stack.
 384  * Used for general scanner.
 385  */
 386 static int nunicharstack = 0;
 387 static int allocunicharstack = 0;
 388 static WCHAR *unicharstack = NULL;
 389
 390 static void empty_unichar_stack(void)
 391 {
 392         nunicharstack = 0;
 393 }
 394
 395 static void push_unichar(int ch)
 396 {
 397         if(nunicharstack == allocunicharstack)
 398         {
 399                 allocunicharstack += 128;
 400                 unicharstack = xrealloc(unicharstack, allocunicharstack * sizeof(*unicharstack));
 401         }
 402         unicharstack[nunicharstack++] = (WCHAR)ch;
 403 }
 404
 405 #if 0
 406 static int tos_unichar_stack(void)
 407 {
 408         if(!nunicharstack)
 409                 return 0;
 410         else
 411                 return (int)(unicharstack[nunicharstack-1] & 0xffff);
 412 }
 413 #endif
 414
 415 static WCHAR *get_unichar_stack(void)
 416 {
 417         return unicharstack;
 418 }
 419
 420 /*
 421  * Number scanner
 422  *
 423  * state |      ch         | next state
 424  * ------+-----------------+--------------------------
 425  *   0   | [0]             | 1
 426  *   0   | [1-9]           | 4
 427  *   0   | .               | error (should never occur)
 428  *   1   | [xX]            | 2
 429  *   1   | [0-7]           | 3
 430  *   1   | [89a-wyzA-WYZ_] | error invalid digit
 431  *   1   | .               | return 0
 432  *   2   | [0-9a-fA-F]     | 2
 433  *   2   | [g-zG-Z_]       | error invalid hex digit
 434  *   2   | .               | return (hex-number) if TOS != [xX] else error
 435  *   3   | [0-7]           | 3
 436  *   3   | [89a-zA-Z_]     | error invalid octal digit
 437  *   3   | .               | return (octal-number)
 438  *   4   | [0-9]           | 4
 439  *   4   | [a-zA-Z_]       | error invalid decimal digit
 440  *   4   | .               | return (decimal-number)
 441  *
 442  * All non-identifier characters [^a-zA-Z_0-9] terminate the scan
 443  * and return the value. This is not entirely correct, but close
 444  * enough (should check punctuators as trailing context, but the
 445  * char_table is not adapted to that and it is questionable whether
 446  * it is worth the trouble).
 447  * All non-iso-8859-1 characters are an error.
 448  */
 449 static int scan_number(int ch)
 450 {
 451         int state = 0;
 452         int base = 10;
 453         empty_char_stack();
 454
 455         while(1)
 456         {
 457                 if(!isisochar(ch))
 458                         xyyerror("Invalid digit\n");
 459
 460                 switch(state)
 461                 {
 462                 case 0:
 463                         if(isdigit(ch))
 464                         {
 465                                 push_char(ch);
 466                                 if(ch == '0')
 467                                         state = 1;
 468                                 else
 469                                         state = 4;
 470                         }
 471                         else
 472                                 internal_error(__FILE__, __LINE__, "Non-digit in first number-scanner state\n");
 473                         break;
 474                 case 1:
 475                         if(ch == 'x' || ch == 'X')
 476                         {
 477                                 push_char(ch);
 478                                 state = 2;
 479                         }
 480                         else if(ch >= '0' && ch <= '7')
 481                         {
 482                                 push_char(ch);
 483                                 state = 3;
 484                         }
 485                         else if(isalpha(ch) || ch == '_')
 486                                 xyyerror("Invalid number digit\n");
 487                         else
 488                         {
 489                                 unget_unichar(ch);
 490                                 mcy_lval.num = 0;
 491                                 return tNUMBER;
 492                         }
 493                         break;
 494                 case 2:
 495                         if(isxdigit(ch))
 496                                 push_char(ch);
 497                         else if(isalpha(ch) || ch == '_' || !isxdigit(tos_char_stack()))
 498                                 xyyerror("Invalid hex digit\n");
 499                         else
 500                         {
 501                                 base = 16;
 502                                 goto finish;
 503                         }
 504                         break;
 505                 case 3:
 506                         if(ch >= '0' && ch <= '7')
 507                                 push_char(ch);
 508                         else if(isalnum(ch) || ch == '_')
 509                                 xyyerror("Invalid octal digit\n");
 510                         else
 511                         {
 512                                 base = 8;
 513                                 goto finish;
 514                         }
 515                         break;
 516                 case 4:
 517                         if(isdigit(ch))
 518                                 push_char(ch);
 519                         else if(isalnum(ch) || ch == '_')
 520                                 xyyerror("Invalid decimal digit\n");
 521                         else
 522                         {
 523                                 base = 10;
 524                                 goto finish;
 525                         }
 526                         break;
 527                 default:
 528                         internal_error(__FILE__, __LINE__, "Invalid state in number-scanner\n");
 529                 }
 530                 ch = get_unichar();
 531         }
 532 finish:
 533         unget_unichar(ch);
 534         push_char(0);
 535         mcy_lval.num = strtoul(get_char_stack(), NULL, base);
 536         return tNUMBER;
 537 }
 538
 539 static void newline(void)
 540 {
 541         line_number++;
 542         char_number = 1;
 543 }
 544
 545 static int unisort(const void *p1, const void *p2)
 546 {
 547         return unistricmp(((const token_t *)p1)->name, ((const token_t *)p2)->name);
 548 }
 549
 550 static token_t *tokentable = NULL;
 551 static int ntokentable = 0;
 552
 553 token_t *lookup_token(const WCHAR *s)
 554 {
 555         token_t tok;
 556
 557         tok.name = s;
 558         return (token_t *)bsearch(&tok, tokentable, ntokentable, sizeof(*tokentable), unisort);
 559 }
 560
 561 void add_token(tok_e type, const WCHAR *name, int tok, int cp, const WCHAR *alias, int fix)
 562 {
 563         ntokentable++;
 564         tokentable = xrealloc(tokentable, ntokentable * sizeof(*tokentable));
 565         tokentable[ntokentable-1].type = type;
 566         tokentable[ntokentable-1].name = name;
 567         tokentable[ntokentable-1].token = tok;
 568         tokentable[ntokentable-1].codepage = cp;
 569         tokentable[ntokentable-1].alias = alias;
 570         tokentable[ntokentable-1].fixed = fix;
 571         qsort(tokentable, ntokentable, sizeof(*tokentable), unisort);
 572 }
 573
 574 void get_tokentable(token_t **tab, int *len)
 575 {
 576         assert(tab != NULL);
 577         assert(len != NULL);
 578         *tab = tokentable;
 579         *len = ntokentable;
 580 }
 581
 582 /*
 583  * The scanner
 584  *
 585  */
 586 int mcy_lex(void)
 587 {
 588         static const WCHAR ustr_dot1[] = { '.', '\n', 0 };
 589         static const WCHAR ustr_dot2[] = { '.', '\r', '\n', 0 };
 590         static int isinit = 0;
 591         int ch;
 592
 593         if(!isinit)
 594         {
 595                 isinit++;
 596                 set_codepage(WMC_DEFAULT_CODEPAGE);
 597                 add_token(tok_keyword,  ustr_codepages,         tCODEPAGE,      0, NULL, 0);
 598                 add_token(tok_keyword,  ustr_facility,          tFACILITY,      0, NULL, 1);
 599                 add_token(tok_keyword,  ustr_facilitynames,     tFACNAMES,      0, NULL, 1);
 600                 add_token(tok_keyword,  ustr_language,          tLANGUAGE,      0, NULL, 1);
 601                 add_token(tok_keyword,  ustr_languagenames,     tLANNAMES,      0, NULL, 1);
 602                 add_token(tok_keyword,  ustr_messageid,         tMSGID,         0, NULL, 1);
 603                 add_token(tok_keyword,  ustr_messageidtypedef,  tTYPEDEF,       0, NULL, 1);
 604                 add_token(tok_keyword,  ustr_outputbase,        tBASE,          0, NULL, 1);
 605                 add_token(tok_keyword,  ustr_severity,          tSEVERITY,      0, NULL, 1);
 606                 add_token(tok_keyword,  ustr_severitynames,     tSEVNAMES,      0, NULL, 1);
 607                 add_token(tok_keyword,  ustr_symbolicname,      tSYMNAME,       0, NULL, 1);
 608                 add_token(tok_severity, ustr_error,             0x03,           0, NULL, 0);
 609                 add_token(tok_severity, ustr_warning,           0x02,           0, NULL, 0);
 610                 add_token(tok_severity, ustr_informational,     0x01,           0, NULL, 0);
 611                 add_token(tok_severity, ustr_success,           0x00,           0, NULL, 0);
 612                 add_token(tok_facility, ustr_application,       0xFFF,          0, NULL, 0);
 613                 add_token(tok_facility, ustr_system,            0x0FF,          0, NULL, 0);
 614                 add_token(tok_language, ustr_english,           0x409,          437, ustr_msg00001, 0);
 615         }
 616
 617         empty_unichar_stack();
 618
 619         while(1)
 620         {
 621                 if(want_line)
 622                 {
 623                         while((ch = get_unichar()) != '\n')
 624                         {
 625                                 if(ch == EOF)
 626                                         xyyerror("Unexpected EOF\n");
 627                                 push_unichar(ch);
 628                         }
 629                         newline();
 630                         push_unichar(ch);
 631                         push_unichar(0);
 632                         if(!unistrcmp(ustr_dot1, get_unichar_stack()) || !unistrcmp(ustr_dot2, get_unichar_stack()))
 633                         {
 634                                 want_line = 0;
 635                                 /* Reset the codepage to our default after each message */
 636                                 set_codepage(WMC_DEFAULT_CODEPAGE);
 637                                 return tMSGEND;
 638                         }
 639                         mcy_lval.str = xunistrdup(get_unichar_stack());
 640                         return tLINE;
 641                 }
 642
 643                 ch = get_unichar();
 644
 645                 if(ch == EOF)
 646                         return EOF;
 647
 648                 if(ch == '\n')
 649                 {
 650                         newline();
 651                         if(want_nl)
 652                         {
 653                                 want_nl = 0;
 654                                 return tNL;
 655                         }
 656                         continue;
 657                 }
 658
 659                 if(isisochar(ch))
 660                 {
 661                         if(want_file)
 662                         {
 663                                 int n = 0;
 664                                 while(n < 8 && isisochar(ch))
 665                                 {
 666                                         int t = char_table[ch];
 667                                         if((t & CH_PUNCT) || !(t & CH_SHORTNAME))
 668                                                 break;
 669
 670                                         push_unichar(ch);
 671                                         n++;
 672                                         ch = get_unichar();
 673                                 }
 674                                 unget_unichar(ch);
 675                                 push_unichar(0);
 676                                 want_file = 0;
 677                                 mcy_lval.str = xunistrdup(get_unichar_stack());
 678                                 return tFILE;
 679                         }
 680
 681                         if(char_table[ch] & CH_IDENT)
 682                         {
 683                                 token_t *tok;
 684                                 while(isisochar(ch) && (char_table[ch] & (CH_IDENT|CH_NUMBER)))
 685                                 {
 686                                         push_unichar(ch);
 687                                         ch = get_unichar();
 688                                 }
 689                                 unget_unichar(ch);
 690                                 push_unichar(0);
 691                                 if(!(tok = lookup_token(get_unichar_stack())))
 692                                 {
 693                                         mcy_lval.str = xunistrdup(get_unichar_stack());
 694                                         return tIDENT;
 695                                 }
 696                                 switch(tok->type)
 697                                 {
 698                                 case tok_keyword:
 699                                         return tok->token;
 700
 701                                 case tok_language:
 702                                         codepage = tok->codepage;
 703                                         /* Fall through */
 704                                 case tok_severity:
 705                                 case tok_facility:
 706                                         mcy_lval.tok = tok;
 707                                         return tTOKEN;
 708
 709                                 default:
 710                                         internal_error(__FILE__, __LINE__, "Invalid token type encountered\n");
 711                                 }
 712                         }
 713
 714                         if(isspace(ch)) /* Ignore space */
 715                                 continue;
 716
 717                         if(isdigit(ch))
 718                                 return scan_number(ch);
 719                 }
 720
 721                 switch(ch)
 722                 {
 723                 case ':':
 724                 case '=':
 725                 case '+':
 726                 case '(':
 727                 case ')':
 728                         return ch;
 729                 case ';':
 730                         while(ch != '\n' && ch != EOF)
 731                         {
 732                                 push_unichar(ch);
 733                                 ch = get_unichar();
 734                         }
 735                         newline();
 736                         push_unichar(ch);       /* Include the newline */
 737                         push_unichar(0);
 738                         mcy_lval.str = xunistrdup(get_unichar_stack());
 739                         return tCOMMENT;
 740                 default:
 741                         xyyerror("Invalid character '%c' (0x%04x)\n", isisochar(ch) && isprint(ch) ? ch : '.', ch);
 742                 }
 743         }
 744 }