src/preproc/eqn/lex.cpp

   1 // -*- C++ -*-
   2 /* Copyright (C) 1989, 1990, 1991, 1992, 2000, 2001, 2002, 2003, 2005, 2007,
   3                  2008
   4    Free Software Foundation, Inc.
   5      Written by James Clark (jjc@jclark.com)
   6
   7 This file is part of groff.
   8
   9 groff is free software; you can redistribute it and/or modify it under
  10 the terms of the GNU General Public License as published by the Free
  11 Software Foundation; either version 2, or (at your option) any later
  12 version.
  13
  14 groff is distributed in the hope that it will be useful, but WITHOUT ANY
  15 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  16 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  17 for more details.
  18
  19 You should have received a copy of the GNU General Public License along
  20 with groff; see the file COPYING.  If not, write to the Free Software
  21 Foundation, 51 Franklin St - Fifth Floor, Boston, MA 02110-1301, USA. */
  22
  23 #include "eqn.h"
  24 #include "eqn_tab.h"
  25 #include "stringclass.h"
  26 #include "ptable.h"
  27 #include "file_case.h"
  28
  29
  30 // declarations to avoid friend name injection problems
  31 int get_char();
  32 int peek_char();
  33 int get_location(char **, int *);
  34
  35 struct definition {
  36   char is_macro;
  37   char is_simple;
  38   union {
  39     int tok;
  40     char *contents;
  41   };
  42   definition();
  43   ~definition();
  44 };
  45
  46 definition::definition() : is_macro(1), is_simple(0)
  47 {
  48   contents = 0;
  49 }
  50
  51 definition::~definition()
  52 {
  53   if (is_macro)
  54     a_delete contents;
  55 }
  56
  57 declare_ptable(definition)
  58 implement_ptable(definition)
  59
  60 PTABLE(definition) macro_table;
  61
  62 static struct {
  63   const char *name;
  64   int token;
  65 } token_table[] = {
  66   { "over", OVER },
  67   { "smallover", SMALLOVER },
  68   { "sqrt", SQRT },
  69   { "sub", SUB },
  70   { "sup", SUP },
  71   { "lpile", LPILE },
  72   { "rpile", RPILE },
  73   { "cpile", CPILE },
  74   { "pile", PILE },
  75   { "left", LEFT },
  76   { "right", RIGHT },
  77   { "to", TO },
  78   { "from", FROM },
  79   { "size", SIZE },
  80   { "font", FONT },
  81   { "roman", ROMAN },
  82   { "bold", BOLD },
  83   { "italic", ITALIC },
  84   { "fat", FAT },
  85   { "bar", BAR },
  86   { "under", UNDER },
  87   { "accent", ACCENT },
  88   { "uaccent", UACCENT },
  89   { "above", ABOVE },
  90   { "fwd", FWD },
  91   { "back", BACK },
  92   { "down", DOWN },
  93   { "up", UP },
  94   { "matrix", MATRIX },
  95   { "col", COL },
  96   { "lcol", LCOL },
  97   { "rcol", RCOL },
  98   { "ccol", CCOL },
  99   { "mark", MARK },
 100   { "lineup", LINEUP },
 101   { "space", SPACE },
 102   { "gfont", GFONT },
 103   { "gsize", GSIZE },
 104   { "define", DEFINE },
 105   { "sdefine", SDEFINE },
 106   { "ndefine", NDEFINE },
 107   { "tdefine", TDEFINE },
 108   { "undef", UNDEF },
 109   { "ifdef", IFDEF },
 110   { "include", INCLUDE },
 111   { "copy", INCLUDE },
 112   { "delim", DELIM },
 113   { "chartype", CHARTYPE },
 114   { "type", TYPE },
 115   { "vcenter", VCENTER },
 116   { "set", SET },
 117   { "opprime", PRIME },
 118   { "grfont", GRFONT },
 119   { "gbfont", GBFONT },
 120   { "split", SPLIT },
 121   { "nosplit", NOSPLIT },
 122   { "special", SPECIAL },
 123 };
 124
 125 struct builtin_def {
 126   const char *name;
 127   const char *def;
 128 };
 129
 130 static struct builtin_def common_defs[] = {
 131   { "ALPHA", "\\(*A" },
 132   { "BETA", "\\(*B" },
 133   { "CHI", "\\(*X" },
 134   { "DELTA", "\\(*D" },
 135   { "EPSILON", "\\(*E" },
 136   { "ETA", "\\(*Y" },
 137   { "GAMMA", "\\(*G" },
 138   { "IOTA", "\\(*I" },
 139   { "KAPPA", "\\(*K" },
 140   { "LAMBDA", "\\(*L" },
 141   { "MU", "\\(*M" },
 142   { "NU", "\\(*N" },
 143   { "OMEGA", "\\(*W" },
 144   { "OMICRON", "\\(*O" },
 145   { "PHI", "\\(*F" },
 146   { "PI", "\\(*P" },
 147   { "PSI", "\\(*Q" },
 148   { "RHO", "\\(*R" },
 149   { "SIGMA", "\\(*S" },
 150   { "TAU", "\\(*T" },
 151   { "THETA", "\\(*H" },
 152   { "UPSILON", "\\(*U" },
 153   { "XI", "\\(*C" },
 154   { "ZETA", "\\(*Z" },
 155   { "Alpha", "\\(*A" },
 156   { "Beta", "\\(*B" },
 157   { "Chi", "\\(*X" },
 158   { "Delta", "\\(*D" },
 159   { "Epsilon", "\\(*E" },
 160   { "Eta", "\\(*Y" },
 161   { "Gamma", "\\(*G" },
 162   { "Iota", "\\(*I" },
 163   { "Kappa", "\\(*K" },
 164   { "Lambda", "\\(*L" },
 165   { "Mu", "\\(*M" },
 166   { "Nu", "\\(*N" },
 167   { "Omega", "\\(*W" },
 168   { "Omicron", "\\(*O" },
 169   { "Phi", "\\(*F" },
 170   { "Pi", "\\(*P" },
 171   { "Psi", "\\(*Q" },
 172   { "Rho", "\\(*R" },
 173   { "Sigma", "\\(*S" },
 174   { "Tau", "\\(*T" },
 175   { "Theta", "\\(*H" },
 176   { "Upsilon", "\\(*U" },
 177   { "Xi", "\\(*C" },
 178   { "Zeta", "\\(*Z" },
 179   { "alpha", "\\(*a" },
 180   { "beta", "\\(*b" },
 181   { "chi", "\\(*x" },
 182   { "delta", "\\(*d" },
 183   { "epsilon", "\\(*e" },
 184   { "eta", "\\(*y" },
 185   { "gamma", "\\(*g" },
 186   { "iota", "\\(*i" },
 187   { "kappa", "\\(*k" },
 188   { "lambda", "\\(*l" },
 189   { "mu", "\\(*m" },
 190   { "nu", "\\(*n" },
 191   { "omega", "\\(*w" },
 192   { "omicron", "\\(*o" },
 193   { "phi", "\\(*f" },
 194   { "pi", "\\(*p" },
 195   { "psi", "\\(*q" },
 196   { "rho", "\\(*r" },
 197   { "sigma", "\\(*s" },
 198   { "tau", "\\(*t" },
 199   { "theta", "\\(*h" },
 200   { "upsilon", "\\(*u" },
 201   { "xi", "\\(*c" },
 202   { "zeta", "\\(*z" },
 203   { "max", "{type \"operator\" roman \"max\"}" },
 204   { "min", "{type \"operator\" roman \"min\"}" },
 205   { "lim", "{type \"operator\" roman \"lim\"}" },
 206   { "sin", "{type \"operator\" roman \"sin\"}" },
 207   { "cos", "{type \"operator\" roman \"cos\"}" },
 208   { "tan", "{type \"operator\" roman \"tan\"}" },
 209   { "sinh", "{type \"operator\" roman \"sinh\"}" },
 210   { "cosh", "{type \"operator\" roman \"cosh\"}" },
 211   { "tanh", "{type \"operator\" roman \"tanh\"}" },
 212   { "arc", "{type \"operator\" roman \"arc\"}" },
 213   { "log", "{type \"operator\" roman \"log\"}" },
 214   { "ln", "{type \"operator\" roman \"ln\"}" },
 215   { "exp", "{type \"operator\" roman \"exp\"}" },
 216   { "Re", "{type \"operator\" roman \"Re\"}" },
 217   { "Im", "{type \"operator\" roman \"Im\"}" },
 218   { "det", "{type \"operator\" roman \"det\"}" },
 219   { "and", "{roman \"and\"}" },
 220   { "if", "{roman \"if\"}" },
 221   { "for", "{roman \"for\"}" },
 222   { "times", "type \"binary\" \\(mu" },
 223   { "ldots", "type \"inner\" { . . . }" },
 224   { "inf", "\\(if" },
 225   { "partial", "\\(pd" },
 226   { "nothing", "\"\"" },
 227   { "half", "{1 smallover 2}" },
 228   { "hat_def", "roman \"^\"" },
 229   { "hat", "accent { hat_def }" },
 230   { "tilde_def", "\"~\"" },
 231   { "tilde", "accent { tilde_def }" },
 232   { "==", "type \"relation\" \\(==" },
 233   { "!=", "type \"relation\" \\(!=" },
 234   { "+-", "type \"binary\" \\(+-" },
 235   { "->", "type \"relation\" \\(->" },
 236   { "<-", "type \"relation\" \\(<-" },
 237   { "<<", "type \"relation\" \\(<<" },
 238   { ">>", "type \"relation\" \\(>>" },
 239   { "prime", "'" },
 240   { "approx", "type \"relation\" \"\\(~=\"" },
 241   { "grad", "\\(gr" },
 242   { "del", "\\(gr" },
 243   { "cdot", "type \"binary\" \\(md" },
 244   { "cdots", "type \"inner\" { \\(md \\(md \\(md }" },
 245   { "dollar", "$" },
 246 };
 247
 248 /* composite definitions that require troff size and motion operators */
 249 static struct builtin_def troff_defs[] = {
 250   { "sum", "{type \"operator\" vcenter size +5 \\(*S}" },
 251   { "prod", "{type \"operator\" vcenter size +5 \\(*P}" },
 252   { "int", "{type \"operator\" vcenter size +8 \\(is}" },
 253   { "union", "{type \"operator\" vcenter size +5 \\(cu}" },
 254   { "inter", "{type \"operator\" vcenter size +5 \\(ca}" },
 255   { "dot_def", "up 52 back 15 \".\"" },
 256   { "dot", "accent { dot_def }" },
 257   { "dotdot_def", "up 52 back 25 \"..\"" },
 258   { "dotdot", "accent { dotdot_def }" },
 259   { "utilde_def", "down 75 \"~\"" },
 260   { "utilde", "uaccent { utilde_def }" },
 261   { "vec_def", "up 52 size -5 \\(->" },
 262   { "vec", "accent { vec_def }" },
 263   { "dyad_def", "up 52 size -5 { \\(<> }" },
 264   { "dyad", "accent { dyad_def }" },
 265   { "...", "type \"inner\" vcenter { . . . }" },
 266 };
 267
 268 /* equivalent definitions for MathML mode */
 269 static struct builtin_def mathml_defs[] = {
 270   { "sum", "{type \"operator\" size big \\(*S}" },
 271   { "prod", "{type \"operator\" size big \\(*P}" },
 272   { "int", "{type \"operator\" size big \\(is}" },
 273   { "union", "{type \"operator\" size big \\(cu}" },
 274   { "inter", "{type \"operator\" size big \\(ca}" },
 275   { "dot", "accent { \".\" }" },
 276   { "dotdot", "accent { \"..\" }" },
 277   { "utilde", "uaccent { \"~\" }" },
 278   { "vec", "accent { \\(-> }" },
 279   { "dyad", "accent { \\(<> }" },
 280   { "...", "type \"inner\" { . . . }" },
 281 };
 282
 283 void init_table(const char *device)
 284 {
 285   unsigned int i;
 286   for (i = 0; i < sizeof(token_table)/sizeof(token_table[0]); i++) {
 287     definition *def = new definition[1];
 288     def->is_macro = 0;
 289     def->tok = token_table[i].token;
 290     macro_table.define(token_table[i].name, def);
 291   }
 292   for (i = 0; i < sizeof(common_defs)/sizeof(common_defs[0]); i++) {
 293     definition *def = new definition[1];
 294     def->is_macro = 1;
 295     def->contents = strsave(common_defs[i].def);
 296     def->is_simple = 1;
 297     macro_table.define(common_defs[i].name, def);
 298   }
 299   if (output_format == troff) {
 300     for (i = 0; i < sizeof(troff_defs)/sizeof(troff_defs[0]); i++) {
 301       definition *def = new definition[1];
 302       def->is_macro = 1;
 303       def->contents = strsave(troff_defs[i].def);
 304       def->is_simple = 1;
 305       macro_table.define(troff_defs[i].name, def);
 306     }
 307   }
 308   else if (output_format == mathml) {
 309     for (i = 0; i < sizeof(mathml_defs)/sizeof(mathml_defs[0]); i++) {
 310       definition *def = new definition[1];
 311       def->is_macro = 1;
 312       def->contents = strsave(mathml_defs[i].def);
 313       def->is_simple = 1;
 314       macro_table.define(mathml_defs[i].name, def);
 315     }
 316   }
 317   definition *def = new definition[1];
 318   def->is_macro = 1;
 319   def->contents = strsave("1");
 320   macro_table.define(device, def);
 321 }
 322
 323 class input {
 324   input *next;
 325 public:
 326   input(input *p);
 327   virtual ~input();
 328   virtual int get() = 0;
 329   virtual int peek() = 0;
 330   virtual int get_location(char **, int *);
 331
 332   friend int get_char();
 333   friend int peek_char();
 334   friend int get_location(char **, int *);
 335   friend void init_lex(const char *str, const char *filename, int lineno);
 336 };
 337
 338 class file_input : public input {
 339   file_case *_fcp;
 340   char *filename;
 341   int lineno;
 342   string line;
 343   const char *ptr;
 344   int read_line();
 345 public:
 346   file_input(file_case *, const char *, input *);
 347   ~file_input();
 348   int get();
 349   int peek();
 350   int get_location(char **, int *);
 351 };
 352
 353
 354 class macro_input : public input {
 355   char *s;
 356   char *p;
 357 public:
 358   macro_input(const char *, input *);
 359   ~macro_input();
 360   int get();
 361   int peek();
 362 };
 363
 364 class top_input : public macro_input {
 365   char *filename;
 366   int lineno;
 367  public:
 368   top_input(const char *, const char *, int, input *);
 369   ~top_input();
 370   int get();
 371   int get_location(char **, int *);
 372 };
 373
 374 class argument_macro_input: public input {
 375   char *s;
 376   char *p;
 377   char *ap;
 378   int argc;
 379   char *argv[9];
 380 public:
 381   argument_macro_input(const char *, int, char **, input *);
 382   ~argument_macro_input();
 383   int get();
 384   int peek();
 385 };
 386
 387 input::input(input *x) : next(x)
 388 {
 389 }
 390
 391 input::~input()
 392 {
 393 }
 394
 395 int input::get_location(char **, int *)
 396 {
 397   return 0;
 398 }
 399
 400 file_input::file_input(file_case *fcp, const char *fn, input *p)
 401 : input(p), _fcp(fcp), lineno(0), ptr("")
 402 {
 403   filename = strsave(fn);
 404 }
 405
 406 file_input::~file_input()
 407 {
 408   a_delete filename;
 409   delete _fcp;
 410 }
 411
 412 int file_input::read_line()
 413 {
 414   for (;;) {
 415     line.clear();
 416     lineno++;
 417     for (;;) {
 418       int c = _fcp->get_c();
 419       if (c == EOF)
 420         break;
 421       else if (invalid_input_char(c))
 422         lex_error("invalid input character code %1", c);
 423       else {
 424         line += char(c);
 425         if (c == '\n')
 426           break;
 427       }
 428     }
 429     if (line.length() == 0)
 430       return 0;
 431     if (!(line.length() >= 3 && line[0] == '.' && line[1] == 'E'
 432           && (line[2] == 'Q' || line[2] == 'N')
 433           && (line.length() == 3 || line[3] == ' ' || line[3] == '\n'
 434               || compatible_flag))) {
 435       line += '\0';
 436       ptr = line.contents();
 437       return 1;
 438     }
 439   }
 440 }
 441
 442 int file_input::get()
 443 {
 444   if (*ptr != '\0' || read_line())
 445     return *ptr++ & 0377;
 446   else
 447     return EOF;
 448 }
 449
 450 int file_input::peek()
 451 {
 452   if (*ptr != '\0' || read_line())
 453     return *ptr;
 454   else
 455     return EOF;
 456 }
 457
 458 int file_input::get_location(char **fnp, int *lnp)
 459 {
 460   *fnp = filename;
 461   *lnp = lineno;
 462   return 1;
 463 }
 464
 465 macro_input::macro_input(const char *str, input *x) : input(x)
 466 {
 467   p = s = strsave(str);
 468 }
 469
 470 macro_input::~macro_input()
 471 {
 472   a_delete s;
 473 }
 474
 475 int macro_input::get()
 476 {
 477   if (p == 0 || *p == '\0')
 478     return EOF;
 479   else
 480     return *p++ & 0377;
 481 }
 482
 483 int macro_input::peek()
 484 {
 485   if (p == 0 || *p == '\0')
 486     return EOF;
 487   else
 488     return *p & 0377;
 489 }
 490
 491 top_input::top_input(const char *str, const char *fn, int ln, input *x)
 492 : macro_input(str, x), lineno(ln)
 493 {
 494   filename = strsave(fn);
 495 }
 496
 497 top_input::~top_input()
 498 {
 499   a_delete filename;
 500 }
 501
 502 int top_input::get()
 503 {
 504   int c = macro_input::get();
 505   if (c == '\n')
 506     lineno++;
 507   return c;
 508 }
 509
 510 int top_input::get_location(char **fnp, int *lnp)
 511 {
 512   *fnp = filename;
 513   *lnp = lineno;
 514   return 1;
 515 }
 516
 517 // Character representing $1.  Must be invalid input character.
 518 #define ARG1 14
 519
 520 argument_macro_input::argument_macro_input(const char *body, int ac,
 521                                            char **av, input *x)
 522 : input(x), ap(0), argc(ac)
 523 {
 524   int i;
 525   for (i = 0; i < argc; i++)
 526     argv[i] = av[i];
 527   p = s = strsave(body);
 528   int j = 0;
 529   for (i = 0; s[i] != '\0'; i++)
 530     if (s[i] == '$' && s[i+1] >= '0' && s[i+1] <= '9') {
 531       if (s[i+1] != '0')
 532         s[j++] = ARG1 + s[++i] - '1';
 533     }
 534     else
 535       s[j++] = s[i];
 536   s[j] = '\0';
 537 }
 538
 539
 540 argument_macro_input::~argument_macro_input()
 541 {
 542   for (int i = 0; i < argc; i++)
 543     a_delete argv[i];
 544   a_delete s;
 545 }
 546
 547 int argument_macro_input::get()
 548 {
 549   if (ap) {
 550     if (*ap != '\0')
 551       return *ap++ & 0377;
 552     ap = 0;
 553   }
 554   if (p == 0)
 555     return EOF;
 556   while (*p >= ARG1 && *p <= ARG1 + 8) {
 557     int i = *p++ - ARG1;
 558     if (i < argc && argv[i] != 0 && argv[i][0] != '\0') {
 559       ap = argv[i];
 560       return *ap++ & 0377;
 561     }
 562   }
 563   if (*p == '\0')
 564     return EOF;
 565   return *p++ & 0377;
 566 }
 567
 568 int argument_macro_input::peek()
 569 {
 570   if (ap) {
 571     if (*ap != '\0')
 572       return *ap & 0377;
 573     ap = 0;
 574   }
 575   if (p == 0)
 576     return EOF;
 577   while (*p >= ARG1 && *p <= ARG1 + 8) {
 578     int i = *p++ - ARG1;
 579     if (i < argc && argv[i] != 0 && argv[i][0] != '\0') {
 580       ap = argv[i];
 581       return *ap & 0377;
 582     }
 583   }
 584   if (*p == '\0')
 585     return EOF;
 586   return *p & 0377;
 587 }
 588
 589 static input *current_input = 0;
 590
 591 /* we insert a newline between input from different levels */
 592
 593 int get_char()
 594 {
 595   if (current_input == 0)
 596     return EOF;
 597   else {
 598     int c = current_input->get();
 599     if (c != EOF)
 600       return c;
 601     else {
 602       input *tem = current_input;
 603       current_input = current_input->next;
 604       delete tem;
 605       return '\n';
 606     }
 607   }
 608 }
 609
 610 int peek_char()
 611 {
 612   if (current_input == 0)
 613     return EOF;
 614   else {
 615     int c = current_input->peek();
 616     if (c != EOF)
 617       return c;
 618     else
 619       return '\n';
 620   }
 621 }
 622
 623 int get_location(char **fnp, int *lnp)
 624 {
 625   for (input *p = current_input; p; p = p->next)
 626     if (p->get_location(fnp, lnp))
 627       return 1;
 628   return 0;
 629 }
 630
 631 string token_buffer;
 632 const int NCONTEXT = 4;
 633 string context_ring[NCONTEXT];
 634 int context_index = 0;
 635
 636 void flush_context()
 637 {
 638   for (int i = 0; i < NCONTEXT; i++)
 639     context_ring[i] = "";
 640   context_index = 0;
 641 }
 642
 643 void show_context()
 644 {
 645   int i = context_index;
 646   fputs(" context is\n\t", stderr);
 647   for (;;) {
 648     int j = (i + 1) % NCONTEXT;
 649     if (j == context_index) {
 650       fputs(">>> ", stderr);
 651       put_string(context_ring[i], stderr);
 652       fputs(" <<<", stderr);
 653       break;
 654     }
 655     else if (context_ring[i].length() > 0) {
 656       put_string(context_ring[i], stderr);
 657       putc(' ', stderr);
 658     }
 659     i = j;
 660   }
 661   putc('\n', stderr);
 662 }
 663
 664 void add_context(const string &s)
 665 {
 666   context_ring[context_index] = s;
 667   context_index = (context_index + 1) % NCONTEXT;
 668 }
 669
 670 void add_context(char c)
 671 {
 672   context_ring[context_index] = c;
 673   context_index = (context_index + 1) % NCONTEXT;
 674 }
 675
 676 void add_quoted_context(const string &s)
 677 {
 678   string &r = context_ring[context_index];
 679   r = '"';
 680   for (int i = 0; i < s.length(); i++)
 681     if (s[i] == '"')
 682       r += "\\\"";
 683     else
 684       r += s[i];
 685   r += '"';
 686   context_index = (context_index + 1) % NCONTEXT;
 687 }
 688
 689 void init_lex(const char *str, const char *filename, int lineno)
 690 {
 691  while (current_input != 0) {
 692     input *tem = current_input;
 693     current_input = current_input->next;
 694     delete tem;
 695   }
 696   current_input = new top_input(str, filename, lineno, 0);
 697   flush_context();
 698 }
 699
 700
 701 void get_delimited_text()
 702 {
 703   char *filename;
 704   int lineno;
 705   int got_location = get_location(&filename, &lineno);
 706   int start = get_char();
 707   while (start == ' ' || start == '\t' || start == '\n')
 708     start = get_char();
 709   token_buffer.clear();
 710   if (start == EOF) {
 711     if (got_location)
 712       error_with_file_and_line(filename, lineno,
 713                                "end of input while defining macro");
 714     else
 715       error("end of input while defining macro");
 716     return;
 717   }
 718   for (;;) {
 719     int c = get_char();
 720     if (c == EOF) {
 721       if (got_location)
 722         error_with_file_and_line(filename, lineno,
 723                                  "end of input while defining macro");
 724       else
 725         error("end of input while defining macro");
 726       add_context(start + token_buffer);
 727       return;
 728     }
 729     if (c == start)
 730       break;
 731     token_buffer += char(c);
 732   }
 733   add_context(start + token_buffer + start);
 734 }
 735
 736 void interpolate_macro_with_args(const char *body)
 737 {
 738   char *argv[9];
 739   int argc = 0;
 740   int i;
 741   for (i = 0; i < 9; i++)
 742     argv[i] = 0;
 743   int level = 0;
 744   int c;
 745   do {
 746     token_buffer.clear();
 747     for (;;) {
 748       c = get_char();
 749       if (c == EOF) {
 750         lex_error("end of input while scanning macro arguments");
 751         break;
 752       }
 753       if (level == 0 && (c == ',' || c == ')')) {
 754         if (token_buffer.length() > 0) {
 755           token_buffer +=  '\0';
 756           argv[argc] = strsave(token_buffer.contents());
 757         }
 758         // for `foo()', argc = 0
 759         if (argc > 0 || c != ')' || i > 0)
 760           argc++;
 761         break;
 762       }
 763       token_buffer += char(c);
 764       if (c == '(')
 765         level++;
 766       else if (c == ')')
 767         level--;
 768     }
 769   } while (c != ')' && c != EOF);
 770   current_input = new argument_macro_input(body, argc, argv, current_input);
 771 }
 772
 773 /* If lookup flag is non-zero the token will be looked up to see
 774 if it is macro. If it's 1, it will looked up to see if it's a token.
 775 */
 776
 777 int get_token(int lookup_flag = 0)
 778 {
 779   for (;;) {
 780     int c = get_char();
 781     while (c == ' ' || c == '\n')
 782       c = get_char();
 783     switch (c) {
 784     case EOF:
 785       {
 786         add_context("end of input");
 787       }
 788       return 0;
 789     case '"':
 790       {
 791         int quoted = 0;
 792         token_buffer.clear();
 793         for (;;) {
 794           c = get_char();
 795           if (c == EOF) {
 796             lex_error("missing \"");
 797             break;
 798           }
 799           else if (c == '\n') {
 800             lex_error("newline before end of quoted text");
 801             break;
 802           }
 803           else if (c == '"') {
 804             if (!quoted)
 805               break;
 806             token_buffer[token_buffer.length() - 1] = '"';
 807             quoted = 0;
 808           }
 809           else {
 810             token_buffer += c;
 811             quoted = quoted ? 0 : c == '\\';
 812           }
 813         }
 814       }
 815       add_quoted_context(token_buffer);
 816       return QUOTED_TEXT;
 817     case '{':
 818     case '}':
 819     case '^':
 820     case '~':
 821     case '\t':
 822       add_context(c);
 823       return c;
 824     default:
 825       {
 826         int break_flag = 0;
 827         int quoted = 0;
 828         token_buffer.clear();
 829         if (c == '\\')
 830           quoted = 1;
 831         else
 832           token_buffer += c;
 833         int done = 0;
 834         while (!done) {
 835           c = peek_char();
 836           if (!quoted && lookup_flag != 0 && c == '(') {
 837             token_buffer += '\0';
 838             definition *def = macro_table.lookup(token_buffer.contents());
 839             if (def && def->is_macro && !def->is_simple) {
 840               (void)get_char(); // skip initial '('
 841               interpolate_macro_with_args(def->contents);
 842               break_flag = 1;
 843               break;
 844             }
 845             token_buffer.set_length(token_buffer.length() - 1);
 846           }
 847           if (quoted) {
 848             quoted = 0;
 849             switch (c) {
 850             case EOF:
 851               lex_error("`\\' ignored at end of equation");
 852               done = 1;
 853               break;
 854             case '\n':
 855               lex_error("`\\' ignored because followed by newline");
 856               done = 1;
 857               break;
 858             case '\t':
 859               lex_error("`\\' ignored because followed by tab");
 860               done = 1;
 861               break;
 862             case '"':
 863               (void)get_char();
 864               token_buffer += '"';
 865               break;
 866             default:
 867               (void)get_char();
 868               token_buffer += '\\';
 869               token_buffer += c;
 870               break;
 871             }
 872           }
 873           else {
 874             switch (c) {
 875             case EOF:
 876             case '{':
 877             case '}':
 878             case '^':
 879             case '~':
 880             case '"':
 881             case ' ':
 882             case '\t':
 883             case '\n':
 884               done = 1;
 885               break;
 886             case '\\':
 887               (void)get_char();
 888               quoted = 1;
 889               break;
 890             default:
 891               (void)get_char();
 892               token_buffer += char(c);
 893               break;
 894             }
 895           }
 896         }
 897         if (break_flag || token_buffer.length() == 0)
 898           break;
 899         if (lookup_flag != 0) {
 900           token_buffer += '\0';
 901           definition *def = macro_table.lookup(token_buffer.contents());
 902           token_buffer.set_length(token_buffer.length() - 1);
 903           if (def) {
 904             if (def->is_macro) {
 905               current_input = new macro_input(def->contents, current_input);
 906               break;
 907             }
 908             else if (lookup_flag == 1) {
 909               add_context(token_buffer);
 910               return def->tok;
 911             }
 912           }
 913         }
 914         add_context(token_buffer);
 915         return TEXT;
 916       }
 917     }
 918   }
 919 }
 920
 921 void do_include()
 922 {
 923   int t = get_token(2);
 924   if (t != TEXT && t != QUOTED_TEXT) {
 925     lex_error("bad filename for include");
 926     return;
 927   }
 928   token_buffer += '\0';
 929   const char *filename = token_buffer.contents();
 930   file_case *fcp = file_case::muxer(filename);
 931   if (fcp == NULL) {
 932     lex_error("can't open included file `%1'", filename);
 933     return;
 934   }
 935   current_input = new file_input(fcp, filename, current_input);
 936 }
 937
 938 void ignore_definition()
 939 {
 940   int t = get_token();
 941   if (t != TEXT) {
 942     lex_error("bad definition");
 943     return;
 944   }
 945   get_delimited_text();
 946 }
 947
 948 void do_definition(int is_simple)
 949 {
 950   int t = get_token();
 951   if (t != TEXT) {
 952     lex_error("bad definition");
 953     return;
 954   }
 955   token_buffer += '\0';
 956   const char *name = token_buffer.contents();
 957   definition *def = macro_table.lookup(name);
 958   if (def == 0) {
 959     def = new definition[1];
 960     macro_table.define(name, def);
 961   }
 962   else if (def->is_macro) {
 963     a_delete def->contents;
 964   }
 965   get_delimited_text();
 966   token_buffer += '\0';
 967   def->is_macro = 1;
 968   def->contents = strsave(token_buffer.contents());
 969   def->is_simple = is_simple;
 970 }
 971
 972 void do_undef()
 973 {
 974   int t = get_token();
 975   if (t != TEXT) {
 976     lex_error("bad undef command");
 977     return;
 978   }
 979   token_buffer += '\0';
 980   macro_table.define(token_buffer.contents(), 0);
 981 }
 982
 983 void do_gsize()
 984 {
 985   int t = get_token(2);
 986   if (t != TEXT && t != QUOTED_TEXT) {
 987     lex_error("bad argument to gsize command");
 988     return;
 989   }
 990   token_buffer += '\0';
 991   if (!set_gsize(token_buffer.contents()))
 992     lex_error("invalid size `%1'", token_buffer.contents());
 993 }
 994
 995 void do_gfont()
 996 {
 997   int t = get_token(2);
 998   if (t != TEXT && t != QUOTED_TEXT) {
 999     lex_error("bad argument to gfont command");
1000     return;
1001   }
1002   token_buffer += '\0';
1003   set_gfont(token_buffer.contents());
1004 }
1005
1006 void do_grfont()
1007 {
1008   int t = get_token(2);
1009   if (t != TEXT && t != QUOTED_TEXT) {
1010     lex_error("bad argument to grfont command");
1011     return;
1012   }
1013   token_buffer += '\0';
1014   set_grfont(token_buffer.contents());
1015 }
1016
1017 void do_gbfont()
1018 {
1019   int t = get_token(2);
1020   if (t != TEXT && t != QUOTED_TEXT) {
1021     lex_error("bad argument to gbfont command");
1022     return;
1023   }
1024   token_buffer += '\0';
1025   set_gbfont(token_buffer.contents());
1026 }
1027
1028 void do_space()
1029 {
1030   int t = get_token(2);
1031   if (t != TEXT && t != QUOTED_TEXT) {
1032     lex_error("bad argument to space command");
1033     return;
1034   }
1035   token_buffer += '\0';
1036   char *ptr;
1037   long n = strtol(token_buffer.contents(), &ptr, 10);
1038   if (n == 0 && ptr == token_buffer.contents())
1039     lex_error("bad argument `%1' to space command", token_buffer.contents());
1040   else
1041     set_space(int(n));
1042 }
1043
1044 void do_ifdef()
1045 {
1046   int t = get_token();
1047   if (t != TEXT) {
1048     lex_error("bad ifdef");
1049     return;
1050   }
1051   token_buffer += '\0';
1052   definition *def = macro_table.lookup(token_buffer.contents());
1053   int result = def && def->is_macro && !def->is_simple;
1054   get_delimited_text();
1055   if (result) {
1056     token_buffer += '\0';
1057     current_input = new macro_input(token_buffer.contents(), current_input);
1058   }
1059 }
1060
1061 void do_delim()
1062 {
1063   int c = get_char();
1064   while (c == ' ' || c == '\n')
1065     c = get_char();
1066   int d;
1067   if (c == EOF || (d = get_char()) == EOF)
1068     lex_error("end of file while reading argument to `delim'");
1069   else {
1070     if (c == 'o' && d == 'f' && peek_char() == 'f') {
1071       (void)get_char();
1072       start_delim = end_delim = '\0';
1073     }
1074     else {
1075       start_delim = c;
1076       end_delim = d;
1077     }
1078   }
1079 }
1080
1081 void do_chartype()
1082 {
1083   int t = get_token(2);
1084   if (t != TEXT && t != QUOTED_TEXT) {
1085     lex_error("bad chartype");
1086     return;
1087   }
1088   token_buffer += '\0';
1089   string type = token_buffer;
1090   t = get_token();
1091   if (t != TEXT && t != QUOTED_TEXT) {
1092     lex_error("bad chartype");
1093     return;
1094   }
1095   token_buffer += '\0';
1096   set_char_type(type.contents(), strsave(token_buffer.contents()));
1097 }
1098
1099 void do_set()
1100 {
1101   int t = get_token(2);
1102   if (t != TEXT && t != QUOTED_TEXT) {
1103     lex_error("bad set");
1104     return;
1105   }
1106   token_buffer += '\0';
1107   string param = token_buffer;
1108   t = get_token();
1109   if (t != TEXT && t != QUOTED_TEXT) {
1110     lex_error("bad set");
1111     return;
1112   }
1113   token_buffer += '\0';
1114   int n;
1115   if (sscanf(&token_buffer[0], "%d", &n) != 1) {
1116     lex_error("bad number `%1'", token_buffer.contents());
1117     return;
1118   }
1119   set_param(param.contents(), n);
1120 }
1121
1122 int yylex()
1123 {
1124   for (;;) {
1125     int tk = get_token(1);
1126     switch(tk) {
1127     case UNDEF:
1128       do_undef();
1129       break;
1130     case SDEFINE:
1131       do_definition(1);
1132       break;
1133     case DEFINE:
1134       do_definition(0);
1135       break;
1136     case TDEFINE:
1137       if (!nroff)
1138         do_definition(0);
1139       else
1140         ignore_definition();
1141       break;
1142     case NDEFINE:
1143       if (nroff)
1144         do_definition(0);
1145       else
1146         ignore_definition();
1147       break;
1148     case GSIZE:
1149       do_gsize();
1150       break;
1151     case GFONT:
1152       do_gfont();
1153       break;
1154     case GRFONT:
1155       do_grfont();
1156       break;
1157     case GBFONT:
1158       do_gbfont();
1159       break;
1160     case SPACE:
1161       do_space();
1162       break;
1163     case INCLUDE:
1164       do_include();
1165       break;
1166     case IFDEF:
1167       do_ifdef();
1168       break;
1169     case DELIM:
1170       do_delim();
1171       break;
1172     case CHARTYPE:
1173       do_chartype();
1174       break;
1175     case SET:
1176       do_set();
1177       break;
1178     case QUOTED_TEXT:
1179     case TEXT:
1180       token_buffer += '\0';
1181       yylval.str = strsave(token_buffer.contents());
1182       // fall through
1183     default:
1184       return tk;
1185     }
1186   }
1187 }
1188
1189 void lex_error(const char *message,
1190                const errarg &arg1,
1191                const errarg &arg2,
1192                const errarg &arg3)
1193 {
1194   char *filename;
1195   int lineno;
1196   if (!get_location(&filename, &lineno))
1197     error(message, arg1, arg2, arg3);
1198   else
1199     error_with_file_and_line(filename, lineno, message, arg1, arg2, arg3);
1200 }
1201
1202 void yyerror(const char *s)
1203 {
1204   char *filename;
1205   int lineno;
1206   if (!get_location(&filename, &lineno))
1207     error(s);
1208   else
1209     error_with_file_and_line(filename, lineno, s);
1210   show_context();
1211 }
1212