src/pre-eqn/lex.cpp

   1 /*@
   2  * Copyright (c) 2014 - 2015 Steffen (Daode) Nurpmeso <sdaoden@users.sf.net>.
   3  *
   4  * Copyright (C) 1989 - 1992, 2000 - 2003, 2005, 2007, 2008
   5  *    Free Software Foundation, Inc.
   6  *      Written by James Clark (jjc@jclark.com)
   7  *
   8  * This is free software; you can redistribute it and/or modify it under
   9  * the terms of the GNU General Public License as published by the Free
  10  * Software Foundation; either version 2, or (at your option) any later
  11  * version.
  12  *
  13  * This is distributed in the hope that it will be useful, but WITHOUT ANY
  14  * WARRANTY; without even the implied warranty of MERCHANTABILITY or
  15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  16  * for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License along
  19  * with groff; see the file COPYING.  If not, write to the Free Software
  20  * Foundation, 51 Franklin St - Fifth Floor, Boston, MA 02110-1301, USA.
  21  */
  22
  23 #include "config.h"
  24 #include "eqn-config.h"
  25
  26 #include "eqn.h"
  27 #include "eqn_tab.h"
  28 #include "file_case.h"
  29 #include "ptable.h"
  30 #include "stringclass.h"
  31
  32 // declarations to avoid friend name injection problems
  33 int get_char();
  34 int peek_char();
  35 int get_location(char **, int *);
  36
  37 class definition
  38 {
  39 public:
  40   char is_macro;
  41   char is_simple;
  42   union {
  43     int tok;
  44     char *contents;
  45   };
  46   definition();
  47   ~definition();
  48 };
  49
  50 definition::definition() : is_macro(1), is_simple(0)
  51 {
  52   contents = 0;
  53 }
  54
  55 definition::~definition()
  56 {
  57   if (is_macro)
  58     a_delete contents;
  59 }
  60
  61 declare_ptable(definition)
  62 implement_ptable(definition)
  63
  64 PTABLE(definition) macro_table;
  65
  66 static struct {
  67   const char *name;
  68   int token;
  69 } token_table[] = { // FIXME const
  70   { "over", OVER },
  71   { "smallover", SMALLOVER },
  72   { "sqrt", SQRT },
  73   { "sub", SUB },
  74   { "sup", SUP },
  75   { "lpile", LPILE },
  76   { "rpile", RPILE },
  77   { "cpile", CPILE },
  78   { "pile", PILE },
  79   { "left", LEFT },
  80   { "right", RIGHT },
  81   { "to", TO },
  82   { "from", FROM },
  83   { "size", SIZE },
  84   { "font", FONT },
  85   { "roman", ROMAN },
  86   { "bold", BOLD },
  87   { "italic", ITALIC },
  88   { "fat", FAT },
  89   { "bar", BAR },
  90   { "under", UNDER },
  91   { "accent", ACCENT },
  92   { "uaccent", UACCENT },
  93   { "above", ABOVE },
  94   { "fwd", FWD },
  95   { "back", BACK },
  96   { "down", DOWN },
  97   { "up", UP },
  98   { "matrix", MATRIX },
  99   { "col", COL },
 100   { "lcol", LCOL },
 101   { "rcol", RCOL },
 102   { "ccol", CCOL },
 103   { "mark", MARK },
 104   { "lineup", LINEUP },
 105   { "space", SPACE },
 106   { "gfont", GFONT },
 107   { "gsize", GSIZE },
 108   { "define", DEFINE },
 109   { "sdefine", SDEFINE },
 110   { "ndefine", NDEFINE },
 111   { "tdefine", TDEFINE },
 112   { "undef", UNDEF },
 113   { "ifdef", IFDEF },
 114   { "include", INCLUDE },
 115   { "copy", INCLUDE },
 116   { "delim", DELIM },
 117   { "chartype", CHARTYPE },
 118   { "type", TYPE },
 119   { "vcenter", VCENTER },
 120   { "set", SET },
 121   { "opprime", PRIME },
 122   { "grfont", GRFONT },
 123   { "gbfont", GBFONT },
 124   { "split", SPLIT },
 125   { "nosplit", NOSPLIT },
 126   { "special", SPECIAL },
 127 };
 128
 129 struct builtin_def {
 130   const char *name;
 131   const char *def;
 132 };
 133
 134 static struct builtin_def common_defs[] = { // FIXME const
 135   { "ALPHA", "\\(*A" },
 136   { "BETA", "\\(*B" },
 137   { "CHI", "\\(*X" },
 138   { "DELTA", "\\(*D" },
 139   { "EPSILON", "\\(*E" },
 140   { "ETA", "\\(*Y" },
 141   { "GAMMA", "\\(*G" },
 142   { "IOTA", "\\(*I" },
 143   { "KAPPA", "\\(*K" },
 144   { "LAMBDA", "\\(*L" },
 145   { "MU", "\\(*M" },
 146   { "NU", "\\(*N" },
 147   { "OMEGA", "\\(*W" },
 148   { "OMICRON", "\\(*O" },
 149   { "PHI", "\\(*F" },
 150   { "PI", "\\(*P" },
 151   { "PSI", "\\(*Q" },
 152   { "RHO", "\\(*R" },
 153   { "SIGMA", "\\(*S" },
 154   { "TAU", "\\(*T" },
 155   { "THETA", "\\(*H" },
 156   { "UPSILON", "\\(*U" },
 157   { "XI", "\\(*C" },
 158   { "ZETA", "\\(*Z" },
 159   { "Alpha", "\\(*A" },
 160   { "Beta", "\\(*B" },
 161   { "Chi", "\\(*X" },
 162   { "Delta", "\\(*D" },
 163   { "Epsilon", "\\(*E" },
 164   { "Eta", "\\(*Y" },
 165   { "Gamma", "\\(*G" },
 166   { "Iota", "\\(*I" },
 167   { "Kappa", "\\(*K" },
 168   { "Lambda", "\\(*L" },
 169   { "Mu", "\\(*M" },
 170   { "Nu", "\\(*N" },
 171   { "Omega", "\\(*W" },
 172   { "Omicron", "\\(*O" },
 173   { "Phi", "\\(*F" },
 174   { "Pi", "\\(*P" },
 175   { "Psi", "\\(*Q" },
 176   { "Rho", "\\(*R" },
 177   { "Sigma", "\\(*S" },
 178   { "Tau", "\\(*T" },
 179   { "Theta", "\\(*H" },
 180   { "Upsilon", "\\(*U" },
 181   { "Xi", "\\(*C" },
 182   { "Zeta", "\\(*Z" },
 183   { "alpha", "\\(*a" },
 184   { "beta", "\\(*b" },
 185   { "chi", "\\(*x" },
 186   { "delta", "\\(*d" },
 187   { "epsilon", "\\(*e" },
 188   { "eta", "\\(*y" },
 189   { "gamma", "\\(*g" },
 190   { "iota", "\\(*i" },
 191   { "kappa", "\\(*k" },
 192   { "lambda", "\\(*l" },
 193   { "mu", "\\(*m" },
 194   { "nu", "\\(*n" },
 195   { "omega", "\\(*w" },
 196   { "omicron", "\\(*o" },
 197   { "phi", "\\(*f" },
 198   { "pi", "\\(*p" },
 199   { "psi", "\\(*q" },
 200   { "rho", "\\(*r" },
 201   { "sigma", "\\(*s" },
 202   { "tau", "\\(*t" },
 203   { "theta", "\\(*h" },
 204   { "upsilon", "\\(*u" },
 205   { "xi", "\\(*c" },
 206   { "zeta", "\\(*z" },
 207   { "max", "{type \"operator\" roman \"max\"}" },
 208   { "min", "{type \"operator\" roman \"min\"}" },
 209   { "lim", "{type \"operator\" roman \"lim\"}" },
 210   { "sin", "{type \"operator\" roman \"sin\"}" },
 211   { "cos", "{type \"operator\" roman \"cos\"}" },
 212   { "tan", "{type \"operator\" roman \"tan\"}" },
 213   { "sinh", "{type \"operator\" roman \"sinh\"}" },
 214   { "cosh", "{type \"operator\" roman \"cosh\"}" },
 215   { "tanh", "{type \"operator\" roman \"tanh\"}" },
 216   { "arc", "{type \"operator\" roman \"arc\"}" },
 217   { "log", "{type \"operator\" roman \"log\"}" },
 218   { "ln", "{type \"operator\" roman \"ln\"}" },
 219   { "exp", "{type \"operator\" roman \"exp\"}" },
 220   { "Re", "{type \"operator\" roman \"Re\"}" },
 221   { "Im", "{type \"operator\" roman \"Im\"}" },
 222   { "det", "{type \"operator\" roman \"det\"}" },
 223   { "and", "{roman \"and\"}" },
 224   { "if", "{roman \"if\"}" },
 225   { "for", "{roman \"for\"}" },
 226   { "times", "type \"binary\" \\(mu" },
 227   { "ldots", "type \"inner\" { . . . }" },
 228   { "inf", "\\(if" },
 229   { "partial", "\\(pd" },
 230   { "nothing", "\"\"" },
 231   { "half", "{1 smallover 2}" },
 232   { "hat_def", "roman \"^\"" },
 233   { "hat", "accent { hat_def }" },
 234   { "tilde_def", "\"~\"" },
 235   { "tilde", "accent { tilde_def }" },
 236   { "==", "type \"relation\" \\(==" },
 237   { "!=", "type \"relation\" \\(!=" },
 238   { "+-", "type \"binary\" \\(+-" },
 239   { "->", "type \"relation\" \\(->" },
 240   { "<-", "type \"relation\" \\(<-" },
 241   { "<<", "type \"relation\" \\(<<" },
 242   { ">>", "type \"relation\" \\(>>" },
 243   { "prime", "'" },
 244   { "approx", "type \"relation\" \"\\(~=\"" },
 245   { "grad", "\\(gr" },
 246   { "del", "\\(gr" },
 247   { "cdot", "type \"binary\" \\(md" },
 248   { "cdots", "type \"inner\" { \\(md \\(md \\(md }" },
 249   { "dollar", "$" },
 250 };
 251
 252 /* composite definitions that require troff size and motion operators */
 253 static struct builtin_def troff_defs[] = { // FIXME const
 254   { "sum", "{type \"operator\" vcenter size +5 \\(*S}" },
 255   { "prod", "{type \"operator\" vcenter size +5 \\(*P}" },
 256   { "int", "{type \"operator\" vcenter size +8 \\(is}" },
 257   { "union", "{type \"operator\" vcenter size +5 \\(cu}" },
 258   { "inter", "{type \"operator\" vcenter size +5 \\(ca}" },
 259   { "dot_def", "up 52 back 15 \".\"" },
 260   { "dot", "accent { dot_def }" },
 261   { "dotdot_def", "up 52 back 25 \"..\"" },
 262   { "dotdot", "accent { dotdot_def }" },
 263   { "utilde_def", "down 75 \"~\"" },
 264   { "utilde", "uaccent { utilde_def }" },
 265   { "vec_def", "up 52 size -5 \\(->" },
 266   { "vec", "accent { vec_def }" },
 267   { "dyad_def", "up 52 size -5 { \\(<> }" },
 268   { "dyad", "accent { dyad_def }" },
 269   { "...", "type \"inner\" vcenter { . . . }" },
 270 };
 271
 272 /* equivalent definitions for MathML mode */
 273 static struct builtin_def mathml_defs[] = { // FIXME const
 274   { "sum", "{type \"operator\" size big \\(*S}" },
 275   { "prod", "{type \"operator\" size big \\(*P}" },
 276   { "int", "{type \"operator\" size big \\(is}" },
 277   { "union", "{type \"operator\" size big \\(cu}" },
 278   { "inter", "{type \"operator\" size big \\(ca}" },
 279   { "dot", "accent { \".\" }" },
 280   { "dotdot", "accent { \"..\" }" },
 281   { "utilde", "uaccent { \"~\" }" },
 282   { "vec", "accent { \\(-> }" },
 283   { "dyad", "accent { \\(<> }" },
 284   { "...", "type \"inner\" { . . . }" },
 285 };
 286
 287 void init_table(const char *device)
 288 {
 289   unsigned int i;
 290   for (i = 0; i < NELEM(token_table); i++) {
 291     definition *def = new definition[1];
 292     def->is_macro = 0;
 293     def->tok = token_table[i].token;
 294     macro_table.define(token_table[i].name, def);
 295   }
 296   for (i = 0; i < NELEM(common_defs); i++) {
 297     definition *def = new definition[1];
 298     def->is_macro = 1;
 299     def->contents = strsave(common_defs[i].def);
 300     def->is_simple = 1;
 301     macro_table.define(common_defs[i].name, def);
 302   }
 303   if (output_format == troff) {
 304     for (i = 0; i < NELEM(troff_defs); i++) {
 305       definition *def = new definition[1];
 306       def->is_macro = 1;
 307       def->contents = strsave(troff_defs[i].def);
 308       def->is_simple = 1;
 309       macro_table.define(troff_defs[i].name, def);
 310     }
 311   }
 312   else if (output_format == mathml) {
 313     for (i = 0; i < NELEM(mathml_defs); i++) {
 314       definition *def = new definition[1];
 315       def->is_macro = 1;
 316       def->contents = strsave(mathml_defs[i].def);
 317       def->is_simple = 1;
 318       macro_table.define(mathml_defs[i].name, def);
 319     }
 320   }
 321   definition *def = new definition[1];
 322   def->is_macro = 1;
 323   def->contents = strsave("1");
 324   macro_table.define(device, def);
 325 }
 326
 327 class input
 328 {
 329   friend int get_char();
 330   friend int peek_char();
 331   friend int get_location(char **, int *);
 332   friend void init_lex(const char *str, const char *filename, int lineno);
 333
 334   input *next;
 335
 336 public:
 337   input(input *p);
 338   virtual ~input();
 339   virtual int get() = 0;
 340   virtual int peek() = 0;
 341   virtual int get_location(char **, int *);
 342 };
 343
 344 class file_input
 345 : public input
 346 {
 347   file_case *_fcp;
 348   char *filename;
 349   int lineno;
 350   string line;
 351   const char *ptr;
 352
 353   int read_line();
 354
 355 public:
 356   file_input(file_case *, const char *, input *);
 357   ~file_input();
 358   int get();
 359   int peek();
 360   int get_location(char **, int *);
 361 };
 362
 363 class macro_input
 364 : public input
 365 {
 366   char *s;
 367   char *p;
 368
 369 public:
 370   macro_input(const char *, input *);
 371   ~macro_input();
 372   int get();
 373   int peek();
 374 };
 375
 376 class top_input
 377 : public macro_input
 378 {
 379   char *filename;
 380   int lineno;
 381
 382 public:
 383   top_input(const char *, const char *, int, input *);
 384   ~top_input();
 385   int get();
 386   int get_location(char **, int *);
 387 };
 388
 389 class argument_macro_input
 390 : public input
 391 {
 392   char *s;
 393   char *p;
 394   char *ap;
 395   int argc;
 396   char *argv[9];
 397
 398 public:
 399   argument_macro_input(const char *, int, char **, input *);
 400   ~argument_macro_input();
 401   int get();
 402   int peek();
 403 };
 404
 405 input::input(input *x) : next(x)
 406 {
 407 }
 408
 409 input::~input()
 410 {
 411 }
 412
 413 int input::get_location(char **, int *)
 414 {
 415   return 0;
 416 }
 417
 418 file_input::file_input(file_case *fcp, const char *fn, input *p)
 419 : input(p), _fcp(fcp), lineno(0), ptr("")
 420 {
 421   filename = strsave(fn);
 422 }
 423
 424 file_input::~file_input()
 425 {
 426   a_delete filename;
 427   delete _fcp;
 428 }
 429
 430 int file_input::read_line() /* TODO lib-roff */
 431 {
 432   for (;;) {
 433     line.clear();
 434     lineno++;
 435     for (;;) {
 436       int c = _fcp->get_c();
 437       if (c == '\r' && (c = _fcp->get_c()) != '\n')
 438         lex_error("invalid input character CR (carriage return)");
 439       if (c == EOF)
 440         break;
 441       else if (invalid_input_char(c))
 442         lex_error("invalid input character code %1", c);
 443       else {
 444         line += char(c);
 445         if (c == '\n')
 446           break;
 447       }
 448     }
 449     if (line.length() == 0)
 450       return 0;
 451     if (!(line.length() >= 3 && line[0] == '.' && line[1] == 'E'
 452           && (line[2] == 'Q' || line[2] == 'N')
 453           && (line.length() == 3 || line[3] == ' ' || line[3] == '\n'
 454               || compatible_flag))) {
 455       line += '\0';
 456       ptr = line.contents();
 457       return 1;
 458     }
 459   }
 460 }
 461
 462 int file_input::get()
 463 {
 464   if (*ptr != '\0' || read_line())
 465     return *ptr++ & 0377;
 466   else
 467     return EOF;
 468 }
 469
 470 int file_input::peek()
 471 {
 472   if (*ptr != '\0' || read_line())
 473     return *ptr;
 474   else
 475     return EOF;
 476 }
 477
 478 int file_input::get_location(char **fnp, int *lnp)
 479 {
 480   *fnp = filename;
 481   *lnp = lineno;
 482   return 1;
 483 }
 484
 485 macro_input::macro_input(const char *str, input *x) : input(x)
 486 {
 487   p = s = strsave(str);
 488 }
 489
 490 macro_input::~macro_input()
 491 {
 492   a_delete s;
 493 }
 494
 495 int macro_input::get()
 496 {
 497   if (p == 0 || *p == '\0')
 498     return EOF;
 499   else
 500     return *p++ & 0377;
 501 }
 502
 503 int macro_input::peek()
 504 {
 505   if (p == 0 || *p == '\0')
 506     return EOF;
 507   else
 508     return *p & 0377;
 509 }
 510
 511 top_input::top_input(const char *str, const char *fn, int ln, input *x)
 512 : macro_input(str, x), lineno(ln)
 513 {
 514   filename = strsave(fn);
 515 }
 516
 517 top_input::~top_input()
 518 {
 519   a_delete filename;
 520 }
 521
 522 int top_input::get()
 523 {
 524   int c = macro_input::get();
 525   if (c == '\n')
 526     lineno++;
 527   return c;
 528 }
 529
 530 int top_input::get_location(char **fnp, int *lnp)
 531 {
 532   *fnp = filename;
 533   *lnp = lineno;
 534   return 1;
 535 }
 536
 537 argument_macro_input::argument_macro_input(const char *body, int ac,
 538                                            char **av, input *x)
 539 : input(x), ap(0), argc(ac)
 540 {
 541   int i;
 542   for (i = 0; i < argc; i++)
 543     argv[i] = av[i];
 544   p = s = strsave(body);
 545   int j = 0;
 546   for (i = 0; s[i] != '\0'; i++)
 547     if (s[i] == '$' && s[i+1] >= '0' && s[i+1] <= '9') {
 548       if (s[i+1] != '0')
 549         s[j++] = ARG1 + s[++i] - '1';
 550     }
 551     else
 552       s[j++] = s[i];
 553   s[j] = '\0';
 554 }
 555
 556 argument_macro_input::~argument_macro_input()
 557 {
 558   for (int i = 0; i < argc; i++)
 559     a_delete argv[i];
 560   a_delete s;
 561 }
 562
 563 int argument_macro_input::get()
 564 {
 565   if (ap) {
 566     if (*ap != '\0')
 567       return *ap++ & 0377;
 568     ap = 0;
 569   }
 570   if (p == 0)
 571     return EOF;
 572   while (*p >= ARG1 && *p <= ARG1 + 8) {
 573     int i = *p++ - ARG1;
 574     if (i < argc && argv[i] != 0 && argv[i][0] != '\0') {
 575       ap = argv[i];
 576       return *ap++ & 0377;
 577     }
 578   }
 579   if (*p == '\0')
 580     return EOF;
 581   return *p++ & 0377;
 582 }
 583
 584 int argument_macro_input::peek()
 585 {
 586   if (ap) {
 587     if (*ap != '\0')
 588       return *ap & 0377;
 589     ap = 0;
 590   }
 591   if (p == 0)
 592     return EOF;
 593   while (*p >= ARG1 && *p <= ARG1 + 8) {
 594     int i = *p++ - ARG1;
 595     if (i < argc && argv[i] != 0 && argv[i][0] != '\0') {
 596       ap = argv[i];
 597       return *ap & 0377;
 598     }
 599   }
 600   if (*p == '\0')
 601     return EOF;
 602   return *p & 0377;
 603 }
 604
 605 static input *current_input = 0;
 606
 607 /* we insert a newline between input from different levels */
 608
 609 int get_char()
 610 {
 611   if (current_input == 0)
 612     return EOF;
 613   else {
 614     int c = current_input->get();
 615     if (c != EOF)
 616       return c;
 617     else {
 618       input *tem = current_input;
 619       current_input = current_input->next;
 620       delete tem;
 621       return '\n';
 622     }
 623   }
 624 }
 625
 626 int peek_char()
 627 {
 628   if (current_input == 0)
 629     return EOF;
 630   else {
 631     int c = current_input->peek();
 632     if (c != EOF)
 633       return c;
 634     else
 635       return '\n';
 636   }
 637 }
 638
 639 int get_location(char **fnp, int *lnp)
 640 {
 641   for (input *p = current_input; p; p = p->next)
 642     if (p->get_location(fnp, lnp))
 643       return 1;
 644   return 0;
 645 }
 646
 647 string token_buffer;
 648 const int NCONTEXT = 4;
 649 string context_ring[NCONTEXT];
 650 int context_index = 0;
 651
 652 void flush_context()
 653 {
 654   for (int i = 0; i < NCONTEXT; i++)
 655     context_ring[i] = "";
 656   context_index = 0;
 657 }
 658
 659 void show_context()
 660 {
 661   int i = context_index;
 662   fputs(" context is\n\t", stderr);
 663   for (;;) {
 664     int j = (i + 1) % NCONTEXT;
 665     if (j == context_index) {
 666       fputs(">>> ", stderr);
 667       put_string(context_ring[i], stderr);
 668       fputs(" <<<", stderr);
 669       break;
 670     }
 671     else if (context_ring[i].length() > 0) {
 672       put_string(context_ring[i], stderr);
 673       putc(' ', stderr);
 674     }
 675     i = j;
 676   }
 677   putc('\n', stderr);
 678 }
 679
 680 void add_context(const string &s)
 681 {
 682   context_ring[context_index] = s;
 683   context_index = (context_index + 1) % NCONTEXT;
 684 }
 685
 686 void add_context(char c)
 687 {
 688   context_ring[context_index] = c;
 689   context_index = (context_index + 1) % NCONTEXT;
 690 }
 691
 692 void add_quoted_context(const string &s)
 693 {
 694   string &r = context_ring[context_index];
 695   r = '"';
 696   for (int i = 0; i < s.length(); i++)
 697     if (s[i] == '"')
 698       r += "\\\"";
 699     else
 700       r += s[i];
 701   r += '"';
 702   context_index = (context_index + 1) % NCONTEXT;
 703 }
 704
 705 void init_lex(const char *str, const char *filename, int lineno)
 706 {
 707  while (current_input != 0) {
 708     input *tem = current_input;
 709     current_input = current_input->next;
 710     delete tem;
 711   }
 712   current_input = new top_input(str, filename, lineno, 0);
 713   flush_context();
 714 }
 715
 716 void get_delimited_text()
 717 {
 718   char *filename;
 719   int lineno;
 720   int got_location = get_location(&filename, &lineno);
 721   int start = get_char();
 722   while (start == ' ' || start == '\t' || start == '\n')
 723     start = get_char();
 724   token_buffer.clear();
 725   if (start == EOF) {
 726     if (got_location)
 727       error_with_file_and_line(filename, lineno,
 728                                "end of input while defining macro");
 729     else
 730       error("end of input while defining macro");
 731     return;
 732   }
 733   for (;;) {
 734     int c = get_char();
 735     if (c == EOF) {
 736       if (got_location)
 737         error_with_file_and_line(filename, lineno,
 738                                  "end of input while defining macro");
 739       else
 740         error("end of input while defining macro");
 741       add_context(start + token_buffer);
 742       return;
 743     }
 744     if (c == start)
 745       break;
 746     token_buffer += char(c);
 747   }
 748   add_context(start + token_buffer + start);
 749 }
 750
 751 void interpolate_macro_with_args(const char *body)
 752 {
 753   char *argv[9];
 754   int argc = 0;
 755   int i;
 756   for (i = 0; i < 9; i++)
 757     argv[i] = 0;
 758   int level = 0;
 759   int c;
 760   do {
 761     token_buffer.clear();
 762     for (;;) {
 763       c = get_char();
 764       if (c == EOF) {
 765         lex_error("end of input while scanning macro arguments");
 766         break;
 767       }
 768       if (level == 0 && (c == ',' || c == ')')) {
 769         if (token_buffer.length() > 0) {
 770           token_buffer +=  '\0';
 771           argv[argc] = strsave(token_buffer.contents());
 772         }
 773         // for `foo()', argc = 0
 774         if (argc > 0 || c != ')' || i > 0)
 775           argc++;
 776         break;
 777       }
 778       token_buffer += char(c);
 779       if (c == '(')
 780         level++;
 781       else if (c == ')')
 782         level--;
 783     }
 784   } while (c != ')' && c != EOF);
 785   current_input = new argument_macro_input(body, argc, argv, current_input);
 786 }
 787
 788 /* If lookup flag is non-zero the token will be looked up to see
 789 if it is macro. If it's 1, it will looked up to see if it's a token.
 790 */
 791
 792 int get_token(int lookup_flag = 0)
 793 {
 794   for (;;) {
 795     int c = get_char();
 796     while (c == ' ' || c == '\n')
 797       c = get_char();
 798     switch (c) {
 799     case EOF:
 800       {
 801         add_context("end of input");
 802       }
 803       return 0;
 804     case '"':
 805       {
 806         int quoted = 0;
 807         token_buffer.clear();
 808         for (;;) {
 809           c = get_char();
 810           if (c == EOF) {
 811             lex_error("missing \"");
 812             break;
 813           }
 814           else if (c == '\n') {
 815             lex_error("newline before end of quoted text");
 816             break;
 817           }
 818           else if (c == '"') {
 819             if (!quoted)
 820               break;
 821             token_buffer[token_buffer.length() - 1] = '"';
 822             quoted = 0;
 823           }
 824           else {
 825             token_buffer += c;
 826             quoted = quoted ? 0 : c == '\\';
 827           }
 828         }
 829       }
 830       add_quoted_context(token_buffer);
 831       return QUOTED_TEXT;
 832     case '{':
 833     case '}':
 834     case '^':
 835     case '~':
 836     case '\t':
 837       add_context(c);
 838       return c;
 839     default:
 840       {
 841         int break_flag = 0;
 842         int quoted = 0;
 843         token_buffer.clear();
 844         if (c == '\\')
 845           quoted = 1;
 846         else
 847           token_buffer += c;
 848         int done = 0;
 849         while (!done) {
 850           c = peek_char();
 851           if (!quoted && lookup_flag != 0 && c == '(') {
 852             token_buffer += '\0';
 853             definition *def = macro_table.lookup(token_buffer.contents());
 854             if (def && def->is_macro && !def->is_simple) {
 855               (void)get_char(); // skip initial '('
 856               interpolate_macro_with_args(def->contents);
 857               break_flag = 1;
 858               break;
 859             }
 860             token_buffer.set_length(token_buffer.length() - 1);
 861           }
 862           if (quoted) {
 863             quoted = 0;
 864             switch (c) {
 865             case EOF:
 866               lex_error("`\\' ignored at end of equation");
 867               done = 1;
 868               break;
 869             case '\n':
 870               lex_error("`\\' ignored because followed by newline");
 871               done = 1;
 872               break;
 873             case '\t':
 874               lex_error("`\\' ignored because followed by tab");
 875               done = 1;
 876               break;
 877             case '"':
 878               (void)get_char();
 879               token_buffer += '"';
 880               break;
 881             default:
 882               (void)get_char();
 883               token_buffer += '\\';
 884               token_buffer += c;
 885               break;
 886             }
 887           }
 888           else {
 889             switch (c) {
 890             case EOF:
 891             case '{':
 892             case '}':
 893             case '^':
 894             case '~':
 895             case '"':
 896             case ' ':
 897             case '\t':
 898             case '\n':
 899               done = 1;
 900               break;
 901             case '\\':
 902               (void)get_char();
 903               quoted = 1;
 904               break;
 905             default:
 906               (void)get_char();
 907               token_buffer += char(c);
 908               break;
 909             }
 910           }
 911         }
 912         if (break_flag || token_buffer.length() == 0)
 913           break;
 914         if (lookup_flag != 0) {
 915           token_buffer += '\0';
 916           definition *def = macro_table.lookup(token_buffer.contents());
 917           token_buffer.set_length(token_buffer.length() - 1);
 918           if (def) {
 919             if (def->is_macro) {
 920               current_input = new macro_input(def->contents, current_input);
 921               break;
 922             }
 923             else if (lookup_flag == 1) {
 924               add_context(token_buffer);
 925               return def->tok;
 926             }
 927           }
 928         }
 929         add_context(token_buffer);
 930         return TEXT;
 931       }
 932     }
 933   }
 934 }
 935
 936 void do_include()
 937 {
 938   int t = get_token(2);
 939   if (t != TEXT && t != QUOTED_TEXT) {
 940     lex_error("bad filename for include");
 941     return;
 942   }
 943   token_buffer += '\0';
 944   const char *filename = token_buffer.contents();
 945   file_case *fcp = file_case::muxer(filename);
 946   if (fcp == NULL) {
 947     lex_error("can't open included file `%1'", filename);
 948     return;
 949   }
 950   current_input = new file_input(fcp, filename, current_input);
 951 }
 952
 953 void ignore_definition()
 954 {
 955   int t = get_token();
 956   if (t != TEXT) {
 957     lex_error("bad definition");
 958     return;
 959   }
 960   get_delimited_text();
 961 }
 962
 963 void do_definition(int is_simple)
 964 {
 965   int t = get_token();
 966   if (t != TEXT) {
 967     lex_error("bad definition");
 968     return;
 969   }
 970   token_buffer += '\0';
 971   const char *name = token_buffer.contents();
 972   definition *def = macro_table.lookup(name);
 973   if (def == 0) {
 974     def = new definition[1];
 975     macro_table.define(name, def);
 976   }
 977   else if (def->is_macro) {
 978     a_delete def->contents;
 979   }
 980   get_delimited_text();
 981   token_buffer += '\0';
 982   def->is_macro = 1;
 983   def->contents = strsave(token_buffer.contents());
 984   def->is_simple = is_simple;
 985 }
 986
 987 void do_undef()
 988 {
 989   int t = get_token();
 990   if (t != TEXT) {
 991     lex_error("bad undef command");
 992     return;
 993   }
 994   token_buffer += '\0';
 995   macro_table.define(token_buffer.contents(), 0);
 996 }
 997
 998 void do_gsize()
 999 {
1000   int t = get_token(2);
1001   if (t != TEXT && t != QUOTED_TEXT) {
1002     lex_error("bad argument to gsize command");
1003     return;
1004   }
1005   token_buffer += '\0';
1006   if (!set_gsize(token_buffer.contents()))
1007     lex_error("invalid size `%1'", token_buffer.contents());
1008 }
1009
1010 void do_gfont()
1011 {
1012   int t = get_token(2);
1013   if (t != TEXT && t != QUOTED_TEXT) {
1014     lex_error("bad argument to gfont command");
1015     return;
1016   }
1017   token_buffer += '\0';
1018   set_gfont(token_buffer.contents());
1019 }
1020
1021 void do_grfont()
1022 {
1023   int t = get_token(2);
1024   if (t != TEXT && t != QUOTED_TEXT) {
1025     lex_error("bad argument to grfont command");
1026     return;
1027   }
1028   token_buffer += '\0';
1029   set_grfont(token_buffer.contents());
1030 }
1031
1032 void do_gbfont()
1033 {
1034   int t = get_token(2);
1035   if (t != TEXT && t != QUOTED_TEXT) {
1036     lex_error("bad argument to gbfont command");
1037     return;
1038   }
1039   token_buffer += '\0';
1040   set_gbfont(token_buffer.contents());
1041 }
1042
1043 void do_space()
1044 {
1045   int t = get_token(2);
1046   if (t != TEXT && t != QUOTED_TEXT) {
1047     lex_error("bad argument to space command");
1048     return;
1049   }
1050   token_buffer += '\0';
1051   char *ptr;
1052   long n = strtol(token_buffer.contents(), &ptr, 10);
1053   if (n == 0 && ptr == token_buffer.contents())
1054     lex_error("bad argument `%1' to space command", token_buffer.contents());
1055   else
1056     set_space(int(n));
1057 }
1058
1059 void do_ifdef()
1060 {
1061   int t = get_token();
1062   if (t != TEXT) {
1063     lex_error("bad ifdef");
1064     return;
1065   }
1066   token_buffer += '\0';
1067   definition *def = macro_table.lookup(token_buffer.contents());
1068   int result = def && def->is_macro && !def->is_simple;
1069   get_delimited_text();
1070   if (result) {
1071     token_buffer += '\0';
1072     current_input = new macro_input(token_buffer.contents(), current_input);
1073   }
1074 }
1075
1076 void do_delim()
1077 {
1078   int c = get_char();
1079   while (c == ' ' || c == '\n')
1080     c = get_char();
1081   int d;
1082   if (c == EOF || (d = get_char()) == EOF)
1083     lex_error("end of file while reading argument to `delim'");
1084   else {
1085     if (c == 'o' && d == 'f' && peek_char() == 'f') {
1086       (void)get_char();
1087       start_delim = end_delim = '\0';
1088     }
1089     else {
1090       start_delim = c;
1091       end_delim = d;
1092     }
1093   }
1094 }
1095
1096 void do_chartype()
1097 {
1098   int t = get_token(2);
1099   if (t != TEXT && t != QUOTED_TEXT) {
1100     lex_error("bad chartype");
1101     return;
1102   }
1103   token_buffer += '\0';
1104   string type = token_buffer;
1105   t = get_token();
1106   if (t != TEXT && t != QUOTED_TEXT) {
1107     lex_error("bad chartype");
1108     return;
1109   }
1110   token_buffer += '\0';
1111   set_char_type(type.contents(), strsave(token_buffer.contents()));
1112 }
1113
1114 void do_set()
1115 {
1116   int t = get_token(2);
1117   if (t != TEXT && t != QUOTED_TEXT) {
1118     lex_error("bad set");
1119     return;
1120   }
1121   token_buffer += '\0';
1122   string param = token_buffer;
1123   t = get_token();
1124   if (t != TEXT && t != QUOTED_TEXT) {
1125     lex_error("bad set");
1126     return;
1127   }
1128   token_buffer += '\0';
1129   int n;
1130   if (sscanf(&token_buffer[0], "%d", &n) != 1) {
1131     lex_error("bad number `%1'", token_buffer.contents());
1132     return;
1133   }
1134   set_param(param.contents(), n);
1135 }
1136
1137 int yylex()
1138 {
1139   for (;;) {
1140     int tk = get_token(1);
1141     switch(tk) {
1142     case UNDEF:
1143       do_undef();
1144       break;
1145     case SDEFINE:
1146       do_definition(1);
1147       break;
1148     case DEFINE:
1149       do_definition(0);
1150       break;
1151     case TDEFINE:
1152       if (!nroff)
1153         do_definition(0);
1154       else
1155         ignore_definition();
1156       break;
1157     case NDEFINE:
1158       if (nroff)
1159         do_definition(0);
1160       else
1161         ignore_definition();
1162       break;
1163     case GSIZE:
1164       do_gsize();
1165       break;
1166     case GFONT:
1167       do_gfont();
1168       break;
1169     case GRFONT:
1170       do_grfont();
1171       break;
1172     case GBFONT:
1173       do_gbfont();
1174       break;
1175     case SPACE:
1176       do_space();
1177       break;
1178     case INCLUDE:
1179       do_include();
1180       break;
1181     case IFDEF:
1182       do_ifdef();
1183       break;
1184     case DELIM:
1185       do_delim();
1186       break;
1187     case CHARTYPE:
1188       do_chartype();
1189       break;
1190     case SET:
1191       do_set();
1192       break;
1193     case QUOTED_TEXT:
1194     case TEXT:
1195       token_buffer += '\0';
1196       yylval.str = strsave(token_buffer.contents());
1197       // fall through
1198     default:
1199       return tk;
1200     }
1201   }
1202 }
1203
1204 void lex_error(const char *message,
1205                const errarg &arg1,
1206                const errarg &arg2,
1207                const errarg &arg3)
1208 {
1209   char *filename;
1210   int lineno;
1211   if (!get_location(&filename, &lineno))
1212     error(message, arg1, arg2, arg3);
1213   else
1214     error_with_file_and_line(filename, lineno, message, arg1, arg2, arg3);
1215 }
1216
1217 void yyerror(const char *s)
1218 {
1219   char *filename;
1220   int lineno;
1221   if (!get_location(&filename, &lineno))
1222     error(s);
1223   else
1224     error_with_file_and_line(filename, lineno, s);
1225   show_context();
1226 }
1227
1228 // s-it2-mode