src/scan.c

   1 /*
   2  * Copyright 1993-2002 Christopher Seiwald and Perforce Software, Inc.
   3  *
   4  * This file is part of Jam - see jam.c for Copyright information.
   5  */
   6 /*
   7  * scan.c - the jam yacc scanner
   8  *
   9  * 12/26/93 (seiwald) - bump buf in yylex to 10240 - yuk.
  10  * 09/16/94 (seiwald) - check for overflows, unmatched {}'s, etc.
  11  *                      Also handle tokens abutting EOF by remembering
  12  *                      to return EOF now matter how many times yylex()
  13  *                      reinvokes yyline().
  14  * 02/11/95 (seiwald) - honor only punctuation keywords if SCAN_PUNCT.
  15  * 07/27/95 (seiwald) - Include jamgram.h after scan.h, so that YYSTYPE is
  16  *                      defined before Linux's yacc tries to redefine it.
  17  * 01/10/01 (seiwald) - \ can now escape any whitespace char
  18  * 11/04/02 (seiwald) - const-ing for string literals
  19  */
  20 #include "jam.h"
  21 #include "lists.h"
  22 #include "parse.h"
  23 #include "scan.h"
  24 #include "jamgram.h"
  25 #include "jambase.h"
  26 #include "newstr.h"
  27
  28
  29 const struct keyword {
  30   const char *word;
  31   int type;
  32 } keywords[] = {
  33 #include "jamgramtab.h"
  34   {0,0}
  35 };
  36
  37
  38 struct include {
  39   struct include *next; /* next serial include file */
  40   const char *string; /* pointer into current line */
  41   char **strings; /* for yyfparse() -- text to parse */
  42   FILE *file; /* for yyfparse() -- file being read */
  43   const char *fname; /* for yyfparse() -- file name */
  44   int line; /* line counter for error messages */
  45   char buf[512]; /* for yyfparse() -- line buffer */
  46 };
  47
  48 static struct include *incp = 0; /* current file; head of chain */
  49
  50 static int scanmode = SCAN_NORMAL;
  51 static int anyerrors = 0;
  52 static char *symdump (YYSTYPE *s);
  53
  54 /* no single token can be larger */
  55 #define BIGGEST_TOKEN  (10240)
  56
  57
  58 /*
  59  * Set parser mode: normal, string, or keyword
  60  */
  61 void yymode (int n) {
  62   scanmode = n;
  63 }
  64
  65
  66 void yyerror (const char *s) {
  67   if (incp) printf("%s: line %d: ", incp->fname, incp->line);
  68   printf("%s at %s\n", s, symdump(&yylval));
  69   ++anyerrors;
  70 }
  71
  72
  73 int yyanyerrors (void) {
  74   return anyerrors != 0;
  75 }
  76
  77
  78 void yyfparse (const char *s) {
  79   struct include *i = (struct include *)malloc(sizeof(*i));
  80   /* push this onto the incp chain */
  81   i->string = "";
  82   i->strings = 0;
  83   i->file = 0;
  84   i->fname = copystr(s);
  85   i->line = 0;
  86   i->next = incp;
  87   incp = i;
  88   /* if the filename is "::Jambase", it means use the internal jambase */
  89   if (!strcmp(s, "::Jambase")) {
  90     jambaseUnpack();
  91     i->strings = jambase;
  92   }
  93 }
  94
  95
  96 /*
  97  * yyline() - read new line and return first character
  98  *
  99  * fabricates a continuous stream of characters across include files, returning EOF at the bitter end
 100  */
 101 int yyline (void) {
 102   struct include *i = incp;
 103   //
 104   if (!incp) return EOF;
 105   /* once we start reading from the input stream, we reset the
 106    * include insertion point so that the next include file becomes
 107    * the head of the list */
 108   /* if there is more data in this line, return it */
 109   if (*i->string) return *i->string++;
 110   /* if we're reading from an internal string list, go to the next string */
 111   if (i->strings) {
 112     if (!*i->strings) goto next;
 113     ++i->line;
 114     i->string = *(i->strings++);
 115     return *i->string++;
 116   }
 117   /* if necessary, open the file */
 118   if (!i->file) {
 119     FILE *f = stdin;
 120     if (strcmp(i->fname, "-") && !(f = fopen(i->fname, "r"))) perror(i->fname);
 121     i->file = f;
 122   }
 123   /* if there's another line in this file, start it */
 124   if (i->file && fgets(i->buf, sizeof(i->buf), i->file)) {
 125     ++i->line;
 126     i->string = i->buf;
 127     return *i->string++;
 128   }
 129 next:
 130   /* this include is done */
 131   /* free it up and return EOF so yyparse() returns to parse_file() */
 132   incp = i->next;
 133   /* close file, free name */
 134   if (i->file && i->file != stdin) fclose(i->file);
 135   freestr(i->fname);
 136   free((char *)i);
 137   return EOF;
 138 }
 139
 140
 141 /*
 142  * yylex() - set yylval to current token; return its type
 143  *
 144  * Macros to move things along:
 145  *
 146  *  yychar() - return and advance character; invalid after EOF
 147  *  yyprev() - back up one character; invalid before yychar()
 148  *
 149  * yychar() returns a continuous stream of characters, until it hits
 150  * the EOF of the current include file.
 151  */
 152 #define yychar()  (*incp->string ? *incp->string++ : yyline())
 153 #define yyprev()  (incp->string--)
 154
 155
 156 /* eat white space */
 157 static int skipSpaces (int c) {
 158   for (;;) {
 159     /* skip past white space */
 160     while (c != EOF && isspace(c)) c = yychar();
 161     /* not a comment? swallow up comment line */
 162     if (c != '#') break;
 163     while ((c = yychar()) != EOF && c != '\n') ;
 164   }
 165   return c;
 166 }
 167
 168
 169 static int digit (int c, int base) {
 170   if (c == EOF) return -1;
 171   if (c >= 'a' && c <= 'z') c -= 32;
 172   if (c < '0' || (c > '9' && c < 'A') || c > 'Z') return -1;
 173   c -= '0';
 174   if (c > 9) c -= 7;
 175   if (c >= base) return -1;
 176   return c;
 177 }
 178
 179
 180 int yylex (void) {
 181   int c;
 182   char buf[BIGGEST_TOKEN];
 183   char *b = buf;
 184   //
 185   if (!incp) goto eof;
 186   /* get first character (whitespace or of token) */
 187   c = yychar();
 188   if (scanmode == SCAN_STRING) {
 189     /* if scanning for a string (action's {}'s), look for the closing brace */
 190     /* we handle matching braces, if they match! */
 191     int nest = 1;
 192     //
 193     while (c != EOF && b < buf+sizeof(buf)) {
 194       if (c == '{') ++nest;
 195       if (c == '}' && !--nest) break;
 196       *b++ = c;
 197       c = yychar();
 198     }
 199     /* we ate the ending brace -- regurgitate it */
 200     if (c != EOF) yyprev();
 201     /* check obvious errors */
 202     if (b == buf+sizeof(buf)) { yyerror("action block too big"); goto eof; }
 203     if (nest) { yyerror("unmatched {} in action block"); goto eof; }
 204     *b = 0;
 205     yylval.type = STRING;
 206     yylval.string = newstr(buf);
 207   } else {
 208     char *b = buf;
 209     const struct keyword *k;
 210     int inquote = 0, notkeyword = 0, n, d, wasNotAlNum = 0;
 211     //
 212     c = skipSpaces(c);
 213     /* c now points to the first character of a token */
 214     if (c == EOF) goto eof;
 215     //printf(":'%c'\n", c);
 216 #if 0
 217     if (!isalpha(c) && c != '$' && c != '_' && c != '"' && c != '\'') {
 218       const struct keyword *kgood = NULL;
 219       /* special chars are delimiters */
 220       while (c != EOF) {
 221         *b++ = c;
 222         *b = 0;
 223         for (k = keywords; k->word != NULL; ++k) if (!isalpha(k->word[0]) && strcmp(buf, k->word) == 0) break;
 224         if (k->word != NULL) {
 225           /* good keyword */
 226           kgood = k;
 227           c = yychar();
 228           continue;
 229         }
 230         /* bad keyword */
 231         break;
 232       }
 233       if (c != EOF) --b; /* remove last char from token buffer */
 234       if (kgood != NULL) {
 235         /* ok, we got it */
 236         printf("![%s]\n", buf);
 237         goto lexdoneback;
 238       }
 239     }
 240     /* bad luck, try it another way */
 241 #else
 242     /* while scanning the word, disqualify it for (expensive)
 243      * keyword lookup when we can: $anything, "anything", \anything */
 244     notkeyword = (c == '$');
 245     if (c == '{' || c == '}' || c == ';' || c == '[' || c == ']') {
 246       *b++ = c;
 247       goto lexdone;
 248     }
 249     if (c == ':') {
 250       /* only ':abc' is good, ':*' is not */
 251       c = yychar();
 252       if (c == EOF || isspace(c) || isalnum(c) || c == '$' || c == '_') {
 253         *b++ = ':';
 254         goto lexdoneback;
 255       }
 256     }
 257 #endif
 258     /* look for white space to delimit word */
 259     /* "'s get stripped but preserve white space */
 260     /* \ protects next character */
 261     while (c != EOF && b < buf+sizeof(buf) && (inquote || !isspace(c))) {
 262       if (c == '"') {
 263         /* begin or end " */
 264         inquote = !inquote;
 265         notkeyword = 1;
 266       } else if (!inquote && (c == '{' || c == '}' || c == ';')) {
 267         /* k8: allow specials to work as delimiters */
 268         break;
 269       } else if (!inquote && !notkeyword && (c == '[' || c == ']')) {
 270         /* k8: allow specials to work as delimiters */
 271         break;
 272       } else if (!inquote && !notkeyword && !wasNotAlNum && c == ':') {
 273         /* k8: allow specials to work as delimiters; '*:' is not good */
 274         /**b = 0; printf("***OUT [%s]! %d\n", buf, incp?incp->line:0);*/
 275         break;
 276       } else if (c != '\\') {
 277         /* normal char */
 278         if (!isalnum(c)) wasNotAlNum = 1;
 279         *b++ = c;
 280       } else if ((c = yychar()) != EOF) {
 281         /* \c */
 282         wasNotAlNum = 1;
 283         if (inquote) {
 284           switch (c) {
 285             case 't': *b++ = '\t'; break;
 286             case 'n': *b++ = '\n'; break;
 287             case 'r': *b++ = '\r'; break;
 288             case 'v': *b++ = '\v'; break;
 289             case 'b': *b++ = '\b'; break;
 290             case 'a': *b++ = '\a'; break;
 291             case 'f': *b++ = '\f'; break;
 292             case 'e': *b++ = '\x1b'; break;
 293             case 'x':
 294               c = yychar(); // first digit
 295               n = digit(c, 16);
 296               if (n < 0) { yyerror("invalid hex escape in quoted string"); goto eof; }
 297               c = yychar(); // second digit
 298               d = digit(c, 16);
 299               if (d < 0) { if (c != EOF) yyprev(); } else n = (n*16)+d;
 300               if (n == 0) { yyerror("invalid hex escape in quoted string"); goto eof; }
 301               *b++ = n;
 302               break;
 303             default: *b++ = c; break;
 304           }
 305         } else {
 306           *b++ = c;
 307         }
 308         notkeyword = 1;
 309       } else {
 310         /* \EOF */
 311         break;
 312       }
 313       c = yychar();
 314     }
 315     /* we looked ahead a character - back up */
 316 lexdoneback:
 317     if (c != EOF) yyprev();
 318 lexdone:
 319     /* check obvious errors */
 320     if (b == buf+sizeof(buf)) { yyerror("string too big"); goto eof; }
 321     if (inquote) { yyerror("unmatched \" in string"); goto eof; }
 322     /* scan token table */
 323     /* don't scan if it's obviously not a keyword or if its */
 324     /* an alphabetic when were looking for punctuation */
 325     *b = 0;
 326     yylval.type = ARG;
 327     if (!notkeyword && !(isalpha(*buf) && scanmode == SCAN_PUNCT)) {
 328       for (k = keywords; k->word; ++k) {
 329         if (*buf == *k->word && strcmp(k->word, buf) == 0) {
 330           yylval.type = k->type;
 331           yylval.string = k->word; /* used by symdump */
 332           break;
 333         }
 334       }
 335     }
 336     if (yylval.type == ARG) yylval.string = newstr(buf);
 337   }
 338   if (DEBUG_SCAN) printf("scan %s\n", symdump(&yylval));
 339   return yylval.type;
 340 eof:
 341   yylval.type = EOF;
 342   return yylval.type;
 343 }
 344
 345
 346 static char *symdump (YYSTYPE *s) {
 347   static char buf[BIGGEST_TOKEN+20];
 348   //
 349   switch (s->type) {
 350     case EOF: sprintf(buf, "EOF"); break;
 351     case 0: sprintf(buf, "unknown symbol %s", s->string); break;
 352     case ARG: sprintf(buf, "argument %s", s->string); break;
 353     case STRING: sprintf(buf, "string \"%s\"", s->string); break;
 354     default: sprintf(buf, "keyword %s", s->string); break;
 355   }
 356   return buf;
 357 }