src/util/scanner.h

   1
   2 #ifndef EL__UTIL_SCANNER_H
   3 #define EL__UTIL_SCANNER_H
   4
   5 #include "util/error.h"
   6
   7 /* Define if you want a talking scanner */
   8 /* #define DEBUG_SCANNER */
   9
  10 /* The {struct scanner_token} describes one scanner state. There are two kinds
  11  * of tokens: char and non-char tokens. Char tokens contains only one char and
  12  * simply have their char value as type. They are tokens having special control
  13  * meaning in the code, like ':', ';', '{', '}' and '*'. Non char tokens has
  14  * one or more chars and contain stuff like number or indentifier strings.  */
  15 struct scanner_token {
  16         /* The type the token */
  17         int type;
  18
  19         /* Some precedence value */
  20         int precedence;
  21
  22         /* The start of the token string and the token length */
  23         unsigned char *string;
  24         int length;
  25 };
  26
  27 /* The naming of these two macros is a bit odd .. we compare often with
  28  * "static" strings (I don't have a better word) so the macro name should
  29  * be short. --jonas */
  30
  31 /* Compare the string of @token with @string */
  32 #define scanner_token_strlcasecmp(token, str, len) \
  33         ((token) && !strlcasecmp((token)->string, (token)->length, str, len))
  34
  35 /* Also compares the token string but using a "static" string */
  36 #define scanner_token_contains(token, str) \
  37         scanner_token_strlcasecmp(token, str, sizeof(str) - 1)
  38
  39
  40 struct scan_table_info {
  41         enum { SCAN_RANGE, SCAN_STRING, SCAN_END } type;
  42         union scan_table_data {
  43                 struct { unsigned char *source; long length; } string;
  44                 struct { unsigned char *start; long end; } range;
  45         } data;
  46         int bits;
  47 };
  48
  49 #define SCAN_TABLE_SIZE 256
  50
  51 #define SCAN_TABLE_INFO(type, data1, data2, bits) \
  52         { (type), { { (data1), (data2) } }, (bits) }
  53
  54 #define SCAN_TABLE_RANGE(from, to, bits) SCAN_TABLE_INFO(SCAN_RANGE, from, to, bits)
  55 #define SCAN_TABLE_STRING(str, bits)     SCAN_TABLE_INFO(SCAN_STRING, str, sizeof(str) - 1, bits)
  56 #define SCAN_TABLE_END                   SCAN_TABLE_INFO(SCAN_END, 0, 0, 0)
  57
  58 struct scanner_string_mapping {
  59         unsigned char *name;
  60         int type;
  61         int base_type;
  62 };
  63
  64 struct scanner;
  65
  66 struct scanner_info {
  67         /* Table containing how to map strings to token types */
  68         const struct scanner_string_mapping *mappings;
  69
  70         /* Information for how to initialize the scanner table */
  71         const struct scan_table_info *scan_table_info;
  72
  73         /* Fills the scanner with tokens. Already scanned tokens which have not
  74          * been requested remain and are moved to the start of the scanners
  75          * token table. */
  76         /* Returns the current token or NULL if there are none. */
  77         struct scanner_token *(*scan)(struct scanner *scanner);
  78
  79         /* The scanner table */
  80         /* Contains bitmaps for the various characters groups.
  81          * Idea sync'ed from mozilla browser. */
  82         int scan_table[SCAN_TABLE_SIZE];
  83
  84         /* Has the scanner info been initialized? */
  85         unsigned int initialized:1;
  86 };
  87
  88
  89 /* Initializes the scanner. */
  90 void init_scanner(struct scanner *scanner, struct scanner_info *scanner_info,
  91                   unsigned char *string, unsigned char *end);
  92
  93 /* The number of tokens in the scanners token table:
  94  * At best it should be big enough to contain properties with space separated
  95  * values and function calls with up to 3 variables like rgb(). At worst it
  96  * should be no less than 2 in order to be able to peek at the next token in
  97  * the scanner. */
  98 #define SCANNER_TOKENS 10
  99
 100 /* The {struct scanner} describes the current state of the scanner. */
 101 struct scanner {
 102         /* The very start of the scanned string, the position in the string
 103          * where to scan next and the end of the string. If position is NULL it
 104          * means that no more tokens can be retrieved from the string. */
 105         unsigned char *string, *position, *end;
 106
 107         /* The current token and number of scanned tokens in the table.
 108          * If the number of scanned tokens is less than SCANNER_TOKENS
 109          * it is because there are no more tokens in the string. */
 110         struct scanner_token *current;
 111         int tokens;
 112
 113         /* The 'meta' scanner information */
 114         struct scanner_info *info;
 115
 116 #ifdef DEBUG_SCANNER
 117         /* Debug info about the caller. */
 118         unsigned char *file;
 119         int line;
 120 #endif
 121
 122         /* Some state indicator only meaningful to the scanner internals */
 123         int state;
 124
 125         /* The table contain already scanned tokens. It is maintained in
 126          * order to optimize the scanning a bit and make it possible to look
 127          * ahead at the next token. You should always use the accessors
 128          * (defined below) for getting tokens from the scanner. */
 129         struct scanner_token table[SCANNER_TOKENS];
 130 };
 131
 132 #define scanner_has_tokens(scanner) \
 133         ((scanner)->tokens > 0 && (scanner)->current < (scanner)->table + (scanner)->tokens)
 134
 135 /* This macro checks if the current scanner state is valid. Meaning if the
 136  * scanners table is full the last token skipping or get_next_scanner_token()
 137  * call made it possible to get the type of the next token. */
 138 #define check_scanner(scanner) \
 139         (scanner->tokens < SCANNER_TOKENS \
 140          || scanner->current + 1 < scanner->table + scanner->tokens)
 141
 142
 143 /* Scanner table accessors and mutators */
 144
 145 /* Checks the type of the next token */
 146 #define check_next_scanner_token(scanner, token_type)                           \
 147         (scanner_has_tokens(scanner)                                    \
 148          && ((scanner)->current + 1 < (scanner)->table + (scanner)->tokens)     \
 149          && (scanner)->current[1].type == (token_type))
 150
 151 /* Access current and next token. Getting the next token might cause
 152  * a rescan so any token pointers that has been stored in a local variable
 153  * might not be valid after the call. */
 154 static inline struct scanner_token *
 155 get_scanner_token(struct scanner *scanner)
 156 {
 157         return scanner_has_tokens(scanner) ? scanner->current : NULL;
 158 }
 159
 160 /* Do a scanning if we do not have also have access to next token. */
 161 static inline struct scanner_token *
 162 get_next_scanner_token(struct scanner *scanner)
 163 {
 164         return (scanner_has_tokens(scanner)
 165                 && (++scanner->current + 1 >= scanner->table + scanner->tokens)
 166                 ? scanner->info->scan(scanner) : get_scanner_token(scanner));
 167 }
 168
 169 /* This should just make the code more understandable .. hopefully */
 170 #define skip_scanner_token(scanner) get_next_scanner_token(scanner)
 171
 172 /* Removes tokens from the scanner until it meets a token of the given type.
 173  * This token will then also be skipped. */
 174 struct scanner_token *
 175 skip_scanner_tokens(struct scanner *scanner, int skipto, int precedence);
 176
 177 /* Looks up the string from @ident to @end to in the scanners string mapping
 178  * table */
 179 int
 180 map_scanner_string(struct scanner *scanner,
 181                    unsigned char *ident, unsigned char *end, int base_type);
 182
 183 #ifdef DEBUG_SCANNER
 184 void dump_scanner(struct scanner *scanner);
 185 #endif
 186
 187 /* The begin_token_scanning() and end_token_scanning() functions provide the
 188  * basic setup and teardown for the rescan function made public via the
 189  * scanner_info->scan member. */
 190
 191 /* Returns NULL if it is not necessary to try to scan for more tokens */
 192 static inline struct scanner_token *
 193 begin_token_scanning(struct scanner *scanner)
 194 {
 195         struct scanner_token *table = scanner->table;
 196         struct scanner_token *table_end = table + scanner->tokens;
 197         int move_to_front = int_max(table_end - scanner->current, 0);
 198         struct scanner_token *current = move_to_front ? scanner->current : table;
 199         size_t moved_size = 0;
 200
 201         assert(scanner->current);
 202
 203         /* Move any untouched tokens */
 204         if (move_to_front) {
 205                 moved_size = move_to_front * sizeof(*table);
 206                 memmove(table, current, moved_size);
 207                 current = &table[move_to_front];
 208         }
 209
 210         /* Clear all unused tokens */
 211         memset(current, 0, sizeof(*table) * SCANNER_TOKENS - moved_size);
 212
 213         if (!scanner->position) {
 214                 scanner->tokens = move_to_front ? move_to_front : -1;
 215                 scanner->current = table;
 216                 assert(check_scanner(scanner));
 217                 return NULL;
 218         }
 219
 220         scanner->tokens = move_to_front;
 221
 222         return table;
 223 }
 224
 225 /* Updates the @scanner struct after scanning has been done. The position
 226  * _after_ the last valid token is taken as the @end argument. */
 227 /* It is ok for @end to be < scanner->table since scanner->tokens will become
 228  * <= 0 anyway. */
 229 static inline struct scanner_token *
 230 end_token_scanning(struct scanner *scanner, struct scanner_token *end)
 231 {
 232         assert(end <= scanner->table + SCANNER_TOKENS);
 233
 234         scanner->tokens = (end - scanner->table);
 235         scanner->current = scanner->table;
 236         if (scanner->position >= scanner->end)
 237                 scanner->position = NULL;
 238
 239         assert(check_scanner(scanner));
 240
 241         return get_scanner_token(scanner);
 242 }
 243
 244 #endif