src/util/scanner.h

   1 #ifndef EL__UTIL_SCANNER_H
   2 #define EL__UTIL_SCANNER_H
   3
   4 #include "util/error.h"
   5
   6 /* Define if you want a talking scanner */
   7 /* #define DEBUG_SCANNER */
   8
   9 /** The struct scanner_token describes one scanner state. There are two kinds
  10  * of tokens: char and non-char tokens. Char tokens contains only one char and
  11  * simply have their char value as type. They are tokens having special control
  12  * meaning in the code, like ':', ';', '{', '}' and '*'. Non char tokens has
  13  * one or more chars and contain stuff like number or indentifier strings.  */
  14 struct scanner_token {
  15         /** The type of the token */
  16         int type;
  17
  18         /** Some precedence value */
  19         int precedence;
  20
  21         /** The start of the token string and the token length */
  22         unsigned char *string;
  23         int length;
  24 };
  25
  26 /* The naming of these two macros is a bit odd .. we compare often with
  27  * "static" strings (I don't have a better word) so the macro name should
  28  * be short. --jonas */
  29
  30 /** Compare the string of @a token with @a str */
  31 #define scanner_token_strlcasecmp(token, str, len) \
  32         ((token) && !strlcasecmp((token)->string, (token)->length, str, len))
  33
  34 /** Also compares the token string but using a "static" string */
  35 #define scanner_token_contains(token, str) \
  36         scanner_token_strlcasecmp(token, str, sizeof(str) - 1)
  37
  38
  39 struct scan_table_info {
  40         enum { SCAN_RANGE, SCAN_STRING, SCAN_END } type;
  41         union scan_table_data {
  42                 struct { unsigned char *source; long length; } string;
  43                 struct { unsigned char *start; long end; } range;
  44         } data;
  45         int bits;
  46 };
  47
  48 #define SCAN_TABLE_SIZE 256
  49
  50 #define SCAN_TABLE_INFO(type, data1, data2, bits) \
  51         { (type), { { (data1), (data2) } }, (bits) }
  52
  53 #define SCAN_TABLE_RANGE(from, to, bits) SCAN_TABLE_INFO(SCAN_RANGE, from, to, bits)
  54 #define SCAN_TABLE_STRING(str, bits)     SCAN_TABLE_INFO(SCAN_STRING, str, sizeof(str) - 1, bits)
  55 #define SCAN_TABLE_END                   SCAN_TABLE_INFO(SCAN_END, 0, 0, 0)
  56
  57 struct scanner_string_mapping {
  58         unsigned char *name;
  59         int type;
  60         int base_type;
  61 };
  62
  63 struct scanner;
  64
  65 struct scanner_info {
  66         /** Table containing how to map strings to token types */
  67         const struct scanner_string_mapping *mappings;
  68
  69         /** Information for how to initialize the scanner table */
  70         const struct scan_table_info *scan_table_info;
  71
  72         /** Fills the scanner with tokens. Already scanned tokens
  73          * which have not been requested remain and are moved to the
  74          * start of the scanners token table.
  75          * @returns the current token or NULL if there are none. */
  76         struct scanner_token *(*scan)(struct scanner *scanner);
  77
  78         /** The scanner table.  Contains bitmaps for the various
  79          * characters groups.  Idea sync'ed from mozilla browser. */
  80         int scan_table[SCAN_TABLE_SIZE];
  81
  82         /** Has the scanner info been initialized? */
  83         unsigned int initialized:1;
  84 };
  85
  86
  87 /** Initializes the scanner.
  88  * @relates scanner */
  89 void init_scanner(struct scanner *scanner, struct scanner_info *scanner_info,
  90                   unsigned char *string, unsigned char *end);
  91
  92 /** The number of tokens in the scanners token table:
  93  * At best it should be big enough to contain properties with space separated
  94  * values and function calls with up to 3 variables like rgb(). At worst it
  95  * should be no less than 2 in order to be able to peek at the next token in
  96  * the scanner. */
  97 #define SCANNER_TOKENS 10
  98
  99 /** The struct scanner describes the current state of the scanner. */
 100 struct scanner {
 101         /** The very start of the scanned string, the position in the string
 102          * where to scan next and the end of the string. If #position is NULL
 103          * it means that no more tokens can be retrieved from the string. */
 104         unsigned char *string, *position, *end;
 105
 106         /** The current token and number of scanned tokens in the table.
 107          * If the number of scanned tokens is less than ::SCANNER_TOKENS
 108          * it is because there are no more tokens in the string. */
 109         struct scanner_token *current;
 110         int tokens;
 111
 112         /** The 'meta' scanner information */
 113         struct scanner_info *info;
 114
 115 #ifdef DEBUG_SCANNER
 116         /** @name Debug info about the caller.
 117          * @{ */
 118         unsigned char *file;
 119         int line;
 120         /** @} */
 121 #endif
 122
 123         /** Some state indicator only meaningful to the scanner internals */
 124         int state;
 125
 126         /** The table contain already scanned tokens. It is maintained in
 127          * order to optimize the scanning a bit and make it possible to look
 128          * ahead at the next token. You should always use the accessors
 129          * (defined below) for getting tokens from the scanner. */
 130         struct scanner_token table[SCANNER_TOKENS];
 131 };
 132
 133 /** @relates scanner  */
 134 #define scanner_has_tokens(scanner) \
 135         ((scanner)->tokens > 0 && (scanner)->current < (scanner)->table + (scanner)->tokens)
 136
 137 /** This macro checks if the current scanner state is valid. Meaning if the
 138  * scanners table is full the last token skipping or get_next_scanner_token()
 139  * call made it possible to get the type of the next token.
 140  * @relates scanner */
 141 #define check_scanner(scanner) \
 142         (scanner->tokens < SCANNER_TOKENS \
 143          || scanner->current + 1 < scanner->table + scanner->tokens)
 144
 145
 146 /** @name Scanner table accessors and mutators
 147  * @{ */
 148
 149 /** Checks the type of the next token
 150  * @relates scanner */
 151 #define check_next_scanner_token(scanner, token_type)                           \
 152         (scanner_has_tokens(scanner)                                    \
 153          && ((scanner)->current + 1 < (scanner)->table + (scanner)->tokens)     \
 154          && (scanner)->current[1].type == (token_type))
 155
 156 /** Access current and next token. Getting the next token might cause
 157  * a rescan so any token pointers that has been stored in a local variable
 158  * might not be valid after the call.
 159  * @relates scanner */
 160 static inline struct scanner_token *
 161 get_scanner_token(struct scanner *scanner)
 162 {
 163         return scanner_has_tokens(scanner) ? scanner->current : NULL;
 164 }
 165
 166 /** Do a scanning if we do not have also have access to next token.
 167  * @relates scanner */
 168 static inline struct scanner_token *
 169 get_next_scanner_token(struct scanner *scanner)
 170 {
 171         return (scanner_has_tokens(scanner)
 172                 && (++scanner->current + 1 >= scanner->table + scanner->tokens)
 173                 ? scanner->info->scan(scanner) : get_scanner_token(scanner));
 174 }
 175
 176 /** This should just make the code more understandable .. hopefully
 177  * @relates scanner */
 178 #define skip_scanner_token(scanner) get_next_scanner_token(scanner)
 179
 180 /** Removes tokens from the scanner until it meets a token of the given type.
 181  * This token will then also be skipped.
 182  * @relates scanner */
 183 struct scanner_token *
 184 skip_scanner_tokens(struct scanner *scanner, int skipto, int precedence);
 185
 186 /** @} */
 187
 188 /** Looks up the string from @a ident to @a end to in the scanners
 189  * string mapping table
 190  * @relates scanner */
 191 int
 192 map_scanner_string(struct scanner *scanner,
 193                    const unsigned char *ident, const unsigned char *end,
 194                    int base_type);
 195
 196 #ifdef DEBUG_SCANNER
 197 /** @relates scanner */
 198 void dump_scanner(struct scanner *scanner);
 199 #endif
 200
 201 /* The begin_token_scanning() and end_token_scanning() functions provide the
 202  * basic setup and teardown for the rescan function made public via the
 203  * scanner_info->scan member.
 204  * @returns NULL if it is not necessary to try to scan for more tokens
 205  * @relates scanner */
 206 static inline struct scanner_token *
 207 begin_token_scanning(struct scanner *scanner)
 208 {
 209         struct scanner_token *table = scanner->table;
 210         struct scanner_token *table_end = table + scanner->tokens;
 211         int move_to_front = int_max(table_end - scanner->current, 0);
 212         struct scanner_token *current = move_to_front ? scanner->current : table;
 213         size_t moved_size = 0;
 214
 215         assert(scanner->current);
 216
 217         /* Move any untouched tokens */
 218         if (move_to_front) {
 219                 moved_size = move_to_front * sizeof(*table);
 220                 memmove(table, current, moved_size);
 221                 current = &table[move_to_front];
 222         }
 223
 224         /* Clear all unused tokens */
 225         memset(current, 0, sizeof(*table) * SCANNER_TOKENS - moved_size);
 226
 227         if (!scanner->position) {
 228                 scanner->tokens = move_to_front ? move_to_front : -1;
 229                 scanner->current = table;
 230                 assert(check_scanner(scanner));
 231                 return NULL;
 232         }
 233
 234         scanner->tokens = move_to_front;
 235
 236         return table;
 237 }
 238
 239 /* Updates the @a scanner struct after scanning has been done. The position
 240  * _after_ the last valid token is taken as the @a end argument.
 241  *
 242  * It is ok for @a end to be < scanner->table since scanner->tokens
 243  * will become <= 0 anyway.
 244  * @relates scanner */
 245 static inline struct scanner_token *
 246 end_token_scanning(struct scanner *scanner, struct scanner_token *end)
 247 {
 248         assert(end <= scanner->table + SCANNER_TOKENS);
 249
 250         scanner->tokens = (end - scanner->table);
 251         scanner->current = scanner->table;
 252         if (scanner->position >= scanner->end)
 253                 scanner->position = NULL;
 254
 255         assert(check_scanner(scanner));
 256
 257         return get_scanner_token(scanner);
 258 }
 259
 260 #endif