src/util/scanner.h

   1 #ifndef EL__UTIL_SCANNER_H
   2 #define EL__UTIL_SCANNER_H
   3
   4 #include "util/error.h"
   5 #include "util/string.h"
   6
   7 /* Define if you want a talking scanner */
   8 /* #define DEBUG_SCANNER */
   9
  10 /** The struct scanner_token describes one scanner state. There are two kinds
  11  * of tokens: char and non-char tokens. Char tokens contains only one char and
  12  * simply have their char value as type. They are tokens having special control
  13  * meaning in the code, like ':', ';', '{', '}' and '*'. Non char tokens has
  14  * one or more chars and contain stuff like number or indentifier strings.  */
  15 struct scanner_token {
  16         /** The type of the token */
  17         int type;
  18
  19         /** Some precedence value */
  20         int precedence;
  21
  22         /** The start of the token string and the token length */
  23         const unsigned char *string;
  24         int length;
  25 };
  26
  27 /* The naming of these two macros is a bit odd .. we compare often with
  28  * "static" strings (I don't have a better word) so the macro name should
  29  * be short. --jonas */
  30
  31 /** Compare the string of @a token with @a str */
  32 #define scanner_token_strlcasecmp(token, str, len) \
  33         ((token) && !c_strlcasecmp((token)->string, (token)->length, str, len))
  34
  35 /** Also compares the token string but using a "static" string */
  36 #define scanner_token_contains(token, str) \
  37         scanner_token_strlcasecmp(token, str, sizeof(str) - 1)
  38
  39 enum scan_type { SCAN_RANGE, SCAN_STRING, SCAN_END };
  40 union scan_table_data {
  41         struct { unsigned char *source; long length; } string;
  42         struct { unsigned char *start; long end; } range;
  43 };
  44
  45 struct scan_table_info {
  46         enum scan_type type;
  47         union scan_table_data data;
  48         int bits;
  49 };
  50
  51 #define SCAN_TABLE_SIZE 256
  52
  53 #define SCAN_TABLE_INFO(type, data1, data2, bits) \
  54         { (type), { { (data1), (data2) } }, (bits) }
  55
  56 #define SCAN_TABLE_RANGE(from, to, bits) SCAN_TABLE_INFO(SCAN_RANGE, from, to, bits)
  57 #define SCAN_TABLE_STRING(str, bits)     SCAN_TABLE_INFO(SCAN_STRING, str, sizeof(str) - 1, bits)
  58 #define SCAN_TABLE_END                   SCAN_TABLE_INFO(SCAN_END, 0, 0, 0)
  59
  60 struct scanner_string_mapping {
  61         unsigned char *name;
  62         int type;
  63         int base_type;
  64 };
  65
  66 struct scanner;
  67
  68 struct scanner_info {
  69         /** Table containing how to map strings to token types */
  70         const struct scanner_string_mapping *mappings;
  71
  72         /** Information for how to initialize the scanner table */
  73         const struct scan_table_info *scan_table_info;
  74
  75         /** Fills the scanner with tokens. Already scanned tokens
  76          * which have not been requested remain and are moved to the
  77          * start of the scanners token table.
  78          * @returns the current token or NULL if there are none. */
  79         struct scanner_token *(*scan)(struct scanner *scanner);
  80
  81         /** The scanner table.  Contains bitmaps for the various
  82          * characters groups.  Idea sync'ed from mozilla browser. */
  83         int scan_table[SCAN_TABLE_SIZE];
  84
  85         /** Has the scanner info been initialized? */
  86         unsigned int initialized:1;
  87 };
  88
  89
  90 /** Initializes the scanner.
  91  * @relates scanner */
  92 void init_scanner(struct scanner *scanner, struct scanner_info *scanner_info,
  93                   const unsigned char *string, const unsigned char *end);
  94
  95 /** The number of tokens in the scanners token table:
  96  * At best it should be big enough to contain properties with space separated
  97  * values and function calls with up to 3 variables like rgb(). At worst it
  98  * should be no less than 2 in order to be able to peek at the next token in
  99  * the scanner. */
 100 #define SCANNER_TOKENS 10
 101
 102 /** The struct scanner describes the current state of the scanner. */
 103 struct scanner {
 104         /** The very start of the scanned string, the position in the string
 105          * where to scan next and the end of the string. If #position is NULL
 106          * it means that no more tokens can be retrieved from the string. */
 107         const unsigned char *string, *position, *end;
 108
 109         /** The current token and number of scanned tokens in the table.
 110          * If the number of scanned tokens is less than ::SCANNER_TOKENS
 111          * it is because there are no more tokens in the string. */
 112         struct scanner_token *current;
 113         int tokens;
 114
 115         /** The 'meta' scanner information */
 116         struct scanner_info *info;
 117
 118 #ifdef DEBUG_SCANNER
 119         /** @name Debug info about the caller.
 120          * @{ */
 121         unsigned char *file;
 122         int line;
 123         /** @} */
 124 #endif
 125
 126         /** Some state indicator only meaningful to the scanner internals */
 127         int state;
 128
 129         /** The table contain already scanned tokens. It is maintained in
 130          * order to optimize the scanning a bit and make it possible to look
 131          * ahead at the next token. You should always use the accessors
 132          * (defined below) for getting tokens from the scanner. */
 133         struct scanner_token table[SCANNER_TOKENS];
 134 };
 135
 136 /** @relates scanner  */
 137 #define scanner_has_tokens(scanner) \
 138         ((scanner)->tokens > 0 && (scanner)->current < (scanner)->table + (scanner)->tokens)
 139
 140 /** This macro checks if the current scanner state is valid. Meaning if the
 141  * scanners table is full the last token skipping or get_next_scanner_token()
 142  * call made it possible to get the type of the next token.
 143  * @relates scanner */
 144 #define check_scanner(scanner) \
 145         (scanner->tokens < SCANNER_TOKENS \
 146          || scanner->current + 1 < scanner->table + scanner->tokens)
 147
 148
 149 /** @name Scanner table accessors and mutators
 150  * @{ */
 151
 152 /** Checks the type of the next token
 153  * @relates scanner */
 154 #define check_next_scanner_token(scanner, token_type)                           \
 155         (scanner_has_tokens(scanner)                                    \
 156          && ((scanner)->current + 1 < (scanner)->table + (scanner)->tokens)     \
 157          && (scanner)->current[1].type == (token_type))
 158
 159 /** Access current and next token. Getting the next token might cause
 160  * a rescan so any token pointers that has been stored in a local variable
 161  * might not be valid after the call.
 162  * @relates scanner */
 163 static inline struct scanner_token *
 164 get_scanner_token(struct scanner *scanner)
 165 {
 166         return scanner_has_tokens(scanner) ? scanner->current : NULL;
 167 }
 168
 169 /** Do a scanning if we do not have also have access to next token.
 170  * @relates scanner */
 171 static inline struct scanner_token *
 172 get_next_scanner_token(struct scanner *scanner)
 173 {
 174         return (scanner_has_tokens(scanner)
 175                 && (++scanner->current + 1 >= scanner->table + scanner->tokens)
 176                 ? scanner->info->scan(scanner) : get_scanner_token(scanner));
 177 }
 178
 179 /** This should just make the code more understandable .. hopefully
 180  * @relates scanner */
 181 #define skip_scanner_token(scanner) get_next_scanner_token(scanner)
 182
 183 /** Removes tokens from the scanner until it meets a token of the given type.
 184  * This token will then also be skipped.
 185  * @relates scanner */
 186 struct scanner_token *
 187 skip_scanner_tokens(struct scanner *scanner, int skipto, int precedence);
 188
 189 /** @} */
 190
 191 /** Looks up the string from @a ident to @a end to in the scanners
 192  * string mapping table
 193  * @relates scanner */
 194 int
 195 map_scanner_string(struct scanner *scanner,
 196                    const unsigned char *ident, const unsigned char *end,
 197                    int base_type);
 198
 199 #ifdef DEBUG_SCANNER
 200 /** @relates scanner */
 201 void dump_scanner(struct scanner *scanner);
 202 #endif
 203
 204 /* The begin_token_scanning() and end_token_scanning() functions provide the
 205  * basic setup and teardown for the rescan function made public via the
 206  * scanner_info->scan member.
 207  * @returns NULL if it is not necessary to try to scan for more tokens
 208  * @relates scanner */
 209 static inline struct scanner_token *
 210 begin_token_scanning(struct scanner *scanner)
 211 {
 212         struct scanner_token *table = scanner->table;
 213         struct scanner_token *table_end = table + scanner->tokens;
 214         int move_to_front = int_max(table_end - scanner->current, 0);
 215         struct scanner_token *current = move_to_front ? scanner->current : table;
 216         size_t moved_size = 0;
 217
 218         assert(scanner->current);
 219
 220         /* Move any untouched tokens */
 221         if (move_to_front) {
 222                 moved_size = move_to_front * sizeof(*table);
 223                 memmove(table, current, moved_size);
 224                 current = &table[move_to_front];
 225         }
 226
 227         /* Clear all unused tokens */
 228         memset(current, 0, sizeof(*table) * SCANNER_TOKENS - moved_size);
 229
 230         if (!scanner->position) {
 231                 scanner->tokens = move_to_front ? move_to_front : -1;
 232                 scanner->current = table;
 233                 assert(check_scanner(scanner));
 234                 return NULL;
 235         }
 236
 237         scanner->tokens = move_to_front;
 238
 239         return table;
 240 }
 241
 242 /* Updates the @a scanner struct after scanning has been done. The position
 243  * _after_ the last valid token is taken as the @a end argument.
 244  *
 245  * It is ok for @a end to be < scanner->table since scanner->tokens
 246  * will become <= 0 anyway.
 247  * @relates scanner */
 248 static inline struct scanner_token *
 249 end_token_scanning(struct scanner *scanner, struct scanner_token *end)
 250 {
 251         assert(end <= scanner->table + SCANNER_TOKENS);
 252
 253         scanner->tokens = (end - scanner->table);
 254         scanner->current = scanner->table;
 255         if (scanner->position >= scanner->end)
 256                 scanner->position = NULL;
 257
 258         assert(check_scanner(scanner));
 259
 260         return get_scanner_token(scanner);
 261 }
 262
 263 #endif