1 #ifndef EL__UTIL_SCANNER_H
2 #define EL__UTIL_SCANNER_H
4 #include "util/error.h"
5 #include "util/string.h"
7 /* Define if you want a talking scanner */
8 /* #define DEBUG_SCANNER */
10 /** The struct scanner_token describes one scanner state. There are two kinds
11 * of tokens: char and non-char tokens. Char tokens contains only one char and
12 * simply have their char value as type. They are tokens having special control
13 * meaning in the code, like ':', ';', '{', '}' and '*'. Non char tokens has
14 * one or more chars and contain stuff like number or indentifier strings. */
15 struct scanner_token
{
16 /** The type of the token */
19 /** Some precedence value */
22 /** The start of the token string and the token length */
23 const unsigned char *string
;
27 /* The naming of these two macros is a bit odd .. we compare often with
28 * "static" strings (I don't have a better word) so the macro name should
29 * be short. --jonas */
31 /** Compare the string of @a token with @a str */
32 #define scanner_token_strlcasecmp(token, str, len) \
33 ((token) && !c_strlcasecmp((token)->string, (token)->length, str, len))
35 /** Also compares the token string but using a "static" string */
36 #define scanner_token_contains(token, str) \
37 scanner_token_strlcasecmp(token, str, sizeof(str) - 1)
39 enum scan_type
{ SCAN_RANGE
, SCAN_STRING
, SCAN_END
};
40 union scan_table_data
{
41 struct { unsigned char *source
; long length
; } string
;
42 struct { unsigned char *start
; long end
; } range
;
45 struct scan_table_info
{
47 union scan_table_data data
;
51 #define SCAN_TABLE_SIZE 256
53 #define SCAN_TABLE_INFO(type, data1, data2, bits) \
54 { (type), { { (data1), (data2) } }, (bits) }
56 #define SCAN_TABLE_RANGE(from, to, bits) SCAN_TABLE_INFO(SCAN_RANGE, from, to, bits)
57 #define SCAN_TABLE_STRING(str, bits) SCAN_TABLE_INFO(SCAN_STRING, str, sizeof(str) - 1, bits)
58 #define SCAN_TABLE_END SCAN_TABLE_INFO(SCAN_END, 0, 0, 0)
60 struct scanner_string_mapping
{
69 /** Table containing how to map strings to token types */
70 const struct scanner_string_mapping
*mappings
;
72 /** Information for how to initialize the scanner table */
73 const struct scan_table_info
*scan_table_info
;
75 /** Fills the scanner with tokens. Already scanned tokens
76 * which have not been requested remain and are moved to the
77 * start of the scanners token table.
78 * @returns the current token or NULL if there are none. */
79 struct scanner_token
*(*scan
)(struct scanner
*scanner
);
81 /** The scanner table. Contains bitmaps for the various
82 * characters groups. Idea sync'ed from mozilla browser. */
83 int scan_table
[SCAN_TABLE_SIZE
];
85 /** Has the scanner info been initialized? */
86 unsigned int initialized
:1;
90 /** Initializes the scanner.
92 void init_scanner(struct scanner
*scanner
, struct scanner_info
*scanner_info
,
93 const unsigned char *string
, const unsigned char *end
);
95 /** The number of tokens in the scanners token table:
96 * At best it should be big enough to contain properties with space separated
97 * values and function calls with up to 3 variables like rgb(). At worst it
98 * should be no less than 2 in order to be able to peek at the next token in
100 #define SCANNER_TOKENS 10
102 /** The struct scanner describes the current state of the scanner. */
104 /** The very start of the scanned string, the position in the string
105 * where to scan next and the end of the string. If #position is NULL
106 * it means that no more tokens can be retrieved from the string. */
107 const unsigned char *string
, *position
, *end
;
109 /** The current token and number of scanned tokens in the table.
110 * If the number of scanned tokens is less than ::SCANNER_TOKENS
111 * it is because there are no more tokens in the string. */
112 struct scanner_token
*current
;
115 /** The 'meta' scanner information */
116 struct scanner_info
*info
;
119 /** @name Debug info about the caller.
126 /** Some state indicator only meaningful to the scanner internals */
129 /** The table contain already scanned tokens. It is maintained in
130 * order to optimize the scanning a bit and make it possible to look
131 * ahead at the next token. You should always use the accessors
132 * (defined below) for getting tokens from the scanner. */
133 struct scanner_token table
[SCANNER_TOKENS
];
136 /** @relates scanner */
137 #define scanner_has_tokens(scanner) \
138 ((scanner)->tokens > 0 && (scanner)->current < (scanner)->table + (scanner)->tokens)
140 /** This macro checks if the current scanner state is valid. Meaning if the
141 * scanners table is full the last token skipping or get_next_scanner_token()
142 * call made it possible to get the type of the next token.
143 * @relates scanner */
144 #define check_scanner(scanner) \
145 (scanner->tokens < SCANNER_TOKENS \
146 || scanner->current + 1 < scanner->table + scanner->tokens)
149 /** @name Scanner table accessors and mutators
152 /** Checks the type of the next token
153 * @relates scanner */
154 #define check_next_scanner_token(scanner, token_type) \
155 (scanner_has_tokens(scanner) \
156 && ((scanner)->current + 1 < (scanner)->table + (scanner)->tokens) \
157 && (scanner)->current[1].type == (token_type))
159 /** Access current and next token. Getting the next token might cause
160 * a rescan so any token pointers that has been stored in a local variable
161 * might not be valid after the call.
162 * @relates scanner */
163 static inline struct scanner_token
*
164 get_scanner_token(struct scanner
*scanner
)
166 return scanner_has_tokens(scanner
) ? scanner
->current
: NULL
;
169 /** Do a scanning if we do not have also have access to next token.
170 * @relates scanner */
171 static inline struct scanner_token
*
172 get_next_scanner_token(struct scanner
*scanner
)
174 return (scanner_has_tokens(scanner
)
175 && (++scanner
->current
+ 1 >= scanner
->table
+ scanner
->tokens
)
176 ? scanner
->info
->scan(scanner
) : get_scanner_token(scanner
));
179 /** This should just make the code more understandable .. hopefully
180 * @relates scanner */
181 #define skip_scanner_token(scanner) get_next_scanner_token(scanner)
183 /** Removes tokens from the scanner until it meets a token of the given type.
184 * This token will then also be skipped.
185 * @relates scanner */
186 struct scanner_token
*
187 skip_scanner_tokens(struct scanner
*scanner
, int skipto
, int precedence
);
191 /** Looks up the string from @a ident to @a end to in the scanners
192 * string mapping table
193 * @relates scanner */
195 map_scanner_string(struct scanner
*scanner
,
196 const unsigned char *ident
, const unsigned char *end
,
200 /** @relates scanner */
201 void dump_scanner(struct scanner
*scanner
);
204 /* The begin_token_scanning() and end_token_scanning() functions provide the
205 * basic setup and teardown for the rescan function made public via the
206 * scanner_info->scan member.
207 * @returns NULL if it is not necessary to try to scan for more tokens
208 * @relates scanner */
209 static inline struct scanner_token
*
210 begin_token_scanning(struct scanner
*scanner
)
212 struct scanner_token
*table
= scanner
->table
;
213 struct scanner_token
*table_end
= table
+ scanner
->tokens
;
214 int move_to_front
= int_max(table_end
- scanner
->current
, 0);
215 struct scanner_token
*current
= move_to_front
? scanner
->current
: table
;
216 size_t moved_size
= 0;
218 assert(scanner
->current
);
220 /* Move any untouched tokens */
222 moved_size
= move_to_front
* sizeof(*table
);
223 memmove(table
, current
, moved_size
);
224 current
= &table
[move_to_front
];
227 /* Clear all unused tokens */
228 memset(current
, 0, sizeof(*table
) * SCANNER_TOKENS
- moved_size
);
230 if (!scanner
->position
) {
231 scanner
->tokens
= move_to_front
? move_to_front
: -1;
232 scanner
->current
= table
;
233 assert(check_scanner(scanner
));
237 scanner
->tokens
= move_to_front
;
242 /* Updates the @a scanner struct after scanning has been done. The position
243 * _after_ the last valid token is taken as the @a end argument.
245 * It is ok for @a end to be < scanner->table since scanner->tokens
246 * will become <= 0 anyway.
247 * @relates scanner */
248 static inline struct scanner_token
*
249 end_token_scanning(struct scanner
*scanner
, struct scanner_token
*end
)
251 assert(end
<= scanner
->table
+ SCANNER_TOKENS
);
253 scanner
->tokens
= (end
- scanner
->table
);
254 scanner
->current
= scanner
->table
;
255 if (scanner
->position
>= scanner
->end
)
256 scanner
->position
= NULL
;
258 assert(check_scanner(scanner
));
260 return get_scanner_token(scanner
);