4 * Basic tokenization structures. NOTE! Those tokens had better
5 * be pretty small, since we're going to keep them all in memory
8 * Copyright (C) 2003 Linus Torvalds, all rights reserved.
11 #include <sys/types.h>
13 extern char *includepath
[];
16 * This describes the pure lexical elements (tokens), with
17 * no semantic meaning. In other words, an identifier doesn't
18 * have a type or meaning, it is only a specific string in
21 * Semantic meaning is handled elsewhere.
28 /* Use these to check for "already parsed" */
29 int constant
, nesting
;
30 struct ident
*protect
;
35 extern int input_stream_nr
;
36 extern struct stream
*input_streams
;
38 extern int ident_hit
, ident_miss
;
41 struct ident
*next
; /* Hash chain of identifiers */
42 struct symbol
*symbols
; /* Pointer to semantic meaning list */
43 unsigned char len
; /* Length of identifier name */
44 char name
[]; /* Actual identifier */
60 /* Combination tokens */
61 #define COMBINATION_STRINGS { \
79 SPECIAL_ADD_ASSIGN
= 256,
105 SPECIAL_ARG_SEPARATOR
114 * This is a very common data structure, it should be kept
115 * as small as humanly possible. Big (rare) types go as
130 unsigned int special
;
131 struct string
*string
;
137 * Last token in the stream - points to itself.
138 * This allows us to not test for NULL pointers
139 * when following the token->next chain..
141 extern struct token eof_token_entry
;
142 #define eof_token(x) ((x) == &eof_token_entry)
144 extern int init_stream(const char *, int fd
);
145 extern struct ident
*hash_ident(struct ident
*);
146 extern struct ident
*built_in_ident(const char *);
147 extern struct token
*built_in_token(int, const char *);
148 extern const char *show_special(int);
149 extern const char *show_ident(const struct ident
*);
150 extern const char *show_token(const struct token
*);
151 extern struct token
* tokenize(const char *, int, struct token
*end
);
152 extern void die(const char *, ...);
153 extern void show_identifier_stats(void);
154 extern struct token
*preprocess(struct token
*);
156 static inline int match_op(struct token
*token
, int op
)
158 return token
->type
== TOKEN_SPECIAL
&& token
->special
== op
;
161 static inline int match_ident(struct token
*token
, struct ident
*id
)
163 return token
->type
== TOKEN_IDENT
&& token
->ident
== id
;