4 * Basic tokenization structures. NOTE! Those tokens had better
5 * be pretty small, since we're going to keep them all in memory
8 * Copyright (C) 2003 Linus Torvalds, all rights reserved.
11 #include <sys/types.h>
14 * This describes the pure lexical elements (tokens), with
15 * no semantic meaning. In other words, an identifier doesn't
16 * have a type or meaning, it is only a specific string in
19 * Semantic meaning is handled elsewhere.
26 /* Use these to check for "already parsed" */
27 int constant
, nesting
;
28 struct ident
*protect
;
33 extern int input_stream_nr
;
34 extern struct stream
*input_streams
;
36 extern int ident_hit
, ident_miss
;
39 struct ident
*next
; /* Hash chain of identifiers */
40 struct symbol
*symbols
; /* Pointer to semantic meaning list */
41 unsigned char len
; /* Length of identifier name */
42 char name
[]; /* Actual identifier */
58 /* Combination tokens */
59 #define COMBINATION_STRINGS { \
77 SPECIAL_ADD_ASSIGN
= 256,
103 SPECIAL_ARG_SEPARATOR
112 * This is a very common data structure, it should be kept
113 * as small as humanly possible. Big (rare) types go as
128 unsigned int special
;
129 struct string
*string
;
135 * Last token in the stream - points to itself.
136 * This allows us to not test for NULL pointers
137 * when following the token->next chain..
139 extern struct token eof_token_entry
;
140 #define eof_token(x) ((x) == &eof_token_entry)
142 extern int init_stream(const char *, int fd
);
143 extern struct ident
*hash_ident(struct ident
*);
144 extern struct ident
*built_in_ident(const char *);
145 extern struct token
*built_in_token(int, const char *);
146 extern const char *show_special(int);
147 extern const char *show_ident(const struct ident
*);
148 extern const char *show_token(const struct token
*);
149 extern struct token
* tokenize(const char *, int, struct token
*end
);
150 extern void die(const char *, ...);
151 extern void show_identifier_stats(void);
152 extern struct token
*preprocess(struct token
*);
154 static inline int match_op(struct token
*token
, int op
)
156 return token
->type
== TOKEN_SPECIAL
&& token
->special
== op
;
159 static inline int match_ident(struct token
*token
, struct ident
*id
)
161 return token
->type
== TOKEN_IDENT
&& token
->ident
== id
;