4 * Basic tokenization structures. NOTE! Those tokens had better
5 * be pretty small, since we're going to keep them all in memory
8 * Copyright (C) 2003 Transmeta Corp.
11 * Licensed under the Open Software License version 1.1
14 #include <sys/types.h>
18 * This describes the pure lexical elements (tokens), with
19 * no semantic meaning. In other words, an identifier doesn't
20 * have a type or meaning, it is only a specific string in
23 * Semantic meaning is handled elsewhere.
30 /* Use these to check for "already parsed" */
31 int constant
, nesting
;
32 struct ident
*protect
;
37 extern int input_stream_nr
;
38 extern struct stream
*input_streams
;
40 extern int ident_hit
, ident_miss
;
43 struct ident
*next
; /* Hash chain of identifiers */
44 struct symbol
*symbols
; /* Pointer to semantic meaning list */
45 unsigned char len
; /* Length of identifier name */
46 unsigned char tainted
;
47 char name
[]; /* Actual identifier */
62 TOKEN_QUOTED_ARGUMENT
,
69 /* Combination tokens */
70 #define COMBINATION_STRINGS { \
84 "<", ">", "<=", ">=" \
89 SPECIAL_ADD_ASSIGN
= SPECIAL_BASE
,
113 SPECIAL_ARG_SEPARATOR
,
116 SPECIAL_UNSIGNED_LTE
,
117 SPECIAL_UNSIGNED_GTE
,
125 /* will fit into 32 bits */
134 * This is a very common data structure, it should be kept
135 * as small as humanly possible. Big (rare) types go as
144 unsigned int special
;
145 struct string
*string
;
148 struct argcount count
;
152 #define MAX_STRING 4095
154 static inline struct token
*containing_token(struct token
**p
)
156 void *addr
= (char*)p
- ((char*)&((struct token
*)0)->next
- (char*)0);
160 #define token_type(x) ((x)->pos.type)
163 * Last token in the stream - points to itself.
164 * This allows us to not test for NULL pointers
165 * when following the token->next chain..
167 extern int preprocessing
, verbose
;
168 extern struct token eof_token_entry
;
169 #define eof_token(x) ((x) == &eof_token_entry)
171 extern int init_stream(const char *, int fd
);
172 extern struct ident
*hash_ident(struct ident
*);
173 extern struct ident
*built_in_ident(const char *);
174 extern struct token
*built_in_token(int, const char *);
175 extern const char *show_special(int);
176 extern const char *show_ident(const struct ident
*);
177 extern const char *show_string(const struct string
*string
);
178 extern const char *show_token(const struct token
*);
179 extern struct token
* tokenize(const char *, int, struct token
*);
180 extern struct token
* tokenize_buffer(unsigned char *, unsigned long, struct token
*);
182 extern void die(const char *, ...);
183 extern void show_identifier_stats(void);
184 extern struct token
*preprocess(struct token
*);
186 static inline int match_op(struct token
*token
, int op
)
188 return token
->pos
.type
== TOKEN_SPECIAL
&& token
->special
== op
;
191 static inline int match_ident(struct token
*token
, struct ident
*id
)
193 return token
->pos
.type
== TOKEN_IDENT
&& token
->ident
== id
;