4 * Basic tokenization structures. NOTE! Those tokens had better
5 * be pretty small, since we're going to keep them all in memory
8 * Copyright (C) 2003 Transmeta Corp, all rights reserved.
11 #include <sys/types.h>
15 * This describes the pure lexical elements (tokens), with
16 * no semantic meaning. In other words, an identifier doesn't
17 * have a type or meaning, it is only a specific string in
20 * Semantic meaning is handled elsewhere.
27 /* Use these to check for "already parsed" */
28 int constant
, nesting
;
29 struct ident
*protect
;
34 extern int input_stream_nr
;
35 extern struct stream
*input_streams
;
37 extern int ident_hit
, ident_miss
;
40 struct ident
*next
; /* Hash chain of identifiers */
41 struct symbol
*symbols
; /* Pointer to semantic meaning list */
42 unsigned char len
; /* Length of identifier name */
43 char name
[]; /* Actual identifier */
59 /* Combination tokens */
60 #define COMBINATION_STRINGS { \
78 SPECIAL_ADD_ASSIGN
= 256,
104 SPECIAL_ARG_SEPARATOR
113 * This is a very common data structure, it should be kept
114 * as small as humanly possible. Big (rare) types go as
124 unsigned int special
;
125 struct string
*string
;
130 #define token_type(x) ((x)->pos.type)
133 * Last token in the stream - points to itself.
134 * This allows us to not test for NULL pointers
135 * when following the token->next chain..
137 extern int preprocessing
;
138 extern struct token eof_token_entry
;
139 #define eof_token(x) ((x) == &eof_token_entry)
141 extern int init_stream(const char *, int fd
);
142 extern struct ident
*hash_ident(struct ident
*);
143 extern struct ident
*built_in_ident(const char *);
144 extern struct token
*built_in_token(int, const char *);
145 extern const char *show_special(int);
146 extern const char *show_ident(const struct ident
*);
147 extern const char *show_string(const struct string
*string
);
148 extern const char *show_token(const struct token
*);
149 extern struct token
* tokenize(const char *, int, struct token
*);
150 extern struct token
* tokenize_buffer(unsigned char *, unsigned long, struct token
*);
152 extern void die(const char *, ...);
153 extern void show_identifier_stats(void);
154 extern struct token
*preprocess(struct token
*);
156 static inline int match_op(struct token
*token
, int op
)
158 return token
->pos
.type
== TOKEN_SPECIAL
&& token
->special
== op
;
161 static inline int match_ident(struct token
*token
, struct ident
*id
)
163 return token
->pos
.type
== TOKEN_IDENT
&& token
->ident
== id
;