4 * Basic tokenization structures. NOTE! Those tokens had better
5 * be pretty small, since we're going to keep them all in memory
8 * Copyright (C) 2003 Transmeta Corp.
11 * Licensed under the Open Software License version 1.1
14 #include <sys/types.h>
18 * This describes the pure lexical elements (tokens), with
19 * no semantic meaning. In other words, an identifier doesn't
20 * have a type or meaning, it is only a specific string in
23 * Semantic meaning is handled elsewhere.
27 CONSTANT_FILE_MAYBE
, // To be determined, not inside any #ifs in this file
28 CONSTANT_FILE_IFNDEF
, // To be determined, currently inside #ifndef
29 CONSTANT_FILE_NOPE
, // No
30 CONSTANT_FILE_YES
// Yes
33 extern const char *includepath
[];
38 const char *path
; // input-file path - see set_stream_include_path()
39 const char **next_path
;
41 /* Use these to check for "already parsed" */
42 enum constantfile constant
;
43 int dirty
, next_stream
, once
;
44 struct ident
*protect
;
49 extern int input_stream_nr
;
50 extern struct stream
*input_streams
;
51 extern unsigned int tabstop
;
52 extern int *hash_stream(const char *name
);
55 struct ident
*next
; /* Hash chain of identifiers */
56 struct symbol
*symbols
; /* Pointer to semantic meaning list */
57 unsigned char len
; /* Length of identifier name */
58 unsigned char tainted
:1,
61 char name
[]; /* Actual identifier */
71 TOKEN_CHAR_EMBEDDED_0
,
72 TOKEN_CHAR_EMBEDDED_1
,
73 TOKEN_CHAR_EMBEDDED_2
,
74 TOKEN_CHAR_EMBEDDED_3
,
76 TOKEN_WIDE_CHAR_EMBEDDED_0
,
77 TOKEN_WIDE_CHAR_EMBEDDED_1
,
78 TOKEN_WIDE_CHAR_EMBEDDED_2
,
79 TOKEN_WIDE_CHAR_EMBEDDED_3
,
87 TOKEN_QUOTED_ARGUMENT
,
97 /* Combination tokens */
98 #define COMBINATION_STRINGS { \
110 "<<=", ">>=", "...", \
112 "<", ">", "<=", ">=" \
115 extern unsigned char combinations
[][4];
119 SPECIAL_ADD_ASSIGN
= SPECIAL_BASE
,
143 SPECIAL_ARG_SEPARATOR
,
146 SPECIAL_UNSIGNED_LTE
,
147 SPECIAL_UNSIGNED_GTE
,
155 /* will fit into 32 bits */
164 * This is a very common data structure, it should be kept
165 * as small as humanly possible. Big (rare) types go as
174 unsigned int special
;
175 struct string
*string
;
177 struct argcount count
;
182 #define MAX_STRING 8191
184 static inline struct token
*containing_token(struct token
**p
)
186 void *addr
= (char *)p
- ((char *)&((struct token
*)0)->next
- (char *)0);
190 #define token_type(x) ((x)->pos.type)
193 * Last token in the stream - points to itself.
194 * This allows us to not test for NULL pointers
195 * when following the token->next chain..
197 extern struct token eof_token_entry
;
198 #define eof_token(x) ((x) == &eof_token_entry)
200 extern int init_stream(const char *, int fd
, const char **next_path
);
201 extern const char *stream_name(int stream
);
202 extern struct ident
*hash_ident(struct ident
*);
203 extern struct ident
*built_in_ident(const char *);
204 extern struct token
*built_in_token(int, const char *);
205 extern const char *show_special(int);
206 extern const char *show_ident(const struct ident
*);
207 extern const char *show_string(const struct string
*string
);
208 extern const char *show_token(const struct token
*);
209 extern const char *quote_token(const struct token
*);
210 extern struct token
* tokenize(const char *, int, struct token
*, const char **next_path
);
211 extern struct token
* tokenize_buffer(void *, unsigned long, struct token
**);
213 extern void show_identifier_stats(void);
214 extern struct token
*preprocess(struct token
*);
216 static inline int match_op(struct token
*token
, int op
)
218 return token
->pos
.type
== TOKEN_SPECIAL
&& token
->special
== op
;
221 static inline int match_ident(struct token
*token
, struct ident
*id
)
223 return token
->pos
.type
== TOKEN_IDENT
&& token
->ident
== id
;