4 * Basic tokenization structures. NOTE! Those tokens had better
5 * be pretty small, since we're going to keep them all in memory
8 * Copyright (C) 2003 Transmeta Corp.
11 * Permission is hereby granted, free of charge, to any person obtaining a copy
12 * of this software and associated documentation files (the "Software"), to deal
13 * in the Software without restriction, including without limitation the rights
14 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
15 * copies of the Software, and to permit persons to whom the Software is
16 * furnished to do so, subject to the following conditions:
18 * The above copyright notice and this permission notice shall be included in
19 * all copies or substantial portions of the Software.
21 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
24 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
30 #include <sys/types.h>
34 * This describes the pure lexical elements (tokens), with
35 * no semantic meaning. In other words, an identifier doesn't
36 * have a type or meaning, it is only a specific string in
39 * Semantic meaning is handled elsewhere.
43 CONSTANT_FILE_MAYBE
, // To be determined, not inside any #ifs in this file
44 CONSTANT_FILE_IFNDEF
, // To be determined, currently inside #ifndef
45 CONSTANT_FILE_NOPE
, // No
46 CONSTANT_FILE_YES
// Yes
49 extern const char *includepath
[];
54 const char *path
; // input-file path - see set_stream_include_path()
55 const char **next_path
;
57 /* Use these to check for "already parsed" */
58 enum constantfile constant
;
59 int dirty
, next_stream
, once
;
60 struct ident
*protect
;
65 extern int input_stream_nr
;
66 extern struct stream
*input_streams
;
67 extern unsigned int tabstop
;
69 extern int *hash_stream(const char *name
);
72 struct ident
*next
; /* Hash chain of identifiers */
73 struct symbol
*symbols
; /* Pointer to semantic meaning list */
74 unsigned char len
; /* Length of identifier name */
75 unsigned char tainted
:1,
78 char name
[]; /* Actual identifier */
89 TOKEN_CHAR_EMBEDDED_0
,
90 TOKEN_CHAR_EMBEDDED_1
,
91 TOKEN_CHAR_EMBEDDED_2
,
92 TOKEN_CHAR_EMBEDDED_3
,
94 TOKEN_WIDE_CHAR_EMBEDDED_0
,
95 TOKEN_WIDE_CHAR_EMBEDDED_1
,
96 TOKEN_WIDE_CHAR_EMBEDDED_2
,
97 TOKEN_WIDE_CHAR_EMBEDDED_3
,
103 TOKEN_MACRO_ARGUMENT
,
105 TOKEN_QUOTED_ARGUMENT
,
115 /* Combination tokens */
116 #define COMBINATION_STRINGS { \
128 "<<=", ">>=", "...", \
130 "<", ">", "<=", ">=" \
133 extern unsigned char combinations
[][4];
137 SPECIAL_ADD_ASSIGN
= SPECIAL_BASE
,
161 SPECIAL_ARG_SEPARATOR
,
164 SPECIAL_UNSIGNED_LTE
,
165 SPECIAL_UNSIGNED_GTE
,
169 unsigned int length
:31;
170 unsigned int immutable
:1;
174 /* will fit into 32 bits */
183 * This is a very common data structure, it should be kept
184 * as small as humanly possible. Big (rare) types go as
193 unsigned int special
;
194 struct string
*string
;
196 struct argcount count
;
201 #define MAX_STRING 8191
203 static inline struct token
*containing_token(struct token
**p
)
205 void *addr
= (char *)p
- ((char *)&((struct token
*)0)->next
- (char *)0);
209 #define token_type(x) ((x)->pos.type)
212 * Last token in the stream - points to itself.
213 * This allows us to not test for NULL pointers
214 * when following the token->next chain..
216 extern struct token eof_token_entry
;
217 #define eof_token(x) ((x) == &eof_token_entry)
219 extern int init_stream(const char *, int fd
, const char **next_path
);
220 extern const char *stream_name(int stream
);
221 extern struct ident
*hash_ident(struct ident
*);
222 extern struct ident
*built_in_ident(const char *);
223 extern struct token
*built_in_token(int, struct ident
*);
224 extern const char *show_special(int);
225 extern const char *show_ident(const struct ident
*);
226 extern const char *show_string(const struct string
*string
);
227 extern const char *show_token(const struct token
*);
228 extern const char *quote_token(const struct token
*);
229 extern struct token
* tokenize(const char *, int, struct token
*, const char **next_path
);
230 extern struct token
* tokenize_buffer(void *, unsigned long, struct token
**);
232 extern void show_identifier_stats(void);
233 extern void init_include_path(void);
234 extern struct token
*preprocess(struct token
*);
236 extern void store_all_tokens(struct token
*token
);
237 extern struct token
*pos_get_token(struct position pos
);
238 extern char *pos_ident(struct position pos
);
240 extern void store_macro_pos(struct token
*);
241 extern char *get_macro_name(struct position pos
);
242 extern char *get_inner_macro(struct position pos
);
243 extern struct string_list
*get_all_macros(struct position pos
);
245 static inline int match_op(struct token
*token
, unsigned int op
)
247 return token
->pos
.type
== TOKEN_SPECIAL
&& token
->special
== op
;
250 static inline int match_ident(struct token
*token
, struct ident
*id
)
252 return token
->pos
.type
== TOKEN_IDENT
&& token
->ident
== id
;