token.h

   1 #ifndef TOKEN_H
   2 #define TOKEN_H
   3 /*
   4  * Basic tokenization structures. NOTE! Those tokens had better
   5  * be pretty small, since we're going to keep them all in memory
   6  * indefinitely.
   7  *
   8  * Copyright (C) 2003 Transmeta Corp.
   9  *               2003 Linus Torvalds
  10  *
  11  * Permission is hereby granted, free of charge, to any person obtaining a copy
  12  * of this software and associated documentation files (the "Software"), to deal
  13  * in the Software without restriction, including without limitation the rights
  14  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  15  * copies of the Software, and to permit persons to whom the Software is
  16  * furnished to do so, subject to the following conditions:
  17  *
  18  * The above copyright notice and this permission notice shall be included in
  19  * all copies or substantial portions of the Software.
  20  *
  21  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  22  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  23  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  24  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  25  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  26  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  27  * THE SOFTWARE.
  28  */
  29
  30 #include <sys/types.h>
  31 #include "lib.h"
  32
  33 /*
  34  * This describes the pure lexical elements (tokens), with
  35  * no semantic meaning. In other words, an identifier doesn't
  36  * have a type or meaning, it is only a specific string in
  37  * the input stream.
  38  *
  39  * Semantic meaning is handled elsewhere.
  40  */
  41
  42 enum constantfile {
  43   CONSTANT_FILE_MAYBE,    // To be determined, not inside any #ifs in this file
  44   CONSTANT_FILE_IFNDEF,   // To be determined, currently inside #ifndef
  45   CONSTANT_FILE_NOPE,     // No
  46   CONSTANT_FILE_YES       // Yes
  47 };
  48
  49 extern const char *includepath[];
  50
  51 struct stream {
  52         int fd;
  53         const char *name;
  54         const char *path;    // input-file path - see set_stream_include_path()
  55         const char **next_path;
  56         struct position pos; //position of the #include, if any
  57
  58         /* Use these to check for "already parsed" */
  59         enum constantfile constant;
  60         int dirty, next_stream, once;
  61         struct ident *protect;
  62         struct token *ifndef;
  63         struct token *top_if;
  64 };
  65
  66 extern int input_stream_nr;
  67 extern struct stream *input_streams;
  68 extern unsigned int tabstop;
  69 extern int no_lineno;
  70 extern int *hash_stream(const char *name);
  71
  72 struct ident {
  73         struct ident *next;     /* Hash chain of identifiers */
  74         struct symbol *symbols; /* Pointer to semantic meaning list */
  75         unsigned char len;      /* Length of identifier name */
  76         unsigned char tainted:1,
  77                       reserved:1,
  78                       keyword:1;
  79         char name[];            /* Actual identifier */
  80 };
  81
  82 enum token_type {
  83         TOKEN_EOF,
  84         TOKEN_BAD,
  85         TOKEN_ERROR,
  86         TOKEN_IDENT,
  87         TOKEN_ZERO_IDENT,
  88         TOKEN_NUMBER,
  89         TOKEN_CHAR,
  90         TOKEN_CHAR_EMBEDDED_0,
  91         TOKEN_CHAR_EMBEDDED_1,
  92         TOKEN_CHAR_EMBEDDED_2,
  93         TOKEN_CHAR_EMBEDDED_3,
  94         TOKEN_WIDE_CHAR,
  95         TOKEN_WIDE_CHAR_EMBEDDED_0,
  96         TOKEN_WIDE_CHAR_EMBEDDED_1,
  97         TOKEN_WIDE_CHAR_EMBEDDED_2,
  98         TOKEN_WIDE_CHAR_EMBEDDED_3,
  99         TOKEN_STRING,
 100         TOKEN_WIDE_STRING,
 101         TOKEN_SPECIAL,
 102         TOKEN_STREAMBEGIN,
 103         TOKEN_STREAMEND,
 104         TOKEN_MACRO_ARGUMENT,
 105         TOKEN_STR_ARGUMENT,
 106         TOKEN_QUOTED_ARGUMENT,
 107         TOKEN_CONCAT,
 108         TOKEN_GNU_KLUDGE,
 109         TOKEN_UNTAINT,
 110         TOKEN_ARG_COUNT,
 111         TOKEN_IF,
 112         TOKEN_SKIP_GROUPS,
 113         TOKEN_ELSE,
 114 };
 115
 116 /* Combination tokens */
 117 #define COMBINATION_STRINGS {   \
 118         "+=", "++",             \
 119         "-=", "--", "->",       \
 120         "*=",                   \
 121         "/=",                   \
 122         "%=",                   \
 123         "<=", ">=",             \
 124         "==", "!=",             \
 125         "&&", "&=",             \
 126         "||", "|=",             \
 127         "^=", "##",             \
 128         "<<", ">>", "..",       \
 129         "<<=", ">>=", "...",    \
 130         "",                     \
 131         "<", ">", "<=", ">="    \
 132 }
 133
 134 extern unsigned char combinations[][4];
 135
 136 enum special_token {
 137         SPECIAL_BASE = 256,
 138         SPECIAL_ADD_ASSIGN = SPECIAL_BASE,
 139         SPECIAL_INCREMENT,
 140         SPECIAL_SUB_ASSIGN,
 141         SPECIAL_DECREMENT,
 142         SPECIAL_DEREFERENCE,
 143         SPECIAL_MUL_ASSIGN,
 144         SPECIAL_DIV_ASSIGN,
 145         SPECIAL_MOD_ASSIGN,
 146         SPECIAL_LTE,
 147         SPECIAL_GTE,
 148         SPECIAL_EQUAL,
 149         SPECIAL_NOTEQUAL,
 150         SPECIAL_LOGICAL_AND,
 151         SPECIAL_AND_ASSIGN,
 152         SPECIAL_LOGICAL_OR,
 153         SPECIAL_OR_ASSIGN,
 154         SPECIAL_XOR_ASSIGN,
 155         SPECIAL_HASHHASH,
 156         SPECIAL_LEFTSHIFT,
 157         SPECIAL_RIGHTSHIFT,
 158         SPECIAL_DOTDOT,
 159         SPECIAL_SHL_ASSIGN,
 160         SPECIAL_SHR_ASSIGN,
 161         SPECIAL_ELLIPSIS,
 162         SPECIAL_ARG_SEPARATOR,
 163         SPECIAL_UNSIGNED_LT,
 164         SPECIAL_UNSIGNED_GT,
 165         SPECIAL_UNSIGNED_LTE,
 166         SPECIAL_UNSIGNED_GTE,
 167 };
 168
 169 struct string {
 170         unsigned int length:31;
 171         unsigned int immutable:1;
 172         char data[];
 173 };
 174
 175 /* will fit into 32 bits */
 176 struct argcount {
 177         unsigned normal:10;
 178         unsigned quoted:10;
 179         unsigned str:10;
 180         unsigned vararg:1;
 181 };
 182
 183 /*
 184  * This is a very common data structure, it should be kept
 185  * as small as humanly possible. Big (rare) types go as
 186  * pointers.
 187  */
 188 struct token {
 189         struct position pos;
 190         struct token *next;
 191         union {
 192                 const char *number;
 193                 struct ident *ident;
 194                 unsigned int special;
 195                 struct string *string;
 196                 int argnum;
 197                 struct argcount count;
 198                 char embedded[4];
 199         };
 200 };
 201
 202 #define MAX_STRING 8191
 203
 204 static inline struct token *containing_token(struct token **p)
 205 {
 206         void *addr = (char *)p - ((char *)&((struct token *)0)->next - (char *)0);
 207         return addr;
 208 }
 209
 210 #define token_type(x) ((x)->pos.type)
 211
 212 /*
 213  * Last token in the stream - points to itself.
 214  * This allows us to not test for NULL pointers
 215  * when following the token->next chain..
 216  */
 217 extern struct token eof_token_entry;
 218 #define eof_token(x) ((x) == &eof_token_entry)
 219
 220 extern int init_stream(const struct position *pos, const char *, int fd, const char **next_path);
 221 extern int stream_prev(int stream);
 222 extern const char *stream_name(int stream);
 223 struct ident *alloc_ident(const char *name, int len);
 224 extern struct ident *hash_ident(struct ident *);
 225 extern struct ident *built_in_ident(const char *);
 226 extern struct token *built_in_token(int, struct ident *);
 227 extern const char *show_special(int);
 228 extern const char *show_ident(const struct ident *);
 229 extern const char *show_string(const struct string *string);
 230 extern const char *show_token(const struct token *);
 231 extern const char *quote_token(const struct token *);
 232 extern struct token * tokenize(const struct position *pos, const char *, int, struct token *, const char **next_path);
 233 extern struct token * tokenize_buffer(void *, unsigned long, struct token **);
 234
 235 extern void show_identifier_stats(void);
 236 extern void init_include_path(void);
 237 extern struct token *preprocess(struct token *);
 238
 239 extern void store_all_tokens(struct token *token);
 240 extern struct token *pos_get_token(struct position pos);
 241 extern char *pos_ident(struct position pos);
 242
 243 extern void store_macro_pos(struct token *);
 244 extern char *get_macro_name(struct position pos);
 245 extern char *get_inner_macro(struct position pos);
 246 extern struct string_list *get_all_macros(struct position pos);
 247
 248 static inline int match_op(struct token *token, unsigned int op)
 249 {
 250         return token->pos.type == TOKEN_SPECIAL && token->special == op;
 251 }
 252
 253 static inline int match_ident(struct token *token, struct ident *id)
 254 {
 255         return token->pos.type == TOKEN_IDENT && token->ident == id;
 256 }
 257
 258 static inline int match_token_zero(struct token *token)
 259 {
 260         if (token_type(token) != TOKEN_NUMBER)
 261                 return false;
 262         return token->number[0] == '0' && !token->number[1];
 263 }
 264
 265 #endif