token.h

   1 #ifndef TOKEN_H
   2 #define TOKEN_H
   3 /*
   4  * Basic tokenization structures. NOTE! Those tokens had better
   5  * be pretty small, since we're going to keep them all in memory
   6  * indefinitely.
   7  *
   8  * Copyright (C) 2003 Transmeta Corp.
   9  *               2003 Linus Torvalds
  10  *
  11  *  Licensed under the Open Software License version 1.1
  12  */
  13
  14 #include <sys/types.h>
  15 #include "lib.h"
  16
  17 /*
  18  * This describes the pure lexical elements (tokens), with
  19  * no semantic meaning. In other words, an identifier doesn't
  20  * have a type or meaning, it is only a specific string in
  21  * the input stream.
  22  *
  23  * Semantic meaning is handled elsewhere.
  24  */
  25
  26 enum constantfile {
  27   CONSTANT_FILE_MAYBE,    // To be determined, not inside any #ifs in this file
  28   CONSTANT_FILE_IFNDEF,   // To be determined, currently inside #ifndef
  29   CONSTANT_FILE_NOPE,     // No
  30   CONSTANT_FILE_YES       // Yes
  31 };
  32
  33 struct stream {
  34         int fd;
  35         const char *name;
  36
  37         /* Use these to check for "already parsed" */
  38         enum constantfile constant;
  39         int nesting;
  40         struct ident *protect;
  41         dev_t dev;
  42         ino_t ino;
  43 };
  44
  45 extern int input_stream_nr;
  46 extern struct stream *input_streams;
  47
  48 extern int ident_hit, ident_miss;
  49
  50 struct ident {
  51         struct ident *next;     /* Hash chain of identifiers */
  52         struct symbol *symbols; /* Pointer to semantic meaning list */
  53         unsigned char len;      /* Length of identifier name */
  54         unsigned char tainted;
  55         char name[];            /* Actual identifier */
  56 };
  57
  58 enum token_type {
  59         TOKEN_EOF,
  60         TOKEN_ERROR,
  61         TOKEN_IDENT,
  62         TOKEN_NUMBER,
  63         TOKEN_CHAR,
  64         TOKEN_STRING,
  65         TOKEN_SPECIAL,
  66         TOKEN_STREAMBEGIN,
  67         TOKEN_STREAMEND,
  68         TOKEN_MACRO_ARGUMENT,
  69         TOKEN_STR_ARGUMENT,
  70         TOKEN_QUOTED_ARGUMENT,
  71         TOKEN_CONCAT,
  72         TOKEN_GNU_KLUDGE,
  73         TOKEN_UNTAINT,
  74         TOKEN_ARG_COUNT,
  75 };
  76
  77 /* Combination tokens */
  78 #define COMBINATION_STRINGS {   \
  79         "+=", "++",             \
  80         "-=", "--", "->",       \
  81         "*=",                   \
  82         "/=",                   \
  83         "%=",                   \
  84         "..", "...",            \
  85         "<=", "<<", "<<=",      \
  86         ">=", ">>", ">>=",      \
  87         "==", "!=",             \
  88         "&&", "&=",             \
  89         "||", "|=",             \
  90         "^=", "##",             \
  91         "",                     \
  92         "<", ">", "<=", ">="    \
  93 }
  94
  95 enum special_token {
  96         SPECIAL_BASE = 256,
  97         SPECIAL_ADD_ASSIGN = SPECIAL_BASE,
  98         SPECIAL_INCREMENT,
  99         SPECIAL_SUB_ASSIGN,
 100         SPECIAL_DECREMENT,
 101         SPECIAL_DEREFERENCE,
 102         SPECIAL_MUL_ASSIGN,
 103         SPECIAL_DIV_ASSIGN,
 104         SPECIAL_MOD_ASSIGN,
 105         SPECIAL_DOTDOT,
 106         SPECIAL_ELLIPSIS,
 107         SPECIAL_LTE,
 108         SPECIAL_LEFTSHIFT,
 109         SPECIAL_SHL_ASSIGN,
 110         SPECIAL_GTE,
 111         SPECIAL_RIGHTSHIFT,
 112         SPECIAL_SHR_ASSIGN,
 113         SPECIAL_EQUAL,
 114         SPECIAL_NOTEQUAL,
 115         SPECIAL_LOGICAL_AND,
 116         SPECIAL_AND_ASSIGN,
 117         SPECIAL_LOGICAL_OR,
 118         SPECIAL_OR_ASSIGN,
 119         SPECIAL_XOR_ASSIGN,
 120         SPECIAL_HASHHASH,
 121         SPECIAL_ARG_SEPARATOR,
 122         SPECIAL_UNSIGNED_LT,
 123         SPECIAL_UNSIGNED_GT,
 124         SPECIAL_UNSIGNED_LTE,
 125         SPECIAL_UNSIGNED_GTE,
 126 };
 127
 128 struct string {
 129         unsigned int length;
 130         char data[];
 131 };
 132
 133 /* will fit into 32 bits */
 134 struct argcount {
 135         unsigned normal:10;
 136         unsigned quoted:10;
 137         unsigned str:10;
 138         unsigned vararg:1;
 139 };
 140
 141 /*
 142  * This is a very common data structure, it should be kept
 143  * as small as humanly possible. Big (rare) types go as
 144  * pointers.
 145  */
 146 struct token {
 147         struct position pos;
 148         struct token *next;
 149         union {
 150                 char *number;
 151                 struct ident *ident;
 152                 unsigned int special;
 153                 struct string *string;
 154                 int character;
 155                 int argnum;
 156                 struct argcount count;
 157         };
 158 };
 159
 160 #define MAX_STRING 4095
 161
 162 static inline struct token *containing_token(struct token **p)
 163 {
 164         void *addr = (char*)p - ((char*)&((struct token *)0)->next - (char*)0);
 165         return addr;
 166 }
 167
 168 #define token_type(x) ((x)->pos.type)
 169
 170 /*
 171  * Last token in the stream - points to itself.
 172  * This allows us to not test for NULL pointers
 173  * when following the token->next chain..
 174  */
 175 extern int preprocessing, verbose;
 176 extern struct token eof_token_entry;
 177 #define eof_token(x) ((x) == &eof_token_entry)
 178
 179 extern int init_stream(const char *, int fd);
 180 extern struct ident *hash_ident(struct ident *);
 181 extern struct ident *built_in_ident(const char *);
 182 extern struct token *built_in_token(int, const char *);
 183 extern const char *show_special(int);
 184 extern const char *show_ident(const struct ident *);
 185 extern const char *show_string(const struct string *string);
 186 extern const char *show_token(const struct token *);
 187 extern struct token * tokenize(const char *, int, struct token *);
 188 extern struct token * tokenize_buffer(unsigned char *, unsigned long, struct token *);
 189
 190 extern void die(const char *, ...);
 191 extern void show_identifier_stats(void);
 192 extern struct token *preprocess(struct token *);
 193
 194 static inline int match_op(struct token *token, int op)
 195 {
 196         return token->pos.type == TOKEN_SPECIAL && token->special == op;
 197 }
 198
 199 static inline int match_ident(struct token *token, struct ident *id)
 200 {
 201         return token->pos.type == TOKEN_IDENT && token->ident == id;
 202 }
 203
 204 #endif