token.h

   1 #ifndef TOKEN_H
   2 #define TOKEN_H
   3 /*
   4  * Basic tokenization structures. NOTE! Those tokens had better
   5  * be pretty small, since we're going to keep them all in memory
   6  * indefinitely.
   7  *
   8  * Copyright (C) 2003 Transmeta Corp.
   9  *               2003 Linus Torvalds
  10  *
  11  *  Licensed under the Open Software License version 1.1
  12  */
  13
  14 #include <sys/types.h>
  15 #include "lib.h"
  16
  17 /*
  18  * This describes the pure lexical elements (tokens), with
  19  * no semantic meaning. In other words, an identifier doesn't
  20  * have a type or meaning, it is only a specific string in
  21  * the input stream.
  22  *
  23  * Semantic meaning is handled elsewhere.
  24  */
  25
  26 struct stream {
  27         int fd;
  28         const char *name;
  29
  30         /* Use these to check for "already parsed" */
  31         int constant, nesting;
  32         struct ident *protect;
  33         dev_t dev;
  34         ino_t ino;
  35 };
  36
  37 extern int input_stream_nr;
  38 extern struct stream *input_streams;
  39
  40 extern int ident_hit, ident_miss;
  41
  42 struct ident {
  43         struct ident *next;     /* Hash chain of identifiers */
  44         struct symbol *symbols; /* Pointer to semantic meaning list */
  45         unsigned char len;      /* Length of identifier name */
  46         unsigned char tainted;
  47         char name[];            /* Actual identifier */
  48 };
  49
  50 enum token_type {
  51         TOKEN_EOF,
  52         TOKEN_ERROR,
  53         TOKEN_IDENT,
  54         TOKEN_NUMBER,
  55         TOKEN_CHAR,
  56         TOKEN_STRING,
  57         TOKEN_SPECIAL,
  58         TOKEN_STREAMBEGIN,
  59         TOKEN_STREAMEND,
  60         TOKEN_MACRO_ARGUMENT,
  61         TOKEN_STR_ARGUMENT,
  62         TOKEN_QUOTED_ARGUMENT,
  63         TOKEN_CONCAT,
  64         TOKEN_GNU_KLUDGE,
  65         TOKEN_UNTAINT,
  66         TOKEN_ARG_COUNT,
  67 };
  68
  69 /* Combination tokens */
  70 #define COMBINATION_STRINGS {   \
  71         "+=", "++",             \
  72         "-=", "--", "->",       \
  73         "*=",                   \
  74         "/=", "/*", "//",       \
  75         "%=",                   \
  76         "..", "...",            \
  77         "<=", "<<", "<<=",      \
  78         ">=", ">>", ">>=",      \
  79         "==", "!=",             \
  80         "&&", "&=",             \
  81         "||", "|=",             \
  82         "^=", "##",             \
  83         " @ ",                  \
  84 }
  85
  86 enum special_token {
  87         SPECIAL_BASE = 256,
  88         SPECIAL_ADD_ASSIGN = 256,
  89         SPECIAL_INCREMENT,
  90         SPECIAL_SUB_ASSIGN,
  91         SPECIAL_DECREMENT,
  92         SPECIAL_DEREFERENCE,
  93         SPECIAL_MUL_ASSIGN,
  94         SPECIAL_DIV_ASSIGN,
  95         SPECIAL_COMMENT,
  96         SPECIAL_CPPCOMMENT,
  97         SPECIAL_MOD_ASSIGN,
  98         SPECIAL_DOTDOT,
  99         SPECIAL_ELLIPSIS,
 100         SPECIAL_LTE,
 101         SPECIAL_LEFTSHIFT,
 102         SPECIAL_SHL_ASSIGN,
 103         SPECIAL_GTE,
 104         SPECIAL_RIGHTSHIFT,
 105         SPECIAL_SHR_ASSIGN,
 106         SPECIAL_EQUAL,
 107         SPECIAL_NOTEQUAL,
 108         SPECIAL_LOGICAL_AND,
 109         SPECIAL_AND_ASSIGN,
 110         SPECIAL_LOGICAL_OR,
 111         SPECIAL_OR_ASSIGN,
 112         SPECIAL_XOR_ASSIGN,
 113         SPECIAL_HASHHASH,
 114         SPECIAL_ARG_SEPARATOR
 115 };
 116
 117 struct string {
 118         unsigned int length;
 119         char data[];
 120 };
 121
 122 /* will fit into 32 bits */
 123 struct argcount {
 124         unsigned normal:10;
 125         unsigned quoted:10;
 126         unsigned str:10;
 127         unsigned vararg:1;
 128 };
 129
 130 /*
 131  * This is a very common data structure, it should be kept
 132  * as small as humanly possible. Big (rare) types go as
 133  * pointers.
 134  */
 135 struct token {
 136         struct position pos;
 137         struct token *next;
 138         union {
 139                 char *number;
 140                 struct ident *ident;
 141                 unsigned int special;
 142                 struct string *string;
 143                 int character;
 144                 int argnum;
 145                 struct argcount count;
 146         };
 147 };
 148
 149 static inline struct token *containing_token(struct token **p)
 150 {
 151         void *addr = (char*)p - ((char*)&((struct token *)0)->next - (char*)0);
 152         return addr;
 153 }
 154
 155 #define token_type(x) ((x)->pos.type)
 156
 157 /*
 158  * Last token in the stream - points to itself.
 159  * This allows us to not test for NULL pointers
 160  * when following the token->next chain..
 161  */
 162 extern int preprocessing, verbose;
 163 extern struct token eof_token_entry;
 164 #define eof_token(x) ((x) == &eof_token_entry)
 165
 166 extern int init_stream(const char *, int fd);
 167 extern struct ident *hash_ident(struct ident *);
 168 extern struct ident *built_in_ident(const char *);
 169 extern struct token *built_in_token(int, const char *);
 170 extern const char *show_special(int);
 171 extern const char *show_ident(const struct ident *);
 172 extern const char *show_string(const struct string *string);
 173 extern const char *show_token(const struct token *);
 174 extern struct token * tokenize(const char *, int, struct token *);
 175 extern struct token * tokenize_buffer(unsigned char *, unsigned long, struct token *);
 176
 177 extern void die(const char *, ...);
 178 extern void show_identifier_stats(void);
 179 extern struct token *preprocess(struct token *);
 180
 181 static inline int match_op(struct token *token, int op)
 182 {
 183         return token->pos.type == TOKEN_SPECIAL && token->special == op;
 184 }
 185
 186 static inline int match_ident(struct token *token, struct ident *id)
 187 {
 188         return token->pos.type == TOKEN_IDENT && token->ident == id;
 189 }
 190
 191 #endif