token.h

   1 #ifndef TOKEN_H
   2 #define TOKEN_H
   3 /*
   4  * Basic tokenization structures. NOTE! Those tokens had better
   5  * be pretty small, since we're going to keep them all in memory
   6  * indefinitely.
   7  *
   8  * Copyright (C) 2003 Linus Torvalds, all rights reserved.
   9  */
  10
  11 #include <sys/types.h>
  12
  13 /*
  14  * This describes the pure lexical elements (tokens), with
  15  * no semantic meaning. In other words, an identifier doesn't
  16  * have a type or meaning, it is only a specific string in
  17  * the input stream.
  18  *
  19  * Semantic meaning is handled elsewhere.
  20  */
  21
  22 struct stream {
  23         int fd;
  24         const char *name;
  25
  26         /* Use these to check for "already parsed" */
  27         int constant, nesting;
  28         struct ident *protect;
  29         dev_t dev;
  30         ino_t ino;
  31 };
  32
  33 extern int input_stream_nr;
  34 extern struct stream *input_streams;
  35
  36 extern int ident_hit, ident_miss;
  37
  38 struct ident {
  39         struct ident *next;     /* Hash chain of identifiers */
  40         struct symbol *symbols; /* Pointer to semantic meaning list */
  41         unsigned char len;      /* Length of identifier name */
  42         char name[];            /* Actual identifier */
  43 };
  44
  45 enum token_type {
  46         TOKEN_EOF,
  47         TOKEN_ERROR,
  48         TOKEN_IDENT,
  49         TOKEN_INTEGER,
  50         TOKEN_FP,
  51         TOKEN_CHAR,
  52         TOKEN_STRING,
  53         TOKEN_SPECIAL,
  54         TOKEN_STREAMBEGIN,
  55         TOKEN_STREAMEND,
  56 };
  57
  58 /* Combination tokens */
  59 #define COMBINATION_STRINGS {   \
  60         "+=", "++",             \
  61         "-=", "--", "->",       \
  62         "*=",                   \
  63         "/=", "/*", "//",       \
  64         "%=",                   \
  65         "..", "...",            \
  66         "<=", "<<", "<<=",      \
  67         ">=", ">>", ">>=",      \
  68         "==", "!=",             \
  69         "&&", "&=",             \
  70         "||", "|=",             \
  71         "^=", "##",             \
  72         " @ ",                  \
  73 }
  74
  75 enum special_token {
  76         SPECIAL_BASE = 256,
  77         SPECIAL_ADD_ASSIGN = 256,
  78         SPECIAL_INCREMENT,
  79         SPECIAL_MINUS_ASSIGN,
  80         SPECIAL_DECREMENT,
  81         SPECIAL_DEREFERENCE,
  82         SPECIAL_TIMES_ASSIGN,
  83         SPECIAL_DIV_ASSIGN,
  84         SPECIAL_COMMENT,
  85         SPECIAL_CPPCOMMENT,
  86         SPECIAL_MOD_ASSIGN,
  87         SPECIAL_DOTDOT,
  88         SPECIAL_ELLIPSIS,
  89         SPECIAL_LTE,
  90         SPECIAL_LEFTSHIFT,
  91         SPECIAL_SHL_ASSIGN,
  92         SPECIAL_GTE,
  93         SPECIAL_RIGHTSHIFT,
  94         SPECIAL_SHR_ASSIGN,
  95         SPECIAL_EQUAL,
  96         SPECIAL_NOTEQUAL,
  97         SPECIAL_LOGICAL_AND,
  98         SPECIAL_AND_ASSIGN,
  99         SPECIAL_LOGICAL_OR,
 100         SPECIAL_OR_ASSIGN,
 101         SPECIAL_XOR_ASSIGN,
 102         SPECIAL_HASHHASH,
 103         SPECIAL_ARG_SEPARATOR
 104 };
 105
 106 struct string {
 107         unsigned int length;
 108         char data[];
 109 };
 110
 111 /*
 112  * This is a very common data structure, it should be kept
 113  * as small as humanly possible. Big (rare) types go as
 114  * pointers.
 115  */
 116 struct token {
 117         unsigned int type:8,
 118                      stream:8,
 119                      pos:14,
 120                      newline:1,
 121                      whitespace:1;
 122         unsigned int line;
 123         struct token *next;
 124         union {
 125                 char *integer;
 126                 char *fp;
 127                 struct ident *ident;
 128                 unsigned int special;
 129                 struct string *string;
 130                 int character;
 131         };
 132 };
 133
 134 /*
 135  * Last token in the stream - points to itself.
 136  * This allows us to not test for NULL pointers
 137  * when following the token->next chain..
 138  */
 139 extern struct token eof_token_entry;
 140 #define eof_token(x) ((x) == &eof_token_entry)
 141
 142 extern int init_stream(const char *, int fd);
 143 extern struct ident *hash_ident(struct ident *);
 144 extern struct ident *built_in_ident(const char *);
 145 extern struct token *built_in_token(int, const char *);
 146 extern const char *show_special(int);
 147 extern const char *show_ident(const struct ident *);
 148 extern const char *show_token(const struct token *);
 149 extern struct token * tokenize(const char *, int, struct token *end);
 150 extern void die(const char *, ...);
 151 extern void show_identifier_stats(void);
 152 extern struct token *preprocess(struct token *);
 153
 154 static inline int match_op(struct token *token, int op)
 155 {
 156         return token->type == TOKEN_SPECIAL && token->special == op;
 157 }
 158
 159 static inline int match_ident(struct token *token, struct ident *id)
 160 {
 161         return token->type == TOKEN_IDENT && token->ident == id;
 162 }
 163
 164 #endif