token.h

   1 #ifndef TOKEN_H
   2 #define TOKEN_H
   3 /*
   4  * Basic tokenization structures. NOTE! Those tokens had better
   5  * be pretty small, since we're going to keep them all in memory
   6  * indefinitely.
   7  *
   8  * Copyright (C) 2003 Transmeta Corp.
   9  *
  10  *  Licensed under the Open Software License version 1.1
  11  */
  12
  13 #include <sys/types.h>
  14 #include "lib.h"
  15
  16 /*
  17  * This describes the pure lexical elements (tokens), with
  18  * no semantic meaning. In other words, an identifier doesn't
  19  * have a type or meaning, it is only a specific string in
  20  * the input stream.
  21  *
  22  * Semantic meaning is handled elsewhere.
  23  */
  24
  25 struct stream {
  26         int fd;
  27         const char *name;
  28
  29         /* Use these to check for "already parsed" */
  30         int constant, nesting;
  31         struct ident *protect;
  32         dev_t dev;
  33         ino_t ino;
  34 };
  35
  36 extern int input_stream_nr;
  37 extern struct stream *input_streams;
  38
  39 extern int ident_hit, ident_miss;
  40
  41 struct ident {
  42         struct ident *next;     /* Hash chain of identifiers */
  43         struct symbol *symbols; /* Pointer to semantic meaning list */
  44         unsigned char len;      /* Length of identifier name */
  45         char name[];            /* Actual identifier */
  46 };
  47
  48 enum token_type {
  49         TOKEN_EOF,
  50         TOKEN_ERROR,
  51         TOKEN_IDENT,
  52         TOKEN_INTEGER,
  53         TOKEN_FP,
  54         TOKEN_CHAR,
  55         TOKEN_STRING,
  56         TOKEN_SPECIAL,
  57         TOKEN_STREAMBEGIN,
  58         TOKEN_STREAMEND,
  59 };
  60
  61 /* Combination tokens */
  62 #define COMBINATION_STRINGS {   \
  63         "+=", "++",             \
  64         "-=", "--", "->",       \
  65         "*=",                   \
  66         "/=", "/*", "//",       \
  67         "%=",                   \
  68         "..", "...",            \
  69         "<=", "<<", "<<=",      \
  70         ">=", ">>", ">>=",      \
  71         "==", "!=",             \
  72         "&&", "&=",             \
  73         "||", "|=",             \
  74         "^=", "##",             \
  75         " @ ",                  \
  76 }
  77
  78 enum special_token {
  79         SPECIAL_BASE = 256,
  80         SPECIAL_ADD_ASSIGN = 256,
  81         SPECIAL_INCREMENT,
  82         SPECIAL_SUB_ASSIGN,
  83         SPECIAL_DECREMENT,
  84         SPECIAL_DEREFERENCE,
  85         SPECIAL_MUL_ASSIGN,
  86         SPECIAL_DIV_ASSIGN,
  87         SPECIAL_COMMENT,
  88         SPECIAL_CPPCOMMENT,
  89         SPECIAL_MOD_ASSIGN,
  90         SPECIAL_DOTDOT,
  91         SPECIAL_ELLIPSIS,
  92         SPECIAL_LTE,
  93         SPECIAL_LEFTSHIFT,
  94         SPECIAL_SHL_ASSIGN,
  95         SPECIAL_GTE,
  96         SPECIAL_RIGHTSHIFT,
  97         SPECIAL_SHR_ASSIGN,
  98         SPECIAL_EQUAL,
  99         SPECIAL_NOTEQUAL,
 100         SPECIAL_LOGICAL_AND,
 101         SPECIAL_AND_ASSIGN,
 102         SPECIAL_LOGICAL_OR,
 103         SPECIAL_OR_ASSIGN,
 104         SPECIAL_XOR_ASSIGN,
 105         SPECIAL_HASHHASH,
 106         SPECIAL_ARG_SEPARATOR
 107 };
 108
 109 struct string {
 110         unsigned int length;
 111         char data[];
 112 };
 113
 114 /*
 115  * This is a very common data structure, it should be kept
 116  * as small as humanly possible. Big (rare) types go as
 117  * pointers.
 118  */
 119 struct token {
 120         struct position pos;
 121         struct token *next;
 122         struct token *parent;
 123         union {
 124                 char *integer;
 125                 char *fp;
 126                 struct ident *ident;
 127                 unsigned int special;
 128                 struct string *string;
 129                 int character;
 130         };
 131 };
 132
 133 #define token_type(x) ((x)->pos.type)
 134
 135 /*
 136  * Last token in the stream - points to itself.
 137  * This allows us to not test for NULL pointers
 138  * when following the token->next chain..
 139  */
 140 extern int preprocessing, verbose;
 141 extern struct token eof_token_entry;
 142 #define eof_token(x) ((x) == &eof_token_entry)
 143
 144 extern int init_stream(const char *, int fd);
 145 extern struct ident *hash_ident(struct ident *);
 146 extern struct ident *built_in_ident(const char *);
 147 extern struct token *built_in_token(int, const char *);
 148 extern const char *show_special(int);
 149 extern const char *show_ident(const struct ident *);
 150 extern const char *show_string(const struct string *string);
 151 extern const char *show_token(const struct token *);
 152 extern struct token * tokenize(const char *, int, struct token *);
 153 extern struct token * tokenize_buffer(unsigned char *, unsigned long, struct token *);
 154
 155 extern void die(const char *, ...);
 156 extern void show_identifier_stats(void);
 157 extern struct token *preprocess(struct token *);
 158
 159 static inline int match_op(struct token *token, int op)
 160 {
 161         return token->pos.type == TOKEN_SPECIAL && token->special == op;
 162 }
 163
 164 static inline int match_ident(struct token *token, struct ident *id)
 165 {
 166         return token->pos.type == TOKEN_IDENT && token->ident == id;
 167 }
 168
 169 #endif