tokenizer.h

   1 #ifndef TOKENIZER_H
   2 #define TOKENIZER_H
   3
   4 #define MAX_TOK_LEN 4096
   5 #define MAX_UNGETC 8
   6
   7 #include <stdint.h>
   8 #include <stddef.h>
   9 #include <stdio.h>
  10
  11 struct tokenizer_getc_buf {
  12         int buf[MAX_UNGETC];
  13         size_t cnt, buffered;
  14 };
  15
  16 enum markertype {
  17         MT_SINGLELINE_COMMENT_START = 0,
  18         MT_MULTILINE_COMMENT_START = 1,
  19         MT_MULTILINE_COMMENT_END = 2,
  20         MT_MAX = MT_MULTILINE_COMMENT_END
  21 };
  22
  23 #define MAX_CUSTOM_TOKENS 32
  24
  25 enum tokentype {
  26         TT_IDENTIFIER = 1,
  27         TT_SQSTRING_LIT,
  28         TT_DQSTRING_LIT,
  29         TT_ELLIPSIS,
  30         TT_HEX_INT_LIT,
  31         TT_OCT_INT_LIT,
  32         TT_DEC_INT_LIT,
  33         TT_FLOAT_LIT,
  34         TT_SEP,
  35         /* errors and similar */
  36         TT_UNKNOWN,
  37         TT_OVERFLOW,
  38         TT_WIDECHAR_LIT,
  39         TT_WIDESTRING_LIT,
  40         TT_EOF,
  41         TT_CUSTOM = 1000 /* start user defined tokentype values */
  42 };
  43
  44 const char* tokentype_to_str(enum tokentype tt);
  45
  46 struct token {
  47         enum tokentype type;
  48         uint32_t line;
  49         uint32_t column;
  50         int value;
  51 };
  52
  53 enum tokenizer_flags {
  54         TF_PARSE_STRINGS = 1 << 0,
  55         TF_PARSE_WIDE_STRINGS = 1 << 1,
  56 };
  57
  58 struct tokenizer {
  59         FILE *input;
  60         uint32_t line;
  61         uint32_t column;
  62         int flags;
  63         int custom_count;
  64         int peeking;
  65         const char *custom_tokens[MAX_CUSTOM_TOKENS];
  66         char buf[MAX_TOK_LEN];
  67         size_t bufsize;
  68         struct tokenizer_getc_buf getc_buf;
  69         const char* marker[MT_MAX+1];
  70         const char* filename;
  71         struct token peek_token;
  72 };
  73
  74 void tokenizer_init(struct tokenizer *t, FILE* in, int flags);
  75 void tokenizer_set_filename(struct tokenizer *t, const char*);
  76 void tokenizer_set_flags(struct tokenizer *t, int flags);
  77 int tokenizer_get_flags(struct tokenizer *t);
  78 off_t tokenizer_ftello(struct tokenizer *t);
  79 void tokenizer_register_marker(struct tokenizer*, enum markertype, const char*);
  80 void tokenizer_register_custom_token(struct tokenizer*, int tokentype, const char*);
  81 int tokenizer_next(struct tokenizer *t, struct token* out);
  82 int tokenizer_peek_token(struct tokenizer *t, struct token* out);
  83 int tokenizer_peek(struct tokenizer *t);
  84 void tokenizer_skip_until(struct tokenizer *t, const char *marker);
  85 int tokenizer_skip_chars(struct tokenizer *t, const char *chars, int *count);
  86 int tokenizer_read_until(struct tokenizer *t, const char* marker, int stop_at_nl);
  87 int tokenizer_rewind(struct tokenizer *t);
  88
  89 #ifdef __GNUC__
  90 #pragma GCC diagnostic ignored "-Wunknown-pragmas"
  91 #endif
  92 #pragma RcB2 DEP "tokenizer.c"
  93
  94 #endif
  95