cpptoxml/parser/lexer.h

   1 /****************************************************************************
   2 **
   3 ** Copyright (C) 1992-2008 Trolltech ASA. All rights reserved.
   4 ** Copyright (C) 2002-2005 Roberto Raggi <roberto@kdevelop.org>
   5 **
   6 ** This file is part of the Qt Script Generator project on Trolltech Labs.
   7 **
   8 ** This file may be used under the terms of the GNU General Public
   9 ** License version 2.0 as published by the Free Software Foundation
  10 ** and appearing in the file LICENSE.GPL included in the packaging of
  11 ** this file.  Please review the following information to ensure GNU
  12 ** General Public Licensing requirements will be met:
  13 ** http://www.trolltech.com/products/qt/opensource.html
  14 **
  15 ** If you are unsure which license is appropriate for your use, please
  16 ** review the following information:
  17 ** http://www.trolltech.com/products/qt/licensing.html or contact the
  18 ** sales department at sales@trolltech.com.
  19 **
  20 ** This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
  21 ** WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
  22 **
  23 ****************************************************************************/
  24
  25
  26 #ifndef LEXER_H
  27 #define LEXER_H
  28
  29 #include "symbol.h"
  30
  31 #include <QtCore/QString>
  32 #include <cstdlib>
  33 #include <cassert>
  34
  35 struct NameSymbol;
  36 class Lexer;
  37 class Control;
  38
  39 typedef void (Lexer::*scan_fun_ptr)();
  40
  41 class Token
  42 {
  43 public:
  44   int kind;
  45   std::size_t position;
  46   std::size_t size;
  47   char const *text;
  48
  49   union
  50   {
  51     const NameSymbol *symbol;
  52     std::size_t right_brace;
  53   } extra;
  54 };
  55
  56 class LocationTable
  57 {
  58 private:
  59   LocationTable(const LocationTable &source);
  60   void operator = (const LocationTable &source);
  61
  62 public:
  63   inline LocationTable(std::size_t size = 1024)
  64     : lines(0),
  65       line_count(0),
  66       current_line(0)
  67   {
  68     resize(size);
  69   }
  70
  71   inline ~LocationTable()
  72   {
  73     std::free(lines);
  74   }
  75
  76   inline std::size_t size() const
  77   { return line_count; }
  78
  79   void resize(std::size_t size)
  80   {
  81     Q_ASSERT(size > 0);
  82     lines = (std::size_t*) std::realloc(lines, sizeof(std::size_t) * size);
  83     line_count = size;
  84   }
  85
  86   void positionAt(std::size_t offset, int *line, int *column) const
  87   { positionAt(offset, (int) current_line, line, column); }
  88
  89   void positionAt(std::size_t offset, int max_line, int *line, int *column) const;
  90
  91   inline std::size_t &operator[](int index)
  92   { return lines[index]; }
  93
  94 private:
  95   std::size_t *lines;
  96   std::size_t line_count;
  97   std::size_t current_line;
  98
  99   friend class Lexer;
 100 };
 101
 102 class TokenStream
 103 {
 104 private:
 105   TokenStream(const TokenStream &);
 106   void operator = (const TokenStream &);
 107
 108 public:
 109   inline TokenStream(std::size_t size = 1024)
 110      : tokens(0),
 111        index(0),
 112        token_count(0)
 113   {
 114     resize(size);
 115   }
 116
 117   inline ~TokenStream()
 118   { std::free(tokens); }
 119
 120   inline std::size_t size() const
 121   { return token_count; }
 122
 123   inline std::size_t cursor() const
 124   { return index; }
 125
 126   inline void rewind(int i)
 127   { index = i; }
 128
 129   void resize(std::size_t size)
 130   {
 131     Q_ASSERT(size > 0);
 132     tokens = (Token*) std::realloc(tokens, sizeof(Token) * size);
 133     token_count = size;
 134   }
 135
 136   inline std::size_t nextToken()
 137   { return index++; }
 138
 139   inline int lookAhead(std::size_t i = 0) const
 140   { return tokens[index + i].kind; }
 141
 142   inline int kind(std::size_t i) const
 143   { return tokens[i].kind; }
 144
 145   inline std::size_t position(std::size_t i) const
 146   { return tokens[i].position; }
 147
 148   inline const NameSymbol *symbol(std::size_t i) const
 149   { return tokens[i].extra.symbol; }
 150
 151   inline std::size_t matchingBrace(std::size_t i) const
 152   { return tokens[i].extra.right_brace; }
 153
 154   inline Token &operator[](int index)
 155   { return tokens[index]; }
 156
 157   inline const Token &token(int index) const
 158   { return tokens[index]; }
 159
 160 private:
 161   Token *tokens;
 162   std::size_t index;
 163   std::size_t token_count;
 164
 165 private:
 166   friend class Lexer;
 167 };
 168
 169 class LocationManager
 170 {
 171   LocationManager(LocationManager const &__other);
 172   void operator = (LocationManager const &__other);
 173
 174 public:
 175   LocationManager (TokenStream &__token_stream,
 176           LocationTable &__location_table,
 177           LocationTable &__line_table):
 178     token_stream (__token_stream),
 179     location_table (__location_table),
 180     line_table (__line_table) {}
 181
 182   void positionAt(std::size_t offset, int *line, int *column,
 183                   QString *filename) const;
 184
 185   void extract_line(int offset, int *line, QString *filename) const;
 186
 187   TokenStream &token_stream;
 188   LocationTable &location_table;
 189   LocationTable &line_table;
 190 };
 191
 192 class Lexer
 193 {
 194 public:
 195   Lexer(LocationManager &__location, Control *__control):
 196     _M_location(__location),
 197     token_stream(_M_location.token_stream),
 198     location_table(_M_location.location_table),
 199     line_table(_M_location.line_table),
 200     control(__control) {}
 201
 202   void tokenize(const char *contents, std::size_t size);
 203
 204   LocationManager &_M_location;
 205   TokenStream &token_stream;
 206   LocationTable &location_table;
 207   LocationTable &line_table;
 208
 209 private:
 210   void reportError(const QString& msg);
 211
 212   void initialize_scan_table();
 213   void scan_newline();
 214   void scan_white_spaces();
 215   void scan_identifier_or_keyword();
 216   void scan_identifier_or_literal();
 217   void scan_int_constant();
 218   void scan_char_constant();
 219   void scan_string_constant();
 220   void scan_invalid_input();
 221   void scan_preprocessor();
 222
 223   // keywords
 224   void scanKeyword0();
 225   void scanKeyword2();
 226   void scanKeyword3();
 227   void scanKeyword4();
 228   void scanKeyword5();
 229   void scanKeyword6();
 230   void scanKeyword7();
 231   void scanKeyword8();
 232   void scanKeyword9();
 233   void scanKeyword10();
 234   void scanKeyword11();
 235   void scanKeyword12();
 236   void scanKeyword13();
 237   void scanKeyword14();
 238   void scanKeyword16();
 239
 240   // operators
 241   void scan_not();
 242   void scan_remainder();
 243   void scan_and();
 244   void scan_left_paren();
 245   void scan_right_paren();
 246   void scan_star();
 247   void scan_plus();
 248   void scan_comma();
 249   void scan_minus();
 250   void scan_dot();
 251   void scan_divide();
 252   void scan_colon();
 253   void scan_semicolon();
 254   void scan_less();
 255   void scan_equal();
 256   void scan_greater();
 257   void scan_question();
 258   void scan_left_bracket();
 259   void scan_right_bracket();
 260   void scan_xor();
 261   void scan_left_brace();
 262   void scan_or();
 263   void scan_right_brace();
 264   void scan_tilde();
 265   void scan_EOF();
 266
 267 private:
 268   Control *control;
 269   const unsigned char *cursor;
 270   const unsigned char *begin_buffer;
 271   const unsigned char *end_buffer;
 272   std::size_t index;
 273
 274   static scan_fun_ptr s_scan_table[];
 275   static scan_fun_ptr s_scan_keyword_table[];
 276   static bool s_initialized;
 277 };
 278
 279 #endif // LEXER_H
 280
 281 // kate: space-indent on; indent-width 2; replace-tabs on;