tools/qdoc3/tokenizer.h

   1 /****************************************************************************
   2 **
   3 ** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
   4 ** All rights reserved.
   5 ** Contact: Nokia Corporation (qt-info@nokia.com)
   6 **
   7 ** This file is part of the tools applications of the Qt Toolkit.
   8 **
   9 ** $QT_BEGIN_LICENSE:LGPL$
  10 ** No Commercial Usage
  11 ** This file contains pre-release code and may not be distributed.
  12 ** You may use this file in accordance with the terms and conditions
  13 ** contained in the Technology Preview License Agreement accompanying
  14 ** this package.
  15 **
  16 ** GNU Lesser General Public License Usage
  17 ** Alternatively, this file may be used under the terms of the GNU Lesser
  18 ** General Public License version 2.1 as published by the Free Software
  19 ** Foundation and appearing in the file LICENSE.LGPL included in the
  20 ** packaging of this file.  Please review the following information to
  21 ** ensure the GNU Lesser General Public License version 2.1 requirements
  22 ** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
  23 **
  24 ** In addition, as a special exception, Nokia gives you certain additional
  25 ** rights.  These rights are described in the Nokia Qt LGPL Exception
  26 ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
  27 **
  28 ** If you have questions regarding the use of this file, please contact
  29 ** Nokia at qt-info@nokia.com.
  30 **
  31 **
  32 **
  33 **
  34 **
  35 **
  36 **
  37 **
  38 ** $QT_END_LICENSE$
  39 **
  40 ****************************************************************************/
  41
  42 /*
  43   tokenizer.h
  44 */
  45
  46 #ifndef TOKENIZER_H
  47 #define TOKENIZER_H
  48
  49 #include <qstack.h>
  50 #include <qstring.h>
  51
  52 #include <stdio.h>
  53
  54 #include "location.h"
  55
  56 QT_BEGIN_NAMESPACE
  57
  58 /*
  59   Here come the C++ tokens we support.  The first part contains
  60   all-purpose tokens; then come keywords.
  61
  62   If you add a keyword, make sure to modify the keyword array in
  63   tokenizer.cpp as well, and possibly adjust Tok_FirstKeyword and
  64   Tok_LastKeyword.
  65 */
  66 enum { Tok_Eoi, Tok_Ampersand, Tok_Aster, Tok_Caret, Tok_LeftParen,
  67        Tok_RightParen, Tok_LeftParenAster, Tok_Equal, Tok_LeftBrace,
  68        Tok_RightBrace, Tok_Semicolon, Tok_Colon, Tok_LeftAngle,
  69        Tok_RightAngle, Tok_Comma, Tok_Ellipsis, Tok_Gulbrandsen,
  70        Tok_LeftBracket, Tok_RightBracket, Tok_Tilde, Tok_SomeOperator,
  71        Tok_Number, Tok_String, Tok_Doc, Tok_Comment, Tok_Ident, Tok_At,
  72        Tok_char, Tok_class, Tok_const, Tok_double, Tok_enum,
  73        Tok_explicit, Tok_friend, Tok_inline, Tok_int, Tok_long,
  74        Tok_namespace, Tok_operator, Tok_private, Tok_protected,
  75        Tok_public, Tok_short, Tok_signals, Tok_signed, Tok_slots,
  76        Tok_static, Tok_struct, Tok_template, Tok_typedef,
  77        Tok_typename, Tok_union, Tok_unsigned, Tok_using, Tok_virtual,
  78        Tok_void, Tok_volatile, Tok_int64, Tok_Q_OBJECT, Tok_Q_OVERRIDE,
  79        Tok_Q_PROPERTY, Tok_Q_DECLARE_SEQUENTIAL_ITERATOR,
  80        Tok_Q_DECLARE_MUTABLE_SEQUENTIAL_ITERATOR,
  81        Tok_Q_DECLARE_ASSOCIATIVE_ITERATOR,
  82        Tok_Q_DECLARE_MUTABLE_ASSOCIATIVE_ITERATOR,
  83        Tok_Q_DECLARE_FLAGS, Tok_Q_SIGNALS, Tok_Q_SLOTS, Tok_QT_COMPAT,
  84        Tok_QT_COMPAT_CONSTRUCTOR, Tok_QT_DEPRECATED, Tok_QT_MOC_COMPAT,
  85        Tok_QT_MODULE, Tok_QT3_SUPPORT, Tok_QT3_SUPPORT_CONSTRUCTOR,
  86        Tok_QT3_MOC_SUPPORT, Tok_QDOC_PROPERTY,
  87        Tok_FirstKeyword = Tok_char, Tok_LastKeyword = Tok_QDOC_PROPERTY };
  88
  89 /*
  90   The Tokenizer class implements lexical analysis of C++ source
  91   files.
  92
  93   Not every operator or keyword of C++ is recognized; only those
  94   that are interesting to us. Some Qt keywords or macros are also
  95   recognized.
  96 */
  97
  98 class Tokenizer
  99 {
 100  public:
 101     Tokenizer(const Location& loc, const QByteArray &in);
 102     Tokenizer(const Location& loc, FILE *in);
 103
 104     ~Tokenizer();
 105
 106     int getToken();
 107     void setParsingFnOrMacro(bool macro) { parsingMacro = macro; }
 108     bool parsingFnOrMacro() const { return parsingMacro; }
 109
 110     const Location &location() const { return yyTokLoc; }
 111     QString previousLexeme() const { return QString(yyPrevLex); }
 112     QString lexeme() const { return QString(yyLex); }
 113     QString version() const { return yyVersion; }
 114     int braceDepth() const { return yyBraceDepth; }
 115     int parenDepth() const { return yyParenDepth; }
 116     int bracketDepth() const { return yyBracketDepth; }
 117
 118     static void initialize(const Config &config);
 119     static void terminate();
 120     static bool isTrue(const QString &condition);
 121
 122  private:
 123     void init();
 124     void start(const Location& loc);
 125     /*
 126       This limit on the length of a lexeme seems fairly high, but a
 127       doc comment can be arbitrarily long. The previous 65,536 limit
 128       was reached by Mark Summerfield.
 129     */
 130     enum { yyLexBufSize = 524288 };
 131
 132     int getch()
 133     {
 134         return yyPos == yyIn.size() ? EOF : yyIn[yyPos++];
 135     }
 136
 137     inline int getChar()
 138     {
 139         if (yyCh == EOF)
 140             return EOF;
 141         if (yyLexLen < yyLexBufSize - 1) {
 142             yyLex[yyLexLen++] = (char) yyCh;
 143             yyLex[yyLexLen] = '\0';
 144         }
 145         yyCurLoc.advance(yyCh);
 146         int ch = getch();
 147         if (ch == EOF)
 148             return EOF;
 149         // cast explicitely to make sure the value of ch
 150         // is in range [0..255] to avoid assert messages
 151         // when using debug CRT that checks its input.
 152         return int(uint(uchar(ch)));
 153     }
 154
 155     int getTokenAfterPreprocessor();
 156     void pushSkipping(bool skip);
 157     bool popSkipping();
 158
 159     Location yyTokLoc;
 160     Location yyCurLoc;
 161     char *yyLexBuf1;
 162     char *yyLexBuf2;
 163     char *yyPrevLex;
 164     char *yyLex;
 165     size_t yyLexLen;
 166     QStack<bool> yyPreprocessorSkipping;
 167     int yyNumPreprocessorSkipping;
 168     int yyBraceDepth;
 169     int yyParenDepth;
 170     int yyBracketDepth;
 171     int yyCh;
 172
 173     QString yyVersion;
 174     bool parsingMacro;
 175
 176  protected:
 177     QByteArray yyIn;
 178     int yyPos;
 179 };
 180
 181 QT_END_NAMESPACE
 182
 183 #endif