From fd7188fb6601014bac6e2df72b1fccdd7781bb53 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Sun, 6 Dec 2009 20:35:01 +0300 Subject: [PATCH] flex source code of lexers is now available as part of CE source --- .../src/com/intellij/lexer/_JavaLexer.flex | 191 +++++++++++++++++++++ .../src/com/intellij/lexer/_JavaDocLexer.flex | 102 +++++++++++ .../template/impl/TemplateTextLexer.flex | 31 ++++ .../codeInsight/template/impl/_MacroLexer.flex | 38 ++++ .../fileTemplates/impl/FileTemplateTextLexer.flex | 32 ++++ .../scope/packageSet/lexer/_ScopesLexer.flex | 49 ++++++ xml/impl/src/com/intellij/lexer/_HtmlLexer.flex | 157 +++++++++++++++++ xml/impl/src/com/intellij/lexer/_OldXmlLexer.flex | 166 ++++++++++++++++++ xml/impl/src/com/intellij/lexer/_XmlLexer.flex | 178 +++++++++++++++++++ 9 files changed, 944 insertions(+) create mode 100644 java/java-impl/src/com/intellij/lexer/_JavaLexer.flex create mode 100644 platform/lang-api/src/com/intellij/lexer/_JavaDocLexer.flex create mode 100644 platform/lang-impl/src/com/intellij/codeInsight/template/impl/TemplateTextLexer.flex create mode 100644 platform/lang-impl/src/com/intellij/codeInsight/template/impl/_MacroLexer.flex create mode 100644 platform/lang-impl/src/com/intellij/ide/fileTemplates/impl/FileTemplateTextLexer.flex create mode 100644 platform/lang-impl/src/com/intellij/psi/search/scope/packageSet/lexer/_ScopesLexer.flex create mode 100644 xml/impl/src/com/intellij/lexer/_HtmlLexer.flex create mode 100644 xml/impl/src/com/intellij/lexer/_OldXmlLexer.flex create mode 100644 xml/impl/src/com/intellij/lexer/_XmlLexer.flex diff --git a/java/java-impl/src/com/intellij/lexer/_JavaLexer.flex b/java/java-impl/src/com/intellij/lexer/_JavaLexer.flex new file mode 100644 index 0000000000..4e79d1a07a --- /dev/null +++ b/java/java-impl/src/com/intellij/lexer/_JavaLexer.flex @@ -0,0 +1,191 @@ +/* It's an automatically generated code. Do not modify it. */ +package com.intellij.lexer; +import com.intellij.psi.*; +import com.intellij.psi.tree.IElementType; + +%% + +%{ + private boolean myAssertKeywordEnabled; + private boolean myJdk15Enabled; + + public _JavaLexer(boolean isAssertKeywordEnabled, boolean jdk15Enabled){ + this((java.io.Reader)null); + myAssertKeywordEnabled = isAssertKeywordEnabled; + myJdk15Enabled = jdk15Enabled; + } + + public void goTo(int offset) { + zzCurrentPos = zzMarkedPos = zzStartRead = offset; + zzPushbackPos = 0; + zzAtEOF = offset < zzEndRead; + } +%} + +%unicode +%class _JavaLexer +%implements FlexLexer +%function advance +%type IElementType +%eof{ return; +%eof} + +DIGIT=[0-9] +OCTAL_DIGIT=[0-7] +HEX_DIGIT=[0-9A-Fa-f] +WHITE_SPACE_CHAR=[\ \n\r\t\f] + +IDENTIFIER=[:jletter:] [:jletterdigit:]* + +C_STYLE_COMMENT=("/*"[^"*"]{COMMENT_TAIL})|"/*" +DOC_COMMENT="/*""*"+("/"|([^"/""*"]{COMMENT_TAIL}))? +COMMENT_TAIL=([^"*"]*("*"+[^"*""/"])?)*("*"+"/")? +END_OF_LINE_COMMENT="/""/"[^\r\n]* + +INTEGER_LITERAL={DECIMAL_INTEGER_LITERAL}|{HEX_INTEGER_LITERAL}|{OCTAL_INTEGER_LITERAL} +DECIMAL_INTEGER_LITERAL=(0|([1-9]({DIGIT})*)) +HEX_INTEGER_LITERAL=0[Xx]({HEX_DIGIT})* +OCTAL_INTEGER_LITERAL=0({OCTAL_DIGIT})+ +LONG_LITERAL=({INTEGER_LITERAL})[Ll] + +FLOAT_LITERAL=(({FLOATING_POINT_LITERAL1})[Ff])|(({FLOATING_POINT_LITERAL2})[Ff])|(({FLOATING_POINT_LITERAL3})[Ff])|(({FLOATING_POINT_LITERAL4})[Ff]) +DOUBLE_LITERAL=(({FLOATING_POINT_LITERAL1})[Dd]?)|(({FLOATING_POINT_LITERAL2})[Dd]?)|(({FLOATING_POINT_LITERAL3})[Dd]?)|(({FLOATING_POINT_LITERAL4})[Dd]) +FLOATING_POINT_LITERAL1=({DIGIT})+"."({DIGIT})*({EXPONENT_PART})? +FLOATING_POINT_LITERAL2="."({DIGIT})+({EXPONENT_PART})? +FLOATING_POINT_LITERAL3=({DIGIT})+({EXPONENT_PART}) +FLOATING_POINT_LITERAL4=({DIGIT})+ +EXPONENT_PART=[Ee]["+""-"]?({DIGIT})* +HEX_FLOAT_LITERAL={HEX_SIGNIFICAND}{BINARY_EXPONENT}[Ff] +HEX_DOUBLE_LITERAL={HEX_SIGNIFICAND}{BINARY_EXPONENT}[Dd]? +BINARY_EXPONENT=[Pp][+-]?{DIGIT}+ +HEX_SIGNIFICAND={HEX_INTEGER_LITERAL}|{HEX_INTEGER_LITERAL}.|0[Xx]{HEX_DIGIT}*.{HEX_DIGIT}+ + +CHARACTER_LITERAL="'"([^\\\'\r\n]|{ESCAPE_SEQUENCE})*("'"|\\)? +STRING_LITERAL=\"([^\\\"\r\n]|{ESCAPE_SEQUENCE})*(\"|\\)? +ESCAPE_SEQUENCE=\\[^\r\n] + +%% + + {WHITE_SPACE_CHAR}+ { return JavaTokenType.WHITE_SPACE; } + + {C_STYLE_COMMENT} { return JavaTokenType.C_STYLE_COMMENT; } + {END_OF_LINE_COMMENT} { return JavaTokenType.END_OF_LINE_COMMENT; } + {DOC_COMMENT} { return JavaTokenType.DOC_COMMENT; } + + {LONG_LITERAL} { return JavaTokenType.LONG_LITERAL; } + {INTEGER_LITERAL} { return JavaTokenType.INTEGER_LITERAL; } + {FLOAT_LITERAL} { return JavaTokenType.FLOAT_LITERAL; } + {HEX_FLOAT_LITERAL} { if (myJdk15Enabled) return JavaTokenType.FLOAT_LITERAL; } + {DOUBLE_LITERAL} { return JavaTokenType.DOUBLE_LITERAL; } + {HEX_DOUBLE_LITERAL} { if (myJdk15Enabled) return JavaTokenType.DOUBLE_LITERAL; } + + {CHARACTER_LITERAL} { return JavaTokenType.CHARACTER_LITERAL; } + {STRING_LITERAL} { return JavaTokenType.STRING_LITERAL; } + + "true" { return JavaTokenType.TRUE_KEYWORD; } + "false" { return JavaTokenType.FALSE_KEYWORD; } + "null" { return JavaTokenType.NULL_KEYWORD; } + + "abstract" { return JavaTokenType.ABSTRACT_KEYWORD; } + "assert" { return myAssertKeywordEnabled ? JavaTokenType.ASSERT_KEYWORD : JavaTokenType.IDENTIFIER; } + "boolean" { return JavaTokenType.BOOLEAN_KEYWORD; } + "break" { return JavaTokenType.BREAK_KEYWORD; } + "byte" { return JavaTokenType.BYTE_KEYWORD; } + "case" { return JavaTokenType.CASE_KEYWORD; } + "catch" { return JavaTokenType.CATCH_KEYWORD; } + "char" { return JavaTokenType.CHAR_KEYWORD; } + "class" { return JavaTokenType.CLASS_KEYWORD; } + "const" { return JavaTokenType.CONST_KEYWORD; } + "continue" { return JavaTokenType.CONTINUE_KEYWORD; } + "default" { return JavaTokenType.DEFAULT_KEYWORD; } + "do" { return JavaTokenType.DO_KEYWORD; } + "double" { return JavaTokenType.DOUBLE_KEYWORD; } + "else" { return JavaTokenType.ELSE_KEYWORD; } + "enum" { return myJdk15Enabled ? JavaTokenType.ENUM_KEYWORD : JavaTokenType.IDENTIFIER; } + "extends" { return JavaTokenType.EXTENDS_KEYWORD; } + "final" { return JavaTokenType.FINAL_KEYWORD; } + "finally" { return JavaTokenType.FINALLY_KEYWORD; } + "float" { return JavaTokenType.FLOAT_KEYWORD; } + "for" { return JavaTokenType.FOR_KEYWORD; } + "goto" { return JavaTokenType.GOTO_KEYWORD; } + "if" { return JavaTokenType.IF_KEYWORD; } + "implements" { return JavaTokenType.IMPLEMENTS_KEYWORD; } + "import" { return JavaTokenType.IMPORT_KEYWORD; } + "instanceof" { return JavaTokenType.INSTANCEOF_KEYWORD; } + "int" { return JavaTokenType.INT_KEYWORD; } + "interface" { return JavaTokenType.INTERFACE_KEYWORD; } + "long" { return JavaTokenType.LONG_KEYWORD; } + "native" { return JavaTokenType.NATIVE_KEYWORD; } + "new" { return JavaTokenType.NEW_KEYWORD; } + "package" { return JavaTokenType.PACKAGE_KEYWORD; } + "private" { return JavaTokenType.PRIVATE_KEYWORD; } + "public" { return JavaTokenType.PUBLIC_KEYWORD; } + "short" { return JavaTokenType.SHORT_KEYWORD; } + "super" { return JavaTokenType.SUPER_KEYWORD; } + "switch" { return JavaTokenType.SWITCH_KEYWORD; } + "synchronized" { return JavaTokenType.SYNCHRONIZED_KEYWORD; } + "this" { return JavaTokenType.THIS_KEYWORD; } + "throw" { return JavaTokenType.THROW_KEYWORD; } + "protected" { return JavaTokenType.PROTECTED_KEYWORD; } + "transient" { return JavaTokenType.TRANSIENT_KEYWORD; } + "return" { return JavaTokenType.RETURN_KEYWORD; } + "void" { return JavaTokenType.VOID_KEYWORD; } + "static" { return JavaTokenType.STATIC_KEYWORD; } + "strictfp" { return JavaTokenType.STRICTFP_KEYWORD; } + "while" { return JavaTokenType.WHILE_KEYWORD; } + "try" { return JavaTokenType.TRY_KEYWORD; } + "volatile" { return JavaTokenType.VOLATILE_KEYWORD; } + "throws" { return JavaTokenType.THROWS_KEYWORD; } + + {IDENTIFIER} { return JavaTokenType.IDENTIFIER; } + + "==" { return JavaTokenType.EQEQ; } + "!=" { return JavaTokenType.NE; } + "||" { return JavaTokenType.OROR; } + "++" { return JavaTokenType.PLUSPLUS; } + "--" { return JavaTokenType.MINUSMINUS; } + + "<" { return JavaTokenType.LT; } + "<=" { return JavaTokenType.LE; } + "<<=" { return JavaTokenType.LTLTEQ; } + "<<" { return JavaTokenType.LTLT; } + ">" { return JavaTokenType.GT; } + "&" { return JavaTokenType.AND; } + "&&" { return JavaTokenType.ANDAND; } + + "+=" { return JavaTokenType.PLUSEQ; } + "-=" { return JavaTokenType.MINUSEQ; } + "*=" { return JavaTokenType.ASTERISKEQ; } + "/=" { return JavaTokenType.DIVEQ; } + "&=" { return JavaTokenType.ANDEQ; } + "|=" { return JavaTokenType.OREQ; } + "^=" { return JavaTokenType.XOREQ; } + "%=" { return JavaTokenType.PERCEQ; } + + "(" { return JavaTokenType.LPARENTH; } + ")" { return JavaTokenType.RPARENTH; } + "{" { return JavaTokenType.LBRACE; } + "}" { return JavaTokenType.RBRACE; } + "[" { return JavaTokenType.LBRACKET; } + "]" { return JavaTokenType.RBRACKET; } + ";" { return JavaTokenType.SEMICOLON; } + "," { return JavaTokenType.COMMA; } + "..." { return JavaTokenType.ELLIPSIS; } + "." { return JavaTokenType.DOT; } + + "=" { return JavaTokenType.EQ; } + "!" { return JavaTokenType.EXCL; } + "~" { return JavaTokenType.TILDE; } + "?" { return JavaTokenType.QUEST; } + ":" { return JavaTokenType.COLON; } + "+" { return JavaTokenType.PLUS; } + "-" { return JavaTokenType.MINUS; } + "*" { return JavaTokenType.ASTERISK; } + "/" { return JavaTokenType.DIV; } + "|" { return JavaTokenType.OR; } + "^" { return JavaTokenType.XOR; } + "%" { return JavaTokenType.PERC; } + "@" { return JavaTokenType.AT; } + + . { return JavaTokenType.BAD_CHARACTER; } + diff --git a/platform/lang-api/src/com/intellij/lexer/_JavaDocLexer.flex b/platform/lang-api/src/com/intellij/lexer/_JavaDocLexer.flex new file mode 100644 index 0000000000..143c67e424 --- /dev/null +++ b/platform/lang-api/src/com/intellij/lexer/_JavaDocLexer.flex @@ -0,0 +1,102 @@ +/* It's an automatically generated code. Do not modify it. */ +package com.intellij.lexer; + +import com.intellij.psi.tree.IElementType; +import com.intellij.psi.*; + +%% + +%{ + private boolean myJdk15Enabled; + private DocCommentTokenTypes myTokenTypes; + + public _JavaDocLexer(boolean isJdk15Enabled, DocCommentTokenTypes tokenTypes) { + this((java.io.Reader)null); + myJdk15Enabled = isJdk15Enabled; + myTokenTypes = tokenTypes; + } + + public boolean checkAhead(char c) { + if (zzMarkedPos >= zzBuffer.length()) return false; + return zzBuffer.charAt(zzMarkedPos) == c; + } + + public void goTo(int offset) { + zzCurrentPos = zzMarkedPos = zzStartRead = offset; + zzPushbackPos = 0; + zzAtEOF = offset < zzEndRead; + } +%} + +%class _JavaDocLexer +%implements FlexLexer +%unicode +%function advance +%type IElementType +%eof{ return; +%eof} + +%state COMMENT_DATA_START +%state COMMENT_DATA +%state TAG_DOC_SPACE +%state PARAM_TAG_SPACE +%state DOC_TAG_VALUE +%state DOC_TAG_VALUE_IN_PAREN +%state DOC_TAG_VALUE_IN_LTGT +%state INLINE_TAG_NAME + +WHITE_DOC_SPACE_CHAR=[\ \t\f\n\r] +WHITE_DOC_SPACE_NO_LR=[\ \t\f] +DIGIT=[0-9] +ALPHA=[:jletter:] +IDENTIFIER={ALPHA}({ALPHA}|{DIGIT}|[":.-"])* + +%% + + "/**/" { return myTokenTypes.commentEnd(); } + "/**" { yybegin(COMMENT_DATA_START); return myTokenTypes.commentStart(); } + {WHITE_DOC_SPACE_CHAR}+ { return myTokenTypes.space(); } + {WHITE_DOC_SPACE_NO_LR}+ { return myTokenTypes.commentData(); } + [\n\r]+{WHITE_DOC_SPACE_CHAR}* { return myTokenTypes.space(); } + + {WHITE_DOC_SPACE_CHAR}+ { yybegin(COMMENT_DATA); return myTokenTypes.space(); } + ({ALPHA}|[_0-9\."$"\[\]])+ { return myTokenTypes.tagValueToken(); } + [\(] { yybegin(DOC_TAG_VALUE_IN_PAREN); return myTokenTypes.tagValueLParen(); } + [\)] { yybegin(DOC_TAG_VALUE); return myTokenTypes.tagValueRParen(); } + [#] { return myTokenTypes.tagValueSharp(); } + [,] { return myTokenTypes.tagValueComma(); } + {WHITE_DOC_SPACE_CHAR}+ { return myTokenTypes.space(); } + + "@param" { yybegin(PARAM_TAG_SPACE); return myTokenTypes.tagName(); } + {WHITE_DOC_SPACE_CHAR}+ {yybegin(DOC_TAG_VALUE); return myTokenTypes.space();} + [\<] { if (myJdk15Enabled) {yybegin(DOC_TAG_VALUE_IN_LTGT); return myTokenTypes.tagValueLT();} else { yybegin(COMMENT_DATA); return myTokenTypes.commentData(); } } + {IDENTIFIER} { return myTokenTypes.tagValueToken(); } + {IDENTIFIER} { return myTokenTypes.tagValueToken(); } + [\>] { yybegin(COMMENT_DATA); return myTokenTypes.tagValueGT(); } + + "{" { + if (checkAhead('@')){ + yybegin(INLINE_TAG_NAME); + return myTokenTypes.inlineTagStart(); + } + else{ + yybegin(COMMENT_DATA); + return myTokenTypes.inlineTagStart(); + } +} + "@"{IDENTIFIER} { yybegin(TAG_DOC_SPACE); return myTokenTypes.tagName(); } + "}" { yybegin(COMMENT_DATA); return myTokenTypes.inlineTagEnd(); } + + . { yybegin(COMMENT_DATA); return myTokenTypes.commentData(); } + "@"{IDENTIFIER} { yybegin(TAG_DOC_SPACE); return myTokenTypes.tagName(); } + {WHITE_DOC_SPACE_CHAR}+ { + + if (checkAhead('<') || checkAhead('\"')) yybegin(COMMENT_DATA); + else if (checkAhead('\u007b') ) yybegin(COMMENT_DATA); //lbrace - there's some error in JLex when typing lbrace directly + else yybegin(DOC_TAG_VALUE); + + return myTokenTypes.space(); +} + +"*"+"/" { return myTokenTypes.commentEnd(); } +[^] { return myTokenTypes.badCharacter(); } diff --git a/platform/lang-impl/src/com/intellij/codeInsight/template/impl/TemplateTextLexer.flex b/platform/lang-impl/src/com/intellij/codeInsight/template/impl/TemplateTextLexer.flex new file mode 100644 index 0000000000..cbcb3b50b2 --- /dev/null +++ b/platform/lang-impl/src/com/intellij/codeInsight/template/impl/TemplateTextLexer.flex @@ -0,0 +1,31 @@ +/* It's an automatically generated code. Do not modify it. */ +package com.intellij.codeInsight.template.impl; + +import com.intellij.lexer.FlexLexer; +import com.intellij.psi.tree.IElementType; + +%% + +%{ + public _TemplateTextLexer() { + this((java.io.Reader)null); + } +%} + +%unicode +%class _TemplateTextLexer +%implements FlexLexer +%function advance +%type IElementType +%eof{ return; +%eof} + +ALPHA=[A-Za-z_] +DIGIT=[0-9] +VARIABLE="$"({ALPHA}|{DIGIT})+"$" + +%% + + "$""$" { return TemplateTokenType.ESCAPE_DOLLAR; } + {VARIABLE} { return TemplateTokenType.VARIABLE; } + [^] { return TemplateTokenType.TEXT; } diff --git a/platform/lang-impl/src/com/intellij/codeInsight/template/impl/_MacroLexer.flex b/platform/lang-impl/src/com/intellij/codeInsight/template/impl/_MacroLexer.flex new file mode 100644 index 0000000000..a1e448c7ca --- /dev/null +++ b/platform/lang-impl/src/com/intellij/codeInsight/template/impl/_MacroLexer.flex @@ -0,0 +1,38 @@ +/* It's an automatically generated code. Do not modify it. */ +package com.intellij.codeInsight.template.impl; + +import com.intellij.psi.TokenType; +import com.intellij.psi.tree.IElementType; +import com.intellij.lexer.FlexLexer; + +%% + +%{ + public _MacroLexer() { + this((java.io.Reader)null); + } +%} + +%unicode +%class _MacroLexer +%implements FlexLexer +%function advance +%type IElementType +%eof{ return; +%eof} + +IDENTIFIER=[:jletter:] [:jletterdigit:]* +WHITE_SPACE_CHAR=[\ \n\r\t\f] +STRING_LITERAL=\"([^\\\"\r\n]|{ESCAPE_SEQUENCE})*(\"|\\)? +ESCAPE_SEQUENCE=\\[^\r\n] + +%% + +{IDENTIFIER} { return MacroTokenType.IDENTIFIER; } +{WHITE_SPACE_CHAR}+ { return MacroTokenType.WHITE_SPACE; } +{STRING_LITERAL} { return MacroTokenType.STRING_LITERAL; } +"(" { return MacroTokenType.LPAREN; } +")" { return MacroTokenType.RPAREN; } +"," { return MacroTokenType.COMMA; } +"=" { return MacroTokenType.EQ; } +. { return TokenType.BAD_CHARACTER; } diff --git a/platform/lang-impl/src/com/intellij/ide/fileTemplates/impl/FileTemplateTextLexer.flex b/platform/lang-impl/src/com/intellij/ide/fileTemplates/impl/FileTemplateTextLexer.flex new file mode 100644 index 0000000000..022fdc8a29 --- /dev/null +++ b/platform/lang-impl/src/com/intellij/ide/fileTemplates/impl/FileTemplateTextLexer.flex @@ -0,0 +1,32 @@ +/* It's an automatically generated code. Do not modify it. */ +package com.intellij.ide.fileTemplates.impl; + +import com.intellij.lexer.FlexLexer; +import com.intellij.psi.tree.IElementType; + +%% + +%{ + public FileTemplateTextLexer() { + this((java.io.Reader)null); + } +%} + +%unicode +%class FileTemplateTextLexer +%implements FlexLexer +%function advance +%type IElementType +%eof{ return; +%eof} + +ALPHA=[A-Za-z_] +DIGIT=[0-9] +MACRO="$"({ALPHA}|{DIGIT})+|"$""{"({ALPHA}|{DIGIT})+"}" +DIRECTIVE="#"{ALPHA}+ + +%% + + {MACRO} { return FileTemplateTokenType.MACRO; } + {DIRECTIVE} { return FileTemplateTokenType.DIRECTIVE; } + [^] { return FileTemplateTokenType.TEXT; } diff --git a/platform/lang-impl/src/com/intellij/psi/search/scope/packageSet/lexer/_ScopesLexer.flex b/platform/lang-impl/src/com/intellij/psi/search/scope/packageSet/lexer/_ScopesLexer.flex new file mode 100644 index 0000000000..7c850c3806 --- /dev/null +++ b/platform/lang-impl/src/com/intellij/psi/search/scope/packageSet/lexer/_ScopesLexer.flex @@ -0,0 +1,49 @@ +/* It's an automatically generated code. Do not modify it. */ +package com.intellij.psi.search.scope.packageSet.lexer; +import com.intellij.psi.tree.IElementType; +import com.intellij.lexer.FlexLexer; + +%% + +%{ + public void goTo(int offset) { + zzCurrentPos = zzMarkedPos = zzStartRead = offset; + zzPushbackPos = 0; + zzAtEOF = offset < zzEndRead; + } +%} + +%unicode +%class _ScopesLexer +%implements FlexLexer +%function advance +%type IElementType +%eof{ return; +%eof} + +IDENTIFIER=[:jletter:] [:jletterdigit:]* +WHITE_SPACE_CHAR=[\ \n\r\t\f] +DIGIT=[0-9]+ + +%% + {IDENTIFIER} { return ScopeTokenTypes.IDENTIFIER; } + {WHITE_SPACE_CHAR}+ { return ScopeTokenTypes.WHITE_SPACE; } + {DIGIT}+ { return ScopeTokenTypes.INTEGER_LITERAL; } + "||" { return ScopeTokenTypes.OROR; } + "&&" { return ScopeTokenTypes.ANDAND; } + "!" { return ScopeTokenTypes.EXCL; } + "$" { return ScopeTokenTypes.IDENTIFIER;} + "-" { return ScopeTokenTypes.MINUS;} + + "[" { return ScopeTokenTypes.LBRACKET; } + "]" { return ScopeTokenTypes.RBRACKET; } + "(" { return ScopeTokenTypes.LPARENTH; } + ")" { return ScopeTokenTypes.RPARENTH; } + "." { return ScopeTokenTypes.DOT; } + + ":" { return ScopeTokenTypes.COLON; } + "*" { return ScopeTokenTypes.ASTERISK; } + "/" { return ScopeTokenTypes.DIV; } + + . { return ScopeTokenTypes.BAD_CHARACTER; } + diff --git a/xml/impl/src/com/intellij/lexer/_HtmlLexer.flex b/xml/impl/src/com/intellij/lexer/_HtmlLexer.flex new file mode 100644 index 0000000000..8def2be62d --- /dev/null +++ b/xml/impl/src/com/intellij/lexer/_HtmlLexer.flex @@ -0,0 +1,157 @@ +/* It's an automatically generated code. Do not modify it. */ +package com.intellij.lexer; + +import com.intellij.psi.tree.IElementType; +import com.intellij.psi.*; +import com.intellij.psi.xml.*; + +%% + +%unicode + +%{ + private IElementType elTokenType = XmlTokenType.XML_DATA_CHARACTERS; + private IElementType elTokenType2 = XmlTokenType.XML_ATTRIBUTE_VALUE_TOKEN; + + public void setElTypes(IElementType _elTokenType,IElementType _elTokenType2) { + elTokenType = _elTokenType; + elTokenType2 = _elTokenType2; + } + + public _HtmlLexer() { + this((java.io.Reader)null); + } +%} + +%class _HtmlLexer +%public +%implements FlexLexer,ELHostLexer +%function advance +%type IElementType +%eof{ return; +%eof} + +%state DOC_TYPE +%state COMMENT +%state START_TAG_NAME +%state END_TAG_NAME +%state TAG_ATTRIBUTES +%state ATTRIBUTE_VALUE_START +%state ATTRIBUTE_VALUE_DQ +%state ATTRIBUTE_VALUE_SQ +%state PROCESSING_INSTRUCTION +%state START_TAG_NAME2 +%state END_TAG_NAME2 +%state TAG_CHARACTERS +%state C_COMMENT_START +%state C_COMMENT_END +/* IMPORTANT! number of states should not exceed 16. See JspHighlightingLexer. */ + +ALPHA=[:letter:] +DIGIT=[0-9] +WHITE_SPACE_CHARS=[ \n\r\t\f]+ + +TAG_NAME=({ALPHA}|"_"|":")({ALPHA}|{DIGIT}|"_"|":"|"."|"-")* +TAG_NAME_FWT=("#")({ALPHA}|{DIGIT}|"_"|":"|"."|"-")* +ATTRIBUTE_NAME=({ALPHA}|"_"|":")({ALPHA}|{DIGIT}|"_"|":"|"."|"-")* + +DTD_REF= "\"" [^\"]* "\"" | "'" [^']* "'" +DOCTYPE= "" + +CONDITIONAL_COMMENT_CONDITION=({ALPHA})({ALPHA}|{WHITE_SPACE_CHARS}|{DIGIT}|"."|"("|")"|"|"|"!"|"&")* +%% + + " "?"? ">" { yybegin(YYINITIAL); return XmlTokenType.XML_PI_END; } + ([^\?\>] | (\?[^\>]))* { return XmlTokenType.XML_PI_TARGET; } + + {DOCTYPE} { yybegin(DOC_TYPE); return XmlTokenType.XML_DOCTYPE_START; } + {HTML} { return XmlTokenType.XML_NAME; } + {PUBLIC} { return XmlTokenType.XML_DOCTYPE_PUBLIC; } + {DTD_REF} { return XmlTokenType.XML_ATTRIBUTE_VALUE_TOKEN;} + ">" { yybegin(YYINITIAL); return XmlTokenType.XML_DOCTYPE_END; } + {WHITE_SPACE_CHARS} { return XmlTokenType.XML_REAL_WHITE_SPACE; } + {WHITE_SPACE_CHARS} { return XmlTokenType.XML_WHITE_SPACE; } + "<" {TAG_NAME} { yybegin(START_TAG_NAME); yypushback(yylength()); } + "<" {TAG_NAME_FWT} { yybegin(START_TAG_NAME2); yypushback(yylength()); } + "<" { return XmlTokenType.XML_START_TAG_START; } + + " " " "" { yybegin(YYINITIAL); return XmlTokenType.XML_COMMENT_END; } + ([^\-]|(-[^\-]))* { return XmlTokenType.XML_COMMENT_CHARACTERS; } + +{S} { return XmlTokenType.XML_WHITE_SPACE; } + + [^<&] { return XmlTokenType.XML_DATA_CHARACTERS; } + +[^] { return XmlTokenType.XML_BAD_CHARACTER; } diff --git a/xml/impl/src/com/intellij/lexer/_XmlLexer.flex b/xml/impl/src/com/intellij/lexer/_XmlLexer.flex new file mode 100644 index 0000000000..5e4d5d2d7b --- /dev/null +++ b/xml/impl/src/com/intellij/lexer/_XmlLexer.flex @@ -0,0 +1,178 @@ + /* It's an automatically generated code. Do not modify it. */ +package com.intellij.lexer; + +import com.intellij.psi.tree.IElementType; +import com.intellij.psi.*; +import com.intellij.psi.xml.*; + +%% + +%{ + private IElementType elTokenType = XmlTokenType.XML_DATA_CHARACTERS; + private IElementType elTokenType2 = XmlTokenType.XML_ATTRIBUTE_VALUE_TOKEN; + private IElementType javaEmbeddedTokenType = XmlTokenType.XML_ATTRIBUTE_VALUE_TOKEN; + private boolean myConditionalCommentsSupport; + + public void setConditionalCommentsSupport(final boolean b) { + myConditionalCommentsSupport = b; + } + + public void setElTypes(IElementType _elTokenType,IElementType _elTokenType2) { + elTokenType = _elTokenType; + elTokenType2 = _elTokenType2; + } + + public void setJavaEmbeddedType(IElementType _tokenType) { + javaEmbeddedTokenType = _tokenType; + } + + private int myPrevState = YYINITIAL; + + public int yyprevstate() { + return myPrevState; + } + + private int popState(){ + final int prev = myPrevState; + myPrevState = YYINITIAL; + return prev; + } + + protected void pushState(int state){ + myPrevState = state; + } +%} + +%unicode +%class __XmlLexer +%public +%implements FlexLexer +%function advance +%type IElementType +%eof{ return; +%eof} + +%state TAG +%state PROCESSING_INSTRUCTION +%state PI_ANY +%state END_TAG +%xstate COMMENT +%state ATTR_LIST +%state ATTR +%state ATTR_VALUE_START +%state ATTR_VALUE_DQ +%state ATTR_VALUE_SQ +%state DTD_MARKUP +%state DOCTYPE +%xstate CDATA +%state C_COMMENT_START +%state C_COMMENT_END + +ALPHA=[:letter:] +DIGIT=[0-9] +WS=[\ \n\r\t\f] +S={WS}+ + +EL_EMBEDMENT_START="${" | "#{" +NAME=({ALPHA}|"_")({ALPHA}|{DIGIT}|"_"|"."|"-")*(":"({ALPHA}|"_")?({ALPHA}|{DIGIT}|"_"|"."|"-")*)? + +END_COMMENT="-->" +CONDITIONAL_COMMENT_CONDITION=({ALPHA})({ALPHA}|{S}|{DIGIT}|"."|"("|")"|"|"|"!"|"&")* + +%% +"{ + "]]>" {yybegin(YYINITIAL); return XmlTokenType.XML_CDATA_END; } + [^] {return XmlTokenType.XML_DATA_CHARACTERS; } +} + +"