java/java-impl/src/com/intellij/psi/impl/source/parsing/ParseUtil.java

   1 /*
   2  * Copyright 2000-2009 JetBrains s.r.o.
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  * http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16 package com.intellij.psi.impl.source.parsing;
  17
  18 import com.intellij.lang.ASTFactory;
  19 import com.intellij.lang.ASTNode;
  20 import com.intellij.lexer.JavaLexer;
  21 import com.intellij.lexer.Lexer;
  22 import com.intellij.lexer.LexerUtil;
  23 import com.intellij.openapi.util.text.StringUtil;
  24 import com.intellij.psi.JavaTokenType;
  25 import com.intellij.psi.TokenType;
  26 import com.intellij.psi.impl.source.ParsingContext;
  27 import com.intellij.psi.impl.source.tree.*;
  28 import com.intellij.psi.impl.source.tree.java.ModifierListElement;
  29 import com.intellij.psi.jsp.AbstractJspJavaLexer;
  30 import com.intellij.psi.tree.IElementType;
  31 import com.intellij.psi.tree.TokenSet;
  32 import com.intellij.util.CharTable;
  33 import com.intellij.util.SmartList;
  34
  35 import java.util.Iterator;
  36 import java.util.List;
  37
  38 public class ParseUtil {
  39   public static TreeElement createTokenElement(Lexer lexer, CharTable table) {
  40     IElementType tokenType = lexer.getTokenType();
  41     if (tokenType == null) return null;
  42     if (tokenType == JavaTokenType.DOC_COMMENT) {
  43       return ASTFactory.lazy(JavaDocElementType.DOC_COMMENT, LexerUtil.internToken(lexer, table));
  44     }
  45     else {
  46       return ASTFactory.leaf(tokenType, LexerUtil.internToken(lexer, table));
  47     }
  48   }
  49
  50   /*public static void insertMissingTokens(CompositeElement root,
  51                                          Lexer lexer,
  52                                          int startOffset,
  53                                          int endOffset,
  54                                          final int state, TokenProcessor processor, ParsingContext context) {
  55     insertMissingTokens(root, lexer, startOffset, endOffset, -1, processor, context);
  56   }*/
  57
  58   public static void insertMissingTokens(CompositeElement root,
  59                                          Lexer lexer,
  60                                          int startOffset,
  61                                          int endOffset,
  62                                          int state,
  63                                          TokenProcessor processor,
  64                                          ParsingContext context) {
  65     final MissingTokenInserter inserter;
  66     if (lexer instanceof JavaLexer || lexer instanceof AbstractJspJavaLexer) {
  67       inserter = new JavaMissingTokenInserter(root, lexer, startOffset, endOffset, state, processor, context);
  68     }
  69     else {
  70       inserter = new MissingTokenInserter(root, lexer, startOffset, endOffset, state, processor, context);
  71     }
  72     inserter.invoke();
  73   }
  74
  75
  76   private static class JavaMissingTokenInserter extends MissingTokenInserter {
  77
  78     public JavaMissingTokenInserter(final CompositeElement root, final Lexer lexer, final int startOffset, final int endOffset, final int state,
  79                                     final TokenProcessor processor,
  80                                     final ParsingContext context) {
  81       super(root, lexer, startOffset, endOffset, state, processor, context);
  82     }
  83
  84     @Override
  85     public void invoke() {
  86       super.invoke();
  87       bindComments(myRoot);
  88     }
  89
  90     @Override
  91     protected IElementType getNextTokenType() {
  92       return GTTokens.getTokenType(myLexer);
  93     }
  94
  95     @Override
  96     protected void advanceLexer(final ASTNode next) {
  97       GTTokens.advance(next.getElementType(), myLexer);
  98     }
  99
 100     private static void bindComments(ASTNode root) {
 101       if (TreeUtil.isLeafOrCollapsedChameleon(root)) return;
 102
 103       final List<ASTNode> comments = new SmartList<ASTNode>();
 104       ((TreeElement)root).acceptTree(new RecursiveTreeElementWalkingVisitor(false) {
 105         @Override
 106         protected void visitNode(TreeElement child) {
 107           IElementType type = child.getElementType();
 108           if (type == JavaDocElementType.DOC_COMMENT ||
 109               type == JavaTokenType.END_OF_LINE_COMMENT ||
 110               type == JavaTokenType.C_STYLE_COMMENT) {
 111             comments.add(child);
 112           }
 113           if (TreeUtil.isLeafOrCollapsedChameleon(child)) return;
 114
 115           super.visitNode(child);
 116         }
 117       });
 118
 119       Iterator<ASTNode> iterator = comments.iterator();
 120       while (iterator.hasNext()) {
 121         ASTNode child = iterator.next();
 122         IElementType type = child.getElementType();
 123         if (type == JavaDocElementType.DOC_COMMENT) {
 124           if (bindDocComment((TreeElement)child)) iterator.remove();
 125         }
 126         // bind "trailing comments" (like "int a; // comment")
 127         else if (type == JavaTokenType.END_OF_LINE_COMMENT || type == JavaTokenType.C_STYLE_COMMENT) {
 128           if (bindTrailingComment((TreeElement)child)) iterator.remove();
 129         }
 130       }
 131
 132       //pass 2: bind preceding comments (like "// comment \n void f();")
 133       for (ASTNode child : comments) {
 134         if (child.getElementType() == JavaTokenType.END_OF_LINE_COMMENT || child.getElementType() == JavaTokenType.C_STYLE_COMMENT) {
 135           TreeElement next = (TreeElement)TreeUtil.skipElements(child, PRECEDING_COMMENT_OR_SPACE_BIT_SET);
 136           bindPrecedingComment((TreeElement)child, next);
 137         }
 138       }
 139     }
 140
 141     /*
 142     private static void bindComments(ASTNode root) {
 143       TreeElement child = (TreeElement)root.getFirstChildNode();
 144       while (child != null) {
 145         if (child.getElementType() == JavaDocElementType.DOC_COMMENT) {
 146           if (bindDocComment(child)) {
 147             child = child.getTreeParent();
 148             continue;
 149           }
 150         }
 151
 152         // bind "trailing comments" (like "int a; // comment")
 153         if (child.getElementType() == JavaTokenType.END_OF_LINE_COMMENT || child.getElementType() == JavaTokenType.C_STYLE_COMMENT) {
 154           if (bindTrailingComment(child)) {
 155             child = child.getTreeParent();
 156             continue;
 157           }
 158         }
 159
 160         bindComments(child);
 161         child = child.getTreeNext();
 162       }
 163
 164       //pass 2: bind preceding comments (like "// comment \n void f();")
 165       child = (TreeElement)root.getFirstChildNode();
 166       while(child != null) {
 167         if (child.getElementType() == JavaTokenType.END_OF_LINE_COMMENT || child.getElementType() == JavaTokenType.C_STYLE_COMMENT) {
 168           TreeElement next = (TreeElement)TreeUtil.skipElements(child, PRECEDING_COMMENT_OR_SPACE_BIT_SET);
 169           bindPrecedingComment(child, next);
 170           child = next;
 171         } else {
 172           child = child.getTreeNext();
 173         }
 174       }
 175     }
 176     */
 177
 178     private static boolean bindDocComment(TreeElement docComment) {
 179       TreeElement element = docComment.getTreeNext();
 180       if (element == null) return false;
 181       TreeElement startSpaces = null;
 182
 183       TreeElement importList = null;
 184       // Bypass meaningless tokens and hold'em in hands
 185       while (element.getElementType() == TokenType.WHITE_SPACE ||
 186              element.getElementType() == JavaTokenType.C_STYLE_COMMENT ||
 187              element.getElementType() == JavaTokenType.END_OF_LINE_COMMENT ||
 188              element.getElementType() == JavaElementType.IMPORT_LIST && element.getTextLength() == 0) {
 189         if (element.getElementType() == JavaElementType.IMPORT_LIST) importList = element;
 190         if (startSpaces == null) startSpaces = element;
 191         element = element.getTreeNext();
 192         if (element == null) return false;
 193       }
 194
 195       if (element.getElementType() == JavaElementType.CLASS ||
 196           element.getElementType() == JavaElementType.FIELD ||
 197           element.getElementType() == JavaElementType.METHOD ||
 198           element.getElementType() == JavaElementType.ENUM_CONSTANT ||
 199           element.getElementType() == JavaElementType.ANNOTATION_METHOD) {
 200         TreeElement first = element.getFirstChildNode();
 201         if (startSpaces != null) {
 202           docComment.rawRemoveUpTo(element);
 203         }
 204         else {
 205           docComment.rawRemove();
 206         }
 207
 208         first.rawInsertBeforeMe(docComment);
 209
 210         if (importList != null) {
 211           importList.rawRemove();
 212           element.rawInsertBeforeMe(importList);
 213         }
 214
 215         return true;
 216       }
 217       return false;
 218     }
 219
 220     private static final TokenSet BIND_TRAILING_COMMENT_BIT_SET = TokenSet.orSet(
 221       TokenSet.create(JavaElementType.FIELD, JavaElementType.METHOD, JavaElementType.CLASS, JavaElementType.CLASS_INITIALIZER,
 222                       JavaElementType.IMPORT_STATEMENT, JavaElementType.IMPORT_STATIC_STATEMENT, JavaElementType.PACKAGE_STATEMENT),
 223       ElementType.JAVA_STATEMENT_BIT_SET);
 224
 225     private static boolean bindTrailingComment(TreeElement comment) {
 226       TreeElement element = comment.getTreePrev();
 227       if (element == null) return false;
 228       TreeElement space = null;
 229       if (element.getElementType() == TokenType.WHITE_SPACE) {
 230         space = element;
 231         element = element.getTreePrev();
 232       }
 233       if (element != null && BIND_TRAILING_COMMENT_BIT_SET.contains(element.getElementType())) {
 234         if (space == null || (!space.textContains('\n') && !space.textContains('\r'))) {
 235           if (!comment.textContains('\n') && !comment.textContains('\r')) {
 236             if (space != null) {
 237               space.rawRemove();
 238               ((CompositeElement)element).rawAddChildren(space);
 239             }
 240             comment.rawRemove();
 241             ((CompositeElement)element).rawAddChildren(comment);
 242             return true;
 243           }
 244         }
 245       }
 246       return false;
 247     }
 248
 249     private static final TokenSet BIND_PRECEDING_COMMENT_BIT_SET = TokenSet.create(JavaElementType.FIELD, JavaElementType.METHOD,
 250                                                                                    JavaElementType.CLASS, JavaElementType.CLASS_INITIALIZER);
 251
 252     private static final TokenSet PRECEDING_COMMENT_OR_SPACE_BIT_SET = TokenSet.create(JavaTokenType.C_STYLE_COMMENT, JavaTokenType.END_OF_LINE_COMMENT,
 253                                                                                        TokenType.WHITE_SPACE);
 254
 255     private static void bindPrecedingComment(TreeElement comment, ASTNode bindTo) {
 256       if (bindTo == null || bindTo.getFirstChildNode() != null && bindTo.getFirstChildNode().getElementType() == JavaTokenType.DOC_COMMENT) return;
 257
 258       if (bindTo.getElementType() == JavaElementType.IMPORT_LIST && bindTo.getTextLength() == 0) {
 259         bindTo = bindTo.getTreeNext();
 260       }
 261
 262       ASTNode toStart = isBindingComment(comment) ? comment : null;
 263       if (bindTo != null && BIND_PRECEDING_COMMENT_BIT_SET.contains(bindTo.getElementType())) {
 264         for (ASTNode child = comment; child != bindTo; child = child.getTreeNext()) {
 265           if (child.getElementType() == TokenType.WHITE_SPACE) {
 266             int count = StringUtil.getLineBreakCount(child.getText());
 267             if (count > 1) toStart = null;
 268           }
 269           else {
 270             if (child.getTreePrev() != null && child.getTreePrev().getElementType() == TokenType.WHITE_SPACE) {
 271               LeafElement prev = (LeafElement)child.getTreePrev();
 272               char lastC = prev.charAt(prev.getTextLength() - 1);
 273               if (lastC == '\n' || lastC == '\r') toStart = isBindingComment(child) ? child : null;
 274             }
 275             else {
 276               return;
 277             }
 278           }
 279         }
 280
 281         if (toStart == null) return;
 282
 283         TreeElement first = (TreeElement)bindTo.getFirstChildNode();
 284         TreeElement child = (TreeElement)toStart;
 285         while (child != bindTo) {
 286           TreeElement next = child.getTreeNext();
 287           if (child.getElementType() != JavaElementType.IMPORT_LIST) {
 288             child.rawRemove();
 289             first.rawInsertBeforeMe(child);
 290           }
 291           child = next;
 292         }
 293       }
 294     }
 295
 296     private static boolean isBindingComment(final ASTNode node) {
 297       ASTNode prev = node.getTreePrev();
 298       if (prev != null) {
 299         if (prev.getElementType() != TokenType.WHITE_SPACE) {
 300           return false;
 301         }
 302         else {
 303           if (!prev.textContains('\n')) return false;
 304         }
 305       }
 306
 307       return true;
 308     }
 309
 310     protected boolean isInsertAfterElement(final TreeElement treeNext) {
 311       return treeNext instanceof ModifierListElement;
 312     }
 313   }
 314 }