platform/lang-impl/src/com/intellij/psi/impl/source/tree/injected/MultiHostRegistrarImpl.java

   1 /*
   2  * Copyright 2000-2009 JetBrains s.r.o.
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  * http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 package com.intellij.psi.impl.source.tree.injected;
  18
  19 import com.intellij.injected.editor.DocumentWindow;
  20 import com.intellij.injected.editor.DocumentWindowImpl;
  21 import com.intellij.injected.editor.VirtualFileWindow;
  22 import com.intellij.injected.editor.VirtualFileWindowImpl;
  23 import com.intellij.lang.ASTNode;
  24 import com.intellij.lang.Language;
  25 import com.intellij.lang.LanguageParserDefinitions;
  26 import com.intellij.lang.ParserDefinition;
  27 import com.intellij.lang.injection.MultiHostRegistrar;
  28 import com.intellij.lexer.Lexer;
  29 import com.intellij.openapi.editor.RangeMarker;
  30 import com.intellij.openapi.editor.ex.DocumentEx;
  31 import com.intellij.openapi.editor.impl.DocumentImpl;
  32 import com.intellij.openapi.fileEditor.impl.FileDocumentManagerImpl;
  33 import com.intellij.openapi.fileTypes.SyntaxHighlighter;
  34 import com.intellij.openapi.fileTypes.SyntaxHighlighterFactory;
  35 import com.intellij.openapi.progress.ProcessCanceledException;
  36 import com.intellij.openapi.project.Project;
  37 import com.intellij.openapi.util.*;
  38 import com.intellij.openapi.util.text.StringUtil;
  39 import com.intellij.openapi.vfs.VirtualFile;
  40 import com.intellij.psi.*;
  41 import com.intellij.psi.impl.PsiDocumentManagerImpl;
  42 import com.intellij.psi.impl.source.PsiFileImpl;
  43 import com.intellij.psi.impl.source.resolve.FileContextUtil;
  44 import com.intellij.psi.impl.source.text.BlockSupportImpl;
  45 import com.intellij.psi.impl.source.tree.*;
  46 import com.intellij.psi.tree.IElementType;
  47 import com.intellij.psi.util.PsiTreeUtil;
  48 import com.intellij.psi.util.PsiUtilBase;
  49 import com.intellij.util.ArrayUtil;
  50 import com.intellij.util.SmartList;
  51 import org.jetbrains.annotations.NonNls;
  52 import org.jetbrains.annotations.NotNull;
  53 import org.jetbrains.annotations.Nullable;
  54
  55 import java.util.ArrayList;
  56 import java.util.List;
  57 import java.util.Map;
  58
  59 /**
  60  * @author cdr
  61 */
  62 public class MultiHostRegistrarImpl implements MultiHostRegistrar {
  63   Places result;
  64   private Language myLanguage;
  65   private List<LiteralTextEscaper<? extends PsiLanguageInjectionHost>> escapers;
  66   private List<PsiLanguageInjectionHost.Shred> shreds;
  67   private StringBuilder outChars;
  68   private boolean isOneLineEditor;
  69   private boolean cleared;
  70   private final Project myProject;
  71   private final PsiManager myPsiManager;
  72   private DocumentEx myHostDocument;
  73   private VirtualFile myHostVirtualFile;
  74   private final PsiElement myContextElement;
  75   private final PsiFile myHostPsiFile;
  76
  77   MultiHostRegistrarImpl(@NotNull Project project, @NotNull PsiFile hostPsiFile, @NotNull PsiElement contextElement) {
  78     myProject = project;
  79     myContextElement = contextElement;
  80     myHostPsiFile = PsiUtilBase.getTemplateLanguageFile(hostPsiFile);
  81     myPsiManager = myHostPsiFile.getManager();
  82     cleared = true;
  83   }
  84
  85   @NotNull
  86   public MultiHostRegistrar startInjecting(@NotNull Language language) {
  87     escapers = new SmartList<LiteralTextEscaper<? extends PsiLanguageInjectionHost>>();
  88     shreds = new SmartList<PsiLanguageInjectionHost.Shred>();
  89     outChars = new StringBuilder();
  90
  91     if (!cleared) {
  92       clear();
  93       throw new IllegalStateException("Seems you haven't called doneInjecting()");
  94     }
  95
  96     if (LanguageParserDefinitions.INSTANCE.forLanguage(language) == null) {
  97       throw new UnsupportedOperationException("Cannot inject language '" + language + "' since its getParserDefinition() returns null");
  98     }
  99     myLanguage = language;
 100
 101     FileViewProvider viewProvider = myHostPsiFile.getViewProvider();
 102     myHostVirtualFile = viewProvider.getVirtualFile();
 103     myHostDocument = (DocumentEx)viewProvider.getDocument();
 104     assert myHostDocument != null : myHostPsiFile + "; " + viewProvider;
 105     return this;
 106   }
 107
 108   private void clear() {
 109     escapers.clear();
 110     shreds.clear();
 111     outChars.setLength(0);
 112     isOneLineEditor = false;
 113     myLanguage = null;
 114
 115     cleared = true;
 116   }
 117
 118   @NotNull
 119   public MultiHostRegistrar addPlace(@NonNls @Nullable String prefix,
 120                                      @NonNls @Nullable String suffix,
 121                                      @NotNull PsiLanguageInjectionHost host,
 122                                      @NotNull TextRange rangeInsideHost) {
 123     ProperTextRange.assertProperRange(rangeInsideHost);
 124
 125     PsiFile containingFile = PsiUtilBase.getTemplateLanguageFile(host);
 126     assert containingFile == myHostPsiFile : exceptionContext("Trying to inject into foreign file: "+containingFile);
 127     TextRange hostTextRange = host.getTextRange();
 128     if (!hostTextRange.contains(rangeInsideHost.shiftRight(hostTextRange.getStartOffset()))) {
 129       clear();
 130       throw new IllegalArgumentException("rangeInsideHost must lie within host text range. rangeInsideHost:"+rangeInsideHost+"; host textRange:"+
 131                                          hostTextRange);
 132     }
 133     if (myLanguage == null) {
 134       clear();
 135       throw new IllegalStateException("Seems you haven't called startInjecting()");
 136     }
 137
 138     if (prefix == null) prefix = "";
 139     if (suffix == null) suffix = "";
 140     cleared = false;
 141     int startOffset = outChars.length();
 142     outChars.append(prefix);
 143     LiteralTextEscaper<? extends PsiLanguageInjectionHost> textEscaper = host.createLiteralTextEscaper();
 144     escapers.add(textEscaper);
 145     isOneLineEditor |= textEscaper.isOneLine();
 146     TextRange relevantRange = textEscaper.getRelevantTextRange().intersection(rangeInsideHost);
 147     if (relevantRange == null) {
 148       relevantRange = TextRange.from(textEscaper.getRelevantTextRange().getStartOffset(), 0);
 149     }
 150     else {
 151       boolean result = textEscaper.decode(relevantRange, outChars);
 152       if (!result) {
 153         // if there are invalid chars, adjust the range
 154         int offsetInHost = textEscaper.getOffsetInHost(outChars.length() - startOffset, rangeInsideHost);
 155         relevantRange = relevantRange.intersection(new ProperTextRange(0, offsetInHost));
 156       }
 157     }
 158     outChars.append(suffix);
 159     int endOffset = outChars.length();
 160     TextRange relevantRangeInHost = relevantRange.shiftRight(hostTextRange.getStartOffset());
 161     RangeMarker relevantMarker = myHostDocument.createRangeMarker(relevantRangeInHost);
 162     relevantMarker.setGreedyToLeft(true);
 163     relevantMarker.setGreedyToRight(true);
 164     shreds.add(new PsiLanguageInjectionHost.Shred(host, relevantMarker, prefix, suffix, new ProperTextRange(startOffset, endOffset)));
 165     return this;
 166   }
 167
 168   public void doneInjecting() {
 169     try {
 170       if (shreds.isEmpty()) {
 171         throw new IllegalStateException("Seems you haven't called addPlace()");
 172       }
 173       PsiDocumentManager documentManager = PsiDocumentManager.getInstance(myProject);
 174       assert ArrayUtil.indexOf(documentManager.getUncommittedDocuments(), myHostDocument) == -1 : "document is uncommitted: "+myHostDocument;
 175       assert myHostPsiFile.getText().equals(myHostDocument.getText()) : "host text mismatch";
 176
 177       Place place = new Place(shreds, null);
 178       DocumentWindowImpl documentWindow = new DocumentWindowImpl(myHostDocument, isOneLineEditor, place);
 179       VirtualFileWindowImpl virtualFile = new VirtualFileWindowImpl(myHostVirtualFile, documentWindow, myLanguage, outChars);
 180       myLanguage = LanguageSubstitutors.INSTANCE.substituteLanguage(myLanguage, virtualFile, myProject);
 181       virtualFile.setLanguage(myLanguage);
 182
 183       DocumentImpl decodedDocument;
 184       if (StringUtil.indexOf(outChars, '\r') == -1) {
 185         decodedDocument = new DocumentImpl(outChars);
 186       }
 187       else {
 188         decodedDocument = new DocumentImpl(true);
 189         decodedDocument.setAcceptSlashR(true);
 190         decodedDocument.replaceString(0,0,outChars);
 191       }
 192       FileDocumentManagerImpl.registerDocument(decodedDocument, virtualFile);
 193
 194       InjectedFileViewProvider viewProvider = new InjectedFileViewProvider(myPsiManager, virtualFile, place, documentWindow, myLanguage);
 195       ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(myLanguage);
 196       assert parserDefinition != null : "Parser definition for language "+myLanguage+" is null";
 197       PsiFile psiFile = parserDefinition.createFile(viewProvider);
 198
 199       SmartPsiElementPointer<PsiLanguageInjectionHost> pointer = createHostSmartPointer(shreds.get(0).host);
 200
 201       synchronized (PsiLock.LOCK) {
 202         final ASTNode parsedNode = keepTreeFromChameleoningBack(psiFile);
 203
 204         assert parsedNode instanceof FileElement : "Parsed to "+parsedNode+" instead of FileElement";
 205
 206         String documentText = documentWindow.getText();
 207         assert outChars.toString().equals(parsedNode.getText()) : exceptionContext("Before patch: doc:\n" + documentText + "\n---PSI:\n" + parsedNode.getText() + "\n---chars:\n"+outChars);
 208         try {
 209           patchLeafs(parsedNode, escapers, place);
 210         }
 211         catch (ProcessCanceledException e) {
 212           throw e;
 213         }
 214         catch (RuntimeException e) {
 215           throw new RuntimeException(exceptionContext("Patch error"), e);
 216         }
 217         assert parsedNode.getText().equals(documentText) : exceptionContext("After patch: doc:\n" + documentText + "\n---PSI:\n" + parsedNode.getText() + "\n---chars:\n"+outChars);
 218
 219         virtualFile.setContent(null, documentWindow.getText(), false);
 220
 221         cacheEverything(place, documentWindow, viewProvider, psiFile, pointer);
 222
 223         PsiFile cachedPsiFile = documentManager.getCachedPsiFile(documentWindow);
 224         assert cachedPsiFile == psiFile : "Cached psi :"+ cachedPsiFile +" instead of "+psiFile;
 225
 226         assert place.isValid();
 227         assert viewProvider.isValid();
 228         PsiFile newFile = registerDocument(documentWindow, psiFile, place, myHostPsiFile, documentManager);
 229         boolean mergeHappened = newFile != psiFile;
 230         if (mergeHappened) {
 231           InjectedLanguageUtil.clearCaches(psiFile);
 232           psiFile = newFile;
 233           viewProvider = (InjectedFileViewProvider)psiFile.getViewProvider();
 234           documentWindow = (DocumentWindowImpl)viewProvider.getDocument();
 235           virtualFile = (VirtualFileWindowImpl)viewProvider.getVirtualFile();
 236           cacheEverything(place, documentWindow, viewProvider, psiFile, pointer);
 237         }
 238
 239         assert psiFile.isValid();
 240         assert place.isValid();
 241         assert viewProvider.isValid();
 242
 243         try {
 244           List<Trinity<IElementType, PsiLanguageInjectionHost, TextRange>> tokens = obtainHighlightTokensFromLexer(myLanguage, outChars, escapers, place, virtualFile, myProject);
 245           psiFile.putUserData(InjectedLanguageUtil.HIGHLIGHT_TOKENS, tokens);
 246         }
 247         catch (ProcessCanceledException e) {
 248           throw e;
 249         }
 250         catch (RuntimeException e) {
 251           throw new RuntimeException(exceptionContext("Obtaining tokens error"), e);
 252         }
 253
 254         addToResults(place);
 255
 256         assertEverythingIsAllright(documentManager, documentWindow, psiFile);
 257       }
 258     }
 259     finally {
 260       clear();
 261     }
 262   }
 263
 264   private static void cacheEverything(Place place,
 265                                       DocumentWindowImpl documentWindow,
 266                                       InjectedFileViewProvider viewProvider,
 267                                       PsiFile psiFile,
 268                                       SmartPsiElementPointer<PsiLanguageInjectionHost> pointer) {
 269     FileDocumentManagerImpl.registerDocument(documentWindow, viewProvider.getVirtualFile());
 270
 271     viewProvider.forceCachedPsi(psiFile);
 272
 273     psiFile.putUserData(FileContextUtil.INJECTED_IN_ELEMENT, pointer);
 274     PsiDocumentManagerImpl.cachePsi(documentWindow, psiFile);
 275
 276     keepTreeFromChameleoningBack(psiFile);
 277     place.setInjectedPsi(psiFile);
 278
 279     viewProvider.setShreds(place);
 280   }
 281
 282
 283   @NonNls
 284   private String exceptionContext(@NonNls String msg) {
 285     return msg + ".\n" +
 286            "Language: " +myLanguage+";\n "+
 287            "Host file: "+myHostPsiFile+" in '" + myHostVirtualFile.getPresentableUrl() + "'\n" +
 288            "Context element "+myContextElement.getTextRange() + ": '" + myContextElement +"'; "+
 289            "Ranges: "+shreds;
 290   }
 291
 292   private static final Key<ASTNode> TREE_HARD_REF = Key.create("TREE_HARD_REF");
 293   private static ASTNode keepTreeFromChameleoningBack(PsiFile psiFile) {
 294     psiFile.getFirstChild();
 295     // need to keep tree reacheable to avoid being garbage-collected (via WeakReference in PsiFileImpl)
 296     // and then being reparsed from wrong (escaped) document content
 297     ASTNode node = psiFile.getNode();
 298     assert !TreeUtil.isCollapsedChameleon(node) : "Chameleon "+node+" is collapsed";
 299     psiFile.putUserData(TREE_HARD_REF, node);
 300     return node;
 301   }
 302
 303   private void assertEverythingIsAllright(PsiDocumentManager documentManager, DocumentWindowImpl documentWindow, PsiFile psiFile) {
 304     boolean isAncestor = false;
 305     for (PsiLanguageInjectionHost.Shred shred : shreds) {
 306       PsiLanguageInjectionHost host = shred.host;
 307       isAncestor |= PsiTreeUtil.isAncestor(myContextElement, host, false);
 308     }
 309     assert isAncestor : exceptionContext(myContextElement + " must be the parent of at least one of injection hosts");
 310
 311     InjectedFileViewProvider injectedFileViewProvider = (InjectedFileViewProvider)psiFile.getViewProvider();
 312     assert injectedFileViewProvider.isValid() : "Invalid view provider: "+injectedFileViewProvider;
 313     assert documentWindow.getText().equals(psiFile.getText()) : "Document window text mismatch";
 314     assert injectedFileViewProvider.getDocument() == documentWindow : "Provider document mismatch";
 315     assert documentManager.getCachedDocument(psiFile) == documentWindow : "Cached document mismatch";
 316     assert psiFile.getVirtualFile() == injectedFileViewProvider.getVirtualFile() : "Virtual file mismatch";
 317     PsiDocumentManagerImpl.checkConsistency(psiFile, documentWindow);
 318   }
 319
 320   private void addToResults(Place place) {
 321     if (result == null) {
 322       result = new Places();
 323     }
 324     result.add(place);
 325   }
 326
 327   private static <T extends PsiLanguageInjectionHost> SmartPsiElementPointer<T> createHostSmartPointer(final T host) {
 328     return host.isPhysical()
 329            ? SmartPointerManager.getInstance(host.getProject()).createSmartPsiElementPointer(host)
 330            : new IdentitySmartPointer<T>(host);
 331   }
 332
 333   private static void patchLeafs(ASTNode parsedNode, List<LiteralTextEscaper<? extends PsiLanguageInjectionHost>> escapers, Place shreds) {
 334     LeafPatcher patcher = new LeafPatcher(shreds, escapers);
 335     ((TreeElement)parsedNode).acceptTree(patcher);
 336
 337     String nodeText = parsedNode.getText();
 338     assert nodeText.equals(patcher.catLeafs.toString()) : "Malformed PSI structure: leaf texts do not add up to the whole file text." +
 339                                                   "\nFile text (from tree)  :'"+nodeText+"'" +
 340                                                   "\nFile text (from PSI)   :'"+parsedNode.getPsi().getText()+"'" +
 341                                                   "\nLeaf texts concatenated:'"+ patcher.catLeafs +"';" +
 342                                                   "\nFile root: "+parsedNode+
 343                                                   "\nLanguage: "+parsedNode.getPsi().getLanguage()+
 344                                                   "\nHost file: "+shreds.get(0).host.getContainingFile().getVirtualFile()
 345         ;
 346     for (Map.Entry<LeafElement, String> entry : patcher.newTexts.entrySet()) {
 347       LeafElement leaf = entry.getKey();
 348       String newText = entry.getValue();
 349       leaf.rawReplaceWithText(newText);
 350     }
 351     ((TreeElement)parsedNode).acceptTree(new RecursiveTreeElementWalkingVisitor(){
 352       protected void visitNode(TreeElement element) {
 353         element.clearCaches();
 354         super.visitNode(element);
 355       }
 356     });
 357   }
 358
 359   private static PsiFile registerDocument(final DocumentWindowImpl documentWindow,
 360                                           final PsiFile injectedPsi,
 361                                           final Place shreds,
 362                                           final PsiFile hostPsiFile,
 363                                           final PsiDocumentManager documentManager) {
 364     DocumentEx hostDocument = documentWindow.getDelegate();
 365     List<DocumentWindow> injected = InjectedLanguageUtil.getCachedInjectedDocuments(hostPsiFile);
 366
 367     for (int i = injected.size()-1; i>=0; i--) {
 368       DocumentWindowImpl oldDocument = (DocumentWindowImpl)injected.get(i);
 369       final PsiFileImpl oldFile = (PsiFileImpl)documentManager.getCachedPsiFile(oldDocument);
 370       FileViewProvider viewProvider;
 371
 372       if (oldFile == null ||
 373           !oldFile.isValid() ||
 374           !((viewProvider = oldFile.getViewProvider()) instanceof InjectedFileViewProvider) ||
 375           ((InjectedFileViewProvider)viewProvider).isDisposed()
 376         ) {
 377         injected.remove(i);
 378         Disposer.dispose(oldDocument);
 379         continue;
 380       }
 381       InjectedFileViewProvider oldViewProvider = (InjectedFileViewProvider)viewProvider;
 382
 383       final ASTNode injectedNode = injectedPsi.getNode();
 384       final ASTNode oldFileNode = oldFile.getNode();
 385       assert injectedNode != null : "New node is null";
 386       assert oldFileNode != null : "Old node is null";
 387       if (oldDocument.areRangesEqual(documentWindow)) {
 388         if (oldFile.getFileType() != injectedPsi.getFileType() || oldFile.getLanguage() != injectedPsi.getLanguage()) {
 389           injected.remove(i);
 390           Disposer.dispose(oldDocument);
 391           continue;
 392         }
 393         oldFile.putUserData(FileContextUtil.INJECTED_IN_ELEMENT, injectedPsi.getUserData(FileContextUtil.INJECTED_IN_ELEMENT));
 394
 395         assert shreds.isValid();
 396         oldViewProvider.performNonPhysically(new Runnable() {
 397           public void run() {
 398             BlockSupportImpl.mergeTrees(oldFile, oldFileNode, injectedNode);
 399           }
 400         });
 401         assert shreds.isValid();
 402
 403         return oldFile;
 404       }
 405     }
 406     injected.add(documentWindow);
 407
 408     cacheInjectedRegion(documentWindow, hostDocument);
 409     return injectedPsi;
 410   }
 411
 412   private static void cacheInjectedRegion(DocumentWindowImpl documentWindow, DocumentEx hostDocument) {
 413     List<RangeMarker> injectedRegions = InjectedLanguageUtil.getCachedInjectedRegions(hostDocument);
 414     RangeMarker newMarker = documentWindow.getHostRanges()[0];
 415     TextRange newRange = InjectedLanguageUtil.toTextRange(newMarker);
 416     for (int i = 0; i < injectedRegions.size(); i++) {
 417       RangeMarker stored = injectedRegions.get(i);
 418       TextRange storedRange = InjectedLanguageUtil.toTextRange(stored);
 419       if (storedRange.intersects(newRange)) {
 420         injectedRegions.set(i, newMarker);
 421         break;
 422       }
 423       if (storedRange.getStartOffset() > newRange.getEndOffset()) {
 424         injectedRegions.add(i, newMarker);
 425         break;
 426       }
 427     }
 428     if (injectedRegions.isEmpty() || newRange.getStartOffset() > injectedRegions.get(injectedRegions.size()-1).getEndOffset()) {
 429       injectedRegions.add(newMarker);
 430     }
 431   }
 432
 433   // returns lexer elemet types with corresponsing ranges in encoded (injection host based) PSI
 434   private static List<Trinity<IElementType, PsiLanguageInjectionHost, TextRange>> obtainHighlightTokensFromLexer(Language language,
 435                                                                                                                  StringBuilder outChars,
 436                                                                                                                  List<LiteralTextEscaper<? extends PsiLanguageInjectionHost>> escapers,
 437                                                                                                                  Place shreds,
 438                                                                                                                  VirtualFileWindow virtualFile,
 439                                                                                                                  Project project) {
 440     List<Trinity<IElementType, PsiLanguageInjectionHost, TextRange>> tokens = new ArrayList<Trinity<IElementType, PsiLanguageInjectionHost, TextRange>>(10);
 441     SyntaxHighlighter syntaxHighlighter = SyntaxHighlighterFactory.getSyntaxHighlighter(language, project, (VirtualFile)virtualFile);
 442     Lexer lexer = syntaxHighlighter.getHighlightingLexer();
 443     lexer.start(outChars);
 444     int hostNum = -1;
 445     int prevHostEndOffset = 0;
 446     PsiLanguageInjectionHost host = null;
 447     LiteralTextEscaper<? extends PsiLanguageInjectionHost> escaper = null;
 448     int prefixLength = 0;
 449     int suffixLength = 0;
 450     TextRange rangeInsideHost = null;
 451     int shredEndOffset = -1;
 452     for (IElementType tokenType = lexer.getTokenType(); tokenType != null; lexer.advance(), tokenType = lexer.getTokenType()) {
 453       TextRange range = new ProperTextRange(lexer.getTokenStart(), lexer.getTokenEnd());
 454       while (range != null && !range.isEmpty()) {
 455         if (range.getStartOffset() >= shredEndOffset) {
 456           hostNum++;
 457           shredEndOffset = shreds.get(hostNum).range.getEndOffset();
 458           prevHostEndOffset = range.getStartOffset();
 459           host = shreds.get(hostNum).host;
 460           escaper = escapers.get(hostNum);
 461           rangeInsideHost = shreds.get(hostNum).getRangeInsideHost();
 462           prefixLength = shreds.get(hostNum).prefix.length();
 463           suffixLength = shreds.get(hostNum).suffix.length();
 464         }
 465         //in prefix/suffix or spills over to next fragment
 466         if (range.getStartOffset() < prevHostEndOffset + prefixLength) {
 467           range = new TextRange(prevHostEndOffset + prefixLength, range.getEndOffset());
 468         }
 469         TextRange spilled = null;
 470         if (range.getEndOffset() >= shredEndOffset - suffixLength) {
 471           spilled = new TextRange(shredEndOffset, range.getEndOffset());
 472           range = new TextRange(range.getStartOffset(), shredEndOffset);
 473         }
 474         if (!range.isEmpty()) {
 475           int start = escaper.getOffsetInHost(range.getStartOffset() - prevHostEndOffset - prefixLength, rangeInsideHost);
 476           if (start == -1) start = rangeInsideHost.getStartOffset();
 477           int end = escaper.getOffsetInHost(range.getEndOffset() - prevHostEndOffset - prefixLength, rangeInsideHost);
 478           if (end == -1) {
 479             end = rangeInsideHost.getEndOffset();
 480             prevHostEndOffset = shredEndOffset;
 481           }
 482           TextRange rangeInHost = new ProperTextRange(start, end);
 483           tokens.add(Trinity.create(tokenType, host, rangeInHost));
 484         }
 485         range = spilled;
 486       }
 487     }
 488     return tokens;
 489   }
 490
 491   void addToResults(Places places) {
 492     for (Place place : places) {
 493       addToResults(place);
 494     }
 495   }
 496 }