platform/lang-impl/src/com/intellij/psi/impl/source/tree/injected/MultiHostRegistrarImpl.java

   1 /*
   2  * Copyright 2000-2009 JetBrains s.r.o.
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  * http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 package com.intellij.psi.impl.source.tree.injected;
  18
  19 import com.intellij.injected.editor.DocumentWindow;
  20 import com.intellij.injected.editor.DocumentWindowImpl;
  21 import com.intellij.injected.editor.VirtualFileWindow;
  22 import com.intellij.injected.editor.VirtualFileWindowImpl;
  23 import com.intellij.lang.ASTNode;
  24 import com.intellij.lang.Language;
  25 import com.intellij.lang.LanguageParserDefinitions;
  26 import com.intellij.lang.ParserDefinition;
  27 import com.intellij.lang.injection.MultiHostRegistrar;
  28 import com.intellij.lexer.Lexer;
  29 import com.intellij.openapi.editor.RangeMarker;
  30 import com.intellij.openapi.editor.ex.DocumentEx;
  31 import com.intellij.openapi.editor.impl.DocumentImpl;
  32 import com.intellij.openapi.fileEditor.impl.FileDocumentManagerImpl;
  33 import com.intellij.openapi.fileTypes.SyntaxHighlighter;
  34 import com.intellij.openapi.fileTypes.SyntaxHighlighterFactory;
  35 import com.intellij.openapi.progress.ProcessCanceledException;
  36 import com.intellij.openapi.project.Project;
  37 import com.intellij.openapi.util.*;
  38 import com.intellij.openapi.util.text.StringUtil;
  39 import com.intellij.openapi.vfs.VirtualFile;
  40 import com.intellij.psi.*;
  41 import com.intellij.psi.impl.PsiDocumentManagerImpl;
  42 import com.intellij.psi.impl.source.PsiFileImpl;
  43 import com.intellij.psi.impl.source.resolve.FileContextUtil;
  44 import com.intellij.psi.impl.source.text.BlockSupportImpl;
  45 import com.intellij.psi.impl.source.tree.*;
  46 import com.intellij.psi.tree.IElementType;
  47 import com.intellij.psi.util.PsiTreeUtil;
  48 import com.intellij.psi.util.PsiUtilBase;
  49 import com.intellij.util.ArrayUtil;
  50 import com.intellij.util.SmartList;
  51 import org.jetbrains.annotations.NonNls;
  52 import org.jetbrains.annotations.NotNull;
  53 import org.jetbrains.annotations.Nullable;
  54
  55 import java.util.ArrayList;
  56 import java.util.List;
  57 import java.util.Map;
  58
  59 /**
  60  * @author cdr
  61 */
  62 public class MultiHostRegistrarImpl implements MultiHostRegistrar {
  63   Places result;
  64   private Language myLanguage;
  65   private List<LiteralTextEscaper<? extends PsiLanguageInjectionHost>> escapers;
  66   private List<PsiLanguageInjectionHost.Shred> shreds;
  67   private StringBuilder outChars;
  68   private boolean isOneLineEditor;
  69   private boolean cleared;
  70   private final Project myProject;
  71   private final PsiManager myPsiManager;
  72   private DocumentEx myHostDocument;
  73   private VirtualFile myHostVirtualFile;
  74   private final PsiElement myContextElement;
  75   private final PsiFile myHostPsiFile;
  76
  77   MultiHostRegistrarImpl(@NotNull Project project, @NotNull PsiFile hostPsiFile, @NotNull PsiElement contextElement) {
  78     myProject = project;
  79     myContextElement = contextElement;
  80     myHostPsiFile = PsiUtilBase.getTemplateLanguageFile(hostPsiFile);
  81     myPsiManager = myHostPsiFile.getManager();
  82     cleared = true;
  83   }
  84
  85   @NotNull
  86   public MultiHostRegistrar startInjecting(@NotNull Language language) {
  87     escapers = new SmartList<LiteralTextEscaper<? extends PsiLanguageInjectionHost>>();
  88     shreds = new SmartList<PsiLanguageInjectionHost.Shred>();
  89     outChars = new StringBuilder();
  90
  91     if (!cleared) {
  92       clear();
  93       throw new IllegalStateException("Seems you haven't called doneInjecting()");
  94     }
  95
  96     if (LanguageParserDefinitions.INSTANCE.forLanguage(language) == null) {
  97       throw new UnsupportedOperationException("Cannot inject language '" + language + "' since its getParserDefinition() returns null");
  98     }
  99     myLanguage = language;
 100
 101     FileViewProvider viewProvider = myHostPsiFile.getViewProvider();
 102     myHostVirtualFile = viewProvider.getVirtualFile();
 103     myHostDocument = (DocumentEx)viewProvider.getDocument();
 104     assert myHostDocument != null : myHostPsiFile + "; " + viewProvider;
 105     return this;
 106   }
 107
 108   private void clear() {
 109     escapers.clear();
 110     shreds.clear();
 111     outChars.setLength(0);
 112     isOneLineEditor = false;
 113     myLanguage = null;
 114
 115     cleared = true;
 116   }
 117
 118   @NotNull
 119   public MultiHostRegistrar addPlace(@NonNls @Nullable String prefix,
 120                                      @NonNls @Nullable String suffix,
 121                                      @NotNull PsiLanguageInjectionHost host,
 122                                      @NotNull TextRange rangeInsideHost) {
 123     ProperTextRange.assertProperRange(rangeInsideHost);
 124
 125     PsiFile containingFile = PsiUtilBase.getTemplateLanguageFile(host);
 126     assert containingFile == myHostPsiFile : exceptionContext("Trying to inject into foreign file: "+containingFile);
 127     TextRange hostTextRange = host.getTextRange();
 128     if (!hostTextRange.contains(rangeInsideHost.shiftRight(hostTextRange.getStartOffset()))) {
 129       clear();
 130       throw new IllegalArgumentException("rangeInsideHost must lie within host text range. rangeInsideHost:"+rangeInsideHost+"; host textRange:"+
 131                                          hostTextRange);
 132     }
 133     if (myLanguage == null) {
 134       clear();
 135       throw new IllegalStateException("Seems you haven't called startInjecting()");
 136     }
 137
 138     if (prefix == null) prefix = "";
 139     if (suffix == null) suffix = "";
 140     cleared = false;
 141     int startOffset = outChars.length();
 142     outChars.append(prefix);
 143     LiteralTextEscaper<? extends PsiLanguageInjectionHost> textEscaper = host.createLiteralTextEscaper();
 144     escapers.add(textEscaper);
 145     isOneLineEditor |= textEscaper.isOneLine();
 146     TextRange relevantRange = textEscaper.getRelevantTextRange().intersection(rangeInsideHost);
 147     if (relevantRange == null) {
 148       relevantRange = TextRange.from(textEscaper.getRelevantTextRange().getStartOffset(), 0);
 149     }
 150     else {
 151       int before = outChars.length();
 152       boolean result = textEscaper.decode(relevantRange, outChars);
 153       int after = outChars.length();
 154       assert after >= before : "Escaper " + textEscaper + "("+textEscaper.getClass()+") must not mangle char buffer";
 155       if (!result) {
 156         // if there are invalid chars, adjust the range
 157         int offsetInHost = textEscaper.getOffsetInHost(outChars.length() - startOffset, rangeInsideHost);
 158         relevantRange = relevantRange.intersection(new ProperTextRange(0, offsetInHost));
 159       }
 160     }
 161     outChars.append(suffix);
 162     int endOffset = outChars.length();
 163     TextRange relevantRangeInHost = relevantRange.shiftRight(hostTextRange.getStartOffset());
 164     RangeMarker relevantMarker = myHostDocument.createRangeMarker(relevantRangeInHost);
 165     relevantMarker.setGreedyToLeft(true);
 166     relevantMarker.setGreedyToRight(true);
 167     shreds.add(new PsiLanguageInjectionHost.Shred(host, relevantMarker, prefix, suffix, new ProperTextRange(startOffset, endOffset)));
 168     return this;
 169   }
 170
 171   public void doneInjecting() {
 172     try {
 173       if (shreds.isEmpty()) {
 174         throw new IllegalStateException("Seems you haven't called addPlace()");
 175       }
 176       PsiDocumentManager documentManager = PsiDocumentManager.getInstance(myProject);
 177       assert ArrayUtil.indexOf(documentManager.getUncommittedDocuments(), myHostDocument) == -1 : "document is uncommitted: "+myHostDocument;
 178       assert myHostPsiFile.getText().equals(myHostDocument.getText()) : "host text mismatch";
 179
 180       Place place = new Place(shreds, null);
 181       DocumentWindowImpl documentWindow = new DocumentWindowImpl(myHostDocument, isOneLineEditor, place);
 182       VirtualFileWindowImpl virtualFile = new VirtualFileWindowImpl(myHostVirtualFile, documentWindow, myLanguage, outChars);
 183       myLanguage = LanguageSubstitutors.INSTANCE.substituteLanguage(myLanguage, virtualFile, myProject);
 184       virtualFile.setLanguage(myLanguage);
 185
 186       DocumentImpl decodedDocument;
 187       if (StringUtil.indexOf(outChars, '\r') == -1) {
 188         decodedDocument = new DocumentImpl(outChars);
 189       }
 190       else {
 191         decodedDocument = new DocumentImpl(true);
 192         decodedDocument.setAcceptSlashR(true);
 193         decodedDocument.replaceString(0,0,outChars);
 194       }
 195       FileDocumentManagerImpl.registerDocument(decodedDocument, virtualFile);
 196
 197       InjectedFileViewProvider viewProvider = new InjectedFileViewProvider(myPsiManager, virtualFile, place, documentWindow, myLanguage);
 198       ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(myLanguage);
 199       assert parserDefinition != null : "Parser definition for language "+myLanguage+" is null";
 200       PsiFile psiFile = parserDefinition.createFile(viewProvider);
 201
 202       SmartPsiElementPointer<PsiLanguageInjectionHost> pointer = createHostSmartPointer(shreds.get(0).host);
 203
 204       synchronized (PsiLock.LOCK) {
 205         final ASTNode parsedNode = keepTreeFromChameleoningBack(psiFile);
 206
 207         assert parsedNode instanceof FileElement : "Parsed to "+parsedNode+" instead of FileElement";
 208
 209         String documentText = documentWindow.getText();
 210         assert outChars.toString().equals(parsedNode.getText()) : exceptionContext("Before patch: doc:\n'" + documentText + "'\n---PSI:\n'" + parsedNode.getText() + "'\n---chars:\n'"+outChars+"'");
 211         try {
 212           patchLeafs(parsedNode, escapers, place);
 213         }
 214         catch (ProcessCanceledException e) {
 215           throw e;
 216         }
 217         catch (RuntimeException e) {
 218           throw new RuntimeException(exceptionContext("Patch error"), e);
 219         }
 220         assert parsedNode.getText().equals(documentText) : exceptionContext("After patch: doc:\n'" + documentText + "'\n---PSI:\n'" + parsedNode.getText() + "'\n---chars:\n'"+outChars+"'");
 221
 222         virtualFile.setContent(null, documentWindow.getText(), false);
 223
 224         cacheEverything(place, documentWindow, viewProvider, psiFile, pointer);
 225
 226         PsiFile cachedPsiFile = documentManager.getCachedPsiFile(documentWindow);
 227         assert cachedPsiFile == psiFile : "Cached psi :"+ cachedPsiFile +" instead of "+psiFile;
 228
 229         assert place.isValid();
 230         assert viewProvider.isValid();
 231         PsiFile newFile = registerDocument(documentWindow, psiFile, place, myHostPsiFile, documentManager);
 232         boolean mergeHappened = newFile != psiFile;
 233         if (mergeHappened) {
 234           InjectedLanguageUtil.clearCaches(psiFile);
 235           psiFile = newFile;
 236           viewProvider = (InjectedFileViewProvider)psiFile.getViewProvider();
 237           documentWindow = (DocumentWindowImpl)viewProvider.getDocument();
 238           virtualFile = (VirtualFileWindowImpl)viewProvider.getVirtualFile();
 239           cacheEverything(place, documentWindow, viewProvider, psiFile, pointer);
 240         }
 241
 242         assert psiFile.isValid();
 243         assert place.isValid();
 244         assert viewProvider.isValid();
 245
 246         try {
 247           List<Trinity<IElementType, PsiLanguageInjectionHost, TextRange>> tokens = obtainHighlightTokensFromLexer(myLanguage, outChars, escapers, place, virtualFile, myProject);
 248           psiFile.putUserData(InjectedLanguageUtil.HIGHLIGHT_TOKENS, tokens);
 249         }
 250         catch (ProcessCanceledException e) {
 251           throw e;
 252         }
 253         catch (RuntimeException e) {
 254           throw new RuntimeException(exceptionContext("Obtaining tokens error"), e);
 255         }
 256
 257         addToResults(place);
 258
 259         assertEverythingIsAllright(documentManager, documentWindow, psiFile);
 260       }
 261     }
 262     finally {
 263       clear();
 264     }
 265   }
 266
 267   private static void cacheEverything(Place place,
 268                                       DocumentWindowImpl documentWindow,
 269                                       InjectedFileViewProvider viewProvider,
 270                                       PsiFile psiFile,
 271                                       SmartPsiElementPointer<PsiLanguageInjectionHost> pointer) {
 272     FileDocumentManagerImpl.registerDocument(documentWindow, viewProvider.getVirtualFile());
 273
 274     viewProvider.forceCachedPsi(psiFile);
 275
 276     psiFile.putUserData(FileContextUtil.INJECTED_IN_ELEMENT, pointer);
 277     PsiDocumentManagerImpl.cachePsi(documentWindow, psiFile);
 278
 279     keepTreeFromChameleoningBack(psiFile);
 280     place.setInjectedPsi(psiFile);
 281
 282     viewProvider.setShreds(place);
 283   }
 284
 285
 286   @NonNls
 287   private String exceptionContext(@NonNls String msg) {
 288     return msg + ".\n" +
 289            myLanguage+";\n "+
 290            "Host file: "+myHostPsiFile+" in '" + myHostVirtualFile.getPresentableUrl() + "'\n" +
 291            "Context element "+myContextElement.getTextRange() + ": '" + myContextElement +"'; "+
 292            "Ranges: "+shreds;
 293   }
 294
 295   private static final Key<ASTNode> TREE_HARD_REF = Key.create("TREE_HARD_REF");
 296   private static ASTNode keepTreeFromChameleoningBack(PsiFile psiFile) {
 297     psiFile.getFirstChild();
 298     // need to keep tree reacheable to avoid being garbage-collected (via WeakReference in PsiFileImpl)
 299     // and then being reparsed from wrong (escaped) document content
 300     ASTNode node = psiFile.getNode();
 301     assert !TreeUtil.isCollapsedChameleon(node) : "Chameleon "+node+" is collapsed";
 302     psiFile.putUserData(TREE_HARD_REF, node);
 303     return node;
 304   }
 305
 306   private void assertEverythingIsAllright(PsiDocumentManager documentManager, DocumentWindowImpl documentWindow, PsiFile psiFile) {
 307     boolean isAncestor = false;
 308     for (PsiLanguageInjectionHost.Shred shred : shreds) {
 309       PsiLanguageInjectionHost host = shred.host;
 310       isAncestor |= PsiTreeUtil.isAncestor(myContextElement, host, false);
 311     }
 312     assert isAncestor : exceptionContext(myContextElement + " must be the parent of at least one of injection hosts");
 313
 314     InjectedFileViewProvider injectedFileViewProvider = (InjectedFileViewProvider)psiFile.getViewProvider();
 315     assert injectedFileViewProvider.isValid() : "Invalid view provider: "+injectedFileViewProvider;
 316     assert documentWindow.getText().equals(psiFile.getText()) : "Document window text mismatch";
 317     assert injectedFileViewProvider.getDocument() == documentWindow : "Provider document mismatch";
 318     assert documentManager.getCachedDocument(psiFile) == documentWindow : "Cached document mismatch";
 319     assert psiFile.getVirtualFile() == injectedFileViewProvider.getVirtualFile() : "Virtual file mismatch";
 320     PsiDocumentManagerImpl.checkConsistency(psiFile, documentWindow);
 321   }
 322
 323   private void addToResults(Place place) {
 324     if (result == null) {
 325       result = new Places();
 326     }
 327     result.add(place);
 328   }
 329
 330   private static <T extends PsiLanguageInjectionHost> SmartPsiElementPointer<T> createHostSmartPointer(final T host) {
 331     return host.isPhysical()
 332            ? SmartPointerManager.getInstance(host.getProject()).createSmartPsiElementPointer(host)
 333            : new IdentitySmartPointer<T>(host);
 334   }
 335
 336   private static void patchLeafs(ASTNode parsedNode, List<LiteralTextEscaper<? extends PsiLanguageInjectionHost>> escapers, Place shreds) {
 337     LeafPatcher patcher = new LeafPatcher(shreds, escapers);
 338     ((TreeElement)parsedNode).acceptTree(patcher);
 339
 340     String nodeText = parsedNode.getText();
 341     assert nodeText.equals(patcher.catLeafs.toString()) : "Malformed PSI structure: leaf texts do not add up to the whole file text." +
 342                                                   "\nFile text (from tree)  :'"+nodeText+"'" +
 343                                                   "\nFile text (from PSI)   :'"+parsedNode.getPsi().getText()+"'" +
 344                                                   "\nLeaf texts concatenated:'"+ patcher.catLeafs +"';" +
 345                                                   "\nFile root: "+parsedNode+
 346                                                   "\nLanguage: "+parsedNode.getPsi().getLanguage()+
 347                                                   "\nHost file: "+shreds.get(0).host.getContainingFile().getVirtualFile()
 348         ;
 349     for (Map.Entry<LeafElement, String> entry : patcher.newTexts.entrySet()) {
 350       LeafElement leaf = entry.getKey();
 351       String newText = entry.getValue();
 352       leaf.rawReplaceWithText(newText);
 353     }
 354     ((TreeElement)parsedNode).acceptTree(new RecursiveTreeElementWalkingVisitor(){
 355       protected void visitNode(TreeElement element) {
 356         element.clearCaches();
 357         super.visitNode(element);
 358       }
 359     });
 360   }
 361
 362   private static PsiFile registerDocument(final DocumentWindowImpl documentWindow,
 363                                           final PsiFile injectedPsi,
 364                                           final Place shreds,
 365                                           final PsiFile hostPsiFile,
 366                                           final PsiDocumentManager documentManager) {
 367     DocumentEx hostDocument = documentWindow.getDelegate();
 368     List<DocumentWindow> injected = InjectedLanguageUtil.getCachedInjectedDocuments(hostPsiFile);
 369
 370     for (int i = injected.size()-1; i>=0; i--) {
 371       DocumentWindowImpl oldDocument = (DocumentWindowImpl)injected.get(i);
 372       final PsiFileImpl oldFile = (PsiFileImpl)documentManager.getCachedPsiFile(oldDocument);
 373       FileViewProvider viewProvider;
 374
 375       if (oldFile == null ||
 376           !oldFile.isValid() ||
 377           !((viewProvider = oldFile.getViewProvider()) instanceof InjectedFileViewProvider) ||
 378           ((InjectedFileViewProvider)viewProvider).isDisposed()
 379         ) {
 380         injected.remove(i);
 381         Disposer.dispose(oldDocument);
 382         continue;
 383       }
 384       InjectedFileViewProvider oldViewProvider = (InjectedFileViewProvider)viewProvider;
 385
 386       final ASTNode injectedNode = injectedPsi.getNode();
 387       final ASTNode oldFileNode = oldFile.getNode();
 388       assert injectedNode != null : "New node is null";
 389       assert oldFileNode != null : "Old node is null";
 390       if (oldDocument.areRangesEqual(documentWindow)) {
 391         if (oldFile.getFileType() != injectedPsi.getFileType() || oldFile.getLanguage() != injectedPsi.getLanguage()) {
 392           injected.remove(i);
 393           Disposer.dispose(oldDocument);
 394           continue;
 395         }
 396         oldFile.putUserData(FileContextUtil.INJECTED_IN_ELEMENT, injectedPsi.getUserData(FileContextUtil.INJECTED_IN_ELEMENT));
 397
 398         assert shreds.isValid();
 399         oldViewProvider.performNonPhysically(new Runnable() {
 400           public void run() {
 401             BlockSupportImpl.mergeTrees(oldFile, oldFileNode, injectedNode);
 402           }
 403         });
 404         assert shreds.isValid();
 405
 406         return oldFile;
 407       }
 408     }
 409     injected.add(documentWindow);
 410
 411     cacheInjectedRegion(documentWindow, hostDocument);
 412     return injectedPsi;
 413   }
 414
 415   private static void cacheInjectedRegion(DocumentWindowImpl documentWindow, DocumentEx hostDocument) {
 416     List<RangeMarker> injectedRegions = InjectedLanguageUtil.getCachedInjectedRegions(hostDocument);
 417     RangeMarker newMarker = documentWindow.getHostRanges()[0];
 418     TextRange newRange = InjectedLanguageUtil.toTextRange(newMarker);
 419     for (int i = 0; i < injectedRegions.size(); i++) {
 420       RangeMarker stored = injectedRegions.get(i);
 421       TextRange storedRange = InjectedLanguageUtil.toTextRange(stored);
 422       if (storedRange.intersects(newRange)) {
 423         injectedRegions.set(i, newMarker);
 424         break;
 425       }
 426       if (storedRange.getStartOffset() > newRange.getEndOffset()) {
 427         injectedRegions.add(i, newMarker);
 428         break;
 429       }
 430     }
 431     if (injectedRegions.isEmpty() || newRange.getStartOffset() > injectedRegions.get(injectedRegions.size()-1).getEndOffset()) {
 432       injectedRegions.add(newMarker);
 433     }
 434   }
 435
 436   // returns lexer elemet types with corresponsing ranges in encoded (injection host based) PSI
 437   private static List<Trinity<IElementType, PsiLanguageInjectionHost, TextRange>> obtainHighlightTokensFromLexer(Language language,
 438                                                                                                                  StringBuilder outChars,
 439                                                                                                                  List<LiteralTextEscaper<? extends PsiLanguageInjectionHost>> escapers,
 440                                                                                                                  Place shreds,
 441                                                                                                                  VirtualFileWindow virtualFile,
 442                                                                                                                  Project project) {
 443     List<Trinity<IElementType, PsiLanguageInjectionHost, TextRange>> tokens = new ArrayList<Trinity<IElementType, PsiLanguageInjectionHost, TextRange>>(10);
 444     SyntaxHighlighter syntaxHighlighter = SyntaxHighlighterFactory.getSyntaxHighlighter(language, project, (VirtualFile)virtualFile);
 445     Lexer lexer = syntaxHighlighter.getHighlightingLexer();
 446     lexer.start(outChars);
 447     int hostNum = -1;
 448     int prevHostEndOffset = 0;
 449     PsiLanguageInjectionHost host = null;
 450     LiteralTextEscaper<? extends PsiLanguageInjectionHost> escaper = null;
 451     int prefixLength = 0;
 452     int suffixLength = 0;
 453     TextRange rangeInsideHost = null;
 454     int shredEndOffset = -1;
 455     for (IElementType tokenType = lexer.getTokenType(); tokenType != null; lexer.advance(), tokenType = lexer.getTokenType()) {
 456       TextRange range = new ProperTextRange(lexer.getTokenStart(), lexer.getTokenEnd());
 457       while (range != null && !range.isEmpty()) {
 458         if (range.getStartOffset() >= shredEndOffset) {
 459           hostNum++;
 460           shredEndOffset = shreds.get(hostNum).range.getEndOffset();
 461           prevHostEndOffset = range.getStartOffset();
 462           host = shreds.get(hostNum).host;
 463           escaper = escapers.get(hostNum);
 464           rangeInsideHost = shreds.get(hostNum).getRangeInsideHost();
 465           prefixLength = shreds.get(hostNum).prefix.length();
 466           suffixLength = shreds.get(hostNum).suffix.length();
 467         }
 468         //in prefix/suffix or spills over to next fragment
 469         if (range.getStartOffset() < prevHostEndOffset + prefixLength) {
 470           range = new TextRange(prevHostEndOffset + prefixLength, range.getEndOffset());
 471         }
 472         TextRange spilled = null;
 473         if (range.getEndOffset() >= shredEndOffset - suffixLength) {
 474           spilled = new TextRange(shredEndOffset, range.getEndOffset());
 475           range = new TextRange(range.getStartOffset(), shredEndOffset);
 476         }
 477         if (!range.isEmpty()) {
 478           int start = escaper.getOffsetInHost(range.getStartOffset() - prevHostEndOffset - prefixLength, rangeInsideHost);
 479           if (start == -1) start = rangeInsideHost.getStartOffset();
 480           int end = escaper.getOffsetInHost(range.getEndOffset() - prevHostEndOffset - prefixLength, rangeInsideHost);
 481           if (end == -1) {
 482             end = rangeInsideHost.getEndOffset();
 483             prevHostEndOffset = shredEndOffset;
 484           }
 485           TextRange rangeInHost = new ProperTextRange(start, end);
 486           tokens.add(Trinity.create(tokenType, host, rangeInHost));
 487         }
 488         range = spilled;
 489       }
 490     }
 491     return tokens;
 492   }
 493
 494   void addToResults(Places places) {
 495     for (Place place : places) {
 496       addToResults(place);
 497     }
 498   }
 499 }