lang-impl/src/com/intellij/util/indexing/FileBasedIndex.java

   1 package com.intellij.util.indexing;
   2
   3 import com.intellij.AppTopics;
   4 import com.intellij.ide.startup.CacheUpdater;
   5 import com.intellij.ide.startup.FileSystemSynchronizer;
   6 import com.intellij.lang.ASTNode;
   7 import com.intellij.openapi.application.*;
   8 import com.intellij.openapi.components.ApplicationComponent;
   9 import com.intellij.openapi.diagnostic.Logger;
  10 import com.intellij.openapi.editor.Document;
  11 import com.intellij.openapi.editor.highlighter.EditorHighlighter;
  12 import com.intellij.openapi.editor.impl.DocumentImpl;
  13 import com.intellij.openapi.extensions.Extensions;
  14 import com.intellij.openapi.fileEditor.FileDocumentManager;
  15 import com.intellij.openapi.fileEditor.impl.LoadTextUtil;
  16 import com.intellij.openapi.fileTypes.FileTypeEvent;
  17 import com.intellij.openapi.fileTypes.FileTypeListener;
  18 import com.intellij.openapi.progress.*;
  19 import com.intellij.openapi.project.Project;
  20 import com.intellij.openapi.project.ProjectManager;
  21 import com.intellij.openapi.roots.CollectingContentIterator;
  22 import com.intellij.openapi.roots.ContentIterator;
  23 import com.intellij.openapi.roots.ProjectFileIndex;
  24 import com.intellij.openapi.roots.ProjectRootManager;
  25 import com.intellij.openapi.util.Key;
  26 import com.intellij.openapi.util.Pair;
  27 import com.intellij.openapi.util.io.FileUtil;
  28 import com.intellij.openapi.vfs.*;
  29 import com.intellij.openapi.vfs.ex.VirtualFileManagerEx;
  30 import com.intellij.openapi.vfs.impl.BulkVirtualFileListenerAdapter;
  31 import com.intellij.openapi.vfs.newvfs.ManagingFS;
  32 import com.intellij.openapi.vfs.newvfs.NewVirtualFile;
  33 import com.intellij.openapi.vfs.newvfs.events.VFileEvent;
  34 import com.intellij.openapi.vfs.newvfs.persistent.PersistentFS;
  35 import com.intellij.psi.*;
  36 import com.intellij.psi.impl.PsiDocumentTransactionListener;
  37 import com.intellij.psi.impl.source.PsiFileImpl;
  38 import com.intellij.psi.stubs.StubUpdatingIndex;
  39 import com.intellij.util.ArrayUtil;
  40 import com.intellij.util.CommonProcessors;
  41 import com.intellij.util.Processor;
  42 import com.intellij.util.concurrency.Semaphore;
  43 import com.intellij.util.containers.ConcurrentHashSet;
  44 import com.intellij.util.io.IOUtil;
  45 import com.intellij.util.messages.MessageBus;
  46 import com.intellij.util.messages.MessageBusConnection;
  47 import gnu.trove.TObjectIntHashMap;
  48 import org.jetbrains.annotations.NonNls;
  49 import org.jetbrains.annotations.NotNull;
  50 import org.jetbrains.annotations.Nullable;
  51
  52 import javax.swing.*;
  53 import java.io.*;
  54 import java.util.*;
  55 import java.util.concurrent.CopyOnWriteArrayList;
  56 import java.util.concurrent.atomic.AtomicInteger;
  57 import java.util.concurrent.locks.Lock;
  58
  59 /**
  60  * @author Eugene Zhuravlev
  61  *         Date: Dec 20, 2007
  62  */
  63
  64 public class FileBasedIndex implements ApplicationComponent {
  65   private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.FileBasedIndex");
  66   @NonNls
  67   private static final String CORRUPTION_MARKER_NAME = "corruption.marker";
  68   private final Map<ID<?, ?>, Pair<UpdatableIndex<?, ?, FileContent>, InputFilter>> myIndices = new HashMap<ID<?, ?>, Pair<UpdatableIndex<?, ?, FileContent>, InputFilter>>();
  69   private final Map<ID<?, ?>, Semaphore> myUnsavedDataIndexingSemaphores = new HashMap<ID<?,?>, Semaphore>();
  70   private final TObjectIntHashMap<ID<?, ?>> myIndexIdToVersionMap = new TObjectIntHashMap<ID<?, ?>>();
  71   private final Set<ID<?, ?>> myNeedContentLoading = new HashSet<ID<?, ?>>();
  72
  73   private final PerIndexDocumentMap<Long> myLastIndexedDocStamps = new PerIndexDocumentMap<Long>() {
  74     @Override
  75     protected Long createDefault(Document document) {
  76       return 0L;
  77     }
  78   };
  79   private final PerIndexDocumentMap<CharSequence> myLastIndexedUnsavedContent = new PerIndexDocumentMap<CharSequence>() {
  80     @Override
  81     protected CharSequence createDefault(Document document) {
  82       return loadContent(FileDocumentManager.getInstance().getFile(document));
  83     }
  84   };
  85
  86   private final ChangedFilesUpdater myChangedFilesUpdater;
  87
  88   private final List<IndexableFileSet> myIndexableSets = new CopyOnWriteArrayList<IndexableFileSet>();
  89
  90   public static final int OK = 1;
  91   public static final int REQUIRES_REBUILD = 2;
  92   public static final int REBUILD_IN_PROGRESS = 3;
  93   private final Map<ID<?, ?>, AtomicInteger> myRebuildStatus = new HashMap<ID<?,?>, AtomicInteger>();
  94
  95   private final VirtualFileManagerEx myVfManager;
  96   private final ConcurrentHashSet<ID<?, ?>> myUpToDateIndices = new ConcurrentHashSet<ID<?, ?>>();
  97   private final Map<Document, PsiFile> myTransactionMap = new HashMap<Document, PsiFile>();
  98
  99   private final FileContentStorage myFileContentAttic;
 100   private final Map<ID<?, ?>, FileBasedIndexExtension<?, ?>> myExtentions = new HashMap<ID<?,?>, FileBasedIndexExtension<?,?>>();
 101
 102   private static final int ALREADY_PROCESSED = 0x02;
 103
 104   public void requestReindex(final VirtualFile file) {
 105     myChangedFilesUpdater.scheduleInvalidation(file, true);
 106   }
 107
 108   public interface InputFilter {
 109     boolean acceptInput(VirtualFile file);
 110   }
 111
 112   public FileBasedIndex(final VirtualFileManagerEx vfManager, MessageBus bus) throws IOException {
 113     myVfManager = vfManager;
 114
 115     final MessageBusConnection connection = bus.connect();
 116     connection.subscribe(PsiDocumentTransactionListener.TOPIC, new PsiDocumentTransactionListener() {
 117       public void transactionStarted(final Document doc, final PsiFile file) {
 118         if (file != null) {
 119           myTransactionMap.put(doc, file);
 120           myUpToDateIndices.clear();
 121         }
 122       }
 123
 124       public void transactionCompleted(final Document doc, final PsiFile file) {
 125         myTransactionMap.remove(doc);
 126       }
 127     });
 128
 129     connection.subscribe(AppTopics.FILE_TYPES, new FileTypeListener() {
 130       public void beforeFileTypesChanged(final FileTypeEvent event) {
 131         cleanupProcessedFlag();
 132         // TODO: temporary solution for tests to avoid 'twin stubs' problem
 133         // Correct index update will require more information from FileType events
 134         if (ApplicationManager.getApplication().isUnitTestMode()) {
 135           requestRebuild(StubUpdatingIndex.INDEX_ID);
 136         }
 137       }
 138
 139       public void fileTypesChanged(final FileTypeEvent event) {
 140       }
 141     });
 142
 143     ApplicationManager.getApplication().addApplicationListener(new ApplicationAdapter() {
 144       public void writeActionStarted(Object action) {
 145         myUpToDateIndices.clear();
 146       }
 147     });
 148
 149     final File workInProgressFile = getMarkerFile();
 150     if (workInProgressFile.exists()) {
 151       // previous IDEA session was closed incorrectly, so drop all indices
 152       FileUtil.delete(PathManager.getIndexRoot());
 153     }
 154
 155     try {
 156       final FileBasedIndexExtension[] extensions = Extensions.getExtensions(FileBasedIndexExtension.EXTENSION_POINT_NAME);
 157       for (FileBasedIndexExtension<?, ?> extension : extensions) {
 158         myRebuildStatus.put(extension.getName(), new AtomicInteger(OK));
 159       }
 160
 161       final File corruptionMarker = new File(PathManager.getIndexRoot(), CORRUPTION_MARKER_NAME);
 162       final boolean currentVersionCorrupted = corruptionMarker.exists();
 163       for (FileBasedIndexExtension<?, ?> extension : extensions) {
 164         registerIndexer(extension, currentVersionCorrupted);
 165       }
 166       FileUtil.delete(corruptionMarker);
 167       dropUnregisteredIndices();
 168
 169       // check if rebuild was requested for any index during registration
 170       for (ID<?, ?> indexId : myIndices.keySet()) {
 171         if (myRebuildStatus.get(indexId).compareAndSet(REQUIRES_REBUILD, OK)) {
 172           try {
 173             clearIndex(indexId);
 174           }
 175           catch (StorageException e) {
 176             requestRebuild(indexId);
 177             LOG.error(e);
 178           }
 179         }
 180       }
 181
 182       myChangedFilesUpdater = new ChangedFilesUpdater();
 183       //vfManager.addVirtualFileListener(myChangedFilesUpdater);
 184       connection.subscribe(VirtualFileManager.VFS_CHANGES, new BulkVirtualFileListenerAdapter(myChangedFilesUpdater) {
 185         public void before(final List<? extends VFileEvent> events) {
 186           if (ApplicationManager.getApplication().isDispatchThread()) {
 187             new Task.Modal(null, "Preparing index invalidation...", false) {
 188               public void run(@NotNull ProgressIndicator indicator) {
 189                 indicator.setIndeterminate(true);
 190                 deliverBeforeEvents(events);
 191               }
 192             }.queue();
 193           }
 194           else {
 195             deliverBeforeEvents(events);
 196           }
 197         }
 198
 199         private void deliverBeforeEvents(List<? extends VFileEvent> events) {
 200           super.before(events);
 201         }
 202       });
 203
 204       vfManager.registerRefreshUpdater(myChangedFilesUpdater);
 205       myFileContentAttic = new FileContentStorage(new File(PathManager.getIndexRoot(), "updates.tmp"));
 206     }
 207     finally {
 208       workInProgressFile.createNewFile();
 209       saveRegisteredInices(myIndices.keySet());
 210     }
 211   }
 212
 213   private static FileBasedIndex ourInstance = CachedSingletonsRegistry.markCachedField(FileBasedIndex.class);
 214   public static FileBasedIndex getInstance() {
 215     if (ourInstance == null) {
 216       ourInstance = ApplicationManager.getApplication().getComponent(FileBasedIndex.class);
 217     }
 218
 219     return ourInstance;
 220   }
 221
 222   /**
 223    * @return true if registered index requires full rebuild for some reason, e.g. is just created or corrupted @param extension
 224    * @param isCurrentVersionCorrupted
 225    */
 226   private <K, V> void registerIndexer(final FileBasedIndexExtension<K, V> extension, final boolean isCurrentVersionCorrupted) throws IOException {
 227     final ID<K, V> name = extension.getName();
 228     final int version = extension.getVersion();
 229     if (extension.dependsOnFileContent()) {
 230       myNeedContentLoading.add(name);
 231     }
 232     myIndexIdToVersionMap.put(name, version);
 233     final File versionFile = IndexInfrastructure.getVersionFile(name);
 234     if (isCurrentVersionCorrupted || IndexInfrastructure.versionDiffers(versionFile, version)) {
 235       FileUtil.delete(IndexInfrastructure.getIndexRootDir(name));
 236       IndexInfrastructure.rewriteVersion(versionFile, version);
 237     }
 238
 239     for (int attempt = 0; attempt < 2; attempt++) {
 240       try {
 241         final MapIndexStorage<K, V> storage = new MapIndexStorage<K, V>(IndexInfrastructure.getStorageFile(name), extension.getKeyDescriptor(), extension.getValueExternalizer(), extension.getCacheSize());
 242         final IndexStorage<K, V> memStorage = new MemoryIndexStorage<K, V>(storage);
 243         final UpdatableIndex<K, V, FileContent> index = createIndex(name, extension, memStorage);
 244         myIndices.put(name, new Pair<UpdatableIndex<?,?, FileContent>, InputFilter>(index, new IndexableFilesFilter(extension.getInputFilter())));
 245         myUnsavedDataIndexingSemaphores.put(name, new Semaphore());
 246         myExtentions.put(name, extension);
 247         break;
 248       }
 249       catch (IOException e) {
 250         FileUtil.delete(IndexInfrastructure.getIndexRootDir(name));
 251         IndexInfrastructure.rewriteVersion(versionFile, version);
 252       }
 253     }
 254   }
 255
 256   private static void saveRegisteredInices(Collection<ID<?, ?>> ids) {
 257     final File file = getRegisteredIndicesFile();
 258     try {
 259       file.getParentFile().mkdirs();
 260       file.createNewFile();
 261       final DataOutputStream os = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(file)));
 262       try {
 263         os.writeInt(ids.size());
 264         for (ID<?, ?> id : ids) {
 265           IOUtil.writeString(id.toString(), os);
 266         }
 267       }
 268       finally {
 269         os.close();
 270       }
 271     }
 272     catch (IOException ignored) {
 273     }
 274   }
 275
 276   private static Set<String> readRegistsredIndexNames() {
 277     final Set<String> result = new HashSet<String>();
 278     try {
 279       final DataInputStream in = new DataInputStream(new BufferedInputStream(new FileInputStream(getRegisteredIndicesFile())));
 280       try {
 281         final int size = in.readInt();
 282         for (int idx = 0; idx < size; idx++) {
 283           result.add(IOUtil.readString(in));
 284         }
 285       }
 286       finally {
 287         in.close();
 288       }
 289     }
 290     catch (IOException ignored) {
 291     }
 292     return result;
 293   }
 294
 295   private static File getRegisteredIndicesFile() {
 296     return new File(PathManager.getIndexRoot(), "registered");
 297   }
 298
 299   private static File getMarkerFile() {
 300     return new File(PathManager.getIndexRoot(), "work_in_progress");
 301   }
 302
 303   private <K, V> UpdatableIndex<K, V, FileContent> createIndex(final ID<K, V> indexId, final FileBasedIndexExtension<K, V> extension, final IndexStorage<K, V> storage) {
 304     if (extension instanceof CustomImplementationFileBasedIndexExtension) {
 305       return ((CustomImplementationFileBasedIndexExtension<K, V, FileContent>)extension).createIndexImplementation(indexId, this, storage);
 306     }
 307     return new MapReduceIndex<K, V, FileContent>(indexId, extension.getIndexer(), storage);
 308   }
 309
 310   @NonNls
 311   @NotNull
 312   public String getComponentName() {
 313     return "FileBasedIndex";
 314   }
 315
 316   public void initComponent() {
 317   }
 318
 319   public void disposeComponent() {
 320     myChangedFilesUpdater.forceUpdate();
 321
 322     for (ID<?, ?> indexId : myIndices.keySet()) {
 323       final UpdatableIndex<?, ?, FileContent> index = getIndex(indexId);
 324       assert index != null;
 325       index.dispose();
 326     }
 327
 328     //myVfManager.removeVirtualFileListener(myChangedFilesUpdater);
 329     myVfManager.unregisterRefreshUpdater(myChangedFilesUpdater);
 330
 331     FileUtil.delete(getMarkerFile());
 332   }
 333
 334   public void flushCaches() {
 335     ApplicationManager.getApplication().executeOnPooledThread(new Runnable() {
 336       public void run() {
 337         for (ID<?, ?> indexId : myIndices.keySet()) {
 338           //noinspection ConstantConditions
 339           try {
 340             getIndex(indexId).flush();
 341           }
 342           catch (StorageException e) {
 343             LOG.info(e);
 344             requestRebuild(indexId);
 345           }
 346         }
 347       }
 348     });
 349   }
 350
 351   @NotNull
 352   public <K> Collection<K> getAllKeys(final ID<K, ?> indexId) {
 353     Set<K> allKeys = new HashSet<K>();
 354     processAllKeys(indexId, new CommonProcessors.CollectProcessor<K>(allKeys));
 355     return allKeys;
 356   }
 357
 358   public <K> boolean processAllKeys(final ID<K, ?> indexId, Processor<K> processor) {
 359     try {
 360       ensureUpToDate(indexId);
 361       final UpdatableIndex<K, ?, FileContent> index = getIndex(indexId);
 362       if (index == null) return true;
 363       return index.processAllKeys(processor);
 364     }
 365     catch (StorageException e) {
 366       scheduleRebuild(indexId, e);
 367     }
 368     catch (RuntimeException e) {
 369       final Throwable cause = e.getCause();
 370       if (cause instanceof StorageException || cause instanceof IOException) {
 371         scheduleRebuild(indexId, cause);
 372       }
 373       else {
 374         throw e;
 375       }
 376     }
 377
 378     return false;
 379   }
 380
 381   private static final ThreadLocal<Integer> myUpToDateCheckState = new ThreadLocal<Integer>();
 382
 383   public void disableUpToDateCheckForCurrentThread() {
 384     final Integer currentValue = myUpToDateCheckState.get();
 385     myUpToDateCheckState.set(currentValue == null? 1 : currentValue.intValue() + 1);
 386   }
 387
 388   public void enableUpToDateCheckForCurrentThread() {
 389     final Integer currentValue = myUpToDateCheckState.get();
 390     if (currentValue != null) {
 391       final int newValue = currentValue.intValue() - 1;
 392       if (newValue != 0) {
 393         myUpToDateCheckState.set(newValue);
 394       }
 395       else {
 396         myUpToDateCheckState.remove();
 397       }
 398     }
 399   }
 400
 401   private boolean isUpToDateCheckEnabled() {
 402     final Integer value = myUpToDateCheckState.get();
 403     return value == null || value.intValue() == 0;
 404   }
 405
 406
 407   private final ThreadLocal<Boolean> myReentrancyGuard = new ThreadLocal<Boolean>() {
 408     protected Boolean initialValue() {
 409       return Boolean.FALSE;
 410     }
 411   };
 412
 413   /**
 414    * DO NOT CALL DIRECTLY IN CLIENT CODE
 415    * The method is internal to indexing engine end is called internally. The method is public due to implementation details
 416    */
 417   public <K> void ensureUpToDate(final ID<K, ?> indexId) {
 418     if (myReentrancyGuard.get().booleanValue()) {
 419       //assert false : "ensureUpToDate() is not reentrant!";
 420       return;
 421     }
 422     myReentrancyGuard.set(Boolean.TRUE);
 423
 424     try {
 425       myChangedFilesUpdater.ensureAllInvalidateTasksCompleted();
 426
 427       if (isUpToDateCheckEnabled()) {
 428         try {
 429           checkRebuild(indexId);
 430           indexUnsavedDocuments(indexId);
 431         }
 432         catch (StorageException e) {
 433           scheduleRebuild(indexId, e);
 434         }
 435         catch (RuntimeException e) {
 436           final Throwable cause = e.getCause();
 437           if (cause instanceof StorageException || cause instanceof IOException) {
 438             scheduleRebuild(indexId, e);
 439           }
 440           else {
 441             throw e;
 442           }
 443         }
 444       }
 445     }
 446     finally {
 447       myReentrancyGuard.set(Boolean.FALSE);
 448     }
 449   }
 450
 451   @NotNull
 452   public <K, V> List<V> getValues(final ID<K, V> indexId, @NotNull K dataKey, final VirtualFileFilter filter) {
 453     final List<V> values = new ArrayList<V>();
 454     processValuesImpl(indexId, dataKey, true, null, new ValueProcessor<V>() {
 455       public boolean process(final VirtualFile file, final V value) {
 456         values.add(value);
 457         return true;
 458       }
 459     }, filter);
 460     return values;
 461   }
 462
 463   @NotNull
 464   public <K, V> Collection<VirtualFile> getContainingFiles(final ID<K, V> indexId, @NotNull K dataKey, final VirtualFileFilter filter) {
 465     final Set<VirtualFile> files = new HashSet<VirtualFile>();
 466     processValuesImpl(indexId, dataKey, false, null, new ValueProcessor<V>() {
 467       public boolean process(final VirtualFile file, final V value) {
 468         files.add(file);
 469         return true;
 470       }
 471     }, filter);
 472     return files;
 473   }
 474
 475
 476   public interface ValueProcessor<V> {
 477     /**
 478      * @param value a value to process
 479      * @param file the file the value came from
 480      * @return false if no further processing is needed, true otherwise
 481      */
 482     boolean process(VirtualFile file, V value);
 483   }
 484
 485   /**
 486    * @return false if ValueProcessor.process() returned false; true otherwise or if ValueProcessor was not called at all
 487    */
 488   public <K, V> boolean processValues(final ID<K, V> indexId, final @NotNull K dataKey, @Nullable final VirtualFile inFile,
 489                                    ValueProcessor<V> processor, final VirtualFileFilter filter) {
 490     return processValuesImpl(indexId, dataKey, false, inFile, processor, filter);
 491   }
 492
 493   private <K, V> boolean processValuesImpl(final ID<K, V> indexId, final K dataKey, boolean ensureValueProcessedOnce,
 494                                         @Nullable final VirtualFile restrictToFile, ValueProcessor<V> processor,
 495                                         final VirtualFileFilter filter) {
 496     try {
 497       ensureUpToDate(indexId);
 498       final UpdatableIndex<K, V, FileContent> index = getIndex(indexId);
 499       if (index == null) {
 500         return true;
 501       }
 502
 503       final Lock readLock = index.getReadLock();
 504       try {
 505         readLock.lock();
 506         final ValueContainer<V> container = index.getData(dataKey);
 507
 508         boolean shouldContinue = true;
 509
 510         if (restrictToFile != null) {
 511           if (restrictToFile instanceof VirtualFileWithId) {
 512             final int restrictedFileId = getFileId(restrictToFile);
 513             for (final Iterator<V> valueIt = container.getValueIterator(); valueIt.hasNext();) {
 514               final V value = valueIt.next();
 515               if (container.isAssociated(value, restrictedFileId)) {
 516                 shouldContinue = processor.process(restrictToFile, value);
 517                 if (!shouldContinue) {
 518                   break;
 519                 }
 520               }
 521             }
 522           }
 523         }
 524         else {
 525           final PersistentFS fs = (PersistentFS)ManagingFS.getInstance();
 526           VALUES_LOOP: for (final Iterator<V> valueIt = container.getValueIterator(); valueIt.hasNext();) {
 527             final V value = valueIt.next();
 528             for (final ValueContainer.IntIterator inputIdsIterator = container.getInputIdsIterator(value); inputIdsIterator.hasNext();) {
 529               final int id = inputIdsIterator.next();
 530               VirtualFile file = IndexInfrastructure.findFileById(fs, id);
 531               if (file != null && filter.accept(file)) {
 532                 shouldContinue = processor.process(file, value);
 533                 if (!shouldContinue) {
 534                   break VALUES_LOOP;
 535                 }
 536                 if (ensureValueProcessedOnce) {
 537                   break; // continue with the next value
 538                 }
 539               }
 540             }
 541           }
 542         }
 543         return shouldContinue;
 544       }
 545       finally {
 546         index.getReadLock().unlock();
 547       }
 548     }
 549     catch (StorageException e) {
 550       scheduleRebuild(indexId, e);
 551     }
 552     catch (RuntimeException e) {
 553       final Throwable cause = e.getCause();
 554       if (cause instanceof StorageException || cause instanceof IOException) {
 555         scheduleRebuild(indexId, cause);
 556       }
 557       else {
 558         throw e;
 559       }
 560     }
 561     return true;
 562   }
 563
 564   public interface AllValuesProcessor<V> {
 565     void process(final int inputId, V value);
 566   }
 567
 568   public <K, V> void processAllValues(final ID<K, V> indexId, AllValuesProcessor<V> processor) {
 569     try {
 570       ensureUpToDate(indexId);
 571       final UpdatableIndex<K, V, FileContent> index = getIndex(indexId);
 572       if (index == null) {
 573         return;
 574       }
 575       try {
 576         index.getReadLock().lock();
 577         for (K dataKey : index.getAllKeys()) {
 578           final ValueContainer<V> container = index.getData(dataKey);
 579           for (final Iterator<V> it = container.getValueIterator(); it.hasNext();) {
 580             final V value = it.next();
 581             for (final ValueContainer.IntIterator inputsIt = container.getInputIdsIterator(value); inputsIt.hasNext();) {
 582               processor.process(inputsIt.next(), value);
 583             }
 584           }
 585         }
 586       }
 587       finally {
 588         index.getReadLock().unlock();
 589       }
 590     }
 591     catch (StorageException e) {
 592       scheduleRebuild(indexId, e);
 593     }
 594     catch (RuntimeException e) {
 595       final Throwable cause = e.getCause();
 596       if (cause instanceof StorageException || cause instanceof IOException) {
 597         scheduleRebuild(indexId, e);
 598       }
 599       else {
 600         throw e;
 601       }
 602     }
 603   }
 604
 605   private <K> void scheduleRebuild(final ID<K, ?> indexId, final Throwable e) {
 606     requestRebuild(indexId);
 607     LOG.info(e);
 608     checkRebuild(indexId);
 609   }
 610
 611   private void checkRebuild(final ID<?, ?> indexId) {
 612     if (myRebuildStatus.get(indexId).compareAndSet(REQUIRES_REBUILD, REBUILD_IN_PROGRESS)) {
 613       cleanupProcessedFlag();
 614
 615       final Runnable rebuildRunnable = new Runnable() {
 616         public void run() {
 617
 618           final FileSystemSynchronizer synchronizer = new FileSystemSynchronizer();
 619           synchronizer.setCancelable(false);
 620           for (Project project : ProjectManager.getInstance().getOpenProjects()) {
 621             synchronizer.registerCacheUpdater(new UnindexedFilesUpdater(project, ProjectRootManager.getInstance(project), FileBasedIndex.this));
 622           }
 623
 624           try {
 625             clearIndex(indexId);
 626             synchronizer.execute();
 627           }
 628           catch (StorageException e) {
 629             requestRebuild(indexId);
 630             LOG.info(e);
 631           }
 632           finally {
 633             myRebuildStatus.get(indexId).compareAndSet(REBUILD_IN_PROGRESS, OK);
 634           }
 635         }
 636       };
 637
 638       final Application application = ApplicationManager.getApplication();
 639       if (application.isUnitTestMode()) {
 640         rebuildRunnable.run();
 641       }
 642       else {
 643         SwingUtilities.invokeLater(new Runnable() {
 644           public void run() {
 645             new Task.Modal(null, "Updating index", false) {
 646               public void run(@NotNull final ProgressIndicator indicator) {
 647                 indicator.setIndeterminate(true);
 648                 rebuildRunnable.run();
 649               }
 650             }.queue();
 651           }
 652         });
 653       }
 654     }
 655
 656     if (myRebuildStatus.get(indexId).get() == REBUILD_IN_PROGRESS) {
 657       throw new ProcessCanceledException();
 658     }
 659   }
 660
 661   private void clearIndex(final ID<?, ?> indexId) throws StorageException {
 662     final UpdatableIndex<?, ?, FileContent> index = getIndex(indexId);
 663     assert index != null;
 664     index.clear();
 665     try {
 666       IndexInfrastructure.rewriteVersion(IndexInfrastructure.getVersionFile(indexId), myIndexIdToVersionMap.get(indexId));
 667     }
 668     catch (IOException e) {
 669       LOG.error(e);
 670     }
 671   }
 672
 673   private Set<Document> getUnsavedOrTransactedDocuments() {
 674     Set<Document> docs = new HashSet<Document>(Arrays.asList(FileDocumentManager.getInstance().getUnsavedDocuments()));
 675     docs.addAll(myTransactionMap.keySet());
 676     return docs;
 677   }
 678
 679   private void indexUnsavedDocuments(ID<?, ?> indexId) throws StorageException {
 680     myChangedFilesUpdater.forceUpdate();
 681
 682     if (myUpToDateIndices.contains(indexId)) {
 683       return; // no need to index unsaved docs
 684     }
 685
 686     final Set<Document> documents = getUnsavedOrTransactedDocuments();
 687     if (!documents.isEmpty()) {
 688       // now index unsaved data
 689       final StorageGuard.Holder guard = setDataBufferingEnabled(true);
 690       try {
 691         final Semaphore semaphore = myUnsavedDataIndexingSemaphores.get(indexId);
 692         semaphore.down();
 693         try {
 694           for (Document document : documents) {
 695             indexUnsavedDocument(document, indexId);
 696           }
 697         }
 698         finally {
 699           semaphore.up();
 700
 701           while (!semaphore.waitFor(500)) { // may need to wait until another thread is done with indexing
 702             if (Thread.holdsLock(PsiLock.LOCK)) {
 703               break; // hack. Most probably that other indexing threads is waiting for PsiLock, which we're are holding.
 704             }
 705           }
 706           myUpToDateIndices.add(indexId); // safe to set the flag here, becase it will be cleared under the WriteAction
 707         }
 708       }
 709       finally {
 710         guard.leave();
 711       }
 712     }
 713   }
 714
 715   private interface DocumentContent {
 716     String getText();
 717     long getModificationStamp();
 718   }
 719
 720   private static class AuthenticContent implements DocumentContent {
 721     private final Document myDocument;
 722
 723     private AuthenticContent(final Document document) {
 724       myDocument = document;
 725     }
 726
 727     public String getText() {
 728       return myDocument.getText();
 729     }
 730
 731     public long getModificationStamp() {
 732       return myDocument.getModificationStamp();
 733     }
 734   }
 735
 736   private static class PsiContent implements DocumentContent {
 737     private final Document myDocument;
 738     private final PsiFile myFile;
 739
 740     private PsiContent(final Document document, final PsiFile file) {
 741       myDocument = document;
 742       myFile = file;
 743     }
 744
 745     public String getText() {
 746       if (myFile.getModificationStamp() != myDocument.getModificationStamp()) {
 747         final ASTNode node = myFile.getNode();
 748         assert node != null;
 749         return node.getText();
 750       }
 751       return myDocument.getText();
 752     }
 753
 754     public long getModificationStamp() {
 755       return myFile.getModificationStamp();
 756     }
 757   }
 758
 759   private void indexUnsavedDocument(final Document document, final ID<?, ?> requestedIndexId) throws StorageException {
 760     final VirtualFile vFile = FileDocumentManager.getInstance().getFile(document);
 761     if (!(vFile instanceof VirtualFileWithId) || !vFile.isValid()) {
 762       return;
 763     }
 764
 765     PsiFile dominantContentFile = findDominantPsiForDocument(document);
 766
 767     DocumentContent content;
 768     if (dominantContentFile != null && dominantContentFile.getModificationStamp() != document.getModificationStamp()) {
 769       content = new PsiContent(document, dominantContentFile);
 770     }
 771     else {
 772       content = new AuthenticContent(document);
 773     }
 774
 775     final long currentDocStamp = content.getModificationStamp();
 776     if (currentDocStamp != myLastIndexedDocStamps.getAndSet(document, requestedIndexId, currentDocStamp).longValue()) {
 777       final FileContent newFc = new FileContent(vFile, content.getText(), vFile.getCharset());
 778
 779       if (dominantContentFile != null) {
 780         dominantContentFile.putUserData(PsiFileImpl.BUILDING_STUB, true);
 781         newFc.putUserData(PSI_FILE, dominantContentFile);
 782       }
 783
 784       if (content instanceof AuthenticContent) {
 785         newFc.putUserData(EDITOR_HIGHLIGHTER, document instanceof DocumentImpl
 786                                               ? ((DocumentImpl)document).getEditorHighlighterForCachesBuilding() : null);
 787       }
 788
 789       CharSequence lastIndexed = myLastIndexedUnsavedContent.get(document, requestedIndexId);
 790       final FileContent oldFc = new FileContent(vFile, lastIndexed, vFile.getCharset());
 791       if (getInputFilter(requestedIndexId).acceptInput(vFile)) {
 792         final int inputId = Math.abs(getFileId(vFile));
 793         getIndex(requestedIndexId).update(inputId, newFc, oldFc);
 794       }
 795
 796       if (dominantContentFile != null) {
 797         dominantContentFile.putUserData(PsiFileImpl.BUILDING_STUB, null);
 798       }
 799
 800       myLastIndexedUnsavedContent.put(document, requestedIndexId, newFc.getContentAsText());
 801     }
 802   }
 803
 804   public static final Key<PsiFile> PSI_FILE = new Key<PsiFile>("PSI for stubs");
 805   public static final Key<EditorHighlighter> EDITOR_HIGHLIGHTER = new Key<EditorHighlighter>("Editor");
 806   public static final Key<Project> PROJECT = new Key<Project>("Context project");
 807   public static final Key<VirtualFile> VIRTUAL_FILE = new Key<VirtualFile>("Context virtual file");
 808
 809   @Nullable
 810   private PsiFile findDominantPsiForDocument(final Document document) {
 811     if (myTransactionMap.containsKey(document)) {
 812       return myTransactionMap.get(document);
 813     }
 814
 815     return findLatestKnownPsiForUncomittedDocument(document);
 816   }
 817
 818   private final StorageGuard myStorageLock = new StorageGuard();
 819
 820   private StorageGuard.Holder setDataBufferingEnabled(final boolean enabled) {
 821     final StorageGuard.Holder holder = myStorageLock.enter(enabled);
 822     if (!enabled) {
 823       synchronized (myLastIndexedDocStamps) {
 824         myLastIndexedDocStamps.clear();
 825         myLastIndexedUnsavedContent.clear();
 826       }
 827     }
 828     for (ID<?, ?> indexId : myIndices.keySet()) {
 829       final MapReduceIndex index = (MapReduceIndex)getIndex(indexId);
 830       assert index != null;
 831       final IndexStorage indexStorage = index.getStorage();
 832       ((MemoryIndexStorage)indexStorage).setBufferingEnabled(enabled);
 833     }
 834     return holder;
 835   }
 836
 837   private void dropUnregisteredIndices() {
 838     final Set<String> indicesToDrop = readRegistsredIndexNames();
 839     for (ID<?, ?> key : myIndices.keySet()) {
 840       indicesToDrop.remove(key.toString());
 841     }
 842     for (String s : indicesToDrop) {
 843       FileUtil.delete(IndexInfrastructure.getIndexRootDir(ID.create(s)));
 844     }
 845   }
 846
 847   public void requestRebuild(ID<?, ?> indexId) {
 848     cleanupProcessedFlag();
 849     LOG.info("Rebuild requested for index " + indexId, new Throwable());
 850     myRebuildStatus.get(indexId).set(REQUIRES_REBUILD);
 851   }
 852
 853   private <K, V> UpdatableIndex<K, V, FileContent> getIndex(ID<K, V> indexId) {
 854     final Pair<UpdatableIndex<?, ?, FileContent>, InputFilter> pair = myIndices.get(indexId);
 855     //noinspection unchecked
 856     return pair != null? (UpdatableIndex<K,V, FileContent>)pair.getFirst() : null;
 857   }
 858
 859   private InputFilter getInputFilter(ID<?, ?> indexId) {
 860     final Pair<UpdatableIndex<?, ?, FileContent>, InputFilter> pair = myIndices.get(indexId);
 861     return pair != null? pair.getSecond() : null;
 862   }
 863
 864   public void indexFileContent(com.intellij.ide.startup.FileContent content) {
 865     myChangedFilesUpdater.ensureAllInvalidateTasksCompleted();
 866     final VirtualFile file = content.getVirtualFile();
 867     FileContent fc = null;
 868     FileContent oldContent = null;
 869     final byte[] oldBytes = myFileContentAttic.remove(file);
 870     final boolean forceIndexing = oldBytes != null;
 871
 872     PsiFile psiFile = null;
 873     for (ID<?, ?> indexId : myIndices.keySet()) {
 874       if (forceIndexing ? getInputFilter(indexId).acceptInput(file) : shouldIndexFile(file, indexId)) {
 875         if (fc == null) {
 876           byte[] currentBytes;
 877           try {
 878             currentBytes = content.getBytes();
 879           }
 880           catch (IOException e) {
 881             currentBytes = ArrayUtil.EMPTY_BYTE_ARRAY;
 882           }
 883           fc = new FileContent(file, currentBytes);
 884
 885           psiFile = content.getUserData(PSI_FILE);
 886           if (psiFile != null) {
 887             psiFile.putUserData(PsiFileImpl.BUILDING_STUB, true);
 888             fc.putUserData(PSI_FILE, psiFile);
 889           }
 890           Project project = content.getUserData(PROJECT);
 891           fc.putUserData(PROJECT, project);
 892           oldContent = oldBytes != null? new FileContent(file, oldBytes) : null;
 893         }
 894
 895         try {
 896           updateSingleIndex(indexId, file, fc, oldContent);
 897         }
 898         catch (StorageException e) {
 899           requestRebuild(indexId);
 900           LOG.info(e);
 901         }
 902       }
 903     }
 904
 905     if (psiFile != null) {
 906       psiFile.putUserData(PsiFileImpl.BUILDING_STUB, null);
 907     }
 908
 909     IndexingStamp.flushCache();
 910   }
 911
 912   private void updateSingleIndex(final ID<?, ?> indexId, final VirtualFile file, final FileContent currentFC, final FileContent oldFC)
 913     throws StorageException {
 914
 915     final StorageGuard.Holder lock = setDataBufferingEnabled(false);
 916
 917     try {
 918       final int inputId = Math.abs(getFileId(file));
 919
 920       final UpdatableIndex<?, ?, FileContent> index = getIndex(indexId);
 921       assert index != null;
 922
 923       index.update(inputId, currentFC, oldFC);
 924       ApplicationManager.getApplication().runReadAction(new Runnable() {
 925         public void run() {
 926           if (file.isValid()) {
 927             if (currentFC != null) {
 928               IndexingStamp.update(file, indexId, IndexInfrastructure.getIndexCreationStamp(indexId));
 929             }
 930             else {
 931               // mark the file as unindexed
 932               IndexingStamp.update(file, indexId, -1L);
 933             }
 934           }
 935         }
 936       });
 937     }
 938     finally {
 939       lock.leave();
 940     }
 941   }
 942
 943   public static int getFileId(final VirtualFile file) {
 944     if (file instanceof VirtualFileWithId) {
 945       return ((VirtualFileWithId)file).getId();
 946     }
 947
 948     throw new IllegalArgumentException("Virtual file doesn't support id: " + file + ", implementation class: " + file.getClass().getName());
 949   }
 950
 951   private static CharSequence loadContent(VirtualFile file) {
 952     return LoadTextUtil.loadText(file, true);
 953   }
 954
 955   private abstract static class InvalidationTask implements Runnable {
 956     private final VirtualFile mySubj;
 957
 958     protected InvalidationTask(final VirtualFile subj) {
 959       mySubj = subj;
 960     }
 961
 962     public VirtualFile getSubj() {
 963       return mySubj;
 964     }
 965   }
 966
 967   private final class ChangedFilesUpdater extends VirtualFileAdapter implements CacheUpdater{
 968     private final Set<VirtualFile> myFilesToUpdate = Collections.synchronizedSet(new HashSet<VirtualFile>());
 969     private final Queue<InvalidationTask> myFutureInvalidations = new LinkedList<InvalidationTask>();
 970     private final ManagingFS myManagingFS = ManagingFS.getInstance();
 971     // No need to react on movement events since files stay valid, their ids don't change and all associated attributes remain intact.
 972
 973     public void fileCreated(final VirtualFileEvent event) {
 974       markDirty(event);
 975     }
 976
 977     public void fileDeleted(final VirtualFileEvent event) {
 978       myFilesToUpdate.remove(event.getFile()); // no need to update it anymore
 979     }
 980
 981     public void fileCopied(final VirtualFileCopyEvent event) {
 982       markDirty(event);
 983     }
 984
 985     public void beforeFileDeletion(final VirtualFileEvent event) {
 986       scheduleInvalidation(event.getFile(), false);
 987     }
 988
 989     public void beforeContentsChange(final VirtualFileEvent event) {
 990       scheduleInvalidation(event.getFile(), true);
 991     }
 992
 993     public void contentsChanged(final VirtualFileEvent event) {
 994       markDirty(event);
 995     }
 996
 997     public void beforePropertyChange(final VirtualFilePropertyEvent event) {
 998       if (event.getPropertyName().equals(VirtualFile.PROP_NAME)) {
 999         // indexes may depend on file name
1000         final VirtualFile file = event.getFile();
1001         if (!file.isDirectory()) {
1002           // name change may lead to filetype change so the file might become not indexable
1003           // in general case have to 'unindex' the file and index it again if needed after the name has been changed
1004           scheduleInvalidation(file, false);
1005         }
1006       }
1007     }
1008
1009     public void propertyChanged(final VirtualFilePropertyEvent event) {
1010       if (event.getPropertyName().equals(VirtualFile.PROP_NAME)) {
1011         // indexes may depend on file name
1012         if (!event.getFile().isDirectory()) {
1013           markDirty(event);
1014         }
1015       }
1016     }
1017
1018     private void markDirty(final VirtualFileEvent event) {
1019       cleanProcessedFlag(event.getFile());
1020       iterateIndexableFiles(event.getFile(), new Processor<VirtualFile>() {
1021         public boolean process(final VirtualFile file) {
1022           FileContent fileContent = null;
1023           final boolean isTooLarge = SingleRootFileViewProvider.isTooLarge(file);
1024           for (ID<?, ?> indexId : myIndices.keySet()) {
1025             if (getInputFilter(indexId).acceptInput(file)) {
1026               if (myNeedContentLoading.contains(indexId)) {
1027                 if (!isTooLarge) {
1028                   myFilesToUpdate.add(file);
1029                 }
1030               }
1031               else {
1032                 try {
1033                   if (fileContent == null) {
1034                     fileContent = new FileContent(file);
1035                   }
1036                   updateSingleIndex(indexId, file, fileContent, null);
1037                 }
1038                 catch (StorageException e) {
1039                   LOG.info(e);
1040                   requestRebuild(indexId);
1041                 }
1042               }
1043             }
1044           }
1045
1046           IndexingStamp.flushCache();
1047           return true;
1048         }
1049       });
1050     }
1051
1052     public void scheduleInvalidation(final VirtualFile file, final boolean saveContent) {
1053       if (file.isDirectory()) {
1054         if (isMock(file) || myManagingFS.wereChildrenAccessed(file)) {
1055           for (VirtualFile child : file.getChildren()) {
1056             scheduleInvalidation(child, saveContent);
1057           }
1058         }
1059       }
1060       else {
1061         cleanProcessedFlag(file);
1062
1063         final boolean isTooLarge = SingleRootFileViewProvider.isTooLarge(file);
1064         final List<ID<?, ?>> affectedIndices = new ArrayList<ID<?, ?>>(myIndices.size());
1065         FileContent fileContent = null;
1066
1067         //final int fileId = getFileId(file);
1068         //LOG.assertTrue(fileId >= 0);
1069         //synchronized (myInvalidationInProgress) {
1070         //  myInvalidationInProgress.add(fileId);
1071         //}
1072
1073         for (ID<?, ?> indexId : myIndices.keySet()) {
1074           if (shouldUpdateIndex(file, indexId)) {
1075             if (myNeedContentLoading.contains(indexId)) {
1076               if (!isTooLarge) {
1077                 affectedIndices.add(indexId);
1078               }
1079             }
1080             else {
1081               // invalidate it synchronously
1082               try {
1083                 if (fileContent == null) {
1084                   fileContent = new FileContent(file);
1085                 }
1086                 updateSingleIndex(indexId, file, null, fileContent);
1087               }
1088               catch (StorageException e) {
1089                 LOG.info(e);
1090                 requestRebuild(indexId);
1091               }
1092             }
1093           }
1094         }
1095         IndexingStamp.flushCache();
1096
1097         if (!affectedIndices.isEmpty()) {
1098           if (saveContent) {
1099             myFileContentAttic.offer(file);
1100             iterateIndexableFiles(file, new Processor<VirtualFile>() {
1101               public boolean process(final VirtualFile file) {
1102                 myFilesToUpdate.add(file);
1103                 return true;
1104               }
1105             });
1106           }
1107           else {
1108             // first check if there is an unprocessed content from previous events
1109             byte[] content = myFileContentAttic.remove(file);
1110             try {
1111               if (content == null) {
1112                 content = file.contentsToByteArray();
1113               }
1114             }
1115             catch (IOException e) {
1116               content = ArrayUtil.EMPTY_BYTE_ARRAY;
1117             }
1118             final FileContent fc = new FileContent(file, content);
1119             synchronized (myFutureInvalidations) {
1120               final InvalidationTask invalidator = new InvalidationTask(file) {
1121                 public void run() {
1122                   Throwable unexpectedError = null;
1123                   for (ID<?, ?> indexId : affectedIndices) {
1124                     try {
1125                       updateSingleIndex(indexId, file, null, fc);
1126                     }
1127                     catch (StorageException e) {
1128                       LOG.info(e);
1129                       requestRebuild(indexId);
1130                     }
1131                     catch (ProcessCanceledException ignored) {
1132                     }
1133                     catch (Throwable e) {
1134                       LOG.info(e);
1135                       if (unexpectedError == null) {
1136                         unexpectedError = e;
1137                       }
1138                     }
1139                   }
1140
1141                   IndexingStamp.flushCache();
1142                   if (unexpectedError != null) {
1143                     LOG.error(unexpectedError);
1144                   }
1145                 }
1146               };
1147
1148               myFutureInvalidations.offer(invalidator);
1149             }
1150           }
1151         }
1152       }
1153     }
1154
1155     public void ensureAllInvalidateTasksCompleted() {
1156       final boolean doProgressThing;
1157
1158       final int size;
1159       synchronized (myFutureInvalidations) {
1160         size = myFutureInvalidations.size();
1161         if (size == 0) return;
1162
1163         doProgressThing = size > 1 && ApplicationManager.getApplication().isDispatchThread();
1164       }
1165
1166       final Task.Modal task = new Task.Modal(null, "Invalidating Index Entries", false) {
1167         public void run(@NotNull final ProgressIndicator indicator) {
1168           indicator.setText("");
1169           int count = 0;
1170           while (true) {
1171             InvalidationTask r;
1172             synchronized (myFutureInvalidations) {
1173               r = myFutureInvalidations.poll();
1174             }
1175
1176             if (r == null) return;
1177             indicator.setFraction(((double)count++)/size);
1178             indicator.setText2(r.getSubj().getPresentableUrl());
1179             r.run();
1180           }
1181         }
1182       };
1183
1184       if (doProgressThing) {
1185         task.queue();
1186       }
1187       else {
1188         ProgressIndicator indicator = ProgressManager.getInstance().getProgressIndicator();
1189         task.run(indicator != null ? indicator : new EmptyProgressIndicator());
1190       }
1191     }
1192
1193     private void iterateIndexableFiles(final VirtualFile file, final Processor<VirtualFile> processor) {
1194       if (file.isDirectory()) {
1195         final ContentIterator iterator = new ContentIterator() {
1196           public boolean processFile(final VirtualFile fileOrDir) {
1197             if (!fileOrDir.isDirectory()) {
1198               processor.process(fileOrDir);
1199             }
1200             return true;
1201           }
1202         };
1203
1204         for (IndexableFileSet set : myIndexableSets) {
1205           set.iterateIndexableFilesIn(file, iterator);
1206         }
1207       }
1208       else {
1209         for (IndexableFileSet set : myIndexableSets) {
1210           if (set.isInSet(file)) {
1211             processor.process(file);
1212             break;
1213           }
1214         }
1215       }
1216     }
1217
1218     public VirtualFile[] queryNeededFiles() {
1219       synchronized (myFilesToUpdate) {
1220         return myFilesToUpdate.toArray(new VirtualFile[myFilesToUpdate.size()]);
1221       }
1222     }
1223
1224     public void processFile(final com.intellij.ide.startup.FileContent fileContent) {
1225       ensureAllInvalidateTasksCompleted();
1226       processFileImpl(fileContent);
1227     }
1228
1229     private final Semaphore myForceUpdateSemaphore = new Semaphore();
1230
1231     public void forceUpdate() {
1232       ensureAllInvalidateTasksCompleted();
1233
1234       final VirtualFile[] files = queryNeededFiles();
1235       if (files.length > 0) {
1236         myForceUpdateSemaphore.down();
1237         try {
1238           for (VirtualFile file: files) {
1239             processFileImpl(new com.intellij.ide.startup.FileContent(file));
1240           }
1241         }
1242         finally {
1243           myForceUpdateSemaphore.up();
1244           myForceUpdateSemaphore.waitFor(); // possibly wait until another thread completes indexing
1245         }
1246       }
1247     }
1248
1249     public void updatingDone() {
1250     }
1251
1252     public void canceled() {
1253     }
1254
1255     private void processFileImpl(final com.intellij.ide.startup.FileContent fileContent) {
1256       final VirtualFile file = fileContent.getVirtualFile();
1257       final boolean reallyRemoved = myFilesToUpdate.remove(file);
1258       if (reallyRemoved && file.isValid()) {
1259         indexFileContent(fileContent);
1260       }
1261     }
1262   }
1263
1264   private class UnindexedFilesFinder implements CollectingContentIterator {
1265     private final List<VirtualFile> myFiles = new ArrayList<VirtualFile>();
1266     private final Collection<ID<?, ?>> myIndexIds;
1267     private final Collection<ID<?, ?>> mySkipContentLoading;
1268     private final ProgressIndicator myProgressIndicator;
1269
1270     private UnindexedFilesFinder(final Collection<ID<?, ?>> indexIds) {
1271       myIndexIds = new ArrayList<ID<?, ?>>();
1272       mySkipContentLoading = new ArrayList<ID<?, ?>>();
1273       for (ID<?, ?> indexId : indexIds) {
1274         if (myNeedContentLoading.contains(indexId))  {
1275           myIndexIds.add(indexId);
1276         }
1277         else {
1278           mySkipContentLoading.add(indexId);
1279         }
1280       }
1281       myProgressIndicator = ProgressManager.getInstance().getProgressIndicator();
1282     }
1283
1284     public List<VirtualFile> getFiles() {
1285       return myFiles;
1286     }
1287
1288     public boolean processFile(final VirtualFile file) {
1289       if (!file.isDirectory()) {
1290         if (file instanceof NewVirtualFile && ((NewVirtualFile)file).getFlag(ALREADY_PROCESSED)) {
1291           return true;
1292         }
1293
1294         if (file instanceof VirtualFileWithId) {
1295           boolean oldStuff = true;
1296           if (!SingleRootFileViewProvider.isTooLarge(file)) {
1297             for (ID<?, ?> indexId : myIndexIds) {
1298               try {
1299                 if (myFileContentAttic.containsContent(file) ? getInputFilter(indexId).acceptInput(file) : shouldIndexFile(file, indexId)) {
1300                   myFiles.add(file);
1301                   oldStuff = false;
1302                   break;
1303                 }
1304               }
1305               catch (RuntimeException e) {
1306                 final Throwable cause = e.getCause();
1307                 if (cause instanceof IOException || cause instanceof StorageException) {
1308                   LOG.info(e);
1309                   requestRebuild(indexId);
1310                 }
1311                 else {
1312                   throw e;
1313                 }
1314               }
1315             }
1316           }
1317           FileContent fileContent = null;
1318           for (ID<?, ?> indexId : mySkipContentLoading) {
1319             if (shouldIndexFile(file, indexId)) {
1320               oldStuff = false;
1321               try {
1322                 if (fileContent == null) {
1323                   fileContent = new FileContent(file);
1324                 }
1325                 updateSingleIndex(indexId, file, fileContent, null);
1326               }
1327               catch (StorageException e) {
1328                 LOG.info(e);
1329                 requestRebuild(indexId);
1330               }
1331             }
1332           }
1333           IndexingStamp.flushCache();
1334
1335           if (oldStuff && file instanceof NewVirtualFile) {
1336             ((NewVirtualFile)file).setFlag(ALREADY_PROCESSED, true);
1337           }
1338         }
1339       }
1340       else {
1341         if (myProgressIndicator != null) {
1342           myProgressIndicator.setText("Scanning files to index");
1343           myProgressIndicator.setText2(file.getPresentableUrl());
1344         }
1345       }
1346       return true;
1347     }
1348   }
1349
1350   private boolean shouldUpdateIndex(final VirtualFile file, final ID<?, ?> indexId) {
1351     return getInputFilter(indexId).acceptInput(file) &&
1352            (isMock(file) || IndexingStamp.isFileIndexed(file, indexId, IndexInfrastructure.getIndexCreationStamp(indexId)));
1353   }
1354
1355   private boolean shouldIndexFile(final VirtualFile file, final ID<?, ?> indexId) {
1356     return getInputFilter(indexId).acceptInput(file) &&
1357            (isMock(file) || !IndexingStamp.isFileIndexed(file, indexId, IndexInfrastructure.getIndexCreationStamp(indexId)));
1358   }
1359
1360   private static boolean isMock(final VirtualFile file) {
1361     return !(file instanceof NewVirtualFile);
1362   }
1363
1364   public CollectingContentIterator createContentIterator() {
1365     return new UnindexedFilesFinder(myIndices.keySet());
1366   }
1367
1368   public void registerIndexableSet(IndexableFileSet set) {
1369     myIndexableSets.add(set);
1370   }
1371
1372   public void removeIndexableSet(IndexableFileSet set) {
1373     myChangedFilesUpdater.forceUpdate();
1374     myIndexableSets.remove(set);
1375   }
1376
1377   @Nullable
1378   private static PsiFile findLatestKnownPsiForUncomittedDocument(Document doc) {
1379     PsiFile target = null;
1380     long modStamp = -1L;
1381
1382     for (Project project : ProjectManager.getInstance().getOpenProjects()) {
1383       final PsiDocumentManager pdm = PsiDocumentManager.getInstance(project);
1384       final PsiFile file = pdm.getCachedPsiFile(doc);
1385       if (file != null && file.getModificationStamp() > modStamp) {
1386         final ProjectFileIndex index = ProjectRootManager.getInstance(project).getFileIndex();
1387         final VirtualFile vFile = file.getVirtualFile();
1388         if (vFile != null && (index.isInContent(vFile) || index.isInLibrarySource(vFile))) {
1389           target = file;
1390           modStamp = file.getModificationStamp();
1391         }
1392       }
1393       else if (file != null && file.getModificationStamp() == modStamp) {
1394         if (target instanceof PsiPlainTextFile && !(file instanceof PsiPlainTextFile)) {
1395           target = file;
1396         }
1397       }
1398     }
1399
1400     return target;
1401   }
1402
1403   private static class IndexableFilesFilter implements InputFilter {
1404     private final InputFilter myDelegate;
1405
1406     private IndexableFilesFilter(InputFilter delegate) {
1407       myDelegate = delegate;
1408     }
1409
1410     public boolean acceptInput(final VirtualFile file) {
1411       return file instanceof VirtualFileWithId && myDelegate.acceptInput(file);
1412     }
1413   }
1414
1415   private static void cleanupProcessedFlag() {
1416     final VirtualFile[] roots = ManagingFS.getInstance().getRoots();
1417     for (VirtualFile root : roots) {
1418       cleanProcessedFlag(root);
1419     }
1420   }
1421
1422   private static void cleanProcessedFlag(final VirtualFile file) {
1423     if (!(file instanceof NewVirtualFile)) return;
1424
1425     final NewVirtualFile nvf = (NewVirtualFile)file;
1426     if (file.isDirectory()) {
1427       for (VirtualFile child : nvf.getCachedChildren()) {
1428         cleanProcessedFlag(child);
1429       }
1430     }
1431     else {
1432 /*      nvf.clearCachedFileType(); */
1433       nvf.setFlag(ALREADY_PROCESSED, false);
1434     }
1435   }
1436
1437   private static class StorageGuard {
1438     private int myHolds = 0;
1439
1440     public interface Holder {
1441       void leave();
1442     }
1443
1444     private final Holder myTrueHolder = new Holder() {
1445       public void leave() {
1446         StorageGuard.this.leave(true);
1447       }
1448     };
1449     private final Holder myFalseHolder = new Holder() {
1450       public void leave() {
1451         StorageGuard.this.leave(false);
1452       }
1453     };
1454
1455     public synchronized Holder enter(boolean mode) {
1456       if (mode) {
1457         while (myHolds < 0) {
1458           try {
1459             wait();
1460           }
1461           catch (InterruptedException ignored) {
1462           }
1463         }
1464         myHolds++;
1465         return myTrueHolder;
1466       }
1467       else {
1468         while (myHolds > 0) {
1469           try {
1470             wait();
1471           }
1472           catch (InterruptedException ignored) {
1473           }
1474         }
1475         myHolds--;
1476         return myFalseHolder;
1477       }
1478     }
1479
1480     private synchronized void leave(boolean mode) {
1481       myHolds += (mode? -1 : 1);
1482       if (myHolds == 0) {
1483         notifyAll();
1484       }
1485     }
1486
1487   }
1488 }