platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndex.java

   1 package com.intellij.util.indexing;
   2
   3 import com.intellij.AppTopics;
   4 import com.intellij.ide.startup.CacheUpdater;
   5 import com.intellij.ide.startup.impl.FileSystemSynchronizerImpl;
   6 import com.intellij.lang.ASTNode;
   7 import com.intellij.openapi.application.*;
   8 import com.intellij.openapi.components.ApplicationComponent;
   9 import com.intellij.openapi.diagnostic.Logger;
  10 import com.intellij.openapi.editor.Document;
  11 import com.intellij.openapi.editor.highlighter.EditorHighlighter;
  12 import com.intellij.openapi.editor.impl.DocumentImpl;
  13 import com.intellij.openapi.extensions.Extensions;
  14 import com.intellij.openapi.fileEditor.FileDocumentManager;
  15 import com.intellij.openapi.fileTypes.*;
  16 import com.intellij.openapi.progress.ProcessCanceledException;
  17 import com.intellij.openapi.progress.ProgressIndicator;
  18 import com.intellij.openapi.progress.ProgressManager;
  19 import com.intellij.openapi.progress.Task;
  20 import com.intellij.openapi.progress.impl.BackgroundableProcessIndicator;
  21 import com.intellij.openapi.project.*;
  22 import com.intellij.openapi.roots.CollectingContentIterator;
  23 import com.intellij.openapi.roots.ContentIterator;
  24 import com.intellij.openapi.roots.ProjectRootManager;
  25 import com.intellij.openapi.util.Key;
  26 import com.intellij.openapi.util.Pair;
  27 import com.intellij.openapi.util.ShutDownTracker;
  28 import com.intellij.openapi.util.io.FileUtil;
  29 import com.intellij.openapi.vfs.*;
  30 import com.intellij.openapi.vfs.ex.VirtualFileManagerEx;
  31 import com.intellij.openapi.vfs.newvfs.ManagingFS;
  32 import com.intellij.openapi.vfs.newvfs.NewVirtualFile;
  33 import com.intellij.openapi.vfs.newvfs.persistent.PersistentFS;
  34 import com.intellij.psi.PsiDocumentManager;
  35 import com.intellij.psi.PsiFile;
  36 import com.intellij.psi.PsiLock;
  37 import com.intellij.psi.SingleRootFileViewProvider;
  38 import com.intellij.psi.impl.PsiDocumentTransactionListener;
  39 import com.intellij.psi.impl.source.PsiFileImpl;
  40 import com.intellij.psi.search.GlobalSearchScope;
  41 import com.intellij.util.ArrayUtil;
  42 import com.intellij.util.CommonProcessors;
  43 import com.intellij.util.Processor;
  44 import com.intellij.util.concurrency.JBLock;
  45 import com.intellij.util.concurrency.JBReentrantReadWriteLock;
  46 import com.intellij.util.concurrency.LockFactory;
  47 import com.intellij.util.concurrency.Semaphore;
  48 import com.intellij.util.containers.ConcurrentHashSet;
  49 import com.intellij.util.containers.ContainerUtil;
  50 import com.intellij.util.io.*;
  51 import com.intellij.util.messages.MessageBus;
  52 import com.intellij.util.messages.MessageBusConnection;
  53 import gnu.trove.TIntHashSet;
  54 import gnu.trove.TIntIterator;
  55 import gnu.trove.TObjectIntHashMap;
  56 import org.jetbrains.annotations.NonNls;
  57 import org.jetbrains.annotations.NotNull;
  58 import org.jetbrains.annotations.Nullable;
  59 import org.jetbrains.annotations.TestOnly;
  60
  61 import javax.swing.*;
  62 import java.io.*;
  63 import java.util.*;
  64 import java.util.concurrent.atomic.AtomicBoolean;
  65 import java.util.concurrent.atomic.AtomicInteger;
  66 import java.util.concurrent.locks.Lock;
  67
  68 /**
  69  * @author Eugene Zhuravlev
  70  *         Date: Dec 20, 2007
  71  */
  72
  73 public class FileBasedIndex implements ApplicationComponent {
  74   private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.FileBasedIndex");
  75   @NonNls
  76   private static final String CORRUPTION_MARKER_NAME = "corruption.marker";
  77   private final Map<ID<?, ?>, Pair<UpdatableIndex<?, ?, FileContent>, InputFilter>> myIndices = new HashMap<ID<?, ?>, Pair<UpdatableIndex<?, ?, FileContent>, InputFilter>>();
  78   private final Map<ID<?, ?>, Semaphore> myUnsavedDataIndexingSemaphores = new HashMap<ID<?,?>, Semaphore>();
  79   private final TObjectIntHashMap<ID<?, ?>> myIndexIdToVersionMap = new TObjectIntHashMap<ID<?, ?>>();
  80   private final Set<ID<?, ?>> myNotRequiringContentIndices = new HashSet<ID<?, ?>>();
  81   private final Set<FileType> myNoLimitCheckTypes = new HashSet<FileType>();
  82
  83   private final PerIndexDocumentMap<Long> myLastIndexedDocStamps = new PerIndexDocumentMap<Long>() {
  84     @Override
  85     protected Long createDefault(Document document) {
  86       return 0L;
  87     }
  88   };
  89
  90   private final ChangedFilesUpdater myChangedFilesUpdater;
  91
  92   private final List<IndexableFileSet> myIndexableSets = ContainerUtil.createEmptyCOWList();
  93
  94   public static final int OK = 1;
  95   public static final int REQUIRES_REBUILD = 2;
  96   public static final int REBUILD_IN_PROGRESS = 3;
  97   private final Map<ID<?, ?>, AtomicInteger> myRebuildStatus = new HashMap<ID<?,?>, AtomicInteger>();
  98
  99   private final VirtualFileManagerEx myVfManager;
 100   private final FileDocumentManager myFileDocumentManager;
 101   private final ConcurrentHashSet<ID<?, ?>> myUpToDateIndices = new ConcurrentHashSet<ID<?, ?>>();
 102   private final Map<Document, PsiFile> myTransactionMap = new HashMap<Document, PsiFile>();
 103
 104   private static final int ALREADY_PROCESSED = 0x02;
 105
 106   public void requestReindex(final VirtualFile file) {
 107     myChangedFilesUpdater.invalidateIndices(file, true);
 108   }
 109
 110   public interface InputFilter {
 111     boolean acceptInput(VirtualFile file);
 112   }
 113
 114   public FileBasedIndex(final VirtualFileManagerEx vfManager, FileDocumentManager fdm, MessageBus bus) throws IOException {
 115     myVfManager = vfManager;
 116     myFileDocumentManager = fdm;
 117
 118     final MessageBusConnection connection = bus.connect();
 119     connection.subscribe(PsiDocumentTransactionListener.TOPIC, new PsiDocumentTransactionListener() {
 120       public void transactionStarted(final Document doc, final PsiFile file) {
 121         if (file != null) {
 122           myTransactionMap.put(doc, file);
 123           myUpToDateIndices.clear();
 124         }
 125       }
 126
 127       public void transactionCompleted(final Document doc, final PsiFile file) {
 128         myTransactionMap.remove(doc);
 129       }
 130     });
 131
 132     connection.subscribe(AppTopics.FILE_TYPES, new FileTypeListener() {
 133       private Map<FileType, Set<String>> myTypeToExtensionMap;
 134       public void beforeFileTypesChanged(final FileTypeEvent event) {
 135         cleanupProcessedFlag();
 136         myTypeToExtensionMap = new HashMap<FileType, Set<String>>();
 137         final FileTypeManager manager = event.getManager();
 138         for (FileType type : manager.getRegisteredFileTypes()) {
 139           myTypeToExtensionMap.put(type, getExtensions(manager, type));
 140         }
 141       }
 142
 143       public void fileTypesChanged(final FileTypeEvent event) {
 144         final Map<FileType, Set<String>> oldExtensions = myTypeToExtensionMap;
 145         myTypeToExtensionMap = null;
 146         if (oldExtensions != null) {
 147           final FileTypeManager manager = event.getManager();
 148           final Map<FileType, Set<String>> newExtensions = new HashMap<FileType, Set<String>>();
 149           for (FileType type : manager.getRegisteredFileTypes()) {
 150             newExtensions.put(type, getExtensions(manager, type));
 151           }
 152           // we are interested only in extension changes or removals.
 153           // addition of an extension is handled separately by RootsChanged event
 154           if (!newExtensions.keySet().containsAll(oldExtensions.keySet())) {
 155             rebuildAllndices();
 156             return;
 157           }
 158           for (FileType type : oldExtensions.keySet()) {
 159             if (!newExtensions.get(type).containsAll(oldExtensions.get(type))) {
 160               rebuildAllndices();
 161               return;
 162             }
 163           }
 164         }
 165       }
 166
 167       private Set<String> getExtensions(FileTypeManager manager, FileType type) {
 168         final Set<String> set = new HashSet<String>();
 169         for (FileNameMatcher matcher : manager.getAssociations(type)) {
 170           set.add(matcher.getPresentableString());
 171         }
 172         return set;
 173       }
 174
 175       private void rebuildAllndices() {
 176         for (ID<?, ?> indexId : myIndices.keySet()) {
 177           requestRebuild(indexId);
 178         }
 179       }
 180     });
 181
 182     ApplicationManager.getApplication().addApplicationListener(new ApplicationAdapter() {
 183       public void writeActionStarted(Object action) {
 184         myUpToDateIndices.clear();
 185       }
 186     });
 187
 188     final File workInProgressFile = getMarkerFile();
 189     if (workInProgressFile.exists()) {
 190       // previous IDEA session was closed incorrectly, so drop all indices
 191       FileUtil.delete(PathManager.getIndexRoot());
 192     }
 193
 194     try {
 195       final FileBasedIndexExtension[] extensions = Extensions.getExtensions(FileBasedIndexExtension.EXTENSION_POINT_NAME);
 196       for (FileBasedIndexExtension<?, ?> extension : extensions) {
 197         myRebuildStatus.put(extension.getName(), new AtomicInteger(OK));
 198       }
 199
 200       final File corruptionMarker = new File(PathManager.getIndexRoot(), CORRUPTION_MARKER_NAME);
 201       final boolean currentVersionCorrupted = corruptionMarker.exists();
 202       for (FileBasedIndexExtension<?, ?> extension : extensions) {
 203         registerIndexer(extension, currentVersionCorrupted);
 204       }
 205       FileUtil.delete(corruptionMarker);
 206       dropUnregisteredIndices();
 207
 208       // check if rebuild was requested for any index during registration
 209       for (ID<?, ?> indexId : myIndices.keySet()) {
 210         if (myRebuildStatus.get(indexId).compareAndSet(REQUIRES_REBUILD, OK)) {
 211           try {
 212             clearIndex(indexId);
 213           }
 214           catch (StorageException e) {
 215             requestRebuild(indexId);
 216             LOG.error(e);
 217           }
 218         }
 219       }
 220
 221       myChangedFilesUpdater = new ChangedFilesUpdater();
 222       vfManager.addVirtualFileListener(myChangedFilesUpdater);
 223
 224       vfManager.registerRefreshUpdater(myChangedFilesUpdater);
 225
 226       registerIndexableSet(new AdditionalIndexableFileSet());
 227     }
 228     finally {
 229       ShutDownTracker.getInstance().registerShutdownTask(new Runnable() {
 230         public void run() {
 231           performShutdown();
 232         }
 233       });
 234       FileUtil.createIfDoesntExist(workInProgressFile);
 235       saveRegisteredIndices(myIndices.keySet());
 236     }
 237   }
 238
 239   private static FileBasedIndex ourInstance = CachedSingletonsRegistry.markCachedField(FileBasedIndex.class);
 240   public static FileBasedIndex getInstance() {
 241     if (ourInstance == null) {
 242       ourInstance = ApplicationManager.getApplication().getComponent(FileBasedIndex.class);
 243     }
 244
 245     return ourInstance;
 246   }
 247
 248   /**
 249    * @return true if registered index requires full rebuild for some reason, e.g. is just created or corrupted @param extension
 250    * @param isCurrentVersionCorrupted
 251    */
 252   private <K, V> void registerIndexer(final FileBasedIndexExtension<K, V> extension, final boolean isCurrentVersionCorrupted) throws IOException {
 253     final ID<K, V> name = extension.getName();
 254     final int version = extension.getVersion();
 255     if (!extension.dependsOnFileContent()) {
 256       myNotRequiringContentIndices.add(name);
 257     }
 258     myIndexIdToVersionMap.put(name, version);
 259     final File versionFile = IndexInfrastructure.getVersionFile(name);
 260     if (isCurrentVersionCorrupted || IndexInfrastructure.versionDiffers(versionFile, version)) {
 261       if (!isCurrentVersionCorrupted) {
 262         LOG.info("Version has changed for index " + extension.getName() + ". The index will be rebuilt.");
 263       }
 264       FileUtil.delete(IndexInfrastructure.getIndexRootDir(name));
 265       IndexInfrastructure.rewriteVersion(versionFile, version);
 266     }
 267
 268     for (int attempt = 0; attempt < 2; attempt++) {
 269       try {
 270         final MapIndexStorage<K, V> storage = new MapIndexStorage<K, V>(IndexInfrastructure.getStorageFile(name), extension.getKeyDescriptor(), extension.getValueExternalizer(), extension.getCacheSize());
 271         final IndexStorage<K, V> memStorage = new MemoryIndexStorage<K, V>(storage);
 272         final UpdatableIndex<K, V, FileContent> index = createIndex(name, extension, memStorage);
 273         myIndices.put(name, new Pair<UpdatableIndex<?,?, FileContent>, InputFilter>(index, new IndexableFilesFilter(extension.getInputFilter())));
 274         myUnsavedDataIndexingSemaphores.put(name, new Semaphore());
 275         myNoLimitCheckTypes.addAll(extension.getFileTypesWithSizeLimitNotApplicable());
 276         break;
 277       }
 278       catch (IOException e) {
 279         LOG.info(e);
 280         FileUtil.delete(IndexInfrastructure.getIndexRootDir(name));
 281         IndexInfrastructure.rewriteVersion(versionFile, version);
 282       }
 283     }
 284   }
 285
 286   private static void saveRegisteredIndices(Collection<ID<?, ?>> ids) {
 287     final File file = getRegisteredIndicesFile();
 288     try {
 289       FileUtil.createIfDoesntExist(file);
 290       final DataOutputStream os = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(file)));
 291       try {
 292         os.writeInt(ids.size());
 293         for (ID<?, ?> id : ids) {
 294           IOUtil.writeString(id.toString(), os);
 295         }
 296       }
 297       finally {
 298         os.close();
 299       }
 300     }
 301     catch (IOException ignored) {
 302     }
 303   }
 304
 305   private static Set<String> readRegistsredIndexNames() {
 306     final Set<String> result = new HashSet<String>();
 307     try {
 308       final DataInputStream in = new DataInputStream(new BufferedInputStream(new FileInputStream(getRegisteredIndicesFile())));
 309       try {
 310         final int size = in.readInt();
 311         for (int idx = 0; idx < size; idx++) {
 312           result.add(IOUtil.readString(in));
 313         }
 314       }
 315       finally {
 316         in.close();
 317       }
 318     }
 319     catch (IOException ignored) {
 320     }
 321     return result;
 322   }
 323
 324   private static File getRegisteredIndicesFile() {
 325     return new File(PathManager.getIndexRoot(), "registered");
 326   }
 327
 328   private static File getMarkerFile() {
 329     return new File(PathManager.getIndexRoot(), "work_in_progress");
 330   }
 331
 332   private <K, V> UpdatableIndex<K, V, FileContent> createIndex(final ID<K, V> indexId, final FileBasedIndexExtension<K, V> extension, final IndexStorage<K, V> storage) throws IOException {
 333     final MapReduceIndex<K, V, FileContent> index;
 334     if (extension instanceof CustomImplementationFileBasedIndexExtension) {
 335       final UpdatableIndex<K, V, FileContent> custom =
 336         ((CustomImplementationFileBasedIndexExtension<K, V, FileContent>)extension).createIndexImplementation(indexId, this, storage);
 337       if (!(custom instanceof MapReduceIndex)) {
 338         return custom;
 339       }
 340       index = (MapReduceIndex<K,V, FileContent>)custom;
 341     }
 342     else {
 343       index = new MapReduceIndex<K, V, FileContent>(indexId, extension.getIndexer(), storage);
 344     }
 345
 346     final KeyDescriptor<K> keyDescriptor = extension.getKeyDescriptor();
 347     index.setInputIdToDataKeysIndex(new PersistentHashMap<Integer, Collection<K>>(IndexInfrastructure.getInputIndexStorageFile(indexId), new EnumeratorIntegerDescriptor(), new DataExternalizer<Collection<K>>() {
 348       public void save(DataOutput out, Collection<K> value) throws IOException {
 349         DataInputOutputUtil.writeINT(out, value.size());
 350         for (K key : value) {
 351           keyDescriptor.save(out, key);
 352         }
 353       }
 354
 355       public Collection<K> read(DataInput in) throws IOException {
 356         final int size = DataInputOutputUtil.readINT(in);
 357         final List<K> list = new ArrayList<K>();
 358         for (int idx = 0; idx < size; idx++) {
 359           list.add(keyDescriptor.read(in));
 360         }
 361         return list;
 362       }
 363     }));
 364
 365     return index;
 366   }
 367
 368   @NonNls
 369   @NotNull
 370   public String getComponentName() {
 371     return "FileBasedIndex";
 372   }
 373
 374   public void initComponent() {
 375   }
 376
 377   public void disposeComponent() {
 378     performShutdown();
 379   }
 380
 381   private AtomicBoolean myShutdownPerformed = new AtomicBoolean(false);
 382
 383   private void performShutdown() {
 384     if (!myShutdownPerformed.compareAndSet(false, true)) {
 385       return; // already shut down
 386     }
 387     myFileDocumentManager.saveAllDocuments();
 388
 389     LOG.info("START INDEX SHUTDOWN");
 390     try {
 391       myChangedFilesUpdater.forceUpdate();
 392
 393       for (ID<?, ?> indexId : myIndices.keySet()) {
 394         final UpdatableIndex<?, ?, FileContent> index = getIndex(indexId);
 395         assert index != null;
 396         checkRebuild(indexId, true); // if the index was scheduled for rebuild, only clean it
 397         //LOG.info("DISPOSING " + indexId);
 398         index.dispose();
 399       }
 400
 401       myVfManager.removeVirtualFileListener(myChangedFilesUpdater);
 402       myVfManager.unregisterRefreshUpdater(myChangedFilesUpdater);
 403
 404       FileUtil.delete(getMarkerFile());
 405     }
 406     catch (Throwable e) {
 407       LOG.info("Problems during index shutdown", e);
 408       throw new RuntimeException(e);
 409     }
 410     LOG.info("END INDEX SHUTDOWN");
 411   }
 412
 413   public void flushCaches() {
 414     IndexingStamp.flushCache();
 415     ApplicationManager.getApplication().executeOnPooledThread(new Runnable() {
 416       public void run() {
 417         for (ID<?, ?> indexId : myIndices.keySet()) {
 418           //noinspection ConstantConditions
 419           try {
 420             getIndex(indexId).flush();
 421           }
 422           catch (StorageException e) {
 423             LOG.info(e);
 424             requestRebuild(indexId);
 425           }
 426         }
 427       }
 428     });
 429   }
 430
 431   @NotNull
 432   public <K> Collection<K> getAllKeys(final ID<K, ?> indexId, @NotNull Project project) {
 433     Set<K> allKeys = new HashSet<K>();
 434     processAllKeys(indexId, new CommonProcessors.CollectProcessor<K>(allKeys), project);
 435     return allKeys;
 436   }
 437
 438   public <K> boolean processAllKeys(final ID<K, ?> indexId, Processor<K> processor, @NotNull Project project) {
 439     try {
 440       ensureUpToDate(indexId, project);
 441       final UpdatableIndex<K, ?, FileContent> index = getIndex(indexId);
 442       if (index == null) return true;
 443       return index.processAllKeys(processor);
 444     }
 445     catch (StorageException e) {
 446       scheduleRebuild(indexId, e);
 447     }
 448     catch (RuntimeException e) {
 449       final Throwable cause = e.getCause();
 450       if (cause instanceof StorageException || cause instanceof IOException) {
 451         scheduleRebuild(indexId, cause);
 452       }
 453       else {
 454         throw e;
 455       }
 456     }
 457
 458     return false;
 459   }
 460
 461   private static final ThreadLocal<Integer> myUpToDateCheckState = new ThreadLocal<Integer>();
 462
 463   public void disableUpToDateCheckForCurrentThread() {
 464     final Integer currentValue = myUpToDateCheckState.get();
 465     myUpToDateCheckState.set(currentValue == null? 1 : currentValue.intValue() + 1);
 466   }
 467
 468   public void enableUpToDateCheckForCurrentThread() {
 469     final Integer currentValue = myUpToDateCheckState.get();
 470     if (currentValue != null) {
 471       final int newValue = currentValue.intValue() - 1;
 472       if (newValue != 0) {
 473         myUpToDateCheckState.set(newValue);
 474       }
 475       else {
 476         myUpToDateCheckState.remove();
 477       }
 478     }
 479   }
 480
 481   private boolean isUpToDateCheckEnabled() {
 482     final Integer value = myUpToDateCheckState.get();
 483     return value == null || value.intValue() == 0;
 484   }
 485
 486
 487   private final ThreadLocal<Boolean> myReentrancyGuard = new ThreadLocal<Boolean>() {
 488     protected Boolean initialValue() {
 489       return Boolean.FALSE;
 490     }
 491   };
 492
 493   /**
 494    * DO NOT CALL DIRECTLY IN CLIENT CODE
 495    * The method is internal to indexing engine end is called internally. The method is public due to implementation details
 496    */
 497   public <K> void ensureUpToDate(final ID<K, ?> indexId, @NotNull Project project) {
 498     if (isDumb(project)) {
 499       handleDumbMode(indexId, project);
 500     }
 501
 502     if (myReentrancyGuard.get().booleanValue()) {
 503       //assert false : "ensureUpToDate() is not reentrant!";
 504       return;
 505     }
 506     myReentrancyGuard.set(Boolean.TRUE);
 507
 508     try {
 509       if (isUpToDateCheckEnabled()) {
 510         try {
 511           checkRebuild(indexId, false);
 512           indexUnsavedDocuments(indexId, project);
 513         }
 514         catch (StorageException e) {
 515           scheduleRebuild(indexId, e);
 516         }
 517         catch (RuntimeException e) {
 518           final Throwable cause = e.getCause();
 519           if (cause instanceof StorageException || cause instanceof IOException) {
 520             scheduleRebuild(indexId, e);
 521           }
 522           else {
 523             throw e;
 524           }
 525         }
 526       }
 527     }
 528     finally {
 529       myReentrancyGuard.set(Boolean.FALSE);
 530     }
 531   }
 532
 533   private void handleDumbMode(final ID<?, ?> indexId, Project project) {
 534     if (myNotRequiringContentIndices.contains(indexId)) {
 535       return; //indexed eagerly in foreground while building unindexed file list
 536     }
 537
 538     final ProgressManager progressManager = ProgressManager.getInstance();
 539     progressManager.checkCanceled(); // DumbModeAction.CANCEL
 540
 541     final ProgressIndicator progressIndicator = progressManager.getProgressIndicator();
 542     if (progressIndicator instanceof BackgroundableProcessIndicator) {
 543       final BackgroundableProcessIndicator indicator = (BackgroundableProcessIndicator)progressIndicator;
 544       if (indicator.getDumbModeAction() == DumbModeAction.WAIT) {
 545         assert !ApplicationManager.getApplication().isDispatchThread();
 546         DumbService.getInstance(project).waitForSmartMode();
 547         return;
 548       }
 549     }
 550
 551     throw new IndexNotReadyException();
 552   }
 553
 554   private static boolean isDumb(Project project) {
 555     return DumbServiceImpl.getInstance(project).isDumb();
 556   }
 557
 558   @NotNull
 559   public <K, V> List<V> getValues(final ID<K, V> indexId, @NotNull K dataKey, @NotNull final GlobalSearchScope filter) {
 560     final List<V> values = new ArrayList<V>();
 561     processValuesImpl(indexId, dataKey, true, null, new ValueProcessor<V>() {
 562       public boolean process(final VirtualFile file, final V value) {
 563         values.add(value);
 564         return true;
 565       }
 566     }, filter);
 567     return values;
 568   }
 569
 570   @NotNull
 571   public <K, V> Collection<VirtualFile> getContainingFiles(final ID<K, V> indexId, @NotNull K dataKey, @NotNull final GlobalSearchScope filter) {
 572     final Set<VirtualFile> files = new HashSet<VirtualFile>();
 573     processValuesImpl(indexId, dataKey, false, null, new ValueProcessor<V>() {
 574       public boolean process(final VirtualFile file, final V value) {
 575         files.add(file);
 576         return true;
 577       }
 578     }, filter);
 579     return files;
 580   }
 581
 582
 583   public interface ValueProcessor<V> {
 584     /**
 585      * @param value a value to process
 586      * @param file the file the value came from
 587      * @return false if no further processing is needed, true otherwise
 588      */
 589     boolean process(VirtualFile file, V value);
 590   }
 591
 592   /**
 593    * @return false if ValueProcessor.process() returned false; true otherwise or if ValueProcessor was not called at all
 594    */
 595   public <K, V> boolean processValues(final ID<K, V> indexId, final @NotNull K dataKey, @Nullable final VirtualFile inFile,
 596                                    ValueProcessor<V> processor, @NotNull final GlobalSearchScope filter) {
 597     return processValuesImpl(indexId, dataKey, false, inFile, processor, filter);
 598   }
 599
 600
 601
 602   private <K, V> boolean processValuesImpl(final ID<K, V> indexId, final K dataKey, boolean ensureValueProcessedOnce,
 603                                         @Nullable final VirtualFile restrictToFile, ValueProcessor<V> processor,
 604                                         final GlobalSearchScope filter) {
 605     try {
 606       final Project project = filter.getProject();
 607       assert project != null : "GlobalSearchScope#getProject() should be not-null for all index queries";
 608       ensureUpToDate(indexId, project);
 609       final UpdatableIndex<K, V, FileContent> index = getIndex(indexId);
 610       if (index == null) {
 611         return true;
 612       }
 613
 614       final Lock readLock = index.getReadLock();
 615       try {
 616         readLock.lock();
 617         final ValueContainer<V> container = index.getData(dataKey);
 618
 619         boolean shouldContinue = true;
 620
 621         if (restrictToFile != null) {
 622           if (restrictToFile instanceof VirtualFileWithId) {
 623             final int restrictedFileId = getFileId(restrictToFile);
 624             for (final Iterator<V> valueIt = container.getValueIterator(); valueIt.hasNext();) {
 625               final V value = valueIt.next();
 626               if (container.isAssociated(value, restrictedFileId)) {
 627                 shouldContinue = processor.process(restrictToFile, value);
 628                 if (!shouldContinue) {
 629                   break;
 630                 }
 631               }
 632             }
 633           }
 634         }
 635         else {
 636           final PersistentFS fs = (PersistentFS)ManagingFS.getInstance();
 637           VALUES_LOOP: for (final Iterator<V> valueIt = container.getValueIterator(); valueIt.hasNext();) {
 638             final V value = valueIt.next();
 639             for (final ValueContainer.IntIterator inputIdsIterator = container.getInputIdsIterator(value); inputIdsIterator.hasNext();) {
 640               final int id = inputIdsIterator.next();
 641               VirtualFile file = IndexInfrastructure.findFileById(fs, id);
 642               if (file != null && filter.accept(file)) {
 643                 shouldContinue = processor.process(file, value);
 644                 if (!shouldContinue) {
 645                   break VALUES_LOOP;
 646                 }
 647                 if (ensureValueProcessedOnce) {
 648                   break; // continue with the next value
 649                 }
 650               }
 651             }
 652           }
 653         }
 654         return shouldContinue;
 655       }
 656       finally {
 657         index.getReadLock().unlock();
 658       }
 659     }
 660     catch (StorageException e) {
 661       scheduleRebuild(indexId, e);
 662     }
 663     catch (RuntimeException e) {
 664       final Throwable cause = e.getCause();
 665       if (cause instanceof StorageException || cause instanceof IOException) {
 666         scheduleRebuild(indexId, cause);
 667       }
 668       else {
 669         throw e;
 670       }
 671     }
 672     return true;
 673   }
 674
 675   public <K, V> boolean getFilesWithKey(final ID<K, V> indexId, final Set<K> dataKeys,
 676                                          Processor<VirtualFile> processor,
 677                                          GlobalSearchScope filter) {
 678     try {
 679       final Project project = filter.getProject();
 680       assert project != null : "GlobalSearchScope#getProject() should be not-null for all index queries";
 681       ensureUpToDate(indexId, project);
 682       final UpdatableIndex<K, V, FileContent> index = getIndex(indexId);
 683       if (index == null) {
 684         return true;
 685       }
 686
 687       final Lock readLock = index.getReadLock();
 688       try {
 689         readLock.lock();
 690         List<TIntHashSet> locals = new ArrayList<TIntHashSet>();
 691         for (K dataKey : dataKeys) {
 692           TIntHashSet local = new TIntHashSet();
 693           locals.add(local);
 694           final ValueContainer<V> container = index.getData(dataKey);
 695
 696           for (final Iterator<V> valueIt = container.getValueIterator(); valueIt.hasNext();) {
 697             final V value = valueIt.next();
 698             for (final ValueContainer.IntIterator inputIdsIterator = container.getInputIdsIterator(value); inputIdsIterator.hasNext();) {
 699               final int id = inputIdsIterator.next();
 700               local.add(id);
 701             }
 702           }
 703         }
 704
 705         if (locals.size() == 0) return true;
 706
 707         Collections.sort(locals, new Comparator<TIntHashSet>() {
 708           public int compare(TIntHashSet o1, TIntHashSet o2) {
 709             return o1.size() - o2.size();
 710           }
 711         });
 712
 713         final PersistentFS fs = (PersistentFS)ManagingFS.getInstance();
 714         TIntIterator ids = join(locals).iterator();
 715         while (ids.hasNext()) {
 716           int id = ids.next();
 717           VirtualFile file = IndexInfrastructure.findFileById(fs, id);
 718           if (file != null && filter.accept(file)) {
 719             if (!processor.process(file)) return false;
 720           }
 721         }
 722       }
 723       finally {
 724         index.getReadLock().unlock();
 725       }
 726     }
 727     catch (StorageException e) {
 728       scheduleRebuild(indexId, e);
 729     }
 730     catch (RuntimeException e) {
 731       final Throwable cause = e.getCause();
 732       if (cause instanceof StorageException || cause instanceof IOException) {
 733         scheduleRebuild(indexId, cause);
 734       }
 735       else {
 736         throw e;
 737       }
 738     }
 739     return true;
 740   }
 741
 742   private static TIntHashSet join(List<TIntHashSet> locals) {
 743     TIntHashSet result = locals.get(0);
 744     if (locals.size() > 1) {
 745       TIntIterator it = result.iterator();
 746
 747       while (it.hasNext()) {
 748         int id = it.next();
 749         for (int i = 1; i < locals.size(); i++) {
 750           if (!locals.get(i).contains(id)) {
 751             it.remove();
 752             break;
 753           }
 754         }
 755       }
 756     }
 757     return result;
 758   }
 759
 760   public interface AllValuesProcessor<V> {
 761     void process(final int inputId, V value);
 762   }
 763
 764   public <K, V> void processAllValues(final ID<K, V> indexId, AllValuesProcessor<V> processor, @NotNull Project project) {
 765     try {
 766       ensureUpToDate(indexId, project);
 767       final UpdatableIndex<K, V, FileContent> index = getIndex(indexId);
 768       if (index == null) {
 769         return;
 770       }
 771       try {
 772         index.getReadLock().lock();
 773         for (K dataKey : index.getAllKeys()) {
 774           final ValueContainer<V> container = index.getData(dataKey);
 775           for (final Iterator<V> it = container.getValueIterator(); it.hasNext();) {
 776             final V value = it.next();
 777             for (final ValueContainer.IntIterator inputsIt = container.getInputIdsIterator(value); inputsIt.hasNext();) {
 778               processor.process(inputsIt.next(), value);
 779             }
 780           }
 781         }
 782       }
 783       finally {
 784         index.getReadLock().unlock();
 785       }
 786     }
 787     catch (StorageException e) {
 788       scheduleRebuild(indexId, e);
 789     }
 790     catch (RuntimeException e) {
 791       final Throwable cause = e.getCause();
 792       if (cause instanceof StorageException || cause instanceof IOException) {
 793         scheduleRebuild(indexId, e);
 794       }
 795       else {
 796         throw e;
 797       }
 798     }
 799   }
 800
 801   private <K> void scheduleRebuild(final ID<K, ?> indexId, final Throwable e) {
 802     requestRebuild(indexId);
 803     LOG.info(e);
 804     checkRebuild(indexId, false);
 805   }
 806
 807   @TestOnly
 808   public boolean isIndexReady(final ID<?, ?> indexId) {
 809     return myRebuildStatus.get(indexId).get() == OK;
 810   }
 811
 812   private void checkRebuild(final ID<?, ?> indexId, final boolean cleanupOnly) {
 813     if (myRebuildStatus.get(indexId).compareAndSet(REQUIRES_REBUILD, REBUILD_IN_PROGRESS)) {
 814       cleanupProcessedFlag();
 815
 816       final Runnable rebuildRunnable = new Runnable() {
 817         public void run() {
 818           try {
 819             clearIndex(indexId);
 820             if (!cleanupOnly) {
 821               final FileSystemSynchronizerImpl synchronizer = new FileSystemSynchronizerImpl();
 822               synchronizer.setCancelable(false);
 823               for (Project project : ProjectManager.getInstance().getOpenProjects()) {
 824                 synchronizer.registerCacheUpdater(new UnindexedFilesUpdater(project, ProjectRootManager.getInstance(project), FileBasedIndex.this));
 825               }
 826               synchronizer.executeFileUpdate();
 827             }
 828           }
 829           catch (StorageException e) {
 830             requestRebuild(indexId);
 831             LOG.info(e);
 832           }
 833           finally {
 834             myRebuildStatus.get(indexId).compareAndSet(REBUILD_IN_PROGRESS, OK);
 835           }
 836         }
 837       };
 838
 839       final Application application = ApplicationManager.getApplication();
 840       if (cleanupOnly || application.isUnitTestMode()) {
 841         rebuildRunnable.run();
 842       }
 843       else {
 844         SwingUtilities.invokeLater(new Runnable() {
 845           public void run() {
 846             new Task.Modal(null, "Updating index", false) {
 847               public void run(@NotNull final ProgressIndicator indicator) {
 848                 indicator.setIndeterminate(true);
 849                 rebuildRunnable.run();
 850               }
 851             }.queue();
 852           }
 853         });
 854       }
 855     }
 856
 857     if (myRebuildStatus.get(indexId).get() == REBUILD_IN_PROGRESS) {
 858       throw new ProcessCanceledException();
 859     }
 860   }
 861
 862   private void clearIndex(final ID<?, ?> indexId) throws StorageException {
 863     final UpdatableIndex<?, ?, FileContent> index = getIndex(indexId);
 864     assert index != null;
 865     index.clear();
 866     try {
 867       IndexInfrastructure.rewriteVersion(IndexInfrastructure.getVersionFile(indexId), myIndexIdToVersionMap.get(indexId));
 868     }
 869     catch (IOException e) {
 870       LOG.error(e);
 871     }
 872   }
 873
 874   private Set<Document> getUnsavedOrTransactedDocuments() {
 875     Set<Document> docs = new HashSet<Document>(Arrays.asList(myFileDocumentManager.getUnsavedDocuments()));
 876     docs.addAll(myTransactionMap.keySet());
 877     return docs;
 878   }
 879
 880   private void indexUnsavedDocuments(ID<?, ?> indexId, Project project) throws StorageException {
 881     myChangedFilesUpdater.forceUpdate();
 882
 883     if (myUpToDateIndices.contains(indexId)) {
 884       return; // no need to index unsaved docs
 885     }
 886
 887     final Set<Document> documents = getUnsavedOrTransactedDocuments();
 888     if (!documents.isEmpty()) {
 889       // now index unsaved data
 890       final StorageGuard.Holder guard = setDataBufferingEnabled(true);
 891       try {
 892         final Semaphore semaphore = myUnsavedDataIndexingSemaphores.get(indexId);
 893         semaphore.down();
 894         try {
 895           for (Document document : documents) {
 896             indexUnsavedDocument(document, indexId, project);
 897           }
 898         }
 899         finally {
 900           semaphore.up();
 901
 902           while (!semaphore.waitFor(500)) { // may need to wait until another thread is done with indexing
 903             if (Thread.holdsLock(PsiLock.LOCK)) {
 904               break; // hack. Most probably that other indexing threads is waiting for PsiLock, which we're are holding.
 905             }
 906           }
 907           myUpToDateIndices.add(indexId); // safe to set the flag here, becase it will be cleared under the WriteAction
 908         }
 909       }
 910       finally {
 911         guard.leave();
 912       }
 913     }
 914   }
 915
 916   private interface DocumentContent {
 917     String getText();
 918     long getModificationStamp();
 919   }
 920
 921   private static class AuthenticContent implements DocumentContent {
 922     private final Document myDocument;
 923
 924     private AuthenticContent(final Document document) {
 925       myDocument = document;
 926     }
 927
 928     public String getText() {
 929       return myDocument.getText();
 930     }
 931
 932     public long getModificationStamp() {
 933       return myDocument.getModificationStamp();
 934     }
 935   }
 936
 937   private static class PsiContent implements DocumentContent {
 938     private final Document myDocument;
 939     private final PsiFile myFile;
 940
 941     private PsiContent(final Document document, final PsiFile file) {
 942       myDocument = document;
 943       myFile = file;
 944     }
 945
 946     public String getText() {
 947       if (myFile.getModificationStamp() != myDocument.getModificationStamp()) {
 948         final ASTNode node = myFile.getNode();
 949         assert node != null;
 950         return node.getText();
 951       }
 952       return myDocument.getText();
 953     }
 954
 955     public long getModificationStamp() {
 956       return myFile.getModificationStamp();
 957     }
 958   }
 959
 960   private void indexUnsavedDocument(final Document document, final ID<?, ?> requestedIndexId, Project project) throws StorageException {
 961     final VirtualFile vFile = myFileDocumentManager.getFile(document);
 962     if (!(vFile instanceof VirtualFileWithId) || !vFile.isValid()) {
 963       return;
 964     }
 965
 966     final PsiFile dominantContentFile = findDominantPsiForDocument(document, project);
 967
 968     DocumentContent content;
 969     if (dominantContentFile != null && dominantContentFile.getModificationStamp() != document.getModificationStamp()) {
 970       content = new PsiContent(document, dominantContentFile);
 971     }
 972     else {
 973       content = new AuthenticContent(document);
 974     }
 975
 976     final long currentDocStamp = content.getModificationStamp();
 977     if (currentDocStamp != myLastIndexedDocStamps.getAndSet(document, requestedIndexId, currentDocStamp).longValue()) {
 978       final FileContent newFc = new FileContent(vFile, content.getText(), vFile.getCharset());
 979
 980       if (dominantContentFile != null) {
 981         dominantContentFile.putUserData(PsiFileImpl.BUILDING_STUB, true);
 982         newFc.putUserData(PSI_FILE, dominantContentFile);
 983       }
 984
 985       if (content instanceof AuthenticContent) {
 986         newFc.putUserData(EDITOR_HIGHLIGHTER, document instanceof DocumentImpl
 987                                               ? ((DocumentImpl)document).getEditorHighlighterForCachesBuilding() : null);
 988       }
 989
 990       if (getInputFilter(requestedIndexId).acceptInput(vFile)) {
 991         newFc.putUserData(PROJECT, project);
 992         final int inputId = Math.abs(getFileId(vFile));
 993         getIndex(requestedIndexId).update(inputId, newFc);
 994       }
 995
 996       if (dominantContentFile != null) {
 997         dominantContentFile.putUserData(PsiFileImpl.BUILDING_STUB, null);
 998       }
 999     }
1000   }
1001
1002   public static final Key<PsiFile> PSI_FILE = new Key<PsiFile>("PSI for stubs");
1003   public static final Key<EditorHighlighter> EDITOR_HIGHLIGHTER = new Key<EditorHighlighter>("Editor");
1004   public static final Key<Project> PROJECT = new Key<Project>("Context project");
1005   public static final Key<VirtualFile> VIRTUAL_FILE = new Key<VirtualFile>("Context virtual file");
1006
1007   @Nullable
1008   private PsiFile findDominantPsiForDocument(final Document document, Project project) {
1009     if (myTransactionMap.containsKey(document)) {
1010       return myTransactionMap.get(document);
1011     }
1012
1013     return findLatestKnownPsiForUncomittedDocument(document, project);
1014   }
1015
1016   private final StorageGuard myStorageLock = new StorageGuard();
1017
1018   private StorageGuard.Holder setDataBufferingEnabled(final boolean enabled) {
1019     final StorageGuard.Holder holder = myStorageLock.enter(enabled);
1020     if (!enabled) {
1021       synchronized (myLastIndexedDocStamps) {
1022         myLastIndexedDocStamps.clear();
1023       }
1024     }
1025     for (ID<?, ?> indexId : myIndices.keySet()) {
1026       final MapReduceIndex index = (MapReduceIndex)getIndex(indexId);
1027       assert index != null;
1028       final IndexStorage indexStorage = index.getStorage();
1029       ((MemoryIndexStorage)indexStorage).setBufferingEnabled(enabled);
1030     }
1031     return holder;
1032   }
1033
1034   private void dropUnregisteredIndices() {
1035     final Set<String> indicesToDrop = readRegistsredIndexNames();
1036     for (ID<?, ?> key : myIndices.keySet()) {
1037       indicesToDrop.remove(key.toString());
1038     }
1039     for (String s : indicesToDrop) {
1040       FileUtil.delete(IndexInfrastructure.getIndexRootDir(ID.create(s)));
1041     }
1042   }
1043
1044   public void requestRebuild(ID<?, ?> indexId) {
1045     cleanupProcessedFlag();
1046     LOG.info("Rebuild requested for index " + indexId, new Throwable());
1047     myRebuildStatus.get(indexId).set(REQUIRES_REBUILD);
1048   }
1049
1050   private <K, V> UpdatableIndex<K, V, FileContent> getIndex(ID<K, V> indexId) {
1051     final Pair<UpdatableIndex<?, ?, FileContent>, InputFilter> pair = myIndices.get(indexId);
1052     //noinspection unchecked
1053     return pair != null? (UpdatableIndex<K,V, FileContent>)pair.getFirst() : null;
1054   }
1055
1056   private InputFilter getInputFilter(ID<?, ?> indexId) {
1057     final Pair<UpdatableIndex<?, ?, FileContent>, InputFilter> pair = myIndices.get(indexId);
1058     return pair != null? pair.getSecond() : null;
1059   }
1060
1061   public void indexFileContent(com.intellij.ide.startup.FileContent content) {
1062     final VirtualFile file = content.getVirtualFile();
1063     FileContent fc = null;
1064
1065     PsiFile psiFile = null;
1066     //final Job<Object> job = JobScheduler.getInstance().createJob("IndexJob", Job.DEFAULT_PRIORITY / 2);
1067
1068     for (final ID<?, ?> indexId : myIndices.keySet()) {
1069       if (shouldIndexFile(file, indexId)) {
1070         if (fc == null) {
1071           byte[] currentBytes;
1072           try {
1073             currentBytes = content.getBytes();
1074           }
1075           catch (IOException e) {
1076             currentBytes = ArrayUtil.EMPTY_BYTE_ARRAY;
1077           }
1078           fc = new FileContent(file, currentBytes);
1079
1080           psiFile = content.getUserData(PSI_FILE);
1081           if (psiFile != null) {
1082             psiFile.putUserData(PsiFileImpl.BUILDING_STUB, true);
1083             fc.putUserData(PSI_FILE, psiFile);
1084           }
1085           Project project = content.getUserData(PROJECT);
1086           if (project == null) {
1087             project = ProjectUtil.guessProjectForFile(file);
1088           }
1089           fc.putUserData(PROJECT, project);
1090         }
1091
1092         final FileContent _fc = fc;
1093         //job.addTask(new Runnable() {
1094         //  public void run() {
1095             try {
1096               updateSingleIndex(indexId, file, _fc);
1097             }
1098             catch (StorageException e) {
1099               requestRebuild(indexId);
1100               LOG.info(e);
1101             }
1102           //}
1103         //});
1104       }
1105     }
1106
1107     //try {
1108     //  job.scheduleAndWaitForResults();
1109     //}
1110     //catch (Throwable throwable) {
1111     //  LOG.info(throwable);
1112     //}
1113
1114     if (psiFile != null) {
1115       psiFile.putUserData(PsiFileImpl.BUILDING_STUB, null);
1116     }
1117   }
1118
1119   private void updateSingleIndex(final ID<?, ?> indexId, final VirtualFile file, final FileContent currentFC)
1120     throws StorageException {
1121     if (myRebuildStatus.get(indexId).get() == REQUIRES_REBUILD) {
1122       return; // the index is scheduled for rebuild, no need to update
1123     }
1124
1125     final StorageGuard.Holder lock = setDataBufferingEnabled(false);
1126
1127     try {
1128       final int inputId = Math.abs(getFileId(file));
1129
1130       final UpdatableIndex<?, ?, FileContent> index = getIndex(indexId);
1131       assert index != null;
1132
1133       index.update(inputId, currentFC);
1134       ApplicationManager.getApplication().runReadAction(new Runnable() {
1135         public void run() {
1136           if (file.isValid()) {
1137             if (currentFC != null) {
1138               IndexingStamp.update(file, indexId, IndexInfrastructure.getIndexCreationStamp(indexId));
1139             }
1140             else {
1141               // mark the file as unindexed
1142               IndexingStamp.update(file, indexId, -1L);
1143             }
1144           }
1145         }
1146       });
1147     }
1148     finally {
1149       lock.leave();
1150     }
1151   }
1152
1153   public static int getFileId(final VirtualFile file) {
1154     if (file instanceof VirtualFileWithId) {
1155       return ((VirtualFileWithId)file).getId();
1156     }
1157
1158     throw new IllegalArgumentException("Virtual file doesn't support id: " + file + ", implementation class: " + file.getClass().getName());
1159   }
1160
1161   private boolean needsFileContentLoading(ID<?, ?> indexId) {
1162     return !myNotRequiringContentIndices.contains(indexId);
1163   }
1164
1165   private final class ChangedFilesUpdater extends VirtualFileAdapter implements CacheUpdater{
1166     private final Set<VirtualFile> myFilesToUpdate = new LinkedHashSet<VirtualFile>();
1167     private final JBReentrantReadWriteLock myLock = LockFactory.createReadWriteLock();
1168     private final JBLock r = myLock.readLock();
1169     private final JBLock w = myLock.writeLock();
1170
1171     private final ManagingFS myManagingFS = ManagingFS.getInstance();
1172     // No need to react on movement events since files stay valid, their ids don't change and all associated attributes remain intact.
1173
1174     public void fileCreated(final VirtualFileEvent event) {
1175       markDirty(event);
1176     }
1177
1178     public void fileDeleted(final VirtualFileEvent event) {
1179       w.lock();
1180       try {
1181         myFilesToUpdate.remove(event.getFile()); // no need to update it anymore
1182       }
1183       finally {
1184         w.unlock();
1185       }
1186     }
1187
1188     public void fileCopied(final VirtualFileCopyEvent event) {
1189       markDirty(event);
1190     }
1191
1192     public void beforeFileDeletion(final VirtualFileEvent event) {
1193       invalidateIndices(event.getFile(), false);
1194     }
1195
1196     public void beforeContentsChange(final VirtualFileEvent event) {
1197       invalidateIndices(event.getFile(), true);
1198     }
1199
1200     public void contentsChanged(final VirtualFileEvent event) {
1201       markDirty(event);
1202     }
1203
1204     public void beforePropertyChange(final VirtualFilePropertyEvent event) {
1205       if (event.getPropertyName().equals(VirtualFile.PROP_NAME)) {
1206         // indexes may depend on file name
1207         final VirtualFile file = event.getFile();
1208         if (!file.isDirectory()) {
1209           // name change may lead to filetype change so the file might become not indexable
1210           // in general case have to 'unindex' the file and index it again if needed after the name has been changed
1211           invalidateIndices(file, false);
1212         }
1213       }
1214     }
1215
1216     public void propertyChanged(final VirtualFilePropertyEvent event) {
1217       if (event.getPropertyName().equals(VirtualFile.PROP_NAME)) {
1218         // indexes may depend on file name
1219         if (!event.getFile().isDirectory()) {
1220           markDirty(event);
1221         }
1222       }
1223     }
1224
1225     private void markDirty(final VirtualFileEvent event) {
1226       cleanProcessedFlag(event.getFile());
1227       iterateIndexableFiles(event.getFile(), new Processor<VirtualFile>() {
1228         public boolean process(final VirtualFile file) {
1229           FileContent fileContent = null;
1230           // handle 'content-less' indices separately
1231           for (ID<?, ?> indexId : myNotRequiringContentIndices) {
1232             if (getInputFilter(indexId).acceptInput(file)) {
1233               try {
1234                 if (fileContent == null) {
1235                   fileContent = new FileContent(file);
1236                 }
1237                 updateSingleIndex(indexId, file, fileContent);
1238               }
1239               catch (StorageException e) {
1240                 LOG.info(e);
1241                 requestRebuild(indexId);
1242               }
1243             }
1244           }
1245           // For 'normal indices' schedule the file for update and stop iteration if at least one index accepts it
1246           if (!isTooLarge(file)) {
1247             for (ID<?, ?> indexId : myIndices.keySet()) {
1248               if (needsFileContentLoading(indexId) && getInputFilter(indexId).acceptInput(file)) {
1249                 w.lock();
1250                 try {
1251                   myFilesToUpdate.add(file);
1252                   break; // no need to iterate further, as the file is already marked
1253                 }
1254                 finally {
1255                   w.unlock();
1256                 }
1257               }
1258             }
1259           }
1260
1261           return true;
1262         }
1263       });
1264       IndexingStamp.flushCache();
1265     }
1266
1267     void invalidateIndices(final VirtualFile file, final boolean markForReindex) {
1268       if (file.isDirectory()) {
1269         if (isMock(file) || myManagingFS.wereChildrenAccessed(file)) {
1270           for (VirtualFile child : file.getChildren()) {
1271             invalidateIndices(child, markForReindex);
1272           }
1273         }
1274       }
1275       else {
1276         cleanProcessedFlag(file);
1277         IndexingStamp.flushCache();
1278         boolean indicesAffected = false;
1279
1280         final boolean isTooLarge = isTooLarge(file);
1281         for (final ID<?, ?> indexId : myIndices.keySet()) {
1282           try {
1283             if (myNotRequiringContentIndices.contains(indexId)) {
1284               if (shouldUpdateIndex(file, indexId)) {
1285                 updateSingleIndex(indexId, file, null);
1286               }
1287             }
1288             else { // the index requires file content
1289               if (!isTooLarge && shouldUpdateIndex(file, indexId)) {
1290                 indicesAffected = true;
1291                 if (markForReindex) {
1292                   // only mark the file as unindexed, reindex will be done lazily
1293                   ApplicationManager.getApplication().runReadAction(new Runnable() {
1294                     public void run() {
1295                       IndexingStamp.update(file, indexId, -1L);
1296                     }
1297                   });
1298                 }
1299                 else {
1300                   updateSingleIndex(indexId, file, null);
1301                 }
1302               }
1303             }
1304           }
1305           catch (StorageException e) {
1306             LOG.info(e);
1307             requestRebuild(indexId);
1308           }
1309         }
1310         IndexingStamp.flushCache();
1311         if (indicesAffected && markForReindex) {
1312           iterateIndexableFiles(file, new Processor<VirtualFile>() {
1313             public boolean process(final VirtualFile file) {
1314               w.lock();
1315               try {
1316                 myFilesToUpdate.add(file);
1317               }
1318               finally {
1319                 w.unlock();
1320               }
1321               return true;
1322             }
1323           });
1324         }
1325       }
1326     }
1327
1328     private void iterateIndexableFiles(final VirtualFile file, final Processor<VirtualFile> processor) {
1329       if (file.isDirectory()) {
1330         final ContentIterator iterator = new ContentIterator() {
1331           public boolean processFile(final VirtualFile fileOrDir) {
1332             if (!fileOrDir.isDirectory()) {
1333               processor.process(fileOrDir);
1334             }
1335             return true;
1336           }
1337         };
1338
1339         for (IndexableFileSet set : myIndexableSets) {
1340           set.iterateIndexableFilesIn(file, iterator);
1341         }
1342       }
1343       else {
1344         for (IndexableFileSet set : myIndexableSets) {
1345           if (set.isInSet(file)) {
1346             processor.process(file);
1347             break;
1348           }
1349         }
1350       }
1351     }
1352
1353     public VirtualFile[] queryNeededFiles() {
1354       r.lock();
1355       try {
1356         if (myFilesToUpdate.isEmpty()) return VirtualFile.EMPTY_ARRAY;
1357         return myFilesToUpdate.toArray(new VirtualFile[myFilesToUpdate.size()]);
1358       }
1359       finally {
1360         r.unlock();
1361       }
1362     }
1363
1364     public void processFile(final com.intellij.ide.startup.FileContent fileContent) {
1365       processFileImpl(fileContent);
1366     }
1367
1368     private final Semaphore myForceUpdateSemaphore = new Semaphore();
1369
1370     public void forceUpdate() {
1371       final VirtualFile[] files = queryNeededFiles();
1372       if (files.length > 0) {
1373         myForceUpdateSemaphore.down();
1374         try {
1375           for (VirtualFile file: files) {
1376             processFileImpl(new com.intellij.ide.startup.FileContent(file));
1377           }
1378         }
1379         finally {
1380           myForceUpdateSemaphore.up();
1381           myForceUpdateSemaphore.waitFor(); // possibly wait until another thread completes indexing
1382         }
1383       }
1384     }
1385
1386     public void updatingDone() {
1387     }
1388
1389     public void canceled() {
1390     }
1391
1392     private void processFileImpl(final com.intellij.ide.startup.FileContent fileContent) {
1393       final VirtualFile file = fileContent.getVirtualFile();
1394       final boolean reallyRemoved;
1395       w.lock();
1396       try {
1397         reallyRemoved = myFilesToUpdate.remove(file);
1398       }
1399       finally {
1400         w.unlock();
1401       }
1402       if (reallyRemoved && file.isValid()) {
1403         indexFileContent(fileContent);
1404         IndexingStamp.flushCache();
1405       }
1406     }
1407   }
1408
1409   private class UnindexedFilesFinder implements CollectingContentIterator {
1410     private final List<VirtualFile> myFiles = new ArrayList<VirtualFile>();
1411     private final ProgressIndicator myProgressIndicator;
1412
1413     private UnindexedFilesFinder() {
1414       myProgressIndicator = ProgressManager.getInstance().getProgressIndicator();
1415     }
1416
1417     public List<VirtualFile> getFiles() {
1418       return myFiles;
1419     }
1420
1421     public boolean processFile(final VirtualFile file) {
1422       if (!file.isDirectory()) {
1423         if (file instanceof NewVirtualFile && ((NewVirtualFile)file).getFlag(ALREADY_PROCESSED)) {
1424           return true;
1425         }
1426
1427         if (file instanceof VirtualFileWithId) {
1428           boolean oldStuff = true;
1429           if (!isTooLarge(file)) {
1430             for (ID<?, ?> indexId : myIndices.keySet()) {
1431               try {
1432                 if (shouldIndexFile(file, indexId)) {
1433                   myFiles.add(file);
1434                   oldStuff = false;
1435                   break;
1436                 }
1437               }
1438               catch (RuntimeException e) {
1439                 final Throwable cause = e.getCause();
1440                 if (cause instanceof IOException || cause instanceof StorageException) {
1441                   LOG.info(e);
1442                   requestRebuild(indexId);
1443                 }
1444                 else {
1445                   throw e;
1446                 }
1447               }
1448             }
1449           }
1450           FileContent fileContent = null;
1451           for (ID<?, ?> indexId : myNotRequiringContentIndices) {
1452             if (shouldIndexFile(file, indexId)) {
1453               oldStuff = false;
1454               try {
1455                 if (fileContent == null) {
1456                   fileContent = new FileContent(file);
1457                 }
1458                 updateSingleIndex(indexId, file, fileContent);
1459               }
1460               catch (StorageException e) {
1461                 LOG.info(e);
1462                 requestRebuild(indexId);
1463               }
1464             }
1465           }
1466           IndexingStamp.flushCache();
1467
1468           if (oldStuff && file instanceof NewVirtualFile) {
1469             ((NewVirtualFile)file).setFlag(ALREADY_PROCESSED, true);
1470           }
1471         }
1472       }
1473       else {
1474         if (myProgressIndicator != null) {
1475           myProgressIndicator.setText("Scanning files to index");
1476           myProgressIndicator.setText2(file.getPresentableUrl());
1477         }
1478       }
1479       return true;
1480     }
1481   }
1482
1483   private boolean shouldUpdateIndex(final VirtualFile file, final ID<?, ?> indexId) {
1484     return getInputFilter(indexId).acceptInput(file) &&
1485            (isMock(file) || IndexingStamp.isFileIndexed(file, indexId, IndexInfrastructure.getIndexCreationStamp(indexId)));
1486   }
1487
1488   private boolean shouldIndexFile(final VirtualFile file, final ID<?, ?> indexId) {
1489     return getInputFilter(indexId).acceptInput(file) &&
1490            (isMock(file) || !IndexingStamp.isFileIndexed(file, indexId, IndexInfrastructure.getIndexCreationStamp(indexId)));
1491   }
1492
1493   private static boolean isMock(final VirtualFile file) {
1494     return !(file instanceof NewVirtualFile);
1495   }
1496
1497   private boolean isTooLarge(VirtualFile file) {
1498     if (SingleRootFileViewProvider.isTooLarge(file)) {
1499       final FileType type = FileTypeManager.getInstance().getFileTypeByFile(file);
1500       return !myNoLimitCheckTypes.contains(type);
1501     }
1502     return false;
1503   }
1504
1505   public CollectingContentIterator createContentIterator() {
1506     return new UnindexedFilesFinder();
1507   }
1508
1509   public void registerIndexableSet(IndexableFileSet set) {
1510     myIndexableSets.add(set);
1511   }
1512
1513   public void removeIndexableSet(IndexableFileSet set) {
1514     myChangedFilesUpdater.forceUpdate();
1515     myIndexableSets.remove(set);
1516   }
1517
1518   @Nullable
1519   private static PsiFile findLatestKnownPsiForUncomittedDocument(Document doc, Project project) {
1520     return PsiDocumentManager.getInstance(project).getCachedPsiFile(doc);
1521   }
1522
1523   private static class IndexableFilesFilter implements InputFilter {
1524     private final InputFilter myDelegate;
1525
1526     private IndexableFilesFilter(InputFilter delegate) {
1527       myDelegate = delegate;
1528     }
1529
1530     public boolean acceptInput(final VirtualFile file) {
1531       return file instanceof VirtualFileWithId && myDelegate.acceptInput(file);
1532     }
1533   }
1534
1535   private static void cleanupProcessedFlag() {
1536     final VirtualFile[] roots = ManagingFS.getInstance().getRoots();
1537     for (VirtualFile root : roots) {
1538       cleanProcessedFlag(root);
1539     }
1540   }
1541
1542   private static void cleanProcessedFlag(final VirtualFile file) {
1543     if (!(file instanceof NewVirtualFile)) return;
1544
1545     final NewVirtualFile nvf = (NewVirtualFile)file;
1546     if (file.isDirectory()) {
1547       for (VirtualFile child : nvf.getCachedChildren()) {
1548         cleanProcessedFlag(child);
1549       }
1550     }
1551     else {
1552 /*      nvf.clearCachedFileType(); */
1553       nvf.setFlag(ALREADY_PROCESSED, false);
1554     }
1555   }
1556
1557   private static class StorageGuard {
1558     private int myHolds = 0;
1559
1560     public interface Holder {
1561       void leave();
1562     }
1563
1564     private final Holder myTrueHolder = new Holder() {
1565       public void leave() {
1566         StorageGuard.this.leave(true);
1567       }
1568     };
1569     private final Holder myFalseHolder = new Holder() {
1570       public void leave() {
1571         StorageGuard.this.leave(false);
1572       }
1573     };
1574
1575     public synchronized Holder enter(boolean mode) {
1576       if (mode) {
1577         while (myHolds < 0) {
1578           try {
1579             wait();
1580           }
1581           catch (InterruptedException ignored) {
1582           }
1583         }
1584         myHolds++;
1585         return myTrueHolder;
1586       }
1587       else {
1588         while (myHolds > 0) {
1589           try {
1590             wait();
1591           }
1592           catch (InterruptedException ignored) {
1593           }
1594         }
1595         myHolds--;
1596         return myFalseHolder;
1597       }
1598     }
1599
1600     private synchronized void leave(boolean mode) {
1601       myHolds += (mode? -1 : 1);
1602       if (myHolds == 0) {
1603         notifyAll();
1604       }
1605     }
1606
1607   }
1608 }