platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndex.java

   1 package com.intellij.util.indexing;
   2
   3 import com.intellij.AppTopics;
   4 import com.intellij.concurrency.Job;
   5 import com.intellij.concurrency.JobScheduler;
   6 import com.intellij.ide.startup.BackgroundableCacheUpdater;
   7 import com.intellij.ide.startup.impl.FileSystemSynchronizerImpl;
   8 import com.intellij.lang.ASTNode;
   9 import com.intellij.openapi.application.*;
  10 import com.intellij.openapi.components.ApplicationComponent;
  11 import com.intellij.openapi.diagnostic.Logger;
  12 import com.intellij.openapi.editor.Document;
  13 import com.intellij.openapi.editor.highlighter.EditorHighlighter;
  14 import com.intellij.openapi.editor.impl.DocumentImpl;
  15 import com.intellij.openapi.extensions.Extensions;
  16 import com.intellij.openapi.fileEditor.FileDocumentManager;
  17 import com.intellij.openapi.fileTypes.*;
  18 import com.intellij.openapi.progress.ProcessCanceledException;
  19 import com.intellij.openapi.progress.ProgressIndicator;
  20 import com.intellij.openapi.progress.ProgressManager;
  21 import com.intellij.openapi.progress.Task;
  22 import com.intellij.openapi.progress.impl.BackgroundableProcessIndicator;
  23 import com.intellij.openapi.project.*;
  24 import com.intellij.openapi.roots.CollectingContentIterator;
  25 import com.intellij.openapi.roots.ContentIterator;
  26 import com.intellij.openapi.roots.ProjectRootManager;
  27 import com.intellij.openapi.util.Key;
  28 import com.intellij.openapi.util.Pair;
  29 import com.intellij.openapi.util.ShutDownTracker;
  30 import com.intellij.openapi.util.io.FileUtil;
  31 import com.intellij.openapi.util.registry.Registry;
  32 import com.intellij.openapi.vfs.*;
  33 import com.intellij.openapi.vfs.ex.VirtualFileManagerEx;
  34 import com.intellij.openapi.vfs.newvfs.ManagingFS;
  35 import com.intellij.openapi.vfs.newvfs.NewVirtualFile;
  36 import com.intellij.openapi.vfs.newvfs.persistent.PersistentFS;
  37 import com.intellij.psi.PsiDocumentManager;
  38 import com.intellij.psi.PsiFile;
  39 import com.intellij.psi.PsiLock;
  40 import com.intellij.psi.SingleRootFileViewProvider;
  41 import com.intellij.psi.impl.PsiDocumentTransactionListener;
  42 import com.intellij.psi.impl.source.PsiFileImpl;
  43 import com.intellij.psi.search.GlobalSearchScope;
  44 import com.intellij.util.ArrayUtil;
  45 import com.intellij.util.CommonProcessors;
  46 import com.intellij.util.Processor;
  47 import com.intellij.util.concurrency.JBLock;
  48 import com.intellij.util.concurrency.JBReentrantReadWriteLock;
  49 import com.intellij.util.concurrency.LockFactory;
  50 import com.intellij.util.concurrency.Semaphore;
  51 import com.intellij.util.containers.ConcurrentHashSet;
  52 import com.intellij.util.containers.ContainerUtil;
  53 import com.intellij.util.io.*;
  54 import com.intellij.util.messages.MessageBus;
  55 import com.intellij.util.messages.MessageBusConnection;
  56 import gnu.trove.TIntHashSet;
  57 import gnu.trove.TIntIterator;
  58 import gnu.trove.TObjectIntHashMap;
  59 import org.jetbrains.annotations.NonNls;
  60 import org.jetbrains.annotations.NotNull;
  61 import org.jetbrains.annotations.Nullable;
  62 import org.jetbrains.annotations.TestOnly;
  63
  64 import javax.swing.*;
  65 import java.io.*;
  66 import java.util.*;
  67 import java.util.concurrent.atomic.AtomicBoolean;
  68 import java.util.concurrent.atomic.AtomicInteger;
  69 import java.util.concurrent.locks.Lock;
  70
  71 /**
  72  * @author Eugene Zhuravlev
  73  *         Date: Dec 20, 2007
  74  */
  75
  76 public class FileBasedIndex implements ApplicationComponent {
  77   private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.FileBasedIndex");
  78   @NonNls
  79   private static final String CORRUPTION_MARKER_NAME = "corruption.marker";
  80   private final Map<ID<?, ?>, Pair<UpdatableIndex<?, ?, FileContent>, InputFilter>> myIndices = new HashMap<ID<?, ?>, Pair<UpdatableIndex<?, ?, FileContent>, InputFilter>>();
  81   private final Map<ID<?, ?>, Semaphore> myUnsavedDataIndexingSemaphores = new HashMap<ID<?,?>, Semaphore>();
  82   private final TObjectIntHashMap<ID<?, ?>> myIndexIdToVersionMap = new TObjectIntHashMap<ID<?, ?>>();
  83   private final Set<ID<?, ?>> myNotRequiringContentIndices = new HashSet<ID<?, ?>>();
  84   private final Set<FileType> myNoLimitCheckTypes = new HashSet<FileType>();
  85
  86   private final PerIndexDocumentMap<Long> myLastIndexedDocStamps = new PerIndexDocumentMap<Long>() {
  87     @Override
  88     protected Long createDefault(Document document) {
  89       return 0L;
  90     }
  91   };
  92
  93   private final ChangedFilesUpdater myChangedFilesUpdater;
  94
  95   private final List<IndexableFileSet> myIndexableSets = ContainerUtil.createEmptyCOWList();
  96
  97   public static final int OK = 1;
  98   public static final int REQUIRES_REBUILD = 2;
  99   public static final int REBUILD_IN_PROGRESS = 3;
 100   private final Map<ID<?, ?>, AtomicInteger> myRebuildStatus = new HashMap<ID<?,?>, AtomicInteger>();
 101
 102   private final VirtualFileManagerEx myVfManager;
 103   private final FileDocumentManager myFileDocumentManager;
 104   private final ConcurrentHashSet<ID<?, ?>> myUpToDateIndices = new ConcurrentHashSet<ID<?, ?>>();
 105   private final Map<Document, PsiFile> myTransactionMap = new HashMap<Document, PsiFile>();
 106
 107   private static final int ALREADY_PROCESSED = 0x02;
 108   private static final String USE_MULTITHREADED_INDEXING = "fileIndex.multithreaded";
 109   private @Nullable String myConfigPath;
 110   private @Nullable String mySystemPath;
 111
 112   public void requestReindex(final VirtualFile file) {
 113     myChangedFilesUpdater.invalidateIndices(file, true);
 114   }
 115
 116   public interface InputFilter {
 117     boolean acceptInput(VirtualFile file);
 118   }
 119
 120   public FileBasedIndex(final VirtualFileManagerEx vfManager, FileDocumentManager fdm, MessageBus bus) throws IOException {
 121     myVfManager = vfManager;
 122     myFileDocumentManager = fdm;
 123
 124     myConfigPath = calcConfigPath(PathManager.getConfigPath());
 125     mySystemPath = calcConfigPath(PathManager.getSystemPath());
 126
 127     final MessageBusConnection connection = bus.connect();
 128     connection.subscribe(PsiDocumentTransactionListener.TOPIC, new PsiDocumentTransactionListener() {
 129       public void transactionStarted(final Document doc, final PsiFile file) {
 130         if (file != null) {
 131           myTransactionMap.put(doc, file);
 132           myUpToDateIndices.clear();
 133         }
 134       }
 135
 136       public void transactionCompleted(final Document doc, final PsiFile file) {
 137         myTransactionMap.remove(doc);
 138       }
 139     });
 140
 141     connection.subscribe(AppTopics.FILE_TYPES, new FileTypeListener() {
 142       private Map<FileType, Set<String>> myTypeToExtensionMap;
 143       public void beforeFileTypesChanged(final FileTypeEvent event) {
 144         cleanupProcessedFlag();
 145         myTypeToExtensionMap = new HashMap<FileType, Set<String>>();
 146         final FileTypeManager manager = event.getManager();
 147         for (FileType type : manager.getRegisteredFileTypes()) {
 148           myTypeToExtensionMap.put(type, getExtensions(manager, type));
 149         }
 150       }
 151
 152       public void fileTypesChanged(final FileTypeEvent event) {
 153         final Map<FileType, Set<String>> oldExtensions = myTypeToExtensionMap;
 154         myTypeToExtensionMap = null;
 155         if (oldExtensions != null) {
 156           final FileTypeManager manager = event.getManager();
 157           final Map<FileType, Set<String>> newExtensions = new HashMap<FileType, Set<String>>();
 158           for (FileType type : manager.getRegisteredFileTypes()) {
 159             newExtensions.put(type, getExtensions(manager, type));
 160           }
 161           // we are interested only in extension changes or removals.
 162           // addition of an extension is handled separately by RootsChanged event
 163           if (!newExtensions.keySet().containsAll(oldExtensions.keySet())) {
 164             rebuildAllndices();
 165             return;
 166           }
 167           for (FileType type : oldExtensions.keySet()) {
 168             if (!newExtensions.get(type).containsAll(oldExtensions.get(type))) {
 169               rebuildAllndices();
 170               return;
 171             }
 172           }
 173         }
 174       }
 175
 176       private Set<String> getExtensions(FileTypeManager manager, FileType type) {
 177         final Set<String> set = new HashSet<String>();
 178         for (FileNameMatcher matcher : manager.getAssociations(type)) {
 179           set.add(matcher.getPresentableString());
 180         }
 181         return set;
 182       }
 183
 184       private void rebuildAllndices() {
 185         for (ID<?, ?> indexId : myIndices.keySet()) {
 186           requestRebuild(indexId);
 187         }
 188       }
 189     });
 190
 191     ApplicationManager.getApplication().addApplicationListener(new ApplicationAdapter() {
 192       public void writeActionStarted(Object action) {
 193         myUpToDateIndices.clear();
 194       }
 195     });
 196
 197     final File workInProgressFile = getMarkerFile();
 198     if (workInProgressFile.exists()) {
 199       // previous IDEA session was closed incorrectly, so drop all indices
 200       FileUtil.delete(PathManager.getIndexRoot());
 201     }
 202
 203     try {
 204       final FileBasedIndexExtension[] extensions = Extensions.getExtensions(FileBasedIndexExtension.EXTENSION_POINT_NAME);
 205       for (FileBasedIndexExtension<?, ?> extension : extensions) {
 206         myRebuildStatus.put(extension.getName(), new AtomicInteger(OK));
 207       }
 208
 209       final File corruptionMarker = new File(PathManager.getIndexRoot(), CORRUPTION_MARKER_NAME);
 210       final boolean currentVersionCorrupted = corruptionMarker.exists();
 211       for (FileBasedIndexExtension<?, ?> extension : extensions) {
 212         registerIndexer(extension, currentVersionCorrupted);
 213       }
 214       FileUtil.delete(corruptionMarker);
 215       dropUnregisteredIndices();
 216
 217       // check if rebuild was requested for any index during registration
 218       for (ID<?, ?> indexId : myIndices.keySet()) {
 219         if (myRebuildStatus.get(indexId).compareAndSet(REQUIRES_REBUILD, OK)) {
 220           try {
 221             clearIndex(indexId);
 222           }
 223           catch (StorageException e) {
 224             requestRebuild(indexId);
 225             LOG.error(e);
 226           }
 227         }
 228       }
 229
 230       myChangedFilesUpdater = new ChangedFilesUpdater();
 231       vfManager.addVirtualFileListener(myChangedFilesUpdater);
 232
 233       vfManager.registerRefreshUpdater(myChangedFilesUpdater);
 234
 235       registerIndexableSet(new AdditionalIndexableFileSet());
 236     }
 237     finally {
 238       ShutDownTracker.getInstance().registerShutdownTask(new Runnable() {
 239         public void run() {
 240           performShutdown();
 241         }
 242       });
 243       FileUtil.createIfDoesntExist(workInProgressFile);
 244       saveRegisteredIndices(myIndices.keySet());
 245     }
 246   }
 247
 248   private String calcConfigPath(final String path) {
 249     try {
 250       final String _path = FileUtil.toSystemIndependentName(new File(path).getCanonicalPath());
 251       return _path.endsWith("/")? _path : _path + "/" ;
 252     }
 253     catch (IOException e) {
 254       LOG.info(e);
 255       return null;
 256     }
 257   }
 258
 259   private static FileBasedIndex ourInstance = CachedSingletonsRegistry.markCachedField(FileBasedIndex.class);
 260   public static FileBasedIndex getInstance() {
 261     if (ourInstance == null) {
 262       ourInstance = ApplicationManager.getApplication().getComponent(FileBasedIndex.class);
 263     }
 264
 265     return ourInstance;
 266   }
 267
 268   /**
 269    * @return true if registered index requires full rebuild for some reason, e.g. is just created or corrupted @param extension
 270    * @param isCurrentVersionCorrupted
 271    */
 272   private <K, V> void registerIndexer(final FileBasedIndexExtension<K, V> extension, final boolean isCurrentVersionCorrupted) throws IOException {
 273     final ID<K, V> name = extension.getName();
 274     final int version = extension.getVersion();
 275     if (!extension.dependsOnFileContent()) {
 276       myNotRequiringContentIndices.add(name);
 277     }
 278     myIndexIdToVersionMap.put(name, version);
 279     final File versionFile = IndexInfrastructure.getVersionFile(name);
 280     if (isCurrentVersionCorrupted || IndexInfrastructure.versionDiffers(versionFile, version)) {
 281       if (!isCurrentVersionCorrupted) {
 282         LOG.info("Version has changed for index " + extension.getName() + ". The index will be rebuilt.");
 283       }
 284       FileUtil.delete(IndexInfrastructure.getIndexRootDir(name));
 285       IndexInfrastructure.rewriteVersion(versionFile, version);
 286     }
 287
 288     for (int attempt = 0; attempt < 2; attempt++) {
 289       try {
 290         final MapIndexStorage<K, V> storage = new MapIndexStorage<K, V>(IndexInfrastructure.getStorageFile(name), extension.getKeyDescriptor(), extension.getValueExternalizer(), extension.getCacheSize());
 291         final IndexStorage<K, V> memStorage = new MemoryIndexStorage<K, V>(storage);
 292         final UpdatableIndex<K, V, FileContent> index = createIndex(name, extension, memStorage);
 293         myIndices.put(name, new Pair<UpdatableIndex<?,?, FileContent>, InputFilter>(index, new IndexableFilesFilter(extension.getInputFilter())));
 294         myUnsavedDataIndexingSemaphores.put(name, new Semaphore());
 295         myNoLimitCheckTypes.addAll(extension.getFileTypesWithSizeLimitNotApplicable());
 296         break;
 297       }
 298       catch (IOException e) {
 299         LOG.info(e);
 300         FileUtil.delete(IndexInfrastructure.getIndexRootDir(name));
 301         IndexInfrastructure.rewriteVersion(versionFile, version);
 302       }
 303     }
 304   }
 305
 306   private static void saveRegisteredIndices(Collection<ID<?, ?>> ids) {
 307     final File file = getRegisteredIndicesFile();
 308     try {
 309       FileUtil.createIfDoesntExist(file);
 310       final DataOutputStream os = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(file)));
 311       try {
 312         os.writeInt(ids.size());
 313         for (ID<?, ?> id : ids) {
 314           IOUtil.writeString(id.toString(), os);
 315         }
 316       }
 317       finally {
 318         os.close();
 319       }
 320     }
 321     catch (IOException ignored) {
 322     }
 323   }
 324
 325   private static Set<String> readRegistsredIndexNames() {
 326     final Set<String> result = new HashSet<String>();
 327     try {
 328       final DataInputStream in = new DataInputStream(new BufferedInputStream(new FileInputStream(getRegisteredIndicesFile())));
 329       try {
 330         final int size = in.readInt();
 331         for (int idx = 0; idx < size; idx++) {
 332           result.add(IOUtil.readString(in));
 333         }
 334       }
 335       finally {
 336         in.close();
 337       }
 338     }
 339     catch (IOException ignored) {
 340     }
 341     return result;
 342   }
 343
 344   private static File getRegisteredIndicesFile() {
 345     return new File(PathManager.getIndexRoot(), "registered");
 346   }
 347
 348   private static File getMarkerFile() {
 349     return new File(PathManager.getIndexRoot(), "work_in_progress");
 350   }
 351
 352   private <K, V> UpdatableIndex<K, V, FileContent> createIndex(final ID<K, V> indexId, final FileBasedIndexExtension<K, V> extension, final IndexStorage<K, V> storage) throws IOException {
 353     final MapReduceIndex<K, V, FileContent> index;
 354     if (extension instanceof CustomImplementationFileBasedIndexExtension) {
 355       final UpdatableIndex<K, V, FileContent> custom =
 356         ((CustomImplementationFileBasedIndexExtension<K, V, FileContent>)extension).createIndexImplementation(indexId, this, storage);
 357       if (!(custom instanceof MapReduceIndex)) {
 358         return custom;
 359       }
 360       index = (MapReduceIndex<K,V, FileContent>)custom;
 361     }
 362     else {
 363       index = new MapReduceIndex<K, V, FileContent>(indexId, extension.getIndexer(), storage);
 364     }
 365
 366     final KeyDescriptor<K> keyDescriptor = extension.getKeyDescriptor();
 367     index.setInputIdToDataKeysIndex(new PersistentHashMap<Integer, Collection<K>>(IndexInfrastructure.getInputIndexStorageFile(indexId), new EnumeratorIntegerDescriptor(), new DataExternalizer<Collection<K>>() {
 368       public void save(DataOutput out, Collection<K> value) throws IOException {
 369         DataInputOutputUtil.writeINT(out, value.size());
 370         for (K key : value) {
 371           keyDescriptor.save(out, key);
 372         }
 373       }
 374
 375       public Collection<K> read(DataInput in) throws IOException {
 376         final int size = DataInputOutputUtil.readINT(in);
 377         final List<K> list = new ArrayList<K>();
 378         for (int idx = 0; idx < size; idx++) {
 379           list.add(keyDescriptor.read(in));
 380         }
 381         return list;
 382       }
 383     }));
 384
 385     return index;
 386   }
 387
 388   @NonNls
 389   @NotNull
 390   public String getComponentName() {
 391     return "FileBasedIndex";
 392   }
 393
 394   public void initComponent() {
 395   }
 396
 397   public void disposeComponent() {
 398     performShutdown();
 399   }
 400
 401   private AtomicBoolean myShutdownPerformed = new AtomicBoolean(false);
 402
 403   private void performShutdown() {
 404     if (!myShutdownPerformed.compareAndSet(false, true)) {
 405       return; // already shut down
 406     }
 407     myFileDocumentManager.saveAllDocuments();
 408
 409     LOG.info("START INDEX SHUTDOWN");
 410     try {
 411       myChangedFilesUpdater.forceUpdate();
 412
 413       for (ID<?, ?> indexId : myIndices.keySet()) {
 414         final UpdatableIndex<?, ?, FileContent> index = getIndex(indexId);
 415         assert index != null;
 416         checkRebuild(indexId, true); // if the index was scheduled for rebuild, only clean it
 417         //LOG.info("DISPOSING " + indexId);
 418         index.dispose();
 419       }
 420
 421       myVfManager.removeVirtualFileListener(myChangedFilesUpdater);
 422       myVfManager.unregisterRefreshUpdater(myChangedFilesUpdater);
 423
 424       FileUtil.delete(getMarkerFile());
 425     }
 426     catch (Throwable e) {
 427       LOG.info("Problems during index shutdown", e);
 428       throw new RuntimeException(e);
 429     }
 430     LOG.info("END INDEX SHUTDOWN");
 431   }
 432
 433   public void flushCaches() {
 434     IndexingStamp.flushCache();
 435     ApplicationManager.getApplication().executeOnPooledThread(new Runnable() {
 436       public void run() {
 437         for (ID<?, ?> indexId : myIndices.keySet()) {
 438           //noinspection ConstantConditions
 439           try {
 440             getIndex(indexId).flush();
 441           }
 442           catch (StorageException e) {
 443             LOG.info(e);
 444             requestRebuild(indexId);
 445           }
 446         }
 447       }
 448     });
 449   }
 450
 451   @NotNull
 452   public <K> Collection<K> getAllKeys(final ID<K, ?> indexId, @NotNull Project project) {
 453     Set<K> allKeys = new HashSet<K>();
 454     processAllKeys(indexId, new CommonProcessors.CollectProcessor<K>(allKeys), project);
 455     return allKeys;
 456   }
 457
 458   public <K> boolean processAllKeys(final ID<K, ?> indexId, Processor<K> processor, @NotNull Project project) {
 459     try {
 460       ensureUpToDate(indexId, project);
 461       final UpdatableIndex<K, ?, FileContent> index = getIndex(indexId);
 462       if (index == null) return true;
 463       return index.processAllKeys(processor);
 464     }
 465     catch (StorageException e) {
 466       scheduleRebuild(indexId, e);
 467     }
 468     catch (RuntimeException e) {
 469       final Throwable cause = e.getCause();
 470       if (cause instanceof StorageException || cause instanceof IOException) {
 471         scheduleRebuild(indexId, cause);
 472       }
 473       else {
 474         throw e;
 475       }
 476     }
 477
 478     return false;
 479   }
 480
 481   private static final ThreadLocal<Integer> myUpToDateCheckState = new ThreadLocal<Integer>();
 482
 483   public void disableUpToDateCheckForCurrentThread() {
 484     final Integer currentValue = myUpToDateCheckState.get();
 485     myUpToDateCheckState.set(currentValue == null? 1 : currentValue.intValue() + 1);
 486   }
 487
 488   public void enableUpToDateCheckForCurrentThread() {
 489     final Integer currentValue = myUpToDateCheckState.get();
 490     if (currentValue != null) {
 491       final int newValue = currentValue.intValue() - 1;
 492       if (newValue != 0) {
 493         myUpToDateCheckState.set(newValue);
 494       }
 495       else {
 496         myUpToDateCheckState.remove();
 497       }
 498     }
 499   }
 500
 501   private boolean isUpToDateCheckEnabled() {
 502     final Integer value = myUpToDateCheckState.get();
 503     return value == null || value.intValue() == 0;
 504   }
 505
 506
 507   private final ThreadLocal<Boolean> myReentrancyGuard = new ThreadLocal<Boolean>() {
 508     protected Boolean initialValue() {
 509       return Boolean.FALSE;
 510     }
 511   };
 512
 513   /**
 514    * DO NOT CALL DIRECTLY IN CLIENT CODE
 515    * The method is internal to indexing engine end is called internally. The method is public due to implementation details
 516    */
 517   public <K> void ensureUpToDate(final ID<K, ?> indexId, @NotNull Project project) {
 518     if (isDumb(project)) {
 519       handleDumbMode(indexId, project);
 520     }
 521
 522     if (myReentrancyGuard.get().booleanValue()) {
 523       //assert false : "ensureUpToDate() is not reentrant!";
 524       return;
 525     }
 526     myReentrancyGuard.set(Boolean.TRUE);
 527
 528     try {
 529       if (isUpToDateCheckEnabled()) {
 530         try {
 531           checkRebuild(indexId, false);
 532           indexUnsavedDocuments(indexId, project);
 533         }
 534         catch (StorageException e) {
 535           scheduleRebuild(indexId, e);
 536         }
 537         catch (RuntimeException e) {
 538           final Throwable cause = e.getCause();
 539           if (cause instanceof StorageException || cause instanceof IOException) {
 540             scheduleRebuild(indexId, e);
 541           }
 542           else {
 543             throw e;
 544           }
 545         }
 546       }
 547     }
 548     finally {
 549       myReentrancyGuard.set(Boolean.FALSE);
 550     }
 551   }
 552
 553   private void handleDumbMode(final ID<?, ?> indexId, Project project) {
 554     if (myNotRequiringContentIndices.contains(indexId)) {
 555       return; //indexed eagerly in foreground while building unindexed file list
 556     }
 557
 558     final ProgressManager progressManager = ProgressManager.getInstance();
 559     progressManager.checkCanceled(); // DumbModeAction.CANCEL
 560
 561     final ProgressIndicator progressIndicator = progressManager.getProgressIndicator();
 562     if (progressIndicator instanceof BackgroundableProcessIndicator) {
 563       final BackgroundableProcessIndicator indicator = (BackgroundableProcessIndicator)progressIndicator;
 564       if (indicator.getDumbModeAction() == DumbModeAction.WAIT) {
 565         assert !ApplicationManager.getApplication().isDispatchThread();
 566         DumbService.getInstance(project).waitForSmartMode();
 567         return;
 568       }
 569     }
 570
 571     throw new IndexNotReadyException();
 572   }
 573
 574   private static boolean isDumb(Project project) {
 575     return DumbServiceImpl.getInstance(project).isDumb();
 576   }
 577
 578   @NotNull
 579   public <K, V> List<V> getValues(final ID<K, V> indexId, @NotNull K dataKey, @NotNull final GlobalSearchScope filter) {
 580     final List<V> values = new ArrayList<V>();
 581     processValuesImpl(indexId, dataKey, true, null, new ValueProcessor<V>() {
 582       public boolean process(final VirtualFile file, final V value) {
 583         values.add(value);
 584         return true;
 585       }
 586     }, filter);
 587     return values;
 588   }
 589
 590   @NotNull
 591   public <K, V> Collection<VirtualFile> getContainingFiles(final ID<K, V> indexId, @NotNull K dataKey, @NotNull final GlobalSearchScope filter) {
 592     final Set<VirtualFile> files = new HashSet<VirtualFile>();
 593     processValuesImpl(indexId, dataKey, false, null, new ValueProcessor<V>() {
 594       public boolean process(final VirtualFile file, final V value) {
 595         files.add(file);
 596         return true;
 597       }
 598     }, filter);
 599     return files;
 600   }
 601
 602
 603   public interface ValueProcessor<V> {
 604     /**
 605      * @param value a value to process
 606      * @param file the file the value came from
 607      * @return false if no further processing is needed, true otherwise
 608      */
 609     boolean process(VirtualFile file, V value);
 610   }
 611
 612   /**
 613    * @return false if ValueProcessor.process() returned false; true otherwise or if ValueProcessor was not called at all
 614    */
 615   public <K, V> boolean processValues(final ID<K, V> indexId, final @NotNull K dataKey, @Nullable final VirtualFile inFile,
 616                                    ValueProcessor<V> processor, @NotNull final GlobalSearchScope filter) {
 617     return processValuesImpl(indexId, dataKey, false, inFile, processor, filter);
 618   }
 619
 620
 621
 622   private <K, V> boolean processValuesImpl(final ID<K, V> indexId, final K dataKey, boolean ensureValueProcessedOnce,
 623                                         @Nullable final VirtualFile restrictToFile, ValueProcessor<V> processor,
 624                                         final GlobalSearchScope filter) {
 625     try {
 626       final Project project = filter.getProject();
 627       assert project != null : "GlobalSearchScope#getProject() should be not-null for all index queries";
 628       ensureUpToDate(indexId, project);
 629       final UpdatableIndex<K, V, FileContent> index = getIndex(indexId);
 630       if (index == null) {
 631         return true;
 632       }
 633
 634       final Lock readLock = index.getReadLock();
 635       try {
 636         readLock.lock();
 637         final ValueContainer<V> container = index.getData(dataKey);
 638
 639         boolean shouldContinue = true;
 640
 641         if (restrictToFile != null) {
 642           if (restrictToFile instanceof VirtualFileWithId) {
 643             final int restrictedFileId = getFileId(restrictToFile);
 644             for (final Iterator<V> valueIt = container.getValueIterator(); valueIt.hasNext();) {
 645               final V value = valueIt.next();
 646               if (container.isAssociated(value, restrictedFileId)) {
 647                 shouldContinue = processor.process(restrictToFile, value);
 648                 if (!shouldContinue) {
 649                   break;
 650                 }
 651               }
 652             }
 653           }
 654         }
 655         else {
 656           final PersistentFS fs = (PersistentFS)ManagingFS.getInstance();
 657           VALUES_LOOP: for (final Iterator<V> valueIt = container.getValueIterator(); valueIt.hasNext();) {
 658             final V value = valueIt.next();
 659             for (final ValueContainer.IntIterator inputIdsIterator = container.getInputIdsIterator(value); inputIdsIterator.hasNext();) {
 660               final int id = inputIdsIterator.next();
 661               VirtualFile file = IndexInfrastructure.findFileById(fs, id);
 662               if (file != null && filter.accept(file)) {
 663                 shouldContinue = processor.process(file, value);
 664                 if (!shouldContinue) {
 665                   break VALUES_LOOP;
 666                 }
 667                 if (ensureValueProcessedOnce) {
 668                   break; // continue with the next value
 669                 }
 670               }
 671             }
 672           }
 673         }
 674         return shouldContinue;
 675       }
 676       finally {
 677         index.getReadLock().unlock();
 678       }
 679     }
 680     catch (StorageException e) {
 681       scheduleRebuild(indexId, e);
 682     }
 683     catch (RuntimeException e) {
 684       final Throwable cause = e.getCause();
 685       if (cause instanceof StorageException || cause instanceof IOException) {
 686         scheduleRebuild(indexId, cause);
 687       }
 688       else {
 689         throw e;
 690       }
 691     }
 692     return true;
 693   }
 694
 695   public <K, V> boolean getFilesWithKey(final ID<K, V> indexId, final Set<K> dataKeys,
 696                                          Processor<VirtualFile> processor,
 697                                          GlobalSearchScope filter) {
 698     try {
 699       final Project project = filter.getProject();
 700       assert project != null : "GlobalSearchScope#getProject() should be not-null for all index queries";
 701       ensureUpToDate(indexId, project);
 702       final UpdatableIndex<K, V, FileContent> index = getIndex(indexId);
 703       if (index == null) {
 704         return true;
 705       }
 706
 707       final Lock readLock = index.getReadLock();
 708       try {
 709         readLock.lock();
 710         List<TIntHashSet> locals = new ArrayList<TIntHashSet>();
 711         for (K dataKey : dataKeys) {
 712           TIntHashSet local = new TIntHashSet();
 713           locals.add(local);
 714           final ValueContainer<V> container = index.getData(dataKey);
 715
 716           for (final Iterator<V> valueIt = container.getValueIterator(); valueIt.hasNext();) {
 717             final V value = valueIt.next();
 718             for (final ValueContainer.IntIterator inputIdsIterator = container.getInputIdsIterator(value); inputIdsIterator.hasNext();) {
 719               final int id = inputIdsIterator.next();
 720               local.add(id);
 721             }
 722           }
 723         }
 724
 725         if (locals.size() == 0) return true;
 726
 727         Collections.sort(locals, new Comparator<TIntHashSet>() {
 728           public int compare(TIntHashSet o1, TIntHashSet o2) {
 729             return o1.size() - o2.size();
 730           }
 731         });
 732
 733         final PersistentFS fs = (PersistentFS)ManagingFS.getInstance();
 734         TIntIterator ids = join(locals).iterator();
 735         while (ids.hasNext()) {
 736           int id = ids.next();
 737           VirtualFile file = IndexInfrastructure.findFileById(fs, id);
 738           if (file != null && filter.accept(file)) {
 739             if (!processor.process(file)) return false;
 740           }
 741         }
 742       }
 743       finally {
 744         index.getReadLock().unlock();
 745       }
 746     }
 747     catch (StorageException e) {
 748       scheduleRebuild(indexId, e);
 749     }
 750     catch (RuntimeException e) {
 751       final Throwable cause = e.getCause();
 752       if (cause instanceof StorageException || cause instanceof IOException) {
 753         scheduleRebuild(indexId, cause);
 754       }
 755       else {
 756         throw e;
 757       }
 758     }
 759     return true;
 760   }
 761
 762   private static TIntHashSet join(List<TIntHashSet> locals) {
 763     TIntHashSet result = locals.get(0);
 764     if (locals.size() > 1) {
 765       TIntIterator it = result.iterator();
 766
 767       while (it.hasNext()) {
 768         int id = it.next();
 769         for (int i = 1; i < locals.size(); i++) {
 770           if (!locals.get(i).contains(id)) {
 771             it.remove();
 772             break;
 773           }
 774         }
 775       }
 776     }
 777     return result;
 778   }
 779
 780   public interface AllValuesProcessor<V> {
 781     void process(final int inputId, V value);
 782   }
 783
 784   public <K, V> void processAllValues(final ID<K, V> indexId, AllValuesProcessor<V> processor, @NotNull Project project) {
 785     try {
 786       ensureUpToDate(indexId, project);
 787       final UpdatableIndex<K, V, FileContent> index = getIndex(indexId);
 788       if (index == null) {
 789         return;
 790       }
 791       try {
 792         index.getReadLock().lock();
 793         for (K dataKey : index.getAllKeys()) {
 794           final ValueContainer<V> container = index.getData(dataKey);
 795           for (final Iterator<V> it = container.getValueIterator(); it.hasNext();) {
 796             final V value = it.next();
 797             for (final ValueContainer.IntIterator inputsIt = container.getInputIdsIterator(value); inputsIt.hasNext();) {
 798               processor.process(inputsIt.next(), value);
 799             }
 800           }
 801         }
 802       }
 803       finally {
 804         index.getReadLock().unlock();
 805       }
 806     }
 807     catch (StorageException e) {
 808       scheduleRebuild(indexId, e);
 809     }
 810     catch (RuntimeException e) {
 811       final Throwable cause = e.getCause();
 812       if (cause instanceof StorageException || cause instanceof IOException) {
 813         scheduleRebuild(indexId, e);
 814       }
 815       else {
 816         throw e;
 817       }
 818     }
 819   }
 820
 821   private <K> void scheduleRebuild(final ID<K, ?> indexId, final Throwable e) {
 822     requestRebuild(indexId);
 823     LOG.info(e);
 824     checkRebuild(indexId, false);
 825   }
 826
 827   @TestOnly
 828   public boolean isIndexReady(final ID<?, ?> indexId) {
 829     return myRebuildStatus.get(indexId).get() == OK;
 830   }
 831
 832   private void checkRebuild(final ID<?, ?> indexId, final boolean cleanupOnly) {
 833     if (myRebuildStatus.get(indexId).compareAndSet(REQUIRES_REBUILD, REBUILD_IN_PROGRESS)) {
 834       cleanupProcessedFlag();
 835
 836       final Runnable rebuildRunnable = new Runnable() {
 837         public void run() {
 838           try {
 839             clearIndex(indexId);
 840             if (!cleanupOnly) {
 841               final FileSystemSynchronizerImpl synchronizer = new FileSystemSynchronizerImpl();
 842               synchronizer.setCancelable(false);
 843               for (Project project : ProjectManager.getInstance().getOpenProjects()) {
 844                 synchronizer.registerCacheUpdater(new UnindexedFilesUpdater(project, ProjectRootManager.getInstance(project), FileBasedIndex.this));
 845               }
 846               synchronizer.executeFileUpdate();
 847             }
 848           }
 849           catch (StorageException e) {
 850             requestRebuild(indexId);
 851             LOG.info(e);
 852           }
 853           finally {
 854             myRebuildStatus.get(indexId).compareAndSet(REBUILD_IN_PROGRESS, OK);
 855           }
 856         }
 857       };
 858
 859       final Application application = ApplicationManager.getApplication();
 860       if (cleanupOnly || application.isUnitTestMode()) {
 861         rebuildRunnable.run();
 862       }
 863       else {
 864         SwingUtilities.invokeLater(new Runnable() {
 865           public void run() {
 866             new Task.Modal(null, "Updating index", false) {
 867               public void run(@NotNull final ProgressIndicator indicator) {
 868                 indicator.setIndeterminate(true);
 869                 rebuildRunnable.run();
 870               }
 871             }.queue();
 872           }
 873         });
 874       }
 875     }
 876
 877     if (myRebuildStatus.get(indexId).get() == REBUILD_IN_PROGRESS) {
 878       throw new ProcessCanceledException();
 879     }
 880   }
 881
 882   private void clearIndex(final ID<?, ?> indexId) throws StorageException {
 883     final UpdatableIndex<?, ?, FileContent> index = getIndex(indexId);
 884     assert index != null;
 885     index.clear();
 886     try {
 887       IndexInfrastructure.rewriteVersion(IndexInfrastructure.getVersionFile(indexId), myIndexIdToVersionMap.get(indexId));
 888     }
 889     catch (IOException e) {
 890       LOG.error(e);
 891     }
 892   }
 893
 894   private Set<Document> getUnsavedOrTransactedDocuments() {
 895     Set<Document> docs = new HashSet<Document>(Arrays.asList(myFileDocumentManager.getUnsavedDocuments()));
 896     docs.addAll(myTransactionMap.keySet());
 897     return docs;
 898   }
 899
 900   private void indexUnsavedDocuments(ID<?, ?> indexId, Project project) throws StorageException {
 901     myChangedFilesUpdater.forceUpdate();
 902
 903     if (myUpToDateIndices.contains(indexId)) {
 904       return; // no need to index unsaved docs
 905     }
 906
 907     final Set<Document> documents = getUnsavedOrTransactedDocuments();
 908     if (!documents.isEmpty()) {
 909       // now index unsaved data
 910       final StorageGuard.Holder guard = setDataBufferingEnabled(true);
 911       try {
 912         final Semaphore semaphore = myUnsavedDataIndexingSemaphores.get(indexId);
 913         semaphore.down();
 914         try {
 915           for (Document document : documents) {
 916             indexUnsavedDocument(document, indexId, project);
 917           }
 918         }
 919         finally {
 920           semaphore.up();
 921
 922           while (!semaphore.waitFor(500)) { // may need to wait until another thread is done with indexing
 923             if (Thread.holdsLock(PsiLock.LOCK)) {
 924               break; // hack. Most probably that other indexing threads is waiting for PsiLock, which we're are holding.
 925             }
 926           }
 927           myUpToDateIndices.add(indexId); // safe to set the flag here, becase it will be cleared under the WriteAction
 928         }
 929       }
 930       finally {
 931         guard.leave();
 932       }
 933     }
 934   }
 935
 936   private interface DocumentContent {
 937     String getText();
 938     long getModificationStamp();
 939   }
 940
 941   private static class AuthenticContent implements DocumentContent {
 942     private final Document myDocument;
 943
 944     private AuthenticContent(final Document document) {
 945       myDocument = document;
 946     }
 947
 948     public String getText() {
 949       return myDocument.getText();
 950     }
 951
 952     public long getModificationStamp() {
 953       return myDocument.getModificationStamp();
 954     }
 955   }
 956
 957   private static class PsiContent implements DocumentContent {
 958     private final Document myDocument;
 959     private final PsiFile myFile;
 960
 961     private PsiContent(final Document document, final PsiFile file) {
 962       myDocument = document;
 963       myFile = file;
 964     }
 965
 966     public String getText() {
 967       if (myFile.getModificationStamp() != myDocument.getModificationStamp()) {
 968         final ASTNode node = myFile.getNode();
 969         assert node != null;
 970         return node.getText();
 971       }
 972       return myDocument.getText();
 973     }
 974
 975     public long getModificationStamp() {
 976       return myFile.getModificationStamp();
 977     }
 978   }
 979
 980   private void indexUnsavedDocument(final Document document, final ID<?, ?> requestedIndexId, Project project) throws StorageException {
 981     final VirtualFile vFile = myFileDocumentManager.getFile(document);
 982     if (!(vFile instanceof VirtualFileWithId) || !vFile.isValid()) {
 983       return;
 984     }
 985
 986     final PsiFile dominantContentFile = findDominantPsiForDocument(document, project);
 987
 988     DocumentContent content;
 989     if (dominantContentFile != null && dominantContentFile.getModificationStamp() != document.getModificationStamp()) {
 990       content = new PsiContent(document, dominantContentFile);
 991     }
 992     else {
 993       content = new AuthenticContent(document);
 994     }
 995
 996     final long currentDocStamp = content.getModificationStamp();
 997     if (currentDocStamp != myLastIndexedDocStamps.getAndSet(document, requestedIndexId, currentDocStamp).longValue()) {
 998       final FileContent newFc = new FileContent(vFile, content.getText(), vFile.getCharset());
 999
1000       if (dominantContentFile != null) {
1001         dominantContentFile.putUserData(PsiFileImpl.BUILDING_STUB, true);
1002         newFc.putUserData(PSI_FILE, dominantContentFile);
1003       }
1004
1005       if (content instanceof AuthenticContent) {
1006         newFc.putUserData(EDITOR_HIGHLIGHTER, document instanceof DocumentImpl
1007                                               ? ((DocumentImpl)document).getEditorHighlighterForCachesBuilding() : null);
1008       }
1009
1010       if (getInputFilter(requestedIndexId).acceptInput(vFile)) {
1011         newFc.putUserData(PROJECT, project);
1012         final int inputId = Math.abs(getFileId(vFile));
1013         getIndex(requestedIndexId).update(inputId, newFc);
1014       }
1015
1016       if (dominantContentFile != null) {
1017         dominantContentFile.putUserData(PsiFileImpl.BUILDING_STUB, null);
1018       }
1019     }
1020   }
1021
1022   public static final Key<PsiFile> PSI_FILE = new Key<PsiFile>("PSI for stubs");
1023   public static final Key<EditorHighlighter> EDITOR_HIGHLIGHTER = new Key<EditorHighlighter>("Editor");
1024   public static final Key<Project> PROJECT = new Key<Project>("Context project");
1025   public static final Key<VirtualFile> VIRTUAL_FILE = new Key<VirtualFile>("Context virtual file");
1026
1027   @Nullable
1028   private PsiFile findDominantPsiForDocument(final Document document, Project project) {
1029     if (myTransactionMap.containsKey(document)) {
1030       return myTransactionMap.get(document);
1031     }
1032
1033     return findLatestKnownPsiForUncomittedDocument(document, project);
1034   }
1035
1036   private final StorageGuard myStorageLock = new StorageGuard();
1037
1038   private StorageGuard.Holder setDataBufferingEnabled(final boolean enabled) {
1039     final StorageGuard.Holder holder = myStorageLock.enter(enabled);
1040     if (!enabled) {
1041       synchronized (myLastIndexedDocStamps) {
1042         myLastIndexedDocStamps.clear();
1043       }
1044     }
1045     for (ID<?, ?> indexId : myIndices.keySet()) {
1046       final MapReduceIndex index = (MapReduceIndex)getIndex(indexId);
1047       assert index != null;
1048       final IndexStorage indexStorage = index.getStorage();
1049       ((MemoryIndexStorage)indexStorage).setBufferingEnabled(enabled);
1050     }
1051     return holder;
1052   }
1053
1054   private void dropUnregisteredIndices() {
1055     final Set<String> indicesToDrop = readRegistsredIndexNames();
1056     for (ID<?, ?> key : myIndices.keySet()) {
1057       indicesToDrop.remove(key.toString());
1058     }
1059     for (String s : indicesToDrop) {
1060       FileUtil.delete(IndexInfrastructure.getIndexRootDir(ID.create(s)));
1061     }
1062   }
1063
1064   public void requestRebuild(ID<?, ?> indexId) {
1065     cleanupProcessedFlag();
1066     LOG.info("Rebuild requested for index " + indexId, new Throwable());
1067     myRebuildStatus.get(indexId).set(REQUIRES_REBUILD);
1068   }
1069
1070   private <K, V> UpdatableIndex<K, V, FileContent> getIndex(ID<K, V> indexId) {
1071     final Pair<UpdatableIndex<?, ?, FileContent>, InputFilter> pair = myIndices.get(indexId);
1072     //noinspection unchecked
1073     return pair != null? (UpdatableIndex<K,V, FileContent>)pair.getFirst() : null;
1074   }
1075
1076   private InputFilter getInputFilter(ID<?, ?> indexId) {
1077     final Pair<UpdatableIndex<?, ?, FileContent>, InputFilter> pair = myIndices.get(indexId);
1078     return pair != null? pair.getSecond() : null;
1079   }
1080
1081   public void indexFileContent(com.intellij.ide.startup.FileContent content) {
1082     final VirtualFile file = content.getVirtualFile();
1083     FileContent fc = null;
1084
1085     PsiFile psiFile = null;
1086
1087     final List<Runnable> tasks = new ArrayList<Runnable>();
1088     for (final ID<?, ?> indexId : myIndices.keySet()) {
1089       if (shouldIndexFile(file, indexId)) {
1090         if (fc == null) {
1091           byte[] currentBytes;
1092           try {
1093             currentBytes = content.getBytes();
1094           }
1095           catch (IOException e) {
1096             currentBytes = ArrayUtil.EMPTY_BYTE_ARRAY;
1097           }
1098           fc = new FileContent(file, currentBytes);
1099
1100           psiFile = content.getUserData(PSI_FILE);
1101           if (psiFile != null) {
1102             psiFile.putUserData(PsiFileImpl.BUILDING_STUB, true);
1103             fc.putUserData(PSI_FILE, psiFile);
1104           }
1105           Project project = content.getUserData(PROJECT);
1106           if (project == null) {
1107             project = ProjectUtil.guessProjectForFile(file);
1108           }
1109           fc.putUserData(PROJECT, project);
1110         }
1111
1112         final FileContent _fc = fc;
1113         tasks.add(new Runnable() {
1114           public void run() {
1115             try {
1116               updateSingleIndex(indexId, file, _fc);
1117             }
1118             catch (ProcessCanceledException ignored) {
1119             }
1120             catch (StorageException e) {
1121               requestRebuild(indexId);
1122               LOG.info(e);
1123             }
1124           }
1125         });
1126       }
1127     }
1128
1129     if (tasks.size() > 0) {
1130       if (Registry.get(USE_MULTITHREADED_INDEXING).asBoolean()) {
1131         final Job<Object> job = JobScheduler.getInstance().createJob("IndexJob", Job.DEFAULT_PRIORITY / 2);
1132         try {
1133           for (Runnable task : tasks) {
1134             job.addTask(task);
1135           }
1136           job.scheduleAndWaitForResults();
1137         }
1138         catch (Throwable throwable) {
1139           LOG.info(throwable);
1140         }
1141       }
1142       else {
1143         for (Runnable task : tasks) {
1144           task.run();
1145         }
1146       }
1147     }
1148
1149     if (psiFile != null) {
1150       psiFile.putUserData(PsiFileImpl.BUILDING_STUB, null);
1151     }
1152   }
1153
1154   private void updateSingleIndex(final ID<?, ?> indexId, final VirtualFile file, final FileContent currentFC)
1155     throws StorageException {
1156     if (myRebuildStatus.get(indexId).get() == REQUIRES_REBUILD) {
1157       return; // the index is scheduled for rebuild, no need to update
1158     }
1159
1160     final StorageGuard.Holder lock = setDataBufferingEnabled(false);
1161
1162     try {
1163       final int inputId = Math.abs(getFileId(file));
1164
1165       final UpdatableIndex<?, ?, FileContent> index = getIndex(indexId);
1166       assert index != null;
1167
1168       index.update(inputId, currentFC);
1169       ApplicationManager.getApplication().runReadAction(new Runnable() {
1170         public void run() {
1171           if (file.isValid()) {
1172             if (currentFC != null) {
1173               IndexingStamp.update(file, indexId, IndexInfrastructure.getIndexCreationStamp(indexId));
1174             }
1175             else {
1176               // mark the file as unindexed
1177               IndexingStamp.update(file, indexId, -1L);
1178             }
1179           }
1180         }
1181       });
1182     }
1183     finally {
1184       lock.leave();
1185     }
1186   }
1187
1188   public static int getFileId(final VirtualFile file) {
1189     if (file instanceof VirtualFileWithId) {
1190       return ((VirtualFileWithId)file).getId();
1191     }
1192
1193     throw new IllegalArgumentException("Virtual file doesn't support id: " + file + ", implementation class: " + file.getClass().getName());
1194   }
1195
1196   private boolean needsFileContentLoading(ID<?, ?> indexId) {
1197     return !myNotRequiringContentIndices.contains(indexId);
1198   }
1199
1200   private final class ChangedFilesUpdater extends VirtualFileAdapter implements BackgroundableCacheUpdater {
1201     private final Set<VirtualFile> myFilesToUpdate = new LinkedHashSet<VirtualFile>();
1202     private final JBReentrantReadWriteLock myLock = LockFactory.createReadWriteLock();
1203     private final JBLock r = myLock.readLock();
1204     private final JBLock w = myLock.writeLock();
1205
1206     private final ManagingFS myManagingFS = ManagingFS.getInstance();
1207     // No need to react on movement events since files stay valid, their ids don't change and all associated attributes remain intact.
1208
1209     public void fileCreated(final VirtualFileEvent event) {
1210       markDirty(event);
1211     }
1212
1213     public void fileDeleted(final VirtualFileEvent event) {
1214       w.lock();
1215       try {
1216         myFilesToUpdate.remove(event.getFile()); // no need to update it anymore
1217       }
1218       finally {
1219         w.unlock();
1220       }
1221     }
1222
1223     public void fileCopied(final VirtualFileCopyEvent event) {
1224       markDirty(event);
1225     }
1226
1227     public void beforeFileDeletion(final VirtualFileEvent event) {
1228       invalidateIndices(event.getFile(), false);
1229     }
1230
1231     public void beforeContentsChange(final VirtualFileEvent event) {
1232       invalidateIndices(event.getFile(), true);
1233     }
1234
1235     public void contentsChanged(final VirtualFileEvent event) {
1236       markDirty(event);
1237     }
1238
1239     public void beforePropertyChange(final VirtualFilePropertyEvent event) {
1240       if (event.getPropertyName().equals(VirtualFile.PROP_NAME)) {
1241         // indexes may depend on file name
1242         final VirtualFile file = event.getFile();
1243         if (!file.isDirectory()) {
1244           // name change may lead to filetype change so the file might become not indexable
1245           // in general case have to 'unindex' the file and index it again if needed after the name has been changed
1246           invalidateIndices(file, false);
1247         }
1248       }
1249     }
1250
1251     public void propertyChanged(final VirtualFilePropertyEvent event) {
1252       if (event.getPropertyName().equals(VirtualFile.PROP_NAME)) {
1253         // indexes may depend on file name
1254         if (!event.getFile().isDirectory()) {
1255           markDirty(event);
1256         }
1257       }
1258     }
1259
1260     private void markDirty(final VirtualFileEvent event) {
1261       cleanProcessedFlag(event.getFile());
1262       iterateIndexableFiles(event.getFile(), new Processor<VirtualFile>() {
1263         public boolean process(final VirtualFile file) {
1264           FileContent fileContent = null;
1265           // handle 'content-less' indices separately
1266           for (ID<?, ?> indexId : myNotRequiringContentIndices) {
1267             if (getInputFilter(indexId).acceptInput(file)) {
1268               try {
1269                 if (fileContent == null) {
1270                   fileContent = new FileContent(file);
1271                 }
1272                 updateSingleIndex(indexId, file, fileContent);
1273               }
1274               catch (StorageException e) {
1275                 LOG.info(e);
1276                 requestRebuild(indexId);
1277               }
1278             }
1279           }
1280           // For 'normal indices' schedule the file for update and stop iteration if at least one index accepts it
1281           if (!isTooLarge(file)) {
1282             for (ID<?, ?> indexId : myIndices.keySet()) {
1283               if (needsFileContentLoading(indexId) && getInputFilter(indexId).acceptInput(file)) {
1284                 w.lock();
1285                 try {
1286                   myFilesToUpdate.add(file);
1287                   break; // no need to iterate further, as the file is already marked
1288                 }
1289                 finally {
1290                   w.unlock();
1291                 }
1292               }
1293             }
1294           }
1295
1296           return true;
1297         }
1298       });
1299       IndexingStamp.flushCache();
1300     }
1301
1302     void invalidateIndices(final VirtualFile file, final boolean markForReindex) {
1303       if (isUnderConfigOrSystem(file)) {
1304         return;
1305       }
1306       if (file.isDirectory()) {
1307         if (isMock(file) || myManagingFS.wereChildrenAccessed(file)) {
1308           for (VirtualFile child : file.getChildren()) {
1309             invalidateIndices(child, markForReindex);
1310           }
1311         }
1312       }
1313       else {
1314         cleanProcessedFlag(file);
1315         IndexingStamp.flushCache();
1316         boolean indicesAffected = false;
1317
1318         final boolean isTooLarge = isTooLarge(file);
1319         for (final ID<?, ?> indexId : myIndices.keySet()) {
1320           try {
1321             if (myNotRequiringContentIndices.contains(indexId)) {
1322               if (shouldUpdateIndex(file, indexId)) {
1323                 updateSingleIndex(indexId, file, null);
1324               }
1325             }
1326             else { // the index requires file content
1327               if (!isTooLarge && shouldUpdateIndex(file, indexId)) {
1328                 indicesAffected = true;
1329                 if (markForReindex) {
1330                   // only mark the file as unindexed, reindex will be done lazily
1331                   ApplicationManager.getApplication().runReadAction(new Runnable() {
1332                     public void run() {
1333                       IndexingStamp.update(file, indexId, -1L);
1334                     }
1335                   });
1336                 }
1337                 else {
1338                   updateSingleIndex(indexId, file, null);
1339                 }
1340               }
1341             }
1342           }
1343           catch (StorageException e) {
1344             LOG.info(e);
1345             requestRebuild(indexId);
1346           }
1347         }
1348         IndexingStamp.flushCache();
1349         if (indicesAffected && markForReindex) {
1350           iterateIndexableFiles(file, new Processor<VirtualFile>() {
1351             public boolean process(final VirtualFile file) {
1352               w.lock();
1353               try {
1354                 myFilesToUpdate.add(file);
1355               }
1356               finally {
1357                 w.unlock();
1358               }
1359               return true;
1360             }
1361           });
1362         }
1363       }
1364     }
1365
1366     private void iterateIndexableFiles(final VirtualFile file, final Processor<VirtualFile> processor) {
1367       if (file.isDirectory()) {
1368         final ContentIterator iterator = new ContentIterator() {
1369           public boolean processFile(final VirtualFile fileOrDir) {
1370             if (!fileOrDir.isDirectory()) {
1371               processor.process(fileOrDir);
1372             }
1373             return true;
1374           }
1375         };
1376
1377         for (IndexableFileSet set : myIndexableSets) {
1378           set.iterateIndexableFilesIn(file, iterator);
1379         }
1380       }
1381       else {
1382         for (IndexableFileSet set : myIndexableSets) {
1383           if (set.isInSet(file)) {
1384             processor.process(file);
1385             break;
1386           }
1387         }
1388       }
1389     }
1390
1391     public boolean initiallyBackgrounded() {
1392       if (ApplicationManager.getApplication().isCommandLine() || ApplicationManager.getApplication().isUnitTestMode()) {
1393         return false;
1394       }
1395       return Registry.get(DumbServiceImpl.FILE_INDEX_BACKGROUND).asBoolean();
1396     }
1397
1398     public boolean canBeSentToBackground(Collection<VirtualFile> remaining) {
1399       return true;
1400     }
1401
1402     public void backgrounded(Collection<VirtualFile> remaining) {
1403       new BackgroundCacheUpdaterRunner(remaining).processFiles(this);
1404     }
1405
1406     public VirtualFile[] queryNeededFiles() {
1407       r.lock();
1408       try {
1409         if (myFilesToUpdate.isEmpty()) return VirtualFile.EMPTY_ARRAY;
1410         return myFilesToUpdate.toArray(new VirtualFile[myFilesToUpdate.size()]);
1411       }
1412       finally {
1413         r.unlock();
1414       }
1415     }
1416
1417     public void processFile(final com.intellij.ide.startup.FileContent fileContent) {
1418       processFileImpl(fileContent);
1419     }
1420
1421     private final Semaphore myForceUpdateSemaphore = new Semaphore();
1422
1423     public void forceUpdate() {
1424       final VirtualFile[] files = queryNeededFiles();
1425       if (files.length > 0) {
1426         myForceUpdateSemaphore.down();
1427         try {
1428           for (VirtualFile file: files) {
1429             processFileImpl(new com.intellij.ide.startup.FileContent(file));
1430           }
1431         }
1432         finally {
1433           myForceUpdateSemaphore.up();
1434           myForceUpdateSemaphore.waitFor(); // possibly wait until another thread completes indexing
1435         }
1436       }
1437     }
1438
1439     public void updatingDone() {
1440     }
1441
1442     public void canceled() {
1443     }
1444
1445     private void processFileImpl(final com.intellij.ide.startup.FileContent fileContent) {
1446       final VirtualFile file = fileContent.getVirtualFile();
1447       final boolean reallyRemoved;
1448       w.lock();
1449       try {
1450         reallyRemoved = myFilesToUpdate.remove(file);
1451       }
1452       finally {
1453         w.unlock();
1454       }
1455       if (reallyRemoved && file.isValid()) {
1456         indexFileContent(fileContent);
1457         IndexingStamp.flushCache();
1458       }
1459     }
1460   }
1461
1462   private class UnindexedFilesFinder implements CollectingContentIterator {
1463     private final List<VirtualFile> myFiles = new ArrayList<VirtualFile>();
1464     private final ProgressIndicator myProgressIndicator;
1465
1466     private UnindexedFilesFinder() {
1467       myProgressIndicator = ProgressManager.getInstance().getProgressIndicator();
1468     }
1469
1470     public List<VirtualFile> getFiles() {
1471       return myFiles;
1472     }
1473
1474     public boolean processFile(final VirtualFile file) {
1475       if (!file.isDirectory()) {
1476         if (file instanceof NewVirtualFile && ((NewVirtualFile)file).getFlag(ALREADY_PROCESSED)) {
1477           return true;
1478         }
1479
1480         if (file instanceof VirtualFileWithId) {
1481           boolean oldStuff = true;
1482           if (!isTooLarge(file)) {
1483             for (ID<?, ?> indexId : myIndices.keySet()) {
1484               try {
1485                 if (shouldIndexFile(file, indexId)) {
1486                   myFiles.add(file);
1487                   oldStuff = false;
1488                   break;
1489                 }
1490               }
1491               catch (RuntimeException e) {
1492                 final Throwable cause = e.getCause();
1493                 if (cause instanceof IOException || cause instanceof StorageException) {
1494                   LOG.info(e);
1495                   requestRebuild(indexId);
1496                 }
1497                 else {
1498                   throw e;
1499                 }
1500               }
1501             }
1502           }
1503           FileContent fileContent = null;
1504           for (ID<?, ?> indexId : myNotRequiringContentIndices) {
1505             if (shouldIndexFile(file, indexId)) {
1506               oldStuff = false;
1507               try {
1508                 if (fileContent == null) {
1509                   fileContent = new FileContent(file);
1510                 }
1511                 updateSingleIndex(indexId, file, fileContent);
1512               }
1513               catch (StorageException e) {
1514                 LOG.info(e);
1515                 requestRebuild(indexId);
1516               }
1517             }
1518           }
1519           IndexingStamp.flushCache();
1520
1521           if (oldStuff && file instanceof NewVirtualFile) {
1522             ((NewVirtualFile)file).setFlag(ALREADY_PROCESSED, true);
1523           }
1524         }
1525       }
1526       else {
1527         if (myProgressIndicator != null) {
1528           myProgressIndicator.setText("Scanning files to index");
1529           myProgressIndicator.setText2(file.getPresentableUrl());
1530         }
1531       }
1532       return true;
1533     }
1534   }
1535
1536   private boolean shouldUpdateIndex(final VirtualFile file, final ID<?, ?> indexId) {
1537     return getInputFilter(indexId).acceptInput(file) &&
1538            (isMock(file) || IndexingStamp.isFileIndexed(file, indexId, IndexInfrastructure.getIndexCreationStamp(indexId)));
1539   }
1540
1541   private boolean shouldIndexFile(final VirtualFile file, final ID<?, ?> indexId) {
1542     return getInputFilter(indexId).acceptInput(file) &&
1543            (isMock(file) || !IndexingStamp.isFileIndexed(file, indexId, IndexInfrastructure.getIndexCreationStamp(indexId)));
1544   }
1545
1546   private boolean isUnderConfigOrSystem(VirtualFile file) {
1547     final String filePath = file.getPath();
1548     if (myConfigPath != null && FileUtil.startsWith(filePath, myConfigPath)) {
1549       return true;
1550     }
1551     if (mySystemPath != null && FileUtil.startsWith(filePath, mySystemPath)) {
1552       return true;
1553     }
1554     return false;
1555   }
1556
1557   private static boolean isMock(final VirtualFile file) {
1558     return !(file instanceof NewVirtualFile);
1559   }
1560
1561   private boolean isTooLarge(VirtualFile file) {
1562     if (SingleRootFileViewProvider.isTooLarge(file)) {
1563       final FileType type = FileTypeManager.getInstance().getFileTypeByFile(file);
1564       return !myNoLimitCheckTypes.contains(type);
1565     }
1566     return false;
1567   }
1568
1569   public CollectingContentIterator createContentIterator() {
1570     return new UnindexedFilesFinder();
1571   }
1572
1573   public void registerIndexableSet(IndexableFileSet set) {
1574     myIndexableSets.add(set);
1575   }
1576
1577   public void removeIndexableSet(IndexableFileSet set) {
1578     myChangedFilesUpdater.forceUpdate();
1579     myIndexableSets.remove(set);
1580   }
1581
1582   @Nullable
1583   private static PsiFile findLatestKnownPsiForUncomittedDocument(Document doc, Project project) {
1584     return PsiDocumentManager.getInstance(project).getCachedPsiFile(doc);
1585   }
1586
1587   private static class IndexableFilesFilter implements InputFilter {
1588     private final InputFilter myDelegate;
1589
1590     private IndexableFilesFilter(InputFilter delegate) {
1591       myDelegate = delegate;
1592     }
1593
1594     public boolean acceptInput(final VirtualFile file) {
1595       return file instanceof VirtualFileWithId && myDelegate.acceptInput(file);
1596     }
1597   }
1598
1599   private static void cleanupProcessedFlag() {
1600     final VirtualFile[] roots = ManagingFS.getInstance().getRoots();
1601     for (VirtualFile root : roots) {
1602       cleanProcessedFlag(root);
1603     }
1604   }
1605
1606   private static void cleanProcessedFlag(final VirtualFile file) {
1607     if (!(file instanceof NewVirtualFile)) return;
1608
1609     final NewVirtualFile nvf = (NewVirtualFile)file;
1610     if (file.isDirectory()) {
1611       for (VirtualFile child : nvf.getCachedChildren()) {
1612         cleanProcessedFlag(child);
1613       }
1614     }
1615     else {
1616 /*      nvf.clearCachedFileType(); */
1617       nvf.setFlag(ALREADY_PROCESSED, false);
1618     }
1619   }
1620
1621   private static class StorageGuard {
1622     private int myHolds = 0;
1623
1624     public interface Holder {
1625       void leave();
1626     }
1627
1628     private final Holder myTrueHolder = new Holder() {
1629       public void leave() {
1630         StorageGuard.this.leave(true);
1631       }
1632     };
1633     private final Holder myFalseHolder = new Holder() {
1634       public void leave() {
1635         StorageGuard.this.leave(false);
1636       }
1637     };
1638
1639     public synchronized Holder enter(boolean mode) {
1640       if (mode) {
1641         while (myHolds < 0) {
1642           try {
1643             wait();
1644           }
1645           catch (InterruptedException ignored) {
1646           }
1647         }
1648         myHolds++;
1649         return myTrueHolder;
1650       }
1651       else {
1652         while (myHolds > 0) {
1653           try {
1654             wait();
1655           }
1656           catch (InterruptedException ignored) {
1657           }
1658         }
1659         myHolds--;
1660         return myFalseHolder;
1661       }
1662     }
1663
1664     private synchronized void leave(boolean mode) {
1665       myHolds += (mode? -1 : 1);
1666       if (myHolds == 0) {
1667         notifyAll();
1668       }
1669     }
1670
1671   }
1672 }