lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java

   1 /*
   2  * @author max
   3  */
   4 package com.intellij.psi.stubs;
   5
   6 import com.intellij.lang.Language;
   7 import com.intellij.lang.LanguageParserDefinitions;
   8 import com.intellij.lang.ParserDefinition;
   9 import com.intellij.openapi.application.Application;
  10 import com.intellij.openapi.application.ApplicationManager;
  11 import com.intellij.openapi.diagnostic.Logger;
  12 import com.intellij.openapi.fileTypes.FileType;
  13 import com.intellij.openapi.fileTypes.FileTypeManager;
  14 import com.intellij.openapi.fileTypes.LanguageFileType;
  15 import com.intellij.openapi.progress.ProgressIndicator;
  16 import com.intellij.openapi.progress.Task;
  17 import com.intellij.openapi.project.Project;
  18 import com.intellij.openapi.vfs.VirtualFile;
  19 import com.intellij.psi.PsiFile;
  20 import com.intellij.psi.tree.IFileElementType;
  21 import com.intellij.psi.tree.IStubFileElementType;
  22 import com.intellij.util.indexing.*;
  23 import com.intellij.util.io.DataExternalizer;
  24 import com.intellij.util.io.KeyDescriptor;
  25 import gnu.trove.TIntArrayList;
  26 import org.jetbrains.annotations.NotNull;
  27 import org.jetbrains.annotations.Nullable;
  28
  29 import javax.swing.*;
  30 import java.io.ByteArrayOutputStream;
  31 import java.io.DataInput;
  32 import java.io.DataOutput;
  33 import java.io.IOException;
  34 import java.util.*;
  35 import java.util.concurrent.locks.Lock;
  36
  37 public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtension<Integer, SerializedStubTree, FileContent> {
  38   private static final Logger LOG = Logger.getInstance("#com.intellij.psi.stubs.StubUpdatingIndex");
  39
  40   public static final ID<Integer, SerializedStubTree> INDEX_ID = ID.create("Stubs");
  41   private static final int VERSION = 17;
  42   private static final DataExternalizer<SerializedStubTree> KEY_EXTERNALIZER = new DataExternalizer<SerializedStubTree>() {
  43     public void save(final DataOutput out, final SerializedStubTree v) throws IOException {
  44       byte[] value = v.getBytes();
  45       out.writeInt(value.length);
  46       out.write(value);
  47     }
  48
  49     public SerializedStubTree read(final DataInput in) throws IOException {
  50       int len = in.readInt();
  51       byte[] result = new byte[len];
  52       in.readFully(result);
  53       return new SerializedStubTree(result);
  54     }
  55   };
  56
  57   private static final FileBasedIndex.InputFilter INPUT_FILTER = new FileBasedIndex.InputFilter() {
  58     public boolean acceptInput(final VirtualFile file) {
  59       return canHaveStub(file);
  60     }
  61   };
  62
  63   public static boolean canHaveStub(VirtualFile file) {
  64     final FileType fileType = file.getFileType();
  65     if (fileType instanceof LanguageFileType) {
  66       Language l = ((LanguageFileType)fileType).getLanguage();
  67       ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(l);
  68       if (parserDefinition == null) return false;
  69
  70       final IFileElementType filetype = parserDefinition.getFileNodeType();
  71       return filetype instanceof IStubFileElementType &&
  72              (((IStubFileElementType)filetype).shouldBuildStubFor(file) ||
  73               IndexingStamp.isFileIndexed(file, INDEX_ID, IndexInfrastructure.getIndexCreationStamp(INDEX_ID)));
  74     }
  75     else if (fileType.isBinary()) {
  76       final BinaryFileStubBuilder builder = BinaryFileStubBuilders.INSTANCE.forFileType(fileType);
  77       return builder != null && builder.acceptsFile(file);
  78     }
  79
  80     return false;
  81   }
  82
  83   private static final KeyDescriptor<Integer> DATA_DESCRIPTOR = new KeyDescriptor<Integer>() {
  84     public int getHashCode(final Integer value) {
  85       return value.hashCode();
  86     }
  87
  88     public boolean isEqual(final Integer val1, final Integer val2) {
  89       return val1.equals(val2);
  90     }
  91
  92     public void save(final DataOutput out, final Integer value) throws IOException {
  93       out.writeInt(value.intValue());
  94     }
  95
  96     public Integer read(final DataInput in) throws IOException {
  97       return in.readInt();
  98     }
  99   };
 100
 101   public ID<Integer, SerializedStubTree> getName() {
 102     return INDEX_ID;
 103   }
 104
 105   public int getCacheSize() {
 106     return 5; // no need to cache many serialized trees
 107   }
 108
 109   public DataIndexer<Integer, SerializedStubTree, FileContent> getIndexer() {
 110     return new DataIndexer<Integer, SerializedStubTree, FileContent>() {
 111       @NotNull
 112       public Map<Integer, SerializedStubTree> map(final FileContent inputData) {
 113         final Map<Integer, SerializedStubTree> result = new HashMap<Integer, SerializedStubTree>();
 114
 115         ApplicationManager.getApplication().runReadAction(new Runnable() {
 116           public void run() {
 117             final StubElement rootStub = buildStubTree(inputData);
 118             if (rootStub == null) return;
 119
 120             final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
 121             SerializationManager.getInstance().serialize(rootStub, bytes);
 122
 123             final int key = Math.abs(FileBasedIndex.getFileId(inputData.getFile()));
 124             result.put(key, new SerializedStubTree(bytes.toByteArray()));
 125           }
 126         });
 127
 128         return result;
 129       }
 130     };
 131   }
 132
 133   @Nullable
 134   static StubElement buildStubTree(final FileContent inputData) {
 135     final FileType fileType = inputData.getFileType();
 136
 137     if (fileType.isBinary()) {
 138       final BinaryFileStubBuilder builder = BinaryFileStubBuilders.INSTANCE.forFileType(fileType);
 139       assert builder != null;
 140
 141       return builder.buildStubTree(inputData.getFile(), inputData.getContent(), inputData.getProject());
 142     }
 143
 144     final LanguageFileType filetype = (LanguageFileType)fileType;
 145     Language l = filetype.getLanguage();
 146     final IFileElementType type = LanguageParserDefinitions.INSTANCE.forLanguage(l).getFileNodeType();
 147
 148     PsiFile psi = inputData.getPsiFile();
 149
 150     return ((IStubFileElementType)type).getBuilder().buildStubTree(psi);
 151   }
 152
 153   public KeyDescriptor<Integer> getKeyDescriptor() {
 154     return DATA_DESCRIPTOR;
 155   }
 156
 157   public DataExternalizer<SerializedStubTree> getValueExternalizer() {
 158     return KEY_EXTERNALIZER;
 159   }
 160
 161   public FileBasedIndex.InputFilter getInputFilter() {
 162     return INPUT_FILTER;
 163   }
 164
 165   public boolean dependsOnFileContent() {
 166     return true;
 167   }
 168
 169   public int getVersion() {
 170     return getCumulativeVersion();
 171   }
 172
 173   private static int getCumulativeVersion() {
 174     int version = VERSION;
 175     for (final FileType fileType : FileTypeManager.getInstance().getRegisteredFileTypes()) {
 176       if (fileType instanceof LanguageFileType) {
 177         Language l = ((LanguageFileType)fileType).getLanguage();
 178         ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(l);
 179         if (parserDefinition != null) {
 180           final IFileElementType type = parserDefinition.getFileNodeType();
 181           if (type instanceof IStubFileElementType) {
 182             version += ((IStubFileElementType)type).getStubVersion();
 183           }
 184         }
 185       }
 186       else if (fileType.isBinary()) {
 187         final BinaryFileStubBuilder builder = BinaryFileStubBuilders.INSTANCE.forFileType(fileType);
 188         if (builder != null ) {
 189           version += builder.getStubVersion();
 190         }
 191       }
 192     }
 193     return version;
 194   }
 195
 196   public UpdatableIndex<Integer, SerializedStubTree, FileContent> createIndexImplementation(final ID<Integer, SerializedStubTree> indexId,
 197                                                                                             final FileBasedIndex owner, IndexStorage<Integer, SerializedStubTree> storage) {
 198     if (storage instanceof MemoryIndexStorage) {
 199       final MemoryIndexStorage<Integer, SerializedStubTree> memStorage = (MemoryIndexStorage<Integer, SerializedStubTree>)storage;
 200       memStorage.addBufferingStateListsner(new MemoryIndexStorage.BufferingStateListener() {
 201         public void bufferingStateChanged(final boolean newState) {
 202           ((StubIndexImpl)StubIndexImpl.getInstance()).setDataBufferingEnabled(newState);
 203         }
 204       });
 205     }
 206     return new MyIndex(indexId, owner, storage, getIndexer());
 207   }
 208
 209   /**
 210    * Schedules asynchronous rebuild
 211    * @param finishCallback
 212    */
 213   public static void scheduleStubIndicesRebuild(@Nullable final Runnable finishCallback, @NotNull final Project project) {
 214     final Runnable rebuildRunnable = new Runnable() {
 215       public void run() {
 216         final StubIndexImpl stubIndex = (StubIndexImpl)StubIndexImpl.getInstance();
 217         final Collection<StubIndexKey> allIndexKeys = stubIndex.getAllStubIndexKeys();
 218         try {
 219           for (StubIndexKey key : allIndexKeys) {
 220             stubIndex.getWriteLock(key).lock();
 221           }
 222           stubIndex.clearAllIndices();
 223           final Map<StubIndexKey, Map<Object, TIntArrayList>> empty = Collections.emptyMap();
 224           FileBasedIndex.getInstance().processAllValues(INDEX_ID, new FileBasedIndex.AllValuesProcessor<SerializedStubTree>() {
 225             public void process(final int inputId, final SerializedStubTree value) {
 226               final Map<StubIndexKey, Map<Object, TIntArrayList>> stubTree = new StubTree((PsiFileStub)value.getStub()).indexStubTree();
 227               updateStubIndices(getAffectedIndices(empty, stubTree), inputId, empty, stubTree);
 228             }
 229           }, project);
 230         }
 231         finally {
 232           for (StubIndexKey key : allIndexKeys) {
 233             stubIndex.getWriteLock(key).unlock();
 234           }
 235           if (finishCallback != null) {
 236             finishCallback.run();
 237           }
 238         }
 239       }
 240     };
 241
 242     final Application application = ApplicationManager.getApplication();
 243     if (application.isUnitTestMode()) {
 244       rebuildRunnable.run();
 245     }
 246     else {
 247       SwingUtilities.invokeLater(new Runnable() {
 248         public void run() {
 249           new Task.Modal(null, "Updating index", false) {
 250             public void run(@NotNull final ProgressIndicator indicator) {
 251               rebuildRunnable.run();
 252             }
 253           }.queue();
 254         }
 255       });
 256     }
 257   }
 258
 259   private static void updateStubIndices(final Collection<StubIndexKey> indexKeys, final int inputId, final Map<StubIndexKey, Map<Object, TIntArrayList>> oldStubTree,
 260                                         final Map<StubIndexKey, Map<Object, TIntArrayList>> newStubTree) {
 261     final StubIndexImpl stubIndex = (StubIndexImpl)StubIndex.getInstance();
 262     for (StubIndexKey key : indexKeys) {
 263       final Map<Object, TIntArrayList> oldMap = oldStubTree.get(key);
 264       final Map<Object, TIntArrayList> newMap = newStubTree.get(key);
 265
 266       final Map<Object, TIntArrayList> _oldMap = oldMap != null ? oldMap : Collections.<Object, TIntArrayList>emptyMap();
 267       final Map<Object, TIntArrayList> _newMap = newMap != null ? newMap : Collections.<Object, TIntArrayList>emptyMap();
 268
 269       stubIndex.updateIndex(key, inputId, _oldMap, _newMap);
 270     }
 271   }
 272
 273   private static Collection<StubIndexKey> getAffectedIndices(final Map<StubIndexKey, Map<Object, TIntArrayList>> oldStubTree, final Map<StubIndexKey, Map<Object, TIntArrayList>> newStubTree) {
 274     Set<StubIndexKey> allIndices = new HashSet<StubIndexKey>();
 275     allIndices.addAll(oldStubTree.keySet());
 276     allIndices.addAll(newStubTree.keySet());
 277     return allIndices;
 278   }
 279
 280   private static class MyIndex extends MapReduceIndex<Integer, SerializedStubTree, FileContent> {
 281     public MyIndex(final ID<Integer, SerializedStubTree> indexId, final FileBasedIndex owner, final IndexStorage<Integer, SerializedStubTree> storage,
 282                    final DataIndexer<Integer, SerializedStubTree, FileContent> indexer) {
 283       super(indexId, indexer, storage);
 284       try {
 285         checkNameStorage();
 286       }
 287       catch (StorageException e) {
 288         LOG.info(e);
 289         owner.requestRebuild(INDEX_ID);
 290       }
 291     }
 292
 293     protected void updateWithMap(final int inputId, final Map<Integer, SerializedStubTree> oldData, final Map<Integer, SerializedStubTree> newData)
 294         throws StorageException {
 295
 296       checkNameStorage();
 297       final Map<StubIndexKey, Map<Object, TIntArrayList>> oldStubTree = getStubTree(oldData);
 298       final Map<StubIndexKey, Map<Object, TIntArrayList>> newStubTree = getStubTree(newData);
 299
 300       final Collection<StubIndexKey> affectedIndices = getAffectedIndices(oldStubTree, newStubTree);
 301       final StubIndexImpl stubIndex = (StubIndexImpl)StubIndex.getInstance();
 302       try {
 303         // first write-lock affected stub indices to avoid deadlocks
 304         for (StubIndexKey key : affectedIndices) {
 305           stubIndex.getWriteLock(key).lock();
 306         }
 307
 308         try {
 309           getWriteLock().lock();
 310           super.updateWithMap(inputId, oldData, newData);
 311           updateStubIndices(affectedIndices, inputId, oldStubTree, newStubTree);
 312         }
 313         finally {
 314           getWriteLock().unlock();
 315         }
 316
 317       }
 318       finally {
 319         for (StubIndexKey key : affectedIndices) {
 320           stubIndex.getWriteLock(key).unlock();
 321         }
 322       }
 323     }
 324
 325     private static void checkNameStorage() throws StorageException {
 326       final SerializationManager serializationManager = SerializationManager.getInstance();
 327       if (serializationManager.isNameStorageCorrupted()) {
 328         serializationManager.repairNameStorage();
 329         //noinspection ThrowFromFinallyBlock
 330         throw new StorageException("NameStorage for stubs serialization has been corrupted");
 331       }
 332     }
 333
 334     private static Map<StubIndexKey, Map<Object, TIntArrayList>> getStubTree(final Map<Integer, SerializedStubTree> data) {
 335       final Map<StubIndexKey, Map<Object, TIntArrayList>> stubTree;
 336       if (!data.isEmpty()) {
 337         final SerializedStubTree stub = data.values().iterator().next();
 338         stubTree = new StubTree((PsiFileStub)stub.getStub()).indexStubTree();
 339       }
 340       else {
 341         stubTree = Collections.emptyMap();
 342       }
 343       return stubTree;
 344     }
 345
 346     protected Map<Integer, SerializedStubTree> mapOld(final FileContent inputData) throws StorageException {
 347       checkNameStorage();
 348       if (inputData == null) {
 349         return Collections.emptyMap();
 350       }
 351       final int key = Math.abs(FileBasedIndex.getFileId(inputData.getFile()));
 352
 353       final Map<Integer, SerializedStubTree> result = new HashMap<Integer, SerializedStubTree>();
 354       final Lock lock = getReadLock();
 355       try {
 356         lock.lock();
 357         final ValueContainer<SerializedStubTree> valueContainer = getData(key);
 358         if (valueContainer.size() != 1) {
 359           LOG.assertTrue(valueContainer.size() == 0);
 360           return result;
 361         }
 362
 363         result.put(key, valueContainer.getValueIterator().next());
 364       }
 365       finally {
 366         lock.unlock();
 367       }
 368
 369       return result;
 370     }
 371
 372     public void clear() throws StorageException {
 373       final StubIndexImpl stubIndex = (StubIndexImpl)StubIndex.getInstance();
 374       try {
 375         for (StubIndexKey key : stubIndex.getAllStubIndexKeys()) {
 376           stubIndex.getWriteLock(key).lock();
 377         }
 378         getWriteLock().lock();
 379         stubIndex.clearAllIndices();
 380         super.clear();
 381       }
 382       finally {
 383         getWriteLock().unlock();
 384         for (StubIndexKey key : stubIndex.getAllStubIndexKeys()) {
 385           stubIndex.getWriteLock(key).unlock();
 386         }
 387       }
 388     }
 389
 390     public void dispose() {
 391       try {
 392         super.dispose();
 393       }
 394       finally {
 395         try {
 396           ((StubIndexImpl)StubIndex.getInstance()).dispose();
 397         }
 398         finally {
 399           ((SerializationManagerImpl)SerializationManager.getInstance()).disposeComponent();
 400         }
 401       }
 402     }
 403   }
 404 }