platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java

   1 /*
   2  * @author max
   3  */
   4 package com.intellij.psi.stubs;
   5
   6 import com.intellij.lang.ASTNode;
   7 import com.intellij.openapi.application.ApplicationManager;
   8 import com.intellij.openapi.application.ModalityState;
   9 import com.intellij.openapi.components.*;
  10 import com.intellij.openapi.diagnostic.Logger;
  11 import com.intellij.openapi.extensions.Extensions;
  12 import com.intellij.openapi.progress.ProcessCanceledException;
  13 import com.intellij.openapi.project.Project;
  14 import com.intellij.openapi.util.io.FileUtil;
  15 import com.intellij.openapi.vfs.VirtualFile;
  16 import com.intellij.openapi.vfs.newvfs.ManagingFS;
  17 import com.intellij.openapi.vfs.newvfs.persistent.PersistentFS;
  18 import com.intellij.psi.PsiElement;
  19 import com.intellij.psi.PsiFile;
  20 import com.intellij.psi.PsiManager;
  21 import com.intellij.psi.PsiPlainTextFile;
  22 import com.intellij.psi.impl.source.PsiFileImpl;
  23 import com.intellij.psi.impl.source.PsiFileWithStubSupport;
  24 import com.intellij.psi.search.GlobalSearchScope;
  25 import com.intellij.psi.tree.IElementType;
  26 import com.intellij.psi.tree.IStubFileElementType;
  27 import com.intellij.util.indexing.*;
  28 import com.intellij.util.io.DataExternalizer;
  29 import com.intellij.util.io.DataInputOutputUtil;
  30 import gnu.trove.TIntArrayList;
  31 import gnu.trove.TObjectIntHashMap;
  32 import org.jetbrains.annotations.NotNull;
  33
  34 import java.io.DataInput;
  35 import java.io.DataOutput;
  36 import java.io.File;
  37 import java.io.IOException;
  38 import java.util.*;
  39 import java.util.concurrent.atomic.AtomicInteger;
  40 import java.util.concurrent.locks.Lock;
  41
  42 @State(
  43   name = "FileBasedIndex",
  44   roamingType = RoamingType.DISABLED,
  45   storages = {
  46   @Storage(
  47     id = "stubIndex",
  48     file = "$APP_CONFIG$/stubIndex.xml")
  49     }
  50 )
  51 public class StubIndexImpl extends StubIndex implements ApplicationComponent, PersistentStateComponent<StubIndexState> {
  52   private static final Logger LOG = Logger.getInstance("#com.intellij.psi.stubs.StubIndexImpl");
  53   private final Map<StubIndexKey<?,?>, MyIndex<?>> myIndices = new HashMap<StubIndexKey<?,?>, MyIndex<?>>();
  54   private final TObjectIntHashMap<ID<?, ?>> myIndexIdToVersionMap = new TObjectIntHashMap<ID<?, ?>>();
  55
  56   public static final int OK = 1;
  57   public static final int NEED_REBUILD = 2;
  58   public static final int REBUILD_IN_PROGRESS = 3;
  59   private final AtomicInteger myRebuildStatus = new AtomicInteger(OK);
  60
  61   private StubIndexState myPreviouslyRegistered;
  62
  63   public StubIndexImpl() throws IOException {
  64     final StubIndexExtension<?, ?>[] extensions = Extensions.getExtensions(StubIndexExtension.EP_NAME);
  65     boolean needRebuild = false;
  66     for (StubIndexExtension extension : extensions) {
  67       //noinspection unchecked
  68       needRebuild |= registerIndexer(extension);
  69     }
  70     if (needRebuild) {
  71       myRebuildStatus.set(NEED_REBUILD);
  72     }
  73     dropUnregisteredIndices();
  74   }
  75
  76   private <K> boolean registerIndexer(final StubIndexExtension<K, ?> extension) throws IOException {
  77     final StubIndexKey<K, ?> indexKey = extension.getKey();
  78     final int version = extension.getVersion();
  79     myIndexIdToVersionMap.put(indexKey, version);
  80     final File versionFile = IndexInfrastructure.getVersionFile(indexKey);
  81     final boolean versionFileExisted = versionFile.exists();
  82     final File indexRootDir = IndexInfrastructure.getIndexRootDir(indexKey);
  83     boolean needRebuild = false;
  84     if (IndexInfrastructure.versionDiffers(versionFile, version)) {
  85       final String[] children = indexRootDir.list();
  86       // rebuild only if there exists what to rebuild
  87       needRebuild = versionFileExisted || children != null && children.length > 0;
  88       if (needRebuild) {
  89         LOG.info("Version has changed for stub index " + extension.getKey() + ". The index will be rebuilt.");
  90       }
  91       FileUtil.delete(indexRootDir);
  92       IndexInfrastructure.rewriteVersion(versionFile, version);
  93     }
  94
  95     for (int attempt = 0; attempt < 2; attempt++) {
  96       try {
  97         final MapIndexStorage<K, TIntArrayList> storage = new MapIndexStorage<K, TIntArrayList>(IndexInfrastructure.getStorageFile(indexKey), extension.getKeyDescriptor(), new StubIdExternalizer(), 2 * 1024);
  98         final MemoryIndexStorage<K, TIntArrayList> memStorage = new MemoryIndexStorage<K, TIntArrayList>(storage);
  99         myIndices.put(indexKey, new MyIndex<K>(memStorage));
 100         break;
 101       }
 102       catch (IOException e) {
 103         LOG.info(e);
 104         needRebuild = true;
 105         FileUtil.delete(indexRootDir);
 106         IndexInfrastructure.rewriteVersion(versionFile, version);
 107       }
 108     }
 109     return needRebuild;
 110   }
 111
 112   private static class StubIdExternalizer implements DataExternalizer<TIntArrayList> {
 113     public void save(final DataOutput out, final TIntArrayList value) throws IOException {
 114       int size = value.size();
 115       if (size == 0) {
 116         DataInputOutputUtil.writeSINT(out, Integer.MAX_VALUE);
 117       }
 118       else if (size == 1) {
 119         DataInputOutputUtil.writeSINT(out, -value.get(0));
 120       }
 121       else {
 122         DataInputOutputUtil.writeSINT(out, size);
 123         for (int i = 0; i < size; i++) {
 124           DataInputOutputUtil.writeINT(out, value.get(i));
 125         }
 126       }
 127     }
 128
 129     public TIntArrayList read(final DataInput in) throws IOException {
 130       int size = DataInputOutputUtil.readSINT(in);
 131       if (size == Integer.MAX_VALUE) {
 132         return new TIntArrayList();
 133       }
 134       else if (size <= 0) {
 135         TIntArrayList result = new TIntArrayList(1);
 136         result.add(-size);
 137         return result;
 138       }
 139       else {
 140         TIntArrayList result = new TIntArrayList(size);
 141         for (int i = 0; i < size; i++) {
 142           result.add(DataInputOutputUtil.readINT(in));
 143         }
 144         return result;
 145       }
 146     }
 147   }
 148
 149   public <Key, Psi extends PsiElement> Collection<Psi> get(@NotNull final StubIndexKey<Key, Psi> indexKey, @NotNull final Key key, final Project project,
 150                                                            final GlobalSearchScope scope) {
 151     checkRebuild(project);
 152
 153     FileBasedIndex.getInstance().ensureUpToDate(StubUpdatingIndex.INDEX_ID, project);
 154
 155     final PersistentFS fs = (PersistentFS)ManagingFS.getInstance();
 156     final PsiManager psiManager = PsiManager.getInstance(project);
 157
 158     final List<Psi> result = new ArrayList<Psi>();
 159     final MyIndex<Key> index = (MyIndex<Key>)myIndices.get(indexKey);
 160
 161     try {
 162       try {
 163         // disable up-to-date check to avoid locks on attempt to acquire index write lock while holding at the same time the readLock for this index
 164         FileBasedIndex.getInstance().disableUpToDateCheckForCurrentThread();
 165         index.getReadLock().lock();
 166         final ValueContainer<TIntArrayList> container = index.getData(key);
 167
 168         container.forEach(new ValueContainer.ContainerAction<TIntArrayList>() {
 169           public void perform(final int id, final TIntArrayList value) {
 170             final VirtualFile file = IndexInfrastructure.findFileById(fs, id);
 171             if (file != null && (scope == null || scope.contains(file))) {
 172               StubTree stubTree = null;
 173
 174               final PsiFile _psifile = psiManager.findFile(file);
 175               PsiFileWithStubSupport psiFile = null;
 176
 177               if (_psifile != null && !(_psifile instanceof PsiPlainTextFile)) {
 178                 if (_psifile instanceof PsiFileWithStubSupport) {
 179                   psiFile = (PsiFileWithStubSupport)_psifile;
 180                   stubTree = psiFile.getStubTree();
 181                   if (stubTree == null && psiFile instanceof PsiFileImpl) {
 182                     stubTree = ((PsiFileImpl)psiFile).calcStubTree();
 183                   }
 184                 }
 185               }
 186
 187               if (stubTree != null || psiFile != null) {
 188                 if (stubTree == null) {
 189                   stubTree = StubTree.readFromVFile(project, file);
 190                   if (stubTree != null) {
 191                     final List<StubElement<?>> plained = stubTree.getPlainList();
 192                     for (int i = 0; i < value.size(); i++) {
 193                       final StubElement<?> stub = plained.get(value.get(i));
 194                       final ASTNode tree = psiFile.findTreeForStub(stubTree, stub);
 195
 196                       if (tree != null) {
 197                         if (tree.getElementType() == stubType(stub)) {
 198                           result.add((Psi)tree.getPsi());
 199                         }
 200                         else {
 201                           String persistedStubTree = ((PsiFileStubImpl)stubTree.getRoot()).printTree();
 202
 203                           String stubTreeJustBuilt =
 204                               ((PsiFileStubImpl)((IStubFileElementType)((PsiFileImpl)psiFile).getContentElementType()).getBuilder()
 205                                   .buildStubTree(psiFile)).printTree();
 206
 207                           StringBuilder builder = new StringBuilder();
 208                           builder.append("Oops\n");
 209
 210
 211                           builder.append("Recorded stub:-----------------------------------\n");
 212                           builder.append(persistedStubTree);
 213                           builder.append("\nAST built stub: ------------------------------------\n");
 214                           builder.append(stubTreeJustBuilt);
 215                           builder.append("\n");
 216                           LOG.info(builder.toString());
 217
 218                           // requestReindex() may want to acquire write lock (for indices not requiring content loading)
 219                           // thus, because here we are under read lock, need to use invoke later
 220                           ApplicationManager.getApplication().invokeLater(new Runnable() {
 221                             public void run() {
 222                               FileBasedIndex.getInstance().requestReindex(file);
 223                             }
 224                           }, ModalityState.NON_MODAL);
 225                         }
 226                       }
 227                     }
 228                   }
 229                 }
 230                 else {
 231                   final List<StubElement<?>> plained = stubTree.getPlainList();
 232                   for (int i = 0; i < value.size(); i++) {
 233                     result.add((Psi)plained.get(value.get(i)).getPsi());
 234                   }
 235                 }
 236               }
 237             }
 238           }
 239         });
 240       }
 241       finally {
 242         index.getReadLock().unlock();
 243         FileBasedIndex.getInstance().enableUpToDateCheckForCurrentThread();
 244       }
 245     }
 246     catch (StorageException e) {
 247       forceRebuild(e, project);
 248     }
 249     catch (RuntimeException e) {
 250       final Throwable cause = e.getCause();
 251       if (cause instanceof IOException || cause instanceof StorageException) {
 252         forceRebuild(e, project);
 253       }
 254       else {
 255         throw e;
 256       }
 257     }
 258
 259     return result;
 260   }
 261
 262   private static IElementType stubType(final StubElement<?> stub) {
 263     if (stub instanceof PsiFileStub) {
 264       return ((PsiFileStub)stub).getType();
 265     }
 266
 267     return stub.getStubType();
 268   }
 269
 270   private void forceRebuild(Throwable e, Project project) {
 271     LOG.info(e);
 272     myRebuildStatus.set(NEED_REBUILD);
 273     checkRebuild(project);
 274   }
 275
 276   private void checkRebuild(@NotNull Project project) {
 277     if (myRebuildStatus.compareAndSet(NEED_REBUILD, REBUILD_IN_PROGRESS)) {
 278       StubUpdatingIndex.scheduleStubIndicesRebuild(new Runnable() {
 279         public void run() {
 280           myRebuildStatus.compareAndSet(REBUILD_IN_PROGRESS, OK);
 281         }
 282       }, project);
 283     }
 284     if (myRebuildStatus.get() == REBUILD_IN_PROGRESS) {
 285       throw new ProcessCanceledException();
 286     }
 287   }
 288
 289   public <K> Collection<K> getAllKeys(final StubIndexKey<K, ?> indexKey, @NotNull Project project) {
 290     checkRebuild(project);
 291     FileBasedIndex.getInstance().ensureUpToDate(StubUpdatingIndex.INDEX_ID, project);
 292
 293     final MyIndex<K> index = (MyIndex<K>)myIndices.get(indexKey);
 294     try {
 295       return index.getAllKeys();
 296     }
 297     catch (StorageException e) {
 298       forceRebuild(e, project);
 299     }
 300     catch (RuntimeException e) {
 301       final Throwable cause = e.getCause();
 302       if (cause instanceof IOException || cause instanceof StorageException) {
 303         forceRebuild(e, project);
 304       }
 305       throw e;
 306     }
 307     return Collections.emptyList();
 308   }
 309
 310   @NotNull
 311   public String getComponentName() {
 312     return "Stub.IndexManager";
 313   }
 314
 315   public void initComponent() {
 316   }
 317
 318   public void disposeComponent() {
 319     // This index must be disposed only after StubUpdatingIndex is disposed
 320     // To ensure this, disposing is done explicitly from StubUpdatingIndex by calling dispose() method
 321     // do not call this method here to avoid double-disposal
 322   }
 323
 324   public void dispose() {
 325     for (UpdatableIndex index : myIndices.values()) {
 326       index.dispose();
 327     }
 328   }
 329
 330   public void setDataBufferingEnabled(final boolean enabled) {
 331     for (UpdatableIndex index : myIndices.values()) {
 332       final IndexStorage indexStorage = ((MapReduceIndex)index).getStorage();
 333       ((MemoryIndexStorage)indexStorage).setBufferingEnabled(enabled);
 334     }
 335   }
 336
 337
 338   public void clearAllIndices() {
 339     for (UpdatableIndex index : myIndices.values()) {
 340       try {
 341         index.clear();
 342       }
 343       catch (StorageException e) {
 344         LOG.error(e);
 345         throw new RuntimeException(e);
 346       }
 347     }
 348   }
 349
 350   private void dropUnregisteredIndices() {
 351     final Set<String> indicesToDrop = new HashSet<String>(myPreviouslyRegistered != null? myPreviouslyRegistered.registeredIndices : Collections.<String>emptyList());
 352     for (ID<?, ?> key : myIndices.keySet()) {
 353       indicesToDrop.remove(key.toString());
 354     }
 355
 356     for (String s : indicesToDrop) {
 357       FileUtil.delete(IndexInfrastructure.getIndexRootDir(ID.create(s)));
 358     }
 359   }
 360
 361   public StubIndexState getState() {
 362     return new StubIndexState(myIndices.keySet());
 363   }
 364
 365   public void loadState(final StubIndexState state) {
 366     myPreviouslyRegistered = state;
 367   }
 368
 369   public Lock getWriteLock(StubIndexKey indexKey) {
 370     return myIndices.get(indexKey).getWriteLock();
 371   }
 372
 373   public Collection<StubIndexKey> getAllStubIndexKeys() {
 374     return Collections.<StubIndexKey>unmodifiableCollection(myIndices.keySet());
 375   }
 376
 377   public <K> void updateIndex(StubIndexKey key, int fileId, Map<K, TIntArrayList> oldValues, Map<K, TIntArrayList> newValues) {
 378     try {
 379       MyIndex<K> index = (MyIndex<K>)myIndices.get(key);
 380       index.updateWithMap(fileId, newValues, oldValues.keySet());
 381     }
 382     catch (StorageException e) {
 383       LOG.info(e);
 384       myRebuildStatus.set(NEED_REBUILD);
 385     }
 386   }
 387
 388   private static class MyIndex<K> extends MapReduceIndex<K, TIntArrayList, Void> {
 389     public MyIndex(final IndexStorage<K, TIntArrayList> storage) {
 390       super(null, null, storage);
 391     }
 392
 393     public void updateWithMap(final int inputId, final Map<K, TIntArrayList> newData, Collection<K> oldKeys) throws StorageException {
 394       getWriteLock().lock();
 395       try {
 396         super.updateWithMap(inputId, newData, oldKeys);
 397       }
 398       finally {
 399         getWriteLock().unlock();
 400       }
 401     }
 402   }
 403
 404 }