platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java

   1 /*
   2  * Copyright 2000-2009 JetBrains s.r.o.
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  * http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 /*
  18  * @author max
  19  */
  20 package com.intellij.psi.stubs;
  21
  22 import com.intellij.lang.ASTNode;
  23 import com.intellij.openapi.application.ApplicationManager;
  24 import com.intellij.openapi.application.ModalityState;
  25 import com.intellij.openapi.components.*;
  26 import com.intellij.openapi.diagnostic.Logger;
  27 import com.intellij.openapi.extensions.Extensions;
  28 import com.intellij.openapi.progress.ProcessCanceledException;
  29 import com.intellij.openapi.project.Project;
  30 import com.intellij.openapi.util.io.FileUtil;
  31 import com.intellij.openapi.vfs.VirtualFile;
  32 import com.intellij.openapi.vfs.newvfs.ManagingFS;
  33 import com.intellij.openapi.vfs.newvfs.persistent.PersistentFS;
  34 import com.intellij.psi.PsiElement;
  35 import com.intellij.psi.PsiFile;
  36 import com.intellij.psi.PsiManager;
  37 import com.intellij.psi.PsiPlainTextFile;
  38 import com.intellij.psi.impl.source.PsiFileImpl;
  39 import com.intellij.psi.impl.source.PsiFileWithStubSupport;
  40 import com.intellij.psi.search.GlobalSearchScope;
  41 import com.intellij.psi.tree.IElementType;
  42 import com.intellij.psi.tree.IStubFileElementType;
  43 import com.intellij.util.indexing.*;
  44 import com.intellij.util.io.DataExternalizer;
  45 import com.intellij.util.io.DataInputOutputUtil;
  46 import gnu.trove.TIntArrayList;
  47 import gnu.trove.TObjectIntHashMap;
  48 import org.jetbrains.annotations.NotNull;
  49
  50 import java.io.DataInput;
  51 import java.io.DataOutput;
  52 import java.io.File;
  53 import java.io.IOException;
  54 import java.util.*;
  55 import java.util.concurrent.Callable;
  56 import java.util.concurrent.atomic.AtomicInteger;
  57 import java.util.concurrent.locks.Lock;
  58
  59 @State(
  60   name = "FileBasedIndex",
  61   roamingType = RoamingType.DISABLED,
  62   storages = {
  63   @Storage(
  64     id = "stubIndex",
  65     file = "$APP_CONFIG$/stubIndex.xml")
  66     }
  67 )
  68 public class StubIndexImpl extends StubIndex implements ApplicationComponent, PersistentStateComponent<StubIndexState> {
  69   private static final Logger LOG = Logger.getInstance("#com.intellij.psi.stubs.StubIndexImpl");
  70   private final Map<StubIndexKey<?,?>, MyIndex<?>> myIndices = new HashMap<StubIndexKey<?,?>, MyIndex<?>>();
  71   private final TObjectIntHashMap<ID<?, ?>> myIndexIdToVersionMap = new TObjectIntHashMap<ID<?, ?>>();
  72
  73   public static final int OK = 1;
  74   public static final int NEED_REBUILD = 2;
  75   public static final int REBUILD_IN_PROGRESS = 3;
  76   private final AtomicInteger myRebuildStatus = new AtomicInteger(OK);
  77
  78   private StubIndexState myPreviouslyRegistered;
  79
  80   public StubIndexImpl() throws IOException {
  81     final StubIndexExtension<?, ?>[] extensions = Extensions.getExtensions(StubIndexExtension.EP_NAME);
  82     boolean needRebuild = false;
  83     for (StubIndexExtension extension : extensions) {
  84       //noinspection unchecked
  85       needRebuild |= registerIndexer(extension);
  86     }
  87     if (needRebuild) {
  88       myRebuildStatus.set(NEED_REBUILD);
  89     }
  90     dropUnregisteredIndices();
  91   }
  92
  93   private <K> boolean registerIndexer(final StubIndexExtension<K, ?> extension) throws IOException {
  94     final StubIndexKey<K, ?> indexKey = extension.getKey();
  95     final int version = extension.getVersion();
  96     myIndexIdToVersionMap.put(indexKey, version);
  97     final File versionFile = IndexInfrastructure.getVersionFile(indexKey);
  98     final boolean versionFileExisted = versionFile.exists();
  99     final File indexRootDir = IndexInfrastructure.getIndexRootDir(indexKey);
 100     boolean needRebuild = false;
 101     if (IndexInfrastructure.versionDiffers(versionFile, version)) {
 102       final String[] children = indexRootDir.list();
 103       // rebuild only if there exists what to rebuild
 104       needRebuild = versionFileExisted || children != null && children.length > 0;
 105       if (needRebuild) {
 106         LOG.info("Version has changed for stub index " + extension.getKey() + ". The index will be rebuilt.");
 107       }
 108       FileUtil.delete(indexRootDir);
 109       IndexInfrastructure.rewriteVersion(versionFile, version);
 110     }
 111
 112     for (int attempt = 0; attempt < 2; attempt++) {
 113       try {
 114         final MapIndexStorage<K, TIntArrayList> storage = new MapIndexStorage<K, TIntArrayList>(IndexInfrastructure.getStorageFile(indexKey), extension.getKeyDescriptor(), new StubIdExternalizer(), 2 * 1024);
 115         final MemoryIndexStorage<K, TIntArrayList> memStorage = new MemoryIndexStorage<K, TIntArrayList>(storage);
 116         myIndices.put(indexKey, new MyIndex<K>(memStorage));
 117         break;
 118       }
 119       catch (IOException e) {
 120         LOG.info(e);
 121         needRebuild = true;
 122         FileUtil.delete(indexRootDir);
 123         IndexInfrastructure.rewriteVersion(versionFile, version);
 124       }
 125     }
 126     return needRebuild;
 127   }
 128
 129   private static class StubIdExternalizer implements DataExternalizer<TIntArrayList> {
 130     public void save(final DataOutput out, final TIntArrayList value) throws IOException {
 131       int size = value.size();
 132       if (size == 0) {
 133         DataInputOutputUtil.writeSINT(out, Integer.MAX_VALUE);
 134       }
 135       else if (size == 1) {
 136         DataInputOutputUtil.writeSINT(out, -value.get(0));
 137       }
 138       else {
 139         DataInputOutputUtil.writeSINT(out, size);
 140         for (int i = 0; i < size; i++) {
 141           DataInputOutputUtil.writeINT(out, value.get(i));
 142         }
 143       }
 144     }
 145
 146     public TIntArrayList read(final DataInput in) throws IOException {
 147       int size = DataInputOutputUtil.readSINT(in);
 148       if (size == Integer.MAX_VALUE) {
 149         return new TIntArrayList();
 150       }
 151       else if (size <= 0) {
 152         TIntArrayList result = new TIntArrayList(1);
 153         result.add(-size);
 154         return result;
 155       }
 156       else {
 157         TIntArrayList result = new TIntArrayList(size);
 158         for (int i = 0; i < size; i++) {
 159           result.add(DataInputOutputUtil.readINT(in));
 160         }
 161         return result;
 162       }
 163     }
 164   }
 165
 166   public <Key, Psi extends PsiElement> Collection<Psi> get(@NotNull final StubIndexKey<Key, Psi> indexKey, @NotNull final Key key, final Project project,
 167                                                            final GlobalSearchScope scope) {
 168     checkRebuild(project);
 169
 170     FileBasedIndex.getInstance().ensureUpToDate(StubUpdatingIndex.INDEX_ID, project, scope);
 171
 172     final PersistentFS fs = (PersistentFS)ManagingFS.getInstance();
 173     final PsiManager psiManager = PsiManager.getInstance(project);
 174
 175     final List<Psi> result = new ArrayList<Psi>();
 176     final MyIndex<Key> index = (MyIndex<Key>)myIndices.get(indexKey);
 177
 178     try {
 179       try {
 180         // disable up-to-date check to avoid locks on attempt to acquire index write lock while holding at the same time the readLock for this index
 181         FileBasedIndex.getInstance().disableUpToDateCheckForCurrentThread();
 182         index.getReadLock().lock();
 183         final ValueContainer<TIntArrayList> container = index.getData(key);
 184
 185         container.forEach(new ValueContainer.ContainerAction<TIntArrayList>() {
 186           public void perform(final int id, final TIntArrayList value) {
 187             final VirtualFile file = IndexInfrastructure.findFileById(fs, id);
 188             if (file != null && (scope == null || scope.contains(file))) {
 189               StubTree stubTree = null;
 190
 191               final PsiFile _psifile = psiManager.findFile(file);
 192               PsiFileWithStubSupport psiFile = null;
 193
 194               if (_psifile != null && !(_psifile instanceof PsiPlainTextFile)) {
 195                 if (_psifile instanceof PsiFileWithStubSupport) {
 196                   psiFile = (PsiFileWithStubSupport)_psifile;
 197                   stubTree = psiFile.getStubTree();
 198                   if (stubTree == null && psiFile instanceof PsiFileImpl) {
 199                     stubTree = ((PsiFileImpl)psiFile).calcStubTree();
 200                   }
 201                 }
 202               }
 203
 204               if (stubTree != null || psiFile != null) {
 205                 if (stubTree == null) {
 206                   stubTree = StubTree.readFromVFile(project, file);
 207                   if (stubTree != null) {
 208                     final List<StubElement<?>> plained = stubTree.getPlainList();
 209                     for (int i = 0; i < value.size(); i++) {
 210                       final StubElement<?> stub = plained.get(value.get(i));
 211                       final ASTNode tree = psiFile.findTreeForStub(stubTree, stub);
 212
 213                       if (tree != null) {
 214                         if (tree.getElementType() == stubType(stub)) {
 215                           result.add((Psi)tree.getPsi());
 216                         }
 217                         else {
 218                           String persistedStubTree = ((PsiFileStubImpl)stubTree.getRoot()).printTree();
 219
 220                           String stubTreeJustBuilt =
 221                               ((PsiFileStubImpl)((IStubFileElementType)((PsiFileImpl)psiFile).getContentElementType()).getBuilder()
 222                                   .buildStubTree(psiFile)).printTree();
 223
 224                           StringBuilder builder = new StringBuilder();
 225                           builder.append("Oops\n");
 226
 227
 228                           builder.append("Recorded stub:-----------------------------------\n");
 229                           builder.append(persistedStubTree);
 230                           builder.append("\nAST built stub: ------------------------------------\n");
 231                           builder.append(stubTreeJustBuilt);
 232                           builder.append("\n");
 233                           LOG.info(builder.toString());
 234
 235                           // requestReindex() may want to acquire write lock (for indices not requiring content loading)
 236                           // thus, because here we are under read lock, need to use invoke later
 237                           ApplicationManager.getApplication().invokeLater(new Runnable() {
 238                             public void run() {
 239                               FileBasedIndex.getInstance().requestReindex(file);
 240                             }
 241                           }, ModalityState.NON_MODAL);
 242                         }
 243                       }
 244                     }
 245                   }
 246                 }
 247                 else {
 248                   final List<StubElement<?>> plained = stubTree.getPlainList();
 249                   for (int i = 0; i < value.size(); i++) {
 250                     result.add((Psi)plained.get(value.get(i)).getPsi());
 251                   }
 252                 }
 253               }
 254             }
 255           }
 256         });
 257       }
 258       finally {
 259         index.getReadLock().unlock();
 260         FileBasedIndex.getInstance().enableUpToDateCheckForCurrentThread();
 261       }
 262     }
 263     catch (StorageException e) {
 264       forceRebuild(e, project);
 265     }
 266     catch (RuntimeException e) {
 267       final Throwable cause = e.getCause();
 268       if (cause instanceof IOException || cause instanceof StorageException) {
 269         forceRebuild(e, project);
 270       }
 271       else {
 272         throw e;
 273       }
 274     }
 275
 276     return result;
 277   }
 278
 279   private static IElementType stubType(final StubElement<?> stub) {
 280     if (stub instanceof PsiFileStub) {
 281       return ((PsiFileStub)stub).getType();
 282     }
 283
 284     return stub.getStubType();
 285   }
 286
 287   private void forceRebuild(Throwable e, Project project) {
 288     LOG.info(e);
 289     myRebuildStatus.set(NEED_REBUILD);
 290     checkRebuild(project);
 291   }
 292
 293   private void checkRebuild(@NotNull Project project) {
 294     if (myRebuildStatus.compareAndSet(NEED_REBUILD, REBUILD_IN_PROGRESS)) {
 295       StubUpdatingIndex.scheduleStubIndicesRebuild(new Runnable() {
 296         public void run() {
 297           myRebuildStatus.compareAndSet(REBUILD_IN_PROGRESS, OK);
 298         }
 299       }, project);
 300     }
 301     if (myRebuildStatus.get() == REBUILD_IN_PROGRESS) {
 302       throw new ProcessCanceledException();
 303     }
 304   }
 305
 306   public <K> Collection<K> getAllKeys(final StubIndexKey<K, ?> indexKey, @NotNull Project project) {
 307     checkRebuild(project);
 308     FileBasedIndex.getInstance().ensureUpToDate(StubUpdatingIndex.INDEX_ID, project, GlobalSearchScope.allScope(project));
 309
 310     final MyIndex<K> index = (MyIndex<K>)myIndices.get(indexKey);
 311     try {
 312       return index.getAllKeys();
 313     }
 314     catch (StorageException e) {
 315       forceRebuild(e, project);
 316     }
 317     catch (RuntimeException e) {
 318       final Throwable cause = e.getCause();
 319       if (cause instanceof IOException || cause instanceof StorageException) {
 320         forceRebuild(e, project);
 321       }
 322       throw e;
 323     }
 324     return Collections.emptyList();
 325   }
 326
 327   @NotNull
 328   public String getComponentName() {
 329     return "Stub.IndexManager";
 330   }
 331
 332   public void initComponent() {
 333   }
 334
 335   public void disposeComponent() {
 336     // This index must be disposed only after StubUpdatingIndex is disposed
 337     // To ensure this, disposing is done explicitly from StubUpdatingIndex by calling dispose() method
 338     // do not call this method here to avoid double-disposal
 339   }
 340
 341   public void dispose() {
 342     for (UpdatableIndex index : myIndices.values()) {
 343       index.dispose();
 344     }
 345   }
 346
 347   public void setDataBufferingEnabled(final boolean enabled) {
 348     for (UpdatableIndex index : myIndices.values()) {
 349       final IndexStorage indexStorage = ((MapReduceIndex)index).getStorage();
 350       ((MemoryIndexStorage)indexStorage).setBufferingEnabled(enabled);
 351     }
 352   }
 353
 354   public void cleanupMemoryStorage() {
 355     for (UpdatableIndex index : myIndices.values()) {
 356       final IndexStorage indexStorage = ((MapReduceIndex)index).getStorage();
 357       index.getWriteLock().lock();
 358       try {
 359         ((MemoryIndexStorage)indexStorage).clearMemoryMap();
 360       }
 361       finally {
 362         index.getWriteLock().unlock();
 363       }
 364     }
 365   }
 366
 367
 368   public void clearAllIndices() {
 369     for (UpdatableIndex index : myIndices.values()) {
 370       try {
 371         index.clear();
 372       }
 373       catch (StorageException e) {
 374         LOG.error(e);
 375         throw new RuntimeException(e);
 376       }
 377     }
 378   }
 379
 380   private void dropUnregisteredIndices() {
 381     final Set<String> indicesToDrop = new HashSet<String>(myPreviouslyRegistered != null? myPreviouslyRegistered.registeredIndices : Collections.<String>emptyList());
 382     for (ID<?, ?> key : myIndices.keySet()) {
 383       indicesToDrop.remove(key.toString());
 384     }
 385
 386     for (String s : indicesToDrop) {
 387       FileUtil.delete(IndexInfrastructure.getIndexRootDir(ID.create(s)));
 388     }
 389   }
 390
 391   public StubIndexState getState() {
 392     return new StubIndexState(myIndices.keySet());
 393   }
 394
 395   public void loadState(final StubIndexState state) {
 396     myPreviouslyRegistered = state;
 397   }
 398
 399   public Lock getWriteLock(StubIndexKey indexKey) {
 400     return myIndices.get(indexKey).getWriteLock();
 401   }
 402
 403   public Collection<StubIndexKey> getAllStubIndexKeys() {
 404     return Collections.<StubIndexKey>unmodifiableCollection(myIndices.keySet());
 405   }
 406
 407   public <K> void updateIndex(StubIndexKey key, int fileId, final Map<K, TIntArrayList> oldValues, Map<K, TIntArrayList> newValues) {
 408     try {
 409       final MyIndex<K> index = (MyIndex<K>)myIndices.get(key);
 410       index.updateWithMap(fileId, newValues, new Callable<Collection<K>>() {
 411         public Collection<K> call() throws Exception {
 412           return oldValues.keySet();
 413         }
 414       });
 415     }
 416     catch (StorageException e) {
 417       LOG.info(e);
 418       myRebuildStatus.set(NEED_REBUILD);
 419     }
 420   }
 421
 422   private static class MyIndex<K> extends MapReduceIndex<K, TIntArrayList, Void> {
 423     public MyIndex(final IndexStorage<K, TIntArrayList> storage) {
 424       super(null, null, storage);
 425     }
 426
 427     public void updateWithMap(final int inputId, final Map<K, TIntArrayList> newData, Callable<Collection<K>> oldKeysGetter) throws StorageException {
 428       super.updateWithMap(inputId, newData, oldKeysGetter);
 429     }
 430   }
 431
 432 }