memory index storage reworked:
[fedora-idea.git] / platform / lang-impl / src / com / intellij / psi / stubs / StubIndexImpl.java
blob9809e3699dd3f1a552595a2fe35e97754557ba9e
1 /*
2 * Copyright 2000-2009 JetBrains s.r.o.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
18 * @author max
20 package com.intellij.psi.stubs;
22 import com.intellij.lang.ASTNode;
23 import com.intellij.openapi.application.ApplicationManager;
24 import com.intellij.openapi.application.ModalityState;
25 import com.intellij.openapi.components.*;
26 import com.intellij.openapi.diagnostic.Logger;
27 import com.intellij.openapi.extensions.Extensions;
28 import com.intellij.openapi.progress.ProcessCanceledException;
29 import com.intellij.openapi.project.Project;
30 import com.intellij.openapi.util.io.FileUtil;
31 import com.intellij.openapi.vfs.VirtualFile;
32 import com.intellij.openapi.vfs.newvfs.ManagingFS;
33 import com.intellij.openapi.vfs.newvfs.persistent.PersistentFS;
34 import com.intellij.psi.PsiElement;
35 import com.intellij.psi.PsiFile;
36 import com.intellij.psi.PsiManager;
37 import com.intellij.psi.PsiPlainTextFile;
38 import com.intellij.psi.impl.source.PsiFileImpl;
39 import com.intellij.psi.impl.source.PsiFileWithStubSupport;
40 import com.intellij.psi.search.GlobalSearchScope;
41 import com.intellij.psi.tree.IElementType;
42 import com.intellij.psi.tree.IStubFileElementType;
43 import com.intellij.util.indexing.*;
44 import com.intellij.util.io.DataExternalizer;
45 import com.intellij.util.io.DataInputOutputUtil;
46 import gnu.trove.TIntArrayList;
47 import gnu.trove.TObjectIntHashMap;
48 import org.jetbrains.annotations.NotNull;
50 import java.io.DataInput;
51 import java.io.DataOutput;
52 import java.io.File;
53 import java.io.IOException;
54 import java.util.*;
55 import java.util.concurrent.Callable;
56 import java.util.concurrent.atomic.AtomicInteger;
57 import java.util.concurrent.locks.Lock;
59 @State(
60 name = "FileBasedIndex",
61 roamingType = RoamingType.DISABLED,
62 storages = {
63 @Storage(
64 id = "stubIndex",
65 file = "$APP_CONFIG$/stubIndex.xml")
68 public class StubIndexImpl extends StubIndex implements ApplicationComponent, PersistentStateComponent<StubIndexState> {
69 private static final Logger LOG = Logger.getInstance("#com.intellij.psi.stubs.StubIndexImpl");
70 private final Map<StubIndexKey<?,?>, MyIndex<?>> myIndices = new HashMap<StubIndexKey<?,?>, MyIndex<?>>();
71 private final TObjectIntHashMap<ID<?, ?>> myIndexIdToVersionMap = new TObjectIntHashMap<ID<?, ?>>();
73 public static final int OK = 1;
74 public static final int NEED_REBUILD = 2;
75 public static final int REBUILD_IN_PROGRESS = 3;
76 private final AtomicInteger myRebuildStatus = new AtomicInteger(OK);
78 private StubIndexState myPreviouslyRegistered;
80 public StubIndexImpl() throws IOException {
81 final StubIndexExtension<?, ?>[] extensions = Extensions.getExtensions(StubIndexExtension.EP_NAME);
82 boolean needRebuild = false;
83 for (StubIndexExtension extension : extensions) {
84 //noinspection unchecked
85 needRebuild |= registerIndexer(extension);
87 if (needRebuild) {
88 myRebuildStatus.set(NEED_REBUILD);
90 dropUnregisteredIndices();
93 private <K> boolean registerIndexer(final StubIndexExtension<K, ?> extension) throws IOException {
94 final StubIndexKey<K, ?> indexKey = extension.getKey();
95 final int version = extension.getVersion();
96 myIndexIdToVersionMap.put(indexKey, version);
97 final File versionFile = IndexInfrastructure.getVersionFile(indexKey);
98 final boolean versionFileExisted = versionFile.exists();
99 final File indexRootDir = IndexInfrastructure.getIndexRootDir(indexKey);
100 boolean needRebuild = false;
101 if (IndexInfrastructure.versionDiffers(versionFile, version)) {
102 final String[] children = indexRootDir.list();
103 // rebuild only if there exists what to rebuild
104 needRebuild = versionFileExisted || children != null && children.length > 0;
105 if (needRebuild) {
106 LOG.info("Version has changed for stub index " + extension.getKey() + ". The index will be rebuilt.");
108 FileUtil.delete(indexRootDir);
109 IndexInfrastructure.rewriteVersion(versionFile, version);
112 for (int attempt = 0; attempt < 2; attempt++) {
113 try {
114 final MapIndexStorage<K, TIntArrayList> storage = new MapIndexStorage<K, TIntArrayList>(IndexInfrastructure.getStorageFile(indexKey), extension.getKeyDescriptor(), new StubIdExternalizer(), 2 * 1024);
115 final MemoryIndexStorage<K, TIntArrayList> memStorage = new MemoryIndexStorage<K, TIntArrayList>(storage);
116 myIndices.put(indexKey, new MyIndex<K>(memStorage));
117 break;
119 catch (IOException e) {
120 LOG.info(e);
121 needRebuild = true;
122 FileUtil.delete(indexRootDir);
123 IndexInfrastructure.rewriteVersion(versionFile, version);
126 return needRebuild;
129 private static class StubIdExternalizer implements DataExternalizer<TIntArrayList> {
130 public void save(final DataOutput out, final TIntArrayList value) throws IOException {
131 int size = value.size();
132 if (size == 0) {
133 DataInputOutputUtil.writeSINT(out, Integer.MAX_VALUE);
135 else if (size == 1) {
136 DataInputOutputUtil.writeSINT(out, -value.get(0));
138 else {
139 DataInputOutputUtil.writeSINT(out, size);
140 for (int i = 0; i < size; i++) {
141 DataInputOutputUtil.writeINT(out, value.get(i));
146 public TIntArrayList read(final DataInput in) throws IOException {
147 int size = DataInputOutputUtil.readSINT(in);
148 if (size == Integer.MAX_VALUE) {
149 return new TIntArrayList();
151 else if (size <= 0) {
152 TIntArrayList result = new TIntArrayList(1);
153 result.add(-size);
154 return result;
156 else {
157 TIntArrayList result = new TIntArrayList(size);
158 for (int i = 0; i < size; i++) {
159 result.add(DataInputOutputUtil.readINT(in));
161 return result;
166 public <Key, Psi extends PsiElement> Collection<Psi> get(@NotNull final StubIndexKey<Key, Psi> indexKey, @NotNull final Key key, final Project project,
167 final GlobalSearchScope scope) {
168 checkRebuild(project);
170 FileBasedIndex.getInstance().ensureUpToDate(StubUpdatingIndex.INDEX_ID, project, scope);
172 final PersistentFS fs = (PersistentFS)ManagingFS.getInstance();
173 final PsiManager psiManager = PsiManager.getInstance(project);
175 final List<Psi> result = new ArrayList<Psi>();
176 final MyIndex<Key> index = (MyIndex<Key>)myIndices.get(indexKey);
178 try {
179 try {
180 // disable up-to-date check to avoid locks on attempt to acquire index write lock while holding at the same time the readLock for this index
181 FileBasedIndex.getInstance().disableUpToDateCheckForCurrentThread();
182 index.getReadLock().lock();
183 final ValueContainer<TIntArrayList> container = index.getData(key);
185 container.forEach(new ValueContainer.ContainerAction<TIntArrayList>() {
186 public void perform(final int id, final TIntArrayList value) {
187 final VirtualFile file = IndexInfrastructure.findFileById(fs, id);
188 if (file != null && (scope == null || scope.contains(file))) {
189 StubTree stubTree = null;
191 final PsiFile _psifile = psiManager.findFile(file);
192 PsiFileWithStubSupport psiFile = null;
194 if (_psifile != null && !(_psifile instanceof PsiPlainTextFile)) {
195 if (_psifile instanceof PsiFileWithStubSupport) {
196 psiFile = (PsiFileWithStubSupport)_psifile;
197 stubTree = psiFile.getStubTree();
198 if (stubTree == null && psiFile instanceof PsiFileImpl) {
199 stubTree = ((PsiFileImpl)psiFile).calcStubTree();
204 if (stubTree != null || psiFile != null) {
205 if (stubTree == null) {
206 stubTree = StubTree.readFromVFile(project, file);
207 if (stubTree != null) {
208 final List<StubElement<?>> plained = stubTree.getPlainList();
209 for (int i = 0; i < value.size(); i++) {
210 final StubElement<?> stub = plained.get(value.get(i));
211 final ASTNode tree = psiFile.findTreeForStub(stubTree, stub);
213 if (tree != null) {
214 if (tree.getElementType() == stubType(stub)) {
215 result.add((Psi)tree.getPsi());
217 else {
218 String persistedStubTree = ((PsiFileStubImpl)stubTree.getRoot()).printTree();
220 String stubTreeJustBuilt =
221 ((PsiFileStubImpl)((IStubFileElementType)((PsiFileImpl)psiFile).getContentElementType()).getBuilder()
222 .buildStubTree(psiFile)).printTree();
224 StringBuilder builder = new StringBuilder();
225 builder.append("Oops\n");
228 builder.append("Recorded stub:-----------------------------------\n");
229 builder.append(persistedStubTree);
230 builder.append("\nAST built stub: ------------------------------------\n");
231 builder.append(stubTreeJustBuilt);
232 builder.append("\n");
233 LOG.info(builder.toString());
235 // requestReindex() may want to acquire write lock (for indices not requiring content loading)
236 // thus, because here we are under read lock, need to use invoke later
237 ApplicationManager.getApplication().invokeLater(new Runnable() {
238 public void run() {
239 FileBasedIndex.getInstance().requestReindex(file);
241 }, ModalityState.NON_MODAL);
247 else {
248 final List<StubElement<?>> plained = stubTree.getPlainList();
249 for (int i = 0; i < value.size(); i++) {
250 result.add((Psi)plained.get(value.get(i)).getPsi());
258 finally {
259 index.getReadLock().unlock();
260 FileBasedIndex.getInstance().enableUpToDateCheckForCurrentThread();
263 catch (StorageException e) {
264 forceRebuild(e, project);
266 catch (RuntimeException e) {
267 final Throwable cause = e.getCause();
268 if (cause instanceof IOException || cause instanceof StorageException) {
269 forceRebuild(e, project);
271 else {
272 throw e;
276 return result;
279 private static IElementType stubType(final StubElement<?> stub) {
280 if (stub instanceof PsiFileStub) {
281 return ((PsiFileStub)stub).getType();
284 return stub.getStubType();
287 private void forceRebuild(Throwable e, Project project) {
288 LOG.info(e);
289 myRebuildStatus.set(NEED_REBUILD);
290 checkRebuild(project);
293 private void checkRebuild(@NotNull Project project) {
294 if (myRebuildStatus.compareAndSet(NEED_REBUILD, REBUILD_IN_PROGRESS)) {
295 StubUpdatingIndex.scheduleStubIndicesRebuild(new Runnable() {
296 public void run() {
297 myRebuildStatus.compareAndSet(REBUILD_IN_PROGRESS, OK);
299 }, project);
301 if (myRebuildStatus.get() == REBUILD_IN_PROGRESS) {
302 throw new ProcessCanceledException();
306 public <K> Collection<K> getAllKeys(final StubIndexKey<K, ?> indexKey, @NotNull Project project) {
307 checkRebuild(project);
308 FileBasedIndex.getInstance().ensureUpToDate(StubUpdatingIndex.INDEX_ID, project, GlobalSearchScope.allScope(project));
310 final MyIndex<K> index = (MyIndex<K>)myIndices.get(indexKey);
311 try {
312 return index.getAllKeys();
314 catch (StorageException e) {
315 forceRebuild(e, project);
317 catch (RuntimeException e) {
318 final Throwable cause = e.getCause();
319 if (cause instanceof IOException || cause instanceof StorageException) {
320 forceRebuild(e, project);
322 throw e;
324 return Collections.emptyList();
327 @NotNull
328 public String getComponentName() {
329 return "Stub.IndexManager";
332 public void initComponent() {
335 public void disposeComponent() {
336 // This index must be disposed only after StubUpdatingIndex is disposed
337 // To ensure this, disposing is done explicitly from StubUpdatingIndex by calling dispose() method
338 // do not call this method here to avoid double-disposal
341 public void dispose() {
342 for (UpdatableIndex index : myIndices.values()) {
343 index.dispose();
347 public void setDataBufferingEnabled(final boolean enabled) {
348 for (UpdatableIndex index : myIndices.values()) {
349 final IndexStorage indexStorage = ((MapReduceIndex)index).getStorage();
350 ((MemoryIndexStorage)indexStorage).setBufferingEnabled(enabled);
354 public void cleanupMemoryStorage() {
355 for (UpdatableIndex index : myIndices.values()) {
356 final IndexStorage indexStorage = ((MapReduceIndex)index).getStorage();
357 index.getWriteLock().lock();
358 try {
359 ((MemoryIndexStorage)indexStorage).clearMemoryMap();
361 finally {
362 index.getWriteLock().unlock();
368 public void clearAllIndices() {
369 for (UpdatableIndex index : myIndices.values()) {
370 try {
371 index.clear();
373 catch (StorageException e) {
374 LOG.error(e);
375 throw new RuntimeException(e);
380 private void dropUnregisteredIndices() {
381 final Set<String> indicesToDrop = new HashSet<String>(myPreviouslyRegistered != null? myPreviouslyRegistered.registeredIndices : Collections.<String>emptyList());
382 for (ID<?, ?> key : myIndices.keySet()) {
383 indicesToDrop.remove(key.toString());
386 for (String s : indicesToDrop) {
387 FileUtil.delete(IndexInfrastructure.getIndexRootDir(ID.create(s)));
391 public StubIndexState getState() {
392 return new StubIndexState(myIndices.keySet());
395 public void loadState(final StubIndexState state) {
396 myPreviouslyRegistered = state;
399 public Lock getWriteLock(StubIndexKey indexKey) {
400 return myIndices.get(indexKey).getWriteLock();
403 public Collection<StubIndexKey> getAllStubIndexKeys() {
404 return Collections.<StubIndexKey>unmodifiableCollection(myIndices.keySet());
407 public <K> void updateIndex(StubIndexKey key, int fileId, final Map<K, TIntArrayList> oldValues, Map<K, TIntArrayList> newValues) {
408 try {
409 final MyIndex<K> index = (MyIndex<K>)myIndices.get(key);
410 index.updateWithMap(fileId, newValues, new Callable<Collection<K>>() {
411 public Collection<K> call() throws Exception {
412 return oldValues.keySet();
416 catch (StorageException e) {
417 LOG.info(e);
418 myRebuildStatus.set(NEED_REBUILD);
422 private static class MyIndex<K> extends MapReduceIndex<K, TIntArrayList, Void> {
423 public MyIndex(final IndexStorage<K, TIntArrayList> storage) {
424 super(null, null, storage);
427 public void updateWithMap(final int inputId, final Map<K, TIntArrayList> newData, Callable<Collection<K>> oldKeysGetter) throws StorageException {
428 super.updateWithMap(inputId, newData, oldKeysGetter);