2 * Copyright 2000-2009 JetBrains s.r.o.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
20 package com
.intellij
.psi
.stubs
;
22 import com
.intellij
.lang
.ASTNode
;
23 import com
.intellij
.openapi
.application
.ApplicationManager
;
24 import com
.intellij
.openapi
.application
.ModalityState
;
25 import com
.intellij
.openapi
.components
.*;
26 import com
.intellij
.openapi
.diagnostic
.Logger
;
27 import com
.intellij
.openapi
.extensions
.Extensions
;
28 import com
.intellij
.openapi
.progress
.ProcessCanceledException
;
29 import com
.intellij
.openapi
.project
.Project
;
30 import com
.intellij
.openapi
.util
.io
.FileUtil
;
31 import com
.intellij
.openapi
.vfs
.VirtualFile
;
32 import com
.intellij
.openapi
.vfs
.newvfs
.ManagingFS
;
33 import com
.intellij
.openapi
.vfs
.newvfs
.persistent
.PersistentFS
;
34 import com
.intellij
.psi
.PsiElement
;
35 import com
.intellij
.psi
.PsiFile
;
36 import com
.intellij
.psi
.PsiManager
;
37 import com
.intellij
.psi
.PsiPlainTextFile
;
38 import com
.intellij
.psi
.impl
.source
.PsiFileImpl
;
39 import com
.intellij
.psi
.impl
.source
.PsiFileWithStubSupport
;
40 import com
.intellij
.psi
.search
.GlobalSearchScope
;
41 import com
.intellij
.psi
.tree
.IElementType
;
42 import com
.intellij
.psi
.tree
.IStubFileElementType
;
43 import com
.intellij
.util
.indexing
.*;
44 import com
.intellij
.util
.io
.DataExternalizer
;
45 import com
.intellij
.util
.io
.DataInputOutputUtil
;
46 import gnu
.trove
.TIntArrayList
;
47 import gnu
.trove
.TObjectIntHashMap
;
48 import org
.jetbrains
.annotations
.NotNull
;
50 import java
.io
.DataInput
;
51 import java
.io
.DataOutput
;
53 import java
.io
.IOException
;
55 import java
.util
.concurrent
.Callable
;
56 import java
.util
.concurrent
.atomic
.AtomicInteger
;
57 import java
.util
.concurrent
.locks
.Lock
;
60 name
= "FileBasedIndex",
61 roamingType
= RoamingType
.DISABLED
,
65 file
= "$APP_CONFIG$/stubIndex.xml")
68 public class StubIndexImpl
extends StubIndex
implements ApplicationComponent
, PersistentStateComponent
<StubIndexState
> {
69 private static final Logger LOG
= Logger
.getInstance("#com.intellij.psi.stubs.StubIndexImpl");
70 private final Map
<StubIndexKey
<?
,?
>, MyIndex
<?
>> myIndices
= new HashMap
<StubIndexKey
<?
,?
>, MyIndex
<?
>>();
71 private final TObjectIntHashMap
<ID
<?
, ?
>> myIndexIdToVersionMap
= new TObjectIntHashMap
<ID
<?
, ?
>>();
73 public static final int OK
= 1;
74 public static final int NEED_REBUILD
= 2;
75 public static final int REBUILD_IN_PROGRESS
= 3;
76 private final AtomicInteger myRebuildStatus
= new AtomicInteger(OK
);
78 private StubIndexState myPreviouslyRegistered
;
80 public StubIndexImpl() throws IOException
{
81 final StubIndexExtension
<?
, ?
>[] extensions
= Extensions
.getExtensions(StubIndexExtension
.EP_NAME
);
82 boolean needRebuild
= false;
83 for (StubIndexExtension extension
: extensions
) {
84 //noinspection unchecked
85 needRebuild
|= registerIndexer(extension
);
88 myRebuildStatus
.set(NEED_REBUILD
);
90 dropUnregisteredIndices();
93 private <K
> boolean registerIndexer(final StubIndexExtension
<K
, ?
> extension
) throws IOException
{
94 final StubIndexKey
<K
, ?
> indexKey
= extension
.getKey();
95 final int version
= extension
.getVersion();
96 myIndexIdToVersionMap
.put(indexKey
, version
);
97 final File versionFile
= IndexInfrastructure
.getVersionFile(indexKey
);
98 final boolean versionFileExisted
= versionFile
.exists();
99 final File indexRootDir
= IndexInfrastructure
.getIndexRootDir(indexKey
);
100 boolean needRebuild
= false;
101 if (IndexInfrastructure
.versionDiffers(versionFile
, version
)) {
102 final String
[] children
= indexRootDir
.list();
103 // rebuild only if there exists what to rebuild
104 needRebuild
= versionFileExisted
|| children
!= null && children
.length
> 0;
106 LOG
.info("Version has changed for stub index " + extension
.getKey() + ". The index will be rebuilt.");
108 FileUtil
.delete(indexRootDir
);
109 IndexInfrastructure
.rewriteVersion(versionFile
, version
);
112 for (int attempt
= 0; attempt
< 2; attempt
++) {
114 final MapIndexStorage
<K
, TIntArrayList
> storage
= new MapIndexStorage
<K
, TIntArrayList
>(IndexInfrastructure
.getStorageFile(indexKey
), extension
.getKeyDescriptor(), new StubIdExternalizer(), 2 * 1024);
115 final MemoryIndexStorage
<K
, TIntArrayList
> memStorage
= new MemoryIndexStorage
<K
, TIntArrayList
>(storage
);
116 myIndices
.put(indexKey
, new MyIndex
<K
>(memStorage
));
119 catch (IOException e
) {
122 FileUtil
.delete(indexRootDir
);
123 IndexInfrastructure
.rewriteVersion(versionFile
, version
);
129 private static class StubIdExternalizer
implements DataExternalizer
<TIntArrayList
> {
130 public void save(final DataOutput out
, final TIntArrayList value
) throws IOException
{
131 int size
= value
.size();
133 DataInputOutputUtil
.writeSINT(out
, Integer
.MAX_VALUE
);
135 else if (size
== 1) {
136 DataInputOutputUtil
.writeSINT(out
, -value
.get(0));
139 DataInputOutputUtil
.writeSINT(out
, size
);
140 for (int i
= 0; i
< size
; i
++) {
141 DataInputOutputUtil
.writeINT(out
, value
.get(i
));
146 public TIntArrayList
read(final DataInput in
) throws IOException
{
147 int size
= DataInputOutputUtil
.readSINT(in
);
148 if (size
== Integer
.MAX_VALUE
) {
149 return new TIntArrayList();
151 else if (size
<= 0) {
152 TIntArrayList result
= new TIntArrayList(1);
157 TIntArrayList result
= new TIntArrayList(size
);
158 for (int i
= 0; i
< size
; i
++) {
159 result
.add(DataInputOutputUtil
.readINT(in
));
166 public <Key
, Psi
extends PsiElement
> Collection
<Psi
> get(@NotNull final StubIndexKey
<Key
, Psi
> indexKey
, @NotNull final Key key
, final Project project
,
167 final GlobalSearchScope scope
) {
168 checkRebuild(project
);
170 FileBasedIndex
.getInstance().ensureUpToDate(StubUpdatingIndex
.INDEX_ID
, project
, scope
);
172 final PersistentFS fs
= (PersistentFS
)ManagingFS
.getInstance();
173 final PsiManager psiManager
= PsiManager
.getInstance(project
);
175 final List
<Psi
> result
= new ArrayList
<Psi
>();
176 final MyIndex
<Key
> index
= (MyIndex
<Key
>)myIndices
.get(indexKey
);
180 // disable up-to-date check to avoid locks on attempt to acquire index write lock while holding at the same time the readLock for this index
181 FileBasedIndex
.getInstance().disableUpToDateCheckForCurrentThread();
182 index
.getReadLock().lock();
183 final ValueContainer
<TIntArrayList
> container
= index
.getData(key
);
185 container
.forEach(new ValueContainer
.ContainerAction
<TIntArrayList
>() {
186 public void perform(final int id
, final TIntArrayList value
) {
187 final VirtualFile file
= IndexInfrastructure
.findFileById(fs
, id
);
188 if (file
!= null && (scope
== null || scope
.contains(file
))) {
189 StubTree stubTree
= null;
191 final PsiFile _psifile
= psiManager
.findFile(file
);
192 PsiFileWithStubSupport psiFile
= null;
194 if (_psifile
!= null && !(_psifile
instanceof PsiPlainTextFile
)) {
195 if (_psifile
instanceof PsiFileWithStubSupport
) {
196 psiFile
= (PsiFileWithStubSupport
)_psifile
;
197 stubTree
= psiFile
.getStubTree();
198 if (stubTree
== null && psiFile
instanceof PsiFileImpl
) {
199 stubTree
= ((PsiFileImpl
)psiFile
).calcStubTree();
204 if (stubTree
!= null || psiFile
!= null) {
205 if (stubTree
== null) {
206 stubTree
= StubTree
.readFromVFile(project
, file
);
207 if (stubTree
!= null) {
208 final List
<StubElement
<?
>> plained
= stubTree
.getPlainList();
209 for (int i
= 0; i
< value
.size(); i
++) {
210 final StubElement
<?
> stub
= plained
.get(value
.get(i
));
211 final ASTNode tree
= psiFile
.findTreeForStub(stubTree
, stub
);
214 if (tree
.getElementType() == stubType(stub
)) {
215 result
.add((Psi
)tree
.getPsi());
218 String persistedStubTree
= ((PsiFileStubImpl
)stubTree
.getRoot()).printTree();
220 String stubTreeJustBuilt
=
221 ((PsiFileStubImpl
)((IStubFileElementType
)((PsiFileImpl
)psiFile
).getContentElementType()).getBuilder()
222 .buildStubTree(psiFile
)).printTree();
224 StringBuilder builder
= new StringBuilder();
225 builder
.append("Oops\n");
228 builder
.append("Recorded stub:-----------------------------------\n");
229 builder
.append(persistedStubTree
);
230 builder
.append("\nAST built stub: ------------------------------------\n");
231 builder
.append(stubTreeJustBuilt
);
232 builder
.append("\n");
233 LOG
.info(builder
.toString());
235 // requestReindex() may want to acquire write lock (for indices not requiring content loading)
236 // thus, because here we are under read lock, need to use invoke later
237 ApplicationManager
.getApplication().invokeLater(new Runnable() {
239 FileBasedIndex
.getInstance().requestReindex(file
);
241 }, ModalityState
.NON_MODAL
);
248 final List
<StubElement
<?
>> plained
= stubTree
.getPlainList();
249 for (int i
= 0; i
< value
.size(); i
++) {
250 result
.add((Psi
)plained
.get(value
.get(i
)).getPsi());
259 index
.getReadLock().unlock();
260 FileBasedIndex
.getInstance().enableUpToDateCheckForCurrentThread();
263 catch (StorageException e
) {
264 forceRebuild(e
, project
);
266 catch (RuntimeException e
) {
267 final Throwable cause
= e
.getCause();
268 if (cause
instanceof IOException
|| cause
instanceof StorageException
) {
269 forceRebuild(e
, project
);
279 private static IElementType
stubType(final StubElement
<?
> stub
) {
280 if (stub
instanceof PsiFileStub
) {
281 return ((PsiFileStub
)stub
).getType();
284 return stub
.getStubType();
287 private void forceRebuild(Throwable e
, Project project
) {
289 myRebuildStatus
.set(NEED_REBUILD
);
290 checkRebuild(project
);
293 private void checkRebuild(@NotNull Project project
) {
294 if (myRebuildStatus
.compareAndSet(NEED_REBUILD
, REBUILD_IN_PROGRESS
)) {
295 StubUpdatingIndex
.scheduleStubIndicesRebuild(new Runnable() {
297 myRebuildStatus
.compareAndSet(REBUILD_IN_PROGRESS
, OK
);
301 if (myRebuildStatus
.get() == REBUILD_IN_PROGRESS
) {
302 throw new ProcessCanceledException();
306 public <K
> Collection
<K
> getAllKeys(final StubIndexKey
<K
, ?
> indexKey
, @NotNull Project project
) {
307 checkRebuild(project
);
308 FileBasedIndex
.getInstance().ensureUpToDate(StubUpdatingIndex
.INDEX_ID
, project
, GlobalSearchScope
.allScope(project
));
310 final MyIndex
<K
> index
= (MyIndex
<K
>)myIndices
.get(indexKey
);
312 return index
.getAllKeys();
314 catch (StorageException e
) {
315 forceRebuild(e
, project
);
317 catch (RuntimeException e
) {
318 final Throwable cause
= e
.getCause();
319 if (cause
instanceof IOException
|| cause
instanceof StorageException
) {
320 forceRebuild(e
, project
);
324 return Collections
.emptyList();
328 public String
getComponentName() {
329 return "Stub.IndexManager";
332 public void initComponent() {
335 public void disposeComponent() {
336 // This index must be disposed only after StubUpdatingIndex is disposed
337 // To ensure this, disposing is done explicitly from StubUpdatingIndex by calling dispose() method
338 // do not call this method here to avoid double-disposal
341 public void dispose() {
342 for (UpdatableIndex index
: myIndices
.values()) {
347 public void setDataBufferingEnabled(final boolean enabled
) {
348 for (UpdatableIndex index
: myIndices
.values()) {
349 final IndexStorage indexStorage
= ((MapReduceIndex
)index
).getStorage();
350 ((MemoryIndexStorage
)indexStorage
).setBufferingEnabled(enabled
);
354 public void cleanupMemoryStorage() {
355 for (UpdatableIndex index
: myIndices
.values()) {
356 final IndexStorage indexStorage
= ((MapReduceIndex
)index
).getStorage();
357 index
.getWriteLock().lock();
359 ((MemoryIndexStorage
)indexStorage
).clearMemoryMap();
362 index
.getWriteLock().unlock();
368 public void clearAllIndices() {
369 for (UpdatableIndex index
: myIndices
.values()) {
373 catch (StorageException e
) {
375 throw new RuntimeException(e
);
380 private void dropUnregisteredIndices() {
381 final Set
<String
> indicesToDrop
= new HashSet
<String
>(myPreviouslyRegistered
!= null? myPreviouslyRegistered
.registeredIndices
: Collections
.<String
>emptyList());
382 for (ID
<?
, ?
> key
: myIndices
.keySet()) {
383 indicesToDrop
.remove(key
.toString());
386 for (String s
: indicesToDrop
) {
387 FileUtil
.delete(IndexInfrastructure
.getIndexRootDir(ID
.create(s
)));
391 public StubIndexState
getState() {
392 return new StubIndexState(myIndices
.keySet());
395 public void loadState(final StubIndexState state
) {
396 myPreviouslyRegistered
= state
;
399 public Lock
getWriteLock(StubIndexKey indexKey
) {
400 return myIndices
.get(indexKey
).getWriteLock();
403 public Collection
<StubIndexKey
> getAllStubIndexKeys() {
404 return Collections
.<StubIndexKey
>unmodifiableCollection(myIndices
.keySet());
407 public <K
> void updateIndex(StubIndexKey key
, int fileId
, final Map
<K
, TIntArrayList
> oldValues
, Map
<K
, TIntArrayList
> newValues
) {
409 final MyIndex
<K
> index
= (MyIndex
<K
>)myIndices
.get(key
);
410 index
.updateWithMap(fileId
, newValues
, new Callable
<Collection
<K
>>() {
411 public Collection
<K
> call() throws Exception
{
412 return oldValues
.keySet();
416 catch (StorageException e
) {
418 myRebuildStatus
.set(NEED_REBUILD
);
422 private static class MyIndex
<K
> extends MapReduceIndex
<K
, TIntArrayList
, Void
> {
423 public MyIndex(final IndexStorage
<K
, TIntArrayList
> storage
) {
424 super(null, null, storage
);
427 public void updateWithMap(final int inputId
, final Map
<K
, TIntArrayList
> newData
, Callable
<Collection
<K
>> oldKeysGetter
) throws StorageException
{
428 super.updateWithMap(inputId
, newData
, oldKeysGetter
);