From a18ac467dd284f80b4e50d7932bf7f74dd175109 Mon Sep 17 00:00:00 2001 From: Eugene Zhuravlev Date: Fri, 4 Sep 2009 15:31:08 +0400 Subject: [PATCH] index update now does not require old content for the file (mapOld() removed) --- .../psi/impl/cache/impl/todo/TodoIndex.java | 7 +- .../src/com/intellij/psi/stubs/StubIndexImpl.java | 12 +- .../com/intellij/psi/stubs/StubUpdatingIndex.java | 14 +- .../indexing/ChangeTrackingValueContainer.java | 29 +- .../com/intellij/util/indexing/FileBasedIndex.java | 294 ++++++--------------- .../util/indexing/IndexInfrastructure.java | 6 +- .../com/intellij/util/indexing/IndexStorage.java | 2 + .../intellij/util/indexing/MapIndexStorage.java | 46 +++- .../com/intellij/util/indexing/MapReduceIndex.java | 127 +++++---- .../intellij/util/indexing/MemoryIndexStorage.java | 9 + .../SingleEntryFileBasedIndexExtension.java | 85 ++---- .../com/intellij/util/indexing/UpdatableIndex.java | 2 +- .../util/indexing/UpdatableValueContainer.java | 2 + .../intellij/util/indexing/ValueContainerImpl.java | 13 + 14 files changed, 280 insertions(+), 368 deletions(-) rewrite platform/lang-impl/src/com/intellij/util/indexing/SingleEntryFileBasedIndexExtension.java (75%) diff --git a/platform/lang-impl/src/com/intellij/psi/impl/cache/impl/todo/TodoIndex.java b/platform/lang-impl/src/com/intellij/psi/impl/cache/impl/todo/TodoIndex.java index c7401a3857..3932249649 100644 --- a/platform/lang-impl/src/com/intellij/psi/impl/cache/impl/todo/TodoIndex.java +++ b/platform/lang-impl/src/com/intellij/psi/impl/cache/impl/todo/TodoIndex.java @@ -32,7 +32,7 @@ import java.util.Map; * @author Eugene Zhuravlev * Date: Jan 20, 2008 */ -public class TodoIndex extends CustomImplementationFileBasedIndexExtension { +public class TodoIndex extends FileBasedIndexExtension { @NonNls public static final ID NAME = ID.create("TodoIndex"); public TodoIndex(TodoConfiguration config) { @@ -139,9 +139,4 @@ public class TodoIndex extends CustomImplementationFileBasedIndexExtension createIndexImplementation(final ID indexId, final FileBasedIndex owner, - IndexStorage storage) { - return new SimpleMapReduceIndex(indexId, myIndexer, storage); - } } diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java index 8c0e3216b5..89c069c38f 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java @@ -377,7 +377,7 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe public void updateIndex(StubIndexKey key, int fileId, Map oldValues, Map newValues) { try { MyIndex index = (MyIndex)myIndices.get(key); - index.updateWithMap(fileId, oldValues, newValues); + index.updateWithMap(fileId, newValues, oldValues.keySet()); } catch (StorageException e) { LOG.info(e); @@ -390,8 +390,14 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe super(null, null, storage); } - public void updateWithMap(final int inputId, final Map oldData, final Map newData) throws StorageException { - super.updateWithMap(inputId, oldData, newData); + public void updateWithMap(final int inputId, final Map newData, Collection oldKeys) throws StorageException { + getWriteLock().lock(); + try { + super.updateWithMap(inputId, newData, oldKeys); + } + finally { + getWriteLock().unlock(); + } } } diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java index 5642fdfaf4..0e313fccad 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java @@ -290,10 +290,11 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi } } - protected void updateWithMap(final int inputId, final Map oldData, final Map newData) + protected void updateWithMap(final int inputId, final Map newData, Collection oldKeys) throws StorageException { checkNameStorage(); + Map oldData = readOldData(inputId); final Map> oldStubTree = getStubTree(oldData); final Map> newStubTree = getStubTree(newData); @@ -307,7 +308,7 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi try { getWriteLock().lock(); - super.updateWithMap(inputId, oldData, newData); + super.updateWithMap(inputId, newData, oldKeys); updateStubIndices(affectedIndices, inputId, oldStubTree, newStubTree); } finally { @@ -343,12 +344,7 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi return stubTree; } - protected Map mapOld(final FileContent inputData) throws StorageException { - checkNameStorage(); - if (inputData == null) { - return Collections.emptyMap(); - } - final int key = Math.abs(FileBasedIndex.getFileId(inputData.getFile())); + private Map readOldData(final int key) throws StorageException { final Map result = new HashMap(); final Lock lock = getReadLock(); @@ -368,7 +364,7 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi return result; } - + public void clear() throws StorageException { final StubIndexImpl stubIndex = (StubIndexImpl)StubIndex.getInstance(); try { diff --git a/platform/lang-impl/src/com/intellij/util/indexing/ChangeTrackingValueContainer.java b/platform/lang-impl/src/com/intellij/util/indexing/ChangeTrackingValueContainer.java index 209090e472..6182e04627 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/ChangeTrackingValueContainer.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/ChangeTrackingValueContainer.java @@ -1,6 +1,8 @@ package com.intellij.util.indexing; import com.intellij.openapi.util.Computable; +import gnu.trove.TIntHashSet; +import gnu.trove.TIntProcedure; import java.util.Iterator; import java.util.List; @@ -12,6 +14,7 @@ import java.util.List; class ChangeTrackingValueContainer extends UpdatableValueContainer{ private final ValueContainerImpl myAdded; private final ValueContainerImpl myRemoved; + private final TIntHashSet myInvalidated; private final Initializer myInitializer; private volatile ValueContainerImpl myMerged = null; @@ -23,6 +26,7 @@ class ChangeTrackingValueContainer extends UpdatableValueContainer myInitializer = initializer; myAdded = new ValueContainerImpl(); myRemoved = new ValueContainerImpl(); + myInvalidated = new TIntHashSet(); } //public void log(String op, int id, final Value value) { @@ -44,12 +48,23 @@ class ChangeTrackingValueContainer extends UpdatableValueContainer } } + public void removeAllValues(int inputId) { + if (myMerged != null) { + myMerged.removeAllValues(inputId); + } + myAdded.removeAllValues(inputId); + myRemoved.removeAllValues(inputId); + myInvalidated.add(inputId); + } + public boolean removeValue(int inputId, Value value) { if (myMerged != null) { myMerged.removeValue(inputId, value); } if (!myAdded.removeValue(inputId, value)) { - myRemoved.addValue(inputId, value); + if (!myInvalidated.contains(inputId)) { + myRemoved.addValue(inputId, value); + } } return true; } @@ -105,6 +120,12 @@ class ChangeTrackingValueContainer extends UpdatableValueContainer }; fromDisk.forEach(addAction); + myInvalidated.forEach(new TIntProcedure() { + public boolean execute(int inputId) { + newMerged.removeAllValues(inputId); + return true; + } + }); myRemoved.forEach(removeAction); myAdded.forEach(addAction); setNeedsCompacting(fromDisk.needsCompacting()); @@ -115,7 +136,7 @@ class ChangeTrackingValueContainer extends UpdatableValueContainer } public boolean isDirty() { - return myAdded.size() > 0 || myRemoved.size() > 0 || needsCompacting(); + return myAdded.size() > 0 || myRemoved.size() > 0 || myInvalidated.size() > 0 || needsCompacting(); } public ValueContainer getAddedDelta() { @@ -125,4 +146,8 @@ class ChangeTrackingValueContainer extends UpdatableValueContainer public ValueContainer getRemovedDelta() { return myRemoved; } + + public TIntHashSet getInvalidated() { + return myInvalidated; + } } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndex.java index 4949436ed6..98e5bcb4e1 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndex.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndex.java @@ -12,9 +12,11 @@ import com.intellij.openapi.editor.highlighter.EditorHighlighter; import com.intellij.openapi.editor.impl.DocumentImpl; import com.intellij.openapi.extensions.Extensions; import com.intellij.openapi.fileEditor.FileDocumentManager; -import com.intellij.openapi.fileEditor.impl.LoadTextUtil; import com.intellij.openapi.fileTypes.*; -import com.intellij.openapi.progress.*; +import com.intellij.openapi.progress.ProcessCanceledException; +import com.intellij.openapi.progress.ProgressIndicator; +import com.intellij.openapi.progress.ProgressManager; +import com.intellij.openapi.progress.Task; import com.intellij.openapi.progress.impl.BackgroundableProcessIndicator; import com.intellij.openapi.project.*; import com.intellij.openapi.roots.CollectingContentIterator; @@ -45,7 +47,7 @@ import com.intellij.util.concurrency.LockFactory; import com.intellij.util.concurrency.Semaphore; import com.intellij.util.containers.ConcurrentHashSet; import com.intellij.util.containers.ContainerUtil; -import com.intellij.util.io.IOUtil; +import com.intellij.util.io.*; import com.intellij.util.messages.MessageBus; import com.intellij.util.messages.MessageBusConnection; import gnu.trove.TIntHashSet; @@ -84,12 +86,6 @@ public class FileBasedIndex implements ApplicationComponent { return 0L; } }; - private final PerIndexDocumentMap myLastIndexedUnsavedContent = new PerIndexDocumentMap() { - @Override - protected CharSequence createDefault(Document document) { - return loadContent(myFileDocumentManager.getFile(document)); - } - }; private final ChangedFilesUpdater myChangedFilesUpdater; @@ -105,12 +101,10 @@ public class FileBasedIndex implements ApplicationComponent { private final ConcurrentHashSet> myUpToDateIndices = new ConcurrentHashSet>(); private final Map myTransactionMap = new HashMap(); - private final FileContentStorage myFileContentAttic; - private static final int ALREADY_PROCESSED = 0x02; public void requestReindex(final VirtualFile file) { - myChangedFilesUpdater.scheduleInvalidation(file, true); + myChangedFilesUpdater.invalidateIndices(file, true); } public interface InputFilter { @@ -228,7 +222,6 @@ public class FileBasedIndex implements ApplicationComponent { vfManager.addVirtualFileListener(myChangedFilesUpdater); vfManager.registerRefreshUpdater(myChangedFilesUpdater); - myFileContentAttic = new FileContentStorage(new File(PathManager.getIndexRoot(), "updates.tmp")); registerIndexableSet(new AdditionalIndexableFileSet()); } @@ -336,11 +329,40 @@ public class FileBasedIndex implements ApplicationComponent { return new File(PathManager.getIndexRoot(), "work_in_progress"); } - private UpdatableIndex createIndex(final ID indexId, final FileBasedIndexExtension extension, final IndexStorage storage) { + private UpdatableIndex createIndex(final ID indexId, final FileBasedIndexExtension extension, final IndexStorage storage) throws IOException { + final MapReduceIndex index; if (extension instanceof CustomImplementationFileBasedIndexExtension) { - return ((CustomImplementationFileBasedIndexExtension)extension).createIndexImplementation(indexId, this, storage); + final UpdatableIndex custom = + ((CustomImplementationFileBasedIndexExtension)extension).createIndexImplementation(indexId, this, storage); + if (!(custom instanceof MapReduceIndex)) { + return custom; + } + index = (MapReduceIndex)custom; } - return new MapReduceIndex(indexId, extension.getIndexer(), storage); + else { + index = new MapReduceIndex(indexId, extension.getIndexer(), storage); + } + + final KeyDescriptor keyDescriptor = extension.getKeyDescriptor(); + index.setInputIdToDataKeysIndex(new PersistentHashMap>(IndexInfrastructure.getInputIndexStorageFile(indexId), new EnumeratorIntegerDescriptor(), new DataExternalizer>() { + public void save(DataOutput out, Collection value) throws IOException { + DataInputOutputUtil.writeINT(out, value.size()); + for (K key : value) { + keyDescriptor.save(out, key); + } + } + + public Collection read(DataInput in) throws IOException { + final int size = DataInputOutputUtil.readINT(in); + final List list = new ArrayList(); + for (int idx = 0; idx < size; idx++) { + list.add(keyDescriptor.read(in)); + } + return list; + } + })); + + return index; } @NonNls @@ -378,7 +400,6 @@ public class FileBasedIndex implements ApplicationComponent { myVfManager.removeVirtualFileListener(myChangedFilesUpdater); myVfManager.unregisterRefreshUpdater(myChangedFilesUpdater); - myFileContentAttic.dispose(); FileUtil.delete(getMarkerFile()); } @@ -485,8 +506,6 @@ public class FileBasedIndex implements ApplicationComponent { myReentrancyGuard.set(Boolean.TRUE); try { - myChangedFilesUpdater.ensureAllInvalidateTasksCompleted(); - if (isUpToDateCheckEnabled()) { try { checkRebuild(indexId, false); @@ -968,20 +987,15 @@ public class FileBasedIndex implements ApplicationComponent { ? ((DocumentImpl)document).getEditorHighlighterForCachesBuilding() : null); } - CharSequence lastIndexed = myLastIndexedUnsavedContent.get(document, requestedIndexId); - final FileContent oldFc = new FileContent(vFile, lastIndexed, vFile.getCharset()); if (getInputFilter(requestedIndexId).acceptInput(vFile)) { - oldFc.putUserData(PROJECT, project); newFc.putUserData(PROJECT, project); final int inputId = Math.abs(getFileId(vFile)); - getIndex(requestedIndexId).update(inputId, newFc, oldFc); + getIndex(requestedIndexId).update(inputId, newFc); } if (dominantContentFile != null) { dominantContentFile.putUserData(PsiFileImpl.BUILDING_STUB, null); } - - myLastIndexedUnsavedContent.put(document, requestedIndexId, newFc.getContentAsText()); } } @@ -1006,7 +1020,6 @@ public class FileBasedIndex implements ApplicationComponent { if (!enabled) { synchronized (myLastIndexedDocStamps) { myLastIndexedDocStamps.clear(); - myLastIndexedUnsavedContent.clear(); } } for (ID indexId : myIndices.keySet()) { @@ -1046,18 +1059,14 @@ public class FileBasedIndex implements ApplicationComponent { } public void indexFileContent(com.intellij.ide.startup.FileContent content) { - myChangedFilesUpdater.ensureAllInvalidateTasksCompleted(); final VirtualFile file = content.getVirtualFile(); FileContent fc = null; - FileContent oldContent = null; - final byte[] oldBytes = myFileContentAttic.remove(file); - final boolean forceIndexing = oldBytes != null; PsiFile psiFile = null; //final Job job = JobScheduler.getInstance().createJob("IndexJob", Job.DEFAULT_PRIORITY / 2); for (final ID indexId : myIndices.keySet()) { - if (forceIndexing ? getInputFilter(indexId).acceptInput(file) : shouldIndexFile(file, indexId)) { + if (shouldIndexFile(file, indexId)) { if (fc == null) { byte[] currentBytes; try { @@ -1078,23 +1087,21 @@ public class FileBasedIndex implements ApplicationComponent { project = ProjectUtil.guessProjectForFile(file); } fc.putUserData(PROJECT, project); - oldContent = oldBytes != null? new FileContent(file, oldBytes) : null; } - final FileContent _oldContent = oldContent; final FileContent _fc = fc; //job.addTask(new Runnable() { // public void run() { try { - updateSingleIndex(indexId, file, _fc, _oldContent); + updateSingleIndex(indexId, file, _fc); } catch (StorageException e) { requestRebuild(indexId); LOG.info(e); } - } + //} //}); - //} + } } //try { @@ -1109,7 +1116,7 @@ public class FileBasedIndex implements ApplicationComponent { } } - private void updateSingleIndex(final ID indexId, final VirtualFile file, final FileContent currentFC, final FileContent oldFC) + private void updateSingleIndex(final ID indexId, final VirtualFile file, final FileContent currentFC) throws StorageException { if (myRebuildStatus.get(indexId).get() == REQUIRES_REBUILD) { return; // the index is scheduled for rebuild, no need to update @@ -1123,10 +1130,7 @@ public class FileBasedIndex implements ApplicationComponent { final UpdatableIndex index = getIndex(indexId); assert index != null; - //if (indexId.toString().equals("js.index")) { - // System.out.println(inputId + "; FILE:" + file.getPath()); - //} - index.update(inputId, currentFC, oldFC); + index.update(inputId, currentFC); ApplicationManager.getApplication().runReadAction(new Runnable() { public void run() { if (file.isValid()) { @@ -1154,30 +1158,12 @@ public class FileBasedIndex implements ApplicationComponent { throw new IllegalArgumentException("Virtual file doesn't support id: " + file + ", implementation class: " + file.getClass().getName()); } - private static CharSequence loadContent(VirtualFile file) { - return LoadTextUtil.loadText(file, true); - } - private boolean needsFileContentLoading(ID indexId) { return !myNotRequiringContentIndices.contains(indexId); } - private abstract static class InvalidationTask implements Runnable { - private final VirtualFile mySubj; - - protected InvalidationTask(final VirtualFile subj) { - mySubj = subj; - } - - public VirtualFile getSubj() { - return mySubj; - } - } - private final class ChangedFilesUpdater extends VirtualFileAdapter implements CacheUpdater{ private final Set myFilesToUpdate = new LinkedHashSet(); - private final Queue myFutureInvalidations = new LinkedList(); - private final JBReentrantReadWriteLock myLock = LockFactory.createReadWriteLock(); private final JBLock r = myLock.readLock(); private final JBLock w = myLock.writeLock(); @@ -1204,11 +1190,11 @@ public class FileBasedIndex implements ApplicationComponent { } public void beforeFileDeletion(final VirtualFileEvent event) { - scheduleInvalidation(event.getFile(), false); + invalidateIndices(event.getFile(), false); } public void beforeContentsChange(final VirtualFileEvent event) { - scheduleInvalidation(event.getFile(), true); + invalidateIndices(event.getFile(), true); } public void contentsChanged(final VirtualFileEvent event) { @@ -1222,7 +1208,7 @@ public class FileBasedIndex implements ApplicationComponent { if (!file.isDirectory()) { // name change may lead to filetype change so the file might become not indexable // in general case have to 'unindex' the file and index it again if needed after the name has been changed - scheduleInvalidation(file, false); + invalidateIndices(file, false); } } } @@ -1248,7 +1234,7 @@ public class FileBasedIndex implements ApplicationComponent { if (fileContent == null) { fileContent = new FileContent(file); } - updateSingleIndex(indexId, file, fileContent, null); + updateSingleIndex(indexId, file, fileContent); } catch (StorageException e) { LOG.info(e); @@ -1278,169 +1264,64 @@ public class FileBasedIndex implements ApplicationComponent { IndexingStamp.flushCache(); } - public void scheduleInvalidation(final VirtualFile file, final boolean saveContent) { + void invalidateIndices(final VirtualFile file, final boolean markForReindex) { if (file.isDirectory()) { if (isMock(file) || myManagingFS.wereChildrenAccessed(file)) { for (VirtualFile child : file.getChildren()) { - scheduleInvalidation(child, saveContent); + invalidateIndices(child, markForReindex); } } } else { cleanProcessedFlag(file); - FileContent fileContent = null; IndexingStamp.flushCache(); + boolean indicesAffected = false; - for (ID indexId : myNotRequiringContentIndices) { - if (shouldUpdateIndex(file, indexId)) { - // invalidate it synchronously - try { - if (fileContent == null) { - fileContent = new FileContent(file); + final boolean isTooLarge = isTooLarge(file); + for (final ID indexId : myIndices.keySet()) { + try { + if (myNotRequiringContentIndices.contains(indexId)) { + if (shouldUpdateIndex(file, indexId)) { + updateSingleIndex(indexId, file, null); } - updateSingleIndex(indexId, file, null, fileContent); } - catch (StorageException e) { - LOG.info(e); - requestRebuild(indexId); - } - } - } - - if (!isTooLarge(file)) { - final List> affectedIndices = new ArrayList>(myIndices.size()); - - for (ID indexId : myIndices.keySet()) { - if (needsFileContentLoading(indexId) && shouldUpdateIndex(file, indexId)) { - if (saveContent) { - try { - myFileContentAttic.offer(file); - iterateIndexableFiles(file, new Processor() { - public boolean process(final VirtualFile file) { - w.lock(); - try { - myFilesToUpdate.add(file); - } - finally { - w.unlock(); - } - return true; + else { // the index requires file content + if (!isTooLarge && shouldUpdateIndex(file, indexId)) { + indicesAffected = true; + if (markForReindex) { + // only mark the file as unindexed, reindex will be done lazily + ApplicationManager.getApplication().runReadAction(new Runnable() { + public void run() { + IndexingStamp.update(file, indexId, -1L); } }); } - catch (IOException e) { - LOG.info(e); - for (ID id : myIndices.keySet()) { - if (needsFileContentLoading(id) && shouldUpdateIndex(file, id)) { - requestRebuild(indexId); - } - } + else { + updateSingleIndex(indexId, file, null); } - break; // no need to iterate further - } - else { - affectedIndices.add(indexId); } } } - - if (!affectedIndices.isEmpty()) { - // first check if there is an unprocessed content from previous events - byte[] content = myFileContentAttic.remove(file); - try { - if (content == null) { - content = file.contentsToByteArray(); - } - } - catch (IOException e) { - content = ArrayUtil.EMPTY_BYTE_ARRAY; - } - final FileContent fc = new FileContent(file, content); - - final InvalidationTask invalidator = new InvalidationTask(file) { - public void run() { - Throwable unexpectedError = null; - for (ID indexId : affectedIndices) { - try { - updateSingleIndex(indexId, file, null, fc); - } - catch (StorageException e) { - LOG.info(e); - requestRebuild(indexId); - } - catch (ProcessCanceledException ignored) { - } - catch (Throwable e) { - LOG.info(e); - if (unexpectedError == null) { - unexpectedError = e; - } - } - } - - IndexingStamp.flushCache(); - if (unexpectedError != null) { - LOG.error(unexpectedError); - } - } - }; - - w.lock(); - try { - myFutureInvalidations.offer(invalidator); - } - finally { - w.unlock(); - } + catch (StorageException e) { + LOG.info(e); + requestRebuild(indexId); } } - } - } - - public void ensureAllInvalidateTasksCompleted() { - final boolean doProgressThing; - final int size; - - r.lock(); - try { - size = myFutureInvalidations.size(); - if (size == 0) return; - - doProgressThing = size > 1 && ApplicationManager.getApplication().isDispatchThread() && !Thread.holdsLock(PsiLock.LOCK); - } - finally { - r.unlock(); - } - - - final Task.Modal task = new Task.Modal(null, "Invalidating Index Entries", false) { - public void run(@NotNull final ProgressIndicator indicator) { - indicator.setText(""); - int count = 0; - while (true) { - InvalidationTask r; - w.lock(); - try { - r = myFutureInvalidations.poll(); - } - finally { - w.unlock(); + IndexingStamp.flushCache(); + if (indicesAffected && markForReindex) { + iterateIndexableFiles(file, new Processor() { + public boolean process(final VirtualFile file) { + w.lock(); + try { + myFilesToUpdate.add(file); + } + finally { + w.unlock(); + } + return true; } - - if (r == null) return; - indicator.setFraction(((double)count++)/size); - indicator.setText2(r.getSubj().getPresentableUrl()); - r.run(); - } + }); } - }; - - if (doProgressThing) { - task.queue(); - } - else { - ProgressIndicator indicator = ProgressManager.getInstance().getProgressIndicator(); - task.run(indicator != null ? indicator : new EmptyProgressIndicator()); } } @@ -1481,15 +1362,12 @@ public class FileBasedIndex implements ApplicationComponent { } public void processFile(final com.intellij.ide.startup.FileContent fileContent) { - ensureAllInvalidateTasksCompleted(); processFileImpl(fileContent); } private final Semaphore myForceUpdateSemaphore = new Semaphore(); public void forceUpdate() { - ensureAllInvalidateTasksCompleted(); - final VirtualFile[] files = queryNeededFiles(); if (files.length > 0) { myForceUpdateSemaphore.down(); @@ -1551,7 +1429,7 @@ public class FileBasedIndex implements ApplicationComponent { if (!isTooLarge(file)) { for (ID indexId : myIndices.keySet()) { try { - if (myFileContentAttic.containsContent(file) ? getInputFilter(indexId).acceptInput(file) : shouldIndexFile(file, indexId)) { + if (shouldIndexFile(file, indexId)) { myFiles.add(file); oldStuff = false; break; @@ -1577,7 +1455,7 @@ public class FileBasedIndex implements ApplicationComponent { if (fileContent == null) { fileContent = new FileContent(file); } - updateSingleIndex(indexId, file, fileContent, null); + updateSingleIndex(indexId, file, fileContent); } catch (StorageException e) { LOG.info(e); diff --git a/platform/lang-impl/src/com/intellij/util/indexing/IndexInfrastructure.java b/platform/lang-impl/src/com/intellij/util/indexing/IndexInfrastructure.java index aebc2ad189..4895a78a2a 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/IndexInfrastructure.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/IndexInfrastructure.java @@ -19,7 +19,7 @@ import java.util.Locale; @SuppressWarnings({"HardCodedStringLiteral"}) public class IndexInfrastructure { - private static final int VERSION = 7; + private static final int VERSION = 8; private static final TObjectLongHashMap> ourIndexIdToCreationStamp = new TObjectLongHashMap>(); private static final boolean ourUnitTestMode = ApplicationManager.getApplication().isUnitTestMode(); @@ -34,6 +34,10 @@ public class IndexInfrastructure { return new File(getIndexRootDir(indexName), indexName.toString()); } + public static File getInputIndexStorageFile(final ID indexName) { + return new File(getIndexRootDir(indexName), indexName.toString()+"_inputs"); + } + public static File getIndexRootDir(final ID indexName) { final String dirName = indexName.toString().toLowerCase(Locale.US); // store StubIndices under StubUpdating index' root to ensure they are deleted diff --git a/platform/lang-impl/src/com/intellij/util/indexing/IndexStorage.java b/platform/lang-impl/src/com/intellij/util/indexing/IndexStorage.java index cb2fe4d16f..89b1ba2dcc 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/IndexStorage.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/IndexStorage.java @@ -16,6 +16,8 @@ public interface IndexStorage extends Flushable{ void removeValue(Key key, int inputId, Value value) throws StorageException; + void removeAllValues(Key key, int inputId) throws StorageException; + void remove(Key key) throws StorageException; void clear() throws StorageException; diff --git a/platform/lang-impl/src/com/intellij/util/indexing/MapIndexStorage.java b/platform/lang-impl/src/com/intellij/util/indexing/MapIndexStorage.java index 21978549cc..50b7ec4840 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/MapIndexStorage.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/MapIndexStorage.java @@ -9,6 +9,7 @@ import com.intellij.util.io.DataExternalizer; import com.intellij.util.io.DataInputOutputUtil; import com.intellij.util.io.KeyDescriptor; import com.intellij.util.io.PersistentHashMap; +import gnu.trove.TIntHashSet; import org.jetbrains.annotations.NotNull; import java.io.*; @@ -77,7 +78,12 @@ public final class MapIndexStorage implements IndexStorage 0) { + for (int inputId : set.toArray()) { + myValueContainerExternalizer.saveInvalidateCommand(_out, inputId); + } + } final ValueContainer toRemove = valueContainer.getRemovedDelta(); if (toRemove.size() > 0) { myValueContainerExternalizer.saveAsRemoved(_out, toRemove); @@ -213,6 +219,10 @@ public final class MapIndexStorage implements IndexStorage implements IndexStorage container, final boolean asRemovedData) throws IOException { + DataInputOutputUtil.writeSINT(out, container.size()); for (final Iterator valueIterator = container.getValueIterator(); valueIterator.hasNext();) { final T value = valueIterator.next(); myExternalizer.save(out, value); @@ -276,16 +291,25 @@ public final class MapIndexStorage implements IndexStorage valueContainer = new ValueContainerImpl(); while (stream.available() > 0) { - final T value = myExternalizer.read(in); - final int idCount = DataInputOutputUtil.readSINT(in); - for (int i = 0; i < idCount; i++) { - final int id = DataInputOutputUtil.readSINT(in); - if (id < 0) { - valueContainer.removeValue(-id, value); - valueContainer.setNeedsCompacting(true); - } - else { - valueContainer.addValue(id, value); + final int valueCount = DataInputOutputUtil.readSINT(in); + if (valueCount < 0) { + valueContainer.removeAllValues(-valueCount); + valueContainer.setNeedsCompacting(true); + } + else { + for (int valueIdx = 0; valueIdx < valueCount; valueIdx++) { + final T value = myExternalizer.read(in); + final int idCount = DataInputOutputUtil.readSINT(in); + for (int i = 0; i < idCount; i++) { + final int id = DataInputOutputUtil.readSINT(in); + if (id < 0) { + valueContainer.removeValue(-id, value); + valueContainer.setNeedsCompacting(true); + } + else { + valueContainer.addValue(id, value); + } + } } } } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java index e1b62d9805..a13e6cac86 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java @@ -4,6 +4,7 @@ import com.intellij.openapi.diagnostic.Logger; import com.intellij.util.Alarm; import com.intellij.util.CommonProcessors; import com.intellij.util.Processor; +import com.intellij.util.io.PersistentHashMap; import com.intellij.util.io.storage.HeavyProcessLatch; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -22,7 +23,8 @@ public class MapReduceIndex implements UpdatableIndex myIndexId; private final DataIndexer myIndexer; private final IndexStorage myStorage; - + private PersistentHashMap> myInputsIndex; + private final ReentrantReadWriteLock myLock = new ReentrantReadWriteLock(); private final Alarm myFlushAlarm = new Alarm(Alarm.ThreadToUse.SHARED_THREAD); @@ -105,7 +107,19 @@ public class MapReduceIndex implements UpdatableIndex implements UpdatableIndex oldData = mapOld(oldContent); + public void setInputIdToDataKeysIndex(PersistentHashMap> metaIndex) { + myInputsIndex = metaIndex; + } + + public final void update(final int inputId, @Nullable Input content) throws StorageException { + //final Map oldData = mapOld(oldContent); + assert myInputsIndex != null; + final Map data = mapNew(content); - updateWithMap(inputId, oldData, data); + Collection oldKeys; + try { + oldKeys = myInputsIndex.get(inputId); + } + catch (IOException e) { + throw new StorageException(e); + } + if (oldKeys == null) { + oldKeys = Collections.emptyList(); + } + getWriteLock().lock(); + try { + // remove outdated values + updateWithMap(inputId, data, oldKeys); + try { + final Set newKeys = data.keySet(); + if (newKeys.size() > 0) { + myInputsIndex.put(inputId, newKeys); + } + else { + myInputsIndex.remove(inputId); + } + } + catch (IOException e) { + throw new StorageException(e); + } + } + finally { + getWriteLock().unlock(); + } } protected Map mapNew(final Input content) throws StorageException { return content != null ? myIndexer.map(content) : Collections.emptyMap(); } - protected Map mapOld(final Input oldContent) throws StorageException { - return oldContent != null ? myIndexer.map(oldContent) : Collections.emptyMap(); - } - - protected void updateWithMap(final int inputId, final Map oldData, final Map newData) throws StorageException { - final Set allKeys = new HashSet(oldData.size() + newData.size()); - allKeys.addAll(oldData.keySet()); - allKeys.addAll(newData.keySet()); - - if (allKeys.size() > 0) { - final Lock writeLock = getWriteLock(); - - //if (getClass().getName().contains("StubUpdatingIndex")) { - // final VirtualFile file = IndexInfrastructure.findFileById((PersistentFS)ManagingFS.getInstance(), inputId); - // System.out.print("FILE [" + inputId+ "]=" + (file != null? file.getPresentableUrl() : "null")); - // if (oldData.containsKey(inputId)) { - // System.out.print(" REMOVE"); - // } - // if (newData.containsKey(inputId)) { - // System.out.print(" ADD"); - // } - // System.out.println(""); - // if (allKeys.size() > 1) { - // System.out.println("More than one key for stub updating index: " + allKeys); - // } - //} - - for (Key key : allKeys) { - assert key != null : "Null keys are not allowed. Index: " + myIndexId; - // remove outdated values - try { - writeLock.lock(); - if (oldData.containsKey(key)) { - final Value oldValue = oldData.get(key); - //if (myIndexId != null && "js.index".equals(myIndexId.toString())) { - // System.out.println(key + ": BEFORE REMOVE: " + myStorage.read(key).toValueList()); - //} - myStorage.removeValue(key, inputId, oldValue); - //if (myIndexId != null && "js.index".equals(myIndexId.toString())) { - // System.out.println(key + ": AFTER REMOVE: " + myStorage.read(key).toValueList()); - //} - } - // add new values - if (newData.containsKey(key)) { - final Value newValue = newData.get(key); - //if (myIndexId != null && "js.index".equals(myIndexId.toString())) { - // System.out.println(key + ": BEFORE ADD: " + myStorage.read(key).toValueList()); - //} - myStorage.addValue(key, inputId, newValue); - //if (myIndexId != null && "js.index".equals(myIndexId.toString())) { - // System.out.println(key + ": AFTER ADD: " + myStorage.read(key).toValueList()); - //} - } - } - finally { - writeLock.unlock(); - } - } - scheduleFlush(); + protected void updateWithMap(final int inputId, final Map newData, Collection oldKeys) throws StorageException { + for (Key key : oldKeys) { + myStorage.removeAllValues(key, inputId); + } + // add new values + for (Key key : newData.keySet()) { + final Value newValue = newData.get(key); + myStorage.addValue(key, inputId, newValue); } + scheduleFlush(); } private void scheduleFlush() { diff --git a/platform/lang-impl/src/com/intellij/util/indexing/MemoryIndexStorage.java b/platform/lang-impl/src/com/intellij/util/indexing/MemoryIndexStorage.java index edcf90deab..64a39b8888 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/MemoryIndexStorage.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/MemoryIndexStorage.java @@ -110,6 +110,15 @@ public class MemoryIndexStorage implements IndexStorage } } + public void removeAllValues(Key key, int inputId) throws StorageException { + if (myBufferingEnabled.get()) { + getMemValueContainer(key).removeAllValues(inputId); + } + else { + myBackendStorage.removeAllValues(key, inputId); + } + } + private synchronized UpdatableValueContainer getMemValueContainer(final Key key) { UpdatableValueContainer valueContainer = myMap.get(key); if (valueContainer == null) { diff --git a/platform/lang-impl/src/com/intellij/util/indexing/SingleEntryFileBasedIndexExtension.java b/platform/lang-impl/src/com/intellij/util/indexing/SingleEntryFileBasedIndexExtension.java dissimilarity index 75% index 5df958ed0d..190ad801b7 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/SingleEntryFileBasedIndexExtension.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/SingleEntryFileBasedIndexExtension.java @@ -1,63 +1,22 @@ -package com.intellij.util.indexing; - -import com.intellij.openapi.diagnostic.Logger; -import com.intellij.util.containers.HashMap; -import com.intellij.util.io.EnumeratorIntegerDescriptor; -import com.intellij.util.io.KeyDescriptor; - -import java.util.Collections; -import java.util.Map; -import java.util.concurrent.locks.Lock; - -/** - * Base implementation for indices that produce single value per single file - * - * @author Eugene Zhuravlev - * Date: Feb 18, 2009 - */ -public abstract class SingleEntryFileBasedIndexExtension extends CustomImplementationFileBasedIndexExtension{ - private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.SingleEntryFileBasedIndexExtension"); - - public final KeyDescriptor getKeyDescriptor() { - return new EnumeratorIntegerDescriptor(); - } - - public boolean dependsOnFileContent() { - return true; - } - - public abstract SingleEntryIndexer getIndexer(); - - // optimization: since there is always only one value per file, we can easily - // get old value from index storage instead of calculating it with the indexer - public final UpdatableIndex createIndexImplementation(ID indexId, - FileBasedIndex owner, - IndexStorage integerVIndexStorage) { - return new MapReduceIndex(indexId, getIndexer(), integerVIndexStorage) { - protected Map mapOld(FileContent inputData) throws StorageException { - if (inputData == null) { - return Collections.emptyMap(); - } - final int key = Math.abs(FileBasedIndex.getFileId(inputData.getFile())); - - final Map result = new HashMap(); - final Lock lock = getReadLock(); - try { - lock.lock(); - final ValueContainer valueContainer = getData(key); - if (valueContainer.size() != 1) { - LOG.assertTrue(valueContainer.size() == 0); - return result; - } - - result.put(key, valueContainer.getValueIterator().next()); - } - finally { - lock.unlock(); - } - - return result; - } - }; - } -} +package com.intellij.util.indexing; + +import com.intellij.util.io.EnumeratorIntegerDescriptor; +import com.intellij.util.io.KeyDescriptor; + +/** + * Base implementation for indices that produce single value per single file + * + * @author Eugene Zhuravlev + * Date: Feb 18, 2009 + */ +public abstract class SingleEntryFileBasedIndexExtension extends FileBasedIndexExtension{ + public final KeyDescriptor getKeyDescriptor() { + return new EnumeratorIntegerDescriptor(); + } + + public boolean dependsOnFileContent() { + return true; + } + + public abstract SingleEntryIndexer getIndexer(); +} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/UpdatableIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/UpdatableIndex.java index c9bac4b12c..686ea81bf9 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/UpdatableIndex.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/UpdatableIndex.java @@ -16,7 +16,7 @@ public interface UpdatableIndex extends AbstractIndex extends ValueContainer{ public abstract void addValue(int inputId, T value); public abstract boolean removeValue(int inputId, T value); + + public abstract void removeAllValues(int inputId); } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/ValueContainerImpl.java b/platform/lang-impl/src/com/intellij/util/indexing/ValueContainerImpl.java index bd3c2b6f13..77235fe9ab 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/ValueContainerImpl.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/ValueContainerImpl.java @@ -41,6 +41,19 @@ class ValueContainerImpl extends UpdatableValueContainer implement return myInputIdMapping.size(); } + public void removeAllValues(int inputId) { + final List toRemove= new ArrayList(); + for (final Iterator valueIterator = getValueIterator(); valueIterator.hasNext();) { + final Value value = valueIterator.next(); + if (isAssociated(value, inputId)) { + toRemove.add(value); + } + } + for (Value value : toRemove) { + removeValue(inputId, value); + } + } + public boolean removeValue(int inputId, Value value) { final Object input = myInputIdMapping.get(value); if (input == null) { -- 2.11.4.GIT