platform/lang-impl/src/com/intellij/util/indexing/MapIndexStorage.java

   1 package com.intellij.util.indexing;
   2
   3 import com.intellij.openapi.diagnostic.Logger;
   4 import com.intellij.openapi.util.io.FileUtil;
   5 import com.intellij.util.CommonProcessors;
   6 import com.intellij.util.Processor;
   7 import com.intellij.util.containers.SLRUCache;
   8 import com.intellij.util.io.DataExternalizer;
   9 import com.intellij.util.io.DataInputOutputUtil;
  10 import com.intellij.util.io.KeyDescriptor;
  11 import com.intellij.util.io.PersistentHashMap;
  12 import gnu.trove.TIntHashSet;
  13 import org.jetbrains.annotations.NotNull;
  14
  15 import java.io.*;
  16 import java.util.ArrayList;
  17 import java.util.Collection;
  18 import java.util.Iterator;
  19 import java.util.List;
  20
  21 /**
  22  * @author Eugene Zhuravlev
  23 *         Date: Dec 20, 2007
  24 */
  25 public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Value>{
  26   private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.MapIndexStorage");
  27   private PersistentHashMap<Key, ValueContainer<Value>> myMap;
  28   private SLRUCache<Key, ChangeTrackingValueContainer<Value>> myCache;
  29   private Key myKeyBeingRemoved = null;
  30   private final File myStorageFile;
  31   private final KeyDescriptor<Key> myKeyDescriptor;
  32   private final ValueContainerExternalizer<Value> myValueContainerExternalizer;
  33   private final int myCacheSize;
  34
  35   public MapIndexStorage(File storageFile, final KeyDescriptor<Key> keyDescriptor, final DataExternalizer<Value> valueExternalizer,
  36                          final int cacheSize) throws IOException {
  37
  38     myStorageFile = storageFile;
  39     myKeyDescriptor = keyDescriptor;
  40     myValueContainerExternalizer = new ValueContainerExternalizer<Value>(valueExternalizer);
  41     myCacheSize = cacheSize;
  42     initMapAndCache();
  43   }
  44
  45   private void initMapAndCache() throws IOException {
  46     final PersistentHashMap<Key, ValueContainer<Value>> map =
  47       new PersistentHashMap<Key, ValueContainer<Value>>(myStorageFile, myKeyDescriptor, myValueContainerExternalizer);
  48     myCache = new SLRUCache<Key, ChangeTrackingValueContainer<Value>>(myCacheSize, (int)(Math.ceil(myCacheSize * 0.25)) /* 25% from the main cache size*/) {
  49       @NotNull
  50       public ChangeTrackingValueContainer<Value> createValue(final Key key) {
  51         return new ChangeTrackingValueContainer<Value>(new ChangeTrackingValueContainer.Initializer<Value>() {
  52           public Object getLock() {
  53             return map;
  54           }
  55
  56           public ValueContainer<Value> compute() {
  57             ValueContainer<Value> value = null;
  58             try {
  59               value = map.get(key);
  60               if (value == null) {
  61                 value = new ValueContainerImpl<Value>();
  62               }
  63             }
  64             catch (IOException e) {
  65               throw new RuntimeException(e);
  66             }
  67             return value;
  68           }
  69         });
  70       }
  71
  72       protected void onDropFromCache(final Key key, final ChangeTrackingValueContainer<Value> valueContainer) {
  73         if (key.equals(myKeyBeingRemoved) || !valueContainer.isDirty()) {
  74           return;
  75         }
  76         try {
  77           if (!valueContainer.needsCompacting()) {
  78             final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
  79             //noinspection IOResourceOpenedButNotSafelyClosed
  80             final DataOutputStream _out = new DataOutputStream(bytes);
  81             final TIntHashSet set = valueContainer.getInvalidated();
  82             if (set.size() > 0) {
  83               for (int inputId : set.toArray()) {
  84                 myValueContainerExternalizer.saveInvalidateCommand(_out, inputId);
  85               }
  86             }
  87             final ValueContainer<Value> toRemove = valueContainer.getRemovedDelta();
  88             if (toRemove.size() > 0) {
  89               myValueContainerExternalizer.saveAsRemoved(_out, toRemove);
  90             }
  91
  92             final ValueContainer<Value> toAppend = valueContainer.getAddedDelta();
  93             if (toAppend.size() > 0) {
  94               myValueContainerExternalizer.save(_out, toAppend);
  95             }
  96
  97             map.appendData(key, new PersistentHashMap.ValueDataAppender() {
  98               public void append(final DataOutput out) throws IOException {
  99                 final byte[] barr = bytes.toByteArray();
 100                 out.write(barr);
 101               }
 102             });
 103           }
 104           else {
 105             // rewrite the value container for defragmentation
 106             map.put(key, valueContainer);
 107           }
 108         }
 109         catch (IOException e) {
 110           throw new RuntimeException(e);
 111         }
 112       }
 113     };
 114
 115     myMap = map;
 116   }
 117
 118   public synchronized void flush() {
 119     if (!myMap.isClosed()) {
 120       myCache.clear();
 121       myMap.force();
 122     }
 123   }
 124
 125   public synchronized void close() throws StorageException {
 126     try {
 127       flush();
 128       myMap.close();
 129     }
 130     catch (IOException e) {
 131       throw new StorageException(e);
 132     }
 133     catch (RuntimeException e) {
 134       final Throwable cause = e.getCause();
 135       if (cause instanceof IOException) {
 136         throw new StorageException(cause);
 137       }
 138       if (cause instanceof StorageException) {
 139         throw (StorageException)cause;
 140       }
 141       throw e;
 142     }
 143   }
 144
 145   public synchronized void clear() throws StorageException{
 146     try {
 147       myMap.close();
 148     }
 149     catch (IOException e) {
 150       LOG.error(e);
 151     }
 152     try {
 153       FileUtil.delete(myStorageFile);
 154       initMapAndCache();
 155     }
 156     catch (IOException e) {
 157       throw new StorageException(e);
 158     }
 159     catch (RuntimeException e) {
 160       final Throwable cause = e.getCause();
 161       if (cause instanceof IOException) {
 162         throw new StorageException(cause);
 163       }
 164       if (cause instanceof StorageException) {
 165         throw (StorageException)cause;
 166       }
 167       throw e;
 168     }
 169   }
 170
 171   public synchronized boolean processKeys(final Processor<Key> processor) throws StorageException {
 172     try {
 173       myCache.clear(); // this will ensure that all new keys are made into the map
 174       return myMap.processKeys(processor);
 175     }
 176     catch (IOException e) {
 177       throw new StorageException(e);
 178     }
 179     catch (RuntimeException e) {
 180       final Throwable cause = e.getCause();
 181       if (cause instanceof IOException) {
 182         throw new StorageException(cause);
 183       }
 184       if (cause instanceof StorageException) {
 185         throw (StorageException)cause;
 186       }
 187       throw e;
 188     }
 189   }
 190
 191   public Collection<Key> getKeys() throws StorageException {
 192     List<Key> keys = new ArrayList<Key>();
 193     processKeys(new CommonProcessors.CollectProcessor<Key>(keys));
 194     return keys;
 195   }
 196
 197   @NotNull
 198   public synchronized ChangeTrackingValueContainer<Value> read(final Key key) throws StorageException {
 199     try {
 200       return myCache.get(key);
 201     }
 202     catch (RuntimeException e) {
 203       final Throwable cause = e.getCause();
 204       if (cause instanceof IOException) {
 205         throw new StorageException(cause);
 206       }
 207       if (cause instanceof StorageException) {
 208         throw (StorageException)cause;
 209       }
 210       throw e;
 211     }
 212   }
 213
 214   public void addValue(final Key key, final int inputId, final Value value) throws StorageException {
 215     read(key).addValue(inputId, value);
 216   }
 217
 218   public void removeValue(final Key key, final int inputId, final Value value) throws StorageException {
 219     read(key).removeValue(inputId, value);
 220   }
 221
 222   public void removeAllValues(Key key, int inputId) throws StorageException {
 223     read(key).removeAllValues(inputId);
 224   }
 225
 226   public synchronized void remove(final Key key) throws StorageException {
 227     try {
 228       myKeyBeingRemoved = key;
 229       myCache.remove(key);
 230       myMap.remove(key);
 231     }
 232     catch (IOException e) {
 233       throw new StorageException(e);
 234     }
 235     catch (RuntimeException e) {
 236       final Throwable cause = e.getCause();
 237       if (cause instanceof IOException) {
 238         throw new StorageException(cause);
 239       }
 240       if (cause instanceof StorageException) {
 241         throw (StorageException)cause;
 242       }
 243       throw e;
 244     }
 245     finally {
 246       myKeyBeingRemoved = null;
 247     }
 248   }
 249
 250   private static final class ValueContainerExternalizer<T> implements DataExternalizer<ValueContainer<T>> {
 251     private final DataExternalizer<T> myExternalizer;
 252
 253     private ValueContainerExternalizer(DataExternalizer<T> externalizer) {
 254       myExternalizer = externalizer;
 255     }
 256
 257     public void save(final DataOutput out, final ValueContainer<T> container) throws IOException {
 258       saveImpl(out, container, false);
 259     }
 260
 261     public void saveAsRemoved(final DataOutput out, final ValueContainer<T> container) throws IOException {
 262       saveImpl(out, container, true);
 263     }
 264
 265     public void saveInvalidateCommand(final DataOutput out, int inputId) throws IOException {
 266       DataInputOutputUtil.writeSINT(out, -inputId);
 267     }
 268
 269     private void saveImpl(final DataOutput out, final ValueContainer<T> container, final boolean asRemovedData) throws IOException {
 270       DataInputOutputUtil.writeSINT(out, container.size());
 271       for (final Iterator<T> valueIterator = container.getValueIterator(); valueIterator.hasNext();) {
 272         final T value = valueIterator.next();
 273         myExternalizer.save(out, value);
 274
 275         final ValueContainer.IntIterator ids = container.getInputIdsIterator(value);
 276         if (ids != null) {
 277           DataInputOutputUtil.writeSINT(out, ids.size());
 278           while (ids.hasNext()) {
 279             final int id = ids.next();
 280             DataInputOutputUtil.writeSINT(out, asRemovedData ? -id : id);
 281           }
 282         }
 283         else {
 284           DataInputOutputUtil.writeSINT(out, 0);
 285         }
 286       }
 287     }
 288
 289     public ValueContainerImpl<T> read(final DataInput in) throws IOException {
 290       DataInputStream stream = (DataInputStream)in;
 291       final ValueContainerImpl<T> valueContainer = new ValueContainerImpl<T>();
 292
 293       while (stream.available() > 0) {
 294         final int valueCount = DataInputOutputUtil.readSINT(in);
 295         if (valueCount < 0) {
 296           valueContainer.removeAllValues(-valueCount);
 297           valueContainer.setNeedsCompacting(true);
 298         }
 299         else {
 300           for (int valueIdx = 0; valueIdx < valueCount; valueIdx++) {
 301             final T value = myExternalizer.read(in);
 302             final int idCount = DataInputOutputUtil.readSINT(in);
 303             for (int i = 0; i < idCount; i++) {
 304               final int id = DataInputOutputUtil.readSINT(in);
 305               if (id < 0) {
 306                 valueContainer.removeValue(-id, value);
 307                 valueContainer.setNeedsCompacting(true);
 308               }
 309               else {
 310                 valueContainer.addValue(id, value);
 311               }
 312             }
 313           }
 314         }
 315       }
 316       return valueContainer;
 317     }
 318   }
 319 }