platform/lang-impl/src/com/intellij/util/indexing/MapIndexStorage.java

   1 /*
   2  * Copyright 2000-2009 JetBrains s.r.o.
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  * http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 package com.intellij.util.indexing;
  18
  19 import com.intellij.openapi.diagnostic.Logger;
  20 import com.intellij.openapi.util.io.FileUtil;
  21 import com.intellij.util.CommonProcessors;
  22 import com.intellij.util.Processor;
  23 import com.intellij.util.containers.SLRUCache;
  24 import com.intellij.util.io.DataExternalizer;
  25 import com.intellij.util.io.DataInputOutputUtil;
  26 import com.intellij.util.io.KeyDescriptor;
  27 import com.intellij.util.io.PersistentHashMap;
  28 import gnu.trove.TIntHashSet;
  29 import org.jetbrains.annotations.NotNull;
  30
  31 import java.io.*;
  32 import java.util.ArrayList;
  33 import java.util.Collection;
  34 import java.util.Iterator;
  35 import java.util.List;
  36 import java.util.concurrent.locks.Lock;
  37 import java.util.concurrent.locks.ReentrantLock;
  38
  39 /**
  40  * @author Eugene Zhuravlev
  41 *         Date: Dec 20, 2007
  42 */
  43 public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Value>{
  44   private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.MapIndexStorage");
  45   private PersistentHashMap<Key, ValueContainer<Value>> myMap;
  46   private SLRUCache<Key, ChangeTrackingValueContainer<Value>> myCache;
  47   private final File myStorageFile;
  48   private final KeyDescriptor<Key> myKeyDescriptor;
  49   private final ValueContainerExternalizer<Value> myValueContainerExternalizer;
  50   private final int myCacheSize;
  51
  52   private final Lock l = new ReentrantLock();
  53
  54   public MapIndexStorage(File storageFile, final KeyDescriptor<Key> keyDescriptor, final DataExternalizer<Value> valueExternalizer,
  55                          final int cacheSize) throws IOException {
  56
  57     myStorageFile = storageFile;
  58     myKeyDescriptor = keyDescriptor;
  59     myValueContainerExternalizer = new ValueContainerExternalizer<Value>(valueExternalizer);
  60     myCacheSize = cacheSize;
  61     initMapAndCache();
  62   }
  63
  64   private void initMapAndCache() throws IOException {
  65     final PersistentHashMap<Key, ValueContainer<Value>> map =
  66       new PersistentHashMap<Key, ValueContainer<Value>>(myStorageFile, myKeyDescriptor, myValueContainerExternalizer);
  67     myCache = new SLRUCache<Key, ChangeTrackingValueContainer<Value>>(myCacheSize, (int)(Math.ceil(myCacheSize * 0.25)) /* 25% from the main cache size*/) {
  68       @NotNull
  69       public ChangeTrackingValueContainer<Value> createValue(final Key key) {
  70         return new ChangeTrackingValueContainer<Value>(new ChangeTrackingValueContainer.Initializer<Value>() {
  71           public Object getLock() {
  72             return map;
  73           }
  74
  75           public ValueContainer<Value> compute() {
  76             ValueContainer<Value> value = null;
  77             try {
  78               value = map.get(key);
  79               if (value == null) {
  80                 value = new ValueContainerImpl<Value>();
  81               }
  82             }
  83             catch (IOException e) {
  84               throw new RuntimeException(e);
  85             }
  86             return value;
  87           }
  88         });
  89       }
  90
  91       protected void onDropFromCache(final Key key, final ChangeTrackingValueContainer<Value> valueContainer) {
  92         if (!valueContainer.isDirty()) {
  93           return;
  94         }
  95         try {
  96           if (!valueContainer.needsCompacting()) {
  97             final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
  98             //noinspection IOResourceOpenedButNotSafelyClosed
  99             final DataOutputStream _out = new DataOutputStream(bytes);
 100             final TIntHashSet set = valueContainer.getInvalidated();
 101             if (set.size() > 0) {
 102               for (int inputId : set.toArray()) {
 103                 myValueContainerExternalizer.saveInvalidateCommand(_out, inputId);
 104               }
 105             }
 106             final ValueContainer<Value> toRemove = valueContainer.getRemovedDelta();
 107             if (toRemove.size() > 0) {
 108               myValueContainerExternalizer.saveAsRemoved(_out, toRemove);
 109             }
 110
 111             final ValueContainer<Value> toAppend = valueContainer.getAddedDelta();
 112             if (toAppend.size() > 0) {
 113               myValueContainerExternalizer.save(_out, toAppend);
 114             }
 115
 116             map.appendData(key, new PersistentHashMap.ValueDataAppender() {
 117               public void append(final DataOutput out) throws IOException {
 118                 final byte[] barr = bytes.toByteArray();
 119                 out.write(barr);
 120               }
 121             });
 122           }
 123           else {
 124             // rewrite the value container for defragmentation
 125             map.put(key, valueContainer);
 126           }
 127         }
 128         catch (IOException e) {
 129           throw new RuntimeException(e);
 130         }
 131       }
 132     };
 133
 134     myMap = map;
 135   }
 136
 137   public void flush() {
 138     l.lock();
 139     try {
 140       if (!myMap.isClosed()) {
 141         myCache.clear();
 142         myMap.force();
 143       }
 144     }
 145     finally {
 146       l.unlock();
 147     }
 148   }
 149
 150   public void close() throws StorageException {
 151     try {
 152       flush();
 153       myMap.close();
 154     }
 155     catch (IOException e) {
 156       throw new StorageException(e);
 157     }
 158     catch (RuntimeException e) {
 159       final Throwable cause = e.getCause();
 160       if (cause instanceof IOException) {
 161         throw new StorageException(cause);
 162       }
 163       if (cause instanceof StorageException) {
 164         throw (StorageException)cause;
 165       }
 166       throw e;
 167     }
 168   }
 169
 170   public void clear() throws StorageException{
 171     try {
 172       myMap.close();
 173     }
 174     catch (IOException e) {
 175       LOG.error(e);
 176     }
 177     try {
 178       FileUtil.delete(myStorageFile);
 179       initMapAndCache();
 180     }
 181     catch (IOException e) {
 182       throw new StorageException(e);
 183     }
 184     catch (RuntimeException e) {
 185       final Throwable cause = e.getCause();
 186       if (cause instanceof IOException) {
 187         throw new StorageException(cause);
 188       }
 189       if (cause instanceof StorageException) {
 190         throw (StorageException)cause;
 191       }
 192       throw e;
 193     }
 194   }
 195
 196   public boolean processKeys(final Processor<Key> processor) throws StorageException {
 197     l.lock();
 198     try {
 199       myCache.clear(); // this will ensure that all new keys are made into the map
 200       return myMap.processKeys(processor);
 201     }
 202     catch (IOException e) {
 203       throw new StorageException(e);
 204     }
 205     catch (RuntimeException e) {
 206       final Throwable cause = e.getCause();
 207       if (cause instanceof IOException) {
 208         throw new StorageException(cause);
 209       }
 210       if (cause instanceof StorageException) {
 211         throw (StorageException)cause;
 212       }
 213       throw e;
 214     }
 215     finally {
 216       l.unlock();
 217     }
 218   }
 219
 220   public Collection<Key> getKeys() throws StorageException {
 221     List<Key> keys = new ArrayList<Key>();
 222     processKeys(new CommonProcessors.CollectProcessor<Key>(keys));
 223     return keys;
 224   }
 225
 226   @NotNull
 227   public ChangeTrackingValueContainer<Value> read(final Key key) throws StorageException {
 228     l.lock();
 229     try {
 230       return myCache.get(key);
 231     }
 232     catch (RuntimeException e) {
 233       final Throwable cause = e.getCause();
 234       if (cause instanceof IOException) {
 235         throw new StorageException(cause);
 236       }
 237       if (cause instanceof StorageException) {
 238         throw (StorageException)cause;
 239       }
 240       throw e;
 241     }
 242     finally {
 243       l.unlock();
 244     }
 245   }
 246
 247   public void addValue(final Key key, final int inputId, final Value value) throws StorageException {
 248     read(key).addValue(inputId, value);
 249   }
 250
 251   public void removeValue(final Key key, final int inputId, final Value value) throws StorageException {
 252     read(key).removeValue(inputId, value);
 253   }
 254
 255   public void removeAllValues(Key key, int inputId) throws StorageException {
 256     // important: assuming the key exists in the index
 257     read(key).removeAllValues(inputId);
 258   }
 259
 260   private static final class ValueContainerExternalizer<T> implements DataExternalizer<ValueContainer<T>> {
 261     private final DataExternalizer<T> myExternalizer;
 262
 263     private ValueContainerExternalizer(DataExternalizer<T> externalizer) {
 264       myExternalizer = externalizer;
 265     }
 266
 267     public void save(final DataOutput out, final ValueContainer<T> container) throws IOException {
 268       saveImpl(out, container, false);
 269     }
 270
 271     public void saveAsRemoved(final DataOutput out, final ValueContainer<T> container) throws IOException {
 272       saveImpl(out, container, true);
 273     }
 274
 275     public void saveInvalidateCommand(final DataOutput out, int inputId) throws IOException {
 276       DataInputOutputUtil.writeSINT(out, -inputId);
 277     }
 278
 279     private void saveImpl(final DataOutput out, final ValueContainer<T> container, final boolean asRemovedData) throws IOException {
 280       DataInputOutputUtil.writeSINT(out, container.size());
 281       for (final Iterator<T> valueIterator = container.getValueIterator(); valueIterator.hasNext();) {
 282         final T value = valueIterator.next();
 283         myExternalizer.save(out, value);
 284
 285         final ValueContainer.IntIterator ids = container.getInputIdsIterator(value);
 286         if (ids != null) {
 287           DataInputOutputUtil.writeSINT(out, ids.size());
 288           while (ids.hasNext()) {
 289             final int id = ids.next();
 290             DataInputOutputUtil.writeSINT(out, asRemovedData ? -id : id);
 291           }
 292         }
 293         else {
 294           DataInputOutputUtil.writeSINT(out, 0);
 295         }
 296       }
 297     }
 298
 299     public ValueContainerImpl<T> read(final DataInput in) throws IOException {
 300       DataInputStream stream = (DataInputStream)in;
 301       final ValueContainerImpl<T> valueContainer = new ValueContainerImpl<T>();
 302
 303       while (stream.available() > 0) {
 304         final int valueCount = DataInputOutputUtil.readSINT(in);
 305         if (valueCount < 0) {
 306           valueContainer.removeAllValues(-valueCount);
 307           valueContainer.setNeedsCompacting(true);
 308         }
 309         else {
 310           for (int valueIdx = 0; valueIdx < valueCount; valueIdx++) {
 311             final T value = myExternalizer.read(in);
 312             final int idCount = DataInputOutputUtil.readSINT(in);
 313             for (int i = 0; i < idCount; i++) {
 314               final int id = DataInputOutputUtil.readSINT(in);
 315               if (id < 0) {
 316                 valueContainer.removeValue(-id, value);
 317                 valueContainer.setNeedsCompacting(true);
 318               }
 319               else {
 320                 valueContainer.addValue(id, value);
 321               }
 322             }
 323           }
 324         }
 325       }
 326       return valueContainer;
 327     }
 328   }
 329 }