platform/lang-impl/src/com/intellij/util/indexing/MapIndexStorage.java

   1 /*
   2  * Copyright 2000-2009 JetBrains s.r.o.
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  * http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 package com.intellij.util.indexing;
  18
  19 import com.intellij.openapi.diagnostic.Logger;
  20 import com.intellij.openapi.util.io.FileUtil;
  21 import com.intellij.util.CommonProcessors;
  22 import com.intellij.util.Processor;
  23 import com.intellij.util.containers.SLRUCache;
  24 import com.intellij.util.io.DataExternalizer;
  25 import com.intellij.util.io.DataInputOutputUtil;
  26 import com.intellij.util.io.KeyDescriptor;
  27 import com.intellij.util.io.PersistentHashMap;
  28 import gnu.trove.TIntHashSet;
  29 import org.jetbrains.annotations.NotNull;
  30
  31 import java.io.*;
  32 import java.util.ArrayList;
  33 import java.util.Collection;
  34 import java.util.Iterator;
  35 import java.util.List;
  36
  37 /**
  38  * @author Eugene Zhuravlev
  39 *         Date: Dec 20, 2007
  40 */
  41 public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Value>{
  42   private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.MapIndexStorage");
  43   private PersistentHashMap<Key, ValueContainer<Value>> myMap;
  44   private SLRUCache<Key, ChangeTrackingValueContainer<Value>> myCache;
  45   private final File myStorageFile;
  46   private final KeyDescriptor<Key> myKeyDescriptor;
  47   private final ValueContainerExternalizer<Value> myValueContainerExternalizer;
  48   private final int myCacheSize;
  49
  50   public MapIndexStorage(File storageFile, final KeyDescriptor<Key> keyDescriptor, final DataExternalizer<Value> valueExternalizer,
  51                          final int cacheSize) throws IOException {
  52
  53     myStorageFile = storageFile;
  54     myKeyDescriptor = keyDescriptor;
  55     myValueContainerExternalizer = new ValueContainerExternalizer<Value>(valueExternalizer);
  56     myCacheSize = cacheSize;
  57     initMapAndCache();
  58   }
  59
  60   private void initMapAndCache() throws IOException {
  61     final PersistentHashMap<Key, ValueContainer<Value>> map =
  62       new PersistentHashMap<Key, ValueContainer<Value>>(myStorageFile, myKeyDescriptor, myValueContainerExternalizer);
  63     myCache = new SLRUCache<Key, ChangeTrackingValueContainer<Value>>(myCacheSize, (int)(Math.ceil(myCacheSize * 0.25)) /* 25% from the main cache size*/) {
  64       @NotNull
  65       public ChangeTrackingValueContainer<Value> createValue(final Key key) {
  66         return new ChangeTrackingValueContainer<Value>(new ChangeTrackingValueContainer.Initializer<Value>() {
  67           public Object getLock() {
  68             return map;
  69           }
  70
  71           public ValueContainer<Value> compute() {
  72             ValueContainer<Value> value = null;
  73             try {
  74               value = map.get(key);
  75               if (value == null) {
  76                 value = new ValueContainerImpl<Value>();
  77               }
  78             }
  79             catch (IOException e) {
  80               throw new RuntimeException(e);
  81             }
  82             return value;
  83           }
  84         });
  85       }
  86
  87       protected void onDropFromCache(final Key key, final ChangeTrackingValueContainer<Value> valueContainer) {
  88         if (!valueContainer.isDirty()) {
  89           return;
  90         }
  91         try {
  92           if (!valueContainer.needsCompacting()) {
  93             final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
  94             //noinspection IOResourceOpenedButNotSafelyClosed
  95             final DataOutputStream _out = new DataOutputStream(bytes);
  96             final TIntHashSet set = valueContainer.getInvalidated();
  97             if (set.size() > 0) {
  98               for (int inputId : set.toArray()) {
  99                 myValueContainerExternalizer.saveInvalidateCommand(_out, inputId);
 100               }
 101             }
 102             final ValueContainer<Value> toRemove = valueContainer.getRemovedDelta();
 103             if (toRemove.size() > 0) {
 104               myValueContainerExternalizer.saveAsRemoved(_out, toRemove);
 105             }
 106
 107             final ValueContainer<Value> toAppend = valueContainer.getAddedDelta();
 108             if (toAppend.size() > 0) {
 109               myValueContainerExternalizer.save(_out, toAppend);
 110             }
 111
 112             map.appendData(key, new PersistentHashMap.ValueDataAppender() {
 113               public void append(final DataOutput out) throws IOException {
 114                 final byte[] barr = bytes.toByteArray();
 115                 out.write(barr);
 116               }
 117             });
 118           }
 119           else {
 120             // rewrite the value container for defragmentation
 121             map.put(key, valueContainer);
 122           }
 123         }
 124         catch (IOException e) {
 125           throw new RuntimeException(e);
 126         }
 127       }
 128     };
 129
 130     myMap = map;
 131   }
 132
 133   public synchronized void flush() {
 134     if (!myMap.isClosed()) {
 135       myCache.clear();
 136       myMap.force();
 137     }
 138   }
 139
 140   public void close() throws StorageException {
 141     try {
 142       flush();
 143       myMap.close();
 144     }
 145     catch (IOException e) {
 146       throw new StorageException(e);
 147     }
 148     catch (RuntimeException e) {
 149       final Throwable cause = e.getCause();
 150       if (cause instanceof IOException) {
 151         throw new StorageException(cause);
 152       }
 153       if (cause instanceof StorageException) {
 154         throw (StorageException)cause;
 155       }
 156       throw e;
 157     }
 158   }
 159
 160   public void clear() throws StorageException{
 161     try {
 162       myMap.close();
 163     }
 164     catch (IOException e) {
 165       LOG.error(e);
 166     }
 167     try {
 168       FileUtil.delete(myStorageFile);
 169       initMapAndCache();
 170     }
 171     catch (IOException e) {
 172       throw new StorageException(e);
 173     }
 174     catch (RuntimeException e) {
 175       final Throwable cause = e.getCause();
 176       if (cause instanceof IOException) {
 177         throw new StorageException(cause);
 178       }
 179       if (cause instanceof StorageException) {
 180         throw (StorageException)cause;
 181       }
 182       throw e;
 183     }
 184   }
 185
 186   public synchronized boolean processKeys(final Processor<Key> processor) throws StorageException {
 187     try {
 188       myCache.clear(); // this will ensure that all new keys are made into the map
 189       return myMap.processKeys(processor);
 190     }
 191     catch (IOException e) {
 192       throw new StorageException(e);
 193     }
 194     catch (RuntimeException e) {
 195       final Throwable cause = e.getCause();
 196       if (cause instanceof IOException) {
 197         throw new StorageException(cause);
 198       }
 199       if (cause instanceof StorageException) {
 200         throw (StorageException)cause;
 201       }
 202       throw e;
 203     }
 204   }
 205
 206   public Collection<Key> getKeys() throws StorageException {
 207     List<Key> keys = new ArrayList<Key>();
 208     processKeys(new CommonProcessors.CollectProcessor<Key>(keys));
 209     return keys;
 210   }
 211
 212   @NotNull
 213   public synchronized ChangeTrackingValueContainer<Value> read(final Key key) throws StorageException {
 214     try {
 215       return myCache.get(key);
 216     }
 217     catch (RuntimeException e) {
 218       final Throwable cause = e.getCause();
 219       if (cause instanceof IOException) {
 220         throw new StorageException(cause);
 221       }
 222       if (cause instanceof StorageException) {
 223         throw (StorageException)cause;
 224       }
 225       throw e;
 226     }
 227   }
 228
 229   public void addValue(final Key key, final int inputId, final Value value) throws StorageException {
 230     read(key).addValue(inputId, value);
 231   }
 232
 233   public void removeValue(final Key key, final int inputId, final Value value) throws StorageException {
 234     read(key).removeValue(inputId, value);
 235   }
 236
 237   public void removeAllValues(Key key, int inputId) throws StorageException {
 238     // important: assuming the key exists in the index
 239     read(key).removeAllValues(inputId);
 240   }
 241
 242   private static final class ValueContainerExternalizer<T> implements DataExternalizer<ValueContainer<T>> {
 243     private final DataExternalizer<T> myExternalizer;
 244
 245     private ValueContainerExternalizer(DataExternalizer<T> externalizer) {
 246       myExternalizer = externalizer;
 247     }
 248
 249     public void save(final DataOutput out, final ValueContainer<T> container) throws IOException {
 250       saveImpl(out, container, false);
 251     }
 252
 253     public void saveAsRemoved(final DataOutput out, final ValueContainer<T> container) throws IOException {
 254       saveImpl(out, container, true);
 255     }
 256
 257     public void saveInvalidateCommand(final DataOutput out, int inputId) throws IOException {
 258       DataInputOutputUtil.writeSINT(out, -inputId);
 259     }
 260
 261     private void saveImpl(final DataOutput out, final ValueContainer<T> container, final boolean asRemovedData) throws IOException {
 262       DataInputOutputUtil.writeSINT(out, container.size());
 263       for (final Iterator<T> valueIterator = container.getValueIterator(); valueIterator.hasNext();) {
 264         final T value = valueIterator.next();
 265         myExternalizer.save(out, value);
 266
 267         final ValueContainer.IntIterator ids = container.getInputIdsIterator(value);
 268         if (ids != null) {
 269           DataInputOutputUtil.writeSINT(out, ids.size());
 270           while (ids.hasNext()) {
 271             final int id = ids.next();
 272             DataInputOutputUtil.writeSINT(out, asRemovedData ? -id : id);
 273           }
 274         }
 275         else {
 276           DataInputOutputUtil.writeSINT(out, 0);
 277         }
 278       }
 279     }
 280
 281     public ValueContainerImpl<T> read(final DataInput in) throws IOException {
 282       DataInputStream stream = (DataInputStream)in;
 283       final ValueContainerImpl<T> valueContainer = new ValueContainerImpl<T>();
 284
 285       while (stream.available() > 0) {
 286         final int valueCount = DataInputOutputUtil.readSINT(in);
 287         if (valueCount < 0) {
 288           valueContainer.removeAllValues(-valueCount);
 289           valueContainer.setNeedsCompacting(true);
 290         }
 291         else {
 292           for (int valueIdx = 0; valueIdx < valueCount; valueIdx++) {
 293             final T value = myExternalizer.read(in);
 294             final int idCount = DataInputOutputUtil.readSINT(in);
 295             for (int i = 0; i < idCount; i++) {
 296               final int id = DataInputOutputUtil.readSINT(in);
 297               if (id < 0) {
 298                 valueContainer.removeValue(-id, value);
 299                 valueContainer.setNeedsCompacting(true);
 300               }
 301               else {
 302                 valueContainer.addValue(id, value);
 303               }
 304             }
 305           }
 306         }
 307       }
 308       return valueContainer;
 309     }
 310   }
 311 }