xapian-core/backends/honey/honey_table.cc

   1
   2 #include <config.h>
   3
   4 #include "honey_table.h"
   5
   6 #include "honey_cursor.h"
   7
   8 using Honey::RootInfo;
   9
  10 using namespace std;
  11
  12 size_t HoneyTable::total_index_size = 0;
  13
  14 void
  15 HoneyTable::create_and_open(int flags_, const RootInfo& root_info)
  16 {
  17     flags = flags_;
  18     compress_min = root_info.get_compress_min();
  19     if (read_only) {
  20         num_entries = root_info.get_num_entries();
  21         root = root_info.get_root();
  22         // FIXME: levels
  23     }
  24     if (!fh.open(path, read_only))
  25         throw Xapian::DatabaseOpeningError("Failed to open HoneyTable", errno);
  26 }
  27
  28 void
  29 HoneyTable::open(int flags_, const RootInfo& root_info, honey_revision_number_t)
  30 {
  31     flags = flags_;
  32     compress_min = root_info.get_compress_min();
  33     num_entries = root_info.get_num_entries();
  34     root = root_info.get_root();
  35     if (!fh.open(path, read_only)) {
  36         if (!lazy)
  37             throw Xapian::DatabaseOpeningError("Failed to open HoneyTable", errno);
  38     }
  39 }
  40
  41 void
  42 HoneyTable::add(const std::string& key,
  43                 const char* val,
  44                 size_t val_size,
  45                 bool compressed)
  46 {
  47     if (!compressed && compress_min > 0 && val_size > compress_min) {
  48         size_t compressed_size = val_size;
  49         CompressionStream comp_stream; // FIXME: reuse
  50         const char* p = comp_stream.compress(val, &compressed_size);
  51         if (p) {
  52             add(key, p, compressed_size, true);
  53             return;
  54         }
  55     }
  56
  57     if (read_only)
  58         throw Xapian::InvalidOperationError("add() on read-only HoneyTable");
  59     if (key.size() == 0 || key.size() > 255)
  60         throw Xapian::InvalidArgumentError("Invalid key size: " + str(key.size()));
  61     if (key <= last_key)
  62         throw Xapian::InvalidOperationError("New key <= previous key");
  63     if (!last_key.empty()) {
  64         size_t len = std::min(last_key.size(), key.size());
  65         size_t i;
  66         for (i = 0; i < len; ++i) {
  67             if (last_key[i] != key[i]) break;
  68         }
  69         fh.write(static_cast<unsigned char>(i));
  70         fh.write(static_cast<unsigned char>(key.size() - i));
  71         fh.write(key.data() + i, key.size() - i);
  72     } else {
  73         fh.write(static_cast<unsigned char>(key.size()));
  74         fh.write(key.data(), key.size());
  75     }
  76     ++num_entries;
  77     index.maybe_add_entry(key, fh.get_pos());
  78
  79     // Encode "compressed?" flag in bottom bit.
  80     // FIXME: Don't do this if a table is uncompressed?  That saves a byte
  81     // for each item where the extra bit pushes the length up by a byte.
  82     size_t val_size_enc = (val_size << 1) | compressed;
  83     std::string val_len;
  84     pack_uint(val_len, val_size_enc);
  85     // FIXME: pass together so we can potentially writev() both?
  86     fh.write(val_len.data(), val_len.size());
  87     fh.write(val, val_size);
  88     last_key = key;
  89 }
  90
  91 void
  92 HoneyTable::commit(honey_revision_number_t, RootInfo* root_info)
  93 {
  94     if (root < 0)
  95         throw Xapian::InvalidOperationError("root not set");
  96
  97     root_info->set_level(1); // FIXME: number of index levels
  98     root_info->set_num_entries(num_entries);
  99     root_info->set_root_is_fake(false);
 100     // Not really meaningful.
 101     root_info->set_sequential(true);
 102     root_info->set_root(root);
 103     // Not really meaningful.
 104     root_info->set_blocksize(2048);
 105     // Not really meaningful.
 106     //root_info->set_free_list(std::string());
 107
 108     read_only = true;
 109     fh.rewind();
 110     last_key = string();
 111 }
 112
 113 bool
 114 HoneyTable::read_item(std::string& key, std::string& val, bool& compressed) const
 115 {
 116     if (!read_only) {
 117         return false;
 118     }
 119
 120     int ch = fh.read();
 121     if (ch == EOF) return false;
 122
 123     size_t reuse = 0;
 124     if (!last_key.empty()) {
 125         reuse = ch;
 126         ch = fh.read();
 127         if (ch == EOF) throw Xapian::DatabaseError("EOF/error while reading key length", errno);
 128     }
 129     size_t key_size = ch;
 130     char buf[4096];
 131     if (!fh.read(buf, key_size))
 132         throw Xapian::DatabaseError("read of " + str(key_size) + " bytes of key data failed", errno);
 133     key.assign(last_key, 0, reuse);
 134     key.append(buf, key_size);
 135     last_key = key;
 136
 137     if (false) {
 138         std::string esc;
 139         description_append(esc, key);
 140         std::cout << "K:" << esc << std::endl;
 141     }
 142
 143     int r;
 144     {
 145         // FIXME: rework to take advantage of buffering that's happening anyway?
 146         char * p = buf;
 147         for (int i = 0; i < 8; ++i) {
 148             int ch2 = fh.read();
 149             if (ch2 == EOF) {
 150                 break;
 151             }
 152             *p++ = ch2;
 153             if (ch2 < 128) break;
 154         }
 155         r = p - buf;
 156     }
 157     const char* p = buf;
 158     const char* end = p + r;
 159     size_t val_size;
 160     if (!unpack_uint(&p, end, &val_size)) {
 161         throw Xapian::DatabaseError("val_size unpack_uint invalid");
 162     }
 163     compressed = val_size & 1;
 164     val_size >>= 1;
 165     val.assign(p, end);
 166     if (p != end) std::abort();
 167     val_size -= (end - p);
 168     while (val_size) {
 169         size_t n = min(val_size, sizeof(buf));
 170         if (!fh.read(buf, n))
 171             throw Xapian::DatabaseError("read of " + str(n) + "/" + str(val_size) + " bytes of value data failed", errno);
 172         val.append(buf, n);
 173         val_size -= n;
 174     }
 175
 176     if (false) {
 177         std::string esc;
 178         description_append(esc, val);
 179         std::cout << "V:" << esc << std::endl;
 180     }
 181
 182     return true;
 183 }
 184
 185 bool
 186 HoneyTable::get_exact_entry(const std::string& key, std::string& tag) const
 187 {
 188     if (!read_only) std::abort();
 189     if (!fh.is_open()) return false;
 190     fh.rewind();
 191     last_key = std::string();
 192     std::string k, v;
 193     bool compressed;
 194     int cmp;
 195     do {
 196         if (!read_item(k, v, compressed)) return false;
 197         cmp = k.compare(key);
 198     } while (cmp < 0);
 199     if (cmp > 0) return false;
 200     if (compressed) {
 201         CompressionStream comp_stream;
 202         comp_stream.decompress_start();
 203         if (!comp_stream.decompress_chunk(v.data(), v.size(), tag)) {
 204             // Decompression didn't complete.
 205             abort();
 206         }
 207     } else {
 208         tag = v;
 209     }
 210     return true;
 211 }
 212
 213 bool
 214 HoneyTable::key_exists(const std::string& key) const
 215 {
 216     if (!read_only) std::abort();
 217     if (!fh.is_open()) return false;
 218     fh.rewind();
 219     last_key = std::string();
 220     std::string k, v;
 221     bool compressed;
 222     int cmp;
 223     do {
 224         // FIXME: avoid reading tag data?
 225         if (!read_item(k, v, compressed)) return false;
 226         cmp = k.compare(key);
 227     } while (cmp < 0);
 228     return (cmp == 0);
 229 }
 230
 231 HoneyCursor*
 232 HoneyTable::cursor_get() const
 233 {
 234     return new HoneyCursor(fh, root);
 235 }