xapian-core/backends/inmemory/inmemory_database.cc

   1 /* inmemory_database.cc
   2  *
   3  * Copyright 1999,2000,2001 BrightStation PLC
   4  * Copyright 2002 Ananova Ltd
   5  * Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2014 Olly Betts
   6  * Copyright 2006,2009 Lemur Consulting Ltd
   7  *
   8  * This program is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU General Public License as
  10  * published by the Free Software Foundation; either version 2 of the
  11  * License, or (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  21  * USA
  22  */
  23
  24 #include <config.h>
  25
  26 #include "inmemory_database.h"
  27
  28 #include "debuglog.h"
  29
  30 #include "expand/expandweight.h"
  31 #include "inmemory_document.h"
  32 #include "inmemory_alltermslist.h"
  33 #include "str.h"
  34 #include "backends/valuestats.h"
  35
  36 #include <string>
  37 #include <vector>
  38 #include <map>
  39
  40 #include <xapian/error.h>
  41 #include <xapian/valueiterator.h>
  42
  43 using std::make_pair;
  44 using Xapian::Internal::intrusive_ptr;
  45
  46 inline void
  47 InMemoryTerm::add_posting(const InMemoryPosting & post)
  48 {
  49     // Add document to right place in list
  50     vector<InMemoryPosting>::iterator p;
  51     p = lower_bound(docs.begin(), docs.end(),
  52                     post, InMemoryPostingLessThan());
  53     if (p == docs.end() || InMemoryPostingLessThan()(post, *p)) {
  54         docs.insert(p, post);
  55     } else if (!p->valid) {
  56         *p = post;
  57     } else {
  58         (*p).merge(post);
  59     }
  60 }
  61
  62 inline void
  63 InMemoryDoc::add_posting(const InMemoryTermEntry & post)
  64 {
  65     // Add document to right place in list
  66     vector<InMemoryTermEntry>::iterator p;
  67     p = lower_bound(terms.begin(), terms.end(),
  68                     post, InMemoryTermEntryLessThan());
  69     if (p == terms.end() || InMemoryTermEntryLessThan()(post, *p)) {
  70         terms.insert(p, post);
  71     } else {
  72         (*p).merge(post);
  73     }
  74 }
  75
  76 //////////////
  77 // Postlist //
  78 //////////////
  79
  80 InMemoryPostList::InMemoryPostList(intrusive_ptr<const InMemoryDatabase> db_,
  81                                    const InMemoryTerm & imterm,
  82                                    const std::string & term_)
  83         : LeafPostList(term_),
  84           pos(imterm.docs.begin()),
  85           end(imterm.docs.end()),
  86           termfreq(imterm.term_freq),
  87           started(false),
  88           db(db_)
  89 {
  90     while (pos != end && !pos->valid) ++pos;
  91 }
  92
  93 Xapian::doccount
  94 InMemoryPostList::get_termfreq() const
  95 {
  96     return termfreq;
  97 }
  98
  99 Xapian::docid
 100 InMemoryPostList::get_docid() const
 101 {
 102     if (db->is_closed()) InMemoryDatabase::throw_database_closed();
 103     Assert(started);
 104     Assert(!at_end());
 105     return (*pos).did;
 106 }
 107
 108 PostList *
 109 InMemoryPostList::next(double /*w_min*/)
 110 {
 111     if (db->is_closed()) InMemoryDatabase::throw_database_closed();
 112     if (started) {
 113         Assert(!at_end());
 114         ++pos;
 115         while (pos != end && !pos->valid) ++pos;
 116     } else {
 117         started = true;
 118     }
 119     return NULL;
 120 }
 121
 122 PostList *
 123 InMemoryPostList::skip_to(Xapian::docid did, double w_min)
 124 {
 125     if (db->is_closed()) InMemoryDatabase::throw_database_closed();
 126     // FIXME - see if we can make more efficient, perhaps using better
 127     // data structure.  Note, though, that a binary search of
 128     // the remaining list may NOT be a good idea (search time is then
 129     // O(log {length of list}), as opposed to O(distance we want to skip)
 130     // Since we will frequently only be skipping a short distance, this
 131     // could well be worse.
 132     started = true;
 133     Assert(!at_end());
 134     while (!at_end() && (*pos).did < did) {
 135         (void) next(w_min);
 136     }
 137     return NULL;
 138 }
 139
 140 bool
 141 InMemoryPostList::at_end() const
 142 {
 143     if (db->is_closed()) InMemoryDatabase::throw_database_closed();
 144     return (pos == end);
 145 }
 146
 147 string
 148 InMemoryPostList::get_description() const
 149 {
 150     return "InMemoryPostList " + str(termfreq);
 151 }
 152
 153 Xapian::termcount
 154 InMemoryPostList::get_doclength() const
 155 {
 156     if (db->is_closed()) InMemoryDatabase::throw_database_closed();
 157     return db->get_doclength(get_docid());
 158 }
 159
 160 Xapian::termcount
 161 InMemoryPostList::get_unique_terms() const
 162 {
 163     return db->get_unique_terms(get_docid());
 164 }
 165
 166 PositionList *
 167 InMemoryPostList::read_position_list()
 168 {
 169     if (db->is_closed()) InMemoryDatabase::throw_database_closed();
 170     mypositions.set_data(pos->positions);
 171     return &mypositions;
 172 }
 173
 174 PositionList *
 175 InMemoryPostList::open_position_list() const
 176 {
 177     if (db->is_closed()) InMemoryDatabase::throw_database_closed();
 178     return new InMemoryPositionList(pos->positions);
 179 }
 180
 181 Xapian::termcount
 182 InMemoryPostList::get_wdf() const
 183 {
 184     if (db->is_closed()) InMemoryDatabase::throw_database_closed();
 185     return (*pos).wdf;
 186 }
 187
 188 //////////////
 189 // Termlist //
 190 //////////////
 191
 192 InMemoryTermList::InMemoryTermList(intrusive_ptr<const InMemoryDatabase> db_,
 193                                    Xapian::docid did_,
 194                                    const InMemoryDoc & doc,
 195                                    Xapian::termcount len)
 196         : pos(doc.terms.begin()), end(doc.terms.end()), terms(doc.terms.size()),
 197           started(false), db(db_), did(did_), document_length(len)
 198 {
 199     LOGLINE(DB, "InMemoryTermList::InMemoryTermList(): " <<
 200                 terms << " terms starting from " << pos->tname);
 201 }
 202
 203 Xapian::termcount
 204 InMemoryTermList::get_wdf() const
 205 {
 206     if (db->is_closed()) InMemoryDatabase::throw_database_closed();
 207     Assert(started);
 208     Assert(!at_end());
 209     return (*pos).wdf;
 210 }
 211
 212 Xapian::doccount
 213 InMemoryTermList::get_termfreq() const
 214 {
 215     if (db->is_closed()) InMemoryDatabase::throw_database_closed();
 216     Assert(started);
 217     Assert(!at_end());
 218
 219     Xapian::doccount tf;
 220     db->get_freqs((*pos).tname, &tf, NULL);
 221     return tf;
 222 }
 223
 224 Xapian::termcount
 225 InMemoryTermList::get_approx_size() const
 226 {
 227     if (db->is_closed()) InMemoryDatabase::throw_database_closed();
 228     return terms;
 229 }
 230
 231 void
 232 InMemoryTermList::accumulate_stats(Xapian::Internal::ExpandStats & stats) const
 233 {
 234     if (db->is_closed()) InMemoryDatabase::throw_database_closed();
 235     Assert(started);
 236     Assert(!at_end());
 237     stats.accumulate(InMemoryTermList::get_wdf(), document_length,
 238                      InMemoryTermList::get_termfreq(),
 239                      db->get_doccount());
 240 }
 241
 242 string
 243 InMemoryTermList::get_termname() const
 244 {
 245     if (db->is_closed()) InMemoryDatabase::throw_database_closed();
 246     Assert(started);
 247     Assert(!at_end());
 248     return (*pos).tname;
 249 }
 250
 251 TermList *
 252 InMemoryTermList::next()
 253 {
 254     if (db->is_closed()) InMemoryDatabase::throw_database_closed();
 255     if (started) {
 256         Assert(!at_end());
 257         ++pos;
 258     } else {
 259         started = true;
 260     }
 261     return NULL;
 262 }
 263
 264 TermList *
 265 InMemoryTermList::skip_to(const string & term)
 266 {
 267     if (rare(db->is_closed()))
 268         InMemoryDatabase::throw_database_closed();
 269
 270     while (pos != end && pos->tname < term) {
 271         ++pos;
 272     }
 273
 274     started = true;
 275     return NULL;
 276 }
 277
 278 bool
 279 InMemoryTermList::at_end() const
 280 {
 281     if (db->is_closed()) InMemoryDatabase::throw_database_closed();
 282     Assert(started);
 283     return (pos == end);
 284 }
 285
 286 Xapian::termcount
 287 InMemoryTermList::positionlist_count() const
 288 {
 289     if (db->is_closed()) InMemoryDatabase::throw_database_closed();
 290     return db->positionlist_count(did, (*pos).tname);
 291 }
 292
 293 Xapian::PositionIterator
 294 InMemoryTermList::positionlist_begin() const
 295 {
 296     if (db->is_closed()) InMemoryDatabase::throw_database_closed();
 297     return Xapian::PositionIterator(db->open_position_list(did, (*pos).tname));
 298 }
 299
 300 /////////////////////////////
 301 // InMemoryAllDocsPostList //
 302 /////////////////////////////
 303
 304 InMemoryAllDocsPostList::InMemoryAllDocsPostList(intrusive_ptr<const InMemoryDatabase> db_)
 305         : LeafPostList(std::string()), did(0), db(db_)
 306 {
 307 }
 308
 309 Xapian::doccount
 310 InMemoryAllDocsPostList::get_termfreq() const
 311 {
 312     if (db->is_closed()) InMemoryDatabase::throw_database_closed();
 313     return db->totdocs;
 314 }
 315
 316 Xapian::docid
 317 InMemoryAllDocsPostList::get_docid() const
 318 {
 319     if (db->is_closed()) InMemoryDatabase::throw_database_closed();
 320     Assert(did > 0);
 321     Assert(did <= db->termlists.size());
 322     Assert(db->termlists[did - 1].is_valid);
 323     return did;
 324 }
 325
 326 Xapian::termcount
 327 InMemoryAllDocsPostList::get_doclength() const
 328 {
 329     if (db->is_closed()) InMemoryDatabase::throw_database_closed();
 330     return db->get_doclength(did);
 331 }
 332
 333 Xapian::termcount
 334 InMemoryAllDocsPostList::get_unique_terms() const
 335 {
 336     return db->get_unique_terms(did);
 337 }
 338
 339 Xapian::termcount
 340 InMemoryAllDocsPostList::get_wdf() const
 341 {
 342     return 1;
 343 }
 344
 345 PositionList *
 346 InMemoryAllDocsPostList::read_position_list()
 347 {
 348     throw Xapian::UnimplementedError("Can't open position list for all docs iterator");
 349 }
 350
 351 PositionList *
 352 InMemoryAllDocsPostList::open_position_list() const
 353 {
 354     throw Xapian::UnimplementedError("Can't open position list for all docs iterator");
 355 }
 356
 357 PostList *
 358 InMemoryAllDocsPostList::next(double /*w_min*/)
 359 {
 360     if (db->is_closed()) InMemoryDatabase::throw_database_closed();
 361     Assert(!at_end());
 362     do {
 363        ++did;
 364     } while (did <= db->termlists.size() && !db->termlists[did - 1].is_valid);
 365     return NULL;
 366 }
 367
 368 PostList *
 369 InMemoryAllDocsPostList::skip_to(Xapian::docid did_, double /*w_min*/)
 370 {
 371     if (db->is_closed()) InMemoryDatabase::throw_database_closed();
 372     Assert(!at_end());
 373     if (did <= did_) {
 374         did = did_;
 375         while (did <= db->termlists.size() && !db->termlists[did - 1].is_valid) {
 376             ++did;
 377         }
 378     }
 379     return NULL;
 380 }
 381
 382 bool
 383 InMemoryAllDocsPostList::at_end() const
 384 {
 385     if (db->is_closed()) InMemoryDatabase::throw_database_closed();
 386     return (did > db->termlists.size());
 387 }
 388
 389 string
 390 InMemoryAllDocsPostList::get_description() const
 391 {
 392     return "InMemoryAllDocsPostList " + str(did);
 393 }
 394
 395 ///////////////////////////
 396 // Actual database class //
 397 ///////////////////////////
 398
 399 InMemoryDatabase::InMemoryDatabase()
 400         : totdocs(0), totlen(0), positions_present(false), closed(false)
 401 {
 402     // Updates are applied immediately so we can't support transactions.
 403     transaction_state = TRANSACTION_UNIMPLEMENTED;
 404
 405     // We keep an empty entry in postlists for convenience of implementing
 406     // allterms iteration and returning a PostList for an absent term.
 407     postlists.insert(make_pair(string(), InMemoryTerm()));
 408 }
 409
 410 InMemoryDatabase::~InMemoryDatabase()
 411 {
 412     dtor_called();
 413 }
 414
 415 bool
 416 InMemoryDatabase::reopen()
 417 {
 418     if (closed) InMemoryDatabase::throw_database_closed();
 419     return false;
 420 }
 421
 422 void
 423 InMemoryDatabase::close()
 424 {
 425     // Free all the resources, and mark the db as closed.
 426     postlists.clear();
 427     termlists.clear();
 428     doclists.clear();
 429     valuelists.clear();
 430     valuestats.clear();
 431     doclengths.clear();
 432     metadata.clear();
 433     closed = true;
 434 }
 435
 436 LeafPostList *
 437 InMemoryDatabase::open_post_list(const string & tname) const
 438 {
 439     if (closed) InMemoryDatabase::throw_database_closed();
 440     if (tname.empty()) {
 441         intrusive_ptr<const InMemoryDatabase> ptrtothis(this);
 442         return new InMemoryAllDocsPostList(ptrtothis);
 443     }
 444     map<string, InMemoryTerm>::const_iterator i = postlists.find(tname);
 445     if (i == postlists.end() || i->second.term_freq == 0) {
 446         i = postlists.begin();
 447         // Check that our dummy entry for string() is present.
 448         Assert(i->first.empty());
 449     }
 450     intrusive_ptr<const InMemoryDatabase> ptrtothis(this);
 451     return new InMemoryPostList(ptrtothis, i->second, tname);
 452 }
 453
 454 bool
 455 InMemoryDatabase::doc_exists(Xapian::docid did) const
 456 {
 457     if (closed) InMemoryDatabase::throw_database_closed();
 458     return (did > 0 && did <= termlists.size() && termlists[did - 1].is_valid);
 459 }
 460
 461 void
 462 InMemoryDatabase::get_freqs(const string & term,
 463                             Xapian::doccount * termfreq_ptr,
 464                             Xapian::termcount * collfreq_ptr) const
 465 {
 466     if (closed) InMemoryDatabase::throw_database_closed();
 467     map<string, InMemoryTerm>::const_iterator i = postlists.find(term);
 468     if (i != postlists.end()) {
 469         if (termfreq_ptr)
 470             *termfreq_ptr = i->second.term_freq;
 471         if (collfreq_ptr)
 472             *collfreq_ptr = i->second.collection_freq;
 473     } else {
 474         if (termfreq_ptr)
 475             *termfreq_ptr = 0;
 476         if (collfreq_ptr)
 477             *collfreq_ptr = 0;
 478     }
 479 }
 480
 481 Xapian::doccount
 482 InMemoryDatabase::get_value_freq(Xapian::valueno slot) const
 483 {
 484     if (closed) InMemoryDatabase::throw_database_closed();
 485     map<Xapian::valueno, ValueStats>::const_iterator i = valuestats.find(slot);
 486     if (i == valuestats.end()) return 0;
 487     return i->second.freq;
 488 }
 489
 490 std::string
 491 InMemoryDatabase::get_value_lower_bound(Xapian::valueno slot) const
 492 {
 493     if (closed) InMemoryDatabase::throw_database_closed();
 494     map<Xapian::valueno, ValueStats>::const_iterator i = valuestats.find(slot);
 495     if (i == valuestats.end()) return string();
 496     return i->second.lower_bound;
 497 }
 498
 499 std::string
 500 InMemoryDatabase::get_value_upper_bound(Xapian::valueno slot) const
 501 {
 502     if (closed) InMemoryDatabase::throw_database_closed();
 503     map<Xapian::valueno, ValueStats>::const_iterator i = valuestats.find(slot);
 504     if (i == valuestats.end()) return string();
 505     return i->second.upper_bound;
 506 }
 507
 508 Xapian::doccount
 509 InMemoryDatabase::get_doccount() const
 510 {
 511     if (closed) InMemoryDatabase::throw_database_closed();
 512     return totdocs;
 513 }
 514
 515 Xapian::docid
 516 InMemoryDatabase::get_lastdocid() const
 517 {
 518     if (closed) InMemoryDatabase::throw_database_closed();
 519     return termlists.size();
 520 }
 521
 522 totlen_t
 523 InMemoryDatabase::get_total_length() const
 524 {
 525     return totlen;
 526 }
 527
 528 Xapian::termcount
 529 InMemoryDatabase::get_doclength(Xapian::docid did) const
 530 {
 531     if (closed) InMemoryDatabase::throw_database_closed();
 532     if (!doc_exists(did)) {
 533         throw Xapian::DocNotFoundError(string("Docid ") + str(did) +
 534                                  string(" not found"));
 535     }
 536     return doclengths[did - 1];
 537 }
 538
 539 Xapian::termcount
 540 InMemoryDatabase::get_unique_terms(Xapian::docid did) const
 541 {
 542     if (closed) InMemoryDatabase::throw_database_closed();
 543     if (did == 0 || did > termlists.size() || !termlists[did - 1].is_valid)
 544         throw Xapian::DocNotFoundError(string("Docid ") + str(did) +
 545                                  string(" not found"));
 546     return termlists[did - 1].terms.size();
 547 }
 548
 549 TermList *
 550 InMemoryDatabase::open_term_list(Xapian::docid did) const
 551 {
 552     if (closed) InMemoryDatabase::throw_database_closed();
 553     Assert(did != 0);
 554     if (!doc_exists(did)) {
 555         // FIXME: the docid in this message will be local, not global
 556         throw Xapian::DocNotFoundError(string("Docid ") + str(did) +
 557                                  string(" not found"));
 558     }
 559     return new InMemoryTermList(intrusive_ptr<const InMemoryDatabase>(this), did,
 560                                 termlists[did - 1], doclengths[did - 1]);
 561 }
 562
 563 Xapian::Document::Internal *
 564 InMemoryDatabase::open_document(Xapian::docid did, bool lazy) const
 565 {
 566     if (closed) InMemoryDatabase::throw_database_closed();
 567     Assert(did != 0);
 568     if (!doc_exists(did)) {
 569         if (lazy) return NULL;
 570         // FIXME: the docid in this message will be local, not global
 571         throw Xapian::DocNotFoundError(string("Docid ") + str(did) +
 572                                  string(" not found"));
 573     }
 574     return new InMemoryDocument(this, did);
 575 }
 576
 577 std::string
 578 InMemoryDatabase::get_metadata(const std::string & key) const
 579 {
 580     if (closed) InMemoryDatabase::throw_database_closed();
 581     map<string, string>::const_iterator i = metadata.find(key);
 582     if (i == metadata.end())
 583         return string();
 584     return i->second;
 585 }
 586
 587 TermList *
 588 InMemoryDatabase::open_metadata_keylist(const string &) const
 589 {
 590     if (closed) InMemoryDatabase::throw_database_closed();
 591     if (metadata.empty()) return NULL;
 592     // FIXME: nobody implemented this yet...
 593     throw Xapian::UnimplementedError("InMemory backend doesn't currently implement Database::metadata_keys_begin()");
 594 }
 595
 596 void
 597 InMemoryDatabase::set_metadata(const std::string & key,
 598                                const std::string & value)
 599 {
 600     if (closed) InMemoryDatabase::throw_database_closed();
 601     if (!value.empty()) {
 602         metadata[key] = value;
 603     } else {
 604         metadata.erase(key);
 605     }
 606 }
 607
 608 Xapian::termcount
 609 InMemoryDatabase::positionlist_count(Xapian::docid did,
 610                                      const string & tname) const
 611 {
 612     if (closed) InMemoryDatabase::throw_database_closed();
 613     if (!doc_exists(did)) {
 614         return 0;
 615     }
 616     const InMemoryDoc &doc = termlists[did - 1];
 617
 618     vector<InMemoryTermEntry>::const_iterator i;
 619     for (i = doc.terms.begin(); i != doc.terms.end(); ++i) {
 620         if (i->tname == tname) {
 621             return i->positions.size();
 622         }
 623     }
 624     return 0;
 625 }
 626
 627 PositionList *
 628 InMemoryDatabase::open_position_list(Xapian::docid did,
 629                                      const string & tname) const
 630 {
 631     if (closed) InMemoryDatabase::throw_database_closed();
 632     if (usual(doc_exists(did))) {
 633         const InMemoryDoc &doc = termlists[did - 1];
 634
 635         vector<InMemoryTermEntry>::const_iterator i;
 636         for (i = doc.terms.begin(); i != doc.terms.end(); ++i) {
 637             if (i->tname == tname) {
 638                 return new InMemoryPositionList(i->positions);
 639             }
 640         }
 641     }
 642     return new InMemoryPositionList(false);
 643 }
 644
 645 void
 646 InMemoryDatabase::add_values(Xapian::docid did,
 647                              const map<Xapian::valueno, string> &values_)
 648 {
 649     if (closed) InMemoryDatabase::throw_database_closed();
 650     if (did > valuelists.size()) {
 651         valuelists.resize(did);
 652     }
 653     valuelists[did - 1] = values_;
 654
 655     // Update the statistics.
 656     map<Xapian::valueno, string>::const_iterator j;
 657     for (j = values_.begin(); j != values_.end(); ++j) {
 658         std::pair<map<Xapian::valueno, ValueStats>::iterator, bool> i;
 659         i = valuestats.insert(make_pair(j->first, ValueStats()));
 660
 661         // Now, modify the stored statistics.
 662         if ((i.first->second.freq)++ == 0) {
 663             // If the value count was previously zero, set the upper and lower
 664             // bounds to the newly added value.
 665             i.first->second.lower_bound = j->second;
 666             i.first->second.upper_bound = j->second;
 667         } else {
 668             // Otherwise, simply make sure they reflect the new value.
 669             if (j->second < i.first->second.lower_bound) {
 670                 i.first->second.lower_bound = j->second;
 671             }
 672             if (j->second > i.first->second.upper_bound) {
 673                 i.first->second.upper_bound = j->second;
 674             }
 675         }
 676     }
 677 }
 678
 679 // We implicitly commit each modification right away, so nothing to do here.
 680 void
 681 InMemoryDatabase::commit()
 682 {
 683 }
 684
 685 // We implicitly commit each modification right away, so nothing to do here.
 686 void
 687 InMemoryDatabase::cancel()
 688 {
 689 }
 690
 691 void
 692 InMemoryDatabase::delete_document(Xapian::docid did)
 693 {
 694     if (closed) InMemoryDatabase::throw_database_closed();
 695     if (!doc_exists(did)) {
 696         throw Xapian::DocNotFoundError(string("Docid ") + str(did) +
 697                                  string(" not found"));
 698     }
 699     termlists[did - 1].is_valid = false;
 700     doclists[did - 1] = string();
 701     map<Xapian::valueno, string>::const_iterator j;
 702     for (j = valuelists[did - 1].begin(); j != valuelists[did - 1].end(); ++j) {
 703         map<Xapian::valueno, ValueStats>::iterator i;
 704         i = valuestats.find(j->first);
 705         if (--(i->second.freq) == 0) {
 706             i->second.lower_bound.resize(0);
 707             i->second.upper_bound.resize(0);
 708         }
 709     }
 710     valuelists[did - 1].clear();
 711
 712     totlen -= doclengths[did - 1];
 713     doclengths[did - 1] = 0;
 714     totdocs--;
 715     // A crude check, but it's hard to be more precise with the current
 716     // InMemory structure without being very inefficient.
 717     if (totdocs == 0) positions_present = false;
 718
 719     vector<InMemoryTermEntry>::const_iterator i;
 720     for (i = termlists[did - 1].terms.begin();
 721          i != termlists[did - 1].terms.end();
 722          ++i) {
 723         map<string, InMemoryTerm>::iterator t = postlists.find(i->tname);
 724         Assert(t != postlists.end());
 725         t->second.collection_freq -= i->wdf;
 726         --t->second.term_freq;
 727         vector<InMemoryPosting>::iterator posting = t->second.docs.begin();
 728         while (posting != t->second.docs.end()) {
 729             // Just zero out erased doc ids - otherwise we need to erase
 730             // in a vector (inefficient) and we break any posting lists
 731             // iterating over this posting list.
 732             if (posting->did == did) posting->valid = false;
 733             ++posting;
 734         }
 735     }
 736     termlists[did - 1].terms.clear();
 737 }
 738
 739 void
 740 InMemoryDatabase::replace_document(Xapian::docid did,
 741                                    const Xapian::Document & document)
 742 {
 743     LOGCALL_VOID(DB, "InMemoryDatabase::replace_document", did | document);
 744
 745     if (closed) InMemoryDatabase::throw_database_closed();
 746
 747     if (doc_exists(did)) {
 748         map<Xapian::valueno, string>::const_iterator j;
 749         for (j = valuelists[did - 1].begin(); j != valuelists[did - 1].end(); ++j) {
 750             map<Xapian::valueno, ValueStats>::iterator i;
 751             i = valuestats.find(j->first);
 752             if (--(i->second.freq) == 0) {
 753                 i->second.lower_bound.resize(0);
 754                 i->second.upper_bound.resize(0);
 755             }
 756         }
 757
 758         totlen -= doclengths[did - 1];
 759         totdocs--;
 760     } else if (did > termlists.size()) {
 761         termlists.resize(did);
 762         termlists[did - 1].is_valid = true;
 763         doclengths.resize(did);
 764         doclists.resize(did);
 765         valuelists.resize(did);
 766     } else {
 767         termlists[did - 1].is_valid = true;
 768     }
 769
 770     vector<InMemoryTermEntry>::const_iterator i;
 771     for (i = termlists[did - 1].terms.begin();
 772          i != termlists[did - 1].terms.end();
 773          ++i) {
 774         map<string, InMemoryTerm>::iterator t = postlists.find(i->tname);
 775         Assert(t != postlists.end());
 776         t->second.collection_freq -= i->wdf;
 777         --t->second.term_freq;
 778         vector<InMemoryPosting>::iterator posting = t->second.docs.begin();
 779         while (posting != t->second.docs.end()) {
 780             // Just invalidate erased doc ids - otherwise we need to erase
 781             // in a vector (inefficient) and we break any posting lists
 782             // iterating over this posting list.
 783             if (posting->did == did) posting->valid = false;
 784             ++posting;
 785         }
 786     }
 787
 788     doclengths[did - 1] = 0;
 789     doclists[did - 1] = document.get_data();
 790
 791     finish_add_doc(did, document);
 792 }
 793
 794 Xapian::docid
 795 InMemoryDatabase::add_document(const Xapian::Document & document)
 796 {
 797     LOGCALL(DB, Xapian::docid, "InMemoryDatabase::add_document", document);
 798     if (closed) InMemoryDatabase::throw_database_closed();
 799
 800     Xapian::docid did = make_doc(document.get_data());
 801
 802     finish_add_doc(did, document);
 803
 804     RETURN(did);
 805 }
 806
 807 void
 808 InMemoryDatabase::finish_add_doc(Xapian::docid did, const Xapian::Document &document)
 809 {
 810     {
 811         map<Xapian::valueno, string> values;
 812         Xapian::ValueIterator k = document.values_begin();
 813         for ( ; k != document.values_end(); ++k) {
 814             values.insert(make_pair(k.get_valueno(), *k));
 815             LOGLINE(DB, "InMemoryDatabase::finish_add_doc(): adding value " <<
 816                         k.get_valueno() << " -> " << *k);
 817         }
 818         add_values(did, values);
 819     }
 820
 821     InMemoryDoc doc(true);
 822     Xapian::TermIterator i = document.termlist_begin();
 823     for ( ; i != document.termlist_end(); ++i) {
 824         make_term(*i);
 825
 826         LOGLINE(DB, "InMemoryDatabase::finish_add_doc(): adding term " << *i);
 827         Xapian::PositionIterator j = i.positionlist_begin();
 828         if (j == i.positionlist_end()) {
 829             /* Make sure the posting exists, even without a position. */
 830             make_posting(&doc, *i, did, 0, i.get_wdf(), false);
 831         } else {
 832             positions_present = true;
 833             for ( ; j != i.positionlist_end(); ++j) {
 834                 make_posting(&doc, *i, did, *j, i.get_wdf());
 835             }
 836         }
 837
 838         Assert(did > 0 && did <= doclengths.size());
 839         doclengths[did - 1] += i.get_wdf();
 840         totlen += i.get_wdf();
 841         postlists[*i].collection_freq += i.get_wdf();
 842         ++postlists[*i].term_freq;
 843     }
 844     swap(termlists[did - 1], doc);
 845
 846     totdocs++;
 847 }
 848
 849 void
 850 InMemoryDatabase::make_term(const string & tname)
 851 {
 852     postlists[tname];  // Initialise, if not already there.
 853 }
 854
 855 Xapian::docid
 856 InMemoryDatabase::make_doc(const string & docdata)
 857 {
 858     termlists.push_back(InMemoryDoc(true));
 859     doclengths.push_back(0);
 860     doclists.push_back(docdata);
 861
 862     AssertEqParanoid(termlists.size(), doclengths.size());
 863
 864     return termlists.size();
 865 }
 866
 867 void InMemoryDatabase::make_posting(InMemoryDoc * doc,
 868                                     const string & tname,
 869                                     Xapian::docid did,
 870                                     Xapian::termpos position,
 871                                     Xapian::termcount wdf,
 872                                     bool use_position)
 873 {
 874     Assert(doc);
 875     Assert(postlists.find(tname) != postlists.end());
 876     Assert(did > 0 && did <= termlists.size());
 877     Assert(did > 0 && did <= doclengths.size());
 878     Assert(doc_exists(did));
 879
 880     // Make the posting
 881     InMemoryPosting posting;
 882     posting.did = did;
 883     if (use_position) {
 884         posting.positions.push_back(position);
 885     }
 886     posting.wdf = wdf;
 887     posting.valid = true;
 888
 889     // Now record the posting
 890     postlists[tname].add_posting(posting);
 891
 892     // Make the termentry
 893     InMemoryTermEntry termentry;
 894     termentry.tname = tname;
 895     if (use_position) {
 896         termentry.positions.push_back(position);
 897     }
 898     termentry.wdf = wdf;
 899
 900     // Now record the termentry
 901     doc->add_posting(termentry);
 902 }
 903
 904 bool
 905 InMemoryDatabase::term_exists(const string & tname) const
 906 {
 907     if (closed) InMemoryDatabase::throw_database_closed();
 908     Assert(!tname.empty());
 909     map<string, InMemoryTerm>::const_iterator i = postlists.find(tname);
 910     if (i == postlists.end()) return false;
 911     return (i->second.term_freq != 0);
 912 }
 913
 914 bool
 915 InMemoryDatabase::has_positions() const
 916 {
 917     if (closed) InMemoryDatabase::throw_database_closed();
 918     return positions_present;
 919 }
 920
 921 TermList *
 922 InMemoryDatabase::open_allterms(const string & prefix) const
 923 {
 924     if (closed) InMemoryDatabase::throw_database_closed();
 925     return new InMemoryAllTermsList(&postlists,
 926                                     intrusive_ptr<const InMemoryDatabase>(this),
 927                                     prefix);
 928 }
 929
 930 void
 931 InMemoryDatabase::throw_database_closed()
 932 {
 933     throw Xapian::DatabaseError("Database has been closed");
 934 }