xapian-core/api/omenquire.cc

   1 /* omenquire.cc: External interface for running queries
   2  *
   3  * Copyright 1999,2000,2001 BrightStation PLC
   4  * Copyright 2001,2002 Ananova Ltd
   5  * Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2013,2014,2015 Olly Betts
   6  * Copyright 2007,2009 Lemur Consulting Ltd
   7  * Copyright 2011, Action Without Borders
   8  *
   9  * This program is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU General Public License as
  11  * published by the Free Software Foundation; either version 2 of the
  12  * License, or (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this program; if not, write to the Free Software
  21  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  22  * USA
  23  */
  24
  25 #include <config.h>
  26 #include "xapian/enquire.h"
  27
  28 #include "xapian/document.h"
  29 #include "xapian/error.h"
  30 #include "xapian/errorhandler.h"
  31 #include "xapian/expanddecider.h"
  32 #include "xapian/matchspy.h"
  33 #include "xapian/termiterator.h"
  34 #include "xapian/weight.h"
  35
  36 #include "vectortermlist.h"
  37
  38 #include "backends/database.h"
  39 #include "debuglog.h"
  40 #include "expand/esetinternal.h"
  41 #include "expand/expandweight.h"
  42 #include "matcher/multimatch.h"
  43 #include "omassert.h"
  44 #include "api/omenquireinternal.h"
  45 #include "str.h"
  46 #include "weight/weightinternal.h"
  47
  48 #include <algorithm>
  49 #include "autoptr.h"
  50 #include <cfloat>
  51 #include <cmath>
  52 #include <vector>
  53
  54 using namespace std;
  55
  56 using Xapian::Internal::ExpandWeight;
  57 using Xapian::Internal::Bo1EWeight;
  58 using Xapian::Internal::TradEWeight;
  59
  60 namespace Xapian {
  61
  62 MatchDecider::~MatchDecider() { }
  63
  64 // Methods for Xapian::RSet
  65
  66 RSet::RSet() : internal(new RSet::Internal)
  67 {
  68 }
  69
  70 RSet::RSet(const RSet &other) : internal(other.internal)
  71 {
  72 }
  73
  74 void
  75 RSet::operator=(const RSet &other)
  76 {
  77     internal = other.internal;
  78 }
  79
  80 RSet::~RSet()
  81 {
  82 }
  83
  84 Xapian::doccount
  85 RSet::size() const
  86 {
  87     return internal->items.size();
  88 }
  89
  90 bool
  91 RSet::empty() const
  92 {
  93     return internal->items.empty();
  94 }
  95
  96 void
  97 RSet::add_document(Xapian::docid did)
  98 {
  99     if (did == 0) throw Xapian::InvalidArgumentError("Docid 0 not valid");
 100     internal->items.insert(did);
 101 }
 102
 103 void
 104 RSet::remove_document(Xapian::docid did)
 105 {
 106     internal->items.erase(did);
 107 }
 108
 109 bool
 110 RSet::contains(Xapian::docid did) const
 111 {
 112     return internal->items.find(did) != internal->items.end();
 113 }
 114
 115 string
 116 RSet::get_description() const
 117 {
 118     return "RSet(" + internal->get_description() + ")";
 119 }
 120
 121 string
 122 RSet::Internal::get_description() const
 123 {
 124     string description("RSet::Internal(");
 125
 126     set<Xapian::docid>::const_iterator i;
 127     for (i = items.begin(); i != items.end(); ++i) {
 128         if (i != items.begin()) description += ", ";
 129         description += str(*i);
 130     }
 131
 132     description += ')';
 133
 134     return description;
 135 }
 136
 137 namespace Internal {
 138
 139 // Methods for Xapian::MSetItem
 140
 141 string
 142 MSetItem::get_description() const
 143 {
 144     string description;
 145
 146     description = str(did) + ", " + str(wt) + ", " +
 147             collapse_key;
 148
 149     description = "Xapian::MSetItem(" + description + ")";
 150
 151     return description;
 152 }
 153
 154 }
 155
 156 // Methods for Xapian::MSet
 157
 158 MSet::MSet() : internal(new MSet::Internal)
 159 {
 160 }
 161
 162 MSet::MSet(MSet::Internal * internal_) : internal(internal_)
 163 {
 164 }
 165
 166 MSet::~MSet()
 167 {
 168 }
 169
 170 MSet::MSet(const MSet & other) : internal(other.internal)
 171 {
 172 }
 173
 174 void
 175 MSet::operator=(const MSet &other)
 176 {
 177     internal = other.internal;
 178 }
 179
 180 void
 181 MSet::fetch(const MSetIterator & beginiter, const MSetIterator & enditer) const
 182 {
 183     LOGCALL_VOID(API, "Xapian::MSet::fetch", beginiter | enditer);
 184     Assert(internal.get() != 0);
 185     if (beginiter != enditer)
 186         internal->fetch_items(beginiter.index, enditer.index - 1);
 187 }
 188
 189 void
 190 MSet::fetch(const MSetIterator & beginiter) const
 191 {
 192     LOGCALL_VOID(API, "Xapian::MSet::fetch", beginiter);
 193     Assert(internal.get() != 0);
 194     internal->fetch_items(beginiter.index, beginiter.index);
 195 }
 196
 197 void
 198 MSet::fetch() const
 199 {
 200     LOGCALL_VOID(API, "Xapian::MSet::fetch", NO_ARGS);
 201     Assert(internal.get() != 0);
 202     if (!internal->items.empty())
 203         internal->fetch_items(0, internal->items.size() - 1);
 204 }
 205
 206 int
 207 MSet::convert_to_percent(double wt) const
 208 {
 209     LOGCALL(API, int, "Xapian::MSet::convert_to_percent", wt);
 210     Assert(internal.get() != 0);
 211     RETURN(internal->convert_to_percent_internal(wt));
 212 }
 213
 214 int
 215 MSet::convert_to_percent(const MSetIterator & it) const
 216 {
 217     LOGCALL(API, int, "Xapian::MSet::convert_to_percent", it);
 218     Assert(internal.get() != 0);
 219     RETURN(internal->convert_to_percent_internal(it.get_weight()));
 220 }
 221
 222 Xapian::doccount
 223 MSet::get_termfreq(const string &tname) const
 224 {
 225     LOGCALL(API, Xapian::doccount, "Xapian::MSet::get_termfreq", tname);
 226     Assert(internal.get() != 0);
 227     if (usual(internal->stats)) {
 228         Xapian::doccount termfreq;
 229         if (internal->stats->get_stats(tname, termfreq))
 230             RETURN(termfreq);
 231     }
 232     if (internal->enquire.get() == 0) {
 233         throw InvalidOperationError("Can't get termfreq from an MSet which is not derived from a query.");
 234     }
 235     RETURN(internal->enquire->get_termfreq(tname));
 236 }
 237
 238 double
 239 MSet::get_termweight(const string &tname) const
 240 {
 241     LOGCALL(API, double, "Xapian::MSet::get_termweight", tname);
 242     Assert(internal.get() != 0);
 243     if (!internal->stats) {
 244         throw InvalidOperationError("Can't get termweight from an MSet which is not derived from a query.");
 245     }
 246     double termweight;
 247     if (!internal->stats->get_termweight(tname, termweight)) {
 248         string msg = tname;
 249         msg += ": termweight not available";
 250         throw InvalidArgumentError(msg);
 251     }
 252     RETURN(termweight);
 253 }
 254
 255 Xapian::doccount
 256 MSet::get_firstitem() const
 257 {
 258     Assert(internal.get() != 0);
 259     return internal->firstitem;
 260 }
 261
 262 Xapian::doccount
 263 MSet::get_matches_lower_bound() const
 264 {
 265     Assert(internal.get() != 0);
 266     return internal->matches_lower_bound;
 267 }
 268
 269 Xapian::doccount
 270 MSet::get_matches_estimated() const
 271 {
 272     Assert(internal.get() != 0);
 273     return internal->matches_estimated;
 274 }
 275
 276 Xapian::doccount
 277 MSet::get_matches_upper_bound() const
 278 {
 279     Assert(internal.get() != 0);
 280     return internal->matches_upper_bound;
 281 }
 282
 283 Xapian::doccount
 284 MSet::get_uncollapsed_matches_lower_bound() const
 285 {
 286     Assert(internal.get() != 0);
 287     return internal->uncollapsed_lower_bound;
 288 }
 289
 290 Xapian::doccount
 291 MSet::get_uncollapsed_matches_estimated() const
 292 {
 293     Assert(internal.get() != 0);
 294     return internal->uncollapsed_estimated;
 295 }
 296
 297 Xapian::doccount
 298 MSet::get_uncollapsed_matches_upper_bound() const
 299 {
 300     Assert(internal.get() != 0);
 301     return internal->uncollapsed_upper_bound;
 302 }
 303
 304 double
 305 MSet::get_max_possible() const
 306 {
 307     Assert(internal.get() != 0);
 308     return internal->max_possible;
 309 }
 310
 311 double
 312 MSet::get_max_attained() const
 313 {
 314     Assert(internal.get() != 0);
 315     return internal->max_attained;
 316 }
 317
 318 Xapian::doccount
 319 MSet::size() const
 320 {
 321     Assert(internal.get() != 0);
 322     return internal->items.size();
 323 }
 324
 325 bool
 326 MSet::empty() const
 327 {
 328     Assert(internal.get() != 0);
 329     return internal->items.empty();
 330 }
 331
 332 void
 333 MSet::swap(MSet & other)
 334 {
 335     std::swap(internal, other.internal);
 336 }
 337
 338 MSetIterator
 339 MSet::begin() const
 340 {
 341     return MSetIterator(0, *this);
 342 }
 343
 344 MSetIterator
 345 MSet::end() const
 346 {
 347     Assert(internal.get() != 0);
 348     return MSetIterator(internal->items.size(), *this);
 349 }
 350
 351 MSetIterator
 352 MSet::operator[](Xapian::doccount i) const
 353 {
 354     // Don't test 0 <= i - that gives a compiler warning if i is unsigned
 355     Assert(0 < (i + 1) && i < size());
 356     return MSetIterator(i, *this);
 357 }
 358
 359 MSetIterator
 360 MSet::back() const
 361 {
 362     Assert(!empty());
 363     Assert(internal.get() != 0);
 364     return MSetIterator(internal->items.size() - 1, *this);
 365 }
 366
 367 string
 368 MSet::get_description() const
 369 {
 370     Assert(internal.get() != 0);
 371     return "Xapian::MSet(" + internal->get_description() + ")";
 372 }
 373
 374 int
 375 MSet::Internal::convert_to_percent_internal(double wt) const
 376 {
 377     LOGCALL(MATCH, int, "Xapian::MSet::Internal::convert_to_percent_internal", wt);
 378     if (percent_factor == 0) RETURN(100);
 379
 380     // Excess precision on x86 can result in a difference here.
 381     double v = wt * percent_factor + 100.0 * DBL_EPSILON;
 382     int pcent = static_cast<int>(v);
 383     LOGLINE(MATCH, "wt = " << wt << ", max_possible = " << max_possible <<
 384                    " =>  pcent = " << pcent);
 385     if (pcent > 100) pcent = 100;
 386     if (pcent < 0) pcent = 0;
 387     if (pcent == 0 && wt > 0) pcent = 1;
 388
 389     RETURN(pcent);
 390 }
 391
 392 Document
 393 MSet::Internal::get_doc_by_index(Xapian::doccount index) const
 394 {
 395     LOGCALL(MATCH, Document, "Xapian::MSet::Internal::get_doc_by_index", index);
 396     index += firstitem;
 397     map<Xapian::doccount, Document>::const_iterator doc;
 398     doc = indexeddocs.find(index);
 399     if (doc != indexeddocs.end()) {
 400         RETURN(doc->second);
 401     }
 402     if (index < firstitem || index >= firstitem + items.size()) {
 403         throw RangeError("The mset returned from the match does not contain the document at index " + str(index));
 404     }
 405     Assert(enquire.get());
 406     if (!requested_docs.empty()) {
 407         // There's already a pending request, so handle that.
 408         read_docs();
 409         // Maybe we just fetched the doc we want.
 410         doc = indexeddocs.find(index);
 411         if (doc != indexeddocs.end()) {
 412             RETURN(doc->second);
 413         }
 414     }
 415
 416     // Don't cache unless fetch() was called by the API user.
 417     // FIXME: This is needed for the remote case, but for a local disk-based
 418     // database it means we try to pre-read right before we actually read,
 419     // which probably isn't useful.
 420     enquire->request_doc(items[index - firstitem]);
 421     RETURN(enquire->read_doc(items[index - firstitem]));
 422 }
 423
 424 void
 425 MSet::Internal::fetch_items(Xapian::doccount first, Xapian::doccount last) const
 426 {
 427     LOGCALL_VOID(MATCH, "Xapian::MSet::Internal::fetch_items", first | last);
 428     if (enquire.get() == 0) {
 429         throw InvalidOperationError("Can't fetch documents from an MSet which is not derived from a query.");
 430     }
 431     for (Xapian::doccount i = first; i <= last; ++i) {
 432         map<Xapian::doccount, Document>::const_iterator doc;
 433         doc = indexeddocs.find(i);
 434         if (doc == indexeddocs.end()) {
 435             /* We don't have the document cached */
 436             set<Xapian::doccount>::const_iterator s;
 437             s = requested_docs.find(i);
 438             if (s == requested_docs.end()) {
 439                 /* We haven't even requested it yet - do so now. */
 440                 enquire->request_doc(items[i - firstitem]);
 441                 requested_docs.insert(i);
 442             }
 443         }
 444     }
 445 }
 446
 447 string
 448 MSet::Internal::get_description() const
 449 {
 450     string description = "Xapian::MSet::Internal(";
 451
 452     description += "firstitem=" + str(firstitem) + ", " +
 453             "matches_lower_bound=" + str(matches_lower_bound) + ", " +
 454             "matches_estimated=" + str(matches_estimated) + ", " +
 455             "matches_upper_bound=" + str(matches_upper_bound) + ", " +
 456             "max_possible=" + str(max_possible) + ", " +
 457             "max_attained=" + str(max_attained);
 458
 459     for (vector<Xapian::Internal::MSetItem>::const_iterator i = items.begin();
 460          i != items.end(); ++i) {
 461         if (!description.empty()) description += ", ";
 462         description += i->get_description();
 463     }
 464
 465     description += ")";
 466
 467     return description;
 468 }
 469
 470 void
 471 MSet::Internal::read_docs() const
 472 {
 473     set<Xapian::doccount>::const_iterator i;
 474     for (i = requested_docs.begin(); i != requested_docs.end(); ++i) {
 475         indexeddocs[*i] = enquire->read_doc(items[*i - firstitem]);
 476         LOGLINE(MATCH, "stored doc at index " << *i << " is " << indexeddocs[*i]);
 477     }
 478     /* Clear list of requested but not fetched documents. */
 479     requested_docs.clear();
 480 }
 481
 482 // Methods for Xapian::ESet
 483
 484 ESet::ESet() : internal(new Internal) { }
 485
 486 ESet::~ESet()
 487 {
 488 }
 489
 490 ESet::ESet(const ESet & other) : internal(other.internal)
 491 {
 492 }
 493
 494 void
 495 ESet::operator=(const ESet &other)
 496 {
 497     internal = other.internal;
 498 }
 499
 500 Xapian::termcount
 501 ESet::get_ebound() const
 502 {
 503     return internal->ebound;
 504 }
 505
 506 Xapian::termcount
 507 ESet::size() const
 508 {
 509     return internal->items.size();
 510 }
 511
 512 bool
 513 ESet::empty() const
 514 {
 515     return internal->items.empty();
 516 }
 517
 518 void
 519 ESet::swap(ESet & other)
 520 {
 521     std::swap(internal, other.internal);
 522 }
 523
 524 ESetIterator
 525 ESet::begin() const
 526 {
 527     return ESetIterator(0, *this);
 528 }
 529
 530 ESetIterator
 531 ESet::end() const
 532 {
 533     Assert(internal.get() != 0);
 534     return ESetIterator(internal->items.size(), *this);
 535 }
 536
 537 ESetIterator
 538 ESet::operator[](Xapian::termcount i) const
 539 {
 540     // Don't test 0 <= i - that gives a compiler warning if i is unsigned
 541     Assert(0 < (i + 1) && i < size());
 542     return ESetIterator(i, *this);
 543 }
 544
 545 ESetIterator
 546 ESet::back() const
 547 {
 548     Assert(!empty());
 549     Assert(internal.get() != 0);
 550     return ESetIterator(internal->items.size() - 1, *this);
 551 }
 552
 553 string
 554 ESet::get_description() const
 555 {
 556     Assert(internal.get() != 0);
 557     return "Xapian::ESet(" + internal->get_description() + ")";
 558 }
 559
 560 // Xapian::ESetIterator
 561
 562 const string &
 563 ESetIterator::operator *() const
 564 {
 565     Assert(eset.internal.get());
 566     AssertRel(index,<,eset.internal->items.size());
 567     return eset.internal->items[index].term;
 568 }
 569
 570 double
 571 ESetIterator::get_weight() const
 572 {
 573     Assert(eset.internal.get());
 574     AssertRel(index,<,eset.internal->items.size());
 575     return eset.internal->items[index].wt;
 576 }
 577
 578 string
 579 ESetIterator::get_description() const
 580 {
 581     return "Xapian::ESetIterator(" + str(index) + ")";
 582 }
 583
 584 // MSetIterator
 585
 586 Xapian::docid
 587 MSetIterator::operator *() const
 588 {
 589     Assert(mset.internal.get());
 590     AssertRel(index,<,mset.internal->items.size());
 591     return mset.internal->items[index].did;
 592 }
 593
 594 Document
 595 MSetIterator::get_document() const
 596 {
 597     Assert(mset.internal.get());
 598     AssertRel(index,<,mset.internal->items.size());
 599     return mset.internal->get_doc_by_index(index);
 600 }
 601
 602 double
 603 MSetIterator::get_weight() const
 604 {
 605     Assert(mset.internal.get());
 606     AssertRel(index,<,mset.internal->items.size());
 607     return mset.internal->items[index].wt;
 608 }
 609
 610 std::string
 611 MSetIterator::get_collapse_key() const
 612 {
 613     Assert(mset.internal.get());
 614     AssertRel(index,<,mset.internal->items.size());
 615     return mset.internal->items[index].collapse_key;
 616 }
 617
 618 Xapian::doccount
 619 MSetIterator::get_collapse_count() const
 620 {
 621     Assert(mset.internal.get());
 622     AssertRel(index,<,mset.internal->items.size());
 623     return mset.internal->items[index].collapse_count;
 624 }
 625
 626 int
 627 MSetIterator::get_percent() const
 628 {
 629     LOGCALL(API, int, "MSetIterator::get_percent", NO_ARGS);
 630     RETURN(mset.internal->convert_to_percent_internal(get_weight()));
 631 }
 632
 633 string
 634 MSetIterator::get_description() const
 635 {
 636     return "Xapian::MSetIterator(" + str(index) + ")";
 637 }
 638
 639 // Methods for Xapian::Enquire::Internal
 640
 641 Enquire::Internal::Internal(const Database &db_, ErrorHandler * errorhandler_)
 642   : db(db_), query(), collapse_key(Xapian::BAD_VALUENO), collapse_max(0),
 643     order(Enquire::ASCENDING), percent_cutoff(0), weight_cutoff(0),
 644     sort_key(Xapian::BAD_VALUENO), sort_by(REL), sort_value_forward(true),
 645     sorter(0), time_limit(0.0), errorhandler(errorhandler_), weight(0),
 646     eweightname("trad"), expand_k(1.0)
 647 {
 648     if (db.internal.empty()) {
 649         throw InvalidArgumentError("Can't make an Enquire object from an uninitialised Database object.");
 650     }
 651 }
 652
 653 Enquire::Internal::~Internal()
 654 {
 655     delete weight;
 656     weight = 0;
 657 }
 658
 659 void
 660 Enquire::Internal::set_query(const Query &query_, termcount qlen_)
 661 {
 662     query = query_;
 663     qlen = qlen_ ? qlen_ : query.get_length();
 664 }
 665
 666 const Query &
 667 Enquire::Internal::get_query() const
 668 {
 669     return query;
 670 }
 671
 672 MSet
 673 Enquire::Internal::get_mset(Xapian::doccount first, Xapian::doccount maxitems,
 674                             Xapian::doccount check_at_least, const RSet *rset,
 675                             const MatchDecider *mdecider) const
 676 {
 677     LOGCALL(MATCH, MSet, "Enquire::Internal::get_mset", first | maxitems | check_at_least | rset | mdecider);
 678
 679     if (percent_cutoff && (sort_by == VAL || sort_by == VAL_REL)) {
 680         throw Xapian::UnimplementedError("Use of a percentage cutoff while sorting primary by value isn't currently supported");
 681     }
 682
 683     if (weight == 0) {
 684         weight = new BM25Weight;
 685     }
 686
 687     Xapian::doccount first_orig = first;
 688     {
 689         Xapian::doccount docs = db.get_doccount();
 690         first = min(first, docs);
 691         maxitems = min(maxitems, docs);
 692         check_at_least = min(check_at_least, docs);
 693         check_at_least = max(check_at_least, maxitems);
 694     }
 695
 696     AutoPtr<Xapian::Weight::Internal> stats(new Xapian::Weight::Internal);
 697     ::MultiMatch match(db, query, qlen, rset,
 698                        collapse_max, collapse_key,
 699                        percent_cutoff, weight_cutoff,
 700                        order, sort_key, sort_by, sort_value_forward,
 701                        time_limit, errorhandler, *(stats.get()), weight, spies,
 702                        (sorter != NULL),
 703                        (mdecider != NULL));
 704     // Run query and put results into supplied Xapian::MSet object.
 705     MSet retval;
 706     match.get_mset(first, maxitems, check_at_least, retval,
 707                    *(stats.get()), mdecider, sorter);
 708     if (first_orig != first && retval.internal.get()) {
 709         retval.internal->firstitem = first_orig;
 710     }
 711
 712     Assert(weight->name() != "bool" || retval.get_max_possible() == 0);
 713
 714     // The Xapian::MSet needs to have a pointer to ourselves, so that it can
 715     // retrieve the documents.  This is set here explicitly to avoid having
 716     // to pass it into the matcher, which gets messy particularly in the
 717     // networked case.
 718     retval.internal->enquire = this;
 719
 720     if (!retval.internal->stats) {
 721         retval.internal->stats = stats.release();
 722     }
 723
 724     RETURN(retval);
 725 }
 726
 727 ESet
 728 Enquire::Internal::get_eset(Xapian::termcount maxitems,
 729                             const RSet & rset, int flags,
 730                             const ExpandDecider * edecider, double min_wt) const
 731 {
 732     LOGCALL(MATCH, ESet, "Enquire::Internal::get_eset", maxitems | rset | flags | edecider | min_wt);
 733
 734     if (maxitems == 0 || rset.empty()) {
 735         // Either we were asked for no results, or wouldn't produce any
 736         // because no documents were marked as relevant.
 737         RETURN(ESet());
 738     }
 739
 740     LOGVALUE(MATCH, rset.size());
 741
 742     /* The AutoPtrs will clean up any dynamically allocated
 743      * expand deciders automatically.
 744      */
 745     AutoPtr<ExpandDecider> decider_noquery;
 746     AutoPtr<ExpandDecider> decider_andnoquery;
 747
 748     if (!query.empty() && !(flags & Enquire::INCLUDE_QUERY_TERMS)) {
 749         decider_noquery.reset(
 750             new ExpandDeciderFilterTerms(query.get_terms_begin(),
 751                                          query.get_terms_end()));
 752         if (edecider) {
 753             decider_andnoquery.reset(
 754                 new ExpandDeciderAnd(decider_noquery.get(), edecider));
 755             edecider = decider_andnoquery.get();
 756         } else {
 757             edecider = decider_noquery.get();
 758         }
 759     }
 760
 761     bool use_exact_termfreq(flags & Enquire::USE_EXACT_TERMFREQ);
 762     Xapian::ESet eset;
 763
 764     if (eweightname == "bo1") {
 765         Bo1EWeight bo1eweight(db, rset.size(), use_exact_termfreq);
 766         eset.internal->expand(maxitems, db, rset, edecider, bo1eweight, min_wt);
 767     } else {
 768         TradEWeight tradeweight(db, rset.size(), use_exact_termfreq, expand_k);
 769         eset.internal->expand(maxitems, db, rset, edecider, tradeweight, min_wt);
 770     }
 771
 772     RETURN(eset);
 773 }
 774
 775 class ByQueryIndexCmp {
 776  private:
 777     typedef map<string, unsigned int> tmap_t;
 778     const tmap_t &tmap;
 779  public:
 780     ByQueryIndexCmp(const tmap_t &tmap_) : tmap(tmap_) {}
 781     bool operator()(const string &left,
 782                     const string &right) const {
 783         tmap_t::const_iterator l, r;
 784         l = tmap.find(left);
 785         r = tmap.find(right);
 786         Assert((l != tmap.end()) && (r != tmap.end()));
 787
 788         return l->second < r->second;
 789     }
 790 };
 791
 792 TermIterator
 793 Enquire::Internal::get_matching_terms(Xapian::docid did) const
 794 {
 795     if (query.empty())
 796         return TermIterator();
 797
 798     // The ordered list of terms in the query.
 799     TermIterator qt = query.get_terms_begin();
 800
 801     // copy the list of query terms into a map for faster access.
 802     // FIXME: a hash would be faster than a map, if this becomes
 803     // a problem.
 804     map<string, unsigned int> tmap;
 805     unsigned int index = 1;
 806     for ( ; qt != query.get_terms_end(); ++qt) {
 807         if (tmap.find(*qt) == tmap.end())
 808             tmap[*qt] = index++;
 809     }
 810
 811     vector<string> matching_terms;
 812
 813     TermIterator docterms = db.termlist_begin(did);
 814     TermIterator docterms_end = db.termlist_end(did);
 815     while (docterms != docterms_end) {
 816         string term = *docterms;
 817         map<string, unsigned int>::iterator t = tmap.find(term);
 818         if (t != tmap.end()) matching_terms.push_back(term);
 819         ++docterms;
 820     }
 821
 822     // sort the resulting list by query position.
 823     sort(matching_terms.begin(), matching_terms.end(), ByQueryIndexCmp(tmap));
 824
 825     return TermIterator(new VectorTermList(matching_terms.begin(),
 826                                            matching_terms.end()));
 827 }
 828
 829 TermIterator
 830 Enquire::Internal::get_matching_terms(const MSetIterator &it) const
 831 {
 832     // FIXME: take advantage of MSetIterator to ensure that database
 833     // doesn't get modified underneath us.
 834     return get_matching_terms(*it);
 835 }
 836
 837 Xapian::doccount
 838 Enquire::Internal::get_termfreq(const string &tname) const
 839 {
 840     return db.get_termfreq(tname);
 841 }
 842
 843 string
 844 Enquire::Internal::get_description() const
 845 {
 846     string description = db.get_description();
 847     description += ", ";
 848     description += query.get_description();
 849     return description;
 850 }
 851
 852 // Private methods for Xapian::Enquire::Internal
 853
 854 void
 855 Enquire::Internal::request_doc(const Xapian::Internal::MSetItem &item) const
 856 {
 857     try {
 858         unsigned int multiplier = db.internal.size();
 859
 860         Xapian::docid realdid = (item.did - 1) / multiplier + 1;
 861         Xapian::doccount dbnumber = (item.did - 1) % multiplier;
 862
 863         db.internal[dbnumber]->request_document(realdid);
 864     } catch (Error & e) {
 865         if (errorhandler) (*errorhandler)(e);
 866         throw;
 867     }
 868 }
 869
 870 Document
 871 Enquire::Internal::read_doc(const Xapian::Internal::MSetItem &item) const
 872 {
 873     try {
 874         unsigned int multiplier = db.internal.size();
 875
 876         Xapian::docid realdid = (item.did - 1) / multiplier + 1;
 877         Xapian::doccount dbnumber = (item.did - 1) % multiplier;
 878
 879         Xapian::Document::Internal *doc;
 880         doc = db.internal[dbnumber]->collect_document(realdid);
 881         return Document(doc);
 882     } catch (Error & e) {
 883         if (errorhandler) (*errorhandler)(e);
 884         throw;
 885     }
 886 }
 887
 888 // Methods of Xapian::Enquire
 889
 890 Enquire::Enquire(const Enquire & other) : internal(other.internal)
 891 {
 892     LOGCALL_CTOR(API, "Enquire", other);
 893 }
 894
 895 void
 896 Enquire::operator=(const Enquire & other)
 897 {
 898     LOGCALL_VOID(API, "Xapian::Enquire::operator=", other);
 899     internal = other.internal;
 900 }
 901
 902 Enquire::Enquire(const Database &databases)
 903     : internal(new Internal(databases, NULL))
 904 {
 905     LOGCALL_CTOR(API, "Enquire", databases);
 906 }
 907
 908 Enquire::Enquire(const Database &databases, ErrorHandler * errorhandler)
 909     : internal(new Internal(databases, errorhandler))
 910 {
 911     LOGCALL_CTOR(API, "Enquire", databases | errorhandler);
 912 }
 913
 914 Enquire::~Enquire()
 915 {
 916     LOGCALL_DTOR(API, "Enquire");
 917 }
 918
 919 void
 920 Enquire::set_query(const Query & query, termcount len)
 921 {
 922     LOGCALL_VOID(API, "Xapian::Enquire::set_query", query | len);
 923     internal->set_query(query, len);
 924 }
 925
 926 const Query &
 927 Enquire::get_query() const
 928 {
 929     LOGCALL(API, const Xapian::Query &, "Xapian::Enquire::get_query", NO_ARGS);
 930     RETURN(internal->get_query());
 931 }
 932
 933 void
 934 Enquire::add_matchspy(MatchSpy * spy) {
 935     LOGCALL_VOID(API, "Xapian::Enquire::add_matchspy", spy);
 936     internal->spies.push_back(spy);
 937 }
 938
 939 void
 940 Enquire::clear_matchspies() {
 941     LOGCALL_VOID(API, "Xapian::Enquire::clear_matchspies", NO_ARGS);
 942     internal->spies.clear();
 943 }
 944
 945 void
 946 Enquire::set_weighting_scheme(const Weight &weight_)
 947 {
 948     LOGCALL_VOID(API, "Xapian::Enquire::set_weighting_scheme", weight_);
 949     // Clone first in case doing so throws an exception.
 950     Weight * wt = weight_.clone();
 951     swap(wt, internal->weight);
 952     delete wt;
 953 }
 954
 955 void
 956 Enquire::set_expansion_scheme(const std::string &eweightname_, double expand_k_) const
 957 {
 958      LOGCALL_VOID(API, "Xapian::Enquire::set_expansion_scheme", eweightname_ | expand_k_);
 959
 960      if (eweightname_ != "bo1" && eweightname_ != "trad") {
 961          throw InvalidArgumentError("Invalid name for query expansion scheme.");
 962      }
 963
 964      internal->eweightname = eweightname_;
 965      internal->expand_k = expand_k_;
 966 }
 967
 968 void
 969 Enquire::set_collapse_key(Xapian::valueno collapse_key, Xapian::doccount collapse_max)
 970 {
 971     if (collapse_key == Xapian::BAD_VALUENO) collapse_max = 0;
 972     internal->collapse_key = collapse_key;
 973     internal->collapse_max = collapse_max;
 974 }
 975
 976 void
 977 Enquire::set_docid_order(Enquire::docid_order order)
 978 {
 979     internal->order = order;
 980 }
 981
 982 void
 983 Enquire::set_cutoff(int percent_cutoff, double weight_cutoff)
 984 {
 985     internal->percent_cutoff = percent_cutoff;
 986     internal->weight_cutoff = weight_cutoff;
 987 }
 988
 989 void
 990 Enquire::set_sort_by_relevance()
 991 {
 992     internal->sort_by = Internal::REL;
 993 }
 994
 995 void
 996 Enquire::set_sort_by_value(valueno sort_key, bool ascending)
 997 {
 998     internal->sorter = NULL;
 999     internal->sort_key = sort_key;
1000     internal->sort_by = Internal::VAL;
1001     internal->sort_value_forward = ascending;
1002 }
1003
1004 void
1005 Enquire::set_sort_by_value_then_relevance(valueno sort_key, bool ascending)
1006 {
1007     internal->sorter = NULL;
1008     internal->sort_key = sort_key;
1009     internal->sort_by = Internal::VAL_REL;
1010     internal->sort_value_forward = ascending;
1011 }
1012
1013 void
1014 Enquire::set_sort_by_relevance_then_value(valueno sort_key, bool ascending)
1015 {
1016     internal->sorter = NULL;
1017     internal->sort_key = sort_key;
1018     internal->sort_by = Internal::REL_VAL;
1019     internal->sort_value_forward = ascending;
1020 }
1021
1022 void
1023 Enquire::set_sort_by_key(KeyMaker * sorter, bool ascending)
1024 {
1025     if (sorter == NULL)
1026         throw InvalidArgumentError("sorter can't be NULL");
1027     internal->sorter = sorter;
1028     internal->sort_by = Internal::VAL;
1029     internal->sort_value_forward = ascending;
1030 }
1031
1032 void
1033 Enquire::set_sort_by_key_then_relevance(KeyMaker * sorter, bool ascending)
1034 {
1035     if (sorter == NULL)
1036         throw InvalidArgumentError("sorter can't be NULL");
1037     internal->sorter = sorter;
1038     internal->sort_by = Internal::VAL_REL;
1039     internal->sort_value_forward = ascending;
1040 }
1041
1042 void
1043 Enquire::set_sort_by_relevance_then_key(KeyMaker * sorter, bool ascending)
1044 {
1045     if (sorter == NULL)
1046         throw Xapian::InvalidArgumentError("sorter can't be NULL");
1047     internal->sorter = sorter;
1048     internal->sort_by = Internal::REL_VAL;
1049     internal->sort_value_forward = ascending;
1050 }
1051
1052 void
1053 Enquire::set_time_limit(double time_limit)
1054 {
1055     internal->time_limit = time_limit;
1056 }
1057
1058 MSet
1059 Enquire::get_mset(Xapian::doccount first, Xapian::doccount maxitems,
1060                   Xapian::doccount check_at_least, const RSet *rset,
1061                   const MatchDecider *mdecider) const
1062 {
1063     LOGCALL(API, Xapian::MSet, "Xapian::Enquire::get_mset", first | maxitems | check_at_least | rset | mdecider);
1064
1065     try {
1066         RETURN(internal->get_mset(first, maxitems, check_at_least, rset,
1067                                   mdecider));
1068     } catch (Error & e) {
1069         if (internal->errorhandler) (*internal->errorhandler)(e);
1070         throw;
1071     }
1072 }
1073
1074 ESet
1075 Enquire::get_eset(Xapian::termcount maxitems, const RSet & rset, int flags,
1076                   const ExpandDecider * edecider, double min_wt) const
1077 {
1078     LOGCALL(API, Xapian::ESet, "Xapian::Enquire::get_eset", maxitems | rset | flags | edecider | min_wt);
1079
1080     try {
1081         RETURN(internal->get_eset(maxitems, rset, flags, edecider, min_wt));
1082     } catch (Error & e) {
1083         if (internal->errorhandler) (*internal->errorhandler)(e);
1084         throw;
1085     }
1086 }
1087
1088 TermIterator
1089 Enquire::get_matching_terms_begin(const MSetIterator &it) const
1090 {
1091     LOGCALL(API, Xapian::TermIterator, "Xapian::Enquire::get_matching_terms_begin", it);
1092     try {
1093         RETURN(internal->get_matching_terms(it));
1094     } catch (Error & e) {
1095         if (internal->errorhandler) (*internal->errorhandler)(e);
1096         throw;
1097     }
1098 }
1099
1100 TermIterator
1101 Enquire::get_matching_terms_begin(Xapian::docid did) const
1102 {
1103     LOGCALL(API, Xapian::TermIterator, "Xapian::Enquire::get_matching_terms_begin", did);
1104     try {
1105         RETURN(internal->get_matching_terms(did));
1106     } catch (Error & e) {
1107         if (internal->errorhandler) (*internal->errorhandler)(e);
1108         throw;
1109     }
1110 }
1111
1112 string
1113 Enquire::get_description() const
1114 {
1115     return "Xapian::Enquire(" + internal->get_description() + ")";
1116 }
1117
1118 }