xapian-core/api/postingsource.cc

   1 /** @file postingsource.cc
   2  * @brief External sources of posting information
   3  */
   4 /* Copyright (C) 2008,2009,2010,2011,2012,2015,2016 Olly Betts
   5  * Copyright (C) 2008,2009 Lemur Consulting Ltd
   6  * Copyright (C) 2010 Richard Boulton
   7  *
   8  * This program is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU General Public License as
  10  * published by the Free Software Foundation; either version 2 of the
  11  * License, or (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  21  */
  22
  23 #include <config.h>
  24
  25 // We need to be able to set deprecated members of ValuePostingSource.
  26 #define XAPIAN_DEPRECATED(X) X
  27 #include "xapian/postingsource.h"
  28
  29 #include "autoptr.h"
  30
  31 #include "backends/database.h"
  32 #include "backends/document.h"
  33 #include "matcher/multimatch.h"
  34
  35 #include "xapian/document.h"
  36 #include "xapian/error.h"
  37 #include "xapian/queryparser.h" // For sortable_unserialise().
  38
  39 #include "omassert.h"
  40 #include "net/length.h"
  41 #include "serialise-double.h"
  42 #include "str.h"
  43
  44 #include <cfloat>
  45
  46 using namespace std;
  47
  48 namespace Xapian {
  49
  50 PostingSource::~PostingSource() { }
  51
  52 void
  53 PostingSource::set_maxweight(double max_weight)
  54 {
  55     if (usual(matcher_)) {
  56         MultiMatch * multimatch = static_cast<MultiMatch*>(matcher_);
  57         multimatch->recalc_maxweight();
  58     }
  59     max_weight_ = max_weight;
  60 }
  61
  62 double
  63 PostingSource::get_weight() const
  64 {
  65     return 0;
  66 }
  67
  68 void
  69 PostingSource::skip_to(Xapian::docid did, double min_wt)
  70 {
  71     while (!at_end() && get_docid() < did) {
  72         next(min_wt);
  73     }
  74 }
  75
  76 bool
  77 PostingSource::check(Xapian::docid did, double min_wt)
  78 {
  79     skip_to(did, min_wt);
  80     return true;
  81 }
  82
  83 PostingSource *
  84 PostingSource::clone() const
  85 {
  86     return NULL;
  87 }
  88
  89 string
  90 PostingSource::name() const
  91 {
  92     return string();
  93 }
  94
  95 string
  96 PostingSource::serialise() const
  97 {
  98     throw Xapian::UnimplementedError("serialise() not supported for this PostingSource");
  99 }
 100
 101 PostingSource *
 102 PostingSource::unserialise(const string &) const
 103 {
 104     throw Xapian::UnimplementedError("unserialise() not supported for this PostingSource");
 105 }
 106
 107 PostingSource *
 108 PostingSource::unserialise_with_registry(const std::string &s,
 109                                          const Registry &) const
 110 {
 111     return unserialise(s);
 112 }
 113
 114 string
 115 PostingSource::get_description() const
 116 {
 117     return "Xapian::PostingSource subclass";
 118 }
 119
 120
 121 ValuePostingSource::ValuePostingSource(Xapian::valueno slot_)
 122         : real_slot(slot_),
 123           db(real_db),
 124           slot(real_slot),
 125           value_it(real_value_it),
 126           started(real_started),
 127           termfreq_min(real_termfreq_min),
 128           termfreq_est(real_termfreq_est),
 129           termfreq_max(real_termfreq_max)
 130 {
 131 }
 132
 133 Xapian::doccount
 134 ValuePostingSource::get_termfreq_min() const
 135 {
 136     return real_termfreq_min;
 137 }
 138
 139 Xapian::doccount
 140 ValuePostingSource::get_termfreq_est() const
 141 {
 142     return real_termfreq_est;
 143 }
 144
 145 Xapian::doccount
 146 ValuePostingSource::get_termfreq_max() const
 147 {
 148     return real_termfreq_max;
 149 }
 150
 151 void
 152 ValuePostingSource::next(double min_wt)
 153 {
 154     if (!real_started) {
 155         real_started = true;
 156         real_value_it = real_db.valuestream_begin(real_slot);
 157     } else {
 158         ++real_value_it;
 159     }
 160
 161     if (real_value_it == real_db.valuestream_end(real_slot)) return;
 162
 163     if (min_wt > get_maxweight()) {
 164         real_value_it = real_db.valuestream_end(real_slot);
 165         return;
 166     }
 167 }
 168
 169 void
 170 ValuePostingSource::skip_to(Xapian::docid min_docid, double min_wt)
 171 {
 172     if (!real_started) {
 173         real_started = true;
 174         real_value_it = real_db.valuestream_begin(real_slot);
 175
 176         if (real_value_it == real_db.valuestream_end(real_slot)) return;
 177     }
 178
 179     if (min_wt > get_maxweight()) {
 180         real_value_it = real_db.valuestream_end(real_slot);
 181         return;
 182     }
 183     real_value_it.skip_to(min_docid);
 184 }
 185
 186 bool
 187 ValuePostingSource::check(Xapian::docid min_docid, double min_wt)
 188 {
 189     if (!real_started) {
 190         real_started = true;
 191         real_value_it = real_db.valuestream_begin(real_slot);
 192
 193         if (real_value_it == real_db.valuestream_end(real_slot)) return true;
 194     }
 195
 196     if (min_wt > get_maxweight()) {
 197         real_value_it = real_db.valuestream_end(real_slot);
 198         return true;
 199     }
 200     return real_value_it.check(min_docid);
 201 }
 202
 203 bool
 204 ValuePostingSource::at_end() const
 205 {
 206     return real_started && real_value_it == real_db.valuestream_end(real_slot);
 207 }
 208
 209 Xapian::docid
 210 ValuePostingSource::get_docid() const
 211 {
 212     return real_value_it.get_docid();
 213 }
 214
 215 void
 216 ValuePostingSource::init(const Database & db_)
 217 {
 218     real_db = db_;
 219     real_started = false;
 220     set_maxweight(DBL_MAX);
 221     try {
 222         real_termfreq_max = real_db.get_value_freq(real_slot);
 223         real_termfreq_est = real_termfreq_max;
 224         real_termfreq_min = real_termfreq_max;
 225     } catch (const Xapian::UnimplementedError &) {
 226         real_termfreq_max = real_db.get_doccount();
 227         real_termfreq_est = real_termfreq_max / 2;
 228         real_termfreq_min = 0;
 229     }
 230 }
 231
 232
 233 ValueWeightPostingSource::ValueWeightPostingSource(Xapian::valueno slot_)
 234         : ValuePostingSource(slot_)
 235 {
 236 }
 237
 238 double
 239 ValueWeightPostingSource::get_weight() const
 240 {
 241     Assert(!at_end());
 242     Assert(get_started());
 243     return sortable_unserialise(get_value());
 244 }
 245
 246 ValueWeightPostingSource *
 247 ValueWeightPostingSource::clone() const
 248 {
 249     return new ValueWeightPostingSource(get_slot());
 250 }
 251
 252 string
 253 ValueWeightPostingSource::name() const
 254 {
 255     return string("Xapian::ValueWeightPostingSource");
 256 }
 257
 258 string
 259 ValueWeightPostingSource::serialise() const
 260 {
 261     return encode_length(get_slot());
 262 }
 263
 264 ValueWeightPostingSource *
 265 ValueWeightPostingSource::unserialise(const string &s) const
 266 {
 267     const char * p = s.data();
 268     const char * end = p + s.size();
 269
 270     Xapian::valueno new_slot;
 271     decode_length(&p, end, new_slot);
 272     if (p != end) {
 273         throw Xapian::NetworkError("Bad serialised ValueWeightPostingSource - junk at end");
 274     }
 275
 276     return new ValueWeightPostingSource(new_slot);
 277 }
 278
 279 void
 280 ValueWeightPostingSource::init(const Database & db_)
 281 {
 282     ValuePostingSource::init(db_);
 283
 284     string upper_bound;
 285     try {
 286         upper_bound = get_database().get_value_upper_bound(get_slot());
 287     } catch (const Xapian::UnimplementedError &) {
 288         // ValuePostingSource::init() set the maxweight to DBL_MAX.
 289         return;
 290     }
 291
 292     if (upper_bound.empty()) {
 293         // This should only happen if there are no entries, in which case the
 294         // maxweight is 0.
 295         set_maxweight(0.0);
 296     } else {
 297         set_maxweight(sortable_unserialise(upper_bound));
 298     }
 299 }
 300
 301 string
 302 ValueWeightPostingSource::get_description() const
 303 {
 304     string desc("Xapian::ValueWeightPostingSource(slot=");
 305     desc += str(get_slot());
 306     desc += ")";
 307     return desc;
 308 }
 309
 310
 311 ValueMapPostingSource::ValueMapPostingSource(Xapian::valueno slot_)
 312         : ValuePostingSource(slot_),
 313           default_weight(0.0),
 314           max_weight_in_map(0.0)
 315 {
 316 }
 317
 318 void
 319 ValueMapPostingSource::add_mapping(const string & key, double wt)
 320 {
 321     weight_map[key] = wt;
 322     max_weight_in_map = max(wt, max_weight_in_map);
 323 }
 324
 325 void
 326 ValueMapPostingSource::clear_mappings()
 327 {
 328     weight_map.clear();
 329     max_weight_in_map = 0.0;
 330 }
 331
 332 void
 333 ValueMapPostingSource::set_default_weight(double wt)
 334 {
 335     default_weight = wt;
 336 }
 337
 338 double
 339 ValueMapPostingSource::get_weight() const
 340 {
 341     map<string, double>::const_iterator wit = weight_map.find(get_value());
 342     if (wit == weight_map.end()) {
 343         return default_weight;
 344     }
 345     return wit->second;
 346 }
 347
 348 ValueMapPostingSource *
 349 ValueMapPostingSource::clone() const
 350 {
 351     AutoPtr<ValueMapPostingSource> res(new ValueMapPostingSource(get_slot()));
 352     map<string, double>::const_iterator i;
 353     for (i = weight_map.begin(); i != weight_map.end(); ++i) {
 354         res->add_mapping(i->first, i->second);
 355     }
 356     res->set_default_weight(default_weight);
 357     return res.release();
 358 }
 359
 360 string
 361 ValueMapPostingSource::name() const
 362 {
 363     return string("Xapian::ValueMapPostingSource");
 364 }
 365
 366 string
 367 ValueMapPostingSource::serialise() const
 368 {
 369     string result = encode_length(get_slot());
 370     result += serialise_double(default_weight);
 371
 372     map<string, double>::const_iterator i;
 373     for (i = weight_map.begin(); i != weight_map.end(); ++i) {
 374         result.append(encode_length(i->first.size()));
 375         result.append(i->first);
 376         result.append(serialise_double(i->second));
 377     }
 378
 379     return result;
 380 }
 381
 382 ValueMapPostingSource *
 383 ValueMapPostingSource::unserialise(const string &s) const
 384 {
 385     const char * p = s.data();
 386     const char * end = p + s.size();
 387
 388     Xapian::valueno new_slot;
 389     decode_length(&p, end, new_slot);
 390     AutoPtr<ValueMapPostingSource> res(new ValueMapPostingSource(new_slot));
 391     res->set_default_weight(unserialise_double(&p, end));
 392     while (p != end) {
 393         size_t keylen;
 394         decode_length_and_check(&p, end, keylen);
 395         string key(p, keylen);
 396         p += keylen;
 397         res->add_mapping(key, unserialise_double(&p, end));
 398     }
 399     return res.release();
 400 }
 401
 402 void
 403 ValueMapPostingSource::init(const Database & db_)
 404 {
 405     ValuePostingSource::init(db_);
 406     set_maxweight(max(max_weight_in_map, default_weight));
 407 }
 408
 409 string
 410 ValueMapPostingSource::get_description() const
 411 {
 412     string desc("Xapian::ValueMapPostingSource(slot=");
 413     desc += str(get_slot());
 414     desc += ")";
 415     return desc;
 416 }
 417
 418 FixedWeightPostingSource::FixedWeightPostingSource(double wt)
 419     : started(false)
 420 {
 421     // The weight is fixed at wt, so that's the maxweight too.  So just store wt
 422     // as the maxweight and we can read it from there when we need it.
 423     set_maxweight(wt);
 424 }
 425
 426 Xapian::doccount
 427 FixedWeightPostingSource::get_termfreq_min() const
 428 {
 429     return termfreq;
 430 }
 431
 432 Xapian::doccount
 433 FixedWeightPostingSource::get_termfreq_est() const
 434 {
 435     return termfreq;
 436 }
 437
 438 Xapian::doccount
 439 FixedWeightPostingSource::get_termfreq_max() const
 440 {
 441     return termfreq;
 442 }
 443
 444 double
 445 FixedWeightPostingSource::get_weight() const
 446 {
 447     return get_maxweight();
 448 }
 449
 450 void
 451 FixedWeightPostingSource::next(double min_wt)
 452 {
 453     if (!started) {
 454         started = true;
 455         it = db.postlist_begin(string());
 456     } else {
 457         ++it;
 458     }
 459
 460     if (it == db.postlist_end(string())) return;
 461
 462     if (check_docid) {
 463         it.skip_to(check_docid + 1);
 464         check_docid = 0;
 465     }
 466
 467     if (min_wt > get_maxweight()) {
 468         it = db.postlist_end(string());
 469     }
 470 }
 471
 472 void
 473 FixedWeightPostingSource::skip_to(Xapian::docid min_docid, double min_wt)
 474 {
 475     if (!started) {
 476         started = true;
 477         it = db.postlist_begin(string());
 478
 479         if (it == db.postlist_end(string())) return;
 480     }
 481
 482     if (check_docid) {
 483         if (min_docid < check_docid)
 484             min_docid = check_docid + 1;
 485         check_docid = 0;
 486     }
 487
 488     if (min_wt > get_maxweight()) {
 489         it = db.postlist_end(string());
 490         return;
 491     }
 492     it.skip_to(min_docid);
 493 }
 494
 495 bool
 496 FixedWeightPostingSource::check(Xapian::docid min_docid, double)
 497 {
 498     // We're guaranteed not to be called if the document doesn't
 499     // exist, so just remember the docid passed, and return true.
 500     check_docid = min_docid;
 501     return true;
 502 }
 503
 504 bool
 505 FixedWeightPostingSource::at_end() const
 506 {
 507     if (check_docid != 0) return false;
 508     return started && it == db.postlist_end(string());
 509 }
 510
 511 Xapian::docid
 512 FixedWeightPostingSource::get_docid() const
 513 {
 514     if (check_docid != 0) return check_docid;
 515     return *it;
 516 }
 517
 518 FixedWeightPostingSource *
 519 FixedWeightPostingSource::clone() const
 520 {
 521     return new FixedWeightPostingSource(get_maxweight());
 522 }
 523
 524 string
 525 FixedWeightPostingSource::name() const
 526 {
 527     return string("Xapian::FixedWeightPostingSource");
 528 }
 529
 530 string
 531 FixedWeightPostingSource::serialise() const
 532 {
 533     return serialise_double(get_maxweight());
 534 }
 535
 536 FixedWeightPostingSource *
 537 FixedWeightPostingSource::unserialise(const string &s) const
 538 {
 539     const char * p = s.data();
 540     const char * s_end = p + s.size();
 541     double new_wt = unserialise_double(&p, s_end);
 542     if (p != s_end) {
 543         throw Xapian::NetworkError("Bad serialised FixedWeightPostingSource - junk at end");
 544     }
 545     return new FixedWeightPostingSource(new_wt);
 546 }
 547
 548 void
 549 FixedWeightPostingSource::init(const Xapian::Database & db_)
 550 {
 551     db = db_;
 552     termfreq = db_.get_doccount();
 553     started = false;
 554     check_docid = 0;
 555 }
 556
 557 string
 558 FixedWeightPostingSource::get_description() const
 559 {
 560     string desc("Xapian::FixedWeightPostingSource(wt=");
 561     desc += str(get_maxweight());
 562     desc += ")";
 563     return desc;
 564 }
 565
 566 }