xapian-core/tests/api_weight.cc

   1 /** @file api_weight.cc
   2  * @brief tests of Xapian::Weight subclasses
   3  */
   4 /* Copyright (C) 2004,2012,2013,2016 Olly Betts
   5  * Copyright (C) 2013 Aarsh Shah
   6  * Copyright (C) 2016 Vivek Pal
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  21  */
  22
  23 #include <config.h>
  24
  25 #include "api_weight.h"
  26 #include <cmath>
  27
  28 #include <xapian.h>
  29
  30 #include "apitest.h"
  31 #include "testutils.h"
  32
  33 using namespace std;
  34
  35 // Test exception for junk after serialised weight.
  36 DEFINE_TESTCASE(tradweight3, !backend) {
  37     Xapian::TradWeight wt(42);
  38     try {
  39         Xapian::TradWeight t;
  40         Xapian::TradWeight * t2 = t.unserialise(wt.serialise() + "X");
  41         // Make sure we actually use the weight.
  42         bool empty = t2->name().empty();
  43         delete t2;
  44         if (empty)
  45             FAIL_TEST("Serialised TradWeight with junk appended unserialised to empty name!");
  46         FAIL_TEST("Serialised TradWeight with junk appended unserialised OK");
  47     } catch (const Xapian::SerialisationError &e) {
  48         // Regression test for error in exception message fixed in 1.2.11 and
  49         // 1.3.1.
  50         TEST(e.get_msg().find("BM25") == string::npos);
  51         TEST(e.get_msg().find("Trad") != string::npos);
  52     }
  53     return true;
  54 }
  55
  56 // Test Exception for junk after serialised weight.
  57 DEFINE_TESTCASE(unigramlmweight3, !backend) {
  58     Xapian::LMWeight wt(79898.0, Xapian::Weight::JELINEK_MERCER_SMOOTHING, 0.5, 1.0);
  59     try {
  60         Xapian::LMWeight t;
  61         Xapian::LMWeight * t2 = t.unserialise(wt.serialise() + "X");
  62         // Make sure we actually use the weight.
  63         bool empty = t2->name().empty();
  64         delete t2;
  65         if (empty)
  66             FAIL_TEST("Serialised LMWeight with junk appended unserialised to empty name!");
  67         FAIL_TEST("Serialised LMWeight with junk appended unserialised OK");
  68     } catch (const Xapian::SerialisationError &e) {
  69         TEST(e.get_msg().find("LM") != string::npos);
  70     }
  71     return true;
  72 }
  73
  74 // Test exception for junk after serialised weight.
  75 DEFINE_TESTCASE(bm25weight3, !backend) {
  76     Xapian::BM25Weight wt(2.0, 0.5, 1.3, 0.6, 0.01);
  77     try {
  78         Xapian::BM25Weight b;
  79         Xapian::BM25Weight * b2 = b.unserialise(wt.serialise() + "X");
  80         // Make sure we actually use the weight.
  81         bool empty = b2->name().empty();
  82         delete b2;
  83         if (empty)
  84             FAIL_TEST("Serialised BM25Weight with junk appended unserialised to empty name!");
  85         FAIL_TEST("Serialised BM25Weight with junk appended unserialised OK");
  86     } catch (const Xapian::SerialisationError &e) {
  87         TEST(e.get_msg().find("BM25") != string::npos);
  88     }
  89     return true;
  90 }
  91
  92 // Test parameter combinations which should be unaffected by doclength.
  93 DEFINE_TESTCASE(bm25weight4, backend) {
  94     Xapian::Database db = get_database("apitest_simpledata");
  95     Xapian::Enquire enquire(db);
  96     enquire.set_query(Xapian::Query("paragraph"));
  97     Xapian::MSet mset;
  98
  99     enquire.set_weighting_scheme(Xapian::BM25Weight(1, 0, 1, 0, 0.5));
 100     mset = enquire.get_mset(0, 10);
 101     TEST_EQUAL(mset.size(), 5);
 102     // Expect: wdf has an effect on weight, but doclen doesn't.
 103     TEST_REL(mset[0].get_weight(),>,mset[1].get_weight());
 104     TEST_EQUAL_DOUBLE(mset[1].get_weight(), mset[2].get_weight());
 105     TEST_REL(mset[2].get_weight(),>,mset[3].get_weight());
 106     TEST_EQUAL_DOUBLE(mset[3].get_weight(), mset[4].get_weight());
 107
 108     enquire.set_weighting_scheme(Xapian::BM25Weight(0, 0, 1, 1, 0.5));
 109     mset = enquire.get_mset(0, 10);
 110     TEST_EQUAL(mset.size(), 5);
 111     // Expect: neither wdf nor doclen affects weight.
 112     TEST_EQUAL_DOUBLE(mset[0].get_weight(), mset[4].get_weight());
 113
 114     return true;
 115 }
 116
 117 /// Test non-zero k2 with zero k1.
 118 // Regression test for bug fixed in 1.2.17 and 1.3.2.
 119 DEFINE_TESTCASE(bm25weight5, backend) {
 120     Xapian::Database db = get_database("apitest_simpledata");
 121     Xapian::Enquire enquire(db);
 122     enquire.set_query(Xapian::Query("paragraph"));
 123     Xapian::MSet mset;
 124
 125     enquire.set_weighting_scheme(Xapian::BM25Weight(0, 1, 1, 0.5, 0.5));
 126     mset = enquire.get_mset(0, 10);
 127     TEST_EQUAL(mset.size(), 5);
 128     // Expect: wdf has no effect on weight; shorter docs rank higher.
 129     mset_expect_order(mset, 3, 5, 1, 4, 2);
 130     TEST_EQUAL_DOUBLE(mset[0].get_weight(), mset[1].get_weight());
 131     TEST_REL(mset[1].get_weight(),>,mset[2].get_weight());
 132     TEST_REL(mset[2].get_weight(),>,mset[3].get_weight());
 133     TEST_REL(mset[3].get_weight(),>,mset[4].get_weight());
 134
 135     return true;
 136 }
 137
 138 // Test exception for junk after serialised weight.
 139 DEFINE_TESTCASE(bm25plusweight1, !backend) {
 140     Xapian::BM25PlusWeight wt(2.0, 0.1, 1.3, 0.6, 0.01, 0.5);
 141     try {
 142         Xapian::BM25PlusWeight b;
 143         Xapian::BM25PlusWeight * b2 = b.unserialise(wt.serialise() + "X");
 144         // Make sure we actually use the weight.
 145         bool empty = b2->name().empty();
 146         delete b2;
 147         if (empty)
 148             FAIL_TEST("Serialised BM25PlusWeight with junk appended unserialised to empty name!");
 149         FAIL_TEST("Serialised BM25PlusWeight with junk appended unserialised OK");
 150     } catch (const Xapian::SerialisationError &e) {
 151         TEST(e.get_msg().find("BM25Plus") != string::npos);
 152     }
 153     return true;
 154 }
 155
 156 // Test parameter combinations which should be unaffected by doclength.
 157 DEFINE_TESTCASE(bm25plusweight2, backend) {
 158     Xapian::Database db = get_database("apitest_simpledata");
 159     Xapian::Enquire enquire(db);
 160     enquire.set_query(Xapian::Query("paragraph"));
 161     Xapian::MSet mset;
 162
 163     enquire.set_weighting_scheme(Xapian::BM25PlusWeight(1, 0, 1, 0, 0.5, 1));
 164     mset = enquire.get_mset(0, 10);
 165     TEST_EQUAL(mset.size(), 5);
 166     // Expect: wdf has an effect on weight, but doclen doesn't.
 167     TEST_REL(mset[0].get_weight(),>,mset[1].get_weight());
 168     TEST_EQUAL_DOUBLE(mset[1].get_weight(), mset[2].get_weight());
 169     TEST_REL(mset[2].get_weight(),>,mset[3].get_weight());
 170     TEST_EQUAL_DOUBLE(mset[3].get_weight(), mset[4].get_weight());
 171
 172     enquire.set_weighting_scheme(Xapian::BM25PlusWeight(0, 0, 1, 1, 0.5, 1));
 173     mset = enquire.get_mset(0, 10);
 174     TEST_EQUAL(mset.size(), 5);
 175     // Expect: neither wdf nor doclen affects weight.
 176     TEST_EQUAL_DOUBLE(mset[0].get_weight(), mset[4].get_weight());
 177
 178     return true;
 179 }
 180
 181 // Regression test for a mistake corrected in the BM25+ implementation.
 182 DEFINE_TESTCASE(bm25plusweight3, backend) {
 183     Xapian::Database db = get_database("apitest_simpledata");
 184     Xapian::Enquire enquire(db);
 185     enquire.set_query(Xapian::Query("paragraph"));
 186     Xapian::MSet mset;
 187
 188     enquire.set_weighting_scheme(Xapian::BM25PlusWeight(1, 0, 1, 0.5, 0.5, 1));
 189     mset = enquire.get_mset(0, 10);
 190     TEST_EQUAL(mset.size(), 5);
 191
 192     // The value of each doc weight calculated manually from the BM25+ formulae
 193     // by using the respective document statistics.
 194     TEST_EQUAL_DOUBLE(mset[0].get_weight(), 0.7920796567487473);
 195     TEST_EQUAL_DOUBLE(mset[1].get_weight(), 0.7846980783848447);
 196     TEST_EQUAL_DOUBLE(mset[2].get_weight(), 0.7558817623365934);
 197     TEST_EQUAL_DOUBLE(mset[3].get_weight(), 0.7210119356168847);
 198     TEST_EQUAL_DOUBLE(mset[4].get_weight(), 0.7210119356168847);
 199
 200     return true;
 201 }
 202
 203 // Test exception for junk after serialised weight.
 204 DEFINE_TESTCASE(inl2weight1, !backend) {
 205     Xapian::InL2Weight wt(2.0);
 206     try {
 207         Xapian::InL2Weight b;
 208         Xapian::InL2Weight * b2 = b.unserialise(wt.serialise() + "X");
 209         // Make sure we actually use the weight.
 210         bool empty = b2->name().empty();
 211         delete b2;
 212         if (empty)
 213             FAIL_TEST("Serialised inl2weight with junk appended unserialised to empty name!");
 214         FAIL_TEST("Serialised inl2weight with junk appended unserialised OK");
 215     } catch (const Xapian::SerialisationError &e) {
 216         TEST(e.get_msg().find("InL2") != string::npos);
 217     }
 218
 219     return true;
 220 }
 221
 222 // Test for invalid values of c.
 223 DEFINE_TESTCASE(inl2weight2, !backend) {
 224     // InvalidArgumentError should be thrown if the parameter c is invalid.
 225     TEST_EXCEPTION(Xapian::InvalidArgumentError,
 226         Xapian::InL2Weight wt(-2.0));
 227
 228     TEST_EXCEPTION(Xapian::InvalidArgumentError,
 229         Xapian::InL2Weight wt2(0.0));
 230
 231     /* Parameter c should be set to 1.0 by constructor if none is given. */
 232     Xapian::InL2Weight weight2;
 233     TEST_EQUAL(weight2.serialise(), Xapian::InL2Weight(1.0).serialise());
 234
 235     return true;
 236 }
 237
 238 // Feature tests for Inl2Weight
 239 DEFINE_TESTCASE(inl2weight3, backend) {
 240     Xapian::Database db = get_database("apitest_simpledata");
 241     Xapian::Enquire enquire(db);
 242     Xapian::Query query("banana");
 243
 244     enquire.set_query(query);
 245     enquire.set_weighting_scheme(Xapian::InL2Weight(2.0));
 246
 247     Xapian::MSet mset1;
 248     mset1 = enquire.get_mset(0, 10);
 249     TEST_EQUAL(mset1.size(), 1);
 250     mset_expect_order(mset1, 6);
 251
 252     /* The value has been calculated in the python interpreter by looking at the
 253      * database statistics. */
 254     TEST_EQUAL_DOUBLE(mset1[0].get_weight(), 1.559711143842063);
 255
 256     // Test with OP_SCALE_WEIGHT.
 257     enquire.set_query(Xapian::Query(Xapian::Query::OP_SCALE_WEIGHT, query, 15.0));
 258     enquire.set_weighting_scheme(Xapian::InL2Weight(2.0));
 259
 260     Xapian::MSet mset2;
 261     mset2 = enquire.get_mset(0, 10);
 262     TEST_EQUAL(mset2.size(), 1);
 263     TEST_NOT_EQUAL_DOUBLE(mset1[0].get_weight(), 0.0);
 264     TEST_EQUAL_DOUBLE(15.0 * mset1[0].get_weight(), mset2[0].get_weight());
 265
 266     return true;
 267 }
 268
 269 // Test exception for junk after serialised weight.
 270 DEFINE_TESTCASE(ifb2weight1, !backend) {
 271     Xapian::IfB2Weight wt(2.0);
 272     try {
 273         Xapian::IfB2Weight b;
 274         Xapian::IfB2Weight * b2 = b.unserialise(wt.serialise() + "X");
 275         // Make sure we actually use the weight.
 276         bool empty = b2->name().empty();
 277         delete b2;
 278         if (empty)
 279             FAIL_TEST("Serialised IfB2Weight with junk appended unserialised to empty name!");
 280         FAIL_TEST("Serialised IfB2Weight with junk appended unserialised OK");
 281     } catch (const Xapian::SerialisationError &e) {
 282         TEST(e.get_msg().find("IfB2") != string::npos);
 283     }
 284     return true;
 285 }
 286
 287 // Test for invalid values of c.
 288 DEFINE_TESTCASE(ifb2weight2, !backend) {
 289     // InvalidArgumentError should be thrown if the parameter c is invalid.
 290     TEST_EXCEPTION(Xapian::InvalidArgumentError,
 291         Xapian::IfB2Weight wt(-2.0));
 292
 293     TEST_EXCEPTION(Xapian::InvalidArgumentError,
 294         Xapian::IfB2Weight wt2(0.0));
 295
 296     /* Parameter c should be set to 1.0 by constructor if none is given. */
 297     Xapian::IfB2Weight weight2;
 298     TEST_EQUAL(weight2.serialise(), Xapian::IfB2Weight(1.0).serialise());
 299
 300     return true;
 301 }
 302
 303 // Feature test
 304 DEFINE_TESTCASE(ifb2weight3, backend) {
 305     Xapian::Database db = get_database("apitest_simpledata");
 306     Xapian::Enquire enquire(db);
 307     Xapian::Query query("banana");
 308
 309     enquire.set_query(query);
 310     enquire.set_weighting_scheme(Xapian::IfB2Weight(2.0));
 311
 312     Xapian::MSet mset1;
 313     mset1 = enquire.get_mset(0, 10);
 314     TEST_EQUAL(mset1.size(), 1);
 315
 316     /* The value of the weight has been manually calculated using the statistics
 317      * of the test database. */
 318     TEST_EQUAL_DOUBLE(mset1[0].get_weight(), 3.119422287684126);
 319
 320     // Test with OP_SCALE_WEIGHT.
 321     enquire.set_query(Xapian::Query(Xapian::Query::OP_SCALE_WEIGHT, query, 15.0));
 322     enquire.set_weighting_scheme(Xapian::IfB2Weight(2.0));
 323
 324     Xapian::MSet mset2;
 325     mset2 = enquire.get_mset(0, 10);
 326     TEST_EQUAL(mset2.size(), 1);
 327     TEST_NOT_EQUAL_DOUBLE(mset1[0].get_weight(), 0.0);
 328     TEST_EQUAL_DOUBLE(15.0 * mset1[0].get_weight(), mset2[0].get_weight());
 329
 330     return true;
 331 }
 332
 333 // Test exception for junk after serialised weight.
 334 DEFINE_TESTCASE(ineb2weight1, !backend) {
 335     Xapian::IneB2Weight wt(2.0);
 336     try {
 337         Xapian::IneB2Weight b;
 338         Xapian::IneB2Weight * b2 = b.unserialise(wt.serialise() + "X");
 339         // Make sure we actually use the weight.
 340         bool empty = b2->name().empty();
 341         delete b2;
 342         if (empty)
 343             FAIL_TEST("Serialised ineb2weight with junk appended unserialised to empty name!");
 344         FAIL_TEST("Serialised ineb2weight with junk appended unserialised OK");
 345     } catch (const Xapian::SerialisationError &e) {
 346         TEST(e.get_msg().find("IneB2") != string::npos);
 347     }
 348
 349     return true;
 350 }
 351
 352 // Test for invalid values of c.
 353 DEFINE_TESTCASE(ineb2weight2, !backend) {
 354     // InvalidArgumentError should be thrown if parameter c is invalid.
 355     TEST_EXCEPTION(Xapian::InvalidArgumentError,
 356         Xapian::IneB2Weight wt(-2.0));
 357
 358     TEST_EXCEPTION(Xapian::InvalidArgumentError,
 359         Xapian::IneB2Weight wt2(0.0));
 360
 361     /* Parameter c should be set to 1.0 by constructor if none is given. */
 362     Xapian::IneB2Weight weight2;
 363     TEST_EQUAL(weight2.serialise(), Xapian::IneB2Weight(1.0).serialise());
 364
 365     return true;
 366 }
 367
 368 // Feature test.
 369 DEFINE_TESTCASE(ineb2weight3, backend) {
 370     Xapian::Database db = get_database("apitest_simpledata");
 371     Xapian::Enquire enquire(db);
 372     Xapian::Query query("paragraph");
 373     enquire.set_query(query);
 374     enquire.set_weighting_scheme(Xapian::IneB2Weight(2.0));
 375
 376     Xapian::MSet mset1;
 377     mset1 = enquire.get_mset(0, 10);
 378     TEST_EQUAL(mset1.size(), 5);
 379
 380     // The third document in the database is 4th in the ranking.
 381     /* The weight value has been manually calculated by using the statistics
 382      * of the test database. */
 383     TEST_EQUAL_DOUBLE(mset1[4].get_weight(), 0.61709730297692400036);
 384
 385     // Test with OP_SCALE_WEIGHT.
 386     enquire.set_query(Xapian::Query(Xapian::Query::OP_SCALE_WEIGHT, query, 15.0));
 387     enquire.set_weighting_scheme(Xapian::IneB2Weight(2.0));
 388
 389     Xapian::MSet mset2;
 390     mset2 = enquire.get_mset(0, 10);
 391     TEST_EQUAL(mset2.size(), 5);
 392
 393     TEST_NOT_EQUAL_DOUBLE(mset1[0].get_weight(), 0.0);
 394     for (int i = 0; i < 5; ++i) {
 395         TEST_EQUAL_DOUBLE(15.0 * mset1[i].get_weight(), mset2[i].get_weight());
 396     }
 397
 398     return true;
 399 }
 400
 401 // Test exception for junk after serialised weight.
 402 DEFINE_TESTCASE(bb2weight1, !backend) {
 403     Xapian::BB2Weight wt(2.0);
 404     try {
 405         Xapian::BB2Weight b;
 406         Xapian::BB2Weight * b2 = b.unserialise(wt.serialise() + "X");
 407         // Make sure we actually use the weight.
 408         bool empty = b2->name().empty();
 409         delete b2;
 410         if (empty)
 411             FAIL_TEST("Serialised BB2Weight with junk appended unserialised to empty name!");
 412         FAIL_TEST("Serialised BB2Weight with junk appended unserialised OK");
 413     } catch (const Xapian::SerialisationError &e) {
 414         TEST(e.get_msg().find("BB2") != string::npos);
 415     }
 416     return true;
 417 }
 418
 419 // Test for invalid values of c.
 420 DEFINE_TESTCASE(bb2weight2, !backend) {
 421     // InvalidArgumentError should be thrown if the parameter c is invalid.
 422     TEST_EXCEPTION(Xapian::InvalidArgumentError,
 423         Xapian::BB2Weight wt(-2.0));
 424
 425     TEST_EXCEPTION(Xapian::InvalidArgumentError,
 426         Xapian::BB2Weight wt2(0.0));
 427
 428     /* Parameter c should be set to 1.0 by constructor if none is given. */
 429     Xapian::BB2Weight weight2;
 430     TEST_EQUAL(weight2.serialise(), Xapian::BB2Weight(1.0).serialise());
 431
 432     return true;
 433 }
 434
 435 // Feature test
 436 DEFINE_TESTCASE(bb2weight3, backend) {
 437     Xapian::Database db = get_database("apitest_simpledata");
 438     Xapian::Enquire enquire(db);
 439     Xapian::Query query("paragraph");
 440
 441     enquire.set_query(query);
 442     enquire.set_weighting_scheme(Xapian::BB2Weight(2.0));
 443
 444     Xapian::MSet mset1;
 445     mset1 = enquire.get_mset(0, 10);
 446     TEST_EQUAL(mset1.size(), 5);
 447     /* The third document in the database has the highest weight and is the
 448      * first in the mset. */
 449     // Value calculated manually by using the statistics of the test database.
 450     TEST_EQUAL_DOUBLE(mset1[0].get_weight(), 1.6823696969784483);
 451
 452     // Test with OP_SCALE_WEIGHT.
 453     enquire.set_query(Xapian::Query(Xapian::Query::OP_SCALE_WEIGHT, query, 15.0));
 454     enquire.set_weighting_scheme(Xapian::BB2Weight(2.0));
 455
 456     Xapian::MSet mset2;
 457     mset2 = enquire.get_mset(0, 10);
 458     TEST_EQUAL(mset2.size(), 5);
 459
 460     TEST_NOT_EQUAL_DOUBLE(mset1[0].get_weight(), 0.0);
 461     for (int i = 0; i < 5; ++i) {
 462         TEST_EQUAL_DOUBLE(15.0 * mset1[i].get_weight(), mset2[i].get_weight());
 463     }
 464
 465     // Test with OP_SCALE_WEIGHT and a small factor (regression test, as we
 466     // were applying the factor to the upper bound twice).
 467     enquire.set_query(Xapian::Query(Xapian::Query::OP_SCALE_WEIGHT, query, 1.0 / 1024));
 468     enquire.set_weighting_scheme(Xapian::BB2Weight(2.0));
 469
 470     Xapian::MSet mset3;
 471     mset3 = enquire.get_mset(0, 10);
 472     TEST_EQUAL(mset3.size(), 5);
 473
 474     for (int i = 0; i < 5; ++i) {
 475         TEST_EQUAL_DOUBLE(mset1[i].get_weight(), mset3[i].get_weight() * 1024);
 476     }
 477
 478     return true;
 479 }
 480
 481 // Regression test: we used to calculate log2(0) when there was only one doc.
 482 DEFINE_TESTCASE(bb2weight4, backend) {
 483     Xapian::Database db = get_database("apitest_onedoc");
 484     Xapian::Enquire enquire(db);
 485     Xapian::Query query("word");
 486
 487     enquire.set_query(query);
 488     enquire.set_weighting_scheme(Xapian::BB2Weight());
 489
 490     Xapian::MSet mset1;
 491     mset1 = enquire.get_mset(0, 10);
 492     TEST_EQUAL(mset1.size(), 1);
 493     // Zero weight is a bit bogus, but what we currently give.
 494     TEST_EQUAL_DOUBLE(mset1[0].get_weight(), 0);
 495
 496     return true;
 497 }
 498
 499 // Feature test.
 500 DEFINE_TESTCASE(dlhweight1, backend) {
 501     Xapian::Database db = get_database("apitest_simpledata");
 502     Xapian::Enquire enquire(db);
 503     Xapian::Query query("a");
 504
 505     enquire.set_query(query);
 506     enquire.set_weighting_scheme(Xapian::DLHWeight());
 507
 508     Xapian::MSet mset1;
 509     mset1 = enquire.get_mset(0, 10);
 510     TEST_EQUAL(mset1.size(), 3);
 511     mset_expect_order(mset1, 3, 1, 2);
 512     // Weights calculated manually using stats from the database.
 513     TEST_EQUAL_DOUBLE(mset1[0].get_weight(), 1.0046477754371292362);
 514     TEST_EQUAL_DOUBLE(mset1[1].get_weight(), 0.97621929514640352757);
 515     // The following weight would be negative but gets clamped to 0.
 516     TEST_EQUAL_DOUBLE(mset1[2].get_weight(), 0.0);
 517
 518     // Test with OP_SCALE_WEIGHT.
 519     enquire.set_query(Xapian::Query(Xapian::Query::OP_SCALE_WEIGHT, query, 15.0));
 520     enquire.set_weighting_scheme(Xapian::DLHWeight());
 521
 522     Xapian::MSet mset2;
 523     mset2 = enquire.get_mset(0, 10);
 524     TEST_EQUAL(mset2.size(), 3);
 525
 526     TEST_NOT_EQUAL_DOUBLE(mset1[0].get_weight(), 0.0);
 527     for (Xapian::doccount i = 0; i < mset2.size(); ++i) {
 528         TEST_EQUAL_DOUBLE(15.0 * mset1[i].get_weight(), mset2[i].get_weight());
 529     }
 530
 531     return true;
 532 }
 533
 534 // Test exception for junk after serialised weight.
 535 DEFINE_TESTCASE(dlhweight2, !backend) {
 536     Xapian::DLHWeight wt;
 537     try {
 538         Xapian::DLHWeight t;
 539         Xapian::DLHWeight * t2 = t.unserialise(wt.serialise() + "X");
 540         // Make sure we actually use the weight.
 541         bool empty = t2->name().empty();
 542         delete t2;
 543         if (empty)
 544             FAIL_TEST("Serialised DLHWeight with junk appended unserialised to empty name!");
 545         FAIL_TEST("Serialised DLHWeight with junk appended unserialised OK");
 546     } catch (const Xapian::SerialisationError &e) {
 547         TEST(e.get_msg().find("DLH") != string::npos);
 548     }
 549     return true;
 550 }
 551
 552 static void
 553 gen_wdf_eq_doclen_db(Xapian::WritableDatabase& db, const string&)
 554 {
 555     Xapian::Document doc;
 556     doc.add_term("solo", 37);
 557     db.add_document(doc);
 558 }
 559
 560 // Test wdf == doclen.
 561 DEFINE_TESTCASE(dlhweight3, generated) {
 562     Xapian::Database db = get_database("wdf_eq_doclen", gen_wdf_eq_doclen_db);
 563     Xapian::Enquire enquire(db);
 564     Xapian::Query query("solo");
 565
 566     enquire.set_query(query);
 567     enquire.set_weighting_scheme(Xapian::DLHWeight());
 568
 569     Xapian::MSet mset1;
 570     mset1 = enquire.get_mset(0, 10);
 571     TEST_EQUAL(mset1.size(), 1);
 572     // Weight gets clamped to zero.
 573     TEST_EQUAL_DOUBLE(mset1[0].get_weight(), 0.0);
 574
 575     return true;
 576 }
 577
 578 // Test exception for junk after serialised weight.
 579 DEFINE_TESTCASE(pl2weight1, !backend) {
 580     Xapian::PL2Weight wt(2.0);
 581     try {
 582         Xapian::PL2Weight b;
 583         Xapian::PL2Weight * b2 = b.unserialise(wt.serialise() + "X");
 584         // Make sure we actually use the weight.
 585         bool empty = b2->name().empty();
 586         delete b2;
 587         if (empty)
 588             FAIL_TEST("Serialised PL2Weight with junk appended unserialised to empty name!");
 589         FAIL_TEST("Serialised PL2Weight with junk appended unserialised OK");
 590     } catch (const Xapian::SerialisationError &e) {
 591         TEST(e.get_msg().find("PL2") != string::npos);
 592     }
 593     return true;
 594 }
 595
 596 // Test for invalid values of c.
 597 DEFINE_TESTCASE(pl2weight2, !backend) {
 598     // InvalidArgumentError should be thrown if parameter c is invalid.
 599     TEST_EXCEPTION(Xapian::InvalidArgumentError,
 600         Xapian::PL2Weight wt(-2.0));
 601
 602     /* Parameter c should be set to 1.0 by constructor if none is given. */
 603     Xapian::PL2Weight weight2;
 604     TEST_EQUAL(weight2.serialise(), Xapian::PL2Weight(1.0).serialise());
 605
 606     return true;
 607 }
 608
 609 // Feature Test.
 610 DEFINE_TESTCASE(pl2weight3, backend) {
 611     Xapian::Database db = get_database("apitest_simpledata");
 612     Xapian::Enquire enquire(db);
 613     Xapian::Query query("paragraph");
 614     enquire.set_query(query);
 615     Xapian::MSet mset;
 616
 617     enquire.set_weighting_scheme(Xapian::PL2Weight(2.0));
 618     mset = enquire.get_mset(0, 10);
 619     TEST_EQUAL(mset.size(), 5);
 620     // Expected weight difference calculated in extended precision using stats
 621     // from the test database.
 622     TEST_EQUAL_DOUBLE(mset[2].get_weight(),
 623                       mset[3].get_weight() + 0.0086861771701328694);
 624
 625     // Test with OP_SCALE_WEIGHT.
 626     enquire.set_query(Xapian::Query(Xapian::Query::OP_SCALE_WEIGHT, query, 15.0));
 627     enquire.set_weighting_scheme(Xapian::PL2Weight(2.0));
 628
 629     Xapian::MSet mset2;
 630     mset2 = enquire.get_mset(0, 10);
 631     TEST_EQUAL(mset2.size(), 5);
 632     TEST_NOT_EQUAL_DOUBLE(mset[0].get_weight(), 0.0);
 633     for (int i = 0; i < 5; ++i) {
 634         TEST_EQUAL_DOUBLE(15.0 * mset[i].get_weight(), mset2[i].get_weight());
 635     }
 636
 637     return true;
 638 }
 639
 640 // Test exception for junk after serialised weight.
 641 DEFINE_TESTCASE(pl2plusweight1, !backend) {
 642     Xapian::PL2PlusWeight wt(2.0, 0.9);
 643     try {
 644         Xapian::PL2PlusWeight b;
 645         Xapian::PL2PlusWeight * b2 = b.unserialise(wt.serialise() + "X");
 646         // Make sure we actually use the weight.
 647         bool empty = b2->name().empty();
 648         delete b2;
 649         if (empty)
 650             FAIL_TEST("Serialised PL2PlusWeight with junk appended unserialised to empty name!");
 651         FAIL_TEST("Serialised PL2PlusWeight with junk appended unserialised OK");
 652     } catch (const Xapian::SerialisationError &e) {
 653         TEST(e.get_msg().find("PL2Plus") != string::npos);
 654     }
 655     return true;
 656 }
 657
 658 // Test for invalid values of parameters, c and delta.
 659 DEFINE_TESTCASE(pl2plusweight2, !backend) {
 660     // InvalidArgumentError should be thrown if parameter c is invalid.
 661     TEST_EXCEPTION(Xapian::InvalidArgumentError,
 662         Xapian::PL2PlusWeight wt(-2.0, 0.9));
 663
 664     // InvalidArgumentError should be thrown if parameter delta is invalid.
 665     TEST_EXCEPTION(Xapian::InvalidArgumentError,
 666         Xapian::PL2PlusWeight wt(1.0, -1.9));
 667
 668     return true;
 669 }
 670
 671 // Test for default values of parameters, c and delta.
 672 DEFINE_TESTCASE(pl2plusweight3, !backend) {
 673     Xapian::PL2PlusWeight weight2;
 674
 675     /* Parameter c should be set to 1.0 by constructor if none is given. */
 676     TEST_EQUAL(weight2.serialise(), Xapian::PL2PlusWeight(1.0, 0.8).serialise());
 677
 678     /* Parameter delta should be set to 0.8 by constructor if none is given. */
 679     TEST_EQUAL(weight2.serialise(), Xapian::PL2PlusWeight(1.0, 0.8).serialise());
 680
 681     return true;
 682 }
 683
 684 // Feature Test 1 for PL2PlusWeight.
 685 DEFINE_TESTCASE(pl2plusweight4, backend) {
 686     Xapian::Database db = get_database("apitest_simpledata");
 687     Xapian::Enquire enquire(db);
 688     enquire.set_query(Xapian::Query("paragraph"));
 689     Xapian::MSet mset;
 690
 691     enquire.set_weighting_scheme(Xapian::PL2PlusWeight(2.0, 0.8));
 692     mset = enquire.get_mset(0, 10);
 693     TEST_EQUAL(mset.size(), 5);
 694     // Expected weight difference calculated in extended precision using stats
 695     // from the test database.
 696     TEST_EQUAL_DOUBLE(mset[2].get_weight(),
 697                       mset[3].get_weight() + 0.0086861771701328694);
 698
 699     return true;
 700 }
 701
 702 // Feature Test 2 for PL2PlusWeight
 703 DEFINE_TESTCASE(pl2plusweight5, backend) {
 704     Xapian::Database db = get_database("apitest_simpledata");
 705     Xapian::Enquire enquire(db);
 706     Xapian::Query query("word");
 707     enquire.set_query(query);
 708     Xapian::MSet mset;
 709
 710     enquire.set_weighting_scheme(Xapian::PL2PlusWeight(1.0, 0.8));
 711     mset = enquire.get_mset(0, 10);
 712     // Expect MSet contains two documents having query "word".
 713     TEST_EQUAL(mset.size(), 2);
 714     // Expect Document 2 has higher weight than document 4 because
 715     // "word" appears more no. of times in document 2 than document 4.
 716     mset_expect_order(mset, 2, 4);
 717
 718     // Test with OP_SCALE_WEIGHT.
 719     enquire.set_query(Xapian::Query(Xapian::Query::OP_SCALE_WEIGHT, query, 15.0));
 720     enquire.set_weighting_scheme(Xapian::PL2PlusWeight(1.0, 0.8));
 721
 722     Xapian::MSet mset2;
 723     mset2 = enquire.get_mset(0, 10);
 724     TEST_EQUAL(mset2.size(), mset.size());
 725     TEST_NOT_EQUAL_DOUBLE(mset[0].get_weight(), 0.0);
 726     for (Xapian::doccount i = 0; i < mset.size(); ++i) {
 727         TEST_EQUAL_DOUBLE(15.0 * mset[i].get_weight(), mset2[i].get_weight());
 728     }
 729
 730     return true;
 731 }
 732
 733 // Feature test
 734 DEFINE_TESTCASE(dphweight1, backend) {
 735     Xapian::Database db = get_database("apitest_simpledata");
 736     Xapian::Enquire enquire(db);
 737     Xapian::Query query("paragraph");
 738
 739     enquire.set_query(query);
 740     enquire.set_weighting_scheme(Xapian::DPHWeight());
 741
 742     Xapian::MSet mset1;
 743     mset1 = enquire.get_mset(0, 10);
 744     TEST_EQUAL(mset1.size(), 5);
 745     /* The weight has been calculated manually by using the statistics of the
 746      * test database. */
 747     TEST_EQUAL_DOUBLE(mset1[2].get_weight() - mset1[4].get_weight(), 0.542623617687990167);
 748
 749     // Test with OP_SCALE_WEIGHT.
 750     enquire.set_query(Xapian::Query(Xapian::Query::OP_SCALE_WEIGHT, query, 15.0));
 751     enquire.set_weighting_scheme(Xapian::DPHWeight());
 752
 753     Xapian::MSet mset2;
 754     mset2 = enquire.get_mset(0, 10);
 755     TEST_EQUAL(mset2.size(), 5);
 756     TEST_NOT_EQUAL_DOUBLE(mset1[0].get_weight(), 0.0);
 757     for (int i = 0; i < 5; ++i) {
 758         TEST_EQUAL_DOUBLE(15.0 * mset1[i].get_weight(), mset2[i].get_weight());
 759     }
 760
 761     return true;
 762 }
 763
 764 // Test exception for junk after serialised weight.
 765 DEFINE_TESTCASE(dphweight2, !backend) {
 766     Xapian::DPHWeight wt;
 767     try {
 768         Xapian::DPHWeight t;
 769         Xapian::DPHWeight * t2 = t.unserialise(wt.serialise() + "X");
 770         // Make sure we actually use the weight.
 771         bool empty = t2->name().empty();
 772         delete t2;
 773         if (empty)
 774             FAIL_TEST("Serialised DPHWeight with junk appended unserialised to empty name!");
 775         FAIL_TEST("Serialised DPHWeight with junk appended unserialised OK");
 776     } catch (const Xapian::SerialisationError &e) {
 777         TEST(e.get_msg().find("DPH") != string::npos);
 778     }
 779     return true;
 780 }
 781
 782 // Test wdf == doclen.
 783 DEFINE_TESTCASE(dphweight3, generated) {
 784     Xapian::Database db = get_database("wdf_eq_doclen", gen_wdf_eq_doclen_db);
 785     Xapian::Enquire enquire(db);
 786     Xapian::Query query("solo");
 787
 788     enquire.set_query(query);
 789     enquire.set_weighting_scheme(Xapian::DPHWeight());
 790
 791     Xapian::MSet mset1;
 792     mset1 = enquire.get_mset(0, 10);
 793     TEST_EQUAL(mset1.size(), 1);
 794     // Weight gets clamped to zero.
 795     TEST_EQUAL_DOUBLE(mset1[0].get_weight(), 0.0);
 796
 797     return true;
 798 }
 799
 800 // Test for various cases of normalization string.
 801 DEFINE_TESTCASE(tfidfweight1, !backend) {
 802     // InvalidArgumentError should be thrown if normalization string is invalid
 803     TEST_EXCEPTION(Xapian::InvalidArgumentError,
 804         Xapian::TfIdfWeight b("JOHN_LENNON"));
 805
 806     TEST_EXCEPTION(Xapian::InvalidArgumentError,
 807         Xapian::TfIdfWeight b("LOL"));
 808
 809     /* Normalization string should be set to "ntn" by constructor if none is
 810       given. */
 811     Xapian::TfIdfWeight weight2;
 812     TEST_EQUAL(weight2.serialise(), Xapian::TfIdfWeight("ntn").serialise());
 813
 814     return true;
 815 }
 816
 817 // Test exception for junk after serialised weight.
 818 DEFINE_TESTCASE(tfidfweight2, !backend) {
 819     Xapian::TfIdfWeight wt("ntn");
 820     try {
 821         Xapian::TfIdfWeight b;
 822         Xapian::TfIdfWeight * b2 = b.unserialise(wt.serialise() + "X");
 823         // Make sure we actually use the weight.
 824         bool empty = b2->name().empty();
 825         delete b2;
 826         if (empty)
 827             FAIL_TEST("Serialised TfIdfWeight with junk appended unserialised to empty name!");
 828         FAIL_TEST("Serialised TfIdfWeight with junk appended unserialised OK");
 829     } catch (const Xapian::SerialisationError &e) {
 830         TEST(e.get_msg().find("TfIdf") != string::npos);
 831     }
 832     return true;
 833 }
 834
 835 // Feature tests for various normalization functions.
 836 DEFINE_TESTCASE(tfidfweight3, backend) {
 837     Xapian::Database db = get_database("apitest_simpledata");
 838     Xapian::Enquire enquire(db);
 839     Xapian::Query query("word");
 840     Xapian::MSet mset;
 841
 842     // Check for "ntn" when termfreq != N
 843     enquire.set_query(query);
 844     enquire.set_weighting_scheme(Xapian::TfIdfWeight("ntn"));
 845     mset = enquire.get_mset(0, 10);
 846     TEST_EQUAL(mset.size(), 2);
 847     // doc 2 should have higher weight than 4 as only tf(wdf) will dominate.
 848     mset_expect_order(mset, 2, 4);
 849     TEST_EQUAL_DOUBLE(mset[0].get_weight(), 8.0 * log(6.0 / 2));
 850
 851     // Check that wqf is taken into account.
 852     enquire.set_query(Xapian::Query("word", 2));
 853     enquire.set_weighting_scheme(Xapian::TfIdfWeight("ntn"));
 854     Xapian::MSet mset2 = enquire.get_mset(0, 10);
 855     TEST_EQUAL(mset2.size(), 2);
 856     // wqf is 2, so weights should be doubled.
 857     TEST_EQUAL_DOUBLE(mset[0].get_weight() * 2, mset2[0].get_weight());
 858     TEST_EQUAL_DOUBLE(mset[1].get_weight() * 2, mset2[1].get_weight());
 859
 860     // Test with OP_SCALE_WEIGHT.
 861     enquire.set_query(Xapian::Query(Xapian::Query::OP_SCALE_WEIGHT, query, 15.0));
 862     enquire.set_weighting_scheme(Xapian::TfIdfWeight("ntn"));
 863     mset2 = enquire.get_mset(0, 10);
 864     TEST_EQUAL(mset2.size(), 2);
 865     // doc 2 should have higher weight than 4 as only tf(wdf) will dominate.
 866     mset_expect_order(mset2, 2, 4);
 867     TEST_NOT_EQUAL_DOUBLE(mset[0].get_weight(), 0.0);
 868     TEST_EQUAL_DOUBLE(15 * mset[0].get_weight(), mset2[0].get_weight());
 869
 870     // check for "nfn" when termfreq != N
 871     enquire.set_query(query);
 872     enquire.set_weighting_scheme(Xapian::TfIdfWeight("nfn"));
 873     mset = enquire.get_mset(0, 10);
 874     TEST_EQUAL(mset.size(), 2);
 875     mset_expect_order(mset, 2, 4);
 876     TEST_EQUAL_DOUBLE(mset[0].get_weight(), 8.0 / 2);
 877
 878     // check for "nsn" when termfreq != N
 879     enquire.set_query(query);
 880     enquire.set_weighting_scheme(Xapian::TfIdfWeight("nsn"));
 881     mset = enquire.get_mset(0, 10);
 882     TEST_EQUAL(mset.size(), 2);
 883     mset_expect_order(mset, 2, 4);
 884     TEST_EQUAL_DOUBLE(mset[0].get_weight(), 8.0 * pow(log(6.0 / 2), 2.0));
 885
 886     // Check for "bnn" and for both branches of 'b'.
 887     enquire.set_query(Xapian::Query("test"));
 888     enquire.set_weighting_scheme(Xapian::TfIdfWeight("bnn"));
 889     mset = enquire.get_mset(0, 10);
 890     TEST_EQUAL(mset.size(), 1);
 891     mset_expect_order(mset, 1);
 892     TEST_EQUAL_DOUBLE(mset[0].get_weight(), 1.0);
 893
 894     // Check for "lnn" and for both branches of 'l'.
 895     enquire.set_query(Xapian::Query("word"));
 896     enquire.set_weighting_scheme(Xapian::TfIdfWeight("lnn"));
 897     mset = enquire.get_mset(0, 10);
 898     TEST_EQUAL(mset.size(), 2);
 899     mset_expect_order(mset, 2, 4);
 900     TEST_EQUAL_DOUBLE(mset[0].get_weight(), 1 + log(8.0)); // idfn=1 and so wt=tfn=1+log(tf)
 901     TEST_EQUAL_DOUBLE(mset[1].get_weight(), 1.0);         // idfn=1 and wt=tfn=1+log(tf)=1+log(1)=1
 902
 903     // Check for "snn"
 904     enquire.set_query(Xapian::Query("paragraph"));
 905     enquire.set_weighting_scheme(Xapian::TfIdfWeight("snn")); // idf=1 and tfn=tf*tf
 906     mset = enquire.get_mset(0, 10);
 907     TEST_EQUAL(mset.size(), 5);
 908     mset_expect_order(mset, 2, 1, 4, 3, 5);
 909     TEST_EQUAL_DOUBLE(mset[0].get_weight(), 9.0);
 910     TEST_EQUAL_DOUBLE(mset[4].get_weight(), 1.0);
 911
 912     // Check for "ntn" when termfreq=N
 913     enquire.set_query(Xapian::Query("this"));  // N=termfreq amd so idfn=0 for "t"
 914     enquire.set_weighting_scheme(Xapian::TfIdfWeight("ntn"));
 915     mset = enquire.get_mset(0, 10);
 916     TEST_EQUAL(mset.size(), 6);
 917     mset_expect_order(mset, 1, 2, 3, 4, 5, 6);
 918     for (int i = 0; i < 6; ++i) {
 919         TEST_EQUAL_DOUBLE(mset[i].get_weight(), 0.0);
 920     }
 921
 922     // Check for "npn" and for both branches of 'p'
 923     enquire.set_query(Xapian::Query("this"));  // N=termfreq and so idfn=0 for "p"
 924     enquire.set_weighting_scheme(Xapian::TfIdfWeight("npn"));
 925     mset = enquire.get_mset(0, 10);
 926     TEST_EQUAL(mset.size(), 6);
 927     mset_expect_order(mset, 1, 2, 3, 4, 5, 6);
 928     for (int i = 0; i < 6; ++i) {
 929         TEST_EQUAL_DOUBLE(mset[i].get_weight(), 0.0);
 930     }
 931
 932     enquire.set_query(Xapian::Query("word"));
 933     enquire.set_weighting_scheme(Xapian::TfIdfWeight("npn"));
 934     mset = enquire.get_mset(0, 10);
 935     TEST_EQUAL(mset.size(), 2);
 936     mset_expect_order(mset, 2, 4);
 937     TEST_EQUAL_DOUBLE(mset[0].get_weight(), 8 * log((6.0 - 2) / 2));
 938     TEST_EQUAL_DOUBLE(mset[1].get_weight(), 1 * log((6.0 - 2) / 2));
 939
 940     return true;
 941 }
 942
 943 class CheckInitWeight : public Xapian::Weight {
 944   public:
 945     double factor;
 946
 947     unsigned & zero_inits, & non_zero_inits;
 948
 949     CheckInitWeight(unsigned &z, unsigned &n)
 950         : factor(-1.0), zero_inits(z), non_zero_inits(n) { }
 951
 952     void init(double factor_) {
 953         factor = factor_;
 954         if (factor == 0.0)
 955             ++zero_inits;
 956         else
 957             ++non_zero_inits;
 958     }
 959
 960     Weight * clone() const {
 961         return new CheckInitWeight(zero_inits, non_zero_inits);
 962     }
 963
 964     double get_sumpart(Xapian::termcount, Xapian::termcount,
 965                        Xapian::termcount) const {
 966         return 1.0;
 967     }
 968
 969     double get_maxpart() const { return 1.0; }
 970
 971     double get_sumextra(Xapian::termcount doclen, Xapian::termcount) const {
 972         return 1.0 / doclen;
 973     }
 974
 975     double get_maxextra() const { return 1.0; }
 976 };
 977
 978 /// Regression test - check init() is called for the term-indep Weight obj.
 979 DEFINE_TESTCASE(checkinitweight1, backend && !multi && !remote) {
 980     Xapian::Database db = get_database("apitest_simpledata");
 981     Xapian::Enquire enquire(db);
 982     Xapian::Query q(Xapian::Query::OP_AND,
 983                     Xapian::Query("this"), Xapian::Query("paragraph"));
 984     enquire.set_query(q);
 985     unsigned zero_inits = 0, non_zero_inits = 0;
 986     CheckInitWeight wt(zero_inits, non_zero_inits);
 987     enquire.set_weighting_scheme(wt);
 988     Xapian::MSet mset = enquire.get_mset(0, 3);
 989     TEST_EQUAL(zero_inits, 1);
 990     TEST_EQUAL(non_zero_inits, 2);
 991     return true;
 992 }
 993
 994 class CheckStatsWeight : public Xapian::Weight {
 995   public:
 996     double factor;
 997
 998     Xapian::Database db;
 999
1000     // When testing OP_SYNONYM, term2 is also set.
1001     // When testing OP_WILDCARD, term2 == "*"
1002     string term1, term2;
1003
1004     Xapian::termcount & sum;
1005     Xapian::termcount & sum_squares;
1006
1007     mutable Xapian::termcount len_upper;
1008     mutable Xapian::termcount len_lower;
1009     mutable Xapian::termcount wdf_upper;
1010
1011     CheckStatsWeight(const Xapian::Database & db_,
1012                      const string & term1_,
1013                      const string & term2_,
1014                      Xapian::termcount & sum_,
1015                      Xapian::termcount & sum_squares_)
1016         : factor(-1.0), db(db_), term1(term1_), term2(term2_),
1017           sum(sum_), sum_squares(sum_squares_),
1018           len_upper(0), len_lower(Xapian::termcount(-1)), wdf_upper(0)
1019     {
1020         need_stat(COLLECTION_SIZE);
1021         need_stat(RSET_SIZE);
1022         need_stat(AVERAGE_LENGTH);
1023         need_stat(TERMFREQ);
1024         need_stat(RELTERMFREQ);
1025         need_stat(QUERY_LENGTH);
1026         need_stat(WQF);
1027         need_stat(WDF);
1028         need_stat(DOC_LENGTH);
1029         need_stat(DOC_LENGTH_MIN);
1030         need_stat(DOC_LENGTH_MAX);
1031         need_stat(WDF_MAX);
1032         need_stat(COLLECTION_FREQ);
1033         need_stat(UNIQUE_TERMS);
1034     }
1035
1036     CheckStatsWeight(const Xapian::Database & db_,
1037                      const string & term_,
1038                      Xapian::termcount & sum_,
1039                      Xapian::termcount & sum_squares_)
1040         : CheckStatsWeight(db_, term_, string(), sum_, sum_squares_) { }
1041
1042     void init(double factor_) {
1043         factor = factor_;
1044     }
1045
1046     Weight * clone() const {
1047         return new CheckStatsWeight(db, term1, term2, sum, sum_squares);
1048     }
1049
1050     double get_sumpart(Xapian::termcount wdf, Xapian::termcount doclen,
1051                        Xapian::termcount uniqueterms) const {
1052         Xapian::doccount num_docs = db.get_doccount();
1053         TEST_EQUAL(get_collection_size(), num_docs);
1054         TEST_EQUAL(get_rset_size(), 0);
1055         TEST_EQUAL(get_average_length(), db.get_avlength());
1056         if (term2.empty()) {
1057             TEST_EQUAL(get_termfreq(), db.get_termfreq(term1));
1058             TEST_EQUAL(get_collection_freq(), db.get_collection_freq(term1));
1059             TEST_EQUAL(get_query_length(), 1);
1060         } else {
1061             Xapian::doccount tfmax = 0, tfsum = 0;
1062             Xapian::termcount cfmax = 0, cfsum = 0;
1063             if (term2 == "*") {
1064                 // OP_WILDCARD case.
1065                 for (auto&& t = db.allterms_begin(term1);
1066                      t != db.allterms_end(term1); ++t) {
1067                     Xapian::doccount tf = t.get_termfreq();
1068                     tout << "->" << *t << " " << tf << endl;
1069                     tfsum += tf;
1070                     tfmax = max(tfmax, tf);
1071                     Xapian::termcount cf = db.get_collection_freq(*t);
1072                     cfsum += cf;
1073                     cfmax = max(cfmax, cf);
1074                 }
1075                 TEST_EQUAL(get_query_length(), 1);
1076             } else {
1077                 // OP_SYNONYM case.
1078                 Xapian::doccount tf1 = db.get_termfreq(term1);
1079                 Xapian::doccount tf2 = db.get_termfreq(term2);
1080                 tfsum = tf1 + tf2;
1081                 tfmax = max(tf1, tf2);
1082                 Xapian::termcount cf1 = db.get_collection_freq(term1);
1083                 Xapian::termcount cf2 = db.get_collection_freq(term2);
1084                 cfsum = cf1 + cf2;
1085                 cfmax = max(cf1, cf2);
1086                 TEST_EQUAL(get_query_length(), 2);
1087             }
1088             // Synonym occurs at least as many times as any term.
1089             TEST_REL(get_termfreq(), >=, tfmax);
1090             TEST_REL(get_collection_freq(), >=, cfmax);
1091             // Synonym can't occur more times than the terms do.
1092             TEST_REL(get_termfreq(), <=, tfsum);
1093             TEST_REL(get_collection_freq(), <=, cfsum);
1094             // Synonym can't occur more times than there are documents/terms.
1095             TEST_REL(get_termfreq(), <=, num_docs);
1096             double total_term_occurences = get_average_length() * num_docs;
1097             TEST_REL(get_collection_freq(), <=, total_term_occurences);
1098         }
1099         TEST_EQUAL(get_reltermfreq(), 0);
1100         TEST_EQUAL(get_wqf(), 1);
1101         TEST_REL(doclen,>=,len_lower);
1102         TEST_REL(doclen,<=,len_upper);
1103         TEST_REL(uniqueterms,>=,1);
1104         TEST_REL(uniqueterms,<=,doclen);
1105         TEST_REL(wdf,<=,wdf_upper);
1106         sum += wdf;
1107         sum_squares += wdf * wdf;
1108         return 1.0;
1109     }
1110
1111     double get_maxpart() const {
1112         if (len_upper == 0) {
1113             len_lower = get_doclength_lower_bound();
1114             len_upper = get_doclength_upper_bound();
1115             wdf_upper = get_wdf_upper_bound();
1116         }
1117         return 1.0;
1118     }
1119
1120     double get_sumextra(Xapian::termcount doclen, Xapian::termcount) const {
1121         return 1.0 / doclen;
1122     }
1123
1124     double get_maxextra() const { return 1.0; }
1125 };
1126
1127 /// Check the weight subclass gets the correct stats.
1128 DEFINE_TESTCASE(checkstatsweight1, backend && !remote) {
1129     Xapian::Database db = get_database("apitest_simpledata");
1130     Xapian::Enquire enquire(db);
1131     Xapian::TermIterator a;
1132     for (a = db.allterms_begin(); a != db.allterms_end(); ++a) {
1133         const string & term = *a;
1134         enquire.set_query(Xapian::Query(term));
1135         Xapian::termcount sum = 0;
1136         Xapian::termcount sum_squares = 0;
1137         CheckStatsWeight wt(db, term, sum, sum_squares);
1138         enquire.set_weighting_scheme(wt);
1139         Xapian::MSet mset = enquire.get_mset(0, db.get_doccount());
1140
1141         // The document order in the multi-db case isn't the same as the
1142         // postlist order on the combined DB, so it's hard to compare the
1143         // wdf for each document in the Weight objects, so we can sum
1144         // the wdfs and the squares of the wdfs which provides a decent
1145         // check that we're not getting the wrong wdf values (it ensures
1146         // they have the right mean and standard deviation).
1147         Xapian::termcount expected_sum = 0;
1148         Xapian::termcount expected_sum_squares = 0;
1149         Xapian::PostingIterator i;
1150         for (i = db.postlist_begin(term); i != db.postlist_end(term); ++i) {
1151             Xapian::termcount wdf = i.get_wdf();
1152             expected_sum += wdf;
1153             expected_sum_squares += wdf * wdf;
1154         }
1155         TEST_EQUAL(sum, expected_sum);
1156         TEST_EQUAL(sum_squares, expected_sum_squares);
1157     }
1158     return true;
1159 }
1160
1161 /// Check the weight subclass gets the correct stats with OP_SYNONYM.
1162 // Regression test for bugs fixed in 1.4.1.
1163 DEFINE_TESTCASE(checkstatsweight2, backend && !remote) {
1164     Xapian::Database db = get_database("apitest_simpledata");
1165     Xapian::Enquire enquire(db);
1166     Xapian::TermIterator a;
1167     for (a = db.allterms_begin(); a != db.allterms_end(); ++a) {
1168         const string & term1 = *a;
1169         if (++a == db.allterms_end()) break;
1170         const string & term2 = *a;
1171         Xapian::Query q(Xapian::Query::OP_SYNONYM,
1172                         Xapian::Query(term1), Xapian::Query(term2));
1173         tout << q.get_description() << endl;
1174         enquire.set_query(q);
1175         Xapian::termcount sum = 0;
1176         Xapian::termcount sum_squares = 0;
1177         CheckStatsWeight wt(db, term1, term2, sum, sum_squares);
1178         enquire.set_weighting_scheme(wt);
1179         Xapian::MSet mset = enquire.get_mset(0, db.get_doccount());
1180
1181         // The document order in the multi-db case isn't the same as the
1182         // postlist order on the combined DB, so it's hard to compare the
1183         // wdf for each document in the Weight objects, so we can sum
1184         // the wdfs and the squares of the wdfs which provides a decent
1185         // check that we're not getting the wrong wdf values (it ensures
1186         // they have the right mean and standard deviation).
1187         Xapian::termcount expected_sum = 0;
1188         Xapian::termcount expected_sum_squares = 0;
1189         Xapian::PostingIterator i = db.postlist_begin(term1);
1190         Xapian::PostingIterator j = db.postlist_begin(term2);
1191         Xapian::docid did1 = *i, did2 = *j;
1192         while (true) {
1193             // To calculate expected_sum_squares correctly we need to square
1194             // the sum per document.
1195             Xapian::termcount wdf;
1196             if (did1 == did2) {
1197                 wdf = i.get_wdf() + j.get_wdf();
1198                 did1 = did2 = 0;
1199             } else if (did1 < did2) {
1200                 wdf = i.get_wdf();
1201                 did1 = 0;
1202             } else {
1203                 wdf = j.get_wdf();
1204                 did2 = 0;
1205             }
1206             expected_sum += wdf;
1207             expected_sum_squares += wdf * wdf;
1208
1209             if (did1 == 0) {
1210                 if (++i != db.postlist_end(term1)) {
1211                     did1 = *i;
1212                 } else {
1213                     if (did2 == Xapian::docid(-1)) break;
1214                     did1 = Xapian::docid(-1);
1215                 }
1216             }
1217             if (did2 == 0) {
1218                 if (++j != db.postlist_end(term2)) {
1219                     did2 = *j;
1220                 } else {
1221                     if (did1 == Xapian::docid(-1)) break;
1222                     did2 = Xapian::docid(-1);
1223                 }
1224             }
1225         }
1226         // The OP_SYNONYM's wdf should be equal to the sum of the wdfs of
1227         // the individual terms.
1228         TEST_EQUAL(sum, expected_sum);
1229         TEST_REL(sum_squares, >=, expected_sum_squares);
1230     }
1231     return true;
1232 }
1233
1234 /// Check the weight subclass gets the correct stats with OP_WILDCARD.
1235 // Regression test for bug fixed in 1.4.1.
1236 // Don't run with multi-database, as the termfreq checks don't work
1237 // there - FIXME: Investigate this - it smells like a bug.
1238 DEFINE_TESTCASE(checkstatsweight3, backend && !remote && !multi) {
1239     struct PlCmp {
1240         bool operator()(const Xapian::PostingIterator& a,
1241                         const Xapian::PostingIterator& b) {
1242             return *a < *b;
1243         }
1244     };
1245
1246     Xapian::Database db = get_database("apitest_simpledata");
1247     Xapian::Enquire enquire(db);
1248     Xapian::TermIterator a;
1249     static const char * testcases[] = {
1250         "a", // a* matches all documents, but no term matches all.
1251         "pa", // Expands to only "paragraph", matching 5.
1252         "zulu", // No matches.
1253         "th", // Term "this" matches all documents.
1254         NULL
1255     };
1256     for (const char ** p = testcases; *p; ++p) {
1257         const char * pattern = *p;
1258         Xapian::Query q(Xapian::Query::OP_WILDCARD, pattern);
1259         tout << q.get_description() << endl;
1260         enquire.set_query(q);
1261         Xapian::termcount sum = 0;
1262         Xapian::termcount sum_squares = 0;
1263         CheckStatsWeight wt(db, pattern, "*", sum, sum_squares);
1264         enquire.set_weighting_scheme(wt);
1265         Xapian::MSet mset = enquire.get_mset(0, db.get_doccount());
1266
1267         // The document order in the multi-db case isn't the same as the
1268         // postlist order on the combined DB, so it's hard to compare the
1269         // wdf for each document in the Weight objects, so we can sum
1270         // the wdfs and the squares of the wdfs which provides a decent
1271         // check that we're not getting the wrong wdf values (it ensures
1272         // they have the right mean and standard deviation).
1273         Xapian::termcount expected_sum = 0;
1274         Xapian::termcount expected_sum_squares = 0;
1275         vector<Xapian::PostingIterator> postlists;
1276         for (auto&& t = db.allterms_begin(pattern);
1277              t != db.allterms_end(pattern); ++t) {
1278             postlists.emplace_back(db.postlist_begin(*t));
1279         }
1280         make_heap(postlists.begin(), postlists.end(), PlCmp());
1281         Xapian::docid did = 0;
1282         Xapian::termcount wdf = 0;
1283         while (!postlists.empty()) {
1284             pop_heap(postlists.begin(), postlists.end(), PlCmp());
1285             Xapian::docid did_new = *postlists.back();
1286             Xapian::termcount wdf_new = postlists.back().get_wdf();
1287             if (++(postlists.back()) == Xapian::PostingIterator()) {
1288                 postlists.pop_back();
1289             } else {
1290                 push_heap(postlists.begin(), postlists.end(), PlCmp());
1291             }
1292             if (did_new != did) {
1293                 expected_sum += wdf;
1294                 expected_sum_squares += wdf * wdf;
1295                 wdf = 0;
1296                 did = did_new;
1297             }
1298             wdf += wdf_new;
1299         }
1300         expected_sum += wdf;
1301         expected_sum_squares += wdf * wdf;
1302         // The OP_SYNONYM's wdf should be equal to the sum of the wdfs of
1303         // the individual terms.
1304         TEST_EQUAL(sum, expected_sum);
1305         TEST_REL(sum_squares, >=, expected_sum_squares);
1306     }
1307     return true;
1308 }
1309
1310 // Two stage should perform same as Jelinek mercer if smoothing parameter for mercer is kept 1 in both.
1311 DEFINE_TESTCASE(unigramlmweight4, backend) {
1312     Xapian::Database db = get_database("apitest_simpledata");
1313     Xapian::Enquire enquire1(db);
1314     Xapian::Enquire enquire2(db);
1315     enquire1.set_query(Xapian::Query("paragraph"));
1316     Xapian::MSet mset1;
1317     enquire2.set_query(Xapian::Query("paragraph"));
1318     Xapian::MSet mset2;
1319     // 5 documents available with term paragraph so mset size should be 5
1320     enquire1.set_weighting_scheme(Xapian::LMWeight(0, Xapian::Weight::TWO_STAGE_SMOOTHING, 1, 0));
1321     enquire2.set_weighting_scheme(Xapian::LMWeight(0, Xapian::Weight::JELINEK_MERCER_SMOOTHING, 1, 0));
1322     mset1 = enquire1.get_mset(0, 10);
1323     mset2 = enquire2.get_mset(0, 10);
1324
1325     TEST_EQUAL(mset1.size(), 5);
1326     TEST_EQUAL_DOUBLE(mset1[1].get_weight(), mset2[1].get_weight());
1327     return true;
1328 }
1329
1330 /* Test for checking if we don't use smoothing all
1331  * of them should give same result i.e wdf_double/len_double */
1332 DEFINE_TESTCASE(unigramlmweight5, backend) {
1333     Xapian::Database db = get_database("apitest_simpledata");
1334     Xapian::Enquire enquire1(db);
1335     Xapian::Enquire enquire2(db);
1336     Xapian::Enquire enquire3(db);
1337     Xapian::Enquire enquire4(db);
1338     enquire1.set_query(Xapian::Query("paragraph"));
1339     Xapian::MSet mset1;
1340     enquire2.set_query(Xapian::Query("paragraph"));
1341     Xapian::MSet mset2;
1342     enquire3.set_query(Xapian::Query("paragraph"));
1343     Xapian::MSet mset3;
1344     enquire4.set_query(Xapian::Query("paragraph"));
1345     Xapian::MSet mset4;
1346     // 5 documents available with term paragraph so mset size should be 5
1347     enquire1.set_weighting_scheme(Xapian::LMWeight(10000.0, Xapian::Weight::TWO_STAGE_SMOOTHING, 0, 0));
1348     enquire2.set_weighting_scheme(Xapian::LMWeight(10000.0, Xapian::Weight::JELINEK_MERCER_SMOOTHING, 0, 0));
1349     enquire3.set_weighting_scheme(Xapian::LMWeight(10000.0, Xapian::Weight::ABSOLUTE_DISCOUNT_SMOOTHING, 0, 0));
1350     enquire4.set_weighting_scheme(Xapian::LMWeight(10000.0, Xapian::Weight::DIRICHLET_SMOOTHING, 0, 0));
1351
1352     mset1 = enquire1.get_mset(0, 10);
1353     mset2 = enquire2.get_mset(0, 10);
1354     mset3 = enquire3.get_mset(0, 10);
1355     mset4 = enquire4.get_mset(0, 10);
1356
1357     TEST_EQUAL(mset1.size(), 5);
1358     TEST_EQUAL(mset2.size(), 5);
1359     TEST_EQUAL(mset3.size(), 5);
1360     TEST_EQUAL(mset4.size(), 5);
1361     for (size_t i = 0; i < 5; i++) {
1362         TEST_EQUAL_DOUBLE(mset3[i].get_weight(), mset4[i].get_weight());
1363         TEST_EQUAL_DOUBLE(mset2[i].get_weight(), mset4[i].get_weight());
1364         TEST_EQUAL_DOUBLE(mset1[i].get_weight(), mset2[i].get_weight());
1365         TEST_EQUAL_DOUBLE(mset3[i].get_weight(), mset2[i].get_weight());
1366         TEST_EQUAL_DOUBLE(mset1[i].get_weight(), mset4[i].get_weight());
1367         TEST_EQUAL_DOUBLE(mset1[i].get_weight(), mset3[i].get_weight());
1368     }
1369     return true;
1370 }
1371
1372 // Test Exception for junk after serialised weight (with Dir+ enabled).
1373 DEFINE_TESTCASE(unigramlmweight6, !backend) {
1374     Xapian::LMWeight wt(0, Xapian::Weight::DIRICHLET_SMOOTHING, 0.5, 1.0);
1375     try {
1376         Xapian::LMWeight d;
1377         Xapian::LMWeight * d2 = d.unserialise(wt.serialise() + "X");
1378         // Make sure we actually use the weight.
1379         bool empty = d2->name().empty();
1380         delete d2;
1381         if (empty)
1382             FAIL_TEST("Serialised LMWeight with junk appended unserialised to empty name!");
1383         FAIL_TEST("Serialised LMWeight with junk appended unserialised OK");
1384     } catch (const Xapian::SerialisationError &e) {
1385         TEST(e.get_msg().find("LM") != string::npos);
1386     }
1387     return true;
1388 }
1389
1390 // Feature test for Dir+ function.
1391 DEFINE_TESTCASE(unigramlmweight7, backend) {
1392     Xapian::Database db = get_database("apitest_simpledata");
1393     Xapian::Enquire enquire1(db);
1394     Xapian::Enquire enquire2(db);
1395     enquire1.set_query(Xapian::Query("paragraph"));
1396     enquire2.set_query(Xapian::Query("paragraph"));
1397     Xapian::MSet mset1;
1398     Xapian::MSet mset2;
1399
1400     enquire1.set_weighting_scheme(Xapian::LMWeight(0, Xapian::Weight::DIRICHLET_SMOOTHING, 2000, 0));
1401     enquire2.set_weighting_scheme(Xapian::LMWeight(0, Xapian::Weight::DIRICHLET_PLUS_SMOOTHING, 2000, 0.05));
1402
1403     mset1 = enquire1.get_mset(0, 10);
1404     mset2 = enquire2.get_mset(0, 10);
1405
1406     // mset size should be 5
1407     TEST_EQUAL(mset1.size(), 5);
1408     TEST_EQUAL(mset2.size(), 5);
1409
1410     // Expect mset weights associated with Dir+ more than mset weights by Dir
1411     // because of the presence of extra weight component in Dir+ function.
1412     TEST_REL(mset2[0].get_weight(),>,mset1[0].get_weight());
1413     TEST_REL(mset2[1].get_weight(),>,mset1[1].get_weight());
1414     TEST_REL(mset2[2].get_weight(),>,mset1[2].get_weight());
1415     TEST_REL(mset2[3].get_weight(),>,mset1[3].get_weight());
1416     TEST_REL(mset2[4].get_weight(),>,mset1[4].get_weight());
1417
1418     return true;
1419 }
1420
1421 // Regression test that OP_SCALE_WEIGHT works with LMWeight (fixed in 1.4.1).
1422 DEFINE_TESTCASE(unigramlmweight8, backend) {
1423     Xapian::Database db = get_database("apitest_simpledata");
1424     Xapian::Enquire enquire(db);
1425     Xapian::Query query("paragraph");
1426
1427     enquire.set_query(query);
1428     enquire.set_weighting_scheme(Xapian::LMWeight(0, Xapian::Weight::DIRICHLET_SMOOTHING, 2000, 0));
1429
1430     Xapian::MSet mset1;
1431     mset1 = enquire.get_mset(0, 10);
1432     TEST_EQUAL(mset1.size(), 5);
1433
1434     enquire.set_query(Xapian::Query(Xapian::Query::OP_SCALE_WEIGHT, query, 15.0));
1435     enquire.set_weighting_scheme(Xapian::LMWeight(0, Xapian::Weight::DIRICHLET_SMOOTHING, 2000, 0));
1436
1437     Xapian::MSet mset2;
1438     mset2 = enquire.get_mset(0, 10);
1439     TEST_EQUAL(mset2.size(), mset1.size());
1440     TEST_NOT_EQUAL_DOUBLE(mset1[0].get_weight(), 0.0);
1441     for (Xapian::doccount i = 0; i < mset1.size(); ++i) {
1442         TEST_EQUAL_DOUBLE(15.0 * mset1[i].get_weight(), mset2[i].get_weight());
1443     }
1444
1445     return true;
1446 }
1447
1448 // Feature test for BoolWeight.
1449 // Test exception for junk after serialised weight.
1450 DEFINE_TESTCASE(boolweight1, !backend) {
1451     Xapian::BoolWeight wt;
1452     try {
1453         Xapian::BoolWeight t;
1454         Xapian::BoolWeight * t2 = t.unserialise(wt.serialise() + "X");
1455         // Make sure we actually use the weight.
1456         bool empty = t2->name().empty();
1457         delete t2;
1458         if (empty)
1459             FAIL_TEST("Serialised BoolWeight with junk appended unserialised to empty name!");
1460         FAIL_TEST("Serialised BoolWeight with junk appended unserialised OK");
1461     } catch (const Xapian::SerialisationError &e) {
1462         TEST(e.get_msg().find("Bool") != string::npos);
1463     }
1464     return true;
1465 }
1466
1467 // Feature test for CoordWeight.
1468 DEFINE_TESTCASE(coordweight1, backend) {
1469     Xapian::Enquire enquire(get_database("apitest_simpledata"));
1470     enquire.set_weighting_scheme(Xapian::CoordWeight());
1471     const char * terms[] = { "this", "line", "paragraph", "rubbish" };
1472     Xapian::Query query(Xapian::Query::OP_OR,
1473                         terms, terms + sizeof(terms) / sizeof(terms[0]));
1474     enquire.set_query(query);
1475     Xapian::MSet mymset1 = enquire.get_mset(0, 100);
1476     // CoordWeight scores 1 for each matching term, so the weight should equal
1477     // the number of matching terms.
1478     for (Xapian::MSetIterator i = mymset1.begin(); i != mymset1.end(); ++i) {
1479         Xapian::termcount matching_terms = 0;
1480         Xapian::TermIterator t = enquire.get_matching_terms_begin(i);
1481         while (t != enquire.get_matching_terms_end(i)) {
1482             ++matching_terms;
1483             ++t;
1484         }
1485         TEST_EQUAL(i.get_weight(), matching_terms);
1486     }
1487
1488     // Test with OP_SCALE_WEIGHT.
1489     enquire.set_query(Xapian::Query(Xapian::Query::OP_SCALE_WEIGHT, query, 15.0));
1490     Xapian::MSet mymset2 = enquire.get_mset(0, 100);
1491     TEST_EQUAL(mymset1.size(), mymset2.size());
1492     for (Xapian::doccount i = 0; i != mymset1.size(); ++i) {
1493         TEST_EQUAL(15.0 * mymset1[i].get_weight(), mymset2[i].get_weight());
1494     }
1495
1496     return true;
1497 }
1498
1499 // Test exception for junk after serialised weight.
1500 DEFINE_TESTCASE(coordweight2, !backend) {
1501     Xapian::CoordWeight wt;
1502     try {
1503         Xapian::CoordWeight t;
1504         Xapian::CoordWeight * t2 = t.unserialise(wt.serialise() + "X");
1505         // Make sure we actually use the weight.
1506         bool empty = t2->name().empty();
1507         delete t2;
1508         if (empty)
1509             FAIL_TEST("Serialised CoordWeight with junk appended unserialised to empty name!");
1510         FAIL_TEST("Serialised CoordWeight with junk appended unserialised OK");
1511     } catch (const Xapian::SerialisationError &e) {
1512         TEST(e.get_msg().find("Coord") != string::npos);
1513     }
1514     return true;
1515 }