xapian-core/tests/api_valuestream.cc

   1 /** @file api_valuestream.cc
   2  * @brief Tests of valuestream functionality.
   3  */
   4 /* Copyright (C) 2008,2009,2010 Olly Betts
   5  * Copyright (C) 2009 Lemur Consulting Ltd
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License as
   9  * published by the Free Software Foundation; either version 2 of the
  10  * License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  20  * USA
  21  */
  22
  23 #include <config.h>
  24
  25 #include "api_valuestream.h"
  26
  27 #include <xapian.h>
  28 #include "testsuite.h"
  29 #include "testutils.h"
  30
  31 #include "apitest.h"
  32
  33 using namespace std;
  34
  35 /// Feature test simple valuestream iteration.
  36 DEFINE_TESTCASE(valuestream1, backend && !multi) {
  37     // FIXME: enable for multi once support is in place.
  38     Xapian::Database db = get_database("apitest_simpledata");
  39
  40     for (Xapian::valueno slot = 0; slot < 15; ++slot) {
  41         tout << "testing valuestream iteration for slot " << slot << endl;
  42         Xapian::ValueIterator it = db.valuestream_begin(slot);
  43         while (it != db.valuestream_end(slot)) {
  44             TEST_EQUAL(it.get_valueno(), slot);
  45             string value = *it;
  46             Xapian::docid did = it.get_docid();
  47
  48             Xapian::Document doc = db.get_document(did);
  49             TEST_EQUAL(doc.get_value(slot), value);
  50
  51             ++it;
  52         }
  53     }
  54
  55     return true;
  56 }
  57
  58 /// Test skip_to() on a valuestream iterator.
  59 DEFINE_TESTCASE(valuestream2, backend) {
  60     Xapian::Database db = get_database("etext");
  61
  62     for (Xapian::valueno slot = 0; slot < 15; ++slot) {
  63         unsigned interval = 1;
  64         while (interval < 1999) {
  65             tout.str(string());
  66             tout << "testing valuestream skip_to for slot " << slot
  67                  << " with interval " << interval << endl;
  68             Xapian::docid did = 1;
  69             Xapian::ValueIterator it = db.valuestream_begin(slot);
  70             if (it == db.valuestream_end(slot)) break;
  71             while (it.skip_to(did), it != db.valuestream_end(slot)) {
  72                 TEST_EQUAL(it.get_valueno(), slot);
  73                 string value = *it;
  74
  75                 // Check that the skipped documents had no values.
  76                 Xapian::docid actual_did = it.get_docid();
  77                 TEST_REL(actual_did,>=,did);
  78                 while (did < actual_did) {
  79                     Xapian::Document doc = db.get_document(did);
  80                     TEST(doc.get_value(slot).empty());
  81                     ++did;
  82                 }
  83
  84                 Xapian::Document doc = db.get_document(actual_did);
  85                 TEST_EQUAL(doc.get_value(slot), value);
  86                 did += interval;
  87             }
  88             interval = interval * 3 - 1;
  89         }
  90     }
  91
  92     return true;
  93 }
  94
  95 /// Test check() on a valuestream iterator.
  96 DEFINE_TESTCASE(valuestream3, backend) {
  97     Xapian::Database db = get_database("etext");
  98
  99     // Check combinations of check with other operations.
 100     typedef enum {
 101         CHECK, CHECK_AND_NEXT, CHECK2, SKIP_TO, CHECK_AND_LOOP
 102     } test_op;
 103     test_op operation = CHECK;
 104
 105     for (Xapian::valueno slot = 0; slot < 15; ++slot) {
 106         unsigned interval = 1;
 107         while (interval < 1999) {
 108             tout << "testing valuestream check for slot " << slot
 109                  << " with interval " << interval << endl;
 110             Xapian::docid did = 1;
 111             Xapian::ValueIterator it = db.valuestream_begin(slot);
 112             if (it == db.valuestream_end(slot)) break;
 113             while (true) {
 114                 bool positioned = true;
 115                 switch (operation) {
 116                     case CHECK_AND_LOOP:
 117                         operation = CHECK;
 118                         // FALLTHRU.
 119                     case CHECK: case CHECK2:
 120                         positioned = it.check(did);
 121                         break;
 122                     case CHECK_AND_NEXT: {
 123                         bool was_skip_to = it.check(did);
 124                         if (!was_skip_to) ++it;
 125                         break;
 126                     }
 127                     case SKIP_TO:
 128                         it.skip_to(did);
 129                         break;
 130                 }
 131                 operation = test_op(operation + 1);
 132                 if (positioned) {
 133                     if (it == db.valuestream_end(slot)) break;
 134                     TEST_EQUAL(it.get_valueno(), slot);
 135                     string value = *it;
 136
 137                     // Check that the skipped documents had no values.
 138                     Xapian::docid actual_did = it.get_docid();
 139                     while (did < actual_did) {
 140                         Xapian::Document doc = db.get_document(did);
 141                         TEST(doc.get_value(slot).empty());
 142                         ++did;
 143                     }
 144
 145                     Xapian::Document doc = db.get_document(actual_did);
 146                     TEST_EQUAL(doc.get_value(slot), value);
 147                 }
 148                 did += interval;
 149             }
 150             interval = interval * 3 - 1;
 151         }
 152     }
 153
 154     return true;
 155 }
 156
 157 /** Check that valueweightsource handles last_docid of 0xffffffff.
 158  *
 159  *  The original implementation went into an infinite loop in this case.
 160  */
 161 DEFINE_TESTCASE(valueweightsource5, writable && valuestats) {
 162     // inmemory's memory use is currently O(last_docid)!
 163     SKIP_TEST_FOR_BACKEND("inmemory");
 164     // remote's value slot iteration is very slow for this case currently
 165     // because it throws and catches DocNotFoundError across the link 2^32-3
 166     // times.
 167     SKIP_TEST_FOR_BACKEND("remote");
 168     Xapian::WritableDatabase db = get_writable_database();
 169     Xapian::Document doc;
 170     doc.add_value(1, Xapian::sortable_serialise(3.14));
 171     db.replace_document(1, doc);
 172     db.replace_document(0xffffffff, doc);
 173     db.commit();
 174
 175     Xapian::ValueWeightPostingSource src(1);
 176     src.init(db);
 177     src.next(0.0);
 178     TEST(!src.at_end());
 179     TEST_EQUAL(src.get_docid(), 1);
 180     src.next(0.0);
 181     TEST(!src.at_end());
 182     TEST_EQUAL(src.get_docid(), 0xffffffff);
 183     src.next(0.0);
 184     TEST(src.at_end());
 185
 186     return true;
 187 }
 188
 189 // Check that ValueMapPostingSource works correctly.
 190 // the test db has value 13 set to:
 191 //      1   Thi
 192 //      2   The
 193 //      3   You
 194 //      4   War
 195 //      5   Fri
 196 //      6   Ins
 197 //      7   Whi
 198 //      8   Com
 199 //      9   A p
 200 //      10  Tel
 201 //      11  Tel
 202 //      12  Enc
 203 //      13  Get
 204 //      14  Doe
 205 //      15  fir
 206 //      16  Pad
 207 //      17  Pad
 208 //
 209 DEFINE_TESTCASE(valuemapsource1, backend) {
 210     Xapian::Database db(get_database("apitest_phrase"));
 211     Xapian::Enquire enq(db);
 212
 213     Xapian::ValueMapPostingSource src(13);
 214     src.add_mapping("Thi", 2.0);
 215     src.add_mapping("The", 1.0);
 216     src.add_mapping("You", 3.0);
 217     src.add_mapping("War", 4.0);
 218     src.add_mapping("Fri", 5.0);
 219
 220     // check mset size and order
 221     enq.set_query(Xapian::Query(&src));
 222     Xapian::MSet mset = enq.get_mset(0, 5);
 223
 224     TEST(mset.size() == 5);
 225     mset_expect_order(mset, 5, 4, 3, 1, 2);
 226
 227     // and with default weight
 228     src.clear_mappings();
 229     src.set_default_weight(3.5);
 230     src.add_mapping("Thi", 2.0);
 231     src.add_mapping("The", 1.0);
 232     src.add_mapping("You", 3.0);
 233     src.add_mapping("War", 4.0);
 234     src.add_mapping("Fri", 5.0);
 235
 236     enq.set_query(Xapian::Query(&src));
 237     mset = enq.get_mset(0, 5);
 238
 239     TEST(mset.size() == 5);
 240     mset_expect_order(mset, 5, 4, 6, 7, 8);
 241
 242     return true;
 243 }
 244
 245 // Regression test for valuepostingsource subclasses: used to segfault if skip_to()
 246 // called on an empty list.
 247 DEFINE_TESTCASE(valuemapsource2, backend && !multi) {
 248     Xapian::Database db(get_database("apitest_phrase"));
 249
 250     {
 251         Xapian::ValueMapPostingSource src(100);
 252         src.init(db);
 253         TEST(src.at_end() == false);
 254         src.next(0.0);
 255         TEST(src.at_end() == true);
 256     }
 257
 258     {
 259         Xapian::ValueMapPostingSource src(100);
 260         src.init(db);
 261         TEST(src.at_end() == false);
 262         src.skip_to(1, 0.0);
 263         TEST(src.at_end() == true);
 264     }
 265
 266     {
 267         Xapian::ValueMapPostingSource src(100);
 268         src.init(db);
 269         TEST(src.at_end() == false);
 270         src.check(1, 0.0);
 271         TEST(src.at_end() == true);
 272     }
 273
 274     return true;
 275 }
 276
 277 // Regression test for fixedweightpostingsource: used to segfault if skip_to()
 278 // called on an empty list.
 279 DEFINE_TESTCASE(fixedweightsource2, !backend) {
 280     Xapian::Database db;
 281
 282     {
 283         Xapian::FixedWeightPostingSource src(5.0);
 284         src.init(db);
 285         TEST(src.at_end() == false);
 286         src.next(0.0);
 287         TEST(src.at_end() == true);
 288     }
 289
 290     {
 291         Xapian::FixedWeightPostingSource src(5.0);
 292         src.init(db);
 293         TEST(src.at_end() == false);
 294         src.skip_to(1, 0.0);
 295         TEST(src.at_end() == true);
 296     }
 297
 298     // No need to test behaviour of check() - check is only allowed to be
 299     // called with document IDs which exist, so can never be called for a
 300     // FixedWeightPostingSource with an empty database.
 301
 302     return true;
 303 }
 304
 305 // Test DecreasingValueWeightPostingSource.
 306 DEFINE_TESTCASE(decvalwtsource1, writable) {
 307     Xapian::WritableDatabase db = get_writable_database();
 308
 309     Xapian::Document doc;
 310     doc.add_value(1, Xapian::sortable_serialise(3));
 311     db.add_document(doc);
 312     doc.add_value(1, Xapian::sortable_serialise(2));
 313     db.add_document(doc);
 314     doc.add_value(1, Xapian::sortable_serialise(1));
 315     db.add_document(doc);
 316     db.commit();
 317
 318     // Check basic function
 319     {
 320         Xapian::DecreasingValueWeightPostingSource src(1);
 321         src.init(db);
 322
 323         src.next(0.0);
 324         TEST(!src.at_end());
 325         TEST_EQUAL(src.get_docid(), 1);
 326
 327         src.next(0.0);
 328         TEST(!src.at_end());
 329         TEST_EQUAL(src.get_docid(), 2);
 330
 331         src.next(0.0);
 332         TEST(!src.at_end());
 333         TEST_EQUAL(src.get_docid(), 3);
 334
 335         src.next(0.0);
 336         TEST(src.at_end());
 337     }
 338
 339     // Check skipping to end of list due to weight
 340     {
 341         Xapian::DecreasingValueWeightPostingSource src(1);
 342         src.init(db);
 343
 344         src.next(1.5);
 345         TEST(!src.at_end());
 346         TEST_EQUAL(src.get_docid(), 1);
 347
 348         src.next(1.5);
 349         TEST(!src.at_end());
 350         TEST_EQUAL(src.get_docid(), 2);
 351
 352         src.next(1.5);
 353         TEST(src.at_end());
 354     }
 355
 356     // Check behaviour with a restricted range
 357     doc.add_value(1, Xapian::sortable_serialise(2));
 358     db.add_document(doc);
 359
 360     {
 361         Xapian::DecreasingValueWeightPostingSource src(1, 1, 3);
 362         src.init(db);
 363
 364         src.next(1.5);
 365         TEST(!src.at_end());
 366         TEST_EQUAL(src.get_docid(), 1);
 367
 368         src.next(1.5);
 369         TEST(!src.at_end());
 370         TEST_EQUAL(src.get_docid(), 2);
 371
 372         src.next(1.5);
 373         TEST(!src.at_end());
 374         TEST_EQUAL(src.get_docid(), 4);
 375
 376         src.next(1.5);
 377         TEST(src.at_end());
 378     }
 379
 380     {
 381         Xapian::DecreasingValueWeightPostingSource src(1, 1, 3);
 382         src.init(db);
 383
 384         src.next(1.5);
 385         TEST(!src.at_end());
 386         TEST_EQUAL(src.get_docid(), 1);
 387
 388         src.skip_to(3, 1.5);
 389         TEST(!src.at_end());
 390         TEST_EQUAL(src.get_docid(), 4);
 391
 392         src.next(1.5);
 393         TEST(src.at_end());
 394     }
 395
 396     {
 397         Xapian::DecreasingValueWeightPostingSource src(1, 1, 3);
 398         src.init(db);
 399
 400         src.next(1.5);
 401         TEST(!src.at_end());
 402         TEST_EQUAL(src.get_docid(), 1);
 403
 404         TEST(src.check(3, 1.5));
 405         TEST(!src.at_end());
 406         TEST_EQUAL(src.get_docid(), 4);
 407
 408         src.next(1.5);
 409         TEST(src.at_end());
 410     }
 411
 412     return true;
 413 }
 414
 415 // Test DecreasingValueWeightPostingSource with out-of-order sections at
 416 // start, and with repeated weights.
 417 DEFINE_TESTCASE(decvalwtsource2, writable) {
 418     Xapian::WritableDatabase db = get_writable_database();
 419
 420     Xapian::Document doc;
 421     doc.add_value(1, Xapian::sortable_serialise(1));
 422     db.add_document(doc);
 423     doc.add_value(1, Xapian::sortable_serialise(3));
 424     db.add_document(doc);
 425     doc.add_value(1, Xapian::sortable_serialise(3));
 426     db.add_document(doc);
 427     doc.add_value(1, Xapian::sortable_serialise(1));
 428     db.add_document(doc);
 429     db.commit();
 430
 431     // Check basic function
 432     {
 433         Xapian::DecreasingValueWeightPostingSource src(1);
 434         src.init(db);
 435
 436         src.next(0.0);
 437         TEST(!src.at_end());
 438         TEST_EQUAL(src.get_docid(), 1);
 439
 440         src.next(0.0);
 441         TEST(!src.at_end());
 442         TEST_EQUAL(src.get_docid(), 2);
 443
 444         src.next(0.0);
 445         TEST(!src.at_end());
 446         TEST_EQUAL(src.get_docid(), 3);
 447
 448         src.next(0.0);
 449         TEST(!src.at_end());
 450         TEST_EQUAL(src.get_docid(), 4);
 451
 452         src.next(0.0);
 453         TEST(src.at_end());
 454     }
 455
 456     // Check skipping to end of list due to weight
 457     {
 458         Xapian::DecreasingValueWeightPostingSource src(1, 2);
 459         src.init(db);
 460
 461         src.next(1.5);
 462         TEST(!src.at_end());
 463         TEST_EQUAL(src.get_docid(), 1);
 464
 465         src.next(1.5);
 466         TEST(!src.at_end());
 467         TEST_EQUAL(src.get_docid(), 2);
 468
 469         src.next(1.5);
 470         TEST(!src.at_end());
 471         TEST_EQUAL(src.get_docid(), 3);
 472
 473         src.next(1.5);
 474         TEST(src.at_end());
 475     }
 476
 477     // Check behaviour with a restricted range
 478     doc.add_value(1, Xapian::sortable_serialise(2));
 479     db.add_document(doc);
 480
 481     {
 482         Xapian::DecreasingValueWeightPostingSource src(1, 2, 4);
 483         src.init(db);
 484
 485         src.next(1.5);
 486         TEST(!src.at_end());
 487         TEST_EQUAL(src.get_docid(), 1);
 488
 489         src.next(1.5);
 490         TEST(!src.at_end());
 491         TEST_EQUAL(src.get_docid(), 2);
 492
 493         src.next(1.5);
 494         TEST(!src.at_end());
 495         TEST_EQUAL(src.get_docid(), 3);
 496
 497         src.next(1.5);
 498         TEST(!src.at_end());
 499         TEST_EQUAL(src.get_docid(), 5);
 500
 501         src.next(1.5);
 502         TEST(src.at_end());
 503     }
 504
 505     {
 506         Xapian::DecreasingValueWeightPostingSource src(1, 2, 4);
 507         src.init(db);
 508
 509         TEST(src.check(1, 1.5));
 510         TEST(!src.at_end());
 511         TEST_EQUAL(src.get_docid(), 1);
 512
 513         src.next(1.5);
 514         TEST(!src.at_end());
 515         TEST_EQUAL(src.get_docid(), 2);
 516
 517         src.skip_to(4, 1.5);
 518         TEST(!src.at_end());
 519         TEST_EQUAL(src.get_docid(), 5);
 520
 521         src.next(1.5);
 522         TEST(src.at_end());
 523     }
 524
 525     {
 526         Xapian::DecreasingValueWeightPostingSource src(1, 2, 4);
 527         src.init(db);
 528
 529         TEST(src.check(1, 1.5));
 530         TEST(!src.at_end());
 531         TEST_EQUAL(src.get_docid(), 1);
 532
 533         src.next(1.5);
 534         TEST(!src.at_end());
 535         TEST_EQUAL(src.get_docid(), 2);
 536
 537         TEST(src.check(4, 1.5));
 538         TEST(!src.at_end());
 539         TEST_EQUAL(src.get_docid(), 5);
 540
 541         src.next(1.5);
 542         TEST(src.at_end());
 543     }
 544
 545     return true;
 546 }
 547
 548 // Test DecreasingValueWeightPostingSource with an actual query.
 549 DEFINE_TESTCASE(decvalwtsource3, writable) {
 550     Xapian::WritableDatabase db = get_writable_database();
 551
 552     Xapian::Document doc;
 553     doc.add_term("foo");
 554     doc.add_value(1, Xapian::sortable_serialise(1));
 555     db.add_document(doc);
 556     doc.add_value(1, Xapian::sortable_serialise(3));
 557     db.add_document(doc);
 558     doc.add_term("bar");
 559     doc.add_value(1, Xapian::sortable_serialise(3));
 560     db.add_document(doc);
 561     doc.add_value(1, Xapian::sortable_serialise(1));
 562     db.add_document(doc);
 563     db.commit();
 564
 565     Xapian::DecreasingValueWeightPostingSource ps(1, 2, 5);
 566     Xapian::Query q(&ps);
 567     Xapian::Enquire enq(db);
 568     enq.set_query(q);
 569
 570     Xapian::MSet mset1(enq.get_mset(0, 1));
 571     Xapian::MSet mset2(enq.get_mset(0, 2));
 572     Xapian::MSet mset3(enq.get_mset(0, 3));
 573     Xapian::MSet mset4(enq.get_mset(0, 4));
 574
 575     TEST_EQUAL(mset1.size(), 1);
 576     TEST_EQUAL(mset2.size(), 2);
 577     TEST_EQUAL(mset3.size(), 3);
 578     TEST_EQUAL(mset4.size(), 4);
 579
 580     TEST(mset_range_is_same(mset1, 0, mset2, 0, 1));
 581     TEST(mset_range_is_same(mset2, 0, mset3, 0, 2));
 582     TEST(mset_range_is_same(mset3, 0, mset4, 0, 3));
 583
 584     return true;
 585 }
 586
 587 // Test DecreasingValueWeightPostingSource with an actual query on a fixed
 588 // dataset (so we can cover the remote backend too).
 589 DEFINE_TESTCASE(decvalwtsource4, backend && !multi) {
 590     Xapian::Database db = get_database("apitest_declen");
 591
 592     Xapian::DecreasingValueWeightPostingSource ps(11, 2, 5);
 593     Xapian::Query q(&ps);
 594     Xapian::Enquire enq(db);
 595     enq.set_query(q);
 596
 597     Xapian::MSet mset1(enq.get_mset(0, 1));
 598     Xapian::MSet mset2(enq.get_mset(0, 2));
 599     Xapian::MSet mset3(enq.get_mset(0, 3));
 600     Xapian::MSet mset4(enq.get_mset(0, 4));
 601
 602     TEST_EQUAL(mset1.size(), 1);
 603     TEST_EQUAL(mset2.size(), 2);
 604     TEST_EQUAL(mset3.size(), 3);
 605     TEST_EQUAL(mset4.size(), 4);
 606
 607     TEST(mset_range_is_same(mset1, 0, mset2, 0, 1));
 608     TEST(mset_range_is_same(mset2, 0, mset3, 0, 2));
 609     TEST(mset_range_is_same(mset3, 0, mset4, 0, 3));
 610
 611     return true;
 612 }
 613
 614 // Regression test - used to get segfaults if
 615 // DecreasingValueWeightPostingSource was pointed at an empty slot.
 616 DEFINE_TESTCASE(decvalwtsource5, writable) {
 617     Xapian::WritableDatabase db = get_writable_database();
 618
 619     Xapian::Document doc;
 620     doc.add_value(1, Xapian::sortable_serialise(1));
 621     db.add_document(doc);
 622     doc.add_value(2, Xapian::sortable_serialise(1));
 623     db.add_document(doc);
 624     db.commit();
 625
 626     {
 627         Xapian::DecreasingValueWeightPostingSource ps(1);
 628         Xapian::Query q(&ps);
 629         Xapian::Enquire enq(db);
 630         enq.set_query(q);
 631         Xapian::MSet mset1(enq.get_mset(0, 3));
 632         TEST_EQUAL(mset1.size(), 2);
 633     }
 634     {
 635         Xapian::DecreasingValueWeightPostingSource ps(2);
 636         Xapian::Query q(&ps);
 637         Xapian::Enquire enq(db);
 638         enq.set_query(q);
 639         Xapian::MSet mset1(enq.get_mset(0, 3));
 640         TEST_EQUAL(mset1.size(), 1);
 641     }
 642     {
 643         Xapian::DecreasingValueWeightPostingSource ps(3);
 644         Xapian::Query q(&ps);
 645         Xapian::Enquire enq(db);
 646         enq.set_query(q);
 647         Xapian::MSet mset1(enq.get_mset(0, 3));
 648         TEST_EQUAL(mset1.size(), 0);
 649     }
 650
 651     return true;
 652 }