xapian-core/tests/api_sorting.cc

   1 /** @file api_sorting.cc
   2  * @brief tests of MSet sorting
   3  */
   4 /* Copyright (C) 2007,2008,2009,2012,2017 Olly Betts
   5  * Copyright (C) 2010 Richard Boulton
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  20  */
  21
  22 #include <config.h>
  23
  24 #include "api_sorting.h"
  25
  26 #include <xapian.h>
  27
  28 #include "apitest.h"
  29 #include "testutils.h"
  30
  31 using namespace std;
  32
  33 DEFINE_TESTCASE(sortfunctor1, backend && !remote) {
  34     Xapian::Enquire enquire(get_database("apitest_sortrel"));
  35     enquire.set_query(Xapian::Query("woman"));
  36
  37     {
  38         static const int keys[] = { 3, 1 };
  39         Xapian::MultiValueKeyMaker sorter(keys, keys + 2);
  40
  41         enquire.set_sort_by_key(&sorter, true);
  42         Xapian::MSet mset = enquire.get_mset(0, 10);
  43         mset_expect_order(mset, 2, 6, 7, 1, 3, 4, 5, 8, 9);
  44
  45         for (auto m = mset.begin(); m != mset.end(); ++m) {
  46             const string& data = m.get_document().get_data();
  47             string exp;
  48             exp += data[3];
  49             exp += string(2, '\0');
  50             exp += data[1];
  51             TEST_EQUAL(m.get_sort_key(), exp);
  52         }
  53     }
  54
  55     {
  56         Xapian::MultiValueKeyMaker sorter;
  57         sorter.add_value(3);
  58         sorter.add_value(1, true);
  59
  60         enquire.set_sort_by_key(&sorter, true);
  61         Xapian::MSet mset = enquire.get_mset(0, 10);
  62         mset_expect_order(mset, 7, 6, 2, 8, 9, 4, 5, 1, 3);
  63
  64         for (auto m = mset.begin(); m != mset.end(); ++m) {
  65             const string& data = m.get_document().get_data();
  66             string exp;
  67             exp += data[3];
  68             exp += string(2, '\0');
  69             exp += char(0xff - data[1]);
  70             exp += string(2, '\xff');
  71             TEST_EQUAL(m.get_sort_key(), exp);
  72         }
  73     }
  74
  75     {
  76         Xapian::MultiValueKeyMaker sorter;
  77         sorter.add_value(100); // Value 100 isn't set.
  78         sorter.add_value(3);
  79         sorter.add_value(1, true);
  80
  81         enquire.set_sort_by_key(&sorter, true);
  82         Xapian::MSet mset = enquire.get_mset(0, 10);
  83         mset_expect_order(mset, 7, 6, 2, 8, 9, 4, 5, 1, 3);
  84
  85         for (auto m = mset.begin(); m != mset.end(); ++m) {
  86             const string& data = m.get_document().get_data();
  87             string exp;
  88             exp += string(2, '\0');
  89             exp += data[3];
  90             exp += string(2, '\0');
  91             exp += char(0xff - data[1]);
  92             exp += string(2, '\xff');
  93             TEST_EQUAL(m.get_sort_key(), exp);
  94         }
  95     }
  96
  97     {
  98         Xapian::MultiValueKeyMaker sorter;
  99         sorter.add_value(10); // Value 10 isn't always set.
 100         sorter.add_value(1, true);
 101
 102         enquire.set_sort_by_key(&sorter, true);
 103         Xapian::MSet mset = enquire.get_mset(0, 10);
 104         mset_expect_order(mset, 8, 9, 4, 5, 1, 3, 7, 6, 2);
 105
 106         for (auto m = mset.begin(); m != mset.end(); ++m) {
 107             const string& data = m.get_document().get_data();
 108             string exp;
 109             if (data.size() > 10) exp += data[10];
 110             exp += string(2, '\0');
 111             exp += char(0xff - data[1]);
 112             exp += string(2, '\xff');
 113             TEST_EQUAL(m.get_sort_key(), exp);
 114         }
 115     }
 116
 117     return true;
 118 }
 119
 120 /// Test reverse sort functor.
 121 DEFINE_TESTCASE(sortfunctor2, writable && !remote) {
 122     Xapian::WritableDatabase db = get_writable_database();
 123     Xapian::Document doc;
 124     doc.add_term("foo");
 125     doc.add_value(0, "ABB");
 126     db.add_document(doc);
 127     doc.add_value(0, "ABC");
 128     db.add_document(doc);
 129     doc.add_value(0, string("ABC", 4));
 130     db.add_document(doc);
 131     doc.add_value(0, "ABCD");
 132     db.add_document(doc);
 133     doc.add_value(0, "ABC\xff");
 134     db.add_document(doc);
 135
 136     Xapian::Enquire enquire(db);
 137     enquire.set_query(Xapian::Query("foo"));
 138
 139     {
 140         Xapian::MultiValueKeyMaker sorter;
 141         sorter.add_value(0);
 142         enquire.set_sort_by_key(&sorter, true);
 143         Xapian::MSet mset = enquire.get_mset(0, 10);
 144         mset_expect_order(mset, 5, 4, 3, 2, 1);
 145     }
 146
 147     {
 148         Xapian::MultiValueKeyMaker sorter;
 149         sorter.add_value(0, true);
 150         enquire.set_sort_by_key(&sorter, true);
 151         Xapian::MSet mset = enquire.get_mset(0, 10);
 152         mset_expect_order(mset, 1, 2, 3, 4, 5);
 153     }
 154
 155     {
 156         Xapian::MultiValueKeyMaker sorter;
 157         sorter.add_value(0);
 158         sorter.add_value(1);
 159         enquire.set_sort_by_key(&sorter, true);
 160         Xapian::MSet mset = enquire.get_mset(0, 10);
 161         mset_expect_order(mset, 5, 4, 3, 2, 1);
 162     }
 163
 164     {
 165         Xapian::MultiValueKeyMaker sorter;
 166         sorter.add_value(0, true);
 167         sorter.add_value(1);
 168         enquire.set_sort_by_key(&sorter, true);
 169         Xapian::MSet mset = enquire.get_mset(0, 10);
 170         mset_expect_order(mset, 1, 2, 3, 4, 5);
 171     }
 172
 173     {
 174         Xapian::MultiValueKeyMaker sorter;
 175         sorter.add_value(0);
 176         sorter.add_value(1, true);
 177         enquire.set_sort_by_key(&sorter, true);
 178         Xapian::MSet mset = enquire.get_mset(0, 10);
 179         mset_expect_order(mset, 5, 4, 3, 2, 1);
 180     }
 181
 182     {
 183         Xapian::MultiValueKeyMaker sorter;
 184         sorter.add_value(0, true);
 185         sorter.add_value(1, true);
 186         enquire.set_sort_by_key(&sorter, true);
 187         Xapian::MSet mset = enquire.get_mset(0, 10);
 188         mset_expect_order(mset, 1, 2, 3, 4, 5);
 189     }
 190
 191     return true;
 192 }
 193
 194 // Test sort functor with some empty values.
 195 DEFINE_TESTCASE(sortfunctor3, backend && !remote && valuestats) {
 196     Xapian::Database db(get_database("apitest_sortrel"));
 197     Xapian::Enquire enquire(db);
 198     enquire.set_query(Xapian::Query("woman"));
 199
 200     // Value 10 is set to 'a' for 1, 3, 4, 5, 8, 9, and not set otherwise.
 201     {
 202         // Test default sort order - missing values come first.
 203         Xapian::MultiValueKeyMaker sorter;
 204         sorter.add_value(10);
 205
 206         enquire.set_sort_by_key(&sorter, false);
 207         Xapian::MSet mset = enquire.get_mset(0, 10);
 208         mset_expect_order(mset, 2, 6, 7, 1, 3, 4, 5, 8, 9);
 209     }
 210
 211     {
 212         // Use a default value to put the missing values to the end.
 213         Xapian::MultiValueKeyMaker sorter;
 214         sorter.add_value(10, false, db.get_value_upper_bound(10) + '\xff');
 215
 216         enquire.set_sort_by_key(&sorter, false);
 217         Xapian::MSet mset = enquire.get_mset(0, 10);
 218         mset_expect_order(mset, 1, 3, 4, 5, 8, 9, 2, 6, 7);
 219     }
 220
 221     {
 222         // Test using a default value and sorting in reverse order
 223         Xapian::MultiValueKeyMaker sorter;
 224         sorter.add_value(10, false, db.get_value_upper_bound(10) + '\xff');
 225
 226         enquire.set_sort_by_key(&sorter, true);
 227         Xapian::MSet mset = enquire.get_mset(0, 10);
 228         mset_expect_order(mset, 2, 6, 7, 1, 3, 4, 5, 8, 9);
 229     }
 230
 231     {
 232         // Test using a default value and generating reverse order keys
 233         Xapian::MultiValueKeyMaker sorter;
 234         sorter.add_value(10, true, db.get_value_upper_bound(10) + '\xff');
 235
 236         enquire.set_sort_by_key(&sorter, false);
 237         Xapian::MSet mset = enquire.get_mset(0, 10);
 238         mset_expect_order(mset, 2, 6, 7, 1, 3, 4, 5, 8, 9);
 239     }
 240
 241     {
 242         // Test using a default value, generating reverse order keys, and
 243         // sorting in reverse order
 244         Xapian::MultiValueKeyMaker sorter;
 245         sorter.add_value(10, true, db.get_value_upper_bound(10) + '\xff');
 246
 247         enquire.set_sort_by_key(&sorter, true);
 248         Xapian::MSet mset = enquire.get_mset(0, 10);
 249         mset_expect_order(mset, 1, 3, 4, 5, 8, 9, 2, 6, 7);
 250     }
 251
 252     return true;
 253 }
 254
 255 class NeverUseMeKeyMaker : public Xapian::KeyMaker {
 256   public:
 257     std::string operator() (const Xapian::Document &) const
 258     {
 259         FAIL_TEST("NeverUseMeKeyMaker was called");
 260     }
 261 };
 262
 263 /// Regression test for changing away from a sorter.
 264 DEFINE_TESTCASE(changesorter1, backend && !remote) {
 265     Xapian::Enquire enquire(get_database("apitest_simpledata"));
 266     enquire.set_query(Xapian::Query("word"));
 267     NeverUseMeKeyMaker sorter;
 268
 269     enquire.set_sort_by_key(&sorter, true);
 270     enquire.set_sort_by_value(0, true);
 271     Xapian::MSet mset = enquire.get_mset(0, 25);
 272     TEST_EQUAL(mset.size(), 2); // Check that search is still doing something.
 273
 274     enquire.set_sort_by_key(&sorter, true);
 275     enquire.set_sort_by_value_then_relevance(0, true);
 276     mset = enquire.get_mset(0, 25);
 277     TEST_EQUAL(mset.size(), 2); // Check that search is still doing something.
 278
 279     enquire.set_sort_by_key(&sorter, true);
 280     enquire.set_sort_by_relevance_then_value(0, true);
 281     mset = enquire.get_mset(0, 25);
 282     TEST_EQUAL(mset.size(), 2); // Check that search is still doing something.
 283
 284     enquire.set_sort_by_key(&sorter, true);
 285     enquire.set_sort_by_relevance();
 286     mset = enquire.get_mset(0, 25);
 287     TEST_EQUAL(mset.size(), 2); // Check that search is still doing something.
 288
 289     // Check that NeverUseMeKeyMaker::operator() would actually cause a test
 290     // failure if called.
 291     try {
 292         sorter(Xapian::Document());
 293         FAIL_TEST("NeverUseMeKeyMaker::operator() didn't throw TestFail");
 294     } catch (const TestFail &) {
 295     }
 296
 297     return true;
 298 }
 299
 300 /// Regression test - an empty MultiValueSorter hung in 1.0.9 and earlier.
 301 DEFINE_TESTCASE(sortfunctorempty1, backend && !remote) {
 302     Xapian::Enquire enquire(get_database("apitest_sortrel"));
 303     enquire.set_query(Xapian::Query("woman"));
 304
 305     {
 306         int i;
 307         Xapian::MultiValueKeyMaker sorter(&i, &i);
 308
 309         enquire.set_sort_by_key(&sorter, true);
 310         Xapian::MSet mset = enquire.get_mset(0, 10);
 311         mset_expect_order(mset, 1, 2, 3, 4, 5, 6, 7, 8, 9);
 312     }
 313
 314     return true;
 315 }
 316
 317 DEFINE_TESTCASE(multivaluekeymaker1, !backend) {
 318     static const int keys[] = { 0, 1, 2, 3 };
 319     Xapian::MultiValueKeyMaker sorter(keys, keys + 4);
 320
 321     Xapian::Document doc;
 322     TEST(sorter(doc).empty());
 323
 324     doc.add_value(1, "foo");
 325     TEST_EQUAL(sorter(doc), string("\0\0foo", 5));
 326     doc.add_value(1, string("f\0o", 3));
 327     TEST_EQUAL(sorter(doc), string("\0\0f\0\xffo", 6));
 328     doc.add_value(3, "xyz");
 329     TEST_EQUAL(sorter(doc), string("\0\0f\0\xffo\0\0\0\0xyz", 13));
 330
 331     // An empty slot at the end, in reverse order, is terminated with \xff\xff
 332     sorter.add_value(4, true);
 333     TEST_EQUAL(sorter(doc), string("\0\0f\0\xffo\0\0\0\0xyz\0\0\xff\xff", 17));
 334
 335     // An empty slot at the end, in ascending order, has no effect
 336     sorter.add_value(0);
 337     TEST_EQUAL(sorter(doc), string("\0\0f\0\xffo\0\0\0\0xyz\0\0\xff\xff", 17));
 338
 339     // An empty slot at the end, with a default value
 340     sorter.add_value(0, false, "hi");
 341     TEST_EQUAL(sorter(doc), string("\0\0f\0\xffo\0\0\0\0xyz\0\0\xff\xff\0\0hi",
 342                                    21));
 343
 344     // An empty slot at the end, with a default value, in reverse sort order
 345     sorter.add_value(0, true, "hi");
 346     TEST_EQUAL(sorter(doc), string("\0\0f\0\xffo\0\0\0\0xyz\0\0\xff\xff\0\0hi"
 347                                    "\0\0\x97\x96\xff\xff", 27));
 348
 349     return true;
 350 }
 351
 352 DEFINE_TESTCASE(sortfunctorremote1, remote) {
 353     Xapian::Enquire enquire(get_database(string()));
 354     NeverUseMeKeyMaker sorter;
 355     enquire.set_query(Xapian::Query("word"));
 356     enquire.set_sort_by_key(&sorter, true);
 357     TEST_EXCEPTION(Xapian::UnimplementedError,
 358         Xapian::MSet mset = enquire.get_mset(0, 10);
 359     );
 360     return true;
 361 }
 362
 363 DEFINE_TESTCASE(replace_weights1, backend) {
 364     Xapian::Database mydb(get_database("apitest_onedoc"));
 365     Xapian::Enquire enquire(mydb);
 366     enquire.set_query(Xapian::Query("word"));
 367     Xapian::MSet mymset = enquire.get_mset(0, 10);
 368     // old_max_possible, max_attained = 0.269763689697702
 369     double old_max_possible = mymset.get_max_possible();
 370     const double new_weight = 0.125;
 371     static const double weights[] = {new_weight};
 372     mymset.replace_weights(begin(weights), end(weights));
 373     Xapian::MSetIterator i = mymset.begin();
 374     TEST(i != mymset.end());
 375     TEST_EQUAL_DOUBLE(i.get_weight(), new_weight);
 376     TEST_EQUAL_DOUBLE(mymset.get_max_attained(), new_weight);
 377     TEST_EQUAL_DOUBLE(mymset.get_max_possible(), old_max_possible);
 378     return true;
 379 }
 380
 381 DEFINE_TESTCASE(replace_weights2, backend) {
 382     Xapian::Database mydb(get_database("apitest_onedoc"));
 383     Xapian::Enquire enquire(mydb);
 384     enquire.set_query(Xapian::Query("word"));
 385     Xapian::MSet mymset = enquire.get_mset(0, 10);
 386     static const double weights[] = {1.0, 2.0};
 387     TEST_EXCEPTION(Xapian::InvalidArgumentError,
 388                    mymset.replace_weights(begin(weights), end(weights)));
 389     return true;
 390 }
 391
 392 DEFINE_TESTCASE(sort_existing_mset_by_relevance, backend) {
 393     Xapian::Database db = get_database("apitest_simpledata");
 394     Xapian::Enquire enquire(db);
 395     enquire.set_query(Xapian::Query("word"));
 396     Xapian::MSet mymset = enquire.get_mset(0, 10);
 397     // old max_possible = 1.17367567757238, max_attained = 1.04648168717725
 398     static const Xapian::docid docids[] = {*mymset[0], *mymset[1]};
 399     static const double weights[] = {1.18, 1.19};
 400     mymset.replace_weights(begin(weights), end(weights));
 401     mymset.sort_by_relevance();
 402     // The order of documents should have been reversed.
 403     int k = 1;
 404     for (Xapian::MSetIterator m = mymset.begin(); m != mymset.end();
 405          ++m, --k) {
 406         TEST_EQUAL(*m, docids[k]);
 407         TEST_EQUAL_DOUBLE(m.get_weight(), weights[k]);
 408     }
 409     TEST_EQUAL_DOUBLE(mymset.get_max_attained(), weights[1]);
 410     TEST_EQUAL_DOUBLE(mymset.get_max_possible(), weights[1]);
 411     return true;
 412 }