xapian-core/tests/api_valuestats.cc

   1 /* api_valuestats.cc: tests of the value statistics functions.
   2  *
   3  * Copyright 2008 Lemur Consulting Ltd
   4  * Copyright 2008,2009,2011 Olly Betts
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License as
   8  * published by the Free Software Foundation; either version 2 of the
   9  * License, or (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  19  * USA
  20  */
  21
  22 #include <config.h>
  23
  24 #include "api_valuestats.h"
  25
  26 #include <xapian.h>
  27 #include "testsuite.h"
  28 #include "testutils.h"
  29
  30 #include "apitest.h"
  31
  32 using namespace std;
  33
  34 // #######################################################################
  35 // # Tests start here
  36
  37 /// Test of value statistics methods.
  38 DEFINE_TESTCASE(valuestats1, writable && valuestats) {
  39     Xapian::WritableDatabase db_w = get_writable_database();
  40
  41     // Check that counts are initially zero.
  42     TEST_EQUAL(db_w.get_value_freq(0), 0);
  43     TEST_EQUAL(db_w.get_value_lower_bound(0), "");
  44     TEST_EQUAL(db_w.get_value_upper_bound(0), "");
  45     TEST_EQUAL(db_w.get_value_freq(1), 0);
  46     TEST_EQUAL(db_w.get_value_lower_bound(1), "");
  47     TEST_EQUAL(db_w.get_value_upper_bound(1), "");
  48
  49     Xapian::Document doc;
  50     doc.add_value(0, "hello");
  51
  52     // Check that statistics for the correct value slot increase when document
  53     // is added.  (Check slot 1 first, so that cache invalidation of the last
  54     // slot read also gets checked.)
  55     db_w.add_document(doc);
  56     TEST_EQUAL(db_w.get_value_freq(1), 0);
  57     TEST_EQUAL(db_w.get_value_lower_bound(1), "");
  58     TEST_EQUAL(db_w.get_value_upper_bound(1), "");
  59     TEST_EQUAL(db_w.get_value_freq(0), 1);
  60     TEST_EQUAL(db_w.get_value_lower_bound(0), "hello");
  61     TEST_EQUAL(db_w.get_value_upper_bound(0), "hello");
  62
  63     // Check that statistics work correctly when second document is added.
  64     doc = Xapian::Document();
  65     doc.add_value(0, "world");
  66     doc.add_value(1, "cheese");
  67     db_w.replace_document(2, doc);
  68     TEST_EQUAL(db_w.get_value_freq(0), 2);
  69     TEST_EQUAL(db_w.get_value_lower_bound(0), "hello");
  70     TEST_EQUAL(db_w.get_value_upper_bound(0), "world");
  71     TEST_EQUAL(db_w.get_value_freq(1), 1);
  72     TEST_EQUAL(db_w.get_value_lower_bound(1), "cheese");
  73     TEST_EQUAL(db_w.get_value_upper_bound(1), "cheese");
  74
  75     // Deleting a document affects the count, but not the bounds.
  76     db_w.delete_document(1);
  77     TEST_EQUAL(db_w.get_value_freq(1), 1);
  78     TEST_EQUAL(db_w.get_value_lower_bound(1), "cheese");
  79     TEST_EQUAL(db_w.get_value_upper_bound(1), "cheese");
  80     TEST_EQUAL(db_w.get_value_freq(0), 1);
  81     TEST_EQUAL(db_w.get_value_lower_bound(0), "hello");
  82     TEST_EQUAL(db_w.get_value_upper_bound(0), "world");
  83
  84     // Deleting all the documents returns the bounds to their original value.
  85     db_w.delete_document(2);
  86     TEST_EQUAL(db_w.get_value_freq(0), 0);
  87     TEST_EQUAL(db_w.get_value_lower_bound(0), "");
  88     TEST_EQUAL(db_w.get_value_upper_bound(0), "");
  89     TEST_EQUAL(db_w.get_value_freq(1), 0);
  90     TEST_EQUAL(db_w.get_value_lower_bound(1), "");
  91     TEST_EQUAL(db_w.get_value_upper_bound(1), "");
  92
  93     // Adding a document with a value in one of the old slots should still
  94     // end up with tight bounds on it.
  95     doc = Xapian::Document();
  96     doc.add_value(1, "newval");
  97     db_w.replace_document(2, doc);
  98     TEST_EQUAL(db_w.get_value_freq(1), 1);
  99     TEST_EQUAL(db_w.get_value_lower_bound(1), "newval");
 100     TEST_EQUAL(db_w.get_value_upper_bound(1), "newval");
 101     TEST_EQUAL(db_w.get_value_freq(0), 0);
 102     TEST_EQUAL(db_w.get_value_lower_bound(0), "");
 103     TEST_EQUAL(db_w.get_value_upper_bound(0), "");
 104
 105     return true;
 106 }
 107
 108 /// Test that value statistics stuff obeys transactions.
 109 DEFINE_TESTCASE(valuestats2, transactions && valuestats) {
 110     Xapian::WritableDatabase db_w = get_writable_database();
 111     Xapian::Database db = get_writable_database_as_database();
 112
 113     // Check that counts are initially zero.
 114     TEST_EQUAL(db_w.get_value_freq(0), 0);
 115     TEST_EQUAL(db_w.get_value_lower_bound(0), "");
 116     TEST_EQUAL(db_w.get_value_upper_bound(0), "");
 117     TEST_EQUAL(db_w.get_value_freq(1), 0);
 118     TEST_EQUAL(db_w.get_value_lower_bound(1), "");
 119     TEST_EQUAL(db_w.get_value_upper_bound(1), "");
 120     TEST_EQUAL(db.get_value_freq(0), 0);
 121     TEST_EQUAL(db.get_value_lower_bound(0), "");
 122     TEST_EQUAL(db.get_value_upper_bound(0), "");
 123     TEST_EQUAL(db.get_value_freq(1), 0);
 124     TEST_EQUAL(db.get_value_lower_bound(1), "");
 125     TEST_EQUAL(db.get_value_upper_bound(1), "");
 126
 127     Xapian::Document doc;
 128     doc.add_value(0, "hello");
 129
 130     // Check that statistics for the correct value slot increase when document
 131     // is added.  (Check slot 1 first, so that cache invalidation of the last
 132     // slot read also gets checked.)
 133     db_w.add_document(doc);
 134     TEST_EQUAL(db_w.get_value_freq(1), 0);
 135     TEST_EQUAL(db_w.get_value_lower_bound(1), "");
 136     TEST_EQUAL(db_w.get_value_upper_bound(1), "");
 137     TEST_EQUAL(db_w.get_value_freq(0), 1);
 138     TEST_EQUAL(db_w.get_value_lower_bound(0), "hello");
 139     TEST_EQUAL(db_w.get_value_upper_bound(0), "hello");
 140
 141     // The readonly database shouldn't change, though.
 142     TEST_EQUAL(db.get_value_freq(1), 0);
 143     TEST_EQUAL(db.get_value_lower_bound(1), "");
 144     TEST_EQUAL(db.get_value_upper_bound(1), "");
 145     TEST_EQUAL(db.get_value_freq(0), 0);
 146     TEST_EQUAL(db.get_value_lower_bound(0), "");
 147     TEST_EQUAL(db.get_value_upper_bound(0), "");
 148
 149     // Check that statistics work correctly when second document is added.
 150     doc = Xapian::Document();
 151     doc.add_value(0, "world");
 152     doc.add_value(1, "cheese");
 153     db_w.replace_document(2, doc);
 154     TEST_EQUAL(db_w.get_value_freq(0), 2);
 155     TEST_EQUAL(db_w.get_value_lower_bound(0), "hello");
 156     TEST_EQUAL(db_w.get_value_upper_bound(0), "world");
 157     TEST_EQUAL(db_w.get_value_freq(1), 1);
 158     TEST_EQUAL(db_w.get_value_lower_bound(1), "cheese");
 159     TEST_EQUAL(db_w.get_value_upper_bound(1), "cheese");
 160
 161     // The readonly database shouldn't change, though.
 162     TEST_EQUAL(db.get_value_freq(0), 0);
 163     TEST_EQUAL(db.get_value_lower_bound(0), "");
 164     TEST_EQUAL(db.get_value_upper_bound(0), "");
 165     TEST_EQUAL(db.get_value_freq(1), 0);
 166     TEST_EQUAL(db.get_value_lower_bound(1), "");
 167     TEST_EQUAL(db.get_value_upper_bound(1), "");
 168
 169     // Check that readonly database catches up when a commit is done.
 170     db_w.commit();
 171     TEST(db.reopen());
 172     TEST_EQUAL(db.get_value_freq(1), 1);
 173     TEST_EQUAL(db.get_value_lower_bound(1), "cheese");
 174     TEST_EQUAL(db.get_value_upper_bound(1), "cheese");
 175     TEST_EQUAL(db.get_value_freq(0), 2);
 176     TEST_EQUAL(db.get_value_lower_bound(0), "hello");
 177     TEST_EQUAL(db.get_value_upper_bound(0), "world");
 178
 179     // Deleting a document affects the count, but not the bounds.
 180     db_w.delete_document(1);
 181     TEST_EQUAL(db_w.get_value_freq(1), 1);
 182     TEST_EQUAL(db_w.get_value_lower_bound(1), "cheese");
 183     TEST_EQUAL(db_w.get_value_upper_bound(1), "cheese");
 184     TEST_EQUAL(db_w.get_value_freq(0), 1);
 185     TEST_EQUAL(db_w.get_value_lower_bound(0), "hello");
 186     TEST_EQUAL(db_w.get_value_upper_bound(0), "world");
 187
 188     // Deleting all the documents returns the bounds to their original value.
 189     db_w.delete_document(2);
 190     TEST_EQUAL(db_w.get_value_freq(0), 0);
 191     TEST_EQUAL(db_w.get_value_lower_bound(0), "");
 192     TEST_EQUAL(db_w.get_value_upper_bound(0), "");
 193     TEST_EQUAL(db_w.get_value_freq(1), 0);
 194     TEST_EQUAL(db_w.get_value_lower_bound(1), "");
 195     TEST_EQUAL(db_w.get_value_upper_bound(1), "");
 196
 197     // Adding a document with a value in one of the old slots should still
 198     // end up with tight bounds on it.
 199     doc = Xapian::Document();
 200     doc.add_value(1, "newval");
 201     db_w.replace_document(2, doc);
 202     TEST_EQUAL(db_w.get_value_freq(1), 1);
 203     TEST_EQUAL(db_w.get_value_lower_bound(1), "newval");
 204     TEST_EQUAL(db_w.get_value_upper_bound(1), "newval");
 205     TEST_EQUAL(db_w.get_value_freq(0), 0);
 206     TEST_EQUAL(db_w.get_value_lower_bound(0), "");
 207     TEST_EQUAL(db_w.get_value_upper_bound(0), "");
 208
 209     // Check that a readonly database gets the right statistics, too.
 210     db_w.commit();
 211     TEST(db.reopen());
 212     TEST_EQUAL(db.get_value_freq(0), 0);
 213     TEST_EQUAL(db.get_value_lower_bound(0), "");
 214     TEST_EQUAL(db.get_value_upper_bound(0), "");
 215     TEST_EQUAL(db.get_value_freq(1), 1);
 216     TEST_EQUAL(db.get_value_lower_bound(1), "newval");
 217     TEST_EQUAL(db.get_value_upper_bound(1), "newval");
 218
 219     return true;
 220 }
 221
 222 /// Test reading value statistics from prebuilt databases.
 223 DEFINE_TESTCASE(valuestats3, valuestats) {
 224     Xapian::Database db = get_database("apitest_simpledata");
 225
 226     TEST_EQUAL(db.get_value_freq(1), 6);
 227     TEST_EQUAL(db.get_value_lower_bound(1), "h");
 228     TEST_EQUAL(db.get_value_upper_bound(1), "n");
 229     TEST_EQUAL(db.get_value_freq(2), 6);
 230     TEST_EQUAL(db.get_value_lower_bound(2), "d");
 231     TEST_EQUAL(db.get_value_upper_bound(2), "i");
 232     TEST_EQUAL(db.get_value_freq(3), 6);
 233     TEST_EQUAL(db.get_value_lower_bound(3), " ");
 234     TEST_EQUAL(db.get_value_upper_bound(3), "s");
 235     TEST_EQUAL(db.get_value_freq(4), 6);
 236     TEST_EQUAL(db.get_value_lower_bound(4), " ");
 237     TEST_EQUAL(db.get_value_upper_bound(4), "y");
 238     TEST_EQUAL(db.get_value_freq(5), 6);
 239     TEST_EQUAL(db.get_value_lower_bound(5), "e");
 240     TEST_EQUAL(db.get_value_upper_bound(5), "p");
 241     TEST_EQUAL(db.get_value_freq(6), 6);
 242     TEST_EQUAL(db.get_value_lower_bound(6), "a");
 243     TEST_EQUAL(db.get_value_upper_bound(6), "t");
 244     TEST_EQUAL(db.get_value_freq(7), 6);
 245     TEST_EQUAL(db.get_value_lower_bound(7), " ");
 246     TEST_EQUAL(db.get_value_upper_bound(7), "r");
 247     TEST_EQUAL(db.get_value_freq(8), 6);
 248     TEST_EQUAL(db.get_value_lower_bound(8), "a");
 249     TEST_EQUAL(db.get_value_upper_bound(8), "t");
 250     TEST_EQUAL(db.get_value_freq(9), 6);
 251     TEST_EQUAL(db.get_value_lower_bound(9), " ");
 252     TEST_EQUAL(db.get_value_upper_bound(9), "n");
 253     TEST_EQUAL(db.get_value_freq(10), 6);
 254     TEST_EQUAL(db.get_value_lower_bound(10), "e");
 255     TEST_EQUAL(db.get_value_upper_bound(10), "w");
 256     TEST_EQUAL(db.get_value_freq(11), 6);
 257     TEST_EQUAL(db.get_value_lower_bound(11), "\xb9P");
 258     TEST_EQUAL(db.get_value_upper_bound(11), "\xc7\x04");
 259
 260     return true;
 261 }
 262
 263 DEFINE_TESTCASE(valuestats4, transactions && valuestats) {
 264     const size_t FLUSH_THRESHOLD = 10000;
 265     {
 266         Xapian::WritableDatabase db_w = get_writable_database();
 267         Xapian::Document doc;
 268         doc.add_value(1, "test");
 269         for (size_t i = 0; i < FLUSH_THRESHOLD; ++i) {
 270             db_w.add_document(doc);
 271         }
 272
 273         Xapian::Database db = get_writable_database_as_database();
 274         // Check that we had an automatic-commit.
 275         TEST_EQUAL(db.get_doccount(), FLUSH_THRESHOLD);
 276         // Check that the value stats are there.
 277         TEST_EQUAL(db.get_value_freq(1), FLUSH_THRESHOLD);
 278         TEST_EQUAL(db.get_value_lower_bound(1), "test");
 279         TEST_EQUAL(db.get_value_upper_bound(1), "test");
 280
 281         db_w.begin_transaction();
 282         doc.add_value(1, "umbrella");
 283         db_w.cancel_transaction();
 284     }
 285
 286     {
 287         Xapian::Database db = get_writable_database_as_database();
 288         // Check that we had an automatic-commit.
 289         TEST_EQUAL(db.get_doccount(), FLUSH_THRESHOLD);
 290         // Check that the value stats are there.
 291         TEST_EQUAL(db.get_value_freq(1), FLUSH_THRESHOLD);
 292         TEST_EQUAL(db.get_value_lower_bound(1), "test");
 293         TEST_EQUAL(db.get_value_upper_bound(1), "test");
 294     }
 295
 296     return true;
 297 }
 298
 299 /// Regression test for bug fixed in 1.1.1 which led to incorrect valuestats.
 300 DEFINE_TESTCASE(valuestats5, !backend) {
 301     Xapian::Document doc;
 302     doc.add_value(0, "zero");
 303     doc.add_value(1, "one");
 304     doc.add_value(2, "two");
 305     doc.add_value(3, "three");
 306     doc.add_value(4, "");
 307     doc.add_value(5, "five");
 308     doc.remove_value(3);
 309     doc.add_value(1, "");
 310
 311     // Check that we don't have any empty values reported.
 312     size_t c = 0;
 313     Xapian::ValueIterator v = doc.values_begin();
 314     while (v != doc.values_end()) {
 315         TEST(!(*v).empty());
 316         ++c;
 317         ++v;
 318     }
 319     TEST_EQUAL(c, 3); // 0, 2, 5
 320
 321     return true;
 322 }