xapian-core/tests/api_db.cc

   1 /* api_db.cc: tests which need a backend
   2  *
   3  * Copyright 1999,2000,2001 BrightStation PLC
   4  * Copyright 2002 Ananova Ltd
   5  * Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2011,2012,2013,2015,2016,2017 Olly Betts
   6  * Copyright 2006,2007,2008,2009 Lemur Consulting Ltd
   7  *
   8  * This program is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU General Public License as
  10  * published by the Free Software Foundation; either version 2 of the
  11  * License, or (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  21  * USA
  22  */
  23
  24 #include <config.h>
  25
  26 #include "api_db.h"
  27
  28 #include <algorithm>
  29 #include <fstream>
  30 #include <map>
  31 #include <string>
  32 #include <vector>
  33 #include "safenetdb.h" // For gai_strerror().
  34 #include "safesysstat.h" // For mkdir().
  35 #include "safeunistd.h" // For sleep().
  36
  37 #include <xapian.h>
  38
  39 #include "backendmanager.h"
  40 #include "backendmanager_local.h"
  41 #include "testsuite.h"
  42 #include "testutils.h"
  43 #include "unixcmds.h"
  44
  45 #include "apitest.h"
  46
  47 using namespace std;
  48
  49 static Xapian::Query
  50 query(const string &t)
  51 {
  52     return Xapian::Query(Xapian::Stem("english")(t));
  53 }
  54
  55 // #######################################################################
  56 // # Tests start here
  57
  58 // tests Xapian::Database::get_termfreq() and Xapian::Database::term_exists()
  59 DEFINE_TESTCASE(termstats, backend) {
  60     Xapian::Database db(get_database("apitest_simpledata"));
  61
  62     TEST(!db.term_exists("corn"));
  63     TEST_EQUAL(db.get_termfreq("corn"), 0);
  64     TEST(db.term_exists("banana"));
  65     TEST_EQUAL(db.get_termfreq("banana"), 1);
  66     TEST(db.term_exists("paragraph"));
  67     TEST_EQUAL(db.get_termfreq("paragraph"), 5);
  68
  69     return true;
  70 }
  71
  72 // Check that stub databases work.
  73 DEFINE_TESTCASE(stubdb1, backend && !inmemory && !remote) {
  74     // Only works for backends which have a path.
  75     mkdir(".stub", 0755);
  76     const char * dbpath = ".stub/stubdb1";
  77     ofstream out(dbpath);
  78     TEST(out.is_open());
  79     out << "auto ../" << get_database_path("apitest_simpledata") << endl;
  80     out.close();
  81
  82     {
  83         Xapian::Database db(dbpath, Xapian::DB_BACKEND_STUB);
  84         Xapian::Enquire enquire(db);
  85         enquire.set_query(Xapian::Query("word"));
  86         enquire.get_mset(0, 10);
  87     }
  88     {
  89         Xapian::Database db(dbpath);
  90         Xapian::Enquire enquire(db);
  91         enquire.set_query(Xapian::Query("word"));
  92         enquire.get_mset(0, 10);
  93     }
  94
  95     return true;
  96 }
  97
  98 // Check that stub databases work remotely.
  99 DEFINE_TESTCASE(stubdb2, backend && !inmemory && !remote) {
 100     // Only works for backends which have a path.
 101     mkdir(".stub", 0755);
 102     const char * dbpath = ".stub/stubdb2";
 103     ofstream out(dbpath);
 104     TEST(out.is_open());
 105     out << "remote :" << BackendManager::get_xapian_progsrv_command()
 106         << ' ' << get_database_path("apitest_simpledata") << endl;
 107     out.close();
 108
 109     try {
 110         Xapian::Database db(dbpath, Xapian::DB_BACKEND_STUB);
 111         Xapian::Enquire enquire(db);
 112         enquire.set_query(Xapian::Query("word"));
 113         enquire.get_mset(0, 10);
 114     } catch (Xapian::FeatureUnavailableError&) {
 115 #ifdef XAPIAN_HAS_REMOTE_BACKEND
 116         throw;
 117 #endif
 118     }
 119
 120     try {
 121         Xapian::Database db(dbpath);
 122         Xapian::Enquire enquire(db);
 123         enquire.set_query(Xapian::Query("word"));
 124         enquire.get_mset(0, 10);
 125     } catch (Xapian::FeatureUnavailableError&) {
 126 #ifdef XAPIAN_HAS_REMOTE_BACKEND
 127         throw;
 128 #endif
 129     }
 130
 131     out.open(dbpath);
 132     TEST(out.is_open());
 133     out << "remote" << endl;
 134     out.close();
 135
 136     // Quietly ignored prior to 1.4.1.
 137     TEST_EXCEPTION(Xapian::DatabaseOpeningError,
 138         Xapian::Database db(dbpath, Xapian::DB_BACKEND_STUB)
 139     );
 140
 141     // Quietly ignored prior to 1.4.1.
 142     TEST_EXCEPTION(Xapian::DatabaseOpeningError,
 143         Xapian::WritableDatabase db(dbpath, Xapian::DB_BACKEND_STUB)
 144     );
 145
 146 #ifdef XAPIAN_HAS_REMOTE_BACKEND
 147 # define EXPECTED_EXCEPTION Xapian::DatabaseOpeningError
 148 #else
 149 # define EXPECTED_EXCEPTION Xapian::FeatureUnavailableError
 150 #endif
 151
 152     out.open(dbpath);
 153     TEST(out.is_open());
 154     out << "remote foo" << endl;
 155     out.close();
 156
 157     // Quietly ignored prior to 1.4.1.
 158     TEST_EXCEPTION(EXPECTED_EXCEPTION,
 159         Xapian::Database db(dbpath, Xapian::DB_BACKEND_STUB)
 160     );
 161
 162     // Quietly ignored prior to 1.4.1.
 163     TEST_EXCEPTION(EXPECTED_EXCEPTION,
 164         Xapian::WritableDatabase db(dbpath, Xapian::DB_BACKEND_STUB)
 165     );
 166
 167 #ifdef XAPIAN_HAS_REMOTE_BACKEND
 168     out.open(dbpath);
 169     TEST(out.is_open());
 170     out << "remote [::1]:80" << endl;
 171     out.close();
 172
 173     try {
 174         Xapian::Database db(dbpath, Xapian::DB_BACKEND_STUB);
 175     } catch (const Xapian::NetworkError& e) {
 176         // 1.4.0 threw (Linux):
 177         //  NetworkError: Couldn't resolve host [ (context: remote:tcp([:0)) (No address associated with hostname)
 178         // 1.4.1 throws (because we don't actually support IPv6 yet) on Linux (EAI_ADDRFAMILY):
 179         //  NetworkError: Couldn't resolve host ::1 (context: remote:tcp(::1:80)) (nodename nor servname provided, or not known)
 180         // or on OS X (EAI_NONAME):
 181         //  NetworkError: Couldn't resolve host ::1 (context: remote:tcp(::1:80)) (Address family for hostname not supported)
 182         // So we test the message instead of the error string for portability.
 183         TEST(e.get_msg().find("host ::1") != string::npos);
 184     }
 185
 186     try {
 187         Xapian::WritableDatabase db(dbpath, Xapian::DB_BACKEND_STUB);
 188     } catch (const Xapian::NetworkError& e) {
 189         // 1.4.0 threw (Linux):
 190         //  NetworkError: Couldn't resolve host [ (context: remote:tcp([:0)) (No address associated with hostname)
 191         // 1.4.1 throws (because we don't actually support IPv6 yet) on Linux (EAI_ADDRFAMILY):
 192         //  NetworkError: Couldn't resolve host ::1 (context: remote:tcp(::1:80)) (nodename nor servname provided, or not known)
 193         // or on OS X (EAI_NONAME):
 194         //  NetworkError: Couldn't resolve host ::1 (context: remote:tcp(::1:80)) (Address family for hostname not supported)
 195         // So we test the message instead of the error string for portability.
 196         TEST(e.get_msg().find("host ::1") != string::npos);
 197     }
 198 #endif
 199
 200     out.open(dbpath);
 201     TEST(out.is_open());
 202     // Invalid - the port number is required.
 203     out << "remote [::1]" << endl;
 204     out.close();
 205
 206     // 1.4.0 threw:
 207     // NetworkError: Couldn't resolve host [ (context: remote:tcp([:0)) (No address associated with hostname)
 208     TEST_EXCEPTION(EXPECTED_EXCEPTION,
 209         Xapian::Database db(dbpath, Xapian::DB_BACKEND_STUB);
 210     );
 211
 212     // 1.4.0 threw:
 213     // NetworkError: Couldn't resolve host [ (context: remote:tcp([:0)) (No address associated with hostname)
 214     TEST_EXCEPTION(EXPECTED_EXCEPTION,
 215         Xapian::WritableDatabase db(dbpath, Xapian::DB_BACKEND_STUB);
 216     );
 217
 218     return true;
 219 }
 220
 221 // Regression test - bad entries were ignored after a good entry prior to 1.0.8.
 222 DEFINE_TESTCASE(stubdb3, backend && !inmemory && !remote) {
 223     // Only works for backends which have a path.
 224     mkdir(".stub", 0755);
 225     const char * dbpath = ".stub/stubdb3";
 226     ofstream out(dbpath);
 227     TEST(out.is_open());
 228     out << "auto ../" << get_database_path("apitest_simpledata") << "\n"
 229            "bad line here\n";
 230     out.close();
 231
 232     TEST_EXCEPTION(Xapian::DatabaseOpeningError,
 233         Xapian::Database db(dbpath, Xapian::DB_BACKEND_STUB));
 234
 235     TEST_EXCEPTION(Xapian::DatabaseOpeningError,
 236         Xapian::Database db(dbpath));
 237
 238     return true;
 239 }
 240
 241 // Test a stub database with just a bad entry.
 242 DEFINE_TESTCASE(stubdb4, backend && !inmemory && !remote) {
 243     // Only works for backends which have a path.
 244     mkdir(".stub", 0755);
 245     const char * dbpath = ".stub/stubdb4";
 246     ofstream out(dbpath);
 247     TEST(out.is_open());
 248     out << "bad line here\n";
 249     out.close();
 250
 251     TEST_EXCEPTION(Xapian::DatabaseOpeningError,
 252         Xapian::Database db(dbpath, Xapian::DB_BACKEND_STUB));
 253
 254     TEST_EXCEPTION(Xapian::DatabaseOpeningError,
 255         Xapian::Database db(dbpath));
 256
 257     return true;
 258 }
 259
 260 // Test a stub database with a bad entry with no spaces (prior to 1.1.0 this
 261 // was deliberately allowed, though not documented.
 262 DEFINE_TESTCASE(stubdb5, backend && !inmemory && !remote) {
 263     // Only works for backends which have a path.
 264     mkdir(".stub", 0755);
 265     const char * dbpath = ".stub/stubdb5";
 266     ofstream out(dbpath);
 267     TEST(out.is_open());
 268     out << "bad\n"
 269            "auto ../" << get_database_path("apitest_simpledata") << endl;
 270     out.close();
 271
 272     TEST_EXCEPTION(Xapian::DatabaseOpeningError,
 273         Xapian::Database db(dbpath, Xapian::DB_BACKEND_STUB));
 274
 275     TEST_EXCEPTION(Xapian::DatabaseOpeningError,
 276         Xapian::Database db(dbpath));
 277
 278     return true;
 279 }
 280
 281 // Test a stub database with an inmemory database (new feature in 1.1.0).
 282 DEFINE_TESTCASE(stubdb6, inmemory) {
 283     mkdir(".stub", 0755);
 284     const char * dbpath = ".stub/stubdb6";
 285     ofstream out(dbpath);
 286     TEST(out.is_open());
 287     out << "inmemory\n";
 288     out.close();
 289
 290     // Read-only tests:
 291     {
 292         Xapian::Database db(dbpath, Xapian::DB_BACKEND_STUB);
 293         TEST_EQUAL(db.get_doccount(), 0);
 294         Xapian::Enquire enquire(db);
 295         enquire.set_query(Xapian::Query("word"));
 296         Xapian::MSet mset = enquire.get_mset(0, 10);
 297         TEST(mset.empty());
 298     }
 299     {
 300         Xapian::Database db(dbpath);
 301         TEST_EQUAL(db.get_doccount(), 0);
 302         Xapian::Enquire enquire(db);
 303         enquire.set_query(Xapian::Query("word"));
 304         Xapian::MSet mset = enquire.get_mset(0, 10);
 305         TEST(mset.empty());
 306     }
 307
 308     // Writable tests:
 309     {
 310         Xapian::WritableDatabase db(dbpath,
 311                 Xapian::DB_OPEN|Xapian::DB_BACKEND_STUB);
 312         TEST_EQUAL(db.get_doccount(), 0);
 313         db.add_document(Xapian::Document());
 314         TEST_EQUAL(db.get_doccount(), 1);
 315     }
 316     {
 317         Xapian::WritableDatabase db(dbpath,
 318                 Xapian::DB_OPEN|Xapian::DB_BACKEND_STUB);
 319         TEST_EQUAL(db.get_doccount(), 0);
 320         db.add_document(Xapian::Document());
 321         TEST_EQUAL(db.get_doccount(), 1);
 322     }
 323
 324     return true;
 325 }
 326
 327 /// Test error running Database::check() on a stub database.
 328 // Regression test - in 1.4.3 and earlier this threw
 329 // Xapian::DatabaseError.
 330 DEFINE_TESTCASE(stubdb8, inmemory) {
 331     mkdir(".stub", 0755);
 332     const char * dbpath = ".stub/stubdb8";
 333     ofstream out(dbpath);
 334     TEST(out.is_open());
 335     out << "inmemory\n";
 336     out.close();
 337
 338     try {
 339         Xapian::Database::check(dbpath);
 340         FAIL_TEST("Managed to check inmemory stub");
 341     } catch (const Xapian::DatabaseOpeningError & e) {
 342         // Check the message is appropriate.
 343         TEST_STRINGS_EQUAL(e.get_msg(),
 344                            "File is not a Xapian database or database table");
 345     }
 346     return true;
 347 }
 348
 349 class GrepMatchDecider : public Xapian::MatchDecider {
 350     string needle;
 351   public:
 352     explicit GrepMatchDecider(const string& needle_)
 353         : needle(needle_) {}
 354
 355     bool operator()(const Xapian::Document &doc) const {
 356         // Note that this is not recommended usage of get_data()
 357         return doc.get_data().find(needle) != string::npos;
 358     }
 359 };
 360
 361 // Test Xapian::MatchDecider functor.
 362 DEFINE_TESTCASE(matchdecider1, backend && !remote) {
 363     Xapian::Database db(get_database("apitest_simpledata"));
 364     Xapian::Enquire enquire(db);
 365     enquire.set_query(Xapian::Query("this"));
 366
 367     GrepMatchDecider myfunctor("This is");
 368
 369     Xapian::MSet mymset = enquire.get_mset(0, 100, 0, &myfunctor);
 370
 371     vector<bool> docid_checked(db.get_lastdocid());
 372
 373     // Check that we get the expected number of matches, and that they
 374     // satisfy the condition.
 375     Xapian::MSetIterator i = mymset.begin();
 376     TEST(i != mymset.end());
 377     TEST_EQUAL(mymset.size(), 3);
 378     TEST_EQUAL(mymset.get_matches_lower_bound(), 3);
 379     TEST_EQUAL(mymset.get_matches_upper_bound(), 3);
 380     TEST_EQUAL(mymset.get_matches_estimated(), 3);
 381     TEST_EQUAL(mymset.get_uncollapsed_matches_lower_bound(), 3);
 382     TEST_EQUAL(mymset.get_uncollapsed_matches_upper_bound(), 3);
 383     TEST_EQUAL(mymset.get_uncollapsed_matches_estimated(), 3);
 384     for ( ; i != mymset.end(); ++i) {
 385         const Xapian::Document doc(i.get_document());
 386         TEST(myfunctor(doc));
 387         docid_checked[*i] = true;
 388     }
 389
 390     // Check that there are some documents which aren't accepted by the match
 391     // decider.
 392     mymset = enquire.get_mset(0, 100);
 393     TEST(mymset.size() > 3);
 394
 395     // Check that the bounds are appropriate even if we don't ask for any
 396     // actual matches.
 397     mymset = enquire.get_mset(0, 0, 0, &myfunctor);
 398     TEST_EQUAL(mymset.size(), 0);
 399     TEST_EQUAL(mymset.get_matches_lower_bound(), 0);
 400     TEST_EQUAL(mymset.get_matches_upper_bound(), 6);
 401     TEST_REL(mymset.get_matches_estimated(),>,0);
 402     TEST_REL(mymset.get_matches_estimated(),<=,6);
 403     TEST_EQUAL(mymset.get_uncollapsed_matches_lower_bound(), 0);
 404     TEST_EQUAL(mymset.get_uncollapsed_matches_upper_bound(), 6);
 405     TEST_REL(mymset.get_uncollapsed_matches_estimated(),>,0);
 406     TEST_REL(mymset.get_uncollapsed_matches_estimated(),<=,6);
 407
 408     // Check that the bounds are appropriate if we ask for only one hit.
 409     // (Regression test - until SVN 10256, we didn't reduce the lower_bound
 410     // appropriately, and returned 6 here.)
 411     mymset = enquire.get_mset(0, 1, 0, &myfunctor);
 412     TEST_EQUAL(mymset.size(), 1);
 413     TEST_REL(mymset.get_matches_lower_bound(),>=,1);
 414     TEST_REL(mymset.get_matches_lower_bound(),<=,3);
 415     TEST_REL(mymset.get_matches_upper_bound(),>=,3);
 416     TEST_REL(mymset.get_matches_upper_bound(),<=,6);
 417     TEST_REL(mymset.get_matches_estimated(),>,0);
 418     TEST_REL(mymset.get_matches_estimated(),<=,6);
 419     TEST_REL(mymset.get_uncollapsed_matches_lower_bound(),>=,1);
 420     TEST_REL(mymset.get_uncollapsed_matches_lower_bound(),<=,3);
 421     TEST_REL(mymset.get_uncollapsed_matches_upper_bound(),>=,3);
 422     TEST_REL(mymset.get_uncollapsed_matches_upper_bound(),<=,6);
 423     TEST_REL(mymset.get_uncollapsed_matches_estimated(),>,0);
 424     TEST_REL(mymset.get_uncollapsed_matches_estimated(),<=,6);
 425
 426     // Check that the other documents don't satisfy the condition.
 427     for (Xapian::docid did = 1; did < docid_checked.size(); ++did) {
 428         if (!docid_checked[did]) {
 429             TEST(!myfunctor(db.get_document(did)));
 430         }
 431     }
 432
 433     // Check that the bounds are appropriate if a collapse key is used.
 434     // Use a value which is never set so we don't actually discard anything.
 435     enquire.set_collapse_key(99);
 436     mymset = enquire.get_mset(0, 1, 0, &myfunctor);
 437     TEST_EQUAL(mymset.size(), 1);
 438     TEST_REL(mymset.get_matches_lower_bound(),>=,1);
 439     TEST_REL(mymset.get_matches_lower_bound(),<=,3);
 440     TEST_REL(mymset.get_matches_upper_bound(),>=,3);
 441     TEST_REL(mymset.get_matches_upper_bound(),<=,6);
 442     TEST_REL(mymset.get_matches_estimated(),>,0);
 443     TEST_REL(mymset.get_matches_estimated(),<=,6);
 444     TEST_REL(mymset.get_uncollapsed_matches_lower_bound(),>=,1);
 445     TEST_REL(mymset.get_uncollapsed_matches_lower_bound(),<=,3);
 446     TEST_REL(mymset.get_uncollapsed_matches_upper_bound(),>=,3);
 447     TEST_REL(mymset.get_uncollapsed_matches_upper_bound(),<=,6);
 448     TEST_REL(mymset.get_uncollapsed_matches_estimated(),>,0);
 449     TEST_REL(mymset.get_uncollapsed_matches_estimated(),<=,6);
 450
 451     // Check that the bounds are appropriate if a percentage cutoff is in
 452     // use.  Set a 1% threshold so we don't actually discard anything.
 453     enquire.set_collapse_key(Xapian::BAD_VALUENO);
 454     enquire.set_cutoff(1);
 455     mymset = enquire.get_mset(0, 1, 0, &myfunctor);
 456     TEST_EQUAL(mymset.size(), 1);
 457     TEST_REL(mymset.get_matches_lower_bound(),>=,1);
 458     TEST_REL(mymset.get_matches_lower_bound(),<=,3);
 459     TEST_REL(mymset.get_matches_upper_bound(),>=,3);
 460     TEST_REL(mymset.get_matches_upper_bound(),<=,6);
 461     TEST_REL(mymset.get_matches_estimated(),>,0);
 462     TEST_REL(mymset.get_matches_estimated(),<=,6);
 463     TEST_REL(mymset.get_uncollapsed_matches_lower_bound(),>=,1);
 464     TEST_REL(mymset.get_uncollapsed_matches_lower_bound(),<=,3);
 465     TEST_REL(mymset.get_uncollapsed_matches_upper_bound(),>=,3);
 466     TEST_REL(mymset.get_uncollapsed_matches_upper_bound(),<=,6);
 467     TEST_REL(mymset.get_uncollapsed_matches_estimated(),>,0);
 468     TEST_REL(mymset.get_uncollapsed_matches_estimated(),<=,6);
 469
 470     // And now with both a collapse key and percentage cutoff.
 471     enquire.set_collapse_key(99);
 472     mymset = enquire.get_mset(0, 1, 0, &myfunctor);
 473     TEST_EQUAL(mymset.size(), 1);
 474     TEST_REL(mymset.get_matches_lower_bound(),>=,1);
 475     TEST_REL(mymset.get_matches_lower_bound(),<=,3);
 476     TEST_REL(mymset.get_matches_upper_bound(),>=,3);
 477     TEST_REL(mymset.get_matches_upper_bound(),<=,6);
 478     TEST_REL(mymset.get_matches_estimated(),>,0);
 479     TEST_REL(mymset.get_matches_estimated(),<=,6);
 480     TEST_REL(mymset.get_uncollapsed_matches_lower_bound(),>=,1);
 481     TEST_REL(mymset.get_uncollapsed_matches_lower_bound(),<=,3);
 482     TEST_REL(mymset.get_uncollapsed_matches_upper_bound(),>=,3);
 483     TEST_REL(mymset.get_uncollapsed_matches_upper_bound(),<=,6);
 484     TEST_REL(mymset.get_uncollapsed_matches_estimated(),>,0);
 485     TEST_REL(mymset.get_uncollapsed_matches_estimated(),<=,6);
 486
 487     return true;
 488 }
 489
 490 // Test Xapian::MatchDecider functor used as a match spy.
 491 DEFINE_TESTCASE(matchdecider2, backend && !remote) {
 492     Xapian::Database db(get_database("apitest_simpledata"));
 493     Xapian::Enquire enquire(db);
 494     enquire.set_query(Xapian::Query("this"));
 495
 496     GrepMatchDecider myfunctor("This is");
 497
 498     Xapian::MSet mymset = enquire.get_mset(0, 100, 0, NULL, &myfunctor);
 499
 500     vector<bool> docid_checked(db.get_lastdocid());
 501
 502     // Check that we get the expected number of matches, and that they
 503     // satisfy the condition.
 504     Xapian::MSetIterator i = mymset.begin();
 505     TEST(i != mymset.end());
 506     TEST_EQUAL(mymset.size(), 3);
 507     for ( ; i != mymset.end(); ++i) {
 508         const Xapian::Document doc(i.get_document());
 509         TEST(myfunctor(doc));
 510         docid_checked[*i] = true;
 511     }
 512
 513     // Check that the other documents don't satisfy the condition.
 514     for (Xapian::docid did = 1; did < docid_checked.size(); ++did) {
 515         if (!docid_checked[did]) {
 516             TEST(!myfunctor(db.get_document(did)));
 517         }
 518     }
 519
 520     return true;
 521 }
 522
 523 // Regression test for lower bound using functor, sorting and collapsing.
 524 DEFINE_TESTCASE(matchdecider3, backend && !remote) {
 525     Xapian::Database db(get_database("etext"));
 526     Xapian::Enquire enquire(db);
 527     enquire.set_query(Xapian::Query(""));
 528     enquire.set_collapse_key(12);
 529     enquire.set_sort_by_value(11, true);
 530
 531     GrepMatchDecider myfunctor("We produce");
 532
 533     Xapian::MSet mset1 = enquire.get_mset(0, 2, 0, NULL, &myfunctor);
 534     Xapian::MSet mset2 = enquire.get_mset(0, 1000, 0, NULL, &myfunctor);
 535
 536     // mset2 should contain all the hits, so the statistics should be exact.
 537     TEST_EQUAL(mset2.get_matches_estimated(), mset2.size());
 538     TEST_EQUAL(mset2.get_matches_lower_bound(), mset2.get_matches_estimated());
 539     TEST_EQUAL(mset2.get_matches_estimated(), mset2.get_matches_upper_bound());
 540
 541     TEST_REL(mset2.get_uncollapsed_matches_lower_bound(),<=,mset2.get_uncollapsed_matches_estimated());
 542     TEST_REL(mset2.get_uncollapsed_matches_estimated(),<=,mset2.get_uncollapsed_matches_upper_bound());
 543
 544     // Check that the lower bound in mset1 is not greater than the known
 545     // number of hits.  This failed until revision 10811.
 546     TEST_REL(mset1.get_matches_lower_bound(),<=,mset2.size());
 547
 548     // Check that the bounds for mset1 make sense.
 549     TEST_REL(mset1.get_matches_lower_bound(),<=,mset1.get_matches_estimated());
 550     TEST_REL(mset1.get_matches_estimated(),<=,mset1.get_matches_upper_bound());
 551     TEST_REL(mset1.size(),<=,mset1.get_matches_upper_bound());
 552
 553     TEST_REL(mset1.get_uncollapsed_matches_lower_bound(),<=,mset1.get_uncollapsed_matches_estimated());
 554     TEST_REL(mset1.get_uncollapsed_matches_estimated(),<=,mset1.get_uncollapsed_matches_upper_bound());
 555
 556     // The uncollapsed match would match all documents but the one the
 557     // matchdecider rejects.
 558     TEST_REL(mset1.get_uncollapsed_matches_upper_bound(),>=,db.get_doccount() - 1);
 559     TEST_REL(mset1.get_uncollapsed_matches_upper_bound(),<=,db.get_doccount());
 560     TEST_REL(mset2.get_uncollapsed_matches_upper_bound(),>=,db.get_doccount() - 1);
 561     TEST_REL(mset2.get_uncollapsed_matches_upper_bound(),<=,db.get_doccount());
 562
 563     return true;
 564 }
 565
 566 // tests that mset iterators on msets compare correctly.
 567 DEFINE_TESTCASE(msetiterator1, backend) {
 568     Xapian::Enquire enquire(get_database("apitest_simpledata"));
 569     enquire.set_query(Xapian::Query("this"));
 570     Xapian::MSet mymset = enquire.get_mset(0, 2);
 571
 572     Xapian::MSetIterator j;
 573     j = mymset.begin();
 574     Xapian::MSetIterator k = mymset.end();
 575     Xapian::MSetIterator l(j);
 576     Xapian::MSetIterator m(k);
 577     Xapian::MSetIterator n = mymset.begin();
 578     Xapian::MSetIterator o = mymset.begin();
 579     TEST_NOT_EQUAL(j, k);
 580     TEST_NOT_EQUAL(l, m);
 581     TEST_EQUAL(k, m);
 582     TEST_EQUAL(j, l);
 583     TEST_EQUAL(j, j);
 584     TEST_EQUAL(k, k);
 585
 586     k = j;
 587     TEST_EQUAL(j, k);
 588     TEST_EQUAL(j, o);
 589     k++;
 590     TEST_NOT_EQUAL(j, k);
 591     TEST_NOT_EQUAL(k, l);
 592     TEST_NOT_EQUAL(k, m);
 593     TEST_NOT_EQUAL(k, o);
 594     o++;
 595     TEST_EQUAL(k, o);
 596     k++;
 597     TEST_NOT_EQUAL(j, k);
 598     TEST_NOT_EQUAL(k, l);
 599     TEST_EQUAL(k, m);
 600     TEST_EQUAL(n, l);
 601
 602     n = m;
 603     TEST_NOT_EQUAL(n, l);
 604     TEST_EQUAL(n, m);
 605     TEST_NOT_EQUAL(n, mymset.begin());
 606     TEST_EQUAL(n, mymset.end());
 607
 608     return true;
 609 }
 610
 611 // tests that mset iterators on empty msets compare equal.
 612 DEFINE_TESTCASE(msetiterator2, backend) {
 613     Xapian::Enquire enquire(get_database("apitest_simpledata"));
 614     enquire.set_query(Xapian::Query("this"));
 615     Xapian::MSet mymset = enquire.get_mset(0, 0);
 616
 617     Xapian::MSetIterator j = mymset.begin();
 618     Xapian::MSetIterator k = mymset.end();
 619     Xapian::MSetIterator l(j);
 620     Xapian::MSetIterator m(k);
 621     TEST_EQUAL(j, k);
 622     TEST_EQUAL(l, m);
 623     TEST_EQUAL(k, m);
 624     TEST_EQUAL(j, l);
 625     TEST_EQUAL(j, j);
 626     TEST_EQUAL(k, k);
 627
 628     return true;
 629 }
 630
 631 // tests that begin().get_document() works when first != 0
 632 DEFINE_TESTCASE(msetiterator3, backend) {
 633     Xapian::Database mydb(get_database("apitest_simpledata"));
 634     Xapian::Enquire enquire(mydb);
 635     enquire.set_query(Xapian::Query("this"));
 636
 637     Xapian::MSet mymset = enquire.get_mset(2, 10);
 638
 639     TEST(!mymset.empty());
 640     Xapian::Document doc(mymset.begin().get_document());
 641     TEST(!doc.get_data().empty());
 642
 643     return true;
 644 }
 645
 646 // tests that eset iterators on empty esets compare equal.
 647 DEFINE_TESTCASE(esetiterator1, backend) {
 648     Xapian::Enquire enquire(get_database("apitest_simpledata"));
 649     enquire.set_query(Xapian::Query("this"));
 650
 651     Xapian::MSet mymset = enquire.get_mset(0, 10);
 652     TEST(mymset.size() >= 2);
 653
 654     Xapian::RSet myrset;
 655     Xapian::MSetIterator i = mymset.begin();
 656     myrset.add_document(*i);
 657     myrset.add_document(*(++i));
 658
 659     Xapian::ESet myeset = enquire.get_eset(2, myrset);
 660     Xapian::ESetIterator j;
 661     j = myeset.begin();
 662     Xapian::ESetIterator k = myeset.end();
 663     Xapian::ESetIterator l(j);
 664     Xapian::ESetIterator m(k);
 665     Xapian::ESetIterator n = myeset.begin();
 666
 667     TEST_NOT_EQUAL(j, k);
 668     TEST_NOT_EQUAL(l, m);
 669     TEST_EQUAL(k, m);
 670     TEST_EQUAL(j, l);
 671     TEST_EQUAL(j, j);
 672     TEST_EQUAL(k, k);
 673
 674     k = j;
 675     TEST_EQUAL(j, k);
 676     k++;
 677     TEST_NOT_EQUAL(j, k);
 678     TEST_NOT_EQUAL(k, l);
 679     TEST_NOT_EQUAL(k, m);
 680     k++;
 681     TEST_NOT_EQUAL(j, k);
 682     TEST_NOT_EQUAL(k, l);
 683     TEST_EQUAL(k, m);
 684     TEST_EQUAL(n, l);
 685
 686     n = m;
 687     TEST_NOT_EQUAL(n, l);
 688     TEST_EQUAL(n, m);
 689     TEST_NOT_EQUAL(n, myeset.begin());
 690     TEST_EQUAL(n, myeset.end());
 691
 692     return true;
 693 }
 694
 695 // tests that eset iterators on empty esets compare equal.
 696 DEFINE_TESTCASE(esetiterator2, backend) {
 697     Xapian::Enquire enquire(get_database("apitest_simpledata"));
 698     enquire.set_query(Xapian::Query("this"));
 699
 700     Xapian::MSet mymset = enquire.get_mset(0, 10);
 701     TEST(mymset.size() >= 2);
 702
 703     Xapian::RSet myrset;
 704     Xapian::MSetIterator i = mymset.begin();
 705     myrset.add_document(*i);
 706     myrset.add_document(*(++i));
 707
 708     Xapian::ESet myeset = enquire.get_eset(0, myrset);
 709     Xapian::ESetIterator j = myeset.begin();
 710     Xapian::ESetIterator k = myeset.end();
 711     Xapian::ESetIterator l(j);
 712     Xapian::ESetIterator m(k);
 713     TEST_EQUAL(j, k);
 714     TEST_EQUAL(l, m);
 715     TEST_EQUAL(k, m);
 716     TEST_EQUAL(j, l);
 717     TEST_EQUAL(j, j);
 718     TEST_EQUAL(k, k);
 719
 720     return true;
 721 }
 722
 723 // tests the collapse-on-key
 724 DEFINE_TESTCASE(collapsekey1, backend) {
 725     Xapian::Enquire enquire(get_database("apitest_simpledata"));
 726     enquire.set_query(Xapian::Query("this"));
 727
 728     Xapian::MSet mymset1 = enquire.get_mset(0, 100);
 729     Xapian::doccount mymsize1 = mymset1.size();
 730
 731     for (Xapian::valueno value_no = 1; value_no < 7; ++value_no) {
 732         enquire.set_collapse_key(value_no);
 733         Xapian::MSet mymset = enquire.get_mset(0, 100);
 734
 735         TEST_AND_EXPLAIN(mymsize1 > mymset.size(),
 736                          "Had no fewer items when performing collapse: don't know whether it worked.");
 737
 738         map<string, Xapian::docid> values;
 739         Xapian::MSetIterator i = mymset.begin();
 740         for ( ; i != mymset.end(); ++i) {
 741             string value = i.get_document().get_value(value_no);
 742             TEST(values[value] == 0 || value.empty());
 743             values[value] = *i;
 744         }
 745     }
 746
 747     return true;
 748 }
 749
 750 // tests that collapse-on-key modifies the predicted bounds for the number of
 751 // matches appropriately.
 752 DEFINE_TESTCASE(collapsekey2, backend) {
 753     SKIP_TEST("Don't have a suitable database currently");
 754     // FIXME: this needs an appropriate database creating, but that's quite
 755     // subtle to do it seems.
 756     Xapian::Enquire enquire(get_database("apitest_simpledata2"));
 757     enquire.set_query(Xapian::Query("this"));
 758
 759     Xapian::MSet mset1 = enquire.get_mset(0, 1);
 760
 761     // Test that if no duplicates are found, then the upper bound remains
 762     // unchanged and the lower bound drops.
 763     {
 764         enquire.set_query(Xapian::Query("this"));
 765         Xapian::valueno value_no = 3;
 766         enquire.set_collapse_key(value_no);
 767         Xapian::MSet mset = enquire.get_mset(0, 1);
 768
 769         TEST_REL(mset.get_matches_lower_bound(),<,mset1.get_matches_lower_bound());
 770         TEST_EQUAL(mset.get_matches_upper_bound(), mset1.get_matches_upper_bound());
 771     }
 772
 773     return true;
 774 }
 775
 776 // tests that collapse-on-key modifies the predicted bounds for the number of
 777 // matches appropriately.
 778 DEFINE_TESTCASE(collapsekey3, backend) {
 779     Xapian::Enquire enquire(get_database("apitest_simpledata"));
 780     enquire.set_query(Xapian::Query("this"));
 781
 782     Xapian::MSet mymset1 = enquire.get_mset(0, 3);
 783
 784     for (Xapian::valueno value_no = 1; value_no < 7; ++value_no) {
 785         enquire.set_collapse_key(value_no);
 786         Xapian::MSet mymset = enquire.get_mset(0, 3);
 787
 788         TEST_AND_EXPLAIN(mymset1.get_matches_lower_bound() > mymset.get_matches_lower_bound(),
 789                          "Lower bound was not lower when performing collapse: don't know whether it worked.");
 790         TEST_AND_EXPLAIN(mymset1.get_matches_upper_bound() > mymset.get_matches_upper_bound(),
 791                          "Upper bound was not lower when performing collapse: don't know whether it worked.");
 792
 793         map<string, Xapian::docid> values;
 794         Xapian::MSetIterator i = mymset.begin();
 795         for ( ; i != mymset.end(); ++i) {
 796             string value = i.get_document().get_value(value_no);
 797             TEST(values[value] == 0 || value.empty());
 798             values[value] = *i;
 799         }
 800     }
 801
 802     // Test that if the collapse value is always empty, then the upper bound
 803     // remains unchanged, and the lower bound is the same or lower (it can be
 804     // lower because the matcher counts the number of documents with empty
 805     // collapse keys, but may have rejected a document because its weight is
 806     // too low for the proto-MSet before it even looks at its collapse key).
 807     {
 808         Xapian::valueno value_no = 1000;
 809         enquire.set_collapse_key(value_no);
 810         Xapian::MSet mymset = enquire.get_mset(0, 3);
 811
 812         TEST(mymset.get_matches_lower_bound() <= mymset1.get_matches_lower_bound());
 813         TEST_EQUAL(mymset.get_matches_upper_bound(), mymset1.get_matches_upper_bound());
 814
 815         map<string, Xapian::docid> values;
 816         Xapian::MSetIterator i = mymset.begin();
 817         for ( ; i != mymset.end(); ++i) {
 818             string value = i.get_document().get_value(value_no);
 819             TEST(values[value] == 0 || value.empty());
 820             values[value] = *i;
 821         }
 822     }
 823
 824     return true;
 825 }
 826
 827 // tests that collapse-on-key modifies the predicted bounds for the number of
 828 // matches appropriately even when no results are requested.
 829 DEFINE_TESTCASE(collapsekey4, backend) {
 830     Xapian::Enquire enquire(get_database("apitest_simpledata"));
 831     enquire.set_query(Xapian::Query("this"));
 832
 833     Xapian::MSet mymset1 = enquire.get_mset(0, 0);
 834
 835     for (Xapian::valueno value_no = 1; value_no < 7; ++value_no) {
 836         enquire.set_collapse_key(value_no);
 837         Xapian::MSet mymset = enquire.get_mset(0, 0);
 838
 839         TEST_AND_EXPLAIN(mymset.get_matches_lower_bound() == 1,
 840                          "Lower bound was not 1 when performing collapse but not asking for any results.");
 841         TEST_AND_EXPLAIN(mymset1.get_matches_upper_bound() == mymset.get_matches_upper_bound(),
 842                          "Upper bound was changed when performing collapse but not asking for any results.");
 843
 844         map<string, Xapian::docid> values;
 845         Xapian::MSetIterator i = mymset.begin();
 846         for ( ; i != mymset.end(); ++i) {
 847             string value = i.get_document().get_value(value_no);
 848             TEST(values[value] == 0 || value.empty());
 849             values[value] = *i;
 850         }
 851     }
 852
 853     return true;
 854 }
 855
 856 // test for keepalives
 857 DEFINE_TESTCASE(keepalive1, remote) {
 858     Xapian::Database db(get_remote_database("apitest_simpledata", 5000));
 859
 860     /* Test that keep-alives work */
 861     for (int i = 0; i < 10; ++i) {
 862         sleep(2);
 863         db.keep_alive();
 864     }
 865     Xapian::Enquire enquire(db);
 866     enquire.set_query(Xapian::Query("word"));
 867     enquire.get_mset(0, 10);
 868
 869     /* Test that things break without keepalives */
 870     sleep(10);
 871     enquire.set_query(Xapian::Query("word"));
 872     TEST_EXCEPTION(Xapian::NetworkError,
 873                    enquire.get_mset(0, 10));
 874
 875     return true;
 876 }
 877
 878 // test that iterating through all terms in a database works.
 879 DEFINE_TESTCASE(allterms1, backend) {
 880     Xapian::Database db(get_database("apitest_allterms"));
 881     Xapian::TermIterator ati = db.allterms_begin();
 882     TEST(ati != db.allterms_end());
 883     TEST_EQUAL(*ati, "one");
 884     TEST_EQUAL(ati.get_termfreq(), 1);
 885
 886     ati++;
 887     TEST(ati != db.allterms_end());
 888     if (verbose) {
 889         tout << "*ati = '" << *ati << "'\n";
 890         tout << "*ati.length = '" << (*ati).length() << "'\n";
 891         tout << "*ati == \"one\" = " << (*ati == "one") << "\n";
 892         tout << "*ati[3] = " << ((*ati)[3]) << "\n";
 893         tout << "*ati = '" << *ati << "'\n";
 894     }
 895     TEST(*ati == "three");
 896     TEST(ati.get_termfreq() == 3);
 897
 898     ++ati;
 899     TEST(ati != db.allterms_end());
 900     TEST(*ati == "two");
 901     TEST(ati.get_termfreq() == 2);
 902
 903     ati++;
 904     TEST(ati == db.allterms_end());
 905
 906     return true;
 907 }
 908
 909 // test that iterating through all terms in two databases works.
 910 DEFINE_TESTCASE(allterms2, backend) {
 911     Xapian::Database db;
 912     db.add_database(get_database("apitest_allterms"));
 913     db.add_database(get_database("apitest_allterms2"));
 914     Xapian::TermIterator ati = db.allterms_begin();
 915
 916     TEST(ati != db.allterms_end());
 917     TEST(*ati == "five");
 918     TEST(ati.get_termfreq() == 2);
 919     ati++;
 920
 921     TEST(ati != db.allterms_end());
 922     TEST(*ati == "four");
 923     TEST(ati.get_termfreq() == 1);
 924
 925     ati++;
 926     TEST(ati != db.allterms_end());
 927     TEST(*ati == "one");
 928     TEST(ati.get_termfreq() == 1);
 929
 930     ++ati;
 931     TEST(ati != db.allterms_end());
 932     TEST(*ati == "six");
 933     TEST(ati.get_termfreq() == 3);
 934
 935     ati++;
 936     TEST(ati != db.allterms_end());
 937     TEST(*ati == "three");
 938     TEST(ati.get_termfreq() == 3);
 939
 940     ati++;
 941     TEST(ati != db.allterms_end());
 942     TEST(*ati == "two");
 943     TEST(ati.get_termfreq() == 2);
 944
 945     ati++;
 946     TEST(ati == db.allterms_end());
 947
 948     return true;
 949 }
 950
 951 // test that skip_to sets at_end (regression test)
 952 DEFINE_TESTCASE(allterms3, backend) {
 953     Xapian::Database db;
 954     db.add_database(get_database("apitest_allterms"));
 955     Xapian::TermIterator ati = db.allterms_begin();
 956
 957     ati.skip_to(string("zzzzzz"));
 958     TEST(ati == db.allterms_end());
 959
 960     return true;
 961 }
 962
 963 // test that next ignores extra entries due to long posting lists being
 964 // chunked (regression test for quartz)
 965 DEFINE_TESTCASE(allterms4, backend) {
 966     // apitest_allterms4 contains 682 documents each containing just the word
 967     // "foo".  682 was the magic number which started to cause Quartz problems.
 968     Xapian::Database db = get_database("apitest_allterms4");
 969
 970     Xapian::TermIterator i = db.allterms_begin();
 971     TEST(i != db.allterms_end());
 972     TEST(*i == "foo");
 973     TEST(i.get_termfreq() == 682);
 974     ++i;
 975     TEST(i == db.allterms_end());
 976
 977     return true;
 978 }
 979
 980 // test that skip_to with an exact match sets the current term (regression test
 981 // for quartz)
 982 DEFINE_TESTCASE(allterms5, backend) {
 983     Xapian::Database db;
 984     db.add_database(get_database("apitest_allterms"));
 985     Xapian::TermIterator ati = db.allterms_begin();
 986     ati.skip_to("three");
 987     TEST(ati != db.allterms_end());
 988     TEST_EQUAL(*ati, "three");
 989
 990     return true;
 991 }
 992
 993 // test allterms iterators with prefixes
 994 DEFINE_TESTCASE(allterms6, backend) {
 995     Xapian::Database db;
 996     db.add_database(get_database("apitest_allterms"));
 997     db.add_database(get_database("apitest_allterms2"));
 998
 999     Xapian::TermIterator ati = db.allterms_begin("three");
1000     TEST(ati != db.allterms_end("three"));
1001     TEST_EQUAL(*ati, "three");
1002     ati.skip_to("three");
1003     TEST(ati != db.allterms_end("three"));
1004     TEST_EQUAL(*ati, "three");
1005     ati++;
1006     TEST(ati == db.allterms_end("three"));
1007
1008     ati = db.allterms_begin("thre");
1009     TEST(ati != db.allterms_end("thre"));
1010     TEST_EQUAL(*ati, "three");
1011     ati.skip_to("three");
1012     TEST(ati != db.allterms_end("thre"));
1013     TEST_EQUAL(*ati, "three");
1014     ati++;
1015     TEST(ati == db.allterms_end("thre"));
1016
1017     ati = db.allterms_begin("f");
1018     TEST(ati != db.allterms_end("f"));
1019     TEST_EQUAL(*ati, "five");
1020     TEST(ati != db.allterms_end("f"));
1021     ati.skip_to("three");
1022     TEST(ati == db.allterms_end("f"));
1023
1024     ati = db.allterms_begin("f");
1025     TEST(ati != db.allterms_end("f"));
1026     TEST_EQUAL(*ati, "five");
1027     ati++;
1028     TEST(ati != db.allterms_end("f"));
1029     TEST_EQUAL(*ati, "four");
1030     ati++;
1031     TEST(ati == db.allterms_end("f"));
1032
1033     ati = db.allterms_begin("absent");
1034     TEST(ati == db.allterms_end("absent"));
1035
1036     return true;
1037 }
1038
1039 // test that searching for a term with a special characters in it works
1040 DEFINE_TESTCASE(specialterms1, backend) {
1041     Xapian::Enquire enquire(get_database("apitest_space"));
1042     Xapian::MSet mymset;
1043     Xapian::doccount count;
1044     Xapian::MSetIterator m;
1045     Xapian::Stem stemmer("english");
1046
1047     enquire.set_query(stemmer("new\nline"));
1048     mymset = enquire.get_mset(0, 10);
1049     TEST_MSET_SIZE(mymset, 1);
1050     count = 0;
1051     for (m = mymset.begin(); m != mymset.end(); ++m) ++count;
1052     TEST_EQUAL(count, 1);
1053
1054     for (Xapian::valueno value_no = 0; value_no < 7; ++value_no) {
1055         string value = mymset.begin().get_document().get_value(value_no);
1056         TEST_NOT_EQUAL(value, "");
1057         if (value_no == 0) {
1058             TEST(value.size() > 263);
1059             TEST_EQUAL(static_cast<unsigned char>(value[262]), 255);
1060             for (int k = 0; k < 256; ++k) {
1061                 TEST_EQUAL(static_cast<unsigned char>(value[k + 7]), k);
1062             }
1063         }
1064     }
1065
1066     enquire.set_query(stemmer(string("big\0zero", 8)));
1067     mymset = enquire.get_mset(0, 10);
1068     TEST_MSET_SIZE(mymset, 1);
1069     count = 0;
1070     for (m = mymset.begin(); m != mymset.end(); ++m) ++count;
1071     TEST_EQUAL(count, 1);
1072
1073     return true;
1074 }
1075
1076 // test that terms with a special characters in appear correctly when iterating
1077 // allterms
1078 DEFINE_TESTCASE(specialterms2, backend) {
1079     Xapian::Database db(get_database("apitest_space"));
1080
1081     // Check the terms are all as expected (after stemming) and that allterms
1082     // copes with iterating over them.
1083     Xapian::TermIterator t;
1084     t = db.allterms_begin();
1085     TEST_EQUAL(*t, "back\\slash"); ++t; TEST_NOT_EQUAL(t, db.allterms_end());
1086     TEST_EQUAL(*t, string("big\0zero", 8)); ++t; TEST_NOT_EQUAL(t, db.allterms_end());
1087     TEST_EQUAL(*t, "new\nlin"); ++t; TEST_NOT_EQUAL(t, db.allterms_end());
1088     TEST_EQUAL(*t, "one\x01on"); ++t; TEST_NOT_EQUAL(t, db.allterms_end());
1089     TEST_EQUAL(*t, "space man"); ++t; TEST_NOT_EQUAL(t, db.allterms_end());
1090     TEST_EQUAL(*t, "tab\tbi"); ++t; TEST_NOT_EQUAL(t, db.allterms_end());
1091     TEST_EQUAL(*t, "tu\x02tu"); ++t; TEST_EQUAL(t, db.allterms_end());
1092
1093     // Now check that skip_to exactly a term containing a zero byte works.
1094     // This is a regression test for flint and quartz - an Assert() used to
1095     // fire in debug builds (the Assert was wrong - the actual code handled
1096     // this OK).
1097     t = db.allterms_begin();
1098     t.skip_to(string("big\0zero", 8));
1099     TEST_NOT_EQUAL(t, db.allterms_end());
1100     TEST_EQUAL(*t, string("big\0zero", 8));
1101
1102     return true;
1103 }
1104
1105 // test that rsets behave correctly with multiDBs
1106 DEFINE_TESTCASE(rsetmultidb2, backend && !multi) {
1107     Xapian::Database mydb1(get_database("apitest_rset", "apitest_simpledata2"));
1108     Xapian::Database mydb2(get_database("apitest_rset"));
1109     mydb2.add_database(get_database("apitest_simpledata2"));
1110
1111     Xapian::Enquire enquire1(mydb1);
1112     Xapian::Enquire enquire2(mydb2);
1113
1114     Xapian::Query myquery = query("is");
1115
1116     enquire1.set_query(myquery);
1117     enquire2.set_query(myquery);
1118
1119     Xapian::RSet myrset1;
1120     Xapian::RSet myrset2;
1121     myrset1.add_document(4);
1122     myrset2.add_document(2);
1123
1124     Xapian::MSet mymset1a = enquire1.get_mset(0, 10);
1125     Xapian::MSet mymset1b = enquire1.get_mset(0, 10, &myrset1);
1126     Xapian::MSet mymset2a = enquire2.get_mset(0, 10);
1127     Xapian::MSet mymset2b = enquire2.get_mset(0, 10, &myrset2);
1128
1129     mset_expect_order(mymset1a, 4, 3);
1130     mset_expect_order(mymset1b, 4, 3);
1131     mset_expect_order(mymset2a, 2, 5);
1132     mset_expect_order(mymset2b, 2, 5);
1133
1134     TEST(mset_range_is_same_weights(mymset1a, 0, mymset2a, 0, 2));
1135     TEST(mset_range_is_same_weights(mymset1b, 0, mymset2b, 0, 2));
1136     TEST_NOT_EQUAL(mymset1a, mymset1b);
1137     TEST_NOT_EQUAL(mymset2a, mymset2b);
1138
1139     return true;
1140 }
1141
1142 // tests an expand across multiple databases
1143 DEFINE_TESTCASE(multiexpand1, backend && !multi) {
1144     Xapian::Database mydb1(get_database("apitest_simpledata", "apitest_simpledata2"));
1145     Xapian::Enquire enquire1(mydb1);
1146
1147     Xapian::Database mydb2(get_database("apitest_simpledata"));
1148     mydb2.add_database(get_database("apitest_simpledata2"));
1149     Xapian::Enquire enquire2(mydb2);
1150
1151     // make simple equivalent rsets, with a document from each database in each.
1152     Xapian::RSet rset1;
1153     Xapian::RSet rset2;
1154     rset1.add_document(1);
1155     rset1.add_document(7);
1156     rset2.add_document(1);
1157     rset2.add_document(2);
1158
1159     // Retrieve all the ESet results in each of the three setups:
1160
1161     // This is the single database one.
1162     Xapian::ESet eset1 = enquire1.get_eset(1000, rset1);
1163
1164     // This is the multi database with approximation
1165     Xapian::ESet eset2 = enquire2.get_eset(1000, rset2);
1166
1167     // This is the multi database without approximation
1168     Xapian::ESet eset3 = enquire2.get_eset(1000, rset2, Xapian::Enquire::USE_EXACT_TERMFREQ);
1169
1170     TEST_EQUAL(eset1.size(), eset3.size());
1171
1172     Xapian::ESetIterator i = eset1.begin();
1173     Xapian::ESetIterator j = eset3.begin();
1174     while (i != eset1.end() && j != eset3.end()) {
1175         TEST_EQUAL(*i, *j);
1176         TEST_EQUAL(i.get_weight(), j.get_weight());
1177         ++i;
1178         ++j;
1179     }
1180     TEST(i == eset1.end());
1181     TEST(j == eset3.end());
1182
1183     bool eset1_eq_eset2 = true;
1184     i = eset1.begin();
1185     j = eset2.begin();
1186     while (i != eset1.end() && j != eset2.end()) {
1187         if (i.get_weight() != j.get_weight()) {
1188             eset1_eq_eset2 = false;
1189             break;
1190         }
1191         ++i;
1192         ++j;
1193     }
1194     TEST(!eset1_eq_eset2);
1195
1196     return true;
1197 }
1198
1199 // tests that opening a non-existent postlist returns an empty list
1200 DEFINE_TESTCASE(postlist1, backend) {
1201     Xapian::Database db(get_database("apitest_simpledata"));
1202
1203     TEST_EQUAL(db.postlist_begin("rosebud"), db.postlist_end("rosebud"));
1204
1205     string s = "let_us_see_if_we_can_break_it_with_a_really_really_long_term.";
1206     for (int i = 0; i < 8; ++i) {
1207         s += s;
1208         TEST_EQUAL(db.postlist_begin(s), db.postlist_end(s));
1209     }
1210
1211     // A regression test (no, really!)
1212     TEST_NOT_EQUAL(db.postlist_begin("a"), db.postlist_end("a"));
1213
1214     return true;
1215 }
1216
1217 // tests that a Xapian::PostingIterator works as an STL iterator
1218 DEFINE_TESTCASE(postlist2, backend) {
1219     Xapian::Database db(get_database("apitest_simpledata"));
1220     Xapian::PostingIterator p;
1221     p = db.postlist_begin("this");
1222     Xapian::PostingIterator pend = db.postlist_end("this");
1223
1224     TEST(p.get_description() != "PostingIterator()");
1225
1226     // test operator= creates a copy which compares equal
1227     Xapian::PostingIterator p_copy = p;
1228     TEST_EQUAL(p, p_copy);
1229
1230     TEST(p_copy.get_description() != "PostingIterator()");
1231
1232     // test copy constructor creates a copy which compares equal
1233     Xapian::PostingIterator p_clone(p);
1234     TEST_EQUAL(p, p_clone);
1235
1236     TEST(p_clone.get_description() != "PostingIterator()");
1237
1238     vector<Xapian::docid> v(p, pend);
1239
1240     p = db.postlist_begin("this");
1241     pend = db.postlist_end("this");
1242     vector<Xapian::docid>::const_iterator i;
1243     for (i = v.begin(); i != v.end(); ++i) {
1244         TEST_NOT_EQUAL(p, pend);
1245         TEST_EQUAL(*i, *p);
1246         p++;
1247     }
1248     TEST_EQUAL(p, pend);
1249
1250     TEST_STRINGS_EQUAL(p.get_description(), "PostingIterator()");
1251     TEST_STRINGS_EQUAL(pend.get_description(), "PostingIterator()");
1252
1253     return true;
1254 }
1255
1256 // tests that a Xapian::PostingIterator still works when the DB is deleted
1257 DEFINE_TESTCASE(postlist3, backend) {
1258     Xapian::PostingIterator u;
1259     {
1260         Xapian::Database db_temp(get_database("apitest_simpledata"));
1261         u = db_temp.postlist_begin("this");
1262     }
1263
1264     Xapian::Database db(get_database("apitest_simpledata"));
1265     Xapian::PostingIterator p = db.postlist_begin("this");
1266     Xapian::PostingIterator pend = db.postlist_end("this");
1267
1268     while (p != pend) {
1269         TEST_EQUAL(*p, *u);
1270         p++;
1271         u++;
1272     }
1273     return true;
1274 }
1275
1276 // tests skip_to
1277 DEFINE_TESTCASE(postlist4, backend) {
1278     Xapian::Database db(get_database("apitest_simpledata"));
1279     Xapian::PostingIterator i = db.postlist_begin("this");
1280     i.skip_to(1);
1281     i.skip_to(999999999);
1282     TEST(i == db.postlist_end("this"));
1283     return true;
1284 }
1285
1286 // tests long postlists
1287 DEFINE_TESTCASE(postlist5, backend) {
1288     Xapian::Database db(get_database("apitest_manydocs"));
1289     // Allow for databases which don't support length
1290     if (db.get_avlength() != 1)
1291         TEST_EQUAL_DOUBLE(db.get_avlength(), 4);
1292     Xapian::PostingIterator i = db.postlist_begin("this");
1293     unsigned int j = 1;
1294     while (i != db.postlist_end("this")) {
1295         TEST_EQUAL(*i, j);
1296         i++;
1297         j++;
1298     }
1299     TEST_EQUAL(j, 513);
1300     return true;
1301 }
1302
1303 // tests document length in postlists
1304 DEFINE_TESTCASE(postlist6, backend) {
1305     Xapian::Database db(get_database("apitest_simpledata"));
1306     Xapian::PostingIterator i = db.postlist_begin("this");
1307     TEST(i != db.postlist_end("this"));
1308     while (i != db.postlist_end("this")) {
1309         TEST_EQUAL(i.get_doclength(), db.get_doclength(*i));
1310         TEST_EQUAL(i.get_unique_terms(), db.get_unique_terms(*i));
1311         TEST_REL(i.get_wdf(),<=,i.get_doclength());
1312         TEST_REL(1,<=,i.get_unique_terms());
1313         // The next two aren't necessarily true if there are terms with wdf=0
1314         // in the document, but that isn't the case here.
1315         TEST_REL(i.get_unique_terms(),<=,i.get_doclength());
1316         TEST_REL(i.get_wdf() + i.get_unique_terms() - 1,<=,i.get_doclength());
1317         ++i;
1318     }
1319     return true;
1320 }
1321
1322 // tests collection frequency
1323 DEFINE_TESTCASE(collfreq1, backend) {
1324     Xapian::Database db(get_database("apitest_simpledata"));
1325
1326     TEST_EQUAL(db.get_collection_freq("this"), 11);
1327     TEST_EQUAL(db.get_collection_freq("first"), 1);
1328     TEST_EQUAL(db.get_collection_freq("last"), 0);
1329     TEST_EQUAL(db.get_collection_freq("word"), 9);
1330
1331     Xapian::Database db1(get_database("apitest_simpledata", "apitest_simpledata2"));
1332     Xapian::Database db2(get_database("apitest_simpledata"));
1333     db2.add_database(get_database("apitest_simpledata2"));
1334
1335     TEST_EQUAL(db1.get_collection_freq("this"), 15);
1336     TEST_EQUAL(db1.get_collection_freq("first"), 1);
1337     TEST_EQUAL(db1.get_collection_freq("last"), 0);
1338     TEST_EQUAL(db1.get_collection_freq("word"), 11);
1339     TEST_EQUAL(db2.get_collection_freq("this"), 15);
1340     TEST_EQUAL(db2.get_collection_freq("first"), 1);
1341     TEST_EQUAL(db2.get_collection_freq("last"), 0);
1342     TEST_EQUAL(db2.get_collection_freq("word"), 11);
1343
1344     return true;
1345 }
1346
1347 // Regression test for split msets being incorrect when sorting
1348 DEFINE_TESTCASE(sortvalue1, backend) {
1349     Xapian::Enquire enquire(get_database("apitest_simpledata"));
1350     enquire.set_query(Xapian::Query("this"));
1351
1352     for (int pass = 1; pass <= 2; ++pass) {
1353         for (Xapian::valueno value_no = 1; value_no < 7; ++value_no) {
1354             tout << "Sorting on value " << value_no << endl;
1355             enquire.set_sort_by_value(value_no, true);
1356             Xapian::MSet allbset = enquire.get_mset(0, 100);
1357             Xapian::MSet partbset1 = enquire.get_mset(0, 3);
1358             Xapian::MSet partbset2 = enquire.get_mset(3, 97);
1359             TEST_EQUAL(allbset.size(), partbset1.size() + partbset2.size());
1360
1361             bool ok = true;
1362             int n = 0;
1363             Xapian::MSetIterator i, j;
1364             j = allbset.begin();
1365             for (i = partbset1.begin(); i != partbset1.end(); ++i) {
1366                 tout << "Entry " << n << ": " << *i << " | " << *j << endl;
1367                 TEST(j != allbset.end());
1368                 if (*i != *j) ok = false;
1369                 ++j;
1370                 ++n;
1371             }
1372             tout << "===\n";
1373             for (i = partbset2.begin(); i != partbset2.end(); ++i) {
1374                 tout << "Entry " << n << ": " << *i << " | " << *j << endl;
1375                 TEST(j != allbset.end());
1376                 if (*i != *j) ok = false;
1377                 ++j;
1378                 ++n;
1379             }
1380             TEST(j == allbset.end());
1381             if (!ok)
1382                 FAIL_TEST("Split msets aren't consistent with unsplit");
1383         }
1384         enquire.set_docid_order(Xapian::Enquire::DESCENDING);
1385     }
1386
1387     return true;
1388 }
1389
1390 // consistency check match - vary mset size and check results agree.
1391 // consistency1 will run on the remote backend, but it's particularly slow
1392 // with that, and testing it there doesn't actually improve the test
1393 // coverage really.
1394 DEFINE_TESTCASE(consistency1, backend && !remote) {
1395     Xapian::Database db(get_database("etext"));
1396     Xapian::Enquire enquire(db);
1397     enquire.set_query(Xapian::Query(Xapian::Query::OP_OR, Xapian::Query("the"), Xapian::Query("sky")));
1398     Xapian::doccount lots = 214;
1399     Xapian::MSet bigmset = enquire.get_mset(0, lots);
1400     TEST_EQUAL(bigmset.size(), lots);
1401     try {
1402         for (Xapian::doccount start = 0; start < lots; ++start) {
1403             for (Xapian::doccount size = 0; size < lots - start; ++size) {
1404                 Xapian::MSet mset = enquire.get_mset(start, size);
1405                 if (mset.size()) {
1406                     TEST_EQUAL(start + mset.size(),
1407                                min(start + size, bigmset.size()));
1408                 } else if (size) {
1409 //              tout << start << mset.size() << bigmset.size() << endl;
1410                     TEST(start >= bigmset.size());
1411                 }
1412                 for (Xapian::doccount i = 0; i < mset.size(); ++i) {
1413                     TEST_EQUAL(*mset[i], *bigmset[start + i]);
1414                     TEST_EQUAL_DOUBLE(mset[i].get_weight(),
1415                                       bigmset[start + i].get_weight());
1416                 }
1417             }
1418         }
1419     } catch (const Xapian::NetworkTimeoutError &) {
1420         // consistency1 is a long test - may timeout with the remote backend...
1421         SKIP_TEST("Test taking too long");
1422     }
1423     return true;
1424 }
1425
1426 // tests that specifying a nonexistent input file throws an exception.
1427 DEFINE_TESTCASE(glassdatabaseopeningerror1, glass) {
1428 #ifdef XAPIAN_HAS_GLASS_BACKEND
1429     mkdir(".glass", 0755);
1430
1431     TEST_EXCEPTION(Xapian::DatabaseOpeningError,
1432             Xapian::Database(".glass/nosuchdirectory",
1433                 Xapian::DB_BACKEND_GLASS));
1434     TEST_EXCEPTION(Xapian::DatabaseOpeningError,
1435             Xapian::WritableDatabase(".glass/nosuchdirectory",
1436                 Xapian::DB_OPEN|Xapian::DB_BACKEND_GLASS));
1437
1438     mkdir(".glass/emptydirectory", 0700);
1439     TEST_EXCEPTION(Xapian::DatabaseOpeningError,
1440             Xapian::Database(".glass/emptydirectory",
1441                 Xapian::DB_BACKEND_GLASS));
1442
1443     touch(".glass/somefile");
1444     TEST_EXCEPTION(Xapian::DatabaseOpeningError,
1445             Xapian::Database(".glass/somefile",
1446                 Xapian::DB_BACKEND_GLASS));
1447     TEST_EXCEPTION(Xapian::DatabaseOpeningError,
1448             Xapian::WritableDatabase(".glass/somefile",
1449                 Xapian::DB_OPEN|Xapian::DB_BACKEND_GLASS));
1450     TEST_EXCEPTION(Xapian::DatabaseCreateError,
1451             Xapian::WritableDatabase(".glass/somefile",
1452                 Xapian::DB_CREATE|Xapian::DB_BACKEND_GLASS));
1453     TEST_EXCEPTION(Xapian::DatabaseCreateError,
1454             Xapian::WritableDatabase(".glass/somefile",
1455                 Xapian::DB_CREATE_OR_OPEN|Xapian::DB_BACKEND_GLASS));
1456     TEST_EXCEPTION(Xapian::DatabaseCreateError,
1457             Xapian::WritableDatabase(".glass/somefile",
1458                 Xapian::DB_CREATE_OR_OVERWRITE|Xapian::DB_BACKEND_GLASS));
1459 #endif
1460
1461     return true;
1462 }
1463
1464 /// Test opening of a glass database
1465 DEFINE_TESTCASE(glassdatabaseopen1, glass) {
1466 #ifdef XAPIAN_HAS_GLASS_BACKEND
1467     const string dbdir = ".glass/test_glassdatabaseopen1";
1468     mkdir(".glass", 0755);
1469
1470     {
1471         rm_rf(dbdir);
1472         Xapian::WritableDatabase wdb(dbdir,
1473                 Xapian::DB_CREATE|Xapian::DB_BACKEND_GLASS);
1474         TEST_EXCEPTION(Xapian::DatabaseLockError,
1475             Xapian::WritableDatabase(dbdir,
1476                 Xapian::DB_OPEN|Xapian::DB_BACKEND_GLASS));
1477         Xapian::Database(dbdir, Xapian::DB_BACKEND_GLASS);
1478     }
1479
1480     {
1481         rm_rf(dbdir);
1482         Xapian::WritableDatabase wdb(dbdir,
1483                 Xapian::DB_CREATE_OR_OPEN|Xapian::DB_BACKEND_GLASS);
1484         TEST_EXCEPTION(Xapian::DatabaseLockError,
1485             Xapian::WritableDatabase(dbdir,
1486                 Xapian::DB_CREATE_OR_OVERWRITE|Xapian::DB_BACKEND_GLASS));
1487         Xapian::Database(dbdir, Xapian::DB_BACKEND_GLASS);
1488     }
1489
1490     {
1491         rm_rf(dbdir);
1492         Xapian::WritableDatabase wdb(dbdir,
1493                 Xapian::DB_CREATE_OR_OVERWRITE|Xapian::DB_BACKEND_GLASS);
1494         TEST_EXCEPTION(Xapian::DatabaseLockError,
1495             Xapian::WritableDatabase(dbdir,
1496                 Xapian::DB_CREATE_OR_OPEN|Xapian::DB_BACKEND_GLASS));
1497         Xapian::Database(dbdir, Xapian::DB_BACKEND_GLASS);
1498     }
1499
1500     {
1501         TEST_EXCEPTION(Xapian::DatabaseCreateError,
1502             Xapian::WritableDatabase(dbdir,
1503                 Xapian::DB_CREATE|Xapian::DB_BACKEND_GLASS));
1504         Xapian::WritableDatabase wdb(dbdir,
1505                 Xapian::DB_CREATE_OR_OVERWRITE|Xapian::DB_BACKEND_GLASS);
1506         Xapian::Database(dbdir, Xapian::DB_BACKEND_GLASS);
1507     }
1508
1509     {
1510         Xapian::WritableDatabase wdb(dbdir,
1511                 Xapian::DB_CREATE_OR_OPEN|Xapian::DB_BACKEND_GLASS);
1512         Xapian::Database(dbdir, Xapian::DB_BACKEND_GLASS);
1513     }
1514
1515     {
1516         Xapian::WritableDatabase wdb(dbdir,
1517                 Xapian::DB_OPEN|Xapian::DB_BACKEND_GLASS);
1518         Xapian::Database(dbdir, Xapian::DB_BACKEND_GLASS);
1519     }
1520 #endif
1521
1522     return true;
1523 }
1524
1525 // feature test for Enquire:
1526 // set_sort_by_value
1527 // set_sort_by_value_then_relevance
1528 // set_sort_by_relevance_then_value
1529 // Prior to 1.2.17 and 1.3.2, order8 and order9 were swapped, and
1530 // set_sort_by_relevance_then_value was buggy, so this testcase now serves as
1531 // a regression test for that bug.
1532 DEFINE_TESTCASE(sortrel1, backend) {
1533     Xapian::Enquire enquire(get_database("apitest_sortrel"));
1534     enquire.set_sort_by_value(1, true);
1535     enquire.set_query(Xapian::Query("woman"));
1536
1537     const Xapian::docid order1[] = { 1,2,3,4,5,6,7,8,9 };
1538     const Xapian::docid order2[] = { 2,1,3,6,5,4,7,9,8 };
1539     const Xapian::docid order3[] = { 3,2,1,6,5,4,9,8,7 };
1540     const Xapian::docid order4[] = { 7,8,9,4,5,6,1,2,3 };
1541     const Xapian::docid order5[] = { 9,8,7,6,5,4,3,2,1 };
1542     const Xapian::docid order6[] = { 7,9,8,6,5,4,2,1,3 };
1543     const Xapian::docid order7[] = { 7,9,8,6,5,4,2,1,3 };
1544     const Xapian::docid order8[] = { 2,6,7,1,5,9,3,4,8 };
1545     const Xapian::docid order9[] = { 7,6,2,9,5,1,8,4,3 };
1546
1547     Xapian::MSet mset;
1548     size_t i;
1549
1550     mset = enquire.get_mset(0, 10);
1551     TEST_EQUAL(mset.size(), sizeof(order1) / sizeof(Xapian::docid));
1552     for (i = 0; i < sizeof(order1) / sizeof(Xapian::docid); ++i) {
1553         TEST_EQUAL(*mset[i], order1[i]);
1554     }
1555
1556     enquire.set_sort_by_value_then_relevance(1, true);
1557
1558     mset = enquire.get_mset(0, 10);
1559     TEST_EQUAL(mset.size(), sizeof(order2) / sizeof(Xapian::docid));
1560     for (i = 0; i < sizeof(order2) / sizeof(Xapian::docid); ++i) {
1561         TEST_EQUAL(*mset[i], order2[i]);
1562     }
1563
1564     enquire.set_sort_by_value(1, true);
1565
1566     mset = enquire.get_mset(0, 10);
1567     TEST_EQUAL(mset.size(), sizeof(order1) / sizeof(Xapian::docid));
1568     for (i = 0; i < sizeof(order1) / sizeof(Xapian::docid); ++i) {
1569         TEST_EQUAL(*mset[i], order1[i]);
1570     }
1571
1572     enquire.set_sort_by_value_then_relevance(1, true);
1573     enquire.set_docid_order(Xapian::Enquire::DESCENDING);
1574
1575     mset = enquire.get_mset(0, 10);
1576     TEST_EQUAL(mset.size(), sizeof(order2) / sizeof(Xapian::docid));
1577     for (i = 0; i < sizeof(order2) / sizeof(Xapian::docid); ++i) {
1578         TEST_EQUAL(*mset[i], order2[i]);
1579     }
1580
1581     enquire.set_sort_by_value(1, true);
1582     enquire.set_docid_order(Xapian::Enquire::DESCENDING);
1583
1584     mset = enquire.get_mset(0, 10);
1585     TEST_EQUAL(mset.size(), sizeof(order3) / sizeof(Xapian::docid));
1586     for (i = 0; i < sizeof(order3) / sizeof(Xapian::docid); ++i) {
1587         TEST_EQUAL(*mset[i], order3[i]);
1588     }
1589
1590     enquire.set_sort_by_value(1, false);
1591     enquire.set_docid_order(Xapian::Enquire::ASCENDING);
1592     mset = enquire.get_mset(0, 10);
1593     TEST_EQUAL(mset.size(), sizeof(order4) / sizeof(Xapian::docid));
1594     for (i = 0; i < sizeof(order4) / sizeof(Xapian::docid); ++i) {
1595         TEST_EQUAL(*mset[i], order4[i]);
1596     }
1597
1598     enquire.set_sort_by_value(1, false);
1599     enquire.set_docid_order(Xapian::Enquire::DESCENDING);
1600     mset = enquire.get_mset(0, 10);
1601     TEST_EQUAL(mset.size(), sizeof(order5) / sizeof(Xapian::docid));
1602     for (i = 0; i < sizeof(order5) / sizeof(Xapian::docid); ++i) {
1603         TEST_EQUAL(*mset[i], order5[i]);
1604     }
1605
1606     enquire.set_sort_by_value_then_relevance(1, false);
1607     enquire.set_docid_order(Xapian::Enquire::ASCENDING);
1608     mset = enquire.get_mset(0, 10);
1609     TEST_EQUAL(mset.size(), sizeof(order6) / sizeof(Xapian::docid));
1610     for (i = 0; i < sizeof(order6) / sizeof(Xapian::docid); ++i) {
1611         TEST_EQUAL(*mset[i], order6[i]);
1612     }
1613
1614     enquire.set_sort_by_value_then_relevance(1, false);
1615     enquire.set_docid_order(Xapian::Enquire::DESCENDING);
1616     mset = enquire.get_mset(0, 10);
1617     TEST_EQUAL(mset.size(), sizeof(order7) / sizeof(Xapian::docid));
1618     for (i = 0; i < sizeof(order7) / sizeof(Xapian::docid); ++i) {
1619         TEST_EQUAL(*mset[i], order7[i]);
1620     }
1621
1622     enquire.set_sort_by_relevance_then_value(1, true);
1623     enquire.set_docid_order(Xapian::Enquire::ASCENDING);
1624     mset = enquire.get_mset(0, 10);
1625     TEST_EQUAL(mset.size(), sizeof(order8) / sizeof(Xapian::docid));
1626     for (i = 0; i < sizeof(order8) / sizeof(Xapian::docid); ++i) {
1627         TEST_EQUAL(*mset[i], order8[i]);
1628     }
1629
1630     enquire.set_sort_by_relevance_then_value(1, true);
1631     enquire.set_docid_order(Xapian::Enquire::DESCENDING);
1632     mset = enquire.get_mset(0, 10);
1633     TEST_EQUAL(mset.size(), sizeof(order8) / sizeof(Xapian::docid));
1634     for (i = 0; i < sizeof(order8) / sizeof(Xapian::docid); ++i) {
1635         TEST_EQUAL(*mset[i], order8[i]);
1636     }
1637
1638     enquire.set_sort_by_relevance_then_value(1, false);
1639     enquire.set_docid_order(Xapian::Enquire::ASCENDING);
1640     mset = enquire.get_mset(0, 10);
1641     TEST_EQUAL(mset.size(), sizeof(order9) / sizeof(Xapian::docid));
1642     for (i = 0; i < sizeof(order9) / sizeof(Xapian::docid); ++i) {
1643         TEST_EQUAL(*mset[i], order9[i]);
1644     }
1645
1646     enquire.set_sort_by_relevance_then_value(1, false);
1647     enquire.set_docid_order(Xapian::Enquire::DESCENDING);
1648     mset = enquire.get_mset(0, 10);
1649     TEST_EQUAL(mset.size(), sizeof(order9) / sizeof(Xapian::docid));
1650     for (i = 0; i < sizeof(order9) / sizeof(Xapian::docid); ++i) {
1651         TEST_EQUAL(*mset[i], order9[i]);
1652     }
1653
1654     return true;
1655 }
1656
1657 // Test network stats and local stats give the same results.
1658 DEFINE_TESTCASE(netstats1, remote) {
1659     BackendManagerLocal local_manager;
1660     local_manager.set_datadir(test_driver::get_srcdir() + "/testdata/");
1661
1662     const char * words[] = { "paragraph", "word" };
1663     Xapian::Query query(Xapian::Query::OP_OR, words, words + 2);
1664     const size_t MSET_SIZE = 10;
1665
1666     Xapian::RSet rset;
1667     rset.add_document(4);
1668     rset.add_document(9);
1669
1670     Xapian::MSet mset_alllocal;
1671     {
1672         Xapian::Database db;
1673         db.add_database(local_manager.get_database("apitest_simpledata"));
1674         db.add_database(local_manager.get_database("apitest_simpledata2"));
1675
1676         Xapian::Enquire enq(db);
1677         enq.set_query(query);
1678         mset_alllocal = enq.get_mset(0, MSET_SIZE, &rset);
1679     }
1680
1681     {
1682         Xapian::Database db;
1683         db.add_database(local_manager.get_database("apitest_simpledata"));
1684         db.add_database(get_database("apitest_simpledata2"));
1685
1686         Xapian::Enquire enq(db);
1687         enq.set_query(query);
1688         Xapian::MSet mset = enq.get_mset(0, MSET_SIZE, &rset);
1689         TEST_EQUAL(mset.get_matches_lower_bound(), mset_alllocal.get_matches_lower_bound());
1690         TEST_EQUAL(mset.get_matches_upper_bound(), mset_alllocal.get_matches_upper_bound());
1691         TEST_EQUAL(mset.get_matches_estimated(), mset_alllocal.get_matches_estimated());
1692         TEST_EQUAL(mset.get_max_attained(), mset_alllocal.get_max_attained());
1693         TEST_EQUAL(mset.size(), mset_alllocal.size());
1694         TEST(mset_range_is_same(mset, 0, mset_alllocal, 0, mset.size()));
1695     }
1696
1697     {
1698         Xapian::Database db;
1699         db.add_database(get_database("apitest_simpledata"));
1700         db.add_database(local_manager.get_database("apitest_simpledata2"));
1701
1702         Xapian::Enquire enq(db);
1703         enq.set_query(query);
1704         Xapian::MSet mset = enq.get_mset(0, MSET_SIZE, &rset);
1705         TEST_EQUAL(mset.get_matches_lower_bound(), mset_alllocal.get_matches_lower_bound());
1706         TEST_EQUAL(mset.get_matches_upper_bound(), mset_alllocal.get_matches_upper_bound());
1707         TEST_EQUAL(mset.get_matches_estimated(), mset_alllocal.get_matches_estimated());
1708         TEST_EQUAL(mset.get_max_attained(), mset_alllocal.get_max_attained());
1709         TEST_EQUAL(mset.size(), mset_alllocal.size());
1710         TEST(mset_range_is_same(mset, 0, mset_alllocal, 0, mset.size()));
1711     }
1712
1713     {
1714         Xapian::Database db;
1715         db.add_database(get_database("apitest_simpledata"));
1716         db.add_database(get_database("apitest_simpledata2"));
1717
1718         Xapian::Enquire enq(db);
1719         enq.set_query(query);
1720         Xapian::MSet mset = enq.get_mset(0, MSET_SIZE, &rset);
1721         TEST_EQUAL(mset.get_matches_lower_bound(), mset_alllocal.get_matches_lower_bound());
1722         TEST_EQUAL(mset.get_matches_upper_bound(), mset_alllocal.get_matches_upper_bound());
1723         TEST_EQUAL(mset.get_matches_estimated(), mset_alllocal.get_matches_estimated());
1724         TEST_EQUAL(mset.get_max_attained(), mset_alllocal.get_max_attained());
1725         TEST_EQUAL(mset.size(), mset_alllocal.size());
1726         TEST(mset_range_is_same(mset, 0, mset_alllocal, 0, mset.size()));
1727     }
1728
1729     return true;
1730 }
1731
1732 // Coordinate matching - scores 1 for each matching term
1733 class MyWeight : public Xapian::Weight {
1734     double scale_factor;
1735
1736   public:
1737     MyWeight * clone() const {
1738         return new MyWeight;
1739     }
1740     void init(double factor) {
1741         scale_factor = factor;
1742     }
1743     MyWeight() { }
1744     ~MyWeight() { }
1745     std::string name() const { return "MyWeight"; }
1746     string serialise() const { return string(); }
1747     MyWeight * unserialise(const string &) const { return new MyWeight; }
1748     double get_sumpart(Xapian::termcount, Xapian::termcount, Xapian::termcount) const {
1749         return scale_factor;
1750     }
1751     double get_maxpart() const { return scale_factor; }
1752
1753     double get_sumextra(Xapian::termcount, Xapian::termcount) const { return 0; }
1754     double get_maxextra() const { return 0; }
1755 };
1756
1757 // tests user weighting scheme.
1758 // Would work with remote if we registered the weighting scheme.
1759 // FIXME: do this so we also test that functionality...
1760 DEFINE_TESTCASE(userweight1, backend && !remote) {
1761     Xapian::Enquire enquire(get_database("apitest_simpledata"));
1762     enquire.set_weighting_scheme(MyWeight());
1763     const char * query[] = { "this", "line", "paragraph", "rubbish" };
1764     enquire.set_query(Xapian::Query(Xapian::Query::OP_OR, query,
1765                                     query + sizeof(query) / sizeof(query[0])));
1766     Xapian::MSet mymset1 = enquire.get_mset(0, 100);
1767     // MyWeight scores 1 for each matching term, so the weight should equal
1768     // the number of matching terms.
1769     for (Xapian::MSetIterator i = mymset1.begin(); i != mymset1.end(); ++i) {
1770         Xapian::termcount matching_terms = 0;
1771         Xapian::TermIterator t = enquire.get_matching_terms_begin(i);
1772         while (t != enquire.get_matching_terms_end(i)) {
1773             ++matching_terms;
1774             ++t;
1775         }
1776         TEST_EQUAL(i.get_weight(), matching_terms);
1777     }
1778
1779     return true;
1780 }
1781
1782 // tests MatchAll queries
1783 // This is a regression test, which failed with assertion failures in
1784 // revision 9094.  Also check that the results aren't ranked by relevance
1785 // (regression test for bug fixed in 1.0.9).
1786 DEFINE_TESTCASE(matchall1, backend) {
1787     Xapian::Database db(get_database("apitest_simpledata"));
1788     Xapian::Enquire enquire(db);
1789     enquire.set_query(Xapian::Query::MatchAll);
1790     Xapian::MSet mset = enquire.get_mset(0, 10);
1791     TEST_EQUAL(mset.get_matches_lower_bound(), db.get_doccount());
1792     TEST_EQUAL(mset.get_uncollapsed_matches_lower_bound(), db.get_doccount());
1793
1794     enquire.set_query(Xapian::Query(Xapian::Query::OP_OR,
1795                                     Xapian::Query("nosuchterm"),
1796                                     Xapian::Query::MatchAll));
1797     mset = enquire.get_mset(0, 10);
1798     TEST_EQUAL(mset.get_matches_lower_bound(), db.get_doccount());
1799     TEST_EQUAL(mset.get_uncollapsed_matches_lower_bound(), db.get_doccount());
1800
1801     // Check that the results aren't ranked by relevance (fixed in 1.0.9).
1802     TEST(mset.size() > 1);
1803     TEST_EQUAL(mset[mset.size() - 1].get_weight(), 0);
1804     TEST_EQUAL(*mset[0], 1);
1805     TEST_EQUAL(*mset[mset.size() - 1], mset.size());
1806
1807     return true;
1808 }
1809
1810 // Test using a ValueSetMatchDecider
1811 DEFINE_TESTCASE(valuesetmatchdecider2, backend && !remote) {
1812     Xapian::Database db(get_database("apitest_phrase"));
1813     Xapian::Enquire enq(db);
1814     enq.set_query(Xapian::Query("leav"));
1815
1816     Xapian::ValueSetMatchDecider vsmd1(1, true);
1817     vsmd1.add_value("n");
1818     Xapian::ValueSetMatchDecider vsmd2(1, false);
1819     vsmd2.add_value("n");
1820
1821     Xapian::MSet mymset = enq.get_mset(0, 20);
1822     mset_expect_order(mymset, 8, 6, 4, 5, 7, 10, 12, 11, 13, 9, 14);
1823     mymset = enq.get_mset(0, 20, 0, NULL, &vsmd1);
1824     mset_expect_order(mymset, 6, 12);
1825     mymset = enq.get_mset(0, 20, 0, NULL, &vsmd2);
1826     mset_expect_order(mymset, 8, 4, 5, 7, 10, 11, 13, 9, 14);
1827
1828     return true;
1829 }