xapian-core/tests/api_wrdb.cc

   1 /* api_wrdb.cc: tests which need a writable backend
   2  *
   3  * Copyright 1999,2000,2001 BrightStation PLC
   4  * Copyright 2001 Hein Ragas
   5  * Copyright 2002 Ananova Ltd
   6  * Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2014,2015 Olly Betts
   7  * Copyright 2006 Richard Boulton
   8  * Copyright 2007 Lemur Consulting Ltd
   9  *
  10  * This program is free software; you can redistribute it and/or
  11  * modify it under the terms of the GNU General Public License as
  12  * published by the Free Software Foundation; either version 2 of the
  13  * License, or (at your option) any later version.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU General Public License
  21  * along with this program; if not, write to the Free Software
  22  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  23  * USA
  24  */
  25
  26 #include <config.h>
  27
  28 #include "api_wrdb.h"
  29
  30 #include <xapian.h>
  31
  32 #include "filetests.h"
  33 #include "omassert.h"
  34 #include "str.h"
  35 #include "stringutils.h"
  36 #include "testsuite.h"
  37 #include "testutils.h"
  38 #include "unixcmds.h"
  39
  40 #include "apitest.h"
  41
  42 #include "safeunistd.h"
  43 #include <cmath>
  44 #include <cstdlib>
  45 #include <map>
  46 #include <string>
  47
  48 using namespace std;
  49
  50 // #######################################################################
  51 // # Tests start here
  52
  53 // test that indexing a term more than once at the same position increases
  54 // the wdf
  55 DEFINE_TESTCASE(adddoc1, writable) {
  56     Xapian::WritableDatabase db = get_writable_database();
  57
  58     Xapian::Document doc1, doc2, doc3;
  59
  60     // doc1 should come top, but if term "foo" gets wdf of 1, doc2 will beat it
  61     // doc3 should beat both
  62     // Note: all docs have same length
  63     doc1.set_data(string("tom"));
  64     doc1.add_posting("foo", 1);
  65     doc1.add_posting("foo", 1);
  66     doc1.add_posting("foo", 1);
  67     doc1.add_posting("bar", 3);
  68     doc1.add_posting("bar", 4);
  69     db.add_document(doc1);
  70
  71     doc2.set_data(string("dick"));
  72     doc2.add_posting("foo", 1);
  73     doc2.add_posting("foo", 2);
  74     doc2.add_posting("bar", 3);
  75     doc2.add_posting("bar", 3);
  76     doc2.add_posting("bar", 3);
  77     db.add_document(doc2);
  78
  79     doc3.set_data(string("harry"));
  80     doc3.add_posting("foo", 1);
  81     doc3.add_posting("foo", 1);
  82     doc3.add_posting("foo", 2);
  83     doc3.add_posting("foo", 2);
  84     doc3.add_posting("bar", 3);
  85     db.add_document(doc3);
  86
  87     Xapian::Query query("foo");
  88
  89     Xapian::Enquire enq(db);
  90     enq.set_query(query);
  91
  92     Xapian::MSet mset = enq.get_mset(0, 10);
  93
  94     mset_expect_order(mset, 3, 1, 2);
  95
  96     return true;
  97 }
  98
  99 // test that removing a posting and removing a term works
 100 DEFINE_TESTCASE(adddoc2, writable) {
 101     Xapian::WritableDatabase db = get_writable_database();
 102
 103     Xapian::Document doc1;
 104
 105     doc1.add_posting("foo", 1);
 106     doc1.add_posting("foo", 1);
 107     doc1.add_posting("foo", 2);
 108     doc1.add_posting("foo", 2);
 109     doc1.add_posting("bar", 3);
 110     doc1.add_posting("gone", 1);
 111     // Quartz had a bug handling a term >= 128 characters longer than the
 112     // preceding term in the sort order - this is "foo" + 130 "X"s
 113     doc1.add_posting("fooXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", 1);
 114     Xapian::docid did;
 115
 116     Xapian::Document doc2 = db.get_document(did = db.add_document(doc1));
 117     TEST_EQUAL(did, 1);
 118
 119     Xapian::TermIterator iter1 = doc1.termlist_begin();
 120     Xapian::TermIterator iter2 = doc2.termlist_begin();
 121     TEST(iter1 != doc1.termlist_end());
 122     TEST(iter2 != doc2.termlist_end());
 123     TEST_EQUAL(*iter1, "bar");
 124     TEST_EQUAL(*iter2, *iter1);
 125     TEST_EQUAL(iter1.get_wdf(), 1);
 126     TEST_EQUAL(iter2.get_wdf(), 1);
 127     //TEST_EQUAL(iter1.get_termfreq(), 0);
 128     TEST_EQUAL(iter2.get_termfreq(), 1);
 129
 130     iter1++;
 131     iter2++;
 132     TEST(iter1 != doc1.termlist_end());
 133     TEST(iter2 != doc2.termlist_end());
 134     TEST_EQUAL(*iter1, "foo");
 135     TEST_EQUAL(*iter2, *iter1);
 136     TEST_EQUAL(iter1.get_wdf(), 4);
 137     TEST_EQUAL(iter2.get_wdf(), 4);
 138     //TEST_EQUAL(iter1.get_termfreq(), 0);
 139     TEST_EQUAL(iter2.get_termfreq(), 1);
 140
 141     iter1++;
 142     iter2++;
 143     TEST(iter1 != doc1.termlist_end());
 144     TEST(iter2 != doc2.termlist_end());
 145     TEST_EQUAL(*iter1, "fooXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX");
 146     TEST_EQUAL(*iter2, *iter1);
 147     TEST_EQUAL(iter1.get_wdf(), 1);
 148     TEST_EQUAL(iter2.get_wdf(), 1);
 149     // assertion fails in debug build! TEST_EQUAL(iter1.get_termfreq(), 0);
 150     TEST_EQUAL(iter2.get_termfreq(), 1);
 151
 152     iter1++;
 153     iter2++;
 154     TEST(iter1 != doc1.termlist_end());
 155     TEST(iter2 != doc2.termlist_end());
 156     TEST_EQUAL(*iter1, "gone");
 157     TEST_EQUAL(*iter2, *iter1);
 158     TEST_EQUAL(iter1.get_wdf(), 1);
 159     TEST_EQUAL(iter2.get_wdf(), 1);
 160     // assertion fails in debug build! TEST_EQUAL(iter1.get_termfreq(), 0);
 161     TEST_EQUAL(iter2.get_termfreq(), 1);
 162
 163     iter1++;
 164     iter2++;
 165     TEST(iter1 == doc1.termlist_end());
 166     TEST(iter2 == doc2.termlist_end());
 167
 168     doc2.remove_posting("foo", 1, 5);
 169     doc2.add_term("bat", 0);
 170     doc2.add_term("bar", 8);
 171     doc2.add_term("bag", 0);
 172     doc2.remove_term("gone");
 173     doc2.remove_term("fooXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX");
 174
 175     // Should have (doc,wdf) pairs: (bag,0)(bar,9)(bat,0)(foo,0)
 176     // positionlists (bag,none)(bar,3)(bat,none)(foo,2)
 177
 178     iter2 = doc2.termlist_begin();
 179     TEST(iter2 != doc2.termlist_end());
 180     TEST_EQUAL(*iter2, "bag");
 181     //TEST_EQUAL(iter2.get_termfreq(), 0);
 182     iter2++;
 183     TEST(iter2 != doc2.termlist_end());
 184     TEST_EQUAL(*iter2, "bar");
 185     //TEST_EQUAL(iter2.get_termfreq(), 0);
 186     iter2++;
 187     TEST(iter2 != doc2.termlist_end());
 188     TEST_EQUAL(*iter2, "bat");
 189     //TEST_EQUAL(iter2.get_termfreq(), 0);
 190     iter2++;
 191     TEST(iter2 != doc2.termlist_end());
 192     TEST_EQUAL(*iter2, "foo");
 193     //TEST_EQUAL(iter2.get_termfreq(), 0);
 194     iter2++;
 195     TEST(iter2 == doc2.termlist_end());
 196
 197     doc1 = db.get_document(did = db.add_document(doc2));
 198     TEST_EQUAL(did, 2);
 199
 200     iter1 = doc1.termlist_begin();
 201     iter2 = doc2.termlist_begin();
 202     TEST(iter1 != doc1.termlist_end());
 203     TEST(iter2 != doc2.termlist_end());
 204     TEST_EQUAL(*iter1, "bag");
 205     TEST_EQUAL(*iter2, *iter1);
 206     TEST_EQUAL(iter1.get_wdf(), 0);
 207     TEST_EQUAL(iter2.get_wdf(), 0);
 208     TEST_EQUAL(iter1.get_termfreq(), 1);
 209     //TEST_EQUAL(iter2.get_termfreq(), 0);
 210     TEST(iter1.positionlist_begin() == iter1.positionlist_end());
 211     TEST(iter2.positionlist_begin() == iter2.positionlist_end());
 212
 213     iter1++;
 214     iter2++;
 215     TEST(iter1 != doc1.termlist_end());
 216     TEST(iter2 != doc2.termlist_end());
 217     TEST_EQUAL(*iter1, "bar");
 218     TEST_EQUAL(*iter2, *iter1);
 219     TEST_EQUAL(iter1.get_wdf(), 9);
 220     TEST_EQUAL(iter2.get_wdf(), 9);
 221     TEST_EQUAL(iter1.get_termfreq(), 2);
 222     //TEST_EQUAL(iter2.get_termfreq(), 0);
 223
 224     Xapian::PositionIterator pi1;
 225     pi1 = iter1.positionlist_begin();
 226     Xapian::PositionIterator pi2 = iter2.positionlist_begin();
 227     TEST_EQUAL(*pi1, 3); pi1++;
 228     TEST_EQUAL(*pi2, 3); pi2++;
 229     TEST(pi1 == iter1.positionlist_end());
 230     TEST(pi2 == iter2.positionlist_end());
 231
 232     iter1++;
 233     iter2++;
 234     TEST(iter1 != doc1.termlist_end());
 235     TEST(iter2 != doc2.termlist_end());
 236     TEST_EQUAL(*iter1, "bat");
 237     TEST_EQUAL(*iter2, *iter1);
 238     TEST_EQUAL(iter1.get_wdf(), 0);
 239     TEST_EQUAL(iter2.get_wdf(), 0);
 240     TEST_EQUAL(iter1.get_termfreq(), 1);
 241     //TEST_EQUAL(iter2.get_termfreq(), 0);
 242     TEST(iter1.positionlist_begin() == iter1.positionlist_end());
 243     TEST(iter2.positionlist_begin() == iter2.positionlist_end());
 244
 245     iter1++;
 246     iter2++;
 247     TEST(iter1 != doc1.termlist_end());
 248     TEST(iter2 != doc2.termlist_end());
 249     TEST_EQUAL(*iter1, "foo");
 250     TEST_EQUAL(*iter2, *iter1);
 251     TEST_EQUAL(iter1.get_wdf(), 0);
 252     TEST_EQUAL(iter2.get_wdf(), 0);
 253     TEST_EQUAL(iter1.get_termfreq(), 2);
 254     //TEST_EQUAL(iter2.get_termfreq(), 0);
 255
 256     Xapian::PositionIterator temp1 = iter1.positionlist_begin();
 257     pi1 = temp1;
 258     Xapian::PositionIterator temp2 = iter2.positionlist_begin();
 259     pi2 = temp2;
 260     TEST_EQUAL(*pi1, 2); pi1++;
 261     TEST_EQUAL(*pi2, 2); pi2++;
 262     TEST(pi1 == iter1.positionlist_end());
 263     TEST(pi2 == iter2.positionlist_end());
 264
 265     iter1++;
 266     iter2++;
 267     TEST(iter1 == doc1.termlist_end());
 268     TEST(iter2 == doc2.termlist_end());
 269
 270     return true;
 271 }
 272
 273 // test that adding lots of documents works, and doesn't leak memory
 274 // REGRESSION FIXED:2003-09-07
 275 DEFINE_TESTCASE(adddoc3, writable) {
 276     Xapian::WritableDatabase db = get_writable_database();
 277
 278     for (Xapian::doccount i = 0; i < 2100; ++i) {
 279         Xapian::Document doc;
 280         for (Xapian::termcount t = 0; t < 100; ++t) {
 281             string term("foo");
 282             term += char(t ^ 70 ^ i);
 283             doc.add_posting(term, t);
 284         }
 285         db.add_document(doc);
 286     }
 287     return true;
 288 }
 289
 290 // We originally wanted to test that a termlist starting with a 48 character
 291 // long term worked since that required special handling in flint for
 292 // historical reasons.  That's no longer relevant, but it seems useful to
 293 // continue to test term lists starting with various term lengths work.
 294 DEFINE_TESTCASE(adddoc4, writable) {
 295     Xapian::WritableDatabase db = get_writable_database();
 296
 297     for (Xapian::doccount i = 1; i <= 240; ++i) {
 298         Xapian::Document doc;
 299         string term(i, 'X');
 300         doc.add_term(term);
 301         db.add_document(doc);
 302     }
 303     db.add_document(Xapian::Document());
 304     db.commit();
 305
 306     for (Xapian::doccount i = 1; i <= 240; ++i) {
 307         Xapian::Document doc = db.get_document(i);
 308         Xapian::TermIterator t = doc.termlist_begin();
 309         TEST(t != doc.termlist_end());
 310         TEST_EQUAL((*t).size(), i);
 311         ++t;
 312         TEST(t == doc.termlist_end());
 313     }
 314
 315     // And test a document with no terms.
 316     Xapian::Document doc = db.get_document(241);
 317     TEST(doc.termlist_begin() == doc.termlist_end());
 318
 319     return true;
 320 }
 321
 322 // Test adding a document, and checking that it got added correctly.
 323 // This testcase used to be adddoc2 in quartztest.
 324 DEFINE_TESTCASE(adddoc5, writable) {
 325     // Inmemory doesn't support get_writable_database_as_database().
 326     SKIP_TEST_FOR_BACKEND("inmemory");
 327
 328     Xapian::docid did;
 329     Xapian::Document document_in;
 330     document_in.set_data("Foobar rising");
 331     document_in.add_value(7, "Value7");
 332     document_in.add_value(13, "Value13");
 333     document_in.add_posting("foobar", 1);
 334     document_in.add_posting("rising", 2);
 335     document_in.add_posting("foobar", 3);
 336
 337     Xapian::Document document_in2;
 338     document_in2.set_data("Foobar falling");
 339     document_in2.add_posting("foobar", 1);
 340     document_in2.add_posting("falling", 2);
 341     {
 342         Xapian::WritableDatabase database(get_writable_database());
 343
 344         TEST_EQUAL(database.get_doccount(), 0);
 345         TEST_EQUAL(database.get_avlength(), 0);
 346
 347         did = database.add_document(document_in);
 348         TEST_EQUAL(database.get_doccount(), 1);
 349         TEST_EQUAL(database.get_avlength(), 3);
 350
 351         TEST_EQUAL(database.get_termfreq("foobar"), 1);
 352         TEST_EQUAL(database.get_collection_freq("foobar"), 2);
 353         TEST_EQUAL(database.get_termfreq("rising"), 1);
 354         TEST_EQUAL(database.get_collection_freq("rising"), 1);
 355         TEST_EQUAL(database.get_termfreq("falling"), 0);
 356         TEST_EQUAL(database.get_collection_freq("falling"), 0);
 357
 358         Xapian::docid did2 = database.add_document(document_in2);
 359         TEST_EQUAL(database.get_doccount(), 2);
 360         TEST_NOT_EQUAL(did, did2);
 361         TEST_EQUAL(database.get_avlength(), 5.0 / 2.0);
 362
 363         TEST_EQUAL(database.get_termfreq("foobar"), 2);
 364         TEST_EQUAL(database.get_collection_freq("foobar"), 3);
 365         TEST_EQUAL(database.get_termfreq("rising"), 1);
 366         TEST_EQUAL(database.get_collection_freq("rising"), 1);
 367         TEST_EQUAL(database.get_termfreq("falling"), 1);
 368         TEST_EQUAL(database.get_collection_freq("falling"), 1);
 369
 370         database.delete_document(did);
 371         TEST_EQUAL(database.get_doccount(), 1);
 372         TEST_EQUAL(database.get_avlength(), 2);
 373
 374         TEST_EQUAL(database.get_termfreq("foobar"), 1);
 375         TEST_EQUAL(database.get_collection_freq("foobar"), 1);
 376         TEST_EQUAL(database.get_termfreq("rising"), 0);
 377         TEST_EQUAL(database.get_collection_freq("rising"), 0);
 378         TEST_EQUAL(database.get_termfreq("falling"), 1);
 379         TEST_EQUAL(database.get_collection_freq("falling"), 1);
 380
 381         did = database.add_document(document_in);
 382         TEST_EQUAL(database.get_doccount(), 2);
 383         TEST_EQUAL(database.get_avlength(), 5.0 / 2.0);
 384
 385         TEST_EQUAL(database.get_termfreq("foobar"), 2);
 386         TEST_EQUAL(database.get_collection_freq("foobar"), 3);
 387         TEST_EQUAL(database.get_termfreq("rising"), 1);
 388         TEST_EQUAL(database.get_collection_freq("rising"), 1);
 389         TEST_EQUAL(database.get_termfreq("falling"), 1);
 390         TEST_EQUAL(database.get_collection_freq("falling"), 1);
 391     }
 392
 393     {
 394         Xapian::Database database(get_writable_database_as_database());
 395         Xapian::Document document_out = database.get_document(did);
 396
 397         TEST_EQUAL(document_in.get_data(), document_out.get_data());
 398
 399         {
 400             Xapian::ValueIterator i(document_in.values_begin());
 401             Xapian::ValueIterator j(document_out.values_begin());
 402             for (; i != document_in.values_end(); i++, j++) {
 403                 TEST_NOT_EQUAL(j, document_out.values_end());
 404                 TEST_EQUAL(*i, *j);
 405                 TEST_EQUAL(i.get_valueno(), j.get_valueno());
 406             }
 407             TEST_EQUAL(j, document_out.values_end());
 408         }
 409
 410         {
 411             // Regression test for bug fixed in 1.0.5 - values_begin() didn't
 412             // ensure that values had been read.  However, values_end() did
 413             // (and so did values_count()) so this wasn't generally an issue
 414             // but it shouldn't happen anyway.
 415             Xapian::Document doc_tmp = database.get_document(did);
 416             Xapian::ValueIterator i = document_in.values_begin();
 417             Xapian::ValueIterator j = doc_tmp.values_begin();
 418             TEST_EQUAL(*i, *j);
 419         }
 420
 421         {
 422             Xapian::TermIterator i(document_in.termlist_begin());
 423             Xapian::TermIterator j(document_out.termlist_begin());
 424             for (; i != document_in.termlist_end(); i++, j++) {
 425                 TEST_NOT_EQUAL(j, document_out.termlist_end());
 426                 TEST_EQUAL(*i, *j);
 427                 TEST_EQUAL(i.get_wdf(), j.get_wdf());
 428                 // Actually use termfreq to stop compiler optimising away the
 429                 // call to get_termfreq().
 430                 TEST_EXCEPTION(Xapian::InvalidOperationError,
 431                                if (i.get_termfreq()) return false);
 432                 TEST_NOT_EQUAL(0, j.get_termfreq());
 433                 if (*i == "foobar") {
 434                     // termfreq of foobar is 2
 435                     TEST_EQUAL(2, j.get_termfreq());
 436                 } else {
 437                     // termfreq of rising is 1
 438                     TEST_EQUAL(*i, "rising");
 439                     TEST_EQUAL(1, j.get_termfreq());
 440                 }
 441                 Xapian::PositionIterator k(i.positionlist_begin());
 442                 Xapian::PositionIterator l(j.positionlist_begin());
 443                 for (; k != i.positionlist_end(); k++, l++) {
 444                     TEST_NOT_EQUAL(l, j.positionlist_end());
 445                     TEST_EQUAL(*k, *l);
 446                 }
 447                 TEST_EQUAL(l, j.positionlist_end());
 448             }
 449             TEST_EQUAL(j, document_out.termlist_end());
 450         }
 451     }
 452
 453     return true;
 454 }
 455
 456 // Test adding a document, and checking that it got added correctly.
 457 // This testcase used to be adddoc3 in quartztest.
 458 DEFINE_TESTCASE(adddoc6, writable) {
 459     // Inmemory doesn't support get_writable_database_again().
 460     SKIP_TEST_FOR_BACKEND("inmemory");
 461
 462     Xapian::docid did;
 463     Xapian::Document document_in;
 464     document_in.set_data("Foobar rising");
 465     document_in.add_value(7, "Value7");
 466     document_in.add_value(13, "Value13");
 467     document_in.add_posting("foo", 1);
 468     document_in.add_posting("bar", 2);
 469
 470     {
 471         Xapian::WritableDatabase database(get_writable_database());
 472
 473         did = database.add_document(document_in);
 474         TEST_EQUAL(did, 1);
 475         TEST_EQUAL(database.get_doccount(), 1);
 476         TEST_EQUAL(database.get_avlength(), 2);
 477     }
 478
 479     {
 480         Xapian::WritableDatabase database(get_writable_database_again());
 481
 482         document_in.remove_term("foo");
 483         document_in.add_posting("baz", 1);
 484
 485         database.replace_document(1, document_in);
 486
 487         database.delete_document(1);
 488
 489         TEST_EQUAL(database.get_doccount(), 0);
 490         TEST_EQUAL(database.get_avlength(), 0);
 491         TEST_EQUAL(database.get_termfreq("foo"), 0);
 492         TEST_EQUAL(database.get_collection_freq("foo"), 0);
 493         TEST_EQUAL(database.get_termfreq("bar"), 0);
 494         TEST_EQUAL(database.get_collection_freq("bar"), 0);
 495         TEST_EQUAL(database.get_termfreq("baz"), 0);
 496         TEST_EQUAL(database.get_collection_freq("baz"), 0);
 497     }
 498
 499     return true;
 500 }
 501
 502 // tests that database destructors commit if it isn't done explicitly
 503 DEFINE_TESTCASE(implicitendsession1, writable) {
 504     Xapian::WritableDatabase db = get_writable_database();
 505
 506     Xapian::Document doc;
 507
 508     doc.set_data(string("top secret"));
 509     doc.add_posting("cia", 1);
 510     doc.add_posting("nsa", 2);
 511     doc.add_posting("fbi", 3);
 512     db.add_document(doc);
 513
 514     return true;
 515 }
 516
 517 // tests that assignment of Xapian::Database and Xapian::WritableDatabase works
 518 // as expected
 519 DEFINE_TESTCASE(databaseassign1, writable) {
 520     Xapian::WritableDatabase wdb = get_writable_database();
 521     Xapian::Database db = get_database("");
 522     Xapian::Database actually_wdb = wdb;
 523     Xapian::WritableDatabase w1(wdb);
 524     w1 = wdb;
 525     Xapian::Database d1(wdb);
 526     Xapian::Database d2(actually_wdb);
 527     d2 = wdb;
 528     d2 = actually_wdb;
 529     wdb = wdb; // check assign to itself works
 530     db = db; // check assign to itself works
 531     return true;
 532 }
 533
 534 // tests that deletion and updating of documents works as expected
 535 DEFINE_TESTCASE(deldoc1, writable) {
 536     Xapian::WritableDatabase db = get_writable_database();
 537
 538     Xapian::Document doc1;
 539
 540     doc1.add_posting("foo", 1);
 541     doc1.add_posting("foo", 1);
 542     doc1.add_posting("foo", 2);
 543     doc1.add_posting("foo", 2);
 544     doc1.add_posting("bar", 3);
 545     doc1.add_posting("gone", 1);
 546
 547     Xapian::docid did = db.add_document(doc1);
 548     TEST_EQUAL(did, 1);
 549
 550     doc1.remove_term("gone");
 551
 552     did = db.add_document(doc1);
 553     TEST_EQUAL(did, 2);
 554
 555     doc1.add_term("new", 1);
 556     did = db.add_document(doc1);
 557     TEST_EQUAL(did, 3);
 558
 559     db.delete_document(1);
 560
 561     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_document(1));
 562
 563     doc1 = db.get_document(2);
 564     doc1.remove_term("foo");
 565     doc1.add_term("fwing");
 566     db.replace_document(2, doc1);
 567
 568     Xapian::Document doc2 = db.get_document(2);
 569     Xapian::TermIterator tit = doc2.termlist_begin();
 570     TEST_NOT_EQUAL(tit, doc2.termlist_end());
 571     TEST_EQUAL(*tit, "bar");
 572     tit++;
 573     TEST_NOT_EQUAL(tit, doc2.termlist_end());
 574     TEST_EQUAL(*tit, "fwing");
 575     tit++;
 576     TEST_EQUAL(tit, doc2.termlist_end());
 577
 578     return true;
 579 }
 580
 581 // tests that deletion and updating of documents works as expected
 582 DEFINE_TESTCASE(deldoc2, writable) {
 583     Xapian::WritableDatabase db = get_writable_database();
 584
 585     Xapian::Document doc1;
 586
 587     doc1.add_posting("one", 1);
 588     doc1.add_posting("two", 2);
 589     doc1.add_posting("two", 3);
 590     Xapian::docid did;
 591
 592     did = db.add_document(doc1);
 593     TEST_EQUAL(did, 1);
 594
 595     doc1.remove_term("one");
 596     doc1.add_posting("three", 4);
 597
 598     did = db.add_document(doc1);
 599     TEST_EQUAL(did, 2);
 600
 601     doc1.add_posting("one", 7);
 602     doc1.remove_term("two");
 603
 604     did = db.add_document(doc1);
 605     TEST_EQUAL(did, 3);
 606
 607     db.commit();
 608
 609     // reopen() on a writable database shouldn't do anything.
 610     TEST(!db.reopen());
 611
 612     db.delete_document(1);
 613     db.delete_document(2);
 614     db.delete_document(3);
 615
 616     db.commit();
 617
 618     // reopen() on a writable database shouldn't do anything.
 619     TEST(!db.reopen());
 620
 621     TEST_EQUAL(db.postlist_begin("one"), db.postlist_end("one"));
 622     TEST_EQUAL(db.postlist_begin("two"), db.postlist_end("two"));
 623     TEST_EQUAL(db.postlist_begin("three"), db.postlist_end("three"));
 624
 625     TEST_EXCEPTION(Xapian::DocNotFoundError, db.termlist_begin(1));
 626     TEST_EXCEPTION(Xapian::DocNotFoundError, db.termlist_begin(2));
 627     TEST_EXCEPTION(Xapian::DocNotFoundError, db.termlist_begin(3));
 628     TEST_EXCEPTION(Xapian::DocNotFoundError, db.termlist_begin(4));
 629
 630     // test positionlist_{begin,end}?
 631
 632     TEST_EQUAL(db.get_doccount(), 0);
 633     TEST_EQUAL(db.get_avlength(), 0);
 634     TEST_EQUAL(db.get_termfreq("one"), 0);
 635     TEST_EQUAL(db.get_termfreq("two"), 0);
 636     TEST_EQUAL(db.get_termfreq("three"), 0);
 637
 638     TEST(!db.term_exists("one"));
 639     TEST(!db.term_exists("two"));
 640     TEST(!db.term_exists("three"));
 641
 642     TEST_EQUAL(db.get_collection_freq("one"), 0);
 643     TEST_EQUAL(db.get_collection_freq("two"), 0);
 644     TEST_EQUAL(db.get_collection_freq("three"), 0);
 645
 646     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_doclength(1));
 647     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_doclength(2));
 648     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_doclength(3));
 649
 650     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_unique_terms(1));
 651     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_unique_terms(2));
 652     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_unique_terms(3));
 653
 654     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_document(1));
 655     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_document(2));
 656     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_document(3));
 657
 658     TEST_EQUAL(db.allterms_begin(), db.allterms_end());
 659
 660     return true;
 661 }
 662
 663 // another test of deletion of documents, a cut-down version of deldoc2
 664 DEFINE_TESTCASE(deldoc3, writable) {
 665     Xapian::WritableDatabase db = get_writable_database();
 666
 667     Xapian::Document doc1;
 668
 669     doc1.add_posting("one", 1);
 670
 671     Xapian::docid did = db.add_document(doc1);
 672     TEST_EQUAL(did, 1);
 673
 674     db.commit();
 675
 676     // reopen() on a writable database shouldn't do anything.
 677     TEST(!db.reopen());
 678
 679     db.delete_document(1);
 680
 681     db.commit();
 682
 683     // reopen() on a writable database shouldn't do anything.
 684     TEST(!db.reopen());
 685
 686     TEST_EQUAL(db.postlist_begin("one"), db.postlist_end("one"));
 687
 688     TEST_EXCEPTION(Xapian::DocNotFoundError, db.termlist_begin(1));
 689     TEST_EXCEPTION(Xapian::DocNotFoundError, db.termlist_begin(2));
 690
 691     // test positionlist_{begin,end}?
 692
 693     TEST_EQUAL(db.get_doccount(), 0);
 694     TEST_EQUAL(db.get_avlength(), 0);
 695     TEST_EQUAL(db.get_termfreq("one"), 0);
 696
 697     TEST(!db.term_exists("one"));
 698
 699     TEST_EQUAL(db.get_collection_freq("one"), 0);
 700
 701     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_doclength(1));
 702     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_doclength(2));
 703
 704     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_unique_terms(1));
 705     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_unique_terms(2));
 706
 707     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_document(1));
 708     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_document(2));
 709
 710     TEST_EQUAL(db.allterms_begin(), db.allterms_end());
 711
 712     return true;
 713 }
 714
 715 // tests that deletion and updating of (lots of) documents works as expected
 716 DEFINE_TESTCASE(deldoc4, writable) {
 717     Xapian::WritableDatabase db = get_writable_database();
 718
 719     Xapian::Document doc1;
 720
 721     doc1.add_posting("one", 1);
 722     doc1.add_posting("two", 2);
 723     doc1.add_posting("two", 3);
 724
 725     Xapian::Document doc2 = doc1;
 726     doc2.remove_term("one");
 727     doc2.add_posting("three", 4);
 728
 729     Xapian::Document doc3 = doc2;
 730     doc3.add_posting("one", 7);
 731     doc3.remove_term("two");
 732
 733     const Xapian::docid maxdoc = 1000 * 3;
 734     Xapian::docid did;
 735     for (Xapian::docid i = 0; i < maxdoc / 3; ++i) {
 736         did = db.add_document(doc1);
 737         TEST_EQUAL(did, i * 3 + 1);
 738         did = db.add_document(doc2);
 739         TEST_EQUAL(did, i * 3 + 2);
 740         did = db.add_document(doc3);
 741         TEST_EQUAL(did, i * 3 + 3);
 742
 743         bool is_power_of_two = ((i & (i - 1)) == 0);
 744         if (is_power_of_two) {
 745             db.commit();
 746             // reopen() on a writable database shouldn't do anything.
 747             TEST(!db.reopen());
 748         }
 749     }
 750     db.commit();
 751     // reopen() on a writable database shouldn't do anything.
 752     TEST(!db.reopen());
 753
 754     /* delete the documents in a peculiar order */
 755     for (Xapian::docid i = 0; i < maxdoc / 3; ++i) {
 756         db.delete_document(maxdoc - i);
 757         db.delete_document(maxdoc / 3 + i + 1);
 758         db.delete_document(i + 1);
 759     }
 760
 761     db.commit();
 762     // reopen() on a writable database shouldn't do anything.
 763     TEST(!db.reopen());
 764
 765     TEST_EQUAL(db.postlist_begin("one"), db.postlist_end("one"));
 766     TEST_EQUAL(db.postlist_begin("two"), db.postlist_end("two"));
 767     TEST_EQUAL(db.postlist_begin("three"), db.postlist_end("three"));
 768
 769     for (Xapian::docid i = 1; i <= maxdoc; ++i) {
 770         // TEST_EXCEPTION writes to tout each time if the test is run
 771         // in verbose mode and some string stream implementations get
 772         // very inefficient with large strings, so clear tout on each pass of
 773         // the loop to speed up the test since the older information isn't
 774         // interesting anyway.
 775         tout.str(string());
 776         TEST_EXCEPTION(Xapian::DocNotFoundError, db.termlist_begin(i));
 777         TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_doclength(i));
 778         TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_unique_terms(i));
 779         TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_document(i));
 780     }
 781
 782     // test positionlist_{begin,end}?
 783
 784     TEST_EQUAL(db.get_doccount(), 0);
 785     TEST_EQUAL(db.get_avlength(), 0);
 786     TEST_EQUAL(db.get_termfreq("one"), 0);
 787     TEST_EQUAL(db.get_termfreq("two"), 0);
 788     TEST_EQUAL(db.get_termfreq("three"), 0);
 789
 790     TEST(!db.term_exists("one"));
 791     TEST(!db.term_exists("two"));
 792     TEST(!db.term_exists("three"));
 793
 794     TEST_EQUAL(db.get_collection_freq("one"), 0);
 795     TEST_EQUAL(db.get_collection_freq("two"), 0);
 796     TEST_EQUAL(db.get_collection_freq("three"), 0);
 797
 798     TEST_EQUAL(db.allterms_begin(), db.allterms_end());
 799
 800     return true;
 801 }
 802
 803 // Test deleting a document which was added in the same batch.
 804 DEFINE_TESTCASE(deldoc5, writable) {
 805     Xapian::WritableDatabase db = get_writable_database();
 806
 807     Xapian::Document doc1;
 808
 809     doc1.add_posting("foo", 1);
 810     doc1.add_posting("bar", 2);
 811     doc1.add_posting("aardvark", 3);
 812
 813     Xapian::docid did = db.add_document(doc1);
 814     TEST_EQUAL(did, 1);
 815
 816     doc1.remove_term("bar");
 817     doc1.add_term("hello");
 818
 819     did = db.add_document(doc1);
 820     TEST_EQUAL(did, 2);
 821
 822     doc1.add_term("world", 1);
 823     did = db.add_document(doc1);
 824     TEST_EQUAL(did, 3);
 825
 826     db.delete_document(2);
 827
 828     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_document(2));
 829
 830     db.commit();
 831
 832     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_document(2));
 833
 834     TEST_EQUAL(db.get_termfreq("foo"), 2);
 835     TEST_EQUAL(db.get_termfreq("aardvark"), 2);
 836     TEST_EQUAL(db.get_termfreq("hello"), 1);
 837
 838     Xapian::PostingIterator p = db.postlist_begin("foo");
 839     TEST_NOT_EQUAL(p, db.postlist_end("foo"));
 840     TEST_EQUAL(*p, 1);
 841     ++p;
 842     TEST_NOT_EQUAL(p, db.postlist_end("foo"));
 843     TEST_EQUAL(*p, 3);
 844     ++p;
 845     TEST_EQUAL(p, db.postlist_end("foo"));
 846
 847     return true;
 848 }
 849
 850 // Regression test for bug in quartz and flint, fixed in 1.0.2.
 851 DEFINE_TESTCASE(deldoc6, writable) {
 852     Xapian::WritableDatabase db = get_writable_database();
 853
 854     Xapian::Document doc1;
 855
 856     doc1.add_posting("foo", 1);
 857     doc1.add_posting("bar", 2);
 858     doc1.add_posting("aardvark", 3);
 859
 860     Xapian::docid did = db.add_document(doc1);
 861     TEST_EQUAL(did, 1);
 862
 863     doc1.remove_term("bar");
 864     doc1.add_term("hello");
 865
 866     did = db.add_document(doc1);
 867     TEST_EQUAL(did, 2);
 868
 869     db.commit();
 870
 871     db.delete_document(2);
 872     TEST_EXCEPTION(Xapian::DocNotFoundError, db.delete_document(3));
 873
 874     db.commit();
 875
 876     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_document(2));
 877
 878     return true;
 879 }
 880
 881 DEFINE_TESTCASE(replacedoc1, writable) {
 882     Xapian::WritableDatabase db = get_writable_database();
 883
 884     Xapian::Document doc1;
 885
 886     doc1.add_posting("foo", 1);
 887     doc1.add_posting("foo", 2);
 888     doc1.add_posting("gone", 3);
 889     doc1.add_posting("bar", 4);
 890     doc1.add_posting("foo", 5);
 891     Xapian::docid did;
 892
 893     did = db.add_document(doc1);
 894     TEST_EQUAL(did, 1);
 895
 896     Xapian::Document doc2;
 897
 898     doc2.add_posting("foo", 1);
 899     doc2.add_posting("pipco", 2);
 900     doc2.add_posting("bar", 4);
 901     doc2.add_posting("foo", 5);
 902
 903     db.replace_document(did, doc2);
 904
 905     Xapian::Document doc3 = db.get_document(did);
 906     Xapian::TermIterator tIter = doc3.termlist_begin();
 907     TEST_EQUAL(*tIter, "bar");
 908     Xapian::PositionIterator pIter = tIter.positionlist_begin();
 909     TEST_EQUAL(*pIter, 4);
 910     ++tIter;
 911     TEST_EQUAL(*tIter, "foo");
 912     Xapian::PositionIterator qIter = tIter.positionlist_begin();
 913     TEST_EQUAL(*qIter, 1);
 914     ++qIter;
 915     TEST_EQUAL(*qIter, 5);
 916     ++tIter;
 917     TEST_EQUAL(*tIter, "pipco");
 918     Xapian::PositionIterator rIter = tIter.positionlist_begin();
 919     TEST_EQUAL(*rIter, 2);
 920     ++tIter;
 921     TEST_EQUAL(tIter, doc3.termlist_end());
 922     return true;
 923 }
 924
 925 // Test of new feature: WritableDatabase::replace_document accepts a docid
 926 // which doesn't yet exist as of Xapian 0.8.2.
 927 DEFINE_TESTCASE(replacedoc2, writable) {
 928     Xapian::WritableDatabase db = get_writable_database();
 929
 930     Xapian::Document doc1;
 931
 932     doc1.add_posting("foo", 1);
 933     doc1.add_posting("foo", 2);
 934     doc1.add_posting("gone", 3);
 935     doc1.add_posting("bar", 4);
 936     doc1.add_posting("foo", 5);
 937     Xapian::docid did = 31770;
 938
 939     db.replace_document(did, doc1);
 940
 941     // Regression tests for bug in the InMemory backend - fixed in 1.0.2.
 942     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_document(1));
 943     Xapian::PostingIterator postit = db.postlist_begin("");
 944     TEST(postit != db.postlist_end(""));
 945     TEST_EQUAL(*postit, 31770);
 946
 947     Xapian::Document doc2;
 948
 949     doc2.add_posting("foo", 1);
 950     doc2.add_posting("pipco", 2);
 951     doc2.add_posting("bar", 4);
 952     doc2.add_posting("foo", 5);
 953
 954     db.replace_document(did, doc2);
 955     TEST_EQUAL(db.get_doccount(), 1);
 956
 957     Xapian::Document doc3 = db.get_document(did);
 958     Xapian::TermIterator tIter = doc3.termlist_begin();
 959     TEST_EQUAL(*tIter, "bar");
 960     Xapian::PositionIterator pIter = tIter.positionlist_begin();
 961     TEST_EQUAL(*pIter, 4);
 962     ++tIter;
 963     TEST_EQUAL(*tIter, "foo");
 964     Xapian::PositionIterator qIter = tIter.positionlist_begin();
 965     TEST_EQUAL(*qIter, 1);
 966     ++qIter;
 967     TEST_EQUAL(*qIter, 5);
 968     ++tIter;
 969     TEST_EQUAL(*tIter, "pipco");
 970     Xapian::PositionIterator rIter = tIter.positionlist_begin();
 971     TEST_EQUAL(*rIter, 2);
 972     ++tIter;
 973     TEST_EQUAL(tIter, doc3.termlist_end());
 974
 975     did = db.add_document(doc1);
 976     TEST_EQUAL(did, 31771);
 977     TEST_EQUAL(db.get_doccount(), 2);
 978
 979     TEST_EXCEPTION(Xapian::InvalidArgumentError, db.replace_document(0, doc2));
 980
 981     return true;
 982 }
 983
 984 // Test replacing a document which was added in the same batch.
 985 DEFINE_TESTCASE(replacedoc3, writable) {
 986     Xapian::WritableDatabase db = get_writable_database();
 987
 988     Xapian::Document doc1;
 989
 990     doc1.add_posting("foo", 1);
 991     doc1.add_posting("bar", 2);
 992     doc1.add_posting("aardvark", 3);
 993
 994     Xapian::docid did = db.add_document(doc1);
 995     TEST_EQUAL(did, 1);
 996
 997     doc1.remove_term("bar");
 998     doc1.add_term("hello");
 999
1000     did = db.add_document(doc1);
1001     TEST_EQUAL(did, 2);
1002
1003     doc1.add_term("world", 1);
1004     did = db.add_document(doc1);
1005     TEST_EQUAL(did, 3);
1006
1007     Xapian::Document doc2;
1008     doc2.add_term("world");
1009     db.replace_document(2, doc2);
1010
1011     db.commit();
1012
1013     // Check that the document exists (no DocNotFoundError).
1014     doc2 = db.get_document(2);
1015
1016     TEST_EQUAL(db.get_termfreq("foo"), 2);
1017     TEST_EQUAL(db.get_termfreq("aardvark"), 2);
1018     TEST_EQUAL(db.get_termfreq("hello"), 1);
1019     TEST_EQUAL(db.get_termfreq("world"), 2);
1020
1021     TEST_EQUAL(db.get_doclength(1), 3);
1022     TEST_EQUAL(db.get_doclength(2), 1);
1023     TEST_EQUAL(db.get_doclength(3), 4);
1024
1025     TEST_EQUAL(db.get_unique_terms(1), 3);
1026     TEST_EQUAL(db.get_unique_terms(2), 1);
1027     TEST_EQUAL(db.get_unique_terms(3), 4);
1028
1029     Xapian::PostingIterator p = db.postlist_begin("foo");
1030     TEST_NOT_EQUAL(p, db.postlist_end("foo"));
1031     TEST_EQUAL(*p, 1);
1032     TEST_EQUAL(p.get_doclength(), 3);
1033     TEST_EQUAL(p.get_unique_terms(), 3);
1034     ++p;
1035     TEST_NOT_EQUAL(p, db.postlist_end("foo"));
1036     TEST_EQUAL(*p, 3);
1037     TEST_EQUAL(p.get_doclength(), 4);
1038     TEST_EQUAL(p.get_unique_terms(), 4);
1039     ++p;
1040     TEST_EQUAL(p, db.postlist_end("foo"));
1041
1042     p = db.postlist_begin("world");
1043     TEST_NOT_EQUAL(p, db.postlist_end("world"));
1044     TEST_EQUAL(*p, 2);
1045     TEST_EQUAL(p.get_doclength(), 1);
1046     TEST_EQUAL(p.get_unique_terms(), 1);
1047     ++p;
1048     TEST_NOT_EQUAL(p, db.postlist_end("world"));
1049     TEST_EQUAL(*p, 3);
1050     TEST_EQUAL(p.get_doclength(), 4);
1051     TEST_EQUAL(p.get_unique_terms(), 4);
1052     ++p;
1053     TEST_EQUAL(p, db.postlist_end("world"));
1054
1055     return true;
1056 }
1057
1058 // Test replacing a document which was deleted in the same batch.
1059 DEFINE_TESTCASE(replacedoc4, writable) {
1060     Xapian::WritableDatabase db = get_writable_database();
1061
1062     Xapian::Document doc1;
1063
1064     doc1.add_posting("foo", 1);
1065     doc1.add_posting("bar", 2);
1066     doc1.add_posting("aardvark", 3);
1067
1068     Xapian::docid did = db.add_document(doc1);
1069     TEST_EQUAL(did, 1);
1070
1071     doc1.remove_term("bar");
1072     doc1.add_term("hello");
1073
1074     did = db.add_document(doc1);
1075     TEST_EQUAL(did, 2);
1076
1077     doc1.add_term("world", 1);
1078     did = db.add_document(doc1);
1079     TEST_EQUAL(did, 3);
1080
1081     db.delete_document(2);
1082
1083     Xapian::Document doc2;
1084     doc2.add_term("world");
1085     db.replace_document(2, doc2);
1086
1087     db.commit();
1088
1089     // Check that the document exists (no DocNotFoundError).
1090     doc2 = db.get_document(2);
1091
1092     TEST_EQUAL(db.get_termfreq("foo"), 2);
1093     TEST_EQUAL(db.get_termfreq("aardvark"), 2);
1094     TEST_EQUAL(db.get_termfreq("hello"), 1);
1095     TEST_EQUAL(db.get_termfreq("world"), 2);
1096
1097     Xapian::PostingIterator p = db.postlist_begin("foo");
1098     TEST_NOT_EQUAL(p, db.postlist_end("foo"));
1099     TEST_EQUAL(*p, 1);
1100     ++p;
1101     TEST_NOT_EQUAL(p, db.postlist_end("foo"));
1102     TEST_EQUAL(*p, 3);
1103     ++p;
1104     TEST_EQUAL(p, db.postlist_end("foo"));
1105
1106     p = db.postlist_begin("world");
1107     TEST_NOT_EQUAL(p, db.postlist_end("world"));
1108     TEST_EQUAL(*p, 2);
1109     ++p;
1110     TEST_NOT_EQUAL(p, db.postlist_end("world"));
1111     TEST_EQUAL(*p, 3);
1112     ++p;
1113     TEST_EQUAL(p, db.postlist_end("world"));
1114
1115     return true;
1116 }
1117
1118 // Test replacing a document with itself without modifying postings.
1119 // Regression test for bug in 0.9.9 and earlier - there flint and quartz
1120 // lost all positional information for the document when you did this.
1121 DEFINE_TESTCASE(replacedoc5, writable) {
1122     Xapian::WritableDatabase db = get_writable_database();
1123
1124     {
1125         Xapian::Document doc;
1126         doc.add_posting("hello", 1);
1127         doc.add_posting("world", 2);
1128
1129         Xapian::docid did = db.add_document(doc);
1130         TEST_EQUAL(did, 1);
1131         db.commit();
1132     }
1133
1134     {
1135         Xapian::Document doc = db.get_document(1);
1136         TEST(db.has_positions());
1137         TEST(db.positionlist_begin(1, "hello") != db.positionlist_end(1, "hello"));
1138         TEST(db.positionlist_begin(1, "world") != db.positionlist_end(1, "world"));
1139         db.replace_document(1, doc);
1140         db.commit();
1141
1142         TEST(db.has_positions());
1143         TEST(db.positionlist_begin(1, "hello") != db.positionlist_end(1, "hello"));
1144         TEST(db.positionlist_begin(1, "world") != db.positionlist_end(1, "world"));
1145     }
1146
1147     // The backends now spot simple cases of replacing the same document and
1148     // don't do needless work.  Force them to actually do the replacement to
1149     // make sure that case works.
1150
1151     {
1152         Xapian::Document doc;
1153         doc.add_term("Q2");
1154         db.add_document(doc);
1155         db.commit();
1156     }
1157
1158     {
1159         Xapian::Document doc = db.get_document(1);
1160         TEST(db.has_positions());
1161         TEST(db.positionlist_begin(1, "hello") != db.positionlist_end(1, "hello"));
1162         TEST(db.positionlist_begin(1, "world") != db.positionlist_end(1, "world"));
1163         (void)db.get_document(2);
1164         db.replace_document(1, doc);
1165         db.commit();
1166
1167         TEST(db.has_positions());
1168         TEST(db.positionlist_begin(1, "hello") != db.positionlist_end(1, "hello"));
1169         TEST(db.positionlist_begin(1, "world") != db.positionlist_end(1, "world"));
1170     }
1171
1172     return true;
1173 }
1174
1175 // Test replacing a document while adding values, without changing anything
1176 // else.  Regression test for a bug introduced while implementing lazy update,
1177 // and also covers a few other code paths.
1178 DEFINE_TESTCASE(replacedoc6, writable) {
1179     Xapian::WritableDatabase db = get_writable_database();
1180
1181     Xapian::Document doc;
1182     Xapian::docid did = db.add_document(doc);
1183     TEST_EQUAL(did, 1);
1184     db.commit();
1185
1186     // Add document
1187     doc = db.get_document(1);
1188     TEST_EQUAL(doc.get_value(1), "");
1189     TEST_EQUAL(doc.get_value(2), "");
1190     doc.add_value(1, "banana1");
1191     db.replace_document(1, doc);
1192
1193     doc = db.get_document(1);
1194     TEST_EQUAL(doc.get_value(1), "banana1");
1195     TEST_EQUAL(doc.get_value(2), "");
1196     db.commit();
1197
1198     doc = db.get_document(1);
1199     TEST_EQUAL(doc.get_value(1), "banana1");
1200     TEST_EQUAL(doc.get_value(2), "");
1201     doc.add_value(2, "banana2");
1202     db.replace_document(1, doc);
1203
1204     TEST_EQUAL(doc.get_value(1), "banana1");
1205     TEST_EQUAL(doc.get_value(2), "banana2");
1206     db.commit();
1207
1208     doc = db.get_document(1);
1209     TEST_EQUAL(doc.get_value(1), "banana1");
1210     TEST_EQUAL(doc.get_value(2), "banana2");
1211
1212     return true;
1213 }
1214
1215 // Test of new feature: WritableDatabase::replace_document and delete_document
1216 // can take a unique termname instead of a document id as of Xapian 0.8.2.
1217 DEFINE_TESTCASE(uniqueterm1, writable) {
1218     Xapian::WritableDatabase db = get_writable_database();
1219
1220     for (int n = 1; n <= 20; ++n) {
1221         Xapian::Document doc;
1222         string uterm = "U" + str(n % 16);
1223         doc.add_term(uterm);
1224         doc.add_term(str(n));
1225         doc.add_term(str(n ^ 9));
1226         doc.add_term("all");
1227         doc.set_data("pass1");
1228         db.add_document(doc);
1229     }
1230
1231     TEST_EQUAL(db.get_doccount(), 20);
1232
1233     static const Xapian::doccount sizes[20] = {
1234         19, 17, 16, 15,
1235         15, 15, 15, 15,
1236         15, 15, 15, 15,
1237         15, 15, 15, 15,
1238         15, 15, 15, 15
1239     };
1240     for (int n = 1; n <= 20; ++n) {
1241         string uterm = "U" + str(n % 16);
1242         if (uterm == "U2") {
1243             db.delete_document(uterm);
1244         } else {
1245             Xapian::Document doc;
1246             doc.add_term(uterm);
1247             doc.add_term(str(n));
1248             doc.add_term(str(n ^ 9));
1249             doc.add_term("all");
1250             doc.set_data("pass2");
1251             db.replace_document(uterm, doc);
1252         }
1253         TEST_EQUAL(db.get_doccount(), sizes[n - 1]);
1254     }
1255
1256     string uterm = "U571";
1257     Xapian::Document doc;
1258     doc.add_term(uterm);
1259     doc.set_data("pass3");
1260     db.replace_document(uterm, doc);
1261
1262     TEST_EQUAL(db.get_doccount(), 16);
1263
1264     db.delete_document("U2");
1265
1266     TEST_EQUAL(db.get_doccount(), 16);
1267
1268     return true;
1269 }
1270
1271 // tests all document postlists
1272 DEFINE_TESTCASE(allpostlist2, writable) {
1273     Xapian::WritableDatabase db(get_writable_database("apitest_manydocs"));
1274     Xapian::PostingIterator i = db.postlist_begin("");
1275     unsigned int j = 1;
1276     while (i != db.postlist_end("")) {
1277         TEST_EQUAL(*i, j);
1278         i++;
1279         j++;
1280     }
1281     TEST_EQUAL(j, 513);
1282
1283     db.delete_document(1);
1284     db.delete_document(50);
1285     db.delete_document(512);
1286
1287     i = db.postlist_begin("");
1288     j = 2;
1289     while (i != db.postlist_end("")) {
1290         TEST_EQUAL(*i, j);
1291         i++;
1292         j++;
1293         if (j == 50) j++;
1294     }
1295     TEST_EQUAL(j, 512);
1296
1297     i = db.postlist_begin("");
1298     j = 2;
1299     while (i != db.postlist_end("")) {
1300         TEST_EQUAL(*i, j);
1301         i++;
1302         j++;
1303         if (j == 40) {
1304             j += 10;
1305             i.skip_to(j);
1306             j++;
1307         }
1308     }
1309     TEST_EQUAL(j, 512);
1310
1311     return true;
1312 }
1313
1314 static void test_emptyterm2_helper(Xapian::WritableDatabase & db)
1315 {
1316     // Don't bother with postlist_begin() because allpostlist tests cover that.
1317     TEST_EXCEPTION(Xapian::InvalidArgumentError, db.positionlist_begin(1, ""));
1318     TEST_EQUAL(db.get_doccount(), db.get_termfreq(""));
1319     TEST_EQUAL(db.get_doccount() != 0, db.term_exists(""));
1320     TEST_EQUAL(db.get_doccount(), db.get_collection_freq(""));
1321 }
1322
1323 // tests results of passing an empty term to various methods
1324 // equivalent of emptyterm1 for a writable database
1325 DEFINE_TESTCASE(emptyterm2, writable) {
1326     {
1327         Xapian::WritableDatabase db(get_writable_database("apitest_manydocs"));
1328         TEST_EQUAL(db.get_doccount(), 512);
1329         test_emptyterm2_helper(db);
1330         db.delete_document(1);
1331         TEST_EQUAL(db.get_doccount(), 511);
1332         test_emptyterm2_helper(db);
1333         db.delete_document(50);
1334         TEST_EQUAL(db.get_doccount(), 510);
1335         test_emptyterm2_helper(db);
1336         db.delete_document(512);
1337         TEST_EQUAL(db.get_doccount(), 509);
1338         test_emptyterm2_helper(db);
1339     }
1340
1341     {
1342         Xapian::WritableDatabase db(get_writable_database("apitest_onedoc"));
1343         TEST_EQUAL(db.get_doccount(), 1);
1344         test_emptyterm2_helper(db);
1345         db.delete_document(1);
1346         TEST_EQUAL(db.get_doccount(), 0);
1347         test_emptyterm2_helper(db);
1348     }
1349
1350     {
1351         Xapian::WritableDatabase db(get_writable_database());
1352         TEST_EQUAL(db.get_doccount(), 0);
1353         test_emptyterm2_helper(db);
1354     }
1355
1356     return true;
1357 }
1358
1359 // Check that PHRASE/NEAR becomes AND if there's no positional info in the
1360 // database.
1361 DEFINE_TESTCASE(phraseorneartoand1, writable) {
1362     Xapian::WritableDatabase db = get_writable_database();
1363
1364     for (int n = 1; n <= 20; ++n) {
1365         Xapian::Document doc;
1366         doc.add_term(str(n));
1367         doc.add_term(str(n ^ 9));
1368         doc.add_term("all");
1369         doc.set_data("pass1");
1370         db.add_document(doc);
1371     }
1372     db.commit();
1373
1374     Xapian::Enquire enquire(db);
1375     Xapian::MSet mymset;
1376
1377     const char * q1[] = { "all", "1" };
1378     enquire.set_query(Xapian::Query(Xapian::Query::OP_PHRASE, q1, q1 + 2));
1379     mymset = enquire.get_mset(0, 10);
1380     TEST_EQUAL(2, mymset.size());
1381
1382     enquire.set_query(Xapian::Query(Xapian::Query::OP_NEAR, q1, q1 + 2));
1383     mymset = enquire.get_mset(0, 10);
1384     TEST_EQUAL(2, mymset.size());
1385
1386     const char * q2[] = { "1", "2" };
1387     enquire.set_query(Xapian::Query(Xapian::Query::OP_PHRASE, q2, q2 + 2));
1388     mymset = enquire.get_mset(0, 10);
1389     TEST_EQUAL(0, mymset.size());
1390
1391     enquire.set_query(Xapian::Query(Xapian::Query::OP_NEAR, q2, q2 + 2));
1392     mymset = enquire.get_mset(0, 10);
1393     TEST_EQUAL(0, mymset.size());
1394
1395     return true;
1396 }
1397
1398 // Check that a large number of position list entries for a particular term
1399 // works - regression test for flint.
1400 DEFINE_TESTCASE(longpositionlist1, writable) {
1401     Xapian::WritableDatabase db = get_writable_database();
1402
1403     Xapian::Document doc;
1404     Xapian::termpos n;
1405     for (n = 1; n <= 2000; ++n) {
1406         doc.add_posting("fork", n * 3);
1407         doc.add_posting("knife", n * unsigned(log(double(n + 2))));
1408         doc.add_posting("spoon", n * n);
1409     }
1410     doc.set_data("cutlery");
1411     Xapian::docid did = db.add_document(doc);
1412     db.commit();
1413
1414     doc = db.get_document(did);
1415
1416     Xapian::TermIterator t, tend;
1417     Xapian::PositionIterator p, pend;
1418
1419     t = doc.termlist_begin();
1420     tend = doc.termlist_end();
1421
1422     TEST(t != tend);
1423     TEST_EQUAL(*t, "fork");
1424     p = t.positionlist_begin();
1425     pend = t.positionlist_end();
1426     for (n = 1; n <= 2000; ++n) {
1427         TEST(p != pend);
1428         TEST_EQUAL(*p, n * 3);
1429         ++p;
1430     }
1431     TEST(p == pend);
1432
1433     ++t;
1434     TEST(t != tend);
1435     TEST_EQUAL(*t, "knife");
1436     p = t.positionlist_begin();
1437     pend = t.positionlist_end();
1438     for (n = 1; n <= 2000; ++n) {
1439         TEST(p != pend);
1440         TEST_EQUAL(*p, n * unsigned(log(double(n + 2))));
1441         ++p;
1442     }
1443     TEST(p == pend);
1444
1445     ++t;
1446     TEST(t != tend);
1447     TEST_EQUAL(*t, "spoon");
1448     p = t.positionlist_begin();
1449     pend = t.positionlist_end();
1450     for (n = 1; n <= 2000; ++n) {
1451         TEST(p != pend);
1452         TEST_EQUAL(*p, n * n);
1453         ++p;
1454     }
1455     TEST(p == pend);
1456
1457     ++t;
1458     TEST(t == tend);
1459
1460     return true;
1461 }
1462
1463 // Regression test for bug#110: Inconsistent sort order between pages with
1464 // set_sort_by_value_then_relevance.
1465 DEFINE_TESTCASE(consistency2, writable) {
1466     Xapian::WritableDatabase db = get_writable_database();
1467     char buf[2] = "X";
1468     int i = 0;
1469
1470     // Add 5 documents indexed by "test" with wdf 1.
1471     for (i = 0; i < 5; ++i) {
1472         Xapian::Document doc;
1473         *buf = '0' + i;
1474         doc.add_value(0, buf);
1475         doc.add_term("test");
1476         db.add_document(doc);
1477     }
1478
1479     // Add 5 documents indexed by "test" with wdf 2.
1480     for (i = 0; i < 5; ++i) {
1481         Xapian::Document doc;
1482         *buf = '0' + i;
1483         doc.add_value(0, buf);
1484         doc.add_term("test", 2);
1485         db.add_document(doc);
1486     }
1487
1488     db.commit();
1489
1490     Xapian::Enquire enq(db);
1491     enq.set_query(Xapian::Query("test"));
1492
1493     enq.set_sort_by_value_then_relevance(0, true);
1494
1495     // 10 results, unpaged.
1496     Xapian::MSet mset1 = enq.get_mset(0, 10);
1497     TEST_EQUAL(mset1.size(), 10);
1498
1499     // 10 results, split.
1500     Xapian::MSet mset2a = enq.get_mset(0, 1);
1501     TEST_EQUAL(mset2a.size(), 1);
1502     Xapian::MSet mset2b = enq.get_mset(1, 1);
1503     TEST_EQUAL(mset2b.size(), 1);
1504     Xapian::MSet mset2c = enq.get_mset(2, 8);
1505     TEST_EQUAL(mset2c.size(), 8);
1506
1507     TEST_EQUAL(*mset1[0], *mset2a[0]);
1508     TEST_EQUAL(*mset1[1], *mset2b[0]);
1509     for (i = 0; i < 8; ++i) {
1510         TEST_EQUAL(*mset1[i + 2], *mset2c[i]);
1511     }
1512
1513     return true;
1514 }
1515
1516 // Check that DatabaseError is thrown if the docid counter would wrap.
1517 // Regression test for bug#152.
1518 DEFINE_TESTCASE(nomoredocids1, writable) {
1519     // The InMemory backend uses a vector for the documents, so trying to add
1520     // document "-1" will fail because we can't allocate enough memory!
1521     SKIP_TEST_FOR_BACKEND("inmemory");
1522
1523     Xapian::WritableDatabase db = get_writable_database();
1524     Xapian::Document doc;
1525     doc.set_data("prose");
1526     doc.add_term("word");
1527
1528     // FIXME: This probably should use the _MAX_DOCID values
1529     Xapian::docid max_id = 0xffffffff;
1530
1531     db.replace_document(max_id, doc);
1532
1533     TEST_EXCEPTION(Xapian::DatabaseError, db.add_document(doc));
1534
1535     return true;
1536 }
1537
1538 // Test synonym iterators.
1539 DEFINE_TESTCASE(synonymitor1, writable && synonyms) {
1540     Xapian::WritableDatabase db = get_writable_database();
1541
1542     // Test iterators for terms which aren't there.
1543     TEST(db.synonyms_begin("abc") == db.synonyms_end("abc"));
1544
1545     // Test iterating the synonym keys when there aren't any.
1546     TEST(db.synonym_keys_begin() == db.synonym_keys_end());
1547
1548     db.add_synonym("hello", "howdy");
1549     db.add_synonym("hello", "hi");
1550     db.add_synonym("goodbye", "bye");
1551     db.add_synonym("goodbye", "farewell");
1552
1553     Xapian::TermIterator t;
1554     string s;
1555
1556     // Try these tests twice - once before committing and once after.
1557     for (int times = 1; times <= 2; ++times) {
1558         // Test iterators for terms which aren't there.
1559         TEST(db.synonyms_begin("abc") == db.synonyms_end("abc"));
1560         TEST(db.synonyms_begin("ghi") == db.synonyms_end("ghi"));
1561         TEST(db.synonyms_begin("zzzzz") == db.synonyms_end("zzzzz"));
1562
1563         s = "|";
1564         t = db.synonyms_begin("hello");
1565         while (t != db.synonyms_end("hello")) {
1566             s += *t++;
1567             s += '|';
1568         }
1569         TEST_STRINGS_EQUAL(s, "|hi|howdy|");
1570
1571         s = "|";
1572         t = db.synonyms_begin("goodbye");
1573         while (t != db.synonyms_end("goodbye")) {
1574             s += *t++;
1575             s += '|';
1576         }
1577         TEST_STRINGS_EQUAL(s, "|bye|farewell|");
1578
1579         s = "|";
1580         t = db.synonym_keys_begin();
1581         while (t != db.synonym_keys_end()) {
1582             s += *t++;
1583             s += '|';
1584         }
1585         TEST_STRINGS_EQUAL(s, "|goodbye|hello|");
1586
1587         db.commit();
1588     }
1589
1590     // Delete a synonym for "hello" and all synonyms for "goodbye".
1591     db.remove_synonym("hello", "hi");
1592     db.clear_synonyms("goodbye");
1593
1594     // Try these tests twice - once before committing and once after.
1595     for (int times = 1; times <= 2; ++times) {
1596         // Test iterators for terms which aren't there.
1597         TEST(db.synonyms_begin("abc") == db.synonyms_end("abc"));
1598         TEST(db.synonyms_begin("ghi") == db.synonyms_end("ghi"));
1599         TEST(db.synonyms_begin("zzzzz") == db.synonyms_end("zzzzz"));
1600
1601         s = "|";
1602         t = db.synonyms_begin("hello");
1603         while (t != db.synonyms_end("hello")) {
1604             s += *t++;
1605             s += '|';
1606         }
1607         TEST_STRINGS_EQUAL(s, "|howdy|");
1608
1609         TEST(db.synonyms_begin("goodbye") == db.synonyms_end("goodbye"));
1610
1611         s = "|";
1612         t = db.synonym_keys_begin();
1613         while (t != db.synonym_keys_end()) {
1614             s += *t++;
1615             s += '|';
1616         }
1617         TEST_STRINGS_EQUAL(s, "|hello|");
1618
1619         db.commit();
1620     }
1621
1622     Xapian::Database db_multi;
1623     db_multi.add_database(db);
1624     db_multi.add_database(get_database("apitest_simpledata"));
1625
1626     // Test iterators for terms which aren't there.
1627     TEST(db_multi.synonyms_begin("abc") == db_multi.synonyms_end("abc"));
1628     TEST(db_multi.synonyms_begin("ghi") == db_multi.synonyms_end("ghi"));
1629     TEST(db_multi.synonyms_begin("zzzzz") == db_multi.synonyms_end("zzzzz"));
1630
1631     s = "|";
1632     t = db_multi.synonyms_begin("hello");
1633     while (t != db_multi.synonyms_end("hello")) {
1634         s += *t++;
1635         s += '|';
1636     }
1637     TEST_STRINGS_EQUAL(s, "|howdy|");
1638
1639     TEST(db_multi.synonyms_begin("goodbye") == db_multi.synonyms_end("goodbye"));
1640
1641     s = "|";
1642     t = db_multi.synonym_keys_begin();
1643     while (t != db_multi.synonym_keys_end()) {
1644         s += *t++;
1645         s += '|';
1646     }
1647     TEST_STRINGS_EQUAL(s, "|hello|");
1648
1649     return true;
1650 }
1651
1652 // Test that adding a document with a really long term gives an error on
1653 // add_document() rather than on commit().
1654 DEFINE_TESTCASE(termtoolong1, writable) {
1655     // Inmemory doesn't impose a limit.
1656     SKIP_TEST_FOR_BACKEND("inmemory");
1657
1658     Xapian::WritableDatabase db = get_writable_database();
1659
1660     for (size_t i = 246; i <= 290; ++i) {
1661         tout.str(string());
1662         tout << "Term length " << i << endl;
1663         Xapian::Document doc;
1664         string term(i, 'X');
1665         doc.add_term(term);
1666         try {
1667             db.add_document(doc);
1668             TEST_AND_EXPLAIN(false, "Expecting exception InvalidArgumentError");
1669         } catch (const Xapian::InvalidArgumentError &e) {
1670             // Check that the max length is correctly expressed in the
1671             // exception message - we've got this wrong in two different ways
1672             // in the past!
1673             tout << e.get_msg() << endl;
1674             TEST(e.get_msg().find("Term too long (> 245)") != string::npos);
1675         }
1676     }
1677
1678     for (size_t j = 240; j <= 245; ++j) {
1679         tout.str(string());
1680         tout << "Term length " << j << endl;
1681         Xapian::Document doc;
1682         string term(j, 'X');
1683         doc.add_term(term);
1684         db.add_document(doc);
1685     }
1686
1687     db.commit();
1688
1689     size_t limit = 255;
1690     {
1691         // Currently glass escapes zero bytes from terms in keys for
1692         // some tables, so a term with 128 zero bytes won't work for glass.
1693         Xapian::Document doc;
1694         doc.add_term(string(limit / 2 + 1, '\0'));
1695         db.add_document(doc);
1696         try {
1697             db.commit();
1698             TEST_AND_EXPLAIN(false, "Expecting exception InvalidArgumentError");
1699         } catch (const Xapian::InvalidArgumentError &e) {
1700             // Check that the max length is correctly expressed in the
1701             // exception message - we've got this wrong in two different ways
1702             // in the past!
1703             tout << e.get_msg() << endl;
1704             string target = " is ";
1705             target += str(limit);
1706             target += " bytes";
1707             TEST(e.get_msg().find(target) != string::npos);
1708         }
1709     }
1710
1711     return true;
1712 }
1713
1714 /// Test playing with a postlist
1715 DEFINE_TESTCASE(postlist7, writable) {
1716     Xapian::WritableDatabase db_w = get_writable_database();
1717
1718     {
1719         Xapian::Document doc;
1720         doc.add_term("foo", 3);
1721         doc.add_term("zz", 4);
1722         db_w.replace_document(5, doc);
1723     }
1724
1725     Xapian::PostingIterator p;
1726     p = db_w.postlist_begin("foo");
1727     TEST(p != db_w.postlist_end("foo"));
1728     TEST_EQUAL(*p, 5);
1729     TEST_EQUAL(p.get_wdf(), 3);
1730     TEST_EQUAL(p.get_doclength(), 7);
1731     TEST_EQUAL(p.get_unique_terms(), 2);
1732     ++p;
1733     TEST(p == db_w.postlist_end("foo"));
1734
1735     {
1736         Xapian::Document doc;
1737         doc.add_term("foo", 1);
1738         doc.add_term("zz", 1);
1739         db_w.replace_document(6, doc);
1740     }
1741
1742     p = db_w.postlist_begin("foo");
1743     TEST(p != db_w.postlist_end("foo"));
1744     TEST_EQUAL(*p, 5);
1745     TEST_EQUAL(p.get_wdf(), 3);
1746     TEST_EQUAL(p.get_doclength(), 7);
1747     TEST_EQUAL(p.get_unique_terms(), 2);
1748     ++p;
1749     TEST(p != db_w.postlist_end("foo"));
1750     TEST_EQUAL(*p, 6);
1751     TEST_EQUAL(p.get_wdf(), 1);
1752     TEST_EQUAL(p.get_doclength(), 2);
1753     TEST_EQUAL(p.get_unique_terms(), 2);
1754     ++p;
1755     TEST(p == db_w.postlist_end("foo"));
1756
1757     return true;
1758 }
1759
1760 DEFINE_TESTCASE(lazytablebug1, glass) {
1761     {
1762         Xapian::WritableDatabase db = get_named_writable_database("lazytablebug1", string());
1763
1764         Xapian::Document doc;
1765         doc.add_term("foo");
1766         db.add_document(doc);
1767         db.commit();
1768
1769         string synonym(255, 'x');
1770         char buf[] = " iamafish!!!!!!!!!!";
1771         for (int i = 33; i < 120; ++i) {
1772             db.add_synonym(buf, synonym);
1773             ++buf[0];
1774         }
1775
1776         db.commit();
1777     }
1778
1779     Xapian::Database db = get_writable_database_as_database();
1780     for (Xapian::TermIterator t = db.synonym_keys_begin(); t != db.synonym_keys_end(); ++t) {
1781         tout << *t << endl;
1782     }
1783
1784     return true;
1785 }
1786
1787 /// Regression test for bug #287 for flint.
1788 DEFINE_TESTCASE(cursordelbug1, glass) {
1789     static const int terms[] = { 219, 221, 222, 223, 224, 225, 226 };
1790     static const int copies[] = { 74, 116, 199, 21, 45, 155, 189 };
1791
1792     Xapian::WritableDatabase db;
1793     db = get_named_writable_database("cursordelbug1", string());
1794
1795     for (size_t i = 0; i < sizeof(terms) / sizeof(terms[0]); ++i) {
1796         Xapian::Document doc;
1797         doc.add_term("XC" + str(terms[i]));
1798         doc.add_term("XTabc");
1799         doc.add_term("XAdef");
1800         doc.add_term("XRghi");
1801         doc.add_term("XYabc");
1802         size_t c = copies[i];
1803         while (c--) db.add_document(doc);
1804     }
1805
1806     db.commit();
1807
1808     for (size_t i = 0; i < sizeof(terms) / sizeof(terms[0]); ++i) {
1809         db.delete_document("XC" + str(terms[i]));
1810     }
1811
1812     db.commit();
1813
1814     const string & db_path = get_named_writable_database_path("cursordelbug1");
1815     return Xapian::Database::check(db_path) == 0;
1816 }
1817
1818 /** Helper function for modifyvalues1.
1819  *
1820  * Check that the values stored in the database match */
1821 static void
1822 check_vals(const Xapian::Database & db, const map<Xapian::docid, string> & vals)
1823 {
1824     TEST_EQUAL(db.get_doccount(), vals.size());
1825     if (vals.empty()) return;
1826     TEST_REL(vals.rbegin()->first,<=,db.get_lastdocid());
1827     map<Xapian::docid, string>::const_iterator i;
1828     for (i = vals.begin(); i != vals.end(); ++i) {
1829         tout.str(string());
1830         tout << "Checking value in doc " << i->first << " - should be '" << i->second << "'\n";
1831         Xapian::Document doc = db.get_document(i->first);
1832         string dbval = doc.get_value(1);
1833         TEST_EQUAL(dbval, i->second);
1834         if (dbval.empty()) {
1835             TEST_EQUAL(0, doc.values_count());
1836             TEST_EQUAL(doc.values_begin(), doc.values_end());
1837         } else {
1838             TEST_EQUAL(1, doc.values_count());
1839             Xapian::ValueIterator valit = doc.values_begin();
1840             TEST_NOT_EQUAL(valit, doc.values_end());
1841             TEST_EQUAL(dbval, *valit);
1842             TEST_EQUAL(1, valit.get_valueno());
1843             ++valit;
1844             TEST_EQUAL(valit, doc.values_end());
1845         }
1846     }
1847 }
1848
1849 /** Regression test for bug in initial streaming values implementation in
1850  *  chert.
1851  */
1852 DEFINE_TESTCASE(modifyvalues1, writable) {
1853     unsigned int seed = 7;
1854     Xapian::WritableDatabase db = get_writable_database();
1855     // Note: doccount must be coprime with 13
1856     const Xapian::doccount doccount = 1000;
1857     static_assert(doccount % 13 != 0, "doccount divisible by 13");
1858
1859     map<Xapian::docid, string> vals;
1860
1861     for (Xapian::doccount num = 1; num <= doccount; ++num) {
1862         tout.str(string());
1863         Xapian::Document doc;
1864         string val = "val" + str(num);
1865         tout << "Setting val '" << val << "' in doc " << num << "\n";
1866         doc.add_value(1, val);
1867         db.add_document(doc);
1868         vals[num] = val;
1869     }
1870     check_vals(db, vals);
1871     db.commit();
1872     check_vals(db, vals);
1873
1874     // Modify one of the values (this is a regression test which failed with
1875     // the initial implementation of streaming values).
1876     {
1877         Xapian::Document doc;
1878         string val = "newval0";
1879         tout << "Setting val '" << val << "' in doc 2\n";
1880         doc.add_value(1, val);
1881         db.replace_document(2, doc);
1882         vals[2] = val;
1883         check_vals(db, vals);
1884         db.commit();
1885         check_vals(db, vals);
1886     }
1887
1888     // Check that value doesn't get lost when replacing a document with itself.
1889     {
1890         tout << "Replacing document 1 with itself\n";
1891         Xapian::Document doc = db.get_document(1);
1892         db.replace_document(1, doc);
1893         check_vals(db, vals);
1894         db.commit();
1895         check_vals(db, vals);
1896     }
1897
1898     // Check that value doesn't get lost when replacing a document with itself,
1899     // accessing another document in the meantime.  This is a regression test
1900     // for a bug in the code which implements lazy updates - this used to
1901     // forget the values in the document in this situation.
1902     {
1903         tout << "Replacing document 1 with itself, after reading doc 2.\n";
1904         Xapian::Document doc = db.get_document(1);
1905         db.get_document(2);
1906         db.replace_document(1, doc);
1907         check_vals(db, vals);
1908         db.commit();
1909         check_vals(db, vals);
1910     }
1911
1912     // Do some random modifications: seed random generator, for repeatable
1913     // results.
1914     tout << "Setting seed to " << seed << "\n";
1915     srand(seed);
1916     for (Xapian::doccount num = 1; num <= doccount * 2; ++num) {
1917         tout.str(string());
1918         Xapian::docid did = ((rand() >> 8) % doccount) + 1;
1919         Xapian::Document doc;
1920         string val;
1921
1922         if (num % 5 != 0) {
1923             val = "newval" + str(num);
1924             tout << "Setting val '" << val << "' in doc " << did << "\n";
1925             doc.add_value(1, val);
1926         } else {
1927             tout << "Adding/replacing empty document " << did << "\n";
1928         }
1929         db.replace_document(did, doc);
1930         vals[did] = val;
1931     }
1932     check_vals(db, vals);
1933     db.commit();
1934     check_vals(db, vals);
1935
1936     // Delete all the remaining values, in a slightly shuffled order.
1937     // This is where it's important that doccount is coprime with 13.
1938     for (Xapian::doccount num = 0; num < doccount * 13; num += 13) {
1939         tout.str(string());
1940         Xapian::docid did = (num % doccount) + 1;
1941         tout << "Clearing val in doc " << did << "\n";
1942         Xapian::Document doc;
1943         db.replace_document(did, doc);
1944         vals[did] = string();
1945     }
1946     check_vals(db, vals);
1947     db.commit();
1948     check_vals(db, vals);
1949
1950     return true;
1951 }