xapian-core/tests/api_wrdb.cc

   1 /** @file
   2  * @brief tests which need a writable backend
   3  */
   4 /* Copyright 1999,2000,2001 BrightStation PLC
   5  * Copyright 2001 Hein Ragas
   6  * Copyright 2002 Ananova Ltd
   7  * Copyright 2002-2023 Olly Betts
   8  * Copyright 2006 Richard Boulton
   9  * Copyright 2007 Lemur Consulting Ltd
  10  *
  11  * This program is free software; you can redistribute it and/or
  12  * modify it under the terms of the GNU General Public License as
  13  * published by the Free Software Foundation; either version 2 of the
  14  * License, or (at your option) any later version.
  15  *
  16  * This program is distributed in the hope that it will be useful,
  17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19  * GNU General Public License for more details.
  20  *
  21  * You should have received a copy of the GNU General Public License
  22  * along with this program; if not, write to the Free Software
  23  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  24  * USA
  25  */
  26
  27 #include <config.h>
  28
  29 #include "api_wrdb.h"
  30
  31 #include <xapian.h>
  32
  33 #include "filetests.h"
  34 #include "negate_unsigned.h"
  35 #include "omassert.h"
  36 #include "str.h"
  37 #include "stringutils.h"
  38 #include "testsuite.h"
  39 #include "testutils.h"
  40 #include "unixcmds.h"
  41
  42 #include "apitest.h"
  43
  44 #include "safeunistd.h"
  45 #include <cmath>
  46 #include <cstdlib>
  47 #include <limits>
  48 #include <map>
  49 #include <string>
  50
  51 using namespace std;
  52
  53 // #######################################################################
  54 // # Tests start here
  55
  56 // test that indexing a term more than once at the same position increases
  57 // the wdf
  58 DEFINE_TESTCASE(adddoc1, writable) {
  59     Xapian::WritableDatabase db = get_writable_database();
  60
  61     Xapian::Document doc1, doc2, doc3;
  62
  63     // doc1 should come top, but if term "foo" gets wdf of 1, doc2 will beat it
  64     // doc3 should beat both
  65     // Note: all docs have same length
  66     doc1.set_data(string("tom"));
  67     doc1.add_posting("foo", 1);
  68     doc1.add_posting("foo", 1);
  69     doc1.add_posting("foo", 1);
  70     doc1.add_posting("bar", 3);
  71     doc1.add_posting("bar", 4);
  72     db.add_document(doc1);
  73
  74     doc2.set_data(string("dick"));
  75     doc2.add_posting("foo", 1);
  76     doc2.add_posting("foo", 2);
  77     doc2.add_posting("bar", 3);
  78     doc2.add_posting("bar", 3);
  79     doc2.add_posting("bar", 3);
  80     db.add_document(doc2);
  81
  82     doc3.set_data(string("harry"));
  83     doc3.add_posting("foo", 1);
  84     doc3.add_posting("foo", 1);
  85     doc3.add_posting("foo", 2);
  86     doc3.add_posting("foo", 2);
  87     doc3.add_posting("bar", 3);
  88     db.add_document(doc3);
  89
  90     Xapian::Query query("foo");
  91
  92     Xapian::Enquire enq(db);
  93     enq.set_query(query);
  94
  95     Xapian::MSet mset = enq.get_mset(0, 10);
  96
  97     mset_expect_order(mset, 3, 1, 2);
  98 }
  99
 100 // test that removing a posting and removing a term works
 101 DEFINE_TESTCASE(adddoc2, writable) {
 102     Xapian::WritableDatabase db = get_writable_database();
 103     // get_termfreq() on a TermIterator from a Document returns the termfreq
 104     // for just the shard the doc is in.
 105     bool sharded = (db.size() > 1);
 106
 107     Xapian::Document doc1;
 108
 109     doc1.add_posting("foo", 1);
 110     doc1.add_posting("foo", 1);
 111     doc1.add_posting("foo", 2);
 112     doc1.add_posting("foo", 2);
 113     doc1.add_posting("bar", 3);
 114     doc1.add_posting("gone", 1);
 115     // Quartz had a bug handling a term >= 128 characters longer than the
 116     // preceding term in the sort order - this is "foo" + 130 "X"s
 117     doc1.add_posting("fooXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", 1);
 118     Xapian::docid did;
 119
 120     Xapian::Document doc2 = db.get_document(did = db.add_document(doc1));
 121     TEST_EQUAL(did, 1);
 122
 123     Xapian::TermIterator iter1 = doc1.termlist_begin();
 124     Xapian::TermIterator iter2 = doc2.termlist_begin();
 125     TEST(iter1 != doc1.termlist_end());
 126     TEST(iter2 != doc2.termlist_end());
 127     TEST_EQUAL(*iter1, "bar");
 128     TEST_EQUAL(*iter2, *iter1);
 129     TEST_EQUAL(iter1.get_wdf(), 1);
 130     TEST_EQUAL(iter2.get_wdf(), 1);
 131     if (!sharded) {
 132         // TEST_EQUAL(iter1.get_termfreq(), 0);
 133         TEST_EQUAL(iter2.get_termfreq(), 1);
 134     }
 135
 136     iter1++;
 137     iter2++;
 138     TEST(iter1 != doc1.termlist_end());
 139     TEST(iter2 != doc2.termlist_end());
 140     TEST_EQUAL(*iter1, "foo");
 141     TEST_EQUAL(*iter2, *iter1);
 142     TEST_EQUAL(iter1.get_wdf(), 4);
 143     TEST_EQUAL(iter2.get_wdf(), 4);
 144     if (!sharded) {
 145         // TEST_EQUAL(iter1.get_termfreq(), 0);
 146         TEST_EQUAL(iter2.get_termfreq(), 1);
 147     }
 148
 149     iter1++;
 150     iter2++;
 151     TEST(iter1 != doc1.termlist_end());
 152     TEST(iter2 != doc2.termlist_end());
 153     TEST_EQUAL(*iter1, "fooXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX");
 154     TEST_EQUAL(*iter2, *iter1);
 155     TEST_EQUAL(iter1.get_wdf(), 1);
 156     TEST_EQUAL(iter2.get_wdf(), 1);
 157     if (!sharded) {
 158         // assertion fails in debug build! TEST_EQUAL(iter1.get_termfreq(), 0);
 159         TEST_EQUAL(iter2.get_termfreq(), 1);
 160     }
 161
 162     iter1++;
 163     iter2++;
 164     TEST(iter1 != doc1.termlist_end());
 165     TEST(iter2 != doc2.termlist_end());
 166     TEST_EQUAL(*iter1, "gone");
 167     TEST_EQUAL(*iter2, *iter1);
 168     TEST_EQUAL(iter1.get_wdf(), 1);
 169     TEST_EQUAL(iter2.get_wdf(), 1);
 170     if (!sharded) {
 171         // assertion fails in debug build! TEST_EQUAL(iter1.get_termfreq(), 0);
 172         TEST_EQUAL(iter2.get_termfreq(), 1);
 173     }
 174
 175     iter1++;
 176     iter2++;
 177     TEST(iter1 == doc1.termlist_end());
 178     TEST(iter2 == doc2.termlist_end());
 179
 180     doc2.remove_posting("foo", 1, 5);
 181     doc2.add_term("bat", 0);
 182     doc2.add_term("bar", 8);
 183     doc2.add_term("bag", 0);
 184     doc2.remove_term("gone");
 185     doc2.remove_term("fooXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX");
 186
 187     // Should have (doc,wdf) pairs: (bag,0)(bar,9)(bat,0)(foo,0)
 188     // positionlists (bag,none)(bar,3)(bat,none)(foo,2)
 189
 190     iter2 = doc2.termlist_begin();
 191     TEST(iter2 != doc2.termlist_end());
 192     TEST_EQUAL(*iter2, "bag");
 193     // TEST_EQUAL(iter2.get_termfreq(), 0);
 194     iter2++;
 195     TEST(iter2 != doc2.termlist_end());
 196     TEST_EQUAL(*iter2, "bar");
 197     // TEST_EQUAL(iter2.get_termfreq(), 0);
 198     iter2++;
 199     TEST(iter2 != doc2.termlist_end());
 200     TEST_EQUAL(*iter2, "bat");
 201     // TEST_EQUAL(iter2.get_termfreq(), 0);
 202     iter2++;
 203     TEST(iter2 != doc2.termlist_end());
 204     TEST_EQUAL(*iter2, "foo");
 205     // TEST_EQUAL(iter2.get_termfreq(), 0);
 206     iter2++;
 207     TEST(iter2 == doc2.termlist_end());
 208
 209     doc1 = db.get_document(did = db.add_document(doc2));
 210     TEST_EQUAL(did, 2);
 211
 212     iter1 = doc1.termlist_begin();
 213     iter2 = doc2.termlist_begin();
 214     TEST(iter1 != doc1.termlist_end());
 215     TEST(iter2 != doc2.termlist_end());
 216     TEST_EQUAL(*iter1, "bag");
 217     TEST_EQUAL(*iter2, *iter1);
 218     TEST_EQUAL(iter1.get_wdf(), 0);
 219     TEST_EQUAL(iter2.get_wdf(), 0);
 220     if (!sharded) {
 221         TEST_EQUAL(iter1.get_termfreq(), 1);
 222         // TEST_EQUAL(iter2.get_termfreq(), 0);
 223     }
 224     TEST(iter1.positionlist_begin() == iter1.positionlist_end());
 225     TEST(iter2.positionlist_begin() == iter2.positionlist_end());
 226
 227     iter1++;
 228     iter2++;
 229     TEST(iter1 != doc1.termlist_end());
 230     TEST(iter2 != doc2.termlist_end());
 231     TEST_EQUAL(*iter1, "bar");
 232     TEST_EQUAL(*iter2, *iter1);
 233     TEST_EQUAL(iter1.get_wdf(), 9);
 234     TEST_EQUAL(iter2.get_wdf(), 9);
 235     if (!sharded) {
 236         TEST_EQUAL(iter1.get_termfreq(), 2);
 237         // TEST_EQUAL(iter2.get_termfreq(), 0);
 238     }
 239
 240     Xapian::PositionIterator pi1;
 241     pi1 = iter1.positionlist_begin();
 242     Xapian::PositionIterator pi2 = iter2.positionlist_begin();
 243     TEST_EQUAL(*pi1, 3); pi1++;
 244     TEST_EQUAL(*pi2, 3); pi2++;
 245     TEST(pi1 == iter1.positionlist_end());
 246     TEST(pi2 == iter2.positionlist_end());
 247
 248     iter1++;
 249     iter2++;
 250     TEST(iter1 != doc1.termlist_end());
 251     TEST(iter2 != doc2.termlist_end());
 252     TEST_EQUAL(*iter1, "bat");
 253     TEST_EQUAL(*iter2, *iter1);
 254     TEST_EQUAL(iter1.get_wdf(), 0);
 255     TEST_EQUAL(iter2.get_wdf(), 0);
 256     if (!sharded) {
 257         TEST_EQUAL(iter1.get_termfreq(), 1);
 258         // TEST_EQUAL(iter2.get_termfreq(), 0);
 259     }
 260     TEST(iter1.positionlist_begin() == iter1.positionlist_end());
 261     TEST(iter2.positionlist_begin() == iter2.positionlist_end());
 262
 263     iter1++;
 264     iter2++;
 265     TEST(iter1 != doc1.termlist_end());
 266     TEST(iter2 != doc2.termlist_end());
 267     TEST_EQUAL(*iter1, "foo");
 268     TEST_EQUAL(*iter2, *iter1);
 269     TEST_EQUAL(iter1.get_wdf(), 0);
 270     TEST_EQUAL(iter2.get_wdf(), 0);
 271     if (!sharded) {
 272         TEST_EQUAL(iter1.get_termfreq(), 2);
 273         // TEST_EQUAL(iter2.get_termfreq(), 0);
 274     }
 275
 276     Xapian::PositionIterator temp1 = iter1.positionlist_begin();
 277     pi1 = temp1;
 278     Xapian::PositionIterator temp2 = iter2.positionlist_begin();
 279     pi2 = temp2;
 280     TEST_EQUAL(*pi1, 2); pi1++;
 281     TEST_EQUAL(*pi2, 2); pi2++;
 282     TEST(pi1 == iter1.positionlist_end());
 283     TEST(pi2 == iter2.positionlist_end());
 284
 285     iter1++;
 286     iter2++;
 287     TEST(iter1 == doc1.termlist_end());
 288     TEST(iter2 == doc2.termlist_end());
 289 }
 290
 291 // test that adding lots of documents works, and doesn't leak memory
 292 // REGRESSION FIXED:2003-09-07
 293 DEFINE_TESTCASE(adddoc3, writable) {
 294     Xapian::WritableDatabase db = get_writable_database();
 295
 296     for (Xapian::doccount i = 0; i < 2100; ++i) {
 297         Xapian::Document doc;
 298         for (Xapian::termcount t = 0; t < 100; ++t) {
 299             string term("foo");
 300             term += char(t ^ 70 ^ i);
 301             doc.add_posting(term, t);
 302         }
 303         db.add_document(doc);
 304     }
 305 }
 306
 307 // We originally wanted to test that a termlist starting with a 48 character
 308 // long term worked since that required special handling in flint for
 309 // historical reasons.  That's no longer relevant, but it seems useful to
 310 // continue to test term lists starting with various term lengths work.
 311 DEFINE_TESTCASE(adddoc4, writable) {
 312     Xapian::WritableDatabase db = get_writable_database();
 313
 314     for (Xapian::doccount i = 1; i <= 240; ++i) {
 315         Xapian::Document doc;
 316         string term(i, 'X');
 317         doc.add_term(term);
 318         db.add_document(doc);
 319     }
 320     db.add_document(Xapian::Document());
 321     db.commit();
 322
 323     for (Xapian::doccount i = 1; i <= 240; ++i) {
 324         Xapian::Document doc = db.get_document(i);
 325         Xapian::TermIterator t = doc.termlist_begin();
 326         TEST(t != doc.termlist_end());
 327         TEST_EQUAL((*t).size(), i);
 328         ++t;
 329         TEST(t == doc.termlist_end());
 330     }
 331
 332     // And test a document with no terms.
 333     Xapian::Document doc = db.get_document(241);
 334     TEST(doc.termlist_begin() == doc.termlist_end());
 335 }
 336
 337 // Test adding a document, and checking that it got added correctly.
 338 // This testcase used to be adddoc2 in quartztest.
 339 DEFINE_TESTCASE(adddoc5, writable) {
 340     // FIXME: With multi, get_termfreq() on a TermIterator from a Document
 341     // currently returns the termfreq for just the shard the doc is in.
 342
 343     // Inmemory doesn't support get_writable_database_as_database().
 344     SKIP_TEST_FOR_BACKEND("inmemory");
 345
 346     Xapian::docid did;
 347     Xapian::Document document_in;
 348     document_in.set_data("Foobar rising");
 349     document_in.add_value(7, "Value7");
 350     document_in.add_value(13, "Value13");
 351     document_in.add_posting("foobar", 1);
 352     document_in.add_posting("rising", 2);
 353     document_in.add_posting("foobar", 3);
 354
 355     Xapian::Document document_in2;
 356     document_in2.set_data("Foobar falling");
 357     document_in2.add_posting("foobar", 1);
 358     document_in2.add_posting("falling", 2);
 359     {
 360         Xapian::WritableDatabase database(get_writable_database());
 361
 362         TEST_EQUAL(database.get_doccount(), 0);
 363         TEST_EQUAL(database.get_avlength(), 0);
 364
 365         did = database.add_document(document_in);
 366         TEST_EQUAL(database.get_doccount(), 1);
 367         TEST_EQUAL(database.get_avlength(), 3);
 368
 369         TEST_EQUAL(database.get_termfreq("foobar"), 1);
 370         TEST_EQUAL(database.get_collection_freq("foobar"), 2);
 371         TEST_EQUAL(database.get_termfreq("rising"), 1);
 372         TEST_EQUAL(database.get_collection_freq("rising"), 1);
 373         TEST_EQUAL(database.get_termfreq("falling"), 0);
 374         TEST_EQUAL(database.get_collection_freq("falling"), 0);
 375
 376         Xapian::docid did2 = database.add_document(document_in2);
 377         TEST_EQUAL(database.get_doccount(), 2);
 378         TEST_NOT_EQUAL(did, did2);
 379         TEST_EQUAL(database.get_avlength(), 5.0 / 2.0);
 380
 381         TEST_EQUAL(database.get_termfreq("foobar"), 2);
 382         TEST_EQUAL(database.get_collection_freq("foobar"), 3);
 383         TEST_EQUAL(database.get_termfreq("rising"), 1);
 384         TEST_EQUAL(database.get_collection_freq("rising"), 1);
 385         TEST_EQUAL(database.get_termfreq("falling"), 1);
 386         TEST_EQUAL(database.get_collection_freq("falling"), 1);
 387
 388         database.delete_document(did);
 389         TEST_EQUAL(database.get_doccount(), 1);
 390         TEST_EQUAL(database.get_avlength(), 2);
 391
 392         TEST_EQUAL(database.get_termfreq("foobar"), 1);
 393         TEST_EQUAL(database.get_collection_freq("foobar"), 1);
 394         TEST_EQUAL(database.get_termfreq("rising"), 0);
 395         TEST_EQUAL(database.get_collection_freq("rising"), 0);
 396         TEST_EQUAL(database.get_termfreq("falling"), 1);
 397         TEST_EQUAL(database.get_collection_freq("falling"), 1);
 398
 399         did = database.add_document(document_in);
 400         TEST_EQUAL(database.get_doccount(), 2);
 401         TEST_EQUAL(database.get_avlength(), 5.0 / 2.0);
 402
 403         TEST_EQUAL(database.get_termfreq("foobar"), 2);
 404         TEST_EQUAL(database.get_collection_freq("foobar"), 3);
 405         TEST_EQUAL(database.get_termfreq("rising"), 1);
 406         TEST_EQUAL(database.get_collection_freq("rising"), 1);
 407         TEST_EQUAL(database.get_termfreq("falling"), 1);
 408         TEST_EQUAL(database.get_collection_freq("falling"), 1);
 409     }
 410
 411     {
 412         Xapian::Database database(get_writable_database_as_database());
 413         Xapian::Document document_out = database.get_document(did);
 414
 415         // get_termfreq() on a TermIterator from a Document returns the
 416         // termfreq for just the shard the doc is in.
 417         bool sharded = (database.size() > 1);
 418
 419         TEST_EQUAL(document_in.get_data(), document_out.get_data());
 420
 421         {
 422             Xapian::ValueIterator i(document_in.values_begin());
 423             Xapian::ValueIterator j(document_out.values_begin());
 424             for (; i != document_in.values_end(); i++, j++) {
 425                 TEST_NOT_EQUAL(j, document_out.values_end());
 426                 TEST_EQUAL(*i, *j);
 427                 TEST_EQUAL(i.get_valueno(), j.get_valueno());
 428             }
 429             TEST_EQUAL(j, document_out.values_end());
 430         }
 431
 432         {
 433             // Regression test for bug fixed in 1.0.5 - values_begin() didn't
 434             // ensure that values had been read.  However, values_end() did
 435             // (and so did values_count()) so this wasn't generally an issue
 436             // but it shouldn't happen anyway.
 437             Xapian::Document doc_tmp = database.get_document(did);
 438             Xapian::ValueIterator i = document_in.values_begin();
 439             Xapian::ValueIterator j = doc_tmp.values_begin();
 440             TEST_EQUAL(*i, *j);
 441         }
 442
 443         {
 444             Xapian::TermIterator i(document_in.termlist_begin());
 445             Xapian::TermIterator j(document_out.termlist_begin());
 446             for (; i != document_in.termlist_end(); i++, j++) {
 447                 TEST_NOT_EQUAL(j, document_out.termlist_end());
 448                 TEST_EQUAL(*i, *j);
 449                 TEST_EQUAL(i.get_wdf(), j.get_wdf());
 450                 if (!sharded) {
 451                     // Actually use termfreq to stop compiler optimising away the
 452                     // call to get_termfreq().
 453                     TEST_EXCEPTION(Xapian::InvalidOperationError,
 454                                    if (i.get_termfreq()) FAIL_TEST("?"));
 455                     TEST_NOT_EQUAL(0, j.get_termfreq());
 456                     if (*i == "foobar") {
 457                         // termfreq of foobar is 2
 458                         TEST_EQUAL(2, j.get_termfreq());
 459                     } else {
 460                         // termfreq of rising is 1
 461                         TEST_EQUAL(*i, "rising");
 462                         TEST_EQUAL(1, j.get_termfreq());
 463                     }
 464                 }
 465                 Xapian::PositionIterator k(i.positionlist_begin());
 466                 Xapian::PositionIterator l(j.positionlist_begin());
 467                 for (; k != i.positionlist_end(); k++, l++) {
 468                     TEST_NOT_EQUAL(l, j.positionlist_end());
 469                     TEST_EQUAL(*k, *l);
 470                 }
 471                 TEST_EQUAL(l, j.positionlist_end());
 472             }
 473             TEST_EQUAL(j, document_out.termlist_end());
 474         }
 475     }
 476 }
 477
 478 // Test adding a document, and checking that it got added correctly.
 479 // This testcase used to be adddoc3 in quartztest.
 480 DEFINE_TESTCASE(adddoc6, writable) {
 481     // Inmemory doesn't support get_writable_database_again().
 482     SKIP_TEST_FOR_BACKEND("inmemory");
 483
 484     Xapian::docid did;
 485     Xapian::Document document_in;
 486     document_in.set_data("Foobar rising");
 487     document_in.add_value(7, "Value7");
 488     document_in.add_value(13, "Value13");
 489     document_in.add_posting("foo", 1);
 490     document_in.add_posting("bar", 2);
 491
 492     {
 493         Xapian::WritableDatabase database(get_writable_database());
 494
 495         did = database.add_document(document_in);
 496         TEST_EQUAL(did, 1);
 497         TEST_EQUAL(database.get_doccount(), 1);
 498         TEST_EQUAL(database.get_avlength(), 2);
 499     }
 500
 501     {
 502         Xapian::WritableDatabase database(get_writable_database_again());
 503
 504         document_in.remove_term("foo");
 505         document_in.add_posting("baz", 1);
 506
 507         database.replace_document(1, document_in);
 508
 509         database.delete_document(1);
 510
 511         TEST_EQUAL(database.get_doccount(), 0);
 512         TEST_EQUAL(database.get_avlength(), 0);
 513         TEST_EQUAL(database.get_termfreq("foo"), 0);
 514         TEST_EQUAL(database.get_collection_freq("foo"), 0);
 515         TEST_EQUAL(database.get_termfreq("bar"), 0);
 516         TEST_EQUAL(database.get_collection_freq("bar"), 0);
 517         TEST_EQUAL(database.get_termfreq("baz"), 0);
 518         TEST_EQUAL(database.get_collection_freq("baz"), 0);
 519     }
 520 }
 521
 522 // tests that database destructors commit if it isn't done explicitly
 523 DEFINE_TESTCASE(implicitendsession1, writable) {
 524     Xapian::WritableDatabase db = get_writable_database();
 525
 526     Xapian::Document doc;
 527
 528     doc.set_data(string("top secret"));
 529     doc.add_posting("cia", 1);
 530     doc.add_posting("nsa", 2);
 531     doc.add_posting("fbi", 3);
 532     db.add_document(doc);
 533 }
 534
 535 // tests that assignment of Xapian::Database and Xapian::WritableDatabase works
 536 // as expected
 537 DEFINE_TESTCASE(databaseassign1, writable) {
 538     Xapian::WritableDatabase wdb = get_writable_database();
 539     Xapian::Database db = get_database("");
 540     Xapian::Database actually_wdb = wdb;
 541     Xapian::WritableDatabase w1(wdb);
 542     w1 = wdb;
 543     Xapian::Database d1(wdb);
 544     Xapian::Database d2(actually_wdb);
 545     d2 = wdb;
 546     d2 = actually_wdb;
 547 #ifdef __has_warning
 548 # if __has_warning("-Wself-assign-overloaded")
 549     // Suppress warning from newer clang about self-assignment so we can
 550     // test that self-assignment works!
 551 #  pragma clang diagnostic push
 552 #  pragma clang diagnostic ignored "-Wself-assign-overloaded"
 553 # endif
 554 #endif
 555     wdb = wdb; // check assign to itself works
 556     db = db; // check assign to itself works
 557 #ifdef __has_warning
 558 # if __has_warning("-Wself-assign-overloaded")
 559 #  pragma clang diagnostic pop
 560 # endif
 561 #endif
 562 }
 563
 564 // tests that deletion and updating of documents works as expected
 565 DEFINE_TESTCASE(deldoc1, writable) {
 566     Xapian::WritableDatabase db = get_writable_database();
 567
 568     Xapian::Document doc1;
 569
 570     doc1.add_posting("foo", 1);
 571     doc1.add_posting("foo", 1);
 572     doc1.add_posting("foo", 2);
 573     doc1.add_posting("foo", 2);
 574     doc1.add_posting("bar", 3);
 575     doc1.add_posting("gone", 1);
 576
 577     Xapian::docid did = db.add_document(doc1);
 578     TEST_EQUAL(did, 1);
 579
 580     doc1.remove_term("gone");
 581
 582     did = db.add_document(doc1);
 583     TEST_EQUAL(did, 2);
 584
 585     doc1.add_term("new", 1);
 586     did = db.add_document(doc1);
 587     TEST_EQUAL(did, 3);
 588
 589     db.delete_document(1);
 590
 591     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_document(1));
 592
 593     doc1 = db.get_document(2);
 594     doc1.remove_term("foo");
 595     doc1.add_term("fwing");
 596     db.replace_document(2, doc1);
 597
 598     Xapian::Document doc2 = db.get_document(2);
 599     Xapian::TermIterator tit = doc2.termlist_begin();
 600     TEST_NOT_EQUAL(tit, doc2.termlist_end());
 601     TEST_EQUAL(*tit, "bar");
 602     tit++;
 603     TEST_NOT_EQUAL(tit, doc2.termlist_end());
 604     TEST_EQUAL(*tit, "fwing");
 605     tit++;
 606     TEST_EQUAL(tit, doc2.termlist_end());
 607 }
 608
 609 // tests that deletion and updating of documents works as expected
 610 DEFINE_TESTCASE(deldoc2, writable) {
 611     Xapian::WritableDatabase db = get_writable_database();
 612
 613     Xapian::Document doc1;
 614
 615     doc1.add_posting("one", 1);
 616     doc1.add_posting("two", 2);
 617     doc1.add_posting("two", 3);
 618     Xapian::docid did;
 619
 620     did = db.add_document(doc1);
 621     TEST_EQUAL(did, 1);
 622
 623     doc1.remove_term("one");
 624     doc1.add_posting("three", 4);
 625
 626     did = db.add_document(doc1);
 627     TEST_EQUAL(did, 2);
 628
 629     doc1.add_posting("one", 7);
 630     doc1.remove_term("two");
 631
 632     did = db.add_document(doc1);
 633     TEST_EQUAL(did, 3);
 634
 635     db.commit();
 636
 637     // reopen() on a writable database shouldn't do anything.
 638     TEST(!db.reopen());
 639
 640     db.delete_document(1);
 641     db.delete_document(2);
 642     db.delete_document(3);
 643
 644     db.commit();
 645
 646     // reopen() on a writable database shouldn't do anything.
 647     TEST(!db.reopen());
 648
 649     TEST_EQUAL(db.postlist_begin("one"), db.postlist_end("one"));
 650     TEST_EQUAL(db.postlist_begin("two"), db.postlist_end("two"));
 651     TEST_EQUAL(db.postlist_begin("three"), db.postlist_end("three"));
 652
 653     TEST_EXCEPTION(Xapian::DocNotFoundError, db.termlist_begin(1));
 654     TEST_EXCEPTION(Xapian::DocNotFoundError, db.termlist_begin(2));
 655     TEST_EXCEPTION(Xapian::DocNotFoundError, db.termlist_begin(3));
 656     TEST_EXCEPTION(Xapian::DocNotFoundError, db.termlist_begin(4));
 657
 658     // test positionlist_{begin,end}?
 659
 660     TEST_EQUAL(db.get_doccount(), 0);
 661     TEST_EQUAL(db.get_avlength(), 0);
 662     TEST_EQUAL(db.get_termfreq("one"), 0);
 663     TEST_EQUAL(db.get_termfreq("two"), 0);
 664     TEST_EQUAL(db.get_termfreq("three"), 0);
 665
 666     TEST(!db.term_exists("one"));
 667     TEST(!db.term_exists("two"));
 668     TEST(!db.term_exists("three"));
 669
 670     TEST_EQUAL(db.get_collection_freq("one"), 0);
 671     TEST_EQUAL(db.get_collection_freq("two"), 0);
 672     TEST_EQUAL(db.get_collection_freq("three"), 0);
 673
 674     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_doclength(1));
 675     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_doclength(2));
 676     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_doclength(3));
 677
 678     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_unique_terms(1));
 679     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_unique_terms(2));
 680     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_unique_terms(3));
 681
 682     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_document(1));
 683     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_document(2));
 684     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_document(3));
 685
 686     TEST_EQUAL(db.allterms_begin(), db.allterms_end());
 687 }
 688
 689 // another test of deletion of documents, a cut-down version of deldoc2
 690 DEFINE_TESTCASE(deldoc3, writable) {
 691     Xapian::WritableDatabase db = get_writable_database();
 692
 693     Xapian::Document doc1;
 694
 695     doc1.add_posting("one", 1);
 696
 697     Xapian::docid did = db.add_document(doc1);
 698     TEST_EQUAL(did, 1);
 699
 700     db.commit();
 701
 702     // reopen() on a writable database shouldn't do anything.
 703     TEST(!db.reopen());
 704
 705     db.delete_document(1);
 706
 707     db.commit();
 708
 709     // reopen() on a writable database shouldn't do anything.
 710     TEST(!db.reopen());
 711
 712     TEST_EQUAL(db.postlist_begin("one"), db.postlist_end("one"));
 713
 714     TEST_EXCEPTION(Xapian::DocNotFoundError, db.termlist_begin(1));
 715     TEST_EXCEPTION(Xapian::DocNotFoundError, db.termlist_begin(2));
 716
 717     // test positionlist_{begin,end}?
 718
 719     TEST_EQUAL(db.get_doccount(), 0);
 720     TEST_EQUAL(db.get_avlength(), 0);
 721     TEST_EQUAL(db.get_termfreq("one"), 0);
 722
 723     TEST(!db.term_exists("one"));
 724
 725     TEST_EQUAL(db.get_collection_freq("one"), 0);
 726
 727     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_doclength(1));
 728     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_doclength(2));
 729
 730     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_unique_terms(1));
 731     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_unique_terms(2));
 732
 733     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_document(1));
 734     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_document(2));
 735
 736     TEST_EQUAL(db.allterms_begin(), db.allterms_end());
 737 }
 738
 739 // tests that deletion and updating of (lots of) documents works as expected
 740 DEFINE_TESTCASE(deldoc4, writable) {
 741     Xapian::WritableDatabase db = get_writable_database();
 742
 743     Xapian::Document doc1;
 744
 745     doc1.add_posting("one", 1);
 746     doc1.add_posting("two", 2);
 747     doc1.add_posting("two", 3);
 748
 749     Xapian::Document doc2 = doc1;
 750     doc2.remove_term("one");
 751     doc2.add_posting("three", 4);
 752
 753     Xapian::Document doc3 = doc2;
 754     doc3.add_posting("one", 7);
 755     doc3.remove_term("two");
 756
 757     const Xapian::docid maxdoc = 1000 * 3;
 758     Xapian::docid did;
 759     for (Xapian::docid i = 0; i < maxdoc / 3; ++i) {
 760         did = db.add_document(doc1);
 761         TEST_EQUAL(did, i * 3 + 1);
 762         did = db.add_document(doc2);
 763         TEST_EQUAL(did, i * 3 + 2);
 764         did = db.add_document(doc3);
 765         TEST_EQUAL(did, i * 3 + 3);
 766
 767         bool is_power_of_two = ((i & negate_unsigned(i)) == 0);
 768         if (is_power_of_two) {
 769             db.commit();
 770             // reopen() on a writable database shouldn't do anything.
 771             TEST(!db.reopen());
 772         }
 773     }
 774     db.commit();
 775     // reopen() on a writable database shouldn't do anything.
 776     TEST(!db.reopen());
 777
 778     /* delete the documents in a peculiar order */
 779     for (Xapian::docid i = 0; i < maxdoc / 3; ++i) {
 780         db.delete_document(maxdoc - i);
 781         db.delete_document(maxdoc / 3 + i + 1);
 782         db.delete_document(i + 1);
 783     }
 784
 785     db.commit();
 786     // reopen() on a writable database shouldn't do anything.
 787     TEST(!db.reopen());
 788
 789     TEST_EQUAL(db.postlist_begin("one"), db.postlist_end("one"));
 790     TEST_EQUAL(db.postlist_begin("two"), db.postlist_end("two"));
 791     TEST_EQUAL(db.postlist_begin("three"), db.postlist_end("three"));
 792
 793     for (Xapian::docid i = 1; i <= maxdoc; ++i) {
 794         // TEST_EXCEPTION writes to tout each time if the test is run
 795         // in verbose mode and some string stream implementations get
 796         // very inefficient with large strings, so clear tout on each pass of
 797         // the loop to speed up the test since the older information isn't
 798         // interesting anyway.
 799         tout.str(string());
 800         TEST_EXCEPTION(Xapian::DocNotFoundError, db.termlist_begin(i));
 801         TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_doclength(i));
 802         TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_unique_terms(i));
 803         TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_document(i));
 804     }
 805
 806     // test positionlist_{begin,end}?
 807
 808     TEST_EQUAL(db.get_doccount(), 0);
 809     TEST_EQUAL(db.get_avlength(), 0);
 810     TEST_EQUAL(db.get_termfreq("one"), 0);
 811     TEST_EQUAL(db.get_termfreq("two"), 0);
 812     TEST_EQUAL(db.get_termfreq("three"), 0);
 813
 814     TEST(!db.term_exists("one"));
 815     TEST(!db.term_exists("two"));
 816     TEST(!db.term_exists("three"));
 817
 818     TEST_EQUAL(db.get_collection_freq("one"), 0);
 819     TEST_EQUAL(db.get_collection_freq("two"), 0);
 820     TEST_EQUAL(db.get_collection_freq("three"), 0);
 821
 822     TEST_EQUAL(db.allterms_begin(), db.allterms_end());
 823 }
 824
 825 // Test deleting a document which was added in the same batch.
 826 DEFINE_TESTCASE(deldoc5, writable) {
 827     Xapian::WritableDatabase db = get_writable_database();
 828
 829     Xapian::Document doc1;
 830
 831     doc1.add_posting("foo", 1);
 832     doc1.add_posting("bar", 2);
 833     doc1.add_posting("aardvark", 3);
 834
 835     Xapian::docid did = db.add_document(doc1);
 836     TEST_EQUAL(did, 1);
 837
 838     doc1.remove_term("bar");
 839     doc1.add_term("hello");
 840
 841     did = db.add_document(doc1);
 842     TEST_EQUAL(did, 2);
 843
 844     doc1.add_term("world", 1);
 845     did = db.add_document(doc1);
 846     TEST_EQUAL(did, 3);
 847
 848     db.delete_document(2);
 849
 850     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_document(2));
 851
 852     db.commit();
 853
 854     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_document(2));
 855
 856     TEST_EQUAL(db.get_termfreq("foo"), 2);
 857     TEST_EQUAL(db.get_termfreq("aardvark"), 2);
 858     TEST_EQUAL(db.get_termfreq("hello"), 1);
 859
 860     Xapian::PostingIterator p = db.postlist_begin("foo");
 861     TEST_NOT_EQUAL(p, db.postlist_end("foo"));
 862     TEST_EQUAL(*p, 1);
 863     ++p;
 864     TEST_NOT_EQUAL(p, db.postlist_end("foo"));
 865     TEST_EQUAL(*p, 3);
 866     ++p;
 867     TEST_EQUAL(p, db.postlist_end("foo"));
 868 }
 869
 870 // Regression test for bug in quartz and flint, fixed in 1.0.2.
 871 DEFINE_TESTCASE(deldoc6, writable) {
 872     Xapian::WritableDatabase db = get_writable_database();
 873
 874     Xapian::Document doc1;
 875
 876     doc1.add_posting("foo", 1);
 877     doc1.add_posting("bar", 2);
 878     doc1.add_posting("aardvark", 3);
 879
 880     Xapian::docid did = db.add_document(doc1);
 881     TEST_EQUAL(did, 1);
 882
 883     doc1.remove_term("bar");
 884     doc1.add_term("hello");
 885
 886     did = db.add_document(doc1);
 887     TEST_EQUAL(did, 2);
 888
 889     db.commit();
 890
 891     db.delete_document(2);
 892     TEST_EXCEPTION(Xapian::DocNotFoundError, db.delete_document(3));
 893
 894     db.commit();
 895
 896     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_document(2));
 897 }
 898
 899 DEFINE_TESTCASE(replacedoc1, writable) {
 900     Xapian::WritableDatabase db = get_writable_database();
 901
 902     Xapian::Document doc1;
 903
 904     doc1.add_posting("foo", 1);
 905     doc1.add_posting("foo", 2);
 906     doc1.add_posting("gone", 3);
 907     doc1.add_posting("bar", 4);
 908     doc1.add_posting("foo", 5);
 909     Xapian::docid did;
 910
 911     did = db.add_document(doc1);
 912     TEST_EQUAL(did, 1);
 913
 914     Xapian::Document doc2;
 915
 916     doc2.add_posting("foo", 1);
 917     doc2.add_posting("pipco", 2);
 918     doc2.add_posting("bar", 4);
 919     doc2.add_posting("foo", 5);
 920
 921     db.replace_document(did, doc2);
 922
 923     Xapian::Document doc3 = db.get_document(did);
 924     Xapian::TermIterator t_iter = doc3.termlist_begin();
 925     TEST_EQUAL(*t_iter, "bar");
 926     Xapian::PositionIterator p_iter = t_iter.positionlist_begin();
 927     TEST_EQUAL(*p_iter, 4);
 928     ++t_iter;
 929     TEST_EQUAL(*t_iter, "foo");
 930     Xapian::PositionIterator q_iter = t_iter.positionlist_begin();
 931     TEST_EQUAL(*q_iter, 1);
 932     ++q_iter;
 933     TEST_EQUAL(*q_iter, 5);
 934     ++t_iter;
 935     TEST_EQUAL(*t_iter, "pipco");
 936     Xapian::PositionIterator r_iter = t_iter.positionlist_begin();
 937     TEST_EQUAL(*r_iter, 2);
 938     ++t_iter;
 939     TEST_EQUAL(t_iter, doc3.termlist_end());
 940 }
 941
 942 // Test of new feature: WritableDatabase::replace_document accepts a docid
 943 // which doesn't yet exist as of Xapian 0.8.2.
 944 DEFINE_TESTCASE(replacedoc2, writable) {
 945     Xapian::WritableDatabase db = get_writable_database();
 946
 947     Xapian::Document doc1;
 948
 949     doc1.add_posting("foo", 1);
 950     doc1.add_posting("foo", 2);
 951     doc1.add_posting("gone", 3);
 952     doc1.add_posting("bar", 4);
 953     doc1.add_posting("foo", 5);
 954     Xapian::docid did = 31770;
 955
 956     db.replace_document(did, doc1);
 957
 958     // Regression tests for bug in the InMemory backend - fixed in 1.0.2.
 959     TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_document(1));
 960     Xapian::PostingIterator postit = db.postlist_begin("");
 961     TEST(postit != db.postlist_end(""));
 962     TEST_EQUAL(*postit, 31770);
 963
 964     Xapian::Document doc2;
 965
 966     doc2.add_posting("foo", 1);
 967     doc2.add_posting("pipco", 2);
 968     doc2.add_posting("bar", 4);
 969     doc2.add_posting("foo", 5);
 970
 971     db.replace_document(did, doc2);
 972     TEST_EQUAL(db.get_doccount(), 1);
 973
 974     Xapian::Document doc3 = db.get_document(did);
 975     Xapian::TermIterator t_iter = doc3.termlist_begin();
 976     TEST_EQUAL(*t_iter, "bar");
 977     Xapian::PositionIterator p_iter = t_iter.positionlist_begin();
 978     TEST_EQUAL(*p_iter, 4);
 979     ++t_iter;
 980     TEST_EQUAL(*t_iter, "foo");
 981     Xapian::PositionIterator q_iter = t_iter.positionlist_begin();
 982     TEST_EQUAL(*q_iter, 1);
 983     ++q_iter;
 984     TEST_EQUAL(*q_iter, 5);
 985     ++t_iter;
 986     TEST_EQUAL(*t_iter, "pipco");
 987     Xapian::PositionIterator r_iter = t_iter.positionlist_begin();
 988     TEST_EQUAL(*r_iter, 2);
 989     ++t_iter;
 990     TEST_EQUAL(t_iter, doc3.termlist_end());
 991
 992     did = db.add_document(doc1);
 993     TEST_EQUAL(did, 31771);
 994     TEST_EQUAL(db.get_doccount(), 2);
 995
 996     TEST_EXCEPTION(Xapian::InvalidArgumentError, db.replace_document(0, doc2));
 997 }
 998
 999 // Test replacing a document which was added in the same batch.
1000 DEFINE_TESTCASE(replacedoc3, writable) {
1001     Xapian::WritableDatabase db = get_writable_database();
1002
1003     Xapian::Document doc1;
1004
1005     doc1.add_posting("foo", 1);
1006     doc1.add_posting("bar", 2);
1007     doc1.add_posting("aardvark", 3);
1008
1009     Xapian::docid did = db.add_document(doc1);
1010     TEST_EQUAL(did, 1);
1011
1012     doc1.remove_term("bar");
1013     doc1.add_term("hello");
1014
1015     did = db.add_document(doc1);
1016     TEST_EQUAL(did, 2);
1017
1018     doc1.add_term("world", 1);
1019     did = db.add_document(doc1);
1020     TEST_EQUAL(did, 3);
1021
1022     Xapian::Document doc2;
1023     doc2.add_term("world");
1024     db.replace_document(2, doc2);
1025
1026     db.commit();
1027
1028     // Check that the document exists (no DocNotFoundError).
1029     doc2 = db.get_document(2);
1030
1031     TEST_EQUAL(db.get_termfreq("foo"), 2);
1032     TEST_EQUAL(db.get_termfreq("aardvark"), 2);
1033     TEST_EQUAL(db.get_termfreq("hello"), 1);
1034     TEST_EQUAL(db.get_termfreq("world"), 2);
1035
1036     TEST_EQUAL(db.get_doclength(1), 3);
1037     TEST_EQUAL(db.get_doclength(2), 1);
1038     TEST_EQUAL(db.get_doclength(3), 4);
1039
1040     TEST_EQUAL(db.get_unique_terms(1), 3);
1041     TEST_EQUAL(db.get_unique_terms(2), 1);
1042     TEST_EQUAL(db.get_unique_terms(3), 4);
1043
1044     Xapian::PostingIterator p = db.postlist_begin("foo");
1045     TEST_NOT_EQUAL(p, db.postlist_end("foo"));
1046     TEST_EQUAL(*p, 1);
1047     TEST_EQUAL(p.get_doclength(), 3);
1048     TEST_EQUAL(p.get_unique_terms(), 3);
1049     ++p;
1050     TEST_NOT_EQUAL(p, db.postlist_end("foo"));
1051     TEST_EQUAL(*p, 3);
1052     TEST_EQUAL(p.get_doclength(), 4);
1053     TEST_EQUAL(p.get_unique_terms(), 4);
1054     ++p;
1055     TEST_EQUAL(p, db.postlist_end("foo"));
1056
1057     p = db.postlist_begin("world");
1058     TEST_NOT_EQUAL(p, db.postlist_end("world"));
1059     TEST_EQUAL(*p, 2);
1060     TEST_EQUAL(p.get_doclength(), 1);
1061     TEST_EQUAL(p.get_unique_terms(), 1);
1062     ++p;
1063     TEST_NOT_EQUAL(p, db.postlist_end("world"));
1064     TEST_EQUAL(*p, 3);
1065     TEST_EQUAL(p.get_doclength(), 4);
1066     TEST_EQUAL(p.get_unique_terms(), 4);
1067     ++p;
1068     TEST_EQUAL(p, db.postlist_end("world"));
1069 }
1070
1071 // Test replacing a document which was deleted in the same batch.
1072 DEFINE_TESTCASE(replacedoc4, writable) {
1073     Xapian::WritableDatabase db = get_writable_database();
1074
1075     Xapian::Document doc1;
1076
1077     doc1.add_posting("foo", 1);
1078     doc1.add_posting("bar", 2);
1079     doc1.add_posting("aardvark", 3);
1080
1081     Xapian::docid did = db.add_document(doc1);
1082     TEST_EQUAL(did, 1);
1083
1084     doc1.remove_term("bar");
1085     doc1.add_term("hello");
1086
1087     did = db.add_document(doc1);
1088     TEST_EQUAL(did, 2);
1089
1090     doc1.add_term("world", 1);
1091     did = db.add_document(doc1);
1092     TEST_EQUAL(did, 3);
1093
1094     db.delete_document(2);
1095
1096     Xapian::Document doc2;
1097     doc2.add_term("world");
1098     db.replace_document(2, doc2);
1099
1100     db.commit();
1101
1102     // Check that the document exists (no DocNotFoundError).
1103     doc2 = db.get_document(2);
1104
1105     TEST_EQUAL(db.get_termfreq("foo"), 2);
1106     TEST_EQUAL(db.get_termfreq("aardvark"), 2);
1107     TEST_EQUAL(db.get_termfreq("hello"), 1);
1108     TEST_EQUAL(db.get_termfreq("world"), 2);
1109
1110     Xapian::PostingIterator p = db.postlist_begin("foo");
1111     TEST_NOT_EQUAL(p, db.postlist_end("foo"));
1112     TEST_EQUAL(*p, 1);
1113     ++p;
1114     TEST_NOT_EQUAL(p, db.postlist_end("foo"));
1115     TEST_EQUAL(*p, 3);
1116     ++p;
1117     TEST_EQUAL(p, db.postlist_end("foo"));
1118
1119     p = db.postlist_begin("world");
1120     TEST_NOT_EQUAL(p, db.postlist_end("world"));
1121     TEST_EQUAL(*p, 2);
1122     ++p;
1123     TEST_NOT_EQUAL(p, db.postlist_end("world"));
1124     TEST_EQUAL(*p, 3);
1125     ++p;
1126     TEST_EQUAL(p, db.postlist_end("world"));
1127 }
1128
1129 // Test replacing a document with itself without modifying postings.
1130 // Regression test for bug in 0.9.9 and earlier - there flint and quartz
1131 // lost all positional information for the document when you did this.
1132 DEFINE_TESTCASE(replacedoc5, writable) {
1133     Xapian::WritableDatabase db = get_writable_database();
1134
1135     {
1136         Xapian::Document doc;
1137         doc.add_posting("hello", 1);
1138         doc.add_posting("world", 2);
1139
1140         Xapian::docid did = db.add_document(doc);
1141         TEST_EQUAL(did, 1);
1142         db.commit();
1143     }
1144
1145     {
1146         Xapian::Document doc = db.get_document(1);
1147         TEST(db.has_positions());
1148         TEST(db.positionlist_begin(1, "hello") != db.positionlist_end(1, "hello"));
1149         TEST(db.positionlist_begin(1, "world") != db.positionlist_end(1, "world"));
1150         db.replace_document(1, doc);
1151         db.commit();
1152
1153         TEST(db.has_positions());
1154         TEST(db.positionlist_begin(1, "hello") != db.positionlist_end(1, "hello"));
1155         TEST(db.positionlist_begin(1, "world") != db.positionlist_end(1, "world"));
1156     }
1157
1158     // The backends now spot simple cases of replacing the same document and
1159     // don't do needless work.  Force them to actually do the replacement to
1160     // make sure that case works.
1161
1162     {
1163         Xapian::Document doc;
1164         doc.add_term("Q2");
1165         db.add_document(doc);
1166         db.commit();
1167     }
1168
1169     {
1170         Xapian::Document doc = db.get_document(1);
1171         TEST(db.has_positions());
1172         TEST(db.positionlist_begin(1, "hello") != db.positionlist_end(1, "hello"));
1173         TEST(db.positionlist_begin(1, "world") != db.positionlist_end(1, "world"));
1174         (void)db.get_document(2);
1175         db.replace_document(1, doc);
1176         db.commit();
1177
1178         TEST(db.has_positions());
1179         TEST(db.positionlist_begin(1, "hello") != db.positionlist_end(1, "hello"));
1180         TEST(db.positionlist_begin(1, "world") != db.positionlist_end(1, "world"));
1181     }
1182 }
1183
1184 // Test replacing a document while adding values, without changing anything
1185 // else.  Regression test for a bug introduced while implementing lazy update,
1186 // and also covers a few other code paths.
1187 DEFINE_TESTCASE(replacedoc6, writable) {
1188     Xapian::WritableDatabase db = get_writable_database();
1189
1190     Xapian::Document doc;
1191     Xapian::docid did = db.add_document(doc);
1192     TEST_EQUAL(did, 1);
1193     db.commit();
1194
1195     // Add document
1196     doc = db.get_document(1);
1197     TEST_EQUAL(doc.get_value(1), "");
1198     TEST_EQUAL(doc.get_value(2), "");
1199     doc.add_value(1, "banana1");
1200     db.replace_document(1, doc);
1201
1202     doc = db.get_document(1);
1203     TEST_EQUAL(doc.get_value(1), "banana1");
1204     TEST_EQUAL(doc.get_value(2), "");
1205     db.commit();
1206
1207     doc = db.get_document(1);
1208     TEST_EQUAL(doc.get_value(1), "banana1");
1209     TEST_EQUAL(doc.get_value(2), "");
1210     doc.add_value(2, "banana2");
1211     db.replace_document(1, doc);
1212
1213     TEST_EQUAL(doc.get_value(1), "banana1");
1214     TEST_EQUAL(doc.get_value(2), "banana2");
1215     db.commit();
1216
1217     doc = db.get_document(1);
1218     TEST_EQUAL(doc.get_value(1), "banana1");
1219     TEST_EQUAL(doc.get_value(2), "banana2");
1220 }
1221
1222 // Test of new feature: WritableDatabase::replace_document and delete_document
1223 // can take a unique termname instead of a document id as of Xapian 0.8.2.
1224 DEFINE_TESTCASE(uniqueterm1, writable) {
1225     Xapian::WritableDatabase db = get_writable_database();
1226
1227     for (int n = 1; n <= 20; ++n) {
1228         Xapian::Document doc;
1229         string uterm = "U" + str(n % 16);
1230         doc.add_term(uterm);
1231         doc.add_term(str(n));
1232         doc.add_term(str(n ^ 9));
1233         doc.add_term("all");
1234         doc.set_data("pass1");
1235         db.add_document(doc);
1236     }
1237
1238     TEST_EQUAL(db.get_doccount(), 20);
1239
1240     static const Xapian::doccount sizes[20] = {
1241         19, 17, 16, 15,
1242         15, 15, 15, 15,
1243         15, 15, 15, 15,
1244         15, 15, 15, 15,
1245         15, 15, 15, 15
1246     };
1247     for (int n = 1; n <= 20; ++n) {
1248         string uterm = "U" + str(n % 16);
1249         if (uterm == "U2") {
1250             db.delete_document(uterm);
1251         } else {
1252             Xapian::Document doc;
1253             doc.add_term(uterm);
1254             doc.add_term(str(n));
1255             doc.add_term(str(n ^ 9));
1256             doc.add_term("all");
1257             doc.set_data("pass2");
1258             db.replace_document(uterm, doc);
1259         }
1260         TEST_EQUAL(db.get_doccount(), sizes[n - 1]);
1261     }
1262
1263     string uterm = "U571";
1264     Xapian::Document doc;
1265     doc.add_term(uterm);
1266     doc.set_data("pass3");
1267     db.replace_document(uterm, doc);
1268
1269     TEST_EQUAL(db.get_doccount(), 16);
1270
1271     db.delete_document("U2");
1272
1273     TEST_EQUAL(db.get_doccount(), 16);
1274 }
1275
1276 // tests all document postlists
1277 DEFINE_TESTCASE(allpostlist2, writable) {
1278     Xapian::WritableDatabase db(get_writable_database("apitest_manydocs"));
1279     Xapian::PostingIterator i = db.postlist_begin("");
1280     unsigned int j = 1;
1281     while (i != db.postlist_end("")) {
1282         TEST_EQUAL(*i, j);
1283         i++;
1284         j++;
1285     }
1286     TEST_EQUAL(j, 513);
1287
1288     db.delete_document(1);
1289     db.delete_document(50);
1290     db.delete_document(512);
1291
1292     i = db.postlist_begin("");
1293     j = 2;
1294     while (i != db.postlist_end("")) {
1295         TEST_EQUAL(*i, j);
1296         i++;
1297         j++;
1298         if (j == 50) j++;
1299     }
1300     TEST_EQUAL(j, 512);
1301
1302     i = db.postlist_begin("");
1303     j = 2;
1304     while (i != db.postlist_end("")) {
1305         TEST_EQUAL(*i, j);
1306         i++;
1307         j++;
1308         if (j == 40) {
1309             j += 10;
1310             i.skip_to(j);
1311             j++;
1312         }
1313     }
1314     TEST_EQUAL(j, 512);
1315 }
1316
1317 static void test_emptyterm2_helper(Xapian::WritableDatabase & db)
1318 {
1319     // Don't bother with postlist_begin() because allpostlist tests cover that.
1320     TEST_EXCEPTION(Xapian::InvalidArgumentError, db.positionlist_begin(1, ""));
1321     TEST_EQUAL(db.get_doccount(), db.get_termfreq(""));
1322     TEST_EQUAL(db.get_doccount() != 0, db.term_exists(""));
1323     TEST_EQUAL(db.get_doccount(), db.get_collection_freq(""));
1324 }
1325
1326 // tests results of passing an empty term to various methods
1327 // equivalent of emptyterm1 for a writable database
1328 DEFINE_TESTCASE(emptyterm2, writable) {
1329     {
1330         Xapian::WritableDatabase db(get_writable_database("apitest_manydocs"));
1331         TEST_EQUAL(db.get_doccount(), 512);
1332         test_emptyterm2_helper(db);
1333         db.delete_document(1);
1334         TEST_EQUAL(db.get_doccount(), 511);
1335         test_emptyterm2_helper(db);
1336         db.delete_document(50);
1337         TEST_EQUAL(db.get_doccount(), 510);
1338         test_emptyterm2_helper(db);
1339         db.delete_document(512);
1340         TEST_EQUAL(db.get_doccount(), 509);
1341         test_emptyterm2_helper(db);
1342     }
1343
1344     {
1345         Xapian::WritableDatabase db(get_writable_database("apitest_onedoc"));
1346         TEST_EQUAL(db.get_doccount(), 1);
1347         test_emptyterm2_helper(db);
1348         db.delete_document(1);
1349         TEST_EQUAL(db.get_doccount(), 0);
1350         test_emptyterm2_helper(db);
1351     }
1352
1353     {
1354         Xapian::WritableDatabase db(get_writable_database());
1355         TEST_EQUAL(db.get_doccount(), 0);
1356         test_emptyterm2_helper(db);
1357     }
1358 }
1359
1360 static void
1361 gen_longpositionlist1_db(Xapian::WritableDatabase& db, const string&)
1362 {
1363     Xapian::Document doc;
1364     for (Xapian::termpos n = 1; n <= 2000; ++n) {
1365         doc.add_posting("fork", n * 3);
1366         doc.add_posting("knife", n * unsigned(log(double(n + 2))));
1367         doc.add_posting("spoon", n * n);
1368         // Exercise positions up to 4 billion.
1369         Xapian::termpos half_cube = n * n / 2 * n;
1370         doc.add_posting("chopsticks", half_cube);
1371         if constexpr(sizeof(Xapian::termpos) >= 8) {
1372             // Exercise 64-bit positions.
1373             doc.add_posting("spork", half_cube * half_cube);
1374         }
1375     }
1376     doc.set_data("cutlery");
1377     db.add_document(doc);
1378 }
1379
1380 // Check that a large number of position list entries for a particular term
1381 // works - regression test for flint.
1382 DEFINE_TESTCASE(longpositionlist1, backend) {
1383     Xapian::Database db = get_database("longpositionlist1",
1384                                        gen_longpositionlist1_db);
1385
1386     Xapian::Document doc = db.get_document(1);
1387
1388     Xapian::TermIterator t, tend;
1389     Xapian::PositionIterator p, pend;
1390
1391     t = doc.termlist_begin();
1392     tend = doc.termlist_end();
1393
1394     TEST(t != tend);
1395     TEST_EQUAL(*t, "chopsticks");
1396     p = t.positionlist_begin();
1397     pend = t.positionlist_end();
1398     for (Xapian::termpos n = 1; n <= 2000; ++n) {
1399         TEST(p != pend);
1400         Xapian::termpos half_cube = n * n / 2 * n;
1401         TEST_EQUAL(*p, half_cube);
1402         ++p;
1403     }
1404     TEST(p == pend);
1405
1406     ++t;
1407     TEST(t != tend);
1408     TEST_EQUAL(*t, "fork");
1409     p = t.positionlist_begin();
1410     pend = t.positionlist_end();
1411     for (Xapian::termpos n = 1; n <= 2000; ++n) {
1412         TEST(p != pend);
1413         TEST_EQUAL(*p, n * 3);
1414         ++p;
1415     }
1416     TEST(p == pend);
1417
1418     ++t;
1419     TEST(t != tend);
1420     TEST_EQUAL(*t, "knife");
1421     p = t.positionlist_begin();
1422     pend = t.positionlist_end();
1423     for (Xapian::termpos n = 1; n <= 2000; ++n) {
1424         TEST(p != pend);
1425         TEST_EQUAL(*p, n * unsigned(log(double(n + 2))));
1426         ++p;
1427     }
1428     TEST(p == pend);
1429
1430     ++t;
1431     TEST(t != tend);
1432     TEST_EQUAL(*t, "spoon");
1433     p = t.positionlist_begin();
1434     pend = t.positionlist_end();
1435     for (Xapian::termpos n = 1; n <= 2000; ++n) {
1436         TEST(p != pend);
1437         TEST_EQUAL(*p, n * n);
1438         ++p;
1439     }
1440     TEST(p == pend);
1441
1442     if constexpr(sizeof(Xapian::termpos) >= 8) {
1443         ++t;
1444         TEST(t != tend);
1445         TEST_EQUAL(*t, "spork");
1446         p = t.positionlist_begin();
1447         pend = t.positionlist_end();
1448         for (Xapian::termpos n = 1; n <= 2000; ++n) {
1449             TEST(p != pend);
1450             Xapian::termpos half_cube = n * n / 2 * n;
1451             TEST_EQUAL(*p, half_cube * half_cube);
1452             ++p;
1453         }
1454         TEST(p == pend);
1455     }
1456
1457     ++t;
1458     TEST(t == tend);
1459 }
1460
1461 static void
1462 gen_consistency2_db(Xapian::WritableDatabase& db, const string&)
1463 {
1464     char buf[2] = "X";
1465
1466     // Add 5 documents indexed by "test" with wdf 1.
1467     for (int i = 0; i < 5; ++i) {
1468         Xapian::Document doc;
1469         *buf = '0' + i;
1470         doc.add_value(0, buf);
1471         doc.add_term("test");
1472         db.add_document(doc);
1473     }
1474
1475     // Add 5 documents indexed by "test" with wdf 2.
1476     for (int i = 0; i < 5; ++i) {
1477         Xapian::Document doc;
1478         *buf = '0' + i;
1479         doc.add_value(0, buf);
1480         doc.add_term("test", 2);
1481         db.add_document(doc);
1482     }
1483 }
1484
1485 // Regression test for bug#110: Inconsistent sort order between pages with
1486 // set_sort_by_value_then_relevance.
1487 DEFINE_TESTCASE(consistency2, backend) {
1488     Xapian::Database db = get_database("consistency2", gen_consistency2_db);
1489
1490     Xapian::Enquire enq(db);
1491     enq.set_query(Xapian::Query("test"));
1492
1493     enq.set_sort_by_value_then_relevance(0, true);
1494
1495     // 10 results, unpaged.
1496     Xapian::MSet mset1 = enq.get_mset(0, 10);
1497     TEST_EQUAL(mset1.size(), 10);
1498
1499     // 10 results, split.
1500     Xapian::MSet mset2a = enq.get_mset(0, 1);
1501     TEST_EQUAL(mset2a.size(), 1);
1502     Xapian::MSet mset2b = enq.get_mset(1, 1);
1503     TEST_EQUAL(mset2b.size(), 1);
1504     Xapian::MSet mset2c = enq.get_mset(2, 8);
1505     TEST_EQUAL(mset2c.size(), 8);
1506
1507     TEST_EQUAL(*mset1[0], *mset2a[0]);
1508     TEST_EQUAL(*mset1[1], *mset2b[0]);
1509     for (Xapian::doccount i = 0; i < 8; ++i) {
1510         TEST_EQUAL(*mset1[i + 2], *mset2c[i]);
1511     }
1512 }
1513
1514 // Check that DatabaseError is thrown if the docid counter would wrap.
1515 // Regression test for bug#152.
1516 DEFINE_TESTCASE(nomoredocids1, writable) {
1517     // The InMemory backend uses a vector for the documents, so trying to add
1518     // a document with a really large docid will fail because we can't allocate
1519     // enough memory!
1520     SKIP_TEST_FOR_BACKEND("inmemory");
1521
1522     Xapian::WritableDatabase db = get_writable_database();
1523     Xapian::Document doc;
1524     doc.set_data("prose");
1525     doc.add_term("word");
1526
1527     Xapian::docid max_id = numeric_limits<Xapian::docid>::max();
1528     db.replace_document(max_id, doc);
1529
1530     TEST_EXCEPTION(Xapian::DatabaseError, db.add_document(doc));
1531
1532     // Also test replace_document() by term which will try to add a new
1533     // document if the term isn't present - that should also fail if the
1534     // docid counter would wrap.
1535     TEST_EXCEPTION(Xapian::DatabaseError, db.replace_document("Q42", doc));
1536 }
1537
1538 static void
1539 gen_synonym_merge1a_db(Xapian::WritableDatabase& db, const string&)
1540 {
1541     db.add_synonym("abc", "asd");
1542     db.add_synonym("abc", "asd2");
1543     db.add_synonym("abc", "asd4");
1544     db.add_synonym("abc", "asd6");
1545 }
1546
1547 static void
1548 gen_synonym_merge1b_db(Xapian::WritableDatabase& db, const string&)
1549 {
1550     db.add_synonym("abc", "asd1");
1551     db.add_synonym("abc", "asd2");
1552     db.add_synonym("abc", "asd3");
1553     db.add_synonym("abc", "asd5");
1554     db.add_synonym("abc", "asd10");
1555 }
1556
1557 // Test synonyms from subdbs are appropriately merged.
1558 DEFINE_TESTCASE(synonym_merge1, synonyms) {
1559     Xapian::Database db_multi;
1560     db_multi.add_database(get_database("synonym_merge1a",
1561                                        gen_synonym_merge1a_db));
1562     db_multi.add_database(get_database("synonym_merge1b",
1563                                        gen_synonym_merge1b_db));
1564
1565     Xapian::TermIterator t;
1566     string s, e;
1567     s = "|";
1568     t = db_multi.synonyms_begin("abc");
1569     while (t != db_multi.synonyms_end("abc")) {
1570         s += *t++;
1571         s += '|';
1572     }
1573     e = "|asd|asd1|asd10|asd2|asd3|asd4|asd5|asd6|";
1574     TEST_STRINGS_EQUAL(s, e);
1575 }
1576
1577
1578 // Test synonym iterators.
1579 DEFINE_TESTCASE(synonymitor1, writable && synonyms) {
1580     Xapian::WritableDatabase db = get_writable_database();
1581
1582     // Test iterators for terms which aren't there.
1583     TEST(db.synonyms_begin("abc") == db.synonyms_end("abc"));
1584
1585     // Test iterating the synonym keys when there aren't any.
1586     TEST(db.synonym_keys_begin() == db.synonym_keys_end());
1587
1588     db.add_synonym("hello", "howdy");
1589     db.add_synonym("hello", "hi");
1590     db.add_synonym("goodbye", "bye");
1591     db.add_synonym("goodbye", "farewell");
1592
1593     Xapian::TermIterator t;
1594     string s;
1595
1596     // Try these tests twice - once before committing and once after.
1597     for (int times = 1; times <= 2; ++times) {
1598         // Test iterators for terms which aren't there.
1599         TEST(db.synonyms_begin("abc") == db.synonyms_end("abc"));
1600         TEST(db.synonyms_begin("ghi") == db.synonyms_end("ghi"));
1601         TEST(db.synonyms_begin("zzzzz") == db.synonyms_end("zzzzz"));
1602
1603         s = "|";
1604         t = db.synonyms_begin("hello");
1605         while (t != db.synonyms_end("hello")) {
1606             s += *t++;
1607             s += '|';
1608         }
1609         TEST_STRINGS_EQUAL(s, "|hi|howdy|");
1610
1611         s = "|";
1612         t = db.synonyms_begin("goodbye");
1613         while (t != db.synonyms_end("goodbye")) {
1614             s += *t++;
1615             s += '|';
1616         }
1617         TEST_STRINGS_EQUAL(s, "|bye|farewell|");
1618
1619         s = "|";
1620         t = db.synonym_keys_begin();
1621         while (t != db.synonym_keys_end()) {
1622             s += *t++;
1623             s += '|';
1624         }
1625         TEST_STRINGS_EQUAL(s, "|goodbye|hello|");
1626
1627         db.commit();
1628     }
1629
1630     // Delete a synonym for "hello" and all synonyms for "goodbye".
1631     db.remove_synonym("hello", "hi");
1632     db.clear_synonyms("goodbye");
1633
1634     // Try these tests twice - once before committing and once after.
1635     for (int times = 1; times <= 2; ++times) {
1636         // Test iterators for terms which aren't there.
1637         TEST(db.synonyms_begin("abc") == db.synonyms_end("abc"));
1638         TEST(db.synonyms_begin("ghi") == db.synonyms_end("ghi"));
1639         TEST(db.synonyms_begin("zzzzz") == db.synonyms_end("zzzzz"));
1640
1641         s = "|";
1642         t = db.synonyms_begin("hello");
1643         while (t != db.synonyms_end("hello")) {
1644             s += *t++;
1645             s += '|';
1646         }
1647         TEST_STRINGS_EQUAL(s, "|howdy|");
1648
1649         TEST(db.synonyms_begin("goodbye") == db.synonyms_end("goodbye"));
1650
1651         s = "|";
1652         t = db.synonym_keys_begin();
1653         while (t != db.synonym_keys_end()) {
1654             s += *t++;
1655             s += '|';
1656         }
1657         TEST_STRINGS_EQUAL(s, "|hello|");
1658
1659         db.commit();
1660     }
1661
1662     Xapian::Database db_multi;
1663     db_multi.add_database(db);
1664     db_multi.add_database(get_database("apitest_simpledata"));
1665
1666     // Test iterators for terms which aren't there.
1667     TEST(db_multi.synonyms_begin("abc") == db_multi.synonyms_end("abc"));
1668     TEST(db_multi.synonyms_begin("ghi") == db_multi.synonyms_end("ghi"));
1669     TEST(db_multi.synonyms_begin("zzzzz") == db_multi.synonyms_end("zzzzz"));
1670
1671     s = "|";
1672     t = db_multi.synonyms_begin("hello");
1673     while (t != db_multi.synonyms_end("hello")) {
1674         s += *t++;
1675         s += '|';
1676     }
1677     TEST_STRINGS_EQUAL(s, "|howdy|");
1678
1679     TEST(db_multi.synonyms_begin("goodbye") == db_multi.synonyms_end("goodbye"));
1680
1681     s = "|";
1682     t = db_multi.synonym_keys_begin();
1683     while (t != db_multi.synonym_keys_end()) {
1684         s += *t++;
1685         s += '|';
1686     }
1687     TEST_STRINGS_EQUAL(s, "|hello|");
1688 }
1689
1690 // Test that adding a document with a really long term gives an error on
1691 // add_document() rather than on commit().
1692 DEFINE_TESTCASE(termtoolong1, writable) {
1693     // Inmemory doesn't impose a limit.
1694     SKIP_TEST_FOR_BACKEND("inmemory");
1695
1696     Xapian::WritableDatabase db = get_writable_database();
1697
1698     for (size_t i = 246; i <= 290; ++i) {
1699         tout.str(string());
1700         tout << "Term length " << i << '\n';
1701         Xapian::Document doc;
1702         string term(i, 'X');
1703         doc.add_term(term);
1704         try {
1705             db.add_document(doc);
1706             TEST_AND_EXPLAIN(false, "Expecting exception InvalidArgumentError");
1707         } catch (const Xapian::InvalidArgumentError &e) {
1708             // Check that the max length is correctly expressed in the
1709             // exception message - we've got this wrong in two different ways
1710             // in the past!
1711             tout << e.get_msg() << '\n';
1712             TEST(e.get_msg().find("Term too long (> 245)") != string::npos);
1713         }
1714     }
1715
1716     for (size_t j = 240; j <= 245; ++j) {
1717         tout.str(string());
1718         tout << "Term length " << j << '\n';
1719         Xapian::Document doc;
1720         string term(j, 'X');
1721         doc.add_term(term);
1722         db.add_document(doc);
1723     }
1724
1725     db.commit();
1726
1727     size_t limit = 255;
1728     {
1729         // Currently glass escapes zero bytes from terms in keys for
1730         // some tables, so a term with 128 zero bytes won't work for glass.
1731         Xapian::Document doc;
1732         doc.add_term(string(limit / 2 + 1, '\0'));
1733         db.add_document(doc);
1734         try {
1735             db.commit();
1736             TEST_AND_EXPLAIN(false, "Expecting exception InvalidArgumentError");
1737         } catch (const Xapian::InvalidArgumentError &e) {
1738             // Check that the max length is correctly expressed in the
1739             // exception message - we've got this wrong in two different ways
1740             // in the past!
1741             tout << e.get_msg() << '\n';
1742             string target = " is ";
1743             target += str(limit);
1744             target += " bytes";
1745             TEST(e.get_msg().find(target) != string::npos);
1746         }
1747     }
1748
1749     // Try adding a document.  Regression test for a bug fixed in 1.4.15 - in
1750     // earlier versions the pending changes which caused the
1751     // InvalidArgumentError were left around and a subsequent commit() on the
1752     // same WritableDatabase would also fail with InvalidArgumentError.
1753     Xapian::Document doc;
1754     doc.add_term("ok");
1755     db.add_document(doc);
1756     db.commit();
1757 }
1758
1759 /// Test playing with a postlist
1760 DEFINE_TESTCASE(postlist7, writable) {
1761     Xapian::WritableDatabase db_w = get_writable_database();
1762
1763     {
1764         Xapian::Document doc;
1765         doc.add_term("foo", 3);
1766         doc.add_term("zz", 4);
1767         db_w.replace_document(5, doc);
1768     }
1769
1770     Xapian::PostingIterator p;
1771     p = db_w.postlist_begin("foo");
1772     TEST(p != db_w.postlist_end("foo"));
1773     TEST_EQUAL(*p, 5);
1774     TEST_EQUAL(p.get_wdf(), 3);
1775     TEST_EQUAL(p.get_doclength(), 7);
1776     TEST_EQUAL(p.get_unique_terms(), 2);
1777     ++p;
1778     TEST(p == db_w.postlist_end("foo"));
1779
1780     {
1781         Xapian::Document doc;
1782         doc.add_term("foo", 1);
1783         doc.add_term("zz", 1);
1784         db_w.replace_document(6, doc);
1785     }
1786
1787     p = db_w.postlist_begin("foo");
1788     TEST(p != db_w.postlist_end("foo"));
1789     TEST_EQUAL(*p, 5);
1790     TEST_EQUAL(p.get_wdf(), 3);
1791     TEST_EQUAL(p.get_doclength(), 7);
1792     TEST_EQUAL(p.get_unique_terms(), 2);
1793     ++p;
1794     TEST(p != db_w.postlist_end("foo"));
1795     TEST_EQUAL(*p, 6);
1796     TEST_EQUAL(p.get_wdf(), 1);
1797     TEST_EQUAL(p.get_doclength(), 2);
1798     TEST_EQUAL(p.get_unique_terms(), 2);
1799     ++p;
1800     TEST(p == db_w.postlist_end("foo"));
1801 }
1802
1803 static void
1804 gen_lazytablebug1_db(Xapian::WritableDatabase& db, const string&)
1805 {
1806     Xapian::Document doc;
1807     doc.add_term("foo");
1808     db.add_document(doc);
1809     db.commit();
1810
1811     string synonym(255, 'x');
1812     char buf[] = " iamafish!!!!!!!!!!";
1813     for (int i = 33; i < 120; ++i) {
1814         db.add_synonym(buf, synonym);
1815         ++buf[0];
1816     }
1817 }
1818
1819 DEFINE_TESTCASE(lazytablebug1, synonyms) {
1820     Xapian::Database db = get_database("lazytablebug1", gen_lazytablebug1_db);
1821     for (Xapian::TermIterator t = db.synonym_keys_begin(); t != db.synonym_keys_end(); ++t) {
1822         tout << *t << '\n';
1823     }
1824 }
1825
1826 /// Regression test for bug #287 for flint.
1827 DEFINE_TESTCASE(cursordelbug1, writable && path) {
1828     static const int terms[] = { 219, 221, 222, 223, 224, 225, 226 };
1829     static const int copies[] = { 74, 116, 199, 21, 45, 155, 189 };
1830
1831     Xapian::WritableDatabase db;
1832     db = get_named_writable_database("cursordelbug1", string());
1833
1834     for (size_t i = 0; i < sizeof(terms) / sizeof(terms[0]); ++i) {
1835         Xapian::Document doc;
1836         doc.add_term("XC" + str(terms[i]));
1837         doc.add_term("XTabc");
1838         doc.add_term("XAdef");
1839         doc.add_term("XRghi");
1840         doc.add_term("XYabc");
1841         for (size_t c = copies[i]; c; --c)
1842             db.add_document(doc);
1843     }
1844
1845     db.commit();
1846
1847     for (size_t i = 0; i < sizeof(terms) / sizeof(terms[0]); ++i) {
1848         db.delete_document("XC" + str(terms[i]));
1849     }
1850
1851     db.commit();
1852
1853     const string & db_path = get_named_writable_database_path("cursordelbug1");
1854     TEST_EQUAL(Xapian::Database::check(db_path), 0);
1855 }
1856
1857 /** Helper function for modifyvalues1.
1858  *
1859  * Check that the values stored in the database match */
1860 static void
1861 check_vals(const Xapian::Database & db, const map<Xapian::docid, string> & vals)
1862 {
1863     TEST_EQUAL(db.get_doccount(), vals.size());
1864     if (vals.empty()) return;
1865     TEST_REL(vals.rbegin()->first,<=,db.get_lastdocid());
1866     map<Xapian::docid, string>::const_iterator i;
1867     for (i = vals.begin(); i != vals.end(); ++i) {
1868         tout.str(string());
1869         tout << "Checking value in doc " << i->first << " - should be '" << i->second << "'\n";
1870         Xapian::Document doc = db.get_document(i->first);
1871         string dbval = doc.get_value(1);
1872         TEST_EQUAL(dbval, i->second);
1873         if (dbval.empty()) {
1874             TEST_EQUAL(0, doc.values_count());
1875             TEST_EQUAL(doc.values_begin(), doc.values_end());
1876         } else {
1877             TEST_EQUAL(1, doc.values_count());
1878             Xapian::ValueIterator valit = doc.values_begin();
1879             TEST_NOT_EQUAL(valit, doc.values_end());
1880             TEST_EQUAL(dbval, *valit);
1881             TEST_EQUAL(1, valit.get_valueno());
1882             ++valit;
1883             TEST_EQUAL(valit, doc.values_end());
1884         }
1885     }
1886 }
1887
1888 /** Regression test for bug in initial streaming values implementation in
1889  *  chert.
1890  */
1891 DEFINE_TESTCASE(modifyvalues1, writable) {
1892     unsigned int seed = 7;
1893     Xapian::WritableDatabase db = get_writable_database();
1894     // Note: doccount must be coprime with 13
1895     const Xapian::doccount doccount = 1000;
1896     static_assert(doccount % 13 != 0, "doccount divisible by 13");
1897
1898     map<Xapian::docid, string> vals;
1899
1900     for (Xapian::doccount num = 1; num <= doccount; ++num) {
1901         tout.str(string());
1902         Xapian::Document doc;
1903         string val = "val" + str(num);
1904         tout << "Setting val '" << val << "' in doc " << num << "\n";
1905         doc.add_value(1, val);
1906         db.add_document(doc);
1907         vals[num] = val;
1908     }
1909     check_vals(db, vals);
1910     db.commit();
1911     check_vals(db, vals);
1912
1913     // Modify one of the values (this is a regression test which failed with
1914     // the initial implementation of streaming values).
1915     {
1916         Xapian::Document doc;
1917         string val = "newval0";
1918         tout << "Setting val '" << val << "' in doc 2\n";
1919         doc.add_value(1, val);
1920         db.replace_document(2, doc);
1921         vals[2] = val;
1922         check_vals(db, vals);
1923         db.commit();
1924         check_vals(db, vals);
1925     }
1926
1927     // Check that value doesn't get lost when replacing a document with itself.
1928     {
1929         tout << "Replacing document 1 with itself\n";
1930         Xapian::Document doc = db.get_document(1);
1931         db.replace_document(1, doc);
1932         check_vals(db, vals);
1933         db.commit();
1934         check_vals(db, vals);
1935     }
1936
1937     // Check that value doesn't get lost when replacing a document with itself,
1938     // accessing another document in the meantime.  This is a regression test
1939     // for a bug in the code which implements lazy updates - this used to
1940     // forget the values in the document in this situation.
1941     {
1942         tout << "Replacing document 1 with itself, after reading doc 2.\n";
1943         Xapian::Document doc = db.get_document(1);
1944         db.get_document(2);
1945         db.replace_document(1, doc);
1946         check_vals(db, vals);
1947         db.commit();
1948         check_vals(db, vals);
1949     }
1950
1951     // Do some random modifications: seed random generator, for repeatable
1952     // results.
1953     tout << "Setting seed to " << seed << "\n";
1954     srand(seed);
1955     for (Xapian::doccount num = 1; num <= doccount * 2; ++num) {
1956         tout.str(string());
1957         Xapian::docid did = ((rand() >> 8) % doccount) + 1;
1958         Xapian::Document doc;
1959         string val;
1960
1961         if (num % 5 != 0) {
1962             val = "newval" + str(num);
1963             tout << "Setting val '" << val << "' in doc " << did << "\n";
1964             doc.add_value(1, val);
1965         } else {
1966             tout << "Adding/replacing empty document " << did << "\n";
1967         }
1968         db.replace_document(did, doc);
1969         vals[did] = val;
1970     }
1971     check_vals(db, vals);
1972     db.commit();
1973     check_vals(db, vals);
1974
1975     // Delete all the remaining values, in a slightly shuffled order.
1976     // This is where it's important that doccount is coprime with 13.
1977     for (Xapian::doccount num = 0; num < doccount * 13; num += 13) {
1978         tout.str(string());
1979         Xapian::docid did = (num % doccount) + 1;
1980         tout << "Clearing val in doc " << did << "\n";
1981         Xapian::Document doc;
1982         db.replace_document(did, doc);
1983         vals[did] = string();
1984     }
1985     check_vals(db, vals);
1986     db.commit();
1987     check_vals(db, vals);
1988 }
1989
1990 /** Regression test for protocol design bug.
1991  *
1992  *  Previously some messages didn't send a reply but could result in an
1993  *  exception being sent over the link.  That exception would then get
1994  *  read as a response to the next message instead of its actual response
1995  *  so we'd be out of step.
1996  *
1997  *  This also affected MSG_DELETEDOCUMENTTERM, MSG_CANCEL, MSG_SETMETADATA
1998  *  and MSG_ADDSPELLING but it's harder to reliably trigger an exception
1999  *  from any of those.
2000  *
2001  *  See #783.  Fixed in 1.4.12.
2002  */
2003 DEFINE_TESTCASE(protocolbug1, remote && writable) {
2004     Xapian::WritableDatabase db = get_writable_database("");
2005     Xapian::Document doc;
2006     doc.add_term(string(300, 'x'));
2007
2008     TEST_EXCEPTION(Xapian::InvalidArgumentError,
2009                    db.replace_document(1, doc));
2010     db.commit();
2011 }
2012
2013 DEFINE_TESTCASE(remotefdleak1, remote && writable) {
2014     Xapian::WritableDatabase wdb = get_writable_database("");
2015     TEST_EXCEPTION(Xapian::DatabaseLockError,
2016                    auto wdb2 = get_writable_database_again());
2017 }