xapian-core/tests/api_replicate.cc

   1 /* api_replicate.cc: tests of replication functionality
   2  *
   3  * Copyright 2008 Lemur Consulting Ltd
   4  * Copyright 2009,2010,2011,2012,2013,2014,2015 Olly Betts
   5  * Copyright 2010 Richard Boulton
   6  * Copyright 2011 Dan Colish
   7  *
   8  * This program is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU General Public License as
  10  * published by the Free Software Foundation; either version 2 of the
  11  * License, or (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  21  * USA
  22  */
  23
  24 #include <config.h>
  25
  26 #include "api_replicate.h"
  27
  28 #include <xapian.h>
  29 #include "api/replication.h"
  30
  31 #include "apitest.h"
  32 #include "dbcheck.h"
  33 #include "fd.h"
  34 #include "filetests.h"
  35 #include "safeerrno.h"
  36 #include "safefcntl.h"
  37 #include "safesysstat.h"
  38 #include "safeunistd.h"
  39 #include "testsuite.h"
  40 #include "testutils.h"
  41 #include "unixcmds.h"
  42
  43 #include <sys/types.h>
  44
  45 #include <cstdlib>
  46 #include <string>
  47
  48 #include <stdlib.h> // For setenv() or putenv()
  49
  50 using namespace std;
  51
  52 static void rmtmpdir(const string & path) {
  53     rm_rf(path);
  54 }
  55
  56 static void mktmpdir(const string & path) {
  57     rmtmpdir(path);
  58     if (mkdir(path.c_str(), 0700) == -1 && errno != EEXIST) {
  59         FAIL_TEST("Can't make temporary directory");
  60     }
  61 }
  62
  63 static off_t get_file_size(const string & path) {
  64     off_t size = file_size(path);
  65     if (errno) {
  66         FAIL_TEST("Can't stat '" + path + "'");
  67     }
  68     return size;
  69 }
  70
  71 static size_t do_read(int fd, char * p, size_t desired)
  72 {
  73     size_t total = 0;
  74     while (desired) {
  75         ssize_t c = read(fd, p, desired);
  76         if (c == 0) return total;
  77         if (c < 0) {
  78             if (errno == EINTR) continue;
  79             FAIL_TEST("Error reading from file");
  80         }
  81         p += c;
  82         total += c;
  83         desired -= c;
  84     }
  85     return total;
  86 }
  87
  88 static void do_write(int fd, const char * p, size_t n)
  89 {
  90     while (n) {
  91         ssize_t c = write(fd, p, n);
  92         if (c < 0) {
  93             if (errno == EINTR) continue;
  94             FAIL_TEST("Error writing to file");
  95         }
  96         p += c;
  97         n -= c;
  98     }
  99 }
 100
 101 // Make a truncated copy of a file.
 102 static off_t
 103 truncated_copy(const string & srcpath, const string & destpath, off_t tocopy)
 104 {
 105     FD fdin(open(srcpath.c_str(), O_RDONLY | O_BINARY));
 106     if (fdin == -1) {
 107         FAIL_TEST("Open failed (when opening '" + srcpath + "')");
 108     }
 109
 110     FD fdout(open(destpath.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666));
 111     if (fdout == -1) {
 112         FAIL_TEST("Open failed (when creating '" + destpath + "')");
 113     }
 114
 115     const int BUFSIZE = 1024;
 116     char buf[BUFSIZE];
 117     size_t total_bytes = 0;
 118     while (tocopy > 0) {
 119         size_t thiscopy = tocopy > BUFSIZE ? BUFSIZE : tocopy;
 120         size_t bytes = do_read(fdin, buf, thiscopy);
 121         if (thiscopy != bytes) {
 122             FAIL_TEST("Couldn't read desired number of bytes from changeset");
 123         }
 124         tocopy -= bytes;
 125         total_bytes += bytes;
 126         do_write(fdout, buf, bytes);
 127     }
 128
 129     if (close(fdout) == -1)
 130         FAIL_TEST("Error closing file");
 131
 132     return total_bytes;
 133 }
 134
 135 static void
 136 get_changeset(const string & changesetpath,
 137               Xapian::DatabaseMaster & master,
 138               Xapian::DatabaseReplica & replica,
 139               int expected_changesets,
 140               int expected_fullcopies,
 141               bool expected_changed,
 142               bool full_copy = false)
 143 {
 144     FD fd(open(changesetpath.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666));
 145     if (fd == -1) {
 146         FAIL_TEST("Open failed (when creating a new changeset file at '"
 147                   + changesetpath + "')");
 148     }
 149     Xapian::ReplicationInfo info1;
 150     master.write_changesets_to_fd(fd,
 151                                   full_copy ? "" : replica.get_revision_info(),
 152                                   &info1);
 153
 154     TEST_EQUAL(info1.changeset_count, expected_changesets);
 155     TEST_EQUAL(info1.fullcopy_count, expected_fullcopies);
 156     TEST_EQUAL(info1.changed, expected_changed);
 157 }
 158
 159 static int
 160 apply_changeset(const string & changesetpath,
 161                 Xapian::DatabaseReplica & replica,
 162                 int expected_changesets,
 163                 int expected_fullcopies,
 164                 bool expected_changed)
 165 {
 166     FD fd(open(changesetpath.c_str(), O_RDONLY | O_BINARY));
 167     if (fd == -1) {
 168         FAIL_TEST("Open failed (when reading changeset file at '"
 169                   + changesetpath + "')");
 170     }
 171
 172     int count = 1;
 173     replica.set_read_fd(fd);
 174     Xapian::ReplicationInfo info1;
 175     Xapian::ReplicationInfo info2;
 176     bool client_changed = false;
 177     while (replica.apply_next_changeset(&info2, 0)) {
 178         ++count;
 179         info1.changeset_count += info2.changeset_count;
 180         info1.fullcopy_count += info2.fullcopy_count;
 181         if (info2.changed)
 182             client_changed = true;
 183     }
 184     info1.changeset_count += info2.changeset_count;
 185     info1.fullcopy_count += info2.fullcopy_count;
 186     if (info2.changed)
 187         client_changed = true;
 188
 189     TEST_EQUAL(info1.changeset_count, expected_changesets);
 190     TEST_EQUAL(info1.fullcopy_count, expected_fullcopies);
 191     TEST_EQUAL(client_changed, expected_changed);
 192     return count;
 193 }
 194
 195 // Replicate from the master to the replica.
 196 // Returns the number of changesets which were applied.
 197 static int
 198 replicate(Xapian::DatabaseMaster & master,
 199           Xapian::DatabaseReplica & replica,
 200           const string & tempdir,
 201           int expected_changesets,
 202           int expected_fullcopies,
 203           bool expected_changed,
 204           bool full_copy = false)
 205 {
 206     string changesetpath = tempdir + "/changeset";
 207     get_changeset(changesetpath, master, replica,
 208                   expected_changesets,
 209                   expected_fullcopies,
 210                   expected_changed,
 211                   full_copy);
 212     return apply_changeset(changesetpath, replica,
 213                            expected_changesets,
 214                            expected_fullcopies,
 215                            expected_changed);
 216 }
 217
 218 // Check that the databases held at the given path are identical.
 219 static void
 220 check_equal_dbs(const string & masterpath, const string & replicapath)
 221 {
 222     Xapian::Database master(masterpath);
 223     Xapian::Database replica(replicapath);
 224
 225     TEST_EQUAL(master.get_uuid(), master.get_uuid());
 226     dbcheck(replica, master.get_doccount(), master.get_lastdocid());
 227
 228     for (Xapian::TermIterator t = master.allterms_begin();
 229          t != master.allterms_end(); ++t) {
 230         TEST_EQUAL(postlist_to_string(master, *t),
 231                    postlist_to_string(replica, *t));
 232     }
 233 }
 234
 235 #if 0 // Dynamic version which we don't currently need.
 236 static void
 237 set_max_changesets(int count) {
 238 #ifdef HAVE__PUTENV_S
 239     _putenv_s("XAPIAN_MAX_CHANGESETS", str(count).c_str());
 240 #elif defined HAVE_SETENV
 241     setenv("XAPIAN_MAX_CHANGESETS", str(count).c_str(), 1);
 242 #else
 243     static char buf[64] = "XAPIAN_MAX_CHANGESETS=";
 244     sprintf(buf + CONST_STRLEN("XAPIAN_MAX_CHANGESETS="), "%d", count);
 245     putenv(buf);
 246 #endif
 247 }
 248 #endif
 249
 250 #ifdef HAVE__PUTENV_S
 251 # define set_max_changesets(N) _putenv_s("XAPIAN_MAX_CHANGESETS", #N)
 252 #elif defined HAVE_SETENV
 253 # define set_max_changesets(N) setenv("XAPIAN_MAX_CHANGESETS", #N, 1)
 254 #else
 255 # define set_max_changesets(N) putenv(const_cast<char*>("XAPIAN_MAX_CHANGESETS="#N))
 256 #endif
 257
 258 struct unset_max_changesets_helper_ {
 259     unset_max_changesets_helper_() { }
 260     ~unset_max_changesets_helper_() { set_max_changesets(0); }
 261 };
 262
 263 // Ensure that we don't leave generation of changesets on for the next
 264 // testcase, even if this one exits with an exception.
 265 #define UNSET_MAX_CHANGESETS_AFTERWARDS unset_max_changesets_helper_ ezlxq
 266
 267 // #######################################################################
 268 // # Tests start here
 269
 270 // Basic test of replication functionality.
 271 DEFINE_TESTCASE(replicate1, replicas) {
 272     UNSET_MAX_CHANGESETS_AFTERWARDS;
 273     string tempdir = ".replicatmp";
 274     mktmpdir(tempdir);
 275     string masterpath = get_named_writable_database_path("master");
 276
 277     set_max_changesets(10);
 278
 279     Xapian::WritableDatabase orig(get_named_writable_database("master"));
 280     Xapian::DatabaseMaster master(masterpath);
 281     string replicapath = tempdir + "/replica";
 282     Xapian::DatabaseReplica replica(replicapath);
 283
 284     // Add a document to the original database.
 285     Xapian::Document doc1;
 286     doc1.set_data(string("doc1"));
 287     doc1.add_posting("doc", 1);
 288     doc1.add_posting("one", 1);
 289     orig.add_document(doc1);
 290     orig.commit();
 291
 292     // Apply the replication - we don't have changesets stored, so this should
 293     // just do a database copy, and return a count of 1.
 294     int count = replicate(master, replica, tempdir, 0, 1, true);
 295     TEST_EQUAL(count, 1);
 296     {
 297         Xapian::Database dbcopy(replicapath);
 298         TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 299     }
 300
 301     // Repeating the replication should return a count of 1, since no further
 302     // changes should need to be applied.
 303     count = replicate(master, replica, tempdir, 0, 0, false);
 304     TEST_EQUAL(count, 1);
 305     {
 306         Xapian::Database dbcopy(replicapath);
 307         TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 308     }
 309
 310     // Regression test - if the replica was reopened, a full copy always used
 311     // to occur, whether it was needed or not.  Fixed in revision #10117.
 312     replica.close();
 313     replica = Xapian::DatabaseReplica(replicapath);
 314     count = replicate(master, replica, tempdir, 0, 0, false);
 315     TEST_EQUAL(count, 1);
 316     {
 317         Xapian::Database dbcopy(replicapath);
 318         TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 319     }
 320
 321     orig.add_document(doc1);
 322     orig.commit();
 323     orig.add_document(doc1);
 324     orig.commit();
 325
 326     count = replicate(master, replica, tempdir, 2, 0, true);
 327     TEST_EQUAL(count, 3);
 328     {
 329         Xapian::Database dbcopy(replicapath);
 330         TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 331     }
 332
 333     check_equal_dbs(masterpath, replicapath);
 334
 335     // Need to close the replica before we remove the temporary directory on
 336     // Windows.
 337     replica.close();
 338     rmtmpdir(tempdir);
 339     return true;
 340 }
 341
 342 // Test replication from a replicated copy.
 343 DEFINE_TESTCASE(replicate2, replicas) {
 344     SKIP_TEST_FOR_BACKEND("glass"); // Glass doesn't currently support this.
 345     UNSET_MAX_CHANGESETS_AFTERWARDS;
 346
 347     string tempdir = ".replicatmp";
 348     mktmpdir(tempdir);
 349     string masterpath = get_named_writable_database_path("master");
 350
 351     set_max_changesets(10);
 352
 353     Xapian::WritableDatabase orig(get_named_writable_database("master"));
 354     Xapian::DatabaseMaster master(masterpath);
 355     string replicapath = tempdir + "/replica";
 356     Xapian::DatabaseReplica replica(replicapath);
 357
 358     Xapian::DatabaseMaster master2(replicapath);
 359     string replica2path = tempdir + "/replica2";
 360     Xapian::DatabaseReplica replica2(replica2path);
 361
 362     // Add a document to the original database.
 363     Xapian::Document doc1;
 364     doc1.set_data(string("doc1"));
 365     doc1.add_posting("doc", 1);
 366     doc1.add_posting("one", 1);
 367     orig.add_document(doc1);
 368     orig.commit();
 369
 370     // Apply the replication - we don't have changesets stored, so this should
 371     // just do a database copy, and return a count of 1.
 372     TEST_EQUAL(replicate(master, replica, tempdir, 0, 1, true), 1);
 373     check_equal_dbs(masterpath, replicapath);
 374
 375     // Replicate from the replica.
 376     TEST_EQUAL(replicate(master2, replica2, tempdir, 0, 1, true), 1);
 377     check_equal_dbs(masterpath, replica2path);
 378
 379     orig.add_document(doc1);
 380     orig.commit();
 381     orig.add_document(doc1);
 382     orig.commit();
 383
 384     // Replicate from the replica - should have no changes.
 385     TEST_EQUAL(replicate(master2, replica2, tempdir, 0, 0, false), 1);
 386     check_equal_dbs(replicapath, replica2path);
 387
 388     // Replicate, and replicate from the replica - should have 2 changes.
 389     TEST_EQUAL(replicate(master, replica, tempdir, 2, 0, 1), 3);
 390     check_equal_dbs(masterpath, replicapath);
 391     TEST_EQUAL(replicate(master2, replica2, tempdir, 2, 0, 1), 3);
 392     check_equal_dbs(masterpath, replica2path);
 393
 394     // Stop writing changesets, and make a modification
 395     set_max_changesets(0);
 396     orig.close();
 397     orig = get_writable_database_again();
 398     orig.add_document(doc1);
 399     orig.commit();
 400
 401     // Replication should do a full copy.
 402     TEST_EQUAL(replicate(master, replica, tempdir, 0, 1, true), 1);
 403     check_equal_dbs(masterpath, replicapath);
 404     TEST_EQUAL(replicate(master2, replica2, tempdir, 0, 1, true), 1);
 405     check_equal_dbs(masterpath, replica2path);
 406
 407     // Start writing changesets, but only keep 1 in history, and make a
 408     // modification.
 409     set_max_changesets(1);
 410     orig.close();
 411     orig = get_writable_database_again();
 412     orig.add_document(doc1);
 413     orig.commit();
 414
 415     // Replicate, and replicate from the replica - should have 1 changes.
 416     TEST_EQUAL(replicate(master, replica, tempdir, 1, 0, 1), 2);
 417     check_equal_dbs(masterpath, replicapath);
 418     TEST_EQUAL(replicate(master2, replica2, tempdir, 1, 0, 1), 2);
 419     check_equal_dbs(masterpath, replica2path);
 420
 421     // Make two changes - only one changeset should be preserved.
 422     orig.add_document(doc1);
 423     orig.commit();
 424
 425     // Replication should do a full copy, since one of the needed changesets
 426     // is missing.
 427
 428     //FIXME - the following tests are commented out because the backends don't currently tidy up old changesets correctly.
 429     //TEST_EQUAL(replicate(master, replica, tempdir, 0, 1, true), 1);
 430     //check_equal_dbs(masterpath, replicapath);
 431     //TEST_EQUAL(replicate(master2, replica2, tempdir, 0, 1, true), 1);
 432     //check_equal_dbs(masterpath, replica2path);
 433
 434     // Need to close the replicas before we remove the temporary directory on
 435     // Windows.
 436     replica.close();
 437     replica2.close();
 438     rmtmpdir(tempdir);
 439     return true;
 440 }
 441
 442 static void
 443 replicate_with_brokenness(Xapian::DatabaseMaster & master,
 444                           Xapian::DatabaseReplica & replica,
 445                           const string & tempdir,
 446                           int expected_changesets,
 447                           int expected_fullcopies,
 448                           bool expected_changed)
 449 {
 450     string changesetpath = tempdir + "/changeset";
 451     get_changeset(changesetpath, master, replica,
 452                   1, 0, 1);
 453
 454     // Try applying truncated changesets of various different lengths.
 455     string brokenchangesetpath = tempdir + "/changeset_broken";
 456     off_t filesize = get_file_size(changesetpath);
 457     off_t len = 10;
 458     off_t copylen;
 459     while (len < filesize) {
 460         copylen = truncated_copy(changesetpath, brokenchangesetpath, len);
 461         TEST_EQUAL(copylen, len);
 462         tout << "Trying replication with a changeset truncated to " << len <<
 463                 " bytes, from " << filesize << " bytes\n";
 464         TEST_EXCEPTION(Xapian::NetworkError,
 465                        apply_changeset(brokenchangesetpath, replica,
 466                                        expected_changesets, expected_fullcopies,
 467                                        expected_changed));
 468         if (len < 30 || len >= filesize - 10) {
 469             // For lengths near the beginning and end, increment size by 1
 470             ++len;
 471         } else {
 472             // Don't bother incrementing by small amounts in the middle of
 473             // the changeset.
 474             len += 1000;
 475             if (len >= filesize - 10) {
 476                 len = filesize - 10;
 477             }
 478         }
 479     }
 480 }
 481
 482 // Test changesets which are truncated (and therefore invalid).
 483 DEFINE_TESTCASE(replicate3, replicas) {
 484     UNSET_MAX_CHANGESETS_AFTERWARDS;
 485     string tempdir = ".replicatmp";
 486     mktmpdir(tempdir);
 487     string masterpath = get_named_writable_database_path("master");
 488
 489     set_max_changesets(10);
 490
 491     Xapian::WritableDatabase orig(get_named_writable_database("master"));
 492     Xapian::DatabaseMaster master(masterpath);
 493     string replicapath = tempdir + "/replica";
 494     Xapian::DatabaseReplica replica(replicapath);
 495
 496     // Add a document to the original database.
 497     Xapian::Document doc1;
 498     doc1.set_data(string("doc1"));
 499     doc1.add_posting("doc", 1);
 500     doc1.add_posting("one", 1);
 501     orig.add_document(doc1);
 502     orig.commit();
 503
 504     TEST_EQUAL(replicate(master, replica, tempdir, 0, 1, true), 1);
 505     check_equal_dbs(masterpath, replicapath);
 506
 507     // Make a changeset.
 508     orig.add_document(doc1);
 509     orig.commit();
 510
 511     replicate_with_brokenness(master, replica, tempdir, 1, 0, true);
 512     // Although it throws an error, the final replication in
 513     // replicate_with_brokenness() updates the database, since it's just the
 514     // end-of-replication message which is missing its body.
 515     check_equal_dbs(masterpath, replicapath);
 516
 517     // Check that the earlier broken replications didn't cause any problems for the
 518     // next replication.
 519     orig.add_document(doc1);
 520     orig.commit();
 521     TEST_EQUAL(replicate(master, replica, tempdir, 1, 0, true), 2);
 522
 523     // Need to close the replicas before we remove the temporary directory on
 524     // Windows.
 525     replica.close();
 526     rmtmpdir(tempdir);
 527     return true;
 528 }
 529
 530 // Tests for max_changesets
 531 DEFINE_TESTCASE(replicate4, replicas) {
 532     UNSET_MAX_CHANGESETS_AFTERWARDS;
 533     string tempdir = ".replicatmp";
 534     mktmpdir(tempdir);
 535     string masterpath = get_named_writable_database_path("master");
 536
 537     set_max_changesets(1);
 538
 539     Xapian::WritableDatabase orig(get_named_writable_database("master"));
 540     Xapian::DatabaseMaster master(masterpath);
 541     string replicapath = tempdir + "/replica";
 542     Xapian::DatabaseReplica replica(replicapath);
 543
 544     // Add a document with no positions to the original database.
 545     Xapian::Document doc1;
 546     doc1.set_data(string("doc1"));
 547     doc1.add_term("nopos");
 548     orig.add_document(doc1);
 549     orig.commit();
 550
 551     // Apply the replication - we don't have changesets stored, so this should
 552     // just do a database copy, and return a count of 1.
 553     int count = replicate(master, replica, tempdir, 0, 1, true);
 554     TEST_EQUAL(count, 1);
 555     {
 556         Xapian::Database dbcopy(replicapath);
 557         TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 558     }
 559
 560     // Add a document with positional information to the original database.
 561     doc1.add_posting("pos", 1);
 562     orig.add_document(doc1);
 563     orig.commit();
 564
 565     // Replicate, and check that we have the positional information.
 566     count = replicate(master, replica, tempdir, 1, 0, true);
 567     TEST_EQUAL(count, 2);
 568     {
 569         Xapian::Database dbcopy(replicapath);
 570         TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 571     }
 572     check_equal_dbs(masterpath, replicapath);
 573
 574     // Add a document with no positions to the original database.
 575     Xapian::Document doc2;
 576     doc2.set_data(string("doc2"));
 577     doc2.add_term("nopos");
 578     orig.add_document(doc2);
 579     if (get_dbtype() != "chert") {
 580         set_max_changesets(0); // FIXME: Needs to be pre-commit for new-glass
 581     }
 582     orig.commit();
 583
 584     // Replicate, and check that we have the positional information.
 585     count = replicate(master, replica, tempdir, 1, 0, true);
 586     TEST_EQUAL(count, 2);
 587     {
 588         Xapian::Database dbcopy(replicapath);
 589         TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 590     }
 591     check_equal_dbs(masterpath, replicapath);
 592     TEST(!file_exists(masterpath + "/changes1"));
 593
 594     // Turn off replication, make sure we dont write anything
 595     if (get_dbtype() == "chert") {
 596         set_max_changesets(0);
 597     }
 598
 599     // Add a document with no positions to the original database.
 600     Xapian::Document doc3;
 601     doc3.set_data(string("doc3"));
 602     doc3.add_term("nonopos");
 603     orig.add_document(doc3);
 604     orig.commit();
 605
 606     // Replicate, and check that we have the positional information.
 607     count = replicate(master, replica, tempdir, 0, 1, true);
 608     TEST_EQUAL(count, 1);
 609     {
 610         Xapian::Database dbcopy(replicapath);
 611         TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 612     }
 613     // Should have pulled a full copy
 614     check_equal_dbs(masterpath, replicapath);
 615     TEST(!file_exists(masterpath + "/changes3"));
 616
 617
 618     // Need to close the replica before we remove the temporary directory on
 619     // Windows.
 620     replica.close();
 621     rmtmpdir(tempdir);
 622     return true;
 623 }
 624
 625
 626 // Tests for max_changesets
 627 DEFINE_TESTCASE(replicate5, replicas) {
 628     SKIP_TEST_FOR_BACKEND("chert");
 629     UNSET_MAX_CHANGESETS_AFTERWARDS;
 630     string tempdir = ".replicatmp";
 631     mktmpdir(tempdir);
 632     string masterpath = get_named_writable_database_path("master");
 633
 634     set_max_changesets(2);
 635
 636     Xapian::WritableDatabase orig(get_named_writable_database("master"));
 637     Xapian::DatabaseMaster master(masterpath);
 638     string replicapath = tempdir + "/replica";
 639     Xapian::DatabaseReplica replica(replicapath);
 640
 641     // Add a document with no positions to the original database.
 642     Xapian::Document doc1;
 643     doc1.set_data(string("doc1"));
 644     doc1.add_term("nopos");
 645     orig.add_document(doc1);
 646     orig.commit();
 647
 648     // Apply the replication - we don't have changesets stored, so this should
 649     // just do a database copy, and return a count of 1.
 650     int count = replicate(master, replica, tempdir, 0, 1, true);
 651     TEST_EQUAL(count, 1);
 652     {
 653         Xapian::Database dbcopy(replicapath);
 654         TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 655     }
 656
 657     // Add a document with positional information to the original database.
 658     doc1.add_posting("pos", 1);
 659     orig.add_document(doc1);
 660     orig.commit();
 661
 662     // Replicate, and check that we have the positional information.
 663     count = replicate(master, replica, tempdir, 1, 0, true);
 664     TEST_EQUAL(count, 2);
 665     {
 666         Xapian::Database dbcopy(replicapath);
 667         TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 668     }
 669     check_equal_dbs(masterpath, replicapath);
 670
 671     // Add a document with no positions to the original database.
 672     Xapian::Document doc2;
 673     doc2.set_data(string("doc2"));
 674     doc2.add_term("nopos");
 675     orig.add_document(doc2);
 676     orig.commit();
 677
 678     // Replicate, and check that we have the positional information.
 679     count = replicate(master, replica, tempdir, 1, 0, true);
 680     TEST_EQUAL(count, 2);
 681     {
 682         Xapian::Database dbcopy(replicapath);
 683         TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 684     }
 685     check_equal_dbs(masterpath, replicapath);
 686
 687     // Add a document with no positions to the original database.
 688     Xapian::Document doc3;
 689     doc3.set_data(string("doc3"));
 690     doc3.add_term("nonopos");
 691     orig.add_document(doc3);
 692     orig.commit();
 693
 694     // Replicate, and check that we have the positional information.
 695     count = replicate(master, replica, tempdir, 1, 0, true);
 696     TEST_EQUAL(count, 2);
 697     {
 698         Xapian::Database dbcopy(replicapath);
 699         TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 700     }
 701     check_equal_dbs(masterpath, replicapath);
 702
 703     // Ensure that only these changesets exists
 704     TEST(!file_exists(masterpath + "/changes1"));
 705     TEST(file_exists(masterpath + "/changes2"));
 706     TEST(file_exists(masterpath + "/changes3"));
 707
 708     set_max_changesets(3);
 709     masterpath = get_named_writable_database_path("master");
 710
 711     // Add a document with no positions to the original database.
 712     Xapian::Document doc4;
 713     doc4.set_data(string("doc4"));
 714     doc4.add_term("nononopos");
 715     orig.add_document(doc4);
 716     orig.commit();
 717
 718     // Replicate, and check that we have the positional information.
 719     count = replicate(master, replica, tempdir, 1, 0, true);
 720     TEST_EQUAL(count, 2);
 721     {
 722         Xapian::Database dbcopy(replicapath);
 723         TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 724     }
 725     check_equal_dbs(masterpath, replicapath);
 726
 727     // Add a document with no positions to the original database.
 728     Xapian::Document doc5;
 729     doc5.set_data(string("doc5"));
 730     doc5.add_term("nonononopos");
 731     orig.add_document(doc5);
 732     orig.commit();
 733
 734     // Replicate, and check that we have the positional information.
 735     count = replicate(master, replica, tempdir, 1, 0, true);
 736     TEST_EQUAL(count, 2);
 737     {
 738         Xapian::Database dbcopy(replicapath);
 739         TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 740     }
 741     check_equal_dbs(masterpath, replicapath);
 742
 743     TEST(!file_exists(masterpath + "/changes2"));
 744     TEST(file_exists(masterpath + "/changes3"));
 745     TEST(file_exists(masterpath + "/changes4"));
 746     TEST(file_exists(masterpath + "/changes5"));
 747
 748     // Need to close the replica before we remove the temporary directory on
 749     // Windows.
 750     replica.close();
 751     rmtmpdir(tempdir);
 752     return true;
 753 }
 754
 755 /// Test --full-copy option.
 756 DEFINE_TESTCASE(replicate6, replicas) {
 757     UNSET_MAX_CHANGESETS_AFTERWARDS;
 758     string tempdir = ".replicatmp";
 759     mktmpdir(tempdir);
 760     string masterpath = get_named_writable_database_path("master");
 761
 762     set_max_changesets(10);
 763
 764     Xapian::WritableDatabase orig(get_named_writable_database("master"));
 765     Xapian::DatabaseMaster master(masterpath);
 766     string replicapath = tempdir + "/replica";
 767     Xapian::DatabaseReplica replica(replicapath);
 768
 769     // Add a document to the original database.
 770     Xapian::Document doc1;
 771     doc1.set_data(string("doc1"));
 772     doc1.add_posting("doc", 1);
 773     doc1.add_posting("one", 1);
 774     orig.add_document(doc1);
 775     orig.commit();
 776
 777     rm_rf(masterpath + "1");
 778     cp_R(masterpath, masterpath + "1");
 779
 780     orig.add_document(doc1);
 781     orig.commit();
 782
 783     // Apply the replication - we don't have changesets stored, so this should
 784     // just do a database copy, and return a count of 1.
 785     int count = replicate(master, replica, tempdir, 0, 1, true);
 786     TEST_EQUAL(count, 1);
 787     {
 788         Xapian::Database dbcopy(replicapath);
 789         TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 790     }
 791
 792     Xapian::DatabaseMaster master1(masterpath + "1");
 793
 794     // Try to replicate an older version of the master.
 795     count = replicate(master1, replica, tempdir, 0, 0, false);
 796     TEST_EQUAL(count, 1);
 797
 798     // Force a full copy.
 799     count = replicate(master1, replica, tempdir, 0, 1, true, true);
 800     TEST_EQUAL(count, 1);
 801
 802     // Test we can still replicate.
 803     orig.add_document(doc1);
 804     orig.commit();
 805
 806     count = replicate(master, replica, tempdir, 2, 0, true);
 807     TEST_EQUAL(count, 3);
 808
 809     check_equal_dbs(masterpath, replicapath);
 810
 811     // Need to close the replica before we remove the temporary directory on
 812     // Windows.
 813     replica.close();
 814     rmtmpdir(tempdir);
 815     return true;
 816 }