xapian-core/tests/api_replicate.cc

   1 /* api_replicate.cc: tests of replication functionality
   2  *
   3  * Copyright 2008 Lemur Consulting Ltd
   4  * Copyright 2009,2010,2011,2012,2013,2014,2015,2016,2017 Olly Betts
   5  * Copyright 2010 Richard Boulton
   6  * Copyright 2011 Dan Colish
   7  *
   8  * This program is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU General Public License as
  10  * published by the Free Software Foundation; either version 2 of the
  11  * License, or (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  21  * USA
  22  */
  23
  24 #include <config.h>
  25
  26 #include "api_replicate.h"
  27
  28 #include <xapian.h>
  29 #include "api/replication.h"
  30
  31 #include "apitest.h"
  32 #include "dbcheck.h"
  33 #include "fd.h"
  34 #include "filetests.h"
  35 #include "safedirent.h"
  36 #include "safefcntl.h"
  37 #include "safesysstat.h"
  38 #include "safeunistd.h"
  39 #include "testsuite.h"
  40 #include "testutils.h"
  41 #include "unixcmds.h"
  42
  43 #include <sys/types.h>
  44
  45 #include <cerrno>
  46 #include <cstdlib>
  47 #include <string>
  48
  49 #include <stdlib.h> // For setenv() or putenv() or _putenv_s()
  50
  51 using namespace std;
  52
  53 #ifdef XAPIAN_HAS_REMOTE_BACKEND
  54
  55 static void rmtmpdir(const string & path) {
  56     rm_rf(path);
  57 }
  58
  59 static void mktmpdir(const string & path) {
  60     rmtmpdir(path);
  61     if (mkdir(path.c_str(), 0700) == -1 && errno != EEXIST) {
  62         FAIL_TEST("Can't make temporary directory");
  63     }
  64 }
  65
  66 static off_t get_file_size(const string & path) {
  67     off_t size = file_size(path);
  68     if (errno) {
  69         FAIL_TEST("Can't stat '" << path << "'");
  70     }
  71     return size;
  72 }
  73
  74 static size_t do_read(int fd, char * p, size_t desired)
  75 {
  76     size_t total = 0;
  77     while (desired) {
  78         ssize_t c = read(fd, p, desired);
  79         if (c == 0) return total;
  80         if (c < 0) {
  81             if (errno == EINTR) continue;
  82             FAIL_TEST("Error reading from file");
  83         }
  84         p += c;
  85         total += c;
  86         desired -= c;
  87     }
  88     return total;
  89 }
  90
  91 static void do_write(int fd, const char * p, size_t n)
  92 {
  93     while (n) {
  94         ssize_t c = write(fd, p, n);
  95         if (c < 0) {
  96             if (errno == EINTR) continue;
  97             FAIL_TEST("Error writing to file");
  98         }
  99         p += c;
 100         n -= c;
 101     }
 102 }
 103
 104 // Make a truncated copy of a file.
 105 static off_t
 106 truncated_copy(const string & srcpath, const string & destpath, off_t tocopy)
 107 {
 108     FD fdin(open(srcpath.c_str(), O_RDONLY | O_BINARY));
 109     if (fdin == -1) {
 110         FAIL_TEST("Open failed (when opening '" << srcpath << "')");
 111     }
 112
 113     FD fdout(open(destpath.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666));
 114     if (fdout == -1) {
 115         FAIL_TEST("Open failed (when creating '" << destpath << "')");
 116     }
 117
 118     const int BUFSIZE = 1024;
 119     char buf[BUFSIZE];
 120     size_t total_bytes = 0;
 121     while (tocopy > 0) {
 122         size_t thiscopy = tocopy > BUFSIZE ? BUFSIZE : tocopy;
 123         size_t bytes = do_read(fdin, buf, thiscopy);
 124         if (thiscopy != bytes) {
 125             FAIL_TEST("Couldn't read desired number of bytes from changeset");
 126         }
 127         tocopy -= bytes;
 128         total_bytes += bytes;
 129         do_write(fdout, buf, bytes);
 130     }
 131
 132     if (close(fdout) == -1)
 133         FAIL_TEST("Error closing file");
 134
 135     return total_bytes;
 136 }
 137
 138 static void
 139 get_changeset(const string & changesetpath,
 140               Xapian::DatabaseMaster & master,
 141               Xapian::DatabaseReplica & replica,
 142               int expected_changesets,
 143               int expected_fullcopies,
 144               bool expected_changed,
 145               bool full_copy = false)
 146 {
 147     FD fd(open(changesetpath.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666));
 148     if (fd == -1) {
 149         FAIL_TEST("Open failed (when creating a new changeset file at '"
 150                   << changesetpath << "')");
 151     }
 152     Xapian::ReplicationInfo info1;
 153     master.write_changesets_to_fd(fd,
 154                                   full_copy ? "" : replica.get_revision_info(),
 155                                   &info1);
 156
 157     TEST_EQUAL(info1.changeset_count, expected_changesets);
 158     TEST_EQUAL(info1.fullcopy_count, expected_fullcopies);
 159     TEST_EQUAL(info1.changed, expected_changed);
 160 }
 161
 162 static int
 163 apply_changeset(const string & changesetpath,
 164                 Xapian::DatabaseReplica & replica,
 165                 int expected_changesets,
 166                 int expected_fullcopies,
 167                 bool expected_changed)
 168 {
 169     FD fd(open(changesetpath.c_str(), O_RDONLY | O_BINARY));
 170     if (fd == -1) {
 171         FAIL_TEST("Open failed (when reading changeset file at '"
 172                   << changesetpath << "')");
 173     }
 174
 175     int count = 1;
 176     replica.set_read_fd(fd);
 177     Xapian::ReplicationInfo info1;
 178     Xapian::ReplicationInfo info2;
 179     bool client_changed = false;
 180     while (replica.apply_next_changeset(&info2, 0)) {
 181         ++count;
 182         info1.changeset_count += info2.changeset_count;
 183         info1.fullcopy_count += info2.fullcopy_count;
 184         if (info2.changed)
 185             client_changed = true;
 186     }
 187     info1.changeset_count += info2.changeset_count;
 188     info1.fullcopy_count += info2.fullcopy_count;
 189     if (info2.changed)
 190         client_changed = true;
 191
 192     TEST_EQUAL(info1.changeset_count, expected_changesets);
 193     TEST_EQUAL(info1.fullcopy_count, expected_fullcopies);
 194     TEST_EQUAL(client_changed, expected_changed);
 195     return count;
 196 }
 197
 198 // Replicate from the master to the replica.
 199 // Returns the number of changesets which were applied.
 200 static int
 201 replicate(Xapian::DatabaseMaster & master,
 202           Xapian::DatabaseReplica & replica,
 203           const string & tempdir,
 204           int expected_changesets,
 205           int expected_fullcopies,
 206           bool expected_changed,
 207           bool full_copy = false)
 208 {
 209     string changesetpath = tempdir + "/changeset";
 210     get_changeset(changesetpath, master, replica,
 211                   expected_changesets,
 212                   expected_fullcopies,
 213                   expected_changed,
 214                   full_copy);
 215     return apply_changeset(changesetpath, replica,
 216                            expected_changesets,
 217                            expected_fullcopies,
 218                            expected_changed);
 219 }
 220
 221 // Check that the databases held at the given path are identical.
 222 static void
 223 check_equal_dbs(const string & masterpath, const string & replicapath)
 224 {
 225     Xapian::Database master(masterpath);
 226     Xapian::Database replica(replicapath);
 227
 228     TEST_EQUAL(master.get_uuid(), master.get_uuid());
 229     dbcheck(replica, master.get_doccount(), master.get_lastdocid());
 230
 231     for (Xapian::TermIterator t = master.allterms_begin();
 232          t != master.allterms_end(); ++t) {
 233         TEST_EQUAL(postlist_to_string(master, *t),
 234                    postlist_to_string(replica, *t));
 235     }
 236 }
 237
 238 #if 0 // Dynamic version which we don't currently need.
 239 static void
 240 set_max_changesets(int count) {
 241 #if HAVE_DECL__PUTENV_S
 242     _putenv_s("XAPIAN_MAX_CHANGESETS", str(count).c_str());
 243 #elif defined HAVE_SETENV
 244     setenv("XAPIAN_MAX_CHANGESETS", str(count).c_str(), 1);
 245 #else
 246     static char buf[64] = "XAPIAN_MAX_CHANGESETS=";
 247     sprintf(buf + CONST_STRLEN("XAPIAN_MAX_CHANGESETS="), "%d", count);
 248     putenv(buf);
 249 #endif
 250 }
 251 #endif
 252
 253 #if HAVE_DECL__PUTENV_S
 254 # define set_max_changesets(N) _putenv_s("XAPIAN_MAX_CHANGESETS", #N)
 255 #elif defined HAVE_SETENV
 256 # define set_max_changesets(N) setenv("XAPIAN_MAX_CHANGESETS", #N, 1)
 257 #else
 258 # define set_max_changesets(N) putenv(const_cast<char*>("XAPIAN_MAX_CHANGESETS="#N))
 259 #endif
 260
 261 struct unset_max_changesets_helper_ {
 262     unset_max_changesets_helper_() { }
 263     ~unset_max_changesets_helper_() { set_max_changesets(0); }
 264 };
 265
 266 // Ensure that we don't leave generation of changesets on for the next
 267 // testcase, even if this one exits with an exception.
 268 #define UNSET_MAX_CHANGESETS_AFTERWARDS unset_max_changesets_helper_ ezlxq
 269
 270 #endif
 271
 272 // #######################################################################
 273 // # Tests start here
 274
 275 // Basic test of replication functionality.
 276 DEFINE_TESTCASE(replicate1, replicas) {
 277 #ifdef XAPIAN_HAS_REMOTE_BACKEND
 278     UNSET_MAX_CHANGESETS_AFTERWARDS;
 279     string tempdir = ".replicatmp";
 280     mktmpdir(tempdir);
 281     string masterpath = get_named_writable_database_path("master");
 282
 283     set_max_changesets(10);
 284
 285     Xapian::Document doc1;
 286     doc1.set_data(string("doc1"));
 287     doc1.add_posting("doc", 1);
 288     doc1.add_posting("one", 1);
 289
 290     Xapian::WritableDatabase orig(get_named_writable_database("master"));
 291     Xapian::DatabaseMaster master(masterpath);
 292     string replicapath = tempdir + "/replica";
 293     {
 294         Xapian::DatabaseReplica replica(replicapath);
 295
 296         // Add a document to the original database.
 297         orig.add_document(doc1);
 298         orig.commit();
 299
 300         // Apply the replication - we don't have changesets stored, so this
 301         // should just do a database copy, and return a count of 1.
 302         int count = replicate(master, replica, tempdir, 0, 1, true);
 303         TEST_EQUAL(count, 1);
 304         {
 305             Xapian::Database dbcopy(replicapath);
 306             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 307         }
 308
 309         // Repeating the replication should return a count of 1, since no
 310         // further changes should need to be applied.
 311         count = replicate(master, replica, tempdir, 0, 0, false);
 312         TEST_EQUAL(count, 1);
 313         {
 314             Xapian::Database dbcopy(replicapath);
 315             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 316         }
 317     }
 318     {
 319         // Regression test - if the replica was reopened, a full copy always
 320         // used to occur, whether it was needed or not.  Fixed in revision
 321         // #10117.
 322         Xapian::DatabaseReplica replica(replicapath);
 323         int count = replicate(master, replica, tempdir, 0, 0, false);
 324         TEST_EQUAL(count, 1);
 325         {
 326             Xapian::Database dbcopy(replicapath);
 327             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 328         }
 329
 330         orig.add_document(doc1);
 331         orig.commit();
 332         orig.add_document(doc1);
 333         orig.commit();
 334
 335         count = replicate(master, replica, tempdir, 2, 0, true);
 336         TEST_EQUAL(count, 3);
 337         {
 338             Xapian::Database dbcopy(replicapath);
 339             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 340         }
 341
 342         check_equal_dbs(masterpath, replicapath);
 343
 344         // We need this inner scope to we close the replica before we remove
 345         // the temporary directory on Windows.
 346     }
 347
 348     rmtmpdir(tempdir);
 349 #endif
 350     return true;
 351 }
 352
 353 // Test replication from a replicated copy.
 354 DEFINE_TESTCASE(replicate2, replicas) {
 355 #ifdef XAPIAN_HAS_REMOTE_BACKEND
 356     SKIP_TEST_FOR_BACKEND("glass"); // Glass doesn't currently support this.
 357     UNSET_MAX_CHANGESETS_AFTERWARDS;
 358
 359     string tempdir = ".replicatmp";
 360     mktmpdir(tempdir);
 361     string masterpath = get_named_writable_database_path("master");
 362
 363     set_max_changesets(10);
 364
 365     {
 366         Xapian::WritableDatabase orig(get_named_writable_database("master"));
 367         Xapian::DatabaseMaster master(masterpath);
 368         string replicapath = tempdir + "/replica";
 369         Xapian::DatabaseReplica replica(replicapath);
 370
 371         Xapian::DatabaseMaster master2(replicapath);
 372         string replica2path = tempdir + "/replica2";
 373         Xapian::DatabaseReplica replica2(replica2path);
 374
 375         // Add a document to the original database.
 376         Xapian::Document doc1;
 377         doc1.set_data(string("doc1"));
 378         doc1.add_posting("doc", 1);
 379         doc1.add_posting("one", 1);
 380         orig.add_document(doc1);
 381         orig.commit();
 382
 383         // Apply the replication - we don't have changesets stored, so this
 384         // should just do a database copy, and return a count of 1.
 385         TEST_EQUAL(replicate(master, replica, tempdir, 0, 1, true), 1);
 386         check_equal_dbs(masterpath, replicapath);
 387
 388         // Replicate from the replica.
 389         TEST_EQUAL(replicate(master2, replica2, tempdir, 0, 1, true), 1);
 390         check_equal_dbs(masterpath, replica2path);
 391
 392         orig.add_document(doc1);
 393         orig.commit();
 394         orig.add_document(doc1);
 395         orig.commit();
 396
 397         // Replicate from the replica - should have no changes.
 398         TEST_EQUAL(replicate(master2, replica2, tempdir, 0, 0, false), 1);
 399         check_equal_dbs(replicapath, replica2path);
 400
 401         // Replicate, and replicate from the replica - should have 2 changes.
 402         TEST_EQUAL(replicate(master, replica, tempdir, 2, 0, 1), 3);
 403         check_equal_dbs(masterpath, replicapath);
 404         TEST_EQUAL(replicate(master2, replica2, tempdir, 2, 0, 1), 3);
 405         check_equal_dbs(masterpath, replica2path);
 406
 407         // Stop writing changesets, and make a modification
 408         set_max_changesets(0);
 409         orig.close();
 410         orig = get_writable_database_again();
 411         orig.add_document(doc1);
 412         orig.commit();
 413
 414         // Replication should do a full copy.
 415         TEST_EQUAL(replicate(master, replica, tempdir, 0, 1, true), 1);
 416         check_equal_dbs(masterpath, replicapath);
 417         TEST_EQUAL(replicate(master2, replica2, tempdir, 0, 1, true), 1);
 418         check_equal_dbs(masterpath, replica2path);
 419
 420         // Start writing changesets, but only keep 1 in history, and make a
 421         // modification.
 422         set_max_changesets(1);
 423         orig.close();
 424         orig = get_writable_database_again();
 425         orig.add_document(doc1);
 426         orig.commit();
 427
 428         // Replicate, and replicate from the replica - should have 1 changes.
 429         TEST_EQUAL(replicate(master, replica, tempdir, 1, 0, 1), 2);
 430         check_equal_dbs(masterpath, replicapath);
 431         TEST_EQUAL(replicate(master2, replica2, tempdir, 1, 0, 1), 2);
 432         check_equal_dbs(masterpath, replica2path);
 433
 434         // Make two changes - only one changeset should be preserved.
 435         orig.add_document(doc1);
 436         orig.commit();
 437
 438         // Replication should do a full copy, since one of the needed
 439         // changesets is missing.
 440
 441         // FIXME - the following tests are commented out because the backends
 442         // don't currently tidy up old changesets correctly.
 443         // TEST_EQUAL(replicate(master, replica, tempdir, 0, 1, true), 1);
 444         // check_equal_dbs(masterpath, replicapath);
 445         // TEST_EQUAL(replicate(master2, replica2, tempdir, 0, 1, true), 1);
 446         // check_equal_dbs(masterpath, replica2path);
 447
 448         // We need this inner scope to we close the replicas before we remove
 449         // the temporary directory on Windows.
 450     }
 451
 452     rmtmpdir(tempdir);
 453 #endif
 454     return true;
 455 }
 456
 457 #ifdef XAPIAN_HAS_REMOTE_BACKEND
 458 static void
 459 replicate_with_brokenness(Xapian::DatabaseMaster & master,
 460                           Xapian::DatabaseReplica & replica,
 461                           const string & tempdir,
 462                           int expected_changesets,
 463                           int expected_fullcopies,
 464                           bool expected_changed)
 465 {
 466     string changesetpath = tempdir + "/changeset";
 467     get_changeset(changesetpath, master, replica,
 468                   1, 0, 1);
 469
 470     // Try applying truncated changesets of various different lengths.
 471     string brokenchangesetpath = tempdir + "/changeset_broken";
 472     off_t filesize = get_file_size(changesetpath);
 473     off_t len = 10;
 474     off_t copylen;
 475     while (len < filesize) {
 476         copylen = truncated_copy(changesetpath, brokenchangesetpath, len);
 477         TEST_EQUAL(copylen, len);
 478         tout << "Trying replication with a changeset truncated to " << len <<
 479                 " bytes, from " << filesize << " bytes\n";
 480         TEST_EXCEPTION(Xapian::NetworkError,
 481                        apply_changeset(brokenchangesetpath, replica,
 482                                        expected_changesets, expected_fullcopies,
 483                                        expected_changed));
 484         if (len < 30 || len >= filesize - 10) {
 485             // For lengths near the beginning and end, increment size by 1
 486             ++len;
 487         } else {
 488             // Don't bother incrementing by small amounts in the middle of
 489             // the changeset.
 490             len += 1000;
 491             if (len >= filesize - 10) {
 492                 len = filesize - 10;
 493             }
 494         }
 495     }
 496 }
 497 #endif
 498
 499 // Test changesets which are truncated (and therefore invalid).
 500 DEFINE_TESTCASE(replicate3, replicas) {
 501 #ifdef XAPIAN_HAS_REMOTE_BACKEND
 502     UNSET_MAX_CHANGESETS_AFTERWARDS;
 503     string tempdir = ".replicatmp";
 504     mktmpdir(tempdir);
 505     string masterpath = get_named_writable_database_path("master");
 506
 507     set_max_changesets(10);
 508
 509     {
 510         Xapian::WritableDatabase orig(get_named_writable_database("master"));
 511         Xapian::DatabaseMaster master(masterpath);
 512         string replicapath = tempdir + "/replica";
 513         Xapian::DatabaseReplica replica(replicapath);
 514
 515         // Add a document to the original database.
 516         Xapian::Document doc1;
 517         doc1.set_data(string("doc1"));
 518         doc1.add_posting("doc", 1);
 519         doc1.add_posting("one", 1);
 520         orig.add_document(doc1);
 521         orig.commit();
 522
 523         TEST_EQUAL(replicate(master, replica, tempdir, 0, 1, true), 1);
 524         check_equal_dbs(masterpath, replicapath);
 525
 526         // Make a changeset.
 527         orig.add_document(doc1);
 528         orig.commit();
 529
 530         replicate_with_brokenness(master, replica, tempdir, 1, 0, true);
 531         // Although it throws an error, the final replication in
 532         // replicate_with_brokenness() updates the database, since it's just
 533         // the end-of-replication message which is missing its body.
 534         check_equal_dbs(masterpath, replicapath);
 535
 536         // Check that the earlier broken replications didn't cause any problems
 537         // for the next replication.
 538         orig.add_document(doc1);
 539         orig.commit();
 540         TEST_EQUAL(replicate(master, replica, tempdir, 1, 0, true), 2);
 541
 542         // We need this inner scope to we close the replica before we remove
 543         // the temporary directory on Windows.
 544     }
 545
 546     rmtmpdir(tempdir);
 547 #endif
 548     return true;
 549 }
 550
 551 // Tests for max_changesets
 552 DEFINE_TESTCASE(replicate4, replicas) {
 553 #ifdef XAPIAN_HAS_REMOTE_BACKEND
 554     UNSET_MAX_CHANGESETS_AFTERWARDS;
 555     string tempdir = ".replicatmp";
 556     mktmpdir(tempdir);
 557     string masterpath = get_named_writable_database_path("master");
 558
 559     set_max_changesets(1);
 560
 561     {
 562         Xapian::WritableDatabase orig(get_named_writable_database("master"));
 563         Xapian::DatabaseMaster master(masterpath);
 564         string replicapath = tempdir + "/replica";
 565         Xapian::DatabaseReplica replica(replicapath);
 566
 567         // Add a document with no positions to the original database.
 568         Xapian::Document doc1;
 569         doc1.set_data(string("doc1"));
 570         doc1.add_term("nopos");
 571         orig.add_document(doc1);
 572         orig.commit();
 573
 574         // Apply the replication - we don't have changesets stored, so this
 575         // should just do a database copy, and return a count of 1.
 576         int count = replicate(master, replica, tempdir, 0, 1, true);
 577         TEST_EQUAL(count, 1);
 578         {
 579             Xapian::Database dbcopy(replicapath);
 580             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 581         }
 582
 583         // Add a document with positional information to the original database.
 584         doc1.add_posting("pos", 1);
 585         orig.add_document(doc1);
 586         orig.commit();
 587
 588         // Replicate, and check that we have the positional information.
 589         count = replicate(master, replica, tempdir, 1, 0, true);
 590         TEST_EQUAL(count, 2);
 591         {
 592             Xapian::Database dbcopy(replicapath);
 593             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 594         }
 595         check_equal_dbs(masterpath, replicapath);
 596
 597         // Add a document with no positions to the original database.
 598         Xapian::Document doc2;
 599         doc2.set_data(string("doc2"));
 600         doc2.add_term("nopos");
 601         orig.add_document(doc2);
 602         if (get_dbtype() == "glass") {
 603             // FIXME: Needs to be pre-commit for glass
 604             set_max_changesets(0);
 605         }
 606         orig.commit();
 607
 608         // Replicate, and check that we have the positional information.
 609         count = replicate(master, replica, tempdir, 1, 0, true);
 610         TEST_EQUAL(count, 2);
 611         {
 612             Xapian::Database dbcopy(replicapath);
 613             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 614         }
 615         check_equal_dbs(masterpath, replicapath);
 616         TEST(!file_exists(masterpath + "/changes1"));
 617
 618         // Turn off replication, make sure we dont write anything
 619         if (get_dbtype() != "glass") {
 620             set_max_changesets(0);
 621         }
 622
 623         // Add a document with no positions to the original database.
 624         Xapian::Document doc3;
 625         doc3.set_data(string("doc3"));
 626         doc3.add_term("nonopos");
 627         orig.add_document(doc3);
 628         orig.commit();
 629
 630         // Replicate, and check that we have the positional information.
 631         count = replicate(master, replica, tempdir, 0, 1, true);
 632         TEST_EQUAL(count, 1);
 633         {
 634             Xapian::Database dbcopy(replicapath);
 635             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 636         }
 637         // Should have pulled a full copy
 638         check_equal_dbs(masterpath, replicapath);
 639         TEST(!file_exists(masterpath + "/changes3"));
 640
 641         // We need this inner scope to we close the replica before we remove
 642         // the temporary directory on Windows.
 643     }
 644
 645     rmtmpdir(tempdir);
 646 #endif
 647     return true;
 648 }
 649
 650 // Tests for max_changesets
 651 DEFINE_TESTCASE(replicate5, replicas) {
 652 #ifdef XAPIAN_HAS_REMOTE_BACKEND
 653     UNSET_MAX_CHANGESETS_AFTERWARDS;
 654     string tempdir = ".replicatmp";
 655     mktmpdir(tempdir);
 656     string masterpath = get_named_writable_database_path("master");
 657
 658     set_max_changesets(2);
 659
 660     {
 661         Xapian::WritableDatabase orig(get_named_writable_database("master"));
 662         Xapian::DatabaseMaster master(masterpath);
 663         string replicapath = tempdir + "/replica";
 664         Xapian::DatabaseReplica replica(replicapath);
 665
 666         // Add a document with no positions to the original database.
 667         Xapian::Document doc1;
 668         doc1.set_data(string("doc1"));
 669         doc1.add_term("nopos");
 670         orig.add_document(doc1);
 671         orig.commit();
 672
 673         // Apply the replication - we don't have changesets stored, so this
 674         // should just do a database copy, and return a count of 1.
 675         int count = replicate(master, replica, tempdir, 0, 1, true);
 676         TEST_EQUAL(count, 1);
 677         {
 678             Xapian::Database dbcopy(replicapath);
 679             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 680         }
 681
 682         // Add a document with positional information to the original database.
 683         doc1.add_posting("pos", 1);
 684         orig.add_document(doc1);
 685         orig.commit();
 686
 687         // Replicate, and check that we have the positional information.
 688         count = replicate(master, replica, tempdir, 1, 0, true);
 689         TEST_EQUAL(count, 2);
 690         {
 691             Xapian::Database dbcopy(replicapath);
 692             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 693         }
 694         check_equal_dbs(masterpath, replicapath);
 695
 696         // Add a document with no positions to the original database.
 697         Xapian::Document doc2;
 698         doc2.set_data(string("doc2"));
 699         doc2.add_term("nopos");
 700         orig.add_document(doc2);
 701         orig.commit();
 702
 703         // Replicate, and check that we have the positional information.
 704         count = replicate(master, replica, tempdir, 1, 0, true);
 705         TEST_EQUAL(count, 2);
 706         {
 707             Xapian::Database dbcopy(replicapath);
 708             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 709         }
 710         check_equal_dbs(masterpath, replicapath);
 711
 712         // Add a document with no positions to the original database.
 713         Xapian::Document doc3;
 714         doc3.set_data(string("doc3"));
 715         doc3.add_term("nonopos");
 716         orig.add_document(doc3);
 717         orig.commit();
 718
 719         // Replicate, and check that we have the positional information.
 720         count = replicate(master, replica, tempdir, 1, 0, true);
 721         TEST_EQUAL(count, 2);
 722         {
 723             Xapian::Database dbcopy(replicapath);
 724             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 725         }
 726         check_equal_dbs(masterpath, replicapath);
 727
 728         // Ensure that only these changesets exists
 729         TEST(!file_exists(masterpath + "/changes1"));
 730         TEST(file_exists(masterpath + "/changes2"));
 731         TEST(file_exists(masterpath + "/changes3"));
 732
 733         set_max_changesets(3);
 734         masterpath = get_named_writable_database_path("master");
 735
 736         // Add a document with no positions to the original database.
 737         Xapian::Document doc4;
 738         doc4.set_data(string("doc4"));
 739         doc4.add_term("nononopos");
 740         orig.add_document(doc4);
 741         orig.commit();
 742
 743         // Replicate, and check that we have the positional information.
 744         count = replicate(master, replica, tempdir, 1, 0, true);
 745         TEST_EQUAL(count, 2);
 746         {
 747             Xapian::Database dbcopy(replicapath);
 748             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 749         }
 750         check_equal_dbs(masterpath, replicapath);
 751
 752         // Add a document with no positions to the original database.
 753         Xapian::Document doc5;
 754         doc5.set_data(string("doc5"));
 755         doc5.add_term("nonononopos");
 756         orig.add_document(doc5);
 757         orig.commit();
 758
 759         // Replicate, and check that we have the positional information.
 760         count = replicate(master, replica, tempdir, 1, 0, true);
 761         TEST_EQUAL(count, 2);
 762         {
 763             Xapian::Database dbcopy(replicapath);
 764             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 765         }
 766         check_equal_dbs(masterpath, replicapath);
 767
 768         TEST(!file_exists(masterpath + "/changes2"));
 769         TEST(file_exists(masterpath + "/changes3"));
 770         TEST(file_exists(masterpath + "/changes4"));
 771         TEST(file_exists(masterpath + "/changes5"));
 772
 773         // We need this inner scope to we close the replica before we remove
 774         // the temporary directory on Windows.
 775     }
 776
 777     rmtmpdir(tempdir);
 778 #endif
 779     return true;
 780 }
 781
 782 /// Test --full-copy option.
 783 DEFINE_TESTCASE(replicate6, replicas) {
 784 #ifdef XAPIAN_HAS_REMOTE_BACKEND
 785     UNSET_MAX_CHANGESETS_AFTERWARDS;
 786     string tempdir = ".replicatmp";
 787     mktmpdir(tempdir);
 788     string masterpath = get_named_writable_database_path("master");
 789
 790     set_max_changesets(10);
 791
 792     {
 793         Xapian::WritableDatabase orig(get_named_writable_database("master"));
 794         Xapian::DatabaseMaster master(masterpath);
 795         string replicapath = tempdir + "/replica";
 796         Xapian::DatabaseReplica replica(replicapath);
 797
 798         // Add a document to the original database.
 799         Xapian::Document doc1;
 800         doc1.set_data(string("doc1"));
 801         doc1.add_posting("doc", 1);
 802         doc1.add_posting("one", 1);
 803         orig.add_document(doc1);
 804         orig.commit();
 805
 806         rm_rf(masterpath + "1");
 807         cp_R(masterpath, masterpath + "1");
 808
 809         orig.add_document(doc1);
 810         orig.commit();
 811
 812         // Apply the replication - we don't have changesets stored, so this
 813         // should just do a database copy, and return a count of 1.
 814         int count = replicate(master, replica, tempdir, 0, 1, true);
 815         TEST_EQUAL(count, 1);
 816         {
 817             Xapian::Database dbcopy(replicapath);
 818             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 819         }
 820
 821         Xapian::DatabaseMaster master1(masterpath + "1");
 822
 823         // Try to replicate an older version of the master.
 824         count = replicate(master1, replica, tempdir, 0, 0, false);
 825         TEST_EQUAL(count, 1);
 826
 827         // Force a full copy.
 828         count = replicate(master1, replica, tempdir, 0, 1, true, true);
 829         TEST_EQUAL(count, 1);
 830
 831         // Test we can still replicate.
 832         orig.add_document(doc1);
 833         orig.commit();
 834
 835         count = replicate(master, replica, tempdir, 2, 0, true);
 836         TEST_EQUAL(count, 3);
 837
 838         check_equal_dbs(masterpath, replicapath);
 839
 840         // We need this inner scope to we close the replica before we remove
 841         // the temporary directory on Windows.
 842     }
 843
 844     rmtmpdir(tempdir);
 845 #endif
 846     return true;
 847 }
 848
 849 /// Test healing a corrupt replica (new in 1.3.5).
 850 DEFINE_TESTCASE(replicate7, replicas) {
 851 #ifdef XAPIAN_HAS_REMOTE_BACKEND
 852     UNSET_MAX_CHANGESETS_AFTERWARDS;
 853     string tempdir = ".replicatmp";
 854     mktmpdir(tempdir);
 855     string masterpath = get_named_writable_database_path("master");
 856
 857     set_max_changesets(10);
 858
 859     Xapian::WritableDatabase orig(get_named_writable_database("master"));
 860     Xapian::DatabaseMaster master(masterpath);
 861     string replicapath = tempdir + "/replica";
 862     {
 863         Xapian::DatabaseReplica replica(replicapath);
 864
 865         // Add a document to the original database.
 866         Xapian::Document doc1;
 867         doc1.set_data(string("doc1"));
 868         doc1.add_posting("doc", 1);
 869         doc1.add_posting("one", 1);
 870         orig.add_document(doc1);
 871         orig.commit();
 872
 873         orig.add_document(doc1);
 874         orig.commit();
 875
 876         // Apply the replication - we don't have changesets stored, so this
 877         // should just do a database copy, and return a count of 1.
 878         int count = replicate(master, replica, tempdir, 0, 1, true);
 879         TEST_EQUAL(count, 1);
 880         {
 881             Xapian::Database dbcopy(replicapath);
 882             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 883         }
 884     }
 885
 886     {
 887         // Corrupt replica by truncating all the files to size 0.
 888         string d = replicapath;
 889         d += "/replica_1";
 890         DIR * dir = opendir(d.c_str());
 891         TEST(dir != NULL);
 892         while (true) {
 893             errno = 0;
 894             struct dirent * entry = readdir(dir);
 895             if (!entry) {
 896                 if (errno == 0)
 897                     break;
 898                 FAIL_TEST("readdir failed: " << strerror(errno));
 899             }
 900
 901             // Skip '.' and '..'.
 902             if (entry->d_name[0] == '.') continue;
 903
 904             string file = d;
 905             file += '/';
 906             file += entry->d_name;
 907             int fd = open(file.c_str(), O_WRONLY|O_TRUNC, 0666);
 908             TEST(fd != -1);
 909             TEST(close(fd) == 0);
 910         }
 911         closedir(dir);
 912     }
 913
 914     {
 915         Xapian::DatabaseReplica replica(replicapath);
 916
 917         // Replication should succeed and perform a full copy.
 918         int count = replicate(master, replica, tempdir, 0, 1, true);
 919         TEST_EQUAL(count, 1);
 920
 921         check_equal_dbs(masterpath, replicapath);
 922
 923         // We need this inner scope to we close the replica before we remove
 924         // the temporary directory on Windows.
 925     }
 926
 927     rmtmpdir(tempdir);
 928 #endif
 929     return true;
 930 }