xapian-core/tests/api_replicate.cc

   1 /* api_replicate.cc: tests of replication functionality
   2  *
   3  * Copyright 2008 Lemur Consulting Ltd
   4  * Copyright 2009,2010,2011,2012,2013,2014,2015,2016,2017 Olly Betts
   5  * Copyright 2010 Richard Boulton
   6  * Copyright 2011 Dan Colish
   7  *
   8  * This program is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU General Public License as
  10  * published by the Free Software Foundation; either version 2 of the
  11  * License, or (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  21  * USA
  22  */
  23
  24 #include <config.h>
  25
  26 #include "api_replicate.h"
  27
  28 #include <xapian.h>
  29 #include "api/replication.h"
  30
  31 #include "apitest.h"
  32 #include "dbcheck.h"
  33 #include "fd.h"
  34 #include "filetests.h"
  35 #include "safedirent.h"
  36 #include "safeerrno.h"
  37 #include "safefcntl.h"
  38 #include "safesysstat.h"
  39 #include "safeunistd.h"
  40 #include "testsuite.h"
  41 #include "testutils.h"
  42 #include "unixcmds.h"
  43
  44 #include <sys/types.h>
  45
  46 #include <cstdlib>
  47 #include <string>
  48
  49 #include <stdlib.h> // For setenv() or putenv() or _putenv_s()
  50
  51 using namespace std;
  52
  53 #ifdef XAPIAN_HAS_REMOTE_BACKEND
  54
  55 static void rmtmpdir(const string & path) {
  56     rm_rf(path);
  57 }
  58
  59 static void mktmpdir(const string & path) {
  60     rmtmpdir(path);
  61     if (mkdir(path.c_str(), 0700) == -1 && errno != EEXIST) {
  62         FAIL_TEST("Can't make temporary directory");
  63     }
  64 }
  65
  66 static off_t get_file_size(const string & path) {
  67     off_t size = file_size(path);
  68     if (errno) {
  69         FAIL_TEST("Can't stat '" << path << "'");
  70     }
  71     return size;
  72 }
  73
  74 static size_t do_read(int fd, char * p, size_t desired)
  75 {
  76     size_t total = 0;
  77     while (desired) {
  78         ssize_t c = read(fd, p, desired);
  79         if (c == 0) return total;
  80         if (c < 0) {
  81             if (errno == EINTR) continue;
  82             FAIL_TEST("Error reading from file");
  83         }
  84         p += c;
  85         total += c;
  86         desired -= c;
  87     }
  88     return total;
  89 }
  90
  91 static void do_write(int fd, const char * p, size_t n)
  92 {
  93     while (n) {
  94         ssize_t c = write(fd, p, n);
  95         if (c < 0) {
  96             if (errno == EINTR) continue;
  97             FAIL_TEST("Error writing to file");
  98         }
  99         p += c;
 100         n -= c;
 101     }
 102 }
 103
 104 // Make a truncated copy of a file.
 105 static off_t
 106 truncated_copy(const string & srcpath, const string & destpath, off_t tocopy)
 107 {
 108     FD fdin(open(srcpath.c_str(), O_RDONLY | O_BINARY));
 109     if (fdin == -1) {
 110         FAIL_TEST("Open failed (when opening '" << srcpath << "')");
 111     }
 112
 113     FD fdout(open(destpath.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666));
 114     if (fdout == -1) {
 115         FAIL_TEST("Open failed (when creating '" << destpath << "')");
 116     }
 117
 118     const int BUFSIZE = 1024;
 119     char buf[BUFSIZE];
 120     size_t total_bytes = 0;
 121     while (tocopy > 0) {
 122         size_t thiscopy = tocopy > BUFSIZE ? BUFSIZE : tocopy;
 123         size_t bytes = do_read(fdin, buf, thiscopy);
 124         if (thiscopy != bytes) {
 125             FAIL_TEST("Couldn't read desired number of bytes from changeset");
 126         }
 127         tocopy -= bytes;
 128         total_bytes += bytes;
 129         do_write(fdout, buf, bytes);
 130     }
 131
 132     if (close(fdout) == -1)
 133         FAIL_TEST("Error closing file");
 134
 135     return total_bytes;
 136 }
 137
 138 static void
 139 get_changeset(const string & changesetpath,
 140               Xapian::DatabaseMaster & master,
 141               Xapian::DatabaseReplica & replica,
 142               int expected_changesets,
 143               int expected_fullcopies,
 144               bool expected_changed,
 145               bool full_copy = false)
 146 {
 147     FD fd(open(changesetpath.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666));
 148     if (fd == -1) {
 149         FAIL_TEST("Open failed (when creating a new changeset file at '"
 150                   << changesetpath << "')");
 151     }
 152     Xapian::ReplicationInfo info1;
 153     master.write_changesets_to_fd(fd,
 154                                   full_copy ? "" : replica.get_revision_info(),
 155                                   &info1);
 156
 157     TEST_EQUAL(info1.changeset_count, expected_changesets);
 158     TEST_EQUAL(info1.fullcopy_count, expected_fullcopies);
 159     TEST_EQUAL(info1.changed, expected_changed);
 160 }
 161
 162 static int
 163 apply_changeset(const string & changesetpath,
 164                 Xapian::DatabaseReplica & replica,
 165                 int expected_changesets,
 166                 int expected_fullcopies,
 167                 bool expected_changed)
 168 {
 169     FD fd(open(changesetpath.c_str(), O_RDONLY | O_BINARY));
 170     if (fd == -1) {
 171         FAIL_TEST("Open failed (when reading changeset file at '"
 172                   << changesetpath << "')");
 173     }
 174
 175     int count = 1;
 176     replica.set_read_fd(fd);
 177     Xapian::ReplicationInfo info1;
 178     Xapian::ReplicationInfo info2;
 179     bool client_changed = false;
 180     while (replica.apply_next_changeset(&info2, 0)) {
 181         ++count;
 182         info1.changeset_count += info2.changeset_count;
 183         info1.fullcopy_count += info2.fullcopy_count;
 184         if (info2.changed)
 185             client_changed = true;
 186     }
 187     info1.changeset_count += info2.changeset_count;
 188     info1.fullcopy_count += info2.fullcopy_count;
 189     if (info2.changed)
 190         client_changed = true;
 191
 192     TEST_EQUAL(info1.changeset_count, expected_changesets);
 193     TEST_EQUAL(info1.fullcopy_count, expected_fullcopies);
 194     TEST_EQUAL(client_changed, expected_changed);
 195     return count;
 196 }
 197
 198 // Replicate from the master to the replica.
 199 // Returns the number of changesets which were applied.
 200 static int
 201 replicate(Xapian::DatabaseMaster & master,
 202           Xapian::DatabaseReplica & replica,
 203           const string & tempdir,
 204           int expected_changesets,
 205           int expected_fullcopies,
 206           bool expected_changed,
 207           bool full_copy = false)
 208 {
 209     string changesetpath = tempdir + "/changeset";
 210     get_changeset(changesetpath, master, replica,
 211                   expected_changesets,
 212                   expected_fullcopies,
 213                   expected_changed,
 214                   full_copy);
 215     return apply_changeset(changesetpath, replica,
 216                            expected_changesets,
 217                            expected_fullcopies,
 218                            expected_changed);
 219 }
 220
 221 // Check that the databases held at the given path are identical.
 222 static void
 223 check_equal_dbs(const string & masterpath, const string & replicapath)
 224 {
 225     Xapian::Database master(masterpath);
 226     Xapian::Database replica(replicapath);
 227
 228     TEST_EQUAL(master.get_uuid(), master.get_uuid());
 229     dbcheck(replica, master.get_doccount(), master.get_lastdocid());
 230
 231     for (Xapian::TermIterator t = master.allterms_begin();
 232          t != master.allterms_end(); ++t) {
 233         TEST_EQUAL(postlist_to_string(master, *t),
 234                    postlist_to_string(replica, *t));
 235     }
 236 }
 237
 238 #if 0 // Dynamic version which we don't currently need.
 239 static void
 240 set_max_changesets(int count) {
 241 #if HAVE_DECL__PUTENV_S
 242     _putenv_s("XAPIAN_MAX_CHANGESETS", str(count).c_str());
 243 #elif defined HAVE_SETENV
 244     setenv("XAPIAN_MAX_CHANGESETS", str(count).c_str(), 1);
 245 #else
 246     static char buf[64] = "XAPIAN_MAX_CHANGESETS=";
 247     sprintf(buf + CONST_STRLEN("XAPIAN_MAX_CHANGESETS="), "%d", count);
 248     putenv(buf);
 249 #endif
 250 }
 251 #endif
 252
 253 #if HAVE_DECL__PUTENV_S
 254 # define set_max_changesets(N) _putenv_s("XAPIAN_MAX_CHANGESETS", #N)
 255 #elif defined HAVE_SETENV
 256 # define set_max_changesets(N) setenv("XAPIAN_MAX_CHANGESETS", #N, 1)
 257 #else
 258 # define set_max_changesets(N) putenv(const_cast<char*>("XAPIAN_MAX_CHANGESETS="#N))
 259 #endif
 260
 261 struct unset_max_changesets_helper_ {
 262     unset_max_changesets_helper_() { }
 263     ~unset_max_changesets_helper_() { set_max_changesets(0); }
 264 };
 265
 266 // Ensure that we don't leave generation of changesets on for the next
 267 // testcase, even if this one exits with an exception.
 268 #define UNSET_MAX_CHANGESETS_AFTERWARDS unset_max_changesets_helper_ ezlxq
 269
 270 #endif
 271
 272 // #######################################################################
 273 // # Tests start here
 274
 275 // Basic test of replication functionality.
 276 DEFINE_TESTCASE(replicate1, replicas) {
 277 #ifdef XAPIAN_HAS_REMOTE_BACKEND
 278     UNSET_MAX_CHANGESETS_AFTERWARDS;
 279     string tempdir = ".replicatmp";
 280     mktmpdir(tempdir);
 281     string masterpath = get_named_writable_database_path("master");
 282
 283     set_max_changesets(10);
 284
 285     Xapian::Document doc1;
 286     doc1.set_data(string("doc1"));
 287     doc1.add_posting("doc", 1);
 288     doc1.add_posting("one", 1);
 289
 290     Xapian::WritableDatabase orig(get_named_writable_database("master"));
 291     Xapian::DatabaseMaster master(masterpath);
 292     string replicapath = tempdir + "/replica";
 293     {
 294         Xapian::DatabaseReplica replica(replicapath);
 295
 296         // Add a document to the original database.
 297         orig.add_document(doc1);
 298         orig.commit();
 299
 300         // Apply the replication - we don't have changesets stored, so this
 301         // should just do a database copy, and return a count of 1.
 302         int count = replicate(master, replica, tempdir, 0, 1, true);
 303         TEST_EQUAL(count, 1);
 304         {
 305             Xapian::Database dbcopy(replicapath);
 306             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 307         }
 308
 309         // Repeating the replication should return a count of 1, since no
 310         // further changes should need to be applied.
 311         count = replicate(master, replica, tempdir, 0, 0, false);
 312         TEST_EQUAL(count, 1);
 313         {
 314             Xapian::Database dbcopy(replicapath);
 315             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 316         }
 317     }
 318     {
 319         // Regression test - if the replica was reopened, a full copy always
 320         // used to occur, whether it was needed or not.  Fixed in revision
 321         // #10117.
 322         Xapian::DatabaseReplica replica(replicapath);
 323         int count = replicate(master, replica, tempdir, 0, 0, false);
 324         TEST_EQUAL(count, 1);
 325         {
 326             Xapian::Database dbcopy(replicapath);
 327             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 328         }
 329
 330         orig.add_document(doc1);
 331         orig.commit();
 332         orig.add_document(doc1);
 333         orig.commit();
 334
 335         count = replicate(master, replica, tempdir, 2, 0, true);
 336         TEST_EQUAL(count, 3);
 337         {
 338             Xapian::Database dbcopy(replicapath);
 339             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 340         }
 341
 342         check_equal_dbs(masterpath, replicapath);
 343
 344         // We need this inner scope to we close the replica before we remove
 345         // the temporary directory on Windows.
 346     }
 347
 348     rmtmpdir(tempdir);
 349 #endif
 350     return true;
 351 }
 352
 353 // Test replication from a replicated copy.
 354 DEFINE_TESTCASE(replicate2, replicas) {
 355 #ifdef XAPIAN_HAS_REMOTE_BACKEND
 356     SKIP_TEST_FOR_BACKEND("glass"); // Glass doesn't currently support this.
 357     UNSET_MAX_CHANGESETS_AFTERWARDS;
 358
 359     string tempdir = ".replicatmp";
 360     mktmpdir(tempdir);
 361     string masterpath = get_named_writable_database_path("master");
 362
 363     set_max_changesets(10);
 364
 365     {
 366         Xapian::WritableDatabase orig(get_named_writable_database("master"));
 367         Xapian::DatabaseMaster master(masterpath);
 368         string replicapath = tempdir + "/replica";
 369         Xapian::DatabaseReplica replica(replicapath);
 370
 371         Xapian::DatabaseMaster master2(replicapath);
 372         string replica2path = tempdir + "/replica2";
 373         Xapian::DatabaseReplica replica2(replica2path);
 374
 375         // Add a document to the original database.
 376         Xapian::Document doc1;
 377         doc1.set_data(string("doc1"));
 378         doc1.add_posting("doc", 1);
 379         doc1.add_posting("one", 1);
 380         orig.add_document(doc1);
 381         orig.commit();
 382
 383         // Apply the replication - we don't have changesets stored, so this
 384         // should just do a database copy, and return a count of 1.
 385         TEST_EQUAL(replicate(master, replica, tempdir, 0, 1, true), 1);
 386         check_equal_dbs(masterpath, replicapath);
 387
 388         // Replicate from the replica.
 389         TEST_EQUAL(replicate(master2, replica2, tempdir, 0, 1, true), 1);
 390         check_equal_dbs(masterpath, replica2path);
 391
 392         orig.add_document(doc1);
 393         orig.commit();
 394         orig.add_document(doc1);
 395         orig.commit();
 396
 397         // Replicate from the replica - should have no changes.
 398         TEST_EQUAL(replicate(master2, replica2, tempdir, 0, 0, false), 1);
 399         check_equal_dbs(replicapath, replica2path);
 400
 401         // Replicate, and replicate from the replica - should have 2 changes.
 402         TEST_EQUAL(replicate(master, replica, tempdir, 2, 0, 1), 3);
 403         check_equal_dbs(masterpath, replicapath);
 404         TEST_EQUAL(replicate(master2, replica2, tempdir, 2, 0, 1), 3);
 405         check_equal_dbs(masterpath, replica2path);
 406
 407         // Stop writing changesets, and make a modification
 408         set_max_changesets(0);
 409         orig.close();
 410         orig = get_writable_database_again();
 411         orig.add_document(doc1);
 412         orig.commit();
 413
 414         // Replication should do a full copy.
 415         TEST_EQUAL(replicate(master, replica, tempdir, 0, 1, true), 1);
 416         check_equal_dbs(masterpath, replicapath);
 417         TEST_EQUAL(replicate(master2, replica2, tempdir, 0, 1, true), 1);
 418         check_equal_dbs(masterpath, replica2path);
 419
 420         // Start writing changesets, but only keep 1 in history, and make a
 421         // modification.
 422         set_max_changesets(1);
 423         orig.close();
 424         orig = get_writable_database_again();
 425         orig.add_document(doc1);
 426         orig.commit();
 427
 428         // Replicate, and replicate from the replica - should have 1 changes.
 429         TEST_EQUAL(replicate(master, replica, tempdir, 1, 0, 1), 2);
 430         check_equal_dbs(masterpath, replicapath);
 431         TEST_EQUAL(replicate(master2, replica2, tempdir, 1, 0, 1), 2);
 432         check_equal_dbs(masterpath, replica2path);
 433
 434         // Make two changes - only one changeset should be preserved.
 435         orig.add_document(doc1);
 436         orig.commit();
 437
 438         // Replication should do a full copy, since one of the needed
 439         // changesets is missing.
 440
 441         // FIXME - the following tests are commented out because the backends
 442         // don't currently tidy up old changesets correctly.
 443         // TEST_EQUAL(replicate(master, replica, tempdir, 0, 1, true), 1);
 444         // check_equal_dbs(masterpath, replicapath);
 445         // TEST_EQUAL(replicate(master2, replica2, tempdir, 0, 1, true), 1);
 446         // check_equal_dbs(masterpath, replica2path);
 447
 448         // We need this inner scope to we close the replicas before we remove
 449         // the temporary directory on Windows.
 450     }
 451
 452     rmtmpdir(tempdir);
 453 #endif
 454     return true;
 455 }
 456
 457 #ifdef XAPIAN_HAS_REMOTE_BACKEND
 458 static void
 459 replicate_with_brokenness(Xapian::DatabaseMaster & master,
 460                           Xapian::DatabaseReplica & replica,
 461                           const string & tempdir,
 462                           int expected_changesets,
 463                           int expected_fullcopies,
 464                           bool expected_changed)
 465 {
 466     string changesetpath = tempdir + "/changeset";
 467     get_changeset(changesetpath, master, replica,
 468                   1, 0, 1);
 469
 470     // Try applying truncated changesets of various different lengths.
 471     string brokenchangesetpath = tempdir + "/changeset_broken";
 472     off_t filesize = get_file_size(changesetpath);
 473     off_t len = 10;
 474     off_t copylen;
 475     while (len < filesize) {
 476         copylen = truncated_copy(changesetpath, brokenchangesetpath, len);
 477         TEST_EQUAL(copylen, len);
 478         tout << "Trying replication with a changeset truncated to " << len <<
 479                 " bytes, from " << filesize << " bytes\n";
 480         TEST_EXCEPTION(Xapian::NetworkError,
 481                        apply_changeset(brokenchangesetpath, replica,
 482                                        expected_changesets, expected_fullcopies,
 483                                        expected_changed));
 484         if (len < 30 || len >= filesize - 10) {
 485             // For lengths near the beginning and end, increment size by 1
 486             ++len;
 487         } else {
 488             // Don't bother incrementing by small amounts in the middle of
 489             // the changeset.
 490             len += 1000;
 491             if (len >= filesize - 10) {
 492                 len = filesize - 10;
 493             }
 494         }
 495     }
 496 }
 497 #endif
 498
 499 // Test changesets which are truncated (and therefore invalid).
 500 DEFINE_TESTCASE(replicate3, replicas) {
 501 #ifdef XAPIAN_HAS_REMOTE_BACKEND
 502     UNSET_MAX_CHANGESETS_AFTERWARDS;
 503     string tempdir = ".replicatmp";
 504     mktmpdir(tempdir);
 505     string masterpath = get_named_writable_database_path("master");
 506
 507     set_max_changesets(10);
 508
 509     {
 510         Xapian::WritableDatabase orig(get_named_writable_database("master"));
 511         Xapian::DatabaseMaster master(masterpath);
 512         string replicapath = tempdir + "/replica";
 513         Xapian::DatabaseReplica replica(replicapath);
 514
 515         // Add a document to the original database.
 516         Xapian::Document doc1;
 517         doc1.set_data(string("doc1"));
 518         doc1.add_posting("doc", 1);
 519         doc1.add_posting("one", 1);
 520         orig.add_document(doc1);
 521         orig.commit();
 522
 523         TEST_EQUAL(replicate(master, replica, tempdir, 0, 1, true), 1);
 524         check_equal_dbs(masterpath, replicapath);
 525
 526         // Make a changeset.
 527         orig.add_document(doc1);
 528         orig.commit();
 529
 530         replicate_with_brokenness(master, replica, tempdir, 1, 0, true);
 531         // Although it throws an error, the final replication in
 532         // replicate_with_brokenness() updates the database, since it's just
 533         // the end-of-replication message which is missing its body.
 534         check_equal_dbs(masterpath, replicapath);
 535
 536         // Check that the earlier broken replications didn't cause any problems
 537         // for the next replication.
 538         orig.add_document(doc1);
 539         orig.commit();
 540         TEST_EQUAL(replicate(master, replica, tempdir, 1, 0, true), 2);
 541
 542         // We need this inner scope to we close the replica before we remove
 543         // the temporary directory on Windows.
 544     }
 545
 546     rmtmpdir(tempdir);
 547 #endif
 548     return true;
 549 }
 550
 551 // Tests for max_changesets
 552 DEFINE_TESTCASE(replicate4, replicas) {
 553 #ifdef XAPIAN_HAS_REMOTE_BACKEND
 554     UNSET_MAX_CHANGESETS_AFTERWARDS;
 555     string tempdir = ".replicatmp";
 556     mktmpdir(tempdir);
 557     string masterpath = get_named_writable_database_path("master");
 558
 559     set_max_changesets(1);
 560
 561     {
 562         Xapian::WritableDatabase orig(get_named_writable_database("master"));
 563         Xapian::DatabaseMaster master(masterpath);
 564         string replicapath = tempdir + "/replica";
 565         Xapian::DatabaseReplica replica(replicapath);
 566
 567         // Add a document with no positions to the original database.
 568         Xapian::Document doc1;
 569         doc1.set_data(string("doc1"));
 570         doc1.add_term("nopos");
 571         orig.add_document(doc1);
 572         orig.commit();
 573
 574         // Apply the replication - we don't have changesets stored, so this
 575         // should just do a database copy, and return a count of 1.
 576         int count = replicate(master, replica, tempdir, 0, 1, true);
 577         TEST_EQUAL(count, 1);
 578         {
 579             Xapian::Database dbcopy(replicapath);
 580             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 581         }
 582
 583         // Add a document with positional information to the original database.
 584         doc1.add_posting("pos", 1);
 585         orig.add_document(doc1);
 586         orig.commit();
 587
 588         // Replicate, and check that we have the positional information.
 589         count = replicate(master, replica, tempdir, 1, 0, true);
 590         TEST_EQUAL(count, 2);
 591         {
 592             Xapian::Database dbcopy(replicapath);
 593             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 594         }
 595         check_equal_dbs(masterpath, replicapath);
 596
 597         // Add a document with no positions to the original database.
 598         Xapian::Document doc2;
 599         doc2.set_data(string("doc2"));
 600         doc2.add_term("nopos");
 601         orig.add_document(doc2);
 602         if (get_dbtype() == "glass") {
 603             // FIXME: Needs to be pre-commit for glass
 604             set_max_changesets(0);
 605         }
 606         orig.commit();
 607
 608         // Replicate, and check that we have the positional information.
 609         count = replicate(master, replica, tempdir, 1, 0, true);
 610         TEST_EQUAL(count, 2);
 611         {
 612             Xapian::Database dbcopy(replicapath);
 613             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 614         }
 615         check_equal_dbs(masterpath, replicapath);
 616         TEST(!file_exists(masterpath + "/changes1"));
 617
 618         // Turn off replication, make sure we dont write anything
 619         if (get_dbtype() != "glass") {
 620             set_max_changesets(0);
 621         }
 622
 623         // Add a document with no positions to the original database.
 624         Xapian::Document doc3;
 625         doc3.set_data(string("doc3"));
 626         doc3.add_term("nonopos");
 627         orig.add_document(doc3);
 628         orig.commit();
 629
 630         // Replicate, and check that we have the positional information.
 631         count = replicate(master, replica, tempdir, 0, 1, true);
 632         TEST_EQUAL(count, 1);
 633         {
 634             Xapian::Database dbcopy(replicapath);
 635             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 636         }
 637         // Should have pulled a full copy
 638         check_equal_dbs(masterpath, replicapath);
 639         TEST(!file_exists(masterpath + "/changes3"));
 640
 641         // We need this inner scope to we close the replica before we remove
 642         // the temporary directory on Windows.
 643     }
 644
 645     rmtmpdir(tempdir);
 646 #endif
 647     return true;
 648 }
 649
 650
 651 // Tests for max_changesets
 652 DEFINE_TESTCASE(replicate5, replicas) {
 653 #ifdef XAPIAN_HAS_REMOTE_BACKEND
 654     UNSET_MAX_CHANGESETS_AFTERWARDS;
 655     string tempdir = ".replicatmp";
 656     mktmpdir(tempdir);
 657     string masterpath = get_named_writable_database_path("master");
 658
 659     set_max_changesets(2);
 660
 661     {
 662         Xapian::WritableDatabase orig(get_named_writable_database("master"));
 663         Xapian::DatabaseMaster master(masterpath);
 664         string replicapath = tempdir + "/replica";
 665         Xapian::DatabaseReplica replica(replicapath);
 666
 667         // Add a document with no positions to the original database.
 668         Xapian::Document doc1;
 669         doc1.set_data(string("doc1"));
 670         doc1.add_term("nopos");
 671         orig.add_document(doc1);
 672         orig.commit();
 673
 674         // Apply the replication - we don't have changesets stored, so this
 675         // should just do a database copy, and return a count of 1.
 676         int count = replicate(master, replica, tempdir, 0, 1, true);
 677         TEST_EQUAL(count, 1);
 678         {
 679             Xapian::Database dbcopy(replicapath);
 680             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 681         }
 682
 683         // Add a document with positional information to the original database.
 684         doc1.add_posting("pos", 1);
 685         orig.add_document(doc1);
 686         orig.commit();
 687
 688         // Replicate, and check that we have the positional information.
 689         count = replicate(master, replica, tempdir, 1, 0, true);
 690         TEST_EQUAL(count, 2);
 691         {
 692             Xapian::Database dbcopy(replicapath);
 693             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 694         }
 695         check_equal_dbs(masterpath, replicapath);
 696
 697         // Add a document with no positions to the original database.
 698         Xapian::Document doc2;
 699         doc2.set_data(string("doc2"));
 700         doc2.add_term("nopos");
 701         orig.add_document(doc2);
 702         orig.commit();
 703
 704         // Replicate, and check that we have the positional information.
 705         count = replicate(master, replica, tempdir, 1, 0, true);
 706         TEST_EQUAL(count, 2);
 707         {
 708             Xapian::Database dbcopy(replicapath);
 709             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 710         }
 711         check_equal_dbs(masterpath, replicapath);
 712
 713         // Add a document with no positions to the original database.
 714         Xapian::Document doc3;
 715         doc3.set_data(string("doc3"));
 716         doc3.add_term("nonopos");
 717         orig.add_document(doc3);
 718         orig.commit();
 719
 720         // Replicate, and check that we have the positional information.
 721         count = replicate(master, replica, tempdir, 1, 0, true);
 722         TEST_EQUAL(count, 2);
 723         {
 724             Xapian::Database dbcopy(replicapath);
 725             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 726         }
 727         check_equal_dbs(masterpath, replicapath);
 728
 729         // Ensure that only these changesets exists
 730         TEST(!file_exists(masterpath + "/changes1"));
 731         TEST(file_exists(masterpath + "/changes2"));
 732         TEST(file_exists(masterpath + "/changes3"));
 733
 734         set_max_changesets(3);
 735         masterpath = get_named_writable_database_path("master");
 736
 737         // Add a document with no positions to the original database.
 738         Xapian::Document doc4;
 739         doc4.set_data(string("doc4"));
 740         doc4.add_term("nononopos");
 741         orig.add_document(doc4);
 742         orig.commit();
 743
 744         // Replicate, and check that we have the positional information.
 745         count = replicate(master, replica, tempdir, 1, 0, true);
 746         TEST_EQUAL(count, 2);
 747         {
 748             Xapian::Database dbcopy(replicapath);
 749             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 750         }
 751         check_equal_dbs(masterpath, replicapath);
 752
 753         // Add a document with no positions to the original database.
 754         Xapian::Document doc5;
 755         doc5.set_data(string("doc5"));
 756         doc5.add_term("nonononopos");
 757         orig.add_document(doc5);
 758         orig.commit();
 759
 760         // Replicate, and check that we have the positional information.
 761         count = replicate(master, replica, tempdir, 1, 0, true);
 762         TEST_EQUAL(count, 2);
 763         {
 764             Xapian::Database dbcopy(replicapath);
 765             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 766         }
 767         check_equal_dbs(masterpath, replicapath);
 768
 769         TEST(!file_exists(masterpath + "/changes2"));
 770         TEST(file_exists(masterpath + "/changes3"));
 771         TEST(file_exists(masterpath + "/changes4"));
 772         TEST(file_exists(masterpath + "/changes5"));
 773
 774         // We need this inner scope to we close the replica before we remove
 775         // the temporary directory on Windows.
 776     }
 777
 778     rmtmpdir(tempdir);
 779 #endif
 780     return true;
 781 }
 782
 783 /// Test --full-copy option.
 784 DEFINE_TESTCASE(replicate6, replicas) {
 785 #ifdef XAPIAN_HAS_REMOTE_BACKEND
 786     UNSET_MAX_CHANGESETS_AFTERWARDS;
 787     string tempdir = ".replicatmp";
 788     mktmpdir(tempdir);
 789     string masterpath = get_named_writable_database_path("master");
 790
 791     set_max_changesets(10);
 792
 793     {
 794         Xapian::WritableDatabase orig(get_named_writable_database("master"));
 795         Xapian::DatabaseMaster master(masterpath);
 796         string replicapath = tempdir + "/replica";
 797         Xapian::DatabaseReplica replica(replicapath);
 798
 799         // Add a document to the original database.
 800         Xapian::Document doc1;
 801         doc1.set_data(string("doc1"));
 802         doc1.add_posting("doc", 1);
 803         doc1.add_posting("one", 1);
 804         orig.add_document(doc1);
 805         orig.commit();
 806
 807         rm_rf(masterpath + "1");
 808         cp_R(masterpath, masterpath + "1");
 809
 810         orig.add_document(doc1);
 811         orig.commit();
 812
 813         // Apply the replication - we don't have changesets stored, so this
 814         // should just do a database copy, and return a count of 1.
 815         int count = replicate(master, replica, tempdir, 0, 1, true);
 816         TEST_EQUAL(count, 1);
 817         {
 818             Xapian::Database dbcopy(replicapath);
 819             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 820         }
 821
 822         Xapian::DatabaseMaster master1(masterpath + "1");
 823
 824         // Try to replicate an older version of the master.
 825         count = replicate(master1, replica, tempdir, 0, 0, false);
 826         TEST_EQUAL(count, 1);
 827
 828         // Force a full copy.
 829         count = replicate(master1, replica, tempdir, 0, 1, true, true);
 830         TEST_EQUAL(count, 1);
 831
 832         // Test we can still replicate.
 833         orig.add_document(doc1);
 834         orig.commit();
 835
 836         count = replicate(master, replica, tempdir, 2, 0, true);
 837         TEST_EQUAL(count, 3);
 838
 839         check_equal_dbs(masterpath, replicapath);
 840
 841         // We need this inner scope to we close the replica before we remove
 842         // the temporary directory on Windows.
 843     }
 844
 845     rmtmpdir(tempdir);
 846 #endif
 847     return true;
 848 }
 849
 850 /// Test healing a corrupt replica (new in 1.3.5).
 851 DEFINE_TESTCASE(replicate7, replicas) {
 852 #ifdef XAPIAN_HAS_REMOTE_BACKEND
 853     UNSET_MAX_CHANGESETS_AFTERWARDS;
 854     string tempdir = ".replicatmp";
 855     mktmpdir(tempdir);
 856     string masterpath = get_named_writable_database_path("master");
 857
 858     set_max_changesets(10);
 859
 860     Xapian::WritableDatabase orig(get_named_writable_database("master"));
 861     Xapian::DatabaseMaster master(masterpath);
 862     string replicapath = tempdir + "/replica";
 863     {
 864         Xapian::DatabaseReplica replica(replicapath);
 865
 866         // Add a document to the original database.
 867         Xapian::Document doc1;
 868         doc1.set_data(string("doc1"));
 869         doc1.add_posting("doc", 1);
 870         doc1.add_posting("one", 1);
 871         orig.add_document(doc1);
 872         orig.commit();
 873
 874         orig.add_document(doc1);
 875         orig.commit();
 876
 877         // Apply the replication - we don't have changesets stored, so this
 878         // should just do a database copy, and return a count of 1.
 879         int count = replicate(master, replica, tempdir, 0, 1, true);
 880         TEST_EQUAL(count, 1);
 881         {
 882             Xapian::Database dbcopy(replicapath);
 883             TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
 884         }
 885     }
 886
 887     {
 888         // Corrupt replica by truncating all the files to size 0.
 889         string d = replicapath;
 890         d += "/replica_1";
 891         DIR * dir = opendir(d.c_str());
 892         TEST(dir != NULL);
 893         while (true) {
 894             errno = 0;
 895             struct dirent * entry = readdir(dir);
 896             if (!entry) {
 897                 if (errno == 0)
 898                     break;
 899                 FAIL_TEST("readdir failed: " << strerror(errno));
 900             }
 901
 902             // Skip '.' and '..'.
 903             if (entry->d_name[0] == '.') continue;
 904
 905             string file = d;
 906             file += '/';
 907             file += entry->d_name;
 908             int fd = open(file.c_str(), O_WRONLY|O_TRUNC, 0666);
 909             TEST(fd != -1);
 910             TEST(close(fd) == 0);
 911         }
 912         closedir(dir);
 913     }
 914
 915     {
 916         Xapian::DatabaseReplica replica(replicapath);
 917
 918         // Replication should succeed and perform a full copy.
 919         int count = replicate(master, replica, tempdir, 0, 1, true);
 920         TEST_EQUAL(count, 1);
 921
 922         check_equal_dbs(masterpath, replicapath);
 923
 924         // We need this inner scope to we close the replica before we remove
 925         // the temporary directory on Windows.
 926     }
 927
 928     rmtmpdir(tempdir);
 929 #endif
 930     return true;
 931 }