xapian-core/tests/api_compactold.cc

   1 /** @file api_compactold.cc
   2  * @brief Tests of old compaction API
   3  */
   4 /* Copyright (C) 2009,2010,2011,2012,2013,2015 Olly Betts
   5  * Copyright (C) 2010 Richard Boulton
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License as
   9  * published by the Free Software Foundation; either version 2 of the
  10  * License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  20  * USA
  21  */
  22
  23 #include <config.h>
  24
  25 #include "api_compactold.h"
  26
  27 #define XAPIAN_DEPRECATED(X) X
  28 #include <xapian.h>
  29
  30 #include "apitest.h"
  31 #include "dbcheck.h"
  32 #include "filetests.h"
  33 #include "str.h"
  34 #include "testsuite.h"
  35 #include "testutils.h"
  36
  37 #include <cstdlib>
  38 #include <fstream>
  39
  40 #include "unixcmds.h"
  41
  42 using namespace std;
  43
  44 static void
  45 make_sparse_db(Xapian::WritableDatabase &db, const string & s)
  46 {
  47     // Need non-const pointer for strtoul(), but data isn't modified.
  48     char * p = const_cast<char *>(s.c_str());
  49
  50     while (*p) {
  51         bool del = (*p == '!');
  52         if (del) ++p;
  53         Xapian::docid first = strtoul(p, &p, 10);
  54         Xapian::docid last = first;
  55         if (*p == '-') {
  56             last = strtoul(p + 1, &p, 10);
  57         }
  58         if (*p && *p != ' ') {
  59             tout << p - s.c_str() << endl;
  60             FAIL_TEST("Bad sparse db spec (expected space): " << s);
  61         }
  62         if (first > last) {
  63             FAIL_TEST("Bad sparse db spec (first > last): " << s);
  64         }
  65
  66         do {
  67             if (del) {
  68                 db.delete_document(first);
  69             } else {
  70                 Xapian::Document doc;
  71                 string id = str(first);
  72                 doc.set_data(id);
  73                 doc.add_term("Q" + str(first));
  74                 doc.add_term(string(first % 7 + 1, char((first % 26) + 'a')));
  75                 db.replace_document(first, doc);
  76             }
  77         } while (first++ < last);
  78
  79         if (*p == '\0') break;
  80         ++p;
  81     }
  82
  83     db.commit();
  84 }
  85
  86 static void
  87 check_sparse_uid_terms(const string & path)
  88 {
  89     Xapian::Database db(path);
  90     Xapian::TermIterator t;
  91     for (t = db.allterms_begin("Q"); t != db.allterms_end("Q"); ++t) {
  92         Xapian::docid did = atoi((*t).c_str() + 1);
  93         Xapian::PostingIterator p = db.postlist_begin(*t);
  94         TEST_EQUAL(*p, did);
  95     }
  96 }
  97
  98 DEFINE_TESTCASE(compactoldnorenumber1, generated) {
  99     string a = get_database_path("compactnorenumber1a", make_sparse_db,
 100                                  "5-7 24 76 987 1023-1027 9999 !9999");
 101     string a_uuid;
 102     {
 103         Xapian::Database db(a);
 104         a_uuid = db.get_uuid();
 105     }
 106     string b = get_database_path("compactnorenumber1b", make_sparse_db,
 107                                  "1027-1030");
 108     string c = get_database_path("compactnorenumber1c", make_sparse_db,
 109                                  "1028-1040");
 110     string d = get_database_path("compactnorenumber1d", make_sparse_db,
 111                                  "3000 999999 !999999");
 112
 113     string out = get_named_writable_database_path("compactnorenumber1out");
 114
 115     rm_rf(out);
 116     {
 117         Xapian::Compactor compact;
 118         compact.set_renumber(false);
 119         compact.set_destdir(out);
 120         compact.add_source(a);
 121         compact.compact();
 122     }
 123
 124     check_sparse_uid_terms(out);
 125
 126     {
 127         TEST(!dir_exists(out + "/donor"));
 128         Xapian::Database db(out);
 129         // xapian-compact should change the UUID of the database, but didn't
 130         // prior to 1.0.18/1.1.4.
 131         string out_uuid = db.get_uuid();
 132         TEST_NOT_EQUAL(a_uuid, out_uuid);
 133         TEST_EQUAL(out_uuid.size(), 36);
 134         TEST_NOT_EQUAL(out_uuid, "00000000-0000-0000-0000-000000000000");
 135
 136         // White box test - ensure that the donor database is removed.
 137         TEST(!dir_exists(out + "/donor"));
 138     }
 139
 140     rm_rf(out);
 141     {
 142         Xapian::Compactor compact;
 143         compact.set_renumber(false);
 144         compact.set_destdir(out);
 145         compact.add_source(a);
 146         compact.add_source(c);
 147         compact.compact();
 148     }
 149     check_sparse_uid_terms(out);
 150     {
 151         // Check that xapian-compact is producing a consistent database.  Also,
 152         // regression test - xapian 1.1.4 set lastdocid to 0 in the output
 153         // database.
 154         Xapian::Database outdb(out);
 155         dbcheck(outdb, 24, 9999);
 156     }
 157
 158     rm_rf(out);
 159     {
 160         Xapian::Compactor compact;
 161         compact.set_renumber(false);
 162         compact.set_destdir(out);
 163         compact.add_source(d);
 164         compact.add_source(a);
 165         compact.add_source(c);
 166         compact.compact();
 167     }
 168     check_sparse_uid_terms(out);
 169
 170     rm_rf(out);
 171     {
 172         Xapian::Compactor compact;
 173         compact.set_renumber(false);
 174         compact.set_destdir(out);
 175         compact.add_source(c);
 176         compact.add_source(a);
 177         compact.add_source(d);
 178         compact.compact();
 179     }
 180     check_sparse_uid_terms(out);
 181
 182     // Should fail.
 183     rm_rf(out);
 184     {
 185         Xapian::Compactor compact;
 186         compact.set_renumber(false);
 187         compact.set_destdir(out);
 188         compact.add_source(a);
 189         compact.add_source(b);
 190         TEST_EXCEPTION(Xapian::InvalidOperationError, compact.compact());
 191     }
 192
 193     // Should fail.
 194     rm_rf(out);
 195     {
 196         Xapian::Compactor compact;
 197         compact.set_renumber(false);
 198         compact.set_destdir(out);
 199         compact.add_source(b);
 200         compact.add_source(a);
 201         TEST_EXCEPTION(Xapian::InvalidOperationError, compact.compact());
 202     }
 203
 204     // Should fail.
 205     rm_rf(out);
 206     {
 207         Xapian::Compactor compact;
 208         compact.set_renumber(false);
 209         compact.set_destdir(out);
 210         compact.add_source(a);
 211         compact.add_source(b);
 212         compact.add_source(d);
 213         TEST_EXCEPTION(Xapian::InvalidOperationError, compact.compact());
 214     }
 215
 216     // Should fail.
 217     rm_rf(out);
 218     {
 219         Xapian::Compactor compact;
 220         compact.set_renumber(false);
 221         compact.set_destdir(out);
 222         compact.add_source(d);
 223         compact.add_source(b);
 224         compact.add_source(a);
 225         TEST_EXCEPTION(Xapian::InvalidOperationError, compact.compact());
 226     }
 227
 228     // Should fail.
 229     rm_rf(out);
 230     {
 231         Xapian::Compactor compact;
 232         compact.set_renumber(false);
 233         compact.set_destdir(out);
 234         compact.add_source(b);
 235         compact.add_source(a);
 236         compact.add_source(d);
 237         TEST_EXCEPTION(Xapian::InvalidOperationError, compact.compact());
 238     }
 239
 240     return true;
 241 }
 242
 243 // Test use of compact to merge two databases.
 244 DEFINE_TESTCASE(compactoldmerge1, glass) {
 245     string indbpath = get_database_path("apitest_simpledata");
 246     string outdbpath = get_named_writable_database_path("compactmerge1out");
 247     rm_rf(outdbpath);
 248
 249     Xapian::Compactor compact;
 250     compact.set_destdir(outdbpath);
 251     compact.add_source(indbpath);
 252     compact.add_source(indbpath);
 253     compact.compact();
 254
 255     Xapian::Database indb(get_database("apitest_simpledata"));
 256     Xapian::Database outdb(outdbpath);
 257
 258     TEST_EQUAL(indb.get_doccount() * 2, outdb.get_doccount());
 259     dbcheck(outdb, outdb.get_doccount(), outdb.get_doccount());
 260
 261     return true;
 262 }
 263
 264 static void
 265 make_multichunk_db(Xapian::WritableDatabase &db, const string &)
 266 {
 267     int count = 10000;
 268
 269     Xapian::Document doc;
 270     doc.add_term("a");
 271     while (count) {
 272         db.add_document(doc);
 273         --count;
 274     }
 275
 276     db.commit();
 277 }
 278
 279 // Test use of compact on a database which has multiple chunks for a term.
 280 // This is a regression test for ticket #427
 281 DEFINE_TESTCASE(compactoldmultichunks1, generated) {
 282     string indbpath = get_database_path("compactmultichunks1in",
 283                                         make_multichunk_db, "");
 284     string outdbpath = get_named_writable_database_path("compactmultichunks1out");
 285     rm_rf(outdbpath);
 286
 287     Xapian::Compactor compact;
 288     compact.set_destdir(outdbpath);
 289     compact.add_source(indbpath);
 290     compact.compact();
 291
 292     Xapian::Database indb(indbpath);
 293     Xapian::Database outdb(outdbpath);
 294
 295     TEST_EQUAL(indb.get_doccount(), outdb.get_doccount());
 296     dbcheck(outdb, outdb.get_doccount(), outdb.get_doccount());
 297
 298     return true;
 299 }
 300
 301 // Test compacting from a stub database directory.
 302 DEFINE_TESTCASE(compactoldstub1, glass) {
 303     const char * stubpath = ".stub/compactstub1";
 304     const char * stubpathfile = ".stub/compactstub1/XAPIANDB";
 305     mkdir(".stub", 0755);
 306     mkdir(stubpath, 0755);
 307     ofstream stub(stubpathfile);
 308     TEST(stub.is_open());
 309     stub << "auto ../../" << get_database_path("apitest_simpledata") << endl;
 310     stub << "auto ../../" << get_database_path("apitest_simpledata2") << endl;
 311     stub.close();
 312
 313     string outdbpath = get_named_writable_database_path("compactstub1out");
 314     rm_rf(outdbpath);
 315
 316     Xapian::Compactor compact;
 317     compact.set_destdir(outdbpath);
 318     compact.add_source(stubpath);
 319     compact.compact();
 320
 321     Xapian::Database indb(stubpath);
 322     Xapian::Database outdb(outdbpath);
 323
 324     TEST_EQUAL(indb.get_doccount(), outdb.get_doccount());
 325     dbcheck(outdb, outdb.get_doccount(), outdb.get_doccount());
 326
 327     return true;
 328 }
 329
 330 // Test compacting from a stub database file.
 331 DEFINE_TESTCASE(compactoldstub2, glass) {
 332     const char * stubpath = ".stub/compactstub2";
 333     mkdir(".stub", 0755);
 334     ofstream stub(stubpath);
 335     TEST(stub.is_open());
 336     stub << "auto ../" << get_database_path("apitest_simpledata") << endl;
 337     stub << "auto ../" << get_database_path("apitest_simpledata2") << endl;
 338     stub.close();
 339
 340     string outdbpath = get_named_writable_database_path("compactstub2out");
 341     rm_rf(outdbpath);
 342
 343     Xapian::Compactor compact;
 344     compact.set_destdir(outdbpath);
 345     compact.add_source(stubpath);
 346     compact.compact();
 347
 348     Xapian::Database indb(stubpath);
 349     Xapian::Database outdb(outdbpath);
 350
 351     TEST_EQUAL(indb.get_doccount(), outdb.get_doccount());
 352     dbcheck(outdb, outdb.get_doccount(), outdb.get_doccount());
 353
 354     return true;
 355 }
 356
 357 // Test compacting a stub database file to itself.
 358 DEFINE_TESTCASE(compactoldstub3, glass) {
 359     const char * stubpath = ".stub/compactstub3";
 360     mkdir(".stub", 0755);
 361     ofstream stub(stubpath);
 362     TEST(stub.is_open());
 363     stub << "auto ../" << get_database_path("apitest_simpledata") << endl;
 364     stub << "auto ../" << get_database_path("apitest_simpledata2") << endl;
 365     stub.close();
 366
 367     Xapian::doccount in_docs;
 368     {
 369         Xapian::Database indb(stubpath);
 370         in_docs = indb.get_doccount();
 371     }
 372
 373     Xapian::Compactor compact;
 374     compact.set_destdir(stubpath);
 375     compact.add_source(stubpath);
 376     compact.compact();
 377
 378     Xapian::Database outdb(stubpath);
 379
 380     TEST_EQUAL(in_docs, outdb.get_doccount());
 381     dbcheck(outdb, outdb.get_doccount(), outdb.get_doccount());
 382
 383     return true;
 384 }
 385
 386 // Test compacting a stub database directory to itself.
 387 DEFINE_TESTCASE(compactoldstub4, glass) {
 388     const char * stubpath = ".stub/compactstub4";
 389     const char * stubpathfile = ".stub/compactstub4/XAPIANDB";
 390     mkdir(".stub", 0755);
 391     mkdir(stubpath, 0755);
 392     ofstream stub(stubpathfile);
 393     TEST(stub.is_open());
 394     stub << "auto ../../" << get_database_path("apitest_simpledata") << endl;
 395     stub << "auto ../../" << get_database_path("apitest_simpledata2") << endl;
 396     stub.close();
 397
 398     Xapian::doccount in_docs;
 399     {
 400         Xapian::Database indb(stubpath);
 401         in_docs = indb.get_doccount();
 402     }
 403
 404     Xapian::Compactor compact;
 405     compact.set_destdir(stubpath);
 406     compact.add_source(stubpath);
 407     compact.compact();
 408
 409     Xapian::Database outdb(stubpath);
 410
 411     TEST_EQUAL(in_docs, outdb.get_doccount());
 412     dbcheck(outdb, outdb.get_doccount(), outdb.get_doccount());
 413
 414     return true;
 415 }
 416
 417 static void
 418 make_all_tables(Xapian::WritableDatabase &db, const string &)
 419 {
 420     Xapian::Document doc;
 421     doc.add_term("foo");
 422     db.add_document(doc);
 423     db.add_spelling("foo");
 424     db.add_synonym("bar", "pub");
 425     db.add_synonym("foobar", "foo");
 426
 427     db.commit();
 428 }
 429
 430 static void
 431 make_missing_tables(Xapian::WritableDatabase &db, const string &)
 432 {
 433     Xapian::Document doc;
 434     doc.add_term("foo");
 435     db.add_document(doc);
 436
 437     db.commit();
 438 }
 439
 440 DEFINE_TESTCASE(compactoldmissingtables1, generated) {
 441     string a = get_database_path("compactmissingtables1a",
 442                                  make_all_tables);
 443     string b = get_database_path("compactmissingtables1b",
 444                                  make_missing_tables);
 445
 446     string out = get_named_writable_database_path("compactmissingtables1out");
 447     rm_rf(out);
 448
 449     Xapian::Compactor compact;
 450     compact.set_destdir(out);
 451     compact.add_source(a);
 452     compact.add_source(b);
 453     compact.compact();
 454
 455     {
 456         Xapian::Database db(out);
 457         TEST_NOT_EQUAL(db.spellings_begin(), db.spellings_end());
 458         TEST_NOT_EQUAL(db.synonym_keys_begin(), db.synonym_keys_end());
 459         // FIXME: arrange for input b to not have a termlist table.
 460 //      TEST_EXCEPTION(Xapian::FeatureUnavailableError, db.termlist_begin(1));
 461     }
 462
 463     return true;
 464 }
 465
 466 static void
 467 make_all_tables2(Xapian::WritableDatabase &db, const string &)
 468 {
 469     Xapian::Document doc;
 470     doc.add_term("bar");
 471     db.add_document(doc);
 472     db.add_spelling("bar");
 473     db.add_synonym("bar", "baa");
 474     db.add_synonym("barfoo", "barbar");
 475     db.add_synonym("foofoo", "barfoo");
 476
 477     db.commit();
 478 }
 479
 480 /// Adds coverage for merging synonym table.
 481 DEFINE_TESTCASE(compactoldmergesynonym1, generated) {
 482     string a = get_database_path("compactmergesynonym1a",
 483                                  make_all_tables);
 484     string b = get_database_path("compactmergesynonym1b",
 485                                  make_all_tables2);
 486
 487     string out = get_named_writable_database_path("compactmergesynonym1out");
 488     rm_rf(out);
 489
 490     Xapian::Compactor compact;
 491     compact.set_destdir(out);
 492     compact.add_source(a);
 493     compact.add_source(b);
 494     compact.compact();
 495
 496     {
 497         Xapian::Database db(out);
 498
 499         Xapian::TermIterator i = db.spellings_begin();
 500         TEST_NOT_EQUAL(i, db.spellings_end());
 501         TEST_EQUAL(*i, "bar");
 502         ++i;
 503         TEST_NOT_EQUAL(i, db.spellings_end());
 504         TEST_EQUAL(*i, "foo");
 505         ++i;
 506         TEST_EQUAL(i, db.spellings_end());
 507
 508         i = db.synonym_keys_begin();
 509         TEST_NOT_EQUAL(i, db.synonym_keys_end());
 510         TEST_EQUAL(*i, "bar");
 511         ++i;
 512         TEST_NOT_EQUAL(i, db.synonym_keys_end());
 513         TEST_EQUAL(*i, "barfoo");
 514         ++i;
 515         TEST_NOT_EQUAL(i, db.synonym_keys_end());
 516         TEST_EQUAL(*i, "foobar");
 517         ++i;
 518         TEST_NOT_EQUAL(i, db.synonym_keys_end());
 519         TEST_EQUAL(*i, "foofoo");
 520         ++i;
 521         TEST_EQUAL(i, db.synonym_keys_end());
 522     }
 523
 524     return true;
 525 }
 526
 527 DEFINE_TESTCASE(compactoldempty1, glass) {
 528     string empty_dbpath = get_database_path(string());
 529     string outdbpath = get_named_writable_database_path("compactempty1out");
 530     rm_rf(outdbpath);
 531
 532     {
 533         // Compacting an empty database tried to divide by zero in 1.3.0.
 534         Xapian::Compactor compact;
 535         compact.set_destdir(outdbpath);
 536         compact.add_source(empty_dbpath);
 537         compact.compact();
 538
 539         Xapian::Database outdb(outdbpath);
 540         TEST_EQUAL(outdb.get_doccount(), 0);
 541         dbcheck(outdb, 0, 0);
 542     }
 543
 544     {
 545         // Check compacting two empty databases together.
 546         Xapian::Compactor compact;
 547         compact.set_destdir(outdbpath);
 548         compact.add_source(empty_dbpath);
 549         compact.add_source(empty_dbpath);
 550         compact.compact();
 551
 552         Xapian::Database outdb(outdbpath);
 553         TEST_EQUAL(outdb.get_doccount(), 0);
 554         dbcheck(outdb, 0, 0);
 555     }
 556
 557     return true;
 558 }
 559
 560 DEFINE_TESTCASE(compactoldmultipass1, glass) {
 561     string outdbpath = get_named_writable_database_path("compactmultipass1");
 562     rm_rf(outdbpath);
 563
 564     string a = get_database_path("compactnorenumber1a", make_sparse_db,
 565                                  "5-7 24 76 987 1023-1027 9999 !9999");
 566     string b = get_database_path("compactnorenumber1b", make_sparse_db,
 567                                  "1027-1030");
 568     string c = get_database_path("compactnorenumber1c", make_sparse_db,
 569                                  "1028-1040");
 570     string d = get_database_path("compactnorenumber1d", make_sparse_db,
 571                                  "3000 999999 !999999");
 572
 573     Xapian::Compactor compact;
 574     compact.set_destdir(outdbpath);
 575     compact.add_source(a);
 576     compact.add_source(b);
 577     compact.add_source(c);
 578     compact.add_source(d);
 579     compact.set_multipass(true);
 580     compact.compact();
 581
 582     Xapian::Database outdb(outdbpath);
 583     dbcheck(outdb, 29, 1041);
 584
 585     return true;
 586 }