xapian-core/tests/api_compact.cc

   1 /** @file api_compact.cc
   2  * @brief Tests of Database::compact()
   3  */
   4 /* Copyright (C) 2009,2010,2011,2012,2013,2015 Olly Betts
   5  * Copyright (C) 2010 Richard Boulton
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License as
   9  * published by the Free Software Foundation; either version 2 of the
  10  * License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  20  * USA
  21  */
  22
  23 #include <config.h>
  24
  25 #include "api_compact.h"
  26
  27 #include <xapian.h>
  28
  29 #include "apitest.h"
  30 #include "dbcheck.h"
  31 #include "filetests.h"
  32 #include "str.h"
  33 #include "testsuite.h"
  34 #include "testutils.h"
  35
  36 #include <cstdlib>
  37 #include <fstream>
  38
  39 #include "unixcmds.h"
  40
  41 using namespace std;
  42
  43 static void
  44 make_sparse_db(Xapian::WritableDatabase &db, const string & s)
  45 {
  46     // Need non-const pointer for strtoul(), but data isn't modified.
  47     char * p = const_cast<char *>(s.c_str());
  48
  49     while (*p) {
  50         bool del = (*p == '!');
  51         if (del) ++p;
  52         Xapian::docid first = strtoul(p, &p, 10);
  53         Xapian::docid last = first;
  54         if (*p == '-') {
  55             last = strtoul(p + 1, &p, 10);
  56         }
  57         if (*p && *p != ' ') {
  58             tout << p - s.c_str() << endl;
  59             FAIL_TEST("Bad sparse db spec (expected space): " << s);
  60         }
  61         if (first > last) {
  62             FAIL_TEST("Bad sparse db spec (first > last): " << s);
  63         }
  64
  65         do {
  66             if (del) {
  67                 db.delete_document(first);
  68             } else {
  69                 Xapian::Document doc;
  70                 string id = str(first);
  71                 doc.set_data(id);
  72                 doc.add_term("Q" + str(first));
  73                 doc.add_term(string(first % 7 + 1, char((first % 26) + 'a')));
  74                 db.replace_document(first, doc);
  75             }
  76         } while (first++ < last);
  77
  78         if (*p == '\0') break;
  79         ++p;
  80     }
  81
  82     db.commit();
  83 }
  84
  85 static void
  86 check_sparse_uid_terms(const string & path)
  87 {
  88     Xapian::Database db(path);
  89     Xapian::TermIterator t;
  90     for (t = db.allterms_begin("Q"); t != db.allterms_end("Q"); ++t) {
  91         Xapian::docid did = atoi((*t).c_str() + 1);
  92         Xapian::PostingIterator p = db.postlist_begin(*t);
  93         TEST_EQUAL(*p, did);
  94     }
  95 }
  96
  97 DEFINE_TESTCASE(compactnorenumber1, generated) {
  98     string a = get_database_path("compactnorenumber1a", make_sparse_db,
  99                                  "5-7 24 76 987 1023-1027 9999 !9999");
 100     string a_uuid;
 101     {
 102         Xapian::Database db(a);
 103         a_uuid = db.get_uuid();
 104     }
 105     string b = get_database_path("compactnorenumber1b", make_sparse_db,
 106                                  "1027-1030");
 107     string c = get_database_path("compactnorenumber1c", make_sparse_db,
 108                                  "1028-1040");
 109     string d = get_database_path("compactnorenumber1d", make_sparse_db,
 110                                  "3000 999999 !999999");
 111
 112     string out = get_named_writable_database_path("compactnorenumber1out");
 113
 114     rm_rf(out);
 115     {
 116         Xapian::Database db(a);
 117         db.compact(out, Xapian::DBCOMPACT_NO_RENUMBER);
 118     }
 119
 120     check_sparse_uid_terms(out);
 121
 122     {
 123         TEST(!dir_exists(out + "/donor"));
 124         Xapian::Database db(out);
 125         // xapian-compact should change the UUID of the database, but didn't
 126         // prior to 1.0.18/1.1.4.
 127         string out_uuid = db.get_uuid();
 128         TEST_NOT_EQUAL(a_uuid, out_uuid);
 129         TEST_EQUAL(out_uuid.size(), 36);
 130         TEST_NOT_EQUAL(out_uuid, "00000000-0000-0000-0000-000000000000");
 131
 132         // White box test - ensure that the donor database is removed.
 133         TEST(!dir_exists(out + "/donor"));
 134     }
 135
 136     rm_rf(out);
 137     {
 138         Xapian::Database db;
 139         db.add_database(Xapian::Database(a));
 140         db.add_database(Xapian::Database(c));
 141         db.compact(out, Xapian::DBCOMPACT_NO_RENUMBER);
 142     }
 143     check_sparse_uid_terms(out);
 144     {
 145         // Check that xapian-compact is producing a consistent database.  Also,
 146         // regression test - xapian 1.1.4 set lastdocid to 0 in the output
 147         // database.
 148         Xapian::Database outdb(out);
 149         dbcheck(outdb, 24, 9999);
 150     }
 151
 152     rm_rf(out);
 153     {
 154         Xapian::Database db;
 155         db.add_database(Xapian::Database(d));
 156         db.add_database(Xapian::Database(a));
 157         db.add_database(Xapian::Database(c));
 158         db.compact(out, Xapian::DBCOMPACT_NO_RENUMBER);
 159     }
 160     check_sparse_uid_terms(out);
 161
 162     rm_rf(out);
 163     {
 164         Xapian::Database db;
 165         db.add_database(Xapian::Database(c));
 166         db.add_database(Xapian::Database(a));
 167         db.add_database(Xapian::Database(d));
 168         db.compact(out, Xapian::DBCOMPACT_NO_RENUMBER);
 169     }
 170     check_sparse_uid_terms(out);
 171
 172     // Should fail.
 173     rm_rf(out);
 174     {
 175         Xapian::Database db;
 176         db.add_database(Xapian::Database(a));
 177         db.add_database(Xapian::Database(b));
 178         TEST_EXCEPTION(Xapian::InvalidOperationError,
 179             db.compact(out, Xapian::DBCOMPACT_NO_RENUMBER)
 180         );
 181     }
 182
 183     // Should fail.
 184     rm_rf(out);
 185     {
 186         Xapian::Database db;
 187         db.add_database(Xapian::Database(b));
 188         db.add_database(Xapian::Database(a));
 189         TEST_EXCEPTION(Xapian::InvalidOperationError,
 190             db.compact(out, Xapian::DBCOMPACT_NO_RENUMBER)
 191         );
 192     }
 193
 194     // Should fail.
 195     rm_rf(out);
 196     {
 197         Xapian::Database db;
 198         db.add_database(Xapian::Database(a));
 199         db.add_database(Xapian::Database(b));
 200         db.add_database(Xapian::Database(d));
 201         TEST_EXCEPTION(Xapian::InvalidOperationError,
 202             db.compact(out, Xapian::DBCOMPACT_NO_RENUMBER)
 203         );
 204     }
 205
 206     // Should fail.
 207     rm_rf(out);
 208     {
 209         Xapian::Database db;
 210         db.add_database(Xapian::Database(d));
 211         db.add_database(Xapian::Database(b));
 212         db.add_database(Xapian::Database(a));
 213         TEST_EXCEPTION(Xapian::InvalidOperationError,
 214             db.compact(out, Xapian::DBCOMPACT_NO_RENUMBER)
 215         );
 216     }
 217
 218     // Should fail.
 219     rm_rf(out);
 220     {
 221         Xapian::Database db;
 222         db.add_database(Xapian::Database(b));
 223         db.add_database(Xapian::Database(a));
 224         db.add_database(Xapian::Database(d));
 225         TEST_EXCEPTION(Xapian::InvalidOperationError,
 226             db.compact(out, Xapian::DBCOMPACT_NO_RENUMBER)
 227         );
 228     }
 229
 230     return true;
 231 }
 232
 233 // Test use of compact to merge two databases.
 234 DEFINE_TESTCASE(compactmerge1, chert || glass) {
 235     string indbpath = get_database_path("apitest_simpledata");
 236     string outdbpath = get_named_writable_database_path("compactmerge1out");
 237     rm_rf(outdbpath);
 238
 239     {
 240         Xapian::Database db;
 241         db.add_database(Xapian::Database(indbpath));
 242         db.add_database(Xapian::Database(indbpath));
 243         db.compact(outdbpath);
 244     }
 245
 246     Xapian::Database indb(get_database("apitest_simpledata"));
 247     Xapian::Database outdb(outdbpath);
 248
 249     TEST_EQUAL(indb.get_doccount() * 2, outdb.get_doccount());
 250     dbcheck(outdb, outdb.get_doccount(), outdb.get_doccount());
 251
 252     return true;
 253 }
 254
 255 static void
 256 make_multichunk_db(Xapian::WritableDatabase &db, const string &)
 257 {
 258     int count = 10000;
 259
 260     Xapian::Document doc;
 261     doc.add_term("a");
 262     while (count) {
 263         db.add_document(doc);
 264         --count;
 265     }
 266
 267     db.commit();
 268 }
 269
 270 // Test use of compact on a database which has multiple chunks for a term.
 271 // This is a regression test for ticket #427
 272 DEFINE_TESTCASE(compactmultichunks1, generated) {
 273     string indbpath = get_database_path("compactmultichunks1in",
 274                                         make_multichunk_db, "");
 275     string outdbpath = get_named_writable_database_path("compactmultichunks1out");
 276     rm_rf(outdbpath);
 277
 278     {
 279         Xapian::Database db(indbpath);
 280         db.compact(outdbpath);
 281     }
 282
 283     Xapian::Database indb(indbpath);
 284     Xapian::Database outdb(outdbpath);
 285
 286     TEST_EQUAL(indb.get_doccount(), outdb.get_doccount());
 287     dbcheck(outdb, outdb.get_doccount(), outdb.get_doccount());
 288
 289     return true;
 290 }
 291
 292 // Test compacting from a stub database directory.
 293 DEFINE_TESTCASE(compactstub1, chert || glass) {
 294     const char * stubpath = ".stub/compactstub1";
 295     const char * stubpathfile = ".stub/compactstub1/XAPIANDB";
 296     mkdir(".stub", 0755);
 297     mkdir(stubpath, 0755);
 298     ofstream stub(stubpathfile);
 299     TEST(stub.is_open());
 300     stub << "auto ../../" << get_database_path("apitest_simpledata") << endl;
 301     stub << "auto ../../" << get_database_path("apitest_simpledata2") << endl;
 302     stub.close();
 303
 304     string outdbpath = get_named_writable_database_path("compactstub1out");
 305     rm_rf(outdbpath);
 306
 307     {
 308         Xapian::Database db(stubpath);
 309         db.compact(outdbpath);
 310     }
 311
 312     Xapian::Database indb(stubpath);
 313     Xapian::Database outdb(outdbpath);
 314
 315     TEST_EQUAL(indb.get_doccount(), outdb.get_doccount());
 316     dbcheck(outdb, outdb.get_doccount(), outdb.get_doccount());
 317
 318     return true;
 319 }
 320
 321 // Test compacting from a stub database file.
 322 DEFINE_TESTCASE(compactstub2, chert || glass) {
 323     const char * stubpath = ".stub/compactstub2";
 324     mkdir(".stub", 0755);
 325     ofstream stub(stubpath);
 326     TEST(stub.is_open());
 327     stub << "auto ../" << get_database_path("apitest_simpledata") << endl;
 328     stub << "auto ../" << get_database_path("apitest_simpledata2") << endl;
 329     stub.close();
 330
 331     string outdbpath = get_named_writable_database_path("compactstub2out");
 332     rm_rf(outdbpath);
 333
 334     {
 335         Xapian::Database db(stubpath);
 336         db.compact(outdbpath);
 337     }
 338
 339     Xapian::Database indb(stubpath);
 340     Xapian::Database outdb(outdbpath);
 341
 342     TEST_EQUAL(indb.get_doccount(), outdb.get_doccount());
 343     dbcheck(outdb, outdb.get_doccount(), outdb.get_doccount());
 344
 345     return true;
 346 }
 347
 348 // Test compacting a stub database file to itself.
 349 DEFINE_TESTCASE(compactstub3, chert || glass) {
 350     const char * stubpath = ".stub/compactstub3";
 351     mkdir(".stub", 0755);
 352     ofstream stub(stubpath);
 353     TEST(stub.is_open());
 354     stub << "auto ../" << get_database_path("apitest_simpledata") << endl;
 355     stub << "auto ../" << get_database_path("apitest_simpledata2") << endl;
 356     stub.close();
 357
 358     Xapian::doccount in_docs;
 359     {
 360         Xapian::Database indb(stubpath);
 361         in_docs = indb.get_doccount();
 362         indb.compact(stubpath);
 363     }
 364
 365     Xapian::Database outdb(stubpath);
 366
 367     TEST_EQUAL(in_docs, outdb.get_doccount());
 368     dbcheck(outdb, outdb.get_doccount(), outdb.get_doccount());
 369
 370     return true;
 371 }
 372
 373 // Test compacting a stub database directory to itself.
 374 DEFINE_TESTCASE(compactstub4, chert || glass) {
 375     const char * stubpath = ".stub/compactstub4";
 376     const char * stubpathfile = ".stub/compactstub4/XAPIANDB";
 377     mkdir(".stub", 0755);
 378     mkdir(stubpath, 0755);
 379     ofstream stub(stubpathfile);
 380     TEST(stub.is_open());
 381     stub << "auto ../../" << get_database_path("apitest_simpledata") << endl;
 382     stub << "auto ../../" << get_database_path("apitest_simpledata2") << endl;
 383     stub.close();
 384
 385     Xapian::doccount in_docs;
 386     {
 387         Xapian::Database indb(stubpath);
 388         in_docs = indb.get_doccount();
 389         indb.compact(stubpath);
 390     }
 391
 392     Xapian::Database outdb(stubpath);
 393
 394     TEST_EQUAL(in_docs, outdb.get_doccount());
 395     dbcheck(outdb, outdb.get_doccount(), outdb.get_doccount());
 396
 397     return true;
 398 }
 399
 400 static void
 401 make_all_tables(Xapian::WritableDatabase &db, const string &)
 402 {
 403     Xapian::Document doc;
 404     doc.add_term("foo");
 405     db.add_document(doc);
 406     db.add_spelling("foo");
 407     db.add_synonym("bar", "pub");
 408     db.add_synonym("foobar", "foo");
 409
 410     db.commit();
 411 }
 412
 413 static void
 414 make_missing_tables(Xapian::WritableDatabase &db, const string &)
 415 {
 416     Xapian::Document doc;
 417     doc.add_term("foo");
 418     db.add_document(doc);
 419
 420     db.commit();
 421 }
 422
 423 DEFINE_TESTCASE(compactmissingtables1, generated) {
 424     string a = get_database_path("compactmissingtables1a",
 425                                  make_all_tables);
 426     string b = get_database_path("compactmissingtables1b",
 427                                  make_missing_tables);
 428
 429     string out = get_named_writable_database_path("compactmissingtables1out");
 430     rm_rf(out);
 431
 432     {
 433         Xapian::Database db;
 434         db.add_database(Xapian::Database(a));
 435         db.add_database(Xapian::Database(b));
 436         db.compact(out);
 437     }
 438
 439     {
 440         Xapian::Database db(out);
 441         TEST_NOT_EQUAL(db.spellings_begin(), db.spellings_end());
 442         TEST_NOT_EQUAL(db.synonym_keys_begin(), db.synonym_keys_end());
 443         // FIXME: arrange for input b to not have a termlist table.
 444 //      TEST_EXCEPTION(Xapian::FeatureUnavailableError, db.termlist_begin(1));
 445     }
 446
 447     return true;
 448 }
 449
 450 static void
 451 make_all_tables2(Xapian::WritableDatabase &db, const string &)
 452 {
 453     Xapian::Document doc;
 454     doc.add_term("bar");
 455     db.add_document(doc);
 456     db.add_spelling("bar");
 457     db.add_synonym("bar", "baa");
 458     db.add_synonym("barfoo", "barbar");
 459     db.add_synonym("foofoo", "barfoo");
 460
 461     db.commit();
 462 }
 463
 464 /// Adds coverage for merging synonym table.
 465 DEFINE_TESTCASE(compactmergesynonym1, generated) {
 466     string a = get_database_path("compactmergesynonym1a",
 467                                  make_all_tables);
 468     string b = get_database_path("compactmergesynonym1b",
 469                                  make_all_tables2);
 470
 471     string out = get_named_writable_database_path("compactmergesynonym1out");
 472     rm_rf(out);
 473
 474     {
 475         Xapian::Database db;
 476         db.add_database(Xapian::Database(a));
 477         db.add_database(Xapian::Database(b));
 478         db.compact(out);
 479     }
 480
 481     {
 482         Xapian::Database db(out);
 483
 484         Xapian::TermIterator i = db.spellings_begin();
 485         TEST_NOT_EQUAL(i, db.spellings_end());
 486         TEST_EQUAL(*i, "bar");
 487         ++i;
 488         TEST_NOT_EQUAL(i, db.spellings_end());
 489         TEST_EQUAL(*i, "foo");
 490         ++i;
 491         TEST_EQUAL(i, db.spellings_end());
 492
 493         i = db.synonym_keys_begin();
 494         TEST_NOT_EQUAL(i, db.synonym_keys_end());
 495         TEST_EQUAL(*i, "bar");
 496         ++i;
 497         TEST_NOT_EQUAL(i, db.synonym_keys_end());
 498         TEST_EQUAL(*i, "barfoo");
 499         ++i;
 500         TEST_NOT_EQUAL(i, db.synonym_keys_end());
 501         TEST_EQUAL(*i, "foobar");
 502         ++i;
 503         TEST_NOT_EQUAL(i, db.synonym_keys_end());
 504         TEST_EQUAL(*i, "foofoo");
 505         ++i;
 506         TEST_EQUAL(i, db.synonym_keys_end());
 507     }
 508
 509     return true;
 510 }
 511
 512 DEFINE_TESTCASE(compactempty1, chert || glass) {
 513     string empty_dbpath = get_database_path(string());
 514     string outdbpath = get_named_writable_database_path("compactempty1out");
 515     rm_rf(outdbpath);
 516
 517     {
 518         // Compacting an empty database tried to divide by zero in 1.3.0.
 519         Xapian::Database db;
 520         db.add_database(Xapian::Database(empty_dbpath));
 521         db.compact(outdbpath);
 522
 523         Xapian::Database outdb(outdbpath);
 524         TEST_EQUAL(outdb.get_doccount(), 0);
 525         dbcheck(outdb, 0, 0);
 526     }
 527
 528     {
 529         // Check compacting two empty databases together.
 530         Xapian::Database db;
 531         db.add_database(Xapian::Database(empty_dbpath));
 532         db.add_database(Xapian::Database(empty_dbpath));
 533         db.compact(outdbpath);
 534
 535         Xapian::Database outdb(outdbpath);
 536         TEST_EQUAL(outdb.get_doccount(), 0);
 537         dbcheck(outdb, 0, 0);
 538     }
 539
 540     return true;
 541 }
 542
 543 DEFINE_TESTCASE(compactmultipass1, chert || glass) {
 544     string outdbpath = get_named_writable_database_path("compactmultipass1");
 545     rm_rf(outdbpath);
 546
 547     string a = get_database_path("compactnorenumber1a", make_sparse_db,
 548                                  "5-7 24 76 987 1023-1027 9999 !9999");
 549     string b = get_database_path("compactnorenumber1b", make_sparse_db,
 550                                  "1027-1030");
 551     string c = get_database_path("compactnorenumber1c", make_sparse_db,
 552                                  "1028-1040");
 553     string d = get_database_path("compactnorenumber1d", make_sparse_db,
 554                                  "3000 999999 !999999");
 555
 556     {
 557         Xapian::Database db;
 558         db.add_database(Xapian::Database(a));
 559         db.add_database(Xapian::Database(b));
 560         db.add_database(Xapian::Database(c));
 561         db.add_database(Xapian::Database(d));
 562         db.compact(outdbpath, Xapian::DBCOMPACT_MULTIPASS);
 563     }
 564
 565     Xapian::Database outdb(outdbpath);
 566     dbcheck(outdb, 29, 1041);
 567
 568     return true;
 569 }