2 * @brief Tests of Database::compact()
4 /* Copyright (C) 2009,2010,2011,2012,2013,2015,2016,2017,2018,2019 Olly Betts
5 * Copyright (C) 2010 Richard Boulton
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License as
9 * published by the Free Software Foundation; either version 2 of the
10 * License, or (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
25 #include "api_compact.h"
31 #include "filetests.h"
32 #include "msvcignoreinvalidparam.h"
34 #include "testsuite.h"
35 #include "testutils.h"
41 #include <sys/types.h>
42 #include "safesysstat.h"
43 #include "safefcntl.h"
44 #include "safeunistd.h"
51 make_sparse_db(Xapian::WritableDatabase
&db
, const string
& s
)
53 // Need non-const pointer for strtoul(), but data isn't modified.
54 char * p
= const_cast<char *>(s
.c_str());
57 bool del
= (*p
== '!');
59 Xapian::docid first
= strtoul(p
, &p
, 10);
60 Xapian::docid last
= first
;
62 last
= strtoul(p
+ 1, &p
, 10);
64 if (*p
&& *p
!= ' ') {
65 tout
<< p
- s
.c_str() << '\n';
66 FAIL_TEST("Bad sparse db spec (expected space): " << s
);
69 FAIL_TEST("Bad sparse db spec (first > last): " << s
);
74 db
.delete_document(first
);
77 string id
= str(first
);
79 doc
.add_term("Q" + str(first
));
80 doc
.add_term(string(first
% 7 + 1, char((first
% 26) + 'a')));
81 db
.replace_document(first
, doc
);
83 } while (first
++ < last
);
85 if (*p
== '\0') break;
93 check_sparse_uid_terms(const string
& path
)
95 Xapian::Database
db(path
);
96 Xapian::TermIterator t
;
97 for (t
= db
.allterms_begin("Q"); t
!= db
.allterms_end("Q"); ++t
) {
98 Xapian::docid did
= atoi((*t
).c_str() + 1);
99 Xapian::PostingIterator p
= db
.postlist_begin(*t
);
104 // With multi the docids in the shards change the behaviour.
105 DEFINE_TESTCASE(compactnorenumber1
, compact
&& !multi
) {
106 string a
= get_database_path("compactnorenumber1a", make_sparse_db
,
107 "5-7 24 76 987 1023-1027 9999 !9999");
110 Xapian::Database
db(a
);
111 a_uuid
= db
.get_uuid();
113 string b
= get_database_path("compactnorenumber1b", make_sparse_db
,
115 string c
= get_database_path("compactnorenumber1c", make_sparse_db
,
117 string d
= get_database_path("compactnorenumber1d", make_sparse_db
,
118 "3000 999999 !999999");
120 string out
= get_compaction_output_path("compactnorenumber1out");
124 Xapian::Database
db(a
);
125 db
.compact(out
, Xapian::DBCOMPACT_NO_RENUMBER
);
128 check_sparse_uid_terms(out
);
131 TEST(!dir_exists(out
+ "/donor"));
132 Xapian::Database
db(out
);
133 // xapian-compact should change the UUID of the database, but didn't
134 // prior to 1.0.18/1.1.4.
135 string out_uuid
= db
.get_uuid();
136 TEST_NOT_EQUAL(a_uuid
, out_uuid
);
137 TEST_EQUAL(out_uuid
.size(), 36);
138 TEST_NOT_EQUAL(out_uuid
, "00000000-0000-0000-0000-000000000000");
140 // White box test - ensure that the donor database is removed.
141 TEST(!dir_exists(out
+ "/donor"));
147 db
.add_database(Xapian::Database(a
));
148 db
.add_database(Xapian::Database(c
));
149 db
.compact(out
, Xapian::DBCOMPACT_NO_RENUMBER
);
151 check_sparse_uid_terms(out
);
153 // Check that xapian-compact is producing a consistent database. Also,
154 // regression test - xapian 1.1.4 set lastdocid to 0 in the output
156 Xapian::Database
outdb(out
);
157 dbcheck(outdb
, 24, 9999);
163 db
.add_database(Xapian::Database(d
));
164 db
.add_database(Xapian::Database(a
));
165 db
.add_database(Xapian::Database(c
));
166 db
.compact(out
, Xapian::DBCOMPACT_NO_RENUMBER
);
168 check_sparse_uid_terms(out
);
173 db
.add_database(Xapian::Database(c
));
174 db
.add_database(Xapian::Database(a
));
175 db
.add_database(Xapian::Database(d
));
176 db
.compact(out
, Xapian::DBCOMPACT_NO_RENUMBER
);
178 check_sparse_uid_terms(out
);
184 db
.add_database(Xapian::Database(a
));
185 db
.add_database(Xapian::Database(b
));
186 TEST_EXCEPTION(Xapian::InvalidOperationError
,
187 db
.compact(out
, Xapian::DBCOMPACT_NO_RENUMBER
)
195 db
.add_database(Xapian::Database(b
));
196 db
.add_database(Xapian::Database(a
));
197 TEST_EXCEPTION(Xapian::InvalidOperationError
,
198 db
.compact(out
, Xapian::DBCOMPACT_NO_RENUMBER
)
206 db
.add_database(Xapian::Database(a
));
207 db
.add_database(Xapian::Database(b
));
208 db
.add_database(Xapian::Database(d
));
209 TEST_EXCEPTION(Xapian::InvalidOperationError
,
210 db
.compact(out
, Xapian::DBCOMPACT_NO_RENUMBER
)
218 db
.add_database(Xapian::Database(d
));
219 db
.add_database(Xapian::Database(b
));
220 db
.add_database(Xapian::Database(a
));
221 TEST_EXCEPTION(Xapian::InvalidOperationError
,
222 db
.compact(out
, Xapian::DBCOMPACT_NO_RENUMBER
)
230 db
.add_database(Xapian::Database(b
));
231 db
.add_database(Xapian::Database(a
));
232 db
.add_database(Xapian::Database(d
));
233 TEST_EXCEPTION(Xapian::InvalidOperationError
,
234 db
.compact(out
, Xapian::DBCOMPACT_NO_RENUMBER
)
239 // Test use of compact to merge two databases.
240 DEFINE_TESTCASE(compactmerge1
, compact
) {
241 string indbpath
= get_database_path("apitest_simpledata");
242 string outdbpath
= get_compaction_output_path("compactmerge1out");
245 bool singlefile
= startswith(get_dbtype(), "singlefile_");
248 db
.add_database(Xapian::Database(indbpath
));
249 db
.add_database(Xapian::Database(indbpath
));
251 db
.compact(outdbpath
, Xapian::DBCOMPACT_SINGLE_FILE
);
253 db
.compact(outdbpath
);
257 Xapian::Database
indb(get_database("apitest_simpledata"));
258 Xapian::Database
outdb(outdbpath
);
260 TEST_EQUAL(indb
.get_doccount() * 2, outdb
.get_doccount());
261 dbcheck(outdb
, outdb
.get_doccount(), outdb
.get_doccount());
264 // Check we actually got a single file out.
265 TEST(file_exists(outdbpath
));
266 TEST_EQUAL(Xapian::Database::check(outdbpath
, 0, &tout
), 0);
267 } else if (indb
.size() > 1) {
268 // Can't check tables for a sharded DB.
269 TEST_EQUAL(Xapian::Database::check(outdbpath
, 0, &tout
), 0);
271 // Check we got a directory out, not a file.
272 TEST(dir_exists(outdbpath
));
273 static const char* const suffixes
[] = {
274 "", "/postlist", "/termlist.", nullptr
276 for (auto s
: suffixes
) {
281 suffix
= "/docdata." + get_dbtype();
284 tout
<< "Trying suffix '" << suffix
<< "'\n";
285 string arg
= outdbpath
;
287 TEST_EQUAL(Xapian::Database::check(arg
, 0, &tout
), 0);
293 make_multichunk_db(Xapian::WritableDatabase
&db
, const string
&)
297 Xapian::Document doc
;
300 db
.add_document(doc
);
307 // Test use of compact on a database which has multiple chunks for a term.
308 // This is a regression test for ticket #427
309 DEFINE_TESTCASE(compactmultichunks1
, compact
) {
310 string indbpath
= get_database_path("compactmultichunks1in",
311 make_multichunk_db
, "");
312 string outdbpath
= get_compaction_output_path("compactmultichunks1out");
316 Xapian::Database
db(indbpath
);
317 db
.compact(outdbpath
);
320 Xapian::Database
indb(indbpath
);
321 Xapian::Database
outdb(outdbpath
);
323 TEST_EQUAL(indb
.get_doccount(), outdb
.get_doccount());
324 dbcheck(outdb
, outdb
.get_doccount(), outdb
.get_doccount());
327 // Test compacting from a stub database directory.
328 DEFINE_TESTCASE(compactstub1
, compact
) {
329 const char * stubpath
= ".stub/compactstub1";
330 const char * stubpathfile
= ".stub/compactstub1/XAPIANDB";
331 mkdir(".stub", 0755);
332 mkdir(stubpath
, 0755);
333 ofstream
stub(stubpathfile
);
334 TEST(stub
.is_open());
335 stub
<< "auto ../../" << get_database_path("apitest_simpledata") << '\n';
336 stub
<< "auto ../../" << get_database_path("apitest_simpledata2") << '\n';
339 string outdbpath
= get_compaction_output_path("compactstub1out");
343 Xapian::Database
db(stubpath
);
344 db
.compact(outdbpath
);
347 Xapian::Database
indb(stubpath
);
348 Xapian::Database
outdb(outdbpath
);
350 TEST_EQUAL(indb
.get_doccount(), outdb
.get_doccount());
351 dbcheck(outdb
, outdb
.get_doccount(), outdb
.get_doccount());
354 // Test compacting from a stub database file.
355 DEFINE_TESTCASE(compactstub2
, compact
) {
356 const char * stubpath
= ".stub/compactstub2";
357 mkdir(".stub", 0755);
358 ofstream
stub(stubpath
);
359 TEST(stub
.is_open());
360 stub
<< "auto ../" << get_database_path("apitest_simpledata") << '\n';
361 stub
<< "auto ../" << get_database_path("apitest_simpledata2") << '\n';
364 string outdbpath
= get_compaction_output_path("compactstub2out");
368 Xapian::Database
db(stubpath
);
369 db
.compact(outdbpath
);
372 Xapian::Database
indb(stubpath
);
373 Xapian::Database
outdb(outdbpath
);
375 TEST_EQUAL(indb
.get_doccount(), outdb
.get_doccount());
376 dbcheck(outdb
, outdb
.get_doccount(), outdb
.get_doccount());
379 // Test compacting a stub database file to itself.
380 DEFINE_TESTCASE(compactstub3
, compact
) {
381 const char * stubpath
= ".stub/compactstub3";
382 mkdir(".stub", 0755);
383 ofstream
stub(stubpath
);
384 TEST(stub
.is_open());
385 stub
<< "auto ../" << get_database_path("apitest_simpledata") << '\n';
386 stub
<< "auto ../" << get_database_path("apitest_simpledata2") << '\n';
389 Xapian::doccount in_docs
;
391 Xapian::Database
indb(stubpath
);
392 in_docs
= indb
.get_doccount();
393 indb
.compact(stubpath
);
396 Xapian::Database
outdb(stubpath
);
398 TEST_EQUAL(in_docs
, outdb
.get_doccount());
399 dbcheck(outdb
, outdb
.get_doccount(), outdb
.get_doccount());
402 // Test compacting a stub database directory to itself.
403 DEFINE_TESTCASE(compactstub4
, compact
) {
404 const char * stubpath
= ".stub/compactstub4";
405 const char * stubpathfile
= ".stub/compactstub4/XAPIANDB";
406 mkdir(".stub", 0755);
407 mkdir(stubpath
, 0755);
408 ofstream
stub(stubpathfile
);
409 TEST(stub
.is_open());
410 stub
<< "auto ../../" << get_database_path("apitest_simpledata") << '\n';
411 stub
<< "auto ../../" << get_database_path("apitest_simpledata2") << '\n';
414 Xapian::doccount in_docs
;
416 Xapian::Database
indb(stubpath
);
417 in_docs
= indb
.get_doccount();
418 indb
.compact(stubpath
);
421 Xapian::Database
outdb(stubpath
);
423 TEST_EQUAL(in_docs
, outdb
.get_doccount());
424 dbcheck(outdb
, outdb
.get_doccount(), outdb
.get_doccount());
428 make_all_tables(Xapian::WritableDatabase
&db
, const string
&)
430 Xapian::Document doc
;
432 db
.add_document(doc
);
433 db
.add_spelling("foo");
434 db
.add_synonym("bar", "pub");
435 db
.add_synonym("foobar", "foo");
441 make_missing_tables(Xapian::WritableDatabase
&db
, const string
&)
443 Xapian::Document doc
;
445 db
.add_document(doc
);
450 DEFINE_TESTCASE(compactmissingtables1
, compact
) {
451 string a
= get_database_path("compactmissingtables1a",
453 string b
= get_database_path("compactmissingtables1b",
454 make_missing_tables
);
456 string out
= get_compaction_output_path("compactmissingtables1out");
461 db
.add_database(Xapian::Database(a
));
462 db
.add_database(Xapian::Database(b
));
467 Xapian::Database
db(out
);
468 TEST_NOT_EQUAL(db
.spellings_begin(), db
.spellings_end());
469 TEST_NOT_EQUAL(db
.synonym_keys_begin(), db
.synonym_keys_end());
470 // FIXME: arrange for input b to not have a termlist table.
471 // TEST_EXCEPTION(Xapian::FeatureUnavailableError, db.termlist_begin(1));
476 make_all_tables2(Xapian::WritableDatabase
&db
, const string
&)
478 Xapian::Document doc
;
480 db
.add_document(doc
);
481 db
.add_spelling("bar");
482 db
.add_synonym("bar", "baa");
483 db
.add_synonym("barfoo", "barbar");
484 db
.add_synonym("foofoo", "barfoo");
489 /// Adds coverage for merging synonym table.
490 DEFINE_TESTCASE(compactmergesynonym1
, compact
) {
491 string a
= get_database_path("compactmergesynonym1a",
493 string b
= get_database_path("compactmergesynonym1b",
496 string out
= get_compaction_output_path("compactmergesynonym1out");
501 db
.add_database(Xapian::Database(a
));
502 db
.add_database(Xapian::Database(b
));
507 Xapian::Database
db(out
);
509 Xapian::TermIterator i
= db
.spellings_begin();
510 TEST_NOT_EQUAL(i
, db
.spellings_end());
511 TEST_EQUAL(*i
, "bar");
513 TEST_NOT_EQUAL(i
, db
.spellings_end());
514 TEST_EQUAL(*i
, "foo");
516 TEST_EQUAL(i
, db
.spellings_end());
518 i
= db
.synonym_keys_begin();
519 TEST_NOT_EQUAL(i
, db
.synonym_keys_end());
520 TEST_EQUAL(*i
, "bar");
522 TEST_NOT_EQUAL(i
, db
.synonym_keys_end());
523 TEST_EQUAL(*i
, "barfoo");
525 TEST_NOT_EQUAL(i
, db
.synonym_keys_end());
526 TEST_EQUAL(*i
, "foobar");
528 TEST_NOT_EQUAL(i
, db
.synonym_keys_end());
529 TEST_EQUAL(*i
, "foofoo");
531 TEST_EQUAL(i
, db
.synonym_keys_end());
535 DEFINE_TESTCASE(compactempty1
, compact
) {
536 string empty_dbpath
= get_database_path(string());
537 string outdbpath
= get_compaction_output_path("compactempty1out");
541 // Compacting an empty database tried to divide by zero in 1.3.0.
543 db
.add_database(Xapian::Database(empty_dbpath
));
544 db
.compact(outdbpath
);
546 Xapian::Database
outdb(outdbpath
);
547 TEST_EQUAL(outdb
.get_doccount(), 0);
548 dbcheck(outdb
, 0, 0);
552 // Check compacting two empty databases together.
554 db
.add_database(Xapian::Database(empty_dbpath
));
555 db
.add_database(Xapian::Database(empty_dbpath
));
556 db
.compact(outdbpath
);
558 Xapian::Database
outdb(outdbpath
);
559 TEST_EQUAL(outdb
.get_doccount(), 0);
560 dbcheck(outdb
, 0, 0);
564 DEFINE_TESTCASE(compactmultipass1
, compact
) {
565 string outdbpath
= get_compaction_output_path("compactmultipass1");
568 string a
= get_database_path("compactnorenumber1a", make_sparse_db
,
569 "5-7 24 76 987 1023-1027 9999 !9999");
570 string b
= get_database_path("compactnorenumber1b", make_sparse_db
,
572 string c
= get_database_path("compactnorenumber1c", make_sparse_db
,
574 string d
= get_database_path("compactnorenumber1d", make_sparse_db
,
575 "3000 999999 !999999");
579 db
.add_database(Xapian::Database(a
));
580 db
.add_database(Xapian::Database(b
));
581 db
.add_database(Xapian::Database(c
));
582 db
.add_database(Xapian::Database(d
));
583 db
.compact(outdbpath
, Xapian::DBCOMPACT_MULTIPASS
);
586 Xapian::Database
outdb(outdbpath
);
587 dbcheck(outdb
, 29, 1041);
590 // Test compacting to an fd.
591 DEFINE_TESTCASE(compacttofd1
, compact
) {
592 Xapian::Database
indb(get_database("apitest_simpledata"));
593 string outdbpath
= get_compaction_output_path("compacttofd1out");
596 int fd
= open(outdbpath
.c_str(), O_CREAT
|O_RDWR
|O_BINARY
, 0666);
600 // Confirm that the fd was closed by Xapian. Set errno first to workaround
601 // a bug in Wine's msvcrt.dll which fails to set errno in this case:
602 // https://bugs.winehq.org/show_bug.cgi?id=43902
605 MSVCIgnoreInvalidParameter invalid_fd_in_close_is_expected
;
606 TEST(close(fd
) == -1);
607 TEST_EQUAL(errno
, EBADF
);
610 Xapian::Database
outdb(outdbpath
);
612 TEST_EQUAL(indb
.get_doccount(), outdb
.get_doccount());
613 dbcheck(outdb
, outdb
.get_doccount(), outdb
.get_doccount());
616 // Test compacting to an fd at at offset.
617 DEFINE_TESTCASE(compacttofd2
, compact
) {
618 Xapian::Database
indb(get_database("apitest_simpledata"));
619 string outdbpath
= get_compaction_output_path("compacttofd2out");
622 int fd
= open(outdbpath
.c_str(), O_CREAT
|O_RDWR
|O_BINARY
, 0666);
624 TEST(lseek(fd
, 8192, SEEK_SET
) == 8192);
627 // Confirm that the fd was closed by Xapian. Set errno first to workaround
628 // a bug in Wine's msvcrt.dll which fails to set errno in this case:
629 // https://bugs.winehq.org/show_bug.cgi?id=43902
632 MSVCIgnoreInvalidParameter invalid_fd_in_close_is_expected
;
633 TEST(close(fd
) == -1);
634 TEST_EQUAL(errno
, EBADF
);
637 fd
= open(outdbpath
.c_str(), O_RDONLY
|O_BINARY
);
640 // Test that the database wasn't just written to the start of the file.
642 size_t n
= sizeof(buf
);
644 ssize_t c
= read(fd
, buf
, n
);
646 for (const char * p
= buf
; p
!= buf
+ c
; ++p
) {
652 TEST(lseek(fd
, 8192, SEEK_SET
) == 8192);
653 Xapian::Database
outdb(fd
);
655 TEST_EQUAL(indb
.get_doccount(), outdb
.get_doccount());
656 dbcheck(outdb
, outdb
.get_doccount(), outdb
.get_doccount());
659 // Regression test for bug fixed in 1.3.5. If you compact a WritableDatabase
660 // with uncommitted changes, you get an inconsistent output.
661 DEFINE_TESTCASE(compactsingle1
, compact
&& writable
) {
662 Xapian::WritableDatabase db
= get_writable_database();
663 Xapian::Document doc
;
667 db
.add_document(doc
);
668 // Include a zero-length document as a regression test for a
669 // Database::check() bug fixed in 1.4.7 (and introduced in 1.4.6). Test it
670 // here so we also have test coverage for compaction for such a document.
671 Xapian::Document doc2
;
672 doc2
.add_boolean_term("Kfoo");
673 db
.add_document(doc2
);
674 // Also test a completely empty document.
675 db
.add_document(Xapian::Document());
677 string output
= get_compaction_output_path("compactsingle1-out");
678 // In 1.3.4, we would hang if the output file already existed, so check
682 TEST_EXCEPTION(Xapian::InvalidOperationError
,
683 db
.compact(output
, Xapian::DBCOMPACT_SINGLE_FILE
));
685 // Check the file wasn't removed by the failed attempt.
686 TEST(file_exists(output
));
689 db
.compact(output
, Xapian::DBCOMPACT_SINGLE_FILE
);
692 TEST_EQUAL(Xapian::Database::check(output
, 0, &tout
), 0);
694 TEST_EQUAL(Xapian::Database(output
).get_doccount(), 3);
697 // Regression test for bug fixed in 1.4.6. Same as above, except not with
698 // a single file database!
699 DEFINE_TESTCASE(compact1
, compact
&& writable
) {
700 Xapian::WritableDatabase db
= get_writable_database();
701 Xapian::Document doc
;
705 db
.add_document(doc
);
706 // Include a zero-length document as a regression test for a
707 // Database::check() bug fixed in 1.4.7 (and introduced in 1.4.6). Test it
708 // here so we also have test coverage for compaction for such a document.
709 Xapian::Document doc2
;
710 doc2
.add_boolean_term("Kfoo");
711 db
.add_document(doc2
);
712 // Also test a completely empty document.
713 db
.add_document(Xapian::Document());
715 string output
= get_compaction_output_path("compact1-out");
718 TEST_EXCEPTION(Xapian::InvalidOperationError
,
725 TEST_EQUAL(Xapian::Database::check(output
, 0, &tout
), 0);
727 TEST_EQUAL(Xapian::Database(output
).get_doccount(), 3);
730 // Regression test for compacting honey databases.
731 DEFINE_TESTCASE(compact2
, compact
) {
732 Xapian::Database db
= get_database("compact2",
733 [](Xapian::WritableDatabase
& wdb
,
735 Xapian::Document doc
;
736 doc
.add_term("test");
737 wdb
.add_document(doc
);
738 doc
.add_term("test");
739 for (int i
= 1; i
< 4000; ++i
) {
740 wdb
.add_document(doc
);
744 db
.add_database(get_database("apitest_simpledata"));
745 auto db_size
= db
.get_doccount();
747 string output
= get_compaction_output_path("compact2-out");
753 TEST_EQUAL(Xapian::Database::check(output
, 0, &tout
), 0);
755 TEST_EQUAL(Xapian::Database(output
).get_doccount(), db_size
);