1 /** @file api_compact.cc
2 * @brief Tests of Database::compact()
4 /* Copyright (C) 2009,2010,2011,2012,2013,2015,2016,2017,2018 Olly Betts
5 * Copyright (C) 2010 Richard Boulton
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License as
9 * published by the Free Software Foundation; either version 2 of the
10 * License, or (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
25 #include "api_compact.h"
31 #include "filetests.h"
32 #include "msvcignoreinvalidparam.h"
34 #include "testsuite.h"
35 #include "testutils.h"
40 #include <sys/types.h>
41 #include "safesysstat.h"
42 #include "safefcntl.h"
43 #include "safeunistd.h"
50 make_sparse_db(Xapian::WritableDatabase
&db
, const string
& s
)
52 // Need non-const pointer for strtoul(), but data isn't modified.
53 char * p
= const_cast<char *>(s
.c_str());
56 bool del
= (*p
== '!');
58 Xapian::docid first
= strtoul(p
, &p
, 10);
59 Xapian::docid last
= first
;
61 last
= strtoul(p
+ 1, &p
, 10);
63 if (*p
&& *p
!= ' ') {
64 tout
<< p
- s
.c_str() << endl
;
65 FAIL_TEST("Bad sparse db spec (expected space): " << s
);
68 FAIL_TEST("Bad sparse db spec (first > last): " << s
);
73 db
.delete_document(first
);
76 string id
= str(first
);
78 doc
.add_term("Q" + str(first
));
79 doc
.add_term(string(first
% 7 + 1, char((first
% 26) + 'a')));
80 db
.replace_document(first
, doc
);
82 } while (first
++ < last
);
84 if (*p
== '\0') break;
92 check_sparse_uid_terms(const string
& path
)
94 Xapian::Database
db(path
);
95 Xapian::TermIterator t
;
96 for (t
= db
.allterms_begin("Q"); t
!= db
.allterms_end("Q"); ++t
) {
97 Xapian::docid did
= atoi((*t
).c_str() + 1);
98 Xapian::PostingIterator p
= db
.postlist_begin(*t
);
103 DEFINE_TESTCASE(compactnorenumber1
, compact
&& generated
) {
104 string a
= get_database_path("compactnorenumber1a", make_sparse_db
,
105 "5-7 24 76 987 1023-1027 9999 !9999");
108 Xapian::Database
db(a
);
109 a_uuid
= db
.get_uuid();
111 string b
= get_database_path("compactnorenumber1b", make_sparse_db
,
113 string c
= get_database_path("compactnorenumber1c", make_sparse_db
,
115 string d
= get_database_path("compactnorenumber1d", make_sparse_db
,
116 "3000 999999 !999999");
118 string out
= get_named_writable_database_path("compactnorenumber1out");
122 Xapian::Database
db(a
);
123 db
.compact(out
, Xapian::DBCOMPACT_NO_RENUMBER
);
126 check_sparse_uid_terms(out
);
129 TEST(!dir_exists(out
+ "/donor"));
130 Xapian::Database
db(out
);
131 // xapian-compact should change the UUID of the database, but didn't
132 // prior to 1.0.18/1.1.4.
133 string out_uuid
= db
.get_uuid();
134 TEST_NOT_EQUAL(a_uuid
, out_uuid
);
135 TEST_EQUAL(out_uuid
.size(), 36);
136 TEST_NOT_EQUAL(out_uuid
, "00000000-0000-0000-0000-000000000000");
138 // White box test - ensure that the donor database is removed.
139 TEST(!dir_exists(out
+ "/donor"));
145 db
.add_database(Xapian::Database(a
));
146 db
.add_database(Xapian::Database(c
));
147 db
.compact(out
, Xapian::DBCOMPACT_NO_RENUMBER
);
149 check_sparse_uid_terms(out
);
151 // Check that xapian-compact is producing a consistent database. Also,
152 // regression test - xapian 1.1.4 set lastdocid to 0 in the output
154 Xapian::Database
outdb(out
);
155 dbcheck(outdb
, 24, 9999);
161 db
.add_database(Xapian::Database(d
));
162 db
.add_database(Xapian::Database(a
));
163 db
.add_database(Xapian::Database(c
));
164 db
.compact(out
, Xapian::DBCOMPACT_NO_RENUMBER
);
166 check_sparse_uid_terms(out
);
171 db
.add_database(Xapian::Database(c
));
172 db
.add_database(Xapian::Database(a
));
173 db
.add_database(Xapian::Database(d
));
174 db
.compact(out
, Xapian::DBCOMPACT_NO_RENUMBER
);
176 check_sparse_uid_terms(out
);
182 db
.add_database(Xapian::Database(a
));
183 db
.add_database(Xapian::Database(b
));
184 TEST_EXCEPTION(Xapian::InvalidOperationError
,
185 db
.compact(out
, Xapian::DBCOMPACT_NO_RENUMBER
)
193 db
.add_database(Xapian::Database(b
));
194 db
.add_database(Xapian::Database(a
));
195 TEST_EXCEPTION(Xapian::InvalidOperationError
,
196 db
.compact(out
, Xapian::DBCOMPACT_NO_RENUMBER
)
204 db
.add_database(Xapian::Database(a
));
205 db
.add_database(Xapian::Database(b
));
206 db
.add_database(Xapian::Database(d
));
207 TEST_EXCEPTION(Xapian::InvalidOperationError
,
208 db
.compact(out
, Xapian::DBCOMPACT_NO_RENUMBER
)
216 db
.add_database(Xapian::Database(d
));
217 db
.add_database(Xapian::Database(b
));
218 db
.add_database(Xapian::Database(a
));
219 TEST_EXCEPTION(Xapian::InvalidOperationError
,
220 db
.compact(out
, Xapian::DBCOMPACT_NO_RENUMBER
)
228 db
.add_database(Xapian::Database(b
));
229 db
.add_database(Xapian::Database(a
));
230 db
.add_database(Xapian::Database(d
));
231 TEST_EXCEPTION(Xapian::InvalidOperationError
,
232 db
.compact(out
, Xapian::DBCOMPACT_NO_RENUMBER
)
239 // Test use of compact to merge two databases.
240 DEFINE_TESTCASE(compactmerge1
, compact
) {
241 string indbpath
= get_database_path("apitest_simpledata");
242 string outdbpath
= get_named_writable_database_path("compactmerge1out");
247 db
.add_database(Xapian::Database(indbpath
));
248 db
.add_database(Xapian::Database(indbpath
));
249 db
.compact(outdbpath
);
252 Xapian::Database
indb(get_database("apitest_simpledata"));
253 Xapian::Database
outdb(outdbpath
);
255 TEST_EQUAL(indb
.get_doccount() * 2, outdb
.get_doccount());
256 dbcheck(outdb
, outdb
.get_doccount(), outdb
.get_doccount());
258 if (file_exists(outdbpath
)) {
260 TEST_EQUAL(Xapian::Database::check(outdbpath
, 0, &tout
), 0);
262 static const char* const suffixes
[] = {
263 "", "/postlist", "/termlist.", nullptr
265 for (auto s
: suffixes
) {
270 suffix
= "/docdata." + get_dbtype();
273 tout
<< "Trying suffix '" << suffix
<< "'" << endl
;
274 string arg
= outdbpath
;
276 TEST_EQUAL(Xapian::Database::check(arg
, 0, &tout
), 0);
284 make_multichunk_db(Xapian::WritableDatabase
&db
, const string
&)
288 Xapian::Document doc
;
291 db
.add_document(doc
);
298 // Test use of compact on a database which has multiple chunks for a term.
299 // This is a regression test for ticket #427
300 DEFINE_TESTCASE(compactmultichunks1
, compact
&& generated
) {
301 string indbpath
= get_database_path("compactmultichunks1in",
302 make_multichunk_db
, "");
303 string outdbpath
= get_named_writable_database_path("compactmultichunks1out");
307 Xapian::Database
db(indbpath
);
308 db
.compact(outdbpath
);
311 Xapian::Database
indb(indbpath
);
312 Xapian::Database
outdb(outdbpath
);
314 TEST_EQUAL(indb
.get_doccount(), outdb
.get_doccount());
315 dbcheck(outdb
, outdb
.get_doccount(), outdb
.get_doccount());
320 // Test compacting from a stub database directory.
321 DEFINE_TESTCASE(compactstub1
, compact
) {
322 const char * stubpath
= ".stub/compactstub1";
323 const char * stubpathfile
= ".stub/compactstub1/XAPIANDB";
324 mkdir(".stub", 0755);
325 mkdir(stubpath
, 0755);
326 ofstream
stub(stubpathfile
);
327 TEST(stub
.is_open());
328 stub
<< "auto ../../" << get_database_path("apitest_simpledata") << endl
;
329 stub
<< "auto ../../" << get_database_path("apitest_simpledata2") << endl
;
332 string outdbpath
= get_named_writable_database_path("compactstub1out");
336 Xapian::Database
db(stubpath
);
337 db
.compact(outdbpath
);
340 Xapian::Database
indb(stubpath
);
341 Xapian::Database
outdb(outdbpath
);
343 TEST_EQUAL(indb
.get_doccount(), outdb
.get_doccount());
344 dbcheck(outdb
, outdb
.get_doccount(), outdb
.get_doccount());
349 // Test compacting from a stub database file.
350 DEFINE_TESTCASE(compactstub2
, compact
) {
351 const char * stubpath
= ".stub/compactstub2";
352 mkdir(".stub", 0755);
353 ofstream
stub(stubpath
);
354 TEST(stub
.is_open());
355 stub
<< "auto ../" << get_database_path("apitest_simpledata") << endl
;
356 stub
<< "auto ../" << get_database_path("apitest_simpledata2") << endl
;
359 string outdbpath
= get_named_writable_database_path("compactstub2out");
363 Xapian::Database
db(stubpath
);
364 db
.compact(outdbpath
);
367 Xapian::Database
indb(stubpath
);
368 Xapian::Database
outdb(outdbpath
);
370 TEST_EQUAL(indb
.get_doccount(), outdb
.get_doccount());
371 dbcheck(outdb
, outdb
.get_doccount(), outdb
.get_doccount());
376 // Test compacting a stub database file to itself.
377 DEFINE_TESTCASE(compactstub3
, compact
) {
378 const char * stubpath
= ".stub/compactstub3";
379 mkdir(".stub", 0755);
380 ofstream
stub(stubpath
);
381 TEST(stub
.is_open());
382 stub
<< "auto ../" << get_database_path("apitest_simpledata") << endl
;
383 stub
<< "auto ../" << get_database_path("apitest_simpledata2") << endl
;
386 Xapian::doccount in_docs
;
388 Xapian::Database
indb(stubpath
);
389 in_docs
= indb
.get_doccount();
390 indb
.compact(stubpath
);
393 Xapian::Database
outdb(stubpath
);
395 TEST_EQUAL(in_docs
, outdb
.get_doccount());
396 dbcheck(outdb
, outdb
.get_doccount(), outdb
.get_doccount());
401 // Test compacting a stub database directory to itself.
402 DEFINE_TESTCASE(compactstub4
, compact
) {
403 const char * stubpath
= ".stub/compactstub4";
404 const char * stubpathfile
= ".stub/compactstub4/XAPIANDB";
405 mkdir(".stub", 0755);
406 mkdir(stubpath
, 0755);
407 ofstream
stub(stubpathfile
);
408 TEST(stub
.is_open());
409 stub
<< "auto ../../" << get_database_path("apitest_simpledata") << endl
;
410 stub
<< "auto ../../" << get_database_path("apitest_simpledata2") << endl
;
413 Xapian::doccount in_docs
;
415 Xapian::Database
indb(stubpath
);
416 in_docs
= indb
.get_doccount();
417 indb
.compact(stubpath
);
420 Xapian::Database
outdb(stubpath
);
422 TEST_EQUAL(in_docs
, outdb
.get_doccount());
423 dbcheck(outdb
, outdb
.get_doccount(), outdb
.get_doccount());
429 make_all_tables(Xapian::WritableDatabase
&db
, const string
&)
431 Xapian::Document doc
;
433 db
.add_document(doc
);
434 db
.add_spelling("foo");
435 db
.add_synonym("bar", "pub");
436 db
.add_synonym("foobar", "foo");
442 make_missing_tables(Xapian::WritableDatabase
&db
, const string
&)
444 Xapian::Document doc
;
446 db
.add_document(doc
);
451 DEFINE_TESTCASE(compactmissingtables1
, compact
&& generated
) {
452 string a
= get_database_path("compactmissingtables1a",
454 string b
= get_database_path("compactmissingtables1b",
455 make_missing_tables
);
457 string out
= get_named_writable_database_path("compactmissingtables1out");
462 db
.add_database(Xapian::Database(a
));
463 db
.add_database(Xapian::Database(b
));
468 Xapian::Database
db(out
);
469 TEST_NOT_EQUAL(db
.spellings_begin(), db
.spellings_end());
470 TEST_NOT_EQUAL(db
.synonym_keys_begin(), db
.synonym_keys_end());
471 // FIXME: arrange for input b to not have a termlist table.
472 // TEST_EXCEPTION(Xapian::FeatureUnavailableError, db.termlist_begin(1));
479 make_all_tables2(Xapian::WritableDatabase
&db
, const string
&)
481 Xapian::Document doc
;
483 db
.add_document(doc
);
484 db
.add_spelling("bar");
485 db
.add_synonym("bar", "baa");
486 db
.add_synonym("barfoo", "barbar");
487 db
.add_synonym("foofoo", "barfoo");
492 /// Adds coverage for merging synonym table.
493 DEFINE_TESTCASE(compactmergesynonym1
, compact
&& generated
) {
494 string a
= get_database_path("compactmergesynonym1a",
496 string b
= get_database_path("compactmergesynonym1b",
499 string out
= get_named_writable_database_path("compactmergesynonym1out");
504 db
.add_database(Xapian::Database(a
));
505 db
.add_database(Xapian::Database(b
));
510 Xapian::Database
db(out
);
512 Xapian::TermIterator i
= db
.spellings_begin();
513 TEST_NOT_EQUAL(i
, db
.spellings_end());
514 TEST_EQUAL(*i
, "bar");
516 TEST_NOT_EQUAL(i
, db
.spellings_end());
517 TEST_EQUAL(*i
, "foo");
519 TEST_EQUAL(i
, db
.spellings_end());
521 i
= db
.synonym_keys_begin();
522 TEST_NOT_EQUAL(i
, db
.synonym_keys_end());
523 TEST_EQUAL(*i
, "bar");
525 TEST_NOT_EQUAL(i
, db
.synonym_keys_end());
526 TEST_EQUAL(*i
, "barfoo");
528 TEST_NOT_EQUAL(i
, db
.synonym_keys_end());
529 TEST_EQUAL(*i
, "foobar");
531 TEST_NOT_EQUAL(i
, db
.synonym_keys_end());
532 TEST_EQUAL(*i
, "foofoo");
534 TEST_EQUAL(i
, db
.synonym_keys_end());
540 DEFINE_TESTCASE(compactempty1
, compact
) {
541 string empty_dbpath
= get_database_path(string());
542 string outdbpath
= get_named_writable_database_path("compactempty1out");
546 // Compacting an empty database tried to divide by zero in 1.3.0.
548 db
.add_database(Xapian::Database(empty_dbpath
));
549 db
.compact(outdbpath
);
551 Xapian::Database
outdb(outdbpath
);
552 TEST_EQUAL(outdb
.get_doccount(), 0);
553 dbcheck(outdb
, 0, 0);
557 // Check compacting two empty databases together.
559 db
.add_database(Xapian::Database(empty_dbpath
));
560 db
.add_database(Xapian::Database(empty_dbpath
));
561 db
.compact(outdbpath
);
563 Xapian::Database
outdb(outdbpath
);
564 TEST_EQUAL(outdb
.get_doccount(), 0);
565 dbcheck(outdb
, 0, 0);
571 DEFINE_TESTCASE(compactmultipass1
, compact
&& generated
) {
572 string outdbpath
= get_named_writable_database_path("compactmultipass1");
575 string a
= get_database_path("compactnorenumber1a", make_sparse_db
,
576 "5-7 24 76 987 1023-1027 9999 !9999");
577 string b
= get_database_path("compactnorenumber1b", make_sparse_db
,
579 string c
= get_database_path("compactnorenumber1c", make_sparse_db
,
581 string d
= get_database_path("compactnorenumber1d", make_sparse_db
,
582 "3000 999999 !999999");
586 db
.add_database(Xapian::Database(a
));
587 db
.add_database(Xapian::Database(b
));
588 db
.add_database(Xapian::Database(c
));
589 db
.add_database(Xapian::Database(d
));
590 db
.compact(outdbpath
, Xapian::DBCOMPACT_MULTIPASS
);
593 Xapian::Database
outdb(outdbpath
);
594 dbcheck(outdb
, 29, 1041);
599 // Test compacting to an fd.
600 DEFINE_TESTCASE(compacttofd1
, compact
) {
601 Xapian::Database
indb(get_database("apitest_simpledata"));
602 string outdbpath
= get_named_writable_database_path("compacttofd1out");
605 int fd
= open(outdbpath
.c_str(), O_CREAT
|O_RDWR
|O_BINARY
, 0666);
609 // Confirm that the fd was closed by Xapian. Set errno first to workaround
610 // a bug in Wine's msvcrt.dll which fails to set errno in this case:
611 // https://bugs.winehq.org/show_bug.cgi?id=43902
614 MSVCIgnoreInvalidParameter invalid_fd_in_close_is_expected
;
615 TEST(close(fd
) == -1);
616 TEST_EQUAL(errno
, EBADF
);
619 Xapian::Database
outdb(outdbpath
);
621 TEST_EQUAL(indb
.get_doccount(), outdb
.get_doccount());
622 dbcheck(outdb
, outdb
.get_doccount(), outdb
.get_doccount());
627 // Test compacting to an fd at at offset.
628 DEFINE_TESTCASE(compacttofd2
, compact
) {
629 Xapian::Database
indb(get_database("apitest_simpledata"));
630 string outdbpath
= get_named_writable_database_path("compacttofd2out");
633 int fd
= open(outdbpath
.c_str(), O_CREAT
|O_RDWR
|O_BINARY
, 0666);
635 TEST(lseek(fd
, 8192, SEEK_SET
) == 8192);
638 // Confirm that the fd was closed by Xapian. Set errno first to workaround
639 // a bug in Wine's msvcrt.dll which fails to set errno in this case:
640 // https://bugs.winehq.org/show_bug.cgi?id=43902
643 MSVCIgnoreInvalidParameter invalid_fd_in_close_is_expected
;
644 TEST(close(fd
) == -1);
645 TEST_EQUAL(errno
, EBADF
);
648 fd
= open(outdbpath
.c_str(), O_RDONLY
|O_BINARY
, 0666);
651 // Test that the database wasn't just written to the start of the file.
653 size_t n
= sizeof(buf
);
655 ssize_t c
= read(fd
, buf
, n
);
657 for (const char * p
= buf
; p
!= buf
+ c
; ++p
) {
663 TEST(lseek(fd
, 8192, SEEK_SET
) == 8192);
664 Xapian::Database
outdb(fd
);
666 TEST_EQUAL(indb
.get_doccount(), outdb
.get_doccount());
667 dbcheck(outdb
, outdb
.get_doccount(), outdb
.get_doccount());
672 // Regression test for bug fixed in 1.3.5. If you compact a WritableDatabase
673 // with uncommitted changes, you get an inconsistent output.
674 DEFINE_TESTCASE(compactsingle1
, compact
) {
675 Xapian::WritableDatabase db
= get_writable_database();
676 Xapian::Document doc
;
680 db
.add_document(doc
);
682 string output
= get_named_writable_database_path("compactsingle1-out");
683 // In 1.3.4, we would hang if the output file already existed, so check
687 TEST_EXCEPTION(Xapian::InvalidOperationError
,
688 db
.compact(output
, Xapian::DBCOMPACT_SINGLE_FILE
));
690 // Check the file wasn't removed by the failed attempt.
691 TEST(file_exists(output
));
694 db
.compact(output
, Xapian::DBCOMPACT_SINGLE_FILE
);
697 TEST_EQUAL(Xapian::Database::check(output
, 0, &tout
), 0);
702 // Regression test for bug fixed in 1.4.6. Same as above, except not with
703 // a single file database!
704 DEFINE_TESTCASE(compact1
, compact
) {
705 Xapian::WritableDatabase db
= get_writable_database();
706 Xapian::Document doc
;
710 db
.add_document(doc
);
712 string output
= get_named_writable_database_path("compact1-out");
715 TEST_EXCEPTION(Xapian::InvalidOperationError
,
722 TEST_EQUAL(Xapian::Database::check(output
, 0, &tout
), 0);