1 /** @file api_compact.cc
2 * @brief Tests of Database::compact()
4 /* Copyright (C) 2009,2010,2011,2012,2013,2015,2016,2017 Olly Betts
5 * Copyright (C) 2010 Richard Boulton
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License as
9 * published by the Free Software Foundation; either version 2 of the
10 * License, or (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
25 #include "api_compact.h"
31 #include "filetests.h"
33 #include "testsuite.h"
34 #include "testutils.h"
39 #include <sys/types.h>
40 #include "safesysstat.h"
41 #include "safefcntl.h"
42 #include "safeunistd.h"
49 make_sparse_db(Xapian::WritableDatabase
&db
, const string
& s
)
51 // Need non-const pointer for strtoul(), but data isn't modified.
52 char * p
= const_cast<char *>(s
.c_str());
55 bool del
= (*p
== '!');
57 Xapian::docid first
= strtoul(p
, &p
, 10);
58 Xapian::docid last
= first
;
60 last
= strtoul(p
+ 1, &p
, 10);
62 if (*p
&& *p
!= ' ') {
63 tout
<< p
- s
.c_str() << endl
;
64 FAIL_TEST("Bad sparse db spec (expected space): " << s
);
67 FAIL_TEST("Bad sparse db spec (first > last): " << s
);
72 db
.delete_document(first
);
75 string id
= str(first
);
77 doc
.add_term("Q" + str(first
));
78 doc
.add_term(string(first
% 7 + 1, char((first
% 26) + 'a')));
79 db
.replace_document(first
, doc
);
81 } while (first
++ < last
);
83 if (*p
== '\0') break;
91 check_sparse_uid_terms(const string
& path
)
93 Xapian::Database
db(path
);
94 Xapian::TermIterator t
;
95 for (t
= db
.allterms_begin("Q"); t
!= db
.allterms_end("Q"); ++t
) {
96 Xapian::docid did
= atoi((*t
).c_str() + 1);
97 Xapian::PostingIterator p
= db
.postlist_begin(*t
);
102 DEFINE_TESTCASE(compactnorenumber1
, generated
) {
103 string a
= get_database_path("compactnorenumber1a", make_sparse_db
,
104 "5-7 24 76 987 1023-1027 9999 !9999");
107 Xapian::Database
db(a
);
108 a_uuid
= db
.get_uuid();
110 string b
= get_database_path("compactnorenumber1b", make_sparse_db
,
112 string c
= get_database_path("compactnorenumber1c", make_sparse_db
,
114 string d
= get_database_path("compactnorenumber1d", make_sparse_db
,
115 "3000 999999 !999999");
117 string out
= get_named_writable_database_path("compactnorenumber1out");
121 Xapian::Database
db(a
);
122 db
.compact(out
, Xapian::DBCOMPACT_NO_RENUMBER
);
125 check_sparse_uid_terms(out
);
128 TEST(!dir_exists(out
+ "/donor"));
129 Xapian::Database
db(out
);
130 // xapian-compact should change the UUID of the database, but didn't
131 // prior to 1.0.18/1.1.4.
132 string out_uuid
= db
.get_uuid();
133 TEST_NOT_EQUAL(a_uuid
, out_uuid
);
134 TEST_EQUAL(out_uuid
.size(), 36);
135 TEST_NOT_EQUAL(out_uuid
, "00000000-0000-0000-0000-000000000000");
137 // White box test - ensure that the donor database is removed.
138 TEST(!dir_exists(out
+ "/donor"));
144 db
.add_database(Xapian::Database(a
));
145 db
.add_database(Xapian::Database(c
));
146 db
.compact(out
, Xapian::DBCOMPACT_NO_RENUMBER
);
148 check_sparse_uid_terms(out
);
150 // Check that xapian-compact is producing a consistent database. Also,
151 // regression test - xapian 1.1.4 set lastdocid to 0 in the output
153 Xapian::Database
outdb(out
);
154 dbcheck(outdb
, 24, 9999);
160 db
.add_database(Xapian::Database(d
));
161 db
.add_database(Xapian::Database(a
));
162 db
.add_database(Xapian::Database(c
));
163 db
.compact(out
, Xapian::DBCOMPACT_NO_RENUMBER
);
165 check_sparse_uid_terms(out
);
170 db
.add_database(Xapian::Database(c
));
171 db
.add_database(Xapian::Database(a
));
172 db
.add_database(Xapian::Database(d
));
173 db
.compact(out
, Xapian::DBCOMPACT_NO_RENUMBER
);
175 check_sparse_uid_terms(out
);
181 db
.add_database(Xapian::Database(a
));
182 db
.add_database(Xapian::Database(b
));
183 TEST_EXCEPTION(Xapian::InvalidOperationError
,
184 db
.compact(out
, Xapian::DBCOMPACT_NO_RENUMBER
)
192 db
.add_database(Xapian::Database(b
));
193 db
.add_database(Xapian::Database(a
));
194 TEST_EXCEPTION(Xapian::InvalidOperationError
,
195 db
.compact(out
, Xapian::DBCOMPACT_NO_RENUMBER
)
203 db
.add_database(Xapian::Database(a
));
204 db
.add_database(Xapian::Database(b
));
205 db
.add_database(Xapian::Database(d
));
206 TEST_EXCEPTION(Xapian::InvalidOperationError
,
207 db
.compact(out
, Xapian::DBCOMPACT_NO_RENUMBER
)
215 db
.add_database(Xapian::Database(d
));
216 db
.add_database(Xapian::Database(b
));
217 db
.add_database(Xapian::Database(a
));
218 TEST_EXCEPTION(Xapian::InvalidOperationError
,
219 db
.compact(out
, Xapian::DBCOMPACT_NO_RENUMBER
)
227 db
.add_database(Xapian::Database(b
));
228 db
.add_database(Xapian::Database(a
));
229 db
.add_database(Xapian::Database(d
));
230 TEST_EXCEPTION(Xapian::InvalidOperationError
,
231 db
.compact(out
, Xapian::DBCOMPACT_NO_RENUMBER
)
238 // Test use of compact to merge two databases.
239 DEFINE_TESTCASE(compactmerge1
, glass
) {
240 string indbpath
= get_database_path("apitest_simpledata");
241 string outdbpath
= get_named_writable_database_path("compactmerge1out");
246 db
.add_database(Xapian::Database(indbpath
));
247 db
.add_database(Xapian::Database(indbpath
));
248 db
.compact(outdbpath
);
251 Xapian::Database
indb(get_database("apitest_simpledata"));
252 Xapian::Database
outdb(outdbpath
);
254 TEST_EQUAL(indb
.get_doccount() * 2, outdb
.get_doccount());
255 dbcheck(outdb
, outdb
.get_doccount(), outdb
.get_doccount());
257 for (const char * suffix
:
258 { "", "/postlist", "/termlist.", "/docdata.glass" }) {
260 tout
<< "Trying suffix '" << suffix
<< "'" << endl
;
261 string arg
= outdbpath
;
263 TEST_EQUAL(Xapian::Database::check(arg
, 0, &tout
), 0);
270 make_multichunk_db(Xapian::WritableDatabase
&db
, const string
&)
274 Xapian::Document doc
;
277 db
.add_document(doc
);
284 // Test use of compact on a database which has multiple chunks for a term.
285 // This is a regression test for ticket #427
286 DEFINE_TESTCASE(compactmultichunks1
, generated
) {
287 string indbpath
= get_database_path("compactmultichunks1in",
288 make_multichunk_db
, "");
289 string outdbpath
= get_named_writable_database_path("compactmultichunks1out");
293 Xapian::Database
db(indbpath
);
294 db
.compact(outdbpath
);
297 Xapian::Database
indb(indbpath
);
298 Xapian::Database
outdb(outdbpath
);
300 TEST_EQUAL(indb
.get_doccount(), outdb
.get_doccount());
301 dbcheck(outdb
, outdb
.get_doccount(), outdb
.get_doccount());
306 // Test compacting from a stub database directory.
307 DEFINE_TESTCASE(compactstub1
, glass
) {
308 const char * stubpath
= ".stub/compactstub1";
309 const char * stubpathfile
= ".stub/compactstub1/XAPIANDB";
310 mkdir(".stub", 0755);
311 mkdir(stubpath
, 0755);
312 ofstream
stub(stubpathfile
);
313 TEST(stub
.is_open());
314 stub
<< "auto ../../" << get_database_path("apitest_simpledata") << endl
;
315 stub
<< "auto ../../" << get_database_path("apitest_simpledata2") << endl
;
318 string outdbpath
= get_named_writable_database_path("compactstub1out");
322 Xapian::Database
db(stubpath
);
323 db
.compact(outdbpath
);
326 Xapian::Database
indb(stubpath
);
327 Xapian::Database
outdb(outdbpath
);
329 TEST_EQUAL(indb
.get_doccount(), outdb
.get_doccount());
330 dbcheck(outdb
, outdb
.get_doccount(), outdb
.get_doccount());
335 // Test compacting from a stub database file.
336 DEFINE_TESTCASE(compactstub2
, glass
) {
337 const char * stubpath
= ".stub/compactstub2";
338 mkdir(".stub", 0755);
339 ofstream
stub(stubpath
);
340 TEST(stub
.is_open());
341 stub
<< "auto ../" << get_database_path("apitest_simpledata") << endl
;
342 stub
<< "auto ../" << get_database_path("apitest_simpledata2") << endl
;
345 string outdbpath
= get_named_writable_database_path("compactstub2out");
349 Xapian::Database
db(stubpath
);
350 db
.compact(outdbpath
);
353 Xapian::Database
indb(stubpath
);
354 Xapian::Database
outdb(outdbpath
);
356 TEST_EQUAL(indb
.get_doccount(), outdb
.get_doccount());
357 dbcheck(outdb
, outdb
.get_doccount(), outdb
.get_doccount());
362 // Test compacting a stub database file to itself.
363 DEFINE_TESTCASE(compactstub3
, glass
) {
364 const char * stubpath
= ".stub/compactstub3";
365 mkdir(".stub", 0755);
366 ofstream
stub(stubpath
);
367 TEST(stub
.is_open());
368 stub
<< "auto ../" << get_database_path("apitest_simpledata") << endl
;
369 stub
<< "auto ../" << get_database_path("apitest_simpledata2") << endl
;
372 Xapian::doccount in_docs
;
374 Xapian::Database
indb(stubpath
);
375 in_docs
= indb
.get_doccount();
376 indb
.compact(stubpath
);
379 Xapian::Database
outdb(stubpath
);
381 TEST_EQUAL(in_docs
, outdb
.get_doccount());
382 dbcheck(outdb
, outdb
.get_doccount(), outdb
.get_doccount());
387 // Test compacting a stub database directory to itself.
388 DEFINE_TESTCASE(compactstub4
, glass
) {
389 const char * stubpath
= ".stub/compactstub4";
390 const char * stubpathfile
= ".stub/compactstub4/XAPIANDB";
391 mkdir(".stub", 0755);
392 mkdir(stubpath
, 0755);
393 ofstream
stub(stubpathfile
);
394 TEST(stub
.is_open());
395 stub
<< "auto ../../" << get_database_path("apitest_simpledata") << endl
;
396 stub
<< "auto ../../" << get_database_path("apitest_simpledata2") << endl
;
399 Xapian::doccount in_docs
;
401 Xapian::Database
indb(stubpath
);
402 in_docs
= indb
.get_doccount();
403 indb
.compact(stubpath
);
406 Xapian::Database
outdb(stubpath
);
408 TEST_EQUAL(in_docs
, outdb
.get_doccount());
409 dbcheck(outdb
, outdb
.get_doccount(), outdb
.get_doccount());
415 make_all_tables(Xapian::WritableDatabase
&db
, const string
&)
417 Xapian::Document doc
;
419 db
.add_document(doc
);
420 db
.add_spelling("foo");
421 db
.add_synonym("bar", "pub");
422 db
.add_synonym("foobar", "foo");
428 make_missing_tables(Xapian::WritableDatabase
&db
, const string
&)
430 Xapian::Document doc
;
432 db
.add_document(doc
);
437 DEFINE_TESTCASE(compactmissingtables1
, generated
) {
438 string a
= get_database_path("compactmissingtables1a",
440 string b
= get_database_path("compactmissingtables1b",
441 make_missing_tables
);
443 string out
= get_named_writable_database_path("compactmissingtables1out");
448 db
.add_database(Xapian::Database(a
));
449 db
.add_database(Xapian::Database(b
));
454 Xapian::Database
db(out
);
455 TEST_NOT_EQUAL(db
.spellings_begin(), db
.spellings_end());
456 TEST_NOT_EQUAL(db
.synonym_keys_begin(), db
.synonym_keys_end());
457 // FIXME: arrange for input b to not have a termlist table.
458 // TEST_EXCEPTION(Xapian::FeatureUnavailableError, db.termlist_begin(1));
465 make_all_tables2(Xapian::WritableDatabase
&db
, const string
&)
467 Xapian::Document doc
;
469 db
.add_document(doc
);
470 db
.add_spelling("bar");
471 db
.add_synonym("bar", "baa");
472 db
.add_synonym("barfoo", "barbar");
473 db
.add_synonym("foofoo", "barfoo");
478 /// Adds coverage for merging synonym table.
479 DEFINE_TESTCASE(compactmergesynonym1
, generated
) {
480 string a
= get_database_path("compactmergesynonym1a",
482 string b
= get_database_path("compactmergesynonym1b",
485 string out
= get_named_writable_database_path("compactmergesynonym1out");
490 db
.add_database(Xapian::Database(a
));
491 db
.add_database(Xapian::Database(b
));
496 Xapian::Database
db(out
);
498 Xapian::TermIterator i
= db
.spellings_begin();
499 TEST_NOT_EQUAL(i
, db
.spellings_end());
500 TEST_EQUAL(*i
, "bar");
502 TEST_NOT_EQUAL(i
, db
.spellings_end());
503 TEST_EQUAL(*i
, "foo");
505 TEST_EQUAL(i
, db
.spellings_end());
507 i
= db
.synonym_keys_begin();
508 TEST_NOT_EQUAL(i
, db
.synonym_keys_end());
509 TEST_EQUAL(*i
, "bar");
511 TEST_NOT_EQUAL(i
, db
.synonym_keys_end());
512 TEST_EQUAL(*i
, "barfoo");
514 TEST_NOT_EQUAL(i
, db
.synonym_keys_end());
515 TEST_EQUAL(*i
, "foobar");
517 TEST_NOT_EQUAL(i
, db
.synonym_keys_end());
518 TEST_EQUAL(*i
, "foofoo");
520 TEST_EQUAL(i
, db
.synonym_keys_end());
526 DEFINE_TESTCASE(compactempty1
, glass
) {
527 string empty_dbpath
= get_database_path(string());
528 string outdbpath
= get_named_writable_database_path("compactempty1out");
532 // Compacting an empty database tried to divide by zero in 1.3.0.
534 db
.add_database(Xapian::Database(empty_dbpath
));
535 db
.compact(outdbpath
);
537 Xapian::Database
outdb(outdbpath
);
538 TEST_EQUAL(outdb
.get_doccount(), 0);
539 dbcheck(outdb
, 0, 0);
543 // Check compacting two empty databases together.
545 db
.add_database(Xapian::Database(empty_dbpath
));
546 db
.add_database(Xapian::Database(empty_dbpath
));
547 db
.compact(outdbpath
);
549 Xapian::Database
outdb(outdbpath
);
550 TEST_EQUAL(outdb
.get_doccount(), 0);
551 dbcheck(outdb
, 0, 0);
557 DEFINE_TESTCASE(compactmultipass1
, glass
) {
558 string outdbpath
= get_named_writable_database_path("compactmultipass1");
561 string a
= get_database_path("compactnorenumber1a", make_sparse_db
,
562 "5-7 24 76 987 1023-1027 9999 !9999");
563 string b
= get_database_path("compactnorenumber1b", make_sparse_db
,
565 string c
= get_database_path("compactnorenumber1c", make_sparse_db
,
567 string d
= get_database_path("compactnorenumber1d", make_sparse_db
,
568 "3000 999999 !999999");
572 db
.add_database(Xapian::Database(a
));
573 db
.add_database(Xapian::Database(b
));
574 db
.add_database(Xapian::Database(c
));
575 db
.add_database(Xapian::Database(d
));
576 db
.compact(outdbpath
, Xapian::DBCOMPACT_MULTIPASS
);
579 Xapian::Database
outdb(outdbpath
);
580 dbcheck(outdb
, 29, 1041);
585 // Test compacting to an fd.
586 DEFINE_TESTCASE(compacttofd1
, glass
) {
587 Xapian::Database
indb(get_database("apitest_simpledata"));
588 string outdbpath
= get_named_writable_database_path("compacttofd1out");
591 int fd
= open(outdbpath
.c_str(), O_CREAT
|O_RDWR
|O_BINARY
, 0666);
595 // Confirm that the fd was closed by Xapian. Set errno first to workaround
596 // a bug in Wine's msvcrt.dll which fails to set errno in this case:
597 // https://bugs.winehq.org/show_bug.cgi?id=43902
599 TEST(close(fd
) == -1);
600 TEST_EQUAL(errno
, EBADF
);
602 Xapian::Database
outdb(outdbpath
);
604 TEST_EQUAL(indb
.get_doccount(), outdb
.get_doccount());
605 dbcheck(outdb
, outdb
.get_doccount(), outdb
.get_doccount());
610 // Test compacting to an fd at at offset.
611 DEFINE_TESTCASE(compacttofd2
, glass
) {
612 Xapian::Database
indb(get_database("apitest_simpledata"));
613 string outdbpath
= get_named_writable_database_path("compacttofd2out");
616 int fd
= open(outdbpath
.c_str(), O_CREAT
|O_RDWR
|O_BINARY
, 0666);
618 TEST(lseek(fd
, 8192, SEEK_SET
) == 8192);
621 // Confirm that the fd was closed by Xapian. Set errno first to workaround
622 // a bug in Wine's msvcrt.dll which fails to set errno in this case:
623 // https://bugs.winehq.org/show_bug.cgi?id=43902
625 TEST(close(fd
) == -1);
626 TEST_EQUAL(errno
, EBADF
);
628 fd
= open(outdbpath
.c_str(), O_RDONLY
|O_BINARY
, 0666);
631 // Test that the database wasn't just written to the start of the file.
633 size_t n
= sizeof(buf
);
635 ssize_t c
= read(fd
, buf
, n
);
637 for (const char * p
= buf
; p
!= buf
+ c
; ++p
) {
643 TEST(lseek(fd
, 8192, SEEK_SET
) == 8192);
644 Xapian::Database
outdb(fd
);
646 TEST_EQUAL(indb
.get_doccount(), outdb
.get_doccount());
647 dbcheck(outdb
, outdb
.get_doccount(), outdb
.get_doccount());
652 // Regression test for bug fixed in 1.3.5. If you commit a WritableDatabase
653 // with uncommitted changes, you get an inconsistent output.
654 DEFINE_TESTCASE(compactsingle1
, glass
) {
655 Xapian::WritableDatabase db
= get_writable_database();
656 Xapian::Document doc
;
660 db
.add_document(doc
);
662 string output
= ".glass/db__compactsingle1-out";
663 // In 1.3.4, we would hang if the output file already existed, so check
667 TEST_EXCEPTION(Xapian::InvalidOperationError
,
668 db
.compact(output
, Xapian::DBCOMPACT_SINGLE_FILE
));
670 // Check the file wasn't removed by the failed attempt.
671 TEST(file_exists(output
));
674 db
.compact(output
, Xapian::DBCOMPACT_SINGLE_FILE
);
677 TEST_EQUAL(Xapian::Database::check(output
, 0, &tout
), 0);