Revert changes accidentally committed
[xapian.git] / xapian-core / tests / api_compact.cc
blob1b29e45c47213bd38152180b2b40a2a6b2aedcc3
1 /** @file api_compact.cc
2 * @brief Tests of Database::compact()
3 */
4 /* Copyright (C) 2009,2010,2011,2012,2013,2015,2016,2017 Olly Betts
5 * Copyright (C) 2010 Richard Boulton
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License as
9 * published by the Free Software Foundation; either version 2 of the
10 * License, or (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
20 * USA
23 #include <config.h>
25 #include "api_compact.h"
27 #include <xapian.h>
29 #include "apitest.h"
30 #include "dbcheck.h"
31 #include "filetests.h"
32 #include "str.h"
33 #include "testsuite.h"
34 #include "testutils.h"
36 #include <cstdlib>
37 #include <fstream>
39 #include <sys/types.h>
40 #include "safesysstat.h"
41 #include "safefcntl.h"
42 #include "safeunistd.h"
44 #include "unixcmds.h"
46 using namespace std;
48 static void
49 make_sparse_db(Xapian::WritableDatabase &db, const string & s)
51 // Need non-const pointer for strtoul(), but data isn't modified.
52 char * p = const_cast<char *>(s.c_str());
54 while (*p) {
55 bool del = (*p == '!');
56 if (del) ++p;
57 Xapian::docid first = strtoul(p, &p, 10);
58 Xapian::docid last = first;
59 if (*p == '-') {
60 last = strtoul(p + 1, &p, 10);
62 if (*p && *p != ' ') {
63 tout << p - s.c_str() << endl;
64 FAIL_TEST("Bad sparse db spec (expected space): " << s);
66 if (first > last) {
67 FAIL_TEST("Bad sparse db spec (first > last): " << s);
70 do {
71 if (del) {
72 db.delete_document(first);
73 } else {
74 Xapian::Document doc;
75 string id = str(first);
76 doc.set_data(id);
77 doc.add_term("Q" + str(first));
78 doc.add_term(string(first % 7 + 1, char((first % 26) + 'a')));
79 db.replace_document(first, doc);
81 } while (first++ < last);
83 if (*p == '\0') break;
84 ++p;
87 db.commit();
90 static void
91 check_sparse_uid_terms(const string & path)
93 Xapian::Database db(path);
94 Xapian::TermIterator t;
95 for (t = db.allterms_begin("Q"); t != db.allterms_end("Q"); ++t) {
96 Xapian::docid did = atoi((*t).c_str() + 1);
97 Xapian::PostingIterator p = db.postlist_begin(*t);
98 TEST_EQUAL(*p, did);
102 DEFINE_TESTCASE(compactnorenumber1, generated) {
103 string a = get_database_path("compactnorenumber1a", make_sparse_db,
104 "5-7 24 76 987 1023-1027 9999 !9999");
105 string a_uuid;
107 Xapian::Database db(a);
108 a_uuid = db.get_uuid();
110 string b = get_database_path("compactnorenumber1b", make_sparse_db,
111 "1027-1030");
112 string c = get_database_path("compactnorenumber1c", make_sparse_db,
113 "1028-1040");
114 string d = get_database_path("compactnorenumber1d", make_sparse_db,
115 "3000 999999 !999999");
117 string out = get_named_writable_database_path("compactnorenumber1out");
119 rm_rf(out);
121 Xapian::Database db(a);
122 db.compact(out, Xapian::DBCOMPACT_NO_RENUMBER);
125 check_sparse_uid_terms(out);
128 TEST(!dir_exists(out + "/donor"));
129 Xapian::Database db(out);
130 // xapian-compact should change the UUID of the database, but didn't
131 // prior to 1.0.18/1.1.4.
132 string out_uuid = db.get_uuid();
133 TEST_NOT_EQUAL(a_uuid, out_uuid);
134 TEST_EQUAL(out_uuid.size(), 36);
135 TEST_NOT_EQUAL(out_uuid, "00000000-0000-0000-0000-000000000000");
137 // White box test - ensure that the donor database is removed.
138 TEST(!dir_exists(out + "/donor"));
141 rm_rf(out);
143 Xapian::Database db;
144 db.add_database(Xapian::Database(a));
145 db.add_database(Xapian::Database(c));
146 db.compact(out, Xapian::DBCOMPACT_NO_RENUMBER);
148 check_sparse_uid_terms(out);
150 // Check that xapian-compact is producing a consistent database. Also,
151 // regression test - xapian 1.1.4 set lastdocid to 0 in the output
152 // database.
153 Xapian::Database outdb(out);
154 dbcheck(outdb, 24, 9999);
157 rm_rf(out);
159 Xapian::Database db;
160 db.add_database(Xapian::Database(d));
161 db.add_database(Xapian::Database(a));
162 db.add_database(Xapian::Database(c));
163 db.compact(out, Xapian::DBCOMPACT_NO_RENUMBER);
165 check_sparse_uid_terms(out);
167 rm_rf(out);
169 Xapian::Database db;
170 db.add_database(Xapian::Database(c));
171 db.add_database(Xapian::Database(a));
172 db.add_database(Xapian::Database(d));
173 db.compact(out, Xapian::DBCOMPACT_NO_RENUMBER);
175 check_sparse_uid_terms(out);
177 // Should fail.
178 rm_rf(out);
180 Xapian::Database db;
181 db.add_database(Xapian::Database(a));
182 db.add_database(Xapian::Database(b));
183 TEST_EXCEPTION(Xapian::InvalidOperationError,
184 db.compact(out, Xapian::DBCOMPACT_NO_RENUMBER)
188 // Should fail.
189 rm_rf(out);
191 Xapian::Database db;
192 db.add_database(Xapian::Database(b));
193 db.add_database(Xapian::Database(a));
194 TEST_EXCEPTION(Xapian::InvalidOperationError,
195 db.compact(out, Xapian::DBCOMPACT_NO_RENUMBER)
199 // Should fail.
200 rm_rf(out);
202 Xapian::Database db;
203 db.add_database(Xapian::Database(a));
204 db.add_database(Xapian::Database(b));
205 db.add_database(Xapian::Database(d));
206 TEST_EXCEPTION(Xapian::InvalidOperationError,
207 db.compact(out, Xapian::DBCOMPACT_NO_RENUMBER)
211 // Should fail.
212 rm_rf(out);
214 Xapian::Database db;
215 db.add_database(Xapian::Database(d));
216 db.add_database(Xapian::Database(b));
217 db.add_database(Xapian::Database(a));
218 TEST_EXCEPTION(Xapian::InvalidOperationError,
219 db.compact(out, Xapian::DBCOMPACT_NO_RENUMBER)
223 // Should fail.
224 rm_rf(out);
226 Xapian::Database db;
227 db.add_database(Xapian::Database(b));
228 db.add_database(Xapian::Database(a));
229 db.add_database(Xapian::Database(d));
230 TEST_EXCEPTION(Xapian::InvalidOperationError,
231 db.compact(out, Xapian::DBCOMPACT_NO_RENUMBER)
235 return true;
238 // Test use of compact to merge two databases.
239 DEFINE_TESTCASE(compactmerge1, glass) {
240 string indbpath = get_database_path("apitest_simpledata");
241 string outdbpath = get_named_writable_database_path("compactmerge1out");
242 rm_rf(outdbpath);
245 Xapian::Database db;
246 db.add_database(Xapian::Database(indbpath));
247 db.add_database(Xapian::Database(indbpath));
248 db.compact(outdbpath);
251 Xapian::Database indb(get_database("apitest_simpledata"));
252 Xapian::Database outdb(outdbpath);
254 TEST_EQUAL(indb.get_doccount() * 2, outdb.get_doccount());
255 dbcheck(outdb, outdb.get_doccount(), outdb.get_doccount());
257 for (const char * suffix :
258 { "", "/postlist", "/termlist.", "/docdata.glass" }) {
259 tout.str(string());
260 tout << "Trying suffix '" << suffix << "'" << endl;
261 string arg = outdbpath;
262 arg += suffix;
263 TEST_EQUAL(Xapian::Database::check(arg, 0, &tout), 0);
266 return true;
269 static void
270 make_multichunk_db(Xapian::WritableDatabase &db, const string &)
272 int count = 10000;
274 Xapian::Document doc;
275 doc.add_term("a");
276 while (count) {
277 db.add_document(doc);
278 --count;
281 db.commit();
284 // Test use of compact on a database which has multiple chunks for a term.
285 // This is a regression test for ticket #427
286 DEFINE_TESTCASE(compactmultichunks1, generated) {
287 string indbpath = get_database_path("compactmultichunks1in",
288 make_multichunk_db, "");
289 string outdbpath = get_named_writable_database_path("compactmultichunks1out");
290 rm_rf(outdbpath);
293 Xapian::Database db(indbpath);
294 db.compact(outdbpath);
297 Xapian::Database indb(indbpath);
298 Xapian::Database outdb(outdbpath);
300 TEST_EQUAL(indb.get_doccount(), outdb.get_doccount());
301 dbcheck(outdb, outdb.get_doccount(), outdb.get_doccount());
303 return true;
306 // Test compacting from a stub database directory.
307 DEFINE_TESTCASE(compactstub1, glass) {
308 const char * stubpath = ".stub/compactstub1";
309 const char * stubpathfile = ".stub/compactstub1/XAPIANDB";
310 mkdir(".stub", 0755);
311 mkdir(stubpath, 0755);
312 ofstream stub(stubpathfile);
313 TEST(stub.is_open());
314 stub << "auto ../../" << get_database_path("apitest_simpledata") << endl;
315 stub << "auto ../../" << get_database_path("apitest_simpledata2") << endl;
316 stub.close();
318 string outdbpath = get_named_writable_database_path("compactstub1out");
319 rm_rf(outdbpath);
322 Xapian::Database db(stubpath);
323 db.compact(outdbpath);
326 Xapian::Database indb(stubpath);
327 Xapian::Database outdb(outdbpath);
329 TEST_EQUAL(indb.get_doccount(), outdb.get_doccount());
330 dbcheck(outdb, outdb.get_doccount(), outdb.get_doccount());
332 return true;
335 // Test compacting from a stub database file.
336 DEFINE_TESTCASE(compactstub2, glass) {
337 const char * stubpath = ".stub/compactstub2";
338 mkdir(".stub", 0755);
339 ofstream stub(stubpath);
340 TEST(stub.is_open());
341 stub << "auto ../" << get_database_path("apitest_simpledata") << endl;
342 stub << "auto ../" << get_database_path("apitest_simpledata2") << endl;
343 stub.close();
345 string outdbpath = get_named_writable_database_path("compactstub2out");
346 rm_rf(outdbpath);
349 Xapian::Database db(stubpath);
350 db.compact(outdbpath);
353 Xapian::Database indb(stubpath);
354 Xapian::Database outdb(outdbpath);
356 TEST_EQUAL(indb.get_doccount(), outdb.get_doccount());
357 dbcheck(outdb, outdb.get_doccount(), outdb.get_doccount());
359 return true;
362 // Test compacting a stub database file to itself.
363 DEFINE_TESTCASE(compactstub3, glass) {
364 const char * stubpath = ".stub/compactstub3";
365 mkdir(".stub", 0755);
366 ofstream stub(stubpath);
367 TEST(stub.is_open());
368 stub << "auto ../" << get_database_path("apitest_simpledata") << endl;
369 stub << "auto ../" << get_database_path("apitest_simpledata2") << endl;
370 stub.close();
372 Xapian::doccount in_docs;
374 Xapian::Database indb(stubpath);
375 in_docs = indb.get_doccount();
376 indb.compact(stubpath);
379 Xapian::Database outdb(stubpath);
381 TEST_EQUAL(in_docs, outdb.get_doccount());
382 dbcheck(outdb, outdb.get_doccount(), outdb.get_doccount());
384 return true;
387 // Test compacting a stub database directory to itself.
388 DEFINE_TESTCASE(compactstub4, glass) {
389 const char * stubpath = ".stub/compactstub4";
390 const char * stubpathfile = ".stub/compactstub4/XAPIANDB";
391 mkdir(".stub", 0755);
392 mkdir(stubpath, 0755);
393 ofstream stub(stubpathfile);
394 TEST(stub.is_open());
395 stub << "auto ../../" << get_database_path("apitest_simpledata") << endl;
396 stub << "auto ../../" << get_database_path("apitest_simpledata2") << endl;
397 stub.close();
399 Xapian::doccount in_docs;
401 Xapian::Database indb(stubpath);
402 in_docs = indb.get_doccount();
403 indb.compact(stubpath);
406 Xapian::Database outdb(stubpath);
408 TEST_EQUAL(in_docs, outdb.get_doccount());
409 dbcheck(outdb, outdb.get_doccount(), outdb.get_doccount());
411 return true;
414 static void
415 make_all_tables(Xapian::WritableDatabase &db, const string &)
417 Xapian::Document doc;
418 doc.add_term("foo");
419 db.add_document(doc);
420 db.add_spelling("foo");
421 db.add_synonym("bar", "pub");
422 db.add_synonym("foobar", "foo");
424 db.commit();
427 static void
428 make_missing_tables(Xapian::WritableDatabase &db, const string &)
430 Xapian::Document doc;
431 doc.add_term("foo");
432 db.add_document(doc);
434 db.commit();
437 DEFINE_TESTCASE(compactmissingtables1, generated) {
438 string a = get_database_path("compactmissingtables1a",
439 make_all_tables);
440 string b = get_database_path("compactmissingtables1b",
441 make_missing_tables);
443 string out = get_named_writable_database_path("compactmissingtables1out");
444 rm_rf(out);
447 Xapian::Database db;
448 db.add_database(Xapian::Database(a));
449 db.add_database(Xapian::Database(b));
450 db.compact(out);
454 Xapian::Database db(out);
455 TEST_NOT_EQUAL(db.spellings_begin(), db.spellings_end());
456 TEST_NOT_EQUAL(db.synonym_keys_begin(), db.synonym_keys_end());
457 // FIXME: arrange for input b to not have a termlist table.
458 // TEST_EXCEPTION(Xapian::FeatureUnavailableError, db.termlist_begin(1));
461 return true;
464 static void
465 make_all_tables2(Xapian::WritableDatabase &db, const string &)
467 Xapian::Document doc;
468 doc.add_term("bar");
469 db.add_document(doc);
470 db.add_spelling("bar");
471 db.add_synonym("bar", "baa");
472 db.add_synonym("barfoo", "barbar");
473 db.add_synonym("foofoo", "barfoo");
475 db.commit();
478 /// Adds coverage for merging synonym table.
479 DEFINE_TESTCASE(compactmergesynonym1, generated) {
480 string a = get_database_path("compactmergesynonym1a",
481 make_all_tables);
482 string b = get_database_path("compactmergesynonym1b",
483 make_all_tables2);
485 string out = get_named_writable_database_path("compactmergesynonym1out");
486 rm_rf(out);
489 Xapian::Database db;
490 db.add_database(Xapian::Database(a));
491 db.add_database(Xapian::Database(b));
492 db.compact(out);
496 Xapian::Database db(out);
498 Xapian::TermIterator i = db.spellings_begin();
499 TEST_NOT_EQUAL(i, db.spellings_end());
500 TEST_EQUAL(*i, "bar");
501 ++i;
502 TEST_NOT_EQUAL(i, db.spellings_end());
503 TEST_EQUAL(*i, "foo");
504 ++i;
505 TEST_EQUAL(i, db.spellings_end());
507 i = db.synonym_keys_begin();
508 TEST_NOT_EQUAL(i, db.synonym_keys_end());
509 TEST_EQUAL(*i, "bar");
510 ++i;
511 TEST_NOT_EQUAL(i, db.synonym_keys_end());
512 TEST_EQUAL(*i, "barfoo");
513 ++i;
514 TEST_NOT_EQUAL(i, db.synonym_keys_end());
515 TEST_EQUAL(*i, "foobar");
516 ++i;
517 TEST_NOT_EQUAL(i, db.synonym_keys_end());
518 TEST_EQUAL(*i, "foofoo");
519 ++i;
520 TEST_EQUAL(i, db.synonym_keys_end());
523 return true;
526 DEFINE_TESTCASE(compactempty1, glass) {
527 string empty_dbpath = get_database_path(string());
528 string outdbpath = get_named_writable_database_path("compactempty1out");
529 rm_rf(outdbpath);
532 // Compacting an empty database tried to divide by zero in 1.3.0.
533 Xapian::Database db;
534 db.add_database(Xapian::Database(empty_dbpath));
535 db.compact(outdbpath);
537 Xapian::Database outdb(outdbpath);
538 TEST_EQUAL(outdb.get_doccount(), 0);
539 dbcheck(outdb, 0, 0);
543 // Check compacting two empty databases together.
544 Xapian::Database db;
545 db.add_database(Xapian::Database(empty_dbpath));
546 db.add_database(Xapian::Database(empty_dbpath));
547 db.compact(outdbpath);
549 Xapian::Database outdb(outdbpath);
550 TEST_EQUAL(outdb.get_doccount(), 0);
551 dbcheck(outdb, 0, 0);
554 return true;
557 DEFINE_TESTCASE(compactmultipass1, glass) {
558 string outdbpath = get_named_writable_database_path("compactmultipass1");
559 rm_rf(outdbpath);
561 string a = get_database_path("compactnorenumber1a", make_sparse_db,
562 "5-7 24 76 987 1023-1027 9999 !9999");
563 string b = get_database_path("compactnorenumber1b", make_sparse_db,
564 "1027-1030");
565 string c = get_database_path("compactnorenumber1c", make_sparse_db,
566 "1028-1040");
567 string d = get_database_path("compactnorenumber1d", make_sparse_db,
568 "3000 999999 !999999");
571 Xapian::Database db;
572 db.add_database(Xapian::Database(a));
573 db.add_database(Xapian::Database(b));
574 db.add_database(Xapian::Database(c));
575 db.add_database(Xapian::Database(d));
576 db.compact(outdbpath, Xapian::DBCOMPACT_MULTIPASS);
579 Xapian::Database outdb(outdbpath);
580 dbcheck(outdb, 29, 1041);
582 return true;
585 // Test compacting to an fd.
586 DEFINE_TESTCASE(compacttofd1, glass) {
587 Xapian::Database indb(get_database("apitest_simpledata"));
588 string outdbpath = get_named_writable_database_path("compacttofd1out");
589 rm_rf(outdbpath);
591 int fd = open(outdbpath.c_str(), O_CREAT|O_RDWR|O_BINARY, 0666);
592 TEST(fd != -1);
593 indb.compact(fd);
595 // Confirm that the fd was closed by Xapian. Set errno first to workaround
596 // a bug in Wine's msvcrt.dll which fails to set errno in this case:
597 // https://bugs.winehq.org/show_bug.cgi?id=43902
598 errno = EBADF;
599 TEST(close(fd) == -1);
600 TEST_EQUAL(errno, EBADF);
602 Xapian::Database outdb(outdbpath);
604 TEST_EQUAL(indb.get_doccount(), outdb.get_doccount());
605 dbcheck(outdb, outdb.get_doccount(), outdb.get_doccount());
607 return true;
610 // Test compacting to an fd at at offset.
611 DEFINE_TESTCASE(compacttofd2, glass) {
612 Xapian::Database indb(get_database("apitest_simpledata"));
613 string outdbpath = get_named_writable_database_path("compacttofd2out");
614 rm_rf(outdbpath);
616 int fd = open(outdbpath.c_str(), O_CREAT|O_RDWR|O_BINARY, 0666);
617 TEST(fd != -1);
618 TEST(lseek(fd, 8192, SEEK_SET) == 8192);
619 indb.compact(fd);
621 // Confirm that the fd was closed by Xapian. Set errno first to workaround
622 // a bug in Wine's msvcrt.dll which fails to set errno in this case:
623 // https://bugs.winehq.org/show_bug.cgi?id=43902
624 errno = EBADF;
625 TEST(close(fd) == -1);
626 TEST_EQUAL(errno, EBADF);
628 fd = open(outdbpath.c_str(), O_RDONLY|O_BINARY, 0666);
629 TEST(fd != -1);
631 // Test that the database wasn't just written to the start of the file.
632 char buf[8192];
633 size_t n = sizeof(buf);
634 while (n) {
635 ssize_t c = read(fd, buf, n);
636 TEST(c > 0);
637 for (const char * p = buf; p != buf + c; ++p) {
638 TEST(*p == 0);
640 n -= c;
643 TEST(lseek(fd, 8192, SEEK_SET) == 8192);
644 Xapian::Database outdb(fd);
646 TEST_EQUAL(indb.get_doccount(), outdb.get_doccount());
647 dbcheck(outdb, outdb.get_doccount(), outdb.get_doccount());
649 return true;
652 // Regression test for bug fixed in 1.3.5. If you compact a WritableDatabase
653 // with uncommitted changes, you get an inconsistent output.
654 DEFINE_TESTCASE(compactsingle1, glass) {
655 Xapian::WritableDatabase db = get_writable_database();
656 Xapian::Document doc;
657 doc.add_term("foo");
658 doc.add_term("bar");
659 doc.add_term("baz");
660 db.add_document(doc);
662 string output = ".glass/db__compactsingle1-out";
663 // In 1.3.4, we would hang if the output file already existed, so check
664 // that works.
665 touch(output);
667 TEST_EXCEPTION(Xapian::InvalidOperationError,
668 db.compact(output, Xapian::DBCOMPACT_SINGLE_FILE));
670 // Check the file wasn't removed by the failed attempt.
671 TEST(file_exists(output));
673 db.commit();
674 db.compact(output, Xapian::DBCOMPACT_SINGLE_FILE);
675 db.close();
677 TEST_EQUAL(Xapian::Database::check(output, 0, &tout), 0);
679 return true;
682 // Regression test for bug fixed in 1.4.6. Same as above, except not with
683 // a single file database!
684 DEFINE_TESTCASE(compact1, glass) {
685 Xapian::WritableDatabase db = get_writable_database();
686 Xapian::Document doc;
687 doc.add_term("foo");
688 doc.add_term("bar");
689 doc.add_term("baz");
690 db.add_document(doc);
692 string output = ".glass/db__compact1-out";
693 rm_rf(output);
695 TEST_EXCEPTION(Xapian::InvalidOperationError,
696 db.compact(output));
698 db.commit();
699 db.compact(output);
700 db.close();
702 TEST_EQUAL(Xapian::Database::check(output, 0, &tout), 0);
704 return true;