2 * @brief Compact a database, or merge and compact several.
4 /* Copyright (C) 2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2015,2016,2017 Olly Betts
5 * Copyright (C) 2008 Lemur Consulting Ltd
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License as
9 * published by the Free Software Foundation; either version 2 of the
10 * License, or (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
25 #include <xapian/compactor.h>
27 #include "safeerrno.h"
35 #include "safesysstat.h"
36 #include <sys/types.h>
38 #include "safeunistd.h"
39 #include "safefcntl.h"
41 #include "backends/backends.h"
42 #include "backends/databaseinternal.h"
44 #include "leafpostlist.h"
46 #include "filetests.h"
47 #include "fileutils.h"
49 #include "stringutils.h"
52 #ifdef XAPIAN_HAS_GLASS_BACKEND
53 #include "backends/glass/glass_database.h"
54 #include "backends/glass/glass_version.h"
57 #include "backends/multi/multi_database.h"
59 #include <xapian/constants.h>
60 #include <xapian/database.h>
61 #include <xapian/error.h>
65 class CmpByFirstUsed
{
66 const vector
<pair
<Xapian::docid
, Xapian::docid
> > & used_ranges
;
70 CmpByFirstUsed(const vector
<pair
<Xapian::docid
, Xapian::docid
> > & ur
)
73 bool operator()(size_t a
, size_t b
) const {
74 return used_ranges
[a
].first
< used_ranges
[b
].first
;
80 Compactor::~Compactor() { }
83 Compactor::set_status(const string
& table
, const string
& status
)
90 Compactor::resolve_duplicate_metadata(const string
& key
,
91 size_t num_tags
, const std::string tags
[])
102 backend_mismatch(const Xapian::Database::Internal
* db
, int backend1
,
103 const string
&dbpath2
, int backend2
)
106 db
->get_backend_info(&dbpath1
);
107 string msg
= "All databases must be the same type ('";
110 msg
+= backend_name(backend1
);
114 msg
+= backend_name(backend2
);
116 throw Xapian::InvalidArgumentError(msg
);
122 Database::compact_(const string
* output_ptr
, int fd
, unsigned flags
,
124 Xapian::Compactor
* compactor
) const
126 LOGCALL_VOID(API
, "Database::compact_", output_ptr
| fd
| flags
| block_size
| compactor
);
128 bool renumber
= !(flags
& DBCOMPACT_NO_RENUMBER
);
130 enum { STUB_NO
, STUB_FILE
, STUB_DIR
} compact_to_stub
= STUB_NO
;
133 // We need a modifiable destdir in this function.
134 destdir
= *output_ptr
;
135 if (!(flags
& DBCOMPACT_SINGLE_FILE
)) {
136 if (file_exists(destdir
)) {
138 compact_to_stub
= STUB_FILE
;
139 } else if (file_exists(destdir
+ "/XAPIANDB")) {
141 compact_to_stub
= STUB_DIR
;
145 // Single file is implied when writing to a file descriptor.
146 flags
|= DBCOMPACT_SINGLE_FILE
;
149 auto n_shards
= internal
->size();
150 Xapian::docid tot_off
= 0;
151 Xapian::docid last_docid
= 0;
153 vector
<Xapian::docid
> offset
;
154 vector
<pair
<Xapian::docid
, Xapian::docid
> > used_ranges
;
155 vector
<const Xapian::Database::Internal
*> internals
;
156 offset
.reserve(n_shards
);
157 used_ranges
.reserve(n_shards
);
158 internals
.reserve(n_shards
);
161 auto multi_db
= static_cast<MultiDatabase
*>(internal
.get());
162 for (auto&& db
: multi_db
->shards
) {
163 internals
.push_back(db
);
166 internals
.push_back(internal
.get());
169 int backend
= BACKEND_UNKNOWN
;
170 for (auto&& shard
: internals
) {
172 int type
= shard
->get_backend_info(&srcdir
);
173 // Check destdir isn't the same as any source directory, unless it
174 // is a stub database.
175 if (!compact_to_stub
&& srcdir
== destdir
) {
176 throw InvalidArgumentError("destination may not be the same as "
177 "any source database, unless it is a "
182 if (backend
!= type
&& backend
!= BACKEND_UNKNOWN
) {
183 backend_mismatch(internals
[0], backend
, srcdir
, type
);
188 throw DatabaseError("Only glass databases can be compacted");
191 Xapian::docid first
= 0, last
= 0;
193 // "Empty" databases might have spelling or synonym data so can't
194 // just be completely ignored.
195 Xapian::doccount num_docs
= shard
->get_doccount();
197 shard
->get_used_docid_range(first
, last
);
199 if (renumber
&& first
) {
200 // Prune any unused docids off the start of this source
203 // tot_off could wrap here, but it's unsigned, so that's
205 tot_off
-= (first
- 1);
208 #ifdef XAPIAN_ASSERTIONS
209 PostList
* pl
= shard
->open_post_list(string());
211 // This test should never fail, since shard->get_doccount() is
213 Assert(!pl
->at_end());
214 AssertEq(pl
->get_docid(), first
);
215 AssertRel(last
,>=,first
);
217 Assert(!pl
->at_end());
218 AssertEq(pl
->get_docid(), last
);
220 Assert(pl
->at_end());
225 offset
.push_back(tot_off
);
228 else if (last_docid
< shard
->get_lastdocid())
229 last_docid
= shard
->get_lastdocid();
230 used_ranges
.push_back(make_pair(first
, last
));
234 last_docid
= tot_off
;
236 if (!renumber
&& n_shards
> 1) {
237 // We want to process the sources in ascending order of first
238 // docid. So we create a vector "order" with ascending integers
239 // and then sort so the indirected order is right.
240 vector
<size_t> order
;
241 order
.reserve(n_shards
);
242 for (size_t i
= 0; i
< n_shards
; ++i
)
245 sort(order
.begin(), order
.end(), CmpByFirstUsed(used_ranges
));
247 // Now use order to reorder internals to be in ascending order by first
248 // docid, and while we're at it check the ranges are disjoint.
249 vector
<const Xapian::Database::Internal
*> internals_
;
250 internals_
.reserve(n_shards
);
251 vector
<pair
<Xapian::docid
, Xapian::docid
> > used_ranges_
;
252 used_ranges_
.reserve(n_shards
);
254 Xapian::docid last_start
= 0, last_end
= 0;
255 for (size_t j
= 0; j
!= order
.size(); ++j
) {
258 internals_
.push_back(internals
[n
]);
259 used_ranges_
.push_back(used_ranges
[n
]);
261 const pair
<Xapian::docid
, Xapian::docid
> p
= used_ranges
[n
];
262 // Skip empty databases.
263 if (p
.first
== 0 && p
.second
== 0)
265 // Check for overlap with the previous database's range.
266 if (p
.first
<= last_end
) {
268 string msg
= "when merging databases, --no-renumber is only currently supported if the databases have disjoint ranges of used document ids: ";
269 internals_
[j
- 1]->get_backend_info(&tmp
);
271 msg
+= " has range ";
272 msg
+= str(last_start
);
274 msg
+= str(last_end
);
276 internals_
[j
]->get_backend_info(&tmp
);
278 msg
+= " has range ";
281 msg
+= str(p
.second
);
282 throw Xapian::InvalidOperationError(msg
);
284 last_start
= p
.first
;
288 swap(internals
, internals_
);
289 swap(used_ranges
, used_ranges_
);
293 if (compact_to_stub
) {
295 if (compact_to_stub
== STUB_DIR
) {
296 stub_file
+= "/XAPIANDB";
301 size_t sfx
= destdir
.size();
302 time_t now
= time(NULL
);
305 destdir
+= str(now
++);
306 if (mkdir(destdir
.c_str(), 0755) == 0)
308 if (errno
!= EEXIST
) {
309 string msg
= destdir
;
310 msg
+= ": mkdir failed";
311 throw Xapian::DatabaseError(msg
, errno
);
314 } else if (!(flags
& Xapian::DBCOMPACT_SINGLE_FILE
)) {
315 // If the destination database directory doesn't exist, create it.
316 if (mkdir(destdir
.c_str(), 0755) < 0) {
317 // Check why mkdir failed. It's ok if the directory already
318 // exists, but we also get EEXIST if there's an existing file with
320 if (errno
== EEXIST
) {
321 if (dir_exists(destdir
))
324 errno
= EEXIST
; // dir_exists() might have changed it
327 string msg
= destdir
;
328 msg
+= ": cannot create directory";
329 throw Xapian::DatabaseError(msg
, errno
);
334 #if defined XAPIAN_HAS_GLASS_BACKEND
335 Xapian::Compactor::compaction_level compaction
=
336 static_cast<Xapian::Compactor::compaction_level
>(flags
& (Xapian::Compactor::STANDARD
|Xapian::Compactor::FULL
|Xapian::Compactor::FULLER
));
342 if (backend
== BACKEND_GLASS
) {
343 #ifdef XAPIAN_HAS_GLASS_BACKEND
345 GlassDatabase::compact(compactor
, destdir
.c_str(), 0,
347 block_size
, compaction
, flags
, last_docid
);
349 GlassDatabase::compact(compactor
, NULL
, fd
,
351 block_size
, compaction
, flags
, last_docid
);
356 throw Xapian::FeatureUnavailableError("Glass backend disabled at build time");
360 if (compact_to_stub
) {
361 string new_stub_file
= destdir
;
362 new_stub_file
+= "/new_stub.tmp";
364 ofstream
new_stub(new_stub_file
.c_str());
365 size_t slash
= destdir
.find_last_of(DIR_SEPS
);
366 new_stub
<< "auto " << destdir
.substr(slash
+ 1) << '\n';
368 if (!io_tmp_rename(new_stub_file
, stub_file
)) {
369 string msg
= "Cannot rename '";
370 msg
+= new_stub_file
;
374 throw Xapian::DatabaseError(msg
, errno
);