2 * @brief Database factories for non-remote databases.
4 /* Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2011,2012,2013,2014,2015,2016,2017 Olly Betts
5 * Copyright 2008 Lemur Consulting Ltd
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License as
9 * published by the Free Software Foundation; either version 2 of the
10 * License, or (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
25 #include "xapian/dbfactory.h"
27 #include "xapian/constants.h"
28 #include "xapian/database.h"
29 #include "xapian/error.h"
30 #include "xapian/version.h" // For XAPIAN_HAS_XXX_BACKEND.
34 #include "filetests.h"
35 #include "fileutils.h"
36 #include "posixy_wrapper.h"
39 #include "safeerrno.h"
40 #include <cstdlib> // For atoi().
42 #ifdef XAPIAN_HAS_GLASS_BACKEND
43 # include "glass/glass_database.h"
45 #include "glass/glass_defs.h"
46 #ifdef XAPIAN_HAS_HONEY_BACKEND
47 # include "honey/honey_database.h"
49 #include "honey/honey_defs.h"
50 #ifdef XAPIAN_HAS_INMEMORY_BACKEND
51 # include "inmemory/inmemory_database.h"
53 // Even if none of the above get included, we still need a definition of
54 // Database::Internal.
55 #include "backends/databaseinternal.h"
62 /** Return a BACKEND_* constant from backends.h.
64 * BACKEND_UNKNOWN : stub file
65 * BACKEND_GLASS : glass single file
66 * BACKEND_HONEY : honey single file
69 check_if_single_file_db(const struct stat
& sb
, const string
& path
,
72 #if defined XAPIAN_HAS_GLASS_BACKEND || \
73 defined XAPIAN_HAS_HONEY_BACKEND
74 if (!S_ISREG(sb
.st_mode
)) return BACKEND_UNKNOWN
;
75 // Look at the size as a clue - if it's less than both GLASS_MIN_BLOCKSIZE
76 // and HONEY_MIN_DB_SIZE then it's not a single-file glass or honey
77 // database. For a larger file, we peek at the start of the file to
78 // determine what it is.
79 if (sb
.st_size
< min(GLASS_MIN_BLOCKSIZE
, HONEY_MIN_DB_SIZE
))
81 int fd
= posixy_open(path
.c_str(), O_RDONLY
|O_BINARY
);
84 // FIXME: Don't duplicate magic check here...
85 if (io_read(fd
, magic_buf
, 14) == 14 &&
86 lseek(fd
, 0, SEEK_SET
) == 0 &&
87 memcmp(magic_buf
, "\x0f\x0dXapian ", 9) == 0) {
90 switch (magic_buf
[9]) {
92 if (memcmp(magic_buf
+ 10, "lass", 4) == 0) {
99 if (memcmp(magic_buf
+ 10, "oney", 4) == 0) {
102 return BACKEND_HONEY
;
108 return BACKEND_UNKNOWN
;
117 return BACKEND_UNKNOWN
;
123 open_stub(Database
&db
, const string
&file
)
125 // A stub database is a text file with one or more lines of this format:
126 // <dbtype> <serialised db object>
128 // Lines which start with a "#" character are ignored.
130 // Any paths specified in stub database files which are relative will be
131 // considered to be relative to the directory containing the stub database.
132 ifstream
stub(file
.c_str());
134 string msg
= "Couldn't open stub database file: ";
136 throw Xapian::DatabaseOpeningError(msg
, errno
);
139 unsigned int line_no
= 0;
140 while (getline(stub
, line
)) {
142 if (line
.empty() || line
[0] == '#')
144 string::size_type space
= line
.find(' ');
145 if (space
== string::npos
) space
= line
.size();
147 string
type(line
, 0, space
);
148 line
.erase(0, space
+ 1);
150 if (type
== "auto") {
151 resolve_relative_path(line
, file
);
152 db
.add_database(Database(line
));
156 if (type
== "glass") {
157 #ifdef XAPIAN_HAS_GLASS_BACKEND
158 resolve_relative_path(line
, file
);
159 db
.add_database(Database(new GlassDatabase(line
)));
162 throw FeatureUnavailableError("Glass backend disabled");
166 if (type
== "honey") {
167 #ifdef XAPIAN_HAS_HONEY_BACKEND
168 resolve_relative_path(line
, file
);
169 db
.add_database(Database(new HoneyDatabase(line
)));
172 throw FeatureUnavailableError("Honey backend disabled");
176 if (type
== "remote" && !line
.empty()) {
177 #ifdef XAPIAN_HAS_REMOTE_BACKEND
178 if (line
[0] == ':') {
181 // Is it a security risk?
182 space
= line
.find(' ');
184 if (space
!= string::npos
) {
185 args
.assign(line
, space
+ 1, string::npos
);
186 line
.assign(line
, 1, space
- 1);
190 db
.add_database(Remote::open(line
, args
));
193 string::size_type colon
= line
.rfind(':');
194 if (colon
!= string::npos
) {
197 // Avoid misparsing an IPv6 address without a port number. The
198 // port number is required, so just leave that case to the
199 // error handling further below.
200 if (!(line
[0] == '[' && line
.back() == ']')) {
201 unsigned int port
= atoi(line
.c_str() + colon
+ 1);
203 if (line
[0] == '[' && line
.back() == ']') {
204 line
.erase(line
.size() - 1, 1);
207 db
.add_database(Remote::open(line
, port
));
212 throw FeatureUnavailableError("Remote backend disabled");
216 if (type
== "inmemory" && line
.empty()) {
217 #ifdef XAPIAN_HAS_INMEMORY_BACKEND
218 db
.add_database(Database(string(), DB_BACKEND_INMEMORY
));
221 throw FeatureUnavailableError("Inmemory backend disabled");
225 if (type
== "chert") {
226 throw FeatureUnavailableError("Chert backend no longer supported");
229 if (type
== "flint") {
230 throw FeatureUnavailableError("Flint backend no longer supported");
233 // Don't include the line itself - that might help an attacker
234 // by revealing part of a sensitive file's contents if they can
235 // arrange for it to be read as a stub database via infelicities in
236 // an application which uses Xapian. The line number is enough
237 // information to identify the problem line.
238 throw DatabaseOpeningError(file
+ ':' + str(line_no
) + ": Bad line");
241 // Allowing a stub database with no databases listed allows things like
242 // a "search all databases" feature to be implemented by generating a
243 // stub database file without having to special case there not being any
246 // 1.0.x throws DatabaseOpeningError here, but with a "Bad line" message
247 // with the line number just past the end of the file, which is a bit odd.
251 open_stub(WritableDatabase
&db
, const string
&file
, int flags
)
253 // A stub database is a text file with one or more lines of this format:
254 // <dbtype> <serialised db object>
256 // Lines which start with a "#" character, and lines which have no spaces
257 // in them, are ignored.
259 // Any paths specified in stub database files which are relative will be
260 // considered to be relative to the directory containing the stub database.
261 ifstream
stub(file
.c_str());
263 string msg
= "Couldn't open stub database file: ";
265 throw Xapian::DatabaseOpeningError(msg
, errno
);
268 unsigned int line_no
= 0;
270 if (!getline(stub
, line
)) break;
273 if (line
.empty() || line
[0] == '#')
275 string::size_type space
= line
.find(' ');
276 if (space
== string::npos
) space
= line
.size();
278 string
type(line
, 0, space
);
279 line
.erase(0, space
+ 1);
281 if (type
== "auto") {
282 resolve_relative_path(line
, file
);
283 db
.add_database(WritableDatabase(line
, flags
));
287 if (type
== "glass") {
288 #ifdef XAPIAN_HAS_GLASS_BACKEND
289 resolve_relative_path(line
, file
);
290 db
.add_database(WritableDatabase(line
, flags
|DB_BACKEND_GLASS
));
293 throw FeatureUnavailableError("Glass backend disabled");
297 if (type
== "remote" && !line
.empty()) {
298 #ifdef XAPIAN_HAS_REMOTE_BACKEND
299 if (line
[0] == ':') {
302 // Is it a security risk?
303 space
= line
.find(' ');
305 if (space
!= string::npos
) {
306 args
.assign(line
, space
+ 1, string::npos
);
307 line
.assign(line
, 1, space
- 1);
311 db
.add_database(Remote::open_writable(line
, args
, 0, flags
));
314 string::size_type colon
= line
.rfind(':');
315 if (colon
!= string::npos
) {
318 // Avoid misparsing an IPv6 address without a port number. The
319 // port number is required, so just leave that case to the
320 // error handling further below.
321 if (!(line
[0] == '[' && line
.back() == ']')) {
322 unsigned int port
= atoi(line
.c_str() + colon
+ 1);
324 if (line
[0] == '[' && line
.back() == ']') {
325 line
.erase(line
.size() - 1, 1);
328 db
.add_database(Remote::open_writable(line
, port
, 0, 10000, flags
));
333 throw FeatureUnavailableError("Remote backend disabled");
337 if (type
== "inmemory" && line
.empty()) {
338 #ifdef XAPIAN_HAS_INMEMORY_BACKEND
339 db
.add_database(WritableDatabase(string(), DB_BACKEND_INMEMORY
));
342 throw FeatureUnavailableError("Inmemory backend disabled");
346 if (type
== "chert") {
347 throw FeatureUnavailableError("Chert backend no longer supported");
350 if (type
== "flint") {
351 throw FeatureUnavailableError("Flint backend no longer supported");
354 // Don't include the line itself - that might help an attacker
355 // by revealing part of a sensitive file's contents if they can
356 // arrange for it to be read as a stub database via infelicities in
357 // an application which uses Xapian. The line number is enough
358 // information to identify the problem line.
359 throw DatabaseOpeningError(file
+ ':' + str(line_no
) + ": Bad line");
362 if (db
.internal
->size() == 0) {
363 throw DatabaseOpeningError(file
+ ": No databases listed");
367 Database::Database(const string
& path
, int flags
)
370 LOGCALL_CTOR(API
, "Database", path
|flags
);
372 int type
= flags
& DB_BACKEND_MASK_
;
374 case DB_BACKEND_CHERT
:
375 throw FeatureUnavailableError("Chert backend no longer supported");
376 case DB_BACKEND_GLASS
:
377 #ifdef XAPIAN_HAS_GLASS_BACKEND
378 internal
= new GlassDatabase(path
);
381 throw FeatureUnavailableError("Glass backend disabled");
383 case DB_BACKEND_HONEY
:
384 #ifdef XAPIAN_HAS_HONEY_BACKEND
385 internal
= new HoneyDatabase(path
);
388 throw FeatureUnavailableError("Honey backend disabled");
390 case DB_BACKEND_STUB
:
391 open_stub(*this, path
);
393 case DB_BACKEND_INMEMORY
:
394 #ifdef XAPIAN_HAS_INMEMORY_BACKEND
395 internal
= new InMemoryDatabase();
398 throw FeatureUnavailableError("Inmemory backend disabled");
403 if (stat(path
.c_str(), &statbuf
) == -1) {
404 throw DatabaseOpeningError("Couldn't stat '" + path
+ "'", errno
);
407 if (S_ISREG(statbuf
.st_mode
)) {
408 // Could be a stub database file, or a single file glass database.
410 // Initialise to avoid bogus warning from GCC 4.9.2 with -Os.
412 switch (check_if_single_file_db(statbuf
, path
, &fd
)) {
414 #ifdef XAPIAN_HAS_GLASS_BACKEND
415 // Single file glass format.
416 internal
= new GlassDatabase(fd
);
419 throw FeatureUnavailableError("Glass backend disabled");
422 #ifdef XAPIAN_HAS_HONEY_BACKEND
423 // Single file honey format.
424 internal
= new HoneyDatabase(fd
);
427 throw FeatureUnavailableError("Honey backend disabled");
431 open_stub(*this, path
);
435 if (rare(!S_ISDIR(statbuf
.st_mode
))) {
436 throw DatabaseOpeningError("Not a regular file or directory: '" + path
+ "'");
439 #ifdef XAPIAN_HAS_GLASS_BACKEND
440 if (file_exists(path
+ "/iamglass")) {
441 internal
= new GlassDatabase(path
);
446 #ifdef XAPIAN_HAS_HONEY_BACKEND
447 if (file_exists(path
+ "/iamhoney")) {
448 internal
= new HoneyDatabase(path
);
453 // Check for "stub directories".
454 string stub_file
= path
;
455 stub_file
+= "/XAPIANDB";
456 if (usual(file_exists(stub_file
))) {
457 open_stub(*this, stub_file
);
461 #ifndef XAPIAN_HAS_GLASS_BACKEND
462 if (file_exists(path
+ "/iamglass")) {
463 throw FeatureUnavailableError("Glass backend disabled");
466 #ifndef XAPIAN_HAS_HONEY_BACKEND
467 if (file_exists(path
+ "/iamhoney")) {
468 throw FeatureUnavailableError("Honey backend disabled");
471 if (file_exists(path
+ "/iamchert")) {
472 throw FeatureUnavailableError("Chert backend no longer supported");
474 if (file_exists(path
+ "/iamflint")) {
475 throw FeatureUnavailableError("Flint backend no longer supported");
478 throw DatabaseOpeningError("Couldn't detect type of database");
481 /** Helper factory function.
483 * This allows us to initialise Database::internal via the constructor's
484 * initialiser list, which we want to be able to do as Database::internal
485 * is an intrusive_ptr_nonnull, so we can't set it to NULL in the initialiser
486 * list and then fill it in later in the constructor body.
488 static Database::Internal
*
489 database_factory(int fd
, int flags
)
492 throw InvalidArgumentError("fd < 0");
494 #ifdef XAPIAN_HAS_GLASS_BACKEND
495 int type
= flags
& DB_BACKEND_MASK_
;
498 case DB_BACKEND_GLASS
:
499 return new GlassDatabase(fd
);
506 throw DatabaseOpeningError("Couldn't detect type of database");
509 Database::Database(int fd
, int flags
)
510 : internal(database_factory(fd
, flags
))
512 LOGCALL_CTOR(API
, "Database", fd
|flags
);
515 #if defined XAPIAN_HAS_GLASS_BACKEND
516 #define HAVE_DISK_BACKEND
519 WritableDatabase::WritableDatabase(const std::string
&path
, int flags
, int block_size
)
522 LOGCALL_CTOR(API
, "WritableDatabase", path
|flags
|block_size
);
523 // Avoid warning if all disk-based backends are disabled.
525 int type
= flags
& DB_BACKEND_MASK_
;
526 // Clear the backend bits, so we just pass on other flags to open_stub, etc.
527 flags
&= ~DB_BACKEND_MASK_
;
530 if (stat(path
.c_str(), &statbuf
) == -1) {
531 // ENOENT probably just means that we need to create the directory.
533 throw DatabaseOpeningError("Couldn't stat '" + path
+ "'", errno
);
535 // File or directory already exists.
537 if (S_ISREG(statbuf
.st_mode
)) {
538 // The path is a file, so assume it is a stub database file.
539 open_stub(*this, path
, flags
);
543 if (rare(!S_ISDIR(statbuf
.st_mode
))) {
544 throw DatabaseOpeningError("Not a regular file or directory: '" + path
+ "'");
547 if (file_exists(path
+ "/iamglass")) {
548 // Existing glass DB.
549 #ifdef XAPIAN_HAS_GLASS_BACKEND
550 type
= DB_BACKEND_GLASS
;
552 throw FeatureUnavailableError("Glass backend disabled");
554 } else if (file_exists(path
+ "/iamhoney")) {
555 // Existing honey DB.
556 throw InvalidOperationError("Honey backend doesn't support "
557 "updating existing databases");
558 } else if (file_exists(path
+ "/iamchert")) {
559 // Existing chert DB.
560 throw FeatureUnavailableError("Chert backend no longer supported");
561 } else if (file_exists(path
+ "/iamflint")) {
562 // Existing flint DB.
563 throw FeatureUnavailableError("Flint backend no longer supported");
565 // Check for "stub directories".
566 string stub_file
= path
;
567 stub_file
+= "/XAPIANDB";
568 if (usual(file_exists(stub_file
))) {
569 open_stub(*this, stub_file
, flags
);
577 case DB_BACKEND_STUB
:
578 open_stub(*this, path
, flags
);
581 // Fall through to first enabled case, so order the remaining cases
583 #ifdef XAPIAN_HAS_GLASS_BACKEND
584 case DB_BACKEND_GLASS
:
585 internal
= new GlassWritableDatabase(path
, flags
, block_size
);
588 case DB_BACKEND_HONEY
:
589 throw InvalidArgumentError("Honey backend doesn't support "
590 "updating existing databases");
591 case DB_BACKEND_CHERT
:
592 throw FeatureUnavailableError("Chert backend no longer supported");
593 case DB_BACKEND_INMEMORY
:
594 #ifdef XAPIAN_HAS_INMEMORY_BACKEND
595 internal
= new InMemoryDatabase();
598 throw FeatureUnavailableError("Inmemory backend disabled");
601 #ifndef HAVE_DISK_BACKEND
602 throw FeatureUnavailableError("No disk-based writable backend is enabled");