xapian-core/backends/dbfactory.cc

   1 /** @file dbfactory.cc
   2  * @brief Database factories for non-remote databases.
   3  */
   4 /* Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2011,2012,2013,2014,2015,2016,2017 Olly Betts
   5  * Copyright 2008 Lemur Consulting Ltd
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License as
   9  * published by the Free Software Foundation; either version 2 of the
  10  * License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  20  * USA
  21  */
  22
  23 #include <config.h>
  24
  25 #include "xapian/dbfactory.h"
  26
  27 #include "xapian/constants.h"
  28 #include "xapian/database.h"
  29 #include "xapian/error.h"
  30 #include "xapian/version.h" // For XAPIAN_HAS_XXX_BACKEND.
  31
  32 #include "debuglog.h"
  33 #include "filetests.h"
  34 #include "fileutils.h"
  35 #include "posixy_wrapper.h"
  36 #include "str.h"
  37
  38 #include "safeerrno.h"
  39 #include <cstdlib> // For atoi().
  40
  41 #ifdef XAPIAN_HAS_GLASS_BACKEND
  42 # include "glass/glass_database.h"
  43 # include "glass/glass_defs.h"
  44 #endif
  45 #ifdef XAPIAN_HAS_INMEMORY_BACKEND
  46 # include "inmemory/inmemory_database.h"
  47 #endif
  48 // Even if none of the above get included, we still need a definition of
  49 // Database::Internal.
  50 #include "backends/databaseinternal.h"
  51
  52 #include <fstream>
  53 #include <string>
  54
  55 using namespace std;
  56
  57 static bool
  58 check_if_single_file_db(const struct stat & sb, const string & path,
  59                         int * fd_ptr = NULL)
  60 {
  61 #ifdef XAPIAN_HAS_GLASS_BACKEND
  62     if (!S_ISREG(sb.st_mode)) return false;
  63     // Look at the size as a clue - if it's 0 or not a multiple of
  64     // GLASS_MIN_BLOCKSIZE, then it's not a single-file glass database.  If it
  65     // is, peek at the start of the file to determine which it is.
  66     if (sb.st_size == 0 || sb.st_size % GLASS_MIN_BLOCKSIZE != 0)
  67         return false;
  68     int fd = posixy_open(path.c_str(), O_RDONLY|O_BINARY);
  69     if (fd != -1) {
  70         char magic_buf[14];
  71         // FIXME: Don't duplicate magic check here...
  72         if (io_read(fd, magic_buf, 14) == 14 &&
  73             lseek(fd, 0, SEEK_SET) == 0 &&
  74             memcmp(magic_buf, "\x0f\x0dXapian Glass", 14) == 0) {
  75             if (fd_ptr) {
  76                 *fd_ptr = fd;
  77             } else {
  78                 ::close(fd);
  79             }
  80             return true;
  81         }
  82         ::close(fd);
  83     }
  84 #else
  85     (void)sb;
  86     (void)path;
  87     (void)fd_ptr;
  88 #endif
  89     return false;
  90 }
  91
  92 namespace Xapian {
  93
  94 static void
  95 open_stub(Database &db, const string &file)
  96 {
  97     // A stub database is a text file with one or more lines of this format:
  98     // <dbtype> <serialised db object>
  99     //
 100     // Lines which start with a "#" character are ignored.
 101     //
 102     // Any paths specified in stub database files which are relative will be
 103     // considered to be relative to the directory containing the stub database.
 104     ifstream stub(file.c_str());
 105     if (!stub) {
 106         string msg = "Couldn't open stub database file: ";
 107         msg += file;
 108         throw Xapian::DatabaseOpeningError(msg, errno);
 109     }
 110     string line;
 111     unsigned int line_no = 0;
 112     while (getline(stub, line)) {
 113         ++line_no;
 114         if (line.empty() || line[0] == '#')
 115             continue;
 116         string::size_type space = line.find(' ');
 117         if (space == string::npos) space = line.size();
 118
 119         string type(line, 0, space);
 120         line.erase(0, space + 1);
 121
 122         if (type == "auto") {
 123             resolve_relative_path(line, file);
 124             db.add_database(Database(line));
 125             continue;
 126         }
 127
 128         if (type == "glass") {
 129 #ifdef XAPIAN_HAS_GLASS_BACKEND
 130             resolve_relative_path(line, file);
 131             db.add_database(Database(new GlassDatabase(line)));
 132             continue;
 133 #else
 134             throw FeatureUnavailableError("Glass backend disabled");
 135 #endif
 136         }
 137
 138         if (type == "remote" && !line.empty()) {
 139 #ifdef XAPIAN_HAS_REMOTE_BACKEND
 140             if (line[0] == ':') {
 141                 // prog
 142                 // FIXME: timeouts
 143                 // Is it a security risk?
 144                 space = line.find(' ');
 145                 string args;
 146                 if (space != string::npos) {
 147                     args.assign(line, space + 1, string::npos);
 148                     line.assign(line, 1, space - 1);
 149                 } else {
 150                     line.erase(0, 1);
 151                 }
 152                 db.add_database(Remote::open(line, args));
 153                 continue;
 154             }
 155             string::size_type colon = line.rfind(':');
 156             if (colon != string::npos) {
 157                 // tcp
 158                 // FIXME: timeouts
 159                 // Avoid misparsing an IPv6 address without a port number.  The
 160                 // port number is required, so just leave that case to the
 161                 // error handling further below.
 162                 if (!(line[0] == '[' && line.back() == ']')) {
 163                     unsigned int port = atoi(line.c_str() + colon + 1);
 164                     line.erase(colon);
 165                     if (line[0] == '[' && line.back() == ']') {
 166                         line.erase(line.size() - 1, 1);
 167                         line.erase(0, 1);
 168                     }
 169                     db.add_database(Remote::open(line, port));
 170                     continue;
 171                 }
 172             }
 173 #else
 174             throw FeatureUnavailableError("Remote backend disabled");
 175 #endif
 176         }
 177
 178         if (type == "inmemory" && line.empty()) {
 179 #ifdef XAPIAN_HAS_INMEMORY_BACKEND
 180             db.add_database(Database(string(), DB_BACKEND_INMEMORY));
 181             continue;
 182 #else
 183             throw FeatureUnavailableError("Inmemory backend disabled");
 184 #endif
 185         }
 186
 187         if (type == "chert") {
 188             throw FeatureUnavailableError("Chert backend no longer supported");
 189         }
 190
 191         if (type == "flint") {
 192             throw FeatureUnavailableError("Flint backend no longer supported");
 193         }
 194
 195         // Don't include the line itself - that might help an attacker
 196         // by revealing part of a sensitive file's contents if they can
 197         // arrange for it to be read as a stub database via infelicities in
 198         // an application which uses Xapian.  The line number is enough
 199         // information to identify the problem line.
 200         throw DatabaseOpeningError(file + ':' + str(line_no) + ": Bad line");
 201     }
 202
 203     // Allowing a stub database with no databases listed allows things like
 204     // a "search all databases" feature to be implemented by generating a
 205     // stub database file without having to special case there not being any
 206     // databases yet.
 207     //
 208     // 1.0.x throws DatabaseOpeningError here, but with a "Bad line" message
 209     // with the line number just past the end of the file, which is a bit odd.
 210 }
 211
 212 static void
 213 open_stub(WritableDatabase &db, const string &file, int flags)
 214 {
 215     // A stub database is a text file with one or more lines of this format:
 216     // <dbtype> <serialised db object>
 217     //
 218     // Lines which start with a "#" character, and lines which have no spaces
 219     // in them, are ignored.
 220     //
 221     // Any paths specified in stub database files which are relative will be
 222     // considered to be relative to the directory containing the stub database.
 223     ifstream stub(file.c_str());
 224     if (!stub) {
 225         string msg = "Couldn't open stub database file: ";
 226         msg += file;
 227         throw Xapian::DatabaseOpeningError(msg, errno);
 228     }
 229     string line;
 230     unsigned int line_no = 0;
 231     while (true) {
 232         if (!getline(stub, line)) break;
 233
 234         ++line_no;
 235         if (line.empty() || line[0] == '#')
 236             continue;
 237         string::size_type space = line.find(' ');
 238         if (space == string::npos) space = line.size();
 239
 240         string type(line, 0, space);
 241         line.erase(0, space + 1);
 242
 243         if (type == "auto") {
 244             resolve_relative_path(line, file);
 245             db.add_database(WritableDatabase(line, flags));
 246             continue;
 247         }
 248
 249         if (type == "glass") {
 250 #ifdef XAPIAN_HAS_GLASS_BACKEND
 251             resolve_relative_path(line, file);
 252             db.add_database(WritableDatabase(line, flags|DB_BACKEND_GLASS));
 253             continue;
 254 #else
 255             throw FeatureUnavailableError("Glass backend disabled");
 256 #endif
 257         }
 258
 259         if (type == "remote" && !line.empty()) {
 260 #ifdef XAPIAN_HAS_REMOTE_BACKEND
 261             if (line[0] == ':') {
 262                 // prog
 263                 // FIXME: timeouts
 264                 // Is it a security risk?
 265                 space = line.find(' ');
 266                 string args;
 267                 if (space != string::npos) {
 268                     args.assign(line, space + 1, string::npos);
 269                     line.assign(line, 1, space - 1);
 270                 } else {
 271                     line.erase(0, 1);
 272                 }
 273                 db.add_database(Remote::open_writable(line, args, 0, flags));
 274                 continue;
 275             }
 276             string::size_type colon = line.rfind(':');
 277             if (colon != string::npos) {
 278                 // tcp
 279                 // FIXME: timeouts
 280                 // Avoid misparsing an IPv6 address without a port number.  The
 281                 // port number is required, so just leave that case to the
 282                 // error handling further below.
 283                 if (!(line[0] == '[' && line.back() == ']')) {
 284                     unsigned int port = atoi(line.c_str() + colon + 1);
 285                     line.erase(colon);
 286                     if (line[0] == '[' && line.back() == ']') {
 287                         line.erase(line.size() - 1, 1);
 288                         line.erase(0, 1);
 289                     }
 290                     db.add_database(Remote::open_writable(line, port, 0, 10000, flags));
 291                     continue;
 292                 }
 293             }
 294 #else
 295             throw FeatureUnavailableError("Remote backend disabled");
 296 #endif
 297         }
 298
 299         if (type == "inmemory" && line.empty()) {
 300 #ifdef XAPIAN_HAS_INMEMORY_BACKEND
 301             db.add_database(WritableDatabase(string(), DB_BACKEND_INMEMORY));
 302             continue;
 303 #else
 304             throw FeatureUnavailableError("Inmemory backend disabled");
 305 #endif
 306         }
 307
 308         if (type == "chert") {
 309             throw FeatureUnavailableError("Chert backend no longer supported");
 310         }
 311
 312         if (type == "flint") {
 313             throw FeatureUnavailableError("Flint backend no longer supported");
 314         }
 315
 316         // Don't include the line itself - that might help an attacker
 317         // by revealing part of a sensitive file's contents if they can
 318         // arrange for it to be read as a stub database via infelicities in
 319         // an application which uses Xapian.  The line number is enough
 320         // information to identify the problem line.
 321         throw DatabaseOpeningError(file + ':' + str(line_no) + ": Bad line");
 322     }
 323
 324     if (db.internal->size() == 0) {
 325         throw DatabaseOpeningError(file + ": No databases listed");
 326     }
 327 }
 328
 329 Database::Database(const string& path, int flags)
 330     : Database()
 331 {
 332     LOGCALL_CTOR(API, "Database", path|flags);
 333
 334     int type = flags & DB_BACKEND_MASK_;
 335     switch (type) {
 336         case DB_BACKEND_CHERT:
 337             throw FeatureUnavailableError("Chert backend no longer supported");
 338         case DB_BACKEND_GLASS:
 339 #ifdef XAPIAN_HAS_GLASS_BACKEND
 340             internal = new GlassDatabase(path);
 341             return;
 342 #else
 343             throw FeatureUnavailableError("Glass backend disabled");
 344 #endif
 345         case DB_BACKEND_STUB:
 346             open_stub(*this, path);
 347             return;
 348         case DB_BACKEND_INMEMORY:
 349 #ifdef XAPIAN_HAS_INMEMORY_BACKEND
 350             internal = new InMemoryDatabase();
 351             return;
 352 #else
 353             throw FeatureUnavailableError("Inmemory backend disabled");
 354 #endif
 355     }
 356
 357     struct stat statbuf;
 358     if (stat(path.c_str(), &statbuf) == -1) {
 359         throw DatabaseOpeningError("Couldn't stat '" + path + "'", errno);
 360     }
 361
 362     if (S_ISREG(statbuf.st_mode)) {
 363         // Could be a stub database file, or a single file glass database.
 364         int fd;
 365         if (check_if_single_file_db(statbuf, path, &fd)) {
 366 #ifdef XAPIAN_HAS_GLASS_BACKEND
 367             // Single file glass format.
 368             internal = new GlassDatabase(fd);
 369             return;
 370 #else
 371             throw FeatureUnavailableError("Glass backend disabled");
 372 #endif
 373         }
 374
 375         open_stub(*this, path);
 376         return;
 377     }
 378
 379     if (rare(!S_ISDIR(statbuf.st_mode))) {
 380         throw DatabaseOpeningError("Not a regular file or directory: '" + path + "'");
 381     }
 382
 383 #ifdef XAPIAN_HAS_GLASS_BACKEND
 384     if (file_exists(path + "/iamglass")) {
 385         internal = new GlassDatabase(path);
 386         return;
 387     }
 388 #endif
 389
 390     // Check for "stub directories".
 391     string stub_file = path;
 392     stub_file += "/XAPIANDB";
 393     if (usual(file_exists(stub_file))) {
 394         open_stub(*this, stub_file);
 395         return;
 396     }
 397
 398 #ifndef XAPIAN_HAS_GLASS_BACKEND
 399     if (file_exists(path + "/iamglass")) {
 400         throw FeatureUnavailableError("Glass backend disabled");
 401     }
 402 #endif
 403     if (file_exists(path + "/iamchert")) {
 404         throw FeatureUnavailableError("Chert backend no longer supported");
 405     }
 406     if (file_exists(path + "/iamflint")) {
 407         throw FeatureUnavailableError("Flint backend no longer supported");
 408     }
 409
 410     throw DatabaseOpeningError("Couldn't detect type of database");
 411 }
 412
 413 /** Helper factory function.
 414  *
 415  *  This allows us to initialise Database::internal via the constructor's
 416  *  initialiser list, which we want to be able to do as Database::internal
 417  *  is an intrusive_ptr_nonnull, so we can't set it to NULL in the initialiser
 418  *  list and then fill it in later in the constructor body.
 419  */
 420 static Database::Internal*
 421 database_factory(int fd, int flags)
 422 {
 423     if (rare(fd < 0))
 424         throw InvalidArgumentError("fd < 0");
 425
 426 #ifdef XAPIAN_HAS_GLASS_BACKEND
 427     int type = flags & DB_BACKEND_MASK_;
 428     switch (type) {
 429         case 0: case DB_BACKEND_GLASS:
 430             return new GlassDatabase(fd);
 431     }
 432 #else
 433     (void)flags;
 434 #endif
 435
 436     (void)::close(fd);
 437     throw DatabaseOpeningError("Couldn't detect type of database");
 438 }
 439
 440 Database::Database(int fd, int flags)
 441     : internal(database_factory(fd, flags))
 442 {
 443     LOGCALL_CTOR(API, "Database", fd|flags);
 444 }
 445
 446 #if defined XAPIAN_HAS_GLASS_BACKEND
 447 #define HAVE_DISK_BACKEND
 448 #endif
 449
 450 WritableDatabase::WritableDatabase(const std::string &path, int flags, int block_size)
 451     : Database()
 452 {
 453     LOGCALL_CTOR(API, "WritableDatabase", path|flags|block_size);
 454     // Avoid warning if all disk-based backends are disabled.
 455     (void)block_size;
 456     int type = flags & DB_BACKEND_MASK_;
 457     // Clear the backend bits, so we just pass on other flags to open_stub, etc.
 458     flags &= ~DB_BACKEND_MASK_;
 459     if (type == 0) {
 460         struct stat statbuf;
 461         if (stat(path.c_str(), &statbuf) == -1) {
 462             // ENOENT probably just means that we need to create the directory.
 463             if (errno != ENOENT)
 464                 throw DatabaseOpeningError("Couldn't stat '" + path + "'", errno);
 465         } else {
 466             // File or directory already exists.
 467
 468             if (S_ISREG(statbuf.st_mode)) {
 469                 // The path is a file, so assume it is a stub database file.
 470                 open_stub(*this, path, flags);
 471                 return;
 472             }
 473
 474             if (rare(!S_ISDIR(statbuf.st_mode))) {
 475                 throw DatabaseOpeningError("Not a regular file or directory: '" + path + "'");
 476             }
 477
 478             if (file_exists(path + "/iamglass")) {
 479                 // Existing glass DB.
 480 #ifdef XAPIAN_HAS_GLASS_BACKEND
 481                 type = DB_BACKEND_GLASS;
 482 #else
 483                 throw FeatureUnavailableError("Glass backend disabled");
 484 #endif
 485             } else if (file_exists(path + "/iamchert")) {
 486                 // Existing chert DB.
 487                 throw FeatureUnavailableError("Chert backend no longer supported");
 488             } else if (file_exists(path + "/iamflint")) {
 489                 // Existing flint DB.
 490                 throw FeatureUnavailableError("Flint backend no longer supported");
 491             } else {
 492                 // Check for "stub directories".
 493                 string stub_file = path;
 494                 stub_file += "/XAPIANDB";
 495                 if (usual(file_exists(stub_file))) {
 496                     open_stub(*this, stub_file, flags);
 497                     return;
 498                 }
 499             }
 500         }
 501     }
 502
 503     switch (type) {
 504         case DB_BACKEND_STUB:
 505             open_stub(*this, path, flags);
 506             return;
 507         case 0:
 508             // Fall through to first enabled case, so order the remaining cases
 509             // by preference.
 510 #ifdef XAPIAN_HAS_GLASS_BACKEND
 511         case DB_BACKEND_GLASS:
 512             internal = new GlassWritableDatabase(path, flags, block_size);
 513             return;
 514 #endif
 515         case DB_BACKEND_CHERT:
 516             throw FeatureUnavailableError("Chert backend no longer supported");
 517         case DB_BACKEND_INMEMORY:
 518 #ifdef XAPIAN_HAS_INMEMORY_BACKEND
 519             internal = new InMemoryDatabase();
 520             return;
 521 #else
 522             throw FeatureUnavailableError("Inmemory backend disabled");
 523 #endif
 524     }
 525 #ifndef HAVE_DISK_BACKEND
 526     throw FeatureUnavailableError("No disk-based writable backend is enabled");
 527 #endif
 528 }
 529
 530 }