xapian-core/backends/dbfactory.cc

   1 /** @file dbfactory.cc
   2  * @brief Database factories for non-remote databases.
   3  */
   4 /* Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2011,2012,2013,2014,2015 Olly Betts
   5  * Copyright 2008 Lemur Consulting Ltd
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License as
   9  * published by the Free Software Foundation; either version 2 of the
  10  * License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  20  * USA
  21  */
  22
  23 #include <config.h>
  24
  25 #include "xapian/dbfactory.h"
  26
  27 #include "xapian/constants.h"
  28 #include "xapian/database.h"
  29 #include "xapian/error.h"
  30 #include "xapian/version.h" // For XAPIAN_HAS_XXX_BACKEND.
  31
  32 #include "debuglog.h"
  33 #include "filetests.h"
  34 #include "fileutils.h"
  35 #include "posixy_wrapper.h"
  36 #include "str.h"
  37
  38 #include "safeerrno.h"
  39 #include <cstdlib> // For atoi().
  40
  41 #ifdef XAPIAN_HAS_GLASS_BACKEND
  42 # include "glass/glass_database.h"
  43 #endif
  44 #ifdef XAPIAN_HAS_CHERT_BACKEND
  45 # include "chert/chert_database.h"
  46 #endif
  47 #ifdef XAPIAN_HAS_INMEMORY_BACKEND
  48 # include "inmemory/inmemory_database.h"
  49 #endif
  50 // Even if none of the above get included, we still need a definition of
  51 // Database::Internal.
  52 #include "backends/database.h"
  53
  54 #include <fstream>
  55 #include <string>
  56
  57 using namespace std;
  58
  59 static bool
  60 check_if_single_file_db(const struct stat & sb, const string & path,
  61                         int * fd_ptr = NULL)
  62 {
  63 #ifdef XAPIAN_HAS_GLASS_BACKEND
  64     if (!S_ISREG(sb.st_mode)) return false;
  65     // Look at the size as a clue - if it's 0 or not a multiple of 2048,
  66     // then it's not a single-file glass database.  If it is, peek at the start
  67     // of the file to determine which it is.
  68     if (sb.st_size == 0 || sb.st_size % 2048 != 0) return false;
  69     int fd = posixy_open(path.c_str(), O_RDONLY|O_BINARY);
  70     if (fd != -1) {
  71         char magic_buf[14];
  72         // FIXME: Don't duplicate magic check here...
  73         if (io_read(fd, magic_buf, 14) == 14 &&
  74             lseek(fd, 0, SEEK_SET) == 0 &&
  75             memcmp(magic_buf, "\x0f\x0dXapian Glass", 14) == 0) {
  76             if (fd_ptr) {
  77                 *fd_ptr = fd;
  78             } else {
  79                 ::close(fd);
  80             }
  81             return true;
  82         }
  83         ::close(fd);
  84     }
  85 #else
  86     (void)sb;
  87     (void)path;
  88     (void)fd_ptr;
  89 #endif
  90     return false;
  91 }
  92
  93 namespace Xapian {
  94
  95 #ifdef XAPIAN_HAS_INMEMORY_BACKEND
  96 WritableDatabase
  97 InMemory::open() {
  98     LOGCALL_STATIC(API, WritableDatabase, "InMemory::open", NO_ARGS);
  99     RETURN(WritableDatabase(new InMemoryDatabase));
 100 }
 101 #endif
 102
 103 static void
 104 open_stub(Database &db, const string &file)
 105 {
 106     // A stub database is a text file with one or more lines of this format:
 107     // <dbtype> <serialised db object>
 108     //
 109     // Lines which start with a "#" character are ignored.
 110     //
 111     // Any paths specified in stub database files which are relative will be
 112     // considered to be relative to the directory containing the stub database.
 113     ifstream stub(file.c_str());
 114     if (!stub) {
 115         string msg = "Couldn't open stub database file: ";
 116         msg += file;
 117         throw Xapian::DatabaseOpeningError(msg, errno);
 118     }
 119     string line;
 120     unsigned int line_no = 0;
 121     while (getline(stub, line)) {
 122         ++line_no;
 123         if (line.empty() || line[0] == '#')
 124             continue;
 125         string::size_type space = line.find(' ');
 126         if (space == string::npos) space = line.size();
 127
 128         string type(line, 0, space);
 129         line.erase(0, space + 1);
 130
 131         if (type == "auto") {
 132             resolve_relative_path(line, file);
 133             db.add_database(Database(line));
 134             continue;
 135         }
 136
 137 #ifdef XAPIAN_HAS_CHERT_BACKEND
 138         if (type == "chert") {
 139             resolve_relative_path(line, file);
 140             db.add_database(Database(new ChertDatabase(line)));
 141             continue;
 142         }
 143 #endif
 144
 145 #ifdef XAPIAN_HAS_GLASS_BACKEND
 146         if (type == "glass") {
 147             resolve_relative_path(line, file);
 148             db.add_database(Database(new GlassDatabase(line)));
 149             continue;
 150         }
 151 #endif
 152
 153 #ifdef XAPIAN_HAS_REMOTE_BACKEND
 154         if (type == "remote") {
 155             string::size_type colon = line.find(':');
 156             if (colon == 0) {
 157                 // prog
 158                 // FIXME: timeouts
 159                 // Is it a security risk?
 160                 space = line.find(' ');
 161                 string args;
 162                 if (space != string::npos) {
 163                     args.assign(line, space + 1, string::npos);
 164                     line.assign(line, 1, space - 1);
 165                 } else {
 166                     line.erase(0, 1);
 167                 }
 168                 db.add_database(Remote::open(line, args));
 169             } else if (colon != string::npos) {
 170                 // tcp
 171                 // FIXME: timeouts
 172                 unsigned int port = atoi(line.c_str() + colon + 1);
 173                 line.erase(colon);
 174                 db.add_database(Remote::open(line, port));
 175             }
 176             continue;
 177         }
 178 #endif
 179
 180 #ifdef XAPIAN_HAS_INMEMORY_BACKEND
 181         if (type == "inmemory" && line.empty()) {
 182             db.add_database(InMemory::open());
 183             continue;
 184         }
 185 #endif
 186
 187         if (type == "flint") {
 188             throw FeatureUnavailableError("Flint backend no longer supported");
 189         }
 190
 191         // Don't include the line itself - that might help an attacker
 192         // by revealing part of a sensitive file's contents if they can
 193         // arrange for it to be read as a stub database via infelicities in
 194         // an application which uses Xapian.  The line number is enough
 195         // information to identify the problem line.
 196         throw DatabaseOpeningError(file + ':' + str(line_no) + ": Bad line");
 197     }
 198
 199     // Allowing a stub database with no databases listed allows things like
 200     // a "search all databases" feature to be implemented by generating a
 201     // stub database file without having to special case there not being any
 202     // databases yet.
 203     //
 204     // 1.0.x throws DatabaseOpeningError here, but with a "Bad line" message
 205     // with the line number just past the end of the file, which is a bit odd.
 206 }
 207
 208 static void
 209 open_stub(WritableDatabase &db, const string &file, int flags)
 210 {
 211     // A stub database is a text file with one or more lines of this format:
 212     // <dbtype> <serialised db object>
 213     //
 214     // Lines which start with a "#" character, and lines which have no spaces
 215     // in them, are ignored.
 216     //
 217     // Any paths specified in stub database files which are relative will be
 218     // considered to be relative to the directory containing the stub database.
 219     ifstream stub(file.c_str());
 220     if (!stub) {
 221         string msg = "Couldn't open stub database file: ";
 222         msg += file;
 223         throw Xapian::DatabaseOpeningError(msg, errno);
 224     }
 225     string line;
 226     unsigned int line_no = 0;
 227     while (true) {
 228         if (!getline(stub, line)) break;
 229
 230         ++line_no;
 231         if (line.empty() || line[0] == '#')
 232             continue;
 233         string::size_type space = line.find(' ');
 234         if (space == string::npos) space = line.size();
 235
 236         string type(line, 0, space);
 237         line.erase(0, space + 1);
 238
 239         if (type == "auto") {
 240             resolve_relative_path(line, file);
 241             db.add_database(WritableDatabase(line, flags));
 242             continue;
 243         }
 244
 245 #ifdef XAPIAN_HAS_CHERT_BACKEND
 246         if (type == "chert") {
 247             resolve_relative_path(line, file);
 248             db.add_database(WritableDatabase(line, flags|DB_BACKEND_CHERT));
 249             continue;
 250         }
 251 #endif
 252
 253 #ifdef XAPIAN_HAS_GLASS_BACKEND
 254         if (type == "glass") {
 255             resolve_relative_path(line, file);
 256             db.add_database(WritableDatabase(line, flags|DB_BACKEND_GLASS));
 257             continue;
 258         }
 259 #endif
 260
 261 #ifdef XAPIAN_HAS_REMOTE_BACKEND
 262         if (type == "remote") {
 263             string::size_type colon = line.find(':');
 264             if (colon == 0) {
 265                 // prog
 266                 // FIXME: timeouts
 267                 // Is it a security risk?
 268                 space = line.find(' ');
 269                 string args;
 270                 if (space != string::npos) {
 271                     args.assign(line, space + 1, string::npos);
 272                     line.assign(line, 1, space - 1);
 273                 } else {
 274                     line.erase(0, 1);
 275                 }
 276                 db.add_database(Remote::open_writable(line, args, 0, flags));
 277             } else if (colon != string::npos) {
 278                 // tcp
 279                 // FIXME: timeouts
 280                 unsigned int port = atoi(line.c_str() + colon + 1);
 281                 line.erase(colon);
 282                 db.add_database(Remote::open_writable(line, port, 0, 10000, flags));
 283             }
 284             continue;
 285         }
 286 #endif
 287
 288 #ifdef XAPIAN_HAS_INMEMORY_BACKEND
 289         if (type == "inmemory" && line.empty()) {
 290             db.add_database(InMemory::open());
 291             continue;
 292         }
 293 #endif
 294
 295         if (type == "flint") {
 296             throw FeatureUnavailableError("Flint backend no longer supported");
 297         }
 298
 299         // Don't include the line itself - that might help an attacker
 300         // by revealing part of a sensitive file's contents if they can
 301         // arrange for it to be read as a stub database via infelicities in
 302         // an application which uses Xapian.  The line number is enough
 303         // information to identify the problem line.
 304         throw DatabaseOpeningError(file + ':' + str(line_no) + ": Bad line");
 305     }
 306
 307     if (db.internal.empty()) {
 308         throw DatabaseOpeningError(file + ": No databases listed");
 309     }
 310 }
 311
 312 Database::Database(const string &path, int flags)
 313 {
 314     LOGCALL_CTOR(API, "Database", path|flags);
 315
 316     int type = flags & DB_BACKEND_MASK_;
 317     switch (type) {
 318         case DB_BACKEND_CHERT:
 319 #ifdef XAPIAN_HAS_CHERT_BACKEND
 320             internal.push_back(new ChertDatabase(path));
 321             return;
 322 #else
 323             throw FeatureUnavailableError("Chert backend disabled");
 324 #endif
 325         case DB_BACKEND_GLASS:
 326 #ifdef XAPIAN_HAS_GLASS_BACKEND
 327             internal.push_back(new GlassDatabase(path));
 328             return;
 329 #else
 330             throw FeatureUnavailableError("Glass backend disabled");
 331 #endif
 332         case DB_BACKEND_STUB:
 333             open_stub(*this, path);
 334             return;
 335     }
 336
 337     struct stat statbuf;
 338     if (stat(path.c_str(), &statbuf) == -1) {
 339         throw DatabaseOpeningError("Couldn't stat '" + path + "'", errno);
 340     }
 341
 342     if (S_ISREG(statbuf.st_mode)) {
 343         // Could be a stub database file, or a single file glass database.
 344         int fd;
 345         if (check_if_single_file_db(statbuf, path, &fd)) {
 346             // Single file glass format.
 347             internal.push_back(new GlassDatabase(fd));
 348             return;
 349         }
 350
 351         open_stub(*this, path);
 352         return;
 353     }
 354
 355     if (rare(!S_ISDIR(statbuf.st_mode))) {
 356         throw DatabaseOpeningError("Not a regular file or directory: '" + path + "'");
 357     }
 358
 359 #ifdef XAPIAN_HAS_CHERT_BACKEND
 360     if (file_exists(path + "/iamchert")) {
 361         internal.push_back(new ChertDatabase(path));
 362         return;
 363     }
 364 #endif
 365
 366 #ifdef XAPIAN_HAS_GLASS_BACKEND
 367     if (file_exists(path + "/iamglass")) {
 368         internal.push_back(new GlassDatabase(path));
 369         return;
 370     }
 371 #endif
 372
 373     // Check for "stub directories".
 374     string stub_file = path;
 375     stub_file += "/XAPIANDB";
 376     if (usual(file_exists(stub_file))) {
 377         open_stub(*this, stub_file);
 378         return;
 379     }
 380
 381 #ifndef XAPIAN_HAS_CHERT_BACKEND
 382     if (file_exists(path + "/iamchert")) {
 383         throw FeatureUnavailableError("Chert backend disabled");
 384     }
 385 #endif
 386 #ifndef XAPIAN_HAS_GLASS_BACKEND
 387     if (file_exists(path + "/iamglass")) {
 388         throw FeatureUnavailableError("Glass backend disabled");
 389     }
 390 #endif
 391     if (file_exists(path + "/iamflint")) {
 392         throw FeatureUnavailableError("Flint backend no longer supported");
 393     }
 394
 395     throw DatabaseOpeningError("Couldn't detect type of database");
 396 }
 397
 398 Database::Database(int fd, int flags)
 399 {
 400     LOGCALL_CTOR(API, "Database", fd|flags);
 401     if (rare(fd < 0))
 402         throw InvalidArgumentError("fd < 0");
 403
 404 #ifdef XAPIAN_HAS_GLASS_BACKEND
 405     int type = flags & DB_BACKEND_MASK_;
 406     switch (type) {
 407         case 0: case DB_BACKEND_GLASS:
 408             internal.push_back(new GlassDatabase(fd));
 409             return;
 410     }
 411 #else
 412     (void)flags;
 413 #endif
 414
 415     (void)::close(fd);
 416     throw DatabaseOpeningError("Couldn't detect type of database");
 417 }
 418
 419 #if defined XAPIAN_HAS_CHERT_BACKEND || \
 420     defined XAPIAN_HAS_GLASS_BACKEND
 421 #define HAVE_DISK_BACKEND
 422 #endif
 423
 424 WritableDatabase::WritableDatabase(const std::string &path, int flags, int block_size)
 425     : Database()
 426 {
 427     LOGCALL_CTOR(API, "WritableDatabase", path|flags|block_size);
 428     // Avoid warning if both chert and glass are disabled.
 429     (void)block_size;
 430     int type = flags & DB_BACKEND_MASK_;
 431     // Clear the backend bits, so we just pass on other flags to open_stub, etc.
 432     flags &= ~DB_BACKEND_MASK_;
 433     if (type == 0) {
 434         struct stat statbuf;
 435         if (stat(path.c_str(), &statbuf) == -1) {
 436             // ENOENT probably just means that we need to create the directory.
 437             if (errno != ENOENT)
 438                 throw DatabaseOpeningError("Couldn't stat '" + path + "'", errno);
 439         } else {
 440             // File or directory already exists.
 441
 442             if (S_ISREG(statbuf.st_mode)) {
 443                 // The path is a file, so assume it is a stub database file.
 444                 open_stub(*this, path, flags);
 445                 return;
 446             }
 447
 448             if (rare(!S_ISDIR(statbuf.st_mode))) {
 449                 throw DatabaseOpeningError("Not a regular file or directory: '" + path + "'");
 450             }
 451
 452             if (file_exists(path + "/iamchert")) {
 453                 // Existing chert DB.
 454 #ifdef XAPIAN_HAS_CHERT_BACKEND
 455                 type = DB_BACKEND_CHERT;
 456 #else
 457                 throw FeatureUnavailableError("Chert backend disabled");
 458 #endif
 459             } else if (file_exists(path + "/iamglass")) {
 460                 // Existing glass DB.
 461 #ifdef XAPIAN_HAS_GLASS_BACKEND
 462                 type = DB_BACKEND_GLASS;
 463 #else
 464                 throw FeatureUnavailableError("Glass backend disabled");
 465 #endif
 466             } else if (file_exists(path + "/iamflint")) {
 467                 // Existing flint DB.
 468                 throw FeatureUnavailableError("Flint backend no longer supported");
 469             } else {
 470                 // Check for "stub directories".
 471                 string stub_file = path;
 472                 stub_file += "/XAPIANDB";
 473                 if (usual(file_exists(stub_file))) {
 474                     open_stub(*this, stub_file, flags);
 475                     return;
 476                 }
 477             }
 478         }
 479     }
 480
 481     switch (type) {
 482         case DB_BACKEND_STUB:
 483             open_stub(*this, path, flags);
 484             return;
 485         case 0:
 486             // Fall through to first enabled case, so order the remaining cases
 487             // by preference.
 488 #ifdef XAPIAN_HAS_GLASS_BACKEND
 489         case DB_BACKEND_GLASS:
 490             internal.push_back(new GlassWritableDatabase(path, flags, block_size));
 491             return;
 492 #endif
 493 #ifdef XAPIAN_HAS_CHERT_BACKEND
 494         case DB_BACKEND_CHERT:
 495             internal.push_back(new ChertWritableDatabase(path, flags, block_size));
 496             return;
 497 #endif
 498     }
 499 #ifndef HAVE_DISK_BACKEND
 500     throw FeatureUnavailableError("No disk-based writable backend is enabled");
 501 #endif
 502 }
 503
 504 }