xapian-core/backends/dbfactory.cc

   1 /** @file dbfactory.cc
   2  * @brief Database factories for non-remote databases.
   3  */
   4 /* Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2011,2012,2013,2014,2015,2016 Olly Betts
   5  * Copyright 2008 Lemur Consulting Ltd
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License as
   9  * published by the Free Software Foundation; either version 2 of the
  10  * License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  20  * USA
  21  */
  22
  23 #include <config.h>
  24
  25 #include "xapian/dbfactory.h"
  26
  27 #include "xapian/constants.h"
  28 #include "xapian/database.h"
  29 #include "xapian/error.h"
  30 #include "xapian/version.h" // For XAPIAN_HAS_XXX_BACKEND.
  31
  32 #include "debuglog.h"
  33 #include "filetests.h"
  34 #include "fileutils.h"
  35 #include "posixy_wrapper.h"
  36 #include "str.h"
  37
  38 #include "safeerrno.h"
  39 #include <cstdlib> // For atoi().
  40
  41 #ifdef XAPIAN_HAS_GLASS_BACKEND
  42 # include "glass/glass_database.h"
  43 #endif
  44 #ifdef XAPIAN_HAS_CHERT_BACKEND
  45 # include "chert/chert_database.h"
  46 #endif
  47 #ifdef XAPIAN_HAS_INMEMORY_BACKEND
  48 # include "inmemory/inmemory_database.h"
  49 #endif
  50 // Even if none of the above get included, we still need a definition of
  51 // Database::Internal.
  52 #include "backends/database.h"
  53
  54 #include <fstream>
  55 #include <string>
  56
  57 using namespace std;
  58
  59 static bool
  60 check_if_single_file_db(const struct stat & sb, const string & path,
  61                         int * fd_ptr = NULL)
  62 {
  63 #ifdef XAPIAN_HAS_GLASS_BACKEND
  64     if (!S_ISREG(sb.st_mode)) return false;
  65     // Look at the size as a clue - if it's 0 or not a multiple of 2048,
  66     // then it's not a single-file glass database.  If it is, peek at the start
  67     // of the file to determine which it is.
  68     if (sb.st_size == 0 || sb.st_size % 2048 != 0) return false;
  69     int fd = posixy_open(path.c_str(), O_RDONLY|O_BINARY);
  70     if (fd != -1) {
  71         char magic_buf[14];
  72         // FIXME: Don't duplicate magic check here...
  73         if (io_read(fd, magic_buf, 14) == 14 &&
  74             lseek(fd, 0, SEEK_SET) == 0 &&
  75             memcmp(magic_buf, "\x0f\x0dXapian Glass", 14) == 0) {
  76             if (fd_ptr) {
  77                 *fd_ptr = fd;
  78             } else {
  79                 ::close(fd);
  80             }
  81             return true;
  82         }
  83         ::close(fd);
  84     }
  85 #else
  86     (void)sb;
  87     (void)path;
  88     (void)fd_ptr;
  89 #endif
  90     return false;
  91 }
  92
  93 namespace Xapian {
  94
  95 static void
  96 open_stub(Database &db, const string &file)
  97 {
  98     // A stub database is a text file with one or more lines of this format:
  99     // <dbtype> <serialised db object>
 100     //
 101     // Lines which start with a "#" character are ignored.
 102     //
 103     // Any paths specified in stub database files which are relative will be
 104     // considered to be relative to the directory containing the stub database.
 105     ifstream stub(file.c_str());
 106     if (!stub) {
 107         string msg = "Couldn't open stub database file: ";
 108         msg += file;
 109         throw Xapian::DatabaseOpeningError(msg, errno);
 110     }
 111     string line;
 112     unsigned int line_no = 0;
 113     while (getline(stub, line)) {
 114         ++line_no;
 115         if (line.empty() || line[0] == '#')
 116             continue;
 117         string::size_type space = line.find(' ');
 118         if (space == string::npos) space = line.size();
 119
 120         string type(line, 0, space);
 121         line.erase(0, space + 1);
 122
 123         if (type == "auto") {
 124             resolve_relative_path(line, file);
 125             db.add_database(Database(line));
 126             continue;
 127         }
 128
 129 #ifdef XAPIAN_HAS_CHERT_BACKEND
 130         if (type == "chert") {
 131             resolve_relative_path(line, file);
 132             db.add_database(Database(new ChertDatabase(line)));
 133             continue;
 134         }
 135 #endif
 136
 137 #ifdef XAPIAN_HAS_GLASS_BACKEND
 138         if (type == "glass") {
 139             resolve_relative_path(line, file);
 140             db.add_database(Database(new GlassDatabase(line)));
 141             continue;
 142         }
 143 #endif
 144
 145 #ifdef XAPIAN_HAS_REMOTE_BACKEND
 146         if (type == "remote") {
 147             string::size_type colon = line.find(':');
 148             if (colon == 0) {
 149                 // prog
 150                 // FIXME: timeouts
 151                 // Is it a security risk?
 152                 space = line.find(' ');
 153                 string args;
 154                 if (space != string::npos) {
 155                     args.assign(line, space + 1, string::npos);
 156                     line.assign(line, 1, space - 1);
 157                 } else {
 158                     line.erase(0, 1);
 159                 }
 160                 db.add_database(Remote::open(line, args));
 161             } else if (colon != string::npos) {
 162                 // tcp
 163                 // FIXME: timeouts
 164                 unsigned int port = atoi(line.c_str() + colon + 1);
 165                 line.erase(colon);
 166                 db.add_database(Remote::open(line, port));
 167             }
 168             continue;
 169         }
 170 #endif
 171
 172 #ifdef XAPIAN_HAS_INMEMORY_BACKEND
 173         if (type == "inmemory" && line.empty()) {
 174             db.add_database(Database(string(), DB_BACKEND_INMEMORY));
 175             continue;
 176         }
 177 #endif
 178
 179         if (type == "flint") {
 180             throw FeatureUnavailableError("Flint backend no longer supported");
 181         }
 182
 183         // Don't include the line itself - that might help an attacker
 184         // by revealing part of a sensitive file's contents if they can
 185         // arrange for it to be read as a stub database via infelicities in
 186         // an application which uses Xapian.  The line number is enough
 187         // information to identify the problem line.
 188         throw DatabaseOpeningError(file + ':' + str(line_no) + ": Bad line");
 189     }
 190
 191     // Allowing a stub database with no databases listed allows things like
 192     // a "search all databases" feature to be implemented by generating a
 193     // stub database file without having to special case there not being any
 194     // databases yet.
 195     //
 196     // 1.0.x throws DatabaseOpeningError here, but with a "Bad line" message
 197     // with the line number just past the end of the file, which is a bit odd.
 198 }
 199
 200 static void
 201 open_stub(WritableDatabase &db, const string &file, int flags)
 202 {
 203     // A stub database is a text file with one or more lines of this format:
 204     // <dbtype> <serialised db object>
 205     //
 206     // Lines which start with a "#" character, and lines which have no spaces
 207     // in them, are ignored.
 208     //
 209     // Any paths specified in stub database files which are relative will be
 210     // considered to be relative to the directory containing the stub database.
 211     ifstream stub(file.c_str());
 212     if (!stub) {
 213         string msg = "Couldn't open stub database file: ";
 214         msg += file;
 215         throw Xapian::DatabaseOpeningError(msg, errno);
 216     }
 217     string line;
 218     unsigned int line_no = 0;
 219     while (true) {
 220         if (!getline(stub, line)) break;
 221
 222         ++line_no;
 223         if (line.empty() || line[0] == '#')
 224             continue;
 225         string::size_type space = line.find(' ');
 226         if (space == string::npos) space = line.size();
 227
 228         string type(line, 0, space);
 229         line.erase(0, space + 1);
 230
 231         if (type == "auto") {
 232             resolve_relative_path(line, file);
 233             db.add_database(WritableDatabase(line, flags));
 234             continue;
 235         }
 236
 237 #ifdef XAPIAN_HAS_CHERT_BACKEND
 238         if (type == "chert") {
 239             resolve_relative_path(line, file);
 240             db.add_database(WritableDatabase(line, flags|DB_BACKEND_CHERT));
 241             continue;
 242         }
 243 #endif
 244
 245 #ifdef XAPIAN_HAS_GLASS_BACKEND
 246         if (type == "glass") {
 247             resolve_relative_path(line, file);
 248             db.add_database(WritableDatabase(line, flags|DB_BACKEND_GLASS));
 249             continue;
 250         }
 251 #endif
 252
 253 #ifdef XAPIAN_HAS_REMOTE_BACKEND
 254         if (type == "remote") {
 255             string::size_type colon = line.find(':');
 256             if (colon == 0) {
 257                 // prog
 258                 // FIXME: timeouts
 259                 // Is it a security risk?
 260                 space = line.find(' ');
 261                 string args;
 262                 if (space != string::npos) {
 263                     args.assign(line, space + 1, string::npos);
 264                     line.assign(line, 1, space - 1);
 265                 } else {
 266                     line.erase(0, 1);
 267                 }
 268                 db.add_database(Remote::open_writable(line, args, 0, flags));
 269             } else if (colon != string::npos) {
 270                 // tcp
 271                 // FIXME: timeouts
 272                 unsigned int port = atoi(line.c_str() + colon + 1);
 273                 line.erase(colon);
 274                 db.add_database(Remote::open_writable(line, port, 0, 10000, flags));
 275             }
 276             continue;
 277         }
 278 #endif
 279
 280 #ifdef XAPIAN_HAS_INMEMORY_BACKEND
 281         if (type == "inmemory" && line.empty()) {
 282             db.add_database(WritableDatabase(string(), DB_BACKEND_INMEMORY));
 283             continue;
 284         }
 285 #endif
 286
 287         if (type == "flint") {
 288             throw FeatureUnavailableError("Flint backend no longer supported");
 289         }
 290
 291         // Don't include the line itself - that might help an attacker
 292         // by revealing part of a sensitive file's contents if they can
 293         // arrange for it to be read as a stub database via infelicities in
 294         // an application which uses Xapian.  The line number is enough
 295         // information to identify the problem line.
 296         throw DatabaseOpeningError(file + ':' + str(line_no) + ": Bad line");
 297     }
 298
 299     if (db.internal.empty()) {
 300         throw DatabaseOpeningError(file + ": No databases listed");
 301     }
 302 }
 303
 304 Database::Database(const string &path, int flags)
 305 {
 306     LOGCALL_CTOR(API, "Database", path|flags);
 307
 308     int type = flags & DB_BACKEND_MASK_;
 309     switch (type) {
 310         case DB_BACKEND_CHERT:
 311 #ifdef XAPIAN_HAS_CHERT_BACKEND
 312             internal.push_back(new ChertDatabase(path));
 313             return;
 314 #else
 315             throw FeatureUnavailableError("Chert backend disabled");
 316 #endif
 317         case DB_BACKEND_GLASS:
 318 #ifdef XAPIAN_HAS_GLASS_BACKEND
 319             internal.push_back(new GlassDatabase(path));
 320             return;
 321 #else
 322             throw FeatureUnavailableError("Glass backend disabled");
 323 #endif
 324         case DB_BACKEND_STUB:
 325             open_stub(*this, path);
 326             return;
 327         case DB_BACKEND_INMEMORY:
 328 #ifdef XAPIAN_HAS_INMEMORY_BACKEND
 329             internal.push_back(new InMemoryDatabase());
 330             return;
 331 #else
 332             throw FeatureUnavailableError("Inmemory backend disabled");
 333 #endif
 334     }
 335
 336     struct stat statbuf;
 337     if (stat(path.c_str(), &statbuf) == -1) {
 338         throw DatabaseOpeningError("Couldn't stat '" + path + "'", errno);
 339     }
 340
 341     if (S_ISREG(statbuf.st_mode)) {
 342         // Could be a stub database file, or a single file glass database.
 343         int fd;
 344         if (check_if_single_file_db(statbuf, path, &fd)) {
 345             // Single file glass format.
 346             internal.push_back(new GlassDatabase(fd));
 347             return;
 348         }
 349
 350         open_stub(*this, path);
 351         return;
 352     }
 353
 354     if (rare(!S_ISDIR(statbuf.st_mode))) {
 355         throw DatabaseOpeningError("Not a regular file or directory: '" + path + "'");
 356     }
 357
 358 #ifdef XAPIAN_HAS_CHERT_BACKEND
 359     if (file_exists(path + "/iamchert")) {
 360         internal.push_back(new ChertDatabase(path));
 361         return;
 362     }
 363 #endif
 364
 365 #ifdef XAPIAN_HAS_GLASS_BACKEND
 366     if (file_exists(path + "/iamglass")) {
 367         internal.push_back(new GlassDatabase(path));
 368         return;
 369     }
 370 #endif
 371
 372     // Check for "stub directories".
 373     string stub_file = path;
 374     stub_file += "/XAPIANDB";
 375     if (usual(file_exists(stub_file))) {
 376         open_stub(*this, stub_file);
 377         return;
 378     }
 379
 380 #ifndef XAPIAN_HAS_CHERT_BACKEND
 381     if (file_exists(path + "/iamchert")) {
 382         throw FeatureUnavailableError("Chert backend disabled");
 383     }
 384 #endif
 385 #ifndef XAPIAN_HAS_GLASS_BACKEND
 386     if (file_exists(path + "/iamglass")) {
 387         throw FeatureUnavailableError("Glass backend disabled");
 388     }
 389 #endif
 390     if (file_exists(path + "/iamflint")) {
 391         throw FeatureUnavailableError("Flint backend no longer supported");
 392     }
 393
 394     throw DatabaseOpeningError("Couldn't detect type of database");
 395 }
 396
 397 Database::Database(int fd, int flags)
 398 {
 399     LOGCALL_CTOR(API, "Database", fd|flags);
 400     if (rare(fd < 0))
 401         throw InvalidArgumentError("fd < 0");
 402
 403 #ifdef XAPIAN_HAS_GLASS_BACKEND
 404     int type = flags & DB_BACKEND_MASK_;
 405     switch (type) {
 406         case 0: case DB_BACKEND_GLASS:
 407             internal.push_back(new GlassDatabase(fd));
 408             return;
 409     }
 410 #else
 411     (void)flags;
 412 #endif
 413
 414     (void)::close(fd);
 415     throw DatabaseOpeningError("Couldn't detect type of database");
 416 }
 417
 418 #if defined XAPIAN_HAS_CHERT_BACKEND || \
 419     defined XAPIAN_HAS_GLASS_BACKEND
 420 #define HAVE_DISK_BACKEND
 421 #endif
 422
 423 WritableDatabase::WritableDatabase(const std::string &path, int flags, int block_size)
 424     : Database()
 425 {
 426     LOGCALL_CTOR(API, "WritableDatabase", path|flags|block_size);
 427     // Avoid warning if both chert and glass are disabled.
 428     (void)block_size;
 429     int type = flags & DB_BACKEND_MASK_;
 430     // Clear the backend bits, so we just pass on other flags to open_stub, etc.
 431     flags &= ~DB_BACKEND_MASK_;
 432     if (type == 0) {
 433         struct stat statbuf;
 434         if (stat(path.c_str(), &statbuf) == -1) {
 435             // ENOENT probably just means that we need to create the directory.
 436             if (errno != ENOENT)
 437                 throw DatabaseOpeningError("Couldn't stat '" + path + "'", errno);
 438         } else {
 439             // File or directory already exists.
 440
 441             if (S_ISREG(statbuf.st_mode)) {
 442                 // The path is a file, so assume it is a stub database file.
 443                 open_stub(*this, path, flags);
 444                 return;
 445             }
 446
 447             if (rare(!S_ISDIR(statbuf.st_mode))) {
 448                 throw DatabaseOpeningError("Not a regular file or directory: '" + path + "'");
 449             }
 450
 451             if (file_exists(path + "/iamchert")) {
 452                 // Existing chert DB.
 453 #ifdef XAPIAN_HAS_CHERT_BACKEND
 454                 type = DB_BACKEND_CHERT;
 455 #else
 456                 throw FeatureUnavailableError("Chert backend disabled");
 457 #endif
 458             } else if (file_exists(path + "/iamglass")) {
 459                 // Existing glass DB.
 460 #ifdef XAPIAN_HAS_GLASS_BACKEND
 461                 type = DB_BACKEND_GLASS;
 462 #else
 463                 throw FeatureUnavailableError("Glass backend disabled");
 464 #endif
 465             } else if (file_exists(path + "/iamflint")) {
 466                 // Existing flint DB.
 467                 throw FeatureUnavailableError("Flint backend no longer supported");
 468             } else {
 469                 // Check for "stub directories".
 470                 string stub_file = path;
 471                 stub_file += "/XAPIANDB";
 472                 if (usual(file_exists(stub_file))) {
 473                     open_stub(*this, stub_file, flags);
 474                     return;
 475                 }
 476             }
 477         }
 478     }
 479
 480     switch (type) {
 481         case DB_BACKEND_STUB:
 482             open_stub(*this, path, flags);
 483             return;
 484         case 0:
 485             // Fall through to first enabled case, so order the remaining cases
 486             // by preference.
 487 #ifdef XAPIAN_HAS_GLASS_BACKEND
 488         case DB_BACKEND_GLASS:
 489             internal.push_back(new GlassWritableDatabase(path, flags, block_size));
 490             return;
 491 #endif
 492 #ifdef XAPIAN_HAS_CHERT_BACKEND
 493         case DB_BACKEND_CHERT:
 494             internal.push_back(new ChertWritableDatabase(path, flags, block_size));
 495             return;
 496 #endif
 497         case DB_BACKEND_INMEMORY:
 498 #ifdef XAPIAN_HAS_INMEMORY_BACKEND
 499             internal.push_back(new InMemoryDatabase());
 500             return;
 501 #else
 502             throw FeatureUnavailableError("Inmemory backend disabled");
 503 #endif
 504     }
 505 #ifndef HAVE_DISK_BACKEND
 506     throw FeatureUnavailableError("No disk-based writable backend is enabled");
 507 #endif
 508 }
 509
 510 }