xapian-core/backends/dbfactory.cc

   1 /** @file dbfactory.cc
   2  * @brief Database factories for non-remote databases.
   3  */
   4 /* Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2011,2012,2013,2014,2015,2016,2017 Olly Betts
   5  * Copyright 2008 Lemur Consulting Ltd
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License as
   9  * published by the Free Software Foundation; either version 2 of the
  10  * License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  20  * USA
  21  */
  22
  23 #include <config.h>
  24
  25 #include "xapian/dbfactory.h"
  26
  27 #include "xapian/constants.h"
  28 #include "xapian/database.h"
  29 #include "xapian/error.h"
  30 #include "xapian/version.h" // For XAPIAN_HAS_XXX_BACKEND.
  31
  32 #include "backends.h"
  33 #include "debuglog.h"
  34 #include "filetests.h"
  35 #include "fileutils.h"
  36 #include "posixy_wrapper.h"
  37 #include "str.h"
  38
  39 #include "safeerrno.h"
  40 #include <cstdlib> // For atoi().
  41
  42 #ifdef XAPIAN_HAS_GLASS_BACKEND
  43 # include "glass/glass_database.h"
  44 #endif
  45 #include "glass/glass_defs.h"
  46 #ifdef XAPIAN_HAS_HONEY_BACKEND
  47 # include "honey/honey_database.h"
  48 #endif
  49 #include "honey/honey_defs.h"
  50 #ifdef XAPIAN_HAS_INMEMORY_BACKEND
  51 # include "inmemory/inmemory_database.h"
  52 #endif
  53 // Even if none of the above get included, we still need a definition of
  54 // Database::Internal.
  55 #include "backends/databaseinternal.h"
  56
  57 #include <fstream>
  58 #include <string>
  59
  60 using namespace std;
  61
  62 /** Return a BACKEND_* constant from backends.h.
  63  *
  64  *  BACKEND_UNKNOWN : stub file
  65  *  BACKEND_GLASS : glass single file
  66  *  BACKEND_HONEY : honey single file
  67  */
  68 static int
  69 check_if_single_file_db(const struct stat & sb, const string & path,
  70                         int * fd_ptr = NULL)
  71 {
  72 #if defined XAPIAN_HAS_GLASS_BACKEND || \
  73     defined XAPIAN_HAS_HONEY_BACKEND
  74     if (!S_ISREG(sb.st_mode)) return BACKEND_UNKNOWN;
  75     // Look at the size as a clue - if it's less than both GLASS_MIN_BLOCKSIZE
  76     // and HONEY_MIN_DB_SIZE then it's not a single-file glass or honey
  77     // database.  For a larger file, we peek at the start of the file to
  78     // determine what it is.
  79     if (sb.st_size < min(GLASS_MIN_BLOCKSIZE, HONEY_MIN_DB_SIZE))
  80         return false;
  81     int fd = posixy_open(path.c_str(), O_RDONLY|O_BINARY);
  82     if (fd != -1) {
  83         char magic_buf[14];
  84         // FIXME: Don't duplicate magic check here...
  85         if (io_read(fd, magic_buf, 14) == 14 &&
  86             lseek(fd, 0, SEEK_SET) == 0 &&
  87             memcmp(magic_buf, "\x0f\x0dXapian ", 9) == 0) {
  88             if (!fd_ptr)
  89                 ::close(fd);
  90             switch (magic_buf[9]) {
  91                 case 'G':
  92                     if (memcmp(magic_buf + 10, "lass", 4) == 0) {
  93                         if (fd_ptr)
  94                             *fd_ptr = fd;
  95                         return BACKEND_GLASS;
  96                     }
  97                     break;
  98                 case 'H':
  99                     if (memcmp(magic_buf + 10, "oney", 4) == 0) {
 100                         if (fd_ptr)
 101                             *fd_ptr = fd;
 102                         return BACKEND_HONEY;
 103                     }
 104                     break;
 105             }
 106             if (fd_ptr)
 107                 ::close(fd);
 108             return BACKEND_UNKNOWN;
 109         }
 110         ::close(fd);
 111     }
 112 #else
 113     (void)sb;
 114     (void)path;
 115     (void)fd_ptr;
 116 #endif
 117     return BACKEND_UNKNOWN;
 118 }
 119
 120 namespace Xapian {
 121
 122 static void
 123 open_stub(Database &db, const string &file)
 124 {
 125     // A stub database is a text file with one or more lines of this format:
 126     // <dbtype> <serialised db object>
 127     //
 128     // Lines which start with a "#" character are ignored.
 129     //
 130     // Any paths specified in stub database files which are relative will be
 131     // considered to be relative to the directory containing the stub database.
 132     ifstream stub(file.c_str());
 133     if (!stub) {
 134         string msg = "Couldn't open stub database file: ";
 135         msg += file;
 136         throw Xapian::DatabaseOpeningError(msg, errno);
 137     }
 138     string line;
 139     unsigned int line_no = 0;
 140     while (getline(stub, line)) {
 141         ++line_no;
 142         if (line.empty() || line[0] == '#')
 143             continue;
 144         string::size_type space = line.find(' ');
 145         if (space == string::npos) space = line.size();
 146
 147         string type(line, 0, space);
 148         line.erase(0, space + 1);
 149
 150         if (type == "auto") {
 151             resolve_relative_path(line, file);
 152             db.add_database(Database(line));
 153             continue;
 154         }
 155
 156         if (type == "glass") {
 157 #ifdef XAPIAN_HAS_GLASS_BACKEND
 158             resolve_relative_path(line, file);
 159             db.add_database(Database(new GlassDatabase(line)));
 160             continue;
 161 #else
 162             throw FeatureUnavailableError("Glass backend disabled");
 163 #endif
 164         }
 165
 166         if (type == "honey") {
 167 #ifdef XAPIAN_HAS_HONEY_BACKEND
 168             resolve_relative_path(line, file);
 169             db.add_database(Database(new HoneyDatabase(line)));
 170             continue;
 171 #else
 172             throw FeatureUnavailableError("Honey backend disabled");
 173 #endif
 174         }
 175
 176         if (type == "remote" && !line.empty()) {
 177 #ifdef XAPIAN_HAS_REMOTE_BACKEND
 178             if (line[0] == ':') {
 179                 // prog
 180                 // FIXME: timeouts
 181                 // Is it a security risk?
 182                 space = line.find(' ');
 183                 string args;
 184                 if (space != string::npos) {
 185                     args.assign(line, space + 1, string::npos);
 186                     line.assign(line, 1, space - 1);
 187                 } else {
 188                     line.erase(0, 1);
 189                 }
 190                 db.add_database(Remote::open(line, args));
 191                 continue;
 192             }
 193             string::size_type colon = line.rfind(':');
 194             if (colon != string::npos) {
 195                 // tcp
 196                 // FIXME: timeouts
 197                 // Avoid misparsing an IPv6 address without a port number.  The
 198                 // port number is required, so just leave that case to the
 199                 // error handling further below.
 200                 if (!(line[0] == '[' && line.back() == ']')) {
 201                     unsigned int port = atoi(line.c_str() + colon + 1);
 202                     line.erase(colon);
 203                     if (line[0] == '[' && line.back() == ']') {
 204                         line.erase(line.size() - 1, 1);
 205                         line.erase(0, 1);
 206                     }
 207                     db.add_database(Remote::open(line, port));
 208                     continue;
 209                 }
 210             }
 211 #else
 212             throw FeatureUnavailableError("Remote backend disabled");
 213 #endif
 214         }
 215
 216         if (type == "inmemory" && line.empty()) {
 217 #ifdef XAPIAN_HAS_INMEMORY_BACKEND
 218             db.add_database(Database(string(), DB_BACKEND_INMEMORY));
 219             continue;
 220 #else
 221             throw FeatureUnavailableError("Inmemory backend disabled");
 222 #endif
 223         }
 224
 225         if (type == "chert") {
 226             throw FeatureUnavailableError("Chert backend no longer supported");
 227         }
 228
 229         if (type == "flint") {
 230             throw FeatureUnavailableError("Flint backend no longer supported");
 231         }
 232
 233         // Don't include the line itself - that might help an attacker
 234         // by revealing part of a sensitive file's contents if they can
 235         // arrange for it to be read as a stub database via infelicities in
 236         // an application which uses Xapian.  The line number is enough
 237         // information to identify the problem line.
 238         throw DatabaseOpeningError(file + ':' + str(line_no) + ": Bad line");
 239     }
 240
 241     // Allowing a stub database with no databases listed allows things like
 242     // a "search all databases" feature to be implemented by generating a
 243     // stub database file without having to special case there not being any
 244     // databases yet.
 245     //
 246     // 1.0.x throws DatabaseOpeningError here, but with a "Bad line" message
 247     // with the line number just past the end of the file, which is a bit odd.
 248 }
 249
 250 static void
 251 open_stub(WritableDatabase &db, const string &file, int flags)
 252 {
 253     // A stub database is a text file with one or more lines of this format:
 254     // <dbtype> <serialised db object>
 255     //
 256     // Lines which start with a "#" character, and lines which have no spaces
 257     // in them, are ignored.
 258     //
 259     // Any paths specified in stub database files which are relative will be
 260     // considered to be relative to the directory containing the stub database.
 261     ifstream stub(file.c_str());
 262     if (!stub) {
 263         string msg = "Couldn't open stub database file: ";
 264         msg += file;
 265         throw Xapian::DatabaseOpeningError(msg, errno);
 266     }
 267     string line;
 268     unsigned int line_no = 0;
 269     while (true) {
 270         if (!getline(stub, line)) break;
 271
 272         ++line_no;
 273         if (line.empty() || line[0] == '#')
 274             continue;
 275         string::size_type space = line.find(' ');
 276         if (space == string::npos) space = line.size();
 277
 278         string type(line, 0, space);
 279         line.erase(0, space + 1);
 280
 281         if (type == "auto") {
 282             resolve_relative_path(line, file);
 283             db.add_database(WritableDatabase(line, flags));
 284             continue;
 285         }
 286
 287         if (type == "glass") {
 288 #ifdef XAPIAN_HAS_GLASS_BACKEND
 289             resolve_relative_path(line, file);
 290             db.add_database(WritableDatabase(line, flags|DB_BACKEND_GLASS));
 291             continue;
 292 #else
 293             throw FeatureUnavailableError("Glass backend disabled");
 294 #endif
 295         }
 296
 297         if (type == "remote" && !line.empty()) {
 298 #ifdef XAPIAN_HAS_REMOTE_BACKEND
 299             if (line[0] == ':') {
 300                 // prog
 301                 // FIXME: timeouts
 302                 // Is it a security risk?
 303                 space = line.find(' ');
 304                 string args;
 305                 if (space != string::npos) {
 306                     args.assign(line, space + 1, string::npos);
 307                     line.assign(line, 1, space - 1);
 308                 } else {
 309                     line.erase(0, 1);
 310                 }
 311                 db.add_database(Remote::open_writable(line, args, 0, flags));
 312                 continue;
 313             }
 314             string::size_type colon = line.rfind(':');
 315             if (colon != string::npos) {
 316                 // tcp
 317                 // FIXME: timeouts
 318                 // Avoid misparsing an IPv6 address without a port number.  The
 319                 // port number is required, so just leave that case to the
 320                 // error handling further below.
 321                 if (!(line[0] == '[' && line.back() == ']')) {
 322                     unsigned int port = atoi(line.c_str() + colon + 1);
 323                     line.erase(colon);
 324                     if (line[0] == '[' && line.back() == ']') {
 325                         line.erase(line.size() - 1, 1);
 326                         line.erase(0, 1);
 327                     }
 328                     db.add_database(Remote::open_writable(line, port, 0, 10000, flags));
 329                     continue;
 330                 }
 331             }
 332 #else
 333             throw FeatureUnavailableError("Remote backend disabled");
 334 #endif
 335         }
 336
 337         if (type == "inmemory" && line.empty()) {
 338 #ifdef XAPIAN_HAS_INMEMORY_BACKEND
 339             db.add_database(WritableDatabase(string(), DB_BACKEND_INMEMORY));
 340             continue;
 341 #else
 342             throw FeatureUnavailableError("Inmemory backend disabled");
 343 #endif
 344         }
 345
 346         if (type == "chert") {
 347             throw FeatureUnavailableError("Chert backend no longer supported");
 348         }
 349
 350         if (type == "flint") {
 351             throw FeatureUnavailableError("Flint backend no longer supported");
 352         }
 353
 354         // Don't include the line itself - that might help an attacker
 355         // by revealing part of a sensitive file's contents if they can
 356         // arrange for it to be read as a stub database via infelicities in
 357         // an application which uses Xapian.  The line number is enough
 358         // information to identify the problem line.
 359         throw DatabaseOpeningError(file + ':' + str(line_no) + ": Bad line");
 360     }
 361
 362     if (db.internal->size() == 0) {
 363         throw DatabaseOpeningError(file + ": No databases listed");
 364     }
 365 }
 366
 367 Database::Database(const string& path, int flags)
 368     : Database()
 369 {
 370     LOGCALL_CTOR(API, "Database", path|flags);
 371
 372     int type = flags & DB_BACKEND_MASK_;
 373     switch (type) {
 374         case DB_BACKEND_CHERT:
 375             throw FeatureUnavailableError("Chert backend no longer supported");
 376         case DB_BACKEND_GLASS:
 377 #ifdef XAPIAN_HAS_GLASS_BACKEND
 378             internal = new GlassDatabase(path);
 379             return;
 380 #else
 381             throw FeatureUnavailableError("Glass backend disabled");
 382 #endif
 383         case DB_BACKEND_HONEY:
 384 #ifdef XAPIAN_HAS_HONEY_BACKEND
 385             internal = new HoneyDatabase(path);
 386             return;
 387 #else
 388             throw FeatureUnavailableError("Honey backend disabled");
 389 #endif
 390         case DB_BACKEND_STUB:
 391             open_stub(*this, path);
 392             return;
 393         case DB_BACKEND_INMEMORY:
 394 #ifdef XAPIAN_HAS_INMEMORY_BACKEND
 395             internal = new InMemoryDatabase();
 396             return;
 397 #else
 398             throw FeatureUnavailableError("Inmemory backend disabled");
 399 #endif
 400     }
 401
 402     struct stat statbuf;
 403     if (stat(path.c_str(), &statbuf) == -1) {
 404         throw DatabaseOpeningError("Couldn't stat '" + path + "'", errno);
 405     }
 406
 407     if (S_ISREG(statbuf.st_mode)) {
 408         // Could be a stub database file, or a single file glass database.
 409
 410         // Initialise to avoid bogus warning from GCC 4.9.2 with -Os.
 411         int fd = -1;
 412         switch (check_if_single_file_db(statbuf, path, &fd)) {
 413             case BACKEND_GLASS:
 414 #ifdef XAPIAN_HAS_GLASS_BACKEND
 415                 // Single file glass format.
 416                 internal = new GlassDatabase(fd);
 417                 return;
 418 #else
 419                 throw FeatureUnavailableError("Glass backend disabled");
 420 #endif
 421             case BACKEND_HONEY:
 422 #ifdef XAPIAN_HAS_HONEY_BACKEND
 423                 // Single file honey format.
 424                 internal = new HoneyDatabase(fd);
 425                 return;
 426 #else
 427                 throw FeatureUnavailableError("Honey backend disabled");
 428 #endif
 429         }
 430
 431         open_stub(*this, path);
 432         return;
 433     }
 434
 435     if (rare(!S_ISDIR(statbuf.st_mode))) {
 436         throw DatabaseOpeningError("Not a regular file or directory: '" + path + "'");
 437     }
 438
 439 #ifdef XAPIAN_HAS_GLASS_BACKEND
 440     if (file_exists(path + "/iamglass")) {
 441         internal = new GlassDatabase(path);
 442         return;
 443     }
 444 #endif
 445
 446 #ifdef XAPIAN_HAS_HONEY_BACKEND
 447     if (file_exists(path + "/iamhoney")) {
 448         internal = new HoneyDatabase(path);
 449         return;
 450     }
 451 #endif
 452
 453     // Check for "stub directories".
 454     string stub_file = path;
 455     stub_file += "/XAPIANDB";
 456     if (usual(file_exists(stub_file))) {
 457         open_stub(*this, stub_file);
 458         return;
 459     }
 460
 461 #ifndef XAPIAN_HAS_GLASS_BACKEND
 462     if (file_exists(path + "/iamglass")) {
 463         throw FeatureUnavailableError("Glass backend disabled");
 464     }
 465 #endif
 466 #ifndef XAPIAN_HAS_HONEY_BACKEND
 467     if (file_exists(path + "/iamhoney")) {
 468         throw FeatureUnavailableError("Honey backend disabled");
 469     }
 470 #endif
 471     if (file_exists(path + "/iamchert")) {
 472         throw FeatureUnavailableError("Chert backend no longer supported");
 473     }
 474     if (file_exists(path + "/iamflint")) {
 475         throw FeatureUnavailableError("Flint backend no longer supported");
 476     }
 477
 478     throw DatabaseOpeningError("Couldn't detect type of database");
 479 }
 480
 481 /** Helper factory function.
 482  *
 483  *  This allows us to initialise Database::internal via the constructor's
 484  *  initialiser list, which we want to be able to do as Database::internal
 485  *  is an intrusive_ptr_nonnull, so we can't set it to NULL in the initialiser
 486  *  list and then fill it in later in the constructor body.
 487  */
 488 static Database::Internal*
 489 database_factory(int fd, int flags)
 490 {
 491     if (rare(fd < 0))
 492         throw InvalidArgumentError("fd < 0");
 493
 494 #ifdef XAPIAN_HAS_GLASS_BACKEND
 495     int type = flags & DB_BACKEND_MASK_;
 496     switch (type) {
 497         case 0:
 498         case DB_BACKEND_GLASS:
 499             return new GlassDatabase(fd);
 500     }
 501 #else
 502     (void)flags;
 503 #endif
 504
 505     (void)::close(fd);
 506     throw DatabaseOpeningError("Couldn't detect type of database");
 507 }
 508
 509 Database::Database(int fd, int flags)
 510     : internal(database_factory(fd, flags))
 511 {
 512     LOGCALL_CTOR(API, "Database", fd|flags);
 513 }
 514
 515 #if defined XAPIAN_HAS_GLASS_BACKEND
 516 #define HAVE_DISK_BACKEND
 517 #endif
 518
 519 WritableDatabase::WritableDatabase(const std::string &path, int flags, int block_size)
 520     : Database()
 521 {
 522     LOGCALL_CTOR(API, "WritableDatabase", path|flags|block_size);
 523     // Avoid warning if all disk-based backends are disabled.
 524     (void)block_size;
 525     int type = flags & DB_BACKEND_MASK_;
 526     // Clear the backend bits, so we just pass on other flags to open_stub, etc.
 527     flags &= ~DB_BACKEND_MASK_;
 528     if (type == 0) {
 529         struct stat statbuf;
 530         if (stat(path.c_str(), &statbuf) == -1) {
 531             // ENOENT probably just means that we need to create the directory.
 532             if (errno != ENOENT)
 533                 throw DatabaseOpeningError("Couldn't stat '" + path + "'", errno);
 534         } else {
 535             // File or directory already exists.
 536
 537             if (S_ISREG(statbuf.st_mode)) {
 538                 // The path is a file, so assume it is a stub database file.
 539                 open_stub(*this, path, flags);
 540                 return;
 541             }
 542
 543             if (rare(!S_ISDIR(statbuf.st_mode))) {
 544                 throw DatabaseOpeningError("Not a regular file or directory: '" + path + "'");
 545             }
 546
 547             if (file_exists(path + "/iamglass")) {
 548                 // Existing glass DB.
 549 #ifdef XAPIAN_HAS_GLASS_BACKEND
 550                 type = DB_BACKEND_GLASS;
 551 #else
 552                 throw FeatureUnavailableError("Glass backend disabled");
 553 #endif
 554             } else if (file_exists(path + "/iamhoney")) {
 555                 // Existing honey DB.
 556                 throw InvalidOperationError("Honey backend doesn't support "
 557                                             "updating existing databases");
 558             } else if (file_exists(path + "/iamchert")) {
 559                 // Existing chert DB.
 560                 throw FeatureUnavailableError("Chert backend no longer supported");
 561             } else if (file_exists(path + "/iamflint")) {
 562                 // Existing flint DB.
 563                 throw FeatureUnavailableError("Flint backend no longer supported");
 564             } else {
 565                 // Check for "stub directories".
 566                 string stub_file = path;
 567                 stub_file += "/XAPIANDB";
 568                 if (usual(file_exists(stub_file))) {
 569                     open_stub(*this, stub_file, flags);
 570                     return;
 571                 }
 572             }
 573         }
 574     }
 575
 576     switch (type) {
 577         case DB_BACKEND_STUB:
 578             open_stub(*this, path, flags);
 579             return;
 580         case 0:
 581             // Fall through to first enabled case, so order the remaining cases
 582             // by preference.
 583 #ifdef XAPIAN_HAS_GLASS_BACKEND
 584         case DB_BACKEND_GLASS:
 585             internal = new GlassWritableDatabase(path, flags, block_size);
 586             return;
 587 #endif
 588         case DB_BACKEND_HONEY:
 589             throw InvalidArgumentError("Honey backend doesn't support "
 590                                        "updating existing databases");
 591         case DB_BACKEND_CHERT:
 592             throw FeatureUnavailableError("Chert backend no longer supported");
 593         case DB_BACKEND_INMEMORY:
 594 #ifdef XAPIAN_HAS_INMEMORY_BACKEND
 595             internal = new InMemoryDatabase();
 596             return;
 597 #else
 598             throw FeatureUnavailableError("Inmemory backend disabled");
 599 #endif
 600     }
 601 #ifndef HAVE_DISK_BACKEND
 602     throw FeatureUnavailableError("No disk-based writable backend is enabled");
 603 #endif
 604 }
 605
 606 }