xapian-core/backends/honey/honey_version.cc

   1 /** @file honey_version.cc
   2  * @brief HoneyVersion class
   3  */
   4 /* Copyright (C) 2006,2007,2008,2009,2010,2013,2014,2015,2016,2017 Olly Betts
   5  * Copyright (C) 2011 Dan Colish
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  20  */
  21
  22 #include <config.h>
  23
  24 #include "honey_version.h"
  25
  26 #include "debuglog.h"
  27 #include "fd.h"
  28 #include "honey_defs.h"
  29 #include "io_utils.h"
  30 #include "omassert.h"
  31 #include "pack.h"
  32 #include "posixy_wrapper.h"
  33 #include "stringutils.h" // For STRINGIZE() and CONST_STRLEN().
  34
  35 #include <cstring> // For memcmp().
  36 #include <string>
  37 #include "safeerrno.h"
  38 #include <sys/types.h>
  39 #include "safesysstat.h"
  40 #include "safefcntl.h"
  41 #include "safeunistd.h"
  42 #include "str.h"
  43 #include "stringutils.h"
  44
  45 #include "common/safeuuid.h"
  46
  47 #include "xapian/constants.h"
  48 #include "xapian/error.h"
  49
  50 using namespace std;
  51
  52 /// Honey format version (date of change):
  53 #define HONEY_FORMAT_VERSION DATE_TO_VERSION(2017,12,30)
  54 // 2017,12,30 1.5.0 Value stats key changes
  55 // 2017,12,29       User metadata key changes
  56 // 2017,12,5        New Honey backend
  57
  58 /// Convert date <-> version number.  Dates up to 2141-12-31 fit in 2 bytes.
  59 #define DATE_TO_VERSION(Y,M,D) \
  60         ((unsigned(Y) - 2014) << 9 | unsigned(M) << 5 | unsigned(D))
  61 #define VERSION_TO_YEAR(V) ((unsigned(V) >> 9) + 2014)
  62 #define VERSION_TO_MONTH(V) ((unsigned(V) >> 5) & 0x0f)
  63 #define VERSION_TO_DAY(V) (unsigned(V) & 0x1f)
  64
  65 #define HONEY_VERSION_MAGIC_LEN 14
  66 #define HONEY_VERSION_MAGIC_AND_VERSION_LEN 16
  67
  68 static const char HONEY_VERSION_MAGIC[HONEY_VERSION_MAGIC_AND_VERSION_LEN] = {
  69     '\x0f', '\x0d', 'X', 'a', 'p', 'i', 'a', 'n', ' ', 'H', 'o', 'n', 'e', 'y',
  70     char((HONEY_FORMAT_VERSION >> 8) & 0xff), char(HONEY_FORMAT_VERSION & 0xff)
  71 };
  72
  73 HoneyVersion::HoneyVersion(int fd_)
  74     : rev(0), fd(fd_), offset(0), db_dir(), changes(NULL),
  75       doccount(0), total_doclen(0), last_docid(0),
  76       doclen_lbound(0), doclen_ubound(0),
  77       wdf_ubound(0), spelling_wordfreq_ubound(0),
  78       oldest_changeset(0)
  79 {
  80     offset = lseek(fd, 0, SEEK_CUR);
  81     if (rare(offset == off_t(-1))) {
  82         string msg = "lseek failed on file descriptor ";
  83         msg += str(fd);
  84         throw Xapian::DatabaseOpeningError(msg, errno);
  85     }
  86 }
  87
  88 HoneyVersion::~HoneyVersion()
  89 {
  90     // Either this is a single-file database, or this fd is from opening a new
  91     // version file in write(), but sync() was never called.
  92     if (fd != -1)
  93         (void)::close(fd);
  94 }
  95
  96 void
  97 HoneyVersion::read()
  98 {
  99     LOGCALL_VOID(DB, "HoneyVersion::read", NO_ARGS);
 100     FD close_fd(-1);
 101     int fd_in;
 102     if (single_file()) {
 103         if (rare(lseek(fd, offset, SEEK_SET) == off_t(-1))) {
 104             string msg = "Failed to rewind file descriptor ";
 105             msg += str(fd);
 106             throw Xapian::DatabaseOpeningError(msg, errno);
 107         }
 108         fd_in = fd;
 109     } else {
 110         string filename = db_dir;
 111         filename += "/iamhoney";
 112         fd_in = posixy_open(filename.c_str(), O_RDONLY|O_BINARY);
 113         if (rare(fd_in < 0)) {
 114             string msg = filename;
 115             msg += ": Failed to open honey revision file for reading";
 116             throw Xapian::DatabaseOpeningError(msg, errno);
 117         }
 118         close_fd = fd_in;
 119     }
 120
 121     char buf[256];
 122
 123     const char * p = buf;
 124     const char * end = p + io_read(fd_in, buf, sizeof(buf), 33);
 125
 126     if (memcmp(buf, HONEY_VERSION_MAGIC, HONEY_VERSION_MAGIC_LEN) != 0)
 127         throw Xapian::DatabaseCorruptError("Rev file magic incorrect");
 128
 129     unsigned version;
 130     version = static_cast<unsigned char>(buf[HONEY_VERSION_MAGIC_LEN]);
 131     version <<= 8;
 132     version |= static_cast<unsigned char>(buf[HONEY_VERSION_MAGIC_LEN + 1]);
 133     if (version != HONEY_FORMAT_VERSION) {
 134         string msg;
 135         if (!single_file()) {
 136             msg = db_dir;
 137             msg += ": ";
 138         }
 139         msg += "Database is format version ";
 140         msg += str(VERSION_TO_YEAR(version) * 10000 +
 141                    VERSION_TO_MONTH(version) * 100 +
 142                    VERSION_TO_DAY(version));
 143         msg += " but I only understand ";
 144         msg += str(VERSION_TO_YEAR(HONEY_FORMAT_VERSION) * 10000 +
 145                    VERSION_TO_MONTH(HONEY_FORMAT_VERSION) * 100 +
 146                    VERSION_TO_DAY(HONEY_FORMAT_VERSION));
 147         throw Xapian::DatabaseVersionError(msg);
 148     }
 149
 150     p += HONEY_VERSION_MAGIC_AND_VERSION_LEN;
 151     memcpy(uuid, p, 16);
 152     p += 16;
 153
 154     if (!unpack_uint(&p, end, &rev))
 155         throw Xapian::DatabaseCorruptError("Rev file failed to decode revision");
 156
 157     for (unsigned table_no = 0; table_no < Honey::MAX_; ++table_no) {
 158         if (!root[table_no].unserialise(&p, end)) {
 159             throw Xapian::DatabaseCorruptError("Rev file root_info missing");
 160         }
 161         old_root[table_no] = root[table_no];
 162     }
 163
 164     // For a single-file database, this will assign extra data.  We read
 165     // sizeof(buf) above, then skip HONEY_VERSION_MAGIC_AND_VERSION_LEN,
 166     // then 16, then the size of the serialised root info.
 167     serialised_stats.assign(p, end);
 168     unserialise_stats();
 169 }
 170
 171 void
 172 HoneyVersion::serialise_stats()
 173 {
 174     serialised_stats.resize(0);
 175     pack_uint(serialised_stats, doccount);
 176     // last_docid must always be >= doccount.
 177     pack_uint(serialised_stats, last_docid - doccount);
 178     pack_uint(serialised_stats, doclen_lbound);
 179     pack_uint(serialised_stats, wdf_ubound);
 180     // doclen_ubound should always be >= wdf_ubound, so we store the
 181     // difference as it may encode smaller.  wdf_ubound is likely to
 182     // be larger than doclen_lbound.
 183     pack_uint(serialised_stats, doclen_ubound - wdf_ubound);
 184     pack_uint(serialised_stats, oldest_changeset);
 185     pack_uint(serialised_stats, total_doclen);
 186     pack_uint(serialised_stats, spelling_wordfreq_ubound);
 187 }
 188
 189 void
 190 HoneyVersion::unserialise_stats()
 191 {
 192     const char * p = serialised_stats.data();
 193     const char * end = p + serialised_stats.size();
 194     if (p == end) {
 195         doccount = 0;
 196         total_doclen = 0;
 197         last_docid = 0;
 198         doclen_lbound = 0;
 199         doclen_ubound = 0;
 200         wdf_ubound = 0;
 201         oldest_changeset = 0;
 202         spelling_wordfreq_ubound = 0;
 203         return;
 204     }
 205
 206     if (!unpack_uint(&p, end, &doccount) ||
 207         !unpack_uint(&p, end, &last_docid) ||
 208         !unpack_uint(&p, end, &doclen_lbound) ||
 209         !unpack_uint(&p, end, &wdf_ubound) ||
 210         !unpack_uint(&p, end, &doclen_ubound) ||
 211         !unpack_uint(&p, end, &oldest_changeset) ||
 212         !unpack_uint(&p, end, &total_doclen) ||
 213         !unpack_uint(&p, end, &spelling_wordfreq_ubound)) {
 214         const char * m = p ?
 215             "Bad serialised DB stats (overflowed)" :
 216             "Bad serialised DB stats (out of data)";
 217         throw Xapian::DatabaseCorruptError(m);
 218     }
 219
 220     // In the single-file DB case, there will be extra data in
 221     // serialised_stats, so suppress this check.
 222     if (p != end && !single_file())
 223         throw Xapian::DatabaseCorruptError("Rev file has junk at end");
 224
 225     // last_docid must always be >= doccount.
 226     last_docid += doccount;
 227     // doclen_ubound should always be >= wdf_ubound, so we store the
 228     // difference as it may encode smaller.  wdf_ubound is likely to
 229     // be larger than doclen_lbound.
 230     doclen_ubound += wdf_ubound;
 231 }
 232
 233 void
 234 HoneyVersion::merge_stats(const HoneyVersion & o)
 235 {
 236     doccount += o.get_doccount();
 237     if (doccount < o.get_doccount()) {
 238         throw Xapian::DatabaseError("doccount overflowed!");
 239     }
 240
 241     Xapian::termcount o_doclen_lbound = o.get_doclength_lower_bound();
 242     if (o_doclen_lbound > 0) {
 243         if (doclen_lbound == 0 || o_doclen_lbound < doclen_lbound)
 244             doclen_lbound = o_doclen_lbound;
 245     }
 246
 247     doclen_ubound = max(doclen_ubound, o.get_doclength_upper_bound());
 248     wdf_ubound = max(wdf_ubound, o.get_wdf_upper_bound());
 249     total_doclen += o.get_total_doclen();
 250     if (total_doclen < o.get_total_doclen()) {
 251         throw Xapian::DatabaseError("Total document length overflowed!");
 252     }
 253
 254     // The upper bounds might be on the same word, so we must sum them.
 255     spelling_wordfreq_ubound += o.get_spelling_wordfreq_upper_bound();
 256 }
 257
 258 void
 259 HoneyVersion::merge_stats(Xapian::doccount o_doccount,
 260                           Xapian::termcount o_doclen_lbound,
 261                           Xapian::termcount o_doclen_ubound,
 262                           Xapian::termcount o_wdf_ubound,
 263                           Xapian::totallength o_total_doclen,
 264                           Xapian::termcount o_spelling_wordfreq_ubound)
 265 {
 266     doccount += o_doccount;
 267     if (doccount < o_doccount) {
 268         throw Xapian::DatabaseError("doccount overflowed!");
 269     }
 270
 271     if (o_doclen_lbound > 0) {
 272         if (doclen_lbound == 0 || o_doclen_lbound < doclen_lbound)
 273             doclen_lbound = o_doclen_lbound;
 274     }
 275
 276     doclen_ubound = max(doclen_ubound, o_doclen_ubound);
 277     wdf_ubound = max(wdf_ubound, o_wdf_ubound);
 278     total_doclen += o_total_doclen;
 279     if (total_doclen < o_total_doclen) {
 280         throw Xapian::DatabaseError("Total document length overflowed!");
 281     }
 282
 283     // The upper bounds might be on the same word, so we must sum them.
 284     spelling_wordfreq_ubound += o_spelling_wordfreq_ubound;
 285 }
 286
 287 void
 288 HoneyVersion::cancel()
 289 {
 290     LOGCALL_VOID(DB, "HoneyVersion::cancel", NO_ARGS);
 291     for (unsigned table_no = 0; table_no < Honey::MAX_; ++table_no) {
 292         root[table_no] = old_root[table_no];
 293     }
 294     unserialise_stats();
 295 }
 296
 297 const string
 298 HoneyVersion::write(honey_revision_number_t new_rev, int flags)
 299 {
 300     LOGCALL(DB, const string, "HoneyVersion::write", new_rev|flags);
 301
 302     string s(HONEY_VERSION_MAGIC, HONEY_VERSION_MAGIC_AND_VERSION_LEN);
 303     s.append(reinterpret_cast<const char *>(uuid), 16);
 304
 305     pack_uint(s, new_rev);
 306
 307     for (unsigned table_no = 0; table_no < Honey::MAX_; ++table_no) {
 308         root[table_no].serialise(s);
 309     }
 310
 311     // Serialise database statistics.
 312     serialise_stats();
 313     s += serialised_stats;
 314
 315     string tmpfile;
 316     if (!single_file()) {
 317         tmpfile = db_dir;
 318         // In dangerous mode, just write the new version file in place.
 319         if (flags & Xapian::DB_DANGEROUS)
 320             tmpfile += "/iamhoney";
 321         else
 322             tmpfile += "/v.tmp";
 323
 324         fd = posixy_open(tmpfile.c_str(), O_CREAT|O_TRUNC|O_WRONLY|O_BINARY, 0666);
 325         if (rare(fd < 0))
 326             throw Xapian::DatabaseOpeningError("Couldn't write new rev file: " + tmpfile,
 327                                                errno);
 328
 329         if (flags & Xapian::DB_DANGEROUS)
 330             tmpfile = string();
 331     }
 332
 333     try {
 334         io_write(fd, s.data(), s.size());
 335     } catch (...) {
 336         if (!single_file())
 337             (void)close(fd);
 338         throw;
 339     }
 340
 341     if (changes) {
 342         string changes_buf;
 343         changes_buf += '\xfe';
 344         pack_uint(changes_buf, new_rev);
 345         pack_uint(changes_buf, s.size());
 346         changes->write_block(changes_buf);
 347         changes->write_block(s);
 348     }
 349
 350     RETURN(tmpfile);
 351 }
 352
 353 bool
 354 HoneyVersion::sync(const string & tmpfile,
 355                    honey_revision_number_t new_rev, int flags)
 356 {
 357     Assert(new_rev > rev || rev == 0);
 358
 359     if (single_file()) {
 360         if ((flags & Xapian::DB_NO_SYNC) == 0 &&
 361             ((flags & Xapian::DB_FULL_SYNC) ?
 362               !io_full_sync(fd) :
 363               !io_sync(fd))) {
 364             // FIXME what to do?
 365         }
 366     } else {
 367         int fd_to_close = fd;
 368         fd = -1;
 369         if ((flags & Xapian::DB_NO_SYNC) == 0 &&
 370             ((flags & Xapian::DB_FULL_SYNC) ?
 371               !io_full_sync(fd_to_close) :
 372               !io_sync(fd_to_close))) {
 373             int save_errno = errno;
 374             (void)close(fd_to_close);
 375             if (!tmpfile.empty())
 376                 (void)unlink(tmpfile.c_str());
 377             errno = save_errno;
 378             return false;
 379         }
 380
 381         if (close(fd_to_close) != 0) {
 382             if (!tmpfile.empty()) {
 383                 int save_errno = errno;
 384                 (void)unlink(tmpfile.c_str());
 385                 errno = save_errno;
 386             }
 387             return false;
 388         }
 389
 390         if (!tmpfile.empty()) {
 391             if (!io_tmp_rename(tmpfile, db_dir + "/iamhoney")) {
 392                 return false;
 393             }
 394         }
 395     }
 396
 397     for (unsigned table_no = 0; table_no < Honey::MAX_; ++table_no) {
 398         old_root[table_no] = root[table_no];
 399     }
 400
 401     rev = new_rev;
 402     return true;
 403 }
 404
 405 // Only try to compress tags longer than this many bytes.
 406 const size_t COMPRESS_MIN = 4;
 407
 408 static const uint4 compress_min_tab[] = {
 409     0, // POSTLIST
 410     COMPRESS_MIN, // DOCDATA
 411     COMPRESS_MIN, // TERMLIST
 412     0, // POSITION
 413     COMPRESS_MIN, // SPELLING
 414     COMPRESS_MIN  // SYNONYM
 415 };
 416
 417 void
 418 HoneyVersion::create(unsigned blocksize)
 419 {
 420     AssertRel(blocksize,>=,HONEY_MIN_BLOCKSIZE);
 421     uuid_generate(uuid);
 422     for (unsigned table_no = 0; table_no < Honey::MAX_; ++table_no) {
 423         root[table_no].init(blocksize, compress_min_tab[table_no]);
 424     }
 425 }
 426
 427 namespace Honey {
 428
 429 void
 430 RootInfo::init(unsigned blocksize_, uint4 compress_min_)
 431 {
 432     AssertRel(blocksize_,>=,HONEY_MIN_BLOCKSIZE);
 433     root = 0;
 434     level = 0;
 435     num_entries = 0;
 436     root_is_fake = true;
 437     sequential = true;
 438     blocksize = blocksize_;
 439     compress_min = compress_min_;
 440     fl_serialised.resize(0);
 441 }
 442
 443 void
 444 RootInfo::serialise(string &s) const
 445 {
 446     pack_uint(s, root);
 447     unsigned val = level << 2;
 448     if (sequential) val |= 0x02;
 449     if (root_is_fake) val |= 0x01;
 450     pack_uint(s, val);
 451     pack_uint(s, num_entries);
 452     pack_uint(s, blocksize >> 11);
 453     pack_uint(s, compress_min);
 454     pack_string(s, fl_serialised);
 455 }
 456
 457 bool
 458 RootInfo::unserialise(const char ** p, const char * end)
 459 {
 460     unsigned val;
 461     if (!unpack_uint(p, end, &root) ||
 462         !unpack_uint(p, end, &val) ||
 463         !unpack_uint(p, end, &num_entries) ||
 464         !unpack_uint(p, end, &blocksize) ||
 465         !unpack_uint(p, end, &compress_min) ||
 466         !unpack_string(p, end, fl_serialised)) return false;
 467     level = val >> 2;
 468     sequential = val & 0x02;
 469     root_is_fake = val & 0x01;
 470     blocksize <<= 11;
 471     AssertRel(blocksize,>=,HONEY_MIN_BLOCKSIZE);
 472     return true;
 473 }
 474
 475 }