xapian-core/backends/honey/honey_version.cc

   1 /** @file honey_version.cc
   2  * @brief HoneyVersion class
   3  */
   4 /* Copyright (C) 2006,2007,2008,2009,2010,2013,2014,2015,2016,2017,2018 Olly Betts
   5  * Copyright (C) 2011 Dan Colish
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  20  */
  21
  22 #include <config.h>
  23
  24 #include "honey_version.h"
  25
  26 #include "debuglog.h"
  27 #include "fd.h"
  28 #include "honey_defs.h"
  29 #include "io_utils.h"
  30 #include "omassert.h"
  31 #include "pack.h"
  32 #include "posixy_wrapper.h"
  33 #include "stringutils.h" // For STRINGIZE() and CONST_STRLEN().
  34
  35 #include <cstring> // For memcmp().
  36 #include <string>
  37 #include "safeerrno.h"
  38 #include <sys/types.h>
  39 #include "safesysstat.h"
  40 #include "safefcntl.h"
  41 #include "safeunistd.h"
  42 #include "str.h"
  43 #include "stringutils.h"
  44
  45 #include "common/safeuuid.h"
  46
  47 #include "xapian/constants.h"
  48 #include "xapian/error.h"
  49
  50 using namespace std;
  51
  52 /// Honey format version (date of change):
  53 #define HONEY_FORMAT_VERSION DATE_TO_VERSION(2018,1,31)
  54 // 2018,1,31  1.5.0 Special case postlist when termfreq==2
  55 // 2018,1,30        More compact postlist chunk headers
  56 // 2018,1,23        Elide last-first for single occurrence terms
  57 // 2018,1,4         Merge values used and terms used
  58 // 2018,1,3         Table start offset in RootInfo
  59 // 2017,12,30       Value stats key changes
  60 // 2017,12,29       User metadata key changes
  61 // 2017,12,5        New Honey backend
  62
  63 /// Convert date <-> version number.  Dates up to 2141-12-31 fit in 2 bytes.
  64 #define DATE_TO_VERSION(Y,M,D) \
  65         ((unsigned(Y) - 2014) << 9 | unsigned(M) << 5 | unsigned(D))
  66 #define VERSION_TO_YEAR(V) ((unsigned(V) >> 9) + 2014)
  67 #define VERSION_TO_MONTH(V) ((unsigned(V) >> 5) & 0x0f)
  68 #define VERSION_TO_DAY(V) (unsigned(V) & 0x1f)
  69
  70 #define HONEY_VERSION_MAGIC_LEN 14
  71 #define HONEY_VERSION_MAGIC_AND_VERSION_LEN 16
  72
  73 static const char HONEY_VERSION_MAGIC[HONEY_VERSION_MAGIC_AND_VERSION_LEN] = {
  74     '\x0f', '\x0d', 'X', 'a', 'p', 'i', 'a', 'n', ' ', 'H', 'o', 'n', 'e', 'y',
  75     char((HONEY_FORMAT_VERSION >> 8) & 0xff), char(HONEY_FORMAT_VERSION & 0xff)
  76 };
  77
  78 HoneyVersion::HoneyVersion(int fd_)
  79     : rev(0), fd(fd_), offset(0), db_dir(), changes(NULL),
  80       doccount(0), total_doclen(0), last_docid(0),
  81       doclen_lbound(0), doclen_ubound(0),
  82       wdf_ubound(0), spelling_wordfreq_ubound(0),
  83       oldest_changeset(0)
  84 {
  85     offset = lseek(fd, 0, SEEK_CUR);
  86     if (rare(offset == off_t(-1))) {
  87         string msg = "lseek failed on file descriptor ";
  88         msg += str(fd);
  89         throw Xapian::DatabaseOpeningError(msg, errno);
  90     }
  91 }
  92
  93 HoneyVersion::~HoneyVersion()
  94 {
  95     // Either this is a single-file database, or this fd is from opening a new
  96     // version file in write(), but sync() was never called.
  97     if (fd != -1)
  98         (void)::close(fd);
  99 }
 100
 101 void
 102 HoneyVersion::read()
 103 {
 104     LOGCALL_VOID(DB, "HoneyVersion::read", NO_ARGS);
 105     FD close_fd(-1);
 106     int fd_in;
 107     if (single_file()) {
 108         if (rare(lseek(fd, offset, SEEK_SET) == off_t(-1))) {
 109             string msg = "Failed to rewind file descriptor ";
 110             msg += str(fd);
 111             throw Xapian::DatabaseOpeningError(msg, errno);
 112         }
 113         fd_in = fd;
 114     } else {
 115         string filename = db_dir;
 116         filename += "/iamhoney";
 117         fd_in = posixy_open(filename.c_str(), O_RDONLY|O_BINARY);
 118         if (rare(fd_in < 0)) {
 119             string msg = filename;
 120             msg += ": Failed to open honey revision file for reading";
 121             throw Xapian::DatabaseOpeningError(msg, errno);
 122         }
 123         close_fd = fd_in;
 124     }
 125
 126     char buf[256];
 127
 128     const char * p = buf;
 129     const char * end = p + io_read(fd_in, buf, sizeof(buf), 33);
 130
 131     if (memcmp(buf, HONEY_VERSION_MAGIC, HONEY_VERSION_MAGIC_LEN) != 0)
 132         throw Xapian::DatabaseCorruptError("Rev file magic incorrect");
 133
 134     unsigned version;
 135     version = static_cast<unsigned char>(buf[HONEY_VERSION_MAGIC_LEN]);
 136     version <<= 8;
 137     version |= static_cast<unsigned char>(buf[HONEY_VERSION_MAGIC_LEN + 1]);
 138     if (version != HONEY_FORMAT_VERSION) {
 139         string msg;
 140         if (!single_file()) {
 141             msg = db_dir;
 142             msg += ": ";
 143         }
 144         msg += "Database is format version ";
 145         msg += str(VERSION_TO_YEAR(version) * 10000 +
 146                    VERSION_TO_MONTH(version) * 100 +
 147                    VERSION_TO_DAY(version));
 148         msg += " but I only understand ";
 149         msg += str(VERSION_TO_YEAR(HONEY_FORMAT_VERSION) * 10000 +
 150                    VERSION_TO_MONTH(HONEY_FORMAT_VERSION) * 100 +
 151                    VERSION_TO_DAY(HONEY_FORMAT_VERSION));
 152         throw Xapian::DatabaseVersionError(msg);
 153     }
 154
 155     p += HONEY_VERSION_MAGIC_AND_VERSION_LEN;
 156     memcpy(uuid, p, 16);
 157     p += 16;
 158
 159     if (!unpack_uint(&p, end, &rev))
 160         throw Xapian::DatabaseCorruptError("Rev file failed to decode revision");
 161
 162     for (unsigned table_no = 0; table_no < Honey::MAX_; ++table_no) {
 163         if (!root[table_no].unserialise(&p, end)) {
 164             throw Xapian::DatabaseCorruptError("Rev file root_info missing");
 165         }
 166         old_root[table_no] = root[table_no];
 167     }
 168
 169     // For a single-file database, this will assign extra data.  We read
 170     // sizeof(buf) above, then skip HONEY_VERSION_MAGIC_AND_VERSION_LEN,
 171     // then 16, then the size of the serialised root info.
 172     serialised_stats.assign(p, end);
 173     unserialise_stats();
 174 }
 175
 176 void
 177 HoneyVersion::serialise_stats()
 178 {
 179     serialised_stats.resize(0);
 180     pack_uint(serialised_stats, doccount);
 181     // last_docid must always be >= doccount.
 182     pack_uint(serialised_stats, last_docid - doccount);
 183     pack_uint(serialised_stats, doclen_lbound);
 184     pack_uint(serialised_stats, wdf_ubound);
 185     // doclen_ubound should always be >= wdf_ubound, so we store the
 186     // difference as it may encode smaller.  wdf_ubound is likely to
 187     // be larger than doclen_lbound.
 188     pack_uint(serialised_stats, doclen_ubound - wdf_ubound);
 189     pack_uint(serialised_stats, oldest_changeset);
 190     pack_uint(serialised_stats, total_doclen);
 191     pack_uint(serialised_stats, spelling_wordfreq_ubound);
 192 }
 193
 194 void
 195 HoneyVersion::unserialise_stats()
 196 {
 197     const char * p = serialised_stats.data();
 198     const char * end = p + serialised_stats.size();
 199     if (p == end) {
 200         doccount = 0;
 201         total_doclen = 0;
 202         last_docid = 0;
 203         doclen_lbound = 0;
 204         doclen_ubound = 0;
 205         wdf_ubound = 0;
 206         oldest_changeset = 0;
 207         spelling_wordfreq_ubound = 0;
 208         return;
 209     }
 210
 211     if (!unpack_uint(&p, end, &doccount) ||
 212         !unpack_uint(&p, end, &last_docid) ||
 213         !unpack_uint(&p, end, &doclen_lbound) ||
 214         !unpack_uint(&p, end, &wdf_ubound) ||
 215         !unpack_uint(&p, end, &doclen_ubound) ||
 216         !unpack_uint(&p, end, &oldest_changeset) ||
 217         !unpack_uint(&p, end, &total_doclen) ||
 218         !unpack_uint(&p, end, &spelling_wordfreq_ubound)) {
 219         const char * m = p ?
 220             "Bad serialised DB stats (overflowed)" :
 221             "Bad serialised DB stats (out of data)";
 222         throw Xapian::DatabaseCorruptError(m);
 223     }
 224
 225     // In the single-file DB case, there will be extra data in
 226     // serialised_stats, so suppress this check.
 227     if (p != end && !single_file())
 228         throw Xapian::DatabaseCorruptError("Rev file has junk at end");
 229
 230     // last_docid must always be >= doccount.
 231     last_docid += doccount;
 232     // doclen_ubound should always be >= wdf_ubound, so we store the
 233     // difference as it may encode smaller.  wdf_ubound is likely to
 234     // be larger than doclen_lbound.
 235     doclen_ubound += wdf_ubound;
 236 }
 237
 238 void
 239 HoneyVersion::merge_stats(const HoneyVersion & o)
 240 {
 241     doccount += o.get_doccount();
 242     if (doccount < o.get_doccount()) {
 243         throw Xapian::DatabaseError("doccount overflowed!");
 244     }
 245
 246     Xapian::termcount o_doclen_lbound = o.get_doclength_lower_bound();
 247     if (o_doclen_lbound > 0) {
 248         if (doclen_lbound == 0 || o_doclen_lbound < doclen_lbound)
 249             doclen_lbound = o_doclen_lbound;
 250     }
 251
 252     doclen_ubound = max(doclen_ubound, o.get_doclength_upper_bound());
 253     wdf_ubound = max(wdf_ubound, o.get_wdf_upper_bound());
 254     total_doclen += o.get_total_doclen();
 255     if (total_doclen < o.get_total_doclen()) {
 256         throw Xapian::DatabaseError("Total document length overflowed!");
 257     }
 258
 259     // The upper bounds might be on the same word, so we must sum them.
 260     spelling_wordfreq_ubound += o.get_spelling_wordfreq_upper_bound();
 261 }
 262
 263 void
 264 HoneyVersion::merge_stats(Xapian::doccount o_doccount,
 265                           Xapian::termcount o_doclen_lbound,
 266                           Xapian::termcount o_doclen_ubound,
 267                           Xapian::termcount o_wdf_ubound,
 268                           Xapian::totallength o_total_doclen,
 269                           Xapian::termcount o_spelling_wordfreq_ubound)
 270 {
 271     doccount += o_doccount;
 272     if (doccount < o_doccount) {
 273         throw Xapian::DatabaseError("doccount overflowed!");
 274     }
 275
 276     if (o_doclen_lbound > 0) {
 277         if (doclen_lbound == 0 || o_doclen_lbound < doclen_lbound)
 278             doclen_lbound = o_doclen_lbound;
 279     }
 280
 281     doclen_ubound = max(doclen_ubound, o_doclen_ubound);
 282     wdf_ubound = max(wdf_ubound, o_wdf_ubound);
 283     total_doclen += o_total_doclen;
 284     if (total_doclen < o_total_doclen) {
 285         throw Xapian::DatabaseError("Total document length overflowed!");
 286     }
 287
 288     // The upper bounds might be on the same word, so we must sum them.
 289     spelling_wordfreq_ubound += o_spelling_wordfreq_ubound;
 290 }
 291
 292 void
 293 HoneyVersion::cancel()
 294 {
 295     LOGCALL_VOID(DB, "HoneyVersion::cancel", NO_ARGS);
 296     for (unsigned table_no = 0; table_no < Honey::MAX_; ++table_no) {
 297         root[table_no] = old_root[table_no];
 298     }
 299     unserialise_stats();
 300 }
 301
 302 const string
 303 HoneyVersion::write(honey_revision_number_t new_rev, int flags)
 304 {
 305     LOGCALL(DB, const string, "HoneyVersion::write", new_rev|flags);
 306
 307     string s(HONEY_VERSION_MAGIC, HONEY_VERSION_MAGIC_AND_VERSION_LEN);
 308     s.append(reinterpret_cast<const char *>(uuid), 16);
 309
 310     pack_uint(s, new_rev);
 311
 312     for (unsigned table_no = 0; table_no < Honey::MAX_; ++table_no) {
 313         root[table_no].serialise(s);
 314     }
 315
 316     // Serialise database statistics.
 317     serialise_stats();
 318     s += serialised_stats;
 319
 320     string tmpfile;
 321     if (!single_file()) {
 322         tmpfile = db_dir;
 323         // In dangerous mode, just write the new version file in place.
 324         if (flags & Xapian::DB_DANGEROUS)
 325             tmpfile += "/iamhoney";
 326         else
 327             tmpfile += "/v.tmp";
 328
 329         fd = posixy_open(tmpfile.c_str(), O_CREAT|O_TRUNC|O_WRONLY|O_BINARY, 0666);
 330         if (rare(fd < 0))
 331             throw Xapian::DatabaseOpeningError("Couldn't write new rev file: " + tmpfile,
 332                                                errno);
 333
 334         if (flags & Xapian::DB_DANGEROUS)
 335             tmpfile = string();
 336     }
 337
 338     try {
 339         io_write(fd, s.data(), s.size());
 340     } catch (...) {
 341         if (!single_file())
 342             (void)close(fd);
 343         throw;
 344     }
 345
 346     if (changes) {
 347         string changes_buf;
 348         changes_buf += '\xfe';
 349         pack_uint(changes_buf, new_rev);
 350         pack_uint(changes_buf, s.size());
 351         changes->write_block(changes_buf);
 352         changes->write_block(s);
 353     }
 354
 355     RETURN(tmpfile);
 356 }
 357
 358 bool
 359 HoneyVersion::sync(const string & tmpfile,
 360                    honey_revision_number_t new_rev, int flags)
 361 {
 362     Assert(new_rev > rev || rev == 0);
 363
 364     if (single_file()) {
 365         if ((flags & Xapian::DB_NO_SYNC) == 0 &&
 366             ((flags & Xapian::DB_FULL_SYNC) ?
 367               !io_full_sync(fd) :
 368               !io_sync(fd))) {
 369             // FIXME what to do?
 370         }
 371     } else {
 372         int fd_to_close = fd;
 373         fd = -1;
 374         if ((flags & Xapian::DB_NO_SYNC) == 0 &&
 375             ((flags & Xapian::DB_FULL_SYNC) ?
 376               !io_full_sync(fd_to_close) :
 377               !io_sync(fd_to_close))) {
 378             int save_errno = errno;
 379             (void)close(fd_to_close);
 380             if (!tmpfile.empty())
 381                 (void)unlink(tmpfile.c_str());
 382             errno = save_errno;
 383             return false;
 384         }
 385
 386         if (close(fd_to_close) != 0) {
 387             if (!tmpfile.empty()) {
 388                 int save_errno = errno;
 389                 (void)unlink(tmpfile.c_str());
 390                 errno = save_errno;
 391             }
 392             return false;
 393         }
 394
 395         if (!tmpfile.empty()) {
 396             if (!io_tmp_rename(tmpfile, db_dir + "/iamhoney")) {
 397                 return false;
 398             }
 399         }
 400     }
 401
 402     for (unsigned table_no = 0; table_no < Honey::MAX_; ++table_no) {
 403         old_root[table_no] = root[table_no];
 404     }
 405
 406     rev = new_rev;
 407     return true;
 408 }
 409
 410 // Only try to compress tags longer than this many bytes.
 411 const size_t COMPRESS_MIN = 4;
 412
 413 static const uint4 compress_min_tab[] = {
 414     0, // POSTLIST
 415     COMPRESS_MIN, // DOCDATA
 416     COMPRESS_MIN, // TERMLIST
 417     0, // POSITION
 418     COMPRESS_MIN, // SPELLING
 419     COMPRESS_MIN  // SYNONYM
 420 };
 421
 422 void
 423 HoneyVersion::create(unsigned blocksize)
 424 {
 425     AssertRel(blocksize,>=,HONEY_MIN_BLOCKSIZE);
 426     uuid_generate(uuid);
 427     for (unsigned table_no = 0; table_no < Honey::MAX_; ++table_no) {
 428         root[table_no].init(blocksize, compress_min_tab[table_no]);
 429     }
 430 }
 431
 432 namespace Honey {
 433
 434 void
 435 RootInfo::init(unsigned blocksize_, uint4 compress_min_)
 436 {
 437     AssertRel(blocksize_,>=,HONEY_MIN_BLOCKSIZE);
 438     offset = 0;
 439     root = 0;
 440     level = 0;
 441     num_entries = 0;
 442     root_is_fake = true;
 443     sequential = true;
 444     blocksize = blocksize_;
 445     compress_min = compress_min_;
 446     fl_serialised.resize(0);
 447 }
 448
 449 void
 450 RootInfo::serialise(string &s) const
 451 {
 452     AssertRel(offset, >=, 0);
 453     std::make_unsigned<off_t>::type uoffset = offset;
 454     AssertRel(root, >=, uoffset);
 455     pack_uint(s, uoffset);
 456     pack_uint(s, root - uoffset);
 457     unsigned val = level << 2;
 458     if (sequential) val |= 0x02;
 459     if (root_is_fake) val |= 0x01;
 460     pack_uint(s, val);
 461     pack_uint(s, num_entries);
 462     pack_uint(s, blocksize >> 11);
 463     pack_uint(s, compress_min);
 464     pack_string(s, fl_serialised);
 465 }
 466
 467 bool
 468 RootInfo::unserialise(const char ** p, const char * end)
 469 {
 470     std::make_unsigned<off_t>::type uoffset;
 471     unsigned val;
 472     if (!unpack_uint(p, end, &uoffset) ||
 473         !unpack_uint(p, end, &root) ||
 474         !unpack_uint(p, end, &val) ||
 475         !unpack_uint(p, end, &num_entries) ||
 476         !unpack_uint(p, end, &blocksize) ||
 477         !unpack_uint(p, end, &compress_min) ||
 478         !unpack_string(p, end, fl_serialised)) return false;
 479     offset = uoffset;
 480     root += uoffset;
 481     level = val >> 2;
 482     sequential = val & 0x02;
 483     root_is_fake = val & 0x01;
 484     blocksize <<= 11;
 485     AssertRel(blocksize,>=,HONEY_MIN_BLOCKSIZE);
 486     return true;
 487 }
 488
 489 }