xapian-core/backends/glass/glass_changes.cc

   1 /** @file glass_changes.cc
   2  * @brief Glass changesets
   3  */
   4 /* Copyright 2014,2016 Olly Betts
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License as
   8  * published by the Free Software Foundation; either version 2 of the
   9  * License, or (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  19  * USA
  20  */
  21
  22 #include <config.h>
  23
  24 #include "glass_changes.h"
  25
  26 #include "glass_defs.h"
  27 #include "glass_replicate_internal.h"
  28 #include "fd.h"
  29 #include "io_utils.h"
  30 #include "pack.h"
  31 #include "posixy_wrapper.h"
  32 #include "str.h"
  33 #include "stringutils.h"
  34 #include "wordaccess.h"
  35 #include "xapian/constants.h"
  36 #include "xapian/error.h"
  37
  38 #include <cerrno>
  39 #include <cstdlib>
  40 #include <string>
  41
  42 using namespace std;
  43
  44 GlassChanges::~GlassChanges()
  45 {
  46     if (changes_fd >= 0) {
  47         ::close(changes_fd);
  48         string changes_tmp = changes_stem;
  49         changes_tmp += "tmp";
  50         io_unlink(changes_tmp);
  51     }
  52 }
  53
  54 GlassChanges *
  55 GlassChanges::start(glass_revision_number_t old_rev,
  56                     glass_revision_number_t rev,
  57                     int flags)
  58 {
  59     if (rev == 0) {
  60         // Don't generate a changeset for the first revision.
  61         return NULL;
  62     }
  63
  64     // Always check max_changesets for modification since last revision.
  65     const char *p = getenv("XAPIAN_MAX_CHANGESETS");
  66     if (p) {
  67         max_changesets = atoi(p);
  68     } else {
  69         max_changesets = 0;
  70     }
  71
  72     if (max_changesets == 0)
  73         return NULL;
  74
  75     string changes_tmp = changes_stem;
  76     changes_tmp += "tmp";
  77     changes_fd = posixy_open(changes_tmp.c_str(),
  78                              O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0666);
  79     if (changes_fd < 0) {
  80         string message = "Couldn't open changeset ";
  81         message += changes_tmp;
  82         message += " to write";
  83         throw Xapian::DatabaseError(message, errno);
  84     }
  85
  86     // Write header for changeset file.
  87     string header = CHANGES_MAGIC_STRING;
  88     header += char(CHANGES_VERSION);
  89     pack_uint(header, old_rev);
  90     pack_uint(header, rev);
  91
  92     if (flags & Xapian::DB_DANGEROUS) {
  93         header += '\x01'; // Changes can't be applied to a live database.
  94     } else {
  95         header += '\x00'; // Changes can be applied to a live database.
  96     }
  97
  98     io_write(changes_fd, header.data(), header.size());
  99     // FIXME: save the block stream as a single zlib stream...
 100
 101     // bool compressed = CHANGES_VERSION != 1; FIXME: always true for glass, but make optional?
 102     return this;
 103 }
 104
 105 void
 106 GlassChanges::write_block(const char * p, size_t len)
 107 {
 108     io_write(changes_fd, p, len);
 109 }
 110
 111 void
 112 GlassChanges::commit(glass_revision_number_t new_rev, int flags)
 113 {
 114     if (changes_fd < 0)
 115         return;
 116
 117     io_write(changes_fd, "\xff", 1);
 118
 119     string changes_tmp = changes_stem;
 120     changes_tmp += "tmp";
 121
 122     if (!(flags & Xapian::DB_NO_SYNC) && !io_sync(changes_fd)) {
 123         int saved_errno = errno;
 124         (void)::close(changes_fd);
 125         changes_fd = -1;
 126         (void)unlink(changes_tmp.c_str());
 127         string m = changes_tmp;
 128         m += ": Failed to sync";
 129         throw Xapian::DatabaseError(m, saved_errno);
 130     }
 131
 132     (void)::close(changes_fd);
 133     changes_fd = -1;
 134
 135     string changes_file = changes_stem;
 136     changes_file += str(new_rev - 1); // FIXME: ?
 137
 138     if (!io_tmp_rename(changes_tmp, changes_file)) {
 139         string m = changes_tmp;
 140         m += ": Failed to rename to ";
 141         m += changes_file;
 142         throw Xapian::DatabaseError(m, errno);
 143     }
 144
 145     if (new_rev <= max_changesets) {
 146         // We can't yet have max_changesets old changesets.
 147         return;
 148     }
 149
 150     // Only remove old changesets if we successfully wrote a new changeset.
 151     // Start at the oldest changeset we know about, and stop at max_changesets
 152     // before new_rev.  If max_changesets is unchanged from the previous
 153     // commit and nothing went wrong, exactly one changeset file should be
 154     // deleted.
 155     glass_revision_number_t stop_changeset = new_rev - max_changesets;
 156     while (oldest_changeset < stop_changeset) {
 157         changes_file.resize(changes_stem.size());
 158         changes_file += str(oldest_changeset);
 159         (void)io_unlink(changes_file);
 160         ++oldest_changeset;
 161     }
 162 }
 163
 164 void
 165 GlassChanges::check(const string & changes_file)
 166 {
 167     FD fd(posixy_open(changes_file.c_str(), O_RDONLY | O_CLOEXEC, 0666));
 168     if (fd < 0) {
 169         string message = "Couldn't open changeset ";
 170         message += changes_file;
 171         throw Xapian::DatabaseError(message, errno);
 172     }
 173
 174     char buf[10240];
 175
 176     size_t n = io_read(fd, buf, sizeof(buf), CONST_STRLEN(CHANGES_MAGIC_STRING) + 4);
 177     if (memcmp(buf, CHANGES_MAGIC_STRING,
 178                CONST_STRLEN(CHANGES_MAGIC_STRING)) != 0) {
 179         throw Xapian::DatabaseError("Changes file has wrong magic");
 180     }
 181
 182     const char * p = buf + CONST_STRLEN(CHANGES_MAGIC_STRING);
 183     if (*p++ != CHANGES_VERSION) {
 184         throw Xapian::DatabaseError("Changes file has unknown version");
 185     }
 186     const char * end = buf + n;
 187
 188     glass_revision_number_t old_rev, rev;
 189     if (!unpack_uint(&p, end, &old_rev))
 190         throw Xapian::DatabaseError("Changes file has bad old_rev");
 191     if (!unpack_uint(&p, end, &rev))
 192         throw Xapian::DatabaseError("Changes file has bad rev");
 193     if (rev <= old_rev)
 194         throw Xapian::DatabaseError("Changes file has rev <= old_rev");
 195     if (p == end || (*p != 0 && *p != 1))
 196         throw Xapian::DatabaseError("Changes file has bad dangerous flag");
 197     ++p;
 198
 199     while (true) {
 200         n -= (p - buf);
 201         memmove(buf, p, n);
 202         n += io_read(fd, buf + n, sizeof(buf) - n);
 203
 204         if (n == 0)
 205             throw Xapian::DatabaseError("Changes file truncated");
 206
 207         p = buf;
 208         end = buf + n;
 209
 210         unsigned char v = *p++;
 211         if (v == 0xff) {
 212             if (p != end)
 213                 throw Xapian::DatabaseError("Changes file - junk at end");
 214             break;
 215         }
 216         if (v == 0xfe) {
 217             // Version file.
 218             glass_revision_number_t version_rev;
 219             if (!unpack_uint(&p, end, &version_rev))
 220                 throw Xapian::DatabaseError("Changes file - bad version file revision");
 221             if (rev != version_rev)
 222                 throw Xapian::DatabaseError("Version file revision != changes file new revision");
 223             size_t len;
 224             if (!unpack_uint(&p, end, &len))
 225                 throw Xapian::DatabaseError("Changes file - bad version file length");
 226             if (len <= size_t(end - p)) {
 227                 p += len;
 228             } else {
 229                 if (lseek(fd, len - (end - p), SEEK_CUR) < 0)
 230                     throw Xapian::DatabaseError("Changes file - version file data truncated");
 231                 p = end = buf;
 232                 n = 0;
 233             }
 234             continue;
 235         }
 236         unsigned table = (v & 0x7);
 237         v >>= 3;
 238         if (table > 5)
 239             throw Xapian::DatabaseError("Changes file - bad table code");
 240         // Changed block.
 241         if (v > 5)
 242             throw Xapian::DatabaseError("Changes file - bad block size");
 243         unsigned block_size = GLASS_MIN_BLOCKSIZE << v;
 244         uint4 block_number;
 245         if (!unpack_uint(&p, end, &block_number))
 246             throw Xapian::DatabaseError("Changes file - bad block number");
 247         // Although the revision number is aligned within the block, the block
 248         // data may not be aligned to a word boundary here.
 249         uint4 block_rev = unaligned_read4(reinterpret_cast<const uint8_t*>(p));
 250         (void)block_rev; // FIXME: Sanity check value.
 251         p += 4;
 252         unsigned level = static_cast<unsigned char>(*p++);
 253         (void)level; // FIXME: Sanity check value.
 254         if (block_size <= unsigned(end - p)) {
 255             p += block_size;
 256         } else {
 257             if (lseek(fd, block_size - (end - p), SEEK_CUR) < 0)
 258                 throw Xapian::DatabaseError("Changes file - block data truncated");
 259             p = end = buf;
 260             n = 0;
 261         }
 262     }
 263 }