xapian-core/backends/glass/glass_changes.cc

   1 /** @file glass_changes.cc
   2  * @brief Glass changesets
   3  */
   4 /* Copyright 2014 Olly Betts
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License as
   8  * published by the Free Software Foundation; either version 2 of the
   9  * License, or (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  19  * USA
  20  */
  21
  22 #include <config.h>
  23
  24 #include "glass_changes.h"
  25
  26 #include "glass_replicate_internal.h"
  27 #include "fd.h"
  28 #include "io_utils.h"
  29 #include "pack.h"
  30 #include "posixy_wrapper.h"
  31 #include "str.h"
  32 #include "stringutils.h"
  33 #include "unaligned.h"
  34 #include "xapian/constants.h"
  35 #include "xapian/error.h"
  36
  37 #include <cstdlib>
  38 #include <string>
  39 #include "safeerrno.h"
  40
  41 using namespace std;
  42
  43 GlassChanges::~GlassChanges()
  44 {
  45     if (changes_fd >= 0) {
  46         ::close(changes_fd);
  47         string changes_tmp = changes_stem;
  48         changes_tmp += "tmp";
  49         io_unlink(changes_tmp);
  50     }
  51 }
  52
  53 GlassChanges *
  54 GlassChanges::start(glass_revision_number_t old_rev,
  55                     glass_revision_number_t rev,
  56                     int flags)
  57 {
  58     if (rev == 0) {
  59         // Don't generate a changeset for the first revision.
  60         return NULL;
  61     }
  62
  63     // Always check max_changesets for modification since last revision.
  64     const char *p = getenv("XAPIAN_MAX_CHANGESETS");
  65     if (p) {
  66         max_changesets = atoi(p);
  67     } else {
  68         max_changesets = 0;
  69     }
  70
  71     if (max_changesets == 0)
  72         return NULL;
  73
  74     string changes_tmp = changes_stem;
  75     changes_tmp += "tmp";
  76     changes_fd = posixy_open(changes_tmp.c_str(),
  77                              O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0666);
  78     if (changes_fd < 0) {
  79         string message = "Couldn't open changeset ";
  80         message += changes_tmp;
  81         message += " to write";
  82         throw Xapian::DatabaseError(message, errno);
  83     }
  84
  85     // Write header for changeset file.
  86     string header = CHANGES_MAGIC_STRING;
  87     header += char(CHANGES_VERSION);
  88     pack_uint(header, old_rev);
  89     pack_uint(header, rev);
  90
  91     if (flags & Xapian::DB_DANGEROUS) {
  92         header += '\x01'; // Changes can't be applied to a live database.
  93     } else {
  94         header += '\x00'; // Changes can be applied to a live database.
  95     }
  96
  97     io_write(changes_fd, header.data(), header.size());
  98     // FIXME: save the block stream as a single zlib stream...
  99
 100     // bool compressed = CHANGES_VERSION != 1; FIXME: always true for glass, but make optional?
 101     return this;
 102 }
 103
 104 void
 105 GlassChanges::write_block(const char * p, size_t len)
 106 {
 107     io_write(changes_fd, p, len);
 108 }
 109
 110 void
 111 GlassChanges::commit(glass_revision_number_t new_rev, int flags)
 112 {
 113     if (changes_fd < 0)
 114         return;
 115
 116     io_write(changes_fd, "\xff", 1);
 117
 118     string changes_tmp = changes_stem;
 119     changes_tmp += "tmp";
 120
 121     if (!(flags & Xapian::DB_NO_SYNC) && !io_sync(changes_fd)) {
 122         int saved_errno = errno;
 123         (void)::close(changes_fd);
 124         changes_fd = -1;
 125         (void)unlink(changes_tmp.c_str());
 126         string m = changes_tmp;
 127         m += ": Failed to sync";
 128         throw Xapian::DatabaseError(m, saved_errno);
 129     }
 130
 131     (void)::close(changes_fd);
 132     changes_fd = -1;
 133
 134     string changes_file = changes_stem;
 135     changes_file += str(new_rev - 1); // FIXME: ?
 136
 137     if (posixy_rename(changes_tmp.c_str(), changes_file.c_str()) < 0) {
 138         // With NFS, rename() failing may just mean that the server crashed
 139         // after successfully renaming, but before reporting this, and then
 140         // the retried operation fails.  So we need to check if the source
 141         // file still exists, which we do by calling unlink(), since we want
 142         // to remove the temporary file anyway.
 143         int saved_errno = errno;
 144         if (unlink(changes_tmp.c_str()) == 0 || errno != ENOENT) {
 145             string m = changes_tmp;
 146             m += ": Failed to rename to ";
 147             m += changes_file;
 148             throw Xapian::DatabaseError(m, saved_errno);
 149         }
 150     }
 151
 152     if (new_rev <= max_changesets) {
 153         // We can't yet have max_changesets old changesets.
 154         return;
 155     }
 156
 157     // Only remove old changesets if we successfully wrote a new changeset.
 158     // Start at the oldest changeset we know about, and stop at max_changesets
 159     // before new_rev.  If max_changesets is unchanged from the previous
 160     // commit and nothing went wrong, exactly one changeset file should be
 161     // deleted.
 162     glass_revision_number_t stop_changeset = new_rev - max_changesets;
 163     while (oldest_changeset < stop_changeset) {
 164         changes_file.resize(changes_stem.size());
 165         changes_file += str(oldest_changeset);
 166         (void)io_unlink(changes_file);
 167         ++oldest_changeset;
 168     }
 169 }
 170
 171 void
 172 GlassChanges::check(const string & changes_file)
 173 {
 174     FD fd(posixy_open(changes_file.c_str(), O_RDONLY | O_CLOEXEC, 0666));
 175     if (fd < 0) {
 176         string message = "Couldn't open changeset ";
 177         message += changes_file;
 178         throw Xapian::DatabaseError(message, errno);
 179     }
 180
 181     char buf[10240];
 182
 183     size_t n = io_read(fd, buf, sizeof(buf), CONST_STRLEN(CHANGES_MAGIC_STRING) + 4);
 184     if (memcmp(buf, CHANGES_MAGIC_STRING,
 185                CONST_STRLEN(CHANGES_MAGIC_STRING)) != 0) {
 186         throw Xapian::DatabaseError("Changes file has wrong magic");
 187     }
 188
 189     const char * p = buf + CONST_STRLEN(CHANGES_MAGIC_STRING);
 190     if (*p++ != CHANGES_VERSION) {
 191         throw Xapian::DatabaseError("Changes file has unknown version");
 192     }
 193     const char * end = buf + n;
 194
 195     glass_revision_number_t old_rev, rev;
 196     if (!unpack_uint(&p, end, &old_rev))
 197         throw Xapian::DatabaseError("Changes file has bad old_rev");
 198     if (!unpack_uint(&p, end, &rev))
 199         throw Xapian::DatabaseError("Changes file has bad rev");
 200     if (rev <= old_rev)
 201         throw Xapian::DatabaseError("Changes file has rev <= old_rev");
 202     if (p == end || (*p != 0 && *p != 1))
 203         throw Xapian::DatabaseError("Changes file has bad dangerous flag");
 204     ++p;
 205
 206     while (true) {
 207         n -= (p - buf);
 208         memmove(buf, p, n);
 209         n += io_read(fd, buf + n, sizeof(buf) - n, 0);
 210
 211         if (n == 0)
 212             throw Xapian::DatabaseError("Changes file truncated");
 213
 214         p = buf;
 215         end = buf + n;
 216
 217         unsigned char v = *p++;
 218         if (v == 0xff) {
 219             if (p != end)
 220                 throw Xapian::DatabaseError("Changes file - junk at end");
 221             break;
 222         }
 223         if (v == 0xfe) {
 224             // Version file.
 225             glass_revision_number_t version_rev;
 226             if (!unpack_uint(&p, end, &version_rev))
 227                 throw Xapian::DatabaseError("Changes file - bad version file revision");
 228             if (rev != version_rev)
 229                 throw Xapian::DatabaseError("Version file revision != changes file new revision");
 230             size_t len;
 231             if (!unpack_uint(&p, end, &len))
 232                 throw Xapian::DatabaseError("Changes file - bad version file length");
 233             if (len <= size_t(end - p)) {
 234                 p += len;
 235             } else {
 236                 if (lseek(fd, len - (end - p), SEEK_CUR) == off_t(-1))
 237                     throw Xapian::DatabaseError("Changes file - version file data truncated");
 238                 p = end = buf;
 239                 n = 0;
 240             }
 241             continue;
 242         }
 243         unsigned table = (v & 0x7);
 244         v >>= 3;
 245         if (table > 5)
 246             throw Xapian::DatabaseError("Changes file - bad table code");
 247         // Changed block.
 248         if (v > 5)
 249             throw Xapian::DatabaseError("Changes file - bad block size");
 250         unsigned block_size = 2048 << v;
 251         uint4 block_number;
 252         if (!unpack_uint(&p, end, &block_number))
 253             throw Xapian::DatabaseError("Changes file - bad block number");
 254         uint4 block_rev = getint4(reinterpret_cast<const unsigned char *>(p), 0);
 255         (void)block_rev; // FIXME: Sanity check value.
 256         unsigned level = (unsigned char)p[4];
 257         (void)level; // FIXME: Sanity check value.
 258         if (block_size <= unsigned(end - p)) {
 259             p += block_size;
 260         } else {
 261             if (lseek(fd, block_size - (end - p), SEEK_CUR) == off_t(-1))
 262                 throw Xapian::DatabaseError("Changes file - block data truncated");
 263             p = end = buf;
 264             n = 0;
 265         }
 266     }
 267 }