xapian-core/bin/xapian-replicate.cc

   1 /** @file xapian-replicate.cc
   2  * @brief Replicate a database from a master server to a local copy.
   3  */
   4 /* Copyright (C) 2008,2011,2012,2015 Olly Betts
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License as
   8  * published by the Free Software Foundation; either version 2 of the
   9  * License, or (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  19  * USA
  20  */
  21
  22 #include <config.h>
  23
  24 #include "net/replicatetcpclient.h"
  25
  26 #include <xapian.h>
  27
  28 #include "gnu_getopt.h"
  29 #include "stringutils.h"
  30 #include "safeunistd.h"
  31
  32 #include <iostream>
  33
  34 using namespace std;
  35
  36 #define PROG_NAME "xapian-replicate"
  37 #define PROG_DESC "Replicate a database from a master server to a local copy"
  38
  39 #define OPT_HELP 1
  40 #define OPT_VERSION 2
  41
  42 // Wait DEFAULT_INTERVAL seconds between updates unless --interval is passed.
  43 #define DEFAULT_INTERVAL 60
  44
  45 // Number of seconds before we assume that a reader will be closed.
  46 #define READER_CLOSE_TIME 30
  47
  48 // Socket level timeout (in seconds).
  49 #define DEFAULT_TIMEOUT 0
  50
  51 static void show_usage() {
  52     cout << "Usage: " PROG_NAME " [OPTIONS] DATABASE\n\n"
  53 "Options:\n"
  54 "  -h, --host=HOST     host to connect to (required)\n"
  55 "  -p, --port=PORT     port to connect to (required)\n"
  56 "  -m, --master=DB     replicate database DB from the master (default: DATABASE)\n"
  57 "  -i, --interval=N    wait N seconds between each connection to the master\n"
  58 "                      (default: " STRINGIZE(DEFAULT_INTERVAL) ")\n"
  59 "  -r, --reader-time=N wait N seconds to allow readers time to close before\n"
  60 "                      applying repeated changesets (default: " STRINGIZE(READER_CLOSE_TIME) ")\n"
  61 "  -t, --timeout=N     set socket timeouts (if supported) to N seconds; N=0 for\n"
  62 "                      no timeout (default: " STRINGIZE(DEFAULT_TIMEOUT) ")\n"
  63 "  -f, --force-copy    force a full copy of the database to be sent (and then\n"
  64 "                      replicate as normal)\n"
  65 "  -o, --one-shot      replicate only once and then exit\n"
  66 "  -q, --quiet         only report errors\n"
  67 "  -v, --verbose       be more verbose\n"
  68 "  --help              display this help and exit\n"
  69 "  --version           output version information and exit" << endl;
  70 }
  71
  72 int
  73 main(int argc, char **argv)
  74 {
  75     const char * opts = "h:p:m:i:r:t:ofqv";
  76     const struct option long_opts[] = {
  77         {"host",        required_argument,      0, 'h'},
  78         {"port",        required_argument,      0, 'p'},
  79         {"master",      required_argument,      0, 'm'},
  80         {"interval",    required_argument,      0, 'i'},
  81         {"reader-time", required_argument,      0, 'r'},
  82         {"timeout",     required_argument,      0, 't'},
  83         {"one-shot",    no_argument,            0, 'o'},
  84         {"force-copy",  no_argument,            0, 'f'},
  85         {"quiet",       no_argument,            0, 'q'},
  86         {"verbose",     no_argument,            0, 'v'},
  87         {"help",        no_argument, 0, OPT_HELP},
  88         {"version",     no_argument, 0, OPT_VERSION},
  89         {NULL,          0, 0, 0}
  90     };
  91
  92     string host;
  93     int port = 0;
  94     string masterdb;
  95     int interval = DEFAULT_INTERVAL;
  96     bool one_shot = false;
  97     enum { NORMAL, VERBOSE, QUIET } verbosity = NORMAL;
  98     bool force_copy = false;
  99     int reader_close_time = READER_CLOSE_TIME;
 100     int timeout = DEFAULT_TIMEOUT;
 101
 102     int c;
 103     while ((c = gnu_getopt_long(argc, argv, opts, long_opts, 0)) != -1) {
 104         switch (c) {
 105             case 'h':
 106                 host.assign(optarg);
 107                 break;
 108             case 'p':
 109                 port = atoi(optarg);
 110                 break;
 111             case 'm':
 112                 masterdb.assign(optarg);
 113                 break;
 114             case 'i':
 115                 interval = atoi(optarg);
 116                 break;
 117             case 'r':
 118                 reader_close_time = atoi(optarg);
 119                 break;
 120             case 't':
 121                 timeout = atoi(optarg);
 122                 break;
 123             case 'f':
 124                 force_copy = true;
 125                 break;
 126             case 'o':
 127                 one_shot = true;
 128                 break;
 129             case 'q':
 130                 verbosity = QUIET;
 131                 break;
 132             case 'v':
 133                 verbosity = VERBOSE;
 134                 break;
 135             case OPT_HELP:
 136                 cout << PROG_NAME " - " PROG_DESC "\n\n";
 137                 show_usage();
 138                 exit(0);
 139             case OPT_VERSION:
 140                 cout << PROG_NAME " - " PACKAGE_STRING << endl;
 141                 exit(0);
 142             default:
 143                 show_usage();
 144                 exit(1);
 145         }
 146     }
 147
 148     if (argc - optind != 1) {
 149         show_usage();
 150         exit(1);
 151     }
 152
 153     if (host.empty()) {
 154         cout << "Host required - specify with --host=HOST\n\n";
 155         show_usage();
 156         exit(1);
 157     }
 158
 159     if (port == 0) {
 160         cout << "Port required - specify with --port=PORT\n\n";
 161         show_usage();
 162         exit(1);
 163     }
 164
 165     // Path to the database to create/update.
 166     string dbpath(argv[optind]);
 167
 168     if (masterdb.empty())
 169         masterdb = dbpath;
 170
 171     while (true) {
 172         try {
 173             if (verbosity == VERBOSE) {
 174                 cout << "Connecting to " << host << ":" << port << endl;
 175             }
 176             ReplicateTcpClient client(host, port, 10.0, timeout);
 177             if (verbosity == VERBOSE) {
 178                 cout << "Getting update for " << dbpath << " from "
 179                      << masterdb << endl;
 180             }
 181             Xapian::ReplicationInfo info;
 182             client.update_from_master(dbpath, masterdb, info,
 183                                       reader_close_time, force_copy);
 184             if (verbosity == VERBOSE) {
 185                 cout << "Update complete: "
 186                      << info.fullcopy_count << " copies, "
 187                      << info.changeset_count << " changesets, "
 188                      << (info.changed ? "new live database"
 189                                       : "no changes to live database")
 190                      << endl;
 191             }
 192             if (verbosity != QUIET) {
 193                 if (info.fullcopy_count > 0 && !info.changed) {
 194                     cout <<
 195 "Replication using a full copy failed.  This usually means that the master\n"
 196 "database is changing too frequently.  Ensure that sufficient changesets are\n"
 197 "present by setting XAPIAN_MAX_CHANGESETS on the master." << endl;
 198                 }
 199             }
 200             force_copy = false;
 201         } catch (const Xapian::NetworkError &error) {
 202             // Don't stop running if there's a network error - just log to
 203             // stderr and retry at next timeout.  This should make the client
 204             // robust against temporary network failures.
 205             cerr << argv[0] << ": " << error.get_description() << endl;
 206
 207             // If we were running as a one-shot client though, we're going to
 208             // exit anyway, so let's make the return value reflect that there
 209             // was a failure.
 210             if (one_shot)
 211                 exit(1);
 212         } catch (const Xapian::Error &error) {
 213             cerr << argv[0] << ": " << error.get_description() << endl;
 214             exit(1);
 215         } catch (const exception &e) {
 216             cerr << "Caught standard exception: " << e.what() << endl;
 217             exit(1);
 218         } catch (...) {
 219             cerr << "Caught unknown exception" << endl;
 220             exit(1);
 221         }
 222         if (one_shot) break;
 223         sleep(interval);
 224     }
 225 }