xapian-bindings/python/replicationtest.py

   1 # Tests of Python-specific parts of the xapian bindings.
   2 #
   3 # Copyright (C) 2007,2008 Lemur Consulting Ltd
   4 # Copyright (C) 2008,2009 Olly Betts
   5 #
   6 # This program is free software; you can redistribute it and/or
   7 # modify it under the terms of the GNU General Public License as
   8 # published by the Free Software Foundation; either version 2 of the
   9 # License, or (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  19 # USA
  20
  21 import os
  22 import shutil
  23 import subprocess
  24 import sys
  25 import time
  26 import xapian
  27
  28 from testsuite import *
  29
  30 def set_master(masterpath, srcpath):
  31     # Take a copy of the source, to make modifications to.
  32     if os.path.exists(masterpath + "_"):
  33         shutil.rmtree(masterpath + "_")
  34     shutil.copytree(srcpath, masterpath + "_")
  35
  36     # Set a new uuid on the copy.  FIXME: Needs updating for glass where you
  37     # can't just update the uuid like this.
  38     sys.exit(1)
  39     xapian.WritableDatabase(masterpath + "__", xapian.DB_CREATE_OR_OVERWRITE)
  40     os.unlink(os.path.join(masterpath + "_", "iamchert"))
  41     os.rename(os.path.join(masterpath + "__", "iamchert"),
  42               os.path.join(masterpath + "_", "iamchert"))
  43     shutil.rmtree(masterpath + "__")
  44
  45     # Replace the current master with the copy of the source.
  46     # Note that this isn't an atomic replace, so we'll sometimes get errors
  47     # such as "NetworkError: Unable to fully synchronise: Can't open database:
  48     # Cannot open tables at consistent revisions" - the replication protocol
  49     # should recover happily from this, though.
  50     if os.path.exists(masterpath):
  51         os.rename(masterpath, masterpath + "__")
  52     os.rename(masterpath + '_', masterpath)
  53     if os.path.exists(masterpath + "__"):
  54         shutil.rmtree(masterpath + "__")
  55
  56 def test_replication_concurrency():
  57     """Test concurrent replication and modification
  58
  59     """
  60
  61     builddir = os.environ['abs_builddir']
  62     dbsdir = os.path.join(builddir, 'dbs_replication')
  63     if not os.path.isdir(dbsdir):
  64         os.makedirs(dbsdir)
  65
  66     masterpath = os.path.join(dbsdir, 'master')
  67     firstpath = os.path.join(dbsdir, 'first')
  68     secondpath = os.path.join(dbsdir, 'second')
  69     slavepath = os.path.join(dbsdir, 'slave')
  70     if os.path.isdir(masterpath):
  71         shutil.rmtree(masterpath)
  72     if os.path.isdir(slavepath):
  73         shutil.rmtree(slavepath)
  74     port = 7876
  75
  76     expect_exception(xapian.DatabaseOpeningError,
  77                      "Couldn't stat '" + dbsdir + "/slave' (No such file or directory)",
  78                      xapian.Database, slavepath)
  79
  80     clientp = None
  81     serverp = subprocess.Popen(('../../xapian-core/bin/xapian-replicate-server',
  82                                 dbsdir,
  83                                 '--port=7876',
  84                                ),
  85                               )
  86
  87     doccount1 = 10000
  88     doccount2 = 1000
  89
  90     starttime = time.time()
  91     if not os.path.isdir(firstpath):
  92         firstdb = xapian.WritableDatabase(firstpath, xapian.DB_CREATE_OR_OVERWRITE)
  93         # Make an initial, large database
  94         print
  95         print "Building initial database ..."
  96         for num in xrange(1, doccount1):
  97             doc=xapian.Document()
  98             val = 'val%d' % num
  99             doc.add_value(1, val)
 100             firstdb.add_document(doc)
 101             if num % 100000 == 0:
 102                 print "%d documents..." % num
 103         firstdb.set_metadata('dbname', '1')
 104         firstdb.commit()
 105         print "built"
 106
 107     # The secondary database gets modified during the test, so needs to be
 108     # cleared now.
 109     shutil.rmtree(secondpath)
 110     if not os.path.isdir(secondpath):
 111         seconddb = xapian.WritableDatabase(secondpath, xapian.DB_CREATE_OR_OVERWRITE)
 112         # Make second, small database
 113         print
 114         print "Building secondary database ..."
 115         for num in xrange(1, doccount2):
 116             doc=xapian.Document()
 117             val = 'val%d' % num
 118             doc.add_value(1, val)
 119             seconddb.add_document(doc)
 120             if num % 100000 == 0:
 121                 print "%d documents..." % num
 122         seconddb.set_metadata('dbname', '2')
 123         seconddb.commit()
 124         print "built"
 125
 126     if time.time() - starttime < 1:
 127         time.sleep(1) # Give server time to start
 128
 129     try:
 130         set_master(masterpath, firstpath)
 131         clientp = subprocess.Popen(('../../xapian-core/bin/xapian-replicate',
 132                                     '--host=127.0.0.1',
 133                                     '--master=master',
 134                                     os.path.join(dbsdir, 'slave'),
 135                                     '--interval=0',
 136                                     '--port=7876',
 137                                     '-r 0',
 138                                    ),
 139                                   )
 140         time.sleep(1) # Give client time to start
 141         expect(xapian.Database(slavepath).get_metadata('dbname'), '1')
 142
 143         for count in xrange(10):
 144             # Test that swapping between databases doesn't confuse replication.
 145             for count2 in xrange(2):
 146                 set_master(masterpath, secondpath)
 147                 time.sleep(0.1)
 148                 set_master(masterpath, firstpath)
 149                 time.sleep(0.1)
 150
 151             # Test making changes to the database.
 152             set_master(masterpath, secondpath)
 153             masterdb = xapian.WritableDatabase(masterpath, xapian.DB_OPEN)
 154             print "making 100 changes"
 155             for num in xrange(100):
 156                 masterdb.set_metadata('num%d' % num, str(num + count))
 157                 masterdb.commit()
 158             print "changes done"
 159             masterdb.close()
 160
 161             # Allow time for the replication client to catch up with the
 162             # changes.
 163             time.sleep(2)
 164             expect(xapian.Database(slavepath).get_metadata('dbname'), '2')
 165             expect(xapian.Database(slavepath).get_metadata('num99'), str(99 + count))
 166
 167     finally:
 168         if clientp is not None:
 169             os.kill(clientp.pid, 9)
 170             clientp.wait()
 171         os.kill(serverp.pid, 9)
 172         serverp.wait()
 173         #shutil.rmtree(dbsdir)
 174
 175 # Run all tests (ie, callables with names starting "test_").
 176 if not runtests(globals(), sys.argv[1:]):
 177     sys.exit(1)
 178
 179 # vim:syntax=python:set expandtab: