xapian-bindings/python/replicationtest.py

   1 # Tests of Python-specific parts of the xapian bindings.
   2 #
   3 # Copyright (C) 2007,2008 Lemur Consulting Ltd
   4 # Copyright (C) 2008,2009 Olly Betts
   5 #
   6 # This program is free software; you can redistribute it and/or
   7 # modify it under the terms of the GNU General Public License as
   8 # published by the Free Software Foundation; either version 2 of the
   9 # License, or (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  19 # USA
  20
  21 import os
  22 import shutil
  23 import subprocess
  24 import sys
  25 import time
  26 import xapian
  27
  28 from testsuite import *
  29
  30 def set_master(masterpath, srcpath):
  31     # Take a copy of the source, to make modifications to.
  32     if os.path.exists(masterpath + "_"):
  33         shutil.rmtree(masterpath + "_")
  34     shutil.copytree(srcpath, masterpath + "_")
  35
  36     # Set a new uuid on the copy.
  37     xapian.WritableDatabase(masterpath + "__", xapian.DB_CREATE_OR_OVERWRITE)
  38     os.unlink(os.path.join(masterpath + "_", "iamchert"))
  39     os.rename(os.path.join(masterpath + "__", "iamchert"),
  40               os.path.join(masterpath + "_", "iamchert"))
  41     shutil.rmtree(masterpath + "__")
  42
  43     # Replace the current master with the copy of the source.
  44     # Note that this isn't an atomic replace, so we'll sometimes get errors
  45     # such as "NetworkError: Unable to fully synchronise: Can't open database:
  46     # Cannot open tables at consistent revisions" - the replication protocol
  47     # should recover happily from this, though.
  48     if os.path.exists(masterpath):
  49         os.rename(masterpath, masterpath + "__")
  50     os.rename(masterpath + '_', masterpath)
  51     if os.path.exists(masterpath + "__"):
  52         shutil.rmtree(masterpath + "__")
  53
  54 def test_replication_concurrency():
  55     """Test concurrent replication and modification
  56
  57     """
  58
  59     builddir = os.environ['abs_builddir']
  60     dbsdir = os.path.join(builddir, 'dbs_replication')
  61     if not os.path.isdir(dbsdir):
  62         os.makedirs(dbsdir)
  63
  64     masterpath = os.path.join(dbsdir, 'master')
  65     firstpath = os.path.join(dbsdir, 'first')
  66     secondpath = os.path.join(dbsdir, 'second')
  67     slavepath = os.path.join(dbsdir, 'slave')
  68     if os.path.isdir(masterpath):
  69         shutil.rmtree(masterpath)
  70     if os.path.isdir(slavepath):
  71         shutil.rmtree(slavepath)
  72     port = 7876
  73
  74     expect_exception(xapian.DatabaseOpeningError,
  75                      "Couldn't stat '" + dbsdir + "/slave' (No such file or directory)",
  76                      xapian.Database, slavepath)
  77
  78     clientp = None
  79     serverp = subprocess.Popen(('../../xapian-core/bin/xapian-replicate-server',
  80                                 dbsdir,
  81                                 '--port=7876',
  82                                ),
  83                               )
  84
  85     doccount1 = 10000
  86     doccount2 = 1000
  87
  88     starttime = time.time()
  89     if not os.path.isdir(firstpath):
  90         firstdb = xapian.WritableDatabase(firstpath, xapian.DB_CREATE_OR_OVERWRITE)
  91         # Make an initial, large database
  92         print
  93         print "Building initial database ..."
  94         for num in xrange(1, doccount1):
  95             doc=xapian.Document()
  96             val = 'val%d' % num
  97             doc.add_value(1, val)
  98             firstdb.add_document(doc)
  99             if num % 100000 == 0:
 100                 print "%d documents..." % num
 101         firstdb.set_metadata('dbname', '1')
 102         firstdb.commit()
 103         print "built"
 104
 105     # The secondary database gets modified during the test, so needs to be
 106     # cleared now.
 107     shutil.rmtree(secondpath)
 108     if not os.path.isdir(secondpath):
 109         seconddb = xapian.WritableDatabase(secondpath, xapian.DB_CREATE_OR_OVERWRITE)
 110         # Make second, small database
 111         print
 112         print "Building secondary database ..."
 113         for num in xrange(1, doccount2):
 114             doc=xapian.Document()
 115             val = 'val%d' % num
 116             doc.add_value(1, val)
 117             seconddb.add_document(doc)
 118             if num % 100000 == 0:
 119                 print "%d documents..." % num
 120         seconddb.set_metadata('dbname', '2')
 121         seconddb.commit()
 122         print "built"
 123
 124     if time.time() - starttime < 1:
 125         time.sleep(1) # Give server time to start
 126
 127     try:
 128         set_master(masterpath, firstpath)
 129         clientp = subprocess.Popen(('../../xapian-core/bin/xapian-replicate',
 130                                     '--host=127.0.0.1',
 131                                     '--master=master',
 132                                     os.path.join(dbsdir, 'slave'),
 133                                     '--interval=0',
 134                                     '--port=7876',
 135                                     '-r 0',
 136                                    ),
 137                                   )
 138         time.sleep(1) # Give client time to start
 139         expect(xapian.Database(slavepath).get_metadata('dbname'), '1')
 140
 141         for count in xrange(10):
 142             # Test that swapping between databases doesn't confuse replication.
 143             for count2 in xrange(2):
 144                 set_master(masterpath, secondpath)
 145                 time.sleep(0.1)
 146                 set_master(masterpath, firstpath)
 147                 time.sleep(0.1)
 148
 149             # Test making changes to the database.
 150             set_master(masterpath, secondpath)
 151             masterdb = xapian.WritableDatabase(masterpath, xapian.DB_OPEN)
 152             print "making 100 changes"
 153             for num in xrange(100):
 154                 masterdb.set_metadata('num%d' % num, str(num + count))
 155                 masterdb.commit()
 156             print "changes done"
 157             masterdb.close()
 158
 159             # Allow time for the replication client to catch up with the
 160             # changes.
 161             time.sleep(2)
 162             expect(xapian.Database(slavepath).get_metadata('dbname'), '2')
 163             expect(xapian.Database(slavepath).get_metadata('num99'), str(99 + count))
 164
 165     finally:
 166         if clientp is not None:
 167             os.kill(clientp.pid, 9)
 168             clientp.wait()
 169         os.kill(serverp.pid, 9)
 170         serverp.wait()
 171         #shutil.rmtree(dbsdir)
 172
 173 # Run all tests (ie, callables with names starting "test_").
 174 if not runtests(globals(), sys.argv[1:]):
 175     sys.exit(1)
 176
 177 # vim:syntax=python:set expandtab: