xapian-bindings/python3/smoketest.py

   1 # Simple test to ensure that we can load the xapian module and exercise basic
   2 # functionality successfully.
   3 #
   4 # Copyright (C) 2004,2005,2006,2007,2008,2010,2011,2012,2013,2014,2015,2016,2017 Olly Betts
   5 # Copyright (C) 2007 Lemur Consulting Ltd
   6 #
   7 # This program is free software; you can redistribute it and/or
   8 # modify it under the terms of the GNU General Public License as
   9 # published by the Free Software Foundation; either version 2 of the
  10 # License, or (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  20 # USA
  21
  22 import sys
  23 import re
  24 import xapian
  25
  26 from testsuite import *
  27
  28 mystemmers = set()
  29 mystemmer_id = 0
  30 # Stemmer which strips English vowels.
  31 class MyStemmer(xapian.StemImplementation):
  32     def __init__(self):
  33         global mystemmers
  34         global mystemmer_id
  35         super(MyStemmer, self).__init__()
  36         mystemmers.add(mystemmer_id)
  37         self._id = mystemmer_id
  38         mystemmer_id += 1
  39
  40     def __call__(self, s):
  41         return re.sub(br'[aeiou]', b'', s)
  42
  43     def __del__(self):
  44         global mystemmers
  45         if self._id not in mystemmers:
  46             raise TestFail("MyStemmer #%d deleted more than once" % self._id)
  47         mystemmers.remove(self._id)
  48
  49 def test_all():
  50     # Test the version number reporting functions give plausible results.
  51     v = "%d.%d.%d" % (xapian.major_version(),
  52                       xapian.minor_version(),
  53                       xapian.revision())
  54     v2 = xapian.version_string()
  55     expect(v2, v, "Unexpected version output")
  56
  57     # A regexp check would be better, but seems to create a bogus "leak" of -1
  58     # objects in Python 3.
  59     expect(len(xapian.__version__.split('.')), 3, 'xapian.__version__ not X.Y.Z')
  60     expect((xapian.__version__.split('.'))[0], '1', 'xapian.__version__ not "1.Y.Z"')
  61
  62     def access_cvar():
  63         res = xapian.cvar
  64         print("Unhandled constants: ", res)
  65         return res
  66
  67     # Check that SWIG isn't generating cvar (regression test for ticket#297).
  68     #
  69     # Python 3.5 generates a different exception message here to earlier
  70     # versions, so we need a check which matches both.
  71     expect_exception(AttributeError,
  72                      lambda msg: msg.find("has no attribute 'cvar'") != -1,
  73                      access_cvar)
  74
  75     stem = xapian.Stem(b"english")
  76     expect(str(stem), "Xapian::Stem(english)", "Unexpected str(stem)")
  77
  78     doc = xapian.Document()
  79     doc.set_data(b"a\0b")
  80     if doc.get_data() == b"a":
  81         raise TestFail("get_data+set_data truncates at a zero byte")
  82     expect(doc.get_data(), b"a\0b", "get_data+set_data doesn't transparently handle a zero byte")
  83     doc.set_data(b"is there anybody out there?")
  84     doc.add_term(b"XYzzy")
  85     doc.add_posting(stem(b"is"), 1)
  86     doc.add_posting(stem(b"there"), 2)
  87     doc.add_posting(stem(b"anybody"), 3)
  88     doc.add_posting(stem(b"out"), 4)
  89     doc.add_posting(stem(b"there"), 5)
  90
  91     db = xapian.WritableDatabase('', xapian.DB_BACKEND_INMEMORY)
  92     db.add_document(doc)
  93     expect(db.get_doccount(), 1, "Unexpected db.get_doccount()")
  94     terms = ["smoke", "test", "terms"]
  95     expect_query(xapian.Query(xapian.Query.OP_OR, [t.encode('utf-8') for t in terms]),
  96                  "(smoke OR test OR terms)")
  97     query1 = xapian.Query(xapian.Query.OP_PHRASE, (b"smoke", b"test", b"tuple"))
  98     query2 = xapian.Query(xapian.Query.OP_XOR, (xapian.Query(b"smoke"), query1, b"string"))
  99     expect_query(query1, "(smoke PHRASE 3 test PHRASE 3 tuple)")
 100     expect_query(query2, "(smoke XOR (smoke PHRASE 3 test PHRASE 3 tuple) XOR string)")
 101     subqs = ["a", "b"]
 102     expect_query(xapian.Query(xapian.Query.OP_OR, [s.encode('utf-8') for s in subqs]), "(a OR b)")
 103     expect_query(xapian.Query(xapian.Query.OP_VALUE_RANGE, 0, b'1', b'4'),
 104                  "VALUE_RANGE 0 1 4")
 105
 106     # Check database factory functions are wrapped as expected (or not wrapped
 107     # in the first cases):
 108
 109     expect_exception(AttributeError,
 110             lambda msg: msg.find("has no attribute 'open_stub'") != -1,
 111             lambda : xapian.open_stub(b"nosuchdir/nosuchdb"))
 112     expect_exception(AttributeError,
 113             lambda msg: msg.find("has no attribute 'open_stub'") != -1,
 114             lambda : xapian.open_stub(b"nosuchdir/nosuchdb", xapian.DB_OPEN))
 115
 116     expect_exception(xapian.DatabaseOpeningError, None,
 117             lambda : xapian.Database(b"nosuchdir/nosuchdb", xapian.DB_BACKEND_STUB))
 118     expect_exception(xapian.DatabaseOpeningError, None,
 119             lambda : xapian.WritableDatabase(b"nosuchdir/nosuchdb", xapian.DB_OPEN|xapian.DB_BACKEND_STUB))
 120
 121     expect_exception(xapian.DatabaseOpeningError, None,
 122             lambda : xapian.Database(b"nosuchdir/nosuchdb", xapian.DB_BACKEND_GLASS))
 123     expect_exception(xapian.DatabaseCreateError, None,
 124             lambda : xapian.WritableDatabase(b"nosuchdir/nosuchdb", xapian.DB_CREATE|xapian.DB_BACKEND_GLASS))
 125
 126     expect_exception(xapian.FeatureUnavailableError, None,
 127             lambda : xapian.Database(b"nosuchdir/nosuchdb", xapian.DB_BACKEND_CHERT))
 128     expect_exception(xapian.FeatureUnavailableError, None,
 129             lambda : xapian.WritableDatabase(b"nosuchdir/nosuchdb", xapian.DB_CREATE|xapian.DB_BACKEND_CHERT))
 130
 131     expect_exception(xapian.NetworkError, None,
 132                      xapian.remote_open, b"/bin/false", b"")
 133     expect_exception(xapian.NetworkError, None,
 134                      xapian.remote_open_writable, b"/bin/false", b"")
 135
 136     expect_exception(xapian.NetworkError, None,
 137                      xapian.remote_open, b"127.0.0.1", 0, 1)
 138     expect_exception(xapian.NetworkError, None,
 139                      xapian.remote_open_writable, b"127.0.0.1", 0, 1)
 140
 141     # Check wrapping of MatchAll and MatchNothing:
 142
 143     expect_query(xapian.Query.MatchAll, "<alldocuments>")
 144     expect_query(xapian.Query.MatchNothing, "")
 145
 146     # Feature test for Query.__iter__
 147     term_count = 0
 148     for term in query2:
 149         term_count += 1
 150     expect(term_count, 4, "Unexpected number of terms in query2")
 151
 152     enq = xapian.Enquire(db)
 153     enq.set_query(xapian.Query(xapian.Query.OP_OR, b"there", b"is"))
 154     mset = enq.get_mset(0, 10)
 155     expect(mset.size(), 1, "Unexpected mset.size()")
 156     expect(len(mset), 1, "Unexpected mset.size()")
 157
 158     # Feature test for Enquire.matching_terms(docid)
 159     term_count = 0
 160     for term in enq.matching_terms(mset.get_hit(0)):
 161         term_count += 1
 162     expect(term_count, 2, "Unexpected number of matching terms")
 163
 164     # Feature test for MSet.__iter__
 165     msize = 0
 166     for match in mset:
 167         msize += 1
 168     expect(msize, mset.size(), "Unexpected number of entries in mset")
 169
 170     terms = b" ".join(enq.matching_terms(mset.get_hit(0)))
 171     expect(terms, b"is there", "Unexpected terms")
 172
 173     # Feature test for ESet.__iter__
 174     rset = xapian.RSet()
 175     rset.add_document(1)
 176     eset = enq.get_eset(10, rset)
 177     term_count = 0
 178     for term in eset:
 179         term_count += 1
 180     expect(term_count, 3, "Unexpected number of expand terms")
 181
 182     # Feature test for Database.__iter__
 183     term_count = 0
 184     for term in db:
 185         term_count += 1
 186     expect(term_count, 5, "Unexpected number of terms in db")
 187
 188     # Feature test for Database.allterms
 189     term_count = 0
 190     for term in db.allterms():
 191         term_count += 1
 192     expect(term_count, 5, "Unexpected number of terms in db.allterms")
 193
 194     # Feature test for Database.postlist
 195     count = 0
 196     for posting in db.postlist(b"there"):
 197         count += 1
 198     expect(count, 1, "Unexpected number of entries in db.postlist('there')")
 199
 200     # Feature test for Database.postlist with empty term (alldocspostlist)
 201     count = 0
 202     for posting in db.postlist(b""):
 203         count += 1
 204     expect(count, 1, "Unexpected number of entries in db.postlist('')")
 205
 206     # Feature test for Database.termlist
 207     count = 0
 208     for term in db.termlist(1):
 209         count += 1
 210     expect(count, 5, "Unexpected number of entries in db.termlist(1)")
 211
 212     # Feature test for Database.positionlist
 213     count = 0
 214     for term in db.positionlist(1, b"there"):
 215         count += 1
 216     expect(count, 2, "Unexpected number of entries in db.positionlist(1, 'there')")
 217
 218     # Feature test for Document.termlist
 219     count = 0
 220     for term in doc.termlist():
 221         count += 1
 222     expect(count, 5, "Unexpected number of entries in doc.termlist()")
 223
 224     # Feature test for TermIter.skip_to
 225     term = doc.termlist()
 226     term.skip_to(b'n')
 227     while True:
 228         try:
 229             x = next(term)
 230         except StopIteration:
 231             break
 232         if x.term < b'n':
 233             raise TestFail("TermIter.skip_to didn't skip term '%s'" % x.term.decode('utf-8'))
 234
 235     # Feature test for Document.values
 236     count = 0
 237     for term in list(doc.values()):
 238         count += 1
 239     expect(count, 0, "Unexpected number of entries in doc.values")
 240
 241     # Check exception handling for Xapian::DocNotFoundError
 242     expect_exception(xapian.DocNotFoundError, "Docid 3 not found", db.get_document, 3)
 243
 244     # Check value of OP_ELITE_SET
 245     expect(xapian.Query.OP_ELITE_SET, 10, "Unexpected value for OP_ELITE_SET")
 246
 247     # Feature test for MatchDecider
 248     doc = xapian.Document()
 249     doc.set_data(b"Two")
 250     doc.add_posting(stem(b"out"), 1)
 251     doc.add_posting(stem(b"outside"), 1)
 252     doc.add_posting(stem(b"source"), 2)
 253     doc.add_value(0, b"yes")
 254     db.add_document(doc)
 255
 256     class testmatchdecider(xapian.MatchDecider):
 257         def __call__(self, doc):
 258             return doc.get_value(0) == b"yes"
 259
 260     query = xapian.Query(stem(b"out"))
 261     enquire = xapian.Enquire(db)
 262     enquire.set_query(query)
 263     mset = enquire.get_mset(0, 10, None, testmatchdecider())
 264     expect(mset.size(), 1, "Unexpected number of documents returned by match decider")
 265     expect(mset.get_docid(0), 2, "MatchDecider mset has wrong docid in")
 266
 267     # Feature test for ExpandDecider
 268     class testexpanddecider(xapian.ExpandDecider):
 269         def __call__(self, term):
 270             return (not term.startswith(b'a'))
 271
 272     enquire = xapian.Enquire(db)
 273     rset = xapian.RSet()
 274     rset.add_document(1)
 275     eset = enquire.get_eset(10, rset, xapian.Enquire.USE_EXACT_TERMFREQ, 1.0, testexpanddecider())
 276     eset_terms = [item.term for item in eset]
 277     expect(len(eset_terms), eset.size(), "Unexpected number of terms returned by expand")
 278     if [t for t in eset_terms if t.startswith(b'a')]:
 279         raise TestFail("ExpandDecider was not used")
 280
 281     # Check min_wt argument to get_eset() works (new in 1.2.5).
 282     eset = enquire.get_eset(100, rset, xapian.Enquire.USE_EXACT_TERMFREQ)
 283     expect([i.weight for i in eset][-1] < 1.9, True, "test get_eset() without min_wt")
 284     eset = enquire.get_eset(100, rset, xapian.Enquire.USE_EXACT_TERMFREQ, 1.0, None, 1.9)
 285     expect([i.weight for i in eset][-1] >= 1.9, True, "test get_eset() min_wt")
 286
 287     # Check QueryParser parsing error.
 288     qp = xapian.QueryParser()
 289     expect_exception(xapian.QueryParserError, "Syntax: <expression> AND <expression>", qp.parse_query, b"test AND")
 290
 291     # Check QueryParser pure NOT option
 292     qp = xapian.QueryParser()
 293     expect_query(qp.parse_query(b"NOT test", qp.FLAG_BOOLEAN + qp.FLAG_PURE_NOT),
 294                  "(<alldocuments> AND_NOT test@1)")
 295
 296     # Check QueryParser partial option
 297     qp = xapian.QueryParser()
 298     qp.set_database(db)
 299     qp.set_default_op(xapian.Query.OP_AND)
 300     qp.set_stemming_strategy(qp.STEM_SOME)
 301     qp.set_stemmer(xapian.Stem(b'en'))
 302     expect_query(qp.parse_query(b"foo o", qp.FLAG_PARTIAL),
 303                  "(Zfoo@1 AND ((SYNONYM WILDCARD OR o) OR Zo@2))")
 304
 305     expect_query(qp.parse_query(b"foo outside", qp.FLAG_PARTIAL),
 306                  "(Zfoo@1 AND ((SYNONYM WILDCARD OR outside) OR Zoutsid@2))")
 307
 308     # Test supplying unicode strings
 309     expect_query(xapian.Query(xapian.Query.OP_OR, (b'foo', b'bar')),
 310                  '(foo OR bar)')
 311     expect_query(xapian.Query(xapian.Query.OP_OR, (b'foo', b'bar\xa3')),
 312                  '(foo OR bar\\xa3)')
 313     expect_query(xapian.Query(xapian.Query.OP_OR, (b'foo', b'bar\xc2\xa3')),
 314                  '(foo OR bar\u00a3)')
 315     expect_query(xapian.Query(xapian.Query.OP_OR, b'foo', b'bar'),
 316                  '(foo OR bar)')
 317
 318     expect_query(qp.parse_query(b"NOT t\xe9st", qp.FLAG_BOOLEAN + qp.FLAG_PURE_NOT),
 319                  "(<alldocuments> AND_NOT Zt\u00e9st@1)")
 320
 321     doc = xapian.Document()
 322     doc.set_data(b"Unicode with an acc\xe9nt")
 323     doc.add_posting(stem(b"out\xe9r"), 1)
 324     expect(doc.get_data(), b"Unicode with an acc\xe9nt")
 325     term = next(doc.termlist()).term
 326     expect(term, b"out\xe9r")
 327
 328     # Check simple stopper
 329     stop = xapian.SimpleStopper()
 330     qp.set_stopper(stop)
 331     expect(stop(b'a'), False)
 332     expect_query(qp.parse_query(b"foo bar a", qp.FLAG_BOOLEAN),
 333                  "(Zfoo@1 AND Zbar@2 AND Za@3)")
 334
 335     stop.add(b'a')
 336     expect(stop(b'a'), True)
 337     expect_query(qp.parse_query(b"foo bar a", qp.FLAG_BOOLEAN),
 338                  "(Zfoo@1 AND Zbar@2)")
 339
 340     # Feature test for custom Stopper
 341     class my_b_stopper(xapian.Stopper):
 342         def __call__(self, term):
 343             return term == b"b"
 344
 345         def get_description(self):
 346             return "my_b_stopper"
 347
 348     stop = my_b_stopper()
 349     expect(stop.get_description(), "my_b_stopper")
 350     qp.set_stopper(stop)
 351     expect(stop(b'a'), False)
 352     expect_query(qp.parse_query(b"foo bar a", qp.FLAG_BOOLEAN),
 353                  "(Zfoo@1 AND Zbar@2 AND Za@3)")
 354
 355     expect(stop(b'b'), True)
 356     expect_query(qp.parse_query(b"foo bar b", qp.FLAG_BOOLEAN),
 357                  "(Zfoo@1 AND Zbar@2)")
 358
 359     # Test TermGenerator
 360     termgen = xapian.TermGenerator()
 361     doc = xapian.Document()
 362     termgen.set_document(doc)
 363     termgen.index_text(b'foo bar baz foo')
 364     expect([(item.term, item.wdf, [pos for pos in item.positer]) for item in doc.termlist()], [(b'bar', 1, [2]), (b'baz', 1, [3]), (b'foo', 2, [1, 4])])
 365
 366
 367     # Check DateValueRangeProcessor works
 368     context("checking that DateValueRangeProcessor works")
 369     qp = xapian.QueryParser()
 370     vrpdate = xapian.DateValueRangeProcessor(1, 1, 1960)
 371     qp.add_valuerangeprocessor(vrpdate)
 372     query = qp.parse_query(b'12/03/99..12/04/01')
 373     expect(str(query), 'Query(VALUE_RANGE 1 19991203 20011204)')
 374
 375     # Regression test for bug#193, fixed in 1.0.3.
 376     context("running regression test for bug#193")
 377     vrp = xapian.NumberValueRangeProcessor(0, b'$', True)
 378     a = '$10'
 379     b = '20'
 380     slot, a, b = vrp(a, b.encode('utf-8'))
 381     expect(slot, 0)
 382     expect(xapian.sortable_unserialise(a), 10)
 383     expect(xapian.sortable_unserialise(b), 20)
 384
 385     # Feature test for xapian.FieldProcessor
 386     context("running feature test for xapian.FieldProcessor")
 387     class testfieldprocessor(xapian.FieldProcessor):
 388         def __call__(self, s):
 389             if s == 'spam':
 390                 raise Exception('already spam')
 391             return xapian.Query("spam")
 392
 393     qp.add_prefix('spam', testfieldprocessor())
 394     qp.add_boolean_prefix('boolspam', testfieldprocessor())
 395     qp.add_boolean_prefix('boolspam2', testfieldprocessor(), False) # Old-style
 396     qp.add_boolean_prefix('boolspam3', testfieldprocessor(), '')
 397     qp.add_boolean_prefix('boolspam4', testfieldprocessor(), 'group')
 398     qp.add_boolean_prefix('boolspam5', testfieldprocessor(), None)
 399     query = qp.parse_query('spam:ignored')
 400     expect(str(query), 'Query(spam)')
 401
 402     # FIXME: This doesn't currently work:
 403     # expect_exception(Exception, 'already spam', qp.parse_query, 'spam:spam')
 404
 405     # Regression tests copied from PHP (probably always worked in python, but
 406     # let's check...)
 407     context("running regression tests for issues which were found in PHP")
 408
 409     # PHP overload resolution involving boolean types failed.
 410     enq.set_sort_by_value(1, True)
 411
 412     # Regression test - fixed in 0.9.10.1.
 413     oqparser = xapian.QueryParser()
 414     oquery = oqparser.parse_query(b"I like tea")
 415
 416     # Regression test for bug fixed in 1.4.4:
 417     # https://bugs.debian.org/849722
 418     oqparser.add_boolean_prefix('tag', 'K', '')
 419     # Make sure other cases also work:
 420     oqparser.add_boolean_prefix('zag', 'XR', False) # Old-style
 421     oqparser.add_boolean_prefix('rag', 'XR', None)
 422     oqparser.add_boolean_prefix('nag', 'XB', '')
 423     oqparser.add_boolean_prefix('bag', 'XB', 'blergh')
 424     oqparser.add_boolean_prefix('gag', 'XB', u'blergh')
 425     oqparser.add_boolean_prefix('jag', 'XB', b'blergh')
 426
 427     # Regression test for bug#192 - fixed in 1.0.3.
 428     enq.set_cutoff(100)
 429
 430     # Test setting and getting metadata
 431     expect(db.get_metadata(b'Foo'), b'')
 432     db.set_metadata(b'Foo', b'Foo')
 433     expect(db.get_metadata(b'Foo'), b'Foo')
 434     expect_exception(xapian.InvalidArgumentError, "Empty metadata keys are invalid", db.get_metadata, b'')
 435     expect_exception(xapian.InvalidArgumentError, "Empty metadata keys are invalid", db.set_metadata, b'', b'Foo')
 436     expect_exception(xapian.InvalidArgumentError, "Empty metadata keys are invalid", db.get_metadata, b'')
 437
 438     # Test OP_SCALE_WEIGHT and corresponding constructor
 439     expect_query(xapian.Query(xapian.Query.OP_SCALE_WEIGHT, xapian.Query(b'foo'), 5),
 440                  "5 * foo")
 441
 442 def test_userstem():
 443     mystem = MyStemmer()
 444     stem = xapian.Stem(mystem)
 445     expect(stem(b'test'), b'tst')
 446     stem2 = xapian.Stem(mystem)
 447     expect(stem2(b'toastie'), b'tst')
 448
 449     indexer = xapian.TermGenerator()
 450     indexer.set_stemmer(xapian.Stem(MyStemmer()))
 451
 452     doc = xapian.Document()
 453     indexer.set_document(doc)
 454     indexer.index_text(b'hello world')
 455
 456     s = '/'
 457     for t in doc.termlist():
 458         s += t.term.decode('utf-8')
 459         s += '/'
 460     expect(s, '/Zhll/Zwrld/hello/world/')
 461
 462     parser = xapian.QueryParser()
 463     parser.set_stemmer(xapian.Stem(MyStemmer()))
 464     parser.set_stemming_strategy(xapian.QueryParser.STEM_ALL)
 465     expect_query(parser.parse_query(b'color television'), '(clr@1 OR tlvsn@2)')
 466
 467 def test_zz9_check_leaks():
 468     import gc
 469     gc.collect()
 470     if len(mystemmers):
 471         raise TestFail("%d MyStemmer objects not deleted" % len(mystemmers))
 472
 473 # Run all tests (ie, callables with names starting "test_").
 474 if not runtests(globals()):
 475     sys.exit(1)
 476
 477 # vim:syntax=python:set expandtab: