xapian-bindings/lua/smoketest.lua

   1 #!/usr/bin/env lua
   2 --
   3 -- Simple test to ensure that we can load the xapian module and exercise
   4 -- basic functionality successfully.
   5 --
   6 -- Copyright (C) 2011 Xiaona Han
   7 -- Copyright (C) 2011,2012,2014,2016,2017 Olly Betts
   8 --
   9 -- This program is free software; you can redistribute it and/or
  10 -- modify it under the terms of the GNU General Public License as
  11 -- published by the Free Software Foundation; either version 2 of the
  12 -- License, or (at your option) any later version.
  13 --
  14 -- This program is distributed in the hope that it will be useful,
  15 -- but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 -- GNU General Public License for more details.
  18 --
  19 -- You should have received a copy of the GNU General Public License
  20 -- along with this program; if not, write to the Free Software
  21 -- Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
  22 -- USA
  23
  24 ---
  25 -- got: obtained value
  26 -- exp: expected value
  27 -- msg: failure message (a string)
  28 function expect(got, exp, msg)
  29   msg = msg and msg..': ' or ''
  30   if type(exp) == "table" then
  31     assert('table' == type(got), msg..'not a table')
  32     assert(#exp == #got, msg..'got: '..tostring(#got)..' want: '..tostring(#exp))
  33     for i = 1, #exp do
  34       expect(got[i], exp[i])
  35     end
  36   else
  37     assert(exp == got, msg..'got: '..(tostring(got) or '???')..' want: '..(tostring(exp) or '???'))
  38   end
  39 end
  40
  41 function run_tests()
  42   local xap = require 'xapian'
  43   -- Check that require sets global "xapian":
  44   expect(nil ~= xapian, true, 'global xapian is nil')
  45   expect(type(xapian), 'table', 'global xapian is not a table')
  46   -- Check that require returns the module table (SWIG 2.0.4 returned 'xapian'):
  47   expect(nil ~= xap, true, "require 'xapian' returned nil")
  48   expect(type(xap), 'table', "require 'xapian' didn't return a table")
  49   expect(xapian, xap, "require 'xapian' return value not the same as global xapian")
  50
  51   stem = xapian.Stem("english")
  52   doc = xapian.Document()
  53   doc:set_data("is there anybody out there?")
  54   doc:add_posting(stem("is"), 1)
  55   doc:add_posting(stem("there"), 2)
  56   doc:add_posting(stem("anybody"), 3)
  57   doc:add_posting(stem("out"), 4)
  58   doc:add_posting(stem("there"), 5)
  59   doc:add_term("XYzzy")
  60   db = xapian.WritableDatabase("", xapian.DB_BACKEND_INMEMORY)
  61   db2 = xapian.WritableDatabase("", xapian.DB_BACKEND_INMEMORY)
  62   db:add_document(doc)
  63   enq = xapian.Enquire(db)
  64
  65   it = db:positionlist_begin(1, "is")
  66   _end = db:positionlist_end(1, "is")
  67   while not it:equals(_end) do
  68     assert(it:get_termpos(), 1)
  69     it:next()
  70   end
  71
  72   -- Test the version number reporting functions give plausible results.
  73   v = string.format("%d.%d.%d", xapian.major_version(), xapian.minor_version(), xapian.revision())
  74   v2 = xapian.version_string()
  75   expect(v, v2)
  76
  77   -- Test stem
  78   expect(tostring(stem), "Xapian::Stem(english)")
  79   expect(stem("is"), "is", 'stem noop "is" fails')
  80   expect(stem("going"), "go", 'stem noop "go" fails')
  81   expect(stem("wanted"), "want", 'stem(english) "wanted" -> "want" fails')
  82   expect(stem("reference"), "refer", 'stem(english) "reference" -> "refer" fails')
  83
  84   -- Test document
  85   expect(doc:termlist_count(), 5)
  86   expect(doc:get_data(), "is there anybody out there?")
  87   doc:add_term("foo")
  88   expect(doc:termlist_count(), 6)
  89
  90   -- Test database
  91   expect(tostring(db), "WritableDatabase(InMemory)")
  92   expect(db:get_doccount(), 1, 'db should have 1 document')
  93
  94   term_count = 0
  95   for term in db:allterms() do
  96    term_count = term_count + 1
  97   end
  98   expect(term_count, 5)
  99
 100   -- Test queries
 101   terms = {"smoke", "test", "terms"}
 102   expect(tostring(xapian.Query(xapian.Query_OP_OR, terms)), "Query((smoke OR test OR terms))")
 103   query1 = xapian.Query(xapian.Query_OP_PHRASE, {"smoke", "test", "tuple"})
 104   query2 = xapian.Query(xapian.Query_OP_XOR, {xapian.Query("smoke"), query1, "string"})
 105   expect(tostring(query1), "Query((smoke PHRASE 3 test PHRASE 3 tuple))")
 106   expect(tostring(query2), "Query((smoke XOR (smoke PHRASE 3 test PHRASE 3 tuple) XOR string))")
 107   subqs = {"a", "b"}
 108   expect(tostring(xapian.Query(xapian.Query_OP_OR, subqs)), "Query((a OR b))")
 109   expect(tostring(xapian.Query(xapian.Query_OP_VALUE_RANGE, 0, '1', '4')), "Query(VALUE_RANGE 0 1 4)")
 110
 111   -- Test MatchAll and MatchNothing
 112   expect(tostring(xapian.Query_MatchAll), "Query(<alldocuments>)")
 113   expect(tostring(xapian.Query_MatchNothing), "Query()")
 114
 115   -- Test enq
 116   query = xapian.Query(xapian.Query_OP_OR, {"there", "is"})
 117   enq:set_query(query)
 118   mset = enq:get_mset(0, 10)
 119   expect(mset:size(), 1)
 120
 121   terms = {}
 122   for term in enq:get_matching_terms(mset:get_hit(0)) do
 123     table.insert(terms, term:get_term())
 124   end
 125   expect(terms, {"is", "there"})
 126
 127   -- Check value of OP_ELITE_SET
 128   expect(xapian.Query_OP_ELITE_SET, 10)
 129
 130   -- Test for MatchDecider
 131   doc = xapian.Document()
 132   doc:set_data("Two")
 133   doc:add_posting(stem("out"), 1)
 134   doc:add_posting(stem("outside"), 1)
 135   doc:add_posting(stem("source"), 2)
 136   doc:add_value(0, "yes");
 137   db:add_document(doc)
 138
 139   function testmatchdecider(doc)
 140     return doc:get_value(0) == "yes"
 141   end
 142
 143   query = xapian.Query("out")
 144   enq = xapian.Enquire(db)
 145   enq:set_query(query)
 146   mset = enq:get_mset(0, 10, None, testmatchdecider)
 147   expect(mset:size(), 1)
 148   expect(mset:get_docid(0), 2)
 149
 150   rset = xapian.RSet()
 151   rset:add_document(1)
 152   eset = enq:get_eset(10, rset)
 153   expect(eset:size(), 4)
 154
 155   eterms={}
 156   for term in eset:terms() do
 157     table.insert(eterms, term:get_term())
 158   end
 159   expect(eterms, {"there", "is", "anybodi", "XYzzy"})
 160
 161   function testexpanddecider(term)
 162     return term ~= "there"
 163   end
 164
 165   eset = enq:get_eset(10, rset, testexpanddecider)
 166   expect(eset:size(), 3)
 167
 168   eterms={}
 169   for term in eset:terms() do
 170     table.insert(eterms, term:get_term())
 171   end
 172   expect(eterms, {"is", "anybodi", "XYzzy"})
 173
 174   -- Regression test - overload resolution involving boolean types failed.
 175   enq:set_sort_by_value(1, true)
 176
 177   -- Regression test - fixed in 0.9.10.1.
 178   oqparser = xapian.QueryParser()
 179   oquery = oqparser:parse_query("I like tea")
 180
 181   -- Regression test for bug#192 - fixed in 1.0.3.
 182   enq:set_cutoff(100)
 183
 184   -- Check DateRangeProcessor works
 185   qp = xapian.QueryParser()
 186   rpdate = xapian.DateRangeProcessor(1, xapian.RP_DATE_PREFER_MDY, 1960)
 187   qp:add_rangeprocessor(rpdate)
 188   query = qp:parse_query("12/03/99..12/04/01")
 189   expect(tostring(query), "Query(VALUE_RANGE 1 19991203 20011204)")
 190
 191   -- Check DateValueRangeProcessor works
 192   qp = xapian.QueryParser()
 193   vrpdate = xapian.DateValueRangeProcessor(1, true, 1960)
 194   qp:add_valuerangeprocessor(vrpdate)
 195   query = qp:parse_query("12/03/99..12/04/01")
 196   expect(tostring(query), "Query(VALUE_RANGE 1 19991203 20011204)")
 197
 198   -- Check FieldProcessor works
 199   qp:add_prefix('test', function (s) return "foo" end)
 200   qp:add_prefix('rev', function (s) return xapian.Query(s:reverse()) end)
 201   query = qp:parse_query("test:ing rev:erse")
 202   expect(tostring(query), "Query((foo OR esre))")
 203
 204   -- Test setting and getting metadata
 205   db:set_metadata('Foo', 'Foo')
 206   expect(db:get_metadata('Foo'), 'Foo')
 207
 208   -- Test OP_SCALE_WEIGHT and corresponding constructor
 209   query4 = xapian.Query(xapian.Query_OP_SCALE_WEIGHT, xapian.Query('foo'), 5.0)
 210   expect(tostring(query4), "Query(5 * foo)")
 211
 212   -- Function to test the order of mset docid
 213   function mset_expect_order(mset, a)
 214     if mset:size() ~= #a then
 215       print(string.format("MSet has %i entries, expected %i\n",  mset:size(), #a))
 216       os.exit(-1)
 217     end
 218
 219     for j = 0, #a -1 do
 220       hit = mset:get_hit(j)
 221       if hit:get_docid() ~= a[j + 1] then
 222         print(string.format("Expected MSet[%i] to be %i, got %i.\n", j, a[j + 1], hit:get_docid()))
 223         os.exit(-1)
 224       end
 225     end
 226   end
 227
 228   -- Test MultiValueKeyMaker
 229   doc = xapian.Document()
 230   doc:add_term("foo")
 231   doc:add_value(0, "ABB")
 232   db2:add_document(doc)
 233   doc:add_value(0, "ABC")
 234   db2:add_document(doc)
 235   doc:add_value(0, "ABC\0")
 236   db2:add_document(doc)
 237   doc:add_value(0, "ABCD")
 238   db2:add_document(doc)
 239   doc:add_value(0, "ABC\xff")
 240   db2:add_document(doc)
 241
 242   enq = xapian.Enquire(db2)
 243   enq:set_query(xapian.Query("foo"))
 244
 245   sorter = xapian.MultiValueKeyMaker()
 246   sorter:add_value(0)
 247   enq:set_sort_by_key(sorter, true)
 248   mset = enq:get_mset(0, 10)
 249   mset_expect_order(mset, {5, 4, 3, 2, 1})
 250
 251   sorter = xapian.MultiValueKeyMaker()
 252   sorter:add_value(0, true)
 253   enq:set_sort_by_key(sorter, true)
 254   mset = enq:get_mset(0, 10)
 255   mset_expect_order(mset, {1, 2, 3, 4, 5})
 256
 257   sorter = xapian.MultiValueKeyMaker()
 258   sorter:add_value(0)
 259   sorter:add_value(1)
 260   enq:set_sort_by_key(sorter, true)
 261   mset = enq:get_mset(0, 10)
 262   mset_expect_order(mset, {5, 4, 3, 2, 1})
 263
 264   sorter = xapian.MultiValueKeyMaker()
 265   sorter:add_value(0, true)
 266   sorter:add_value(1)
 267   enq:set_sort_by_key(sorter, true)
 268   mset = enq:get_mset(0, 10)
 269   mset_expect_order(mset, {1, 2, 3, 4, 5})
 270
 271   sorter = xapian.MultiValueKeyMaker()
 272   sorter:add_value(0)
 273   sorter:add_value(1, true)
 274   enq:set_sort_by_key(sorter, true)
 275   mset = enq:get_mset(0, 10)
 276   mset_expect_order(mset, {5, 4, 3, 2, 1})
 277
 278   sorter = xapian.MultiValueKeyMaker()
 279   sorter:add_value(0, true)
 280   sorter:add_value(1, true)
 281   enq:set_sort_by_key(sorter, true)
 282   mset = enq:get_mset(0, 10)
 283   mset_expect_order(mset, {1, 2, 3, 4, 5})
 284
 285   -- Feature test for ValueSetMatchDecider:
 286   md = xapian.ValueSetMatchDecider(0, true)
 287   md:add_value("ABC")
 288   doc = xapian.Document()
 289   doc:add_value(0, "ABCD")
 290   if md(doc) then
 291     print "Unexpected result from ValueSetMatchDecider() expected false\n"
 292     os.exit(-1)
 293   end
 294
 295   doc = xapian.Document()
 296   doc:add_value(0, "ABC")
 297   if not md(doc) then
 298     print "Unexpected result from ValueSetMatchDecider() expected true\n"
 299     os.exit(-1)
 300   end
 301
 302   mset = enq:get_mset(0, 10, 0, nil, md)
 303   mset_expect_order(mset, {2})
 304
 305   md = xapian.ValueSetMatchDecider(0, false)
 306   md:add_value("ABC")
 307   mset = enq:get_mset(0, 10, 0, nil, md)
 308   mset_expect_order(mset, {1, 3, 4, 5})
 309
 310   -- Feature tests for Query "term" constructor optional arguments:
 311   query_wqf = xapian.Query('wqf', 3)
 312   if tostring(query_wqf) ~= 'Query(wqf#3)' then
 313     print "Unexpected query_wqf->tostring():\n"
 314     print(tostring(query_wqf) .."\n")
 315     os.exit(-1)
 316   end
 317
 318   query = xapian.Query(xapian.Query_OP_VALUE_GE, 0, "100")
 319   if tostring(query) ~= 'Query(VALUE_GE 0 100)' then
 320     print "Unexpected query->tostring():\n"
 321     print(tostring(query) .. "\n")
 322     os.exit(-1)
 323   end
 324
 325   -- Test access to matchspy values:
 326   matchspy = xapian.ValueCountMatchSpy(0)
 327   enq:add_matchspy(matchspy)
 328   enq:get_mset(0, 10)
 329   beg = matchspy:values_begin()
 330   _end = matchspy:values_end()
 331
 332   values = {}
 333   while not beg:equals(_end) do
 334     values[beg:get_term()] = beg:get_termfreq()
 335     beg:next()
 336   end
 337
 338   expected = {["ABB"] = 1, ["ABC"] = 1, ["ABC\0"] = 1, ["ABCD"] =1, ["ABC\xff"] = 1}
 339   expect(values, expected)
 340
 341   mset:get_hit(0)
 342
 343   ---Test preservation of stopper set on query parser.
 344   function make_qp()
 345     queryparser = xapian.QueryParser()
 346     stopper = xapian.SimpleStopper()
 347     stopper:add('to')
 348     stopper:add('not')
 349     queryparser:set_stopper(stopper)
 350     return queryparser
 351   end
 352   queryparser = make_qp()
 353   query = queryparser:parse_query('to be')
 354   terms = {}
 355   for term in queryparser:stoplist() do
 356     table.insert(terms, term:get_term())
 357   end
 358   expect(terms, {"to"})
 359
 360   -- Test preservation of stopper set on term generator.
 361   function make_tg()
 362     termgen = xapian.TermGenerator()
 363     termgen:set_stemmer(xapian.Stem('en'))
 364     stopper = xapian.SimpleStopper()
 365     stopper:add('to')
 366     stopper:add('not')
 367     termgen:set_stopper(stopper)
 368     return termgen
 369   end
 370   termgen = make_tg()
 371   termgen:index_text('to be')
 372   doc = termgen:get_document()
 373   terms = {}
 374   for term in doc:termlist() do
 375     table.insert(terms, term:get_term())
 376   end
 377   expect(terms, {"Zbe", "be", "to"})
 378
 379   -- Test use of matchspies
 380   function setup_database()
 381     -- Set up and return an inmemory database with 5 documents.
 382     db = xapian.WritableDatabase("", xapian.DB_BACKEND_INMEMORY)
 383
 384     doc = xapian.Document()
 385     doc:set_data("is it cold?")
 386     doc:add_term("is")
 387     doc:add_posting("it", 1)
 388     doc:add_posting("cold", 2)
 389     db:add_document(doc)
 390
 391     doc = xapian.Document()
 392     doc:set_data("was it warm?")
 393     doc:add_posting("was", 1)
 394     doc:add_posting("it", 2)
 395     doc:add_posting("warm", 3)
 396     db:add_document(doc)
 397     doc:set_data("was it warm? two")
 398     doc:add_term("two", 2)
 399     doc:add_value(0, xapian.sortable_serialise(2))
 400     db:add_document(doc)
 401     doc:set_data("was it warm? three")
 402     doc:add_term("three", 3)
 403     doc:add_value(0, xapian.sortable_serialise(1.5))
 404     db:add_document(doc)
 405     doc:set_data("was it warm? four it")
 406     doc:add_term("four", 4)
 407     doc:add_term("it", 6)
 408     doc:add_posting("it", 7)
 409     doc:add_value(5, 'five')
 410     doc:add_value(9, 'nine')
 411     doc:add_value(0, xapian.sortable_serialise(2))
 412     db:add_document(doc)
 413
 414     expect(db:get_doccount(), 5)
 415     return db
 416   end
 417   db = setup_database()
 418   query = xapian.Query(xapian.Query_OP_OR, "was", "it")
 419   enq = xapian.Enquire(db)
 420   enq:set_query(query)
 421   function set_matchspy_deref(enq)
 422     -- Set a matchspy, and then drop the reference, to check that it
 423     -- doesn't get deleted too soon.
 424     spy = xapian.ValueCountMatchSpy(0)
 425     enq:add_matchspy(spy)
 426   end
 427   set_matchspy_deref(enq)
 428   mset = enq:get_mset(0, 10)
 429   expect(mset:size(), 5)
 430
 431   spy = xapian.ValueCountMatchSpy(0)
 432   enq:add_matchspy(spy)
 433   enq:clear_matchspies()
 434   mset = enq:get_mset(0, 10)
 435   spy:values()
 436   items = {}
 437   for item in spy:values() do
 438     table.insert(items, item:get_term())
 439   end
 440   expect(items, {})
 441
 442   enq:add_matchspy(spy)
 443   mset = enq:get_mset(0, 10)
 444   expect(spy:get_total(), 5)
 445   items = {}
 446   for item in spy:values() do
 447     table.insert(items, {item:get_term(), item:get_termfreq()})
 448   end
 449   expect(items, {{xapian.sortable_serialise(1.5), 1}, {xapian.sortable_serialise(2), 2}})
 450
 451   items = {}
 452   for item in spy:top_values(10) do
 453     table.insert(items, {item:get_term(), item:get_termfreq()})
 454   end
 455   expect(items, {{xapian.sortable_serialise(2), 2}, {xapian.sortable_serialise(1.5), 1}})
 456
 457   -- Test exceptions
 458   ok,res = pcall(db.get_document, db, 0)
 459   expect(ok, false)
 460   expect(tostring(res), "InvalidArgumentError: Document ID 0 is invalid")
 461   expect(res:get_type(), "InvalidArgumentError")
 462 end
 463
 464 ok, e = pcall(run_tests)
 465 if not ok then
 466   print("Unexpected exception:", tostring(e))
 467   os.exit(-1)
 468 end