3 # The author disclaims copyright to this source code. In place of
4 # a legal notice, here is a blessing:
6 # May you do good and not evil.
7 # May you find forgiveness for yourself and forgive others.
8 # May you share freely, never taking more than you give.
10 #***********************************************************************
11 # This file implements regression tests for the FTS3 module. The focus
12 # of this file is tables created with the "matchinfo=fts3" option.
15 set testdir [file dirname $argv0]
16 source $testdir/tester.tcl
18 # If SQLITE_ENABLE_FTS3 is not defined, omit this file.
19 ifcapable !fts3 { finish_test ; return }
21 set testprefix fts3matchinfo
22 set sqlite_fts3_enable_parentheses 0
25 set scan(littleEndian) i*
26 set scan(bigEndian) I*
27 binary scan $blob $scan($::tcl_platform(byteOrder)) r
33 CREATE VIRTUAL TABLE t1 USING fts4(matchinfo=fts3);
34 SELECT name FROM sqlite_master WHERE type = 'table';
35 } {t1 t1_content t1_segments t1_segdir t1_stat}
38 INSERT INTO t1(content) VALUES('I wandered lonely as a cloud');
39 INSERT INTO t1(content) VALUES('That floats on high o''er vales and hills,');
40 INSERT INTO t1(content) VALUES('When all at once I saw a crowd,');
41 INSERT INTO t1(content) VALUES('A host, of golden daffodils,');
42 SELECT mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'I';
43 } {{1 1 1 2 2} {1 1 1 2 2}}
45 # Now create an FTS4 table that does not specify matchinfo=fts3.
48 CREATE VIRTUAL TABLE t2 USING fts4;
49 INSERT INTO t2 SELECT * FROM t1;
50 SELECT mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'I';
51 } {{1 1 1 2 2} {1 1 1 2 2}}
53 # Test some syntax-error handling.
55 do_catchsql_test 2.0 {
56 CREATE VIRTUAL TABLE x1 USING fts4(matchinfo=fs3);
57 } {1 {unrecognized matchinfo: fs3}}
58 do_catchsql_test 2.1 {
59 CREATE VIRTUAL TABLE x2 USING fts4(mtchinfo=fts3);
60 } {1 {unrecognized parameter: mtchinfo=fts3}}
61 do_catchsql_test 2.2 {
62 CREATE VIRTUAL TABLE x2 USING fts4(matchinfo=fts5);
63 } {1 {unrecognized matchinfo: fts5}}
65 # Check that with fts3, the "=" character is permitted in column definitions.
68 CREATE VIRTUAL TABLE t3 USING fts3(mtchinfo=fts3);
69 INSERT INTO t3(mtchinfo) VALUES('Beside the lake, beneath the trees');
70 SELECT mtchinfo FROM t3;
71 } {{Beside the lake, beneath the trees}}
74 CREATE VIRTUAL TABLE xx USING FTS4;
77 SELECT * FROM xx WHERE xx MATCH 'abc';
80 SELECT * FROM xx WHERE xx MATCH 'a b c';
84 #--------------------------------------------------------------------------
85 # Proc [do_matchinfo_test] is used to test the FTSX matchinfo() function.
87 # The first argument - $tn - is a test identifier. This may be either a
88 # full identifier (i.e. "fts3matchinfo-1.1") or, if global var $testprefix
89 # is set, just the numeric component (i.e. "1.1").
91 # The second argument is the name of an FTSX table. The third is the
92 # full text of a WHERE/MATCH expression to query the table for
93 # (i.e. "t1 MATCH 'abc'"). The final argument - $results - should be a
94 # key-value list (serialized array) with matchinfo() format specifiers
95 # as keys, and the results of executing the statement:
97 # SELECT matchinfo($tbl, '$key') FROM $tbl WHERE $expr
101 # CREATE VIRTUAL TABLE t1 USING fts4;
102 # INSERT INTO t1 VALUES('abc');
103 # INSERT INTO t1 VALUES('def');
104 # INSERT INTO t1 VALUES('abc abc');
106 # do_matchinfo_test 1.1 t1 "t1 MATCH 'abc'" {
110 # x {{1 3 2} {2 3 2}}
113 # If the $results list contains keys mapped to "-" instead of a matchinfo()
114 # result, then this command computes the expected results based on other
115 # mappings to test the matchinfo() function. For example, the command above
116 # could be changed to:
118 # do_matchinfo_test 1.1 t1 "t1 MATCH 'abc'" {
119 # n {3 3} p {1 1} c {1 1} x {{1 3 2} {2 3 2}}
123 # And this command would compute the expected results for matchinfo(t1, 'pcx')
124 # based on the results of matchinfo(t1, 'p'), matchinfo(t1, 'c') and
125 # matchinfo(t1, 'x') in order to test 'pcx'.
127 proc do_matchinfo_test {tn tbl expr results} {
129 foreach {fmt res} $results {
130 if {$res == "-"} continue
131 set resarray($fmt) $res
135 foreach {fmt res} [array get resarray] {
136 if {[llength $res]>$nRow} { set nRow [llength $res] }
139 # Construct expected results for any formats for which the caller
140 # supplied result is "-".
142 foreach {fmt res} $results {
145 for {set iRow 0} {$iRow<$nRow} {incr iRow} {
147 foreach c [split $fmt ""] {
148 set rowres [concat $rowres [lindex $resarray($c) $iRow]]
152 set resarray($fmt) $res
156 # Test each matchinfo() request individually.
158 foreach {fmt res} [array get resarray] {
159 set sql "SELECT mit(matchinfo($tbl, '$fmt')) FROM $tbl WHERE $expr"
160 do_execsql_test $tn.$fmt $sql [normalize2 $res]
163 # Test them all executed together (multiple invocations of matchinfo()).
166 foreach {format res} [array get resarray] {
167 lappend exprlist "mit(matchinfo($tbl, '$format'))"
170 for {set iRow 0} {$iRow<$nRow} {incr iRow} {
171 foreach {format res} [array get resarray] {
172 lappend allres [lindex $res $iRow]
175 set sql "SELECT [join $exprlist ,] FROM $tbl WHERE $expr"
176 do_execsql_test $tn.multi $sql [normalize2 $allres]
178 proc normalize2 {list_of_lists} {
180 foreach elem $list_of_lists {
181 lappend res [list {*}$elem]
187 do_execsql_test 4.1.0 {
188 CREATE VIRTUAL TABLE t4 USING fts4(x, y);
189 INSERT INTO t4 VALUES('a b c d e', 'f g h i j');
190 INSERT INTO t4 VALUES('f g h i j', 'a b c d e');
193 do_matchinfo_test 4.1.1 t4 {t4 MATCH 'a b c'} {
197 {1 1 1 0 1 1 1 1 1 0 1 1 1 1 1 0 1 1}
198 {0 1 1 1 1 1 0 1 1 1 1 1 0 1 1 1 1 1}
206 xxxxxxxxxxxxxxxxxx - pcx - xpc - ccc - pppxpcpcx - laxnpc -
210 do_matchinfo_test 4.1.2 t4 {t4 MATCH '"g h i"'} {
223 xxxxxxxxxxxxxxxxxx - pcx - xpc - ccc - pppxpcpcx - laxnpc -
227 do_matchinfo_test 4.1.3 t4 {t4 MATCH 'a b'} { s {{2 0} {0 2}} }
228 do_matchinfo_test 4.1.4 t4 {t4 MATCH '"a b" c'} { s {{2 0} {0 2}} }
229 do_matchinfo_test 4.1.5 t4 {t4 MATCH 'a "b c"'} { s {{2 0} {0 2}} }
230 do_matchinfo_test 4.1.6 t4 {t4 MATCH 'd d'} { s {{1 0} {0 1}} }
231 do_matchinfo_test 4.1.7 t4 {t4 MATCH 'f OR abcd'} {
233 {0 1 1 1 1 1 0 0 0 0 0 0}
234 {1 1 1 0 1 1 0 0 0 0 0 0}
237 do_matchinfo_test 4.1.8 t4 {t4 MATCH 'f -abcd'} {
244 do_execsql_test 4.2.0 {
245 CREATE VIRTUAL TABLE t5 USING fts4;
246 INSERT INTO t5 VALUES('a a a a a');
247 INSERT INTO t5 VALUES('a b a b a');
248 INSERT INTO t5 VALUES('c b c b c');
249 INSERT INTO t5 VALUES('x x x x x');
251 do_matchinfo_test 4.2.1 t5 {t5 MATCH 'a a'} {
252 x {{5 8 2 5 8 2} {3 8 2 3 8 2}}
255 do_matchinfo_test 4.2.2 t5 {t5 MATCH 'a b'} { s {2} }
256 do_matchinfo_test 4.2.3 t5 {t5 MATCH 'a b a'} { s {3} }
257 do_matchinfo_test 4.2.4 t5 {t5 MATCH 'a a a'} { s {3 1} }
258 do_matchinfo_test 4.2.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
259 do_matchinfo_test 4.2.6 t5 {t5 MATCH 'a OR b'} { s {1 2 1} }
261 do_execsql_test 4.3.0 "INSERT INTO t5 VALUES('x y [string repeat {b } 50000]')";
263 # It used to be that the second 'a' token would be deferred. That doesn't
266 do_matchinfo_test 4.3.1 t5 {t5 MATCH 'a a'} {
267 x {{5 8 2 5 5 5} {3 8 2 3 5 5}}
272 do_matchinfo_test 4.3.2 t5 {t5 MATCH 'a b'} { s {2} }
273 do_matchinfo_test 4.3.3 t5 {t5 MATCH 'a b a'} { s {3} }
274 do_matchinfo_test 4.3.4 t5 {t5 MATCH 'a a a'} { s {3 1} }
275 do_matchinfo_test 4.3.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
276 do_matchinfo_test 4.3.6 t5 {t5 MATCH 'a OR b'} { s {1 2 1 1} }
278 do_execsql_test 4.4.0.1 { INSERT INTO t5(t5) VALUES('optimize') }
280 ifcapable fts4_deferred {
281 sqlite3_db_config db DEFENSIVE 0
282 do_execsql_test 4.4.0.2 {
284 SET block = zeroblob(length(block))
285 WHERE length(block)>10000;
289 do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'} { s {2} }
290 do_matchinfo_test 4.4.1 t5 {t5 MATCH 'a a'} { s {2 1} }
291 do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'} { s {2} }
292 do_matchinfo_test 4.4.3 t5 {t5 MATCH 'a b a'} { s {3} }
293 do_matchinfo_test 4.4.4 t5 {t5 MATCH 'a a a'} { s {3 1} }
294 do_matchinfo_test 4.4.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
296 do_execsql_test 4.5.0 {
297 CREATE VIRTUAL TABLE t6 USING fts4(a, b, c);
298 INSERT INTO t6 VALUES('a', 'b', 'c');
300 do_matchinfo_test 4.5.1 t6 {t6 MATCH 'a b c'} { s {{1 1 1}} }
303 #-------------------------------------------------------------------------
304 # Check the following restrictions:
306 # + Matchinfo flags 'a', 'l' and 'n' can only be used with fts4, not fts3.
307 # + Matchinfo flag 'l' cannot be used with matchinfo=fts3.
309 do_execsql_test 5.1 {
310 CREATE VIRTUAL TABLE t7 USING fts3(a, b);
311 INSERT INTO t7 VALUES('u v w', 'x y z');
313 CREATE VIRTUAL TABLE t8 USING fts4(a, b, matchinfo=fts3);
314 INSERT INTO t8 VALUES('u v w', 'x y z');
317 do_catchsql_test 5.2.1 {
318 SELECT matchinfo(t7, 'a') FROM t7 WHERE t7 MATCH 'x y'
319 } {1 {unrecognized matchinfo request: a}}
320 do_catchsql_test 5.2.2 {
321 SELECT matchinfo(t7, 'l') FROM t7 WHERE t7 MATCH 'x y'
322 } {1 {unrecognized matchinfo request: l}}
323 do_catchsql_test 5.2.3 {
324 SELECT matchinfo(t7, 'n') FROM t7 WHERE t7 MATCH 'x y'
325 } {1 {unrecognized matchinfo request: n}}
327 do_catchsql_test 5.3.1 {
328 SELECT matchinfo(t8, 'l') FROM t8 WHERE t8 MATCH 'x y'
329 } {1 {unrecognized matchinfo request: l}}
331 #-------------------------------------------------------------------------
332 # Test that the offsets() function handles corruption in the %_content
335 do_execsql_test 6.1 {
336 CREATE VIRTUAL TABLE t9 USING fts4;
337 INSERT INTO t9 VALUES(
338 'this record is used to try to dectect corruption'
340 SELECT offsets(t9) FROM t9 WHERE t9 MATCH 'to';
341 } {{0 0 20 2 0 0 27 2}}
343 sqlite3_db_config db DEFENSIVE 0
344 do_catchsql_test 6.2 {
345 UPDATE t9_content SET c0content = 'this record is used to';
346 SELECT offsets(t9) FROM t9 WHERE t9 MATCH 'to';
347 } {1 {database disk image is malformed}}
349 #-------------------------------------------------------------------------
350 # Test the outcome of matchinfo() when used within a query that does not
351 # use the full-text index (i.e. lookup by rowid or full-table scan).
353 do_execsql_test 7.1 {
354 CREATE VIRTUAL TABLE t10 USING fts4;
355 INSERT INTO t10 VALUES('first record');
356 INSERT INTO t10 VALUES('second record');
358 do_execsql_test 7.2 {
359 SELECT typeof(matchinfo(t10)), length(matchinfo(t10)) FROM t10;
361 do_execsql_test 7.3 {
362 SELECT typeof(matchinfo(t10)), length(matchinfo(t10)) FROM t10 WHERE docid=1;
364 do_execsql_test 7.4 {
365 SELECT typeof(matchinfo(t10)), length(matchinfo(t10))
366 FROM t10 WHERE t10 MATCH 'record'
369 #-------------------------------------------------------------------------
370 # Test a special case - matchinfo('nxa') with many zero length documents.
371 # Special because "x" internally uses a statement used by both "n" and "a".
372 # This was causing a problem at one point in the obscure case where the
373 # total number of bytes of data stored in an fts3 table was greater than
374 # the number of rows. i.e. when the following query returns true:
376 # SELECT sum(length(content)) < count(*) FROM fts4table;
378 do_execsql_test 8.1 {
379 CREATE VIRTUAL TABLE t11 USING fts4;
380 INSERT INTO t11(t11) VALUES('nodesize=24');
381 INSERT INTO t11 VALUES('quitealongstringoftext');
382 INSERT INTO t11 VALUES('anotherquitealongstringoftext');
383 INSERT INTO t11 VALUES('athirdlongstringoftext');
384 INSERT INTO t11 VALUES('andonemoreforgoodluck');
387 for {set i 0} {$i < 200} {incr i} {
388 execsql { INSERT INTO t11 VALUES('') }
390 execsql { INSERT INTO t11(t11) VALUES('optimize') }
392 do_execsql_test 8.3 {
393 SELECT mit(matchinfo(t11, 'nxa')) FROM t11 WHERE t11 MATCH 'a*'
394 } {{204 1 3 3 0} {204 1 3 3 0} {204 1 3 3 0}}
396 # Corruption related tests.
397 sqlite3_db_config db DEFENSIVE 0
398 do_execsql_test 8.4.1.1 { UPDATE t11_stat SET value = X'0000'; }
399 do_catchsql_test 8.5.1.2 {
400 SELECT mit(matchinfo(t11, 'nxa')) FROM t11 WHERE t11 MATCH 'a*'
401 } {1 {database disk image is malformed}}
403 do_execsql_test 8.4.2.1 { UPDATE t11_stat SET value = X'00'; }
404 do_catchsql_test 8.5.2.2 {
405 SELECT mit(matchinfo(t11, 'nxa')) FROM t11 WHERE t11 MATCH 'a*'
406 } {1 {database disk image is malformed}}
408 do_execsql_test 8.4.3.1 { UPDATE t11_stat SET value = NULL; }
409 do_catchsql_test 8.5.3.2 {
410 SELECT mit(matchinfo(t11, 'nxa')) FROM t11 WHERE t11 MATCH 'a*'
411 } {1 {database disk image is malformed}}
413 #-------------------------------------------------------------------------
414 do_execsql_test 8.1 {
415 CREATE VIRTUAL TABLE t12 USING fts4;
416 INSERT INTO t12 VALUES('a b c d');
417 SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'a NEAR/1 d OR a';
418 } {{0 0 0 0 0 0 1 1 1}}
419 do_execsql_test 8.2 {
420 INSERT INTO t12 VALUES('a d c d');
421 SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'a NEAR/1 d OR a';
423 {0 1 1 0 1 1 1 2 2} {1 1 1 1 1 1 1 2 2}
425 do_execsql_test 8.3 {
426 INSERT INTO t12 VALUES('a d d a');
427 SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'a NEAR/1 d OR a';
429 {0 3 2 0 3 2 1 4 3} {1 3 2 1 3 2 1 4 3} {2 3 2 2 3 2 2 4 3}
432 do_execsql_test 9.1 {
433 CREATE VIRTUAL TABLE ft2 USING fts4;
434 INSERT INTO ft2 VALUES('a b c d e');
435 INSERT INTO ft2 VALUES('f a b c d');
436 SELECT snippet(ft2, '[', ']', '', -1, 1) FROM ft2 WHERE ft2 MATCH 'c';
439 #---------------------------------------------------------------------------
440 # Test for a memory leak
442 do_execsql_test 10.1 {
444 CREATE VIRTUAL TABLE t10 USING fts4(idx, value);
445 INSERT INTO t10 values (1, 'one'),(2, 'two'),(3, 'three');
448 JOIN (SELECT 1 AS idx UNION SELECT 2 UNION SELECT 3) AS x
449 WHERE t10 MATCH x.idx
450 AND matchinfo(t10) not null
453 } {1 1 one 2 2 two 3 3 three}
455 #---------------------------------------------------------------------------
456 # Test the 'y' matchinfo flag
458 set sqlite_fts3_enable_parentheses 1
460 do_execsql_test 11.0 {
461 CREATE VIRTUAL TABLE tt USING fts3(x, y);
462 INSERT INTO tt VALUES('c d a c d d', 'e a g b d a'); -- 1
463 INSERT INTO tt VALUES('c c g a e b', 'c g d g e c'); -- 2
464 INSERT INTO tt VALUES('b e f d e g', 'b a c b c g'); -- 3
465 INSERT INTO tt VALUES('a c f f g d', 'd b f d e g'); -- 4
466 INSERT INTO tt VALUES('g a c f c f', 'd g g b c c'); -- 5
467 INSERT INTO tt VALUES('g a c e b b', 'd b f b g g'); -- 6
468 INSERT INTO tt VALUES('f d a a f c', 'e e a d c f'); -- 7
469 INSERT INTO tt VALUES('a c b b g f', 'a b a e d f'); -- 8
470 INSERT INTO tt VALUES('b a f e c c', 'f d b b a b'); -- 9
471 INSERT INTO tt VALUES('f d c e a c', 'f a f a a f'); -- 10
475 foreach {tn expr res} {
477 1 {1 2} 2 {1 0} 3 {0 1} 4 {1 0} 5 {1 0}
478 6 {1 0} 7 {2 1} 8 {1 2} 9 {1 1} 10 {1 3}
482 1 {0 1} 2 {1 0} 3 {1 2} 4 {0 1} 5 {0 1}
483 6 {2 2} 8 {2 1} 9 {1 3}
488 7 {0 1} 8 {0 2} 9 {0 1} 10 {0 3}
492 1 {1 0} 2 {1 0} 4 {1 0} 5 {1 0}
493 6 {1 0} 7 {2 0} 8 {1 0} 9 {1 0} 10 {1 0}
497 1 {1 2 0 1} 2 {1 0 1 0} 3 {0 1 1 2} 4 {1 0 0 1} 5 {1 0 0 1}
498 6 {1 0 2 2} 7 {2 1 0 0} 8 {1 2 2 1} 9 {1 1 1 3} 10 {1 3 0 0}
502 1 {1 2 0 1} 2 {1 0 1 0} 3 {0 1 1 2} 4 {1 0 0 1} 5 {1 0 0 1}
503 6 {1 0 2 2} 8 {1 2 2 1} 9 {1 1 1 3}
507 1 {1 2 1 2 0 1} 2 {1 0 1 0 1 0} 3 {0 1 0 1 1 2} 4 {1 0 1 0 0 1}
508 5 {1 0 1 0 0 1} 6 {1 0 1 0 2 2} 7 {2 1 0 0 0 0} 8 {1 2 1 2 2 1}
509 9 {1 1 1 1 1 3} 10 {1 3 0 0 0 0}
513 do_execsql_test 11.1.$tn.1 {
514 SELECT rowid, mit(matchinfo(tt, 'y')) FROM tt WHERE tt MATCH $expr
518 foreach {rowid L} $res {
522 lappend M [expr ($a ? 1 : 0) + ($b ? 2 : 0)]
527 do_execsql_test 11.1.$tn.2 {
528 SELECT rowid, mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH $expr
531 do_execsql_test 11.1.$tn.2 {
532 SELECT rowid, mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH $expr
535 set sqlite_fts3_enable_parentheses 0
537 #---------------------------------------------------------------------------
538 # Test the 'b' matchinfo flag
540 set sqlite_fts3_enable_parentheses 1
546 for {set i 0} {$i < 50} {incr i} { lappend cols "c$i" }
547 execsql "CREATE VIRTUAL TABLE tt USING fts3([join $cols ,])"
550 do_execsql_test 12.1 {
551 INSERT INTO tt (rowid, c4, c45) VALUES(1, 'abc', 'abc');
552 SELECT mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH 'abc';
553 } [list [list [expr 1<<4] [expr 1<<(45-32)]]]
555 set sqlite_fts3_enable_parentheses 0