From 7a5ef06a13be4a052166cc33d7d813fe1410f5f8 Mon Sep 17 00:00:00 2001
From: Dan Kennedy <danielk1977@gmail.com>
Date: Thu, 7 Dec 2023 20:46:40 +0000
Subject: [PATCH] Avoid an assert() failure when querying an fts5vocab table
 that accesses a tokendata=1 fts5 table with corrupt %_data records.

---
 ext/fts5/fts5_index.c         |  5 ++++-
 ext/fts5/test/fts5vocab2.test | 24 ++++++++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 735d06c29f..902af552a6 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -7031,6 +7031,10 @@ int sqlite3Fts5IndexQuery(
     int bTokendata = pConfig->bTokendata;
     if( nToken>0 ) memcpy(&buf.p[1], pToken, nToken);
 
+    if( flags & (FTS5INDEX_QUERY_NOTOKENDATA|FTS5INDEX_QUERY_SCAN) ){
+      bTokendata = 0;
+    }
+
     /* Figure out which index to search and set iIdx accordingly. If this
     ** is a prefix query for which there is no prefix index, set iIdx to
     ** greater than pConfig->nPrefix to indicate that the query will be
@@ -7042,7 +7046,6 @@ int sqlite3Fts5IndexQuery(
     ** for internal sanity checking by the integrity-check in debug 
     ** mode only.  */
 #ifdef SQLITE_DEBUG
-    if( flags & FTS5INDEX_QUERY_NOTOKENDATA ) bTokendata = 0;
     if( pConfig->bPrefixIndex==0 || (flags & FTS5INDEX_QUERY_TEST_NOIDX) ){
       assert( flags & FTS5INDEX_QUERY_PREFIX );
       iIdx = 1+pConfig->nPrefix;
diff --git a/ext/fts5/test/fts5vocab2.test b/ext/fts5/test/fts5vocab2.test
index 6f7aad329c..ecacc50dab 100644
--- a/ext/fts5/test/fts5vocab2.test
+++ b/ext/fts5/test/fts5vocab2.test
@@ -280,6 +280,30 @@ do_catchsql_test 5.2 {
   INSERT INTO t1 SELECT randomblob(3000) FROM v1
 } {1 {query aborted}}
 
+#-------------------------------------------------------------------------
+reset_db
+sqlite3_fts5_may_be_corrupt 1
+
+do_execsql_test 6.0 {
+  BEGIN TRANSACTION;
+    CREATE VIRTUAL TABLE t1 USING fts5(a,b unindexed,c,tokenize="porter ascii",tokendata=1);
+    REPLACE INTO t1_data VALUES(1,X'03090009');
+    REPLACE INTO t1_data VALUES(10,X'000000000103030003010101020101030101');
+    REPLACE INTO t1_data VALUES(137438953473,X'0000002e023061010202010162010203010163010204010167010601020201016801060102030101690106010204040606060808');
+    REPLACE INTO t1_data VALUES(274877906945,X'0000001f013067020802010202010168020803010203010169020804010204040909');
+    REPLACE INTO t1_data VALUES(412316860417,X'0000002e023061030202010162030203010163030204010167030601020201016803060102030101690306010204040606060808');
+  COMMIT;
+}
+
+do_execsql_test 6.1 {
+  CREATE VIRTUAL TABLE t3 USING fts5vocab('t1', 'row');
+}
+
+do_catchsql_test 6.2 {
+  SELECT * FROM t3;
+} {1 {database disk image is malformed}}
+
+sqlite3_fts5_may_be_corrupt 0
 
 finish_test
 
-- 
2.11.4.GIT