4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing:
7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give.
11 ******************************************************************************
18 #include "fts5parse.h"
21 ** All token types in the generated fts5parse.h file are greater than 0.
25 #define FTS5_LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32))
27 typedef struct Fts5ExprTerm Fts5ExprTerm
;
30 ** Functions generated by lemon from fts5parse.y.
32 void *sqlite3Fts5ParserAlloc(void *(*mallocProc
)(u64
));
33 void sqlite3Fts5ParserFree(void*, void (*freeProc
)(void*));
34 void sqlite3Fts5Parser(void*, int, Fts5Token
, Fts5Parse
*);
37 void sqlite3Fts5ParserTrace(FILE*, char*);
39 int sqlite3Fts5ParserFallback(int);
46 int bDesc
; /* Iterate in descending rowid order */
47 int nPhrase
; /* Number of phrases in expression */
48 Fts5ExprPhrase
**apExprPhrase
; /* Pointers to phrase objects */
53 ** Expression node type. Always one of:
55 ** FTS5_AND (nChild, apChild valid)
56 ** FTS5_OR (nChild, apChild valid)
57 ** FTS5_NOT (nChild, apChild valid)
58 ** FTS5_STRING (pNear valid)
59 ** FTS5_TERM (pNear valid)
62 int eType
; /* Node type */
63 int bEof
; /* True at EOF */
64 int bNomatch
; /* True if entry is not a match */
66 /* Next method for this node. */
67 int (*xNext
)(Fts5Expr
*, Fts5ExprNode
*, int, i64
);
69 i64 iRowid
; /* Current rowid */
70 Fts5ExprNearset
*pNear
; /* For FTS5_STRING - cluster of phrases */
72 /* Child nodes. For a NOT node, this array always contains 2 entries. For
73 ** AND or OR nodes, it contains 2 or more entries. */
74 int nChild
; /* Number of child nodes */
75 Fts5ExprNode
*apChild
[1]; /* Array of child nodes */
78 #define Fts5NodeIsString(p) ((p)->eType==FTS5_TERM || (p)->eType==FTS5_STRING)
81 ** Invoke the xNext method of an Fts5ExprNode object. This macro should be
82 ** used as if it has the same signature as the xNext() methods themselves.
84 #define fts5ExprNodeNext(a,b,c,d) (b)->xNext((a), (b), (c), (d))
87 ** An instance of the following structure represents a single search term
91 u8 bPrefix
; /* True for a prefix term */
92 u8 bFirst
; /* True if token must be first in column */
93 char *zTerm
; /* nul-terminated term */
94 Fts5IndexIter
*pIter
; /* Iterator for this term */
95 Fts5ExprTerm
*pSynonym
; /* Pointer to first in list of synonyms */
99 ** A phrase. One or more terms that must appear in a contiguous sequence
100 ** within a document for it to match.
102 struct Fts5ExprPhrase
{
103 Fts5ExprNode
*pNode
; /* FTS5_STRING node this phrase is part of */
104 Fts5Buffer poslist
; /* Current position list */
105 int nTerm
; /* Number of entries in aTerm[] */
106 Fts5ExprTerm aTerm
[1]; /* Terms that make up this phrase */
110 ** One or more phrases that must appear within a certain token distance of
111 ** each other within each matching document.
113 struct Fts5ExprNearset
{
114 int nNear
; /* NEAR parameter */
115 Fts5Colset
*pColset
; /* Columns to search (NULL -> all columns) */
116 int nPhrase
; /* Number of entries in aPhrase[] array */
117 Fts5ExprPhrase
*apPhrase
[1]; /* Array of phrase pointers */
128 int nPhrase
; /* Size of apPhrase array */
129 Fts5ExprPhrase
**apPhrase
; /* Array of all phrases */
130 Fts5ExprNode
*pExpr
; /* Result of a successful parse */
131 int bPhraseToAnd
; /* Convert "a+b" to "a AND b" */
134 void sqlite3Fts5ParseError(Fts5Parse
*pParse
, const char *zFmt
, ...){
137 if( pParse
->rc
==SQLITE_OK
){
138 assert( pParse
->zErr
==0 );
139 pParse
->zErr
= sqlite3_vmprintf(zFmt
, ap
);
140 pParse
->rc
= SQLITE_ERROR
;
145 static int fts5ExprIsspace(char t
){
146 return t
==' ' || t
=='\t' || t
=='\n' || t
=='\r';
150 ** Read the first token from the nul-terminated string at *pz.
152 static int fts5ExprGetToken(
154 const char **pz
, /* IN/OUT: Pointer into buffer */
160 /* Skip past any whitespace */
161 while( fts5ExprIsspace(*z
) ) z
++;
166 case '(': tok
= FTS5_LP
; break;
167 case ')': tok
= FTS5_RP
; break;
168 case '{': tok
= FTS5_LCP
; break;
169 case '}': tok
= FTS5_RCP
; break;
170 case ':': tok
= FTS5_COLON
; break;
171 case ',': tok
= FTS5_COMMA
; break;
172 case '+': tok
= FTS5_PLUS
; break;
173 case '*': tok
= FTS5_STAR
; break;
174 case '-': tok
= FTS5_MINUS
; break;
175 case '^': tok
= FTS5_CARET
; break;
176 case '\0': tok
= FTS5_EOF
; break;
182 for(z2
=&z
[1]; 1; z2
++){
185 if( z2
[0]!='"' ) break;
188 sqlite3Fts5ParseError(pParse
, "unterminated string");
192 pToken
->n
= (z2
- z
);
198 if( sqlite3Fts5IsBareword(z
[0])==0 ){
199 sqlite3Fts5ParseError(pParse
, "fts5: syntax error near \"%.1s\"", z
);
203 for(z2
=&z
[1]; sqlite3Fts5IsBareword(*z2
); z2
++);
204 pToken
->n
= (z2
- z
);
205 if( pToken
->n
==2 && memcmp(pToken
->p
, "OR", 2)==0 ) tok
= FTS5_OR
;
206 if( pToken
->n
==3 && memcmp(pToken
->p
, "NOT", 3)==0 ) tok
= FTS5_NOT
;
207 if( pToken
->n
==3 && memcmp(pToken
->p
, "AND", 3)==0 ) tok
= FTS5_AND
;
212 *pz
= &pToken
->p
[pToken
->n
];
216 static void *fts5ParseAlloc(u64 t
){ return sqlite3_malloc64((sqlite3_int64
)t
);}
217 static void fts5ParseFree(void *p
){ sqlite3_free(p
); }
219 int sqlite3Fts5ExprNew(
220 Fts5Config
*pConfig
, /* FTS5 Configuration */
223 const char *zExpr
, /* Expression text */
229 const char *z
= zExpr
;
230 int t
; /* Next token type */
236 memset(&sParse
, 0, sizeof(sParse
));
237 sParse
.bPhraseToAnd
= bPhraseToAnd
;
238 pEngine
= sqlite3Fts5ParserAlloc(fts5ParseAlloc
);
239 if( pEngine
==0 ){ return SQLITE_NOMEM
; }
240 sParse
.pConfig
= pConfig
;
243 t
= fts5ExprGetToken(&sParse
, &z
, &token
);
244 sqlite3Fts5Parser(pEngine
, t
, token
, &sParse
);
245 }while( sParse
.rc
==SQLITE_OK
&& t
!=FTS5_EOF
);
246 sqlite3Fts5ParserFree(pEngine
, fts5ParseFree
);
248 /* If the LHS of the MATCH expression was a user column, apply the
249 ** implicit column-filter. */
250 if( iCol
<pConfig
->nCol
&& sParse
.pExpr
&& sParse
.rc
==SQLITE_OK
){
251 int n
= sizeof(Fts5Colset
);
252 Fts5Colset
*pColset
= (Fts5Colset
*)sqlite3Fts5MallocZero(&sParse
.rc
, n
);
255 pColset
->aiCol
[0] = iCol
;
256 sqlite3Fts5ParseSetColset(&sParse
, sParse
.pExpr
, pColset
);
260 assert( sParse
.rc
!=SQLITE_OK
|| sParse
.zErr
==0 );
261 if( sParse
.rc
==SQLITE_OK
){
262 *ppNew
= pNew
= sqlite3_malloc(sizeof(Fts5Expr
));
264 sParse
.rc
= SQLITE_NOMEM
;
265 sqlite3Fts5ParseNodeFree(sParse
.pExpr
);
268 const int nByte
= sizeof(Fts5ExprNode
);
269 pNew
->pRoot
= (Fts5ExprNode
*)sqlite3Fts5MallocZero(&sParse
.rc
, nByte
);
271 pNew
->pRoot
->bEof
= 1;
274 pNew
->pRoot
= sParse
.pExpr
;
277 pNew
->pConfig
= pConfig
;
278 pNew
->apExprPhrase
= sParse
.apPhrase
;
279 pNew
->nPhrase
= sParse
.nPhrase
;
284 sqlite3Fts5ParseNodeFree(sParse
.pExpr
);
287 sqlite3_free(sParse
.apPhrase
);
288 *pzErr
= sParse
.zErr
;
293 ** This function is only called when using the special 'trigram' tokenizer.
294 ** Argument zText contains the text of a LIKE or GLOB pattern matched
295 ** against column iCol. This function creates and compiles an FTS5 MATCH
296 ** expression that will match a superset of the rows matched by the LIKE or
297 ** GLOB. If successful, SQLITE_OK is returned. Otherwise, an SQLite error
300 int sqlite3Fts5ExprPattern(
301 Fts5Config
*pConfig
, int bGlob
, int iCol
, const char *zText
, Fts5Expr
**pp
303 i64 nText
= strlen(zText
);
304 char *zExpr
= (char*)sqlite3_malloc64(nText
*4 + 1);
327 || zText
[i
]==aSpec
[0] || zText
[i
]==aSpec
[1] || zText
[i
]==aSpec
[2]
332 for(jj
=iFirst
; jj
<i
; jj
++){
333 zExpr
[iOut
++] = zText
[jj
];
334 if( zText
[jj
]=='"' ) zExpr
[iOut
++] = '"';
339 if( zText
[i
]==aSpec
[2] ){
341 if( zText
[i
-1]=='^' ) i
++;
342 while( i
<nText
&& zText
[i
]!=']' ) i
++;
350 if( pConfig
->eDetail
!=FTS5_DETAIL_FULL
){
352 if( pConfig
->eDetail
==FTS5_DETAIL_NONE
){
353 iCol
= pConfig
->nCol
;
357 rc
= sqlite3Fts5ExprNew(pConfig
, bAnd
, iCol
, zExpr
, pp
,pConfig
->pzErrmsg
);
368 ** Free the expression node object passed as the only argument.
370 void sqlite3Fts5ParseNodeFree(Fts5ExprNode
*p
){
373 for(i
=0; i
<p
->nChild
; i
++){
374 sqlite3Fts5ParseNodeFree(p
->apChild
[i
]);
376 sqlite3Fts5ParseNearsetFree(p
->pNear
);
382 ** Free the expression object passed as the only argument.
384 void sqlite3Fts5ExprFree(Fts5Expr
*p
){
386 sqlite3Fts5ParseNodeFree(p
->pRoot
);
387 sqlite3_free(p
->apExprPhrase
);
392 int sqlite3Fts5ExprAnd(Fts5Expr
**pp1
, Fts5Expr
*p2
){
394 memset(&sParse
, 0, sizeof(sParse
));
398 int nPhrase
= p1
->nPhrase
+ p2
->nPhrase
;
400 p1
->pRoot
= sqlite3Fts5ParseNode(&sParse
, FTS5_AND
, p1
->pRoot
, p2
->pRoot
,0);
403 if( sParse
.rc
==SQLITE_OK
){
404 Fts5ExprPhrase
**ap
= (Fts5ExprPhrase
**)sqlite3_realloc(
405 p1
->apExprPhrase
, nPhrase
* sizeof(Fts5ExprPhrase
*)
408 sParse
.rc
= SQLITE_NOMEM
;
411 memmove(&ap
[p2
->nPhrase
], ap
, p1
->nPhrase
*sizeof(Fts5ExprPhrase
*));
412 for(i
=0; i
<p2
->nPhrase
; i
++){
413 ap
[i
] = p2
->apExprPhrase
[i
];
415 p1
->nPhrase
= nPhrase
;
416 p1
->apExprPhrase
= ap
;
419 sqlite3_free(p2
->apExprPhrase
);
429 ** Argument pTerm must be a synonym iterator. Return the current rowid
430 ** that it points to.
432 static i64
fts5ExprSynonymRowid(Fts5ExprTerm
*pTerm
, int bDesc
, int *pbEof
){
438 assert( pTerm
->pSynonym
);
439 assert( bDesc
==0 || bDesc
==1 );
440 for(p
=pTerm
; p
; p
=p
->pSynonym
){
441 if( 0==sqlite3Fts5IterEof(p
->pIter
) ){
442 i64 iRowid
= p
->pIter
->iRowid
;
443 if( bRetValid
==0 || (bDesc
!=(iRowid
<iRet
)) ){
450 if( pbEof
&& bRetValid
==0 ) *pbEof
= 1;
455 ** Argument pTerm must be a synonym iterator.
457 static int fts5ExprSynonymList(
460 Fts5Buffer
*pBuf
, /* Use this buffer for space if required */
463 Fts5PoslistReader aStatic
[4];
464 Fts5PoslistReader
*aIter
= aStatic
;
470 assert( pTerm
->pSynonym
);
471 for(p
=pTerm
; p
; p
=p
->pSynonym
){
472 Fts5IndexIter
*pIter
= p
->pIter
;
473 if( sqlite3Fts5IterEof(pIter
)==0 && pIter
->iRowid
==iRowid
){
474 if( pIter
->nData
==0 ) continue;
476 sqlite3_int64 nByte
= sizeof(Fts5PoslistReader
) * nAlloc
* 2;
477 Fts5PoslistReader
*aNew
= (Fts5PoslistReader
*)sqlite3_malloc64(nByte
);
480 goto synonym_poslist_out
;
482 memcpy(aNew
, aIter
, sizeof(Fts5PoslistReader
) * nIter
);
484 if( aIter
!=aStatic
) sqlite3_free(aIter
);
487 sqlite3Fts5PoslistReaderInit(pIter
->pData
, pIter
->nData
, &aIter
[nIter
]);
488 assert( aIter
[nIter
].bEof
==0 );
494 *pa
= (u8
*)aIter
[0].a
;
497 Fts5PoslistWriter writer
= {0};
499 fts5BufferZero(pBuf
);
502 i64 iMin
= FTS5_LARGEST_INT64
;
503 for(i
=0; i
<nIter
; i
++){
504 if( aIter
[i
].bEof
==0 ){
505 if( aIter
[i
].iPos
==iPrev
){
506 if( sqlite3Fts5PoslistReaderNext(&aIter
[i
]) ) continue;
508 if( aIter
[i
].iPos
<iMin
){
509 iMin
= aIter
[i
].iPos
;
513 if( iMin
==FTS5_LARGEST_INT64
|| rc
!=SQLITE_OK
) break;
514 rc
= sqlite3Fts5PoslistWriterAppend(pBuf
, &writer
, iMin
);
524 if( aIter
!=aStatic
) sqlite3_free(aIter
);
530 ** All individual term iterators in pPhrase are guaranteed to be valid and
531 ** pointing to the same rowid when this function is called. This function
532 ** checks if the current rowid really is a match, and if so populates
533 ** the pPhrase->poslist buffer accordingly. Output parameter *pbMatch
534 ** is set to true if this is really a match, or false otherwise.
536 ** SQLITE_OK is returned if an error occurs, or an SQLite error code
537 ** otherwise. It is not considered an error code if the current rowid is
540 static int fts5ExprPhraseIsMatch(
541 Fts5ExprNode
*pNode
, /* Node pPhrase belongs to */
542 Fts5ExprPhrase
*pPhrase
, /* Phrase object to initialize */
543 int *pbMatch
/* OUT: Set to true if really a match */
545 Fts5PoslistWriter writer
= {0};
546 Fts5PoslistReader aStatic
[4];
547 Fts5PoslistReader
*aIter
= aStatic
;
550 int bFirst
= pPhrase
->aTerm
[0].bFirst
;
552 fts5BufferZero(&pPhrase
->poslist
);
554 /* If the aStatic[] array is not large enough, allocate a large array
555 ** using sqlite3_malloc(). This approach could be improved upon. */
556 if( pPhrase
->nTerm
>ArraySize(aStatic
) ){
557 sqlite3_int64 nByte
= sizeof(Fts5PoslistReader
) * pPhrase
->nTerm
;
558 aIter
= (Fts5PoslistReader
*)sqlite3_malloc64(nByte
);
559 if( !aIter
) return SQLITE_NOMEM
;
561 memset(aIter
, 0, sizeof(Fts5PoslistReader
) * pPhrase
->nTerm
);
563 /* Initialize a term iterator for each term in the phrase */
564 for(i
=0; i
<pPhrase
->nTerm
; i
++){
565 Fts5ExprTerm
*pTerm
= &pPhrase
->aTerm
[i
];
569 if( pTerm
->pSynonym
){
570 Fts5Buffer buf
= {0, 0, 0};
571 rc
= fts5ExprSynonymList(pTerm
, pNode
->iRowid
, &buf
, &a
, &n
);
576 if( a
==buf
.p
) bFlag
= 1;
578 a
= (u8
*)pTerm
->pIter
->pData
;
579 n
= pTerm
->pIter
->nData
;
581 sqlite3Fts5PoslistReaderInit(a
, n
, &aIter
[i
]);
582 aIter
[i
].bFlag
= (u8
)bFlag
;
583 if( aIter
[i
].bEof
) goto ismatch_out
;
588 i64 iPos
= aIter
[0].iPos
;
591 for(i
=0; i
<pPhrase
->nTerm
; i
++){
592 Fts5PoslistReader
*pPos
= &aIter
[i
];
594 if( pPos
->iPos
!=iAdj
){
596 while( pPos
->iPos
<iAdj
){
597 if( sqlite3Fts5PoslistReaderNext(pPos
) ) goto ismatch_out
;
599 if( pPos
->iPos
>iAdj
) iPos
= pPos
->iPos
-i
;
604 /* Append position iPos to the output */
605 if( bFirst
==0 || FTS5_POS2OFFSET(iPos
)==0 ){
606 rc
= sqlite3Fts5PoslistWriterAppend(&pPhrase
->poslist
, &writer
, iPos
);
607 if( rc
!=SQLITE_OK
) goto ismatch_out
;
610 for(i
=0; i
<pPhrase
->nTerm
; i
++){
611 if( sqlite3Fts5PoslistReaderNext(&aIter
[i
]) ) goto ismatch_out
;
616 *pbMatch
= (pPhrase
->poslist
.n
>0);
617 for(i
=0; i
<pPhrase
->nTerm
; i
++){
618 if( aIter
[i
].bFlag
) sqlite3_free((u8
*)aIter
[i
].a
);
620 if( aIter
!=aStatic
) sqlite3_free(aIter
);
624 typedef struct Fts5LookaheadReader Fts5LookaheadReader
;
625 struct Fts5LookaheadReader
{
626 const u8
*a
; /* Buffer containing position list */
627 int n
; /* Size of buffer a[] in bytes */
628 int i
; /* Current offset in position list */
629 i64 iPos
; /* Current position */
630 i64 iLookahead
; /* Next position */
633 #define FTS5_LOOKAHEAD_EOF (((i64)1) << 62)
635 static int fts5LookaheadReaderNext(Fts5LookaheadReader
*p
){
636 p
->iPos
= p
->iLookahead
;
637 if( sqlite3Fts5PoslistNext64(p
->a
, p
->n
, &p
->i
, &p
->iLookahead
) ){
638 p
->iLookahead
= FTS5_LOOKAHEAD_EOF
;
640 return (p
->iPos
==FTS5_LOOKAHEAD_EOF
);
643 static int fts5LookaheadReaderInit(
644 const u8
*a
, int n
, /* Buffer to read position list from */
645 Fts5LookaheadReader
*p
/* Iterator object to initialize */
647 memset(p
, 0, sizeof(Fts5LookaheadReader
));
650 fts5LookaheadReaderNext(p
);
651 return fts5LookaheadReaderNext(p
);
654 typedef struct Fts5NearTrimmer Fts5NearTrimmer
;
655 struct Fts5NearTrimmer
{
656 Fts5LookaheadReader reader
; /* Input iterator */
657 Fts5PoslistWriter writer
; /* Writer context */
658 Fts5Buffer
*pOut
; /* Output poslist */
662 ** The near-set object passed as the first argument contains more than
663 ** one phrase. All phrases currently point to the same row. The
664 ** Fts5ExprPhrase.poslist buffers are populated accordingly. This function
665 ** tests if the current row contains instances of each phrase sufficiently
666 ** close together to meet the NEAR constraint. Non-zero is returned if it
667 ** does, or zero otherwise.
669 ** If in/out parameter (*pRc) is set to other than SQLITE_OK when this
670 ** function is called, it is a no-op. Or, if an error (e.g. SQLITE_NOMEM)
671 ** occurs within this function (*pRc) is set accordingly before returning.
672 ** The return value is undefined in both these cases.
674 ** If no error occurs and non-zero (a match) is returned, the position-list
675 ** of each phrase object is edited to contain only those entries that
676 ** meet the constraint before returning.
678 static int fts5ExprNearIsMatch(int *pRc
, Fts5ExprNearset
*pNear
){
679 Fts5NearTrimmer aStatic
[4];
680 Fts5NearTrimmer
*a
= aStatic
;
681 Fts5ExprPhrase
**apPhrase
= pNear
->apPhrase
;
687 assert( pNear
->nPhrase
>1 );
689 /* If the aStatic[] array is not large enough, allocate a large array
690 ** using sqlite3_malloc(). This approach could be improved upon. */
691 if( pNear
->nPhrase
>ArraySize(aStatic
) ){
692 sqlite3_int64 nByte
= sizeof(Fts5NearTrimmer
) * pNear
->nPhrase
;
693 a
= (Fts5NearTrimmer
*)sqlite3Fts5MallocZero(&rc
, nByte
);
695 memset(aStatic
, 0, sizeof(aStatic
));
702 /* Initialize a lookahead iterator for each phrase. After passing the
703 ** buffer and buffer size to the lookaside-reader init function, zero
704 ** the phrase poslist buffer. The new poslist for the phrase (containing
705 ** the same entries as the original with some entries removed on account
706 ** of the NEAR constraint) is written over the original even as it is
707 ** being read. This is safe as the entries for the new poslist are a
708 ** subset of the old, so it is not possible for data yet to be read to
709 ** be overwritten. */
710 for(i
=0; i
<pNear
->nPhrase
; i
++){
711 Fts5Buffer
*pPoslist
= &apPhrase
[i
]->poslist
;
712 fts5LookaheadReaderInit(pPoslist
->p
, pPoslist
->n
, &a
[i
].reader
);
714 a
[i
].pOut
= pPoslist
;
722 /* This block advances the phrase iterators until they point to a set of
723 ** entries that together comprise a match. */
724 iMax
= a
[0].reader
.iPos
;
727 for(i
=0; i
<pNear
->nPhrase
; i
++){
728 Fts5LookaheadReader
*pPos
= &a
[i
].reader
;
729 iMin
= iMax
- pNear
->apPhrase
[i
]->nTerm
- pNear
->nNear
;
730 if( pPos
->iPos
<iMin
|| pPos
->iPos
>iMax
){
732 while( pPos
->iPos
<iMin
){
733 if( fts5LookaheadReaderNext(pPos
) ) goto ismatch_out
;
735 if( pPos
->iPos
>iMax
) iMax
= pPos
->iPos
;
740 /* Add an entry to each output position list */
741 for(i
=0; i
<pNear
->nPhrase
; i
++){
742 i64 iPos
= a
[i
].reader
.iPos
;
743 Fts5PoslistWriter
*pWriter
= &a
[i
].writer
;
744 if( a
[i
].pOut
->n
==0 || iPos
!=pWriter
->iPrev
){
745 sqlite3Fts5PoslistWriterAppend(a
[i
].pOut
, pWriter
, iPos
);
750 iMin
= a
[0].reader
.iLookahead
;
751 for(i
=0; i
<pNear
->nPhrase
; i
++){
752 if( a
[i
].reader
.iLookahead
< iMin
){
753 iMin
= a
[i
].reader
.iLookahead
;
757 if( fts5LookaheadReaderNext(&a
[iAdv
].reader
) ) goto ismatch_out
;
761 int bRet
= a
[0].pOut
->n
>0;
763 if( a
!=aStatic
) sqlite3_free(a
);
769 ** Advance iterator pIter until it points to a value equal to or laster
770 ** than the initial value of *piLast. If this means the iterator points
771 ** to a value laster than *piLast, update *piLast to the new lastest value.
773 ** If the iterator reaches EOF, set *pbEof to true before returning. If
774 ** an error occurs, set *pRc to an error code. If either *pbEof or *pRc
775 ** are set, return a non-zero value. Otherwise, return zero.
777 static int fts5ExprAdvanceto(
778 Fts5IndexIter
*pIter
, /* Iterator to advance */
779 int bDesc
, /* True if iterator is "rowid DESC" */
780 i64
*piLast
, /* IN/OUT: Lastest rowid seen so far */
781 int *pRc
, /* OUT: Error code */
782 int *pbEof
/* OUT: Set to true if EOF */
787 iRowid
= pIter
->iRowid
;
788 if( (bDesc
==0 && iLast
>iRowid
) || (bDesc
&& iLast
<iRowid
) ){
789 int rc
= sqlite3Fts5IterNextFrom(pIter
, iLast
);
790 if( rc
|| sqlite3Fts5IterEof(pIter
) ){
795 iRowid
= pIter
->iRowid
;
796 assert( (bDesc
==0 && iRowid
>=iLast
) || (bDesc
==1 && iRowid
<=iLast
) );
803 static int fts5ExprSynonymAdvanceto(
804 Fts5ExprTerm
*pTerm
, /* Term iterator to advance */
805 int bDesc
, /* True if iterator is "rowid DESC" */
806 i64
*piLast
, /* IN/OUT: Lastest rowid seen so far */
807 int *pRc
/* OUT: Error code */
814 for(p
=pTerm
; rc
==SQLITE_OK
&& p
; p
=p
->pSynonym
){
815 if( sqlite3Fts5IterEof(p
->pIter
)==0 ){
816 i64 iRowid
= p
->pIter
->iRowid
;
817 if( (bDesc
==0 && iLast
>iRowid
) || (bDesc
&& iLast
<iRowid
) ){
818 rc
= sqlite3Fts5IterNextFrom(p
->pIter
, iLast
);
827 *piLast
= fts5ExprSynonymRowid(pTerm
, bDesc
, &bEof
);
833 static int fts5ExprNearTest(
835 Fts5Expr
*pExpr
, /* Expression that pNear is a part of */
836 Fts5ExprNode
*pNode
/* The "NEAR" node (FTS5_STRING) */
838 Fts5ExprNearset
*pNear
= pNode
->pNear
;
841 if( pExpr
->pConfig
->eDetail
!=FTS5_DETAIL_FULL
){
843 Fts5ExprPhrase
*pPhrase
= pNear
->apPhrase
[0];
844 pPhrase
->poslist
.n
= 0;
845 for(pTerm
=&pPhrase
->aTerm
[0]; pTerm
; pTerm
=pTerm
->pSynonym
){
846 Fts5IndexIter
*pIter
= pTerm
->pIter
;
847 if( sqlite3Fts5IterEof(pIter
)==0 ){
848 if( pIter
->iRowid
==pNode
->iRowid
&& pIter
->nData
>0 ){
849 pPhrase
->poslist
.n
= 1;
853 return pPhrase
->poslist
.n
;
857 /* Check that each phrase in the nearset matches the current row.
858 ** Populate the pPhrase->poslist buffers at the same time. If any
859 ** phrase is not a match, break out of the loop early. */
860 for(i
=0; rc
==SQLITE_OK
&& i
<pNear
->nPhrase
; i
++){
861 Fts5ExprPhrase
*pPhrase
= pNear
->apPhrase
[i
];
862 if( pPhrase
->nTerm
>1 || pPhrase
->aTerm
[0].pSynonym
863 || pNear
->pColset
|| pPhrase
->aTerm
[0].bFirst
866 rc
= fts5ExprPhraseIsMatch(pNode
, pPhrase
, &bMatch
);
867 if( bMatch
==0 ) break;
869 Fts5IndexIter
*pIter
= pPhrase
->aTerm
[0].pIter
;
870 fts5BufferSet(&rc
, &pPhrase
->poslist
, pIter
->nData
, pIter
->pData
);
875 if( i
==pNear
->nPhrase
&& (i
==1 || fts5ExprNearIsMatch(pRc
, pNear
)) ){
884 ** Initialize all term iterators in the pNear object. If any term is found
885 ** to match no documents at all, return immediately without initializing any
886 ** further iterators.
888 ** If an error occurs, return an SQLite error code. Otherwise, return
889 ** SQLITE_OK. It is not considered an error if some term matches zero
892 static int fts5ExprNearInitAll(
896 Fts5ExprNearset
*pNear
= pNode
->pNear
;
899 assert( pNode
->bNomatch
==0 );
900 for(i
=0; i
<pNear
->nPhrase
; i
++){
901 Fts5ExprPhrase
*pPhrase
= pNear
->apPhrase
[i
];
902 if( pPhrase
->nTerm
==0 ){
907 for(j
=0; j
<pPhrase
->nTerm
; j
++){
908 Fts5ExprTerm
*pTerm
= &pPhrase
->aTerm
[j
];
912 for(p
=pTerm
; p
; p
=p
->pSynonym
){
915 sqlite3Fts5IterClose(p
->pIter
);
918 rc
= sqlite3Fts5IndexQuery(
919 pExpr
->pIndex
, p
->zTerm
, (int)strlen(p
->zTerm
),
920 (pTerm
->bPrefix
? FTS5INDEX_QUERY_PREFIX
: 0) |
921 (pExpr
->bDesc
? FTS5INDEX_QUERY_DESC
: 0),
925 assert( (rc
==SQLITE_OK
)==(p
->pIter
!=0) );
926 if( rc
!=SQLITE_OK
) return rc
;
927 if( 0==sqlite3Fts5IterEof(p
->pIter
) ){
945 ** If pExpr is an ASC iterator, this function returns a value with the
950 ** Otherwise, if this is a DESC iterator, the opposite is returned:
954 static int fts5RowidCmp(
959 assert( pExpr
->bDesc
==0 || pExpr
->bDesc
==1 );
960 if( pExpr
->bDesc
==0 ){
961 if( iLhs
<iRhs
) return -1;
962 return (iLhs
> iRhs
);
964 if( iLhs
>iRhs
) return -1;
965 return (iLhs
< iRhs
);
969 static void fts5ExprSetEof(Fts5ExprNode
*pNode
){
973 for(i
=0; i
<pNode
->nChild
; i
++){
974 fts5ExprSetEof(pNode
->apChild
[i
]);
978 static void fts5ExprNodeZeroPoslist(Fts5ExprNode
*pNode
){
979 if( pNode
->eType
==FTS5_STRING
|| pNode
->eType
==FTS5_TERM
){
980 Fts5ExprNearset
*pNear
= pNode
->pNear
;
982 for(i
=0; i
<pNear
->nPhrase
; i
++){
983 Fts5ExprPhrase
*pPhrase
= pNear
->apPhrase
[i
];
984 pPhrase
->poslist
.n
= 0;
988 for(i
=0; i
<pNode
->nChild
; i
++){
989 fts5ExprNodeZeroPoslist(pNode
->apChild
[i
]);
997 ** Compare the values currently indicated by the two nodes as follows:
999 ** res = (*p1) - (*p2)
1001 ** Nodes that point to values that come later in the iteration order are
1002 ** considered to be larger. Nodes at EOF are the largest of all.
1004 ** This means that if the iteration order is ASC, then numerically larger
1005 ** rowids are considered larger. Or if it is the default DESC, numerically
1006 ** smaller rowids are larger.
1008 static int fts5NodeCompare(
1013 if( p2
->bEof
) return -1;
1014 if( p1
->bEof
) return +1;
1015 return fts5RowidCmp(pExpr
, p1
->iRowid
, p2
->iRowid
);
1019 ** All individual term iterators in pNear are guaranteed to be valid when
1020 ** this function is called. This function checks if all term iterators
1021 ** point to the same rowid, and if not, advances them until they do.
1022 ** If an EOF is reached before this happens, *pbEof is set to true before
1025 ** SQLITE_OK is returned if an error occurs, or an SQLite error code
1026 ** otherwise. It is not considered an error code if an iterator reaches
1029 static int fts5ExprNodeTest_STRING(
1030 Fts5Expr
*pExpr
, /* Expression pPhrase belongs to */
1033 Fts5ExprNearset
*pNear
= pNode
->pNear
;
1034 Fts5ExprPhrase
*pLeft
= pNear
->apPhrase
[0];
1036 i64 iLast
; /* Lastest rowid any iterator points to */
1037 int i
, j
; /* Phrase and token index, respectively */
1038 int bMatch
; /* True if all terms are at the same rowid */
1039 const int bDesc
= pExpr
->bDesc
;
1041 /* Check that this node should not be FTS5_TERM */
1042 assert( pNear
->nPhrase
>1
1043 || pNear
->apPhrase
[0]->nTerm
>1
1044 || pNear
->apPhrase
[0]->aTerm
[0].pSynonym
1045 || pNear
->apPhrase
[0]->aTerm
[0].bFirst
1048 /* Initialize iLast, the "lastest" rowid any iterator points to. If the
1049 ** iterator skips through rowids in the default ascending order, this means
1050 ** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it
1051 ** means the minimum rowid. */
1052 if( pLeft
->aTerm
[0].pSynonym
){
1053 iLast
= fts5ExprSynonymRowid(&pLeft
->aTerm
[0], bDesc
, 0);
1055 iLast
= pLeft
->aTerm
[0].pIter
->iRowid
;
1060 for(i
=0; i
<pNear
->nPhrase
; i
++){
1061 Fts5ExprPhrase
*pPhrase
= pNear
->apPhrase
[i
];
1062 for(j
=0; j
<pPhrase
->nTerm
; j
++){
1063 Fts5ExprTerm
*pTerm
= &pPhrase
->aTerm
[j
];
1064 if( pTerm
->pSynonym
){
1065 i64 iRowid
= fts5ExprSynonymRowid(pTerm
, bDesc
, 0);
1066 if( iRowid
==iLast
) continue;
1068 if( fts5ExprSynonymAdvanceto(pTerm
, bDesc
, &iLast
, &rc
) ){
1069 pNode
->bNomatch
= 0;
1074 Fts5IndexIter
*pIter
= pPhrase
->aTerm
[j
].pIter
;
1075 if( pIter
->iRowid
==iLast
|| pIter
->bEof
) continue;
1077 if( fts5ExprAdvanceto(pIter
, bDesc
, &iLast
, &rc
, &pNode
->bEof
) ){
1083 }while( bMatch
==0 );
1085 pNode
->iRowid
= iLast
;
1086 pNode
->bNomatch
= ((0==fts5ExprNearTest(&rc
, pExpr
, pNode
)) && rc
==SQLITE_OK
);
1087 assert( pNode
->bEof
==0 || pNode
->bNomatch
==0 );
1093 ** Advance the first term iterator in the first phrase of pNear. Set output
1094 ** variable *pbEof to true if it reaches EOF or if an error occurs.
1096 ** Return SQLITE_OK if successful, or an SQLite error code if an error
1099 static int fts5ExprNodeNext_STRING(
1100 Fts5Expr
*pExpr
, /* Expression pPhrase belongs to */
1101 Fts5ExprNode
*pNode
, /* FTS5_STRING or FTS5_TERM node */
1105 Fts5ExprTerm
*pTerm
= &pNode
->pNear
->apPhrase
[0]->aTerm
[0];
1108 pNode
->bNomatch
= 0;
1109 if( pTerm
->pSynonym
){
1113 /* Find the firstest rowid any synonym points to. */
1114 i64 iRowid
= fts5ExprSynonymRowid(pTerm
, pExpr
->bDesc
, 0);
1116 /* Advance each iterator that currently points to iRowid. Or, if iFrom
1117 ** is valid - each iterator that points to a rowid before iFrom. */
1118 for(p
=pTerm
; p
; p
=p
->pSynonym
){
1119 if( sqlite3Fts5IterEof(p
->pIter
)==0 ){
1120 i64 ii
= p
->pIter
->iRowid
;
1122 || (bFromValid
&& ii
!=iFrom
&& (ii
>iFrom
)==pExpr
->bDesc
)
1125 rc
= sqlite3Fts5IterNextFrom(p
->pIter
, iFrom
);
1127 rc
= sqlite3Fts5IterNext(p
->pIter
);
1129 if( rc
!=SQLITE_OK
) break;
1130 if( sqlite3Fts5IterEof(p
->pIter
)==0 ){
1139 /* Set the EOF flag if either all synonym iterators are at EOF or an
1140 ** error has occurred. */
1141 pNode
->bEof
= (rc
|| bEof
);
1143 Fts5IndexIter
*pIter
= pTerm
->pIter
;
1145 assert( Fts5NodeIsString(pNode
) );
1147 rc
= sqlite3Fts5IterNextFrom(pIter
, iFrom
);
1149 rc
= sqlite3Fts5IterNext(pIter
);
1152 pNode
->bEof
= (rc
|| sqlite3Fts5IterEof(pIter
));
1155 if( pNode
->bEof
==0 ){
1156 assert( rc
==SQLITE_OK
);
1157 rc
= fts5ExprNodeTest_STRING(pExpr
, pNode
);
1164 static int fts5ExprNodeTest_TERM(
1165 Fts5Expr
*pExpr
, /* Expression that pNear is a part of */
1166 Fts5ExprNode
*pNode
/* The "NEAR" node (FTS5_TERM) */
1168 /* As this "NEAR" object is actually a single phrase that consists
1169 ** of a single term only, grab pointers into the poslist managed by the
1170 ** fts5_index.c iterator object. This is much faster than synthesizing
1171 ** a new poslist the way we have to for more complicated phrase or NEAR
1173 Fts5ExprPhrase
*pPhrase
= pNode
->pNear
->apPhrase
[0];
1174 Fts5IndexIter
*pIter
= pPhrase
->aTerm
[0].pIter
;
1176 assert( pNode
->eType
==FTS5_TERM
);
1177 assert( pNode
->pNear
->nPhrase
==1 && pPhrase
->nTerm
==1 );
1178 assert( pPhrase
->aTerm
[0].pSynonym
==0 );
1180 pPhrase
->poslist
.n
= pIter
->nData
;
1181 if( pExpr
->pConfig
->eDetail
==FTS5_DETAIL_FULL
){
1182 pPhrase
->poslist
.p
= (u8
*)pIter
->pData
;
1184 pNode
->iRowid
= pIter
->iRowid
;
1185 pNode
->bNomatch
= (pPhrase
->poslist
.n
==0);
1190 ** xNext() method for a node of type FTS5_TERM.
1192 static int fts5ExprNodeNext_TERM(
1194 Fts5ExprNode
*pNode
,
1199 Fts5IndexIter
*pIter
= pNode
->pNear
->apPhrase
[0]->aTerm
[0].pIter
;
1201 assert( pNode
->bEof
==0 );
1203 rc
= sqlite3Fts5IterNextFrom(pIter
, iFrom
);
1205 rc
= sqlite3Fts5IterNext(pIter
);
1207 if( rc
==SQLITE_OK
&& sqlite3Fts5IterEof(pIter
)==0 ){
1208 rc
= fts5ExprNodeTest_TERM(pExpr
, pNode
);
1211 pNode
->bNomatch
= 0;
1216 static void fts5ExprNodeTest_OR(
1217 Fts5Expr
*pExpr
, /* Expression of which pNode is a part */
1218 Fts5ExprNode
*pNode
/* Expression node to test */
1220 Fts5ExprNode
*pNext
= pNode
->apChild
[0];
1223 for(i
=1; i
<pNode
->nChild
; i
++){
1224 Fts5ExprNode
*pChild
= pNode
->apChild
[i
];
1225 int cmp
= fts5NodeCompare(pExpr
, pNext
, pChild
);
1226 if( cmp
>0 || (cmp
==0 && pChild
->bNomatch
==0) ){
1230 pNode
->iRowid
= pNext
->iRowid
;
1231 pNode
->bEof
= pNext
->bEof
;
1232 pNode
->bNomatch
= pNext
->bNomatch
;
1235 static int fts5ExprNodeNext_OR(
1237 Fts5ExprNode
*pNode
,
1242 i64 iLast
= pNode
->iRowid
;
1244 for(i
=0; i
<pNode
->nChild
; i
++){
1245 Fts5ExprNode
*p1
= pNode
->apChild
[i
];
1246 assert( p1
->bEof
|| fts5RowidCmp(pExpr
, p1
->iRowid
, iLast
)>=0 );
1248 if( (p1
->iRowid
==iLast
)
1249 || (bFromValid
&& fts5RowidCmp(pExpr
, p1
->iRowid
, iFrom
)<0)
1251 int rc
= fts5ExprNodeNext(pExpr
, p1
, bFromValid
, iFrom
);
1252 if( rc
!=SQLITE_OK
){
1253 pNode
->bNomatch
= 0;
1260 fts5ExprNodeTest_OR(pExpr
, pNode
);
1265 ** Argument pNode is an FTS5_AND node.
1267 static int fts5ExprNodeTest_AND(
1268 Fts5Expr
*pExpr
, /* Expression pPhrase belongs to */
1269 Fts5ExprNode
*pAnd
/* FTS5_AND node to advance */
1272 i64 iLast
= pAnd
->iRowid
;
1276 assert( pAnd
->bEof
==0 );
1280 for(iChild
=0; iChild
<pAnd
->nChild
; iChild
++){
1281 Fts5ExprNode
*pChild
= pAnd
->apChild
[iChild
];
1282 int cmp
= fts5RowidCmp(pExpr
, iLast
, pChild
->iRowid
);
1284 /* Advance pChild until it points to iLast or laster */
1285 rc
= fts5ExprNodeNext(pExpr
, pChild
, 1, iLast
);
1286 if( rc
!=SQLITE_OK
){
1292 /* If the child node is now at EOF, so is the parent AND node. Otherwise,
1293 ** the child node is guaranteed to have advanced at least as far as
1294 ** rowid iLast. So if it is not at exactly iLast, pChild->iRowid is the
1295 ** new lastest rowid seen so far. */
1296 assert( pChild
->bEof
|| fts5RowidCmp(pExpr
, iLast
, pChild
->iRowid
)<=0 );
1298 fts5ExprSetEof(pAnd
);
1301 }else if( iLast
!=pChild
->iRowid
){
1303 iLast
= pChild
->iRowid
;
1306 if( pChild
->bNomatch
){
1310 }while( bMatch
==0 );
1312 if( pAnd
->bNomatch
&& pAnd
!=pExpr
->pRoot
){
1313 fts5ExprNodeZeroPoslist(pAnd
);
1315 pAnd
->iRowid
= iLast
;
1319 static int fts5ExprNodeNext_AND(
1321 Fts5ExprNode
*pNode
,
1325 int rc
= fts5ExprNodeNext(pExpr
, pNode
->apChild
[0], bFromValid
, iFrom
);
1326 if( rc
==SQLITE_OK
){
1327 rc
= fts5ExprNodeTest_AND(pExpr
, pNode
);
1329 pNode
->bNomatch
= 0;
1334 static int fts5ExprNodeTest_NOT(
1335 Fts5Expr
*pExpr
, /* Expression pPhrase belongs to */
1336 Fts5ExprNode
*pNode
/* FTS5_NOT node to advance */
1339 Fts5ExprNode
*p1
= pNode
->apChild
[0];
1340 Fts5ExprNode
*p2
= pNode
->apChild
[1];
1341 assert( pNode
->nChild
==2 );
1343 while( rc
==SQLITE_OK
&& p1
->bEof
==0 ){
1344 int cmp
= fts5NodeCompare(pExpr
, p1
, p2
);
1346 rc
= fts5ExprNodeNext(pExpr
, p2
, 1, p1
->iRowid
);
1347 cmp
= fts5NodeCompare(pExpr
, p1
, p2
);
1349 assert( rc
!=SQLITE_OK
|| cmp
<=0 );
1350 if( cmp
|| p2
->bNomatch
) break;
1351 rc
= fts5ExprNodeNext(pExpr
, p1
, 0, 0);
1353 pNode
->bEof
= p1
->bEof
;
1354 pNode
->bNomatch
= p1
->bNomatch
;
1355 pNode
->iRowid
= p1
->iRowid
;
1357 fts5ExprNodeZeroPoslist(p2
);
1362 static int fts5ExprNodeNext_NOT(
1364 Fts5ExprNode
*pNode
,
1368 int rc
= fts5ExprNodeNext(pExpr
, pNode
->apChild
[0], bFromValid
, iFrom
);
1369 if( rc
==SQLITE_OK
){
1370 rc
= fts5ExprNodeTest_NOT(pExpr
, pNode
);
1372 if( rc
!=SQLITE_OK
){
1373 pNode
->bNomatch
= 0;
1379 ** If pNode currently points to a match, this function returns SQLITE_OK
1380 ** without modifying it. Otherwise, pNode is advanced until it does point
1381 ** to a match or EOF is reached.
1383 static int fts5ExprNodeTest(
1384 Fts5Expr
*pExpr
, /* Expression of which pNode is a part */
1385 Fts5ExprNode
*pNode
/* Expression node to test */
1388 if( pNode
->bEof
==0 ){
1389 switch( pNode
->eType
){
1392 rc
= fts5ExprNodeTest_STRING(pExpr
, pNode
);
1397 rc
= fts5ExprNodeTest_TERM(pExpr
, pNode
);
1402 rc
= fts5ExprNodeTest_AND(pExpr
, pNode
);
1407 fts5ExprNodeTest_OR(pExpr
, pNode
);
1411 default: assert( pNode
->eType
==FTS5_NOT
); {
1412 rc
= fts5ExprNodeTest_NOT(pExpr
, pNode
);
1422 ** Set node pNode, which is part of expression pExpr, to point to the first
1423 ** match. If there are no matches, set the Node.bEof flag to indicate EOF.
1425 ** Return an SQLite error code if an error occurs, or SQLITE_OK otherwise.
1426 ** It is not an error if there are no matches.
1428 static int fts5ExprNodeFirst(Fts5Expr
*pExpr
, Fts5ExprNode
*pNode
){
1431 pNode
->bNomatch
= 0;
1433 if( Fts5NodeIsString(pNode
) ){
1434 /* Initialize all term iterators in the NEAR object. */
1435 rc
= fts5ExprNearInitAll(pExpr
, pNode
);
1436 }else if( pNode
->xNext
==0 ){
1441 for(i
=0; i
<pNode
->nChild
&& rc
==SQLITE_OK
; i
++){
1442 Fts5ExprNode
*pChild
= pNode
->apChild
[i
];
1443 rc
= fts5ExprNodeFirst(pExpr
, pNode
->apChild
[i
]);
1444 assert( pChild
->bEof
==0 || pChild
->bEof
==1 );
1445 nEof
+= pChild
->bEof
;
1447 pNode
->iRowid
= pNode
->apChild
[0]->iRowid
;
1449 switch( pNode
->eType
){
1451 if( nEof
>0 ) fts5ExprSetEof(pNode
);
1455 if( pNode
->nChild
==nEof
) fts5ExprSetEof(pNode
);
1459 assert( pNode
->eType
==FTS5_NOT
);
1460 pNode
->bEof
= pNode
->apChild
[0]->bEof
;
1465 if( rc
==SQLITE_OK
){
1466 rc
= fts5ExprNodeTest(pExpr
, pNode
);
1473 ** Begin iterating through the set of documents in index pIdx matched by
1474 ** the MATCH expression passed as the first argument. If the "bDesc"
1475 ** parameter is passed a non-zero value, iteration is in descending rowid
1476 ** order. Or, if it is zero, in ascending order.
1478 ** If iterating in ascending rowid order (bDesc==0), the first document
1479 ** visited is that with the smallest rowid that is larger than or equal
1480 ** to parameter iFirst. Or, if iterating in ascending order (bDesc==1),
1481 ** then the first document visited must have a rowid smaller than or
1484 ** Return SQLITE_OK if successful, or an SQLite error code otherwise. It
1485 ** is not considered an error if the query does not match any documents.
1487 int sqlite3Fts5ExprFirst(Fts5Expr
*p
, Fts5Index
*pIdx
, i64 iFirst
, int bDesc
){
1488 Fts5ExprNode
*pRoot
= p
->pRoot
;
1489 int rc
; /* Return code */
1493 rc
= fts5ExprNodeFirst(p
, pRoot
);
1495 /* If not at EOF but the current rowid occurs earlier than iFirst in
1496 ** the iteration order, move to document iFirst or later. */
1499 && fts5RowidCmp(p
, pRoot
->iRowid
, iFirst
)<0
1501 rc
= fts5ExprNodeNext(p
, pRoot
, 1, iFirst
);
1504 /* If the iterator is not at a real match, skip forward until it is. */
1505 while( pRoot
->bNomatch
&& rc
==SQLITE_OK
){
1506 assert( pRoot
->bEof
==0 );
1507 rc
= fts5ExprNodeNext(p
, pRoot
, 0, 0);
1513 ** Move to the next document
1515 ** Return SQLITE_OK if successful, or an SQLite error code otherwise. It
1516 ** is not considered an error if the query does not match any documents.
1518 int sqlite3Fts5ExprNext(Fts5Expr
*p
, i64 iLast
){
1520 Fts5ExprNode
*pRoot
= p
->pRoot
;
1521 assert( pRoot
->bEof
==0 && pRoot
->bNomatch
==0 );
1523 rc
= fts5ExprNodeNext(p
, pRoot
, 0, 0);
1524 assert( pRoot
->bNomatch
==0 || (rc
==SQLITE_OK
&& pRoot
->bEof
==0) );
1525 }while( pRoot
->bNomatch
);
1526 if( fts5RowidCmp(p
, pRoot
->iRowid
, iLast
)>0 ){
1532 int sqlite3Fts5ExprEof(Fts5Expr
*p
){
1533 return p
->pRoot
->bEof
;
1536 i64
sqlite3Fts5ExprRowid(Fts5Expr
*p
){
1537 return p
->pRoot
->iRowid
;
1540 static int fts5ParseStringFromToken(Fts5Token
*pToken
, char **pz
){
1542 *pz
= sqlite3Fts5Strndup(&rc
, pToken
->p
, pToken
->n
);
1547 ** Free the phrase object passed as the only argument.
1549 static void fts5ExprPhraseFree(Fts5ExprPhrase
*pPhrase
){
1552 for(i
=0; i
<pPhrase
->nTerm
; i
++){
1554 Fts5ExprTerm
*pNext
;
1555 Fts5ExprTerm
*pTerm
= &pPhrase
->aTerm
[i
];
1556 sqlite3_free(pTerm
->zTerm
);
1557 sqlite3Fts5IterClose(pTerm
->pIter
);
1558 for(pSyn
=pTerm
->pSynonym
; pSyn
; pSyn
=pNext
){
1559 pNext
= pSyn
->pSynonym
;
1560 sqlite3Fts5IterClose(pSyn
->pIter
);
1561 fts5BufferFree((Fts5Buffer
*)&pSyn
[1]);
1565 if( pPhrase
->poslist
.nSpace
>0 ) fts5BufferFree(&pPhrase
->poslist
);
1566 sqlite3_free(pPhrase
);
1571 ** Set the "bFirst" flag on the first token of the phrase passed as the
1574 void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase
*pPhrase
){
1575 if( pPhrase
&& pPhrase
->nTerm
){
1576 pPhrase
->aTerm
[0].bFirst
= 1;
1581 ** If argument pNear is NULL, then a new Fts5ExprNearset object is allocated
1582 ** and populated with pPhrase. Or, if pNear is not NULL, phrase pPhrase is
1583 ** appended to it and the results returned.
1585 ** If an OOM error occurs, both the pNear and pPhrase objects are freed and
1588 Fts5ExprNearset
*sqlite3Fts5ParseNearset(
1589 Fts5Parse
*pParse
, /* Parse context */
1590 Fts5ExprNearset
*pNear
, /* Existing nearset, or NULL */
1591 Fts5ExprPhrase
*pPhrase
/* Recently parsed phrase */
1593 const int SZALLOC
= 8;
1594 Fts5ExprNearset
*pRet
= 0;
1596 if( pParse
->rc
==SQLITE_OK
){
1601 sqlite3_int64 nByte
;
1602 nByte
= sizeof(Fts5ExprNearset
) + SZALLOC
* sizeof(Fts5ExprPhrase
*);
1603 pRet
= sqlite3_malloc64(nByte
);
1605 pParse
->rc
= SQLITE_NOMEM
;
1607 memset(pRet
, 0, (size_t)nByte
);
1609 }else if( (pNear
->nPhrase
% SZALLOC
)==0 ){
1610 int nNew
= pNear
->nPhrase
+ SZALLOC
;
1611 sqlite3_int64 nByte
;
1613 nByte
= sizeof(Fts5ExprNearset
) + nNew
* sizeof(Fts5ExprPhrase
*);
1614 pRet
= (Fts5ExprNearset
*)sqlite3_realloc64(pNear
, nByte
);
1616 pParse
->rc
= SQLITE_NOMEM
;
1624 assert( pParse
->rc
!=SQLITE_OK
);
1625 sqlite3Fts5ParseNearsetFree(pNear
);
1626 sqlite3Fts5ParsePhraseFree(pPhrase
);
1628 if( pRet
->nPhrase
>0 ){
1629 Fts5ExprPhrase
*pLast
= pRet
->apPhrase
[pRet
->nPhrase
-1];
1630 assert( pLast
==pParse
->apPhrase
[pParse
->nPhrase
-2] );
1631 if( pPhrase
->nTerm
==0 ){
1632 fts5ExprPhraseFree(pPhrase
);
1636 }else if( pLast
->nTerm
==0 ){
1637 fts5ExprPhraseFree(pLast
);
1638 pParse
->apPhrase
[pParse
->nPhrase
-2] = pPhrase
;
1643 pRet
->apPhrase
[pRet
->nPhrase
++] = pPhrase
;
1648 typedef struct TokenCtx TokenCtx
;
1650 Fts5ExprPhrase
*pPhrase
;
1655 ** Callback for tokenizing terms used by ParseTerm().
1657 static int fts5ParseTokenize(
1658 void *pContext
, /* Pointer to Fts5InsertCtx object */
1659 int tflags
, /* Mask of FTS5_TOKEN_* flags */
1660 const char *pToken
, /* Buffer containing token */
1661 int nToken
, /* Size of token in bytes */
1662 int iUnused1
, /* Start offset of token */
1663 int iUnused2
/* End offset of token */
1666 const int SZALLOC
= 8;
1667 TokenCtx
*pCtx
= (TokenCtx
*)pContext
;
1668 Fts5ExprPhrase
*pPhrase
= pCtx
->pPhrase
;
1670 UNUSED_PARAM2(iUnused1
, iUnused2
);
1672 /* If an error has already occurred, this is a no-op */
1673 if( pCtx
->rc
!=SQLITE_OK
) return pCtx
->rc
;
1674 if( nToken
>FTS5_MAX_TOKEN_SIZE
) nToken
= FTS5_MAX_TOKEN_SIZE
;
1676 if( pPhrase
&& pPhrase
->nTerm
>0 && (tflags
& FTS5_TOKEN_COLOCATED
) ){
1678 sqlite3_int64 nByte
= sizeof(Fts5ExprTerm
) + sizeof(Fts5Buffer
) + nToken
+1;
1679 pSyn
= (Fts5ExprTerm
*)sqlite3_malloc64(nByte
);
1683 memset(pSyn
, 0, (size_t)nByte
);
1684 pSyn
->zTerm
= ((char*)pSyn
) + sizeof(Fts5ExprTerm
) + sizeof(Fts5Buffer
);
1685 memcpy(pSyn
->zTerm
, pToken
, nToken
);
1686 pSyn
->pSynonym
= pPhrase
->aTerm
[pPhrase
->nTerm
-1].pSynonym
;
1687 pPhrase
->aTerm
[pPhrase
->nTerm
-1].pSynonym
= pSyn
;
1690 Fts5ExprTerm
*pTerm
;
1691 if( pPhrase
==0 || (pPhrase
->nTerm
% SZALLOC
)==0 ){
1692 Fts5ExprPhrase
*pNew
;
1693 int nNew
= SZALLOC
+ (pPhrase
? pPhrase
->nTerm
: 0);
1695 pNew
= (Fts5ExprPhrase
*)sqlite3_realloc64(pPhrase
,
1696 sizeof(Fts5ExprPhrase
) + sizeof(Fts5ExprTerm
) * nNew
1701 if( pPhrase
==0 ) memset(pNew
, 0, sizeof(Fts5ExprPhrase
));
1702 pCtx
->pPhrase
= pPhrase
= pNew
;
1703 pNew
->nTerm
= nNew
- SZALLOC
;
1707 if( rc
==SQLITE_OK
){
1708 pTerm
= &pPhrase
->aTerm
[pPhrase
->nTerm
++];
1709 memset(pTerm
, 0, sizeof(Fts5ExprTerm
));
1710 pTerm
->zTerm
= sqlite3Fts5Strndup(&rc
, pToken
, nToken
);
1720 ** Free the phrase object passed as the only argument.
1722 void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase
*pPhrase
){
1723 fts5ExprPhraseFree(pPhrase
);
1727 ** Free the phrase object passed as the second argument.
1729 void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset
*pNear
){
1732 for(i
=0; i
<pNear
->nPhrase
; i
++){
1733 fts5ExprPhraseFree(pNear
->apPhrase
[i
]);
1735 sqlite3_free(pNear
->pColset
);
1736 sqlite3_free(pNear
);
1740 void sqlite3Fts5ParseFinished(Fts5Parse
*pParse
, Fts5ExprNode
*p
){
1741 assert( pParse
->pExpr
==0 );
1745 static int parseGrowPhraseArray(Fts5Parse
*pParse
){
1746 if( (pParse
->nPhrase
% 8)==0 ){
1747 sqlite3_int64 nByte
= sizeof(Fts5ExprPhrase
*) * (pParse
->nPhrase
+ 8);
1748 Fts5ExprPhrase
**apNew
;
1749 apNew
= (Fts5ExprPhrase
**)sqlite3_realloc64(pParse
->apPhrase
, nByte
);
1751 pParse
->rc
= SQLITE_NOMEM
;
1752 return SQLITE_NOMEM
;
1754 pParse
->apPhrase
= apNew
;
1760 ** This function is called by the parser to process a string token. The
1761 ** string may or may not be quoted. In any case it is tokenized and a
1762 ** phrase object consisting of all tokens returned.
1764 Fts5ExprPhrase
*sqlite3Fts5ParseTerm(
1765 Fts5Parse
*pParse
, /* Parse context */
1766 Fts5ExprPhrase
*pAppend
, /* Phrase to append to */
1767 Fts5Token
*pToken
, /* String to tokenize */
1768 int bPrefix
/* True if there is a trailing "*" */
1770 Fts5Config
*pConfig
= pParse
->pConfig
;
1771 TokenCtx sCtx
; /* Context object passed to callback */
1772 int rc
; /* Tokenize return code */
1775 memset(&sCtx
, 0, sizeof(TokenCtx
));
1776 sCtx
.pPhrase
= pAppend
;
1778 rc
= fts5ParseStringFromToken(pToken
, &z
);
1779 if( rc
==SQLITE_OK
){
1780 int flags
= FTS5_TOKENIZE_QUERY
| (bPrefix
? FTS5_TOKENIZE_PREFIX
: 0);
1782 sqlite3Fts5Dequote(z
);
1784 rc
= sqlite3Fts5Tokenize(pConfig
, flags
, z
, n
, &sCtx
, fts5ParseTokenize
);
1787 if( rc
|| (rc
= sCtx
.rc
) ){
1789 fts5ExprPhraseFree(sCtx
.pPhrase
);
1794 if( parseGrowPhraseArray(pParse
) ){
1795 fts5ExprPhraseFree(sCtx
.pPhrase
);
1801 if( sCtx
.pPhrase
==0 ){
1802 /* This happens when parsing a token or quoted phrase that contains
1803 ** no token characters at all. (e.g ... MATCH '""'). */
1804 sCtx
.pPhrase
= sqlite3Fts5MallocZero(&pParse
->rc
, sizeof(Fts5ExprPhrase
));
1805 }else if( sCtx
.pPhrase
->nTerm
){
1806 sCtx
.pPhrase
->aTerm
[sCtx
.pPhrase
->nTerm
-1].bPrefix
= (u8
)bPrefix
;
1808 pParse
->apPhrase
[pParse
->nPhrase
-1] = sCtx
.pPhrase
;
1811 return sCtx
.pPhrase
;
1815 ** Create a new FTS5 expression by cloning phrase iPhrase of the
1816 ** expression passed as the second argument.
1818 int sqlite3Fts5ExprClonePhrase(
1823 int rc
= SQLITE_OK
; /* Return code */
1824 Fts5ExprPhrase
*pOrig
; /* The phrase extracted from pExpr */
1825 Fts5Expr
*pNew
= 0; /* Expression to return via *ppNew */
1826 TokenCtx sCtx
= {0,0}; /* Context object for fts5ParseTokenize */
1828 pOrig
= pExpr
->apExprPhrase
[iPhrase
];
1829 pNew
= (Fts5Expr
*)sqlite3Fts5MallocZero(&rc
, sizeof(Fts5Expr
));
1830 if( rc
==SQLITE_OK
){
1831 pNew
->apExprPhrase
= (Fts5ExprPhrase
**)sqlite3Fts5MallocZero(&rc
,
1832 sizeof(Fts5ExprPhrase
*));
1834 if( rc
==SQLITE_OK
){
1835 pNew
->pRoot
= (Fts5ExprNode
*)sqlite3Fts5MallocZero(&rc
,
1836 sizeof(Fts5ExprNode
));
1838 if( rc
==SQLITE_OK
){
1839 pNew
->pRoot
->pNear
= (Fts5ExprNearset
*)sqlite3Fts5MallocZero(&rc
,
1840 sizeof(Fts5ExprNearset
) + sizeof(Fts5ExprPhrase
*));
1842 if( rc
==SQLITE_OK
){
1843 Fts5Colset
*pColsetOrig
= pOrig
->pNode
->pNear
->pColset
;
1845 sqlite3_int64 nByte
;
1846 Fts5Colset
*pColset
;
1847 nByte
= sizeof(Fts5Colset
) + (pColsetOrig
->nCol
-1) * sizeof(int);
1848 pColset
= (Fts5Colset
*)sqlite3Fts5MallocZero(&rc
, nByte
);
1850 memcpy(pColset
, pColsetOrig
, (size_t)nByte
);
1852 pNew
->pRoot
->pNear
->pColset
= pColset
;
1857 int i
; /* Used to iterate through phrase terms */
1858 for(i
=0; rc
==SQLITE_OK
&& i
<pOrig
->nTerm
; i
++){
1861 for(p
=&pOrig
->aTerm
[i
]; p
&& rc
==SQLITE_OK
; p
=p
->pSynonym
){
1862 const char *zTerm
= p
->zTerm
;
1863 rc
= fts5ParseTokenize((void*)&sCtx
, tflags
, zTerm
, (int)strlen(zTerm
),
1865 tflags
= FTS5_TOKEN_COLOCATED
;
1867 if( rc
==SQLITE_OK
){
1868 sCtx
.pPhrase
->aTerm
[i
].bPrefix
= pOrig
->aTerm
[i
].bPrefix
;
1869 sCtx
.pPhrase
->aTerm
[i
].bFirst
= pOrig
->aTerm
[i
].bFirst
;
1873 /* This happens when parsing a token or quoted phrase that contains
1874 ** no token characters at all. (e.g ... MATCH '""'). */
1875 sCtx
.pPhrase
= sqlite3Fts5MallocZero(&rc
, sizeof(Fts5ExprPhrase
));
1878 if( rc
==SQLITE_OK
&& ALWAYS(sCtx
.pPhrase
) ){
1879 /* All the allocations succeeded. Put the expression object together. */
1880 pNew
->pIndex
= pExpr
->pIndex
;
1881 pNew
->pConfig
= pExpr
->pConfig
;
1883 pNew
->apExprPhrase
[0] = sCtx
.pPhrase
;
1884 pNew
->pRoot
->pNear
->apPhrase
[0] = sCtx
.pPhrase
;
1885 pNew
->pRoot
->pNear
->nPhrase
= 1;
1886 sCtx
.pPhrase
->pNode
= pNew
->pRoot
;
1889 && pOrig
->aTerm
[0].pSynonym
==0
1890 && pOrig
->aTerm
[0].bFirst
==0
1892 pNew
->pRoot
->eType
= FTS5_TERM
;
1893 pNew
->pRoot
->xNext
= fts5ExprNodeNext_TERM
;
1895 pNew
->pRoot
->eType
= FTS5_STRING
;
1896 pNew
->pRoot
->xNext
= fts5ExprNodeNext_STRING
;
1899 sqlite3Fts5ExprFree(pNew
);
1900 fts5ExprPhraseFree(sCtx
.pPhrase
);
1910 ** Token pTok has appeared in a MATCH expression where the NEAR operator
1911 ** is expected. If token pTok does not contain "NEAR", store an error
1912 ** in the pParse object.
1914 void sqlite3Fts5ParseNear(Fts5Parse
*pParse
, Fts5Token
*pTok
){
1915 if( pTok
->n
!=4 || memcmp("NEAR", pTok
->p
, 4) ){
1916 sqlite3Fts5ParseError(
1917 pParse
, "fts5: syntax error near \"%.*s\"", pTok
->n
, pTok
->p
1922 void sqlite3Fts5ParseSetDistance(
1924 Fts5ExprNearset
*pNear
,
1931 for(i
=0; i
<p
->n
; i
++){
1932 char c
= (char)p
->p
[i
];
1933 if( c
<'0' || c
>'9' ){
1934 sqlite3Fts5ParseError(
1935 pParse
, "expected integer, got \"%.*s\"", p
->n
, p
->p
1939 nNear
= nNear
* 10 + (p
->p
[i
] - '0');
1942 nNear
= FTS5_DEFAULT_NEARDIST
;
1944 pNear
->nNear
= nNear
;
1949 ** The second argument passed to this function may be NULL, or it may be
1950 ** an existing Fts5Colset object. This function returns a pointer to
1951 ** a new colset object containing the contents of (p) with new value column
1952 ** number iCol appended.
1954 ** If an OOM error occurs, store an error code in pParse and return NULL.
1955 ** The old colset object (if any) is not freed in this case.
1957 static Fts5Colset
*fts5ParseColset(
1958 Fts5Parse
*pParse
, /* Store SQLITE_NOMEM here if required */
1959 Fts5Colset
*p
, /* Existing colset object */
1960 int iCol
/* New column to add to colset object */
1962 int nCol
= p
? p
->nCol
: 0; /* Num. columns already in colset object */
1963 Fts5Colset
*pNew
; /* New colset object to return */
1965 assert( pParse
->rc
==SQLITE_OK
);
1966 assert( iCol
>=0 && iCol
<pParse
->pConfig
->nCol
);
1968 pNew
= sqlite3_realloc64(p
, sizeof(Fts5Colset
) + sizeof(int)*nCol
);
1970 pParse
->rc
= SQLITE_NOMEM
;
1972 int *aiCol
= pNew
->aiCol
;
1974 for(i
=0; i
<nCol
; i
++){
1975 if( aiCol
[i
]==iCol
) return pNew
;
1976 if( aiCol
[i
]>iCol
) break;
1978 for(j
=nCol
; j
>i
; j
--){
1979 aiCol
[j
] = aiCol
[j
-1];
1982 pNew
->nCol
= nCol
+1;
1985 /* Check that the array is in order and contains no duplicate entries. */
1986 for(i
=1; i
<pNew
->nCol
; i
++) assert( pNew
->aiCol
[i
]>pNew
->aiCol
[i
-1] );
1994 ** Allocate and return an Fts5Colset object specifying the inverse of
1995 ** the colset passed as the second argument. Free the colset passed
1996 ** as the second argument before returning.
1998 Fts5Colset
*sqlite3Fts5ParseColsetInvert(Fts5Parse
*pParse
, Fts5Colset
*p
){
2000 int nCol
= pParse
->pConfig
->nCol
;
2002 pRet
= (Fts5Colset
*)sqlite3Fts5MallocZero(&pParse
->rc
,
2003 sizeof(Fts5Colset
) + sizeof(int)*nCol
2008 for(i
=0; i
<nCol
; i
++){
2009 if( iOld
>=p
->nCol
|| p
->aiCol
[iOld
]!=i
){
2010 pRet
->aiCol
[pRet
->nCol
++] = i
;
2021 Fts5Colset
*sqlite3Fts5ParseColset(
2022 Fts5Parse
*pParse
, /* Store SQLITE_NOMEM here if required */
2023 Fts5Colset
*pColset
, /* Existing colset object */
2026 Fts5Colset
*pRet
= 0;
2028 char *z
; /* Dequoted copy of token p */
2030 z
= sqlite3Fts5Strndup(&pParse
->rc
, p
->p
, p
->n
);
2031 if( pParse
->rc
==SQLITE_OK
){
2032 Fts5Config
*pConfig
= pParse
->pConfig
;
2033 sqlite3Fts5Dequote(z
);
2034 for(iCol
=0; iCol
<pConfig
->nCol
; iCol
++){
2035 if( 0==sqlite3_stricmp(pConfig
->azCol
[iCol
], z
) ) break;
2037 if( iCol
==pConfig
->nCol
){
2038 sqlite3Fts5ParseError(pParse
, "no such column: %s", z
);
2040 pRet
= fts5ParseColset(pParse
, pColset
, iCol
);
2046 assert( pParse
->rc
!=SQLITE_OK
);
2047 sqlite3_free(pColset
);
2054 ** If argument pOrig is NULL, or if (*pRc) is set to anything other than
2055 ** SQLITE_OK when this function is called, NULL is returned.
2057 ** Otherwise, a copy of (*pOrig) is made into memory obtained from
2058 ** sqlite3Fts5MallocZero() and a pointer to it returned. If the allocation
2059 ** fails, (*pRc) is set to SQLITE_NOMEM and NULL is returned.
2061 static Fts5Colset
*fts5CloneColset(int *pRc
, Fts5Colset
*pOrig
){
2064 sqlite3_int64 nByte
= sizeof(Fts5Colset
) + (pOrig
->nCol
-1) * sizeof(int);
2065 pRet
= (Fts5Colset
*)sqlite3Fts5MallocZero(pRc
, nByte
);
2067 memcpy(pRet
, pOrig
, (size_t)nByte
);
2076 ** Remove from colset pColset any columns that are not also in colset pMerge.
2078 static void fts5MergeColset(Fts5Colset
*pColset
, Fts5Colset
*pMerge
){
2079 int iIn
= 0; /* Next input in pColset */
2080 int iMerge
= 0; /* Next input in pMerge */
2081 int iOut
= 0; /* Next output slot in pColset */
2083 while( iIn
<pColset
->nCol
&& iMerge
<pMerge
->nCol
){
2084 int iDiff
= pColset
->aiCol
[iIn
] - pMerge
->aiCol
[iMerge
];
2086 pColset
->aiCol
[iOut
++] = pMerge
->aiCol
[iMerge
];
2089 }else if( iDiff
>0 ){
2095 pColset
->nCol
= iOut
;
2099 ** Recursively apply colset pColset to expression node pNode and all of
2100 ** its decendents. If (*ppFree) is not NULL, it contains a spare copy
2101 ** of pColset. This function may use the spare copy and set (*ppFree) to
2102 ** zero, or it may create copies of pColset using fts5CloneColset().
2104 static void fts5ParseSetColset(
2106 Fts5ExprNode
*pNode
,
2107 Fts5Colset
*pColset
,
2110 if( pParse
->rc
==SQLITE_OK
){
2111 assert( pNode
->eType
==FTS5_TERM
|| pNode
->eType
==FTS5_STRING
2112 || pNode
->eType
==FTS5_AND
|| pNode
->eType
==FTS5_OR
2113 || pNode
->eType
==FTS5_NOT
|| pNode
->eType
==FTS5_EOF
2115 if( pNode
->eType
==FTS5_STRING
|| pNode
->eType
==FTS5_TERM
){
2116 Fts5ExprNearset
*pNear
= pNode
->pNear
;
2117 if( pNear
->pColset
){
2118 fts5MergeColset(pNear
->pColset
, pColset
);
2119 if( pNear
->pColset
->nCol
==0 ){
2120 pNode
->eType
= FTS5_EOF
;
2123 }else if( *ppFree
){
2124 pNear
->pColset
= pColset
;
2127 pNear
->pColset
= fts5CloneColset(&pParse
->rc
, pColset
);
2131 assert( pNode
->eType
!=FTS5_EOF
|| pNode
->nChild
==0 );
2132 for(i
=0; i
<pNode
->nChild
; i
++){
2133 fts5ParseSetColset(pParse
, pNode
->apChild
[i
], pColset
, ppFree
);
2140 ** Apply colset pColset to expression node pExpr and all of its descendents.
2142 void sqlite3Fts5ParseSetColset(
2144 Fts5ExprNode
*pExpr
,
2147 Fts5Colset
*pFree
= pColset
;
2148 if( pParse
->pConfig
->eDetail
==FTS5_DETAIL_NONE
){
2149 sqlite3Fts5ParseError(pParse
,
2150 "fts5: column queries are not supported (detail=none)"
2153 fts5ParseSetColset(pParse
, pExpr
, pColset
, &pFree
);
2155 sqlite3_free(pFree
);
2158 static void fts5ExprAssignXNext(Fts5ExprNode
*pNode
){
2159 switch( pNode
->eType
){
2161 Fts5ExprNearset
*pNear
= pNode
->pNear
;
2162 if( pNear
->nPhrase
==1 && pNear
->apPhrase
[0]->nTerm
==1
2163 && pNear
->apPhrase
[0]->aTerm
[0].pSynonym
==0
2164 && pNear
->apPhrase
[0]->aTerm
[0].bFirst
==0
2166 pNode
->eType
= FTS5_TERM
;
2167 pNode
->xNext
= fts5ExprNodeNext_TERM
;
2169 pNode
->xNext
= fts5ExprNodeNext_STRING
;
2175 pNode
->xNext
= fts5ExprNodeNext_OR
;
2180 pNode
->xNext
= fts5ExprNodeNext_AND
;
2184 default: assert( pNode
->eType
==FTS5_NOT
); {
2185 pNode
->xNext
= fts5ExprNodeNext_NOT
;
2191 static void fts5ExprAddChildren(Fts5ExprNode
*p
, Fts5ExprNode
*pSub
){
2192 if( p
->eType
!=FTS5_NOT
&& pSub
->eType
==p
->eType
){
2193 int nByte
= sizeof(Fts5ExprNode
*) * pSub
->nChild
;
2194 memcpy(&p
->apChild
[p
->nChild
], pSub
->apChild
, nByte
);
2195 p
->nChild
+= pSub
->nChild
;
2198 p
->apChild
[p
->nChild
++] = pSub
;
2203 ** This function is used when parsing LIKE or GLOB patterns against
2204 ** trigram indexes that specify either detail=column or detail=none.
2205 ** It converts a phrase:
2209 ** into an AND tree:
2211 ** abc AND def AND ghi
2213 static Fts5ExprNode
*fts5ParsePhraseToAnd(
2215 Fts5ExprNearset
*pNear
2217 int nTerm
= pNear
->apPhrase
[0]->nTerm
;
2222 assert( pNear
->nPhrase
==1 );
2223 assert( pParse
->bPhraseToAnd
);
2225 nByte
= sizeof(Fts5ExprNode
) + nTerm
*sizeof(Fts5ExprNode
*);
2226 pRet
= (Fts5ExprNode
*)sqlite3Fts5MallocZero(&pParse
->rc
, nByte
);
2228 pRet
->eType
= FTS5_AND
;
2229 pRet
->nChild
= nTerm
;
2230 fts5ExprAssignXNext(pRet
);
2232 for(ii
=0; ii
<nTerm
; ii
++){
2233 Fts5ExprPhrase
*pPhrase
= (Fts5ExprPhrase
*)sqlite3Fts5MallocZero(
2234 &pParse
->rc
, sizeof(Fts5ExprPhrase
)
2237 if( parseGrowPhraseArray(pParse
) ){
2238 fts5ExprPhraseFree(pPhrase
);
2240 pParse
->apPhrase
[pParse
->nPhrase
++] = pPhrase
;
2242 pPhrase
->aTerm
[0].zTerm
= sqlite3Fts5Strndup(
2243 &pParse
->rc
, pNear
->apPhrase
[0]->aTerm
[ii
].zTerm
, -1
2245 pRet
->apChild
[ii
] = sqlite3Fts5ParseNode(pParse
, FTS5_STRING
,
2246 0, 0, sqlite3Fts5ParseNearset(pParse
, 0, pPhrase
)
2253 sqlite3Fts5ParseNodeFree(pRet
);
2256 sqlite3Fts5ParseNearsetFree(pNear
);
2264 ** Allocate and return a new expression object. If anything goes wrong (i.e.
2265 ** OOM error), leave an error code in pParse and return NULL.
2267 Fts5ExprNode
*sqlite3Fts5ParseNode(
2268 Fts5Parse
*pParse
, /* Parse context */
2269 int eType
, /* FTS5_STRING, AND, OR or NOT */
2270 Fts5ExprNode
*pLeft
, /* Left hand child expression */
2271 Fts5ExprNode
*pRight
, /* Right hand child expression */
2272 Fts5ExprNearset
*pNear
/* For STRING expressions, the near cluster */
2274 Fts5ExprNode
*pRet
= 0;
2276 if( pParse
->rc
==SQLITE_OK
){
2277 int nChild
= 0; /* Number of children of returned node */
2278 sqlite3_int64 nByte
; /* Bytes of space to allocate for this node */
2280 assert( (eType
!=FTS5_STRING
&& !pNear
)
2281 || (eType
==FTS5_STRING
&& !pLeft
&& !pRight
)
2283 if( eType
==FTS5_STRING
&& pNear
==0 ) return 0;
2284 if( eType
!=FTS5_STRING
&& pLeft
==0 ) return pRight
;
2285 if( eType
!=FTS5_STRING
&& pRight
==0 ) return pLeft
;
2287 if( eType
==FTS5_STRING
2288 && pParse
->bPhraseToAnd
2289 && pNear
->apPhrase
[0]->nTerm
>1
2291 pRet
= fts5ParsePhraseToAnd(pParse
, pNear
);
2293 if( eType
==FTS5_NOT
){
2295 }else if( eType
==FTS5_AND
|| eType
==FTS5_OR
){
2297 if( pLeft
->eType
==eType
) nChild
+= pLeft
->nChild
-1;
2298 if( pRight
->eType
==eType
) nChild
+= pRight
->nChild
-1;
2301 nByte
= sizeof(Fts5ExprNode
) + sizeof(Fts5ExprNode
*)*(nChild
-1);
2302 pRet
= (Fts5ExprNode
*)sqlite3Fts5MallocZero(&pParse
->rc
, nByte
);
2305 pRet
->eType
= eType
;
2306 pRet
->pNear
= pNear
;
2307 fts5ExprAssignXNext(pRet
);
2308 if( eType
==FTS5_STRING
){
2310 for(iPhrase
=0; iPhrase
<pNear
->nPhrase
; iPhrase
++){
2311 pNear
->apPhrase
[iPhrase
]->pNode
= pRet
;
2312 if( pNear
->apPhrase
[iPhrase
]->nTerm
==0 ){
2314 pRet
->eType
= FTS5_EOF
;
2318 if( pParse
->pConfig
->eDetail
!=FTS5_DETAIL_FULL
){
2319 Fts5ExprPhrase
*pPhrase
= pNear
->apPhrase
[0];
2320 if( pNear
->nPhrase
!=1
2322 || (pPhrase
->nTerm
>0 && pPhrase
->aTerm
[0].bFirst
)
2324 sqlite3Fts5ParseError(pParse
,
2325 "fts5: %s queries are not supported (detail!=full)",
2326 pNear
->nPhrase
==1 ? "phrase": "NEAR"
2333 fts5ExprAddChildren(pRet
, pLeft
);
2334 fts5ExprAddChildren(pRet
, pRight
);
2341 assert( pParse
->rc
!=SQLITE_OK
);
2342 sqlite3Fts5ParseNodeFree(pLeft
);
2343 sqlite3Fts5ParseNodeFree(pRight
);
2344 sqlite3Fts5ParseNearsetFree(pNear
);
2349 Fts5ExprNode
*sqlite3Fts5ParseImplicitAnd(
2350 Fts5Parse
*pParse
, /* Parse context */
2351 Fts5ExprNode
*pLeft
, /* Left hand child expression */
2352 Fts5ExprNode
*pRight
/* Right hand child expression */
2354 Fts5ExprNode
*pRet
= 0;
2355 Fts5ExprNode
*pPrev
;
2358 sqlite3Fts5ParseNodeFree(pLeft
);
2359 sqlite3Fts5ParseNodeFree(pRight
);
2362 assert( pLeft
->eType
==FTS5_STRING
2363 || pLeft
->eType
==FTS5_TERM
2364 || pLeft
->eType
==FTS5_EOF
2365 || pLeft
->eType
==FTS5_AND
2367 assert( pRight
->eType
==FTS5_STRING
2368 || pRight
->eType
==FTS5_TERM
2369 || pRight
->eType
==FTS5_EOF
2372 if( pLeft
->eType
==FTS5_AND
){
2373 pPrev
= pLeft
->apChild
[pLeft
->nChild
-1];
2377 assert( pPrev
->eType
==FTS5_STRING
2378 || pPrev
->eType
==FTS5_TERM
2379 || pPrev
->eType
==FTS5_EOF
2382 if( pRight
->eType
==FTS5_EOF
){
2383 assert( pParse
->apPhrase
[pParse
->nPhrase
-1]==pRight
->pNear
->apPhrase
[0] );
2384 sqlite3Fts5ParseNodeFree(pRight
);
2388 else if( pPrev
->eType
==FTS5_EOF
){
2389 Fts5ExprPhrase
**ap
;
2394 pLeft
->apChild
[pLeft
->nChild
-1] = pRight
;
2398 ap
= &pParse
->apPhrase
[pParse
->nPhrase
-1-pRight
->pNear
->nPhrase
];
2399 assert( ap
[0]==pPrev
->pNear
->apPhrase
[0] );
2400 memmove(ap
, &ap
[1], sizeof(Fts5ExprPhrase
*)*pRight
->pNear
->nPhrase
);
2403 sqlite3Fts5ParseNodeFree(pPrev
);
2406 pRet
= sqlite3Fts5ParseNode(pParse
, FTS5_AND
, pLeft
, pRight
, 0);
2414 static char *fts5ExprTermPrint(Fts5ExprTerm
*pTerm
){
2415 sqlite3_int64 nByte
= 0;
2419 /* Determine the maximum amount of space required. */
2420 for(p
=pTerm
; p
; p
=p
->pSynonym
){
2421 nByte
+= (int)strlen(pTerm
->zTerm
) * 2 + 3 + 2;
2423 zQuoted
= sqlite3_malloc64(nByte
);
2427 for(p
=pTerm
; p
; p
=p
->pSynonym
){
2428 char *zIn
= p
->zTerm
;
2431 if( *zIn
=='"' ) zQuoted
[i
++] = '"';
2432 zQuoted
[i
++] = *zIn
++;
2435 if( p
->pSynonym
) zQuoted
[i
++] = '|';
2437 if( pTerm
->bPrefix
){
2441 zQuoted
[i
++] = '\0';
2446 static char *fts5PrintfAppend(char *zApp
, const char *zFmt
, ...){
2450 zNew
= sqlite3_vmprintf(zFmt
, ap
);
2453 char *zNew2
= sqlite3_mprintf("%s%s", zApp
, zNew
);
2462 ** Compose a tcl-readable representation of expression pExpr. Return a
2463 ** pointer to a buffer containing that representation. It is the
2464 ** responsibility of the caller to at some point free the buffer using
2467 static char *fts5ExprPrintTcl(
2468 Fts5Config
*pConfig
,
2469 const char *zNearsetCmd
,
2473 if( pExpr
->eType
==FTS5_STRING
|| pExpr
->eType
==FTS5_TERM
){
2474 Fts5ExprNearset
*pNear
= pExpr
->pNear
;
2478 zRet
= fts5PrintfAppend(zRet
, "%s ", zNearsetCmd
);
2479 if( zRet
==0 ) return 0;
2480 if( pNear
->pColset
){
2481 int *aiCol
= pNear
->pColset
->aiCol
;
2482 int nCol
= pNear
->pColset
->nCol
;
2484 zRet
= fts5PrintfAppend(zRet
, "-col %d ", aiCol
[0]);
2486 zRet
= fts5PrintfAppend(zRet
, "-col {%d", aiCol
[0]);
2487 for(i
=1; i
<pNear
->pColset
->nCol
; i
++){
2488 zRet
= fts5PrintfAppend(zRet
, " %d", aiCol
[i
]);
2490 zRet
= fts5PrintfAppend(zRet
, "} ");
2492 if( zRet
==0 ) return 0;
2495 if( pNear
->nPhrase
>1 ){
2496 zRet
= fts5PrintfAppend(zRet
, "-near %d ", pNear
->nNear
);
2497 if( zRet
==0 ) return 0;
2500 zRet
= fts5PrintfAppend(zRet
, "--");
2501 if( zRet
==0 ) return 0;
2503 for(i
=0; i
<pNear
->nPhrase
; i
++){
2504 Fts5ExprPhrase
*pPhrase
= pNear
->apPhrase
[i
];
2506 zRet
= fts5PrintfAppend(zRet
, " {");
2507 for(iTerm
=0; zRet
&& iTerm
<pPhrase
->nTerm
; iTerm
++){
2508 char *zTerm
= pPhrase
->aTerm
[iTerm
].zTerm
;
2509 zRet
= fts5PrintfAppend(zRet
, "%s%s", iTerm
==0?"":" ", zTerm
);
2510 if( pPhrase
->aTerm
[iTerm
].bPrefix
){
2511 zRet
= fts5PrintfAppend(zRet
, "*");
2515 if( zRet
) zRet
= fts5PrintfAppend(zRet
, "}");
2516 if( zRet
==0 ) return 0;
2520 char const *zOp
= 0;
2522 switch( pExpr
->eType
){
2523 case FTS5_AND
: zOp
= "AND"; break;
2524 case FTS5_NOT
: zOp
= "NOT"; break;
2526 assert( pExpr
->eType
==FTS5_OR
);
2531 zRet
= sqlite3_mprintf("%s", zOp
);
2532 for(i
=0; zRet
&& i
<pExpr
->nChild
; i
++){
2533 char *z
= fts5ExprPrintTcl(pConfig
, zNearsetCmd
, pExpr
->apChild
[i
]);
2538 zRet
= fts5PrintfAppend(zRet
, " [%z]", z
);
2546 static char *fts5ExprPrint(Fts5Config
*pConfig
, Fts5ExprNode
*pExpr
){
2548 if( pExpr
->eType
==0 ){
2549 return sqlite3_mprintf("\"\"");
2551 if( pExpr
->eType
==FTS5_STRING
|| pExpr
->eType
==FTS5_TERM
){
2552 Fts5ExprNearset
*pNear
= pExpr
->pNear
;
2556 if( pNear
->pColset
){
2558 Fts5Colset
*pColset
= pNear
->pColset
;
2559 if( pColset
->nCol
>1 ) zRet
= fts5PrintfAppend(zRet
, "{");
2560 for(ii
=0; ii
<pColset
->nCol
; ii
++){
2561 zRet
= fts5PrintfAppend(zRet
, "%s%s",
2562 pConfig
->azCol
[pColset
->aiCol
[ii
]], ii
==pColset
->nCol
-1 ? "" : " "
2566 zRet
= fts5PrintfAppend(zRet
, "%s : ", pColset
->nCol
>1 ? "}" : "");
2568 if( zRet
==0 ) return 0;
2571 if( pNear
->nPhrase
>1 ){
2572 zRet
= fts5PrintfAppend(zRet
, "NEAR(");
2573 if( zRet
==0 ) return 0;
2576 for(i
=0; i
<pNear
->nPhrase
; i
++){
2577 Fts5ExprPhrase
*pPhrase
= pNear
->apPhrase
[i
];
2579 zRet
= fts5PrintfAppend(zRet
, " ");
2580 if( zRet
==0 ) return 0;
2582 for(iTerm
=0; iTerm
<pPhrase
->nTerm
; iTerm
++){
2583 char *zTerm
= fts5ExprTermPrint(&pPhrase
->aTerm
[iTerm
]);
2585 zRet
= fts5PrintfAppend(zRet
, "%s%s", iTerm
==0?"":" + ", zTerm
);
2586 sqlite3_free(zTerm
);
2588 if( zTerm
==0 || zRet
==0 ){
2595 if( pNear
->nPhrase
>1 ){
2596 zRet
= fts5PrintfAppend(zRet
, ", %d)", pNear
->nNear
);
2597 if( zRet
==0 ) return 0;
2601 char const *zOp
= 0;
2604 switch( pExpr
->eType
){
2605 case FTS5_AND
: zOp
= " AND "; break;
2606 case FTS5_NOT
: zOp
= " NOT "; break;
2608 assert( pExpr
->eType
==FTS5_OR
);
2613 for(i
=0; i
<pExpr
->nChild
; i
++){
2614 char *z
= fts5ExprPrint(pConfig
, pExpr
->apChild
[i
]);
2619 int e
= pExpr
->apChild
[i
]->eType
;
2620 int b
= (e
!=FTS5_STRING
&& e
!=FTS5_TERM
&& e
!=FTS5_EOF
);
2621 zRet
= fts5PrintfAppend(zRet
, "%s%s%z%s",
2623 (b
?"(":""), z
, (b
?")":"")
2626 if( zRet
==0 ) break;
2634 ** The implementation of user-defined scalar functions fts5_expr() (bTcl==0)
2635 ** and fts5_expr_tcl() (bTcl!=0).
2637 static void fts5ExprFunction(
2638 sqlite3_context
*pCtx
, /* Function call context */
2639 int nArg
, /* Number of args */
2640 sqlite3_value
**apVal
, /* Function arguments */
2643 Fts5Global
*pGlobal
= (Fts5Global
*)sqlite3_user_data(pCtx
);
2644 sqlite3
*db
= sqlite3_context_db_handle(pCtx
);
2645 const char *zExpr
= 0;
2647 Fts5Expr
*pExpr
= 0;
2651 const char **azConfig
; /* Array of arguments for Fts5Config */
2652 const char *zNearsetCmd
= "nearset";
2653 int nConfig
; /* Size of azConfig[] */
2654 Fts5Config
*pConfig
= 0;
2658 zErr
= sqlite3_mprintf("wrong number of arguments to function %s",
2659 bTcl
? "fts5_expr_tcl" : "fts5_expr"
2661 sqlite3_result_error(pCtx
, zErr
, -1);
2666 if( bTcl
&& nArg
>1 ){
2667 zNearsetCmd
= (const char*)sqlite3_value_text(apVal
[1]);
2671 nConfig
= 3 + (nArg
-iArg
);
2672 azConfig
= (const char**)sqlite3_malloc64(sizeof(char*) * nConfig
);
2674 sqlite3_result_error_nomem(pCtx
);
2678 azConfig
[1] = "main";
2679 azConfig
[2] = "tbl";
2680 for(i
=3; iArg
<nArg
; iArg
++){
2681 const char *z
= (const char*)sqlite3_value_text(apVal
[iArg
]);
2682 azConfig
[i
++] = (z
? z
: "");
2685 zExpr
= (const char*)sqlite3_value_text(apVal
[0]);
2686 if( zExpr
==0 ) zExpr
= "";
2688 rc
= sqlite3Fts5ConfigParse(pGlobal
, db
, nConfig
, azConfig
, &pConfig
, &zErr
);
2689 if( rc
==SQLITE_OK
){
2690 rc
= sqlite3Fts5ExprNew(pConfig
, 0, pConfig
->nCol
, zExpr
, &pExpr
, &zErr
);
2692 if( rc
==SQLITE_OK
){
2694 if( pExpr
->pRoot
->xNext
==0 ){
2695 zText
= sqlite3_mprintf("");
2697 zText
= fts5ExprPrintTcl(pConfig
, zNearsetCmd
, pExpr
->pRoot
);
2699 zText
= fts5ExprPrint(pConfig
, pExpr
->pRoot
);
2704 sqlite3_result_text(pCtx
, zText
, -1, SQLITE_TRANSIENT
);
2705 sqlite3_free(zText
);
2709 if( rc
!=SQLITE_OK
){
2711 sqlite3_result_error(pCtx
, zErr
, -1);
2714 sqlite3_result_error_code(pCtx
, rc
);
2717 sqlite3_free((void *)azConfig
);
2718 sqlite3Fts5ConfigFree(pConfig
);
2719 sqlite3Fts5ExprFree(pExpr
);
2722 static void fts5ExprFunctionHr(
2723 sqlite3_context
*pCtx
, /* Function call context */
2724 int nArg
, /* Number of args */
2725 sqlite3_value
**apVal
/* Function arguments */
2727 fts5ExprFunction(pCtx
, nArg
, apVal
, 0);
2729 static void fts5ExprFunctionTcl(
2730 sqlite3_context
*pCtx
, /* Function call context */
2731 int nArg
, /* Number of args */
2732 sqlite3_value
**apVal
/* Function arguments */
2734 fts5ExprFunction(pCtx
, nArg
, apVal
, 1);
2738 ** The implementation of an SQLite user-defined-function that accepts a
2739 ** single integer as an argument. If the integer is an alpha-numeric
2740 ** unicode code point, 1 is returned. Otherwise 0.
2742 static void fts5ExprIsAlnum(
2743 sqlite3_context
*pCtx
, /* Function call context */
2744 int nArg
, /* Number of args */
2745 sqlite3_value
**apVal
/* Function arguments */
2750 sqlite3_result_error(pCtx
,
2751 "wrong number of arguments to function fts5_isalnum", -1
2755 memset(aArr
, 0, sizeof(aArr
));
2756 sqlite3Fts5UnicodeCatParse("L*", aArr
);
2757 sqlite3Fts5UnicodeCatParse("N*", aArr
);
2758 sqlite3Fts5UnicodeCatParse("Co", aArr
);
2759 iCode
= sqlite3_value_int(apVal
[0]);
2760 sqlite3_result_int(pCtx
, aArr
[sqlite3Fts5UnicodeCategory((u32
)iCode
)]);
2763 static void fts5ExprFold(
2764 sqlite3_context
*pCtx
, /* Function call context */
2765 int nArg
, /* Number of args */
2766 sqlite3_value
**apVal
/* Function arguments */
2768 if( nArg
!=1 && nArg
!=2 ){
2769 sqlite3_result_error(pCtx
,
2770 "wrong number of arguments to function fts5_fold", -1
2774 int bRemoveDiacritics
= 0;
2775 iCode
= sqlite3_value_int(apVal
[0]);
2776 if( nArg
==2 ) bRemoveDiacritics
= sqlite3_value_int(apVal
[1]);
2777 sqlite3_result_int(pCtx
, sqlite3Fts5UnicodeFold(iCode
, bRemoveDiacritics
));
2780 #endif /* ifdef SQLITE_TEST */
2783 ** This is called during initialization to register the fts5_expr() scalar
2784 ** UDF with the SQLite handle passed as the only argument.
2786 int sqlite3Fts5ExprInit(Fts5Global
*pGlobal
, sqlite3
*db
){
2788 struct Fts5ExprFunc
{
2790 void (*x
)(sqlite3_context
*,int,sqlite3_value
**);
2792 { "fts5_expr", fts5ExprFunctionHr
},
2793 { "fts5_expr_tcl", fts5ExprFunctionTcl
},
2794 { "fts5_isalnum", fts5ExprIsAlnum
},
2795 { "fts5_fold", fts5ExprFold
},
2799 void *pCtx
= (void*)pGlobal
;
2801 for(i
=0; rc
==SQLITE_OK
&& i
<ArraySize(aFunc
); i
++){
2802 struct Fts5ExprFunc
*p
= &aFunc
[i
];
2803 rc
= sqlite3_create_function(db
, p
->z
, -1, SQLITE_UTF8
, pCtx
, p
->x
, 0, 0);
2807 UNUSED_PARAM2(pGlobal
,db
);
2810 /* Avoid warnings indicating that sqlite3Fts5ParserTrace() and
2811 ** sqlite3Fts5ParserFallback() are unused */
2813 (void)sqlite3Fts5ParserTrace
;
2815 (void)sqlite3Fts5ParserFallback
;
2821 ** Return the number of phrases in expression pExpr.
2823 int sqlite3Fts5ExprPhraseCount(Fts5Expr
*pExpr
){
2824 return (pExpr
? pExpr
->nPhrase
: 0);
2828 ** Return the number of terms in the iPhrase'th phrase in pExpr.
2830 int sqlite3Fts5ExprPhraseSize(Fts5Expr
*pExpr
, int iPhrase
){
2831 if( iPhrase
<0 || iPhrase
>=pExpr
->nPhrase
) return 0;
2832 return pExpr
->apExprPhrase
[iPhrase
]->nTerm
;
2836 ** This function is used to access the current position list for phrase
2839 int sqlite3Fts5ExprPoslist(Fts5Expr
*pExpr
, int iPhrase
, const u8
**pa
){
2841 Fts5ExprPhrase
*pPhrase
= pExpr
->apExprPhrase
[iPhrase
];
2842 Fts5ExprNode
*pNode
= pPhrase
->pNode
;
2843 if( pNode
->bEof
==0 && pNode
->iRowid
==pExpr
->pRoot
->iRowid
){
2844 *pa
= pPhrase
->poslist
.p
;
2845 nRet
= pPhrase
->poslist
.n
;
2853 struct Fts5PoslistPopulator
{
2854 Fts5PoslistWriter writer
;
2855 int bOk
; /* True if ok to populate */
2860 ** Clear the position lists associated with all phrases in the expression
2861 ** passed as the first argument. Argument bLive is true if the expression
2862 ** might be pointing to a real entry, otherwise it has just been reset.
2864 ** At present this function is only used for detail=col and detail=none
2865 ** fts5 tables. This implies that all phrases must be at most 1 token
2866 ** in size, as phrase matches are not supported without detail=full.
2868 Fts5PoslistPopulator
*sqlite3Fts5ExprClearPoslists(Fts5Expr
*pExpr
, int bLive
){
2869 Fts5PoslistPopulator
*pRet
;
2870 pRet
= sqlite3_malloc64(sizeof(Fts5PoslistPopulator
)*pExpr
->nPhrase
);
2873 memset(pRet
, 0, sizeof(Fts5PoslistPopulator
)*pExpr
->nPhrase
);
2874 for(i
=0; i
<pExpr
->nPhrase
; i
++){
2875 Fts5Buffer
*pBuf
= &pExpr
->apExprPhrase
[i
]->poslist
;
2876 Fts5ExprNode
*pNode
= pExpr
->apExprPhrase
[i
]->pNode
;
2877 assert( pExpr
->apExprPhrase
[i
]->nTerm
<=1 );
2879 (pBuf
->n
==0 || pNode
->iRowid
!=pExpr
->pRoot
->iRowid
|| pNode
->bEof
)
2890 struct Fts5ExprCtx
{
2892 Fts5PoslistPopulator
*aPopulator
;
2895 typedef struct Fts5ExprCtx Fts5ExprCtx
;
2898 ** TODO: Make this more efficient!
2900 static int fts5ExprColsetTest(Fts5Colset
*pColset
, int iCol
){
2902 for(i
=0; i
<pColset
->nCol
; i
++){
2903 if( pColset
->aiCol
[i
]==iCol
) return 1;
2908 static int fts5ExprPopulatePoslistsCb(
2909 void *pCtx
, /* Copy of 2nd argument to xTokenize() */
2910 int tflags
, /* Mask of FTS5_TOKEN_* flags */
2911 const char *pToken
, /* Pointer to buffer containing token */
2912 int nToken
, /* Size of token in bytes */
2913 int iUnused1
, /* Byte offset of token within input text */
2914 int iUnused2
/* Byte offset of end of token within input text */
2916 Fts5ExprCtx
*p
= (Fts5ExprCtx
*)pCtx
;
2917 Fts5Expr
*pExpr
= p
->pExpr
;
2920 UNUSED_PARAM2(iUnused1
, iUnused2
);
2922 if( nToken
>FTS5_MAX_TOKEN_SIZE
) nToken
= FTS5_MAX_TOKEN_SIZE
;
2923 if( (tflags
& FTS5_TOKEN_COLOCATED
)==0 ) p
->iOff
++;
2924 for(i
=0; i
<pExpr
->nPhrase
; i
++){
2925 Fts5ExprTerm
*pTerm
;
2926 if( p
->aPopulator
[i
].bOk
==0 ) continue;
2927 for(pTerm
=&pExpr
->apExprPhrase
[i
]->aTerm
[0]; pTerm
; pTerm
=pTerm
->pSynonym
){
2928 int nTerm
= (int)strlen(pTerm
->zTerm
);
2929 if( (nTerm
==nToken
|| (nTerm
<nToken
&& pTerm
->bPrefix
))
2930 && memcmp(pTerm
->zTerm
, pToken
, nTerm
)==0
2932 int rc
= sqlite3Fts5PoslistWriterAppend(
2933 &pExpr
->apExprPhrase
[i
]->poslist
, &p
->aPopulator
[i
].writer
, p
->iOff
2943 int sqlite3Fts5ExprPopulatePoslists(
2944 Fts5Config
*pConfig
,
2946 Fts5PoslistPopulator
*aPopulator
,
2948 const char *z
, int n
2953 sCtx
.aPopulator
= aPopulator
;
2954 sCtx
.iOff
= (((i64
)iCol
) << 32) - 1;
2956 for(i
=0; i
<pExpr
->nPhrase
; i
++){
2957 Fts5ExprNode
*pNode
= pExpr
->apExprPhrase
[i
]->pNode
;
2958 Fts5Colset
*pColset
= pNode
->pNear
->pColset
;
2959 if( (pColset
&& 0==fts5ExprColsetTest(pColset
, iCol
))
2960 || aPopulator
[i
].bMiss
2962 aPopulator
[i
].bOk
= 0;
2964 aPopulator
[i
].bOk
= 1;
2968 return sqlite3Fts5Tokenize(pConfig
,
2969 FTS5_TOKENIZE_DOCUMENT
, z
, n
, (void*)&sCtx
, fts5ExprPopulatePoslistsCb
2973 static void fts5ExprClearPoslists(Fts5ExprNode
*pNode
){
2974 if( pNode
->eType
==FTS5_TERM
|| pNode
->eType
==FTS5_STRING
){
2975 pNode
->pNear
->apPhrase
[0]->poslist
.n
= 0;
2978 for(i
=0; i
<pNode
->nChild
; i
++){
2979 fts5ExprClearPoslists(pNode
->apChild
[i
]);
2984 static int fts5ExprCheckPoslists(Fts5ExprNode
*pNode
, i64 iRowid
){
2985 pNode
->iRowid
= iRowid
;
2987 switch( pNode
->eType
){
2990 return (pNode
->pNear
->apPhrase
[0]->poslist
.n
>0);
2994 for(i
=0; i
<pNode
->nChild
; i
++){
2995 if( fts5ExprCheckPoslists(pNode
->apChild
[i
], iRowid
)==0 ){
2996 fts5ExprClearPoslists(pNode
);
3006 for(i
=0; i
<pNode
->nChild
; i
++){
3007 if( fts5ExprCheckPoslists(pNode
->apChild
[i
], iRowid
) ){
3015 assert( pNode
->eType
==FTS5_NOT
);
3016 if( 0==fts5ExprCheckPoslists(pNode
->apChild
[0], iRowid
)
3017 || 0!=fts5ExprCheckPoslists(pNode
->apChild
[1], iRowid
)
3019 fts5ExprClearPoslists(pNode
);
3028 void sqlite3Fts5ExprCheckPoslists(Fts5Expr
*pExpr
, i64 iRowid
){
3029 fts5ExprCheckPoslists(pExpr
->pRoot
, iRowid
);
3033 ** This function is only called for detail=columns tables.
3035 int sqlite3Fts5ExprPhraseCollist(
3038 const u8
**ppCollist
,
3041 Fts5ExprPhrase
*pPhrase
= pExpr
->apExprPhrase
[iPhrase
];
3042 Fts5ExprNode
*pNode
= pPhrase
->pNode
;
3045 assert( iPhrase
>=0 && iPhrase
<pExpr
->nPhrase
);
3046 assert( pExpr
->pConfig
->eDetail
==FTS5_DETAIL_COLUMNS
);
3049 && pNode
->iRowid
==pExpr
->pRoot
->iRowid
3050 && pPhrase
->poslist
.n
>0
3052 Fts5ExprTerm
*pTerm
= &pPhrase
->aTerm
[0];
3053 if( pTerm
->pSynonym
){
3054 Fts5Buffer
*pBuf
= (Fts5Buffer
*)&pTerm
->pSynonym
[1];
3055 rc
= fts5ExprSynonymList(
3056 pTerm
, pNode
->iRowid
, pBuf
, (u8
**)ppCollist
, pnCollist
3059 *ppCollist
= pPhrase
->aTerm
[0].pIter
->pData
;
3060 *pnCollist
= pPhrase
->aTerm
[0].pIter
->nData
;