4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing:
7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give.
11 *************************************************************************
12 ** This module contains C code that generates VDBE code used to process
13 ** the WHERE clause of SQL statements. This module is responsible for
14 ** generating the code that loops through a table looking for applicable
15 ** rows. Indices are selected and used to speed the search when doing
16 ** so is applicable. Because this module is responsible for selecting
17 ** indices, you might also think of this module as the "query optimizer".
19 #include "sqliteInt.h"
23 ** Extra information appended to the end of sqlite3_index_info but not
24 ** visible to the xBestIndex function, at least not directly. The
25 ** sqlite3_vtab_collation() interface knows how to reach it, however.
27 ** This object is not an API and can be changed from one release to the
28 ** next. As long as allocateIndexInfo() and sqlite3_vtab_collation()
29 ** agree on the structure, all will be well.
31 typedef struct HiddenIndexInfo HiddenIndexInfo
;
32 struct HiddenIndexInfo
{
33 WhereClause
*pWC
; /* The Where clause being analyzed */
34 Parse
*pParse
; /* The parsing context */
35 int eDistinct
; /* Value to return from sqlite3_vtab_distinct() */
36 u32 mIn
; /* Mask of terms that are <col> IN (...) */
37 u32 mHandleIn
; /* Terms that vtab will handle as <col> IN (...) */
38 sqlite3_value
*aRhs
[1]; /* RHS values for constraints. MUST BE LAST
39 ** because extra space is allocated to hold up
40 ** to nTerm such values */
43 /* Forward declaration of methods */
44 static int whereLoopResize(sqlite3
*, WhereLoop
*, int);
47 ** Return the estimated number of output rows from a WHERE clause
49 LogEst
sqlite3WhereOutputRowCount(WhereInfo
*pWInfo
){
50 return pWInfo
->nRowOut
;
54 ** Return one of the WHERE_DISTINCT_xxxxx values to indicate how this
55 ** WHERE clause returns outputs for DISTINCT processing.
57 int sqlite3WhereIsDistinct(WhereInfo
*pWInfo
){
58 return pWInfo
->eDistinct
;
62 ** Return the number of ORDER BY terms that are satisfied by the
63 ** WHERE clause. A return of 0 means that the output must be
64 ** completely sorted. A return equal to the number of ORDER BY
65 ** terms means that no sorting is needed at all. A return that
66 ** is positive but less than the number of ORDER BY terms means that
67 ** block sorting is required.
69 int sqlite3WhereIsOrdered(WhereInfo
*pWInfo
){
70 return pWInfo
->nOBSat
<0 ? 0 : pWInfo
->nOBSat
;
74 ** In the ORDER BY LIMIT optimization, if the inner-most loop is known
75 ** to emit rows in increasing order, and if the last row emitted by the
76 ** inner-most loop did not fit within the sorter, then we can skip all
77 ** subsequent rows for the current iteration of the inner loop (because they
78 ** will not fit in the sorter either) and continue with the second inner
79 ** loop - the loop immediately outside the inner-most.
81 ** When a row does not fit in the sorter (because the sorter already
82 ** holds LIMIT+OFFSET rows that are smaller), then a jump is made to the
83 ** label returned by this function.
85 ** If the ORDER BY LIMIT optimization applies, the jump destination should
86 ** be the continuation for the second-inner-most loop. If the ORDER BY
87 ** LIMIT optimization does not apply, then the jump destination should
88 ** be the continuation for the inner-most loop.
90 ** It is always safe for this routine to return the continuation of the
91 ** inner-most loop, in the sense that a correct answer will result.
92 ** Returning the continuation the second inner loop is an optimization
93 ** that might make the code run a little faster, but should not change
96 int sqlite3WhereOrderByLimitOptLabel(WhereInfo
*pWInfo
){
98 if( !pWInfo
->bOrderedInnerLoop
){
99 /* The ORDER BY LIMIT optimization does not apply. Jump to the
100 ** continuation of the inner-most loop. */
101 return pWInfo
->iContinue
;
103 pInner
= &pWInfo
->a
[pWInfo
->nLevel
-1];
104 assert( pInner
->addrNxt
!=0 );
105 return pInner
->pRJ
? pWInfo
->iContinue
: pInner
->addrNxt
;
109 ** While generating code for the min/max optimization, after handling
110 ** the aggregate-step call to min() or max(), check to see if any
111 ** additional looping is required. If the output order is such that
112 ** we are certain that the correct answer has already been found, then
113 ** code an OP_Goto to by pass subsequent processing.
115 ** Any extra OP_Goto that is coded here is an optimization. The
116 ** correct answer should be obtained regardless. This OP_Goto just
117 ** makes the answer appear faster.
119 void sqlite3WhereMinMaxOptEarlyOut(Vdbe
*v
, WhereInfo
*pWInfo
){
122 if( !pWInfo
->bOrderedInnerLoop
) return;
123 if( pWInfo
->nOBSat
==0 ) return;
124 for(i
=pWInfo
->nLevel
-1; i
>=0; i
--){
125 pInner
= &pWInfo
->a
[i
];
126 if( (pInner
->pWLoop
->wsFlags
& WHERE_COLUMN_IN
)!=0 ){
127 sqlite3VdbeGoto(v
, pInner
->addrNxt
);
131 sqlite3VdbeGoto(v
, pWInfo
->iBreak
);
135 ** Return the VDBE address or label to jump to in order to continue
136 ** immediately with the next row of a WHERE clause.
138 int sqlite3WhereContinueLabel(WhereInfo
*pWInfo
){
139 assert( pWInfo
->iContinue
!=0 );
140 return pWInfo
->iContinue
;
144 ** Return the VDBE address or label to jump to in order to break
145 ** out of a WHERE loop.
147 int sqlite3WhereBreakLabel(WhereInfo
*pWInfo
){
148 return pWInfo
->iBreak
;
152 ** Return ONEPASS_OFF (0) if an UPDATE or DELETE statement is unable to
153 ** operate directly on the rowids returned by a WHERE clause. Return
154 ** ONEPASS_SINGLE (1) if the statement can operation directly because only
155 ** a single row is to be changed. Return ONEPASS_MULTI (2) if the one-pass
156 ** optimization can be used on multiple
158 ** If the ONEPASS optimization is used (if this routine returns true)
159 ** then also write the indices of open cursors used by ONEPASS
160 ** into aiCur[0] and aiCur[1]. iaCur[0] gets the cursor of the data
161 ** table and iaCur[1] gets the cursor used by an auxiliary index.
162 ** Either value may be -1, indicating that cursor is not used.
163 ** Any cursors returned will have been opened for writing.
165 ** aiCur[0] and aiCur[1] both get -1 if the where-clause logic is
166 ** unable to use the ONEPASS optimization.
168 int sqlite3WhereOkOnePass(WhereInfo
*pWInfo
, int *aiCur
){
169 memcpy(aiCur
, pWInfo
->aiCurOnePass
, sizeof(int)*2);
170 #ifdef WHERETRACE_ENABLED
171 if( sqlite3WhereTrace
&& pWInfo
->eOnePass
!=ONEPASS_OFF
){
172 sqlite3DebugPrintf("%s cursors: %d %d\n",
173 pWInfo
->eOnePass
==ONEPASS_SINGLE
? "ONEPASS_SINGLE" : "ONEPASS_MULTI",
177 return pWInfo
->eOnePass
;
181 ** Return TRUE if the WHERE loop uses the OP_DeferredSeek opcode to move
182 ** the data cursor to the row selected by the index cursor.
184 int sqlite3WhereUsesDeferredSeek(WhereInfo
*pWInfo
){
185 return pWInfo
->bDeferredSeek
;
189 ** Move the content of pSrc into pDest
191 static void whereOrMove(WhereOrSet
*pDest
, WhereOrSet
*pSrc
){
193 memcpy(pDest
->a
, pSrc
->a
, pDest
->n
*sizeof(pDest
->a
[0]));
197 ** Try to insert a new prerequisite/cost entry into the WhereOrSet pSet.
199 ** The new entry might overwrite an existing entry, or it might be
200 ** appended, or it might be discarded. Do whatever is the right thing
201 ** so that pSet keeps the N_OR_COST best entries seen so far.
203 static int whereOrInsert(
204 WhereOrSet
*pSet
, /* The WhereOrSet to be updated */
205 Bitmask prereq
, /* Prerequisites of the new entry */
206 LogEst rRun
, /* Run-cost of the new entry */
207 LogEst nOut
/* Number of outputs for the new entry */
211 for(i
=pSet
->n
, p
=pSet
->a
; i
>0; i
--, p
++){
212 if( rRun
<=p
->rRun
&& (prereq
& p
->prereq
)==prereq
){
213 goto whereOrInsert_done
;
215 if( p
->rRun
<=rRun
&& (p
->prereq
& prereq
)==p
->prereq
){
219 if( pSet
->n
<N_OR_COST
){
220 p
= &pSet
->a
[pSet
->n
++];
224 for(i
=1; i
<pSet
->n
; i
++){
225 if( p
->rRun
>pSet
->a
[i
].rRun
) p
= pSet
->a
+ i
;
227 if( p
->rRun
<=rRun
) return 0;
232 if( p
->nOut
>nOut
) p
->nOut
= nOut
;
237 ** Return the bitmask for the given cursor number. Return 0 if
238 ** iCursor is not in the set.
240 Bitmask
sqlite3WhereGetMask(WhereMaskSet
*pMaskSet
, int iCursor
){
242 assert( pMaskSet
->n
<=(int)sizeof(Bitmask
)*8 );
243 assert( pMaskSet
->n
>0 || pMaskSet
->ix
[0]<0 );
244 assert( iCursor
>=-1 );
245 if( pMaskSet
->ix
[0]==iCursor
){
248 for(i
=1; i
<pMaskSet
->n
; i
++){
249 if( pMaskSet
->ix
[i
]==iCursor
){
256 /* Allocate memory that is automatically freed when pWInfo is freed.
258 void *sqlite3WhereMalloc(WhereInfo
*pWInfo
, u64 nByte
){
259 WhereMemBlock
*pBlock
;
260 pBlock
= sqlite3DbMallocRawNN(pWInfo
->pParse
->db
, nByte
+sizeof(*pBlock
));
262 pBlock
->pNext
= pWInfo
->pMemToFree
;
264 pWInfo
->pMemToFree
= pBlock
;
267 return (void*)pBlock
;
269 void *sqlite3WhereRealloc(WhereInfo
*pWInfo
, void *pOld
, u64 nByte
){
270 void *pNew
= sqlite3WhereMalloc(pWInfo
, nByte
);
272 WhereMemBlock
*pOldBlk
= (WhereMemBlock
*)pOld
;
274 assert( pOldBlk
->sz
<nByte
);
275 memcpy(pNew
, pOld
, pOldBlk
->sz
);
281 ** Create a new mask for cursor iCursor.
283 ** There is one cursor per table in the FROM clause. The number of
284 ** tables in the FROM clause is limited by a test early in the
285 ** sqlite3WhereBegin() routine. So we know that the pMaskSet->ix[]
286 ** array will never overflow.
288 static void createMask(WhereMaskSet
*pMaskSet
, int iCursor
){
289 assert( pMaskSet
->n
< ArraySize(pMaskSet
->ix
) );
290 pMaskSet
->ix
[pMaskSet
->n
++] = iCursor
;
294 ** If the right-hand branch of the expression is a TK_COLUMN, then return
295 ** a pointer to the right-hand branch. Otherwise, return NULL.
297 static Expr
*whereRightSubexprIsColumn(Expr
*p
){
298 p
= sqlite3ExprSkipCollateAndLikely(p
->pRight
);
299 if( ALWAYS(p
!=0) && p
->op
==TK_COLUMN
&& !ExprHasProperty(p
, EP_FixedCol
) ){
306 ** Advance to the next WhereTerm that matches according to the criteria
307 ** established when the pScan object was initialized by whereScanInit().
308 ** Return NULL if there are no more matching WhereTerms.
310 static WhereTerm
*whereScanNext(WhereScan
*pScan
){
311 int iCur
; /* The cursor on the LHS of the term */
312 i16 iColumn
; /* The column on the LHS of the term. -1 for IPK */
313 Expr
*pX
; /* An expression being tested */
314 WhereClause
*pWC
; /* Shorthand for pScan->pWC */
315 WhereTerm
*pTerm
; /* The term being tested */
316 int k
= pScan
->k
; /* Where to start scanning */
318 assert( pScan
->iEquiv
<=pScan
->nEquiv
);
321 iColumn
= pScan
->aiColumn
[pScan
->iEquiv
-1];
322 iCur
= pScan
->aiCur
[pScan
->iEquiv
-1];
326 for(pTerm
=pWC
->a
+k
; k
<pWC
->nTerm
; k
++, pTerm
++){
327 assert( (pTerm
->eOperator
& (WO_OR
|WO_AND
))==0 || pTerm
->leftCursor
<0 );
328 if( pTerm
->leftCursor
==iCur
329 && pTerm
->u
.x
.leftColumn
==iColumn
331 || sqlite3ExprCompareSkip(pTerm
->pExpr
->pLeft
,
332 pScan
->pIdxExpr
,iCur
)==0)
333 && (pScan
->iEquiv
<=1 || !ExprHasProperty(pTerm
->pExpr
, EP_OuterON
))
335 if( (pTerm
->eOperator
& WO_EQUIV
)!=0
336 && pScan
->nEquiv
<ArraySize(pScan
->aiCur
)
337 && (pX
= whereRightSubexprIsColumn(pTerm
->pExpr
))!=0
340 for(j
=0; j
<pScan
->nEquiv
; j
++){
341 if( pScan
->aiCur
[j
]==pX
->iTable
342 && pScan
->aiColumn
[j
]==pX
->iColumn
){
346 if( j
==pScan
->nEquiv
){
347 pScan
->aiCur
[j
] = pX
->iTable
;
348 pScan
->aiColumn
[j
] = pX
->iColumn
;
352 if( (pTerm
->eOperator
& pScan
->opMask
)!=0 ){
353 /* Verify the affinity and collating sequence match */
354 if( pScan
->zCollName
&& (pTerm
->eOperator
& WO_ISNULL
)==0 ){
356 Parse
*pParse
= pWC
->pWInfo
->pParse
;
358 if( !sqlite3IndexAffinityOk(pX
, pScan
->idxaff
) ){
362 pColl
= sqlite3ExprCompareCollSeq(pParse
, pX
);
363 if( pColl
==0 ) pColl
= pParse
->db
->pDfltColl
;
364 if( sqlite3StrICmp(pColl
->zName
, pScan
->zCollName
) ){
368 if( (pTerm
->eOperator
& (WO_EQ
|WO_IS
))!=0
369 && (pX
= pTerm
->pExpr
->pRight
, ALWAYS(pX
!=0))
371 && pX
->iTable
==pScan
->aiCur
[0]
372 && pX
->iColumn
==pScan
->aiColumn
[0]
374 testcase( pTerm
->eOperator
& WO_IS
);
379 #ifdef WHERETRACE_ENABLED
380 if( sqlite3WhereTrace
& 0x20000 ){
382 sqlite3DebugPrintf("SCAN-TERM %p: nEquiv=%d",
383 pTerm
, pScan
->nEquiv
);
384 for(ii
=0; ii
<pScan
->nEquiv
; ii
++){
385 sqlite3DebugPrintf(" {%d:%d}",
386 pScan
->aiCur
[ii
], pScan
->aiColumn
[ii
]);
388 sqlite3DebugPrintf("\n");
398 if( pScan
->iEquiv
>=pScan
->nEquiv
) break;
399 pWC
= pScan
->pOrigWC
;
407 ** This is whereScanInit() for the case of an index on an expression.
408 ** It is factored out into a separate tail-recursion subroutine so that
409 ** the normal whereScanInit() routine, which is a high-runner, does not
410 ** need to push registers onto the stack as part of its prologue.
412 static SQLITE_NOINLINE WhereTerm
*whereScanInitIndexExpr(WhereScan
*pScan
){
413 pScan
->idxaff
= sqlite3ExprAffinity(pScan
->pIdxExpr
);
414 return whereScanNext(pScan
);
418 ** Initialize a WHERE clause scanner object. Return a pointer to the
419 ** first match. Return NULL if there are no matches.
421 ** The scanner will be searching the WHERE clause pWC. It will look
422 ** for terms of the form "X <op> <expr>" where X is column iColumn of table
423 ** iCur. Or if pIdx!=0 then X is column iColumn of index pIdx. pIdx
424 ** must be one of the indexes of table iCur.
426 ** The <op> must be one of the operators described by opMask.
428 ** If the search is for X and the WHERE clause contains terms of the
429 ** form X=Y then this routine might also return terms of the form
430 ** "Y <op> <expr>". The number of levels of transitivity is limited,
431 ** but is enough to handle most commonly occurring SQL statements.
433 ** If X is not the INTEGER PRIMARY KEY then X must be compatible with
436 static WhereTerm
*whereScanInit(
437 WhereScan
*pScan
, /* The WhereScan object being initialized */
438 WhereClause
*pWC
, /* The WHERE clause to be scanned */
439 int iCur
, /* Cursor to scan for */
440 int iColumn
, /* Column to scan for */
441 u32 opMask
, /* Operator(s) to scan for */
442 Index
*pIdx
/* Must be compatible with this index */
444 pScan
->pOrigWC
= pWC
;
448 pScan
->zCollName
= 0;
449 pScan
->opMask
= opMask
;
451 pScan
->aiCur
[0] = iCur
;
456 iColumn
= pIdx
->aiColumn
[j
];
457 if( iColumn
==pIdx
->pTable
->iPKey
){
459 }else if( iColumn
>=0 ){
460 pScan
->idxaff
= pIdx
->pTable
->aCol
[iColumn
].affinity
;
461 pScan
->zCollName
= pIdx
->azColl
[j
];
462 }else if( iColumn
==XN_EXPR
){
463 pScan
->pIdxExpr
= pIdx
->aColExpr
->a
[j
].pExpr
;
464 pScan
->zCollName
= pIdx
->azColl
[j
];
465 pScan
->aiColumn
[0] = XN_EXPR
;
466 return whereScanInitIndexExpr(pScan
);
468 }else if( iColumn
==XN_EXPR
){
471 pScan
->aiColumn
[0] = iColumn
;
472 return whereScanNext(pScan
);
476 ** Search for a term in the WHERE clause that is of the form "X <op> <expr>"
477 ** where X is a reference to the iColumn of table iCur or of index pIdx
478 ** if pIdx!=0 and <op> is one of the WO_xx operator codes specified by
479 ** the op parameter. Return a pointer to the term. Return 0 if not found.
481 ** If pIdx!=0 then it must be one of the indexes of table iCur.
482 ** Search for terms matching the iColumn-th column of pIdx
483 ** rather than the iColumn-th column of table iCur.
485 ** The term returned might by Y=<expr> if there is another constraint in
486 ** the WHERE clause that specifies that X=Y. Any such constraints will be
487 ** identified by the WO_EQUIV bit in the pTerm->eOperator field. The
488 ** aiCur[]/iaColumn[] arrays hold X and all its equivalents. There are 11
489 ** slots in aiCur[]/aiColumn[] so that means we can look for X plus up to 10
490 ** other equivalent values. Hence a search for X will return <expr> if X=A1
491 ** and A1=A2 and A2=A3 and ... and A9=A10 and A10=<expr>.
493 ** If there are multiple terms in the WHERE clause of the form "X <op> <expr>"
494 ** then try for the one with no dependencies on <expr> - in other words where
495 ** <expr> is a constant expression of some kind. Only return entries of
496 ** the form "X <op> Y" where Y is a column in another table if no terms of
497 ** the form "X <op> <const-expr>" exist. If no terms with a constant RHS
498 ** exist, try to return a term that does not use WO_EQUIV.
500 WhereTerm
*sqlite3WhereFindTerm(
501 WhereClause
*pWC
, /* The WHERE clause to be searched */
502 int iCur
, /* Cursor number of LHS */
503 int iColumn
, /* Column number of LHS */
504 Bitmask notReady
, /* RHS must not overlap with this mask */
505 u32 op
, /* Mask of WO_xx values describing operator */
506 Index
*pIdx
/* Must be compatible with this index, if not NULL */
508 WhereTerm
*pResult
= 0;
512 p
= whereScanInit(&scan
, pWC
, iCur
, iColumn
, op
, pIdx
);
515 if( (p
->prereqRight
& notReady
)==0 ){
516 if( p
->prereqRight
==0 && (p
->eOperator
&op
)!=0 ){
517 testcase( p
->eOperator
& WO_IS
);
520 if( pResult
==0 ) pResult
= p
;
522 p
= whereScanNext(&scan
);
528 ** This function searches pList for an entry that matches the iCol-th column
531 ** If such an expression is found, its index in pList->a[] is returned. If
532 ** no expression is found, -1 is returned.
534 static int findIndexCol(
535 Parse
*pParse
, /* Parse context */
536 ExprList
*pList
, /* Expression list to search */
537 int iBase
, /* Cursor for table associated with pIdx */
538 Index
*pIdx
, /* Index to match column of */
539 int iCol
/* Column of index to match */
542 const char *zColl
= pIdx
->azColl
[iCol
];
544 for(i
=0; i
<pList
->nExpr
; i
++){
545 Expr
*p
= sqlite3ExprSkipCollateAndLikely(pList
->a
[i
].pExpr
);
547 && (p
->op
==TK_COLUMN
|| p
->op
==TK_AGG_COLUMN
)
548 && p
->iColumn
==pIdx
->aiColumn
[iCol
]
551 CollSeq
*pColl
= sqlite3ExprNNCollSeq(pParse
, pList
->a
[i
].pExpr
);
552 if( 0==sqlite3StrICmp(pColl
->zName
, zColl
) ){
562 ** Return TRUE if the iCol-th column of index pIdx is NOT NULL
564 static int indexColumnNotNull(Index
*pIdx
, int iCol
){
567 assert( iCol
>=0 && iCol
<pIdx
->nColumn
);
568 j
= pIdx
->aiColumn
[iCol
];
570 return pIdx
->pTable
->aCol
[j
].notNull
;
575 return 0; /* Assume an indexed expression can always yield a NULL */
581 ** Return true if the DISTINCT expression-list passed as the third argument
584 ** A DISTINCT list is redundant if any subset of the columns in the
585 ** DISTINCT list are collectively unique and individually non-null.
587 static int isDistinctRedundant(
588 Parse
*pParse
, /* Parsing context */
589 SrcList
*pTabList
, /* The FROM clause */
590 WhereClause
*pWC
, /* The WHERE clause */
591 ExprList
*pDistinct
/* The result set that needs to be DISTINCT */
598 /* If there is more than one table or sub-select in the FROM clause of
599 ** this query, then it will not be possible to show that the DISTINCT
600 ** clause is redundant. */
601 if( pTabList
->nSrc
!=1 ) return 0;
602 iBase
= pTabList
->a
[0].iCursor
;
603 pTab
= pTabList
->a
[0].pTab
;
605 /* If any of the expressions is an IPK column on table iBase, then return
606 ** true. Note: The (p->iTable==iBase) part of this test may be false if the
607 ** current SELECT is a correlated sub-query.
609 for(i
=0; i
<pDistinct
->nExpr
; i
++){
610 Expr
*p
= sqlite3ExprSkipCollateAndLikely(pDistinct
->a
[i
].pExpr
);
611 if( NEVER(p
==0) ) continue;
612 if( p
->op
!=TK_COLUMN
&& p
->op
!=TK_AGG_COLUMN
) continue;
613 if( p
->iTable
==iBase
&& p
->iColumn
<0 ) return 1;
616 /* Loop through all indices on the table, checking each to see if it makes
617 ** the DISTINCT qualifier redundant. It does so if:
619 ** 1. The index is itself UNIQUE, and
621 ** 2. All of the columns in the index are either part of the pDistinct
622 ** list, or else the WHERE clause contains a term of the form "col=X",
623 ** where X is a constant value. The collation sequences of the
624 ** comparison and select-list expressions must match those of the index.
626 ** 3. All of those index columns for which the WHERE clause does not
627 ** contain a "col=X" term are subject to a NOT NULL constraint.
629 for(pIdx
=pTab
->pIndex
; pIdx
; pIdx
=pIdx
->pNext
){
630 if( !IsUniqueIndex(pIdx
) ) continue;
631 if( pIdx
->pPartIdxWhere
) continue;
632 for(i
=0; i
<pIdx
->nKeyCol
; i
++){
633 if( 0==sqlite3WhereFindTerm(pWC
, iBase
, i
, ~(Bitmask
)0, WO_EQ
, pIdx
) ){
634 if( findIndexCol(pParse
, pDistinct
, iBase
, pIdx
, i
)<0 ) break;
635 if( indexColumnNotNull(pIdx
, i
)==0 ) break;
638 if( i
==pIdx
->nKeyCol
){
639 /* This index implies that the DISTINCT qualifier is redundant. */
649 ** Estimate the logarithm of the input value to base 2.
651 static LogEst
estLog(LogEst N
){
652 return N
<=10 ? 0 : sqlite3LogEst(N
) - 33;
656 ** Convert OP_Column opcodes to OP_Copy in previously generated code.
658 ** This routine runs over generated VDBE code and translates OP_Column
659 ** opcodes into OP_Copy when the table is being accessed via co-routine
660 ** instead of via table lookup.
662 ** If the iAutoidxCur is not zero, then any OP_Rowid instructions on
663 ** cursor iTabCur are transformed into OP_Sequence opcode for the
664 ** iAutoidxCur cursor, in order to generate unique rowids for the
665 ** automatic index being generated.
667 static void translateColumnToCopy(
668 Parse
*pParse
, /* Parsing context */
669 int iStart
, /* Translate from this opcode to the end */
670 int iTabCur
, /* OP_Column/OP_Rowid references to this table */
671 int iRegister
, /* The first column is in this register */
672 int iAutoidxCur
/* If non-zero, cursor of autoindex being generated */
674 Vdbe
*v
= pParse
->pVdbe
;
675 VdbeOp
*pOp
= sqlite3VdbeGetOp(v
, iStart
);
676 int iEnd
= sqlite3VdbeCurrentAddr(v
);
677 if( pParse
->db
->mallocFailed
) return;
678 for(; iStart
<iEnd
; iStart
++, pOp
++){
679 if( pOp
->p1
!=iTabCur
) continue;
680 if( pOp
->opcode
==OP_Column
){
681 pOp
->opcode
= OP_Copy
;
682 pOp
->p1
= pOp
->p2
+ iRegister
;
685 pOp
->p5
= 2; /* Cause the MEM_Subtype flag to be cleared */
686 }else if( pOp
->opcode
==OP_Rowid
){
687 pOp
->opcode
= OP_Sequence
;
688 pOp
->p1
= iAutoidxCur
;
689 #ifdef SQLITE_ALLOW_ROWID_IN_VIEW
690 if( iAutoidxCur
==0 ){
691 pOp
->opcode
= OP_Null
;
700 ** Two routines for printing the content of an sqlite3_index_info
701 ** structure. Used for testing and debugging only. If neither
702 ** SQLITE_TEST or SQLITE_DEBUG are defined, then these routines
705 #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED)
706 static void whereTraceIndexInfoInputs(sqlite3_index_info
*p
){
708 if( (sqlite3WhereTrace
& 0x10)==0 ) return;
709 for(i
=0; i
<p
->nConstraint
; i
++){
711 " constraint[%d]: col=%d termid=%d op=%d usabled=%d collseq=%s\n",
713 p
->aConstraint
[i
].iColumn
,
714 p
->aConstraint
[i
].iTermOffset
,
715 p
->aConstraint
[i
].op
,
716 p
->aConstraint
[i
].usable
,
717 sqlite3_vtab_collation(p
,i
));
719 for(i
=0; i
<p
->nOrderBy
; i
++){
720 sqlite3DebugPrintf(" orderby[%d]: col=%d desc=%d\n",
722 p
->aOrderBy
[i
].iColumn
,
723 p
->aOrderBy
[i
].desc
);
726 static void whereTraceIndexInfoOutputs(sqlite3_index_info
*p
){
728 if( (sqlite3WhereTrace
& 0x10)==0 ) return;
729 for(i
=0; i
<p
->nConstraint
; i
++){
730 sqlite3DebugPrintf(" usage[%d]: argvIdx=%d omit=%d\n",
732 p
->aConstraintUsage
[i
].argvIndex
,
733 p
->aConstraintUsage
[i
].omit
);
735 sqlite3DebugPrintf(" idxNum=%d\n", p
->idxNum
);
736 sqlite3DebugPrintf(" idxStr=%s\n", p
->idxStr
);
737 sqlite3DebugPrintf(" orderByConsumed=%d\n", p
->orderByConsumed
);
738 sqlite3DebugPrintf(" estimatedCost=%g\n", p
->estimatedCost
);
739 sqlite3DebugPrintf(" estimatedRows=%lld\n", p
->estimatedRows
);
742 #define whereTraceIndexInfoInputs(A)
743 #define whereTraceIndexInfoOutputs(A)
747 ** We know that pSrc is an operand of an outer join. Return true if
748 ** pTerm is a constraint that is compatible with that join.
750 ** pTerm must be EP_OuterON if pSrc is the right operand of an
751 ** outer join. pTerm can be either EP_OuterON or EP_InnerON if pSrc
752 ** is the left operand of a RIGHT join.
754 ** See https://sqlite.org/forum/forumpost/206d99a16dd9212f
755 ** for an example of a WHERE clause constraints that may not be used on
756 ** the right table of a RIGHT JOIN because the constraint implies a
757 ** not-NULL condition on the left table of the RIGHT JOIN.
759 static int constraintCompatibleWithOuterJoin(
760 const WhereTerm
*pTerm
, /* WHERE clause term to check */
761 const SrcItem
*pSrc
/* Table we are trying to access */
763 assert( (pSrc
->fg
.jointype
&(JT_LEFT
|JT_LTORJ
|JT_RIGHT
))!=0 ); /* By caller */
764 testcase( (pSrc
->fg
.jointype
& (JT_LEFT
|JT_LTORJ
|JT_RIGHT
))==JT_LEFT
);
765 testcase( (pSrc
->fg
.jointype
& (JT_LEFT
|JT_LTORJ
|JT_RIGHT
))==JT_LTORJ
);
766 testcase( ExprHasProperty(pTerm
->pExpr
, EP_OuterON
) )
767 testcase( ExprHasProperty(pTerm
->pExpr
, EP_InnerON
) );
768 if( !ExprHasProperty(pTerm
->pExpr
, EP_OuterON
|EP_InnerON
)
769 || pTerm
->pExpr
->w
.iJoin
!= pSrc
->iCursor
773 if( (pSrc
->fg
.jointype
& (JT_LEFT
|JT_RIGHT
))!=0
774 && ExprHasProperty(pTerm
->pExpr
, EP_InnerON
)
783 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
785 ** Return TRUE if the WHERE clause term pTerm is of a form where it
786 ** could be used with an index to access pSrc, assuming an appropriate
789 static int termCanDriveIndex(
790 const WhereTerm
*pTerm
, /* WHERE clause term to check */
791 const SrcItem
*pSrc
, /* Table we are trying to access */
792 const Bitmask notReady
/* Tables in outer loops of the join */
795 if( pTerm
->leftCursor
!=pSrc
->iCursor
) return 0;
796 if( (pTerm
->eOperator
& (WO_EQ
|WO_IS
))==0 ) return 0;
797 assert( (pSrc
->fg
.jointype
& JT_RIGHT
)==0 );
798 if( (pSrc
->fg
.jointype
& (JT_LEFT
|JT_LTORJ
|JT_RIGHT
))!=0
799 && !constraintCompatibleWithOuterJoin(pTerm
,pSrc
)
801 return 0; /* See https://sqlite.org/forum/forumpost/51e6959f61 */
803 if( (pTerm
->prereqRight
& notReady
)!=0 ) return 0;
804 assert( (pTerm
->eOperator
& (WO_OR
|WO_AND
))==0 );
805 if( pTerm
->u
.x
.leftColumn
<0 ) return 0;
806 aff
= pSrc
->pTab
->aCol
[pTerm
->u
.x
.leftColumn
].affinity
;
807 if( !sqlite3IndexAffinityOk(pTerm
->pExpr
, aff
) ) return 0;
808 testcase( pTerm
->pExpr
->op
==TK_IS
);
814 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
816 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
818 ** Argument pIdx represents an automatic index that the current statement
819 ** will create and populate. Add an OP_Explain with text of the form:
821 ** CREATE AUTOMATIC INDEX ON <table>(<cols>) [WHERE <expr>]
823 ** This is only required if sqlite3_stmt_scanstatus() is enabled, to
824 ** associate an SQLITE_SCANSTAT_NCYCLE and SQLITE_SCANSTAT_NLOOP
825 ** values with. In order to avoid breaking legacy code and test cases,
826 ** the OP_Explain is not added if this is an EXPLAIN QUERY PLAN command.
828 static void explainAutomaticIndex(
830 Index
*pIdx
, /* Automatic index to explain */
831 int bPartial
, /* True if pIdx is a partial index */
832 int *pAddrExplain
/* OUT: Address of OP_Explain */
834 if( IS_STMT_SCANSTATUS(pParse
->db
) && pParse
->explain
!=2 ){
835 Table
*pTab
= pIdx
->pTable
;
836 const char *zSep
= "";
839 sqlite3_str
*pStr
= sqlite3_str_new(pParse
->db
);
840 sqlite3_str_appendf(pStr
,"CREATE AUTOMATIC INDEX ON %s(", pTab
->zName
);
841 assert( pIdx
->nColumn
>1 );
842 assert( pIdx
->aiColumn
[pIdx
->nColumn
-1]==XN_ROWID
);
843 for(ii
=0; ii
<(pIdx
->nColumn
-1); ii
++){
844 const char *zName
= 0;
845 int iCol
= pIdx
->aiColumn
[ii
];
847 zName
= pTab
->aCol
[iCol
].zCnName
;
848 sqlite3_str_appendf(pStr
, "%s%s", zSep
, zName
);
851 zText
= sqlite3_str_finish(pStr
);
853 sqlite3OomFault(pParse
->db
);
855 *pAddrExplain
= sqlite3VdbeExplain(
856 pParse
, 0, "%s)%s", zText
, (bPartial
? " WHERE <expr>" : "")
863 # define explainAutomaticIndex(a,b,c,d)
867 ** Generate code to construct the Index object for an automatic index
868 ** and to set up the WhereLevel object pLevel so that the code generator
869 ** makes use of the automatic index.
871 static SQLITE_NOINLINE
void constructAutomaticIndex(
872 Parse
*pParse
, /* The parsing context */
873 WhereClause
*pWC
, /* The WHERE clause */
874 const Bitmask notReady
, /* Mask of cursors that are not available */
875 WhereLevel
*pLevel
/* Write new index here */
877 int nKeyCol
; /* Number of columns in the constructed index */
878 WhereTerm
*pTerm
; /* A single term of the WHERE clause */
879 WhereTerm
*pWCEnd
; /* End of pWC->a[] */
880 Index
*pIdx
; /* Object describing the transient index */
881 Vdbe
*v
; /* Prepared statement under construction */
882 int addrInit
; /* Address of the initialization bypass jump */
883 Table
*pTable
; /* The table being indexed */
884 int addrTop
; /* Top of the index fill loop */
885 int regRecord
; /* Register holding an index record */
886 int n
; /* Column counter */
887 int i
; /* Loop counter */
888 int mxBitCol
; /* Maximum column in pSrc->colUsed */
889 CollSeq
*pColl
; /* Collating sequence to on a column */
890 WhereLoop
*pLoop
; /* The Loop object */
891 char *zNotUsed
; /* Extra space on the end of pIdx */
892 Bitmask idxCols
; /* Bitmap of columns used for indexing */
893 Bitmask extraCols
; /* Bitmap of additional columns */
894 u8 sentWarning
= 0; /* True if a warning has been issued */
895 u8 useBloomFilter
= 0; /* True to also add a Bloom filter */
896 Expr
*pPartial
= 0; /* Partial Index Expression */
897 int iContinue
= 0; /* Jump here to skip excluded rows */
898 SrcList
*pTabList
; /* The complete FROM clause */
899 SrcItem
*pSrc
; /* The FROM clause term to get the next index */
900 int addrCounter
= 0; /* Address where integer counter is initialized */
901 int regBase
; /* Array of registers where record is assembled */
902 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
903 int addrExp
= 0; /* Address of OP_Explain */
906 /* Generate code to skip over the creation and initialization of the
907 ** transient index on 2nd and subsequent iterations of the loop. */
910 addrInit
= sqlite3VdbeAddOp0(v
, OP_Once
); VdbeCoverage(v
);
912 /* Count the number of columns that will be added to the index
913 ** and used to match WHERE clause constraints */
915 pTabList
= pWC
->pWInfo
->pTabList
;
916 pSrc
= &pTabList
->a
[pLevel
->iFrom
];
918 pWCEnd
= &pWC
->a
[pWC
->nTerm
];
919 pLoop
= pLevel
->pWLoop
;
921 for(pTerm
=pWC
->a
; pTerm
<pWCEnd
; pTerm
++){
922 Expr
*pExpr
= pTerm
->pExpr
;
923 /* Make the automatic index a partial index if there are terms in the
924 ** WHERE clause (or the ON clause of a LEFT join) that constrain which
925 ** rows of the target table (pSrc) that can be used. */
926 if( (pTerm
->wtFlags
& TERM_VIRTUAL
)==0
927 && sqlite3ExprIsSingleTableConstraint(pExpr
, pTabList
, pLevel
->iFrom
)
929 pPartial
= sqlite3ExprAnd(pParse
, pPartial
,
930 sqlite3ExprDup(pParse
->db
, pExpr
, 0));
932 if( termCanDriveIndex(pTerm
, pSrc
, notReady
) ){
935 assert( (pTerm
->eOperator
& (WO_OR
|WO_AND
))==0 );
936 iCol
= pTerm
->u
.x
.leftColumn
;
937 cMask
= iCol
>=BMS
? MASKBIT(BMS
-1) : MASKBIT(iCol
);
938 testcase( iCol
==BMS
);
939 testcase( iCol
==BMS
-1 );
941 sqlite3_log(SQLITE_WARNING_AUTOINDEX
,
942 "automatic index on %s(%s)", pTable
->zName
,
943 pTable
->aCol
[iCol
].zCnName
);
946 if( (idxCols
& cMask
)==0 ){
947 if( whereLoopResize(pParse
->db
, pLoop
, nKeyCol
+1) ){
948 goto end_auto_index_create
;
950 pLoop
->aLTerm
[nKeyCol
++] = pTerm
;
955 assert( nKeyCol
>0 || pParse
->db
->mallocFailed
);
956 pLoop
->u
.btree
.nEq
= pLoop
->nLTerm
= nKeyCol
;
957 pLoop
->wsFlags
= WHERE_COLUMN_EQ
| WHERE_IDX_ONLY
| WHERE_INDEXED
960 /* Count the number of additional columns needed to create a
961 ** covering index. A "covering index" is an index that contains all
962 ** columns that are needed by the query. With a covering index, the
963 ** original table never needs to be accessed. Automatic indices must
964 ** be a covering index because the index will not be updated if the
965 ** original table changes and the index and table cannot both be used
966 ** if they go out of sync.
968 if( IsView(pTable
) ){
971 extraCols
= pSrc
->colUsed
& (~idxCols
| MASKBIT(BMS
-1));
973 mxBitCol
= MIN(BMS
-1,pTable
->nCol
);
974 testcase( pTable
->nCol
==BMS
-1 );
975 testcase( pTable
->nCol
==BMS
-2 );
976 for(i
=0; i
<mxBitCol
; i
++){
977 if( extraCols
& MASKBIT(i
) ) nKeyCol
++;
979 if( pSrc
->colUsed
& MASKBIT(BMS
-1) ){
980 nKeyCol
+= pTable
->nCol
- BMS
+ 1;
983 /* Construct the Index object to describe this index */
984 pIdx
= sqlite3AllocateIndexObject(pParse
->db
, nKeyCol
+1, 0, &zNotUsed
);
985 if( pIdx
==0 ) goto end_auto_index_create
;
986 pLoop
->u
.btree
.pIndex
= pIdx
;
987 pIdx
->zName
= "auto-index";
988 pIdx
->pTable
= pTable
;
991 for(pTerm
=pWC
->a
; pTerm
<pWCEnd
; pTerm
++){
992 if( termCanDriveIndex(pTerm
, pSrc
, notReady
) ){
995 assert( (pTerm
->eOperator
& (WO_OR
|WO_AND
))==0 );
996 iCol
= pTerm
->u
.x
.leftColumn
;
997 cMask
= iCol
>=BMS
? MASKBIT(BMS
-1) : MASKBIT(iCol
);
998 testcase( iCol
==BMS
-1 );
999 testcase( iCol
==BMS
);
1000 if( (idxCols
& cMask
)==0 ){
1001 Expr
*pX
= pTerm
->pExpr
;
1003 pIdx
->aiColumn
[n
] = pTerm
->u
.x
.leftColumn
;
1004 pColl
= sqlite3ExprCompareCollSeq(pParse
, pX
);
1005 assert( pColl
!=0 || pParse
->nErr
>0 ); /* TH3 collate01.800 */
1006 pIdx
->azColl
[n
] = pColl
? pColl
->zName
: sqlite3StrBINARY
;
1008 if( ALWAYS(pX
->pLeft
!=0)
1009 && sqlite3ExprAffinity(pX
->pLeft
)!=SQLITE_AFF_TEXT
1011 /* TUNING: only use a Bloom filter on an automatic index
1012 ** if one or more key columns has the ability to hold numeric
1013 ** values, since strings all have the same hash in the Bloom
1014 ** filter implementation and hence a Bloom filter on a text column
1015 ** is not usually helpful. */
1021 assert( (u32
)n
==pLoop
->u
.btree
.nEq
);
1023 /* Add additional columns needed to make the automatic index into
1024 ** a covering index */
1025 for(i
=0; i
<mxBitCol
; i
++){
1026 if( extraCols
& MASKBIT(i
) ){
1027 pIdx
->aiColumn
[n
] = i
;
1028 pIdx
->azColl
[n
] = sqlite3StrBINARY
;
1032 if( pSrc
->colUsed
& MASKBIT(BMS
-1) ){
1033 for(i
=BMS
-1; i
<pTable
->nCol
; i
++){
1034 pIdx
->aiColumn
[n
] = i
;
1035 pIdx
->azColl
[n
] = sqlite3StrBINARY
;
1039 assert( n
==nKeyCol
);
1040 pIdx
->aiColumn
[n
] = XN_ROWID
;
1041 pIdx
->azColl
[n
] = sqlite3StrBINARY
;
1043 /* Create the automatic index */
1044 explainAutomaticIndex(pParse
, pIdx
, pPartial
!=0, &addrExp
);
1045 assert( pLevel
->iIdxCur
>=0 );
1046 pLevel
->iIdxCur
= pParse
->nTab
++;
1047 sqlite3VdbeAddOp2(v
, OP_OpenAutoindex
, pLevel
->iIdxCur
, nKeyCol
+1);
1048 sqlite3VdbeSetP4KeyInfo(pParse
, pIdx
);
1049 VdbeComment((v
, "for %s", pTable
->zName
));
1050 if( OptimizationEnabled(pParse
->db
, SQLITE_BloomFilter
) && useBloomFilter
){
1051 sqlite3WhereExplainBloomFilter(pParse
, pWC
->pWInfo
, pLevel
);
1052 pLevel
->regFilter
= ++pParse
->nMem
;
1053 sqlite3VdbeAddOp2(v
, OP_Blob
, 10000, pLevel
->regFilter
);
1056 /* Fill the automatic index with content */
1057 assert( pSrc
== &pWC
->pWInfo
->pTabList
->a
[pLevel
->iFrom
] );
1058 if( pSrc
->fg
.viaCoroutine
){
1059 int regYield
= pSrc
->regReturn
;
1060 addrCounter
= sqlite3VdbeAddOp2(v
, OP_Integer
, 0, 0);
1061 sqlite3VdbeAddOp3(v
, OP_InitCoroutine
, regYield
, 0, pSrc
->addrFillSub
);
1062 addrTop
= sqlite3VdbeAddOp1(v
, OP_Yield
, regYield
);
1064 VdbeComment((v
, "next row of %s", pSrc
->pTab
->zName
));
1066 addrTop
= sqlite3VdbeAddOp1(v
, OP_Rewind
, pLevel
->iTabCur
); VdbeCoverage(v
);
1069 iContinue
= sqlite3VdbeMakeLabel(pParse
);
1070 sqlite3ExprIfFalse(pParse
, pPartial
, iContinue
, SQLITE_JUMPIFNULL
);
1071 pLoop
->wsFlags
|= WHERE_PARTIALIDX
;
1073 regRecord
= sqlite3GetTempReg(pParse
);
1074 regBase
= sqlite3GenerateIndexKey(
1075 pParse
, pIdx
, pLevel
->iTabCur
, regRecord
, 0, 0, 0, 0
1077 if( pLevel
->regFilter
){
1078 sqlite3VdbeAddOp4Int(v
, OP_FilterAdd
, pLevel
->regFilter
, 0,
1079 regBase
, pLoop
->u
.btree
.nEq
);
1081 sqlite3VdbeScanStatusCounters(v
, addrExp
, addrExp
, sqlite3VdbeCurrentAddr(v
));
1082 sqlite3VdbeAddOp2(v
, OP_IdxInsert
, pLevel
->iIdxCur
, regRecord
);
1083 sqlite3VdbeChangeP5(v
, OPFLAG_USESEEKRESULT
);
1084 if( pPartial
) sqlite3VdbeResolveLabel(v
, iContinue
);
1085 if( pSrc
->fg
.viaCoroutine
){
1086 sqlite3VdbeChangeP2(v
, addrCounter
, regBase
+n
);
1087 testcase( pParse
->db
->mallocFailed
);
1088 assert( pLevel
->iIdxCur
>0 );
1089 translateColumnToCopy(pParse
, addrTop
, pLevel
->iTabCur
,
1090 pSrc
->regResult
, pLevel
->iIdxCur
);
1091 sqlite3VdbeGoto(v
, addrTop
);
1092 pSrc
->fg
.viaCoroutine
= 0;
1094 sqlite3VdbeAddOp2(v
, OP_Next
, pLevel
->iTabCur
, addrTop
+1); VdbeCoverage(v
);
1095 sqlite3VdbeChangeP5(v
, SQLITE_STMTSTATUS_AUTOINDEX
);
1097 sqlite3VdbeJumpHere(v
, addrTop
);
1098 sqlite3ReleaseTempReg(pParse
, regRecord
);
1100 /* Jump here when skipping the initialization */
1101 sqlite3VdbeJumpHere(v
, addrInit
);
1102 sqlite3VdbeScanStatusRange(v
, addrExp
, addrExp
, -1);
1104 end_auto_index_create
:
1105 sqlite3ExprDelete(pParse
->db
, pPartial
);
1107 #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */
1110 ** Generate bytecode that will initialize a Bloom filter that is appropriate
1113 ** If there are inner loops within pLevel that have the WHERE_BLOOMFILTER
1114 ** flag set, initialize a Bloomfilter for them as well. Except don't do
1115 ** this recursive initialization if the SQLITE_BloomPulldown optimization has
1118 ** When the Bloom filter is initialized, the WHERE_BLOOMFILTER flag is cleared
1119 ** from the loop, but the regFilter value is set to a register that implements
1120 ** the Bloom filter. When regFilter is positive, the
1121 ** sqlite3WhereCodeOneLoopStart() will generate code to test the Bloom filter
1122 ** and skip the subsequence B-Tree seek if the Bloom filter indicates that
1123 ** no matching rows exist.
1125 ** This routine may only be called if it has previously been determined that
1126 ** the loop would benefit from a Bloom filter, and the WHERE_BLOOMFILTER bit
1129 static SQLITE_NOINLINE
void sqlite3ConstructBloomFilter(
1130 WhereInfo
*pWInfo
, /* The WHERE clause */
1131 int iLevel
, /* Index in pWInfo->a[] that is pLevel */
1132 WhereLevel
*pLevel
, /* Make a Bloom filter for this FROM term */
1133 Bitmask notReady
/* Loops that are not ready */
1135 int addrOnce
; /* Address of opening OP_Once */
1136 int addrTop
; /* Address of OP_Rewind */
1137 int addrCont
; /* Jump here to skip a row */
1138 const WhereTerm
*pTerm
; /* For looping over WHERE clause terms */
1139 const WhereTerm
*pWCEnd
; /* Last WHERE clause term */
1140 Parse
*pParse
= pWInfo
->pParse
; /* Parsing context */
1141 Vdbe
*v
= pParse
->pVdbe
; /* VDBE under construction */
1142 WhereLoop
*pLoop
= pLevel
->pWLoop
; /* The loop being coded */
1143 int iCur
; /* Cursor for table getting the filter */
1144 IndexedExpr
*saved_pIdxEpr
; /* saved copy of Parse.pIdxEpr */
1146 saved_pIdxEpr
= pParse
->pIdxEpr
;
1147 pParse
->pIdxEpr
= 0;
1151 assert( pLoop
->wsFlags
& WHERE_BLOOMFILTER
);
1152 assert( (pLoop
->wsFlags
& WHERE_IDX_ONLY
)==0 );
1154 addrOnce
= sqlite3VdbeAddOp0(v
, OP_Once
); VdbeCoverage(v
);
1156 const SrcList
*pTabList
;
1157 const SrcItem
*pItem
;
1161 sqlite3WhereExplainBloomFilter(pParse
, pWInfo
, pLevel
);
1162 addrCont
= sqlite3VdbeMakeLabel(pParse
);
1163 iCur
= pLevel
->iTabCur
;
1164 pLevel
->regFilter
= ++pParse
->nMem
;
1166 /* The Bloom filter is a Blob held in a register. Initialize it
1167 ** to zero-filled blob of at least 80K bits, but maybe more if the
1168 ** estimated size of the table is larger. We could actually
1169 ** measure the size of the table at run-time using OP_Count with
1170 ** P3==1 and use that value to initialize the blob. But that makes
1171 ** testing complicated. By basing the blob size on the value in the
1172 ** sqlite_stat1 table, testing is much easier.
1174 pTabList
= pWInfo
->pTabList
;
1175 iSrc
= pLevel
->iFrom
;
1176 pItem
= &pTabList
->a
[iSrc
];
1180 sz
= sqlite3LogEstToInt(pTab
->nRowLogEst
);
1183 }else if( sz
>10000000 ){
1186 sqlite3VdbeAddOp2(v
, OP_Blob
, (int)sz
, pLevel
->regFilter
);
1188 addrTop
= sqlite3VdbeAddOp1(v
, OP_Rewind
, iCur
); VdbeCoverage(v
);
1189 pWCEnd
= &pWInfo
->sWC
.a
[pWInfo
->sWC
.nTerm
];
1190 for(pTerm
=pWInfo
->sWC
.a
; pTerm
<pWCEnd
; pTerm
++){
1191 Expr
*pExpr
= pTerm
->pExpr
;
1192 if( (pTerm
->wtFlags
& TERM_VIRTUAL
)==0
1193 && sqlite3ExprIsSingleTableConstraint(pExpr
, pTabList
, iSrc
)
1195 sqlite3ExprIfFalse(pParse
, pTerm
->pExpr
, addrCont
, SQLITE_JUMPIFNULL
);
1198 if( pLoop
->wsFlags
& WHERE_IPK
){
1199 int r1
= sqlite3GetTempReg(pParse
);
1200 sqlite3VdbeAddOp2(v
, OP_Rowid
, iCur
, r1
);
1201 sqlite3VdbeAddOp4Int(v
, OP_FilterAdd
, pLevel
->regFilter
, 0, r1
, 1);
1202 sqlite3ReleaseTempReg(pParse
, r1
);
1204 Index
*pIdx
= pLoop
->u
.btree
.pIndex
;
1205 int n
= pLoop
->u
.btree
.nEq
;
1206 int r1
= sqlite3GetTempRange(pParse
, n
);
1208 for(jj
=0; jj
<n
; jj
++){
1209 assert( pIdx
->pTable
==pItem
->pTab
);
1210 sqlite3ExprCodeLoadIndexColumn(pParse
, pIdx
, iCur
, jj
, r1
+jj
);
1212 sqlite3VdbeAddOp4Int(v
, OP_FilterAdd
, pLevel
->regFilter
, 0, r1
, n
);
1213 sqlite3ReleaseTempRange(pParse
, r1
, n
);
1215 sqlite3VdbeResolveLabel(v
, addrCont
);
1216 sqlite3VdbeAddOp2(v
, OP_Next
, pLevel
->iTabCur
, addrTop
+1);
1218 sqlite3VdbeJumpHere(v
, addrTop
);
1219 pLoop
->wsFlags
&= ~WHERE_BLOOMFILTER
;
1220 if( OptimizationDisabled(pParse
->db
, SQLITE_BloomPulldown
) ) break;
1221 while( ++iLevel
< pWInfo
->nLevel
){
1222 const SrcItem
*pTabItem
;
1223 pLevel
= &pWInfo
->a
[iLevel
];
1224 pTabItem
= &pWInfo
->pTabList
->a
[pLevel
->iFrom
];
1225 if( pTabItem
->fg
.jointype
& (JT_LEFT
|JT_LTORJ
) ) continue;
1226 pLoop
= pLevel
->pWLoop
;
1227 if( NEVER(pLoop
==0) ) continue;
1228 if( pLoop
->prereq
& notReady
) continue;
1229 if( (pLoop
->wsFlags
& (WHERE_BLOOMFILTER
|WHERE_COLUMN_IN
))
1232 /* This is a candidate for bloom-filter pull-down (early evaluation).
1233 ** The test that WHERE_COLUMN_IN is omitted is important, as we are
1234 ** not able to do early evaluation of bloom filters that make use of
1235 ** the IN operator */
1239 }while( iLevel
< pWInfo
->nLevel
);
1240 sqlite3VdbeJumpHere(v
, addrOnce
);
1241 pParse
->pIdxEpr
= saved_pIdxEpr
;
1245 #ifndef SQLITE_OMIT_VIRTUALTABLE
1247 ** Allocate and populate an sqlite3_index_info structure. It is the
1248 ** responsibility of the caller to eventually release the structure
1249 ** by passing the pointer returned by this function to freeIndexInfo().
1251 static sqlite3_index_info
*allocateIndexInfo(
1252 WhereInfo
*pWInfo
, /* The WHERE clause */
1253 WhereClause
*pWC
, /* The WHERE clause being analyzed */
1254 Bitmask mUnusable
, /* Ignore terms with these prereqs */
1255 SrcItem
*pSrc
, /* The FROM clause term that is the vtab */
1256 u16
*pmNoOmit
/* Mask of terms not to omit */
1260 Parse
*pParse
= pWInfo
->pParse
;
1261 struct sqlite3_index_constraint
*pIdxCons
;
1262 struct sqlite3_index_orderby
*pIdxOrderBy
;
1263 struct sqlite3_index_constraint_usage
*pUsage
;
1264 struct HiddenIndexInfo
*pHidden
;
1267 sqlite3_index_info
*pIdxInfo
;
1271 ExprList
*pOrderBy
= pWInfo
->pOrderBy
;
1276 assert( IsVirtual(pTab
) );
1278 /* Find all WHERE clause constraints referring to this virtual table.
1279 ** Mark each term with the TERM_OK flag. Set nTerm to the number of
1282 for(i
=nTerm
=0, pTerm
=pWC
->a
; i
<pWC
->nTerm
; i
++, pTerm
++){
1283 pTerm
->wtFlags
&= ~TERM_OK
;
1284 if( pTerm
->leftCursor
!= pSrc
->iCursor
) continue;
1285 if( pTerm
->prereqRight
& mUnusable
) continue;
1286 assert( IsPowerOfTwo(pTerm
->eOperator
& ~WO_EQUIV
) );
1287 testcase( pTerm
->eOperator
& WO_IN
);
1288 testcase( pTerm
->eOperator
& WO_ISNULL
);
1289 testcase( pTerm
->eOperator
& WO_IS
);
1290 testcase( pTerm
->eOperator
& WO_ALL
);
1291 if( (pTerm
->eOperator
& ~(WO_EQUIV
))==0 ) continue;
1292 if( pTerm
->wtFlags
& TERM_VNULL
) continue;
1294 assert( (pTerm
->eOperator
& (WO_OR
|WO_AND
))==0 );
1295 assert( pTerm
->u
.x
.leftColumn
>=XN_ROWID
);
1296 assert( pTerm
->u
.x
.leftColumn
<pTab
->nCol
);
1297 if( (pSrc
->fg
.jointype
& (JT_LEFT
|JT_LTORJ
|JT_RIGHT
))!=0
1298 && !constraintCompatibleWithOuterJoin(pTerm
,pSrc
)
1303 pTerm
->wtFlags
|= TERM_OK
;
1306 /* If the ORDER BY clause contains only columns in the current
1307 ** virtual table then allocate space for the aOrderBy part of
1308 ** the sqlite3_index_info structure.
1312 int n
= pOrderBy
->nExpr
;
1314 Expr
*pExpr
= pOrderBy
->a
[i
].pExpr
;
1317 /* Skip over constant terms in the ORDER BY clause */
1318 if( sqlite3ExprIsConstant(pExpr
) ){
1322 /* Virtual tables are unable to deal with NULLS FIRST */
1323 if( pOrderBy
->a
[i
].fg
.sortFlags
& KEYINFO_ORDER_BIGNULL
) break;
1325 /* First case - a direct column references without a COLLATE operator */
1326 if( pExpr
->op
==TK_COLUMN
&& pExpr
->iTable
==pSrc
->iCursor
){
1327 assert( pExpr
->iColumn
>=XN_ROWID
&& pExpr
->iColumn
<pTab
->nCol
);
1331 /* 2nd case - a column reference with a COLLATE operator. Only match
1332 ** of the COLLATE operator matches the collation of the column. */
1333 if( pExpr
->op
==TK_COLLATE
1334 && (pE2
= pExpr
->pLeft
)->op
==TK_COLUMN
1335 && pE2
->iTable
==pSrc
->iCursor
1337 const char *zColl
; /* The collating sequence name */
1338 assert( !ExprHasProperty(pExpr
, EP_IntValue
) );
1339 assert( pExpr
->u
.zToken
!=0 );
1340 assert( pE2
->iColumn
>=XN_ROWID
&& pE2
->iColumn
<pTab
->nCol
);
1341 pExpr
->iColumn
= pE2
->iColumn
;
1342 if( pE2
->iColumn
<0 ) continue; /* Collseq does not matter for rowid */
1343 zColl
= sqlite3ColumnColl(&pTab
->aCol
[pE2
->iColumn
]);
1344 if( zColl
==0 ) zColl
= sqlite3StrBINARY
;
1345 if( sqlite3_stricmp(pExpr
->u
.zToken
, zColl
)==0 ) continue;
1348 /* No matches cause a break out of the loop */
1353 if( (pWInfo
->wctrlFlags
& WHERE_DISTINCTBY
) ){
1354 eDistinct
= 2 + ((pWInfo
->wctrlFlags
& WHERE_SORTBYGROUP
)!=0);
1355 }else if( pWInfo
->wctrlFlags
& WHERE_GROUPBY
){
1361 /* Allocate the sqlite3_index_info structure
1363 pIdxInfo
= sqlite3DbMallocZero(pParse
->db
, sizeof(*pIdxInfo
)
1364 + (sizeof(*pIdxCons
) + sizeof(*pUsage
))*nTerm
1365 + sizeof(*pIdxOrderBy
)*nOrderBy
+ sizeof(*pHidden
)
1366 + sizeof(sqlite3_value
*)*nTerm
);
1368 sqlite3ErrorMsg(pParse
, "out of memory");
1371 pHidden
= (struct HiddenIndexInfo
*)&pIdxInfo
[1];
1372 pIdxCons
= (struct sqlite3_index_constraint
*)&pHidden
->aRhs
[nTerm
];
1373 pIdxOrderBy
= (struct sqlite3_index_orderby
*)&pIdxCons
[nTerm
];
1374 pUsage
= (struct sqlite3_index_constraint_usage
*)&pIdxOrderBy
[nOrderBy
];
1375 pIdxInfo
->aConstraint
= pIdxCons
;
1376 pIdxInfo
->aOrderBy
= pIdxOrderBy
;
1377 pIdxInfo
->aConstraintUsage
= pUsage
;
1379 pHidden
->pParse
= pParse
;
1380 pHidden
->eDistinct
= eDistinct
;
1382 for(i
=j
=0, pTerm
=pWC
->a
; i
<pWC
->nTerm
; i
++, pTerm
++){
1384 if( (pTerm
->wtFlags
& TERM_OK
)==0 ) continue;
1385 pIdxCons
[j
].iColumn
= pTerm
->u
.x
.leftColumn
;
1386 pIdxCons
[j
].iTermOffset
= i
;
1387 op
= pTerm
->eOperator
& WO_ALL
;
1389 if( (pTerm
->wtFlags
& TERM_SLICE
)==0 ){
1390 pHidden
->mIn
|= SMASKBIT32(j
);
1395 pIdxCons
[j
].op
= pTerm
->eMatchOp
;
1396 }else if( op
& (WO_ISNULL
|WO_IS
) ){
1397 if( op
==WO_ISNULL
){
1398 pIdxCons
[j
].op
= SQLITE_INDEX_CONSTRAINT_ISNULL
;
1400 pIdxCons
[j
].op
= SQLITE_INDEX_CONSTRAINT_IS
;
1403 pIdxCons
[j
].op
= (u8
)op
;
1404 /* The direct assignment in the previous line is possible only because
1405 ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The
1406 ** following asserts verify this fact. */
1407 assert( WO_EQ
==SQLITE_INDEX_CONSTRAINT_EQ
);
1408 assert( WO_LT
==SQLITE_INDEX_CONSTRAINT_LT
);
1409 assert( WO_LE
==SQLITE_INDEX_CONSTRAINT_LE
);
1410 assert( WO_GT
==SQLITE_INDEX_CONSTRAINT_GT
);
1411 assert( WO_GE
==SQLITE_INDEX_CONSTRAINT_GE
);
1412 assert( pTerm
->eOperator
&(WO_IN
|WO_EQ
|WO_LT
|WO_LE
|WO_GT
|WO_GE
|WO_AUX
) );
1414 if( op
& (WO_LT
|WO_LE
|WO_GT
|WO_GE
)
1415 && sqlite3ExprIsVector(pTerm
->pExpr
->pRight
)
1418 if( j
<16 ) mNoOmit
|= (1 << j
);
1419 if( op
==WO_LT
) pIdxCons
[j
].op
= WO_LE
;
1420 if( op
==WO_GT
) pIdxCons
[j
].op
= WO_GE
;
1427 pIdxInfo
->nConstraint
= j
;
1428 for(i
=j
=0; i
<nOrderBy
; i
++){
1429 Expr
*pExpr
= pOrderBy
->a
[i
].pExpr
;
1430 if( sqlite3ExprIsConstant(pExpr
) ) continue;
1431 assert( pExpr
->op
==TK_COLUMN
1432 || (pExpr
->op
==TK_COLLATE
&& pExpr
->pLeft
->op
==TK_COLUMN
1433 && pExpr
->iColumn
==pExpr
->pLeft
->iColumn
) );
1434 pIdxOrderBy
[j
].iColumn
= pExpr
->iColumn
;
1435 pIdxOrderBy
[j
].desc
= pOrderBy
->a
[i
].fg
.sortFlags
& KEYINFO_ORDER_DESC
;
1438 pIdxInfo
->nOrderBy
= j
;
1440 *pmNoOmit
= mNoOmit
;
1445 ** Free an sqlite3_index_info structure allocated by allocateIndexInfo()
1446 ** and possibly modified by xBestIndex methods.
1448 static void freeIndexInfo(sqlite3
*db
, sqlite3_index_info
*pIdxInfo
){
1449 HiddenIndexInfo
*pHidden
;
1451 assert( pIdxInfo
!=0 );
1452 pHidden
= (HiddenIndexInfo
*)&pIdxInfo
[1];
1453 assert( pHidden
->pParse
!=0 );
1454 assert( pHidden
->pParse
->db
==db
);
1455 for(i
=0; i
<pIdxInfo
->nConstraint
; i
++){
1456 sqlite3ValueFree(pHidden
->aRhs
[i
]); /* IMP: R-14553-25174 */
1457 pHidden
->aRhs
[i
] = 0;
1459 sqlite3DbFree(db
, pIdxInfo
);
1463 ** The table object reference passed as the second argument to this function
1464 ** must represent a virtual table. This function invokes the xBestIndex()
1465 ** method of the virtual table with the sqlite3_index_info object that
1466 ** comes in as the 3rd argument to this function.
1468 ** If an error occurs, pParse is populated with an error message and an
1469 ** appropriate error code is returned. A return of SQLITE_CONSTRAINT from
1470 ** xBestIndex is not considered an error. SQLITE_CONSTRAINT indicates that
1471 ** the current configuration of "unusable" flags in sqlite3_index_info can
1472 ** not result in a valid plan.
1474 ** Whether or not an error is returned, it is the responsibility of the
1475 ** caller to eventually free p->idxStr if p->needToFreeIdxStr indicates
1476 ** that this is required.
1478 static int vtabBestIndex(Parse
*pParse
, Table
*pTab
, sqlite3_index_info
*p
){
1479 sqlite3_vtab
*pVtab
= sqlite3GetVTable(pParse
->db
, pTab
)->pVtab
;
1482 whereTraceIndexInfoInputs(p
);
1483 pParse
->db
->nSchemaLock
++;
1484 rc
= pVtab
->pModule
->xBestIndex(pVtab
, p
);
1485 pParse
->db
->nSchemaLock
--;
1486 whereTraceIndexInfoOutputs(p
);
1488 if( rc
!=SQLITE_OK
&& rc
!=SQLITE_CONSTRAINT
){
1489 if( rc
==SQLITE_NOMEM
){
1490 sqlite3OomFault(pParse
->db
);
1491 }else if( !pVtab
->zErrMsg
){
1492 sqlite3ErrorMsg(pParse
, "%s", sqlite3ErrStr(rc
));
1494 sqlite3ErrorMsg(pParse
, "%s", pVtab
->zErrMsg
);
1497 if( pTab
->u
.vtab
.p
->bAllSchemas
){
1498 sqlite3VtabUsesAllSchemas(pParse
);
1500 sqlite3_free(pVtab
->zErrMsg
);
1504 #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */
1506 #ifdef SQLITE_ENABLE_STAT4
1508 ** Estimate the location of a particular key among all keys in an
1509 ** index. Store the results in aStat as follows:
1511 ** aStat[0] Est. number of rows less than pRec
1512 ** aStat[1] Est. number of rows equal to pRec
1514 ** Return the index of the sample that is the smallest sample that
1515 ** is greater than or equal to pRec. Note that this index is not an index
1516 ** into the aSample[] array - it is an index into a virtual set of samples
1517 ** based on the contents of aSample[] and the number of fields in record
1520 static int whereKeyStats(
1521 Parse
*pParse
, /* Database connection */
1522 Index
*pIdx
, /* Index to consider domain of */
1523 UnpackedRecord
*pRec
, /* Vector of values to consider */
1524 int roundUp
, /* Round up if true. Round down if false */
1525 tRowcnt
*aStat
/* OUT: stats written here */
1527 IndexSample
*aSample
= pIdx
->aSample
;
1528 int iCol
; /* Index of required stats in anEq[] etc. */
1529 int i
; /* Index of first sample >= pRec */
1530 int iSample
; /* Smallest sample larger than or equal to pRec */
1531 int iMin
= 0; /* Smallest sample not yet tested */
1532 int iTest
; /* Next sample to test */
1533 int res
; /* Result of comparison operation */
1534 int nField
; /* Number of fields in pRec */
1535 tRowcnt iLower
= 0; /* anLt[] + anEq[] of largest sample pRec is > */
1537 #ifndef SQLITE_DEBUG
1538 UNUSED_PARAMETER( pParse
);
1541 assert( pIdx
->nSample
>0 );
1542 assert( pRec
->nField
>0 );
1545 /* Do a binary search to find the first sample greater than or equal
1546 ** to pRec. If pRec contains a single field, the set of samples to search
1547 ** is simply the aSample[] array. If the samples in aSample[] contain more
1548 ** than one fields, all fields following the first are ignored.
1550 ** If pRec contains N fields, where N is more than one, then as well as the
1551 ** samples in aSample[] (truncated to N fields), the search also has to
1552 ** consider prefixes of those samples. For example, if the set of samples
1555 ** aSample[0] = (a, 5)
1556 ** aSample[1] = (a, 10)
1557 ** aSample[2] = (b, 5)
1558 ** aSample[3] = (c, 100)
1559 ** aSample[4] = (c, 105)
1561 ** Then the search space should ideally be the samples above and the
1562 ** unique prefixes [a], [b] and [c]. But since that is hard to organize,
1563 ** the code actually searches this set:
1576 ** For each sample in the aSample[] array, N samples are present in the
1577 ** effective sample array. In the above, samples 0 and 1 are based on
1578 ** sample aSample[0]. Samples 2 and 3 on aSample[1] etc.
1580 ** Often, sample i of each block of N effective samples has (i+1) fields.
1581 ** Except, each sample may be extended to ensure that it is greater than or
1582 ** equal to the previous sample in the array. For example, in the above,
1583 ** sample 2 is the first sample of a block of N samples, so at first it
1584 ** appears that it should be 1 field in size. However, that would make it
1585 ** smaller than sample 1, so the binary search would not work. As a result,
1586 ** it is extended to two fields. The duplicates that this creates do not
1587 ** cause any problems.
1589 if( !HasRowid(pIdx
->pTable
) && IsPrimaryKeyIndex(pIdx
) ){
1590 nField
= pIdx
->nKeyCol
;
1592 nField
= pIdx
->nColumn
;
1594 nField
= MIN(pRec
->nField
, nField
);
1596 iSample
= pIdx
->nSample
* nField
;
1598 int iSamp
; /* Index in aSample[] of test sample */
1599 int n
; /* Number of fields in test sample */
1601 iTest
= (iMin
+iSample
)/2;
1602 iSamp
= iTest
/ nField
;
1604 /* The proposed effective sample is a prefix of sample aSample[iSamp].
1605 ** Specifically, the shortest prefix of at least (1 + iTest%nField)
1606 ** fields that is greater than the previous effective sample. */
1607 for(n
=(iTest
% nField
) + 1; n
<nField
; n
++){
1608 if( aSample
[iSamp
-1].anLt
[n
-1]!=aSample
[iSamp
].anLt
[n
-1] ) break;
1615 res
= sqlite3VdbeRecordCompare(aSample
[iSamp
].n
, aSample
[iSamp
].p
, pRec
);
1617 iLower
= aSample
[iSamp
].anLt
[n
-1] + aSample
[iSamp
].anEq
[n
-1];
1619 }else if( res
==0 && n
<nField
){
1620 iLower
= aSample
[iSamp
].anLt
[n
-1];
1627 }while( res
&& iMin
<iSample
);
1628 i
= iSample
/ nField
;
1631 /* The following assert statements check that the binary search code
1632 ** above found the right answer. This block serves no purpose other
1633 ** than to invoke the asserts. */
1634 if( pParse
->db
->mallocFailed
==0 ){
1636 /* If (res==0) is true, then pRec must be equal to sample i. */
1637 assert( i
<pIdx
->nSample
);
1638 assert( iCol
==nField
-1 );
1639 pRec
->nField
= nField
;
1640 assert( 0==sqlite3VdbeRecordCompare(aSample
[i
].n
, aSample
[i
].p
, pRec
)
1641 || pParse
->db
->mallocFailed
1644 /* Unless i==pIdx->nSample, indicating that pRec is larger than
1645 ** all samples in the aSample[] array, pRec must be smaller than the
1646 ** (iCol+1) field prefix of sample i. */
1647 assert( i
<=pIdx
->nSample
&& i
>=0 );
1648 pRec
->nField
= iCol
+1;
1649 assert( i
==pIdx
->nSample
1650 || sqlite3VdbeRecordCompare(aSample
[i
].n
, aSample
[i
].p
, pRec
)>0
1651 || pParse
->db
->mallocFailed
);
1653 /* if i==0 and iCol==0, then record pRec is smaller than all samples
1654 ** in the aSample[] array. Otherwise, if (iCol>0) then pRec must
1655 ** be greater than or equal to the (iCol) field prefix of sample i.
1656 ** If (i>0), then pRec must also be greater than sample (i-1). */
1658 pRec
->nField
= iCol
;
1659 assert( sqlite3VdbeRecordCompare(aSample
[i
].n
, aSample
[i
].p
, pRec
)<=0
1660 || pParse
->db
->mallocFailed
|| CORRUPT_DB
);
1663 pRec
->nField
= nField
;
1664 assert( sqlite3VdbeRecordCompare(aSample
[i
-1].n
, aSample
[i
-1].p
, pRec
)<0
1665 || pParse
->db
->mallocFailed
|| CORRUPT_DB
);
1669 #endif /* ifdef SQLITE_DEBUG */
1672 /* Record pRec is equal to sample i */
1673 assert( iCol
==nField
-1 );
1674 aStat
[0] = aSample
[i
].anLt
[iCol
];
1675 aStat
[1] = aSample
[i
].anEq
[iCol
];
1677 /* At this point, the (iCol+1) field prefix of aSample[i] is the first
1678 ** sample that is greater than pRec. Or, if i==pIdx->nSample then pRec
1679 ** is larger than all samples in the array. */
1680 tRowcnt iUpper
, iGap
;
1681 if( i
>=pIdx
->nSample
){
1682 iUpper
= pIdx
->nRowEst0
;
1684 iUpper
= aSample
[i
].anLt
[iCol
];
1687 if( iLower
>=iUpper
){
1690 iGap
= iUpper
- iLower
;
1697 aStat
[0] = iLower
+ iGap
;
1698 aStat
[1] = pIdx
->aAvgEq
[nField
-1];
1701 /* Restore the pRec->nField value before returning. */
1702 pRec
->nField
= nField
;
1705 #endif /* SQLITE_ENABLE_STAT4 */
1708 ** If it is not NULL, pTerm is a term that provides an upper or lower
1709 ** bound on a range scan. Without considering pTerm, it is estimated
1710 ** that the scan will visit nNew rows. This function returns the number
1711 ** estimated to be visited after taking pTerm into account.
1713 ** If the user explicitly specified a likelihood() value for this term,
1714 ** then the return value is the likelihood multiplied by the number of
1715 ** input rows. Otherwise, this function assumes that an "IS NOT NULL" term
1716 ** has a likelihood of 0.50, and any other term a likelihood of 0.25.
1718 static LogEst
whereRangeAdjust(WhereTerm
*pTerm
, LogEst nNew
){
1721 if( pTerm
->truthProb
<=0 ){
1722 nRet
+= pTerm
->truthProb
;
1723 }else if( (pTerm
->wtFlags
& TERM_VNULL
)==0 ){
1724 nRet
-= 20; assert( 20==sqlite3LogEst(4) );
1731 #ifdef SQLITE_ENABLE_STAT4
1733 ** Return the affinity for a single column of an index.
1735 char sqlite3IndexColumnAffinity(sqlite3
*db
, Index
*pIdx
, int iCol
){
1736 assert( iCol
>=0 && iCol
<pIdx
->nColumn
);
1737 if( !pIdx
->zColAff
){
1738 if( sqlite3IndexAffinityStr(db
, pIdx
)==0 ) return SQLITE_AFF_BLOB
;
1740 assert( pIdx
->zColAff
[iCol
]!=0 );
1741 return pIdx
->zColAff
[iCol
];
1746 #ifdef SQLITE_ENABLE_STAT4
1748 ** This function is called to estimate the number of rows visited by a
1749 ** range-scan on a skip-scan index. For example:
1751 ** CREATE INDEX i1 ON t1(a, b, c);
1752 ** SELECT * FROM t1 WHERE a=? AND c BETWEEN ? AND ?;
1754 ** Value pLoop->nOut is currently set to the estimated number of rows
1755 ** visited for scanning (a=? AND b=?). This function reduces that estimate
1756 ** by some factor to account for the (c BETWEEN ? AND ?) expression based
1757 ** on the stat4 data for the index. this scan will be performed multiple
1758 ** times (once for each (a,b) combination that matches a=?) is dealt with
1761 ** It does this by scanning through all stat4 samples, comparing values
1762 ** extracted from pLower and pUpper with the corresponding column in each
1763 ** sample. If L and U are the number of samples found to be less than or
1764 ** equal to the values extracted from pLower and pUpper respectively, and
1765 ** N is the total number of samples, the pLoop->nOut value is adjusted
1768 ** nOut = nOut * ( min(U - L, 1) / N )
1770 ** If pLower is NULL, or a value cannot be extracted from the term, L is
1771 ** set to zero. If pUpper is NULL, or a value cannot be extracted from it,
1774 ** Normally, this function sets *pbDone to 1 before returning. However,
1775 ** if no value can be extracted from either pLower or pUpper (and so the
1776 ** estimate of the number of rows delivered remains unchanged), *pbDone
1779 ** If an error occurs, an SQLite error code is returned. Otherwise,
1782 static int whereRangeSkipScanEst(
1783 Parse
*pParse
, /* Parsing & code generating context */
1784 WhereTerm
*pLower
, /* Lower bound on the range. ex: "x>123" Might be NULL */
1785 WhereTerm
*pUpper
, /* Upper bound on the range. ex: "x<455" Might be NULL */
1786 WhereLoop
*pLoop
, /* Update the .nOut value of this loop */
1787 int *pbDone
/* Set to true if at least one expr. value extracted */
1789 Index
*p
= pLoop
->u
.btree
.pIndex
;
1790 int nEq
= pLoop
->u
.btree
.nEq
;
1791 sqlite3
*db
= pParse
->db
;
1793 int nUpper
= p
->nSample
+1;
1795 u8 aff
= sqlite3IndexColumnAffinity(db
, p
, nEq
);
1798 sqlite3_value
*p1
= 0; /* Value extracted from pLower */
1799 sqlite3_value
*p2
= 0; /* Value extracted from pUpper */
1800 sqlite3_value
*pVal
= 0; /* Value extracted from record */
1802 pColl
= sqlite3LocateCollSeq(pParse
, p
->azColl
[nEq
]);
1804 rc
= sqlite3Stat4ValueFromExpr(pParse
, pLower
->pExpr
->pRight
, aff
, &p1
);
1807 if( pUpper
&& rc
==SQLITE_OK
){
1808 rc
= sqlite3Stat4ValueFromExpr(pParse
, pUpper
->pExpr
->pRight
, aff
, &p2
);
1809 nUpper
= p2
? 0 : p
->nSample
;
1815 for(i
=0; rc
==SQLITE_OK
&& i
<p
->nSample
; i
++){
1816 rc
= sqlite3Stat4Column(db
, p
->aSample
[i
].p
, p
->aSample
[i
].n
, nEq
, &pVal
);
1817 if( rc
==SQLITE_OK
&& p1
){
1818 int res
= sqlite3MemCompare(p1
, pVal
, pColl
);
1819 if( res
>=0 ) nLower
++;
1821 if( rc
==SQLITE_OK
&& p2
){
1822 int res
= sqlite3MemCompare(p2
, pVal
, pColl
);
1823 if( res
>=0 ) nUpper
++;
1826 nDiff
= (nUpper
- nLower
);
1827 if( nDiff
<=0 ) nDiff
= 1;
1829 /* If there is both an upper and lower bound specified, and the
1830 ** comparisons indicate that they are close together, use the fallback
1831 ** method (assume that the scan visits 1/64 of the rows) for estimating
1832 ** the number of rows visited. Otherwise, estimate the number of rows
1833 ** using the method described in the header comment for this function. */
1834 if( nDiff
!=1 || pUpper
==0 || pLower
==0 ){
1835 int nAdjust
= (sqlite3LogEst(p
->nSample
) - sqlite3LogEst(nDiff
));
1836 pLoop
->nOut
-= nAdjust
;
1838 WHERETRACE(0x20, ("range skip-scan regions: %u..%u adjust=%d est=%d\n",
1839 nLower
, nUpper
, nAdjust
*-1, pLoop
->nOut
));
1843 assert( *pbDone
==0 );
1846 sqlite3ValueFree(p1
);
1847 sqlite3ValueFree(p2
);
1848 sqlite3ValueFree(pVal
);
1852 #endif /* SQLITE_ENABLE_STAT4 */
1855 ** This function is used to estimate the number of rows that will be visited
1856 ** by scanning an index for a range of values. The range may have an upper
1857 ** bound, a lower bound, or both. The WHERE clause terms that set the upper
1858 ** and lower bounds are represented by pLower and pUpper respectively. For
1859 ** example, assuming that index p is on t1(a):
1861 ** ... FROM t1 WHERE a > ? AND a < ? ...
1866 ** If either of the upper or lower bound is not present, then NULL is passed in
1867 ** place of the corresponding WhereTerm.
1869 ** The value in (pBuilder->pNew->u.btree.nEq) is the number of the index
1870 ** column subject to the range constraint. Or, equivalently, the number of
1871 ** equality constraints optimized by the proposed index scan. For example,
1872 ** assuming index p is on t1(a, b), and the SQL query is:
1874 ** ... FROM t1 WHERE a = ? AND b > ? AND b < ? ...
1876 ** then nEq is set to 1 (as the range restricted column, b, is the second
1877 ** left-most column of the index). Or, if the query is:
1879 ** ... FROM t1 WHERE a > ? AND a < ? ...
1881 ** then nEq is set to 0.
1883 ** When this function is called, *pnOut is set to the sqlite3LogEst() of the
1884 ** number of rows that the index scan is expected to visit without
1885 ** considering the range constraints. If nEq is 0, then *pnOut is the number of
1886 ** rows in the index. Assuming no error occurs, *pnOut is adjusted (reduced)
1887 ** to account for the range constraints pLower and pUpper.
1889 ** In the absence of sqlite_stat4 ANALYZE data, or if such data cannot be
1890 ** used, a single range inequality reduces the search space by a factor of 4.
1891 ** and a pair of constraints (x>? AND x<?) reduces the expected number of
1892 ** rows visited by a factor of 64.
1894 static int whereRangeScanEst(
1895 Parse
*pParse
, /* Parsing & code generating context */
1896 WhereLoopBuilder
*pBuilder
,
1897 WhereTerm
*pLower
, /* Lower bound on the range. ex: "x>123" Might be NULL */
1898 WhereTerm
*pUpper
, /* Upper bound on the range. ex: "x<455" Might be NULL */
1899 WhereLoop
*pLoop
/* Modify the .nOut and maybe .rRun fields */
1902 int nOut
= pLoop
->nOut
;
1905 #ifdef SQLITE_ENABLE_STAT4
1906 Index
*p
= pLoop
->u
.btree
.pIndex
;
1907 int nEq
= pLoop
->u
.btree
.nEq
;
1909 if( p
->nSample
>0 && ALWAYS(nEq
<p
->nSampleCol
)
1910 && OptimizationEnabled(pParse
->db
, SQLITE_Stat4
)
1912 if( nEq
==pBuilder
->nRecValid
){
1913 UnpackedRecord
*pRec
= pBuilder
->pRec
;
1915 int nBtm
= pLoop
->u
.btree
.nBtm
;
1916 int nTop
= pLoop
->u
.btree
.nTop
;
1918 /* Variable iLower will be set to the estimate of the number of rows in
1919 ** the index that are less than the lower bound of the range query. The
1920 ** lower bound being the concatenation of $P and $L, where $P is the
1921 ** key-prefix formed by the nEq values matched against the nEq left-most
1922 ** columns of the index, and $L is the value in pLower.
1924 ** Or, if pLower is NULL or $L cannot be extracted from it (because it
1925 ** is not a simple variable or literal value), the lower bound of the
1926 ** range is $P. Due to a quirk in the way whereKeyStats() works, even
1927 ** if $L is available, whereKeyStats() is called for both ($P) and
1928 ** ($P:$L) and the larger of the two returned values is used.
1930 ** Similarly, iUpper is to be set to the estimate of the number of rows
1931 ** less than the upper bound of the range query. Where the upper bound
1932 ** is either ($P) or ($P:$U). Again, even if $U is available, both values
1933 ** of iUpper are requested of whereKeyStats() and the smaller used.
1935 ** The number of rows between the two bounds is then just iUpper-iLower.
1937 tRowcnt iLower
; /* Rows less than the lower bound */
1938 tRowcnt iUpper
; /* Rows less than the upper bound */
1939 int iLwrIdx
= -2; /* aSample[] for the lower bound */
1940 int iUprIdx
= -1; /* aSample[] for the upper bound */
1943 testcase( pRec
->nField
!=pBuilder
->nRecValid
);
1944 pRec
->nField
= pBuilder
->nRecValid
;
1946 /* Determine iLower and iUpper using ($P) only. */
1949 iUpper
= p
->nRowEst0
;
1951 /* Note: this call could be optimized away - since the same values must
1952 ** have been requested when testing key $P in whereEqualScanEst(). */
1953 whereKeyStats(pParse
, p
, pRec
, 0, a
);
1955 iUpper
= a
[0] + a
[1];
1958 assert( pLower
==0 || (pLower
->eOperator
& (WO_GT
|WO_GE
))!=0 );
1959 assert( pUpper
==0 || (pUpper
->eOperator
& (WO_LT
|WO_LE
))!=0 );
1960 assert( p
->aSortOrder
!=0 );
1961 if( p
->aSortOrder
[nEq
] ){
1962 /* The roles of pLower and pUpper are swapped for a DESC index */
1963 SWAP(WhereTerm
*, pLower
, pUpper
);
1964 SWAP(int, nBtm
, nTop
);
1967 /* If possible, improve on the iLower estimate using ($P:$L). */
1969 int n
; /* Values extracted from pExpr */
1970 Expr
*pExpr
= pLower
->pExpr
->pRight
;
1971 rc
= sqlite3Stat4ProbeSetValue(pParse
, p
, &pRec
, pExpr
, nBtm
, nEq
, &n
);
1972 if( rc
==SQLITE_OK
&& n
){
1974 u16 mask
= WO_GT
|WO_LE
;
1975 if( sqlite3ExprVectorSize(pExpr
)>n
) mask
= (WO_LE
|WO_LT
);
1976 iLwrIdx
= whereKeyStats(pParse
, p
, pRec
, 0, a
);
1977 iNew
= a
[0] + ((pLower
->eOperator
& mask
) ? a
[1] : 0);
1978 if( iNew
>iLower
) iLower
= iNew
;
1984 /* If possible, improve on the iUpper estimate using ($P:$U). */
1986 int n
; /* Values extracted from pExpr */
1987 Expr
*pExpr
= pUpper
->pExpr
->pRight
;
1988 rc
= sqlite3Stat4ProbeSetValue(pParse
, p
, &pRec
, pExpr
, nTop
, nEq
, &n
);
1989 if( rc
==SQLITE_OK
&& n
){
1991 u16 mask
= WO_GT
|WO_LE
;
1992 if( sqlite3ExprVectorSize(pExpr
)>n
) mask
= (WO_LE
|WO_LT
);
1993 iUprIdx
= whereKeyStats(pParse
, p
, pRec
, 1, a
);
1994 iNew
= a
[0] + ((pUpper
->eOperator
& mask
) ? a
[1] : 0);
1995 if( iNew
<iUpper
) iUpper
= iNew
;
2001 pBuilder
->pRec
= pRec
;
2002 if( rc
==SQLITE_OK
){
2003 if( iUpper
>iLower
){
2004 nNew
= sqlite3LogEst(iUpper
- iLower
);
2005 /* TUNING: If both iUpper and iLower are derived from the same
2006 ** sample, then assume they are 4x more selective. This brings
2007 ** the estimated selectivity more in line with what it would be
2008 ** if estimated without the use of STAT4 tables. */
2009 if( iLwrIdx
==iUprIdx
) nNew
-= 20; assert( 20==sqlite3LogEst(4) );
2011 nNew
= 10; assert( 10==sqlite3LogEst(2) );
2016 WHERETRACE(0x20, ("STAT4 range scan: %u..%u est=%d\n",
2017 (u32
)iLower
, (u32
)iUpper
, nOut
));
2021 rc
= whereRangeSkipScanEst(pParse
, pLower
, pUpper
, pLoop
, &bDone
);
2022 if( bDone
) return rc
;
2026 UNUSED_PARAMETER(pParse
);
2027 UNUSED_PARAMETER(pBuilder
);
2028 assert( pLower
|| pUpper
);
2030 assert( pUpper
==0 || (pUpper
->wtFlags
& TERM_VNULL
)==0 || pParse
->nErr
>0 );
2031 nNew
= whereRangeAdjust(pLower
, nOut
);
2032 nNew
= whereRangeAdjust(pUpper
, nNew
);
2034 /* TUNING: If there is both an upper and lower limit and neither limit
2035 ** has an application-defined likelihood(), assume the range is
2036 ** reduced by an additional 75%. This means that, by default, an open-ended
2037 ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the
2038 ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to
2039 ** match 1/64 of the index. */
2040 if( pLower
&& pLower
->truthProb
>0 && pUpper
&& pUpper
->truthProb
>0 ){
2044 nOut
-= (pLower
!=0) + (pUpper
!=0);
2045 if( nNew
<10 ) nNew
= 10;
2046 if( nNew
<nOut
) nOut
= nNew
;
2047 #if defined(WHERETRACE_ENABLED)
2048 if( pLoop
->nOut
>nOut
){
2049 WHERETRACE(0x20,("Range scan lowers nOut from %d to %d\n",
2050 pLoop
->nOut
, nOut
));
2053 pLoop
->nOut
= (LogEst
)nOut
;
2057 #ifdef SQLITE_ENABLE_STAT4
2059 ** Estimate the number of rows that will be returned based on
2060 ** an equality constraint x=VALUE and where that VALUE occurs in
2061 ** the histogram data. This only works when x is the left-most
2062 ** column of an index and sqlite_stat4 histogram data is available
2063 ** for that index. When pExpr==NULL that means the constraint is
2064 ** "x IS NULL" instead of "x=VALUE".
2066 ** Write the estimated row count into *pnRow and return SQLITE_OK.
2067 ** If unable to make an estimate, leave *pnRow unchanged and return
2070 ** This routine can fail if it is unable to load a collating sequence
2071 ** required for string comparison, or if unable to allocate memory
2072 ** for a UTF conversion required for comparison. The error is stored
2073 ** in the pParse structure.
2075 static int whereEqualScanEst(
2076 Parse
*pParse
, /* Parsing & code generating context */
2077 WhereLoopBuilder
*pBuilder
,
2078 Expr
*pExpr
, /* Expression for VALUE in the x=VALUE constraint */
2079 tRowcnt
*pnRow
/* Write the revised row estimate here */
2081 Index
*p
= pBuilder
->pNew
->u
.btree
.pIndex
;
2082 int nEq
= pBuilder
->pNew
->u
.btree
.nEq
;
2083 UnpackedRecord
*pRec
= pBuilder
->pRec
;
2084 int rc
; /* Subfunction return code */
2085 tRowcnt a
[2]; /* Statistics */
2089 assert( nEq
<=p
->nColumn
);
2090 assert( p
->aSample
!=0 );
2091 assert( p
->nSample
>0 );
2092 assert( pBuilder
->nRecValid
<nEq
);
2094 /* If values are not available for all fields of the index to the left
2095 ** of this one, no estimate can be made. Return SQLITE_NOTFOUND. */
2096 if( pBuilder
->nRecValid
<(nEq
-1) ){
2097 return SQLITE_NOTFOUND
;
2100 /* This is an optimization only. The call to sqlite3Stat4ProbeSetValue()
2101 ** below would return the same value. */
2102 if( nEq
>=p
->nColumn
){
2107 rc
= sqlite3Stat4ProbeSetValue(pParse
, p
, &pRec
, pExpr
, 1, nEq
-1, &bOk
);
2108 pBuilder
->pRec
= pRec
;
2109 if( rc
!=SQLITE_OK
) return rc
;
2110 if( bOk
==0 ) return SQLITE_NOTFOUND
;
2111 pBuilder
->nRecValid
= nEq
;
2113 whereKeyStats(pParse
, p
, pRec
, 0, a
);
2114 WHERETRACE(0x20,("equality scan regions %s(%d): %d\n",
2115 p
->zName
, nEq
-1, (int)a
[1]));
2120 #endif /* SQLITE_ENABLE_STAT4 */
2122 #ifdef SQLITE_ENABLE_STAT4
2124 ** Estimate the number of rows that will be returned based on
2125 ** an IN constraint where the right-hand side of the IN operator
2126 ** is a list of values. Example:
2128 ** WHERE x IN (1,2,3,4)
2130 ** Write the estimated row count into *pnRow and return SQLITE_OK.
2131 ** If unable to make an estimate, leave *pnRow unchanged and return
2134 ** This routine can fail if it is unable to load a collating sequence
2135 ** required for string comparison, or if unable to allocate memory
2136 ** for a UTF conversion required for comparison. The error is stored
2137 ** in the pParse structure.
2139 static int whereInScanEst(
2140 Parse
*pParse
, /* Parsing & code generating context */
2141 WhereLoopBuilder
*pBuilder
,
2142 ExprList
*pList
, /* The value list on the RHS of "x IN (v1,v2,v3,...)" */
2143 tRowcnt
*pnRow
/* Write the revised row estimate here */
2145 Index
*p
= pBuilder
->pNew
->u
.btree
.pIndex
;
2146 i64 nRow0
= sqlite3LogEstToInt(p
->aiRowLogEst
[0]);
2147 int nRecValid
= pBuilder
->nRecValid
;
2148 int rc
= SQLITE_OK
; /* Subfunction return code */
2149 tRowcnt nEst
; /* Number of rows for a single term */
2150 tRowcnt nRowEst
= 0; /* New estimate of the number of rows */
2151 int i
; /* Loop counter */
2153 assert( p
->aSample
!=0 );
2154 for(i
=0; rc
==SQLITE_OK
&& i
<pList
->nExpr
; i
++){
2156 rc
= whereEqualScanEst(pParse
, pBuilder
, pList
->a
[i
].pExpr
, &nEst
);
2158 pBuilder
->nRecValid
= nRecValid
;
2161 if( rc
==SQLITE_OK
){
2162 if( nRowEst
> (tRowcnt
)nRow0
) nRowEst
= nRow0
;
2164 WHERETRACE(0x20,("IN row estimate: est=%d\n", nRowEst
));
2166 assert( pBuilder
->nRecValid
==nRecValid
);
2169 #endif /* SQLITE_ENABLE_STAT4 */
2172 #ifdef WHERETRACE_ENABLED
2174 ** Print the content of a WhereTerm object
2176 void sqlite3WhereTermPrint(WhereTerm
*pTerm
, int iTerm
){
2178 sqlite3DebugPrintf("TERM-%-3d NULL\n", iTerm
);
2182 memcpy(zType
, "....", 5);
2183 if( pTerm
->wtFlags
& TERM_VIRTUAL
) zType
[0] = 'V';
2184 if( pTerm
->eOperator
& WO_EQUIV
) zType
[1] = 'E';
2185 if( ExprHasProperty(pTerm
->pExpr
, EP_OuterON
) ) zType
[2] = 'L';
2186 if( pTerm
->wtFlags
& TERM_CODED
) zType
[3] = 'C';
2187 if( pTerm
->eOperator
& WO_SINGLE
){
2188 assert( (pTerm
->eOperator
& (WO_OR
|WO_AND
))==0 );
2189 sqlite3_snprintf(sizeof(zLeft
),zLeft
,"left={%d:%d}",
2190 pTerm
->leftCursor
, pTerm
->u
.x
.leftColumn
);
2191 }else if( (pTerm
->eOperator
& WO_OR
)!=0 && pTerm
->u
.pOrInfo
!=0 ){
2192 sqlite3_snprintf(sizeof(zLeft
),zLeft
,"indexable=0x%llx",
2193 pTerm
->u
.pOrInfo
->indexable
);
2195 sqlite3_snprintf(sizeof(zLeft
),zLeft
,"left=%d", pTerm
->leftCursor
);
2198 "TERM-%-3d %p %s %-12s op=%03x wtFlags=%04x",
2199 iTerm
, pTerm
, zType
, zLeft
, pTerm
->eOperator
, pTerm
->wtFlags
);
2200 /* The 0x10000 .wheretrace flag causes extra information to be
2201 ** shown about each Term */
2202 if( sqlite3WhereTrace
& 0x10000 ){
2203 sqlite3DebugPrintf(" prob=%-3d prereq=%llx,%llx",
2204 pTerm
->truthProb
, (u64
)pTerm
->prereqAll
, (u64
)pTerm
->prereqRight
);
2206 if( (pTerm
->eOperator
& (WO_OR
|WO_AND
))==0 && pTerm
->u
.x
.iField
){
2207 sqlite3DebugPrintf(" iField=%d", pTerm
->u
.x
.iField
);
2209 if( pTerm
->iParent
>=0 ){
2210 sqlite3DebugPrintf(" iParent=%d", pTerm
->iParent
);
2212 sqlite3DebugPrintf("\n");
2213 sqlite3TreeViewExpr(0, pTerm
->pExpr
, 0);
2218 #ifdef WHERETRACE_ENABLED
2220 ** Show the complete content of a WhereClause
2222 void sqlite3WhereClausePrint(WhereClause
*pWC
){
2224 for(i
=0; i
<pWC
->nTerm
; i
++){
2225 sqlite3WhereTermPrint(&pWC
->a
[i
], i
);
2230 #ifdef WHERETRACE_ENABLED
2232 ** Print a WhereLoop object for debugging purposes
2234 void sqlite3WhereLoopPrint(WhereLoop
*p
, WhereClause
*pWC
){
2235 WhereInfo
*pWInfo
= pWC
->pWInfo
;
2236 int nb
= 1+(pWInfo
->pTabList
->nSrc
+3)/4;
2237 SrcItem
*pItem
= pWInfo
->pTabList
->a
+ p
->iTab
;
2238 Table
*pTab
= pItem
->pTab
;
2239 Bitmask mAll
= (((Bitmask
)1)<<(nb
*4)) - 1;
2240 sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p
->cId
,
2241 p
->iTab
, nb
, p
->maskSelf
, nb
, p
->prereq
& mAll
);
2242 sqlite3DebugPrintf(" %12s",
2243 pItem
->zAlias
? pItem
->zAlias
: pTab
->zName
);
2244 if( (p
->wsFlags
& WHERE_VIRTUALTABLE
)==0 ){
2246 if( p
->u
.btree
.pIndex
&& (zName
= p
->u
.btree
.pIndex
->zName
)!=0 ){
2247 if( strncmp(zName
, "sqlite_autoindex_", 17)==0 ){
2248 int i
= sqlite3Strlen30(zName
) - 1;
2249 while( zName
[i
]!='_' ) i
--;
2252 sqlite3DebugPrintf(".%-16s %2d", zName
, p
->u
.btree
.nEq
);
2254 sqlite3DebugPrintf("%20s","");
2258 if( p
->u
.vtab
.idxStr
){
2259 z
= sqlite3_mprintf("(%d,\"%s\",%#x)",
2260 p
->u
.vtab
.idxNum
, p
->u
.vtab
.idxStr
, p
->u
.vtab
.omitMask
);
2262 z
= sqlite3_mprintf("(%d,%x)", p
->u
.vtab
.idxNum
, p
->u
.vtab
.omitMask
);
2264 sqlite3DebugPrintf(" %-19s", z
);
2267 if( p
->wsFlags
& WHERE_SKIPSCAN
){
2268 sqlite3DebugPrintf(" f %06x %d-%d", p
->wsFlags
, p
->nLTerm
,p
->nSkip
);
2270 sqlite3DebugPrintf(" f %06x N %d", p
->wsFlags
, p
->nLTerm
);
2272 sqlite3DebugPrintf(" cost %d,%d,%d\n", p
->rSetup
, p
->rRun
, p
->nOut
);
2273 if( p
->nLTerm
&& (sqlite3WhereTrace
& 0x4000)!=0 ){
2275 for(i
=0; i
<p
->nLTerm
; i
++){
2276 sqlite3WhereTermPrint(p
->aLTerm
[i
], i
);
2283 ** Convert bulk memory into a valid WhereLoop that can be passed
2284 ** to whereLoopClear harmlessly.
2286 static void whereLoopInit(WhereLoop
*p
){
2287 p
->aLTerm
= p
->aLTermSpace
;
2289 p
->nLSlot
= ArraySize(p
->aLTermSpace
);
2294 ** Clear the WhereLoop.u union. Leave WhereLoop.pLTerm intact.
2296 static void whereLoopClearUnion(sqlite3
*db
, WhereLoop
*p
){
2297 if( p
->wsFlags
& (WHERE_VIRTUALTABLE
|WHERE_AUTO_INDEX
) ){
2298 if( (p
->wsFlags
& WHERE_VIRTUALTABLE
)!=0 && p
->u
.vtab
.needFree
){
2299 sqlite3_free(p
->u
.vtab
.idxStr
);
2300 p
->u
.vtab
.needFree
= 0;
2301 p
->u
.vtab
.idxStr
= 0;
2302 }else if( (p
->wsFlags
& WHERE_AUTO_INDEX
)!=0 && p
->u
.btree
.pIndex
!=0 ){
2303 sqlite3DbFree(db
, p
->u
.btree
.pIndex
->zColAff
);
2304 sqlite3DbFreeNN(db
, p
->u
.btree
.pIndex
);
2305 p
->u
.btree
.pIndex
= 0;
2311 ** Deallocate internal memory used by a WhereLoop object. Leave the
2312 ** object in an initialized state, as if it had been newly allocated.
2314 static void whereLoopClear(sqlite3
*db
, WhereLoop
*p
){
2315 if( p
->aLTerm
!=p
->aLTermSpace
){
2316 sqlite3DbFreeNN(db
, p
->aLTerm
);
2317 p
->aLTerm
= p
->aLTermSpace
;
2318 p
->nLSlot
= ArraySize(p
->aLTermSpace
);
2320 whereLoopClearUnion(db
, p
);
2326 ** Increase the memory allocation for pLoop->aLTerm[] to be at least n.
2328 static int whereLoopResize(sqlite3
*db
, WhereLoop
*p
, int n
){
2330 if( p
->nLSlot
>=n
) return SQLITE_OK
;
2332 paNew
= sqlite3DbMallocRawNN(db
, sizeof(p
->aLTerm
[0])*n
);
2333 if( paNew
==0 ) return SQLITE_NOMEM_BKPT
;
2334 memcpy(paNew
, p
->aLTerm
, sizeof(p
->aLTerm
[0])*p
->nLSlot
);
2335 if( p
->aLTerm
!=p
->aLTermSpace
) sqlite3DbFreeNN(db
, p
->aLTerm
);
2342 ** Transfer content from the second pLoop into the first.
2344 static int whereLoopXfer(sqlite3
*db
, WhereLoop
*pTo
, WhereLoop
*pFrom
){
2345 whereLoopClearUnion(db
, pTo
);
2346 if( pFrom
->nLTerm
> pTo
->nLSlot
2347 && whereLoopResize(db
, pTo
, pFrom
->nLTerm
)
2349 memset(pTo
, 0, WHERE_LOOP_XFER_SZ
);
2350 return SQLITE_NOMEM_BKPT
;
2352 memcpy(pTo
, pFrom
, WHERE_LOOP_XFER_SZ
);
2353 memcpy(pTo
->aLTerm
, pFrom
->aLTerm
, pTo
->nLTerm
*sizeof(pTo
->aLTerm
[0]));
2354 if( pFrom
->wsFlags
& WHERE_VIRTUALTABLE
){
2355 pFrom
->u
.vtab
.needFree
= 0;
2356 }else if( (pFrom
->wsFlags
& WHERE_AUTO_INDEX
)!=0 ){
2357 pFrom
->u
.btree
.pIndex
= 0;
2363 ** Delete a WhereLoop object
2365 static void whereLoopDelete(sqlite3
*db
, WhereLoop
*p
){
2367 whereLoopClear(db
, p
);
2368 sqlite3DbNNFreeNN(db
, p
);
2372 ** Free a WhereInfo structure
2374 static void whereInfoFree(sqlite3
*db
, WhereInfo
*pWInfo
){
2375 assert( pWInfo
!=0 );
2377 sqlite3WhereClauseClear(&pWInfo
->sWC
);
2378 while( pWInfo
->pLoops
){
2379 WhereLoop
*p
= pWInfo
->pLoops
;
2380 pWInfo
->pLoops
= p
->pNextLoop
;
2381 whereLoopDelete(db
, p
);
2383 while( pWInfo
->pMemToFree
){
2384 WhereMemBlock
*pNext
= pWInfo
->pMemToFree
->pNext
;
2385 sqlite3DbNNFreeNN(db
, pWInfo
->pMemToFree
);
2386 pWInfo
->pMemToFree
= pNext
;
2388 sqlite3DbNNFreeNN(db
, pWInfo
);
2392 ** Return TRUE if all of the following are true:
2394 ** (1) X has the same or lower cost, or returns the same or fewer rows,
2396 ** (2) X uses fewer WHERE clause terms than Y
2397 ** (3) Every WHERE clause term used by X is also used by Y
2398 ** (4) X skips at least as many columns as Y
2399 ** (5) If X is a covering index, than Y is too
2401 ** Conditions (2) and (3) mean that X is a "proper subset" of Y.
2402 ** If X is a proper subset of Y then Y is a better choice and ought
2403 ** to have a lower cost. This routine returns TRUE when that cost
2404 ** relationship is inverted and needs to be adjusted. Constraint (4)
2405 ** was added because if X uses skip-scan less than Y it still might
2406 ** deserve a lower cost even if it is a proper subset of Y. Constraint (5)
2407 ** was added because a covering index probably deserves to have a lower cost
2408 ** than a non-covering index even if it is a proper subset.
2410 static int whereLoopCheaperProperSubset(
2411 const WhereLoop
*pX
, /* First WhereLoop to compare */
2412 const WhereLoop
*pY
/* Compare against this WhereLoop */
2415 if( pX
->nLTerm
-pX
->nSkip
>= pY
->nLTerm
-pY
->nSkip
){
2416 return 0; /* X is not a subset of Y */
2418 if( pX
->rRun
>pY
->rRun
&& pX
->nOut
>pY
->nOut
) return 0;
2419 if( pY
->nSkip
> pX
->nSkip
) return 0;
2420 for(i
=pX
->nLTerm
-1; i
>=0; i
--){
2421 if( pX
->aLTerm
[i
]==0 ) continue;
2422 for(j
=pY
->nLTerm
-1; j
>=0; j
--){
2423 if( pY
->aLTerm
[j
]==pX
->aLTerm
[i
] ) break;
2425 if( j
<0 ) return 0; /* X not a subset of Y since term X[i] not used by Y */
2427 if( (pX
->wsFlags
&WHERE_IDX_ONLY
)!=0
2428 && (pY
->wsFlags
&WHERE_IDX_ONLY
)==0 ){
2429 return 0; /* Constraint (5) */
2431 return 1; /* All conditions meet */
2435 ** Try to adjust the cost and number of output rows of WhereLoop pTemplate
2436 ** upwards or downwards so that:
2438 ** (1) pTemplate costs less than any other WhereLoops that are a proper
2439 ** subset of pTemplate
2441 ** (2) pTemplate costs more than any other WhereLoops for which pTemplate
2442 ** is a proper subset.
2444 ** To say "WhereLoop X is a proper subset of Y" means that X uses fewer
2445 ** WHERE clause terms than Y and that every WHERE clause term used by X is
2448 static void whereLoopAdjustCost(const WhereLoop
*p
, WhereLoop
*pTemplate
){
2449 if( (pTemplate
->wsFlags
& WHERE_INDEXED
)==0 ) return;
2450 for(; p
; p
=p
->pNextLoop
){
2451 if( p
->iTab
!=pTemplate
->iTab
) continue;
2452 if( (p
->wsFlags
& WHERE_INDEXED
)==0 ) continue;
2453 if( whereLoopCheaperProperSubset(p
, pTemplate
) ){
2454 /* Adjust pTemplate cost downward so that it is cheaper than its
2456 WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n",
2457 pTemplate
->rRun
, pTemplate
->nOut
,
2458 MIN(p
->rRun
, pTemplate
->rRun
),
2459 MIN(p
->nOut
- 1, pTemplate
->nOut
)));
2460 pTemplate
->rRun
= MIN(p
->rRun
, pTemplate
->rRun
);
2461 pTemplate
->nOut
= MIN(p
->nOut
- 1, pTemplate
->nOut
);
2462 }else if( whereLoopCheaperProperSubset(pTemplate
, p
) ){
2463 /* Adjust pTemplate cost upward so that it is costlier than p since
2464 ** pTemplate is a proper subset of p */
2465 WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n",
2466 pTemplate
->rRun
, pTemplate
->nOut
,
2467 MAX(p
->rRun
, pTemplate
->rRun
),
2468 MAX(p
->nOut
+ 1, pTemplate
->nOut
)));
2469 pTemplate
->rRun
= MAX(p
->rRun
, pTemplate
->rRun
);
2470 pTemplate
->nOut
= MAX(p
->nOut
+ 1, pTemplate
->nOut
);
2476 ** Search the list of WhereLoops in *ppPrev looking for one that can be
2477 ** replaced by pTemplate.
2479 ** Return NULL if pTemplate does not belong on the WhereLoop list.
2480 ** In other words if pTemplate ought to be dropped from further consideration.
2482 ** If pX is a WhereLoop that pTemplate can replace, then return the
2483 ** link that points to pX.
2485 ** If pTemplate cannot replace any existing element of the list but needs
2486 ** to be added to the list as a new entry, then return a pointer to the
2487 ** tail of the list.
2489 static WhereLoop
**whereLoopFindLesser(
2491 const WhereLoop
*pTemplate
2494 for(p
=(*ppPrev
); p
; ppPrev
=&p
->pNextLoop
, p
=*ppPrev
){
2495 if( p
->iTab
!=pTemplate
->iTab
|| p
->iSortIdx
!=pTemplate
->iSortIdx
){
2496 /* If either the iTab or iSortIdx values for two WhereLoop are different
2497 ** then those WhereLoops need to be considered separately. Neither is
2498 ** a candidate to replace the other. */
2501 /* In the current implementation, the rSetup value is either zero
2502 ** or the cost of building an automatic index (NlogN) and the NlogN
2503 ** is the same for compatible WhereLoops. */
2504 assert( p
->rSetup
==0 || pTemplate
->rSetup
==0
2505 || p
->rSetup
==pTemplate
->rSetup
);
2507 /* whereLoopAddBtree() always generates and inserts the automatic index
2508 ** case first. Hence compatible candidate WhereLoops never have a larger
2509 ** rSetup. Call this SETUP-INVARIANT */
2510 assert( p
->rSetup
>=pTemplate
->rSetup
);
2512 /* Any loop using an application-defined index (or PRIMARY KEY or
2513 ** UNIQUE constraint) with one or more == constraints is better
2514 ** than an automatic index. Unless it is a skip-scan. */
2515 if( (p
->wsFlags
& WHERE_AUTO_INDEX
)!=0
2516 && (pTemplate
->nSkip
)==0
2517 && (pTemplate
->wsFlags
& WHERE_INDEXED
)!=0
2518 && (pTemplate
->wsFlags
& WHERE_COLUMN_EQ
)!=0
2519 && (p
->prereq
& pTemplate
->prereq
)==pTemplate
->prereq
2524 /* If existing WhereLoop p is better than pTemplate, pTemplate can be
2525 ** discarded. WhereLoop p is better if:
2526 ** (1) p has no more dependencies than pTemplate, and
2527 ** (2) p has an equal or lower cost than pTemplate
2529 if( (p
->prereq
& pTemplate
->prereq
)==p
->prereq
/* (1) */
2530 && p
->rSetup
<=pTemplate
->rSetup
/* (2a) */
2531 && p
->rRun
<=pTemplate
->rRun
/* (2b) */
2532 && p
->nOut
<=pTemplate
->nOut
/* (2c) */
2534 return 0; /* Discard pTemplate */
2537 /* If pTemplate is always better than p, then cause p to be overwritten
2538 ** with pTemplate. pTemplate is better than p if:
2539 ** (1) pTemplate has no more dependencies than p, and
2540 ** (2) pTemplate has an equal or lower cost than p.
2542 if( (p
->prereq
& pTemplate
->prereq
)==pTemplate
->prereq
/* (1) */
2543 && p
->rRun
>=pTemplate
->rRun
/* (2a) */
2544 && p
->nOut
>=pTemplate
->nOut
/* (2b) */
2546 assert( p
->rSetup
>=pTemplate
->rSetup
); /* SETUP-INVARIANT above */
2547 break; /* Cause p to be overwritten by pTemplate */
2554 ** Insert or replace a WhereLoop entry using the template supplied.
2556 ** An existing WhereLoop entry might be overwritten if the new template
2557 ** is better and has fewer dependencies. Or the template will be ignored
2558 ** and no insert will occur if an existing WhereLoop is faster and has
2559 ** fewer dependencies than the template. Otherwise a new WhereLoop is
2560 ** added based on the template.
2562 ** If pBuilder->pOrSet is not NULL then we care about only the
2563 ** prerequisites and rRun and nOut costs of the N best loops. That
2564 ** information is gathered in the pBuilder->pOrSet object. This special
2565 ** processing mode is used only for OR clause processing.
2567 ** When accumulating multiple loops (when pBuilder->pOrSet is NULL) we
2568 ** still might overwrite similar loops with the new template if the
2569 ** new template is better. Loops may be overwritten if the following
2570 ** conditions are met:
2572 ** (1) They have the same iTab.
2573 ** (2) They have the same iSortIdx.
2574 ** (3) The template has same or fewer dependencies than the current loop
2575 ** (4) The template has the same or lower cost than the current loop
2577 static int whereLoopInsert(WhereLoopBuilder
*pBuilder
, WhereLoop
*pTemplate
){
2578 WhereLoop
**ppPrev
, *p
;
2579 WhereInfo
*pWInfo
= pBuilder
->pWInfo
;
2580 sqlite3
*db
= pWInfo
->pParse
->db
;
2583 /* Stop the search once we hit the query planner search limit */
2584 if( pBuilder
->iPlanLimit
==0 ){
2585 WHERETRACE(0xffffffff,("=== query planner search limit reached ===\n"));
2586 if( pBuilder
->pOrSet
) pBuilder
->pOrSet
->n
= 0;
2589 pBuilder
->iPlanLimit
--;
2591 whereLoopAdjustCost(pWInfo
->pLoops
, pTemplate
);
2593 /* If pBuilder->pOrSet is defined, then only keep track of the costs
2596 if( pBuilder
->pOrSet
!=0 ){
2597 if( pTemplate
->nLTerm
){
2598 #if WHERETRACE_ENABLED
2599 u16 n
= pBuilder
->pOrSet
->n
;
2602 whereOrInsert(pBuilder
->pOrSet
, pTemplate
->prereq
, pTemplate
->rRun
,
2604 #if WHERETRACE_ENABLED /* 0x8 */
2605 if( sqlite3WhereTrace
& 0x8 ){
2606 sqlite3DebugPrintf(x
?" or-%d: ":" or-X: ", n
);
2607 sqlite3WhereLoopPrint(pTemplate
, pBuilder
->pWC
);
2614 /* Look for an existing WhereLoop to replace with pTemplate
2616 ppPrev
= whereLoopFindLesser(&pWInfo
->pLoops
, pTemplate
);
2619 /* There already exists a WhereLoop on the list that is better
2620 ** than pTemplate, so just ignore pTemplate */
2621 #if WHERETRACE_ENABLED /* 0x8 */
2622 if( sqlite3WhereTrace
& 0x8 ){
2623 sqlite3DebugPrintf(" skip: ");
2624 sqlite3WhereLoopPrint(pTemplate
, pBuilder
->pWC
);
2632 /* If we reach this point it means that either p[] should be overwritten
2633 ** with pTemplate[] if p[] exists, or if p==NULL then allocate a new
2634 ** WhereLoop and insert it.
2636 #if WHERETRACE_ENABLED /* 0x8 */
2637 if( sqlite3WhereTrace
& 0x8 ){
2639 sqlite3DebugPrintf("replace: ");
2640 sqlite3WhereLoopPrint(p
, pBuilder
->pWC
);
2641 sqlite3DebugPrintf(" with: ");
2643 sqlite3DebugPrintf(" add: ");
2645 sqlite3WhereLoopPrint(pTemplate
, pBuilder
->pWC
);
2649 /* Allocate a new WhereLoop to add to the end of the list */
2650 *ppPrev
= p
= sqlite3DbMallocRawNN(db
, sizeof(WhereLoop
));
2651 if( p
==0 ) return SQLITE_NOMEM_BKPT
;
2655 /* We will be overwriting WhereLoop p[]. But before we do, first
2656 ** go through the rest of the list and delete any other entries besides
2657 ** p[] that are also supplanted by pTemplate */
2658 WhereLoop
**ppTail
= &p
->pNextLoop
;
2661 ppTail
= whereLoopFindLesser(ppTail
, pTemplate
);
2662 if( ppTail
==0 ) break;
2664 if( pToDel
==0 ) break;
2665 *ppTail
= pToDel
->pNextLoop
;
2666 #if WHERETRACE_ENABLED /* 0x8 */
2667 if( sqlite3WhereTrace
& 0x8 ){
2668 sqlite3DebugPrintf(" delete: ");
2669 sqlite3WhereLoopPrint(pToDel
, pBuilder
->pWC
);
2672 whereLoopDelete(db
, pToDel
);
2675 rc
= whereLoopXfer(db
, p
, pTemplate
);
2676 if( (p
->wsFlags
& WHERE_VIRTUALTABLE
)==0 ){
2677 Index
*pIndex
= p
->u
.btree
.pIndex
;
2678 if( pIndex
&& pIndex
->idxType
==SQLITE_IDXTYPE_IPK
){
2679 p
->u
.btree
.pIndex
= 0;
2686 ** Adjust the WhereLoop.nOut value downward to account for terms of the
2687 ** WHERE clause that reference the loop but which are not used by an
2690 ** For every WHERE clause term that is not used by the index
2691 ** and which has a truth probability assigned by one of the likelihood(),
2692 ** likely(), or unlikely() SQL functions, reduce the estimated number
2693 ** of output rows by the probability specified.
2695 ** TUNING: For every WHERE clause term that is not used by the index
2696 ** and which does not have an assigned truth probability, heuristics
2697 ** described below are used to try to estimate the truth probability.
2698 ** TODO --> Perhaps this is something that could be improved by better
2699 ** table statistics.
2701 ** Heuristic 1: Estimate the truth probability as 93.75%. The 93.75%
2702 ** value corresponds to -1 in LogEst notation, so this means decrement
2703 ** the WhereLoop.nOut field for every such WHERE clause term.
2705 ** Heuristic 2: If there exists one or more WHERE clause terms of the
2706 ** form "x==EXPR" and EXPR is not a constant 0 or 1, then make sure the
2707 ** final output row estimate is no greater than 1/4 of the total number
2708 ** of rows in the table. In other words, assume that x==EXPR will filter
2709 ** out at least 3 out of 4 rows. If EXPR is -1 or 0 or 1, then maybe the
2710 ** "x" column is boolean or else -1 or 0 or 1 is a common default value
2711 ** on the "x" column and so in that case only cap the output row estimate
2712 ** at 1/2 instead of 1/4.
2714 static void whereLoopOutputAdjust(
2715 WhereClause
*pWC
, /* The WHERE clause */
2716 WhereLoop
*pLoop
, /* The loop to adjust downward */
2717 LogEst nRow
/* Number of rows in the entire table */
2719 WhereTerm
*pTerm
, *pX
;
2720 Bitmask notAllowed
= ~(pLoop
->prereq
|pLoop
->maskSelf
);
2722 LogEst iReduce
= 0; /* pLoop->nOut should not exceed nRow-iReduce */
2724 assert( (pLoop
->wsFlags
& WHERE_AUTO_INDEX
)==0 );
2725 for(i
=pWC
->nBase
, pTerm
=pWC
->a
; i
>0; i
--, pTerm
++){
2727 if( (pTerm
->prereqAll
& notAllowed
)!=0 ) continue;
2728 if( (pTerm
->prereqAll
& pLoop
->maskSelf
)==0 ) continue;
2729 if( (pTerm
->wtFlags
& TERM_VIRTUAL
)!=0 ) continue;
2730 for(j
=pLoop
->nLTerm
-1; j
>=0; j
--){
2731 pX
= pLoop
->aLTerm
[j
];
2732 if( pX
==0 ) continue;
2733 if( pX
==pTerm
) break;
2734 if( pX
->iParent
>=0 && (&pWC
->a
[pX
->iParent
])==pTerm
) break;
2737 sqlite3ProgressCheck(pWC
->pWInfo
->pParse
);
2738 if( pLoop
->maskSelf
==pTerm
->prereqAll
){
2739 /* If there are extra terms in the WHERE clause not used by an index
2740 ** that depend only on the table being scanned, and that will tend to
2741 ** cause many rows to be omitted, then mark that table as
2744 ** 2022-03-24: Self-culling only applies if either the extra terms
2745 ** are straight comparison operators that are non-true with NULL
2746 ** operand, or if the loop is not an OUTER JOIN.
2748 if( (pTerm
->eOperator
& 0x3f)!=0
2749 || (pWC
->pWInfo
->pTabList
->a
[pLoop
->iTab
].fg
.jointype
2750 & (JT_LEFT
|JT_LTORJ
))==0
2752 pLoop
->wsFlags
|= WHERE_SELFCULL
;
2755 if( pTerm
->truthProb
<=0 ){
2756 /* If a truth probability is specified using the likelihood() hints,
2757 ** then use the probability provided by the application. */
2758 pLoop
->nOut
+= pTerm
->truthProb
;
2760 /* In the absence of explicit truth probabilities, use heuristics to
2761 ** guess a reasonable truth probability. */
2763 if( (pTerm
->eOperator
&(WO_EQ
|WO_IS
))!=0
2764 && (pTerm
->wtFlags
& TERM_HIGHTRUTH
)==0 /* tag-20200224-1 */
2766 Expr
*pRight
= pTerm
->pExpr
->pRight
;
2768 testcase( pTerm
->pExpr
->op
==TK_IS
);
2769 if( sqlite3ExprIsInteger(pRight
, &k
) && k
>=(-1) && k
<=1 ){
2775 pTerm
->wtFlags
|= TERM_HEURTRUTH
;
2782 if( pLoop
->nOut
> nRow
-iReduce
){
2783 pLoop
->nOut
= nRow
- iReduce
;
2788 ** Term pTerm is a vector range comparison operation. The first comparison
2789 ** in the vector can be optimized using column nEq of the index. This
2790 ** function returns the total number of vector elements that can be used
2791 ** as part of the range comparison.
2793 ** For example, if the query is:
2795 ** WHERE a = ? AND (b, c, d) > (?, ?, ?)
2799 ** CREATE INDEX ... ON (a, b, c, d, e)
2801 ** then this function would be invoked with nEq=1. The value returned in
2804 static int whereRangeVectorLen(
2805 Parse
*pParse
, /* Parsing context */
2806 int iCur
, /* Cursor open on pIdx */
2807 Index
*pIdx
, /* The index to be used for a inequality constraint */
2808 int nEq
, /* Number of prior equality constraints on same index */
2809 WhereTerm
*pTerm
/* The vector inequality constraint */
2811 int nCmp
= sqlite3ExprVectorSize(pTerm
->pExpr
->pLeft
);
2814 nCmp
= MIN(nCmp
, (pIdx
->nColumn
- nEq
));
2815 for(i
=1; i
<nCmp
; i
++){
2816 /* Test if comparison i of pTerm is compatible with column (i+nEq)
2817 ** of the index. If not, exit the loop. */
2818 char aff
; /* Comparison affinity */
2819 char idxaff
= 0; /* Indexed columns affinity */
2820 CollSeq
*pColl
; /* Comparison collation sequence */
2823 assert( ExprUseXList(pTerm
->pExpr
->pLeft
) );
2824 pLhs
= pTerm
->pExpr
->pLeft
->x
.pList
->a
[i
].pExpr
;
2825 pRhs
= pTerm
->pExpr
->pRight
;
2826 if( ExprUseXSelect(pRhs
) ){
2827 pRhs
= pRhs
->x
.pSelect
->pEList
->a
[i
].pExpr
;
2829 pRhs
= pRhs
->x
.pList
->a
[i
].pExpr
;
2832 /* Check that the LHS of the comparison is a column reference to
2833 ** the right column of the right source table. And that the sort
2834 ** order of the index column is the same as the sort order of the
2835 ** leftmost index column. */
2836 if( pLhs
->op
!=TK_COLUMN
2837 || pLhs
->iTable
!=iCur
2838 || pLhs
->iColumn
!=pIdx
->aiColumn
[i
+nEq
]
2839 || pIdx
->aSortOrder
[i
+nEq
]!=pIdx
->aSortOrder
[nEq
]
2844 testcase( pLhs
->iColumn
==XN_ROWID
);
2845 aff
= sqlite3CompareAffinity(pRhs
, sqlite3ExprAffinity(pLhs
));
2846 idxaff
= sqlite3TableColumnAffinity(pIdx
->pTable
, pLhs
->iColumn
);
2847 if( aff
!=idxaff
) break;
2849 pColl
= sqlite3BinaryCompareCollSeq(pParse
, pLhs
, pRhs
);
2850 if( pColl
==0 ) break;
2851 if( sqlite3StrICmp(pColl
->zName
, pIdx
->azColl
[i
+nEq
]) ) break;
2857 ** Adjust the cost C by the costMult factor T. This only occurs if
2858 ** compiled with -DSQLITE_ENABLE_COSTMULT
2860 #ifdef SQLITE_ENABLE_COSTMULT
2861 # define ApplyCostMultiplier(C,T) C += T
2863 # define ApplyCostMultiplier(C,T)
2867 ** We have so far matched pBuilder->pNew->u.btree.nEq terms of the
2868 ** index pIndex. Try to match one more.
2870 ** When this function is called, pBuilder->pNew->nOut contains the
2871 ** number of rows expected to be visited by filtering using the nEq
2872 ** terms only. If it is modified, this value is restored before this
2873 ** function returns.
2875 ** If pProbe->idxType==SQLITE_IDXTYPE_IPK, that means pIndex is
2876 ** a fake index used for the INTEGER PRIMARY KEY.
2878 static int whereLoopAddBtreeIndex(
2879 WhereLoopBuilder
*pBuilder
, /* The WhereLoop factory */
2880 SrcItem
*pSrc
, /* FROM clause term being analyzed */
2881 Index
*pProbe
, /* An index on pSrc */
2882 LogEst nInMul
/* log(Number of iterations due to IN) */
2884 WhereInfo
*pWInfo
= pBuilder
->pWInfo
; /* WHERE analyze context */
2885 Parse
*pParse
= pWInfo
->pParse
; /* Parsing context */
2886 sqlite3
*db
= pParse
->db
; /* Database connection malloc context */
2887 WhereLoop
*pNew
; /* Template WhereLoop under construction */
2888 WhereTerm
*pTerm
; /* A WhereTerm under consideration */
2889 int opMask
; /* Valid operators for constraints */
2890 WhereScan scan
; /* Iterator for WHERE terms */
2891 Bitmask saved_prereq
; /* Original value of pNew->prereq */
2892 u16 saved_nLTerm
; /* Original value of pNew->nLTerm */
2893 u16 saved_nEq
; /* Original value of pNew->u.btree.nEq */
2894 u16 saved_nBtm
; /* Original value of pNew->u.btree.nBtm */
2895 u16 saved_nTop
; /* Original value of pNew->u.btree.nTop */
2896 u16 saved_nSkip
; /* Original value of pNew->nSkip */
2897 u32 saved_wsFlags
; /* Original value of pNew->wsFlags */
2898 LogEst saved_nOut
; /* Original value of pNew->nOut */
2899 int rc
= SQLITE_OK
; /* Return code */
2900 LogEst rSize
; /* Number of rows in the table */
2901 LogEst rLogSize
; /* Logarithm of table size */
2902 WhereTerm
*pTop
= 0, *pBtm
= 0; /* Top and bottom range constraints */
2904 pNew
= pBuilder
->pNew
;
2905 assert( db
->mallocFailed
==0 || pParse
->nErr
>0 );
2909 WHERETRACE(0x800, ("BEGIN %s.addBtreeIdx(%s), nEq=%d, nSkip=%d, rRun=%d\n",
2910 pProbe
->pTable
->zName
,pProbe
->zName
,
2911 pNew
->u
.btree
.nEq
, pNew
->nSkip
, pNew
->rRun
));
2913 assert( (pNew
->wsFlags
& WHERE_VIRTUALTABLE
)==0 );
2914 assert( (pNew
->wsFlags
& WHERE_TOP_LIMIT
)==0 );
2915 if( pNew
->wsFlags
& WHERE_BTM_LIMIT
){
2916 opMask
= WO_LT
|WO_LE
;
2918 assert( pNew
->u
.btree
.nBtm
==0 );
2919 opMask
= WO_EQ
|WO_IN
|WO_GT
|WO_GE
|WO_LT
|WO_LE
|WO_ISNULL
|WO_IS
;
2921 if( pProbe
->bUnordered
) opMask
&= ~(WO_GT
|WO_GE
|WO_LT
|WO_LE
);
2923 assert( pNew
->u
.btree
.nEq
<pProbe
->nColumn
);
2924 assert( pNew
->u
.btree
.nEq
<pProbe
->nKeyCol
2925 || pProbe
->idxType
!=SQLITE_IDXTYPE_PRIMARYKEY
);
2927 saved_nEq
= pNew
->u
.btree
.nEq
;
2928 saved_nBtm
= pNew
->u
.btree
.nBtm
;
2929 saved_nTop
= pNew
->u
.btree
.nTop
;
2930 saved_nSkip
= pNew
->nSkip
;
2931 saved_nLTerm
= pNew
->nLTerm
;
2932 saved_wsFlags
= pNew
->wsFlags
;
2933 saved_prereq
= pNew
->prereq
;
2934 saved_nOut
= pNew
->nOut
;
2935 pTerm
= whereScanInit(&scan
, pBuilder
->pWC
, pSrc
->iCursor
, saved_nEq
,
2938 rSize
= pProbe
->aiRowLogEst
[0];
2939 rLogSize
= estLog(rSize
);
2940 for(; rc
==SQLITE_OK
&& pTerm
!=0; pTerm
= whereScanNext(&scan
)){
2941 u16 eOp
= pTerm
->eOperator
; /* Shorthand for pTerm->eOperator */
2943 LogEst nOutUnadjusted
; /* nOut before IN() and WHERE adjustments */
2945 #ifdef SQLITE_ENABLE_STAT4
2946 int nRecValid
= pBuilder
->nRecValid
;
2948 if( (eOp
==WO_ISNULL
|| (pTerm
->wtFlags
&TERM_VNULL
)!=0)
2949 && indexColumnNotNull(pProbe
, saved_nEq
)
2951 continue; /* ignore IS [NOT] NULL constraints on NOT NULL columns */
2953 if( pTerm
->prereqRight
& pNew
->maskSelf
) continue;
2955 /* Do not allow the upper bound of a LIKE optimization range constraint
2956 ** to mix with a lower range bound from some other source */
2957 if( pTerm
->wtFlags
& TERM_LIKEOPT
&& pTerm
->eOperator
==WO_LT
) continue;
2959 if( (pSrc
->fg
.jointype
& (JT_LEFT
|JT_LTORJ
|JT_RIGHT
))!=0
2960 && !constraintCompatibleWithOuterJoin(pTerm
,pSrc
)
2964 if( IsUniqueIndex(pProbe
) && saved_nEq
==pProbe
->nKeyCol
-1 ){
2965 pBuilder
->bldFlags1
|= SQLITE_BLDF1_UNIQUE
;
2967 pBuilder
->bldFlags1
|= SQLITE_BLDF1_INDEXED
;
2969 pNew
->wsFlags
= saved_wsFlags
;
2970 pNew
->u
.btree
.nEq
= saved_nEq
;
2971 pNew
->u
.btree
.nBtm
= saved_nBtm
;
2972 pNew
->u
.btree
.nTop
= saved_nTop
;
2973 pNew
->nLTerm
= saved_nLTerm
;
2974 if( pNew
->nLTerm
>=pNew
->nLSlot
2975 && whereLoopResize(db
, pNew
, pNew
->nLTerm
+1)
2977 break; /* OOM while trying to enlarge the pNew->aLTerm array */
2979 pNew
->aLTerm
[pNew
->nLTerm
++] = pTerm
;
2980 pNew
->prereq
= (saved_prereq
| pTerm
->prereqRight
) & ~pNew
->maskSelf
;
2983 || (pNew
->wsFlags
& WHERE_COLUMN_NULL
)!=0
2984 || (pNew
->wsFlags
& WHERE_COLUMN_IN
)!=0
2985 || (pNew
->wsFlags
& WHERE_SKIPSCAN
)!=0
2989 Expr
*pExpr
= pTerm
->pExpr
;
2990 if( ExprUseXSelect(pExpr
) ){
2991 /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */
2993 nIn
= 46; assert( 46==sqlite3LogEst(25) );
2995 /* The expression may actually be of the form (x, y) IN (SELECT...).
2996 ** In this case there is a separate term for each of (x) and (y).
2997 ** However, the nIn multiplier should only be applied once, not once
2998 ** for each such term. The following loop checks that pTerm is the
2999 ** first such term in use, and sets nIn back to 0 if it is not. */
3000 for(i
=0; i
<pNew
->nLTerm
-1; i
++){
3001 if( pNew
->aLTerm
[i
] && pNew
->aLTerm
[i
]->pExpr
==pExpr
) nIn
= 0;
3003 }else if( ALWAYS(pExpr
->x
.pList
&& pExpr
->x
.pList
->nExpr
) ){
3004 /* "x IN (value, value, ...)" */
3005 nIn
= sqlite3LogEst(pExpr
->x
.pList
->nExpr
);
3007 if( pProbe
->hasStat1
&& rLogSize
>=10 ){
3010 ** N = the total number of rows in the table
3011 ** K = the number of entries on the RHS of the IN operator
3012 ** M = the number of rows in the table that match terms to the
3013 ** to the left in the same index. If the IN operator is on
3014 ** the left-most index column, M==N.
3016 ** Given the definitions above, it is better to omit the IN operator
3017 ** from the index lookup and instead do a scan of the M elements,
3018 ** testing each scanned row against the IN operator separately, if:
3020 ** M*log(K) < K*log(N)
3022 ** Our estimates for M, K, and N might be inaccurate, so we build in
3023 ** a safety margin of 2 (LogEst: 10) that favors using the IN operator
3024 ** with the index, as using an index has better worst-case behavior.
3025 ** If we do not have real sqlite_stat1 data, always prefer to use
3026 ** the index. Do not bother with this optimization on very small
3027 ** tables (less than 2 rows) as it is pointless in that case.
3029 M
= pProbe
->aiRowLogEst
[saved_nEq
];
3031 /* TUNING v----- 10 to bias toward indexed IN */
3032 x
= M
+ logK
+ 10 - (nIn
+ rLogSize
);
3035 ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d) "
3036 "prefers indexed lookup\n",
3037 saved_nEq
, M
, logK
, nIn
, rLogSize
, x
));
3038 }else if( nInMul
<2 && OptimizationEnabled(db
, SQLITE_SeekScan
) ){
3040 ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d"
3041 " nInMul=%d) prefers skip-scan\n",
3042 saved_nEq
, M
, logK
, nIn
, rLogSize
, x
, nInMul
));
3043 pNew
->wsFlags
|= WHERE_IN_SEEKSCAN
;
3046 ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d"
3047 " nInMul=%d) prefers normal scan\n",
3048 saved_nEq
, M
, logK
, nIn
, rLogSize
, x
, nInMul
));
3052 pNew
->wsFlags
|= WHERE_COLUMN_IN
;
3053 }else if( eOp
& (WO_EQ
|WO_IS
) ){
3054 int iCol
= pProbe
->aiColumn
[saved_nEq
];
3055 pNew
->wsFlags
|= WHERE_COLUMN_EQ
;
3056 assert( saved_nEq
==pNew
->u
.btree
.nEq
);
3058 || (iCol
>=0 && nInMul
==0 && saved_nEq
==pProbe
->nKeyCol
-1)
3060 if( iCol
==XN_ROWID
|| pProbe
->uniqNotNull
3061 || (pProbe
->nKeyCol
==1 && pProbe
->onError
&& eOp
==WO_EQ
)
3063 pNew
->wsFlags
|= WHERE_ONEROW
;
3065 pNew
->wsFlags
|= WHERE_UNQ_WANTED
;
3068 if( scan
.iEquiv
>1 ) pNew
->wsFlags
|= WHERE_TRANSCONS
;
3069 }else if( eOp
& WO_ISNULL
){
3070 pNew
->wsFlags
|= WHERE_COLUMN_NULL
;
3072 int nVecLen
= whereRangeVectorLen(
3073 pParse
, pSrc
->iCursor
, pProbe
, saved_nEq
, pTerm
3075 if( eOp
& (WO_GT
|WO_GE
) ){
3076 testcase( eOp
& WO_GT
);
3077 testcase( eOp
& WO_GE
);
3078 pNew
->wsFlags
|= WHERE_COLUMN_RANGE
|WHERE_BTM_LIMIT
;
3079 pNew
->u
.btree
.nBtm
= nVecLen
;
3082 if( pTerm
->wtFlags
& TERM_LIKEOPT
){
3083 /* Range constraints that come from the LIKE optimization are
3084 ** always used in pairs. */
3086 assert( (pTop
-(pTerm
->pWC
->a
))<pTerm
->pWC
->nTerm
);
3087 assert( pTop
->wtFlags
& TERM_LIKEOPT
);
3088 assert( pTop
->eOperator
==WO_LT
);
3089 if( whereLoopResize(db
, pNew
, pNew
->nLTerm
+1) ) break; /* OOM */
3090 pNew
->aLTerm
[pNew
->nLTerm
++] = pTop
;
3091 pNew
->wsFlags
|= WHERE_TOP_LIMIT
;
3092 pNew
->u
.btree
.nTop
= 1;
3095 assert( eOp
& (WO_LT
|WO_LE
) );
3096 testcase( eOp
& WO_LT
);
3097 testcase( eOp
& WO_LE
);
3098 pNew
->wsFlags
|= WHERE_COLUMN_RANGE
|WHERE_TOP_LIMIT
;
3099 pNew
->u
.btree
.nTop
= nVecLen
;
3101 pBtm
= (pNew
->wsFlags
& WHERE_BTM_LIMIT
)!=0 ?
3102 pNew
->aLTerm
[pNew
->nLTerm
-2] : 0;
3106 /* At this point pNew->nOut is set to the number of rows expected to
3107 ** be visited by the index scan before considering term pTerm, or the
3108 ** values of nIn and nInMul. In other words, assuming that all
3109 ** "x IN(...)" terms are replaced with "x = ?". This block updates
3110 ** the value of pNew->nOut to account for pTerm (but not nIn/nInMul). */
3111 assert( pNew
->nOut
==saved_nOut
);
3112 if( pNew
->wsFlags
& WHERE_COLUMN_RANGE
){
3113 /* Adjust nOut using stat4 data. Or, if there is no stat4
3114 ** data, using some other estimate. */
3115 whereRangeScanEst(pParse
, pBuilder
, pBtm
, pTop
, pNew
);
3117 int nEq
= ++pNew
->u
.btree
.nEq
;
3118 assert( eOp
& (WO_ISNULL
|WO_EQ
|WO_IN
|WO_IS
) );
3120 assert( pNew
->nOut
==saved_nOut
);
3121 if( pTerm
->truthProb
<=0 && pProbe
->aiColumn
[saved_nEq
]>=0 ){
3122 assert( (eOp
& WO_IN
) || nIn
==0 );
3123 testcase( eOp
& WO_IN
);
3124 pNew
->nOut
+= pTerm
->truthProb
;
3127 #ifdef SQLITE_ENABLE_STAT4
3131 && ALWAYS(pNew
->u
.btree
.nEq
<=pProbe
->nSampleCol
)
3132 && ((eOp
& WO_IN
)==0 || ExprUseXList(pTerm
->pExpr
))
3133 && OptimizationEnabled(db
, SQLITE_Stat4
)
3135 Expr
*pExpr
= pTerm
->pExpr
;
3136 if( (eOp
& (WO_EQ
|WO_ISNULL
|WO_IS
))!=0 ){
3137 testcase( eOp
& WO_EQ
);
3138 testcase( eOp
& WO_IS
);
3139 testcase( eOp
& WO_ISNULL
);
3140 rc
= whereEqualScanEst(pParse
, pBuilder
, pExpr
->pRight
, &nOut
);
3142 rc
= whereInScanEst(pParse
, pBuilder
, pExpr
->x
.pList
, &nOut
);
3144 if( rc
==SQLITE_NOTFOUND
) rc
= SQLITE_OK
;
3145 if( rc
!=SQLITE_OK
) break; /* Jump out of the pTerm loop */
3147 pNew
->nOut
= sqlite3LogEst(nOut
);
3149 /* TUNING: Mark terms as "low selectivity" if they seem likely
3150 ** to be true for half or more of the rows in the table.
3151 ** See tag-202002240-1 */
3152 && pNew
->nOut
+10 > pProbe
->aiRowLogEst
[0]
3154 #if WHERETRACE_ENABLED /* 0x01 */
3155 if( sqlite3WhereTrace
& 0x20 ){
3157 "STAT4 determines term has low selectivity:\n");
3158 sqlite3WhereTermPrint(pTerm
, 999);
3161 pTerm
->wtFlags
|= TERM_HIGHTRUTH
;
3162 if( pTerm
->wtFlags
& TERM_HEURTRUTH
){
3163 /* If the term has previously been used with an assumption of
3164 ** higher selectivity, then set the flag to rerun the
3165 ** loop computations. */
3166 pBuilder
->bldFlags2
|= SQLITE_BLDF2_2NDPASS
;
3169 if( pNew
->nOut
>saved_nOut
) pNew
->nOut
= saved_nOut
;
3176 pNew
->nOut
+= (pProbe
->aiRowLogEst
[nEq
] - pProbe
->aiRowLogEst
[nEq
-1]);
3177 if( eOp
& WO_ISNULL
){
3178 /* TUNING: If there is no likelihood() value, assume that a
3179 ** "col IS NULL" expression matches twice as many rows
3187 /* Set rCostIdx to the cost of visiting selected rows in index. Add
3188 ** it to pNew->rRun, which is currently set to the cost of the index
3189 ** seek only. Then, if this is a non-covering index, add the cost of
3190 ** visiting the rows in the main table. */
3191 assert( pSrc
->pTab
->szTabRow
>0 );
3192 if( pProbe
->idxType
==SQLITE_IDXTYPE_IPK
){
3193 /* The pProbe->szIdxRow is low for an IPK table since the interior
3194 ** pages are small. Thus szIdxRow gives a good estimate of seek cost.
3195 ** But the leaf pages are full-size, so pProbe->szIdxRow would badly
3196 ** under-estimate the scanning cost. */
3197 rCostIdx
= pNew
->nOut
+ 16;
3199 rCostIdx
= pNew
->nOut
+ 1 + (15*pProbe
->szIdxRow
)/pSrc
->pTab
->szTabRow
;
3201 pNew
->rRun
= sqlite3LogEstAdd(rLogSize
, rCostIdx
);
3202 if( (pNew
->wsFlags
& (WHERE_IDX_ONLY
|WHERE_IPK
|WHERE_EXPRIDX
))==0 ){
3203 pNew
->rRun
= sqlite3LogEstAdd(pNew
->rRun
, pNew
->nOut
+ 16);
3205 ApplyCostMultiplier(pNew
->rRun
, pProbe
->pTable
->costMult
);
3207 nOutUnadjusted
= pNew
->nOut
;
3208 pNew
->rRun
+= nInMul
+ nIn
;
3209 pNew
->nOut
+= nInMul
+ nIn
;
3210 whereLoopOutputAdjust(pBuilder
->pWC
, pNew
, rSize
);
3211 rc
= whereLoopInsert(pBuilder
, pNew
);
3213 if( pNew
->wsFlags
& WHERE_COLUMN_RANGE
){
3214 pNew
->nOut
= saved_nOut
;
3216 pNew
->nOut
= nOutUnadjusted
;
3219 if( (pNew
->wsFlags
& WHERE_TOP_LIMIT
)==0
3220 && pNew
->u
.btree
.nEq
<pProbe
->nColumn
3221 && (pNew
->u
.btree
.nEq
<pProbe
->nKeyCol
||
3222 pProbe
->idxType
!=SQLITE_IDXTYPE_PRIMARYKEY
)
3224 if( pNew
->u
.btree
.nEq
>3 ){
3225 sqlite3ProgressCheck(pParse
);
3227 whereLoopAddBtreeIndex(pBuilder
, pSrc
, pProbe
, nInMul
+nIn
);
3229 pNew
->nOut
= saved_nOut
;
3230 #ifdef SQLITE_ENABLE_STAT4
3231 pBuilder
->nRecValid
= nRecValid
;
3234 pNew
->prereq
= saved_prereq
;
3235 pNew
->u
.btree
.nEq
= saved_nEq
;
3236 pNew
->u
.btree
.nBtm
= saved_nBtm
;
3237 pNew
->u
.btree
.nTop
= saved_nTop
;
3238 pNew
->nSkip
= saved_nSkip
;
3239 pNew
->wsFlags
= saved_wsFlags
;
3240 pNew
->nOut
= saved_nOut
;
3241 pNew
->nLTerm
= saved_nLTerm
;
3243 /* Consider using a skip-scan if there are no WHERE clause constraints
3244 ** available for the left-most terms of the index, and if the average
3245 ** number of repeats in the left-most terms is at least 18.
3247 ** The magic number 18 is selected on the basis that scanning 17 rows
3248 ** is almost always quicker than an index seek (even though if the index
3249 ** contains fewer than 2^17 rows we assume otherwise in other parts of
3250 ** the code). And, even if it is not, it should not be too much slower.
3251 ** On the other hand, the extra seeks could end up being significantly
3252 ** more expensive. */
3253 assert( 42==sqlite3LogEst(18) );
3254 if( saved_nEq
==saved_nSkip
3255 && saved_nEq
+1<pProbe
->nKeyCol
3256 && saved_nEq
==pNew
->nLTerm
3257 && pProbe
->noSkipScan
==0
3258 && pProbe
->hasStat1
!=0
3259 && OptimizationEnabled(db
, SQLITE_SkipScan
)
3260 && pProbe
->aiRowLogEst
[saved_nEq
+1]>=42 /* TUNING: Minimum for skip-scan */
3261 && (rc
= whereLoopResize(db
, pNew
, pNew
->nLTerm
+1))==SQLITE_OK
3264 pNew
->u
.btree
.nEq
++;
3266 pNew
->aLTerm
[pNew
->nLTerm
++] = 0;
3267 pNew
->wsFlags
|= WHERE_SKIPSCAN
;
3268 nIter
= pProbe
->aiRowLogEst
[saved_nEq
] - pProbe
->aiRowLogEst
[saved_nEq
+1];
3269 pNew
->nOut
-= nIter
;
3270 /* TUNING: Because uncertainties in the estimates for skip-scan queries,
3271 ** add a 1.375 fudge factor to make skip-scan slightly less likely. */
3273 whereLoopAddBtreeIndex(pBuilder
, pSrc
, pProbe
, nIter
+ nInMul
);
3274 pNew
->nOut
= saved_nOut
;
3275 pNew
->u
.btree
.nEq
= saved_nEq
;
3276 pNew
->nSkip
= saved_nSkip
;
3277 pNew
->wsFlags
= saved_wsFlags
;
3280 WHERETRACE(0x800, ("END %s.addBtreeIdx(%s), nEq=%d, rc=%d\n",
3281 pProbe
->pTable
->zName
, pProbe
->zName
, saved_nEq
, rc
));
3286 ** Return True if it is possible that pIndex might be useful in
3287 ** implementing the ORDER BY clause in pBuilder.
3289 ** Return False if pBuilder does not contain an ORDER BY clause or
3290 ** if there is no way for pIndex to be useful in implementing that
3293 static int indexMightHelpWithOrderBy(
3294 WhereLoopBuilder
*pBuilder
,
3302 if( pIndex
->bUnordered
) return 0;
3303 if( (pOB
= pBuilder
->pWInfo
->pOrderBy
)==0 ) return 0;
3304 for(ii
=0; ii
<pOB
->nExpr
; ii
++){
3305 Expr
*pExpr
= sqlite3ExprSkipCollateAndLikely(pOB
->a
[ii
].pExpr
);
3306 if( NEVER(pExpr
==0) ) continue;
3307 if( pExpr
->op
==TK_COLUMN
&& pExpr
->iTable
==iCursor
){
3308 if( pExpr
->iColumn
<0 ) return 1;
3309 for(jj
=0; jj
<pIndex
->nKeyCol
; jj
++){
3310 if( pExpr
->iColumn
==pIndex
->aiColumn
[jj
] ) return 1;
3312 }else if( (aColExpr
= pIndex
->aColExpr
)!=0 ){
3313 for(jj
=0; jj
<pIndex
->nKeyCol
; jj
++){
3314 if( pIndex
->aiColumn
[jj
]!=XN_EXPR
) continue;
3315 if( sqlite3ExprCompareSkip(pExpr
,aColExpr
->a
[jj
].pExpr
,iCursor
)==0 ){
3324 /* Check to see if a partial index with pPartIndexWhere can be used
3325 ** in the current query. Return true if it can be and false if not.
3327 static int whereUsablePartialIndex(
3328 int iTab
, /* The table for which we want an index */
3329 u8 jointype
, /* The JT_* flags on the join */
3330 WhereClause
*pWC
, /* The WHERE clause of the query */
3331 Expr
*pWhere
/* The WHERE clause from the partial index */
3337 if( jointype
& JT_LTORJ
) return 0;
3338 pParse
= pWC
->pWInfo
->pParse
;
3339 while( pWhere
->op
==TK_AND
){
3340 if( !whereUsablePartialIndex(iTab
,jointype
,pWC
,pWhere
->pLeft
) ) return 0;
3341 pWhere
= pWhere
->pRight
;
3343 if( pParse
->db
->flags
& SQLITE_EnableQPSG
) pParse
= 0;
3344 for(i
=0, pTerm
=pWC
->a
; i
<pWC
->nTerm
; i
++, pTerm
++){
3346 pExpr
= pTerm
->pExpr
;
3347 if( (!ExprHasProperty(pExpr
, EP_OuterON
) || pExpr
->w
.iJoin
==iTab
)
3348 && ((jointype
& JT_OUTER
)==0 || ExprHasProperty(pExpr
, EP_OuterON
))
3349 && sqlite3ExprImpliesExpr(pParse
, pExpr
, pWhere
, iTab
)
3350 && (pTerm
->wtFlags
& TERM_VNULL
)==0
3359 ** pIdx is an index containing expressions. Check it see if any of the
3360 ** expressions in the index match the pExpr expression.
3362 static int exprIsCoveredByIndex(
3368 for(i
=0; i
<pIdx
->nColumn
; i
++){
3369 if( pIdx
->aiColumn
[i
]==XN_EXPR
3370 && sqlite3ExprCompare(0, pExpr
, pIdx
->aColExpr
->a
[i
].pExpr
, iTabCur
)==0
3379 ** Structure passed to the whereIsCoveringIndex Walker callback.
3381 typedef struct CoveringIndexCheck CoveringIndexCheck
;
3382 struct CoveringIndexCheck
{
3383 Index
*pIdx
; /* The index */
3384 int iTabCur
; /* Cursor number for the corresponding table */
3385 u8 bExpr
; /* Uses an indexed expression */
3386 u8 bUnidx
; /* Uses an unindexed column not within an indexed expr */
3390 ** Information passed in is pWalk->u.pCovIdxCk. Call it pCk.
3392 ** If the Expr node references the table with cursor pCk->iTabCur, then
3393 ** make sure that column is covered by the index pCk->pIdx. We know that
3394 ** all columns less than 63 (really BMS-1) are covered, so we don't need
3395 ** to check them. But we do need to check any column at 63 or greater.
3397 ** If the index does not cover the column, then set pWalk->eCode to
3398 ** non-zero and return WRC_Abort to stop the search.
3400 ** If this node does not disprove that the index can be a covering index,
3401 ** then just return WRC_Continue, to continue the search.
3403 ** If pCk->pIdx contains indexed expressions and one of those expressions
3404 ** matches pExpr, then prune the search.
3406 static int whereIsCoveringIndexWalkCallback(Walker
*pWalk
, Expr
*pExpr
){
3407 int i
; /* Loop counter */
3408 const Index
*pIdx
; /* The index of interest */
3409 const i16
*aiColumn
; /* Columns contained in the index */
3410 u16 nColumn
; /* Number of columns in the index */
3411 CoveringIndexCheck
*pCk
; /* Info about this search */
3413 pCk
= pWalk
->u
.pCovIdxCk
;
3415 if( (pExpr
->op
==TK_COLUMN
|| pExpr
->op
==TK_AGG_COLUMN
) ){
3416 /* if( pExpr->iColumn<(BMS-1) && pIdx->bHasExpr==0 ) return WRC_Continue;*/
3417 if( pExpr
->iTable
!=pCk
->iTabCur
) return WRC_Continue
;
3418 pIdx
= pWalk
->u
.pCovIdxCk
->pIdx
;
3419 aiColumn
= pIdx
->aiColumn
;
3420 nColumn
= pIdx
->nColumn
;
3421 for(i
=0; i
<nColumn
; i
++){
3422 if( aiColumn
[i
]==pExpr
->iColumn
) return WRC_Continue
;
3426 }else if( pIdx
->bHasExpr
3427 && exprIsCoveredByIndex(pExpr
, pIdx
, pWalk
->u
.pCovIdxCk
->iTabCur
) ){
3431 return WRC_Continue
;
3436 ** pIdx is an index that covers all of the low-number columns used by
3437 ** pWInfo->pSelect (columns from 0 through 62) or an index that has
3438 ** expressions terms. Hence, we cannot determine whether or not it is
3439 ** a covering index by using the colUsed bitmasks. We have to do a search
3440 ** to see if the index is covering. This routine does that search.
3442 ** The return value is one of these:
3444 ** 0 The index is definitely not a covering index
3446 ** WHERE_IDX_ONLY The index is definitely a covering index
3448 ** WHERE_EXPRIDX The index is likely a covering index, but it is
3449 ** difficult to determine precisely because of the
3450 ** expressions that are indexed. Score it as a
3451 ** covering index, but still keep the main table open
3452 ** just in case we need it.
3454 ** This routine is an optimization. It is always safe to return zero.
3455 ** But returning one of the other two values when zero should have been
3456 ** returned can lead to incorrect bytecode and assertion faults.
3458 static SQLITE_NOINLINE u32
whereIsCoveringIndex(
3459 WhereInfo
*pWInfo
, /* The WHERE clause context */
3460 Index
*pIdx
, /* Index that is being tested */
3461 int iTabCur
/* Cursor for the table being indexed */
3464 struct CoveringIndexCheck ck
;
3466 if( pWInfo
->pSelect
==0 ){
3467 /* We don't have access to the full query, so we cannot check to see
3468 ** if pIdx is covering. Assume it is not. */
3471 if( pIdx
->bHasExpr
==0 ){
3472 for(i
=0; i
<pIdx
->nColumn
; i
++){
3473 if( pIdx
->aiColumn
[i
]>=BMS
-1 ) break;
3475 if( i
>=pIdx
->nColumn
){
3476 /* pIdx does not index any columns greater than 62, but we know from
3477 ** colMask that columns greater than 62 are used, so this is not a
3478 ** covering index */
3483 ck
.iTabCur
= iTabCur
;
3486 memset(&w
, 0, sizeof(w
));
3487 w
.xExprCallback
= whereIsCoveringIndexWalkCallback
;
3488 w
.xSelectCallback
= sqlite3SelectWalkNoop
;
3489 w
.u
.pCovIdxCk
= &ck
;
3490 sqlite3WalkSelect(&w
, pWInfo
->pSelect
);
3493 }else if( ck
.bExpr
){
3496 rc
= WHERE_IDX_ONLY
;
3502 ** Add all WhereLoop objects for a single table of the join where the table
3503 ** is identified by pBuilder->pNew->iTab. That table is guaranteed to be
3504 ** a b-tree table, not a virtual table.
3506 ** The costs (WhereLoop.rRun) of the b-tree loops added by this function
3507 ** are calculated as follows:
3509 ** For a full scan, assuming the table (or index) contains nRow rows:
3511 ** cost = nRow * 3.0 // full-table scan
3512 ** cost = nRow * K // scan of covering index
3513 ** cost = nRow * (K+3.0) // scan of non-covering index
3515 ** where K is a value between 1.1 and 3.0 set based on the relative
3516 ** estimated average size of the index and table records.
3518 ** For an index scan, where nVisit is the number of index rows visited
3519 ** by the scan, and nSeek is the number of seek operations required on
3520 ** the index b-tree:
3522 ** cost = nSeek * (log(nRow) + K * nVisit) // covering index
3523 ** cost = nSeek * (log(nRow) + (K+3.0) * nVisit) // non-covering index
3525 ** Normally, nSeek is 1. nSeek values greater than 1 come about if the
3526 ** WHERE clause includes "x IN (....)" terms used in place of "x=?". Or when
3527 ** implicit "x IN (SELECT x FROM tbl)" terms are added for skip-scans.
3529 ** The estimated values (nRow, nVisit, nSeek) often contain a large amount
3530 ** of uncertainty. For this reason, scoring is designed to pick plans that
3531 ** "do the least harm" if the estimates are inaccurate. For example, a
3532 ** log(nRow) factor is omitted from a non-covering index scan in order to
3533 ** bias the scoring in favor of using an index, since the worst-case
3534 ** performance of using an index is far better than the worst-case performance
3535 ** of a full table scan.
3537 static int whereLoopAddBtree(
3538 WhereLoopBuilder
*pBuilder
, /* WHERE clause information */
3539 Bitmask mPrereq
/* Extra prerequisites for using this table */
3541 WhereInfo
*pWInfo
; /* WHERE analysis context */
3542 Index
*pProbe
; /* An index we are evaluating */
3543 Index sPk
; /* A fake index object for the primary key */
3544 LogEst aiRowEstPk
[2]; /* The aiRowLogEst[] value for the sPk index */
3545 i16 aiColumnPk
= -1; /* The aColumn[] value for the sPk index */
3546 SrcList
*pTabList
; /* The FROM clause */
3547 SrcItem
*pSrc
; /* The FROM clause btree term to add */
3548 WhereLoop
*pNew
; /* Template WhereLoop object */
3549 int rc
= SQLITE_OK
; /* Return code */
3550 int iSortIdx
= 1; /* Index number */
3551 int b
; /* A boolean value */
3552 LogEst rSize
; /* number of rows in the table */
3553 WhereClause
*pWC
; /* The parsed WHERE clause */
3554 Table
*pTab
; /* Table being queried */
3556 pNew
= pBuilder
->pNew
;
3557 pWInfo
= pBuilder
->pWInfo
;
3558 pTabList
= pWInfo
->pTabList
;
3559 pSrc
= pTabList
->a
+ pNew
->iTab
;
3561 pWC
= pBuilder
->pWC
;
3562 assert( !IsVirtual(pSrc
->pTab
) );
3564 if( pSrc
->fg
.isIndexedBy
){
3565 assert( pSrc
->fg
.isCte
==0 );
3566 /* An INDEXED BY clause specifies a particular index to use */
3567 pProbe
= pSrc
->u2
.pIBIndex
;
3568 }else if( !HasRowid(pTab
) ){
3569 pProbe
= pTab
->pIndex
;
3571 /* There is no INDEXED BY clause. Create a fake Index object in local
3572 ** variable sPk to represent the rowid primary key index. Make this
3573 ** fake index the first in a chain of Index objects with all of the real
3574 ** indices to follow */
3575 Index
*pFirst
; /* First of real indices on the table */
3576 memset(&sPk
, 0, sizeof(Index
));
3579 sPk
.aiColumn
= &aiColumnPk
;
3580 sPk
.aiRowLogEst
= aiRowEstPk
;
3581 sPk
.onError
= OE_Replace
;
3583 sPk
.szIdxRow
= 3; /* TUNING: Interior rows of IPK table are very small */
3584 sPk
.idxType
= SQLITE_IDXTYPE_IPK
;
3585 aiRowEstPk
[0] = pTab
->nRowLogEst
;
3587 pFirst
= pSrc
->pTab
->pIndex
;
3588 if( pSrc
->fg
.notIndexed
==0 ){
3589 /* The real indices of the table are only considered if the
3590 ** NOT INDEXED qualifier is omitted from the FROM clause */
3595 rSize
= pTab
->nRowLogEst
;
3597 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
3598 /* Automatic indexes */
3599 if( !pBuilder
->pOrSet
/* Not part of an OR optimization */
3600 && (pWInfo
->wctrlFlags
& (WHERE_RIGHT_JOIN
|WHERE_OR_SUBCLAUSE
))==0
3601 && (pWInfo
->pParse
->db
->flags
& SQLITE_AutoIndex
)!=0
3602 && !pSrc
->fg
.isIndexedBy
/* Has no INDEXED BY clause */
3603 && !pSrc
->fg
.notIndexed
/* Has no NOT INDEXED clause */
3604 && HasRowid(pTab
) /* Not WITHOUT ROWID table. (FIXME: Why not?) */
3605 && !pSrc
->fg
.isCorrelated
/* Not a correlated subquery */
3606 && !pSrc
->fg
.isRecursive
/* Not a recursive common table expression. */
3607 && (pSrc
->fg
.jointype
& JT_RIGHT
)==0 /* Not the right tab of a RIGHT JOIN */
3609 /* Generate auto-index WhereLoops */
3610 LogEst rLogSize
; /* Logarithm of the number of rows in the table */
3612 WhereTerm
*pWCEnd
= pWC
->a
+ pWC
->nTerm
;
3613 rLogSize
= estLog(rSize
);
3614 for(pTerm
=pWC
->a
; rc
==SQLITE_OK
&& pTerm
<pWCEnd
; pTerm
++){
3615 if( pTerm
->prereqRight
& pNew
->maskSelf
) continue;
3616 if( termCanDriveIndex(pTerm
, pSrc
, 0) ){
3617 pNew
->u
.btree
.nEq
= 1;
3619 pNew
->u
.btree
.pIndex
= 0;
3621 pNew
->aLTerm
[0] = pTerm
;
3622 /* TUNING: One-time cost for computing the automatic index is
3623 ** estimated to be X*N*log2(N) where N is the number of rows in
3624 ** the table being indexed and where X is 7 (LogEst=28) for normal
3625 ** tables or 0.5 (LogEst=-10) for views and subqueries. The value
3626 ** of X is smaller for views and subqueries so that the query planner
3627 ** will be more aggressive about generating automatic indexes for
3628 ** those objects, since there is no opportunity to add schema
3629 ** indexes on subqueries and views. */
3630 pNew
->rSetup
= rLogSize
+ rSize
;
3631 if( !IsView(pTab
) && (pTab
->tabFlags
& TF_Ephemeral
)==0 ){
3634 pNew
->rSetup
-= 25; /* Greatly reduced setup cost for auto indexes
3635 ** on ephemeral materializations of views */
3637 ApplyCostMultiplier(pNew
->rSetup
, pTab
->costMult
);
3638 if( pNew
->rSetup
<0 ) pNew
->rSetup
= 0;
3639 /* TUNING: Each index lookup yields 20 rows in the table. This
3640 ** is more than the usual guess of 10 rows, since we have no way
3641 ** of knowing how selective the index will ultimately be. It would
3642 ** not be unreasonable to make this value much larger. */
3643 pNew
->nOut
= 43; assert( 43==sqlite3LogEst(20) );
3644 pNew
->rRun
= sqlite3LogEstAdd(rLogSize
,pNew
->nOut
);
3645 pNew
->wsFlags
= WHERE_AUTO_INDEX
;
3646 pNew
->prereq
= mPrereq
| pTerm
->prereqRight
;
3647 rc
= whereLoopInsert(pBuilder
, pNew
);
3651 #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */
3653 /* Loop over all indices. If there was an INDEXED BY clause, then only
3654 ** consider index pProbe. */
3655 for(; rc
==SQLITE_OK
&& pProbe
;
3656 pProbe
=(pSrc
->fg
.isIndexedBy
? 0 : pProbe
->pNext
), iSortIdx
++
3658 if( pProbe
->pPartIdxWhere
!=0
3659 && !whereUsablePartialIndex(pSrc
->iCursor
, pSrc
->fg
.jointype
, pWC
,
3660 pProbe
->pPartIdxWhere
)
3662 testcase( pNew
->iTab
!=pSrc
->iCursor
); /* See ticket [98d973b8f5] */
3663 continue; /* Partial index inappropriate for this query */
3665 if( pProbe
->bNoQuery
) continue;
3666 rSize
= pProbe
->aiRowLogEst
[0];
3667 pNew
->u
.btree
.nEq
= 0;
3668 pNew
->u
.btree
.nBtm
= 0;
3669 pNew
->u
.btree
.nTop
= 0;
3674 pNew
->prereq
= mPrereq
;
3676 pNew
->u
.btree
.pIndex
= pProbe
;
3677 b
= indexMightHelpWithOrderBy(pBuilder
, pProbe
, pSrc
->iCursor
);
3679 /* The ONEPASS_DESIRED flags never occurs together with ORDER BY */
3680 assert( (pWInfo
->wctrlFlags
& WHERE_ONEPASS_DESIRED
)==0 || b
==0 );
3681 if( pProbe
->idxType
==SQLITE_IDXTYPE_IPK
){
3682 /* Integer primary key index */
3683 pNew
->wsFlags
= WHERE_IPK
;
3685 /* Full table scan */
3686 pNew
->iSortIdx
= b
? iSortIdx
: 0;
3687 /* TUNING: Cost of full table scan is 3.0*N. The 3.0 factor is an
3688 ** extra cost designed to discourage the use of full table scans,
3689 ** since index lookups have better worst-case performance if our
3690 ** stat guesses are wrong. Reduce the 3.0 penalty slightly
3691 ** (to 2.75) if we have valid STAT4 information for the table.
3692 ** At 2.75, a full table scan is preferred over using an index on
3693 ** a column with just two distinct values where each value has about
3694 ** an equal number of appearances. Without STAT4 data, we still want
3695 ** to use an index in that case, since the constraint might be for
3696 ** the scarcer of the two values, and in that case an index lookup is
3699 #ifdef SQLITE_ENABLE_STAT4
3700 pNew
->rRun
= rSize
+ 16 - 2*((pTab
->tabFlags
& TF_HasStat4
)!=0);
3702 pNew
->rRun
= rSize
+ 16;
3704 ApplyCostMultiplier(pNew
->rRun
, pTab
->costMult
);
3705 whereLoopOutputAdjust(pWC
, pNew
, rSize
);
3706 rc
= whereLoopInsert(pBuilder
, pNew
);
3711 if( pProbe
->isCovering
){
3713 pNew
->wsFlags
= WHERE_IDX_ONLY
| WHERE_INDEXED
;
3715 m
= pSrc
->colUsed
& pProbe
->colNotIdxed
;
3716 pNew
->wsFlags
= WHERE_INDEXED
;
3717 if( m
==TOPBIT
|| (pProbe
->bHasExpr
&& !pProbe
->bHasVCol
&& m
!=0) ){
3718 u32 isCov
= whereIsCoveringIndex(pWInfo
, pProbe
, pSrc
->iCursor
);
3721 ("-> %s is not a covering index"
3722 " according to whereIsCoveringIndex()\n", pProbe
->zName
));
3726 pNew
->wsFlags
|= isCov
;
3727 if( isCov
& WHERE_IDX_ONLY
){
3729 ("-> %s is a covering expression index"
3730 " according to whereIsCoveringIndex()\n", pProbe
->zName
));
3732 assert( isCov
==WHERE_EXPRIDX
);
3734 ("-> %s might be a covering expression index"
3735 " according to whereIsCoveringIndex()\n", pProbe
->zName
));
3740 ("-> %s a covering index according to bitmasks\n",
3741 pProbe
->zName
, m
==0 ? "is" : "is not"));
3742 pNew
->wsFlags
= WHERE_IDX_ONLY
| WHERE_INDEXED
;
3746 /* Full scan via index */
3749 || pProbe
->pPartIdxWhere
!=0
3750 || pSrc
->fg
.isIndexedBy
3752 && pProbe
->bUnordered
==0
3753 && (pProbe
->szIdxRow
<pTab
->szTabRow
)
3754 && (pWInfo
->wctrlFlags
& WHERE_ONEPASS_DESIRED
)==0
3755 && sqlite3GlobalConfig
.bUseCis
3756 && OptimizationEnabled(pWInfo
->pParse
->db
, SQLITE_CoverIdxScan
)
3759 pNew
->iSortIdx
= b
? iSortIdx
: 0;
3761 /* The cost of visiting the index rows is N*K, where K is
3762 ** between 1.1 and 3.0, depending on the relative sizes of the
3763 ** index and table rows. */
3764 pNew
->rRun
= rSize
+ 1 + (15*pProbe
->szIdxRow
)/pTab
->szTabRow
;
3766 /* If this is a non-covering index scan, add in the cost of
3767 ** doing table lookups. The cost will be 3x the number of
3768 ** lookups. Take into account WHERE clause terms that can be
3769 ** satisfied using just the index, and that do not require a
3771 LogEst nLookup
= rSize
+ 16; /* Base cost: N*3 */
3773 int iCur
= pSrc
->iCursor
;
3774 WhereClause
*pWC2
= &pWInfo
->sWC
;
3775 for(ii
=0; ii
<pWC2
->nTerm
; ii
++){
3776 WhereTerm
*pTerm
= &pWC2
->a
[ii
];
3777 if( !sqlite3ExprCoveredByIndex(pTerm
->pExpr
, iCur
, pProbe
) ){
3780 /* pTerm can be evaluated using just the index. So reduce
3781 ** the expected number of table lookups accordingly */
3782 if( pTerm
->truthProb
<=0 ){
3783 nLookup
+= pTerm
->truthProb
;
3786 if( pTerm
->eOperator
& (WO_EQ
|WO_IS
) ) nLookup
-= 19;
3790 pNew
->rRun
= sqlite3LogEstAdd(pNew
->rRun
, nLookup
);
3792 ApplyCostMultiplier(pNew
->rRun
, pTab
->costMult
);
3793 whereLoopOutputAdjust(pWC
, pNew
, rSize
);
3794 if( (pSrc
->fg
.jointype
& JT_RIGHT
)!=0 && pProbe
->aColExpr
){
3795 /* Do not do an SCAN of a index-on-expression in a RIGHT JOIN
3796 ** because the cursor used to access the index might not be
3797 ** positioned to the correct row during the right-join no-match
3800 rc
= whereLoopInsert(pBuilder
, pNew
);
3807 pBuilder
->bldFlags1
= 0;
3808 rc
= whereLoopAddBtreeIndex(pBuilder
, pSrc
, pProbe
, 0);
3809 if( pBuilder
->bldFlags1
==SQLITE_BLDF1_INDEXED
){
3810 /* If a non-unique index is used, or if a prefix of the key for
3811 ** unique index is used (making the index functionally non-unique)
3812 ** then the sqlite_stat1 data becomes important for scoring the
3814 pTab
->tabFlags
|= TF_StatsUsed
;
3816 #ifdef SQLITE_ENABLE_STAT4
3817 sqlite3Stat4ProbeFree(pBuilder
->pRec
);
3818 pBuilder
->nRecValid
= 0;
3825 #ifndef SQLITE_OMIT_VIRTUALTABLE
3828 ** Return true if pTerm is a virtual table LIMIT or OFFSET term.
3830 static int isLimitTerm(WhereTerm
*pTerm
){
3831 assert( pTerm
->eOperator
==WO_AUX
|| pTerm
->eMatchOp
==0 );
3832 return pTerm
->eMatchOp
>=SQLITE_INDEX_CONSTRAINT_LIMIT
3833 && pTerm
->eMatchOp
<=SQLITE_INDEX_CONSTRAINT_OFFSET
;
3837 ** Argument pIdxInfo is already populated with all constraints that may
3838 ** be used by the virtual table identified by pBuilder->pNew->iTab. This
3839 ** function marks a subset of those constraints usable, invokes the
3840 ** xBestIndex method and adds the returned plan to pBuilder.
3842 ** A constraint is marked usable if:
3844 ** * Argument mUsable indicates that its prerequisites are available, and
3846 ** * It is not one of the operators specified in the mExclude mask passed
3847 ** as the fourth argument (which in practice is either WO_IN or 0).
3849 ** Argument mPrereq is a mask of tables that must be scanned before the
3850 ** virtual table in question. These are added to the plans prerequisites
3851 ** before it is added to pBuilder.
3853 ** Output parameter *pbIn is set to true if the plan added to pBuilder
3854 ** uses one or more WO_IN terms, or false otherwise.
3856 static int whereLoopAddVirtualOne(
3857 WhereLoopBuilder
*pBuilder
,
3858 Bitmask mPrereq
, /* Mask of tables that must be used. */
3859 Bitmask mUsable
, /* Mask of usable tables */
3860 u16 mExclude
, /* Exclude terms using these operators */
3861 sqlite3_index_info
*pIdxInfo
, /* Populated object for xBestIndex */
3862 u16 mNoOmit
, /* Do not omit these constraints */
3863 int *pbIn
, /* OUT: True if plan uses an IN(...) op */
3864 int *pbRetryLimit
/* OUT: Retry without LIMIT/OFFSET */
3866 WhereClause
*pWC
= pBuilder
->pWC
;
3867 HiddenIndexInfo
*pHidden
= (HiddenIndexInfo
*)&pIdxInfo
[1];
3868 struct sqlite3_index_constraint
*pIdxCons
;
3869 struct sqlite3_index_constraint_usage
*pUsage
= pIdxInfo
->aConstraintUsage
;
3873 WhereLoop
*pNew
= pBuilder
->pNew
;
3874 Parse
*pParse
= pBuilder
->pWInfo
->pParse
;
3875 SrcItem
*pSrc
= &pBuilder
->pWInfo
->pTabList
->a
[pNew
->iTab
];
3876 int nConstraint
= pIdxInfo
->nConstraint
;
3878 assert( (mUsable
& mPrereq
)==mPrereq
);
3880 pNew
->prereq
= mPrereq
;
3882 /* Set the usable flag on the subset of constraints identified by
3883 ** arguments mUsable and mExclude. */
3884 pIdxCons
= *(struct sqlite3_index_constraint
**)&pIdxInfo
->aConstraint
;
3885 for(i
=0; i
<nConstraint
; i
++, pIdxCons
++){
3886 WhereTerm
*pTerm
= &pWC
->a
[pIdxCons
->iTermOffset
];
3887 pIdxCons
->usable
= 0;
3888 if( (pTerm
->prereqRight
& mUsable
)==pTerm
->prereqRight
3889 && (pTerm
->eOperator
& mExclude
)==0
3890 && (pbRetryLimit
|| !isLimitTerm(pTerm
))
3892 pIdxCons
->usable
= 1;
3896 /* Initialize the output fields of the sqlite3_index_info structure */
3897 memset(pUsage
, 0, sizeof(pUsage
[0])*nConstraint
);
3898 assert( pIdxInfo
->needToFreeIdxStr
==0 );
3899 pIdxInfo
->idxStr
= 0;
3900 pIdxInfo
->idxNum
= 0;
3901 pIdxInfo
->orderByConsumed
= 0;
3902 pIdxInfo
->estimatedCost
= SQLITE_BIG_DBL
/ (double)2;
3903 pIdxInfo
->estimatedRows
= 25;
3904 pIdxInfo
->idxFlags
= 0;
3905 pIdxInfo
->colUsed
= (sqlite3_int64
)pSrc
->colUsed
;
3906 pHidden
->mHandleIn
= 0;
3908 /* Invoke the virtual table xBestIndex() method */
3909 rc
= vtabBestIndex(pParse
, pSrc
->pTab
, pIdxInfo
);
3911 if( rc
==SQLITE_CONSTRAINT
){
3912 /* If the xBestIndex method returns SQLITE_CONSTRAINT, that means
3913 ** that the particular combination of parameters provided is unusable.
3914 ** Make no entries in the loop table.
3916 WHERETRACE(0xffffffff, (" ^^^^--- non-viable plan rejected!\n"));
3923 assert( pNew
->nLSlot
>=nConstraint
);
3924 memset(pNew
->aLTerm
, 0, sizeof(pNew
->aLTerm
[0])*nConstraint
);
3925 memset(&pNew
->u
.vtab
, 0, sizeof(pNew
->u
.vtab
));
3926 pIdxCons
= *(struct sqlite3_index_constraint
**)&pIdxInfo
->aConstraint
;
3927 for(i
=0; i
<nConstraint
; i
++, pIdxCons
++){
3929 if( (iTerm
= pUsage
[i
].argvIndex
- 1)>=0 ){
3931 int j
= pIdxCons
->iTermOffset
;
3932 if( iTerm
>=nConstraint
3935 || pNew
->aLTerm
[iTerm
]!=0
3936 || pIdxCons
->usable
==0
3938 sqlite3ErrorMsg(pParse
,"%s.xBestIndex malfunction",pSrc
->pTab
->zName
);
3939 testcase( pIdxInfo
->needToFreeIdxStr
);
3940 return SQLITE_ERROR
;
3942 testcase( iTerm
==nConstraint
-1 );
3944 testcase( j
==pWC
->nTerm
-1 );
3946 pNew
->prereq
|= pTerm
->prereqRight
;
3947 assert( iTerm
<pNew
->nLSlot
);
3948 pNew
->aLTerm
[iTerm
] = pTerm
;
3949 if( iTerm
>mxTerm
) mxTerm
= iTerm
;
3950 testcase( iTerm
==15 );
3951 testcase( iTerm
==16 );
3952 if( pUsage
[i
].omit
){
3953 if( i
<16 && ((1<<i
)&mNoOmit
)==0 ){
3954 testcase( i
!=iTerm
);
3955 pNew
->u
.vtab
.omitMask
|= 1<<iTerm
;
3957 testcase( i
!=iTerm
);
3959 if( pTerm
->eMatchOp
==SQLITE_INDEX_CONSTRAINT_OFFSET
){
3960 pNew
->u
.vtab
.bOmitOffset
= 1;
3963 if( SMASKBIT32(i
) & pHidden
->mHandleIn
){
3964 pNew
->u
.vtab
.mHandleIn
|= MASKBIT32(iTerm
);
3965 }else if( (pTerm
->eOperator
& WO_IN
)!=0 ){
3966 /* A virtual table that is constrained by an IN clause may not
3967 ** consume the ORDER BY clause because (1) the order of IN terms
3968 ** is not necessarily related to the order of output terms and
3969 ** (2) Multiple outputs from a single IN value will not merge
3971 pIdxInfo
->orderByConsumed
= 0;
3972 pIdxInfo
->idxFlags
&= ~SQLITE_INDEX_SCAN_UNIQUE
;
3973 *pbIn
= 1; assert( (mExclude
& WO_IN
)==0 );
3976 assert( pbRetryLimit
|| !isLimitTerm(pTerm
) );
3977 if( isLimitTerm(pTerm
) && *pbIn
){
3978 /* If there is an IN(...) term handled as an == (separate call to
3979 ** xFilter for each value on the RHS of the IN) and a LIMIT or
3980 ** OFFSET term handled as well, the plan is unusable. Set output
3981 ** variable *pbRetryLimit to true to tell the caller to retry with
3982 ** LIMIT and OFFSET disabled. */
3983 if( pIdxInfo
->needToFreeIdxStr
){
3984 sqlite3_free(pIdxInfo
->idxStr
);
3985 pIdxInfo
->idxStr
= 0;
3986 pIdxInfo
->needToFreeIdxStr
= 0;
3994 pNew
->nLTerm
= mxTerm
+1;
3995 for(i
=0; i
<=mxTerm
; i
++){
3996 if( pNew
->aLTerm
[i
]==0 ){
3997 /* The non-zero argvIdx values must be contiguous. Raise an
3998 ** error if they are not */
3999 sqlite3ErrorMsg(pParse
,"%s.xBestIndex malfunction",pSrc
->pTab
->zName
);
4000 testcase( pIdxInfo
->needToFreeIdxStr
);
4001 return SQLITE_ERROR
;
4004 assert( pNew
->nLTerm
<=pNew
->nLSlot
);
4005 pNew
->u
.vtab
.idxNum
= pIdxInfo
->idxNum
;
4006 pNew
->u
.vtab
.needFree
= pIdxInfo
->needToFreeIdxStr
;
4007 pIdxInfo
->needToFreeIdxStr
= 0;
4008 pNew
->u
.vtab
.idxStr
= pIdxInfo
->idxStr
;
4009 pNew
->u
.vtab
.isOrdered
= (i8
)(pIdxInfo
->orderByConsumed
?
4010 pIdxInfo
->nOrderBy
: 0);
4012 pNew
->rRun
= sqlite3LogEstFromDouble(pIdxInfo
->estimatedCost
);
4013 pNew
->nOut
= sqlite3LogEst(pIdxInfo
->estimatedRows
);
4015 /* Set the WHERE_ONEROW flag if the xBestIndex() method indicated
4016 ** that the scan will visit at most one row. Clear it otherwise. */
4017 if( pIdxInfo
->idxFlags
& SQLITE_INDEX_SCAN_UNIQUE
){
4018 pNew
->wsFlags
|= WHERE_ONEROW
;
4020 pNew
->wsFlags
&= ~WHERE_ONEROW
;
4022 rc
= whereLoopInsert(pBuilder
, pNew
);
4023 if( pNew
->u
.vtab
.needFree
){
4024 sqlite3_free(pNew
->u
.vtab
.idxStr
);
4025 pNew
->u
.vtab
.needFree
= 0;
4027 WHERETRACE(0xffffffff, (" bIn=%d prereqIn=%04llx prereqOut=%04llx\n",
4028 *pbIn
, (sqlite3_uint64
)mPrereq
,
4029 (sqlite3_uint64
)(pNew
->prereq
& ~mPrereq
)));
4035 ** Return the collating sequence for a constraint passed into xBestIndex.
4037 ** pIdxInfo must be an sqlite3_index_info structure passed into xBestIndex.
4038 ** This routine depends on there being a HiddenIndexInfo structure immediately
4039 ** following the sqlite3_index_info structure.
4041 ** Return a pointer to the collation name:
4043 ** 1. If there is an explicit COLLATE operator on the constraint, return it.
4045 ** 2. Else, if the column has an alternative collation, return that.
4047 ** 3. Otherwise, return "BINARY".
4049 const char *sqlite3_vtab_collation(sqlite3_index_info
*pIdxInfo
, int iCons
){
4050 HiddenIndexInfo
*pHidden
= (HiddenIndexInfo
*)&pIdxInfo
[1];
4051 const char *zRet
= 0;
4052 if( iCons
>=0 && iCons
<pIdxInfo
->nConstraint
){
4054 int iTerm
= pIdxInfo
->aConstraint
[iCons
].iTermOffset
;
4055 Expr
*pX
= pHidden
->pWC
->a
[iTerm
].pExpr
;
4057 pC
= sqlite3ExprCompareCollSeq(pHidden
->pParse
, pX
);
4059 zRet
= (pC
? pC
->zName
: sqlite3StrBINARY
);
4065 ** Return true if constraint iCons is really an IN(...) constraint, or
4066 ** false otherwise. If iCons is an IN(...) constraint, set (if bHandle!=0)
4067 ** or clear (if bHandle==0) the flag to handle it using an iterator.
4069 int sqlite3_vtab_in(sqlite3_index_info
*pIdxInfo
, int iCons
, int bHandle
){
4070 HiddenIndexInfo
*pHidden
= (HiddenIndexInfo
*)&pIdxInfo
[1];
4071 u32 m
= SMASKBIT32(iCons
);
4072 if( m
& pHidden
->mIn
){
4074 pHidden
->mHandleIn
&= ~m
;
4075 }else if( bHandle
>0 ){
4076 pHidden
->mHandleIn
|= m
;
4084 ** This interface is callable from within the xBestIndex callback only.
4086 ** If possible, set (*ppVal) to point to an object containing the value
4087 ** on the right-hand-side of constraint iCons.
4089 int sqlite3_vtab_rhs_value(
4090 sqlite3_index_info
*pIdxInfo
, /* Copy of first argument to xBestIndex */
4091 int iCons
, /* Constraint for which RHS is wanted */
4092 sqlite3_value
**ppVal
/* Write value extracted here */
4094 HiddenIndexInfo
*pH
= (HiddenIndexInfo
*)&pIdxInfo
[1];
4095 sqlite3_value
*pVal
= 0;
4097 if( iCons
<0 || iCons
>=pIdxInfo
->nConstraint
){
4098 rc
= SQLITE_MISUSE_BKPT
; /* EV: R-30545-25046 */
4100 if( pH
->aRhs
[iCons
]==0 ){
4101 WhereTerm
*pTerm
= &pH
->pWC
->a
[pIdxInfo
->aConstraint
[iCons
].iTermOffset
];
4102 rc
= sqlite3ValueFromExpr(
4103 pH
->pParse
->db
, pTerm
->pExpr
->pRight
, ENC(pH
->pParse
->db
),
4104 SQLITE_AFF_BLOB
, &pH
->aRhs
[iCons
]
4106 testcase( rc
!=SQLITE_OK
);
4108 pVal
= pH
->aRhs
[iCons
];
4112 if( rc
==SQLITE_OK
&& pVal
==0 ){ /* IMP: R-19933-32160 */
4113 rc
= SQLITE_NOTFOUND
; /* IMP: R-36424-56542 */
4120 ** Return true if ORDER BY clause may be handled as DISTINCT.
4122 int sqlite3_vtab_distinct(sqlite3_index_info
*pIdxInfo
){
4123 HiddenIndexInfo
*pHidden
= (HiddenIndexInfo
*)&pIdxInfo
[1];
4124 assert( pHidden
->eDistinct
>=0 && pHidden
->eDistinct
<=3 );
4125 return pHidden
->eDistinct
;
4129 ** Cause the prepared statement that is associated with a call to
4130 ** xBestIndex to potentially use all schemas. If the statement being
4131 ** prepared is read-only, then just start read transactions on all
4132 ** schemas. But if this is a write operation, start writes on all
4135 ** This is used by the (built-in) sqlite_dbpage virtual table.
4137 void sqlite3VtabUsesAllSchemas(Parse
*pParse
){
4138 int nDb
= pParse
->db
->nDb
;
4140 for(i
=0; i
<nDb
; i
++){
4141 sqlite3CodeVerifySchema(pParse
, i
);
4143 if( DbMaskNonZero(pParse
->writeMask
) ){
4144 for(i
=0; i
<nDb
; i
++){
4145 sqlite3BeginWriteOperation(pParse
, 0, i
);
4151 ** Add all WhereLoop objects for a table of the join identified by
4152 ** pBuilder->pNew->iTab. That table is guaranteed to be a virtual table.
4154 ** If there are no LEFT or CROSS JOIN joins in the query, both mPrereq and
4155 ** mUnusable are set to 0. Otherwise, mPrereq is a mask of all FROM clause
4156 ** entries that occur before the virtual table in the FROM clause and are
4157 ** separated from it by at least one LEFT or CROSS JOIN. Similarly, the
4158 ** mUnusable mask contains all FROM clause entries that occur after the
4159 ** virtual table and are separated from it by at least one LEFT or
4162 ** For example, if the query were:
4164 ** ... FROM t1, t2 LEFT JOIN t3, t4, vt CROSS JOIN t5, t6;
4166 ** then mPrereq corresponds to (t1, t2) and mUnusable to (t5, t6).
4168 ** All the tables in mPrereq must be scanned before the current virtual
4169 ** table. So any terms for which all prerequisites are satisfied by
4170 ** mPrereq may be specified as "usable" in all calls to xBestIndex.
4171 ** Conversely, all tables in mUnusable must be scanned after the current
4172 ** virtual table, so any terms for which the prerequisites overlap with
4173 ** mUnusable should always be configured as "not-usable" for xBestIndex.
4175 static int whereLoopAddVirtual(
4176 WhereLoopBuilder
*pBuilder
, /* WHERE clause information */
4177 Bitmask mPrereq
, /* Tables that must be scanned before this one */
4178 Bitmask mUnusable
/* Tables that must be scanned after this one */
4180 int rc
= SQLITE_OK
; /* Return code */
4181 WhereInfo
*pWInfo
; /* WHERE analysis context */
4182 Parse
*pParse
; /* The parsing context */
4183 WhereClause
*pWC
; /* The WHERE clause */
4184 SrcItem
*pSrc
; /* The FROM clause term to search */
4185 sqlite3_index_info
*p
; /* Object to pass to xBestIndex() */
4186 int nConstraint
; /* Number of constraints in p */
4187 int bIn
; /* True if plan uses IN(...) operator */
4189 Bitmask mBest
; /* Tables used by best possible plan */
4191 int bRetry
= 0; /* True to retry with LIMIT/OFFSET disabled */
4193 assert( (mPrereq
& mUnusable
)==0 );
4194 pWInfo
= pBuilder
->pWInfo
;
4195 pParse
= pWInfo
->pParse
;
4196 pWC
= pBuilder
->pWC
;
4197 pNew
= pBuilder
->pNew
;
4198 pSrc
= &pWInfo
->pTabList
->a
[pNew
->iTab
];
4199 assert( IsVirtual(pSrc
->pTab
) );
4200 p
= allocateIndexInfo(pWInfo
, pWC
, mUnusable
, pSrc
, &mNoOmit
);
4201 if( p
==0 ) return SQLITE_NOMEM_BKPT
;
4203 pNew
->wsFlags
= WHERE_VIRTUALTABLE
;
4205 pNew
->u
.vtab
.needFree
= 0;
4206 nConstraint
= p
->nConstraint
;
4207 if( whereLoopResize(pParse
->db
, pNew
, nConstraint
) ){
4208 freeIndexInfo(pParse
->db
, p
);
4209 return SQLITE_NOMEM_BKPT
;
4212 /* First call xBestIndex() with all constraints usable. */
4213 WHERETRACE(0x800, ("BEGIN %s.addVirtual()\n", pSrc
->pTab
->zName
));
4214 WHERETRACE(0x800, (" VirtualOne: all usable\n"));
4215 rc
= whereLoopAddVirtualOne(
4216 pBuilder
, mPrereq
, ALLBITS
, 0, p
, mNoOmit
, &bIn
, &bRetry
4219 assert( rc
==SQLITE_OK
);
4220 rc
= whereLoopAddVirtualOne(
4221 pBuilder
, mPrereq
, ALLBITS
, 0, p
, mNoOmit
, &bIn
, 0
4225 /* If the call to xBestIndex() with all terms enabled produced a plan
4226 ** that does not require any source tables (IOW: a plan with mBest==0)
4227 ** and does not use an IN(...) operator, then there is no point in making
4228 ** any further calls to xBestIndex() since they will all return the same
4229 ** result (if the xBestIndex() implementation is sane). */
4230 if( rc
==SQLITE_OK
&& ((mBest
= (pNew
->prereq
& ~mPrereq
))!=0 || bIn
) ){
4231 int seenZero
= 0; /* True if a plan with no prereqs seen */
4232 int seenZeroNoIN
= 0; /* Plan with no prereqs and no IN(...) seen */
4234 Bitmask mBestNoIn
= 0;
4236 /* If the plan produced by the earlier call uses an IN(...) term, call
4237 ** xBestIndex again, this time with IN(...) terms disabled. */
4239 WHERETRACE(0x800, (" VirtualOne: all usable w/o IN\n"));
4240 rc
= whereLoopAddVirtualOne(
4241 pBuilder
, mPrereq
, ALLBITS
, WO_IN
, p
, mNoOmit
, &bIn
, 0);
4243 mBestNoIn
= pNew
->prereq
& ~mPrereq
;
4250 /* Call xBestIndex once for each distinct value of (prereqRight & ~mPrereq)
4251 ** in the set of terms that apply to the current virtual table. */
4252 while( rc
==SQLITE_OK
){
4254 Bitmask mNext
= ALLBITS
;
4256 for(i
=0; i
<nConstraint
; i
++){
4258 pWC
->a
[p
->aConstraint
[i
].iTermOffset
].prereqRight
& ~mPrereq
4260 if( mThis
>mPrev
&& mThis
<mNext
) mNext
= mThis
;
4263 if( mNext
==ALLBITS
) break;
4264 if( mNext
==mBest
|| mNext
==mBestNoIn
) continue;
4265 WHERETRACE(0x800, (" VirtualOne: mPrev=%04llx mNext=%04llx\n",
4266 (sqlite3_uint64
)mPrev
, (sqlite3_uint64
)mNext
));
4267 rc
= whereLoopAddVirtualOne(
4268 pBuilder
, mPrereq
, mNext
|mPrereq
, 0, p
, mNoOmit
, &bIn
, 0);
4269 if( pNew
->prereq
==mPrereq
){
4271 if( bIn
==0 ) seenZeroNoIN
= 1;
4275 /* If the calls to xBestIndex() in the above loop did not find a plan
4276 ** that requires no source tables at all (i.e. one guaranteed to be
4277 ** usable), make a call here with all source tables disabled */
4278 if( rc
==SQLITE_OK
&& seenZero
==0 ){
4279 WHERETRACE(0x800, (" VirtualOne: all disabled\n"));
4280 rc
= whereLoopAddVirtualOne(
4281 pBuilder
, mPrereq
, mPrereq
, 0, p
, mNoOmit
, &bIn
, 0);
4282 if( bIn
==0 ) seenZeroNoIN
= 1;
4285 /* If the calls to xBestIndex() have so far failed to find a plan
4286 ** that requires no source tables at all and does not use an IN(...)
4287 ** operator, make a final call to obtain one here. */
4288 if( rc
==SQLITE_OK
&& seenZeroNoIN
==0 ){
4289 WHERETRACE(0x800, (" VirtualOne: all disabled and w/o IN\n"));
4290 rc
= whereLoopAddVirtualOne(
4291 pBuilder
, mPrereq
, mPrereq
, WO_IN
, p
, mNoOmit
, &bIn
, 0);
4295 if( p
->needToFreeIdxStr
) sqlite3_free(p
->idxStr
);
4296 freeIndexInfo(pParse
->db
, p
);
4297 WHERETRACE(0x800, ("END %s.addVirtual(), rc=%d\n", pSrc
->pTab
->zName
, rc
));
4300 #endif /* SQLITE_OMIT_VIRTUALTABLE */
4303 ** Add WhereLoop entries to handle OR terms. This works for either
4304 ** btrees or virtual tables.
4306 static int whereLoopAddOr(
4307 WhereLoopBuilder
*pBuilder
,
4311 WhereInfo
*pWInfo
= pBuilder
->pWInfo
;
4314 WhereTerm
*pTerm
, *pWCEnd
;
4318 WhereLoopBuilder sSubBuild
;
4319 WhereOrSet sSum
, sCur
;
4322 pWC
= pBuilder
->pWC
;
4323 pWCEnd
= pWC
->a
+ pWC
->nTerm
;
4324 pNew
= pBuilder
->pNew
;
4325 memset(&sSum
, 0, sizeof(sSum
));
4326 pItem
= pWInfo
->pTabList
->a
+ pNew
->iTab
;
4327 iCur
= pItem
->iCursor
;
4329 /* The multi-index OR optimization does not work for RIGHT and FULL JOIN */
4330 if( pItem
->fg
.jointype
& JT_RIGHT
) return SQLITE_OK
;
4332 for(pTerm
=pWC
->a
; pTerm
<pWCEnd
&& rc
==SQLITE_OK
; pTerm
++){
4333 if( (pTerm
->eOperator
& WO_OR
)!=0
4334 && (pTerm
->u
.pOrInfo
->indexable
& pNew
->maskSelf
)!=0
4336 WhereClause
* const pOrWC
= &pTerm
->u
.pOrInfo
->wc
;
4337 WhereTerm
* const pOrWCEnd
= &pOrWC
->a
[pOrWC
->nTerm
];
4342 sSubBuild
= *pBuilder
;
4343 sSubBuild
.pOrSet
= &sCur
;
4345 WHERETRACE(0x400, ("Begin processing OR-clause %p\n", pTerm
));
4346 for(pOrTerm
=pOrWC
->a
; pOrTerm
<pOrWCEnd
; pOrTerm
++){
4347 if( (pOrTerm
->eOperator
& WO_AND
)!=0 ){
4348 sSubBuild
.pWC
= &pOrTerm
->u
.pAndInfo
->wc
;
4349 }else if( pOrTerm
->leftCursor
==iCur
){
4350 tempWC
.pWInfo
= pWC
->pWInfo
;
4351 tempWC
.pOuter
= pWC
;
4356 sSubBuild
.pWC
= &tempWC
;
4361 #ifdef WHERETRACE_ENABLED
4362 WHERETRACE(0x400, ("OR-term %d of %p has %d subterms:\n",
4363 (int)(pOrTerm
-pOrWC
->a
), pTerm
, sSubBuild
.pWC
->nTerm
));
4364 if( sqlite3WhereTrace
& 0x20000 ){
4365 sqlite3WhereClausePrint(sSubBuild
.pWC
);
4368 #ifndef SQLITE_OMIT_VIRTUALTABLE
4369 if( IsVirtual(pItem
->pTab
) ){
4370 rc
= whereLoopAddVirtual(&sSubBuild
, mPrereq
, mUnusable
);
4374 rc
= whereLoopAddBtree(&sSubBuild
, mPrereq
);
4376 if( rc
==SQLITE_OK
){
4377 rc
= whereLoopAddOr(&sSubBuild
, mPrereq
, mUnusable
);
4379 testcase( rc
==SQLITE_NOMEM
&& sCur
.n
>0 );
4380 testcase( rc
==SQLITE_DONE
);
4385 whereOrMove(&sSum
, &sCur
);
4389 whereOrMove(&sPrev
, &sSum
);
4391 for(i
=0; i
<sPrev
.n
; i
++){
4392 for(j
=0; j
<sCur
.n
; j
++){
4393 whereOrInsert(&sSum
, sPrev
.a
[i
].prereq
| sCur
.a
[j
].prereq
,
4394 sqlite3LogEstAdd(sPrev
.a
[i
].rRun
, sCur
.a
[j
].rRun
),
4395 sqlite3LogEstAdd(sPrev
.a
[i
].nOut
, sCur
.a
[j
].nOut
));
4401 pNew
->aLTerm
[0] = pTerm
;
4402 pNew
->wsFlags
= WHERE_MULTI_OR
;
4405 memset(&pNew
->u
, 0, sizeof(pNew
->u
));
4406 for(i
=0; rc
==SQLITE_OK
&& i
<sSum
.n
; i
++){
4407 /* TUNING: Currently sSum.a[i].rRun is set to the sum of the costs
4408 ** of all sub-scans required by the OR-scan. However, due to rounding
4409 ** errors, it may be that the cost of the OR-scan is equal to its
4410 ** most expensive sub-scan. Add the smallest possible penalty
4411 ** (equivalent to multiplying the cost by 1.07) to ensure that
4412 ** this does not happen. Otherwise, for WHERE clauses such as the
4413 ** following where there is an index on "y":
4415 ** WHERE likelihood(x=?, 0.99) OR y=?
4417 ** the planner may elect to "OR" together a full-table scan and an
4418 ** index lookup. And other similarly odd results. */
4419 pNew
->rRun
= sSum
.a
[i
].rRun
+ 1;
4420 pNew
->nOut
= sSum
.a
[i
].nOut
;
4421 pNew
->prereq
= sSum
.a
[i
].prereq
;
4422 rc
= whereLoopInsert(pBuilder
, pNew
);
4424 WHERETRACE(0x400, ("End processing OR-clause %p\n", pTerm
));
4431 ** Add all WhereLoop objects for all tables
4433 static int whereLoopAddAll(WhereLoopBuilder
*pBuilder
){
4434 WhereInfo
*pWInfo
= pBuilder
->pWInfo
;
4435 Bitmask mPrereq
= 0;
4438 SrcList
*pTabList
= pWInfo
->pTabList
;
4440 SrcItem
*pEnd
= &pTabList
->a
[pWInfo
->nLevel
];
4441 sqlite3
*db
= pWInfo
->pParse
->db
;
4443 int bFirstPastRJ
= 0;
4444 int hasRightJoin
= 0;
4448 /* Loop over the tables in the join, from left to right */
4449 pNew
= pBuilder
->pNew
;
4451 /* Verify that pNew has already been initialized */
4452 assert( pNew
->nLTerm
==0 );
4453 assert( pNew
->wsFlags
==0 );
4454 assert( pNew
->nLSlot
>=ArraySize(pNew
->aLTermSpace
) );
4455 assert( pNew
->aLTerm
!=0 );
4457 pBuilder
->iPlanLimit
= SQLITE_QUERY_PLANNER_LIMIT
;
4458 for(iTab
=0, pItem
=pTabList
->a
; pItem
<pEnd
; iTab
++, pItem
++){
4459 Bitmask mUnusable
= 0;
4461 pBuilder
->iPlanLimit
+= SQLITE_QUERY_PLANNER_LIMIT_INCR
;
4462 pNew
->maskSelf
= sqlite3WhereGetMask(&pWInfo
->sMaskSet
, pItem
->iCursor
);
4464 || (pItem
->fg
.jointype
& (JT_OUTER
|JT_CROSS
|JT_LTORJ
))!=0
4466 /* Add prerequisites to prevent reordering of FROM clause terms
4467 ** across CROSS joins and outer joins. The bFirstPastRJ boolean
4468 ** prevents the right operand of a RIGHT JOIN from being swapped with
4469 ** other elements even further to the right.
4471 ** The JT_LTORJ case and the hasRightJoin flag work together to
4472 ** prevent FROM-clause terms from moving from the right side of
4473 ** a LEFT JOIN over to the left side of that join if the LEFT JOIN
4474 ** is itself on the left side of a RIGHT JOIN.
4476 if( pItem
->fg
.jointype
& JT_LTORJ
) hasRightJoin
= 1;
4478 bFirstPastRJ
= (pItem
->fg
.jointype
& JT_RIGHT
)!=0;
4479 }else if( !hasRightJoin
){
4482 #ifndef SQLITE_OMIT_VIRTUALTABLE
4483 if( IsVirtual(pItem
->pTab
) ){
4485 for(p
=&pItem
[1]; p
<pEnd
; p
++){
4486 if( mUnusable
|| (p
->fg
.jointype
& (JT_OUTER
|JT_CROSS
)) ){
4487 mUnusable
|= sqlite3WhereGetMask(&pWInfo
->sMaskSet
, p
->iCursor
);
4490 rc
= whereLoopAddVirtual(pBuilder
, mPrereq
, mUnusable
);
4492 #endif /* SQLITE_OMIT_VIRTUALTABLE */
4494 rc
= whereLoopAddBtree(pBuilder
, mPrereq
);
4496 if( rc
==SQLITE_OK
&& pBuilder
->pWC
->hasOr
){
4497 rc
= whereLoopAddOr(pBuilder
, mPrereq
, mUnusable
);
4499 mPrior
|= pNew
->maskSelf
;
4500 if( rc
|| db
->mallocFailed
){
4501 if( rc
==SQLITE_DONE
){
4502 /* We hit the query planner search limit set by iPlanLimit */
4503 sqlite3_log(SQLITE_WARNING
, "abbreviated query algorithm search");
4511 whereLoopClear(db
, pNew
);
4516 ** Examine a WherePath (with the addition of the extra WhereLoop of the 6th
4517 ** parameters) to see if it outputs rows in the requested ORDER BY
4518 ** (or GROUP BY) without requiring a separate sort operation. Return N:
4520 ** N>0: N terms of the ORDER BY clause are satisfied
4521 ** N==0: No terms of the ORDER BY clause are satisfied
4522 ** N<0: Unknown yet how many terms of ORDER BY might be satisfied.
4524 ** Note that processing for WHERE_GROUPBY and WHERE_DISTINCTBY is not as
4525 ** strict. With GROUP BY and DISTINCT the only requirement is that
4526 ** equivalent rows appear immediately adjacent to one another. GROUP BY
4527 ** and DISTINCT do not require rows to appear in any particular order as long
4528 ** as equivalent rows are grouped together. Thus for GROUP BY and DISTINCT
4529 ** the pOrderBy terms can be matched in any order. With ORDER BY, the
4530 ** pOrderBy terms must be matched in strict left-to-right order.
4532 static i8
wherePathSatisfiesOrderBy(
4533 WhereInfo
*pWInfo
, /* The WHERE clause */
4534 ExprList
*pOrderBy
, /* ORDER BY or GROUP BY or DISTINCT clause to check */
4535 WherePath
*pPath
, /* The WherePath to check */
4536 u16 wctrlFlags
, /* WHERE_GROUPBY or _DISTINCTBY or _ORDERBY_LIMIT */
4537 u16 nLoop
, /* Number of entries in pPath->aLoop[] */
4538 WhereLoop
*pLast
, /* Add this WhereLoop to the end of pPath->aLoop[] */
4539 Bitmask
*pRevMask
/* OUT: Mask of WhereLoops to run in reverse order */
4541 u8 revSet
; /* True if rev is known */
4542 u8 rev
; /* Composite sort order */
4543 u8 revIdx
; /* Index sort order */
4544 u8 isOrderDistinct
; /* All prior WhereLoops are order-distinct */
4545 u8 distinctColumns
; /* True if the loop has UNIQUE NOT NULL columns */
4546 u8 isMatch
; /* iColumn matches a term of the ORDER BY clause */
4547 u16 eqOpMask
; /* Allowed equality operators */
4548 u16 nKeyCol
; /* Number of key columns in pIndex */
4549 u16 nColumn
; /* Total number of ordered columns in the index */
4550 u16 nOrderBy
; /* Number terms in the ORDER BY clause */
4551 int iLoop
; /* Index of WhereLoop in pPath being processed */
4552 int i
, j
; /* Loop counters */
4553 int iCur
; /* Cursor number for current WhereLoop */
4554 int iColumn
; /* A column number within table iCur */
4555 WhereLoop
*pLoop
= 0; /* Current WhereLoop being processed. */
4556 WhereTerm
*pTerm
; /* A single term of the WHERE clause */
4557 Expr
*pOBExpr
; /* An expression from the ORDER BY clause */
4558 CollSeq
*pColl
; /* COLLATE function from an ORDER BY clause term */
4559 Index
*pIndex
; /* The index associated with pLoop */
4560 sqlite3
*db
= pWInfo
->pParse
->db
; /* Database connection */
4561 Bitmask obSat
= 0; /* Mask of ORDER BY terms satisfied so far */
4562 Bitmask obDone
; /* Mask of all ORDER BY terms */
4563 Bitmask orderDistinctMask
; /* Mask of all well-ordered loops */
4564 Bitmask ready
; /* Mask of inner loops */
4567 ** We say the WhereLoop is "one-row" if it generates no more than one
4568 ** row of output. A WhereLoop is one-row if all of the following are true:
4569 ** (a) All index columns match with WHERE_COLUMN_EQ.
4570 ** (b) The index is unique
4571 ** Any WhereLoop with an WHERE_COLUMN_EQ constraint on the rowid is one-row.
4572 ** Every one-row WhereLoop will have the WHERE_ONEROW bit set in wsFlags.
4574 ** We say the WhereLoop is "order-distinct" if the set of columns from
4575 ** that WhereLoop that are in the ORDER BY clause are different for every
4576 ** row of the WhereLoop. Every one-row WhereLoop is automatically
4577 ** order-distinct. A WhereLoop that has no columns in the ORDER BY clause
4578 ** is not order-distinct. To be order-distinct is not quite the same as being
4579 ** UNIQUE since a UNIQUE column or index can have multiple rows that
4580 ** are NULL and NULL values are equivalent for the purpose of order-distinct.
4581 ** To be order-distinct, the columns must be UNIQUE and NOT NULL.
4583 ** The rowid for a table is always UNIQUE and NOT NULL so whenever the
4584 ** rowid appears in the ORDER BY clause, the corresponding WhereLoop is
4585 ** automatically order-distinct.
4588 assert( pOrderBy
!=0 );
4589 if( nLoop
&& OptimizationDisabled(db
, SQLITE_OrderByIdxJoin
) ) return 0;
4591 nOrderBy
= pOrderBy
->nExpr
;
4592 testcase( nOrderBy
==BMS
-1 );
4593 if( nOrderBy
>BMS
-1 ) return 0; /* Cannot optimize overly large ORDER BYs */
4594 isOrderDistinct
= 1;
4595 obDone
= MASKBIT(nOrderBy
)-1;
4596 orderDistinctMask
= 0;
4598 eqOpMask
= WO_EQ
| WO_IS
| WO_ISNULL
;
4599 if( wctrlFlags
& (WHERE_ORDERBY_LIMIT
|WHERE_ORDERBY_MAX
|WHERE_ORDERBY_MIN
) ){
4602 for(iLoop
=0; isOrderDistinct
&& obSat
<obDone
&& iLoop
<=nLoop
; iLoop
++){
4603 if( iLoop
>0 ) ready
|= pLoop
->maskSelf
;
4605 pLoop
= pPath
->aLoop
[iLoop
];
4606 if( wctrlFlags
& WHERE_ORDERBY_LIMIT
) continue;
4610 if( pLoop
->wsFlags
& WHERE_VIRTUALTABLE
){
4611 if( pLoop
->u
.vtab
.isOrdered
4612 && ((wctrlFlags
&(WHERE_DISTINCTBY
|WHERE_SORTBYGROUP
))!=WHERE_DISTINCTBY
)
4617 }else if( wctrlFlags
& WHERE_DISTINCTBY
){
4618 pLoop
->u
.btree
.nDistinctCol
= 0;
4620 iCur
= pWInfo
->pTabList
->a
[pLoop
->iTab
].iCursor
;
4622 /* Mark off any ORDER BY term X that is a column in the table of
4623 ** the current loop for which there is term in the WHERE
4624 ** clause of the form X IS NULL or X=? that reference only outer
4627 for(i
=0; i
<nOrderBy
; i
++){
4628 if( MASKBIT(i
) & obSat
) continue;
4629 pOBExpr
= sqlite3ExprSkipCollateAndLikely(pOrderBy
->a
[i
].pExpr
);
4630 if( NEVER(pOBExpr
==0) ) continue;
4631 if( pOBExpr
->op
!=TK_COLUMN
&& pOBExpr
->op
!=TK_AGG_COLUMN
) continue;
4632 if( pOBExpr
->iTable
!=iCur
) continue;
4633 pTerm
= sqlite3WhereFindTerm(&pWInfo
->sWC
, iCur
, pOBExpr
->iColumn
,
4634 ~ready
, eqOpMask
, 0);
4635 if( pTerm
==0 ) continue;
4636 if( pTerm
->eOperator
==WO_IN
){
4637 /* IN terms are only valid for sorting in the ORDER BY LIMIT
4638 ** optimization, and then only if they are actually used
4639 ** by the query plan */
4640 assert( wctrlFlags
&
4641 (WHERE_ORDERBY_LIMIT
|WHERE_ORDERBY_MIN
|WHERE_ORDERBY_MAX
) );
4642 for(j
=0; j
<pLoop
->nLTerm
&& pTerm
!=pLoop
->aLTerm
[j
]; j
++){}
4643 if( j
>=pLoop
->nLTerm
) continue;
4645 if( (pTerm
->eOperator
&(WO_EQ
|WO_IS
))!=0 && pOBExpr
->iColumn
>=0 ){
4646 Parse
*pParse
= pWInfo
->pParse
;
4647 CollSeq
*pColl1
= sqlite3ExprNNCollSeq(pParse
, pOrderBy
->a
[i
].pExpr
);
4648 CollSeq
*pColl2
= sqlite3ExprCompareCollSeq(pParse
, pTerm
->pExpr
);
4650 if( pColl2
==0 || sqlite3StrICmp(pColl1
->zName
, pColl2
->zName
) ){
4653 testcase( pTerm
->pExpr
->op
==TK_IS
);
4655 obSat
|= MASKBIT(i
);
4658 if( (pLoop
->wsFlags
& WHERE_ONEROW
)==0 ){
4659 if( pLoop
->wsFlags
& WHERE_IPK
){
4663 }else if( (pIndex
= pLoop
->u
.btree
.pIndex
)==0 || pIndex
->bUnordered
){
4666 nKeyCol
= pIndex
->nKeyCol
;
4667 nColumn
= pIndex
->nColumn
;
4668 assert( nColumn
==nKeyCol
+1 || !HasRowid(pIndex
->pTable
) );
4669 assert( pIndex
->aiColumn
[nColumn
-1]==XN_ROWID
4670 || !HasRowid(pIndex
->pTable
));
4671 /* All relevant terms of the index must also be non-NULL in order
4672 ** for isOrderDistinct to be true. So the isOrderDistint value
4673 ** computed here might be a false positive. Corrections will be
4674 ** made at tag-20210426-1 below */
4675 isOrderDistinct
= IsUniqueIndex(pIndex
)
4676 && (pLoop
->wsFlags
& WHERE_SKIPSCAN
)==0;
4679 /* Loop through all columns of the index and deal with the ones
4680 ** that are not constrained by == or IN.
4683 distinctColumns
= 0;
4684 for(j
=0; j
<nColumn
; j
++){
4685 u8 bOnce
= 1; /* True to run the ORDER BY search loop */
4687 assert( j
>=pLoop
->u
.btree
.nEq
4688 || (pLoop
->aLTerm
[j
]==0)==(j
<pLoop
->nSkip
)
4690 if( j
<pLoop
->u
.btree
.nEq
&& j
>=pLoop
->nSkip
){
4691 u16 eOp
= pLoop
->aLTerm
[j
]->eOperator
;
4693 /* Skip over == and IS and ISNULL terms. (Also skip IN terms when
4694 ** doing WHERE_ORDERBY_LIMIT processing). Except, IS and ISNULL
4695 ** terms imply that the index is not UNIQUE NOT NULL in which case
4696 ** the loop need to be marked as not order-distinct because it can
4697 ** have repeated NULL rows.
4699 ** If the current term is a column of an ((?,?) IN (SELECT...))
4700 ** expression for which the SELECT returns more than one column,
4701 ** check that it is the only column used by this loop. Otherwise,
4702 ** if it is one of two or more, none of the columns can be
4703 ** considered to match an ORDER BY term.
4705 if( (eOp
& eqOpMask
)!=0 ){
4706 if( eOp
& (WO_ISNULL
|WO_IS
) ){
4707 testcase( eOp
& WO_ISNULL
);
4708 testcase( eOp
& WO_IS
);
4709 testcase( isOrderDistinct
);
4710 isOrderDistinct
= 0;
4713 }else if( ALWAYS(eOp
& WO_IN
) ){
4714 /* ALWAYS() justification: eOp is an equality operator due to the
4715 ** j<pLoop->u.btree.nEq constraint above. Any equality other
4716 ** than WO_IN is captured by the previous "if". So this one
4717 ** always has to be WO_IN. */
4718 Expr
*pX
= pLoop
->aLTerm
[j
]->pExpr
;
4719 for(i
=j
+1; i
<pLoop
->u
.btree
.nEq
; i
++){
4720 if( pLoop
->aLTerm
[i
]->pExpr
==pX
){
4721 assert( (pLoop
->aLTerm
[i
]->eOperator
& WO_IN
) );
4729 /* Get the column number in the table (iColumn) and sort order
4730 ** (revIdx) for the j-th column of the index.
4733 iColumn
= pIndex
->aiColumn
[j
];
4734 revIdx
= pIndex
->aSortOrder
[j
] & KEYINFO_ORDER_DESC
;
4735 if( iColumn
==pIndex
->pTable
->iPKey
) iColumn
= XN_ROWID
;
4741 /* An unconstrained column that might be NULL means that this
4742 ** WhereLoop is not well-ordered. tag-20210426-1
4744 if( isOrderDistinct
){
4746 && j
>=pLoop
->u
.btree
.nEq
4747 && pIndex
->pTable
->aCol
[iColumn
].notNull
==0
4749 isOrderDistinct
= 0;
4751 if( iColumn
==XN_EXPR
){
4752 isOrderDistinct
= 0;
4756 /* Find the ORDER BY term that corresponds to the j-th column
4757 ** of the index and mark that ORDER BY term off
4760 for(i
=0; bOnce
&& i
<nOrderBy
; i
++){
4761 if( MASKBIT(i
) & obSat
) continue;
4762 pOBExpr
= sqlite3ExprSkipCollateAndLikely(pOrderBy
->a
[i
].pExpr
);
4763 testcase( wctrlFlags
& WHERE_GROUPBY
);
4764 testcase( wctrlFlags
& WHERE_DISTINCTBY
);
4765 if( NEVER(pOBExpr
==0) ) continue;
4766 if( (wctrlFlags
& (WHERE_GROUPBY
|WHERE_DISTINCTBY
))==0 ) bOnce
= 0;
4767 if( iColumn
>=XN_ROWID
){
4768 if( pOBExpr
->op
!=TK_COLUMN
&& pOBExpr
->op
!=TK_AGG_COLUMN
) continue;
4769 if( pOBExpr
->iTable
!=iCur
) continue;
4770 if( pOBExpr
->iColumn
!=iColumn
) continue;
4772 Expr
*pIxExpr
= pIndex
->aColExpr
->a
[j
].pExpr
;
4773 if( sqlite3ExprCompareSkip(pOBExpr
, pIxExpr
, iCur
) ){
4777 if( iColumn
!=XN_ROWID
){
4778 pColl
= sqlite3ExprNNCollSeq(pWInfo
->pParse
, pOrderBy
->a
[i
].pExpr
);
4779 if( sqlite3StrICmp(pColl
->zName
, pIndex
->azColl
[j
])!=0 ) continue;
4781 if( wctrlFlags
& WHERE_DISTINCTBY
){
4782 pLoop
->u
.btree
.nDistinctCol
= j
+1;
4787 if( isMatch
&& (wctrlFlags
& WHERE_GROUPBY
)==0 ){
4788 /* Make sure the sort order is compatible in an ORDER BY clause.
4789 ** Sort order is irrelevant for a GROUP BY clause. */
4792 != (pOrderBy
->a
[i
].fg
.sortFlags
&KEYINFO_ORDER_DESC
)
4797 rev
= revIdx
^ (pOrderBy
->a
[i
].fg
.sortFlags
& KEYINFO_ORDER_DESC
);
4798 if( rev
) *pRevMask
|= MASKBIT(iLoop
);
4802 if( isMatch
&& (pOrderBy
->a
[i
].fg
.sortFlags
& KEYINFO_ORDER_BIGNULL
) ){
4803 if( j
==pLoop
->u
.btree
.nEq
){
4804 pLoop
->wsFlags
|= WHERE_BIGNULL_SORT
;
4810 if( iColumn
==XN_ROWID
){
4811 testcase( distinctColumns
==0 );
4812 distinctColumns
= 1;
4814 obSat
|= MASKBIT(i
);
4816 /* No match found */
4817 if( j
==0 || j
<nKeyCol
){
4818 testcase( isOrderDistinct
!=0 );
4819 isOrderDistinct
= 0;
4823 } /* end Loop over all index columns */
4824 if( distinctColumns
){
4825 testcase( isOrderDistinct
==0 );
4826 isOrderDistinct
= 1;
4828 } /* end-if not one-row */
4830 /* Mark off any other ORDER BY terms that reference pLoop */
4831 if( isOrderDistinct
){
4832 orderDistinctMask
|= pLoop
->maskSelf
;
4833 for(i
=0; i
<nOrderBy
; i
++){
4836 if( MASKBIT(i
) & obSat
) continue;
4837 p
= pOrderBy
->a
[i
].pExpr
;
4838 mTerm
= sqlite3WhereExprUsage(&pWInfo
->sMaskSet
,p
);
4839 if( mTerm
==0 && !sqlite3ExprIsConstant(p
) ) continue;
4840 if( (mTerm
&~orderDistinctMask
)==0 ){
4841 obSat
|= MASKBIT(i
);
4845 } /* End the loop over all WhereLoops from outer-most down to inner-most */
4846 if( obSat
==obDone
) return (i8
)nOrderBy
;
4847 if( !isOrderDistinct
){
4848 for(i
=nOrderBy
-1; i
>0; i
--){
4849 Bitmask m
= ALWAYS(i
<BMS
) ? MASKBIT(i
) - 1 : 0;
4850 if( (obSat
&m
)==m
) return i
;
4859 ** If the WHERE_GROUPBY flag is set in the mask passed to sqlite3WhereBegin(),
4860 ** the planner assumes that the specified pOrderBy list is actually a GROUP
4861 ** BY clause - and so any order that groups rows as required satisfies the
4864 ** Normally, in this case it is not possible for the caller to determine
4865 ** whether or not the rows are really being delivered in sorted order, or
4866 ** just in some other order that provides the required grouping. However,
4867 ** if the WHERE_SORTBYGROUP flag is also passed to sqlite3WhereBegin(), then
4868 ** this function may be called on the returned WhereInfo object. It returns
4869 ** true if the rows really will be sorted in the specified order, or false
4872 ** For example, assuming:
4874 ** CREATE INDEX i1 ON t1(x, Y);
4878 ** SELECT * FROM t1 GROUP BY x,y ORDER BY x,y; -- IsSorted()==1
4879 ** SELECT * FROM t1 GROUP BY y,x ORDER BY y,x; -- IsSorted()==0
4881 int sqlite3WhereIsSorted(WhereInfo
*pWInfo
){
4882 assert( pWInfo
->wctrlFlags
& (WHERE_GROUPBY
|WHERE_DISTINCTBY
) );
4883 assert( pWInfo
->wctrlFlags
& WHERE_SORTBYGROUP
);
4884 return pWInfo
->sorted
;
4887 #ifdef WHERETRACE_ENABLED
4888 /* For debugging use only: */
4889 static const char *wherePathName(WherePath
*pPath
, int nLoop
, WhereLoop
*pLast
){
4890 static char zName
[65];
4892 for(i
=0; i
<nLoop
; i
++){ zName
[i
] = pPath
->aLoop
[i
]->cId
; }
4893 if( pLast
) zName
[i
++] = pLast
->cId
;
4900 ** Return the cost of sorting nRow rows, assuming that the keys have
4901 ** nOrderby columns and that the first nSorted columns are already in
4904 static LogEst
whereSortingCost(
4905 WhereInfo
*pWInfo
, /* Query planning context */
4906 LogEst nRow
, /* Estimated number of rows to sort */
4907 int nOrderBy
, /* Number of ORDER BY clause terms */
4908 int nSorted
/* Number of initial ORDER BY terms naturally in order */
4910 /* Estimated cost of a full external sort, where N is
4911 ** the number of rows to sort is:
4913 ** cost = (K * N * log(N)).
4915 ** Or, if the order-by clause has X terms but only the last Y
4916 ** terms are out of order, then block-sorting will reduce the
4919 ** cost = (K * N * log(N)) * (Y/X)
4921 ** The constant K is at least 2.0 but will be larger if there are a
4922 ** large number of columns to be sorted, as the sorting time is
4923 ** proportional to the amount of content to be sorted. The algorithm
4924 ** does not currently distinguish between fat columns (BLOBs and TEXTs)
4925 ** and skinny columns (INTs). It just uses the number of columns as
4926 ** an approximation for the row width.
4928 ** And extra factor of 2.0 or 3.0 is added to the sorting cost if the sort
4929 ** is built using OP_IdxInsert and OP_Sort rather than with OP_SorterInsert.
4931 LogEst rSortCost
, nCol
;
4932 assert( pWInfo
->pSelect
!=0 );
4933 assert( pWInfo
->pSelect
->pEList
!=0 );
4934 /* TUNING: sorting cost proportional to the number of output columns: */
4935 nCol
= sqlite3LogEst((pWInfo
->pSelect
->pEList
->nExpr
+59)/30);
4936 rSortCost
= nRow
+ nCol
;
4938 /* Scale the result by (Y/X) */
4939 rSortCost
+= sqlite3LogEst((nOrderBy
-nSorted
)*100/nOrderBy
) - 66;
4942 /* Multiple by log(M) where M is the number of output rows.
4943 ** Use the LIMIT for M if it is smaller. Or if this sort is for
4944 ** a DISTINCT operator, M will be the number of distinct output
4945 ** rows, so fudge it downwards a bit.
4947 if( (pWInfo
->wctrlFlags
& WHERE_USE_LIMIT
)!=0 ){
4948 rSortCost
+= 10; /* TUNING: Extra 2.0x if using LIMIT */
4950 rSortCost
+= 6; /* TUNING: Extra 1.5x if also using partial sort */
4952 if( pWInfo
->iLimit
<nRow
){
4953 nRow
= pWInfo
->iLimit
;
4955 }else if( (pWInfo
->wctrlFlags
& WHERE_WANT_DISTINCT
) ){
4956 /* TUNING: In the sort for a DISTINCT operator, assume that the DISTINCT
4957 ** reduces the number of output rows by a factor of 2 */
4958 if( nRow
>10 ){ nRow
-= 10; assert( 10==sqlite3LogEst(2) ); }
4960 rSortCost
+= estLog(nRow
);
4965 ** Given the list of WhereLoop objects at pWInfo->pLoops, this routine
4966 ** attempts to find the lowest cost path that visits each WhereLoop
4967 ** once. This path is then loaded into the pWInfo->a[].pWLoop fields.
4969 ** Assume that the total number of output rows that will need to be sorted
4970 ** will be nRowEst (in the 10*log2 representation). Or, ignore sorting
4971 ** costs if nRowEst==0.
4973 ** Return SQLITE_OK on success or SQLITE_NOMEM of a memory allocation
4976 static int wherePathSolver(WhereInfo
*pWInfo
, LogEst nRowEst
){
4977 int mxChoice
; /* Maximum number of simultaneous paths tracked */
4978 int nLoop
; /* Number of terms in the join */
4979 Parse
*pParse
; /* Parsing context */
4980 int iLoop
; /* Loop counter over the terms of the join */
4981 int ii
, jj
; /* Loop counters */
4982 int mxI
= 0; /* Index of next entry to replace */
4983 int nOrderBy
; /* Number of ORDER BY clause terms */
4984 LogEst mxCost
= 0; /* Maximum cost of a set of paths */
4985 LogEst mxUnsorted
= 0; /* Maximum unsorted cost of a set of path */
4986 int nTo
, nFrom
; /* Number of valid entries in aTo[] and aFrom[] */
4987 WherePath
*aFrom
; /* All nFrom paths at the previous level */
4988 WherePath
*aTo
; /* The nTo best paths at the current level */
4989 WherePath
*pFrom
; /* An element of aFrom[] that we are working on */
4990 WherePath
*pTo
; /* An element of aTo[] that we are working on */
4991 WhereLoop
*pWLoop
; /* One of the WhereLoop objects */
4992 WhereLoop
**pX
; /* Used to divy up the pSpace memory */
4993 LogEst
*aSortCost
= 0; /* Sorting and partial sorting costs */
4994 char *pSpace
; /* Temporary memory used by this routine */
4995 int nSpace
; /* Bytes of space allocated at pSpace */
4997 pParse
= pWInfo
->pParse
;
4998 nLoop
= pWInfo
->nLevel
;
4999 /* TUNING: For simple queries, only the best path is tracked.
5000 ** For 2-way joins, the 5 best paths are followed.
5001 ** For joins of 3 or more tables, track the 10 best paths */
5002 mxChoice
= (nLoop
<=1) ? 1 : (nLoop
==2 ? 5 : 10);
5003 assert( nLoop
<=pWInfo
->pTabList
->nSrc
);
5004 WHERETRACE(0x002, ("---- begin solver. (nRowEst=%d, nQueryLoop=%d)\n",
5005 nRowEst
, pParse
->nQueryLoop
));
5007 /* If nRowEst is zero and there is an ORDER BY clause, ignore it. In this
5008 ** case the purpose of this call is to estimate the number of rows returned
5009 ** by the overall query. Once this estimate has been obtained, the caller
5010 ** will invoke this function a second time, passing the estimate as the
5011 ** nRowEst parameter. */
5012 if( pWInfo
->pOrderBy
==0 || nRowEst
==0 ){
5015 nOrderBy
= pWInfo
->pOrderBy
->nExpr
;
5018 /* Allocate and initialize space for aTo, aFrom and aSortCost[] */
5019 nSpace
= (sizeof(WherePath
)+sizeof(WhereLoop
*)*nLoop
)*mxChoice
*2;
5020 nSpace
+= sizeof(LogEst
) * nOrderBy
;
5021 pSpace
= sqlite3StackAllocRawNN(pParse
->db
, nSpace
);
5022 if( pSpace
==0 ) return SQLITE_NOMEM_BKPT
;
5023 aTo
= (WherePath
*)pSpace
;
5024 aFrom
= aTo
+mxChoice
;
5025 memset(aFrom
, 0, sizeof(aFrom
[0]));
5026 pX
= (WhereLoop
**)(aFrom
+mxChoice
);
5027 for(ii
=mxChoice
*2, pFrom
=aTo
; ii
>0; ii
--, pFrom
++, pX
+= nLoop
){
5031 /* If there is an ORDER BY clause and it is not being ignored, set up
5032 ** space for the aSortCost[] array. Each element of the aSortCost array
5033 ** is either zero - meaning it has not yet been initialized - or the
5034 ** cost of sorting nRowEst rows of data where the first X terms of
5035 ** the ORDER BY clause are already in order, where X is the array
5037 aSortCost
= (LogEst
*)pX
;
5038 memset(aSortCost
, 0, sizeof(LogEst
) * nOrderBy
);
5040 assert( aSortCost
==0 || &pSpace
[nSpace
]==(char*)&aSortCost
[nOrderBy
] );
5041 assert( aSortCost
!=0 || &pSpace
[nSpace
]==(char*)pX
);
5043 /* Seed the search with a single WherePath containing zero WhereLoops.
5045 ** TUNING: Do not let the number of iterations go above 28. If the cost
5046 ** of computing an automatic index is not paid back within the first 28
5047 ** rows, then do not use the automatic index. */
5048 aFrom
[0].nRow
= MIN(pParse
->nQueryLoop
, 48); assert( 48==sqlite3LogEst(28) );
5050 assert( aFrom
[0].isOrdered
==0 );
5052 /* If nLoop is zero, then there are no FROM terms in the query. Since
5053 ** in this case the query may return a maximum of one row, the results
5054 ** are already in the requested order. Set isOrdered to nOrderBy to
5055 ** indicate this. Or, if nLoop is greater than zero, set isOrdered to
5056 ** -1, indicating that the result set may or may not be ordered,
5057 ** depending on the loops added to the current plan. */
5058 aFrom
[0].isOrdered
= nLoop
>0 ? -1 : nOrderBy
;
5061 /* Compute successively longer WherePaths using the previous generation
5062 ** of WherePaths as the basis for the next. Keep track of the mxChoice
5063 ** best paths at each generation */
5064 for(iLoop
=0; iLoop
<nLoop
; iLoop
++){
5066 for(ii
=0, pFrom
=aFrom
; ii
<nFrom
; ii
++, pFrom
++){
5067 for(pWLoop
=pWInfo
->pLoops
; pWLoop
; pWLoop
=pWLoop
->pNextLoop
){
5068 LogEst nOut
; /* Rows visited by (pFrom+pWLoop) */
5069 LogEst rCost
; /* Cost of path (pFrom+pWLoop) */
5070 LogEst rUnsorted
; /* Unsorted cost of (pFrom+pWLoop) */
5071 i8 isOrdered
; /* isOrdered for (pFrom+pWLoop) */
5072 Bitmask maskNew
; /* Mask of src visited by (..) */
5073 Bitmask revMask
; /* Mask of rev-order loops for (..) */
5075 if( (pWLoop
->prereq
& ~pFrom
->maskLoop
)!=0 ) continue;
5076 if( (pWLoop
->maskSelf
& pFrom
->maskLoop
)!=0 ) continue;
5077 if( (pWLoop
->wsFlags
& WHERE_AUTO_INDEX
)!=0 && pFrom
->nRow
<3 ){
5078 /* Do not use an automatic index if the this loop is expected
5079 ** to run less than 1.25 times. It is tempting to also exclude
5080 ** automatic index usage on an outer loop, but sometimes an automatic
5081 ** index is useful in the outer loop of a correlated subquery. */
5082 assert( 10==sqlite3LogEst(2) );
5086 /* At this point, pWLoop is a candidate to be the next loop.
5087 ** Compute its cost */
5088 rUnsorted
= sqlite3LogEstAdd(pWLoop
->rSetup
,pWLoop
->rRun
+ pFrom
->nRow
);
5089 rUnsorted
= sqlite3LogEstAdd(rUnsorted
, pFrom
->rUnsorted
);
5090 nOut
= pFrom
->nRow
+ pWLoop
->nOut
;
5091 maskNew
= pFrom
->maskLoop
| pWLoop
->maskSelf
;
5092 isOrdered
= pFrom
->isOrdered
;
5095 isOrdered
= wherePathSatisfiesOrderBy(pWInfo
,
5096 pWInfo
->pOrderBy
, pFrom
, pWInfo
->wctrlFlags
,
5097 iLoop
, pWLoop
, &revMask
);
5099 revMask
= pFrom
->revLoop
;
5101 if( isOrdered
>=0 && isOrdered
<nOrderBy
){
5102 if( aSortCost
[isOrdered
]==0 ){
5103 aSortCost
[isOrdered
] = whereSortingCost(
5104 pWInfo
, nRowEst
, nOrderBy
, isOrdered
5107 /* TUNING: Add a small extra penalty (3) to sorting as an
5108 ** extra encouragement to the query planner to select a plan
5109 ** where the rows emerge in the correct order without any sorting
5111 rCost
= sqlite3LogEstAdd(rUnsorted
, aSortCost
[isOrdered
]) + 3;
5114 ("---- sort cost=%-3d (%d/%d) increases cost %3d to %-3d\n",
5115 aSortCost
[isOrdered
], (nOrderBy
-isOrdered
), nOrderBy
,
5119 rUnsorted
-= 2; /* TUNING: Slight bias in favor of no-sort plans */
5122 /* Check to see if pWLoop should be added to the set of
5123 ** mxChoice best-so-far paths.
5125 ** First look for an existing path among best-so-far paths
5126 ** that covers the same set of loops and has the same isOrdered
5127 ** setting as the current path candidate.
5129 ** The term "((pTo->isOrdered^isOrdered)&0x80)==0" is equivalent
5130 ** to (pTo->isOrdered==(-1))==(isOrdered==(-1))" for the range
5131 ** of legal values for isOrdered, -1..64.
5133 for(jj
=0, pTo
=aTo
; jj
<nTo
; jj
++, pTo
++){
5134 if( pTo
->maskLoop
==maskNew
5135 && ((pTo
->isOrdered
^isOrdered
)&0x80)==0
5137 testcase( jj
==nTo
-1 );
5142 /* None of the existing best-so-far paths match the candidate. */
5144 && (rCost
>mxCost
|| (rCost
==mxCost
&& rUnsorted
>=mxUnsorted
))
5146 /* The current candidate is no better than any of the mxChoice
5147 ** paths currently in the best-so-far buffer. So discard
5148 ** this candidate as not viable. */
5149 #ifdef WHERETRACE_ENABLED /* 0x4 */
5150 if( sqlite3WhereTrace
&0x4 ){
5151 sqlite3DebugPrintf("Skip %s cost=%-3d,%3d,%3d order=%c\n",
5152 wherePathName(pFrom
, iLoop
, pWLoop
), rCost
, nOut
, rUnsorted
,
5153 isOrdered
>=0 ? isOrdered
+'0' : '?');
5158 /* If we reach this points it means that the new candidate path
5159 ** needs to be added to the set of best-so-far paths. */
5161 /* Increase the size of the aTo set by one */
5164 /* New path replaces the prior worst to keep count below mxChoice */
5168 #ifdef WHERETRACE_ENABLED /* 0x4 */
5169 if( sqlite3WhereTrace
&0x4 ){
5170 sqlite3DebugPrintf("New %s cost=%-3d,%3d,%3d order=%c\n",
5171 wherePathName(pFrom
, iLoop
, pWLoop
), rCost
, nOut
, rUnsorted
,
5172 isOrdered
>=0 ? isOrdered
+'0' : '?');
5176 /* Control reaches here if best-so-far path pTo=aTo[jj] covers the
5177 ** same set of loops and has the same isOrdered setting as the
5178 ** candidate path. Check to see if the candidate should replace
5179 ** pTo or if the candidate should be skipped.
5181 ** The conditional is an expanded vector comparison equivalent to:
5182 ** (pTo->rCost,pTo->nRow,pTo->rUnsorted) <= (rCost,nOut,rUnsorted)
5184 if( pTo
->rCost
<rCost
5185 || (pTo
->rCost
==rCost
5187 || (pTo
->nRow
==nOut
&& pTo
->rUnsorted
<=rUnsorted
)
5191 #ifdef WHERETRACE_ENABLED /* 0x4 */
5192 if( sqlite3WhereTrace
&0x4 ){
5194 "Skip %s cost=%-3d,%3d,%3d order=%c",
5195 wherePathName(pFrom
, iLoop
, pWLoop
), rCost
, nOut
, rUnsorted
,
5196 isOrdered
>=0 ? isOrdered
+'0' : '?');
5197 sqlite3DebugPrintf(" vs %s cost=%-3d,%3d,%3d order=%c\n",
5198 wherePathName(pTo
, iLoop
+1, 0), pTo
->rCost
, pTo
->nRow
,
5199 pTo
->rUnsorted
, pTo
->isOrdered
>=0 ? pTo
->isOrdered
+'0' : '?');
5202 /* Discard the candidate path from further consideration */
5203 testcase( pTo
->rCost
==rCost
);
5206 testcase( pTo
->rCost
==rCost
+1 );
5207 /* Control reaches here if the candidate path is better than the
5208 ** pTo path. Replace pTo with the candidate. */
5209 #ifdef WHERETRACE_ENABLED /* 0x4 */
5210 if( sqlite3WhereTrace
&0x4 ){
5212 "Update %s cost=%-3d,%3d,%3d order=%c",
5213 wherePathName(pFrom
, iLoop
, pWLoop
), rCost
, nOut
, rUnsorted
,
5214 isOrdered
>=0 ? isOrdered
+'0' : '?');
5215 sqlite3DebugPrintf(" was %s cost=%-3d,%3d,%3d order=%c\n",
5216 wherePathName(pTo
, iLoop
+1, 0), pTo
->rCost
, pTo
->nRow
,
5217 pTo
->rUnsorted
, pTo
->isOrdered
>=0 ? pTo
->isOrdered
+'0' : '?');
5221 /* pWLoop is a winner. Add it to the set of best so far */
5222 pTo
->maskLoop
= pFrom
->maskLoop
| pWLoop
->maskSelf
;
5223 pTo
->revLoop
= revMask
;
5226 pTo
->rUnsorted
= rUnsorted
;
5227 pTo
->isOrdered
= isOrdered
;
5228 memcpy(pTo
->aLoop
, pFrom
->aLoop
, sizeof(WhereLoop
*)*iLoop
);
5229 pTo
->aLoop
[iLoop
] = pWLoop
;
5230 if( nTo
>=mxChoice
){
5232 mxCost
= aTo
[0].rCost
;
5233 mxUnsorted
= aTo
[0].nRow
;
5234 for(jj
=1, pTo
=&aTo
[1]; jj
<mxChoice
; jj
++, pTo
++){
5235 if( pTo
->rCost
>mxCost
5236 || (pTo
->rCost
==mxCost
&& pTo
->rUnsorted
>mxUnsorted
)
5238 mxCost
= pTo
->rCost
;
5239 mxUnsorted
= pTo
->rUnsorted
;
5247 #ifdef WHERETRACE_ENABLED /* >=2 */
5248 if( sqlite3WhereTrace
& 0x02 ){
5249 sqlite3DebugPrintf("---- after round %d ----\n", iLoop
);
5250 for(ii
=0, pTo
=aTo
; ii
<nTo
; ii
++, pTo
++){
5251 sqlite3DebugPrintf(" %s cost=%-3d nrow=%-3d order=%c",
5252 wherePathName(pTo
, iLoop
+1, 0), pTo
->rCost
, pTo
->nRow
,
5253 pTo
->isOrdered
>=0 ? (pTo
->isOrdered
+'0') : '?');
5254 if( pTo
->isOrdered
>0 ){
5255 sqlite3DebugPrintf(" rev=0x%llx\n", pTo
->revLoop
);
5257 sqlite3DebugPrintf("\n");
5263 /* Swap the roles of aFrom and aTo for the next generation */
5271 sqlite3ErrorMsg(pParse
, "no query solution");
5272 sqlite3StackFreeNN(pParse
->db
, pSpace
);
5273 return SQLITE_ERROR
;
5276 /* Find the lowest cost path. pFrom will be left pointing to that path */
5278 for(ii
=1; ii
<nFrom
; ii
++){
5279 if( pFrom
->rCost
>aFrom
[ii
].rCost
) pFrom
= &aFrom
[ii
];
5281 assert( pWInfo
->nLevel
==nLoop
);
5282 /* Load the lowest cost path into pWInfo */
5283 for(iLoop
=0; iLoop
<nLoop
; iLoop
++){
5284 WhereLevel
*pLevel
= pWInfo
->a
+ iLoop
;
5285 pLevel
->pWLoop
= pWLoop
= pFrom
->aLoop
[iLoop
];
5286 pLevel
->iFrom
= pWLoop
->iTab
;
5287 pLevel
->iTabCur
= pWInfo
->pTabList
->a
[pLevel
->iFrom
].iCursor
;
5289 if( (pWInfo
->wctrlFlags
& WHERE_WANT_DISTINCT
)!=0
5290 && (pWInfo
->wctrlFlags
& WHERE_DISTINCTBY
)==0
5291 && pWInfo
->eDistinct
==WHERE_DISTINCT_NOOP
5295 int rc
= wherePathSatisfiesOrderBy(pWInfo
, pWInfo
->pResultSet
, pFrom
,
5296 WHERE_DISTINCTBY
, nLoop
-1, pFrom
->aLoop
[nLoop
-1], ¬Used
);
5297 if( rc
==pWInfo
->pResultSet
->nExpr
){
5298 pWInfo
->eDistinct
= WHERE_DISTINCT_ORDERED
;
5301 pWInfo
->bOrderedInnerLoop
= 0;
5302 if( pWInfo
->pOrderBy
){
5303 pWInfo
->nOBSat
= pFrom
->isOrdered
;
5304 if( pWInfo
->wctrlFlags
& WHERE_DISTINCTBY
){
5305 if( pFrom
->isOrdered
==pWInfo
->pOrderBy
->nExpr
){
5306 pWInfo
->eDistinct
= WHERE_DISTINCT_ORDERED
;
5308 if( pWInfo
->pSelect
->pOrderBy
5309 && pWInfo
->nOBSat
> pWInfo
->pSelect
->pOrderBy
->nExpr
){
5310 pWInfo
->nOBSat
= pWInfo
->pSelect
->pOrderBy
->nExpr
;
5313 pWInfo
->revMask
= pFrom
->revLoop
;
5314 if( pWInfo
->nOBSat
<=0 ){
5317 u32 wsFlags
= pFrom
->aLoop
[nLoop
-1]->wsFlags
;
5318 if( (wsFlags
& WHERE_ONEROW
)==0
5319 && (wsFlags
&(WHERE_IPK
|WHERE_COLUMN_IN
))!=(WHERE_IPK
|WHERE_COLUMN_IN
)
5322 int rc
= wherePathSatisfiesOrderBy(pWInfo
, pWInfo
->pOrderBy
, pFrom
,
5323 WHERE_ORDERBY_LIMIT
, nLoop
-1, pFrom
->aLoop
[nLoop
-1], &m
);
5324 testcase( wsFlags
& WHERE_IPK
);
5325 testcase( wsFlags
& WHERE_COLUMN_IN
);
5326 if( rc
==pWInfo
->pOrderBy
->nExpr
){
5327 pWInfo
->bOrderedInnerLoop
= 1;
5328 pWInfo
->revMask
= m
;
5333 && pWInfo
->nOBSat
==1
5334 && (pWInfo
->wctrlFlags
& (WHERE_ORDERBY_MIN
|WHERE_ORDERBY_MAX
))!=0
5336 pWInfo
->bOrderedInnerLoop
= 1;
5339 if( (pWInfo
->wctrlFlags
& WHERE_SORTBYGROUP
)
5340 && pWInfo
->nOBSat
==pWInfo
->pOrderBy
->nExpr
&& nLoop
>0
5342 Bitmask revMask
= 0;
5343 int nOrder
= wherePathSatisfiesOrderBy(pWInfo
, pWInfo
->pOrderBy
,
5344 pFrom
, 0, nLoop
-1, pFrom
->aLoop
[nLoop
-1], &revMask
5346 assert( pWInfo
->sorted
==0 );
5347 if( nOrder
==pWInfo
->pOrderBy
->nExpr
){
5349 pWInfo
->revMask
= revMask
;
5355 pWInfo
->nRowOut
= pFrom
->nRow
;
5357 /* Free temporary memory and return success */
5358 sqlite3StackFreeNN(pParse
->db
, pSpace
);
5363 ** Most queries use only a single table (they are not joins) and have
5364 ** simple == constraints against indexed fields. This routine attempts
5365 ** to plan those simple cases using much less ceremony than the
5366 ** general-purpose query planner, and thereby yield faster sqlite3_prepare()
5367 ** times for the common case.
5369 ** Return non-zero on success, if this query can be handled by this
5370 ** no-frills query planner. Return zero if this query needs the
5371 ** general-purpose query planner.
5373 static int whereShortCut(WhereLoopBuilder
*pBuilder
){
5385 pWInfo
= pBuilder
->pWInfo
;
5386 if( pWInfo
->wctrlFlags
& WHERE_OR_SUBCLAUSE
) return 0;
5387 assert( pWInfo
->pTabList
->nSrc
>=1 );
5388 pItem
= pWInfo
->pTabList
->a
;
5390 if( IsVirtual(pTab
) ) return 0;
5391 if( pItem
->fg
.isIndexedBy
|| pItem
->fg
.notIndexed
){
5392 testcase( pItem
->fg
.isIndexedBy
);
5393 testcase( pItem
->fg
.notIndexed
);
5396 iCur
= pItem
->iCursor
;
5398 pLoop
= pBuilder
->pNew
;
5401 pTerm
= whereScanInit(&scan
, pWC
, iCur
, -1, WO_EQ
|WO_IS
, 0);
5402 while( pTerm
&& pTerm
->prereqRight
) pTerm
= whereScanNext(&scan
);
5404 testcase( pTerm
->eOperator
& WO_IS
);
5405 pLoop
->wsFlags
= WHERE_COLUMN_EQ
|WHERE_IPK
|WHERE_ONEROW
;
5406 pLoop
->aLTerm
[0] = pTerm
;
5408 pLoop
->u
.btree
.nEq
= 1;
5409 /* TUNING: Cost of a rowid lookup is 10 */
5410 pLoop
->rRun
= 33; /* 33==sqlite3LogEst(10) */
5412 for(pIdx
=pTab
->pIndex
; pIdx
; pIdx
=pIdx
->pNext
){
5414 assert( pLoop
->aLTermSpace
==pLoop
->aLTerm
);
5415 if( !IsUniqueIndex(pIdx
)
5416 || pIdx
->pPartIdxWhere
!=0
5417 || pIdx
->nKeyCol
>ArraySize(pLoop
->aLTermSpace
)
5419 opMask
= pIdx
->uniqNotNull
? (WO_EQ
|WO_IS
) : WO_EQ
;
5420 for(j
=0; j
<pIdx
->nKeyCol
; j
++){
5421 pTerm
= whereScanInit(&scan
, pWC
, iCur
, j
, opMask
, pIdx
);
5422 while( pTerm
&& pTerm
->prereqRight
) pTerm
= whereScanNext(&scan
);
5423 if( pTerm
==0 ) break;
5424 testcase( pTerm
->eOperator
& WO_IS
);
5425 pLoop
->aLTerm
[j
] = pTerm
;
5427 if( j
!=pIdx
->nKeyCol
) continue;
5428 pLoop
->wsFlags
= WHERE_COLUMN_EQ
|WHERE_ONEROW
|WHERE_INDEXED
;
5429 if( pIdx
->isCovering
|| (pItem
->colUsed
& pIdx
->colNotIdxed
)==0 ){
5430 pLoop
->wsFlags
|= WHERE_IDX_ONLY
;
5433 pLoop
->u
.btree
.nEq
= j
;
5434 pLoop
->u
.btree
.pIndex
= pIdx
;
5435 /* TUNING: Cost of a unique index lookup is 15 */
5436 pLoop
->rRun
= 39; /* 39==sqlite3LogEst(15) */
5440 if( pLoop
->wsFlags
){
5441 pLoop
->nOut
= (LogEst
)1;
5442 pWInfo
->a
[0].pWLoop
= pLoop
;
5443 assert( pWInfo
->sMaskSet
.n
==1 && iCur
==pWInfo
->sMaskSet
.ix
[0] );
5444 pLoop
->maskSelf
= 1; /* sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); */
5445 pWInfo
->a
[0].iTabCur
= iCur
;
5446 pWInfo
->nRowOut
= 1;
5447 if( pWInfo
->pOrderBy
) pWInfo
->nOBSat
= pWInfo
->pOrderBy
->nExpr
;
5448 if( pWInfo
->wctrlFlags
& WHERE_WANT_DISTINCT
){
5449 pWInfo
->eDistinct
= WHERE_DISTINCT_UNIQUE
;
5451 if( scan
.iEquiv
>1 ) pLoop
->wsFlags
|= WHERE_TRANSCONS
;
5455 #ifdef WHERETRACE_ENABLED
5456 if( sqlite3WhereTrace
& 0x02 ){
5457 sqlite3DebugPrintf("whereShortCut() used to compute solution\n");
5466 ** Helper function for exprIsDeterministic().
5468 static int exprNodeIsDeterministic(Walker
*pWalker
, Expr
*pExpr
){
5469 if( pExpr
->op
==TK_FUNCTION
&& ExprHasProperty(pExpr
, EP_ConstFunc
)==0 ){
5473 return WRC_Continue
;
5477 ** Return true if the expression contains no non-deterministic SQL
5478 ** functions. Do not consider non-deterministic SQL functions that are
5479 ** part of sub-select statements.
5481 static int exprIsDeterministic(Expr
*p
){
5483 memset(&w
, 0, sizeof(w
));
5485 w
.xExprCallback
= exprNodeIsDeterministic
;
5486 w
.xSelectCallback
= sqlite3SelectWalkFail
;
5487 sqlite3WalkExpr(&w
, p
);
5492 #ifdef WHERETRACE_ENABLED
5494 ** Display all WhereLoops in pWInfo
5496 static void showAllWhereLoops(WhereInfo
*pWInfo
, WhereClause
*pWC
){
5497 if( sqlite3WhereTrace
){ /* Display all of the WhereLoop objects */
5500 static const char zLabel
[] = "0123456789abcdefghijklmnopqrstuvwyxz"
5501 "ABCDEFGHIJKLMNOPQRSTUVWYXZ";
5502 for(p
=pWInfo
->pLoops
, i
=0; p
; p
=p
->pNextLoop
, i
++){
5503 p
->cId
= zLabel
[i
%(sizeof(zLabel
)-1)];
5504 sqlite3WhereLoopPrint(p
, pWC
);
5508 # define WHERETRACE_ALL_LOOPS(W,C) showAllWhereLoops(W,C)
5510 # define WHERETRACE_ALL_LOOPS(W,C)
5513 /* Attempt to omit tables from a join that do not affect the result.
5514 ** For a table to not affect the result, the following must be true:
5516 ** 1) The query must not be an aggregate.
5517 ** 2) The table must be the RHS of a LEFT JOIN.
5518 ** 3) Either the query must be DISTINCT, or else the ON or USING clause
5519 ** must contain a constraint that limits the scan of the table to
5520 ** at most a single row.
5521 ** 4) The table must not be referenced by any part of the query apart
5522 ** from its own USING or ON clause.
5523 ** 5) The table must not have an inner-join ON or USING clause if there is
5524 ** a RIGHT JOIN anywhere in the query. Otherwise the ON/USING clause
5525 ** might move from the right side to the left side of the RIGHT JOIN.
5526 ** Note: Due to (2), this condition can only arise if the table is
5527 ** the right-most table of a subquery that was flattened into the
5528 ** main query and that subquery was the right-hand operand of an
5529 ** inner join that held an ON or USING clause.
5531 ** For example, given:
5533 ** CREATE TABLE t1(ipk INTEGER PRIMARY KEY, v1);
5534 ** CREATE TABLE t2(ipk INTEGER PRIMARY KEY, v2);
5535 ** CREATE TABLE t3(ipk INTEGER PRIMARY KEY, v3);
5537 ** then table t2 can be omitted from the following:
5539 ** SELECT v1, v3 FROM t1
5540 ** LEFT JOIN t2 ON (t1.ipk=t2.ipk)
5541 ** LEFT JOIN t3 ON (t1.ipk=t3.ipk)
5545 ** SELECT DISTINCT v1, v3 FROM t1
5547 ** LEFT JOIN t3 ON (t1.ipk=t3.ipk)
5549 static SQLITE_NOINLINE Bitmask
whereOmitNoopJoin(
5557 /* Preconditions checked by the caller */
5558 assert( pWInfo
->nLevel
>=2 );
5559 assert( OptimizationEnabled(pWInfo
->pParse
->db
, SQLITE_OmitNoopJoin
) );
5561 /* These two preconditions checked by the caller combine to guarantee
5562 ** condition (1) of the header comment */
5563 assert( pWInfo
->pResultSet
!=0 );
5564 assert( 0==(pWInfo
->wctrlFlags
& WHERE_AGG_DISTINCT
) );
5566 tabUsed
= sqlite3WhereExprListUsage(&pWInfo
->sMaskSet
, pWInfo
->pResultSet
);
5567 if( pWInfo
->pOrderBy
){
5568 tabUsed
|= sqlite3WhereExprListUsage(&pWInfo
->sMaskSet
, pWInfo
->pOrderBy
);
5570 hasRightJoin
= (pWInfo
->pTabList
->a
[0].fg
.jointype
& JT_LTORJ
)!=0;
5571 for(i
=pWInfo
->nLevel
-1; i
>=1; i
--){
5572 WhereTerm
*pTerm
, *pEnd
;
5575 pLoop
= pWInfo
->a
[i
].pWLoop
;
5576 pItem
= &pWInfo
->pTabList
->a
[pLoop
->iTab
];
5577 if( (pItem
->fg
.jointype
& (JT_LEFT
|JT_RIGHT
))!=JT_LEFT
) continue;
5578 if( (pWInfo
->wctrlFlags
& WHERE_WANT_DISTINCT
)==0
5579 && (pLoop
->wsFlags
& WHERE_ONEROW
)==0
5583 if( (tabUsed
& pLoop
->maskSelf
)!=0 ) continue;
5584 pEnd
= pWInfo
->sWC
.a
+ pWInfo
->sWC
.nTerm
;
5585 for(pTerm
=pWInfo
->sWC
.a
; pTerm
<pEnd
; pTerm
++){
5586 if( (pTerm
->prereqAll
& pLoop
->maskSelf
)!=0 ){
5587 if( !ExprHasProperty(pTerm
->pExpr
, EP_OuterON
)
5588 || pTerm
->pExpr
->w
.iJoin
!=pItem
->iCursor
5594 && ExprHasProperty(pTerm
->pExpr
, EP_InnerON
)
5595 && pTerm
->pExpr
->w
.iJoin
==pItem
->iCursor
5597 break; /* restriction (5) */
5600 if( pTerm
<pEnd
) continue;
5601 WHERETRACE(0xffffffff, ("-> drop loop %c not used\n", pLoop
->cId
));
5602 notReady
&= ~pLoop
->maskSelf
;
5603 for(pTerm
=pWInfo
->sWC
.a
; pTerm
<pEnd
; pTerm
++){
5604 if( (pTerm
->prereqAll
& pLoop
->maskSelf
)!=0 ){
5605 pTerm
->wtFlags
|= TERM_CODED
;
5608 if( i
!=pWInfo
->nLevel
-1 ){
5609 int nByte
= (pWInfo
->nLevel
-1-i
) * sizeof(WhereLevel
);
5610 memmove(&pWInfo
->a
[i
], &pWInfo
->a
[i
+1], nByte
);
5613 assert( pWInfo
->nLevel
>0 );
5619 ** Check to see if there are any SEARCH loops that might benefit from
5620 ** using a Bloom filter. Consider a Bloom filter if:
5622 ** (1) The SEARCH happens more than N times where N is the number
5623 ** of rows in the table that is being considered for the Bloom
5625 ** (2) Some searches are expected to find zero rows. (This is determined
5626 ** by the WHERE_SELFCULL flag on the term.)
5627 ** (3) Bloom-filter processing is not disabled. (Checked by the
5629 ** (4) The size of the table being searched is known by ANALYZE.
5631 ** This block of code merely checks to see if a Bloom filter would be
5632 ** appropriate, and if so sets the WHERE_BLOOMFILTER flag on the
5633 ** WhereLoop. The implementation of the Bloom filter comes further
5634 ** down where the code for each WhereLoop is generated.
5636 static SQLITE_NOINLINE
void whereCheckIfBloomFilterIsUseful(
5637 const WhereInfo
*pWInfo
5642 assert( pWInfo
->nLevel
>=2 );
5643 assert( OptimizationEnabled(pWInfo
->pParse
->db
, SQLITE_BloomFilter
) );
5644 for(i
=0; i
<pWInfo
->nLevel
; i
++){
5645 WhereLoop
*pLoop
= pWInfo
->a
[i
].pWLoop
;
5646 const unsigned int reqFlags
= (WHERE_SELFCULL
|WHERE_COLUMN_EQ
);
5647 SrcItem
*pItem
= &pWInfo
->pTabList
->a
[pLoop
->iTab
];
5648 Table
*pTab
= pItem
->pTab
;
5649 if( (pTab
->tabFlags
& TF_HasStat1
)==0 ) break;
5650 pTab
->tabFlags
|= TF_StatsUsed
;
5652 && (pLoop
->wsFlags
& reqFlags
)==reqFlags
5653 /* vvvvvv--- Always the case if WHERE_COLUMN_EQ is defined */
5654 && ALWAYS((pLoop
->wsFlags
& (WHERE_IPK
|WHERE_INDEXED
))!=0)
5656 if( nSearch
> pTab
->nRowLogEst
){
5657 testcase( pItem
->fg
.jointype
& JT_LEFT
);
5658 pLoop
->wsFlags
|= WHERE_BLOOMFILTER
;
5659 pLoop
->wsFlags
&= ~WHERE_IDX_ONLY
;
5660 WHERETRACE(0xffffffff, (
5661 "-> use Bloom-filter on loop %c because there are ~%.1e "
5662 "lookups into %s which has only ~%.1e rows\n",
5663 pLoop
->cId
, (double)sqlite3LogEstToInt(nSearch
), pTab
->zName
,
5664 (double)sqlite3LogEstToInt(pTab
->nRowLogEst
)));
5667 nSearch
+= pLoop
->nOut
;
5672 ** This is an sqlite3ParserAddCleanup() callback that is invoked to
5673 ** free the Parse->pIdxEpr list when the Parse object is destroyed.
5675 static void whereIndexedExprCleanup(sqlite3
*db
, void *pObject
){
5676 IndexedExpr
**pp
= (IndexedExpr
**)pObject
;
5678 IndexedExpr
*p
= *pp
;
5680 sqlite3ExprDelete(db
, p
->pExpr
);
5681 sqlite3DbFreeNN(db
, p
);
5686 ** The index pIdx is used by a query and contains one or more expressions.
5687 ** In other words pIdx is an index on an expression. iIdxCur is the cursor
5688 ** number for the index and iDataCur is the cursor number for the corresponding
5691 ** This routine adds IndexedExpr entries to the Parse->pIdxEpr field for
5692 ** each of the expressions in the index so that the expression code generator
5693 ** will know to replace occurrences of the indexed expression with
5694 ** references to the corresponding column of the index.
5696 static SQLITE_NOINLINE
void whereAddIndexedExpr(
5697 Parse
*pParse
, /* Add IndexedExpr entries to pParse->pIdxEpr */
5698 Index
*pIdx
, /* The index-on-expression that contains the expressions */
5699 int iIdxCur
, /* Cursor number for pIdx */
5700 SrcItem
*pTabItem
/* The FROM clause entry for the table */
5705 assert( pIdx
->bHasExpr
);
5706 pTab
= pIdx
->pTable
;
5707 for(i
=0; i
<pIdx
->nColumn
; i
++){
5709 int j
= pIdx
->aiColumn
[i
];
5712 pExpr
= pIdx
->aColExpr
->a
[i
].pExpr
;
5713 testcase( pTabItem
->fg
.jointype
& JT_LEFT
);
5714 testcase( pTabItem
->fg
.jointype
& JT_RIGHT
);
5715 testcase( pTabItem
->fg
.jointype
& JT_LTORJ
);
5716 bMaybeNullRow
= (pTabItem
->fg
.jointype
& (JT_LEFT
|JT_LTORJ
|JT_RIGHT
))!=0;
5717 }else if( j
>=0 && (pTab
->aCol
[j
].colFlags
& COLFLAG_VIRTUAL
)!=0 ){
5718 pExpr
= sqlite3ColumnExpr(pTab
, &pTab
->aCol
[j
]);
5723 if( sqlite3ExprIsConstant(pExpr
) ) continue;
5724 p
= sqlite3DbMallocRaw(pParse
->db
, sizeof(IndexedExpr
));
5726 p
->pIENext
= pParse
->pIdxEpr
;
5727 #ifdef WHERETRACE_ENABLED
5728 if( sqlite3WhereTrace
& 0x200 ){
5729 sqlite3DebugPrintf("New pParse->pIdxEpr term {%d,%d}\n", iIdxCur
, i
);
5730 if( sqlite3WhereTrace
& 0x5000 ) sqlite3ShowExpr(pExpr
);
5733 p
->pExpr
= sqlite3ExprDup(pParse
->db
, pExpr
, 0);
5734 p
->iDataCur
= pTabItem
->iCursor
;
5735 p
->iIdxCur
= iIdxCur
;
5737 p
->bMaybeNullRow
= bMaybeNullRow
;
5738 if( sqlite3IndexAffinityStr(pParse
->db
, pIdx
) ){
5739 p
->aff
= pIdx
->zColAff
[i
];
5741 #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS
5742 p
->zIdxName
= pIdx
->zName
;
5744 pParse
->pIdxEpr
= p
;
5745 if( p
->pIENext
==0 ){
5746 void *pArg
= (void*)&pParse
->pIdxEpr
;
5747 sqlite3ParserAddCleanup(pParse
, whereIndexedExprCleanup
, pArg
);
5753 ** This function is called for a partial index - one with a WHERE clause - in
5754 ** two scenarios. In both cases, it determines whether or not the WHERE
5755 ** clause on the index implies that a column of the table may be safely
5756 ** replaced by a constant expression. For example, in the following
5759 ** CREATE INDEX i1 ON t1(b, c) WHERE a=<expr>;
5760 ** SELECT a, b, c FROM t1 WHERE a=<expr> AND b=?;
5762 ** The "a" in the select-list may be replaced by <expr>, iff:
5764 ** (a) <expr> is a constant expression, and
5765 ** (b) The (a=<expr>) comparison uses the BINARY collation sequence, and
5766 ** (c) Column "a" has an affinity other than NONE or BLOB.
5768 ** If argument pTabItem is NULL, then this function is being called as part
5769 ** of parsing the CREATE INDEX statement. In that case the Index.colNotIdxed
5770 ** mask is updated to mark any columns that will be replaced by constant
5771 ** values as indexed.
5773 ** Otherwise, if pTabItem is not NULL, then this function is being called
5774 ** as part of coding a loop that uses index pIdx. In this case, add entries
5775 ** to the Parse.pIdxPartExpr list for each column that can be replaced
5778 void sqlite3WherePartIdxExpr(
5779 Parse
*pParse
, /* Parse context */
5780 Index
*pIdx
, /* Partial index being processed */
5781 Expr
*pPart
, /* WHERE clause being processed */
5782 int iIdxCur
, /* Cursor number for index */
5783 SrcItem
*pTabItem
/* The FROM clause entry for the table */
5785 assert( pTabItem
==0 || (pTabItem
->fg
.jointype
& JT_RIGHT
)==0 );
5786 if( pPart
->op
==TK_AND
){
5787 sqlite3WherePartIdxExpr(pParse
, pIdx
, pPart
->pRight
, iIdxCur
, pTabItem
);
5788 pPart
= pPart
->pLeft
;
5791 if( (pPart
->op
==TK_EQ
|| pPart
->op
==TK_IS
) ){
5792 Expr
*pLeft
= pPart
->pLeft
;
5793 Expr
*pRight
= pPart
->pRight
;
5796 if( pRight
->op
==TK_COLUMN
){
5797 SWAP(Expr
*, pLeft
, pRight
);
5800 if( pLeft
->op
!=TK_COLUMN
) return;
5801 if( !sqlite3ExprIsConstant(pRight
) ) return;
5802 if( !sqlite3IsBinary(sqlite3ExprCompareCollSeq(pParse
, pPart
)) ) return;
5803 if( pLeft
->iColumn
<0 ) return;
5804 aff
= pIdx
->pTable
->aCol
[pLeft
->iColumn
].affinity
;
5805 if( aff
>=SQLITE_AFF_TEXT
){
5807 sqlite3
*db
= pParse
->db
;
5808 IndexedExpr
*p
= (IndexedExpr
*)sqlite3DbMallocZero(db
, sizeof(*p
));
5810 int bNullRow
= (pTabItem
->fg
.jointype
&(JT_LEFT
|JT_LTORJ
))!=0;
5811 p
->pExpr
= sqlite3ExprDup(db
, pRight
, 0);
5812 p
->iDataCur
= pTabItem
->iCursor
;
5813 p
->iIdxCur
= iIdxCur
;
5814 p
->iIdxCol
= pLeft
->iColumn
;
5815 p
->bMaybeNullRow
= bNullRow
;
5816 p
->pIENext
= pParse
->pIdxPartExpr
;
5818 pParse
->pIdxPartExpr
= p
;
5819 if( p
->pIENext
==0 ){
5820 void *pArg
= (void*)&pParse
->pIdxPartExpr
;
5821 sqlite3ParserAddCleanup(pParse
, whereIndexedExprCleanup
, pArg
);
5824 }else if( pLeft
->iColumn
<(BMS
-1) ){
5825 pIdx
->colNotIdxed
&= ~((Bitmask
)1 << pLeft
->iColumn
);
5832 ** Set the reverse-scan order mask to one for all tables in the query
5833 ** with the exception of MATERIALIZED common table expressions that have
5834 ** their own internal ORDER BY clauses.
5836 ** This implements the PRAGMA reverse_unordered_selects=ON setting.
5837 ** (Also SQLITE_DBCONFIG_REVERSE_SCANORDER).
5839 static SQLITE_NOINLINE
void whereReverseScanOrder(WhereInfo
*pWInfo
){
5841 for(ii
=0; ii
<pWInfo
->pTabList
->nSrc
; ii
++){
5842 SrcItem
*pItem
= &pWInfo
->pTabList
->a
[ii
];
5843 if( !pItem
->fg
.isCte
5844 || pItem
->u2
.pCteUse
->eM10d
!=M10d_Yes
5845 || NEVER(pItem
->pSelect
==0)
5846 || pItem
->pSelect
->pOrderBy
==0
5848 pWInfo
->revMask
|= MASKBIT(ii
);
5854 ** Generate the beginning of the loop used for WHERE clause processing.
5855 ** The return value is a pointer to an opaque structure that contains
5856 ** information needed to terminate the loop. Later, the calling routine
5857 ** should invoke sqlite3WhereEnd() with the return value of this function
5858 ** in order to complete the WHERE clause processing.
5860 ** If an error occurs, this routine returns NULL.
5862 ** The basic idea is to do a nested loop, one loop for each table in
5863 ** the FROM clause of a select. (INSERT and UPDATE statements are the
5864 ** same as a SELECT with only a single table in the FROM clause.) For
5865 ** example, if the SQL is this:
5867 ** SELECT * FROM t1, t2, t3 WHERE ...;
5869 ** Then the code generated is conceptually like the following:
5871 ** foreach row1 in t1 do \ Code generated
5872 ** foreach row2 in t2 do |-- by sqlite3WhereBegin()
5873 ** foreach row3 in t3 do /
5875 ** end \ Code generated
5876 ** end |-- by sqlite3WhereEnd()
5879 ** Note that the loops might not be nested in the order in which they
5880 ** appear in the FROM clause if a different order is better able to make
5881 ** use of indices. Note also that when the IN operator appears in
5882 ** the WHERE clause, it might result in additional nested loops for
5883 ** scanning through all values on the right-hand side of the IN.
5885 ** There are Btree cursors associated with each table. t1 uses cursor
5886 ** number pTabList->a[0].iCursor. t2 uses the cursor pTabList->a[1].iCursor.
5887 ** And so forth. This routine generates code to open those VDBE cursors
5888 ** and sqlite3WhereEnd() generates the code to close them.
5890 ** The code that sqlite3WhereBegin() generates leaves the cursors named
5891 ** in pTabList pointing at their appropriate entries. The [...] code
5892 ** can use OP_Column and OP_Rowid opcodes on these cursors to extract
5893 ** data from the various tables of the loop.
5895 ** If the WHERE clause is empty, the foreach loops must each scan their
5896 ** entire tables. Thus a three-way join is an O(N^3) operation. But if
5897 ** the tables have indices and there are terms in the WHERE clause that
5898 ** refer to those indices, a complete table scan can be avoided and the
5899 ** code will run much faster. Most of the work of this routine is checking
5900 ** to see if there are indices that can be used to speed up the loop.
5902 ** Terms of the WHERE clause are also used to limit which rows actually
5903 ** make it to the "..." in the middle of the loop. After each "foreach",
5904 ** terms of the WHERE clause that use only terms in that loop and outer
5905 ** loops are evaluated and if false a jump is made around all subsequent
5906 ** inner loops (or around the "..." if the test occurs within the inner-
5911 ** An outer join of tables t1 and t2 is conceptually coded as follows:
5913 ** foreach row1 in t1 do
5915 ** foreach row2 in t2 do
5921 ** move the row2 cursor to a null row
5926 ** ORDER BY CLAUSE PROCESSING
5928 ** pOrderBy is a pointer to the ORDER BY clause (or the GROUP BY clause
5929 ** if the WHERE_GROUPBY flag is set in wctrlFlags) of a SELECT statement
5930 ** if there is one. If there is no ORDER BY clause or if this routine
5931 ** is called from an UPDATE or DELETE statement, then pOrderBy is NULL.
5933 ** The iIdxCur parameter is the cursor number of an index. If
5934 ** WHERE_OR_SUBCLAUSE is set, iIdxCur is the cursor number of an index
5935 ** to use for OR clause processing. The WHERE clause should use this
5936 ** specific cursor. If WHERE_ONEPASS_DESIRED is set, then iIdxCur is
5937 ** the first cursor in an array of cursors for all indices. iIdxCur should
5938 ** be used to compute the appropriate cursor depending on which index is
5941 WhereInfo
*sqlite3WhereBegin(
5942 Parse
*pParse
, /* The parser context */
5943 SrcList
*pTabList
, /* FROM clause: A list of all tables to be scanned */
5944 Expr
*pWhere
, /* The WHERE clause */
5945 ExprList
*pOrderBy
, /* An ORDER BY (or GROUP BY) clause, or NULL */
5946 ExprList
*pResultSet
, /* Query result set. Req'd for DISTINCT */
5947 Select
*pSelect
, /* The entire SELECT statement */
5948 u16 wctrlFlags
, /* The WHERE_* flags defined in sqliteInt.h */
5949 int iAuxArg
/* If WHERE_OR_SUBCLAUSE is set, index cursor number
5950 ** If WHERE_USE_LIMIT, then the limit amount */
5952 int nByteWInfo
; /* Num. bytes allocated for WhereInfo struct */
5953 int nTabList
; /* Number of elements in pTabList */
5954 WhereInfo
*pWInfo
; /* Will become the return value of this function */
5955 Vdbe
*v
= pParse
->pVdbe
; /* The virtual database engine */
5956 Bitmask notReady
; /* Cursors that are not yet positioned */
5957 WhereLoopBuilder sWLB
; /* The WhereLoop builder */
5958 WhereMaskSet
*pMaskSet
; /* The expression mask set */
5959 WhereLevel
*pLevel
; /* A single level in pWInfo->a[] */
5960 WhereLoop
*pLoop
; /* Pointer to a single WhereLoop object */
5961 int ii
; /* Loop counter */
5962 sqlite3
*db
; /* Database connection */
5963 int rc
; /* Return code */
5964 u8 bFordelete
= 0; /* OPFLAG_FORDELETE or zero, as appropriate */
5966 assert( (wctrlFlags
& WHERE_ONEPASS_MULTIROW
)==0 || (
5967 (wctrlFlags
& WHERE_ONEPASS_DESIRED
)!=0
5968 && (wctrlFlags
& WHERE_OR_SUBCLAUSE
)==0
5971 /* Only one of WHERE_OR_SUBCLAUSE or WHERE_USE_LIMIT */
5972 assert( (wctrlFlags
& WHERE_OR_SUBCLAUSE
)==0
5973 || (wctrlFlags
& WHERE_USE_LIMIT
)==0 );
5975 /* Variable initialization */
5977 memset(&sWLB
, 0, sizeof(sWLB
));
5979 /* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */
5980 testcase( pOrderBy
&& pOrderBy
->nExpr
==BMS
-1 );
5981 if( pOrderBy
&& pOrderBy
->nExpr
>=BMS
) pOrderBy
= 0;
5983 /* The number of tables in the FROM clause is limited by the number of
5984 ** bits in a Bitmask
5986 testcase( pTabList
->nSrc
==BMS
);
5987 if( pTabList
->nSrc
>BMS
){
5988 sqlite3ErrorMsg(pParse
, "at most %d tables in a join", BMS
);
5992 /* This function normally generates a nested loop for all tables in
5993 ** pTabList. But if the WHERE_OR_SUBCLAUSE flag is set, then we should
5994 ** only generate code for the first table in pTabList and assume that
5995 ** any cursors associated with subsequent tables are uninitialized.
5997 nTabList
= (wctrlFlags
& WHERE_OR_SUBCLAUSE
) ? 1 : pTabList
->nSrc
;
5999 /* Allocate and initialize the WhereInfo structure that will become the
6000 ** return value. A single allocation is used to store the WhereInfo
6001 ** struct, the contents of WhereInfo.a[], the WhereClause structure
6002 ** and the WhereMaskSet structure. Since WhereClause contains an 8-byte
6003 ** field (type Bitmask) it must be aligned on an 8-byte boundary on
6004 ** some architectures. Hence the ROUND8() below.
6006 nByteWInfo
= ROUND8P(sizeof(WhereInfo
)+(nTabList
-1)*sizeof(WhereLevel
));
6007 pWInfo
= sqlite3DbMallocRawNN(db
, nByteWInfo
+ sizeof(WhereLoop
));
6008 if( db
->mallocFailed
){
6009 sqlite3DbFree(db
, pWInfo
);
6011 goto whereBeginError
;
6013 pWInfo
->pParse
= pParse
;
6014 pWInfo
->pTabList
= pTabList
;
6015 pWInfo
->pOrderBy
= pOrderBy
;
6016 #if WHERETRACE_ENABLED
6017 pWInfo
->pWhere
= pWhere
;
6019 pWInfo
->pResultSet
= pResultSet
;
6020 pWInfo
->aiCurOnePass
[0] = pWInfo
->aiCurOnePass
[1] = -1;
6021 pWInfo
->nLevel
= nTabList
;
6022 pWInfo
->iBreak
= pWInfo
->iContinue
= sqlite3VdbeMakeLabel(pParse
);
6023 pWInfo
->wctrlFlags
= wctrlFlags
;
6024 pWInfo
->iLimit
= iAuxArg
;
6025 pWInfo
->savedNQueryLoop
= pParse
->nQueryLoop
;
6026 pWInfo
->pSelect
= pSelect
;
6027 memset(&pWInfo
->nOBSat
, 0,
6028 offsetof(WhereInfo
,sWC
) - offsetof(WhereInfo
,nOBSat
));
6029 memset(&pWInfo
->a
[0], 0, sizeof(WhereLoop
)+nTabList
*sizeof(WhereLevel
));
6030 assert( pWInfo
->eOnePass
==ONEPASS_OFF
); /* ONEPASS defaults to OFF */
6031 pMaskSet
= &pWInfo
->sMaskSet
;
6033 pMaskSet
->ix
[0] = -99; /* Initialize ix[0] to a value that can never be
6034 ** a valid cursor number, to avoid an initial
6035 ** test for pMaskSet->n==0 in sqlite3WhereGetMask() */
6036 sWLB
.pWInfo
= pWInfo
;
6037 sWLB
.pWC
= &pWInfo
->sWC
;
6038 sWLB
.pNew
= (WhereLoop
*)(((char*)pWInfo
)+nByteWInfo
);
6039 assert( EIGHT_BYTE_ALIGNMENT(sWLB
.pNew
) );
6040 whereLoopInit(sWLB
.pNew
);
6042 sWLB
.pNew
->cId
= '*';
6045 /* Split the WHERE clause into separate subexpressions where each
6046 ** subexpression is separated by an AND operator.
6048 sqlite3WhereClauseInit(&pWInfo
->sWC
, pWInfo
);
6049 sqlite3WhereSplit(&pWInfo
->sWC
, pWhere
, TK_AND
);
6051 /* Special case: No FROM clause
6054 if( pOrderBy
) pWInfo
->nOBSat
= pOrderBy
->nExpr
;
6055 if( (wctrlFlags
& WHERE_WANT_DISTINCT
)!=0
6056 && OptimizationEnabled(db
, SQLITE_DistinctOpt
)
6058 pWInfo
->eDistinct
= WHERE_DISTINCT_UNIQUE
;
6060 ExplainQueryPlan((pParse
, 0, "SCAN CONSTANT ROW"));
6062 /* Assign a bit from the bitmask to every term in the FROM clause.
6064 ** The N-th term of the FROM clause is assigned a bitmask of 1<<N.
6066 ** The rule of the previous sentence ensures that if X is the bitmask for
6067 ** a table T, then X-1 is the bitmask for all other tables to the left of T.
6068 ** Knowing the bitmask for all tables to the left of a left join is
6069 ** important. Ticket #3015.
6071 ** Note that bitmasks are created for all pTabList->nSrc tables in
6072 ** pTabList, not just the first nTabList tables. nTabList is normally
6073 ** equal to pTabList->nSrc but might be shortened to 1 if the
6074 ** WHERE_OR_SUBCLAUSE flag is set.
6078 createMask(pMaskSet
, pTabList
->a
[ii
].iCursor
);
6079 sqlite3WhereTabFuncArgs(pParse
, &pTabList
->a
[ii
], &pWInfo
->sWC
);
6080 }while( (++ii
)<pTabList
->nSrc
);
6084 for(ii
=0; ii
<pTabList
->nSrc
; ii
++){
6085 Bitmask m
= sqlite3WhereGetMask(pMaskSet
, pTabList
->a
[ii
].iCursor
);
6093 /* Analyze all of the subexpressions. */
6094 sqlite3WhereExprAnalyze(pTabList
, &pWInfo
->sWC
);
6095 if( pSelect
&& pSelect
->pLimit
){
6096 sqlite3WhereAddLimit(&pWInfo
->sWC
, pSelect
);
6098 if( pParse
->nErr
) goto whereBeginError
;
6100 /* The False-WHERE-Term-Bypass optimization:
6102 ** If there are WHERE terms that are false, then no rows will be output,
6103 ** so skip over all of the code generated here.
6107 ** (1) The WHERE term must not refer to any tables in the join.
6108 ** (2) The term must not come from an ON clause on the
6109 ** right-hand side of a LEFT or FULL JOIN.
6110 ** (3) The term must not come from an ON clause, or there must be
6111 ** no RIGHT or FULL OUTER joins in pTabList.
6112 ** (4) If the expression contains non-deterministic functions
6113 ** that are not within a sub-select. This is not required
6114 ** for correctness but rather to preserves SQLite's legacy
6115 ** behaviour in the following two cases:
6117 ** WHERE random()>0; -- eval random() once per row
6118 ** WHERE (SELECT random())>0; -- eval random() just once overall
6120 ** Note that the Where term need not be a constant in order for this
6121 ** optimization to apply, though it does need to be constant relative to
6122 ** the current subquery (condition 1). The term might include variables
6123 ** from outer queries so that the value of the term changes from one
6124 ** invocation of the current subquery to the next.
6126 for(ii
=0; ii
<sWLB
.pWC
->nBase
; ii
++){
6127 WhereTerm
*pT
= &sWLB
.pWC
->a
[ii
]; /* A term of the WHERE clause */
6128 Expr
*pX
; /* The expression of pT */
6129 if( pT
->wtFlags
& TERM_VIRTUAL
) continue;
6132 assert( pT
->prereqAll
!=0 || !ExprHasProperty(pX
, EP_OuterON
) );
6133 if( pT
->prereqAll
==0 /* Conditions (1) and (2) */
6134 && (nTabList
==0 || exprIsDeterministic(pX
)) /* Condition (4) */
6135 && !(ExprHasProperty(pX
, EP_InnerON
) /* Condition (3) */
6136 && (pTabList
->a
[0].fg
.jointype
& JT_LTORJ
)!=0 )
6138 sqlite3ExprIfFalse(pParse
, pX
, pWInfo
->iBreak
, SQLITE_JUMPIFNULL
);
6139 pT
->wtFlags
|= TERM_CODED
;
6143 if( wctrlFlags
& WHERE_WANT_DISTINCT
){
6144 if( OptimizationDisabled(db
, SQLITE_DistinctOpt
) ){
6145 /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via
6146 ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */
6147 wctrlFlags
&= ~WHERE_WANT_DISTINCT
;
6148 pWInfo
->wctrlFlags
&= ~WHERE_WANT_DISTINCT
;
6149 }else if( isDistinctRedundant(pParse
, pTabList
, &pWInfo
->sWC
, pResultSet
) ){
6150 /* The DISTINCT marking is pointless. Ignore it. */
6151 pWInfo
->eDistinct
= WHERE_DISTINCT_UNIQUE
;
6152 }else if( pOrderBy
==0 ){
6153 /* Try to ORDER BY the result set to make distinct processing easier */
6154 pWInfo
->wctrlFlags
|= WHERE_DISTINCTBY
;
6155 pWInfo
->pOrderBy
= pResultSet
;
6159 /* Construct the WhereLoop objects */
6160 #if defined(WHERETRACE_ENABLED)
6161 if( sqlite3WhereTrace
& 0xffffffff ){
6162 sqlite3DebugPrintf("*** Optimizer Start *** (wctrlFlags: 0x%x",wctrlFlags
);
6163 if( wctrlFlags
& WHERE_USE_LIMIT
){
6164 sqlite3DebugPrintf(", limit: %d", iAuxArg
);
6166 sqlite3DebugPrintf(")\n");
6167 if( sqlite3WhereTrace
& 0x8000 ){
6169 memset(&sSelect
, 0, sizeof(sSelect
));
6170 sSelect
.selFlags
= SF_WhereBegin
;
6171 sSelect
.pSrc
= pTabList
;
6172 sSelect
.pWhere
= pWhere
;
6173 sSelect
.pOrderBy
= pOrderBy
;
6174 sSelect
.pEList
= pResultSet
;
6175 sqlite3TreeViewSelect(0, &sSelect
, 0);
6177 if( sqlite3WhereTrace
& 0x4000 ){ /* Display all WHERE clause terms */
6178 sqlite3DebugPrintf("---- WHERE clause at start of analysis:\n");
6179 sqlite3WhereClausePrint(sWLB
.pWC
);
6184 if( nTabList
!=1 || whereShortCut(&sWLB
)==0 ){
6185 rc
= whereLoopAddAll(&sWLB
);
6186 if( rc
) goto whereBeginError
;
6188 #ifdef SQLITE_ENABLE_STAT4
6189 /* If one or more WhereTerm.truthProb values were used in estimating
6190 ** loop parameters, but then those truthProb values were subsequently
6191 ** changed based on STAT4 information while computing subsequent loops,
6192 ** then we need to rerun the whole loop building process so that all
6193 ** loops will be built using the revised truthProb values. */
6194 if( sWLB
.bldFlags2
& SQLITE_BLDF2_2NDPASS
){
6195 WHERETRACE_ALL_LOOPS(pWInfo
, sWLB
.pWC
);
6196 WHERETRACE(0xffffffff,
6197 ("**** Redo all loop computations due to"
6198 " TERM_HIGHTRUTH changes ****\n"));
6199 while( pWInfo
->pLoops
){
6200 WhereLoop
*p
= pWInfo
->pLoops
;
6201 pWInfo
->pLoops
= p
->pNextLoop
;
6202 whereLoopDelete(db
, p
);
6204 rc
= whereLoopAddAll(&sWLB
);
6205 if( rc
) goto whereBeginError
;
6208 WHERETRACE_ALL_LOOPS(pWInfo
, sWLB
.pWC
);
6210 wherePathSolver(pWInfo
, 0);
6211 if( db
->mallocFailed
) goto whereBeginError
;
6212 if( pWInfo
->pOrderBy
){
6213 wherePathSolver(pWInfo
, pWInfo
->nRowOut
+1);
6214 if( db
->mallocFailed
) goto whereBeginError
;
6217 /* TUNING: Assume that a DISTINCT clause on a subquery reduces
6218 ** the output size by a factor of 8 (LogEst -30).
6220 if( (pWInfo
->wctrlFlags
& WHERE_WANT_DISTINCT
)!=0 ){
6221 WHERETRACE(0x0080,("nRowOut reduced from %d to %d due to DISTINCT\n",
6222 pWInfo
->nRowOut
, pWInfo
->nRowOut
-30));
6223 pWInfo
->nRowOut
-= 30;
6227 assert( pWInfo
->pTabList
!=0 );
6228 if( pWInfo
->pOrderBy
==0 && (db
->flags
& SQLITE_ReverseOrder
)!=0 ){
6229 whereReverseScanOrder(pWInfo
);
6232 goto whereBeginError
;
6234 assert( db
->mallocFailed
==0 );
6235 #ifdef WHERETRACE_ENABLED
6236 if( sqlite3WhereTrace
){
6237 sqlite3DebugPrintf("---- Solution nRow=%d", pWInfo
->nRowOut
);
6238 if( pWInfo
->nOBSat
>0 ){
6239 sqlite3DebugPrintf(" ORDERBY=%d,0x%llx", pWInfo
->nOBSat
, pWInfo
->revMask
);
6241 switch( pWInfo
->eDistinct
){
6242 case WHERE_DISTINCT_UNIQUE
: {
6243 sqlite3DebugPrintf(" DISTINCT=unique");
6246 case WHERE_DISTINCT_ORDERED
: {
6247 sqlite3DebugPrintf(" DISTINCT=ordered");
6250 case WHERE_DISTINCT_UNORDERED
: {
6251 sqlite3DebugPrintf(" DISTINCT=unordered");
6255 sqlite3DebugPrintf("\n");
6256 for(ii
=0; ii
<pWInfo
->nLevel
; ii
++){
6257 sqlite3WhereLoopPrint(pWInfo
->a
[ii
].pWLoop
, sWLB
.pWC
);
6262 /* Attempt to omit tables from a join that do not affect the result.
6263 ** See the comment on whereOmitNoopJoin() for further information.
6265 ** This query optimization is factored out into a separate "no-inline"
6266 ** procedure to keep the sqlite3WhereBegin() procedure from becoming
6267 ** too large. If sqlite3WhereBegin() becomes too large, that prevents
6268 ** some C-compiler optimizers from in-lining the
6269 ** sqlite3WhereCodeOneLoopStart() procedure, and it is important to
6270 ** in-line sqlite3WhereCodeOneLoopStart() for performance reasons.
6272 notReady
= ~(Bitmask
)0;
6273 if( pWInfo
->nLevel
>=2
6274 && pResultSet
!=0 /* these two combine to guarantee */
6275 && 0==(wctrlFlags
& WHERE_AGG_DISTINCT
) /* condition (1) above */
6276 && OptimizationEnabled(db
, SQLITE_OmitNoopJoin
)
6278 notReady
= whereOmitNoopJoin(pWInfo
, notReady
);
6279 nTabList
= pWInfo
->nLevel
;
6280 assert( nTabList
>0 );
6283 /* Check to see if there are any SEARCH loops that might benefit from
6284 ** using a Bloom filter.
6286 if( pWInfo
->nLevel
>=2
6287 && OptimizationEnabled(db
, SQLITE_BloomFilter
)
6289 whereCheckIfBloomFilterIsUseful(pWInfo
);
6292 #if defined(WHERETRACE_ENABLED)
6293 if( sqlite3WhereTrace
& 0x4000 ){ /* Display all terms of the WHERE clause */
6294 sqlite3DebugPrintf("---- WHERE clause at end of analysis:\n");
6295 sqlite3WhereClausePrint(sWLB
.pWC
);
6297 WHERETRACE(0xffffffff,("*** Optimizer Finished ***\n"));
6299 pWInfo
->pParse
->nQueryLoop
+= pWInfo
->nRowOut
;
6301 /* If the caller is an UPDATE or DELETE statement that is requesting
6302 ** to use a one-pass algorithm, determine if this is appropriate.
6304 ** A one-pass approach can be used if the caller has requested one
6305 ** and either (a) the scan visits at most one row or (b) each
6306 ** of the following are true:
6308 ** * the caller has indicated that a one-pass approach can be used
6309 ** with multiple rows (by setting WHERE_ONEPASS_MULTIROW), and
6310 ** * the table is not a virtual table, and
6311 ** * either the scan does not use the OR optimization or the caller
6312 ** is a DELETE operation (WHERE_DUPLICATES_OK is only specified
6315 ** The last qualification is because an UPDATE statement uses
6316 ** WhereInfo.aiCurOnePass[1] to determine whether or not it really can
6317 ** use a one-pass approach, and this is not set accurately for scans
6318 ** that use the OR optimization.
6320 assert( (wctrlFlags
& WHERE_ONEPASS_DESIRED
)==0 || pWInfo
->nLevel
==1 );
6321 if( (wctrlFlags
& WHERE_ONEPASS_DESIRED
)!=0 ){
6322 int wsFlags
= pWInfo
->a
[0].pWLoop
->wsFlags
;
6323 int bOnerow
= (wsFlags
& WHERE_ONEROW
)!=0;
6324 assert( !(wsFlags
& WHERE_VIRTUALTABLE
) || IsVirtual(pTabList
->a
[0].pTab
) );
6326 0!=(wctrlFlags
& WHERE_ONEPASS_MULTIROW
)
6327 && !IsVirtual(pTabList
->a
[0].pTab
)
6328 && (0==(wsFlags
& WHERE_MULTI_OR
) || (wctrlFlags
& WHERE_DUPLICATES_OK
))
6329 && OptimizationEnabled(db
, SQLITE_OnePass
)
6331 pWInfo
->eOnePass
= bOnerow
? ONEPASS_SINGLE
: ONEPASS_MULTI
;
6332 if( HasRowid(pTabList
->a
[0].pTab
) && (wsFlags
& WHERE_IDX_ONLY
) ){
6333 if( wctrlFlags
& WHERE_ONEPASS_MULTIROW
){
6334 bFordelete
= OPFLAG_FORDELETE
;
6336 pWInfo
->a
[0].pWLoop
->wsFlags
= (wsFlags
& ~WHERE_IDX_ONLY
);
6341 /* Open all tables in the pTabList and any indices selected for
6342 ** searching those tables.
6344 for(ii
=0, pLevel
=pWInfo
->a
; ii
<nTabList
; ii
++, pLevel
++){
6345 Table
*pTab
; /* Table to open */
6346 int iDb
; /* Index of database containing table/index */
6349 pTabItem
= &pTabList
->a
[pLevel
->iFrom
];
6350 pTab
= pTabItem
->pTab
;
6351 iDb
= sqlite3SchemaToIndex(db
, pTab
->pSchema
);
6352 pLoop
= pLevel
->pWLoop
;
6353 if( (pTab
->tabFlags
& TF_Ephemeral
)!=0 || IsView(pTab
) ){
6356 #ifndef SQLITE_OMIT_VIRTUALTABLE
6357 if( (pLoop
->wsFlags
& WHERE_VIRTUALTABLE
)!=0 ){
6358 const char *pVTab
= (const char *)sqlite3GetVTable(db
, pTab
);
6359 int iCur
= pTabItem
->iCursor
;
6360 sqlite3VdbeAddOp4(v
, OP_VOpen
, iCur
, 0, 0, pVTab
, P4_VTAB
);
6361 }else if( IsVirtual(pTab
) ){
6365 if( ((pLoop
->wsFlags
& WHERE_IDX_ONLY
)==0
6366 && (wctrlFlags
& WHERE_OR_SUBCLAUSE
)==0)
6367 || (pTabItem
->fg
.jointype
& (JT_LTORJ
|JT_RIGHT
))!=0
6369 int op
= OP_OpenRead
;
6370 if( pWInfo
->eOnePass
!=ONEPASS_OFF
){
6372 pWInfo
->aiCurOnePass
[0] = pTabItem
->iCursor
;
6374 sqlite3OpenTable(pParse
, pTabItem
->iCursor
, iDb
, pTab
, op
);
6375 assert( pTabItem
->iCursor
==pLevel
->iTabCur
);
6376 testcase( pWInfo
->eOnePass
==ONEPASS_OFF
&& pTab
->nCol
==BMS
-1 );
6377 testcase( pWInfo
->eOnePass
==ONEPASS_OFF
&& pTab
->nCol
==BMS
);
6378 if( pWInfo
->eOnePass
==ONEPASS_OFF
6380 && (pTab
->tabFlags
& (TF_HasGenerated
|TF_WithoutRowid
))==0
6381 && (pLoop
->wsFlags
& (WHERE_AUTO_INDEX
|WHERE_BLOOMFILTER
))==0
6383 /* If we know that only a prefix of the record will be used,
6384 ** it is advantageous to reduce the "column count" field in
6385 ** the P4 operand of the OP_OpenRead/Write opcode. */
6386 Bitmask b
= pTabItem
->colUsed
;
6388 for(; b
; b
=b
>>1, n
++){}
6389 sqlite3VdbeChangeP4(v
, -1, SQLITE_INT_TO_PTR(n
), P4_INT32
);
6390 assert( n
<=pTab
->nCol
);
6392 #ifdef SQLITE_ENABLE_CURSOR_HINTS
6393 if( pLoop
->u
.btree
.pIndex
!=0 && (pTab
->tabFlags
& TF_WithoutRowid
)==0 ){
6394 sqlite3VdbeChangeP5(v
, OPFLAG_SEEKEQ
|bFordelete
);
6398 sqlite3VdbeChangeP5(v
, bFordelete
);
6400 #ifdef SQLITE_ENABLE_COLUMN_USED_MASK
6401 sqlite3VdbeAddOp4Dup8(v
, OP_ColumnsUsed
, pTabItem
->iCursor
, 0, 0,
6402 (const u8
*)&pTabItem
->colUsed
, P4_INT64
);
6405 sqlite3TableLock(pParse
, iDb
, pTab
->tnum
, 0, pTab
->zName
);
6407 if( pLoop
->wsFlags
& WHERE_INDEXED
){
6408 Index
*pIx
= pLoop
->u
.btree
.pIndex
;
6410 int op
= OP_OpenRead
;
6411 /* iAuxArg is always set to a positive value if ONEPASS is possible */
6412 assert( iAuxArg
!=0 || (pWInfo
->wctrlFlags
& WHERE_ONEPASS_DESIRED
)==0 );
6413 if( !HasRowid(pTab
) && IsPrimaryKeyIndex(pIx
)
6414 && (wctrlFlags
& WHERE_OR_SUBCLAUSE
)!=0
6416 /* This is one term of an OR-optimization using the PRIMARY KEY of a
6417 ** WITHOUT ROWID table. No need for a separate index */
6418 iIndexCur
= pLevel
->iTabCur
;
6420 }else if( pWInfo
->eOnePass
!=ONEPASS_OFF
){
6421 Index
*pJ
= pTabItem
->pTab
->pIndex
;
6422 iIndexCur
= iAuxArg
;
6423 assert( wctrlFlags
& WHERE_ONEPASS_DESIRED
);
6424 while( ALWAYS(pJ
) && pJ
!=pIx
){
6429 pWInfo
->aiCurOnePass
[1] = iIndexCur
;
6430 }else if( iAuxArg
&& (wctrlFlags
& WHERE_OR_SUBCLAUSE
)!=0 ){
6431 iIndexCur
= iAuxArg
;
6434 iIndexCur
= pParse
->nTab
++;
6435 if( pIx
->bHasExpr
&& OptimizationEnabled(db
, SQLITE_IndexedExpr
) ){
6436 whereAddIndexedExpr(pParse
, pIx
, iIndexCur
, pTabItem
);
6438 if( pIx
->pPartIdxWhere
&& (pTabItem
->fg
.jointype
& JT_RIGHT
)==0 ){
6439 sqlite3WherePartIdxExpr(
6440 pParse
, pIx
, pIx
->pPartIdxWhere
, iIndexCur
, pTabItem
6444 pLevel
->iIdxCur
= iIndexCur
;
6446 assert( pIx
->pSchema
==pTab
->pSchema
);
6447 assert( iIndexCur
>=0 );
6449 sqlite3VdbeAddOp3(v
, op
, iIndexCur
, pIx
->tnum
, iDb
);
6450 sqlite3VdbeSetP4KeyInfo(pParse
, pIx
);
6451 if( (pLoop
->wsFlags
& WHERE_CONSTRAINT
)!=0
6452 && (pLoop
->wsFlags
& (WHERE_COLUMN_RANGE
|WHERE_SKIPSCAN
))==0
6453 && (pLoop
->wsFlags
& WHERE_BIGNULL_SORT
)==0
6454 && (pLoop
->wsFlags
& WHERE_IN_SEEKSCAN
)==0
6455 && (pWInfo
->wctrlFlags
&WHERE_ORDERBY_MIN
)==0
6456 && pWInfo
->eDistinct
!=WHERE_DISTINCT_ORDERED
6458 sqlite3VdbeChangeP5(v
, OPFLAG_SEEKEQ
);
6460 VdbeComment((v
, "%s", pIx
->zName
));
6461 #ifdef SQLITE_ENABLE_COLUMN_USED_MASK
6465 for(ii
=0; ii
<pIx
->nColumn
; ii
++){
6466 jj
= pIx
->aiColumn
[ii
];
6467 if( jj
<0 ) continue;
6468 if( jj
>63 ) jj
= 63;
6469 if( (pTabItem
->colUsed
& MASKBIT(jj
))==0 ) continue;
6470 colUsed
|= ((u64
)1)<<(ii
<63 ? ii
: 63);
6472 sqlite3VdbeAddOp4Dup8(v
, OP_ColumnsUsed
, iIndexCur
, 0, 0,
6473 (u8
*)&colUsed
, P4_INT64
);
6475 #endif /* SQLITE_ENABLE_COLUMN_USED_MASK */
6478 if( iDb
>=0 ) sqlite3CodeVerifySchema(pParse
, iDb
);
6479 if( (pTabItem
->fg
.jointype
& JT_RIGHT
)!=0
6480 && (pLevel
->pRJ
= sqlite3WhereMalloc(pWInfo
, sizeof(WhereRightJoin
)))!=0
6482 WhereRightJoin
*pRJ
= pLevel
->pRJ
;
6483 pRJ
->iMatch
= pParse
->nTab
++;
6484 pRJ
->regBloom
= ++pParse
->nMem
;
6485 sqlite3VdbeAddOp2(v
, OP_Blob
, 65536, pRJ
->regBloom
);
6486 pRJ
->regReturn
= ++pParse
->nMem
;
6487 sqlite3VdbeAddOp2(v
, OP_Null
, 0, pRJ
->regReturn
);
6488 assert( pTab
==pTabItem
->pTab
);
6489 if( HasRowid(pTab
) ){
6491 sqlite3VdbeAddOp2(v
, OP_OpenEphemeral
, pRJ
->iMatch
, 1);
6492 pInfo
= sqlite3KeyInfoAlloc(pParse
->db
, 1, 0);
6494 pInfo
->aColl
[0] = 0;
6495 pInfo
->aSortFlags
[0] = 0;
6496 sqlite3VdbeAppendP4(v
, pInfo
, P4_KEYINFO
);
6499 Index
*pPk
= sqlite3PrimaryKeyIndex(pTab
);
6500 sqlite3VdbeAddOp2(v
, OP_OpenEphemeral
, pRJ
->iMatch
, pPk
->nKeyCol
);
6501 sqlite3VdbeSetP4KeyInfo(pParse
, pPk
);
6503 pLoop
->wsFlags
&= ~WHERE_IDX_ONLY
;
6504 /* The nature of RIGHT JOIN processing is such that it messes up
6505 ** the output order. So omit any ORDER BY/GROUP BY elimination
6506 ** optimizations. We need to do an actual sort for RIGHT JOIN. */
6508 pWInfo
->eDistinct
= WHERE_DISTINCT_UNORDERED
;
6511 pWInfo
->iTop
= sqlite3VdbeCurrentAddr(v
);
6512 if( db
->mallocFailed
) goto whereBeginError
;
6514 /* Generate the code to do the search. Each iteration of the for
6515 ** loop below generates code for a single nested loop of the VM
6518 for(ii
=0; ii
<nTabList
; ii
++){
6522 if( pParse
->nErr
) goto whereBeginError
;
6523 pLevel
= &pWInfo
->a
[ii
];
6524 wsFlags
= pLevel
->pWLoop
->wsFlags
;
6525 pSrc
= &pTabList
->a
[pLevel
->iFrom
];
6526 if( pSrc
->fg
.isMaterialized
){
6527 if( pSrc
->fg
.isCorrelated
){
6528 sqlite3VdbeAddOp2(v
, OP_Gosub
, pSrc
->regReturn
, pSrc
->addrFillSub
);
6530 int iOnce
= sqlite3VdbeAddOp0(v
, OP_Once
); VdbeCoverage(v
);
6531 sqlite3VdbeAddOp2(v
, OP_Gosub
, pSrc
->regReturn
, pSrc
->addrFillSub
);
6532 sqlite3VdbeJumpHere(v
, iOnce
);
6535 assert( pTabList
== pWInfo
->pTabList
);
6536 if( (wsFlags
& (WHERE_AUTO_INDEX
|WHERE_BLOOMFILTER
))!=0 ){
6537 if( (wsFlags
& WHERE_AUTO_INDEX
)!=0 ){
6538 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
6539 constructAutomaticIndex(pParse
, &pWInfo
->sWC
, notReady
, pLevel
);
6542 sqlite3ConstructBloomFilter(pWInfo
, ii
, pLevel
, notReady
);
6544 if( db
->mallocFailed
) goto whereBeginError
;
6546 addrExplain
= sqlite3WhereExplainOneScan(
6547 pParse
, pTabList
, pLevel
, wctrlFlags
6549 pLevel
->addrBody
= sqlite3VdbeCurrentAddr(v
);
6550 notReady
= sqlite3WhereCodeOneLoopStart(pParse
,v
,pWInfo
,ii
,pLevel
,notReady
);
6551 pWInfo
->iContinue
= pLevel
->addrCont
;
6552 if( (wsFlags
&WHERE_MULTI_OR
)==0 && (wctrlFlags
&WHERE_OR_SUBCLAUSE
)==0 ){
6553 sqlite3WhereAddScanStatus(v
, pTabList
, pLevel
, addrExplain
);
6558 VdbeModuleComment((v
, "Begin WHERE-core"));
6559 pWInfo
->iEndWhere
= sqlite3VdbeCurrentAddr(v
);
6562 /* Jump here if malloc fails */
6565 pParse
->nQueryLoop
= pWInfo
->savedNQueryLoop
;
6566 whereInfoFree(db
, pWInfo
);
6572 ** Part of sqlite3WhereEnd() will rewrite opcodes to reference the
6573 ** index rather than the main table. In SQLITE_DEBUG mode, we want
6574 ** to trace those changes if PRAGMA vdbe_addoptrace=on. This routine
6577 #ifndef SQLITE_DEBUG
6578 # define OpcodeRewriteTrace(D,K,P) /* no-op */
6580 # define OpcodeRewriteTrace(D,K,P) sqlite3WhereOpcodeRewriteTrace(D,K,P)
6581 static void sqlite3WhereOpcodeRewriteTrace(
6586 if( (db
->flags
& SQLITE_VdbeAddopTrace
)==0 ) return;
6587 sqlite3VdbePrintOp(0, pc
, pOp
);
6593 ** Return true if cursor iCur is opened by instruction k of the
6594 ** bytecode. Used inside of assert() only.
6596 static int cursorIsOpen(Vdbe
*v
, int iCur
, int k
){
6598 VdbeOp
*pOp
= sqlite3VdbeGetOp(v
,k
--);
6599 if( pOp
->p1
!=iCur
) continue;
6600 if( pOp
->opcode
==OP_Close
) return 0;
6601 if( pOp
->opcode
==OP_OpenRead
) return 1;
6602 if( pOp
->opcode
==OP_OpenWrite
) return 1;
6603 if( pOp
->opcode
==OP_OpenDup
) return 1;
6604 if( pOp
->opcode
==OP_OpenAutoindex
) return 1;
6605 if( pOp
->opcode
==OP_OpenEphemeral
) return 1;
6609 #endif /* SQLITE_DEBUG */
6612 ** Generate the end of the WHERE loop. See comments on
6613 ** sqlite3WhereBegin() for additional information.
6615 void sqlite3WhereEnd(WhereInfo
*pWInfo
){
6616 Parse
*pParse
= pWInfo
->pParse
;
6617 Vdbe
*v
= pParse
->pVdbe
;
6621 SrcList
*pTabList
= pWInfo
->pTabList
;
6622 sqlite3
*db
= pParse
->db
;
6623 int iEnd
= sqlite3VdbeCurrentAddr(v
);
6626 /* Generate loop termination code.
6628 VdbeModuleComment((v
, "End WHERE-core"));
6629 for(i
=pWInfo
->nLevel
-1; i
>=0; i
--){
6631 pLevel
= &pWInfo
->a
[i
];
6633 /* Terminate the subroutine that forms the interior of the loop of
6634 ** the RIGHT JOIN table */
6635 WhereRightJoin
*pRJ
= pLevel
->pRJ
;
6636 sqlite3VdbeResolveLabel(v
, pLevel
->addrCont
);
6637 pLevel
->addrCont
= 0;
6638 pRJ
->endSubrtn
= sqlite3VdbeCurrentAddr(v
);
6639 sqlite3VdbeAddOp3(v
, OP_Return
, pRJ
->regReturn
, pRJ
->addrSubrtn
, 1);
6643 pLoop
= pLevel
->pWLoop
;
6644 if( pLevel
->op
!=OP_Noop
){
6645 #ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT
6649 if( pWInfo
->eDistinct
==WHERE_DISTINCT_ORDERED
6650 && i
==pWInfo
->nLevel
-1 /* Ticket [ef9318757b152e3] 2017-10-21 */
6651 && (pLoop
->wsFlags
& WHERE_INDEXED
)!=0
6652 && (pIdx
= pLoop
->u
.btree
.pIndex
)->hasStat1
6653 && (n
= pLoop
->u
.btree
.nDistinctCol
)>0
6654 && pIdx
->aiRowLogEst
[n
]>=36
6656 int r1
= pParse
->nMem
+1;
6659 sqlite3VdbeAddOp3(v
, OP_Column
, pLevel
->iIdxCur
, j
, r1
+j
);
6661 pParse
->nMem
+= n
+1;
6662 op
= pLevel
->op
==OP_Prev
? OP_SeekLT
: OP_SeekGT
;
6663 addrSeek
= sqlite3VdbeAddOp4Int(v
, op
, pLevel
->iIdxCur
, 0, r1
, n
);
6664 VdbeCoverageIf(v
, op
==OP_SeekLT
);
6665 VdbeCoverageIf(v
, op
==OP_SeekGT
);
6666 sqlite3VdbeAddOp2(v
, OP_Goto
, 1, pLevel
->p2
);
6668 #endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */
6669 /* The common case: Advance to the next row */
6670 if( pLevel
->addrCont
) sqlite3VdbeResolveLabel(v
, pLevel
->addrCont
);
6671 sqlite3VdbeAddOp3(v
, pLevel
->op
, pLevel
->p1
, pLevel
->p2
, pLevel
->p3
);
6672 sqlite3VdbeChangeP5(v
, pLevel
->p5
);
6674 VdbeCoverageIf(v
, pLevel
->op
==OP_Next
);
6675 VdbeCoverageIf(v
, pLevel
->op
==OP_Prev
);
6676 VdbeCoverageIf(v
, pLevel
->op
==OP_VNext
);
6677 if( pLevel
->regBignull
){
6678 sqlite3VdbeResolveLabel(v
, pLevel
->addrBignull
);
6679 sqlite3VdbeAddOp2(v
, OP_DecrJumpZero
, pLevel
->regBignull
, pLevel
->p2
-1);
6682 #ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT
6683 if( addrSeek
) sqlite3VdbeJumpHere(v
, addrSeek
);
6685 }else if( pLevel
->addrCont
){
6686 sqlite3VdbeResolveLabel(v
, pLevel
->addrCont
);
6688 if( (pLoop
->wsFlags
& WHERE_IN_ABLE
)!=0 && pLevel
->u
.in
.nIn
>0 ){
6691 sqlite3VdbeResolveLabel(v
, pLevel
->addrNxt
);
6692 for(j
=pLevel
->u
.in
.nIn
, pIn
=&pLevel
->u
.in
.aInLoop
[j
-1]; j
>0; j
--, pIn
--){
6693 assert( sqlite3VdbeGetOp(v
, pIn
->addrInTop
+1)->opcode
==OP_IsNull
6694 || pParse
->db
->mallocFailed
);
6695 sqlite3VdbeJumpHere(v
, pIn
->addrInTop
+1);
6696 if( pIn
->eEndLoopOp
!=OP_Noop
){
6699 (pLoop
->wsFlags
& WHERE_VIRTUALTABLE
)==0
6700 && (pLoop
->wsFlags
& WHERE_IN_EARLYOUT
)!=0;
6701 if( pLevel
->iLeftJoin
){
6702 /* For LEFT JOIN queries, cursor pIn->iCur may not have been
6703 ** opened yet. This occurs for WHERE clauses such as
6704 ** "a = ? AND b IN (...)", where the index is on (a, b). If
6705 ** the RHS of the (a=?) is NULL, then the "b IN (...)" may
6706 ** never have been coded, but the body of the loop run to
6707 ** return the null-row. So, if the cursor is not open yet,
6708 ** jump over the OP_Next or OP_Prev instruction about to
6710 sqlite3VdbeAddOp2(v
, OP_IfNotOpen
, pIn
->iCur
,
6711 sqlite3VdbeCurrentAddr(v
) + 2 + bEarlyOut
);
6715 sqlite3VdbeAddOp4Int(v
, OP_IfNoHope
, pLevel
->iIdxCur
,
6716 sqlite3VdbeCurrentAddr(v
)+2,
6717 pIn
->iBase
, pIn
->nPrefix
);
6719 /* Retarget the OP_IsNull against the left operand of IN so
6720 ** it jumps past the OP_IfNoHope. This is because the
6721 ** OP_IsNull also bypasses the OP_Affinity opcode that is
6722 ** required by OP_IfNoHope. */
6723 sqlite3VdbeJumpHere(v
, pIn
->addrInTop
+1);
6726 sqlite3VdbeAddOp2(v
, pIn
->eEndLoopOp
, pIn
->iCur
, pIn
->addrInTop
);
6728 VdbeCoverageIf(v
, pIn
->eEndLoopOp
==OP_Prev
);
6729 VdbeCoverageIf(v
, pIn
->eEndLoopOp
==OP_Next
);
6731 sqlite3VdbeJumpHere(v
, pIn
->addrInTop
-1);
6734 sqlite3VdbeResolveLabel(v
, pLevel
->addrBrk
);
6736 sqlite3VdbeAddOp3(v
, OP_Return
, pLevel
->pRJ
->regReturn
, 0, 1);
6739 if( pLevel
->addrSkip
){
6740 sqlite3VdbeGoto(v
, pLevel
->addrSkip
);
6741 VdbeComment((v
, "next skip-scan on %s", pLoop
->u
.btree
.pIndex
->zName
));
6742 sqlite3VdbeJumpHere(v
, pLevel
->addrSkip
);
6743 sqlite3VdbeJumpHere(v
, pLevel
->addrSkip
-2);
6745 #ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS
6746 if( pLevel
->addrLikeRep
){
6747 sqlite3VdbeAddOp2(v
, OP_DecrJumpZero
, (int)(pLevel
->iLikeRepCntr
>>1),
6748 pLevel
->addrLikeRep
);
6752 if( pLevel
->iLeftJoin
){
6753 int ws
= pLoop
->wsFlags
;
6754 addr
= sqlite3VdbeAddOp1(v
, OP_IfPos
, pLevel
->iLeftJoin
); VdbeCoverage(v
);
6755 assert( (ws
& WHERE_IDX_ONLY
)==0 || (ws
& WHERE_INDEXED
)!=0 );
6756 if( (ws
& WHERE_IDX_ONLY
)==0 ){
6757 assert( pLevel
->iTabCur
==pTabList
->a
[pLevel
->iFrom
].iCursor
);
6758 sqlite3VdbeAddOp1(v
, OP_NullRow
, pLevel
->iTabCur
);
6760 if( (ws
& WHERE_INDEXED
)
6761 || ((ws
& WHERE_MULTI_OR
) && pLevel
->u
.pCoveringIdx
)
6763 if( ws
& WHERE_MULTI_OR
){
6764 Index
*pIx
= pLevel
->u
.pCoveringIdx
;
6765 int iDb
= sqlite3SchemaToIndex(db
, pIx
->pSchema
);
6766 sqlite3VdbeAddOp3(v
, OP_ReopenIdx
, pLevel
->iIdxCur
, pIx
->tnum
, iDb
);
6767 sqlite3VdbeSetP4KeyInfo(pParse
, pIx
);
6769 sqlite3VdbeAddOp1(v
, OP_NullRow
, pLevel
->iIdxCur
);
6771 if( pLevel
->op
==OP_Return
){
6772 sqlite3VdbeAddOp2(v
, OP_Gosub
, pLevel
->p1
, pLevel
->addrFirst
);
6774 sqlite3VdbeGoto(v
, pLevel
->addrFirst
);
6776 sqlite3VdbeJumpHere(v
, addr
);
6778 VdbeModuleComment((v
, "End WHERE-loop%d: %s", i
,
6779 pWInfo
->pTabList
->a
[pLevel
->iFrom
].pTab
->zName
));
6782 assert( pWInfo
->nLevel
<=pTabList
->nSrc
);
6783 for(i
=0, pLevel
=pWInfo
->a
; i
<pWInfo
->nLevel
; i
++, pLevel
++){
6785 VdbeOp
*pOp
, *pLastOp
;
6787 SrcItem
*pTabItem
= &pTabList
->a
[pLevel
->iFrom
];
6788 Table
*pTab
= pTabItem
->pTab
;
6790 pLoop
= pLevel
->pWLoop
;
6792 /* Do RIGHT JOIN processing. Generate code that will output the
6793 ** unmatched rows of the right operand of the RIGHT JOIN with
6794 ** all of the columns of the left operand set to NULL.
6797 sqlite3WhereRightJoinLoop(pWInfo
, i
, pLevel
);
6801 /* For a co-routine, change all OP_Column references to the table of
6802 ** the co-routine into OP_Copy of result contained in a register.
6803 ** OP_Rowid becomes OP_Null.
6805 if( pTabItem
->fg
.viaCoroutine
){
6806 testcase( pParse
->db
->mallocFailed
);
6807 translateColumnToCopy(pParse
, pLevel
->addrBody
, pLevel
->iTabCur
,
6808 pTabItem
->regResult
, 0);
6812 /* If this scan uses an index, make VDBE code substitutions to read data
6813 ** from the index instead of from the table where possible. In some cases
6814 ** this optimization prevents the table from ever being read, which can
6815 ** yield a significant performance boost.
6817 ** Calls to the code generator in between sqlite3WhereBegin and
6818 ** sqlite3WhereEnd will have created code that references the table
6819 ** directly. This loop scans all that code looking for opcodes
6820 ** that reference the table and converts them into opcodes that
6821 ** reference the index.
6823 if( pLoop
->wsFlags
& (WHERE_INDEXED
|WHERE_IDX_ONLY
) ){
6824 pIdx
= pLoop
->u
.btree
.pIndex
;
6825 }else if( pLoop
->wsFlags
& WHERE_MULTI_OR
){
6826 pIdx
= pLevel
->u
.pCoveringIdx
;
6829 && !db
->mallocFailed
6831 if( pWInfo
->eOnePass
==ONEPASS_OFF
|| !HasRowid(pIdx
->pTable
) ){
6834 last
= pWInfo
->iEndWhere
;
6836 if( pIdx
->bHasExpr
){
6837 IndexedExpr
*p
= pParse
->pIdxEpr
;
6839 if( p
->iIdxCur
==pLevel
->iIdxCur
){
6840 #ifdef WHERETRACE_ENABLED
6841 if( sqlite3WhereTrace
& 0x200 ){
6842 sqlite3DebugPrintf("Disable pParse->pIdxEpr term {%d,%d}\n",
6843 p
->iIdxCur
, p
->iIdxCol
);
6844 if( sqlite3WhereTrace
& 0x5000 ) sqlite3ShowExpr(p
->pExpr
);
6853 k
= pLevel
->addrBody
+ 1;
6855 if( db
->flags
& SQLITE_VdbeAddopTrace
){
6856 printf("TRANSLATE cursor %d->%d in opcode range %d..%d\n",
6857 pLevel
->iTabCur
, pLevel
->iIdxCur
, k
, last
-1);
6859 /* Proof that the "+1" on the k value above is safe */
6860 pOp
= sqlite3VdbeGetOp(v
, k
- 1);
6861 assert( pOp
->opcode
!=OP_Column
|| pOp
->p1
!=pLevel
->iTabCur
);
6862 assert( pOp
->opcode
!=OP_Rowid
|| pOp
->p1
!=pLevel
->iTabCur
);
6863 assert( pOp
->opcode
!=OP_IfNullRow
|| pOp
->p1
!=pLevel
->iTabCur
);
6865 pOp
= sqlite3VdbeGetOp(v
, k
);
6866 pLastOp
= pOp
+ (last
- k
);
6867 assert( pOp
<=pLastOp
);
6869 if( pOp
->p1
!=pLevel
->iTabCur
){
6871 }else if( pOp
->opcode
==OP_Column
6872 #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
6873 || pOp
->opcode
==OP_Offset
6877 assert( pIdx
->pTable
==pTab
);
6878 #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
6879 if( pOp
->opcode
==OP_Offset
){
6880 /* Do not need to translate the column number */
6883 if( !HasRowid(pTab
) ){
6884 Index
*pPk
= sqlite3PrimaryKeyIndex(pTab
);
6885 x
= pPk
->aiColumn
[x
];
6888 testcase( x
!=sqlite3StorageColumnToTable(pTab
,x
) );
6889 x
= sqlite3StorageColumnToTable(pTab
,x
);
6891 x
= sqlite3TableColumnToIndex(pIdx
, x
);
6894 pOp
->p1
= pLevel
->iIdxCur
;
6895 OpcodeRewriteTrace(db
, k
, pOp
);
6897 /* Unable to translate the table reference into an index
6898 ** reference. Verify that this is harmless - that the
6899 ** table being referenced really is open.
6901 #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
6902 assert( (pLoop
->wsFlags
& WHERE_IDX_ONLY
)==0
6903 || cursorIsOpen(v
,pOp
->p1
,k
)
6904 || pOp
->opcode
==OP_Offset
6907 assert( (pLoop
->wsFlags
& WHERE_IDX_ONLY
)==0
6908 || cursorIsOpen(v
,pOp
->p1
,k
)
6912 }else if( pOp
->opcode
==OP_Rowid
){
6913 pOp
->p1
= pLevel
->iIdxCur
;
6914 pOp
->opcode
= OP_IdxRowid
;
6915 OpcodeRewriteTrace(db
, k
, pOp
);
6916 }else if( pOp
->opcode
==OP_IfNullRow
){
6917 pOp
->p1
= pLevel
->iIdxCur
;
6918 OpcodeRewriteTrace(db
, k
, pOp
);
6923 }while( (++pOp
)<pLastOp
);
6925 if( db
->flags
& SQLITE_VdbeAddopTrace
) printf("TRANSLATE complete\n");
6930 /* The "break" point is here, just past the end of the outer loop.
6933 sqlite3VdbeResolveLabel(v
, pWInfo
->iBreak
);
6937 pParse
->nQueryLoop
= pWInfo
->savedNQueryLoop
;
6938 whereInfoFree(db
, pWInfo
);
6939 pParse
->withinRJSubrtn
-= nRJ
;