blob: 81cc1f131b295df30a419855a815964f41f3b711 [file] [log] [blame]
drh75897232000-05-29 14:26:00 +00001/*
drhb19a2bc2001-09-16 00:13:26 +00002** 2001 September 15
drh75897232000-05-29 14:26:00 +00003**
drhb19a2bc2001-09-16 00:13:26 +00004** The author disclaims copyright to this source code. In place of
5** a legal notice, here is a blessing:
drh75897232000-05-29 14:26:00 +00006**
drhb19a2bc2001-09-16 00:13:26 +00007** May you do good and not evil.
8** May you find forgiveness for yourself and forgive others.
9** May you share freely, never taking more than you give.
drh75897232000-05-29 14:26:00 +000010**
11*************************************************************************
12** This module contains C code that generates VDBE code used to process
drh909626d2008-05-30 14:58:37 +000013** the WHERE clause of SQL statements. This module is responsible for
drh51669862004-12-18 18:40:26 +000014** generating the code that loops through a table looking for applicable
15** rows. Indices are selected and used to speed the search when doing
16** so is applicable. Because this module is responsible for selecting
17** indices, you might also think of this module as the "query optimizer".
drh75897232000-05-29 14:26:00 +000018*/
19#include "sqliteInt.h"
drhe54df422013-11-12 18:37:25 +000020#include "whereInt.h"
drh51147ba2005-07-23 22:59:55 +000021
drh6f82e852015-06-06 20:12:09 +000022/* Forward declaration of methods */
23static int whereLoopResize(sqlite3*, WhereLoop*, int);
24
25/* Test variable that can be set to enable WHERE tracing */
26#if defined(SQLITE_TEST) || defined(SQLITE_DEBUG)
27/***/ int sqlite3WhereTrace = 0;
28#endif
29
30
drh51147ba2005-07-23 22:59:55 +000031/*
drh6f328482013-06-05 23:39:34 +000032** Return the estimated number of output rows from a WHERE clause
33*/
drhc3489bb2016-02-25 16:04:59 +000034LogEst sqlite3WhereOutputRowCount(WhereInfo *pWInfo){
35 return pWInfo->nRowOut;
drh6f328482013-06-05 23:39:34 +000036}
37
38/*
39** Return one of the WHERE_DISTINCT_xxxxx values to indicate how this
40** WHERE clause returns outputs for DISTINCT processing.
41*/
42int sqlite3WhereIsDistinct(WhereInfo *pWInfo){
43 return pWInfo->eDistinct;
44}
45
46/*
47** Return TRUE if the WHERE clause returns rows in ORDER BY order.
48** Return FALSE if the output needs to be sorted.
49*/
50int sqlite3WhereIsOrdered(WhereInfo *pWInfo){
drhddba0c22014-03-18 20:33:42 +000051 return pWInfo->nOBSat;
drh6f328482013-06-05 23:39:34 +000052}
53
54/*
drha536df42016-05-19 22:13:37 +000055** Return TRUE if the innermost loop of the WHERE clause implementation
56** returns rows in ORDER BY order for complete run of the inner loop.
57**
58** Across multiple iterations of outer loops, the output rows need not be
59** sorted. As long as rows are sorted for just the innermost loop, this
60** routine can return TRUE.
61*/
62int sqlite3WhereOrderedInnerLoop(WhereInfo *pWInfo){
63 return pWInfo->bOrderedInnerLoop;
64}
65
66/*
drh6f328482013-06-05 23:39:34 +000067** Return the VDBE address or label to jump to in order to continue
68** immediately with the next row of a WHERE clause.
69*/
70int sqlite3WhereContinueLabel(WhereInfo *pWInfo){
drha22a75e2014-03-21 18:16:23 +000071 assert( pWInfo->iContinue!=0 );
drh6f328482013-06-05 23:39:34 +000072 return pWInfo->iContinue;
73}
74
75/*
76** Return the VDBE address or label to jump to in order to break
77** out of a WHERE loop.
78*/
79int sqlite3WhereBreakLabel(WhereInfo *pWInfo){
80 return pWInfo->iBreak;
81}
82
83/*
drhb0264ee2015-09-14 14:45:50 +000084** Return ONEPASS_OFF (0) if an UPDATE or DELETE statement is unable to
85** operate directly on the rowis returned by a WHERE clause. Return
86** ONEPASS_SINGLE (1) if the statement can operation directly because only
87** a single row is to be changed. Return ONEPASS_MULTI (2) if the one-pass
88** optimization can be used on multiple
drhfc8d4f92013-11-08 15:19:46 +000089**
90** If the ONEPASS optimization is used (if this routine returns true)
91** then also write the indices of open cursors used by ONEPASS
92** into aiCur[0] and aiCur[1]. iaCur[0] gets the cursor of the data
93** table and iaCur[1] gets the cursor used by an auxiliary index.
94** Either value may be -1, indicating that cursor is not used.
95** Any cursors returned will have been opened for writing.
96**
97** aiCur[0] and aiCur[1] both get -1 if the where-clause logic is
98** unable to use the ONEPASS optimization.
drh6f328482013-06-05 23:39:34 +000099*/
drhfc8d4f92013-11-08 15:19:46 +0000100int sqlite3WhereOkOnePass(WhereInfo *pWInfo, int *aiCur){
101 memcpy(aiCur, pWInfo->aiCurOnePass, sizeof(int)*2);
drha7228212015-09-28 17:05:22 +0000102#ifdef WHERETRACE_ENABLED
103 if( sqlite3WhereTrace && pWInfo->eOnePass!=ONEPASS_OFF ){
104 sqlite3DebugPrintf("%s cursors: %d %d\n",
105 pWInfo->eOnePass==ONEPASS_SINGLE ? "ONEPASS_SINGLE" : "ONEPASS_MULTI",
106 aiCur[0], aiCur[1]);
107 }
108#endif
drhb0264ee2015-09-14 14:45:50 +0000109 return pWInfo->eOnePass;
drh6f328482013-06-05 23:39:34 +0000110}
111
112/*
drhaa32e3c2013-07-16 21:31:23 +0000113** Move the content of pSrc into pDest
114*/
115static void whereOrMove(WhereOrSet *pDest, WhereOrSet *pSrc){
116 pDest->n = pSrc->n;
117 memcpy(pDest->a, pSrc->a, pDest->n*sizeof(pDest->a[0]));
118}
119
120/*
121** Try to insert a new prerequisite/cost entry into the WhereOrSet pSet.
122**
123** The new entry might overwrite an existing entry, or it might be
124** appended, or it might be discarded. Do whatever is the right thing
125** so that pSet keeps the N_OR_COST best entries seen so far.
126*/
127static int whereOrInsert(
128 WhereOrSet *pSet, /* The WhereOrSet to be updated */
129 Bitmask prereq, /* Prerequisites of the new entry */
drhbf539c42013-10-05 18:16:02 +0000130 LogEst rRun, /* Run-cost of the new entry */
131 LogEst nOut /* Number of outputs for the new entry */
drhaa32e3c2013-07-16 21:31:23 +0000132){
133 u16 i;
134 WhereOrCost *p;
135 for(i=pSet->n, p=pSet->a; i>0; i--, p++){
136 if( rRun<=p->rRun && (prereq & p->prereq)==prereq ){
137 goto whereOrInsert_done;
138 }
139 if( p->rRun<=rRun && (p->prereq & prereq)==p->prereq ){
140 return 0;
141 }
142 }
143 if( pSet->n<N_OR_COST ){
144 p = &pSet->a[pSet->n++];
145 p->nOut = nOut;
146 }else{
147 p = pSet->a;
148 for(i=1; i<pSet->n; i++){
149 if( p->rRun>pSet->a[i].rRun ) p = pSet->a + i;
150 }
151 if( p->rRun<=rRun ) return 0;
152 }
153whereOrInsert_done:
154 p->prereq = prereq;
155 p->rRun = rRun;
156 if( p->nOut>nOut ) p->nOut = nOut;
157 return 1;
158}
159
160/*
drh1398ad32005-01-19 23:24:50 +0000161** Return the bitmask for the given cursor number. Return 0 if
162** iCursor is not in the set.
drh6a3ea0e2003-05-02 14:32:12 +0000163*/
drh6f82e852015-06-06 20:12:09 +0000164Bitmask sqlite3WhereGetMask(WhereMaskSet *pMaskSet, int iCursor){
drh6a3ea0e2003-05-02 14:32:12 +0000165 int i;
drhfcd71b62011-04-05 22:08:24 +0000166 assert( pMaskSet->n<=(int)sizeof(Bitmask)*8 );
drh6a3ea0e2003-05-02 14:32:12 +0000167 for(i=0; i<pMaskSet->n; i++){
drh51669862004-12-18 18:40:26 +0000168 if( pMaskSet->ix[i]==iCursor ){
drh7699d1c2013-06-04 12:42:29 +0000169 return MASKBIT(i);
drh51669862004-12-18 18:40:26 +0000170 }
drh6a3ea0e2003-05-02 14:32:12 +0000171 }
drh6a3ea0e2003-05-02 14:32:12 +0000172 return 0;
173}
174
175/*
drh1398ad32005-01-19 23:24:50 +0000176** Create a new mask for cursor iCursor.
drh0fcef5e2005-07-19 17:38:22 +0000177**
178** There is one cursor per table in the FROM clause. The number of
179** tables in the FROM clause is limited by a test early in the
drhb6fb62d2005-09-20 08:47:20 +0000180** sqlite3WhereBegin() routine. So we know that the pMaskSet->ix[]
drh0fcef5e2005-07-19 17:38:22 +0000181** array will never overflow.
drh1398ad32005-01-19 23:24:50 +0000182*/
drh111a6a72008-12-21 03:51:16 +0000183static void createMask(WhereMaskSet *pMaskSet, int iCursor){
drhcad651e2007-04-20 12:22:01 +0000184 assert( pMaskSet->n < ArraySize(pMaskSet->ix) );
drh0fcef5e2005-07-19 17:38:22 +0000185 pMaskSet->ix[pMaskSet->n++] = iCursor;
drh1398ad32005-01-19 23:24:50 +0000186}
187
188/*
drh1c8148f2013-05-04 20:25:23 +0000189** Advance to the next WhereTerm that matches according to the criteria
190** established when the pScan object was initialized by whereScanInit().
191** Return NULL if there are no more matching WhereTerms.
192*/
danb2cfc142013-07-05 11:10:54 +0000193static WhereTerm *whereScanNext(WhereScan *pScan){
drh1c8148f2013-05-04 20:25:23 +0000194 int iCur; /* The cursor on the LHS of the term */
drha3f108e2015-08-26 21:08:04 +0000195 i16 iColumn; /* The column on the LHS of the term. -1 for IPK */
drh1c8148f2013-05-04 20:25:23 +0000196 Expr *pX; /* An expression being tested */
197 WhereClause *pWC; /* Shorthand for pScan->pWC */
198 WhereTerm *pTerm; /* The term being tested */
drh43b85ef2013-06-10 12:34:45 +0000199 int k = pScan->k; /* Where to start scanning */
drh1c8148f2013-05-04 20:25:23 +0000200
drh392ddeb2016-10-26 17:57:40 +0000201 assert( pScan->iEquiv<=pScan->nEquiv );
202 pWC = pScan->pWC;
203 while(1){
drha3f108e2015-08-26 21:08:04 +0000204 iColumn = pScan->aiColumn[pScan->iEquiv-1];
drh392ddeb2016-10-26 17:57:40 +0000205 iCur = pScan->aiCur[pScan->iEquiv-1];
206 assert( pWC!=0 );
207 do{
drh43b85ef2013-06-10 12:34:45 +0000208 for(pTerm=pWC->a+k; k<pWC->nTerm; k++, pTerm++){
drhe1a086e2013-10-28 20:15:56 +0000209 if( pTerm->leftCursor==iCur
210 && pTerm->u.leftColumn==iColumn
drh4b92f982015-09-29 17:20:14 +0000211 && (iColumn!=XN_EXPR
drh76226dd2015-09-24 17:38:01 +0000212 || sqlite3ExprCompare(pTerm->pExpr->pLeft,pScan->pIdxExpr,iCur)==0)
drha3f108e2015-08-26 21:08:04 +0000213 && (pScan->iEquiv<=1 || !ExprHasProperty(pTerm->pExpr, EP_FromJoin))
drhe1a086e2013-10-28 20:15:56 +0000214 ){
drh1c8148f2013-05-04 20:25:23 +0000215 if( (pTerm->eOperator & WO_EQUIV)!=0
drha3f108e2015-08-26 21:08:04 +0000216 && pScan->nEquiv<ArraySize(pScan->aiCur)
drh72694432015-09-24 11:26:45 +0000217 && (pX = sqlite3ExprSkipCollate(pTerm->pExpr->pRight))->op==TK_COLUMN
drh1c8148f2013-05-04 20:25:23 +0000218 ){
219 int j;
drha3f108e2015-08-26 21:08:04 +0000220 for(j=0; j<pScan->nEquiv; j++){
221 if( pScan->aiCur[j]==pX->iTable
222 && pScan->aiColumn[j]==pX->iColumn ){
drh1c8148f2013-05-04 20:25:23 +0000223 break;
224 }
225 }
226 if( j==pScan->nEquiv ){
drha3f108e2015-08-26 21:08:04 +0000227 pScan->aiCur[j] = pX->iTable;
228 pScan->aiColumn[j] = pX->iColumn;
229 pScan->nEquiv++;
drh1c8148f2013-05-04 20:25:23 +0000230 }
231 }
232 if( (pTerm->eOperator & pScan->opMask)!=0 ){
233 /* Verify the affinity and collating sequence match */
234 if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){
235 CollSeq *pColl;
drh70d18342013-06-06 19:16:33 +0000236 Parse *pParse = pWC->pWInfo->pParse;
drh1c8148f2013-05-04 20:25:23 +0000237 pX = pTerm->pExpr;
238 if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){
239 continue;
240 }
241 assert(pX->pLeft);
drh70d18342013-06-06 19:16:33 +0000242 pColl = sqlite3BinaryCompareCollSeq(pParse,
drh1c8148f2013-05-04 20:25:23 +0000243 pX->pLeft, pX->pRight);
drh70d18342013-06-06 19:16:33 +0000244 if( pColl==0 ) pColl = pParse->db->pDfltColl;
drh1c8148f2013-05-04 20:25:23 +0000245 if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){
246 continue;
247 }
248 }
drhe8d0c612015-05-14 01:05:25 +0000249 if( (pTerm->eOperator & (WO_EQ|WO_IS))!=0
drha184fb82013-05-08 04:22:59 +0000250 && (pX = pTerm->pExpr->pRight)->op==TK_COLUMN
drha3f108e2015-08-26 21:08:04 +0000251 && pX->iTable==pScan->aiCur[0]
252 && pX->iColumn==pScan->aiColumn[0]
drha184fb82013-05-08 04:22:59 +0000253 ){
drhe8d0c612015-05-14 01:05:25 +0000254 testcase( pTerm->eOperator & WO_IS );
drha184fb82013-05-08 04:22:59 +0000255 continue;
256 }
drh392ddeb2016-10-26 17:57:40 +0000257 pScan->pWC = pWC;
drh43b85ef2013-06-10 12:34:45 +0000258 pScan->k = k+1;
drh1c8148f2013-05-04 20:25:23 +0000259 return pTerm;
260 }
261 }
262 }
drh392ddeb2016-10-26 17:57:40 +0000263 pWC = pWC->pOuter;
drh43b85ef2013-06-10 12:34:45 +0000264 k = 0;
drh392ddeb2016-10-26 17:57:40 +0000265 }while( pWC!=0 );
266 if( pScan->iEquiv>=pScan->nEquiv ) break;
267 pWC = pScan->pOrigWC;
drh43b85ef2013-06-10 12:34:45 +0000268 k = 0;
drha3f108e2015-08-26 21:08:04 +0000269 pScan->iEquiv++;
drh1c8148f2013-05-04 20:25:23 +0000270 }
drh1c8148f2013-05-04 20:25:23 +0000271 return 0;
272}
273
274/*
275** Initialize a WHERE clause scanner object. Return a pointer to the
276** first match. Return NULL if there are no matches.
277**
278** The scanner will be searching the WHERE clause pWC. It will look
279** for terms of the form "X <op> <expr>" where X is column iColumn of table
drha3fd75d2016-05-06 18:47:23 +0000280** iCur. Or if pIdx!=0 then X is column iColumn of index pIdx. pIdx
281** must be one of the indexes of table iCur.
282**
283** The <op> must be one of the operators described by opMask.
drh1c8148f2013-05-04 20:25:23 +0000284**
drh3b48e8c2013-06-12 20:18:16 +0000285** If the search is for X and the WHERE clause contains terms of the
286** form X=Y then this routine might also return terms of the form
287** "Y <op> <expr>". The number of levels of transitivity is limited,
288** but is enough to handle most commonly occurring SQL statements.
289**
drh1c8148f2013-05-04 20:25:23 +0000290** If X is not the INTEGER PRIMARY KEY then X must be compatible with
291** index pIdx.
292*/
danb2cfc142013-07-05 11:10:54 +0000293static WhereTerm *whereScanInit(
drh1c8148f2013-05-04 20:25:23 +0000294 WhereScan *pScan, /* The WhereScan object being initialized */
295 WhereClause *pWC, /* The WHERE clause to be scanned */
296 int iCur, /* Cursor to scan for */
297 int iColumn, /* Column to scan for */
298 u32 opMask, /* Operator(s) to scan for */
299 Index *pIdx /* Must be compatible with this index */
300){
drh1c8148f2013-05-04 20:25:23 +0000301 pScan->pOrigWC = pWC;
302 pScan->pWC = pWC;
drh6860e6f2015-08-27 18:24:02 +0000303 pScan->pIdxExpr = 0;
drh99042982016-10-26 18:41:43 +0000304 pScan->idxaff = 0;
305 pScan->zCollName = 0;
drhbb523082015-08-27 15:58:51 +0000306 if( pIdx ){
drh99042982016-10-26 18:41:43 +0000307 int j = iColumn;
drhbb523082015-08-27 15:58:51 +0000308 iColumn = pIdx->aiColumn[j];
drh99042982016-10-26 18:41:43 +0000309 if( iColumn==XN_EXPR ){
310 pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr;
311 }else if( iColumn==pIdx->pTable->iPKey ){
312 iColumn = XN_ROWID;
313 }else if( iColumn>=0 ){
314 pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity;
315 pScan->zCollName = pIdx->azColl[j];
316 }
317 }else if( iColumn==XN_EXPR ){
318 return 0;
drh1c8148f2013-05-04 20:25:23 +0000319 }
320 pScan->opMask = opMask;
drhe9d935a2013-06-05 16:19:59 +0000321 pScan->k = 0;
drha3f108e2015-08-26 21:08:04 +0000322 pScan->aiCur[0] = iCur;
323 pScan->aiColumn[0] = iColumn;
324 pScan->nEquiv = 1;
325 pScan->iEquiv = 1;
drh1c8148f2013-05-04 20:25:23 +0000326 return whereScanNext(pScan);
327}
328
329/*
drhfe05af82005-07-21 03:14:59 +0000330** Search for a term in the WHERE clause that is of the form "X <op> <expr>"
drha3fd75d2016-05-06 18:47:23 +0000331** where X is a reference to the iColumn of table iCur or of index pIdx
332** if pIdx!=0 and <op> is one of the WO_xx operator codes specified by
333** the op parameter. Return a pointer to the term. Return 0 if not found.
drh58eb1c02013-01-17 00:08:42 +0000334**
drha3fd75d2016-05-06 18:47:23 +0000335** If pIdx!=0 then it must be one of the indexes of table iCur.
336** Search for terms matching the iColumn-th column of pIdx
drhbb523082015-08-27 15:58:51 +0000337** rather than the iColumn-th column of table iCur.
338**
drh58eb1c02013-01-17 00:08:42 +0000339** The term returned might by Y=<expr> if there is another constraint in
340** the WHERE clause that specifies that X=Y. Any such constraints will be
341** identified by the WO_EQUIV bit in the pTerm->eOperator field. The
drha3f108e2015-08-26 21:08:04 +0000342** aiCur[]/iaColumn[] arrays hold X and all its equivalents. There are 11
343** slots in aiCur[]/aiColumn[] so that means we can look for X plus up to 10
344** other equivalent values. Hence a search for X will return <expr> if X=A1
345** and A1=A2 and A2=A3 and ... and A9=A10 and A10=<expr>.
drh58eb1c02013-01-17 00:08:42 +0000346**
347** If there are multiple terms in the WHERE clause of the form "X <op> <expr>"
348** then try for the one with no dependencies on <expr> - in other words where
349** <expr> is a constant expression of some kind. Only return entries of
350** the form "X <op> Y" where Y is a column in another table if no terms of
drh459f63e2013-03-06 01:55:27 +0000351** the form "X <op> <const-expr>" exist. If no terms with a constant RHS
352** exist, try to return a term that does not use WO_EQUIV.
drhfe05af82005-07-21 03:14:59 +0000353*/
drh6f82e852015-06-06 20:12:09 +0000354WhereTerm *sqlite3WhereFindTerm(
drhfe05af82005-07-21 03:14:59 +0000355 WhereClause *pWC, /* The WHERE clause to be searched */
356 int iCur, /* Cursor number of LHS */
357 int iColumn, /* Column number of LHS */
358 Bitmask notReady, /* RHS must not overlap with this mask */
drhec1724e2008-12-09 01:32:03 +0000359 u32 op, /* Mask of WO_xx values describing operator */
drhfe05af82005-07-21 03:14:59 +0000360 Index *pIdx /* Must be compatible with this index, if not NULL */
361){
drh1c8148f2013-05-04 20:25:23 +0000362 WhereTerm *pResult = 0;
363 WhereTerm *p;
364 WhereScan scan;
drh7a5bcc02013-01-16 17:08:58 +0000365
drh1c8148f2013-05-04 20:25:23 +0000366 p = whereScanInit(&scan, pWC, iCur, iColumn, op, pIdx);
drhe8d0c612015-05-14 01:05:25 +0000367 op &= WO_EQ|WO_IS;
drh1c8148f2013-05-04 20:25:23 +0000368 while( p ){
369 if( (p->prereqRight & notReady)==0 ){
drhe8d0c612015-05-14 01:05:25 +0000370 if( p->prereqRight==0 && (p->eOperator&op)!=0 ){
371 testcase( p->eOperator & WO_IS );
drh1c8148f2013-05-04 20:25:23 +0000372 return p;
drhfe05af82005-07-21 03:14:59 +0000373 }
drh1c8148f2013-05-04 20:25:23 +0000374 if( pResult==0 ) pResult = p;
drhfe05af82005-07-21 03:14:59 +0000375 }
drh1c8148f2013-05-04 20:25:23 +0000376 p = whereScanNext(&scan);
drhfe05af82005-07-21 03:14:59 +0000377 }
drh7a5bcc02013-01-16 17:08:58 +0000378 return pResult;
drhfe05af82005-07-21 03:14:59 +0000379}
380
drh7b4fc6a2007-02-06 13:26:32 +0000381/*
peter.d.reid60ec9142014-09-06 16:39:46 +0000382** This function searches pList for an entry that matches the iCol-th column
drh3b48e8c2013-06-12 20:18:16 +0000383** of index pIdx.
dan6f343962011-07-01 18:26:40 +0000384**
385** If such an expression is found, its index in pList->a[] is returned. If
386** no expression is found, -1 is returned.
387*/
388static int findIndexCol(
389 Parse *pParse, /* Parse context */
390 ExprList *pList, /* Expression list to search */
391 int iBase, /* Cursor for table associated with pIdx */
392 Index *pIdx, /* Index to match column of */
393 int iCol /* Column of index to match */
394){
395 int i;
396 const char *zColl = pIdx->azColl[iCol];
397
398 for(i=0; i<pList->nExpr; i++){
drh580c8c12012-12-08 03:34:04 +0000399 Expr *p = sqlite3ExprSkipCollate(pList->a[i].pExpr);
drhf1d3e322011-07-09 13:00:41 +0000400 if( p->op==TK_COLUMN
401 && p->iColumn==pIdx->aiColumn[iCol]
402 && p->iTable==iBase
403 ){
drh580c8c12012-12-08 03:34:04 +0000404 CollSeq *pColl = sqlite3ExprCollSeq(pParse, pList->a[i].pExpr);
drh65df68e2015-04-15 05:31:02 +0000405 if( pColl && 0==sqlite3StrICmp(pColl->zName, zColl) ){
dan6f343962011-07-01 18:26:40 +0000406 return i;
407 }
408 }
409 }
410
411 return -1;
412}
413
414/*
drhbb523082015-08-27 15:58:51 +0000415** Return TRUE if the iCol-th column of index pIdx is NOT NULL
416*/
417static int indexColumnNotNull(Index *pIdx, int iCol){
418 int j;
419 assert( pIdx!=0 );
420 assert( iCol>=0 && iCol<pIdx->nColumn );
421 j = pIdx->aiColumn[iCol];
422 if( j>=0 ){
423 return pIdx->pTable->aCol[j].notNull;
424 }else if( j==(-1) ){
425 return 1;
426 }else{
427 assert( j==(-2) );
drh84926532015-08-31 19:38:42 +0000428 return 0; /* Assume an indexed expression can always yield a NULL */
drh7d3d9da2015-09-01 00:42:52 +0000429
drhbb523082015-08-27 15:58:51 +0000430 }
431}
432
433/*
dan6f343962011-07-01 18:26:40 +0000434** Return true if the DISTINCT expression-list passed as the third argument
drh4f402f22013-06-11 18:59:38 +0000435** is redundant.
436**
drhb121dd12015-06-06 18:30:17 +0000437** A DISTINCT list is redundant if any subset of the columns in the
438** DISTINCT list are collectively unique and individually non-null.
dan6f343962011-07-01 18:26:40 +0000439*/
440static int isDistinctRedundant(
drh4f402f22013-06-11 18:59:38 +0000441 Parse *pParse, /* Parsing context */
442 SrcList *pTabList, /* The FROM clause */
443 WhereClause *pWC, /* The WHERE clause */
444 ExprList *pDistinct /* The result set that needs to be DISTINCT */
dan6f343962011-07-01 18:26:40 +0000445){
446 Table *pTab;
447 Index *pIdx;
448 int i;
449 int iBase;
450
451 /* If there is more than one table or sub-select in the FROM clause of
452 ** this query, then it will not be possible to show that the DISTINCT
453 ** clause is redundant. */
454 if( pTabList->nSrc!=1 ) return 0;
455 iBase = pTabList->a[0].iCursor;
456 pTab = pTabList->a[0].pTab;
457
dan94e08d92011-07-02 06:44:05 +0000458 /* If any of the expressions is an IPK column on table iBase, then return
459 ** true. Note: The (p->iTable==iBase) part of this test may be false if the
460 ** current SELECT is a correlated sub-query.
461 */
dan6f343962011-07-01 18:26:40 +0000462 for(i=0; i<pDistinct->nExpr; i++){
drh580c8c12012-12-08 03:34:04 +0000463 Expr *p = sqlite3ExprSkipCollate(pDistinct->a[i].pExpr);
dan94e08d92011-07-02 06:44:05 +0000464 if( p->op==TK_COLUMN && p->iTable==iBase && p->iColumn<0 ) return 1;
dan6f343962011-07-01 18:26:40 +0000465 }
466
467 /* Loop through all indices on the table, checking each to see if it makes
468 ** the DISTINCT qualifier redundant. It does so if:
469 **
470 ** 1. The index is itself UNIQUE, and
471 **
472 ** 2. All of the columns in the index are either part of the pDistinct
473 ** list, or else the WHERE clause contains a term of the form "col=X",
474 ** where X is a constant value. The collation sequences of the
475 ** comparison and select-list expressions must match those of the index.
dan6a36f432012-04-20 16:59:24 +0000476 **
477 ** 3. All of those index columns for which the WHERE clause does not
478 ** contain a "col=X" term are subject to a NOT NULL constraint.
dan6f343962011-07-01 18:26:40 +0000479 */
480 for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
drh5f1d1d92014-07-31 22:59:04 +0000481 if( !IsUniqueIndex(pIdx) ) continue;
drhbbbdc832013-10-22 18:01:40 +0000482 for(i=0; i<pIdx->nKeyCol; i++){
drhbb523082015-08-27 15:58:51 +0000483 if( 0==sqlite3WhereFindTerm(pWC, iBase, i, ~(Bitmask)0, WO_EQ, pIdx) ){
484 if( findIndexCol(pParse, pDistinct, iBase, pIdx, i)<0 ) break;
485 if( indexColumnNotNull(pIdx, i)==0 ) break;
dan6f343962011-07-01 18:26:40 +0000486 }
487 }
drhbbbdc832013-10-22 18:01:40 +0000488 if( i==pIdx->nKeyCol ){
dan6f343962011-07-01 18:26:40 +0000489 /* This index implies that the DISTINCT qualifier is redundant. */
490 return 1;
491 }
492 }
493
494 return 0;
495}
drh0fcef5e2005-07-19 17:38:22 +0000496
drh8636e9c2013-06-11 01:50:08 +0000497
drh75897232000-05-29 14:26:00 +0000498/*
drh3b48e8c2013-06-12 20:18:16 +0000499** Estimate the logarithm of the input value to base 2.
drh28c4cf42005-07-27 20:41:43 +0000500*/
drhbf539c42013-10-05 18:16:02 +0000501static LogEst estLog(LogEst N){
drh696964d2014-06-12 15:46:46 +0000502 return N<=10 ? 0 : sqlite3LogEst(N) - 33;
drh28c4cf42005-07-27 20:41:43 +0000503}
504
drh6d209d82006-06-27 01:54:26 +0000505/*
drh7b3aa082015-05-29 13:55:33 +0000506** Convert OP_Column opcodes to OP_Copy in previously generated code.
507**
508** This routine runs over generated VDBE code and translates OP_Column
danfb785b22015-10-24 20:31:22 +0000509** opcodes into OP_Copy when the table is being accessed via co-routine
510** instead of via table lookup.
511**
512** If the bIncrRowid parameter is 0, then any OP_Rowid instructions on
513** cursor iTabCur are transformed into OP_Null. Or, if bIncrRowid is non-zero,
514** then each OP_Rowid is transformed into an instruction to increment the
515** value stored in its output register.
drh7b3aa082015-05-29 13:55:33 +0000516*/
517static void translateColumnToCopy(
518 Vdbe *v, /* The VDBE containing code to translate */
519 int iStart, /* Translate from this opcode to the end */
520 int iTabCur, /* OP_Column/OP_Rowid references to this table */
danfb785b22015-10-24 20:31:22 +0000521 int iRegister, /* The first column is in this register */
522 int bIncrRowid /* If non-zero, transform OP_rowid to OP_AddImm(1) */
drh7b3aa082015-05-29 13:55:33 +0000523){
524 VdbeOp *pOp = sqlite3VdbeGetOp(v, iStart);
525 int iEnd = sqlite3VdbeCurrentAddr(v);
526 for(; iStart<iEnd; iStart++, pOp++){
527 if( pOp->p1!=iTabCur ) continue;
528 if( pOp->opcode==OP_Column ){
529 pOp->opcode = OP_Copy;
530 pOp->p1 = pOp->p2 + iRegister;
531 pOp->p2 = pOp->p3;
532 pOp->p3 = 0;
533 }else if( pOp->opcode==OP_Rowid ){
danfb785b22015-10-24 20:31:22 +0000534 if( bIncrRowid ){
535 /* Increment the value stored in the P2 operand of the OP_Rowid. */
536 pOp->opcode = OP_AddImm;
537 pOp->p1 = pOp->p2;
538 pOp->p2 = 1;
539 }else{
540 pOp->opcode = OP_Null;
541 pOp->p1 = 0;
542 pOp->p3 = 0;
543 }
drh7b3aa082015-05-29 13:55:33 +0000544 }
545 }
546}
547
548/*
drh6d209d82006-06-27 01:54:26 +0000549** Two routines for printing the content of an sqlite3_index_info
550** structure. Used for testing and debugging only. If neither
551** SQLITE_TEST or SQLITE_DEBUG are defined, then these routines
552** are no-ops.
553*/
drhd15cb172013-05-21 19:23:10 +0000554#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED)
drh6d209d82006-06-27 01:54:26 +0000555static void TRACE_IDX_INPUTS(sqlite3_index_info *p){
556 int i;
mlcreech3a00f902008-03-04 17:45:01 +0000557 if( !sqlite3WhereTrace ) return;
drh6d209d82006-06-27 01:54:26 +0000558 for(i=0; i<p->nConstraint; i++){
559 sqlite3DebugPrintf(" constraint[%d]: col=%d termid=%d op=%d usabled=%d\n",
560 i,
561 p->aConstraint[i].iColumn,
562 p->aConstraint[i].iTermOffset,
563 p->aConstraint[i].op,
564 p->aConstraint[i].usable);
565 }
566 for(i=0; i<p->nOrderBy; i++){
567 sqlite3DebugPrintf(" orderby[%d]: col=%d desc=%d\n",
568 i,
569 p->aOrderBy[i].iColumn,
570 p->aOrderBy[i].desc);
571 }
572}
573static void TRACE_IDX_OUTPUTS(sqlite3_index_info *p){
574 int i;
mlcreech3a00f902008-03-04 17:45:01 +0000575 if( !sqlite3WhereTrace ) return;
drh6d209d82006-06-27 01:54:26 +0000576 for(i=0; i<p->nConstraint; i++){
577 sqlite3DebugPrintf(" usage[%d]: argvIdx=%d omit=%d\n",
578 i,
579 p->aConstraintUsage[i].argvIndex,
580 p->aConstraintUsage[i].omit);
581 }
582 sqlite3DebugPrintf(" idxNum=%d\n", p->idxNum);
583 sqlite3DebugPrintf(" idxStr=%s\n", p->idxStr);
584 sqlite3DebugPrintf(" orderByConsumed=%d\n", p->orderByConsumed);
585 sqlite3DebugPrintf(" estimatedCost=%g\n", p->estimatedCost);
dana9f58152013-11-11 19:01:33 +0000586 sqlite3DebugPrintf(" estimatedRows=%lld\n", p->estimatedRows);
drh6d209d82006-06-27 01:54:26 +0000587}
588#else
589#define TRACE_IDX_INPUTS(A)
590#define TRACE_IDX_OUTPUTS(A)
591#endif
592
drhc6339082010-04-07 16:54:58 +0000593#ifndef SQLITE_OMIT_AUTOMATIC_INDEX
drh8b307fb2010-04-06 15:57:05 +0000594/*
drh4139c992010-04-07 14:59:45 +0000595** Return TRUE if the WHERE clause term pTerm is of a form where it
596** could be used with an index to access pSrc, assuming an appropriate
597** index existed.
598*/
599static int termCanDriveIndex(
600 WhereTerm *pTerm, /* WHERE clause term to check */
601 struct SrcList_item *pSrc, /* Table we are trying to access */
602 Bitmask notReady /* Tables in outer loops of the join */
603){
604 char aff;
605 if( pTerm->leftCursor!=pSrc->iCursor ) return 0;
drhe8d0c612015-05-14 01:05:25 +0000606 if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) return 0;
drh4139c992010-04-07 14:59:45 +0000607 if( (pTerm->prereqRight & notReady)!=0 ) return 0;
drh23f98da2013-05-21 15:52:07 +0000608 if( pTerm->u.leftColumn<0 ) return 0;
drh4139c992010-04-07 14:59:45 +0000609 aff = pSrc->pTab->aCol[pTerm->u.leftColumn].affinity;
610 if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0;
drhe0cc3c22015-05-13 17:54:08 +0000611 testcase( pTerm->pExpr->op==TK_IS );
drh4139c992010-04-07 14:59:45 +0000612 return 1;
613}
drhc6339082010-04-07 16:54:58 +0000614#endif
drh4139c992010-04-07 14:59:45 +0000615
drhc6339082010-04-07 16:54:58 +0000616
617#ifndef SQLITE_OMIT_AUTOMATIC_INDEX
drh8b307fb2010-04-06 15:57:05 +0000618/*
drhc6339082010-04-07 16:54:58 +0000619** Generate code to construct the Index object for an automatic index
620** and to set up the WhereLevel object pLevel so that the code generator
621** makes use of the automatic index.
drh8b307fb2010-04-06 15:57:05 +0000622*/
drhc6339082010-04-07 16:54:58 +0000623static void constructAutomaticIndex(
drh8b307fb2010-04-06 15:57:05 +0000624 Parse *pParse, /* The parsing context */
625 WhereClause *pWC, /* The WHERE clause */
626 struct SrcList_item *pSrc, /* The FROM clause term to get the next index */
627 Bitmask notReady, /* Mask of cursors that are not available */
628 WhereLevel *pLevel /* Write new index here */
629){
drhbbbdc832013-10-22 18:01:40 +0000630 int nKeyCol; /* Number of columns in the constructed index */
drh8b307fb2010-04-06 15:57:05 +0000631 WhereTerm *pTerm; /* A single term of the WHERE clause */
632 WhereTerm *pWCEnd; /* End of pWC->a[] */
drh8b307fb2010-04-06 15:57:05 +0000633 Index *pIdx; /* Object describing the transient index */
634 Vdbe *v; /* Prepared statement under construction */
drh8b307fb2010-04-06 15:57:05 +0000635 int addrInit; /* Address of the initialization bypass jump */
636 Table *pTable; /* The table being indexed */
drh8b307fb2010-04-06 15:57:05 +0000637 int addrTop; /* Top of the index fill loop */
638 int regRecord; /* Register holding an index record */
639 int n; /* Column counter */
drh4139c992010-04-07 14:59:45 +0000640 int i; /* Loop counter */
641 int mxBitCol; /* Maximum column in pSrc->colUsed */
drh424aab82010-04-06 18:28:20 +0000642 CollSeq *pColl; /* Collating sequence to on a column */
drh7ba39a92013-05-30 17:43:19 +0000643 WhereLoop *pLoop; /* The Loop object */
drh77e57df2013-10-22 14:28:02 +0000644 char *zNotUsed; /* Extra space on the end of pIdx */
drh4139c992010-04-07 14:59:45 +0000645 Bitmask idxCols; /* Bitmap of columns used for indexing */
646 Bitmask extraCols; /* Bitmap of additional columns */
drh8d56e202013-06-28 23:55:45 +0000647 u8 sentWarning = 0; /* True if a warnning has been issued */
drh059b2d52014-10-24 19:28:09 +0000648 Expr *pPartial = 0; /* Partial Index Expression */
649 int iContinue = 0; /* Jump here to skip excluded rows */
drh7b3aa082015-05-29 13:55:33 +0000650 struct SrcList_item *pTabItem; /* FROM clause term being indexed */
drh4dd83a22015-10-26 14:54:32 +0000651 int addrCounter = 0; /* Address where integer counter is initialized */
danfb785b22015-10-24 20:31:22 +0000652 int regBase; /* Array of registers where record is assembled */
drh8b307fb2010-04-06 15:57:05 +0000653
654 /* Generate code to skip over the creation and initialization of the
655 ** transient index on 2nd and subsequent iterations of the loop. */
656 v = pParse->pVdbe;
657 assert( v!=0 );
drh511f9e82016-09-22 18:53:13 +0000658 addrInit = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
drh8b307fb2010-04-06 15:57:05 +0000659
drh4139c992010-04-07 14:59:45 +0000660 /* Count the number of columns that will be added to the index
661 ** and used to match WHERE clause constraints */
drhbbbdc832013-10-22 18:01:40 +0000662 nKeyCol = 0;
drh424aab82010-04-06 18:28:20 +0000663 pTable = pSrc->pTab;
drh8b307fb2010-04-06 15:57:05 +0000664 pWCEnd = &pWC->a[pWC->nTerm];
drh7ba39a92013-05-30 17:43:19 +0000665 pLoop = pLevel->pWLoop;
drh4139c992010-04-07 14:59:45 +0000666 idxCols = 0;
drh81186b42013-06-18 01:52:41 +0000667 for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){
drh13cc90c2015-02-25 00:24:41 +0000668 Expr *pExpr = pTerm->pExpr;
669 assert( !ExprHasProperty(pExpr, EP_FromJoin) /* prereq always non-zero */
670 || pExpr->iRightJoinTable!=pSrc->iCursor /* for the right-hand */
671 || pLoop->prereq!=0 ); /* table of a LEFT JOIN */
drh059b2d52014-10-24 19:28:09 +0000672 if( pLoop->prereq==0
drh051575c2014-10-25 12:28:25 +0000673 && (pTerm->wtFlags & TERM_VIRTUAL)==0
drh13cc90c2015-02-25 00:24:41 +0000674 && !ExprHasProperty(pExpr, EP_FromJoin)
675 && sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor) ){
drh059b2d52014-10-24 19:28:09 +0000676 pPartial = sqlite3ExprAnd(pParse->db, pPartial,
drh13cc90c2015-02-25 00:24:41 +0000677 sqlite3ExprDup(pParse->db, pExpr, 0));
drh059b2d52014-10-24 19:28:09 +0000678 }
drh4139c992010-04-07 14:59:45 +0000679 if( termCanDriveIndex(pTerm, pSrc, notReady) ){
680 int iCol = pTerm->u.leftColumn;
drh7699d1c2013-06-04 12:42:29 +0000681 Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol);
drh52ff8ea2010-04-08 14:15:56 +0000682 testcase( iCol==BMS );
683 testcase( iCol==BMS-1 );
drh8d56e202013-06-28 23:55:45 +0000684 if( !sentWarning ){
685 sqlite3_log(SQLITE_WARNING_AUTOINDEX,
686 "automatic index on %s(%s)", pTable->zName,
687 pTable->aCol[iCol].zName);
688 sentWarning = 1;
689 }
drh0013e722010-04-08 00:40:15 +0000690 if( (idxCols & cMask)==0 ){
drh059b2d52014-10-24 19:28:09 +0000691 if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ){
692 goto end_auto_index_create;
693 }
drhbbbdc832013-10-22 18:01:40 +0000694 pLoop->aLTerm[nKeyCol++] = pTerm;
drh0013e722010-04-08 00:40:15 +0000695 idxCols |= cMask;
696 }
drh8b307fb2010-04-06 15:57:05 +0000697 }
698 }
drhbbbdc832013-10-22 18:01:40 +0000699 assert( nKeyCol>0 );
700 pLoop->u.btree.nEq = pLoop->nLTerm = nKeyCol;
drh53b52f72013-05-31 11:57:39 +0000701 pLoop->wsFlags = WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WHERE_INDEXED
drh986b3872013-06-28 21:12:20 +0000702 | WHERE_AUTO_INDEX;
drh4139c992010-04-07 14:59:45 +0000703
704 /* Count the number of additional columns needed to create a
705 ** covering index. A "covering index" is an index that contains all
706 ** columns that are needed by the query. With a covering index, the
707 ** original table never needs to be accessed. Automatic indices must
708 ** be a covering index because the index will not be updated if the
709 ** original table changes and the index and table cannot both be used
710 ** if they go out of sync.
711 */
drh7699d1c2013-06-04 12:42:29 +0000712 extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1));
drhc3ef4fa2014-10-28 15:58:50 +0000713 mxBitCol = MIN(BMS-1,pTable->nCol);
drh52ff8ea2010-04-08 14:15:56 +0000714 testcase( pTable->nCol==BMS-1 );
715 testcase( pTable->nCol==BMS-2 );
drh4139c992010-04-07 14:59:45 +0000716 for(i=0; i<mxBitCol; i++){
drhbbbdc832013-10-22 18:01:40 +0000717 if( extraCols & MASKBIT(i) ) nKeyCol++;
drh4139c992010-04-07 14:59:45 +0000718 }
drh7699d1c2013-06-04 12:42:29 +0000719 if( pSrc->colUsed & MASKBIT(BMS-1) ){
drhbbbdc832013-10-22 18:01:40 +0000720 nKeyCol += pTable->nCol - BMS + 1;
drh4139c992010-04-07 14:59:45 +0000721 }
drh8b307fb2010-04-06 15:57:05 +0000722
723 /* Construct the Index object to describe this index */
drhbbbdc832013-10-22 18:01:40 +0000724 pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+1, 0, &zNotUsed);
drh059b2d52014-10-24 19:28:09 +0000725 if( pIdx==0 ) goto end_auto_index_create;
drh7ba39a92013-05-30 17:43:19 +0000726 pLoop->u.btree.pIndex = pIdx;
drh8b307fb2010-04-06 15:57:05 +0000727 pIdx->zName = "auto-index";
drh424aab82010-04-06 18:28:20 +0000728 pIdx->pTable = pTable;
drh8b307fb2010-04-06 15:57:05 +0000729 n = 0;
drh0013e722010-04-08 00:40:15 +0000730 idxCols = 0;
drh8b307fb2010-04-06 15:57:05 +0000731 for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){
drh4139c992010-04-07 14:59:45 +0000732 if( termCanDriveIndex(pTerm, pSrc, notReady) ){
drh0013e722010-04-08 00:40:15 +0000733 int iCol = pTerm->u.leftColumn;
drh7699d1c2013-06-04 12:42:29 +0000734 Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol);
drh7963b0e2013-06-17 21:37:40 +0000735 testcase( iCol==BMS-1 );
736 testcase( iCol==BMS );
drh0013e722010-04-08 00:40:15 +0000737 if( (idxCols & cMask)==0 ){
738 Expr *pX = pTerm->pExpr;
739 idxCols |= cMask;
740 pIdx->aiColumn[n] = pTerm->u.leftColumn;
741 pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight);
drhf19aa5f2015-12-30 16:51:20 +0000742 pIdx->azColl[n] = pColl ? pColl->zName : sqlite3StrBINARY;
drh0013e722010-04-08 00:40:15 +0000743 n++;
744 }
drh8b307fb2010-04-06 15:57:05 +0000745 }
746 }
drh7ba39a92013-05-30 17:43:19 +0000747 assert( (u32)n==pLoop->u.btree.nEq );
drh4139c992010-04-07 14:59:45 +0000748
drhc6339082010-04-07 16:54:58 +0000749 /* Add additional columns needed to make the automatic index into
750 ** a covering index */
drh4139c992010-04-07 14:59:45 +0000751 for(i=0; i<mxBitCol; i++){
drh7699d1c2013-06-04 12:42:29 +0000752 if( extraCols & MASKBIT(i) ){
drh4139c992010-04-07 14:59:45 +0000753 pIdx->aiColumn[n] = i;
drhf19aa5f2015-12-30 16:51:20 +0000754 pIdx->azColl[n] = sqlite3StrBINARY;
drh4139c992010-04-07 14:59:45 +0000755 n++;
756 }
757 }
drh7699d1c2013-06-04 12:42:29 +0000758 if( pSrc->colUsed & MASKBIT(BMS-1) ){
drh4139c992010-04-07 14:59:45 +0000759 for(i=BMS-1; i<pTable->nCol; i++){
760 pIdx->aiColumn[n] = i;
drhf19aa5f2015-12-30 16:51:20 +0000761 pIdx->azColl[n] = sqlite3StrBINARY;
drh4139c992010-04-07 14:59:45 +0000762 n++;
763 }
764 }
drhbbbdc832013-10-22 18:01:40 +0000765 assert( n==nKeyCol );
drh4b92f982015-09-29 17:20:14 +0000766 pIdx->aiColumn[n] = XN_ROWID;
drhf19aa5f2015-12-30 16:51:20 +0000767 pIdx->azColl[n] = sqlite3StrBINARY;
drh8b307fb2010-04-06 15:57:05 +0000768
drhc6339082010-04-07 16:54:58 +0000769 /* Create the automatic index */
drh8b307fb2010-04-06 15:57:05 +0000770 assert( pLevel->iIdxCur>=0 );
drha1f41242013-05-31 20:00:58 +0000771 pLevel->iIdxCur = pParse->nTab++;
drh2ec2fb22013-11-06 19:59:23 +0000772 sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1);
773 sqlite3VdbeSetP4KeyInfo(pParse, pIdx);
drha21a64d2010-04-06 22:33:55 +0000774 VdbeComment((v, "for %s", pTable->zName));
drh8b307fb2010-04-06 15:57:05 +0000775
drhc6339082010-04-07 16:54:58 +0000776 /* Fill the automatic index with content */
drh059b2d52014-10-24 19:28:09 +0000777 sqlite3ExprCachePush(pParse);
drh7b3aa082015-05-29 13:55:33 +0000778 pTabItem = &pWC->pWInfo->pTabList->a[pLevel->iFrom];
drh8a48b9c2015-08-19 15:20:00 +0000779 if( pTabItem->fg.viaCoroutine ){
drh7b3aa082015-05-29 13:55:33 +0000780 int regYield = pTabItem->regReturn;
danfb785b22015-10-24 20:31:22 +0000781 addrCounter = sqlite3VdbeAddOp2(v, OP_Integer, 0, 0);
drh7b3aa082015-05-29 13:55:33 +0000782 sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub);
783 addrTop = sqlite3VdbeAddOp1(v, OP_Yield, regYield);
784 VdbeCoverage(v);
785 VdbeComment((v, "next row of \"%s\"", pTabItem->pTab->zName));
786 }else{
787 addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v);
788 }
drh059b2d52014-10-24 19:28:09 +0000789 if( pPartial ){
790 iContinue = sqlite3VdbeMakeLabel(v);
791 sqlite3ExprIfFalse(pParse, pPartial, iContinue, SQLITE_JUMPIFNULL);
drh051575c2014-10-25 12:28:25 +0000792 pLoop->wsFlags |= WHERE_PARTIALIDX;
drh059b2d52014-10-24 19:28:09 +0000793 }
drh8b307fb2010-04-06 15:57:05 +0000794 regRecord = sqlite3GetTempReg(pParse);
danfb785b22015-10-24 20:31:22 +0000795 regBase = sqlite3GenerateIndexKey(
796 pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0
797 );
drh8b307fb2010-04-06 15:57:05 +0000798 sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord);
799 sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
drh059b2d52014-10-24 19:28:09 +0000800 if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue);
drh8a48b9c2015-08-19 15:20:00 +0000801 if( pTabItem->fg.viaCoroutine ){
danfb785b22015-10-24 20:31:22 +0000802 sqlite3VdbeChangeP2(v, addrCounter, regBase+n);
803 translateColumnToCopy(v, addrTop, pLevel->iTabCur, pTabItem->regResult, 1);
drh076e85f2015-09-03 13:46:12 +0000804 sqlite3VdbeGoto(v, addrTop);
drh8a48b9c2015-08-19 15:20:00 +0000805 pTabItem->fg.viaCoroutine = 0;
drh7b3aa082015-05-29 13:55:33 +0000806 }else{
807 sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v);
808 }
drha21a64d2010-04-06 22:33:55 +0000809 sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX);
drh8b307fb2010-04-06 15:57:05 +0000810 sqlite3VdbeJumpHere(v, addrTop);
811 sqlite3ReleaseTempReg(pParse, regRecord);
drh059b2d52014-10-24 19:28:09 +0000812 sqlite3ExprCachePop(pParse);
drh8b307fb2010-04-06 15:57:05 +0000813
814 /* Jump here when skipping the initialization */
815 sqlite3VdbeJumpHere(v, addrInit);
drh059b2d52014-10-24 19:28:09 +0000816
817end_auto_index_create:
818 sqlite3ExprDelete(pParse->db, pPartial);
drh8b307fb2010-04-06 15:57:05 +0000819}
drhc6339082010-04-07 16:54:58 +0000820#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */
drh8b307fb2010-04-06 15:57:05 +0000821
drh9eff6162006-06-12 21:59:13 +0000822#ifndef SQLITE_OMIT_VIRTUALTABLE
823/*
danielk19771d461462009-04-21 09:02:45 +0000824** Allocate and populate an sqlite3_index_info structure. It is the
825** responsibility of the caller to eventually release the structure
826** by passing the pointer returned by this function to sqlite3_free().
827*/
drh5346e952013-05-08 14:14:26 +0000828static sqlite3_index_info *allocateIndexInfo(
829 Parse *pParse,
830 WhereClause *pWC,
dan4f20cd42015-06-08 18:05:54 +0000831 Bitmask mUnusable, /* Ignore terms with these prereqs */
drh5346e952013-05-08 14:14:26 +0000832 struct SrcList_item *pSrc,
dan6256c1c2016-08-08 20:15:41 +0000833 ExprList *pOrderBy,
834 u16 *pmNoOmit /* Mask of terms not to omit */
drh5346e952013-05-08 14:14:26 +0000835){
danielk19771d461462009-04-21 09:02:45 +0000836 int i, j;
837 int nTerm;
838 struct sqlite3_index_constraint *pIdxCons;
839 struct sqlite3_index_orderby *pIdxOrderBy;
840 struct sqlite3_index_constraint_usage *pUsage;
841 WhereTerm *pTerm;
842 int nOrderBy;
843 sqlite3_index_info *pIdxInfo;
dan6256c1c2016-08-08 20:15:41 +0000844 u16 mNoOmit = 0;
danielk19771d461462009-04-21 09:02:45 +0000845
danielk19771d461462009-04-21 09:02:45 +0000846 /* Count the number of possible WHERE clause constraints referring
847 ** to this virtual table */
848 for(i=nTerm=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
849 if( pTerm->leftCursor != pSrc->iCursor ) continue;
dan4f20cd42015-06-08 18:05:54 +0000850 if( pTerm->prereqRight & mUnusable ) continue;
drh7a5bcc02013-01-16 17:08:58 +0000851 assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) );
852 testcase( pTerm->eOperator & WO_IN );
853 testcase( pTerm->eOperator & WO_ISNULL );
drhee145872015-05-14 13:18:47 +0000854 testcase( pTerm->eOperator & WO_IS );
dana4ff8252014-01-20 19:55:33 +0000855 testcase( pTerm->eOperator & WO_ALL );
drhee145872015-05-14 13:18:47 +0000856 if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV|WO_IS))==0 ) continue;
drhb4256992011-08-02 01:57:39 +0000857 if( pTerm->wtFlags & TERM_VNULL ) continue;
drh7d3d9da2015-09-01 00:42:52 +0000858 assert( pTerm->u.leftColumn>=(-1) );
danielk19771d461462009-04-21 09:02:45 +0000859 nTerm++;
860 }
861
862 /* If the ORDER BY clause contains only columns in the current
863 ** virtual table then allocate space for the aOrderBy part of
864 ** the sqlite3_index_info structure.
865 */
866 nOrderBy = 0;
867 if( pOrderBy ){
drh56f1b992012-09-25 14:29:39 +0000868 int n = pOrderBy->nExpr;
869 for(i=0; i<n; i++){
danielk19771d461462009-04-21 09:02:45 +0000870 Expr *pExpr = pOrderBy->a[i].pExpr;
871 if( pExpr->op!=TK_COLUMN || pExpr->iTable!=pSrc->iCursor ) break;
872 }
drh56f1b992012-09-25 14:29:39 +0000873 if( i==n){
874 nOrderBy = n;
danielk19771d461462009-04-21 09:02:45 +0000875 }
876 }
877
878 /* Allocate the sqlite3_index_info structure
879 */
880 pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo)
881 + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm
882 + sizeof(*pIdxOrderBy)*nOrderBy );
883 if( pIdxInfo==0 ){
884 sqlite3ErrorMsg(pParse, "out of memory");
danielk19771d461462009-04-21 09:02:45 +0000885 return 0;
886 }
887
888 /* Initialize the structure. The sqlite3_index_info structure contains
889 ** many fields that are declared "const" to prevent xBestIndex from
890 ** changing them. We have to do some funky casting in order to
891 ** initialize those fields.
892 */
893 pIdxCons = (struct sqlite3_index_constraint*)&pIdxInfo[1];
894 pIdxOrderBy = (struct sqlite3_index_orderby*)&pIdxCons[nTerm];
895 pUsage = (struct sqlite3_index_constraint_usage*)&pIdxOrderBy[nOrderBy];
896 *(int*)&pIdxInfo->nConstraint = nTerm;
897 *(int*)&pIdxInfo->nOrderBy = nOrderBy;
898 *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint = pIdxCons;
899 *(struct sqlite3_index_orderby**)&pIdxInfo->aOrderBy = pIdxOrderBy;
900 *(struct sqlite3_index_constraint_usage**)&pIdxInfo->aConstraintUsage =
901 pUsage;
902
903 for(i=j=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
drh281bbe22012-10-16 23:17:14 +0000904 u8 op;
danielk19771d461462009-04-21 09:02:45 +0000905 if( pTerm->leftCursor != pSrc->iCursor ) continue;
dan4f20cd42015-06-08 18:05:54 +0000906 if( pTerm->prereqRight & mUnusable ) continue;
drh7a5bcc02013-01-16 17:08:58 +0000907 assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) );
908 testcase( pTerm->eOperator & WO_IN );
drhee145872015-05-14 13:18:47 +0000909 testcase( pTerm->eOperator & WO_IS );
drh7a5bcc02013-01-16 17:08:58 +0000910 testcase( pTerm->eOperator & WO_ISNULL );
dana4ff8252014-01-20 19:55:33 +0000911 testcase( pTerm->eOperator & WO_ALL );
drhe8d0c612015-05-14 01:05:25 +0000912 if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV|WO_IS))==0 ) continue;
drhb4256992011-08-02 01:57:39 +0000913 if( pTerm->wtFlags & TERM_VNULL ) continue;
drh7d3d9da2015-09-01 00:42:52 +0000914 assert( pTerm->u.leftColumn>=(-1) );
danielk19771d461462009-04-21 09:02:45 +0000915 pIdxCons[j].iColumn = pTerm->u.leftColumn;
916 pIdxCons[j].iTermOffset = i;
drh7a5bcc02013-01-16 17:08:58 +0000917 op = (u8)pTerm->eOperator & WO_ALL;
drh281bbe22012-10-16 23:17:14 +0000918 if( op==WO_IN ) op = WO_EQ;
dan07bdba82015-11-23 21:09:54 +0000919 if( op==WO_MATCH ){
920 op = pTerm->eMatchOp;
921 }
drh281bbe22012-10-16 23:17:14 +0000922 pIdxCons[j].op = op;
danielk19771d461462009-04-21 09:02:45 +0000923 /* The direct assignment in the previous line is possible only because
924 ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The
925 ** following asserts verify this fact. */
926 assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ );
927 assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT );
928 assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE );
929 assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT );
930 assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE );
931 assert( WO_MATCH==SQLITE_INDEX_CONSTRAINT_MATCH );
drh281bbe22012-10-16 23:17:14 +0000932 assert( pTerm->eOperator & (WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_MATCH) );
dan6256c1c2016-08-08 20:15:41 +0000933
934 if( op & (WO_LT|WO_LE|WO_GT|WO_GE)
935 && sqlite3ExprIsVector(pTerm->pExpr->pRight)
936 ){
937 if( i<16 ) mNoOmit |= (1 << i);
938 if( op==WO_LT ) pIdxCons[j].op = WO_LE;
939 if( op==WO_GT ) pIdxCons[j].op = WO_GE;
940 }
941
danielk19771d461462009-04-21 09:02:45 +0000942 j++;
943 }
944 for(i=0; i<nOrderBy; i++){
945 Expr *pExpr = pOrderBy->a[i].pExpr;
946 pIdxOrderBy[i].iColumn = pExpr->iColumn;
947 pIdxOrderBy[i].desc = pOrderBy->a[i].sortOrder;
948 }
949
dan6256c1c2016-08-08 20:15:41 +0000950 *pmNoOmit = mNoOmit;
danielk19771d461462009-04-21 09:02:45 +0000951 return pIdxInfo;
952}
953
954/*
955** The table object reference passed as the second argument to this function
956** must represent a virtual table. This function invokes the xBestIndex()
drh3b48e8c2013-06-12 20:18:16 +0000957** method of the virtual table with the sqlite3_index_info object that
958** comes in as the 3rd argument to this function.
danielk19771d461462009-04-21 09:02:45 +0000959**
960** If an error occurs, pParse is populated with an error message and a
961** non-zero value is returned. Otherwise, 0 is returned and the output
962** part of the sqlite3_index_info structure is left populated.
963**
964** Whether or not an error is returned, it is the responsibility of the
965** caller to eventually free p->idxStr if p->needToFreeIdxStr indicates
966** that this is required.
967*/
968static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){
danielk1977595a5232009-07-24 17:58:53 +0000969 sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab;
danielk19771d461462009-04-21 09:02:45 +0000970 int rc;
971
danielk19771d461462009-04-21 09:02:45 +0000972 TRACE_IDX_INPUTS(p);
973 rc = pVtab->pModule->xBestIndex(pVtab, p);
974 TRACE_IDX_OUTPUTS(p);
danielk19771d461462009-04-21 09:02:45 +0000975
976 if( rc!=SQLITE_OK ){
977 if( rc==SQLITE_NOMEM ){
drh4a642b62016-02-05 01:55:27 +0000978 sqlite3OomFault(pParse->db);
danielk19771d461462009-04-21 09:02:45 +0000979 }else if( !pVtab->zErrMsg ){
980 sqlite3ErrorMsg(pParse, "%s", sqlite3ErrStr(rc));
981 }else{
982 sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg);
983 }
984 }
drhb9755982010-07-24 16:34:37 +0000985 sqlite3_free(pVtab->zErrMsg);
danielk19771d461462009-04-21 09:02:45 +0000986 pVtab->zErrMsg = 0;
987
drh6de32e72016-03-08 02:59:33 +0000988#if 0
989 /* This error is now caught by the caller.
990 ** Search for "xBestIndex malfunction" below */
danielk19771d461462009-04-21 09:02:45 +0000991 for(i=0; i<p->nConstraint; i++){
992 if( !p->aConstraint[i].usable && p->aConstraintUsage[i].argvIndex>0 ){
993 sqlite3ErrorMsg(pParse,
994 "table %s: xBestIndex returned an invalid plan", pTab->zName);
995 }
996 }
drh6de32e72016-03-08 02:59:33 +0000997#endif
danielk19771d461462009-04-21 09:02:45 +0000998
999 return pParse->nErr;
1000}
drh7ba39a92013-05-30 17:43:19 +00001001#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */
danielk19771d461462009-04-21 09:02:45 +00001002
drh1435a9a2013-08-27 23:15:44 +00001003#ifdef SQLITE_ENABLE_STAT3_OR_STAT4
drh28c4cf42005-07-27 20:41:43 +00001004/*
drhfaacf172011-08-12 01:51:45 +00001005** Estimate the location of a particular key among all keys in an
1006** index. Store the results in aStat as follows:
drhe847d322011-01-20 02:56:37 +00001007**
dana3d0c132015-03-14 18:59:58 +00001008** aStat[0] Est. number of rows less than pRec
1009** aStat[1] Est. number of rows equal to pRec
dan02fa4692009-08-17 17:06:58 +00001010**
drh6d3f91d2014-11-05 19:26:12 +00001011** Return the index of the sample that is the smallest sample that
dana3d0c132015-03-14 18:59:58 +00001012** is greater than or equal to pRec. Note that this index is not an index
1013** into the aSample[] array - it is an index into a virtual set of samples
1014** based on the contents of aSample[] and the number of fields in record
1015** pRec.
dan02fa4692009-08-17 17:06:58 +00001016*/
drh6d3f91d2014-11-05 19:26:12 +00001017static int whereKeyStats(
dan02fa4692009-08-17 17:06:58 +00001018 Parse *pParse, /* Database connection */
1019 Index *pIdx, /* Index to consider domain of */
dan7a419232013-08-06 20:01:43 +00001020 UnpackedRecord *pRec, /* Vector of values to consider */
drhfaacf172011-08-12 01:51:45 +00001021 int roundUp, /* Round up if true. Round down if false */
1022 tRowcnt *aStat /* OUT: stats written here */
dan02fa4692009-08-17 17:06:58 +00001023){
danf52bb8d2013-08-03 20:24:58 +00001024 IndexSample *aSample = pIdx->aSample;
drhfbc38de2013-09-03 19:26:22 +00001025 int iCol; /* Index of required stats in anEq[] etc. */
dana3d0c132015-03-14 18:59:58 +00001026 int i; /* Index of first sample >= pRec */
1027 int iSample; /* Smallest sample larger than or equal to pRec */
dan84c309b2013-08-08 16:17:12 +00001028 int iMin = 0; /* Smallest sample not yet tested */
dan84c309b2013-08-08 16:17:12 +00001029 int iTest; /* Next sample to test */
1030 int res; /* Result of comparison operation */
dana3d0c132015-03-14 18:59:58 +00001031 int nField; /* Number of fields in pRec */
1032 tRowcnt iLower = 0; /* anLt[] + anEq[] of largest sample pRec is > */
dan02fa4692009-08-17 17:06:58 +00001033
drh4f991892013-10-11 15:05:05 +00001034#ifndef SQLITE_DEBUG
1035 UNUSED_PARAMETER( pParse );
1036#endif
drh7f594752013-12-03 19:49:55 +00001037 assert( pRec!=0 );
drh5c624862011-09-22 18:46:34 +00001038 assert( pIdx->nSample>0 );
dana3d0c132015-03-14 18:59:58 +00001039 assert( pRec->nField>0 && pRec->nField<=pIdx->nSampleCol );
1040
1041 /* Do a binary search to find the first sample greater than or equal
1042 ** to pRec. If pRec contains a single field, the set of samples to search
1043 ** is simply the aSample[] array. If the samples in aSample[] contain more
1044 ** than one fields, all fields following the first are ignored.
1045 **
1046 ** If pRec contains N fields, where N is more than one, then as well as the
1047 ** samples in aSample[] (truncated to N fields), the search also has to
1048 ** consider prefixes of those samples. For example, if the set of samples
1049 ** in aSample is:
1050 **
1051 ** aSample[0] = (a, 5)
1052 ** aSample[1] = (a, 10)
1053 ** aSample[2] = (b, 5)
1054 ** aSample[3] = (c, 100)
1055 ** aSample[4] = (c, 105)
1056 **
1057 ** Then the search space should ideally be the samples above and the
1058 ** unique prefixes [a], [b] and [c]. But since that is hard to organize,
1059 ** the code actually searches this set:
1060 **
1061 ** 0: (a)
1062 ** 1: (a, 5)
1063 ** 2: (a, 10)
1064 ** 3: (a, 10)
1065 ** 4: (b)
1066 ** 5: (b, 5)
1067 ** 6: (c)
1068 ** 7: (c, 100)
1069 ** 8: (c, 105)
1070 ** 9: (c, 105)
1071 **
1072 ** For each sample in the aSample[] array, N samples are present in the
1073 ** effective sample array. In the above, samples 0 and 1 are based on
1074 ** sample aSample[0]. Samples 2 and 3 on aSample[1] etc.
1075 **
1076 ** Often, sample i of each block of N effective samples has (i+1) fields.
1077 ** Except, each sample may be extended to ensure that it is greater than or
1078 ** equal to the previous sample in the array. For example, in the above,
1079 ** sample 2 is the first sample of a block of N samples, so at first it
1080 ** appears that it should be 1 field in size. However, that would make it
1081 ** smaller than sample 1, so the binary search would not work. As a result,
1082 ** it is extended to two fields. The duplicates that this creates do not
1083 ** cause any problems.
1084 */
1085 nField = pRec->nField;
1086 iCol = 0;
1087 iSample = pIdx->nSample * nField;
dan84c309b2013-08-08 16:17:12 +00001088 do{
dana3d0c132015-03-14 18:59:58 +00001089 int iSamp; /* Index in aSample[] of test sample */
1090 int n; /* Number of fields in test sample */
1091
1092 iTest = (iMin+iSample)/2;
1093 iSamp = iTest / nField;
1094 if( iSamp>0 ){
1095 /* The proposed effective sample is a prefix of sample aSample[iSamp].
1096 ** Specifically, the shortest prefix of at least (1 + iTest%nField)
1097 ** fields that is greater than the previous effective sample. */
1098 for(n=(iTest % nField) + 1; n<nField; n++){
1099 if( aSample[iSamp-1].anLt[n-1]!=aSample[iSamp].anLt[n-1] ) break;
1100 }
dan84c309b2013-08-08 16:17:12 +00001101 }else{
dana3d0c132015-03-14 18:59:58 +00001102 n = iTest + 1;
dan02fa4692009-08-17 17:06:58 +00001103 }
dana3d0c132015-03-14 18:59:58 +00001104
1105 pRec->nField = n;
1106 res = sqlite3VdbeRecordCompare(aSample[iSamp].n, aSample[iSamp].p, pRec);
1107 if( res<0 ){
1108 iLower = aSample[iSamp].anLt[n-1] + aSample[iSamp].anEq[n-1];
1109 iMin = iTest+1;
1110 }else if( res==0 && n<nField ){
1111 iLower = aSample[iSamp].anLt[n-1];
1112 iMin = iTest+1;
1113 res = -1;
1114 }else{
1115 iSample = iTest;
1116 iCol = n-1;
1117 }
1118 }while( res && iMin<iSample );
1119 i = iSample / nField;
drh51147ba2005-07-23 22:59:55 +00001120
dan84c309b2013-08-08 16:17:12 +00001121#ifdef SQLITE_DEBUG
1122 /* The following assert statements check that the binary search code
1123 ** above found the right answer. This block serves no purpose other
1124 ** than to invoke the asserts. */
dana3d0c132015-03-14 18:59:58 +00001125 if( pParse->db->mallocFailed==0 ){
1126 if( res==0 ){
1127 /* If (res==0) is true, then pRec must be equal to sample i. */
1128 assert( i<pIdx->nSample );
1129 assert( iCol==nField-1 );
1130 pRec->nField = nField;
1131 assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)
1132 || pParse->db->mallocFailed
1133 );
1134 }else{
1135 /* Unless i==pIdx->nSample, indicating that pRec is larger than
1136 ** all samples in the aSample[] array, pRec must be smaller than the
1137 ** (iCol+1) field prefix of sample i. */
1138 assert( i<=pIdx->nSample && i>=0 );
1139 pRec->nField = iCol+1;
1140 assert( i==pIdx->nSample
1141 || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)>0
1142 || pParse->db->mallocFailed );
1143
1144 /* if i==0 and iCol==0, then record pRec is smaller than all samples
1145 ** in the aSample[] array. Otherwise, if (iCol>0) then pRec must
1146 ** be greater than or equal to the (iCol) field prefix of sample i.
1147 ** If (i>0), then pRec must also be greater than sample (i-1). */
1148 if( iCol>0 ){
1149 pRec->nField = iCol;
1150 assert( sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)<=0
1151 || pParse->db->mallocFailed );
1152 }
1153 if( i>0 ){
1154 pRec->nField = nField;
1155 assert( sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec)<0
1156 || pParse->db->mallocFailed );
1157 }
1158 }
drhfaacf172011-08-12 01:51:45 +00001159 }
dan84c309b2013-08-08 16:17:12 +00001160#endif /* ifdef SQLITE_DEBUG */
dan02fa4692009-08-17 17:06:58 +00001161
dan84c309b2013-08-08 16:17:12 +00001162 if( res==0 ){
dana3d0c132015-03-14 18:59:58 +00001163 /* Record pRec is equal to sample i */
1164 assert( iCol==nField-1 );
daneea568d2013-08-07 19:46:15 +00001165 aStat[0] = aSample[i].anLt[iCol];
1166 aStat[1] = aSample[i].anEq[iCol];
drhfaacf172011-08-12 01:51:45 +00001167 }else{
dana3d0c132015-03-14 18:59:58 +00001168 /* At this point, the (iCol+1) field prefix of aSample[i] is the first
1169 ** sample that is greater than pRec. Or, if i==pIdx->nSample then pRec
1170 ** is larger than all samples in the array. */
1171 tRowcnt iUpper, iGap;
1172 if( i>=pIdx->nSample ){
1173 iUpper = sqlite3LogEstToInt(pIdx->aiRowLogEst[0]);
drhfaacf172011-08-12 01:51:45 +00001174 }else{
dana3d0c132015-03-14 18:59:58 +00001175 iUpper = aSample[i].anLt[iCol];
drhfaacf172011-08-12 01:51:45 +00001176 }
dana3d0c132015-03-14 18:59:58 +00001177
drhfaacf172011-08-12 01:51:45 +00001178 if( iLower>=iUpper ){
1179 iGap = 0;
1180 }else{
1181 iGap = iUpper - iLower;
drhfaacf172011-08-12 01:51:45 +00001182 }
1183 if( roundUp ){
1184 iGap = (iGap*2)/3;
1185 }else{
1186 iGap = iGap/3;
1187 }
1188 aStat[0] = iLower + iGap;
dana3d0c132015-03-14 18:59:58 +00001189 aStat[1] = pIdx->aAvgEq[iCol];
dan02fa4692009-08-17 17:06:58 +00001190 }
dana3d0c132015-03-14 18:59:58 +00001191
1192 /* Restore the pRec->nField value before returning. */
1193 pRec->nField = nField;
drh6d3f91d2014-11-05 19:26:12 +00001194 return i;
dan02fa4692009-08-17 17:06:58 +00001195}
drh1435a9a2013-08-27 23:15:44 +00001196#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */
dan937d0de2009-10-15 18:35:38 +00001197
1198/*
danaa9933c2014-04-24 20:04:49 +00001199** If it is not NULL, pTerm is a term that provides an upper or lower
1200** bound on a range scan. Without considering pTerm, it is estimated
1201** that the scan will visit nNew rows. This function returns the number
1202** estimated to be visited after taking pTerm into account.
1203**
1204** If the user explicitly specified a likelihood() value for this term,
1205** then the return value is the likelihood multiplied by the number of
1206** input rows. Otherwise, this function assumes that an "IS NOT NULL" term
1207** has a likelihood of 0.50, and any other term a likelihood of 0.25.
1208*/
1209static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){
1210 LogEst nRet = nNew;
1211 if( pTerm ){
1212 if( pTerm->truthProb<=0 ){
1213 nRet += pTerm->truthProb;
dan7de2a1f2014-04-28 20:11:20 +00001214 }else if( (pTerm->wtFlags & TERM_VNULL)==0 ){
danaa9933c2014-04-24 20:04:49 +00001215 nRet -= 20; assert( 20==sqlite3LogEst(4) );
1216 }
1217 }
1218 return nRet;
1219}
1220
drh567cc1e2015-08-25 19:42:28 +00001221
1222#ifdef SQLITE_ENABLE_STAT3_OR_STAT4
1223/*
1224** Return the affinity for a single column of an index.
1225*/
dand66e5792016-08-03 16:14:33 +00001226char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCol){
drh8ffddeb2015-09-25 01:09:27 +00001227 assert( iCol>=0 && iCol<pIdx->nColumn );
drh567cc1e2015-08-25 19:42:28 +00001228 if( !pIdx->zColAff ){
1229 if( sqlite3IndexAffinityStr(db, pIdx)==0 ) return SQLITE_AFF_BLOB;
1230 }
1231 return pIdx->zColAff[iCol];
1232}
1233#endif
1234
1235
mistachkin2d84ac42014-06-26 21:32:09 +00001236#ifdef SQLITE_ENABLE_STAT3_OR_STAT4
danb0b82902014-06-26 20:21:46 +00001237/*
1238** This function is called to estimate the number of rows visited by a
1239** range-scan on a skip-scan index. For example:
1240**
1241** CREATE INDEX i1 ON t1(a, b, c);
1242** SELECT * FROM t1 WHERE a=? AND c BETWEEN ? AND ?;
1243**
1244** Value pLoop->nOut is currently set to the estimated number of rows
1245** visited for scanning (a=? AND b=?). This function reduces that estimate
1246** by some factor to account for the (c BETWEEN ? AND ?) expression based
1247** on the stat4 data for the index. this scan will be peformed multiple
1248** times (once for each (a,b) combination that matches a=?) is dealt with
1249** by the caller.
1250**
1251** It does this by scanning through all stat4 samples, comparing values
1252** extracted from pLower and pUpper with the corresponding column in each
1253** sample. If L and U are the number of samples found to be less than or
1254** equal to the values extracted from pLower and pUpper respectively, and
1255** N is the total number of samples, the pLoop->nOut value is adjusted
1256** as follows:
1257**
1258** nOut = nOut * ( min(U - L, 1) / N )
1259**
1260** If pLower is NULL, or a value cannot be extracted from the term, L is
1261** set to zero. If pUpper is NULL, or a value cannot be extracted from it,
1262** U is set to N.
1263**
1264** Normally, this function sets *pbDone to 1 before returning. However,
1265** if no value can be extracted from either pLower or pUpper (and so the
1266** estimate of the number of rows delivered remains unchanged), *pbDone
1267** is left as is.
1268**
1269** If an error occurs, an SQLite error code is returned. Otherwise,
1270** SQLITE_OK.
1271*/
1272static int whereRangeSkipScanEst(
1273 Parse *pParse, /* Parsing & code generating context */
1274 WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */
1275 WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */
1276 WhereLoop *pLoop, /* Update the .nOut value of this loop */
1277 int *pbDone /* Set to true if at least one expr. value extracted */
1278){
1279 Index *p = pLoop->u.btree.pIndex;
1280 int nEq = pLoop->u.btree.nEq;
1281 sqlite3 *db = pParse->db;
dan4e42ba42014-06-27 20:14:25 +00001282 int nLower = -1;
1283 int nUpper = p->nSample+1;
danb0b82902014-06-26 20:21:46 +00001284 int rc = SQLITE_OK;
drh8ffddeb2015-09-25 01:09:27 +00001285 u8 aff = sqlite3IndexColumnAffinity(db, p, nEq);
danb0b82902014-06-26 20:21:46 +00001286 CollSeq *pColl;
1287
1288 sqlite3_value *p1 = 0; /* Value extracted from pLower */
1289 sqlite3_value *p2 = 0; /* Value extracted from pUpper */
1290 sqlite3_value *pVal = 0; /* Value extracted from record */
1291
1292 pColl = sqlite3LocateCollSeq(pParse, p->azColl[nEq]);
1293 if( pLower ){
1294 rc = sqlite3Stat4ValueFromExpr(pParse, pLower->pExpr->pRight, aff, &p1);
dan4e42ba42014-06-27 20:14:25 +00001295 nLower = 0;
danb0b82902014-06-26 20:21:46 +00001296 }
1297 if( pUpper && rc==SQLITE_OK ){
1298 rc = sqlite3Stat4ValueFromExpr(pParse, pUpper->pExpr->pRight, aff, &p2);
dan4e42ba42014-06-27 20:14:25 +00001299 nUpper = p2 ? 0 : p->nSample;
danb0b82902014-06-26 20:21:46 +00001300 }
1301
1302 if( p1 || p2 ){
1303 int i;
1304 int nDiff;
1305 for(i=0; rc==SQLITE_OK && i<p->nSample; i++){
1306 rc = sqlite3Stat4Column(db, p->aSample[i].p, p->aSample[i].n, nEq, &pVal);
1307 if( rc==SQLITE_OK && p1 ){
1308 int res = sqlite3MemCompare(p1, pVal, pColl);
dan4e42ba42014-06-27 20:14:25 +00001309 if( res>=0 ) nLower++;
danb0b82902014-06-26 20:21:46 +00001310 }
1311 if( rc==SQLITE_OK && p2 ){
1312 int res = sqlite3MemCompare(p2, pVal, pColl);
dan4e42ba42014-06-27 20:14:25 +00001313 if( res>=0 ) nUpper++;
danb0b82902014-06-26 20:21:46 +00001314 }
1315 }
danb0b82902014-06-26 20:21:46 +00001316 nDiff = (nUpper - nLower);
1317 if( nDiff<=0 ) nDiff = 1;
dan4e42ba42014-06-27 20:14:25 +00001318
1319 /* If there is both an upper and lower bound specified, and the
1320 ** comparisons indicate that they are close together, use the fallback
1321 ** method (assume that the scan visits 1/64 of the rows) for estimating
1322 ** the number of rows visited. Otherwise, estimate the number of rows
1323 ** using the method described in the header comment for this function. */
1324 if( nDiff!=1 || pUpper==0 || pLower==0 ){
1325 int nAdjust = (sqlite3LogEst(p->nSample) - sqlite3LogEst(nDiff));
1326 pLoop->nOut -= nAdjust;
1327 *pbDone = 1;
1328 WHERETRACE(0x10, ("range skip-scan regions: %u..%u adjust=%d est=%d\n",
danfa887452014-06-28 15:26:10 +00001329 nLower, nUpper, nAdjust*-1, pLoop->nOut));
dan4e42ba42014-06-27 20:14:25 +00001330 }
1331
danb0b82902014-06-26 20:21:46 +00001332 }else{
1333 assert( *pbDone==0 );
1334 }
1335
1336 sqlite3ValueFree(p1);
1337 sqlite3ValueFree(p2);
1338 sqlite3ValueFree(pVal);
1339
1340 return rc;
1341}
mistachkin2d84ac42014-06-26 21:32:09 +00001342#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */
danb0b82902014-06-26 20:21:46 +00001343
danaa9933c2014-04-24 20:04:49 +00001344/*
dan02fa4692009-08-17 17:06:58 +00001345** This function is used to estimate the number of rows that will be visited
1346** by scanning an index for a range of values. The range may have an upper
1347** bound, a lower bound, or both. The WHERE clause terms that set the upper
1348** and lower bounds are represented by pLower and pUpper respectively. For
1349** example, assuming that index p is on t1(a):
1350**
1351** ... FROM t1 WHERE a > ? AND a < ? ...
1352** |_____| |_____|
1353** | |
1354** pLower pUpper
1355**
drh98cdf622009-08-20 18:14:42 +00001356** If either of the upper or lower bound is not present, then NULL is passed in
drhcdaca552009-08-20 13:45:07 +00001357** place of the corresponding WhereTerm.
dan02fa4692009-08-17 17:06:58 +00001358**
drh6d3f91d2014-11-05 19:26:12 +00001359** The value in (pBuilder->pNew->u.btree.nEq) is the number of the index
dan6cb8d762013-08-08 11:48:57 +00001360** column subject to the range constraint. Or, equivalently, the number of
1361** equality constraints optimized by the proposed index scan. For example,
1362** assuming index p is on t1(a, b), and the SQL query is:
dan02fa4692009-08-17 17:06:58 +00001363**
1364** ... FROM t1 WHERE a = ? AND b > ? AND b < ? ...
1365**
dan6cb8d762013-08-08 11:48:57 +00001366** then nEq is set to 1 (as the range restricted column, b, is the second
1367** left-most column of the index). Or, if the query is:
dan02fa4692009-08-17 17:06:58 +00001368**
1369** ... FROM t1 WHERE a > ? AND a < ? ...
1370**
dan6cb8d762013-08-08 11:48:57 +00001371** then nEq is set to 0.
dan02fa4692009-08-17 17:06:58 +00001372**
drhbf539c42013-10-05 18:16:02 +00001373** When this function is called, *pnOut is set to the sqlite3LogEst() of the
dan6cb8d762013-08-08 11:48:57 +00001374** number of rows that the index scan is expected to visit without
drh6d3f91d2014-11-05 19:26:12 +00001375** considering the range constraints. If nEq is 0, then *pnOut is the number of
dan6cb8d762013-08-08 11:48:57 +00001376** rows in the index. Assuming no error occurs, *pnOut is adjusted (reduced)
peter.d.reid60ec9142014-09-06 16:39:46 +00001377** to account for the range constraints pLower and pUpper.
dan6cb8d762013-08-08 11:48:57 +00001378**
1379** In the absence of sqlite_stat4 ANALYZE data, or if such data cannot be
drh94aa7e02014-06-06 17:09:52 +00001380** used, a single range inequality reduces the search space by a factor of 4.
1381** and a pair of constraints (x>? AND x<?) reduces the expected number of
1382** rows visited by a factor of 64.
dan02fa4692009-08-17 17:06:58 +00001383*/
1384static int whereRangeScanEst(
drhcdaca552009-08-20 13:45:07 +00001385 Parse *pParse, /* Parsing & code generating context */
dan7a419232013-08-06 20:01:43 +00001386 WhereLoopBuilder *pBuilder,
drhcdaca552009-08-20 13:45:07 +00001387 WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */
1388 WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */
drh186ad8c2013-10-08 18:40:37 +00001389 WhereLoop *pLoop /* Modify the .nOut and maybe .rRun fields */
dan02fa4692009-08-17 17:06:58 +00001390){
dan69188d92009-08-19 08:18:32 +00001391 int rc = SQLITE_OK;
drh186ad8c2013-10-08 18:40:37 +00001392 int nOut = pLoop->nOut;
drhbf539c42013-10-05 18:16:02 +00001393 LogEst nNew;
dan69188d92009-08-19 08:18:32 +00001394
drh1435a9a2013-08-27 23:15:44 +00001395#ifdef SQLITE_ENABLE_STAT3_OR_STAT4
drh186ad8c2013-10-08 18:40:37 +00001396 Index *p = pLoop->u.btree.pIndex;
drh4f991892013-10-11 15:05:05 +00001397 int nEq = pLoop->u.btree.nEq;
dan02fa4692009-08-17 17:06:58 +00001398
drh6d3f91d2014-11-05 19:26:12 +00001399 if( p->nSample>0 && nEq<p->nSampleCol ){
danb0b82902014-06-26 20:21:46 +00001400 if( nEq==pBuilder->nRecValid ){
1401 UnpackedRecord *pRec = pBuilder->pRec;
1402 tRowcnt a[2];
dand66e5792016-08-03 16:14:33 +00001403 int nBtm = pLoop->u.btree.nBtm;
1404 int nTop = pLoop->u.btree.nTop;
drh98cdf622009-08-20 18:14:42 +00001405
danb0b82902014-06-26 20:21:46 +00001406 /* Variable iLower will be set to the estimate of the number of rows in
1407 ** the index that are less than the lower bound of the range query. The
1408 ** lower bound being the concatenation of $P and $L, where $P is the
1409 ** key-prefix formed by the nEq values matched against the nEq left-most
1410 ** columns of the index, and $L is the value in pLower.
1411 **
1412 ** Or, if pLower is NULL or $L cannot be extracted from it (because it
1413 ** is not a simple variable or literal value), the lower bound of the
1414 ** range is $P. Due to a quirk in the way whereKeyStats() works, even
1415 ** if $L is available, whereKeyStats() is called for both ($P) and
drh6d3f91d2014-11-05 19:26:12 +00001416 ** ($P:$L) and the larger of the two returned values is used.
danb0b82902014-06-26 20:21:46 +00001417 **
1418 ** Similarly, iUpper is to be set to the estimate of the number of rows
1419 ** less than the upper bound of the range query. Where the upper bound
1420 ** is either ($P) or ($P:$U). Again, even if $U is available, both values
1421 ** of iUpper are requested of whereKeyStats() and the smaller used.
drh6d3f91d2014-11-05 19:26:12 +00001422 **
1423 ** The number of rows between the two bounds is then just iUpper-iLower.
danb0b82902014-06-26 20:21:46 +00001424 */
drh6d3f91d2014-11-05 19:26:12 +00001425 tRowcnt iLower; /* Rows less than the lower bound */
1426 tRowcnt iUpper; /* Rows less than the upper bound */
1427 int iLwrIdx = -2; /* aSample[] for the lower bound */
1428 int iUprIdx = -1; /* aSample[] for the upper bound */
danb3c02e22013-08-08 19:38:40 +00001429
drhb34fc5b2014-08-28 17:20:37 +00001430 if( pRec ){
1431 testcase( pRec->nField!=pBuilder->nRecValid );
1432 pRec->nField = pBuilder->nRecValid;
1433 }
danb0b82902014-06-26 20:21:46 +00001434 /* Determine iLower and iUpper using ($P) only. */
1435 if( nEq==0 ){
1436 iLower = 0;
drh9f07cf72014-10-22 15:27:05 +00001437 iUpper = p->nRowEst0;
danb0b82902014-06-26 20:21:46 +00001438 }else{
1439 /* Note: this call could be optimized away - since the same values must
1440 ** have been requested when testing key $P in whereEqualScanEst(). */
1441 whereKeyStats(pParse, p, pRec, 0, a);
1442 iLower = a[0];
1443 iUpper = a[0] + a[1];
dan6cb8d762013-08-08 11:48:57 +00001444 }
danb0b82902014-06-26 20:21:46 +00001445
drh69afd992014-10-08 02:53:25 +00001446 assert( pLower==0 || (pLower->eOperator & (WO_GT|WO_GE))!=0 );
1447 assert( pUpper==0 || (pUpper->eOperator & (WO_LT|WO_LE))!=0 );
drh681fca02014-10-10 15:01:46 +00001448 assert( p->aSortOrder!=0 );
1449 if( p->aSortOrder[nEq] ){
drh69afd992014-10-08 02:53:25 +00001450 /* The roles of pLower and pUpper are swapped for a DESC index */
1451 SWAP(WhereTerm*, pLower, pUpper);
dand66e5792016-08-03 16:14:33 +00001452 SWAP(int, nBtm, nTop);
drh69afd992014-10-08 02:53:25 +00001453 }
1454
danb0b82902014-06-26 20:21:46 +00001455 /* If possible, improve on the iLower estimate using ($P:$L). */
1456 if( pLower ){
dand66e5792016-08-03 16:14:33 +00001457 int n; /* Values extracted from pExpr */
danb0b82902014-06-26 20:21:46 +00001458 Expr *pExpr = pLower->pExpr->pRight;
dand66e5792016-08-03 16:14:33 +00001459 rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nBtm, nEq, &n);
1460 if( rc==SQLITE_OK && n ){
danb0b82902014-06-26 20:21:46 +00001461 tRowcnt iNew;
dand66e5792016-08-03 16:14:33 +00001462 u16 mask = WO_GT|WO_LE;
1463 if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT);
drh6d3f91d2014-11-05 19:26:12 +00001464 iLwrIdx = whereKeyStats(pParse, p, pRec, 0, a);
dand66e5792016-08-03 16:14:33 +00001465 iNew = a[0] + ((pLower->eOperator & mask) ? a[1] : 0);
danb0b82902014-06-26 20:21:46 +00001466 if( iNew>iLower ) iLower = iNew;
1467 nOut--;
danf741e042014-08-25 18:29:38 +00001468 pLower = 0;
danb0b82902014-06-26 20:21:46 +00001469 }
1470 }
1471
1472 /* If possible, improve on the iUpper estimate using ($P:$U). */
1473 if( pUpper ){
dand66e5792016-08-03 16:14:33 +00001474 int n; /* Values extracted from pExpr */
danb0b82902014-06-26 20:21:46 +00001475 Expr *pExpr = pUpper->pExpr->pRight;
dand66e5792016-08-03 16:14:33 +00001476 rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nTop, nEq, &n);
1477 if( rc==SQLITE_OK && n ){
danb0b82902014-06-26 20:21:46 +00001478 tRowcnt iNew;
dand66e5792016-08-03 16:14:33 +00001479 u16 mask = WO_GT|WO_LE;
1480 if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT);
drh6d3f91d2014-11-05 19:26:12 +00001481 iUprIdx = whereKeyStats(pParse, p, pRec, 1, a);
dand66e5792016-08-03 16:14:33 +00001482 iNew = a[0] + ((pUpper->eOperator & mask) ? a[1] : 0);
danb0b82902014-06-26 20:21:46 +00001483 if( iNew<iUpper ) iUpper = iNew;
1484 nOut--;
danf741e042014-08-25 18:29:38 +00001485 pUpper = 0;
danb0b82902014-06-26 20:21:46 +00001486 }
1487 }
1488
1489 pBuilder->pRec = pRec;
1490 if( rc==SQLITE_OK ){
1491 if( iUpper>iLower ){
1492 nNew = sqlite3LogEst(iUpper - iLower);
drh6d3f91d2014-11-05 19:26:12 +00001493 /* TUNING: If both iUpper and iLower are derived from the same
1494 ** sample, then assume they are 4x more selective. This brings
1495 ** the estimated selectivity more in line with what it would be
1496 ** if estimated without the use of STAT3/4 tables. */
1497 if( iLwrIdx==iUprIdx ) nNew -= 20; assert( 20==sqlite3LogEst(4) );
danb0b82902014-06-26 20:21:46 +00001498 }else{
1499 nNew = 10; assert( 10==sqlite3LogEst(2) );
1500 }
1501 if( nNew<nOut ){
1502 nOut = nNew;
1503 }
drhae914d72014-08-28 19:38:22 +00001504 WHERETRACE(0x10, ("STAT4 range scan: %u..%u est=%d\n",
danb0b82902014-06-26 20:21:46 +00001505 (u32)iLower, (u32)iUpper, nOut));
danb0b82902014-06-26 20:21:46 +00001506 }
1507 }else{
1508 int bDone = 0;
1509 rc = whereRangeSkipScanEst(pParse, pLower, pUpper, pLoop, &bDone);
1510 if( bDone ) return rc;
drh98cdf622009-08-20 18:14:42 +00001511 }
dan02fa4692009-08-17 17:06:58 +00001512 }
drh3f022182009-09-09 16:10:50 +00001513#else
1514 UNUSED_PARAMETER(pParse);
dan7a419232013-08-06 20:01:43 +00001515 UNUSED_PARAMETER(pBuilder);
dan02fa4692009-08-17 17:06:58 +00001516 assert( pLower || pUpper );
danf741e042014-08-25 18:29:38 +00001517#endif
dan7de2a1f2014-04-28 20:11:20 +00001518 assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 );
danaa9933c2014-04-24 20:04:49 +00001519 nNew = whereRangeAdjust(pLower, nOut);
1520 nNew = whereRangeAdjust(pUpper, nNew);
dan7de2a1f2014-04-28 20:11:20 +00001521
drh4dd96a82014-10-24 15:26:29 +00001522 /* TUNING: If there is both an upper and lower limit and neither limit
1523 ** has an application-defined likelihood(), assume the range is
dan42685f22014-04-28 19:34:06 +00001524 ** reduced by an additional 75%. This means that, by default, an open-ended
1525 ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the
1526 ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to
1527 ** match 1/64 of the index. */
drh4dd96a82014-10-24 15:26:29 +00001528 if( pLower && pLower->truthProb>0 && pUpper && pUpper->truthProb>0 ){
1529 nNew -= 20;
1530 }
dan7de2a1f2014-04-28 20:11:20 +00001531
danaa9933c2014-04-24 20:04:49 +00001532 nOut -= (pLower!=0) + (pUpper!=0);
drhabfa6d52013-09-11 03:53:22 +00001533 if( nNew<10 ) nNew = 10;
1534 if( nNew<nOut ) nOut = nNew;
drhae914d72014-08-28 19:38:22 +00001535#if defined(WHERETRACE_ENABLED)
1536 if( pLoop->nOut>nOut ){
1537 WHERETRACE(0x10,("Range scan lowers nOut from %d to %d\n",
1538 pLoop->nOut, nOut));
1539 }
1540#endif
drh186ad8c2013-10-08 18:40:37 +00001541 pLoop->nOut = (LogEst)nOut;
dan02fa4692009-08-17 17:06:58 +00001542 return rc;
1543}
1544
drh1435a9a2013-08-27 23:15:44 +00001545#ifdef SQLITE_ENABLE_STAT3_OR_STAT4
drh82759752011-01-20 16:52:09 +00001546/*
1547** Estimate the number of rows that will be returned based on
1548** an equality constraint x=VALUE and where that VALUE occurs in
1549** the histogram data. This only works when x is the left-most
drhfaacf172011-08-12 01:51:45 +00001550** column of an index and sqlite_stat3 histogram data is available
drhac8eb112011-03-17 01:58:21 +00001551** for that index. When pExpr==NULL that means the constraint is
1552** "x IS NULL" instead of "x=VALUE".
drh82759752011-01-20 16:52:09 +00001553**
drh0c50fa02011-01-21 16:27:18 +00001554** Write the estimated row count into *pnRow and return SQLITE_OK.
1555** If unable to make an estimate, leave *pnRow unchanged and return
1556** non-zero.
drh9b3eb0a2011-01-21 14:37:04 +00001557**
1558** This routine can fail if it is unable to load a collating sequence
1559** required for string comparison, or if unable to allocate memory
1560** for a UTF conversion required for comparison. The error is stored
1561** in the pParse structure.
drh82759752011-01-20 16:52:09 +00001562*/
drh041e09f2011-04-07 19:56:21 +00001563static int whereEqualScanEst(
drh82759752011-01-20 16:52:09 +00001564 Parse *pParse, /* Parsing & code generating context */
dan7a419232013-08-06 20:01:43 +00001565 WhereLoopBuilder *pBuilder,
drh0c50fa02011-01-21 16:27:18 +00001566 Expr *pExpr, /* Expression for VALUE in the x=VALUE constraint */
drhb8a8e8a2013-06-10 19:12:39 +00001567 tRowcnt *pnRow /* Write the revised row estimate here */
drh82759752011-01-20 16:52:09 +00001568){
dan7a419232013-08-06 20:01:43 +00001569 Index *p = pBuilder->pNew->u.btree.pIndex;
1570 int nEq = pBuilder->pNew->u.btree.nEq;
1571 UnpackedRecord *pRec = pBuilder->pRec;
drh82759752011-01-20 16:52:09 +00001572 int rc; /* Subfunction return code */
drhfaacf172011-08-12 01:51:45 +00001573 tRowcnt a[2]; /* Statistics */
dan7a419232013-08-06 20:01:43 +00001574 int bOk;
drh82759752011-01-20 16:52:09 +00001575
dan7a419232013-08-06 20:01:43 +00001576 assert( nEq>=1 );
danfd984b82014-06-30 18:02:20 +00001577 assert( nEq<=p->nColumn );
drh82759752011-01-20 16:52:09 +00001578 assert( p->aSample!=0 );
drh5c624862011-09-22 18:46:34 +00001579 assert( p->nSample>0 );
dan7a419232013-08-06 20:01:43 +00001580 assert( pBuilder->nRecValid<nEq );
1581
1582 /* If values are not available for all fields of the index to the left
1583 ** of this one, no estimate can be made. Return SQLITE_NOTFOUND. */
1584 if( pBuilder->nRecValid<(nEq-1) ){
1585 return SQLITE_NOTFOUND;
drh1f9c7662011-03-17 01:34:26 +00001586 }
dan7a419232013-08-06 20:01:43 +00001587
dandd6e1f12013-08-10 19:08:30 +00001588 /* This is an optimization only. The call to sqlite3Stat4ProbeSetValue()
1589 ** below would return the same value. */
danfd984b82014-06-30 18:02:20 +00001590 if( nEq>=p->nColumn ){
dan7a419232013-08-06 20:01:43 +00001591 *pnRow = 1;
1592 return SQLITE_OK;
drh82759752011-01-20 16:52:09 +00001593 }
dan7a419232013-08-06 20:01:43 +00001594
dand66e5792016-08-03 16:14:33 +00001595 rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, 1, nEq-1, &bOk);
dan87cd9322013-08-07 15:52:41 +00001596 pBuilder->pRec = pRec;
dan7a419232013-08-06 20:01:43 +00001597 if( rc!=SQLITE_OK ) return rc;
1598 if( bOk==0 ) return SQLITE_NOTFOUND;
dan7a419232013-08-06 20:01:43 +00001599 pBuilder->nRecValid = nEq;
dan7a419232013-08-06 20:01:43 +00001600
danb3c02e22013-08-08 19:38:40 +00001601 whereKeyStats(pParse, p, pRec, 0, a);
drh4fb48e42016-03-01 22:41:27 +00001602 WHERETRACE(0x10,("equality scan regions %s(%d): %d\n",
1603 p->zName, nEq-1, (int)a[1]));
danb3c02e22013-08-08 19:38:40 +00001604 *pnRow = a[1];
daneea568d2013-08-07 19:46:15 +00001605
drh0c50fa02011-01-21 16:27:18 +00001606 return rc;
1607}
drh1435a9a2013-08-27 23:15:44 +00001608#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */
drh0c50fa02011-01-21 16:27:18 +00001609
drh1435a9a2013-08-27 23:15:44 +00001610#ifdef SQLITE_ENABLE_STAT3_OR_STAT4
drh0c50fa02011-01-21 16:27:18 +00001611/*
1612** Estimate the number of rows that will be returned based on
drh5ac06072011-01-21 18:18:13 +00001613** an IN constraint where the right-hand side of the IN operator
1614** is a list of values. Example:
1615**
1616** WHERE x IN (1,2,3,4)
drh0c50fa02011-01-21 16:27:18 +00001617**
1618** Write the estimated row count into *pnRow and return SQLITE_OK.
1619** If unable to make an estimate, leave *pnRow unchanged and return
1620** non-zero.
1621**
1622** This routine can fail if it is unable to load a collating sequence
1623** required for string comparison, or if unable to allocate memory
1624** for a UTF conversion required for comparison. The error is stored
1625** in the pParse structure.
1626*/
drh041e09f2011-04-07 19:56:21 +00001627static int whereInScanEst(
drh0c50fa02011-01-21 16:27:18 +00001628 Parse *pParse, /* Parsing & code generating context */
dan7a419232013-08-06 20:01:43 +00001629 WhereLoopBuilder *pBuilder,
drh0c50fa02011-01-21 16:27:18 +00001630 ExprList *pList, /* The value list on the RHS of "x IN (v1,v2,v3,...)" */
drhb8a8e8a2013-06-10 19:12:39 +00001631 tRowcnt *pnRow /* Write the revised row estimate here */
drh0c50fa02011-01-21 16:27:18 +00001632){
dan7a419232013-08-06 20:01:43 +00001633 Index *p = pBuilder->pNew->u.btree.pIndex;
dancfc9df72014-04-25 15:01:01 +00001634 i64 nRow0 = sqlite3LogEstToInt(p->aiRowLogEst[0]);
dan7a419232013-08-06 20:01:43 +00001635 int nRecValid = pBuilder->nRecValid;
drhb8a8e8a2013-06-10 19:12:39 +00001636 int rc = SQLITE_OK; /* Subfunction return code */
1637 tRowcnt nEst; /* Number of rows for a single term */
1638 tRowcnt nRowEst = 0; /* New estimate of the number of rows */
1639 int i; /* Loop counter */
drh0c50fa02011-01-21 16:27:18 +00001640
1641 assert( p->aSample!=0 );
drhfaacf172011-08-12 01:51:45 +00001642 for(i=0; rc==SQLITE_OK && i<pList->nExpr; i++){
dancfc9df72014-04-25 15:01:01 +00001643 nEst = nRow0;
dan7a419232013-08-06 20:01:43 +00001644 rc = whereEqualScanEst(pParse, pBuilder, pList->a[i].pExpr, &nEst);
drhfaacf172011-08-12 01:51:45 +00001645 nRowEst += nEst;
dan7a419232013-08-06 20:01:43 +00001646 pBuilder->nRecValid = nRecValid;
drh0c50fa02011-01-21 16:27:18 +00001647 }
dan7a419232013-08-06 20:01:43 +00001648
drh0c50fa02011-01-21 16:27:18 +00001649 if( rc==SQLITE_OK ){
dancfc9df72014-04-25 15:01:01 +00001650 if( nRowEst > nRow0 ) nRowEst = nRow0;
drh0c50fa02011-01-21 16:27:18 +00001651 *pnRow = nRowEst;
drh5418b122014-08-28 13:42:13 +00001652 WHERETRACE(0x10,("IN row estimate: est=%d\n", nRowEst));
drh0c50fa02011-01-21 16:27:18 +00001653 }
dan7a419232013-08-06 20:01:43 +00001654 assert( pBuilder->nRecValid==nRecValid );
drh0c50fa02011-01-21 16:27:18 +00001655 return rc;
drh82759752011-01-20 16:52:09 +00001656}
drh1435a9a2013-08-27 23:15:44 +00001657#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */
drh82759752011-01-20 16:52:09 +00001658
drh111a6a72008-12-21 03:51:16 +00001659
drhd15cb172013-05-21 19:23:10 +00001660#ifdef WHERETRACE_ENABLED
drha18f3d22013-05-08 03:05:41 +00001661/*
drhc90713d2014-09-30 13:46:49 +00001662** Print the content of a WhereTerm object
1663*/
1664static void whereTermPrint(WhereTerm *pTerm, int iTerm){
drh0a99ba32014-09-30 17:03:35 +00001665 if( pTerm==0 ){
1666 sqlite3DebugPrintf("TERM-%-3d NULL\n", iTerm);
1667 }else{
1668 char zType[4];
drhc84a4022016-05-27 12:30:20 +00001669 char zLeft[50];
drh0a99ba32014-09-30 17:03:35 +00001670 memcpy(zType, "...", 4);
1671 if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V';
1672 if( pTerm->eOperator & WO_EQUIV ) zType[1] = 'E';
1673 if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) zType[2] = 'L';
drhc84a4022016-05-27 12:30:20 +00001674 if( pTerm->eOperator & WO_SINGLE ){
1675 sqlite3_snprintf(sizeof(zLeft),zLeft,"left={%d:%d}",
1676 pTerm->leftCursor, pTerm->u.leftColumn);
1677 }else if( (pTerm->eOperator & WO_OR)!=0 && pTerm->u.pOrInfo!=0 ){
1678 sqlite3_snprintf(sizeof(zLeft),zLeft,"indexable=0x%lld",
1679 pTerm->u.pOrInfo->indexable);
1680 }else{
1681 sqlite3_snprintf(sizeof(zLeft),zLeft,"left=%d", pTerm->leftCursor);
1682 }
drhfcd49532015-05-13 15:24:07 +00001683 sqlite3DebugPrintf(
drha15a8bc2016-08-19 18:40:17 +00001684 "TERM-%-3d %p %s %-12s prob=%-3d op=0x%03x wtFlags=0x%04x",
drhc84a4022016-05-27 12:30:20 +00001685 iTerm, pTerm, zType, zLeft, pTerm->truthProb,
drhfcd49532015-05-13 15:24:07 +00001686 pTerm->eOperator, pTerm->wtFlags);
drha15a8bc2016-08-19 18:40:17 +00001687 if( pTerm->iField ){
1688 sqlite3DebugPrintf(" iField=%d\n", pTerm->iField);
1689 }else{
1690 sqlite3DebugPrintf("\n");
1691 }
drh0a99ba32014-09-30 17:03:35 +00001692 sqlite3TreeViewExpr(0, pTerm->pExpr, 0);
1693 }
drhc90713d2014-09-30 13:46:49 +00001694}
1695#endif
1696
1697#ifdef WHERETRACE_ENABLED
1698/*
drhc84a4022016-05-27 12:30:20 +00001699** Show the complete content of a WhereClause
1700*/
1701void sqlite3WhereClausePrint(WhereClause *pWC){
1702 int i;
1703 for(i=0; i<pWC->nTerm; i++){
1704 whereTermPrint(&pWC->a[i], i);
1705 }
1706}
1707#endif
1708
1709#ifdef WHERETRACE_ENABLED
1710/*
drha18f3d22013-05-08 03:05:41 +00001711** Print a WhereLoop object for debugging purposes
1712*/
drhc1ba2e72013-10-28 19:03:21 +00001713static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){
1714 WhereInfo *pWInfo = pWC->pWInfo;
drh53801ef2016-04-09 14:36:07 +00001715 int nb = 1+(pWInfo->pTabList->nSrc+3)/4;
drh989578e2013-10-28 14:34:35 +00001716 struct SrcList_item *pItem = pWInfo->pTabList->a + p->iTab;
drha18f3d22013-05-08 03:05:41 +00001717 Table *pTab = pItem->pTab;
drh53801ef2016-04-09 14:36:07 +00001718 Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1;
drh6457a352013-06-21 00:35:37 +00001719 sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId,
drh53801ef2016-04-09 14:36:07 +00001720 p->iTab, nb, p->maskSelf, nb, p->prereq & mAll);
drh6457a352013-06-21 00:35:37 +00001721 sqlite3DebugPrintf(" %12s",
drha18f3d22013-05-08 03:05:41 +00001722 pItem->zAlias ? pItem->zAlias : pTab->zName);
drh5346e952013-05-08 14:14:26 +00001723 if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){
drhf3f69ac2014-08-20 23:38:07 +00001724 const char *zName;
1725 if( p->u.btree.pIndex && (zName = p->u.btree.pIndex->zName)!=0 ){
drh319f6772013-05-14 15:31:07 +00001726 if( strncmp(zName, "sqlite_autoindex_", 17)==0 ){
1727 int i = sqlite3Strlen30(zName) - 1;
1728 while( zName[i]!='_' ) i--;
1729 zName += i;
1730 }
drh6457a352013-06-21 00:35:37 +00001731 sqlite3DebugPrintf(".%-16s %2d", zName, p->u.btree.nEq);
drh5346e952013-05-08 14:14:26 +00001732 }else{
drh6457a352013-06-21 00:35:37 +00001733 sqlite3DebugPrintf("%20s","");
drh5346e952013-05-08 14:14:26 +00001734 }
drha18f3d22013-05-08 03:05:41 +00001735 }else{
drh5346e952013-05-08 14:14:26 +00001736 char *z;
1737 if( p->u.vtab.idxStr ){
drh3bd26f02013-05-24 14:52:03 +00001738 z = sqlite3_mprintf("(%d,\"%s\",%x)",
1739 p->u.vtab.idxNum, p->u.vtab.idxStr, p->u.vtab.omitMask);
drh5346e952013-05-08 14:14:26 +00001740 }else{
drh3bd26f02013-05-24 14:52:03 +00001741 z = sqlite3_mprintf("(%d,%x)", p->u.vtab.idxNum, p->u.vtab.omitMask);
drh5346e952013-05-08 14:14:26 +00001742 }
drh6457a352013-06-21 00:35:37 +00001743 sqlite3DebugPrintf(" %-19s", z);
drh5346e952013-05-08 14:14:26 +00001744 sqlite3_free(z);
drha18f3d22013-05-08 03:05:41 +00001745 }
drhf3f69ac2014-08-20 23:38:07 +00001746 if( p->wsFlags & WHERE_SKIPSCAN ){
drhc8bbce12014-10-21 01:05:09 +00001747 sqlite3DebugPrintf(" f %05x %d-%d", p->wsFlags, p->nLTerm,p->nSkip);
drhf3f69ac2014-08-20 23:38:07 +00001748 }else{
1749 sqlite3DebugPrintf(" f %05x N %d", p->wsFlags, p->nLTerm);
1750 }
drhb8a8e8a2013-06-10 19:12:39 +00001751 sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut);
drhc90713d2014-09-30 13:46:49 +00001752 if( p->nLTerm && (sqlite3WhereTrace & 0x100)!=0 ){
1753 int i;
1754 for(i=0; i<p->nLTerm; i++){
drh0a99ba32014-09-30 17:03:35 +00001755 whereTermPrint(p->aLTerm[i], i);
drhc90713d2014-09-30 13:46:49 +00001756 }
1757 }
drha18f3d22013-05-08 03:05:41 +00001758}
1759#endif
1760
drhf1b5f5b2013-05-02 00:15:01 +00001761/*
drh4efc9292013-06-06 23:02:03 +00001762** Convert bulk memory into a valid WhereLoop that can be passed
1763** to whereLoopClear harmlessly.
drh5346e952013-05-08 14:14:26 +00001764*/
drh4efc9292013-06-06 23:02:03 +00001765static void whereLoopInit(WhereLoop *p){
1766 p->aLTerm = p->aLTermSpace;
1767 p->nLTerm = 0;
1768 p->nLSlot = ArraySize(p->aLTermSpace);
1769 p->wsFlags = 0;
1770}
1771
1772/*
1773** Clear the WhereLoop.u union. Leave WhereLoop.pLTerm intact.
1774*/
1775static void whereLoopClearUnion(sqlite3 *db, WhereLoop *p){
drh986b3872013-06-28 21:12:20 +00001776 if( p->wsFlags & (WHERE_VIRTUALTABLE|WHERE_AUTO_INDEX) ){
drh13e11b42013-06-06 23:44:25 +00001777 if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 && p->u.vtab.needFree ){
1778 sqlite3_free(p->u.vtab.idxStr);
1779 p->u.vtab.needFree = 0;
1780 p->u.vtab.idxStr = 0;
drh986b3872013-06-28 21:12:20 +00001781 }else if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 && p->u.btree.pIndex!=0 ){
drh13e11b42013-06-06 23:44:25 +00001782 sqlite3DbFree(db, p->u.btree.pIndex->zColAff);
1783 sqlite3DbFree(db, p->u.btree.pIndex);
1784 p->u.btree.pIndex = 0;
1785 }
drh5346e952013-05-08 14:14:26 +00001786 }
1787}
1788
drh4efc9292013-06-06 23:02:03 +00001789/*
1790** Deallocate internal memory used by a WhereLoop object
1791*/
1792static void whereLoopClear(sqlite3 *db, WhereLoop *p){
1793 if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFree(db, p->aLTerm);
1794 whereLoopClearUnion(db, p);
1795 whereLoopInit(p);
1796}
1797
1798/*
1799** Increase the memory allocation for pLoop->aLTerm[] to be at least n.
1800*/
1801static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){
1802 WhereTerm **paNew;
1803 if( p->nLSlot>=n ) return SQLITE_OK;
1804 n = (n+7)&~7;
drh575fad62016-02-05 13:38:36 +00001805 paNew = sqlite3DbMallocRawNN(db, sizeof(p->aLTerm[0])*n);
mistachkinfad30392016-02-13 23:43:46 +00001806 if( paNew==0 ) return SQLITE_NOMEM_BKPT;
drh4efc9292013-06-06 23:02:03 +00001807 memcpy(paNew, p->aLTerm, sizeof(p->aLTerm[0])*p->nLSlot);
1808 if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFree(db, p->aLTerm);
1809 p->aLTerm = paNew;
1810 p->nLSlot = n;
1811 return SQLITE_OK;
1812}
1813
1814/*
1815** Transfer content from the second pLoop into the first.
1816*/
1817static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){
drh4efc9292013-06-06 23:02:03 +00001818 whereLoopClearUnion(db, pTo);
drh0d31dc32013-09-06 00:40:59 +00001819 if( whereLoopResize(db, pTo, pFrom->nLTerm) ){
1820 memset(&pTo->u, 0, sizeof(pTo->u));
mistachkinfad30392016-02-13 23:43:46 +00001821 return SQLITE_NOMEM_BKPT;
drh0d31dc32013-09-06 00:40:59 +00001822 }
drha2014152013-06-07 00:29:23 +00001823 memcpy(pTo, pFrom, WHERE_LOOP_XFER_SZ);
1824 memcpy(pTo->aLTerm, pFrom->aLTerm, pTo->nLTerm*sizeof(pTo->aLTerm[0]));
drh4efc9292013-06-06 23:02:03 +00001825 if( pFrom->wsFlags & WHERE_VIRTUALTABLE ){
1826 pFrom->u.vtab.needFree = 0;
drh986b3872013-06-28 21:12:20 +00001827 }else if( (pFrom->wsFlags & WHERE_AUTO_INDEX)!=0 ){
drh4efc9292013-06-06 23:02:03 +00001828 pFrom->u.btree.pIndex = 0;
1829 }
1830 return SQLITE_OK;
1831}
1832
drh5346e952013-05-08 14:14:26 +00001833/*
drhf1b5f5b2013-05-02 00:15:01 +00001834** Delete a WhereLoop object
1835*/
1836static void whereLoopDelete(sqlite3 *db, WhereLoop *p){
drh5346e952013-05-08 14:14:26 +00001837 whereLoopClear(db, p);
drhf1b5f5b2013-05-02 00:15:01 +00001838 sqlite3DbFree(db, p);
1839}
drh84bfda42005-07-15 13:05:21 +00001840
drh9eff6162006-06-12 21:59:13 +00001841/*
1842** Free a WhereInfo structure
1843*/
drh10fe8402008-10-11 16:47:35 +00001844static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){
drh52ff8ea2010-04-08 14:15:56 +00001845 if( ALWAYS(pWInfo) ){
danf89aa472015-04-25 12:20:24 +00001846 int i;
1847 for(i=0; i<pWInfo->nLevel; i++){
1848 WhereLevel *pLevel = &pWInfo->a[i];
1849 if( pLevel->pWLoop && (pLevel->pWLoop->wsFlags & WHERE_IN_ABLE) ){
1850 sqlite3DbFree(db, pLevel->u.in.aInLoop);
1851 }
1852 }
drh6c1f4ef2015-06-08 14:23:15 +00001853 sqlite3WhereClauseClear(&pWInfo->sWC);
drhf1b5f5b2013-05-02 00:15:01 +00001854 while( pWInfo->pLoops ){
1855 WhereLoop *p = pWInfo->pLoops;
1856 pWInfo->pLoops = p->pNextLoop;
1857 whereLoopDelete(db, p);
1858 }
drh633e6d52008-07-28 19:34:53 +00001859 sqlite3DbFree(db, pWInfo);
drh9eff6162006-06-12 21:59:13 +00001860 }
1861}
1862
drhf1b5f5b2013-05-02 00:15:01 +00001863/*
drhe0de8762014-11-05 13:13:13 +00001864** Return TRUE if all of the following are true:
drhb355c2c2014-04-18 22:20:31 +00001865**
1866** (1) X has the same or lower cost that Y
1867** (2) X is a proper subset of Y
drhe0de8762014-11-05 13:13:13 +00001868** (3) X skips at least as many columns as Y
drhb355c2c2014-04-18 22:20:31 +00001869**
1870** By "proper subset" we mean that X uses fewer WHERE clause terms
1871** than Y and that every WHERE clause term used by X is also used
1872** by Y.
1873**
1874** If X is a proper subset of Y then Y is a better choice and ought
1875** to have a lower cost. This routine returns TRUE when that cost
drhe0de8762014-11-05 13:13:13 +00001876** relationship is inverted and needs to be adjusted. The third rule
1877** was added because if X uses skip-scan less than Y it still might
1878** deserve a lower cost even if it is a proper subset of Y.
drh3fb183d2014-03-31 19:49:00 +00001879*/
drhb355c2c2014-04-18 22:20:31 +00001880static int whereLoopCheaperProperSubset(
1881 const WhereLoop *pX, /* First WhereLoop to compare */
1882 const WhereLoop *pY /* Compare against this WhereLoop */
1883){
drh3fb183d2014-03-31 19:49:00 +00001884 int i, j;
drhc8bbce12014-10-21 01:05:09 +00001885 if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){
1886 return 0; /* X is not a subset of Y */
1887 }
drhe0de8762014-11-05 13:13:13 +00001888 if( pY->nSkip > pX->nSkip ) return 0;
drhb355c2c2014-04-18 22:20:31 +00001889 if( pX->rRun >= pY->rRun ){
1890 if( pX->rRun > pY->rRun ) return 0; /* X costs more than Y */
1891 if( pX->nOut > pY->nOut ) return 0; /* X costs more than Y */
drh3fb183d2014-03-31 19:49:00 +00001892 }
drh9ee88102014-05-07 20:33:17 +00001893 for(i=pX->nLTerm-1; i>=0; i--){
drhc8bbce12014-10-21 01:05:09 +00001894 if( pX->aLTerm[i]==0 ) continue;
drhb355c2c2014-04-18 22:20:31 +00001895 for(j=pY->nLTerm-1; j>=0; j--){
1896 if( pY->aLTerm[j]==pX->aLTerm[i] ) break;
1897 }
1898 if( j<0 ) return 0; /* X not a subset of Y since term X[i] not used by Y */
1899 }
1900 return 1; /* All conditions meet */
drh3fb183d2014-03-31 19:49:00 +00001901}
1902
1903/*
1904** Try to adjust the cost of WhereLoop pTemplate upwards or downwards so
1905** that:
drh53cd10a2014-03-31 18:24:18 +00001906**
drh3fb183d2014-03-31 19:49:00 +00001907** (1) pTemplate costs less than any other WhereLoops that are a proper
1908** subset of pTemplate
drh53cd10a2014-03-31 18:24:18 +00001909**
drh3fb183d2014-03-31 19:49:00 +00001910** (2) pTemplate costs more than any other WhereLoops for which pTemplate
1911** is a proper subset.
drh53cd10a2014-03-31 18:24:18 +00001912**
drh3fb183d2014-03-31 19:49:00 +00001913** To say "WhereLoop X is a proper subset of Y" means that X uses fewer
1914** WHERE clause terms than Y and that every WHERE clause term used by X is
1915** also used by Y.
drh53cd10a2014-03-31 18:24:18 +00001916*/
1917static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){
1918 if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return;
drh53cd10a2014-03-31 18:24:18 +00001919 for(; p; p=p->pNextLoop){
drh3fb183d2014-03-31 19:49:00 +00001920 if( p->iTab!=pTemplate->iTab ) continue;
1921 if( (p->wsFlags & WHERE_INDEXED)==0 ) continue;
drhb355c2c2014-04-18 22:20:31 +00001922 if( whereLoopCheaperProperSubset(p, pTemplate) ){
1923 /* Adjust pTemplate cost downward so that it is cheaper than its
drhe0de8762014-11-05 13:13:13 +00001924 ** subset p. */
drh1b131b72014-10-21 16:01:40 +00001925 WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n",
1926 pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut-1));
drh3fb183d2014-03-31 19:49:00 +00001927 pTemplate->rRun = p->rRun;
1928 pTemplate->nOut = p->nOut - 1;
drhb355c2c2014-04-18 22:20:31 +00001929 }else if( whereLoopCheaperProperSubset(pTemplate, p) ){
1930 /* Adjust pTemplate cost upward so that it is costlier than p since
1931 ** pTemplate is a proper subset of p */
drh1b131b72014-10-21 16:01:40 +00001932 WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n",
1933 pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut+1));
drh3fb183d2014-03-31 19:49:00 +00001934 pTemplate->rRun = p->rRun;
1935 pTemplate->nOut = p->nOut + 1;
drh53cd10a2014-03-31 18:24:18 +00001936 }
1937 }
1938}
1939
1940/*
drh7a4b1642014-03-29 21:16:07 +00001941** Search the list of WhereLoops in *ppPrev looking for one that can be
1942** supplanted by pTemplate.
drhf1b5f5b2013-05-02 00:15:01 +00001943**
drh7a4b1642014-03-29 21:16:07 +00001944** Return NULL if the WhereLoop list contains an entry that can supplant
1945** pTemplate, in other words if pTemplate does not belong on the list.
drh23f98da2013-05-21 15:52:07 +00001946**
drh7a4b1642014-03-29 21:16:07 +00001947** If pX is a WhereLoop that pTemplate can supplant, then return the
1948** link that points to pX.
drh23f98da2013-05-21 15:52:07 +00001949**
drh7a4b1642014-03-29 21:16:07 +00001950** If pTemplate cannot supplant any existing element of the list but needs
1951** to be added to the list, then return a pointer to the tail of the list.
drhf1b5f5b2013-05-02 00:15:01 +00001952*/
drh7a4b1642014-03-29 21:16:07 +00001953static WhereLoop **whereLoopFindLesser(
1954 WhereLoop **ppPrev,
1955 const WhereLoop *pTemplate
1956){
1957 WhereLoop *p;
1958 for(p=(*ppPrev); p; ppPrev=&p->pNextLoop, p=*ppPrev){
drhdbb80232013-06-19 12:34:13 +00001959 if( p->iTab!=pTemplate->iTab || p->iSortIdx!=pTemplate->iSortIdx ){
1960 /* If either the iTab or iSortIdx values for two WhereLoop are different
1961 ** then those WhereLoops need to be considered separately. Neither is
1962 ** a candidate to replace the other. */
1963 continue;
1964 }
1965 /* In the current implementation, the rSetup value is either zero
1966 ** or the cost of building an automatic index (NlogN) and the NlogN
1967 ** is the same for compatible WhereLoops. */
1968 assert( p->rSetup==0 || pTemplate->rSetup==0
1969 || p->rSetup==pTemplate->rSetup );
1970
1971 /* whereLoopAddBtree() always generates and inserts the automatic index
1972 ** case first. Hence compatible candidate WhereLoops never have a larger
1973 ** rSetup. Call this SETUP-INVARIANT */
1974 assert( p->rSetup>=pTemplate->rSetup );
1975
drhdabe36d2014-06-17 20:16:43 +00001976 /* Any loop using an appliation-defined index (or PRIMARY KEY or
1977 ** UNIQUE constraint) with one or more == constraints is better
dan70273d02014-11-14 19:34:20 +00001978 ** than an automatic index. Unless it is a skip-scan. */
drhdabe36d2014-06-17 20:16:43 +00001979 if( (p->wsFlags & WHERE_AUTO_INDEX)!=0
dan70273d02014-11-14 19:34:20 +00001980 && (pTemplate->nSkip)==0
drhdabe36d2014-06-17 20:16:43 +00001981 && (pTemplate->wsFlags & WHERE_INDEXED)!=0
1982 && (pTemplate->wsFlags & WHERE_COLUMN_EQ)!=0
1983 && (p->prereq & pTemplate->prereq)==pTemplate->prereq
1984 ){
1985 break;
1986 }
1987
drh53cd10a2014-03-31 18:24:18 +00001988 /* If existing WhereLoop p is better than pTemplate, pTemplate can be
1989 ** discarded. WhereLoop p is better if:
1990 ** (1) p has no more dependencies than pTemplate, and
1991 ** (2) p has an equal or lower cost than pTemplate
1992 */
1993 if( (p->prereq & pTemplate->prereq)==p->prereq /* (1) */
1994 && p->rSetup<=pTemplate->rSetup /* (2a) */
1995 && p->rRun<=pTemplate->rRun /* (2b) */
1996 && p->nOut<=pTemplate->nOut /* (2c) */
drhf1b5f5b2013-05-02 00:15:01 +00001997 ){
drh53cd10a2014-03-31 18:24:18 +00001998 return 0; /* Discard pTemplate */
drhf1b5f5b2013-05-02 00:15:01 +00001999 }
drh53cd10a2014-03-31 18:24:18 +00002000
2001 /* If pTemplate is always better than p, then cause p to be overwritten
2002 ** with pTemplate. pTemplate is better than p if:
2003 ** (1) pTemplate has no more dependences than p, and
2004 ** (2) pTemplate has an equal or lower cost than p.
2005 */
2006 if( (p->prereq & pTemplate->prereq)==pTemplate->prereq /* (1) */
2007 && p->rRun>=pTemplate->rRun /* (2a) */
2008 && p->nOut>=pTemplate->nOut /* (2b) */
drhf1b5f5b2013-05-02 00:15:01 +00002009 ){
drhadd5ce32013-09-07 00:29:06 +00002010 assert( p->rSetup>=pTemplate->rSetup ); /* SETUP-INVARIANT above */
drh53cd10a2014-03-31 18:24:18 +00002011 break; /* Cause p to be overwritten by pTemplate */
drhf1b5f5b2013-05-02 00:15:01 +00002012 }
2013 }
drh7a4b1642014-03-29 21:16:07 +00002014 return ppPrev;
2015}
2016
2017/*
drh94a11212004-09-25 13:12:14 +00002018** Insert or replace a WhereLoop entry using the template supplied.
2019**
2020** An existing WhereLoop entry might be overwritten if the new template
2021** is better and has fewer dependencies. Or the template will be ignored
2022** and no insert will occur if an existing WhereLoop is faster and has
2023** fewer dependencies than the template. Otherwise a new WhereLoop is
danielk1977b3bce662005-01-29 08:32:43 +00002024** added based on the template.
drh94a11212004-09-25 13:12:14 +00002025**
drh7a4b1642014-03-29 21:16:07 +00002026** If pBuilder->pOrSet is not NULL then we care about only the
drh94a11212004-09-25 13:12:14 +00002027** prerequisites and rRun and nOut costs of the N best loops. That
2028** information is gathered in the pBuilder->pOrSet object. This special
2029** processing mode is used only for OR clause processing.
danielk1977b3bce662005-01-29 08:32:43 +00002030**
drh94a11212004-09-25 13:12:14 +00002031** When accumulating multiple loops (when pBuilder->pOrSet is NULL) we
danielk1977b3bce662005-01-29 08:32:43 +00002032** still might overwrite similar loops with the new template if the
drh53cd10a2014-03-31 18:24:18 +00002033** new template is better. Loops may be overwritten if the following
drh94a11212004-09-25 13:12:14 +00002034** conditions are met:
2035**
2036** (1) They have the same iTab.
2037** (2) They have the same iSortIdx.
2038** (3) The template has same or fewer dependencies than the current loop
2039** (4) The template has the same or lower cost than the current loop
drh94a11212004-09-25 13:12:14 +00002040*/
2041static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){
drh7a4b1642014-03-29 21:16:07 +00002042 WhereLoop **ppPrev, *p;
drh94a11212004-09-25 13:12:14 +00002043 WhereInfo *pWInfo = pBuilder->pWInfo;
2044 sqlite3 *db = pWInfo->pParse->db;
drhbacbbcc2016-03-09 12:35:18 +00002045 int rc;
drh94a11212004-09-25 13:12:14 +00002046
2047 /* If pBuilder->pOrSet is defined, then only keep track of the costs
2048 ** and prereqs.
2049 */
2050 if( pBuilder->pOrSet!=0 ){
drh2dc29292015-08-27 23:18:55 +00002051 if( pTemplate->nLTerm ){
drh94a11212004-09-25 13:12:14 +00002052#if WHERETRACE_ENABLED
drh2dc29292015-08-27 23:18:55 +00002053 u16 n = pBuilder->pOrSet->n;
2054 int x =
drh94a11212004-09-25 13:12:14 +00002055#endif
drh2dc29292015-08-27 23:18:55 +00002056 whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun,
drh94a11212004-09-25 13:12:14 +00002057 pTemplate->nOut);
2058#if WHERETRACE_ENABLED /* 0x8 */
drh2dc29292015-08-27 23:18:55 +00002059 if( sqlite3WhereTrace & 0x8 ){
2060 sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n);
2061 whereLoopPrint(pTemplate, pBuilder->pWC);
2062 }
drh94a11212004-09-25 13:12:14 +00002063#endif
drh2dc29292015-08-27 23:18:55 +00002064 }
danielk1977b3bce662005-01-29 08:32:43 +00002065 return SQLITE_OK;
drh9012bcb2004-12-19 00:11:35 +00002066 }
drh94a11212004-09-25 13:12:14 +00002067
drh7a4b1642014-03-29 21:16:07 +00002068 /* Look for an existing WhereLoop to replace with pTemplate
drh51669862004-12-18 18:40:26 +00002069 */
drh53cd10a2014-03-31 18:24:18 +00002070 whereLoopAdjustCost(pWInfo->pLoops, pTemplate);
drh7a4b1642014-03-29 21:16:07 +00002071 ppPrev = whereLoopFindLesser(&pWInfo->pLoops, pTemplate);
drhf1b5f5b2013-05-02 00:15:01 +00002072
drh7a4b1642014-03-29 21:16:07 +00002073 if( ppPrev==0 ){
2074 /* There already exists a WhereLoop on the list that is better
2075 ** than pTemplate, so just ignore pTemplate */
2076#if WHERETRACE_ENABLED /* 0x8 */
2077 if( sqlite3WhereTrace & 0x8 ){
drh9a7b41d2014-10-08 00:08:08 +00002078 sqlite3DebugPrintf(" skip: ");
drh7a4b1642014-03-29 21:16:07 +00002079 whereLoopPrint(pTemplate, pBuilder->pWC);
drhf1b5f5b2013-05-02 00:15:01 +00002080 }
drh7a4b1642014-03-29 21:16:07 +00002081#endif
2082 return SQLITE_OK;
2083 }else{
2084 p = *ppPrev;
drhf1b5f5b2013-05-02 00:15:01 +00002085 }
2086
2087 /* If we reach this point it means that either p[] should be overwritten
2088 ** with pTemplate[] if p[] exists, or if p==NULL then allocate a new
2089 ** WhereLoop and insert it.
2090 */
drh989578e2013-10-28 14:34:35 +00002091#if WHERETRACE_ENABLED /* 0x8 */
drhae70cf12013-05-31 15:18:46 +00002092 if( sqlite3WhereTrace & 0x8 ){
2093 if( p!=0 ){
drh9a7b41d2014-10-08 00:08:08 +00002094 sqlite3DebugPrintf("replace: ");
drhc1ba2e72013-10-28 19:03:21 +00002095 whereLoopPrint(p, pBuilder->pWC);
drhae70cf12013-05-31 15:18:46 +00002096 }
drh9a7b41d2014-10-08 00:08:08 +00002097 sqlite3DebugPrintf(" add: ");
drhc1ba2e72013-10-28 19:03:21 +00002098 whereLoopPrint(pTemplate, pBuilder->pWC);
drhae70cf12013-05-31 15:18:46 +00002099 }
2100#endif
drhf1b5f5b2013-05-02 00:15:01 +00002101 if( p==0 ){
drh7a4b1642014-03-29 21:16:07 +00002102 /* Allocate a new WhereLoop to add to the end of the list */
drh575fad62016-02-05 13:38:36 +00002103 *ppPrev = p = sqlite3DbMallocRawNN(db, sizeof(WhereLoop));
mistachkinfad30392016-02-13 23:43:46 +00002104 if( p==0 ) return SQLITE_NOMEM_BKPT;
drh4efc9292013-06-06 23:02:03 +00002105 whereLoopInit(p);
drh7a4b1642014-03-29 21:16:07 +00002106 p->pNextLoop = 0;
2107 }else{
2108 /* We will be overwriting WhereLoop p[]. But before we do, first
2109 ** go through the rest of the list and delete any other entries besides
2110 ** p[] that are also supplated by pTemplate */
2111 WhereLoop **ppTail = &p->pNextLoop;
2112 WhereLoop *pToDel;
2113 while( *ppTail ){
2114 ppTail = whereLoopFindLesser(ppTail, pTemplate);
drhdabe36d2014-06-17 20:16:43 +00002115 if( ppTail==0 ) break;
drh7a4b1642014-03-29 21:16:07 +00002116 pToDel = *ppTail;
2117 if( pToDel==0 ) break;
2118 *ppTail = pToDel->pNextLoop;
2119#if WHERETRACE_ENABLED /* 0x8 */
2120 if( sqlite3WhereTrace & 0x8 ){
drh9a7b41d2014-10-08 00:08:08 +00002121 sqlite3DebugPrintf(" delete: ");
drh7a4b1642014-03-29 21:16:07 +00002122 whereLoopPrint(pToDel, pBuilder->pWC);
2123 }
2124#endif
2125 whereLoopDelete(db, pToDel);
2126 }
drhf1b5f5b2013-05-02 00:15:01 +00002127 }
drhbacbbcc2016-03-09 12:35:18 +00002128 rc = whereLoopXfer(db, p, pTemplate);
drh5346e952013-05-08 14:14:26 +00002129 if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){
drhef866372013-05-22 20:49:02 +00002130 Index *pIndex = p->u.btree.pIndex;
2131 if( pIndex && pIndex->tnum==0 ){
drhcf8fa7a2013-05-10 20:26:22 +00002132 p->u.btree.pIndex = 0;
2133 }
drh5346e952013-05-08 14:14:26 +00002134 }
drhbacbbcc2016-03-09 12:35:18 +00002135 return rc;
drhf1b5f5b2013-05-02 00:15:01 +00002136}
2137
2138/*
drhcca9f3d2013-09-06 15:23:29 +00002139** Adjust the WhereLoop.nOut value downward to account for terms of the
2140** WHERE clause that reference the loop but which are not used by an
2141** index.
drh7a1bca72014-11-22 18:50:44 +00002142*
2143** For every WHERE clause term that is not used by the index
2144** and which has a truth probability assigned by one of the likelihood(),
2145** likely(), or unlikely() SQL functions, reduce the estimated number
2146** of output rows by the probability specified.
drhcca9f3d2013-09-06 15:23:29 +00002147**
drh7a1bca72014-11-22 18:50:44 +00002148** TUNING: For every WHERE clause term that is not used by the index
2149** and which does not have an assigned truth probability, heuristics
2150** described below are used to try to estimate the truth probability.
2151** TODO --> Perhaps this is something that could be improved by better
2152** table statistics.
2153**
drhab4624d2014-11-22 19:52:10 +00002154** Heuristic 1: Estimate the truth probability as 93.75%. The 93.75%
2155** value corresponds to -1 in LogEst notation, so this means decrement
drh7a1bca72014-11-22 18:50:44 +00002156** the WhereLoop.nOut field for every such WHERE clause term.
2157**
2158** Heuristic 2: If there exists one or more WHERE clause terms of the
2159** form "x==EXPR" and EXPR is not a constant 0 or 1, then make sure the
2160** final output row estimate is no greater than 1/4 of the total number
2161** of rows in the table. In other words, assume that x==EXPR will filter
2162** out at least 3 out of 4 rows. If EXPR is -1 or 0 or 1, then maybe the
2163** "x" column is boolean or else -1 or 0 or 1 is a common default value
2164** on the "x" column and so in that case only cap the output row estimate
2165** at 1/2 instead of 1/4.
drhcca9f3d2013-09-06 15:23:29 +00002166*/
drhd8b77e22014-09-06 01:35:57 +00002167static void whereLoopOutputAdjust(
2168 WhereClause *pWC, /* The WHERE clause */
2169 WhereLoop *pLoop, /* The loop to adjust downward */
2170 LogEst nRow /* Number of rows in the entire table */
2171){
drh7d9e7d82013-09-11 17:39:09 +00002172 WhereTerm *pTerm, *pX;
drhcca9f3d2013-09-06 15:23:29 +00002173 Bitmask notAllowed = ~(pLoop->prereq|pLoop->maskSelf);
drh7a1bca72014-11-22 18:50:44 +00002174 int i, j, k;
2175 LogEst iReduce = 0; /* pLoop->nOut should not exceed nRow-iReduce */
drhadd5ce32013-09-07 00:29:06 +00002176
drha3898252014-11-22 12:22:13 +00002177 assert( (pLoop->wsFlags & WHERE_AUTO_INDEX)==0 );
drhcca9f3d2013-09-06 15:23:29 +00002178 for(i=pWC->nTerm, pTerm=pWC->a; i>0; i--, pTerm++){
drh7d9e7d82013-09-11 17:39:09 +00002179 if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) break;
drhcca9f3d2013-09-06 15:23:29 +00002180 if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue;
2181 if( (pTerm->prereqAll & notAllowed)!=0 ) continue;
drh7d9e7d82013-09-11 17:39:09 +00002182 for(j=pLoop->nLTerm-1; j>=0; j--){
2183 pX = pLoop->aLTerm[j];
drhd2447442013-11-13 19:01:41 +00002184 if( pX==0 ) continue;
drh7d9e7d82013-09-11 17:39:09 +00002185 if( pX==pTerm ) break;
2186 if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break;
2187 }
danaa9933c2014-04-24 20:04:49 +00002188 if( j<0 ){
drhd8b77e22014-09-06 01:35:57 +00002189 if( pTerm->truthProb<=0 ){
drh7a1bca72014-11-22 18:50:44 +00002190 /* If a truth probability is specified using the likelihood() hints,
2191 ** then use the probability provided by the application. */
drhd8b77e22014-09-06 01:35:57 +00002192 pLoop->nOut += pTerm->truthProb;
2193 }else{
drh7a1bca72014-11-22 18:50:44 +00002194 /* In the absence of explicit truth probabilities, use heuristics to
2195 ** guess a reasonable truth probability. */
drhd8b77e22014-09-06 01:35:57 +00002196 pLoop->nOut--;
drhe8d0c612015-05-14 01:05:25 +00002197 if( pTerm->eOperator&(WO_EQ|WO_IS) ){
drh7a1bca72014-11-22 18:50:44 +00002198 Expr *pRight = pTerm->pExpr->pRight;
drhe0cc3c22015-05-13 17:54:08 +00002199 testcase( pTerm->pExpr->op==TK_IS );
drh7a1bca72014-11-22 18:50:44 +00002200 if( sqlite3ExprIsInteger(pRight, &k) && k>=(-1) && k<=1 ){
2201 k = 10;
2202 }else{
2203 k = 20;
2204 }
2205 if( iReduce<k ) iReduce = k;
2206 }
drhd8b77e22014-09-06 01:35:57 +00002207 }
danaa9933c2014-04-24 20:04:49 +00002208 }
drhcca9f3d2013-09-06 15:23:29 +00002209 }
drh7a1bca72014-11-22 18:50:44 +00002210 if( pLoop->nOut > nRow-iReduce ) pLoop->nOut = nRow - iReduce;
drhcca9f3d2013-09-06 15:23:29 +00002211}
2212
dan71c57db2016-07-09 20:23:55 +00002213/*
2214** Term pTerm is a vector range comparison operation. The first comparison
dan553168c2016-08-01 20:14:31 +00002215** in the vector can be optimized using column nEq of the index. This
2216** function returns the total number of vector elements that can be used
2217** as part of the range comparison.
2218**
2219** For example, if the query is:
2220**
2221** WHERE a = ? AND (b, c, d) > (?, ?, ?)
2222**
2223** and the index:
2224**
2225** CREATE INDEX ... ON (a, b, c, d, e)
2226**
2227** then this function would be invoked with nEq=1. The value returned in
2228** this case is 3.
dan71c57db2016-07-09 20:23:55 +00002229*/
dan320d4c32016-10-08 11:55:12 +00002230static int whereRangeVectorLen(
drh64bcb8c2016-08-26 03:42:57 +00002231 Parse *pParse, /* Parsing context */
2232 int iCur, /* Cursor open on pIdx */
2233 Index *pIdx, /* The index to be used for a inequality constraint */
2234 int nEq, /* Number of prior equality constraints on same index */
2235 WhereTerm *pTerm /* The vector inequality constraint */
dan71c57db2016-07-09 20:23:55 +00002236){
2237 int nCmp = sqlite3ExprVectorSize(pTerm->pExpr->pLeft);
2238 int i;
2239
2240 nCmp = MIN(nCmp, (pIdx->nColumn - nEq));
2241 for(i=1; i<nCmp; i++){
2242 /* Test if comparison i of pTerm is compatible with column (i+nEq)
2243 ** of the index. If not, exit the loop. */
2244 char aff; /* Comparison affinity */
2245 char idxaff = 0; /* Indexed columns affinity */
2246 CollSeq *pColl; /* Comparison collation sequence */
2247 Expr *pLhs = pTerm->pExpr->pLeft->x.pList->a[i].pExpr;
2248 Expr *pRhs = pTerm->pExpr->pRight;
2249 if( pRhs->flags & EP_xIsSelect ){
2250 pRhs = pRhs->x.pSelect->pEList->a[i].pExpr;
2251 }else{
2252 pRhs = pRhs->x.pList->a[i].pExpr;
2253 }
2254
2255 /* Check that the LHS of the comparison is a column reference to
dand05a7142016-08-02 17:07:51 +00002256 ** the right column of the right source table. And that the sort
2257 ** order of the index column is the same as the sort order of the
2258 ** leftmost index column. */
dan71c57db2016-07-09 20:23:55 +00002259 if( pLhs->op!=TK_COLUMN
2260 || pLhs->iTable!=iCur
2261 || pLhs->iColumn!=pIdx->aiColumn[i+nEq]
dan2c628ea2016-08-03 16:39:04 +00002262 || pIdx->aSortOrder[i+nEq]!=pIdx->aSortOrder[nEq]
dan71c57db2016-07-09 20:23:55 +00002263 ){
2264 break;
2265 }
2266
drh0c36fca2016-08-26 18:17:08 +00002267 testcase( pLhs->iColumn==XN_ROWID );
dan71c57db2016-07-09 20:23:55 +00002268 aff = sqlite3CompareAffinity(pRhs, sqlite3ExprAffinity(pLhs));
drh0dfa4f62016-08-26 13:19:49 +00002269 idxaff = sqlite3TableColumnAffinity(pIdx->pTable, pLhs->iColumn);
dan71c57db2016-07-09 20:23:55 +00002270 if( aff!=idxaff ) break;
2271
2272 pColl = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs);
drh0dfa4f62016-08-26 13:19:49 +00002273 if( pColl==0 ) break;
drh64bcb8c2016-08-26 03:42:57 +00002274 if( sqlite3StrICmp(pColl->zName, pIdx->azColl[i+nEq]) ) break;
dan71c57db2016-07-09 20:23:55 +00002275 }
2276 return i;
2277}
2278
drhcca9f3d2013-09-06 15:23:29 +00002279/*
drhdbd94862014-07-23 23:57:42 +00002280** Adjust the cost C by the costMult facter T. This only occurs if
2281** compiled with -DSQLITE_ENABLE_COSTMULT
2282*/
2283#ifdef SQLITE_ENABLE_COSTMULT
2284# define ApplyCostMultiplier(C,T) C += T
2285#else
2286# define ApplyCostMultiplier(C,T)
2287#endif
2288
2289/*
dan4a6b8a02014-04-30 14:47:01 +00002290** We have so far matched pBuilder->pNew->u.btree.nEq terms of the
2291** index pIndex. Try to match one more.
2292**
2293** When this function is called, pBuilder->pNew->nOut contains the
2294** number of rows expected to be visited by filtering using the nEq
2295** terms only. If it is modified, this value is restored before this
2296** function returns.
drh1c8148f2013-05-04 20:25:23 +00002297**
2298** If pProbe->tnum==0, that means pIndex is a fake index used for the
2299** INTEGER PRIMARY KEY.
2300*/
drh5346e952013-05-08 14:14:26 +00002301static int whereLoopAddBtreeIndex(
drh1c8148f2013-05-04 20:25:23 +00002302 WhereLoopBuilder *pBuilder, /* The WhereLoop factory */
2303 struct SrcList_item *pSrc, /* FROM clause term being analyzed */
2304 Index *pProbe, /* An index on pSrc */
drhbf539c42013-10-05 18:16:02 +00002305 LogEst nInMul /* log(Number of iterations due to IN) */
drh1c8148f2013-05-04 20:25:23 +00002306){
drh70d18342013-06-06 19:16:33 +00002307 WhereInfo *pWInfo = pBuilder->pWInfo; /* WHERE analyse context */
2308 Parse *pParse = pWInfo->pParse; /* Parsing context */
2309 sqlite3 *db = pParse->db; /* Database connection malloc context */
drh1c8148f2013-05-04 20:25:23 +00002310 WhereLoop *pNew; /* Template WhereLoop under construction */
2311 WhereTerm *pTerm; /* A WhereTerm under consideration */
drh43fe25f2013-05-07 23:06:23 +00002312 int opMask; /* Valid operators for constraints */
drh1c8148f2013-05-04 20:25:23 +00002313 WhereScan scan; /* Iterator for WHERE terms */
drh4efc9292013-06-06 23:02:03 +00002314 Bitmask saved_prereq; /* Original value of pNew->prereq */
2315 u16 saved_nLTerm; /* Original value of pNew->nLTerm */
drhcd8629e2013-11-13 12:27:25 +00002316 u16 saved_nEq; /* Original value of pNew->u.btree.nEq */
dan71c57db2016-07-09 20:23:55 +00002317 u16 saved_nBtm; /* Original value of pNew->u.btree.nBtm */
2318 u16 saved_nTop; /* Original value of pNew->u.btree.nTop */
drhc8bbce12014-10-21 01:05:09 +00002319 u16 saved_nSkip; /* Original value of pNew->nSkip */
drh4efc9292013-06-06 23:02:03 +00002320 u32 saved_wsFlags; /* Original value of pNew->wsFlags */
drhbf539c42013-10-05 18:16:02 +00002321 LogEst saved_nOut; /* Original value of pNew->nOut */
drh5346e952013-05-08 14:14:26 +00002322 int rc = SQLITE_OK; /* Return code */
drhd8b77e22014-09-06 01:35:57 +00002323 LogEst rSize; /* Number of rows in the table */
drhbf539c42013-10-05 18:16:02 +00002324 LogEst rLogSize; /* Logarithm of table size */
drhc7f0d222013-06-19 03:27:12 +00002325 WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */
drh1c8148f2013-05-04 20:25:23 +00002326
drh1c8148f2013-05-04 20:25:23 +00002327 pNew = pBuilder->pNew;
mistachkinfad30392016-02-13 23:43:46 +00002328 if( db->mallocFailed ) return SQLITE_NOMEM_BKPT;
drh4313f042016-09-16 14:50:38 +00002329 WHERETRACE(0x800, ("BEGIN addBtreeIdx(%s), nEq=%d\n",
2330 pProbe->zName, pNew->u.btree.nEq));
drh1c8148f2013-05-04 20:25:23 +00002331
drh5346e952013-05-08 14:14:26 +00002332 assert( (pNew->wsFlags & WHERE_VIRTUALTABLE)==0 );
drh43fe25f2013-05-07 23:06:23 +00002333 assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 );
2334 if( pNew->wsFlags & WHERE_BTM_LIMIT ){
2335 opMask = WO_LT|WO_LE;
drh1c8148f2013-05-04 20:25:23 +00002336 }else{
dan71c57db2016-07-09 20:23:55 +00002337 assert( pNew->u.btree.nBtm==0 );
drhe8d0c612015-05-14 01:05:25 +00002338 opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE|WO_ISNULL|WO_IS;
drh1c8148f2013-05-04 20:25:23 +00002339 }
drhef866372013-05-22 20:49:02 +00002340 if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE);
drh1c8148f2013-05-04 20:25:23 +00002341
dan39129ce2014-06-30 15:23:57 +00002342 assert( pNew->u.btree.nEq<pProbe->nColumn );
dan39129ce2014-06-30 15:23:57 +00002343
drh4efc9292013-06-06 23:02:03 +00002344 saved_nEq = pNew->u.btree.nEq;
dan71c57db2016-07-09 20:23:55 +00002345 saved_nBtm = pNew->u.btree.nBtm;
2346 saved_nTop = pNew->u.btree.nTop;
drhc8bbce12014-10-21 01:05:09 +00002347 saved_nSkip = pNew->nSkip;
drh4efc9292013-06-06 23:02:03 +00002348 saved_nLTerm = pNew->nLTerm;
2349 saved_wsFlags = pNew->wsFlags;
2350 saved_prereq = pNew->prereq;
2351 saved_nOut = pNew->nOut;
drhbb523082015-08-27 15:58:51 +00002352 pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, saved_nEq,
2353 opMask, pProbe);
drhb8a8e8a2013-06-10 19:12:39 +00002354 pNew->rSetup = 0;
drhd8b77e22014-09-06 01:35:57 +00002355 rSize = pProbe->aiRowLogEst[0];
2356 rLogSize = estLog(rSize);
drh5346e952013-05-08 14:14:26 +00002357 for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){
dan8ad1d8b2014-04-25 20:22:45 +00002358 u16 eOp = pTerm->eOperator; /* Shorthand for pTerm->eOperator */
danaa9933c2014-04-24 20:04:49 +00002359 LogEst rCostIdx;
dan8ad1d8b2014-04-25 20:22:45 +00002360 LogEst nOutUnadjusted; /* nOut before IN() and WHERE adjustments */
drhb8a8e8a2013-06-10 19:12:39 +00002361 int nIn = 0;
drh1435a9a2013-08-27 23:15:44 +00002362#ifdef SQLITE_ENABLE_STAT3_OR_STAT4
dan7a419232013-08-06 20:01:43 +00002363 int nRecValid = pBuilder->nRecValid;
drhb5246e52013-07-08 21:12:57 +00002364#endif
dan8ad1d8b2014-04-25 20:22:45 +00002365 if( (eOp==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0)
drhbb523082015-08-27 15:58:51 +00002366 && indexColumnNotNull(pProbe, saved_nEq)
dan8bff07a2013-08-29 14:56:14 +00002367 ){
2368 continue; /* ignore IS [NOT] NULL constraints on NOT NULL columns */
2369 }
dan7a419232013-08-06 20:01:43 +00002370 if( pTerm->prereqRight & pNew->maskSelf ) continue;
2371
drha40da622015-03-09 12:11:56 +00002372 /* Do not allow the upper bound of a LIKE optimization range constraint
2373 ** to mix with a lower range bound from some other source */
2374 if( pTerm->wtFlags & TERM_LIKEOPT && pTerm->eOperator==WO_LT ) continue;
2375
drh5996a772016-03-31 20:40:28 +00002376 /* Do not allow IS constraints from the WHERE clause to be used by the
2377 ** right table of a LEFT JOIN. Only constraints in the ON clause are
2378 ** allowed */
2379 if( (pSrc->fg.jointype & JT_LEFT)!=0
2380 && !ExprHasProperty(pTerm->pExpr, EP_FromJoin)
2381 && (eOp & (WO_IS|WO_ISNULL))!=0
2382 ){
2383 testcase( eOp & WO_IS );
2384 testcase( eOp & WO_ISNULL );
2385 continue;
2386 }
2387
drh4efc9292013-06-06 23:02:03 +00002388 pNew->wsFlags = saved_wsFlags;
2389 pNew->u.btree.nEq = saved_nEq;
dan71c57db2016-07-09 20:23:55 +00002390 pNew->u.btree.nBtm = saved_nBtm;
2391 pNew->u.btree.nTop = saved_nTop;
drh4efc9292013-06-06 23:02:03 +00002392 pNew->nLTerm = saved_nLTerm;
2393 if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */
2394 pNew->aLTerm[pNew->nLTerm++] = pTerm;
2395 pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf;
dan8ad1d8b2014-04-25 20:22:45 +00002396
2397 assert( nInMul==0
2398 || (pNew->wsFlags & WHERE_COLUMN_NULL)!=0
2399 || (pNew->wsFlags & WHERE_COLUMN_IN)!=0
2400 || (pNew->wsFlags & WHERE_SKIPSCAN)!=0
2401 );
2402
2403 if( eOp & WO_IN ){
drha18f3d22013-05-08 03:05:41 +00002404 Expr *pExpr = pTerm->pExpr;
2405 pNew->wsFlags |= WHERE_COLUMN_IN;
2406 if( ExprHasProperty(pExpr, EP_xIsSelect) ){
drhe1e2e9a2013-06-13 15:16:53 +00002407 /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */
dan3d1fb1d2016-08-03 18:00:49 +00002408 int i;
drhbf539c42013-10-05 18:16:02 +00002409 nIn = 46; assert( 46==sqlite3LogEst(25) );
dan3d1fb1d2016-08-03 18:00:49 +00002410
2411 /* The expression may actually be of the form (x, y) IN (SELECT...).
2412 ** In this case there is a separate term for each of (x) and (y).
2413 ** However, the nIn multiplier should only be applied once, not once
2414 ** for each such term. The following loop checks that pTerm is the
2415 ** first such term in use, and sets nIn back to 0 if it is not. */
2416 for(i=0; i<pNew->nLTerm-1; i++){
drh9a2e5162016-09-19 11:00:42 +00002417 if( pNew->aLTerm[i] && pNew->aLTerm[i]->pExpr==pExpr ) nIn = 0;
dan3d1fb1d2016-08-03 18:00:49 +00002418 }
drha18f3d22013-05-08 03:05:41 +00002419 }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){
2420 /* "x IN (value, value, ...)" */
drhbf539c42013-10-05 18:16:02 +00002421 nIn = sqlite3LogEst(pExpr->x.pList->nExpr);
dan3d1fb1d2016-08-03 18:00:49 +00002422 assert( nIn>0 ); /* RHS always has 2 or more terms... The parser
2423 ** changes "x IN (?)" into "x=?". */
drhf1645f02013-05-07 19:44:38 +00002424 }
drhe8d0c612015-05-14 01:05:25 +00002425 }else if( eOp & (WO_EQ|WO_IS) ){
drhbb523082015-08-27 15:58:51 +00002426 int iCol = pProbe->aiColumn[saved_nEq];
drha18f3d22013-05-08 03:05:41 +00002427 pNew->wsFlags |= WHERE_COLUMN_EQ;
drhbb523082015-08-27 15:58:51 +00002428 assert( saved_nEq==pNew->u.btree.nEq );
drh4b92f982015-09-29 17:20:14 +00002429 if( iCol==XN_ROWID
2430 || (iCol>0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1)
2431 ){
dan2813bde2015-04-11 11:44:27 +00002432 if( iCol>=0 && pProbe->uniqNotNull==0 ){
drhe39a7322014-02-03 14:04:11 +00002433 pNew->wsFlags |= WHERE_UNQ_WANTED;
2434 }else{
2435 pNew->wsFlags |= WHERE_ONEROW;
2436 }
drh21f7ff72013-06-03 15:07:23 +00002437 }
dan2dd3cdc2014-04-26 20:21:14 +00002438 }else if( eOp & WO_ISNULL ){
2439 pNew->wsFlags |= WHERE_COLUMN_NULL;
dan8ad1d8b2014-04-25 20:22:45 +00002440 }else if( eOp & (WO_GT|WO_GE) ){
2441 testcase( eOp & WO_GT );
2442 testcase( eOp & WO_GE );
drha18f3d22013-05-08 03:05:41 +00002443 pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT;
dan71c57db2016-07-09 20:23:55 +00002444 pNew->u.btree.nBtm = whereRangeVectorLen(
2445 pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm
2446 );
drh6f2bfad2013-06-03 17:35:22 +00002447 pBtm = pTerm;
2448 pTop = 0;
drha40da622015-03-09 12:11:56 +00002449 if( pTerm->wtFlags & TERM_LIKEOPT ){
drh80314622015-03-09 13:01:02 +00002450 /* Range contraints that come from the LIKE optimization are
2451 ** always used in pairs. */
drha40da622015-03-09 12:11:56 +00002452 pTop = &pTerm[1];
2453 assert( (pTop-(pTerm->pWC->a))<pTerm->pWC->nTerm );
2454 assert( pTop->wtFlags & TERM_LIKEOPT );
2455 assert( pTop->eOperator==WO_LT );
2456 if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */
2457 pNew->aLTerm[pNew->nLTerm++] = pTop;
2458 pNew->wsFlags |= WHERE_TOP_LIMIT;
dan71c57db2016-07-09 20:23:55 +00002459 pNew->u.btree.nTop = 1;
drha40da622015-03-09 12:11:56 +00002460 }
dan2dd3cdc2014-04-26 20:21:14 +00002461 }else{
dan8ad1d8b2014-04-25 20:22:45 +00002462 assert( eOp & (WO_LT|WO_LE) );
2463 testcase( eOp & WO_LT );
2464 testcase( eOp & WO_LE );
drha18f3d22013-05-08 03:05:41 +00002465 pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT;
dan71c57db2016-07-09 20:23:55 +00002466 pNew->u.btree.nTop = whereRangeVectorLen(
2467 pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm
2468 );
drh6f2bfad2013-06-03 17:35:22 +00002469 pTop = pTerm;
2470 pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ?
drh4efc9292013-06-06 23:02:03 +00002471 pNew->aLTerm[pNew->nLTerm-2] : 0;
drh1c8148f2013-05-04 20:25:23 +00002472 }
dan8ad1d8b2014-04-25 20:22:45 +00002473
2474 /* At this point pNew->nOut is set to the number of rows expected to
2475 ** be visited by the index scan before considering term pTerm, or the
2476 ** values of nIn and nInMul. In other words, assuming that all
2477 ** "x IN(...)" terms are replaced with "x = ?". This block updates
2478 ** the value of pNew->nOut to account for pTerm (but not nIn/nInMul). */
2479 assert( pNew->nOut==saved_nOut );
drh6f2bfad2013-06-03 17:35:22 +00002480 if( pNew->wsFlags & WHERE_COLUMN_RANGE ){
danaa9933c2014-04-24 20:04:49 +00002481 /* Adjust nOut using stat3/stat4 data. Or, if there is no stat3/stat4
2482 ** data, using some other estimate. */
drh186ad8c2013-10-08 18:40:37 +00002483 whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew);
dan8ad1d8b2014-04-25 20:22:45 +00002484 }else{
2485 int nEq = ++pNew->u.btree.nEq;
drhe8d0c612015-05-14 01:05:25 +00002486 assert( eOp & (WO_ISNULL|WO_EQ|WO_IN|WO_IS) );
dan8ad1d8b2014-04-25 20:22:45 +00002487
2488 assert( pNew->nOut==saved_nOut );
drhbb523082015-08-27 15:58:51 +00002489 if( pTerm->truthProb<=0 && pProbe->aiColumn[saved_nEq]>=0 ){
dan8ad1d8b2014-04-25 20:22:45 +00002490 assert( (eOp & WO_IN) || nIn==0 );
drhc5f246e2014-05-01 20:24:21 +00002491 testcase( eOp & WO_IN );
dan8ad1d8b2014-04-25 20:22:45 +00002492 pNew->nOut += pTerm->truthProb;
2493 pNew->nOut -= nIn;
dan8ad1d8b2014-04-25 20:22:45 +00002494 }else{
drh1435a9a2013-08-27 23:15:44 +00002495#ifdef SQLITE_ENABLE_STAT3_OR_STAT4
dan8ad1d8b2014-04-25 20:22:45 +00002496 tRowcnt nOut = 0;
2497 if( nInMul==0
2498 && pProbe->nSample
2499 && pNew->u.btree.nEq<=pProbe->nSampleCol
dan8ad1d8b2014-04-25 20:22:45 +00002500 && ((eOp & WO_IN)==0 || !ExprHasProperty(pTerm->pExpr, EP_xIsSelect))
dan8ad1d8b2014-04-25 20:22:45 +00002501 ){
2502 Expr *pExpr = pTerm->pExpr;
drhe8d0c612015-05-14 01:05:25 +00002503 if( (eOp & (WO_EQ|WO_ISNULL|WO_IS))!=0 ){
2504 testcase( eOp & WO_EQ );
2505 testcase( eOp & WO_IS );
dan8ad1d8b2014-04-25 20:22:45 +00002506 testcase( eOp & WO_ISNULL );
2507 rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut);
2508 }else{
2509 rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut);
2510 }
dan8ad1d8b2014-04-25 20:22:45 +00002511 if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK;
2512 if( rc!=SQLITE_OK ) break; /* Jump out of the pTerm loop */
2513 if( nOut ){
2514 pNew->nOut = sqlite3LogEst(nOut);
2515 if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut;
2516 pNew->nOut -= nIn;
2517 }
2518 }
2519 if( nOut==0 )
2520#endif
2521 {
2522 pNew->nOut += (pProbe->aiRowLogEst[nEq] - pProbe->aiRowLogEst[nEq-1]);
2523 if( eOp & WO_ISNULL ){
2524 /* TUNING: If there is no likelihood() value, assume that a
2525 ** "col IS NULL" expression matches twice as many rows
2526 ** as (col=?). */
2527 pNew->nOut += 10;
2528 }
2529 }
dan6cb8d762013-08-08 11:48:57 +00002530 }
drh6f2bfad2013-06-03 17:35:22 +00002531 }
dan8ad1d8b2014-04-25 20:22:45 +00002532
danaa9933c2014-04-24 20:04:49 +00002533 /* Set rCostIdx to the cost of visiting selected rows in index. Add
2534 ** it to pNew->rRun, which is currently set to the cost of the index
2535 ** seek only. Then, if this is a non-covering index, add the cost of
2536 ** visiting the rows in the main table. */
2537 rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow;
dan8ad1d8b2014-04-25 20:22:45 +00002538 pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx);
drhe217efc2013-06-12 03:48:41 +00002539 if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){
danaa9933c2014-04-24 20:04:49 +00002540 pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16);
drheb04de32013-05-10 15:16:30 +00002541 }
drhdbd94862014-07-23 23:57:42 +00002542 ApplyCostMultiplier(pNew->rRun, pProbe->pTable->costMult);
danaa9933c2014-04-24 20:04:49 +00002543
dan8ad1d8b2014-04-25 20:22:45 +00002544 nOutUnadjusted = pNew->nOut;
2545 pNew->rRun += nInMul + nIn;
2546 pNew->nOut += nInMul + nIn;
drhd8b77e22014-09-06 01:35:57 +00002547 whereLoopOutputAdjust(pBuilder->pWC, pNew, rSize);
drhcf8fa7a2013-05-10 20:26:22 +00002548 rc = whereLoopInsert(pBuilder, pNew);
dan440e6ff2014-04-28 08:49:54 +00002549
2550 if( pNew->wsFlags & WHERE_COLUMN_RANGE ){
2551 pNew->nOut = saved_nOut;
2552 }else{
2553 pNew->nOut = nOutUnadjusted;
2554 }
dan8ad1d8b2014-04-25 20:22:45 +00002555
drh5346e952013-05-08 14:14:26 +00002556 if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0
dan39129ce2014-06-30 15:23:57 +00002557 && pNew->u.btree.nEq<pProbe->nColumn
drh5346e952013-05-08 14:14:26 +00002558 ){
drhb8a8e8a2013-06-10 19:12:39 +00002559 whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn);
drha18f3d22013-05-08 03:05:41 +00002560 }
danad45ed72013-08-08 12:21:32 +00002561 pNew->nOut = saved_nOut;
drh1435a9a2013-08-27 23:15:44 +00002562#ifdef SQLITE_ENABLE_STAT3_OR_STAT4
dan7a419232013-08-06 20:01:43 +00002563 pBuilder->nRecValid = nRecValid;
dan7a419232013-08-06 20:01:43 +00002564#endif
drh1c8148f2013-05-04 20:25:23 +00002565 }
drh4efc9292013-06-06 23:02:03 +00002566 pNew->prereq = saved_prereq;
2567 pNew->u.btree.nEq = saved_nEq;
dan71c57db2016-07-09 20:23:55 +00002568 pNew->u.btree.nBtm = saved_nBtm;
2569 pNew->u.btree.nTop = saved_nTop;
drhc8bbce12014-10-21 01:05:09 +00002570 pNew->nSkip = saved_nSkip;
drh4efc9292013-06-06 23:02:03 +00002571 pNew->wsFlags = saved_wsFlags;
2572 pNew->nOut = saved_nOut;
2573 pNew->nLTerm = saved_nLTerm;
drhc8bbce12014-10-21 01:05:09 +00002574
2575 /* Consider using a skip-scan if there are no WHERE clause constraints
2576 ** available for the left-most terms of the index, and if the average
2577 ** number of repeats in the left-most terms is at least 18.
2578 **
2579 ** The magic number 18 is selected on the basis that scanning 17 rows
2580 ** is almost always quicker than an index seek (even though if the index
2581 ** contains fewer than 2^17 rows we assume otherwise in other parts of
2582 ** the code). And, even if it is not, it should not be too much slower.
2583 ** On the other hand, the extra seeks could end up being significantly
2584 ** more expensive. */
2585 assert( 42==sqlite3LogEst(18) );
2586 if( saved_nEq==saved_nSkip
2587 && saved_nEq+1<pProbe->nKeyCol
drhf9df2fb2014-11-15 19:08:13 +00002588 && pProbe->noSkipScan==0
drhc8bbce12014-10-21 01:05:09 +00002589 && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */
2590 && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK
2591 ){
2592 LogEst nIter;
2593 pNew->u.btree.nEq++;
2594 pNew->nSkip++;
2595 pNew->aLTerm[pNew->nLTerm++] = 0;
2596 pNew->wsFlags |= WHERE_SKIPSCAN;
drh99bbcc82016-07-29 01:32:36 +00002597 nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1];
drhc8bbce12014-10-21 01:05:09 +00002598 pNew->nOut -= nIter;
2599 /* TUNING: Because uncertainties in the estimates for skip-scan queries,
2600 ** add a 1.375 fudge factor to make skip-scan slightly less likely. */
drh99bbcc82016-07-29 01:32:36 +00002601 nIter += 5;
drhc8bbce12014-10-21 01:05:09 +00002602 whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul);
2603 pNew->nOut = saved_nOut;
2604 pNew->u.btree.nEq = saved_nEq;
2605 pNew->nSkip = saved_nSkip;
2606 pNew->wsFlags = saved_wsFlags;
2607 }
2608
drh4313f042016-09-16 14:50:38 +00002609 WHERETRACE(0x800, ("END addBtreeIdx(%s), nEq=%d, rc=%d\n",
2610 pProbe->zName, saved_nEq, rc));
drh5346e952013-05-08 14:14:26 +00002611 return rc;
drh1c8148f2013-05-04 20:25:23 +00002612}
2613
2614/*
drh23f98da2013-05-21 15:52:07 +00002615** Return True if it is possible that pIndex might be useful in
2616** implementing the ORDER BY clause in pBuilder.
2617**
2618** Return False if pBuilder does not contain an ORDER BY clause or
2619** if there is no way for pIndex to be useful in implementing that
2620** ORDER BY clause.
2621*/
2622static int indexMightHelpWithOrderBy(
2623 WhereLoopBuilder *pBuilder,
2624 Index *pIndex,
2625 int iCursor
2626){
2627 ExprList *pOB;
drhdae26fe2015-09-24 18:47:59 +00002628 ExprList *aColExpr;
drh6d381472013-06-13 17:58:08 +00002629 int ii, jj;
drh23f98da2013-05-21 15:52:07 +00002630
drh53cfbe92013-06-13 17:28:22 +00002631 if( pIndex->bUnordered ) return 0;
drh70d18342013-06-06 19:16:33 +00002632 if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0;
drh23f98da2013-05-21 15:52:07 +00002633 for(ii=0; ii<pOB->nExpr; ii++){
drh45c154a2013-06-03 20:46:35 +00002634 Expr *pExpr = sqlite3ExprSkipCollate(pOB->a[ii].pExpr);
drhdae26fe2015-09-24 18:47:59 +00002635 if( pExpr->op==TK_COLUMN && pExpr->iTable==iCursor ){
drh137fd4f2014-09-19 02:01:37 +00002636 if( pExpr->iColumn<0 ) return 1;
drhbbbdc832013-10-22 18:01:40 +00002637 for(jj=0; jj<pIndex->nKeyCol; jj++){
drh6d381472013-06-13 17:58:08 +00002638 if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1;
2639 }
drhdae26fe2015-09-24 18:47:59 +00002640 }else if( (aColExpr = pIndex->aColExpr)!=0 ){
2641 for(jj=0; jj<pIndex->nKeyCol; jj++){
drh4b92f982015-09-29 17:20:14 +00002642 if( pIndex->aiColumn[jj]!=XN_EXPR ) continue;
drhdae26fe2015-09-24 18:47:59 +00002643 if( sqlite3ExprCompare(pExpr,aColExpr->a[jj].pExpr,iCursor)==0 ){
2644 return 1;
2645 }
2646 }
drh23f98da2013-05-21 15:52:07 +00002647 }
2648 }
2649 return 0;
2650}
2651
2652/*
drh92a121f2013-06-10 12:15:47 +00002653** Return a bitmask where 1s indicate that the corresponding column of
2654** the table is used by an index. Only the first 63 columns are considered.
2655*/
drhfd5874d2013-06-12 14:52:39 +00002656static Bitmask columnsInIndex(Index *pIdx){
drh92a121f2013-06-10 12:15:47 +00002657 Bitmask m = 0;
2658 int j;
drhec95c442013-10-23 01:57:32 +00002659 for(j=pIdx->nColumn-1; j>=0; j--){
drh92a121f2013-06-10 12:15:47 +00002660 int x = pIdx->aiColumn[j];
drhec95c442013-10-23 01:57:32 +00002661 if( x>=0 ){
2662 testcase( x==BMS-1 );
2663 testcase( x==BMS-2 );
2664 if( x<BMS-1 ) m |= MASKBIT(x);
2665 }
drh92a121f2013-06-10 12:15:47 +00002666 }
2667 return m;
2668}
2669
drh4bd5f732013-07-31 23:22:39 +00002670/* Check to see if a partial index with pPartIndexWhere can be used
2671** in the current query. Return true if it can be and false if not.
2672*/
2673static int whereUsablePartialIndex(int iTab, WhereClause *pWC, Expr *pWhere){
2674 int i;
2675 WhereTerm *pTerm;
drhcf599b62015-08-07 20:57:00 +00002676 while( pWhere->op==TK_AND ){
2677 if( !whereUsablePartialIndex(iTab,pWC,pWhere->pLeft) ) return 0;
2678 pWhere = pWhere->pRight;
2679 }
drh4bd5f732013-07-31 23:22:39 +00002680 for(i=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
dan2a45cb52015-02-24 20:10:49 +00002681 Expr *pExpr = pTerm->pExpr;
2682 if( sqlite3ExprImpliesExpr(pExpr, pWhere, iTab)
2683 && (!ExprHasProperty(pExpr, EP_FromJoin) || pExpr->iRightJoinTable==iTab)
drh077f06e2015-02-24 16:48:59 +00002684 ){
2685 return 1;
2686 }
drh4bd5f732013-07-31 23:22:39 +00002687 }
2688 return 0;
2689}
drh92a121f2013-06-10 12:15:47 +00002690
2691/*
dan51576f42013-07-02 10:06:15 +00002692** Add all WhereLoop objects for a single table of the join where the table
dan71c57db2016-07-09 20:23:55 +00002693** is identified by pBuilder->pNew->iTab. That table is guaranteed to be
drh0823c892013-05-11 00:06:23 +00002694** a b-tree table, not a virtual table.
dan81647222014-04-30 15:00:16 +00002695**
2696** The costs (WhereLoop.rRun) of the b-tree loops added by this function
2697** are calculated as follows:
2698**
2699** For a full scan, assuming the table (or index) contains nRow rows:
2700**
2701** cost = nRow * 3.0 // full-table scan
2702** cost = nRow * K // scan of covering index
2703** cost = nRow * (K+3.0) // scan of non-covering index
2704**
2705** where K is a value between 1.1 and 3.0 set based on the relative
2706** estimated average size of the index and table records.
2707**
2708** For an index scan, where nVisit is the number of index rows visited
2709** by the scan, and nSeek is the number of seek operations required on
2710** the index b-tree:
2711**
2712** cost = nSeek * (log(nRow) + K * nVisit) // covering index
2713** cost = nSeek * (log(nRow) + (K+3.0) * nVisit) // non-covering index
2714**
2715** Normally, nSeek is 1. nSeek values greater than 1 come about if the
2716** WHERE clause includes "x IN (....)" terms used in place of "x=?". Or when
2717** implicit "x IN (SELECT x FROM tbl)" terms are added for skip-scans.
drh83a305f2014-07-22 12:05:32 +00002718**
2719** The estimated values (nRow, nVisit, nSeek) often contain a large amount
2720** of uncertainty. For this reason, scoring is designed to pick plans that
2721** "do the least harm" if the estimates are inaccurate. For example, a
2722** log(nRow) factor is omitted from a non-covering index scan in order to
2723** bias the scoring in favor of using an index, since the worst-case
2724** performance of using an index is far better than the worst-case performance
2725** of a full table scan.
drhf1b5f5b2013-05-02 00:15:01 +00002726*/
drh5346e952013-05-08 14:14:26 +00002727static int whereLoopAddBtree(
drh1c8148f2013-05-04 20:25:23 +00002728 WhereLoopBuilder *pBuilder, /* WHERE clause information */
drh599d5762016-03-08 01:11:51 +00002729 Bitmask mPrereq /* Extra prerequesites for using this table */
drhf1b5f5b2013-05-02 00:15:01 +00002730){
drh70d18342013-06-06 19:16:33 +00002731 WhereInfo *pWInfo; /* WHERE analysis context */
drh1c8148f2013-05-04 20:25:23 +00002732 Index *pProbe; /* An index we are evaluating */
drh1c8148f2013-05-04 20:25:23 +00002733 Index sPk; /* A fake index object for the primary key */
dancfc9df72014-04-25 15:01:01 +00002734 LogEst aiRowEstPk[2]; /* The aiRowLogEst[] value for the sPk index */
drhbbbdc832013-10-22 18:01:40 +00002735 i16 aiColumnPk = -1; /* The aColumn[] value for the sPk index */
drh70d18342013-06-06 19:16:33 +00002736 SrcList *pTabList; /* The FROM clause */
drh1c8148f2013-05-04 20:25:23 +00002737 struct SrcList_item *pSrc; /* The FROM clause btree term to add */
drh1c8148f2013-05-04 20:25:23 +00002738 WhereLoop *pNew; /* Template WhereLoop object */
drh5346e952013-05-08 14:14:26 +00002739 int rc = SQLITE_OK; /* Return code */
drhd044d202013-05-31 12:43:55 +00002740 int iSortIdx = 1; /* Index number */
drh23f98da2013-05-21 15:52:07 +00002741 int b; /* A boolean value */
drhbf539c42013-10-05 18:16:02 +00002742 LogEst rSize; /* number of rows in the table */
2743 LogEst rLogSize; /* Logarithm of the number of rows in the table */
drh4bd5f732013-07-31 23:22:39 +00002744 WhereClause *pWC; /* The parsed WHERE clause */
drh3495d202013-10-07 17:32:15 +00002745 Table *pTab; /* Table being queried */
drh23f98da2013-05-21 15:52:07 +00002746
drh1c8148f2013-05-04 20:25:23 +00002747 pNew = pBuilder->pNew;
drh70d18342013-06-06 19:16:33 +00002748 pWInfo = pBuilder->pWInfo;
2749 pTabList = pWInfo->pTabList;
2750 pSrc = pTabList->a + pNew->iTab;
drh3495d202013-10-07 17:32:15 +00002751 pTab = pSrc->pTab;
drh4bd5f732013-07-31 23:22:39 +00002752 pWC = pBuilder->pWC;
drh0823c892013-05-11 00:06:23 +00002753 assert( !IsVirtual(pSrc->pTab) );
drh1c8148f2013-05-04 20:25:23 +00002754
drh8a48b9c2015-08-19 15:20:00 +00002755 if( pSrc->pIBIndex ){
drh1c8148f2013-05-04 20:25:23 +00002756 /* An INDEXED BY clause specifies a particular index to use */
drh8a48b9c2015-08-19 15:20:00 +00002757 pProbe = pSrc->pIBIndex;
drhec95c442013-10-23 01:57:32 +00002758 }else if( !HasRowid(pTab) ){
2759 pProbe = pTab->pIndex;
drh1c8148f2013-05-04 20:25:23 +00002760 }else{
2761 /* There is no INDEXED BY clause. Create a fake Index object in local
2762 ** variable sPk to represent the rowid primary key index. Make this
2763 ** fake index the first in a chain of Index objects with all of the real
2764 ** indices to follow */
2765 Index *pFirst; /* First of real indices on the table */
2766 memset(&sPk, 0, sizeof(Index));
drhbbbdc832013-10-22 18:01:40 +00002767 sPk.nKeyCol = 1;
dan39129ce2014-06-30 15:23:57 +00002768 sPk.nColumn = 1;
drh1c8148f2013-05-04 20:25:23 +00002769 sPk.aiColumn = &aiColumnPk;
dancfc9df72014-04-25 15:01:01 +00002770 sPk.aiRowLogEst = aiRowEstPk;
drh1c8148f2013-05-04 20:25:23 +00002771 sPk.onError = OE_Replace;
drh3495d202013-10-07 17:32:15 +00002772 sPk.pTable = pTab;
danaa9933c2014-04-24 20:04:49 +00002773 sPk.szIdxRow = pTab->szTabRow;
dancfc9df72014-04-25 15:01:01 +00002774 aiRowEstPk[0] = pTab->nRowLogEst;
2775 aiRowEstPk[1] = 0;
drh1c8148f2013-05-04 20:25:23 +00002776 pFirst = pSrc->pTab->pIndex;
drh8a48b9c2015-08-19 15:20:00 +00002777 if( pSrc->fg.notIndexed==0 ){
drh1c8148f2013-05-04 20:25:23 +00002778 /* The real indices of the table are only considered if the
2779 ** NOT INDEXED qualifier is omitted from the FROM clause */
2780 sPk.pNext = pFirst;
2781 }
2782 pProbe = &sPk;
2783 }
dancfc9df72014-04-25 15:01:01 +00002784 rSize = pTab->nRowLogEst;
drheb04de32013-05-10 15:16:30 +00002785 rLogSize = estLog(rSize);
2786
drhfeb56e02013-08-23 17:33:46 +00002787#ifndef SQLITE_OMIT_AUTOMATIC_INDEX
drheb04de32013-05-10 15:16:30 +00002788 /* Automatic indexes */
drh8a48b9c2015-08-19 15:20:00 +00002789 if( !pBuilder->pOrSet /* Not part of an OR optimization */
drhce943bc2016-05-19 18:56:33 +00002790 && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0
drh4fe425a2013-06-12 17:08:06 +00002791 && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0
drh8a48b9c2015-08-19 15:20:00 +00002792 && pSrc->pIBIndex==0 /* Has no INDEXED BY clause */
2793 && !pSrc->fg.notIndexed /* Has no NOT INDEXED clause */
drh76226dd2015-09-24 17:38:01 +00002794 && HasRowid(pTab) /* Not WITHOUT ROWID table. (FIXME: Why not?) */
drh8a48b9c2015-08-19 15:20:00 +00002795 && !pSrc->fg.isCorrelated /* Not a correlated subquery */
2796 && !pSrc->fg.isRecursive /* Not a recursive common table expression. */
drheb04de32013-05-10 15:16:30 +00002797 ){
2798 /* Generate auto-index WhereLoops */
drheb04de32013-05-10 15:16:30 +00002799 WhereTerm *pTerm;
2800 WhereTerm *pWCEnd = pWC->a + pWC->nTerm;
2801 for(pTerm=pWC->a; rc==SQLITE_OK && pTerm<pWCEnd; pTerm++){
drh79a13bf2013-05-31 20:28:28 +00002802 if( pTerm->prereqRight & pNew->maskSelf ) continue;
drheb04de32013-05-10 15:16:30 +00002803 if( termCanDriveIndex(pTerm, pSrc, 0) ){
2804 pNew->u.btree.nEq = 1;
drhc8bbce12014-10-21 01:05:09 +00002805 pNew->nSkip = 0;
drhef866372013-05-22 20:49:02 +00002806 pNew->u.btree.pIndex = 0;
drh4efc9292013-06-06 23:02:03 +00002807 pNew->nLTerm = 1;
2808 pNew->aLTerm[0] = pTerm;
drhe1e2e9a2013-06-13 15:16:53 +00002809 /* TUNING: One-time cost for computing the automatic index is
drh7e074332014-09-22 14:30:51 +00002810 ** estimated to be X*N*log2(N) where N is the number of rows in
2811 ** the table being indexed and where X is 7 (LogEst=28) for normal
2812 ** tables or 1.375 (LogEst=4) for views and subqueries. The value
2813 ** of X is smaller for views and subqueries so that the query planner
2814 ** will be more aggressive about generating automatic indexes for
2815 ** those objects, since there is no opportunity to add schema
2816 ** indexes on subqueries and views. */
2817 pNew->rSetup = rLogSize + rSize + 4;
2818 if( pTab->pSelect==0 && (pTab->tabFlags & TF_Ephemeral)==0 ){
2819 pNew->rSetup += 24;
2820 }
drhdbd94862014-07-23 23:57:42 +00002821 ApplyCostMultiplier(pNew->rSetup, pTab->costMult);
drh05d1bad2016-06-26 04:06:28 +00002822 if( pNew->rSetup<0 ) pNew->rSetup = 0;
drh986b3872013-06-28 21:12:20 +00002823 /* TUNING: Each index lookup yields 20 rows in the table. This
2824 ** is more than the usual guess of 10 rows, since we have no way
peter.d.reid60ec9142014-09-06 16:39:46 +00002825 ** of knowing how selective the index will ultimately be. It would
drh986b3872013-06-28 21:12:20 +00002826 ** not be unreasonable to make this value much larger. */
drhbf539c42013-10-05 18:16:02 +00002827 pNew->nOut = 43; assert( 43==sqlite3LogEst(20) );
drhb50596d2013-10-08 20:42:41 +00002828 pNew->rRun = sqlite3LogEstAdd(rLogSize,pNew->nOut);
drh986b3872013-06-28 21:12:20 +00002829 pNew->wsFlags = WHERE_AUTO_INDEX;
drh599d5762016-03-08 01:11:51 +00002830 pNew->prereq = mPrereq | pTerm->prereqRight;
drhcf8fa7a2013-05-10 20:26:22 +00002831 rc = whereLoopInsert(pBuilder, pNew);
drheb04de32013-05-10 15:16:30 +00002832 }
2833 }
2834 }
drhfeb56e02013-08-23 17:33:46 +00002835#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */
drh1c8148f2013-05-04 20:25:23 +00002836
2837 /* Loop over all indices
2838 */
drh23f98da2013-05-21 15:52:07 +00002839 for(; rc==SQLITE_OK && pProbe; pProbe=pProbe->pNext, iSortIdx++){
drh4bd5f732013-07-31 23:22:39 +00002840 if( pProbe->pPartIdxWhere!=0
dan08291692014-08-27 17:37:20 +00002841 && !whereUsablePartialIndex(pSrc->iCursor, pWC, pProbe->pPartIdxWhere) ){
2842 testcase( pNew->iTab!=pSrc->iCursor ); /* See ticket [98d973b8f5] */
drh4bd5f732013-07-31 23:22:39 +00002843 continue; /* Partial index inappropriate for this query */
2844 }
dan7de2a1f2014-04-28 20:11:20 +00002845 rSize = pProbe->aiRowLogEst[0];
drh5346e952013-05-08 14:14:26 +00002846 pNew->u.btree.nEq = 0;
dan71c57db2016-07-09 20:23:55 +00002847 pNew->u.btree.nBtm = 0;
2848 pNew->u.btree.nTop = 0;
drhc8bbce12014-10-21 01:05:09 +00002849 pNew->nSkip = 0;
drh4efc9292013-06-06 23:02:03 +00002850 pNew->nLTerm = 0;
drh23f98da2013-05-21 15:52:07 +00002851 pNew->iSortIdx = 0;
drhb8a8e8a2013-06-10 19:12:39 +00002852 pNew->rSetup = 0;
drh599d5762016-03-08 01:11:51 +00002853 pNew->prereq = mPrereq;
drh74f91d42013-06-19 18:01:44 +00002854 pNew->nOut = rSize;
drh23f98da2013-05-21 15:52:07 +00002855 pNew->u.btree.pIndex = pProbe;
2856 b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor);
drh53cfbe92013-06-13 17:28:22 +00002857 /* The ONEPASS_DESIRED flags never occurs together with ORDER BY */
2858 assert( (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || b==0 );
drh43fe25f2013-05-07 23:06:23 +00002859 if( pProbe->tnum<=0 ){
2860 /* Integer primary key index */
2861 pNew->wsFlags = WHERE_IPK;
drh23f98da2013-05-21 15:52:07 +00002862
2863 /* Full table scan */
drhd044d202013-05-31 12:43:55 +00002864 pNew->iSortIdx = b ? iSortIdx : 0;
danaa9933c2014-04-24 20:04:49 +00002865 /* TUNING: Cost of full table scan is (N*3.0). */
2866 pNew->rRun = rSize + 16;
drhdbd94862014-07-23 23:57:42 +00002867 ApplyCostMultiplier(pNew->rRun, pTab->costMult);
drhd8b77e22014-09-06 01:35:57 +00002868 whereLoopOutputAdjust(pWC, pNew, rSize);
drh23f98da2013-05-21 15:52:07 +00002869 rc = whereLoopInsert(pBuilder, pNew);
drhcca9f3d2013-09-06 15:23:29 +00002870 pNew->nOut = rSize;
drh23f98da2013-05-21 15:52:07 +00002871 if( rc ) break;
drh43fe25f2013-05-07 23:06:23 +00002872 }else{
drhec95c442013-10-23 01:57:32 +00002873 Bitmask m;
2874 if( pProbe->isCovering ){
2875 pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED;
2876 m = 0;
2877 }else{
2878 m = pSrc->colUsed & ~columnsInIndex(pProbe);
2879 pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED;
2880 }
drh1c8148f2013-05-04 20:25:23 +00002881
drh23f98da2013-05-21 15:52:07 +00002882 /* Full scan via index */
drh53cfbe92013-06-13 17:28:22 +00002883 if( b
drh702ba9f2013-11-07 21:25:13 +00002884 || !HasRowid(pTab)
drh8dc570b2016-06-08 18:07:21 +00002885 || pProbe->pPartIdxWhere!=0
drh53cfbe92013-06-13 17:28:22 +00002886 || ( m==0
2887 && pProbe->bUnordered==0
drh702ba9f2013-11-07 21:25:13 +00002888 && (pProbe->szIdxRow<pTab->szTabRow)
drh53cfbe92013-06-13 17:28:22 +00002889 && (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0
2890 && sqlite3GlobalConfig.bUseCis
2891 && OptimizationEnabled(pWInfo->pParse->db, SQLITE_CoverIdxScan)
2892 )
drhe3b7c922013-06-03 19:17:40 +00002893 ){
drh23f98da2013-05-21 15:52:07 +00002894 pNew->iSortIdx = b ? iSortIdx : 0;
danaa9933c2014-04-24 20:04:49 +00002895
2896 /* The cost of visiting the index rows is N*K, where K is
2897 ** between 1.1 and 3.0, depending on the relative sizes of the
drh2409f8a2016-07-27 18:27:02 +00002898 ** index and table rows. */
danaa9933c2014-04-24 20:04:49 +00002899 pNew->rRun = rSize + 1 + (15*pProbe->szIdxRow)/pTab->szTabRow;
2900 if( m!=0 ){
drh2409f8a2016-07-27 18:27:02 +00002901 /* If this is a non-covering index scan, add in the cost of
2902 ** doing table lookups. The cost will be 3x the number of
2903 ** lookups. Take into account WHERE clause terms that can be
2904 ** satisfied using just the index, and that do not require a
2905 ** table lookup. */
2906 LogEst nLookup = rSize + 16; /* Base cost: N*3 */
2907 int ii;
2908 int iCur = pSrc->iCursor;
mistachkin19e76b22016-07-30 18:54:54 +00002909 WhereClause *pWC2 = &pWInfo->sWC;
2910 for(ii=0; ii<pWC2->nTerm; ii++){
2911 WhereTerm *pTerm = &pWC2->a[ii];
drh2409f8a2016-07-27 18:27:02 +00002912 if( !sqlite3ExprCoveredByIndex(pTerm->pExpr, iCur, pProbe) ){
2913 break;
2914 }
2915 /* pTerm can be evaluated using just the index. So reduce
2916 ** the expected number of table lookups accordingly */
2917 if( pTerm->truthProb<=0 ){
2918 nLookup += pTerm->truthProb;
2919 }else{
2920 nLookup--;
2921 if( pTerm->eOperator & (WO_EQ|WO_IS) ) nLookup -= 19;
2922 }
2923 }
2924
2925 pNew->rRun = sqlite3LogEstAdd(pNew->rRun, nLookup);
drhe1e2e9a2013-06-13 15:16:53 +00002926 }
drhdbd94862014-07-23 23:57:42 +00002927 ApplyCostMultiplier(pNew->rRun, pTab->costMult);
drhd8b77e22014-09-06 01:35:57 +00002928 whereLoopOutputAdjust(pWC, pNew, rSize);
drh23f98da2013-05-21 15:52:07 +00002929 rc = whereLoopInsert(pBuilder, pNew);
drhcca9f3d2013-09-06 15:23:29 +00002930 pNew->nOut = rSize;
drh23f98da2013-05-21 15:52:07 +00002931 if( rc ) break;
2932 }
2933 }
dan7a419232013-08-06 20:01:43 +00002934
drhb8a8e8a2013-06-10 19:12:39 +00002935 rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0);
drh1435a9a2013-08-27 23:15:44 +00002936#ifdef SQLITE_ENABLE_STAT3_OR_STAT4
dan87cd9322013-08-07 15:52:41 +00002937 sqlite3Stat4ProbeFree(pBuilder->pRec);
2938 pBuilder->nRecValid = 0;
2939 pBuilder->pRec = 0;
danddc2d6e2013-08-06 20:15:06 +00002940#endif
drh1c8148f2013-05-04 20:25:23 +00002941
2942 /* If there was an INDEXED BY clause, then only that one index is
2943 ** considered. */
drh8a48b9c2015-08-19 15:20:00 +00002944 if( pSrc->pIBIndex ) break;
drh1c8148f2013-05-04 20:25:23 +00002945 }
drh5346e952013-05-08 14:14:26 +00002946 return rc;
drhf1b5f5b2013-05-02 00:15:01 +00002947}
2948
drh8636e9c2013-06-11 01:50:08 +00002949#ifndef SQLITE_OMIT_VIRTUALTABLE
dan115305f2016-03-05 17:29:08 +00002950
2951/*
2952** Argument pIdxInfo is already populated with all constraints that may
2953** be used by the virtual table identified by pBuilder->pNew->iTab. This
2954** function marks a subset of those constraints usable, invokes the
2955** xBestIndex method and adds the returned plan to pBuilder.
2956**
2957** A constraint is marked usable if:
2958**
2959** * Argument mUsable indicates that its prerequisites are available, and
2960**
2961** * It is not one of the operators specified in the mExclude mask passed
2962** as the fourth argument (which in practice is either WO_IN or 0).
2963**
drh599d5762016-03-08 01:11:51 +00002964** Argument mPrereq is a mask of tables that must be scanned before the
dan115305f2016-03-05 17:29:08 +00002965** virtual table in question. These are added to the plans prerequisites
2966** before it is added to pBuilder.
2967**
2968** Output parameter *pbIn is set to true if the plan added to pBuilder
2969** uses one or more WO_IN terms, or false otherwise.
2970*/
2971static int whereLoopAddVirtualOne(
2972 WhereLoopBuilder *pBuilder,
drh8426e362016-03-08 01:32:30 +00002973 Bitmask mPrereq, /* Mask of tables that must be used. */
2974 Bitmask mUsable, /* Mask of usable tables */
2975 u16 mExclude, /* Exclude terms using these operators */
dan115305f2016-03-05 17:29:08 +00002976 sqlite3_index_info *pIdxInfo, /* Populated object for xBestIndex */
dan6256c1c2016-08-08 20:15:41 +00002977 u16 mNoOmit, /* Do not omit these constraints */
dan115305f2016-03-05 17:29:08 +00002978 int *pbIn /* OUT: True if plan uses an IN(...) op */
2979){
2980 WhereClause *pWC = pBuilder->pWC;
2981 struct sqlite3_index_constraint *pIdxCons;
2982 struct sqlite3_index_constraint_usage *pUsage = pIdxInfo->aConstraintUsage;
2983 int i;
2984 int mxTerm;
2985 int rc = SQLITE_OK;
2986 WhereLoop *pNew = pBuilder->pNew;
2987 Parse *pParse = pBuilder->pWInfo->pParse;
2988 struct SrcList_item *pSrc = &pBuilder->pWInfo->pTabList->a[pNew->iTab];
2989 int nConstraint = pIdxInfo->nConstraint;
2990
drh599d5762016-03-08 01:11:51 +00002991 assert( (mUsable & mPrereq)==mPrereq );
dan115305f2016-03-05 17:29:08 +00002992 *pbIn = 0;
drh599d5762016-03-08 01:11:51 +00002993 pNew->prereq = mPrereq;
dan115305f2016-03-05 17:29:08 +00002994
2995 /* Set the usable flag on the subset of constraints identified by
2996 ** arguments mUsable and mExclude. */
2997 pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint;
2998 for(i=0; i<nConstraint; i++, pIdxCons++){
2999 WhereTerm *pTerm = &pWC->a[pIdxCons->iTermOffset];
3000 pIdxCons->usable = 0;
3001 if( (pTerm->prereqRight & mUsable)==pTerm->prereqRight
3002 && (pTerm->eOperator & mExclude)==0
3003 ){
3004 pIdxCons->usable = 1;
3005 }
3006 }
3007
3008 /* Initialize the output fields of the sqlite3_index_info structure */
3009 memset(pUsage, 0, sizeof(pUsage[0])*nConstraint);
drhd1cca3b2016-03-08 23:44:48 +00003010 assert( pIdxInfo->needToFreeIdxStr==0 );
dan115305f2016-03-05 17:29:08 +00003011 pIdxInfo->idxStr = 0;
3012 pIdxInfo->idxNum = 0;
dan115305f2016-03-05 17:29:08 +00003013 pIdxInfo->orderByConsumed = 0;
3014 pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2;
3015 pIdxInfo->estimatedRows = 25;
3016 pIdxInfo->idxFlags = 0;
3017 pIdxInfo->colUsed = (sqlite3_int64)pSrc->colUsed;
3018
3019 /* Invoke the virtual table xBestIndex() method */
3020 rc = vtabBestIndex(pParse, pSrc->pTab, pIdxInfo);
3021 if( rc ) return rc;
3022
3023 mxTerm = -1;
3024 assert( pNew->nLSlot>=nConstraint );
3025 for(i=0; i<nConstraint; i++) pNew->aLTerm[i] = 0;
3026 pNew->u.vtab.omitMask = 0;
3027 pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint;
3028 for(i=0; i<nConstraint; i++, pIdxCons++){
3029 int iTerm;
3030 if( (iTerm = pUsage[i].argvIndex - 1)>=0 ){
3031 WhereTerm *pTerm;
3032 int j = pIdxCons->iTermOffset;
3033 if( iTerm>=nConstraint
3034 || j<0
3035 || j>=pWC->nTerm
3036 || pNew->aLTerm[iTerm]!=0
drh6de32e72016-03-08 02:59:33 +00003037 || pIdxCons->usable==0
dan115305f2016-03-05 17:29:08 +00003038 ){
3039 rc = SQLITE_ERROR;
drh6de32e72016-03-08 02:59:33 +00003040 sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pTab->zName);
dan115305f2016-03-05 17:29:08 +00003041 return rc;
3042 }
3043 testcase( iTerm==nConstraint-1 );
3044 testcase( j==0 );
3045 testcase( j==pWC->nTerm-1 );
3046 pTerm = &pWC->a[j];
3047 pNew->prereq |= pTerm->prereqRight;
3048 assert( iTerm<pNew->nLSlot );
3049 pNew->aLTerm[iTerm] = pTerm;
3050 if( iTerm>mxTerm ) mxTerm = iTerm;
3051 testcase( iTerm==15 );
3052 testcase( iTerm==16 );
3053 if( iTerm<16 && pUsage[i].omit ) pNew->u.vtab.omitMask |= 1<<iTerm;
3054 if( (pTerm->eOperator & WO_IN)!=0 ){
3055 /* A virtual table that is constrained by an IN clause may not
3056 ** consume the ORDER BY clause because (1) the order of IN terms
3057 ** is not necessarily related to the order of output terms and
3058 ** (2) Multiple outputs from a single IN value will not merge
3059 ** together. */
3060 pIdxInfo->orderByConsumed = 0;
3061 pIdxInfo->idxFlags &= ~SQLITE_INDEX_SCAN_UNIQUE;
drh6de32e72016-03-08 02:59:33 +00003062 *pbIn = 1; assert( (mExclude & WO_IN)==0 );
dan115305f2016-03-05 17:29:08 +00003063 }
3064 }
3065 }
dan6256c1c2016-08-08 20:15:41 +00003066 pNew->u.vtab.omitMask &= ~mNoOmit;
dan115305f2016-03-05 17:29:08 +00003067
3068 pNew->nLTerm = mxTerm+1;
3069 assert( pNew->nLTerm<=pNew->nLSlot );
3070 pNew->u.vtab.idxNum = pIdxInfo->idxNum;
3071 pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr;
3072 pIdxInfo->needToFreeIdxStr = 0;
3073 pNew->u.vtab.idxStr = pIdxInfo->idxStr;
3074 pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ?
3075 pIdxInfo->nOrderBy : 0);
3076 pNew->rSetup = 0;
3077 pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost);
3078 pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows);
3079
3080 /* Set the WHERE_ONEROW flag if the xBestIndex() method indicated
3081 ** that the scan will visit at most one row. Clear it otherwise. */
3082 if( pIdxInfo->idxFlags & SQLITE_INDEX_SCAN_UNIQUE ){
3083 pNew->wsFlags |= WHERE_ONEROW;
3084 }else{
3085 pNew->wsFlags &= ~WHERE_ONEROW;
3086 }
drhbacbbcc2016-03-09 12:35:18 +00003087 rc = whereLoopInsert(pBuilder, pNew);
dan115305f2016-03-05 17:29:08 +00003088 if( pNew->u.vtab.needFree ){
3089 sqlite3_free(pNew->u.vtab.idxStr);
3090 pNew->u.vtab.needFree = 0;
3091 }
drh3349d9b2016-03-08 23:18:51 +00003092 WHERETRACE(0xffff, (" bIn=%d prereqIn=%04llx prereqOut=%04llx\n",
3093 *pbIn, (sqlite3_uint64)mPrereq,
3094 (sqlite3_uint64)(pNew->prereq & ~mPrereq)));
dan115305f2016-03-05 17:29:08 +00003095
drhbacbbcc2016-03-09 12:35:18 +00003096 return rc;
dan115305f2016-03-05 17:29:08 +00003097}
3098
3099
drhf1b5f5b2013-05-02 00:15:01 +00003100/*
drh0823c892013-05-11 00:06:23 +00003101** Add all WhereLoop objects for a table of the join identified by
3102** pBuilder->pNew->iTab. That table is guaranteed to be a virtual table.
dan4f20cd42015-06-08 18:05:54 +00003103**
drh599d5762016-03-08 01:11:51 +00003104** If there are no LEFT or CROSS JOIN joins in the query, both mPrereq and
3105** mUnusable are set to 0. Otherwise, mPrereq is a mask of all FROM clause
dan4f20cd42015-06-08 18:05:54 +00003106** entries that occur before the virtual table in the FROM clause and are
3107** separated from it by at least one LEFT or CROSS JOIN. Similarly, the
3108** mUnusable mask contains all FROM clause entries that occur after the
3109** virtual table and are separated from it by at least one LEFT or
3110** CROSS JOIN.
3111**
3112** For example, if the query were:
3113**
3114** ... FROM t1, t2 LEFT JOIN t3, t4, vt CROSS JOIN t5, t6;
3115**
drh599d5762016-03-08 01:11:51 +00003116** then mPrereq corresponds to (t1, t2) and mUnusable to (t5, t6).
dan4f20cd42015-06-08 18:05:54 +00003117**
drh599d5762016-03-08 01:11:51 +00003118** All the tables in mPrereq must be scanned before the current virtual
dan4f20cd42015-06-08 18:05:54 +00003119** table. So any terms for which all prerequisites are satisfied by
drh599d5762016-03-08 01:11:51 +00003120** mPrereq may be specified as "usable" in all calls to xBestIndex.
dan4f20cd42015-06-08 18:05:54 +00003121** Conversely, all tables in mUnusable must be scanned after the current
3122** virtual table, so any terms for which the prerequisites overlap with
3123** mUnusable should always be configured as "not-usable" for xBestIndex.
drhf1b5f5b2013-05-02 00:15:01 +00003124*/
drh5346e952013-05-08 14:14:26 +00003125static int whereLoopAddVirtual(
danff4b23b2013-11-12 12:17:16 +00003126 WhereLoopBuilder *pBuilder, /* WHERE clause information */
drh599d5762016-03-08 01:11:51 +00003127 Bitmask mPrereq, /* Tables that must be scanned before this one */
dan4f20cd42015-06-08 18:05:54 +00003128 Bitmask mUnusable /* Tables that must be scanned after this one */
drhf1b5f5b2013-05-02 00:15:01 +00003129){
dan115305f2016-03-05 17:29:08 +00003130 int rc = SQLITE_OK; /* Return code */
drh70d18342013-06-06 19:16:33 +00003131 WhereInfo *pWInfo; /* WHERE analysis context */
drh5346e952013-05-08 14:14:26 +00003132 Parse *pParse; /* The parsing context */
3133 WhereClause *pWC; /* The WHERE clause */
3134 struct SrcList_item *pSrc; /* The FROM clause term to search */
dan115305f2016-03-05 17:29:08 +00003135 sqlite3_index_info *p; /* Object to pass to xBestIndex() */
3136 int nConstraint; /* Number of constraints in p */
3137 int bIn; /* True if plan uses IN(...) operator */
drh5346e952013-05-08 14:14:26 +00003138 WhereLoop *pNew;
dan115305f2016-03-05 17:29:08 +00003139 Bitmask mBest; /* Tables used by best possible plan */
dan6256c1c2016-08-08 20:15:41 +00003140 u16 mNoOmit;
drh5346e952013-05-08 14:14:26 +00003141
drh599d5762016-03-08 01:11:51 +00003142 assert( (mPrereq & mUnusable)==0 );
drh70d18342013-06-06 19:16:33 +00003143 pWInfo = pBuilder->pWInfo;
3144 pParse = pWInfo->pParse;
drh5346e952013-05-08 14:14:26 +00003145 pWC = pBuilder->pWC;
drh5346e952013-05-08 14:14:26 +00003146 pNew = pBuilder->pNew;
drh70d18342013-06-06 19:16:33 +00003147 pSrc = &pWInfo->pTabList->a[pNew->iTab];
dan115305f2016-03-05 17:29:08 +00003148 assert( IsVirtual(pSrc->pTab) );
dan6256c1c2016-08-08 20:15:41 +00003149 p = allocateIndexInfo(pParse, pWC, mUnusable, pSrc, pBuilder->pOrderBy,
3150 &mNoOmit);
dan115305f2016-03-05 17:29:08 +00003151 if( p==0 ) return SQLITE_NOMEM_BKPT;
drh5346e952013-05-08 14:14:26 +00003152 pNew->rSetup = 0;
3153 pNew->wsFlags = WHERE_VIRTUALTABLE;
drh4efc9292013-06-06 23:02:03 +00003154 pNew->nLTerm = 0;
drh5346e952013-05-08 14:14:26 +00003155 pNew->u.vtab.needFree = 0;
dan115305f2016-03-05 17:29:08 +00003156 nConstraint = p->nConstraint;
3157 if( whereLoopResize(pParse->db, pNew, nConstraint) ){
3158 sqlite3DbFree(pParse->db, p);
mistachkinfad30392016-02-13 23:43:46 +00003159 return SQLITE_NOMEM_BKPT;
drh7963b0e2013-06-17 21:37:40 +00003160 }
drh5346e952013-05-08 14:14:26 +00003161
dan115305f2016-03-05 17:29:08 +00003162 /* First call xBestIndex() with all constraints usable. */
drh3349d9b2016-03-08 23:18:51 +00003163 WHERETRACE(0x40, (" VirtualOne: all usable\n"));
dan6256c1c2016-08-08 20:15:41 +00003164 rc = whereLoopAddVirtualOne(pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn);
dan076e0f92015-09-28 15:20:58 +00003165
dan115305f2016-03-05 17:29:08 +00003166 /* If the call to xBestIndex() with all terms enabled produced a plan
drh3349d9b2016-03-08 23:18:51 +00003167 ** that does not require any source tables (IOW: a plan with mBest==0),
3168 ** then there is no point in making any further calls to xBestIndex()
3169 ** since they will all return the same result (if the xBestIndex()
3170 ** implementation is sane). */
3171 if( rc==SQLITE_OK && (mBest = (pNew->prereq & ~mPrereq))!=0 ){
dan115305f2016-03-05 17:29:08 +00003172 int seenZero = 0; /* True if a plan with no prereqs seen */
3173 int seenZeroNoIN = 0; /* Plan with no prereqs and no IN(...) seen */
3174 Bitmask mPrev = 0;
3175 Bitmask mBestNoIn = 0;
3176
3177 /* If the plan produced by the earlier call uses an IN(...) term, call
3178 ** xBestIndex again, this time with IN(...) terms disabled. */
drh3349d9b2016-03-08 23:18:51 +00003179 if( bIn ){
3180 WHERETRACE(0x40, (" VirtualOne: all usable w/o IN\n"));
dan6256c1c2016-08-08 20:15:41 +00003181 rc = whereLoopAddVirtualOne(
3182 pBuilder, mPrereq, ALLBITS, WO_IN, p, mNoOmit, &bIn);
drh6de32e72016-03-08 02:59:33 +00003183 assert( bIn==0 );
drh599d5762016-03-08 01:11:51 +00003184 mBestNoIn = pNew->prereq & ~mPrereq;
dan115305f2016-03-05 17:29:08 +00003185 if( mBestNoIn==0 ){
3186 seenZero = 1;
drh6de32e72016-03-08 02:59:33 +00003187 seenZeroNoIN = 1;
drh5346e952013-05-08 14:14:26 +00003188 }
3189 }
drh5346e952013-05-08 14:14:26 +00003190
drh599d5762016-03-08 01:11:51 +00003191 /* Call xBestIndex once for each distinct value of (prereqRight & ~mPrereq)
dan115305f2016-03-05 17:29:08 +00003192 ** in the set of terms that apply to the current virtual table. */
3193 while( rc==SQLITE_OK ){
3194 int i;
drh8426e362016-03-08 01:32:30 +00003195 Bitmask mNext = ALLBITS;
dan115305f2016-03-05 17:29:08 +00003196 assert( mNext>0 );
3197 for(i=0; i<nConstraint; i++){
3198 Bitmask mThis = (
drh599d5762016-03-08 01:11:51 +00003199 pWC->a[p->aConstraint[i].iTermOffset].prereqRight & ~mPrereq
dan115305f2016-03-05 17:29:08 +00003200 );
3201 if( mThis>mPrev && mThis<mNext ) mNext = mThis;
3202 }
3203 mPrev = mNext;
drh8426e362016-03-08 01:32:30 +00003204 if( mNext==ALLBITS ) break;
dan115305f2016-03-05 17:29:08 +00003205 if( mNext==mBest || mNext==mBestNoIn ) continue;
drh3349d9b2016-03-08 23:18:51 +00003206 WHERETRACE(0x40, (" VirtualOne: mPrev=%04llx mNext=%04llx\n",
3207 (sqlite3_uint64)mPrev, (sqlite3_uint64)mNext));
dan6256c1c2016-08-08 20:15:41 +00003208 rc = whereLoopAddVirtualOne(
3209 pBuilder, mPrereq, mNext|mPrereq, 0, p, mNoOmit, &bIn);
drh599d5762016-03-08 01:11:51 +00003210 if( pNew->prereq==mPrereq ){
dan115305f2016-03-05 17:29:08 +00003211 seenZero = 1;
3212 if( bIn==0 ) seenZeroNoIN = 1;
3213 }
3214 }
3215
3216 /* If the calls to xBestIndex() in the above loop did not find a plan
3217 ** that requires no source tables at all (i.e. one guaranteed to be
3218 ** usable), make a call here with all source tables disabled */
3219 if( rc==SQLITE_OK && seenZero==0 ){
drh3349d9b2016-03-08 23:18:51 +00003220 WHERETRACE(0x40, (" VirtualOne: all disabled\n"));
dan6256c1c2016-08-08 20:15:41 +00003221 rc = whereLoopAddVirtualOne(
3222 pBuilder, mPrereq, mPrereq, 0, p, mNoOmit, &bIn);
dan115305f2016-03-05 17:29:08 +00003223 if( bIn==0 ) seenZeroNoIN = 1;
3224 }
3225
3226 /* If the calls to xBestIndex() have so far failed to find a plan
3227 ** that requires no source tables at all and does not use an IN(...)
3228 ** operator, make a final call to obtain one here. */
3229 if( rc==SQLITE_OK && seenZeroNoIN==0 ){
drh3349d9b2016-03-08 23:18:51 +00003230 WHERETRACE(0x40, (" VirtualOne: all disabled and w/o IN\n"));
dan6256c1c2016-08-08 20:15:41 +00003231 rc = whereLoopAddVirtualOne(
3232 pBuilder, mPrereq, mPrereq, WO_IN, p, mNoOmit, &bIn);
dan115305f2016-03-05 17:29:08 +00003233 }
3234 }
3235
3236 if( p->needToFreeIdxStr ) sqlite3_free(p->idxStr);
3237 sqlite3DbFree(pParse->db, p);
drh5346e952013-05-08 14:14:26 +00003238 return rc;
drhf1b5f5b2013-05-02 00:15:01 +00003239}
drh8636e9c2013-06-11 01:50:08 +00003240#endif /* SQLITE_OMIT_VIRTUALTABLE */
drhf1b5f5b2013-05-02 00:15:01 +00003241
3242/*
drhcf8fa7a2013-05-10 20:26:22 +00003243** Add WhereLoop entries to handle OR terms. This works for either
3244** btrees or virtual tables.
3245*/
dan4f20cd42015-06-08 18:05:54 +00003246static int whereLoopAddOr(
3247 WhereLoopBuilder *pBuilder,
drh599d5762016-03-08 01:11:51 +00003248 Bitmask mPrereq,
dan4f20cd42015-06-08 18:05:54 +00003249 Bitmask mUnusable
3250){
drh70d18342013-06-06 19:16:33 +00003251 WhereInfo *pWInfo = pBuilder->pWInfo;
drhcf8fa7a2013-05-10 20:26:22 +00003252 WhereClause *pWC;
3253 WhereLoop *pNew;
3254 WhereTerm *pTerm, *pWCEnd;
3255 int rc = SQLITE_OK;
3256 int iCur;
3257 WhereClause tempWC;
3258 WhereLoopBuilder sSubBuild;
dan5da73e12014-04-30 18:11:55 +00003259 WhereOrSet sSum, sCur;
drhcf8fa7a2013-05-10 20:26:22 +00003260 struct SrcList_item *pItem;
3261
drhcf8fa7a2013-05-10 20:26:22 +00003262 pWC = pBuilder->pWC;
drhcf8fa7a2013-05-10 20:26:22 +00003263 pWCEnd = pWC->a + pWC->nTerm;
3264 pNew = pBuilder->pNew;
drh77dfd5b2013-08-19 11:15:48 +00003265 memset(&sSum, 0, sizeof(sSum));
drh186ad8c2013-10-08 18:40:37 +00003266 pItem = pWInfo->pTabList->a + pNew->iTab;
3267 iCur = pItem->iCursor;
drhcf8fa7a2013-05-10 20:26:22 +00003268
3269 for(pTerm=pWC->a; pTerm<pWCEnd && rc==SQLITE_OK; pTerm++){
3270 if( (pTerm->eOperator & WO_OR)!=0
3271 && (pTerm->u.pOrInfo->indexable & pNew->maskSelf)!=0
3272 ){
3273 WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc;
3274 WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm];
3275 WhereTerm *pOrTerm;
drhaa32e3c2013-07-16 21:31:23 +00003276 int once = 1;
3277 int i, j;
drh783dece2013-06-05 17:53:43 +00003278
drh783dece2013-06-05 17:53:43 +00003279 sSubBuild = *pBuilder;
3280 sSubBuild.pOrderBy = 0;
drhaa32e3c2013-07-16 21:31:23 +00003281 sSubBuild.pOrSet = &sCur;
drhcf8fa7a2013-05-10 20:26:22 +00003282
drh0a99ba32014-09-30 17:03:35 +00003283 WHERETRACE(0x200, ("Begin processing OR-clause %p\n", pTerm));
drhc7f0d222013-06-19 03:27:12 +00003284 for(pOrTerm=pOrWC->a; pOrTerm<pOrWCEnd; pOrTerm++){
drh783dece2013-06-05 17:53:43 +00003285 if( (pOrTerm->eOperator & WO_AND)!=0 ){
drhcf8fa7a2013-05-10 20:26:22 +00003286 sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc;
3287 }else if( pOrTerm->leftCursor==iCur ){
drh70d18342013-06-06 19:16:33 +00003288 tempWC.pWInfo = pWC->pWInfo;
drh783dece2013-06-05 17:53:43 +00003289 tempWC.pOuter = pWC;
3290 tempWC.op = TK_AND;
drh783dece2013-06-05 17:53:43 +00003291 tempWC.nTerm = 1;
drhcf8fa7a2013-05-10 20:26:22 +00003292 tempWC.a = pOrTerm;
3293 sSubBuild.pWC = &tempWC;
3294 }else{
3295 continue;
3296 }
drhaa32e3c2013-07-16 21:31:23 +00003297 sCur.n = 0;
drh52651492014-09-30 14:14:19 +00003298#ifdef WHERETRACE_ENABLED
drh0a99ba32014-09-30 17:03:35 +00003299 WHERETRACE(0x200, ("OR-term %d of %p has %d subterms:\n",
3300 (int)(pOrTerm-pOrWC->a), pTerm, sSubBuild.pWC->nTerm));
3301 if( sqlite3WhereTrace & 0x400 ){
drhc84a4022016-05-27 12:30:20 +00003302 sqlite3WhereClausePrint(sSubBuild.pWC);
drh52651492014-09-30 14:14:19 +00003303 }
3304#endif
drh8636e9c2013-06-11 01:50:08 +00003305#ifndef SQLITE_OMIT_VIRTUALTABLE
drhcf8fa7a2013-05-10 20:26:22 +00003306 if( IsVirtual(pItem->pTab) ){
drh599d5762016-03-08 01:11:51 +00003307 rc = whereLoopAddVirtual(&sSubBuild, mPrereq, mUnusable);
drh8636e9c2013-06-11 01:50:08 +00003308 }else
3309#endif
3310 {
drh599d5762016-03-08 01:11:51 +00003311 rc = whereLoopAddBtree(&sSubBuild, mPrereq);
drhcf8fa7a2013-05-10 20:26:22 +00003312 }
drh36be4c42014-09-30 17:31:23 +00003313 if( rc==SQLITE_OK ){
drh599d5762016-03-08 01:11:51 +00003314 rc = whereLoopAddOr(&sSubBuild, mPrereq, mUnusable);
drh36be4c42014-09-30 17:31:23 +00003315 }
drhaa32e3c2013-07-16 21:31:23 +00003316 assert( rc==SQLITE_OK || sCur.n==0 );
3317 if( sCur.n==0 ){
3318 sSum.n = 0;
3319 break;
3320 }else if( once ){
3321 whereOrMove(&sSum, &sCur);
3322 once = 0;
3323 }else{
dan5da73e12014-04-30 18:11:55 +00003324 WhereOrSet sPrev;
drhaa32e3c2013-07-16 21:31:23 +00003325 whereOrMove(&sPrev, &sSum);
3326 sSum.n = 0;
3327 for(i=0; i<sPrev.n; i++){
3328 for(j=0; j<sCur.n; j++){
3329 whereOrInsert(&sSum, sPrev.a[i].prereq | sCur.a[j].prereq,
drhbf539c42013-10-05 18:16:02 +00003330 sqlite3LogEstAdd(sPrev.a[i].rRun, sCur.a[j].rRun),
3331 sqlite3LogEstAdd(sPrev.a[i].nOut, sCur.a[j].nOut));
drhaa32e3c2013-07-16 21:31:23 +00003332 }
3333 }
3334 }
drhcf8fa7a2013-05-10 20:26:22 +00003335 }
drhaa32e3c2013-07-16 21:31:23 +00003336 pNew->nLTerm = 1;
3337 pNew->aLTerm[0] = pTerm;
3338 pNew->wsFlags = WHERE_MULTI_OR;
3339 pNew->rSetup = 0;
3340 pNew->iSortIdx = 0;
3341 memset(&pNew->u, 0, sizeof(pNew->u));
3342 for(i=0; rc==SQLITE_OK && i<sSum.n; i++){
dan5da73e12014-04-30 18:11:55 +00003343 /* TUNING: Currently sSum.a[i].rRun is set to the sum of the costs
3344 ** of all sub-scans required by the OR-scan. However, due to rounding
3345 ** errors, it may be that the cost of the OR-scan is equal to its
3346 ** most expensive sub-scan. Add the smallest possible penalty
3347 ** (equivalent to multiplying the cost by 1.07) to ensure that
3348 ** this does not happen. Otherwise, for WHERE clauses such as the
3349 ** following where there is an index on "y":
3350 **
3351 ** WHERE likelihood(x=?, 0.99) OR y=?
3352 **
3353 ** the planner may elect to "OR" together a full-table scan and an
3354 ** index lookup. And other similarly odd results. */
3355 pNew->rRun = sSum.a[i].rRun + 1;
drhaa32e3c2013-07-16 21:31:23 +00003356 pNew->nOut = sSum.a[i].nOut;
3357 pNew->prereq = sSum.a[i].prereq;
drhfd5874d2013-06-12 14:52:39 +00003358 rc = whereLoopInsert(pBuilder, pNew);
3359 }
drh0a99ba32014-09-30 17:03:35 +00003360 WHERETRACE(0x200, ("End processing OR-clause %p\n", pTerm));
drhcf8fa7a2013-05-10 20:26:22 +00003361 }
3362 }
3363 return rc;
3364}
3365
3366/*
drhf1b5f5b2013-05-02 00:15:01 +00003367** Add all WhereLoop objects for all tables
3368*/
drh5346e952013-05-08 14:14:26 +00003369static int whereLoopAddAll(WhereLoopBuilder *pBuilder){
drh70d18342013-06-06 19:16:33 +00003370 WhereInfo *pWInfo = pBuilder->pWInfo;
drh599d5762016-03-08 01:11:51 +00003371 Bitmask mPrereq = 0;
drhf1b5f5b2013-05-02 00:15:01 +00003372 Bitmask mPrior = 0;
3373 int iTab;
drh70d18342013-06-06 19:16:33 +00003374 SrcList *pTabList = pWInfo->pTabList;
drhf1b5f5b2013-05-02 00:15:01 +00003375 struct SrcList_item *pItem;
dan4f20cd42015-06-08 18:05:54 +00003376 struct SrcList_item *pEnd = &pTabList->a[pWInfo->nLevel];
drh70d18342013-06-06 19:16:33 +00003377 sqlite3 *db = pWInfo->pParse->db;
drh5346e952013-05-08 14:14:26 +00003378 int rc = SQLITE_OK;
drhb8a8e8a2013-06-10 19:12:39 +00003379 WhereLoop *pNew;
dan35175bf2015-06-08 18:48:29 +00003380 u8 priorJointype = 0;
drhf1b5f5b2013-05-02 00:15:01 +00003381
3382 /* Loop over the tables in the join, from left to right */
drhb8a8e8a2013-06-10 19:12:39 +00003383 pNew = pBuilder->pNew;
drha2014152013-06-07 00:29:23 +00003384 whereLoopInit(pNew);
dan4f20cd42015-06-08 18:05:54 +00003385 for(iTab=0, pItem=pTabList->a; pItem<pEnd; iTab++, pItem++){
3386 Bitmask mUnusable = 0;
drhb2a90f02013-05-10 03:30:49 +00003387 pNew->iTab = iTab;
drh6f82e852015-06-06 20:12:09 +00003388 pNew->maskSelf = sqlite3WhereGetMask(&pWInfo->sMaskSet, pItem->iCursor);
drh8a48b9c2015-08-19 15:20:00 +00003389 if( ((pItem->fg.jointype|priorJointype) & (JT_LEFT|JT_CROSS))!=0 ){
dan4f20cd42015-06-08 18:05:54 +00003390 /* This condition is true when pItem is the FROM clause term on the
3391 ** right-hand-side of a LEFT or CROSS JOIN. */
drh599d5762016-03-08 01:11:51 +00003392 mPrereq = mPrior;
drhf1b5f5b2013-05-02 00:15:01 +00003393 }
drh8a48b9c2015-08-19 15:20:00 +00003394 priorJointype = pItem->fg.jointype;
drhec593592016-06-23 12:35:04 +00003395#ifndef SQLITE_OMIT_VIRTUALTABLE
drhb2a90f02013-05-10 03:30:49 +00003396 if( IsVirtual(pItem->pTab) ){
dan4f20cd42015-06-08 18:05:54 +00003397 struct SrcList_item *p;
3398 for(p=&pItem[1]; p<pEnd; p++){
drh8a48b9c2015-08-19 15:20:00 +00003399 if( mUnusable || (p->fg.jointype & (JT_LEFT|JT_CROSS)) ){
dan4f20cd42015-06-08 18:05:54 +00003400 mUnusable |= sqlite3WhereGetMask(&pWInfo->sMaskSet, p->iCursor);
3401 }
3402 }
drh599d5762016-03-08 01:11:51 +00003403 rc = whereLoopAddVirtual(pBuilder, mPrereq, mUnusable);
drhec593592016-06-23 12:35:04 +00003404 }else
3405#endif /* SQLITE_OMIT_VIRTUALTABLE */
3406 {
drh599d5762016-03-08 01:11:51 +00003407 rc = whereLoopAddBtree(pBuilder, mPrereq);
drhb2a90f02013-05-10 03:30:49 +00003408 }
drhb2a90f02013-05-10 03:30:49 +00003409 if( rc==SQLITE_OK ){
drh599d5762016-03-08 01:11:51 +00003410 rc = whereLoopAddOr(pBuilder, mPrereq, mUnusable);
drhb2a90f02013-05-10 03:30:49 +00003411 }
drhb2a90f02013-05-10 03:30:49 +00003412 mPrior |= pNew->maskSelf;
drh5346e952013-05-08 14:14:26 +00003413 if( rc || db->mallocFailed ) break;
drhf1b5f5b2013-05-02 00:15:01 +00003414 }
dan4f20cd42015-06-08 18:05:54 +00003415
drha2014152013-06-07 00:29:23 +00003416 whereLoopClear(db, pNew);
drh5346e952013-05-08 14:14:26 +00003417 return rc;
drhf1b5f5b2013-05-02 00:15:01 +00003418}
3419
drha18f3d22013-05-08 03:05:41 +00003420/*
drh7699d1c2013-06-04 12:42:29 +00003421** Examine a WherePath (with the addition of the extra WhereLoop of the 5th
drh319f6772013-05-14 15:31:07 +00003422** parameters) to see if it outputs rows in the requested ORDER BY
drh0401ace2014-03-18 15:30:27 +00003423** (or GROUP BY) without requiring a separate sort operation. Return N:
drh319f6772013-05-14 15:31:07 +00003424**
drh0401ace2014-03-18 15:30:27 +00003425** N>0: N terms of the ORDER BY clause are satisfied
3426** N==0: No terms of the ORDER BY clause are satisfied
3427** N<0: Unknown yet how many terms of ORDER BY might be satisfied.
drh319f6772013-05-14 15:31:07 +00003428**
drh94433422013-07-01 11:05:50 +00003429** Note that processing for WHERE_GROUPBY and WHERE_DISTINCTBY is not as
3430** strict. With GROUP BY and DISTINCT the only requirement is that
3431** equivalent rows appear immediately adjacent to one another. GROUP BY
dan374cd782014-04-21 13:21:56 +00003432** and DISTINCT do not require rows to appear in any particular order as long
peter.d.reid60ec9142014-09-06 16:39:46 +00003433** as equivalent rows are grouped together. Thus for GROUP BY and DISTINCT
drh94433422013-07-01 11:05:50 +00003434** the pOrderBy terms can be matched in any order. With ORDER BY, the
3435** pOrderBy terms must be matched in strict left-to-right order.
drh6b7157b2013-05-10 02:00:35 +00003436*/
drh0401ace2014-03-18 15:30:27 +00003437static i8 wherePathSatisfiesOrderBy(
drh6b7157b2013-05-10 02:00:35 +00003438 WhereInfo *pWInfo, /* The WHERE clause */
drh4f402f22013-06-11 18:59:38 +00003439 ExprList *pOrderBy, /* ORDER BY or GROUP BY or DISTINCT clause to check */
drh6b7157b2013-05-10 02:00:35 +00003440 WherePath *pPath, /* The WherePath to check */
drhd711e522016-05-19 22:40:04 +00003441 u16 wctrlFlags, /* WHERE_GROUPBY or _DISTINCTBY or _ORDERBY_LIMIT */
drh4f402f22013-06-11 18:59:38 +00003442 u16 nLoop, /* Number of entries in pPath->aLoop[] */
drh319f6772013-05-14 15:31:07 +00003443 WhereLoop *pLast, /* Add this WhereLoop to the end of pPath->aLoop[] */
drh4f402f22013-06-11 18:59:38 +00003444 Bitmask *pRevMask /* OUT: Mask of WhereLoops to run in reverse order */
drh6b7157b2013-05-10 02:00:35 +00003445){
drh88da6442013-05-27 17:59:37 +00003446 u8 revSet; /* True if rev is known */
3447 u8 rev; /* Composite sort order */
3448 u8 revIdx; /* Index sort order */
drhe353ee32013-06-04 23:40:53 +00003449 u8 isOrderDistinct; /* All prior WhereLoops are order-distinct */
3450 u8 distinctColumns; /* True if the loop has UNIQUE NOT NULL columns */
3451 u8 isMatch; /* iColumn matches a term of the ORDER BY clause */
drhd711e522016-05-19 22:40:04 +00003452 u16 eqOpMask; /* Allowed equality operators */
drh416846a2013-11-06 12:56:04 +00003453 u16 nKeyCol; /* Number of key columns in pIndex */
3454 u16 nColumn; /* Total number of ordered columns in the index */
drh7699d1c2013-06-04 12:42:29 +00003455 u16 nOrderBy; /* Number terms in the ORDER BY clause */
3456 int iLoop; /* Index of WhereLoop in pPath being processed */
3457 int i, j; /* Loop counters */
3458 int iCur; /* Cursor number for current WhereLoop */
3459 int iColumn; /* A column number within table iCur */
drhe8ae5832013-06-19 13:32:46 +00003460 WhereLoop *pLoop = 0; /* Current WhereLoop being processed. */
drh7699d1c2013-06-04 12:42:29 +00003461 WhereTerm *pTerm; /* A single term of the WHERE clause */
3462 Expr *pOBExpr; /* An expression from the ORDER BY clause */
3463 CollSeq *pColl; /* COLLATE function from an ORDER BY clause term */
3464 Index *pIndex; /* The index associated with pLoop */
3465 sqlite3 *db = pWInfo->pParse->db; /* Database connection */
3466 Bitmask obSat = 0; /* Mask of ORDER BY terms satisfied so far */
3467 Bitmask obDone; /* Mask of all ORDER BY terms */
drhe353ee32013-06-04 23:40:53 +00003468 Bitmask orderDistinctMask; /* Mask of all well-ordered loops */
drhb8916be2013-06-14 02:51:48 +00003469 Bitmask ready; /* Mask of inner loops */
drh319f6772013-05-14 15:31:07 +00003470
3471 /*
drh7699d1c2013-06-04 12:42:29 +00003472 ** We say the WhereLoop is "one-row" if it generates no more than one
3473 ** row of output. A WhereLoop is one-row if all of the following are true:
drh319f6772013-05-14 15:31:07 +00003474 ** (a) All index columns match with WHERE_COLUMN_EQ.
3475 ** (b) The index is unique
drh7699d1c2013-06-04 12:42:29 +00003476 ** Any WhereLoop with an WHERE_COLUMN_EQ constraint on the rowid is one-row.
3477 ** Every one-row WhereLoop will have the WHERE_ONEROW bit set in wsFlags.
drh319f6772013-05-14 15:31:07 +00003478 **
drhe353ee32013-06-04 23:40:53 +00003479 ** We say the WhereLoop is "order-distinct" if the set of columns from
3480 ** that WhereLoop that are in the ORDER BY clause are different for every
3481 ** row of the WhereLoop. Every one-row WhereLoop is automatically
3482 ** order-distinct. A WhereLoop that has no columns in the ORDER BY clause
3483 ** is not order-distinct. To be order-distinct is not quite the same as being
3484 ** UNIQUE since a UNIQUE column or index can have multiple rows that
3485 ** are NULL and NULL values are equivalent for the purpose of order-distinct.
3486 ** To be order-distinct, the columns must be UNIQUE and NOT NULL.
3487 **
3488 ** The rowid for a table is always UNIQUE and NOT NULL so whenever the
3489 ** rowid appears in the ORDER BY clause, the corresponding WhereLoop is
3490 ** automatically order-distinct.
drh319f6772013-05-14 15:31:07 +00003491 */
3492
3493 assert( pOrderBy!=0 );
drh7699d1c2013-06-04 12:42:29 +00003494 if( nLoop && OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return 0;
drh319f6772013-05-14 15:31:07 +00003495
drh319f6772013-05-14 15:31:07 +00003496 nOrderBy = pOrderBy->nExpr;
drh7963b0e2013-06-17 21:37:40 +00003497 testcase( nOrderBy==BMS-1 );
drhe353ee32013-06-04 23:40:53 +00003498 if( nOrderBy>BMS-1 ) return 0; /* Cannot optimize overly large ORDER BYs */
3499 isOrderDistinct = 1;
drh7699d1c2013-06-04 12:42:29 +00003500 obDone = MASKBIT(nOrderBy)-1;
drhe353ee32013-06-04 23:40:53 +00003501 orderDistinctMask = 0;
drhb8916be2013-06-14 02:51:48 +00003502 ready = 0;
drhd711e522016-05-19 22:40:04 +00003503 eqOpMask = WO_EQ | WO_IS | WO_ISNULL;
3504 if( wctrlFlags & WHERE_ORDERBY_LIMIT ) eqOpMask |= WO_IN;
drhe353ee32013-06-04 23:40:53 +00003505 for(iLoop=0; isOrderDistinct && obSat<obDone && iLoop<=nLoop; iLoop++){
drhb8916be2013-06-14 02:51:48 +00003506 if( iLoop>0 ) ready |= pLoop->maskSelf;
drhd711e522016-05-19 22:40:04 +00003507 if( iLoop<nLoop ){
3508 pLoop = pPath->aLoop[iLoop];
3509 if( wctrlFlags & WHERE_ORDERBY_LIMIT ) continue;
3510 }else{
3511 pLoop = pLast;
3512 }
drh9dfaf622014-04-25 14:42:17 +00003513 if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){
3514 if( pLoop->u.vtab.isOrdered ) obSat = obDone;
3515 break;
3516 }
drh319f6772013-05-14 15:31:07 +00003517 iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor;
drhb8916be2013-06-14 02:51:48 +00003518
3519 /* Mark off any ORDER BY term X that is a column in the table of
3520 ** the current loop for which there is term in the WHERE
3521 ** clause of the form X IS NULL or X=? that reference only outer
3522 ** loops.
3523 */
3524 for(i=0; i<nOrderBy; i++){
3525 if( MASKBIT(i) & obSat ) continue;
3526 pOBExpr = sqlite3ExprSkipCollate(pOrderBy->a[i].pExpr);
3527 if( pOBExpr->op!=TK_COLUMN ) continue;
3528 if( pOBExpr->iTable!=iCur ) continue;
drh6f82e852015-06-06 20:12:09 +00003529 pTerm = sqlite3WhereFindTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn,
drhd711e522016-05-19 22:40:04 +00003530 ~ready, eqOpMask, 0);
drhb8916be2013-06-14 02:51:48 +00003531 if( pTerm==0 ) continue;
drh57a8c612016-09-07 01:51:46 +00003532 if( pTerm->eOperator==WO_IN ){
3533 /* IN terms are only valid for sorting in the ORDER BY LIMIT
3534 ** optimization, and then only if they are actually used
3535 ** by the query plan */
3536 assert( wctrlFlags & WHERE_ORDERBY_LIMIT );
3537 for(j=0; j<pLoop->nLTerm && pTerm!=pLoop->aLTerm[j]; j++){}
3538 if( j>=pLoop->nLTerm ) continue;
3539 }
drhe8d0c612015-05-14 01:05:25 +00003540 if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 && pOBExpr->iColumn>=0 ){
drhb8916be2013-06-14 02:51:48 +00003541 const char *z1, *z2;
3542 pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr);
3543 if( !pColl ) pColl = db->pDfltColl;
3544 z1 = pColl->zName;
3545 pColl = sqlite3ExprCollSeq(pWInfo->pParse, pTerm->pExpr);
3546 if( !pColl ) pColl = db->pDfltColl;
3547 z2 = pColl->zName;
3548 if( sqlite3StrICmp(z1, z2)!=0 ) continue;
drhe0cc3c22015-05-13 17:54:08 +00003549 testcase( pTerm->pExpr->op==TK_IS );
drhb8916be2013-06-14 02:51:48 +00003550 }
3551 obSat |= MASKBIT(i);
3552 }
3553
drh7699d1c2013-06-04 12:42:29 +00003554 if( (pLoop->wsFlags & WHERE_ONEROW)==0 ){
3555 if( pLoop->wsFlags & WHERE_IPK ){
3556 pIndex = 0;
drhbbbdc832013-10-22 18:01:40 +00003557 nKeyCol = 0;
drh416846a2013-11-06 12:56:04 +00003558 nColumn = 1;
drh7699d1c2013-06-04 12:42:29 +00003559 }else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){
drh1b0f0262013-05-30 22:27:09 +00003560 return 0;
drh7699d1c2013-06-04 12:42:29 +00003561 }else{
drhbbbdc832013-10-22 18:01:40 +00003562 nKeyCol = pIndex->nKeyCol;
drh416846a2013-11-06 12:56:04 +00003563 nColumn = pIndex->nColumn;
3564 assert( nColumn==nKeyCol+1 || !HasRowid(pIndex->pTable) );
drh4b92f982015-09-29 17:20:14 +00003565 assert( pIndex->aiColumn[nColumn-1]==XN_ROWID
3566 || !HasRowid(pIndex->pTable));
drh5f1d1d92014-07-31 22:59:04 +00003567 isOrderDistinct = IsUniqueIndex(pIndex);
drh1b0f0262013-05-30 22:27:09 +00003568 }
drh7699d1c2013-06-04 12:42:29 +00003569
drh7699d1c2013-06-04 12:42:29 +00003570 /* Loop through all columns of the index and deal with the ones
3571 ** that are not constrained by == or IN.
3572 */
3573 rev = revSet = 0;
drhe353ee32013-06-04 23:40:53 +00003574 distinctColumns = 0;
drh416846a2013-11-06 12:56:04 +00003575 for(j=0; j<nColumn; j++){
dand49fd4e2016-07-27 19:33:04 +00003576 u8 bOnce = 1; /* True to run the ORDER BY search loop */
drh7699d1c2013-06-04 12:42:29 +00003577
dand49fd4e2016-07-27 19:33:04 +00003578 assert( j>=pLoop->u.btree.nEq
3579 || (pLoop->aLTerm[j]==0)==(j<pLoop->nSkip)
3580 );
3581 if( j<pLoop->u.btree.nEq && j>=pLoop->nSkip ){
3582 u16 eOp = pLoop->aLTerm[j]->eOperator;
3583
3584 /* Skip over == and IS and ISNULL terms. (Also skip IN terms when
3585 ** doing WHERE_ORDERBY_LIMIT processing).
3586 **
3587 ** If the current term is a column of an ((?,?) IN (SELECT...))
3588 ** expression for which the SELECT returns more than one column,
3589 ** check that it is the only column used by this loop. Otherwise,
3590 ** if it is one of two or more, none of the columns can be
3591 ** considered to match an ORDER BY term. */
3592 if( (eOp & eqOpMask)!=0 ){
3593 if( eOp & WO_ISNULL ){
3594 testcase( isOrderDistinct );
3595 isOrderDistinct = 0;
3596 }
3597 continue;
drh64bcb8c2016-08-26 03:42:57 +00003598 }else if( ALWAYS(eOp & WO_IN) ){
3599 /* ALWAYS() justification: eOp is an equality operator due to the
3600 ** j<pLoop->u.btree.nEq constraint above. Any equality other
3601 ** than WO_IN is captured by the previous "if". So this one
3602 ** always has to be WO_IN. */
dand49fd4e2016-07-27 19:33:04 +00003603 Expr *pX = pLoop->aLTerm[j]->pExpr;
3604 for(i=j+1; i<pLoop->u.btree.nEq; i++){
3605 if( pLoop->aLTerm[i]->pExpr==pX ){
3606 assert( (pLoop->aLTerm[i]->eOperator & WO_IN) );
3607 bOnce = 0;
3608 break;
3609 }
3610 }
drh7963b0e2013-06-17 21:37:40 +00003611 }
drh7699d1c2013-06-04 12:42:29 +00003612 }
3613
drhe353ee32013-06-04 23:40:53 +00003614 /* Get the column number in the table (iColumn) and sort order
3615 ** (revIdx) for the j-th column of the index.
drh7699d1c2013-06-04 12:42:29 +00003616 */
drh416846a2013-11-06 12:56:04 +00003617 if( pIndex ){
drh7699d1c2013-06-04 12:42:29 +00003618 iColumn = pIndex->aiColumn[j];
3619 revIdx = pIndex->aSortOrder[j];
3620 if( iColumn==pIndex->pTable->iPKey ) iColumn = -1;
drhdc3cd4b2013-05-30 23:21:20 +00003621 }else{
drh4b92f982015-09-29 17:20:14 +00003622 iColumn = XN_ROWID;
drh7699d1c2013-06-04 12:42:29 +00003623 revIdx = 0;
drhdc3cd4b2013-05-30 23:21:20 +00003624 }
drh7699d1c2013-06-04 12:42:29 +00003625
3626 /* An unconstrained column that might be NULL means that this
drh416846a2013-11-06 12:56:04 +00003627 ** WhereLoop is not well-ordered
drh7699d1c2013-06-04 12:42:29 +00003628 */
drhe353ee32013-06-04 23:40:53 +00003629 if( isOrderDistinct
3630 && iColumn>=0
drh7699d1c2013-06-04 12:42:29 +00003631 && j>=pLoop->u.btree.nEq
3632 && pIndex->pTable->aCol[iColumn].notNull==0
3633 ){
drhe353ee32013-06-04 23:40:53 +00003634 isOrderDistinct = 0;
drh7699d1c2013-06-04 12:42:29 +00003635 }
3636
3637 /* Find the ORDER BY term that corresponds to the j-th column
dan374cd782014-04-21 13:21:56 +00003638 ** of the index and mark that ORDER BY term off
drh7699d1c2013-06-04 12:42:29 +00003639 */
drhe353ee32013-06-04 23:40:53 +00003640 isMatch = 0;
drh7699d1c2013-06-04 12:42:29 +00003641 for(i=0; bOnce && i<nOrderBy; i++){
3642 if( MASKBIT(i) & obSat ) continue;
3643 pOBExpr = sqlite3ExprSkipCollate(pOrderBy->a[i].pExpr);
drh93ec45d2013-06-17 18:20:48 +00003644 testcase( wctrlFlags & WHERE_GROUPBY );
3645 testcase( wctrlFlags & WHERE_DISTINCTBY );
drh4f402f22013-06-11 18:59:38 +00003646 if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0;
drhdae26fe2015-09-24 18:47:59 +00003647 if( iColumn>=(-1) ){
3648 if( pOBExpr->op!=TK_COLUMN ) continue;
3649 if( pOBExpr->iTable!=iCur ) continue;
3650 if( pOBExpr->iColumn!=iColumn ) continue;
3651 }else{
3652 if( sqlite3ExprCompare(pOBExpr,pIndex->aColExpr->a[j].pExpr,iCur) ){
3653 continue;
3654 }
3655 }
drh7699d1c2013-06-04 12:42:29 +00003656 if( iColumn>=0 ){
3657 pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr);
3658 if( !pColl ) pColl = db->pDfltColl;
3659 if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) continue;
3660 }
drhe353ee32013-06-04 23:40:53 +00003661 isMatch = 1;
drh7699d1c2013-06-04 12:42:29 +00003662 break;
3663 }
drh49290472014-10-11 02:12:58 +00003664 if( isMatch && (wctrlFlags & WHERE_GROUPBY)==0 ){
drh59b8f2e2014-03-22 00:27:14 +00003665 /* Make sure the sort order is compatible in an ORDER BY clause.
3666 ** Sort order is irrelevant for a GROUP BY clause. */
3667 if( revSet ){
3668 if( (rev ^ revIdx)!=pOrderBy->a[i].sortOrder ) isMatch = 0;
3669 }else{
3670 rev = revIdx ^ pOrderBy->a[i].sortOrder;
3671 if( rev ) *pRevMask |= MASKBIT(iLoop);
3672 revSet = 1;
3673 }
3674 }
drhe353ee32013-06-04 23:40:53 +00003675 if( isMatch ){
dan90b2fe62016-10-10 14:34:00 +00003676 if( iColumn==XN_ROWID ){
drh7963b0e2013-06-17 21:37:40 +00003677 testcase( distinctColumns==0 );
3678 distinctColumns = 1;
3679 }
drh7699d1c2013-06-04 12:42:29 +00003680 obSat |= MASKBIT(i);
drh7699d1c2013-06-04 12:42:29 +00003681 }else{
3682 /* No match found */
drhbbbdc832013-10-22 18:01:40 +00003683 if( j==0 || j<nKeyCol ){
drh7963b0e2013-06-17 21:37:40 +00003684 testcase( isOrderDistinct!=0 );
3685 isOrderDistinct = 0;
3686 }
drh7699d1c2013-06-04 12:42:29 +00003687 break;
3688 }
3689 } /* end Loop over all index columns */
drh81186b42013-06-18 01:52:41 +00003690 if( distinctColumns ){
3691 testcase( isOrderDistinct==0 );
3692 isOrderDistinct = 1;
3693 }
drh7699d1c2013-06-04 12:42:29 +00003694 } /* end-if not one-row */
3695
3696 /* Mark off any other ORDER BY terms that reference pLoop */
drhe353ee32013-06-04 23:40:53 +00003697 if( isOrderDistinct ){
3698 orderDistinctMask |= pLoop->maskSelf;
drh7699d1c2013-06-04 12:42:29 +00003699 for(i=0; i<nOrderBy; i++){
3700 Expr *p;
drh434a9312014-02-26 02:26:09 +00003701 Bitmask mTerm;
drh7699d1c2013-06-04 12:42:29 +00003702 if( MASKBIT(i) & obSat ) continue;
3703 p = pOrderBy->a[i].pExpr;
drh6c1f4ef2015-06-08 14:23:15 +00003704 mTerm = sqlite3WhereExprUsage(&pWInfo->sMaskSet,p);
drh434a9312014-02-26 02:26:09 +00003705 if( mTerm==0 && !sqlite3ExprIsConstant(p) ) continue;
3706 if( (mTerm&~orderDistinctMask)==0 ){
drh7699d1c2013-06-04 12:42:29 +00003707 obSat |= MASKBIT(i);
3708 }
drh0afb4232013-05-31 13:36:32 +00003709 }
drh319f6772013-05-14 15:31:07 +00003710 }
drhb8916be2013-06-14 02:51:48 +00003711 } /* End the loop over all WhereLoops from outer-most down to inner-most */
drh36ed0342014-03-28 12:56:57 +00003712 if( obSat==obDone ) return (i8)nOrderBy;
drhd2de8612014-03-18 18:59:07 +00003713 if( !isOrderDistinct ){
3714 for(i=nOrderBy-1; i>0; i--){
3715 Bitmask m = MASKBIT(i) - 1;
3716 if( (obSat&m)==m ) return i;
3717 }
3718 return 0;
3719 }
drh319f6772013-05-14 15:31:07 +00003720 return -1;
drh6b7157b2013-05-10 02:00:35 +00003721}
3722
dan374cd782014-04-21 13:21:56 +00003723
3724/*
3725** If the WHERE_GROUPBY flag is set in the mask passed to sqlite3WhereBegin(),
3726** the planner assumes that the specified pOrderBy list is actually a GROUP
3727** BY clause - and so any order that groups rows as required satisfies the
3728** request.
3729**
3730** Normally, in this case it is not possible for the caller to determine
3731** whether or not the rows are really being delivered in sorted order, or
3732** just in some other order that provides the required grouping. However,
3733** if the WHERE_SORTBYGROUP flag is also passed to sqlite3WhereBegin(), then
3734** this function may be called on the returned WhereInfo object. It returns
3735** true if the rows really will be sorted in the specified order, or false
3736** otherwise.
3737**
3738** For example, assuming:
3739**
3740** CREATE INDEX i1 ON t1(x, Y);
3741**
3742** then
3743**
3744** SELECT * FROM t1 GROUP BY x,y ORDER BY x,y; -- IsSorted()==1
3745** SELECT * FROM t1 GROUP BY y,x ORDER BY y,x; -- IsSorted()==0
3746*/
3747int sqlite3WhereIsSorted(WhereInfo *pWInfo){
3748 assert( pWInfo->wctrlFlags & WHERE_GROUPBY );
3749 assert( pWInfo->wctrlFlags & WHERE_SORTBYGROUP );
3750 return pWInfo->sorted;
3751}
3752
drhd15cb172013-05-21 19:23:10 +00003753#ifdef WHERETRACE_ENABLED
3754/* For debugging use only: */
3755static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){
3756 static char zName[65];
3757 int i;
3758 for(i=0; i<nLoop; i++){ zName[i] = pPath->aLoop[i]->cId; }
3759 if( pLast ) zName[i++] = pLast->cId;
3760 zName[i] = 0;
3761 return zName;
3762}
3763#endif
3764
drh6b7157b2013-05-10 02:00:35 +00003765/*
dan50ae31e2014-08-08 16:52:28 +00003766** Return the cost of sorting nRow rows, assuming that the keys have
3767** nOrderby columns and that the first nSorted columns are already in
3768** order.
3769*/
3770static LogEst whereSortingCost(
drhc3489bb2016-02-25 16:04:59 +00003771 WhereInfo *pWInfo,
dan50ae31e2014-08-08 16:52:28 +00003772 LogEst nRow,
3773 int nOrderBy,
3774 int nSorted
3775){
3776 /* TUNING: Estimated cost of a full external sort, where N is
3777 ** the number of rows to sort is:
3778 **
3779 ** cost = (3.0 * N * log(N)).
3780 **
3781 ** Or, if the order-by clause has X terms but only the last Y
3782 ** terms are out of order, then block-sorting will reduce the
3783 ** sorting cost to:
3784 **
3785 ** cost = (3.0 * N * log(N)) * (Y/X)
3786 **
3787 ** The (Y/X) term is implemented using stack variable rScale
3788 ** below. */
3789 LogEst rScale, rSortCost;
3790 assert( nOrderBy>0 && 66==sqlite3LogEst(100) );
3791 rScale = sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66;
drhc3489bb2016-02-25 16:04:59 +00003792 rSortCost = nRow + rScale + 16;
3793
3794 /* Multiple by log(M) where M is the number of output rows.
3795 ** Use the LIMIT for M if it is smaller */
drh8c098e62016-02-25 23:21:41 +00003796 if( (pWInfo->wctrlFlags & WHERE_USE_LIMIT)!=0 && pWInfo->iLimit<nRow ){
3797 nRow = pWInfo->iLimit;
drhc3489bb2016-02-25 16:04:59 +00003798 }
3799 rSortCost += estLog(nRow);
dan50ae31e2014-08-08 16:52:28 +00003800 return rSortCost;
3801}
3802
3803/*
dan51576f42013-07-02 10:06:15 +00003804** Given the list of WhereLoop objects at pWInfo->pLoops, this routine
drha18f3d22013-05-08 03:05:41 +00003805** attempts to find the lowest cost path that visits each WhereLoop
3806** once. This path is then loaded into the pWInfo->a[].pWLoop fields.
3807**
drhc7f0d222013-06-19 03:27:12 +00003808** Assume that the total number of output rows that will need to be sorted
3809** will be nRowEst (in the 10*log2 representation). Or, ignore sorting
3810** costs if nRowEst==0.
3811**
drha18f3d22013-05-08 03:05:41 +00003812** Return SQLITE_OK on success or SQLITE_NOMEM of a memory allocation
3813** error occurs.
3814*/
drhbf539c42013-10-05 18:16:02 +00003815static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
drh783dece2013-06-05 17:53:43 +00003816 int mxChoice; /* Maximum number of simultaneous paths tracked */
drha18f3d22013-05-08 03:05:41 +00003817 int nLoop; /* Number of terms in the join */
drhe1e2e9a2013-06-13 15:16:53 +00003818 Parse *pParse; /* Parsing context */
drha18f3d22013-05-08 03:05:41 +00003819 sqlite3 *db; /* The database connection */
3820 int iLoop; /* Loop counter over the terms of the join */
3821 int ii, jj; /* Loop counters */
drhfde1e6b2013-09-06 17:45:42 +00003822 int mxI = 0; /* Index of next entry to replace */
drhd2de8612014-03-18 18:59:07 +00003823 int nOrderBy; /* Number of ORDER BY clause terms */
drhbf539c42013-10-05 18:16:02 +00003824 LogEst mxCost = 0; /* Maximum cost of a set of paths */
dan50ae31e2014-08-08 16:52:28 +00003825 LogEst mxUnsorted = 0; /* Maximum unsorted cost of a set of path */
drha18f3d22013-05-08 03:05:41 +00003826 int nTo, nFrom; /* Number of valid entries in aTo[] and aFrom[] */
3827 WherePath *aFrom; /* All nFrom paths at the previous level */
3828 WherePath *aTo; /* The nTo best paths at the current level */
3829 WherePath *pFrom; /* An element of aFrom[] that we are working on */
3830 WherePath *pTo; /* An element of aTo[] that we are working on */
3831 WhereLoop *pWLoop; /* One of the WhereLoop objects */
3832 WhereLoop **pX; /* Used to divy up the pSpace memory */
dan50ae31e2014-08-08 16:52:28 +00003833 LogEst *aSortCost = 0; /* Sorting and partial sorting costs */
drha18f3d22013-05-08 03:05:41 +00003834 char *pSpace; /* Temporary memory used by this routine */
dane2c27852014-08-08 17:25:33 +00003835 int nSpace; /* Bytes of space allocated at pSpace */
drha18f3d22013-05-08 03:05:41 +00003836
drhe1e2e9a2013-06-13 15:16:53 +00003837 pParse = pWInfo->pParse;
3838 db = pParse->db;
drha18f3d22013-05-08 03:05:41 +00003839 nLoop = pWInfo->nLevel;
drhe1e2e9a2013-06-13 15:16:53 +00003840 /* TUNING: For simple queries, only the best path is tracked.
3841 ** For 2-way joins, the 5 best paths are followed.
3842 ** For joins of 3 or more tables, track the 10 best paths */
drh2504c6c2014-06-02 11:26:33 +00003843 mxChoice = (nLoop<=1) ? 1 : (nLoop==2 ? 5 : 10);
drha18f3d22013-05-08 03:05:41 +00003844 assert( nLoop<=pWInfo->pTabList->nSrc );
drhddef5dc2014-08-07 16:50:00 +00003845 WHERETRACE(0x002, ("---- begin solver. (nRowEst=%d)\n", nRowEst));
drha18f3d22013-05-08 03:05:41 +00003846
dan50ae31e2014-08-08 16:52:28 +00003847 /* If nRowEst is zero and there is an ORDER BY clause, ignore it. In this
3848 ** case the purpose of this call is to estimate the number of rows returned
3849 ** by the overall query. Once this estimate has been obtained, the caller
3850 ** will invoke this function a second time, passing the estimate as the
3851 ** nRowEst parameter. */
3852 if( pWInfo->pOrderBy==0 || nRowEst==0 ){
3853 nOrderBy = 0;
3854 }else{
3855 nOrderBy = pWInfo->pOrderBy->nExpr;
3856 }
3857
3858 /* Allocate and initialize space for aTo, aFrom and aSortCost[] */
dane2c27852014-08-08 17:25:33 +00003859 nSpace = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2;
3860 nSpace += sizeof(LogEst) * nOrderBy;
drh575fad62016-02-05 13:38:36 +00003861 pSpace = sqlite3DbMallocRawNN(db, nSpace);
mistachkinfad30392016-02-13 23:43:46 +00003862 if( pSpace==0 ) return SQLITE_NOMEM_BKPT;
drha18f3d22013-05-08 03:05:41 +00003863 aTo = (WherePath*)pSpace;
3864 aFrom = aTo+mxChoice;
3865 memset(aFrom, 0, sizeof(aFrom[0]));
3866 pX = (WhereLoop**)(aFrom+mxChoice);
drhe9d935a2013-06-05 16:19:59 +00003867 for(ii=mxChoice*2, pFrom=aTo; ii>0; ii--, pFrom++, pX += nLoop){
drha18f3d22013-05-08 03:05:41 +00003868 pFrom->aLoop = pX;
3869 }
dan50ae31e2014-08-08 16:52:28 +00003870 if( nOrderBy ){
3871 /* If there is an ORDER BY clause and it is not being ignored, set up
3872 ** space for the aSortCost[] array. Each element of the aSortCost array
3873 ** is either zero - meaning it has not yet been initialized - or the
3874 ** cost of sorting nRowEst rows of data where the first X terms of
3875 ** the ORDER BY clause are already in order, where X is the array
3876 ** index. */
3877 aSortCost = (LogEst*)pX;
dane2c27852014-08-08 17:25:33 +00003878 memset(aSortCost, 0, sizeof(LogEst) * nOrderBy);
dan50ae31e2014-08-08 16:52:28 +00003879 }
dane2c27852014-08-08 17:25:33 +00003880 assert( aSortCost==0 || &pSpace[nSpace]==(char*)&aSortCost[nOrderBy] );
3881 assert( aSortCost!=0 || &pSpace[nSpace]==(char*)pX );
drha18f3d22013-05-08 03:05:41 +00003882
drhe1e2e9a2013-06-13 15:16:53 +00003883 /* Seed the search with a single WherePath containing zero WhereLoops.
3884 **
danf104abb2015-03-16 20:40:00 +00003885 ** TUNING: Do not let the number of iterations go above 28. If the cost
3886 ** of computing an automatic index is not paid back within the first 28
drhe1e2e9a2013-06-13 15:16:53 +00003887 ** rows, then do not use the automatic index. */
danf104abb2015-03-16 20:40:00 +00003888 aFrom[0].nRow = MIN(pParse->nQueryLoop, 48); assert( 48==sqlite3LogEst(28) );
drha18f3d22013-05-08 03:05:41 +00003889 nFrom = 1;
dan50ae31e2014-08-08 16:52:28 +00003890 assert( aFrom[0].isOrdered==0 );
3891 if( nOrderBy ){
3892 /* If nLoop is zero, then there are no FROM terms in the query. Since
3893 ** in this case the query may return a maximum of one row, the results
3894 ** are already in the requested order. Set isOrdered to nOrderBy to
3895 ** indicate this. Or, if nLoop is greater than zero, set isOrdered to
3896 ** -1, indicating that the result set may or may not be ordered,
3897 ** depending on the loops added to the current plan. */
3898 aFrom[0].isOrdered = nLoop>0 ? -1 : nOrderBy;
drh6b7157b2013-05-10 02:00:35 +00003899 }
3900
3901 /* Compute successively longer WherePaths using the previous generation
3902 ** of WherePaths as the basis for the next. Keep track of the mxChoice
3903 ** best paths at each generation */
drha18f3d22013-05-08 03:05:41 +00003904 for(iLoop=0; iLoop<nLoop; iLoop++){
3905 nTo = 0;
3906 for(ii=0, pFrom=aFrom; ii<nFrom; ii++, pFrom++){
3907 for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){
dan50ae31e2014-08-08 16:52:28 +00003908 LogEst nOut; /* Rows visited by (pFrom+pWLoop) */
3909 LogEst rCost; /* Cost of path (pFrom+pWLoop) */
3910 LogEst rUnsorted; /* Unsorted cost of (pFrom+pWLoop) */
3911 i8 isOrdered = pFrom->isOrdered; /* isOrdered for (pFrom+pWLoop) */
3912 Bitmask maskNew; /* Mask of src visited by (..) */
3913 Bitmask revMask = 0; /* Mask of rev-order loops for (..) */
3914
drha18f3d22013-05-08 03:05:41 +00003915 if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue;
3916 if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue;
drh5a6f5ed2016-02-25 18:22:09 +00003917 if( (pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 && pFrom->nRow<10 ){
3918 /* Do not use an automatic index if the this loop is expected
3919 ** to run less than 2 times. */
3920 assert( 10==sqlite3LogEst(2) );
drh87eb9192016-02-25 18:03:38 +00003921 continue;
3922 }
drh6b7157b2013-05-10 02:00:35 +00003923 /* At this point, pWLoop is a candidate to be the next loop.
3924 ** Compute its cost */
dan50ae31e2014-08-08 16:52:28 +00003925 rUnsorted = sqlite3LogEstAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow);
3926 rUnsorted = sqlite3LogEstAdd(rUnsorted, pFrom->rUnsorted);
drhfde1e6b2013-09-06 17:45:42 +00003927 nOut = pFrom->nRow + pWLoop->nOut;
drha18f3d22013-05-08 03:05:41 +00003928 maskNew = pFrom->maskLoop | pWLoop->maskSelf;
drh0401ace2014-03-18 15:30:27 +00003929 if( isOrdered<0 ){
3930 isOrdered = wherePathSatisfiesOrderBy(pWInfo,
drh4f402f22013-06-11 18:59:38 +00003931 pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags,
drh0401ace2014-03-18 15:30:27 +00003932 iLoop, pWLoop, &revMask);
drh3a5ba8b2013-06-03 15:34:48 +00003933 }else{
3934 revMask = pFrom->revLoop;
drh6b7157b2013-05-10 02:00:35 +00003935 }
dan50ae31e2014-08-08 16:52:28 +00003936 if( isOrdered>=0 && isOrdered<nOrderBy ){
3937 if( aSortCost[isOrdered]==0 ){
3938 aSortCost[isOrdered] = whereSortingCost(
drhc3489bb2016-02-25 16:04:59 +00003939 pWInfo, nRowEst, nOrderBy, isOrdered
dan50ae31e2014-08-08 16:52:28 +00003940 );
3941 }
3942 rCost = sqlite3LogEstAdd(rUnsorted, aSortCost[isOrdered]);
3943
3944 WHERETRACE(0x002,
3945 ("---- sort cost=%-3d (%d/%d) increases cost %3d to %-3d\n",
3946 aSortCost[isOrdered], (nOrderBy-isOrdered), nOrderBy,
3947 rUnsorted, rCost));
3948 }else{
3949 rCost = rUnsorted;
3950 }
3951
drhddef5dc2014-08-07 16:50:00 +00003952 /* Check to see if pWLoop should be added to the set of
3953 ** mxChoice best-so-far paths.
3954 **
3955 ** First look for an existing path among best-so-far paths
3956 ** that covers the same set of loops and has the same isOrdered
3957 ** setting as the current path candidate.
drhf2a90302014-08-07 20:37:01 +00003958 **
3959 ** The term "((pTo->isOrdered^isOrdered)&0x80)==0" is equivalent
3960 ** to (pTo->isOrdered==(-1))==(isOrdered==(-1))" for the range
3961 ** of legal values for isOrdered, -1..64.
drhddef5dc2014-08-07 16:50:00 +00003962 */
drh6b7157b2013-05-10 02:00:35 +00003963 for(jj=0, pTo=aTo; jj<nTo; jj++, pTo++){
drhfde1e6b2013-09-06 17:45:42 +00003964 if( pTo->maskLoop==maskNew
drhf2a90302014-08-07 20:37:01 +00003965 && ((pTo->isOrdered^isOrdered)&0x80)==0
drhfde1e6b2013-09-06 17:45:42 +00003966 ){
drh7963b0e2013-06-17 21:37:40 +00003967 testcase( jj==nTo-1 );
drh6b7157b2013-05-10 02:00:35 +00003968 break;
3969 }
3970 }
drha18f3d22013-05-08 03:05:41 +00003971 if( jj>=nTo ){
drhddef5dc2014-08-07 16:50:00 +00003972 /* None of the existing best-so-far paths match the candidate. */
drhddef5dc2014-08-07 16:50:00 +00003973 if( nTo>=mxChoice
dan50ae31e2014-08-08 16:52:28 +00003974 && (rCost>mxCost || (rCost==mxCost && rUnsorted>=mxUnsorted))
drhddef5dc2014-08-07 16:50:00 +00003975 ){
3976 /* The current candidate is no better than any of the mxChoice
3977 ** paths currently in the best-so-far buffer. So discard
3978 ** this candidate as not viable. */
drh989578e2013-10-28 14:34:35 +00003979#ifdef WHERETRACE_ENABLED /* 0x4 */
drhae70cf12013-05-31 15:18:46 +00003980 if( sqlite3WhereTrace&0x4 ){
drhfde1e6b2013-09-06 17:45:42 +00003981 sqlite3DebugPrintf("Skip %s cost=%-3d,%3d order=%c\n",
3982 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut,
drh0401ace2014-03-18 15:30:27 +00003983 isOrdered>=0 ? isOrdered+'0' : '?');
drhd15cb172013-05-21 19:23:10 +00003984 }
3985#endif
3986 continue;
3987 }
drhddef5dc2014-08-07 16:50:00 +00003988 /* If we reach this points it means that the new candidate path
3989 ** needs to be added to the set of best-so-far paths. */
drha18f3d22013-05-08 03:05:41 +00003990 if( nTo<mxChoice ){
drhd15cb172013-05-21 19:23:10 +00003991 /* Increase the size of the aTo set by one */
drha18f3d22013-05-08 03:05:41 +00003992 jj = nTo++;
3993 }else{
drhd15cb172013-05-21 19:23:10 +00003994 /* New path replaces the prior worst to keep count below mxChoice */
drhfde1e6b2013-09-06 17:45:42 +00003995 jj = mxI;
drha18f3d22013-05-08 03:05:41 +00003996 }
3997 pTo = &aTo[jj];
drh989578e2013-10-28 14:34:35 +00003998#ifdef WHERETRACE_ENABLED /* 0x4 */
drhae70cf12013-05-31 15:18:46 +00003999 if( sqlite3WhereTrace&0x4 ){
drhfde1e6b2013-09-06 17:45:42 +00004000 sqlite3DebugPrintf("New %s cost=%-3d,%3d order=%c\n",
4001 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut,
drh0401ace2014-03-18 15:30:27 +00004002 isOrdered>=0 ? isOrdered+'0' : '?');
drhd15cb172013-05-21 19:23:10 +00004003 }
4004#endif
drhf204dac2013-05-08 03:22:07 +00004005 }else{
drhddef5dc2014-08-07 16:50:00 +00004006 /* Control reaches here if best-so-far path pTo=aTo[jj] covers the
4007 ** same set of loops and has the sam isOrdered setting as the
4008 ** candidate path. Check to see if the candidate should replace
4009 ** pTo or if the candidate should be skipped */
4010 if( pTo->rCost<rCost || (pTo->rCost==rCost && pTo->nRow<=nOut) ){
drh989578e2013-10-28 14:34:35 +00004011#ifdef WHERETRACE_ENABLED /* 0x4 */
drhae70cf12013-05-31 15:18:46 +00004012 if( sqlite3WhereTrace&0x4 ){
drhd15cb172013-05-21 19:23:10 +00004013 sqlite3DebugPrintf(
drhfde1e6b2013-09-06 17:45:42 +00004014 "Skip %s cost=%-3d,%3d order=%c",
4015 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut,
drh0401ace2014-03-18 15:30:27 +00004016 isOrdered>=0 ? isOrdered+'0' : '?');
drhfde1e6b2013-09-06 17:45:42 +00004017 sqlite3DebugPrintf(" vs %s cost=%-3d,%d order=%c\n",
4018 wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
drh0401ace2014-03-18 15:30:27 +00004019 pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?');
drhd15cb172013-05-21 19:23:10 +00004020 }
4021#endif
drhddef5dc2014-08-07 16:50:00 +00004022 /* Discard the candidate path from further consideration */
drh7963b0e2013-06-17 21:37:40 +00004023 testcase( pTo->rCost==rCost );
drhd15cb172013-05-21 19:23:10 +00004024 continue;
4025 }
drh7963b0e2013-06-17 21:37:40 +00004026 testcase( pTo->rCost==rCost+1 );
drhddef5dc2014-08-07 16:50:00 +00004027 /* Control reaches here if the candidate path is better than the
4028 ** pTo path. Replace pTo with the candidate. */
drh989578e2013-10-28 14:34:35 +00004029#ifdef WHERETRACE_ENABLED /* 0x4 */
drhae70cf12013-05-31 15:18:46 +00004030 if( sqlite3WhereTrace&0x4 ){
drhd15cb172013-05-21 19:23:10 +00004031 sqlite3DebugPrintf(
drhfde1e6b2013-09-06 17:45:42 +00004032 "Update %s cost=%-3d,%3d order=%c",
4033 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut,
drh0401ace2014-03-18 15:30:27 +00004034 isOrdered>=0 ? isOrdered+'0' : '?');
drhfde1e6b2013-09-06 17:45:42 +00004035 sqlite3DebugPrintf(" was %s cost=%-3d,%3d order=%c\n",
4036 wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
drh0401ace2014-03-18 15:30:27 +00004037 pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?');
drhd15cb172013-05-21 19:23:10 +00004038 }
4039#endif
drha18f3d22013-05-08 03:05:41 +00004040 }
drh6b7157b2013-05-10 02:00:35 +00004041 /* pWLoop is a winner. Add it to the set of best so far */
drha18f3d22013-05-08 03:05:41 +00004042 pTo->maskLoop = pFrom->maskLoop | pWLoop->maskSelf;
drh319f6772013-05-14 15:31:07 +00004043 pTo->revLoop = revMask;
drhfde1e6b2013-09-06 17:45:42 +00004044 pTo->nRow = nOut;
drha18f3d22013-05-08 03:05:41 +00004045 pTo->rCost = rCost;
dan50ae31e2014-08-08 16:52:28 +00004046 pTo->rUnsorted = rUnsorted;
drh6b7157b2013-05-10 02:00:35 +00004047 pTo->isOrdered = isOrdered;
drha18f3d22013-05-08 03:05:41 +00004048 memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop);
4049 pTo->aLoop[iLoop] = pWLoop;
4050 if( nTo>=mxChoice ){
drhfde1e6b2013-09-06 17:45:42 +00004051 mxI = 0;
drha18f3d22013-05-08 03:05:41 +00004052 mxCost = aTo[0].rCost;
dan50ae31e2014-08-08 16:52:28 +00004053 mxUnsorted = aTo[0].nRow;
drha18f3d22013-05-08 03:05:41 +00004054 for(jj=1, pTo=&aTo[1]; jj<mxChoice; jj++, pTo++){
dan50ae31e2014-08-08 16:52:28 +00004055 if( pTo->rCost>mxCost
4056 || (pTo->rCost==mxCost && pTo->rUnsorted>mxUnsorted)
4057 ){
drhfde1e6b2013-09-06 17:45:42 +00004058 mxCost = pTo->rCost;
dan50ae31e2014-08-08 16:52:28 +00004059 mxUnsorted = pTo->rUnsorted;
drhfde1e6b2013-09-06 17:45:42 +00004060 mxI = jj;
4061 }
drha18f3d22013-05-08 03:05:41 +00004062 }
4063 }
4064 }
4065 }
4066
drh989578e2013-10-28 14:34:35 +00004067#ifdef WHERETRACE_ENABLED /* >=2 */
drh1b131b72014-10-21 16:01:40 +00004068 if( sqlite3WhereTrace & 0x02 ){
drha50ef112013-05-22 02:06:59 +00004069 sqlite3DebugPrintf("---- after round %d ----\n", iLoop);
drhd15cb172013-05-21 19:23:10 +00004070 for(ii=0, pTo=aTo; ii<nTo; ii++, pTo++){
drhb8a8e8a2013-06-10 19:12:39 +00004071 sqlite3DebugPrintf(" %s cost=%-3d nrow=%-3d order=%c",
drha50ef112013-05-22 02:06:59 +00004072 wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
drh0401ace2014-03-18 15:30:27 +00004073 pTo->isOrdered>=0 ? (pTo->isOrdered+'0') : '?');
4074 if( pTo->isOrdered>0 ){
drh88da6442013-05-27 17:59:37 +00004075 sqlite3DebugPrintf(" rev=0x%llx\n", pTo->revLoop);
4076 }else{
4077 sqlite3DebugPrintf("\n");
4078 }
drhf204dac2013-05-08 03:22:07 +00004079 }
4080 }
4081#endif
4082
drh6b7157b2013-05-10 02:00:35 +00004083 /* Swap the roles of aFrom and aTo for the next generation */
drha18f3d22013-05-08 03:05:41 +00004084 pFrom = aTo;
4085 aTo = aFrom;
4086 aFrom = pFrom;
4087 nFrom = nTo;
4088 }
4089
drh75b93402013-05-31 20:43:57 +00004090 if( nFrom==0 ){
drhe1e2e9a2013-06-13 15:16:53 +00004091 sqlite3ErrorMsg(pParse, "no query solution");
drh75b93402013-05-31 20:43:57 +00004092 sqlite3DbFree(db, pSpace);
4093 return SQLITE_ERROR;
4094 }
drha18f3d22013-05-08 03:05:41 +00004095
drh6b7157b2013-05-10 02:00:35 +00004096 /* Find the lowest cost path. pFrom will be left pointing to that path */
drha18f3d22013-05-08 03:05:41 +00004097 pFrom = aFrom;
4098 for(ii=1; ii<nFrom; ii++){
4099 if( pFrom->rCost>aFrom[ii].rCost ) pFrom = &aFrom[ii];
4100 }
4101 assert( pWInfo->nLevel==nLoop );
drh6b7157b2013-05-10 02:00:35 +00004102 /* Load the lowest cost path into pWInfo */
drha18f3d22013-05-08 03:05:41 +00004103 for(iLoop=0; iLoop<nLoop; iLoop++){
drh7ba39a92013-05-30 17:43:19 +00004104 WhereLevel *pLevel = pWInfo->a + iLoop;
4105 pLevel->pWLoop = pWLoop = pFrom->aLoop[iLoop];
drhe217efc2013-06-12 03:48:41 +00004106 pLevel->iFrom = pWLoop->iTab;
drh7ba39a92013-05-30 17:43:19 +00004107 pLevel->iTabCur = pWInfo->pTabList->a[pLevel->iFrom].iCursor;
drha18f3d22013-05-08 03:05:41 +00004108 }
drhfd636c72013-06-21 02:05:06 +00004109 if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0
4110 && (pWInfo->wctrlFlags & WHERE_DISTINCTBY)==0
4111 && pWInfo->eDistinct==WHERE_DISTINCT_NOOP
drh4f402f22013-06-11 18:59:38 +00004112 && nRowEst
4113 ){
4114 Bitmask notUsed;
drhf1b5ff72016-04-14 13:35:26 +00004115 int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pDistinctSet, pFrom,
drh93ec45d2013-06-17 18:20:48 +00004116 WHERE_DISTINCTBY, nLoop-1, pFrom->aLoop[nLoop-1], &notUsed);
drhf1b5ff72016-04-14 13:35:26 +00004117 if( rc==pWInfo->pDistinctSet->nExpr ){
drh0401ace2014-03-18 15:30:27 +00004118 pWInfo->eDistinct = WHERE_DISTINCT_ORDERED;
4119 }
drh4f402f22013-06-11 18:59:38 +00004120 }
drh079a3072014-03-19 14:10:55 +00004121 if( pWInfo->pOrderBy ){
drh4f402f22013-06-11 18:59:38 +00004122 if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){
drh079a3072014-03-19 14:10:55 +00004123 if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){
4124 pWInfo->eDistinct = WHERE_DISTINCT_ORDERED;
4125 }
drh4f402f22013-06-11 18:59:38 +00004126 }else{
drhddba0c22014-03-18 20:33:42 +00004127 pWInfo->nOBSat = pFrom->isOrdered;
drh4f402f22013-06-11 18:59:38 +00004128 pWInfo->revMask = pFrom->revLoop;
drha536df42016-05-19 22:13:37 +00004129 if( pWInfo->nOBSat<=0 ){
4130 pWInfo->nOBSat = 0;
drhc436a032016-10-12 18:55:53 +00004131 if( nLoop>0 ){
4132 u32 wsFlags = pFrom->aLoop[nLoop-1]->wsFlags;
4133 if( (wsFlags & WHERE_ONEROW)==0
4134 && (wsFlags&(WHERE_IPK|WHERE_COLUMN_IN))!=(WHERE_IPK|WHERE_COLUMN_IN)
4135 ){
4136 Bitmask m = 0;
4137 int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom,
drhd711e522016-05-19 22:40:04 +00004138 WHERE_ORDERBY_LIMIT, nLoop-1, pFrom->aLoop[nLoop-1], &m);
drhc436a032016-10-12 18:55:53 +00004139 testcase( wsFlags & WHERE_IPK );
4140 testcase( wsFlags & WHERE_COLUMN_IN );
4141 if( rc==pWInfo->pOrderBy->nExpr ){
4142 pWInfo->bOrderedInnerLoop = 1;
4143 pWInfo->revMask = m;
4144 }
drhd711e522016-05-19 22:40:04 +00004145 }
4146 }
drha536df42016-05-19 22:13:37 +00004147 }
drh4f402f22013-06-11 18:59:38 +00004148 }
dan374cd782014-04-21 13:21:56 +00004149 if( (pWInfo->wctrlFlags & WHERE_SORTBYGROUP)
drh11b04812015-04-12 01:22:04 +00004150 && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr && nLoop>0
dan374cd782014-04-21 13:21:56 +00004151 ){
danb6453202014-10-10 20:52:53 +00004152 Bitmask revMask = 0;
dan374cd782014-04-21 13:21:56 +00004153 int nOrder = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy,
danb6453202014-10-10 20:52:53 +00004154 pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], &revMask
dan374cd782014-04-21 13:21:56 +00004155 );
4156 assert( pWInfo->sorted==0 );
danb6453202014-10-10 20:52:53 +00004157 if( nOrder==pWInfo->pOrderBy->nExpr ){
4158 pWInfo->sorted = 1;
4159 pWInfo->revMask = revMask;
4160 }
dan374cd782014-04-21 13:21:56 +00004161 }
drh6b7157b2013-05-10 02:00:35 +00004162 }
dan374cd782014-04-21 13:21:56 +00004163
4164
drha50ef112013-05-22 02:06:59 +00004165 pWInfo->nRowOut = pFrom->nRow;
drha18f3d22013-05-08 03:05:41 +00004166
4167 /* Free temporary memory and return success */
4168 sqlite3DbFree(db, pSpace);
4169 return SQLITE_OK;
4170}
drh94a11212004-09-25 13:12:14 +00004171
4172/*
drh60c96cd2013-06-09 17:21:25 +00004173** Most queries use only a single table (they are not joins) and have
4174** simple == constraints against indexed fields. This routine attempts
4175** to plan those simple cases using much less ceremony than the
4176** general-purpose query planner, and thereby yield faster sqlite3_prepare()
4177** times for the common case.
4178**
4179** Return non-zero on success, if this query can be handled by this
4180** no-frills query planner. Return zero if this query needs the
4181** general-purpose query planner.
4182*/
drhb8a8e8a2013-06-10 19:12:39 +00004183static int whereShortCut(WhereLoopBuilder *pBuilder){
drh60c96cd2013-06-09 17:21:25 +00004184 WhereInfo *pWInfo;
4185 struct SrcList_item *pItem;
4186 WhereClause *pWC;
4187 WhereTerm *pTerm;
4188 WhereLoop *pLoop;
4189 int iCur;
drh92a121f2013-06-10 12:15:47 +00004190 int j;
drh60c96cd2013-06-09 17:21:25 +00004191 Table *pTab;
4192 Index *pIdx;
drh892ffcc2016-03-16 18:26:54 +00004193
drh60c96cd2013-06-09 17:21:25 +00004194 pWInfo = pBuilder->pWInfo;
drhce943bc2016-05-19 18:56:33 +00004195 if( pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE ) return 0;
drh60c96cd2013-06-09 17:21:25 +00004196 assert( pWInfo->pTabList->nSrc>=1 );
4197 pItem = pWInfo->pTabList->a;
4198 pTab = pItem->pTab;
4199 if( IsVirtual(pTab) ) return 0;
drh8a48b9c2015-08-19 15:20:00 +00004200 if( pItem->fg.isIndexedBy ) return 0;
drh60c96cd2013-06-09 17:21:25 +00004201 iCur = pItem->iCursor;
4202 pWC = &pWInfo->sWC;
4203 pLoop = pBuilder->pNew;
drh60c96cd2013-06-09 17:21:25 +00004204 pLoop->wsFlags = 0;
drhc8bbce12014-10-21 01:05:09 +00004205 pLoop->nSkip = 0;
drh6f82e852015-06-06 20:12:09 +00004206 pTerm = sqlite3WhereFindTerm(pWC, iCur, -1, 0, WO_EQ|WO_IS, 0);
drh60c96cd2013-06-09 17:21:25 +00004207 if( pTerm ){
drhe8d0c612015-05-14 01:05:25 +00004208 testcase( pTerm->eOperator & WO_IS );
drh60c96cd2013-06-09 17:21:25 +00004209 pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW;
4210 pLoop->aLTerm[0] = pTerm;
4211 pLoop->nLTerm = 1;
4212 pLoop->u.btree.nEq = 1;
drhe1e2e9a2013-06-13 15:16:53 +00004213 /* TUNING: Cost of a rowid lookup is 10 */
drhbf539c42013-10-05 18:16:02 +00004214 pLoop->rRun = 33; /* 33==sqlite3LogEst(10) */
drh60c96cd2013-06-09 17:21:25 +00004215 }else{
4216 for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
mistachkin4e5bef82015-05-15 20:14:00 +00004217 int opMask;
dancd40abb2013-08-29 10:46:05 +00004218 assert( pLoop->aLTermSpace==pLoop->aLTerm );
drh5f1d1d92014-07-31 22:59:04 +00004219 if( !IsUniqueIndex(pIdx)
dancd40abb2013-08-29 10:46:05 +00004220 || pIdx->pPartIdxWhere!=0
drhbbbdc832013-10-22 18:01:40 +00004221 || pIdx->nKeyCol>ArraySize(pLoop->aLTermSpace)
dancd40abb2013-08-29 10:46:05 +00004222 ) continue;
mistachkin4e5bef82015-05-15 20:14:00 +00004223 opMask = pIdx->uniqNotNull ? (WO_EQ|WO_IS) : WO_EQ;
drhbbbdc832013-10-22 18:01:40 +00004224 for(j=0; j<pIdx->nKeyCol; j++){
drhbb523082015-08-27 15:58:51 +00004225 pTerm = sqlite3WhereFindTerm(pWC, iCur, j, 0, opMask, pIdx);
drh60c96cd2013-06-09 17:21:25 +00004226 if( pTerm==0 ) break;
dan3072b532015-05-15 19:59:23 +00004227 testcase( pTerm->eOperator & WO_IS );
drh60c96cd2013-06-09 17:21:25 +00004228 pLoop->aLTerm[j] = pTerm;
4229 }
drhbbbdc832013-10-22 18:01:40 +00004230 if( j!=pIdx->nKeyCol ) continue;
drh92a121f2013-06-10 12:15:47 +00004231 pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_ONEROW|WHERE_INDEXED;
drhec95c442013-10-23 01:57:32 +00004232 if( pIdx->isCovering || (pItem->colUsed & ~columnsInIndex(pIdx))==0 ){
drh92a121f2013-06-10 12:15:47 +00004233 pLoop->wsFlags |= WHERE_IDX_ONLY;
4234 }
drh60c96cd2013-06-09 17:21:25 +00004235 pLoop->nLTerm = j;
4236 pLoop->u.btree.nEq = j;
4237 pLoop->u.btree.pIndex = pIdx;
drhe1e2e9a2013-06-13 15:16:53 +00004238 /* TUNING: Cost of a unique index lookup is 15 */
drhbf539c42013-10-05 18:16:02 +00004239 pLoop->rRun = 39; /* 39==sqlite3LogEst(15) */
drh60c96cd2013-06-09 17:21:25 +00004240 break;
4241 }
4242 }
drh3b75ffa2013-06-10 14:56:25 +00004243 if( pLoop->wsFlags ){
drhbf539c42013-10-05 18:16:02 +00004244 pLoop->nOut = (LogEst)1;
drh3b75ffa2013-06-10 14:56:25 +00004245 pWInfo->a[0].pWLoop = pLoop;
drh6f82e852015-06-06 20:12:09 +00004246 pLoop->maskSelf = sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur);
drh3b75ffa2013-06-10 14:56:25 +00004247 pWInfo->a[0].iTabCur = iCur;
4248 pWInfo->nRowOut = 1;
drhddba0c22014-03-18 20:33:42 +00004249 if( pWInfo->pOrderBy ) pWInfo->nOBSat = pWInfo->pOrderBy->nExpr;
drh6457a352013-06-21 00:35:37 +00004250 if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){
4251 pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE;
4252 }
drh3b75ffa2013-06-10 14:56:25 +00004253#ifdef SQLITE_DEBUG
4254 pLoop->cId = '0';
4255#endif
4256 return 1;
4257 }
4258 return 0;
drh60c96cd2013-06-09 17:21:25 +00004259}
4260
4261/*
drhe3184742002-06-19 14:27:05 +00004262** Generate the beginning of the loop used for WHERE clause processing.
drhacf3b982005-01-03 01:27:18 +00004263** The return value is a pointer to an opaque structure that contains
drh75897232000-05-29 14:26:00 +00004264** information needed to terminate the loop. Later, the calling routine
danielk19774adee202004-05-08 08:23:19 +00004265** should invoke sqlite3WhereEnd() with the return value of this function
drh75897232000-05-29 14:26:00 +00004266** in order to complete the WHERE clause processing.
4267**
4268** If an error occurs, this routine returns NULL.
drhc27a1ce2002-06-14 20:58:45 +00004269**
4270** The basic idea is to do a nested loop, one loop for each table in
4271** the FROM clause of a select. (INSERT and UPDATE statements are the
4272** same as a SELECT with only a single table in the FROM clause.) For
4273** example, if the SQL is this:
4274**
4275** SELECT * FROM t1, t2, t3 WHERE ...;
4276**
4277** Then the code generated is conceptually like the following:
4278**
4279** foreach row1 in t1 do \ Code generated
danielk19774adee202004-05-08 08:23:19 +00004280** foreach row2 in t2 do |-- by sqlite3WhereBegin()
drhc27a1ce2002-06-14 20:58:45 +00004281** foreach row3 in t3 do /
4282** ...
4283** end \ Code generated
danielk19774adee202004-05-08 08:23:19 +00004284** end |-- by sqlite3WhereEnd()
drhc27a1ce2002-06-14 20:58:45 +00004285** end /
4286**
drh29dda4a2005-07-21 18:23:20 +00004287** Note that the loops might not be nested in the order in which they
4288** appear in the FROM clause if a different order is better able to make
drh51147ba2005-07-23 22:59:55 +00004289** use of indices. Note also that when the IN operator appears in
4290** the WHERE clause, it might result in additional nested loops for
4291** scanning through all values on the right-hand side of the IN.
drh29dda4a2005-07-21 18:23:20 +00004292**
drhc27a1ce2002-06-14 20:58:45 +00004293** There are Btree cursors associated with each table. t1 uses cursor
drh6a3ea0e2003-05-02 14:32:12 +00004294** number pTabList->a[0].iCursor. t2 uses the cursor pTabList->a[1].iCursor.
4295** And so forth. This routine generates code to open those VDBE cursors
danielk19774adee202004-05-08 08:23:19 +00004296** and sqlite3WhereEnd() generates the code to close them.
drhc27a1ce2002-06-14 20:58:45 +00004297**
drhe6f85e72004-12-25 01:03:13 +00004298** The code that sqlite3WhereBegin() generates leaves the cursors named
4299** in pTabList pointing at their appropriate entries. The [...] code
drhf0863fe2005-06-12 21:35:51 +00004300** can use OP_Column and OP_Rowid opcodes on these cursors to extract
drhe6f85e72004-12-25 01:03:13 +00004301** data from the various tables of the loop.
4302**
drhc27a1ce2002-06-14 20:58:45 +00004303** If the WHERE clause is empty, the foreach loops must each scan their
4304** entire tables. Thus a three-way join is an O(N^3) operation. But if
4305** the tables have indices and there are terms in the WHERE clause that
4306** refer to those indices, a complete table scan can be avoided and the
4307** code will run much faster. Most of the work of this routine is checking
4308** to see if there are indices that can be used to speed up the loop.
4309**
4310** Terms of the WHERE clause are also used to limit which rows actually
4311** make it to the "..." in the middle of the loop. After each "foreach",
4312** terms of the WHERE clause that use only terms in that loop and outer
4313** loops are evaluated and if false a jump is made around all subsequent
4314** inner loops (or around the "..." if the test occurs within the inner-
4315** most loop)
4316**
4317** OUTER JOINS
4318**
4319** An outer join of tables t1 and t2 is conceptally coded as follows:
4320**
4321** foreach row1 in t1 do
4322** flag = 0
4323** foreach row2 in t2 do
4324** start:
4325** ...
4326** flag = 1
4327** end
drhe3184742002-06-19 14:27:05 +00004328** if flag==0 then
4329** move the row2 cursor to a null row
4330** goto start
4331** fi
drhc27a1ce2002-06-14 20:58:45 +00004332** end
4333**
drhe3184742002-06-19 14:27:05 +00004334** ORDER BY CLAUSE PROCESSING
4335**
drh94433422013-07-01 11:05:50 +00004336** pOrderBy is a pointer to the ORDER BY clause (or the GROUP BY clause
4337** if the WHERE_GROUPBY flag is set in wctrlFlags) of a SELECT statement
drhe3184742002-06-19 14:27:05 +00004338** if there is one. If there is no ORDER BY clause or if this routine
drh46ec5b62012-09-24 15:30:54 +00004339** is called from an UPDATE or DELETE statement, then pOrderBy is NULL.
drhfc8d4f92013-11-08 15:19:46 +00004340**
4341** The iIdxCur parameter is the cursor number of an index. If
drhce943bc2016-05-19 18:56:33 +00004342** WHERE_OR_SUBCLAUSE is set, iIdxCur is the cursor number of an index
drhfc8d4f92013-11-08 15:19:46 +00004343** to use for OR clause processing. The WHERE clause should use this
4344** specific cursor. If WHERE_ONEPASS_DESIRED is set, then iIdxCur is
4345** the first cursor in an array of cursors for all indices. iIdxCur should
4346** be used to compute the appropriate cursor depending on which index is
4347** used.
drh75897232000-05-29 14:26:00 +00004348*/
danielk19774adee202004-05-08 08:23:19 +00004349WhereInfo *sqlite3WhereBegin(
drhf1b5ff72016-04-14 13:35:26 +00004350 Parse *pParse, /* The parser context */
4351 SrcList *pTabList, /* FROM clause: A list of all tables to be scanned */
4352 Expr *pWhere, /* The WHERE clause */
4353 ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */
4354 ExprList *pDistinctSet, /* Try not to output two rows that duplicate these */
4355 u16 wctrlFlags, /* The WHERE_* flags defined in sqliteInt.h */
drhce943bc2016-05-19 18:56:33 +00004356 int iAuxArg /* If WHERE_OR_SUBCLAUSE is set, index cursor number
drhf1b5ff72016-04-14 13:35:26 +00004357 ** If WHERE_USE_LIMIT, then the limit amount */
drh75897232000-05-29 14:26:00 +00004358){
danielk1977be229652009-03-20 14:18:51 +00004359 int nByteWInfo; /* Num. bytes allocated for WhereInfo struct */
drhc01a3c12009-12-16 22:10:49 +00004360 int nTabList; /* Number of elements in pTabList */
drh75897232000-05-29 14:26:00 +00004361 WhereInfo *pWInfo; /* Will become the return value of this function */
4362 Vdbe *v = pParse->pVdbe; /* The virtual database engine */
drhfe05af82005-07-21 03:14:59 +00004363 Bitmask notReady; /* Cursors that are not yet positioned */
drh1c8148f2013-05-04 20:25:23 +00004364 WhereLoopBuilder sWLB; /* The WhereLoop builder */
drh111a6a72008-12-21 03:51:16 +00004365 WhereMaskSet *pMaskSet; /* The expression mask set */
drh56f1b992012-09-25 14:29:39 +00004366 WhereLevel *pLevel; /* A single level in pWInfo->a[] */
drhfd636c72013-06-21 02:05:06 +00004367 WhereLoop *pLoop; /* Pointer to a single WhereLoop object */
drh9cd1c992012-09-25 20:43:35 +00004368 int ii; /* Loop counter */
drh17435752007-08-16 04:30:38 +00004369 sqlite3 *db; /* Database connection */
drh5346e952013-05-08 14:14:26 +00004370 int rc; /* Return code */
drh9c0c57a2016-01-21 15:55:37 +00004371 u8 bFordelete = 0; /* OPFLAG_FORDELETE or zero, as appropriate */
drh75897232000-05-29 14:26:00 +00004372
danf0ee1d32015-09-12 19:26:11 +00004373 assert( (wctrlFlags & WHERE_ONEPASS_MULTIROW)==0 || (
4374 (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0
drhce943bc2016-05-19 18:56:33 +00004375 && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0
danf0ee1d32015-09-12 19:26:11 +00004376 ));
drh56f1b992012-09-25 14:29:39 +00004377
drhce943bc2016-05-19 18:56:33 +00004378 /* Only one of WHERE_OR_SUBCLAUSE or WHERE_USE_LIMIT */
4379 assert( (wctrlFlags & WHERE_OR_SUBCLAUSE)==0
drhc3489bb2016-02-25 16:04:59 +00004380 || (wctrlFlags & WHERE_USE_LIMIT)==0 );
4381
drh56f1b992012-09-25 14:29:39 +00004382 /* Variable initialization */
drhfd636c72013-06-21 02:05:06 +00004383 db = pParse->db;
drh1c8148f2013-05-04 20:25:23 +00004384 memset(&sWLB, 0, sizeof(sWLB));
drh0401ace2014-03-18 15:30:27 +00004385
4386 /* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */
4387 testcase( pOrderBy && pOrderBy->nExpr==BMS-1 );
4388 if( pOrderBy && pOrderBy->nExpr>=BMS ) pOrderBy = 0;
drh1c8148f2013-05-04 20:25:23 +00004389 sWLB.pOrderBy = pOrderBy;
drh56f1b992012-09-25 14:29:39 +00004390
drhfd636c72013-06-21 02:05:06 +00004391 /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via
4392 ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */
4393 if( OptimizationDisabled(db, SQLITE_DistinctOpt) ){
4394 wctrlFlags &= ~WHERE_WANT_DISTINCT;
4395 }
4396
drh29dda4a2005-07-21 18:23:20 +00004397 /* The number of tables in the FROM clause is limited by the number of
drh1398ad32005-01-19 23:24:50 +00004398 ** bits in a Bitmask
4399 */
drh67ae0cb2010-04-08 14:38:51 +00004400 testcase( pTabList->nSrc==BMS );
drh29dda4a2005-07-21 18:23:20 +00004401 if( pTabList->nSrc>BMS ){
4402 sqlite3ErrorMsg(pParse, "at most %d tables in a join", BMS);
drh1398ad32005-01-19 23:24:50 +00004403 return 0;
4404 }
4405
drhc01a3c12009-12-16 22:10:49 +00004406 /* This function normally generates a nested loop for all tables in
drhce943bc2016-05-19 18:56:33 +00004407 ** pTabList. But if the WHERE_OR_SUBCLAUSE flag is set, then we should
drhc01a3c12009-12-16 22:10:49 +00004408 ** only generate code for the first table in pTabList and assume that
4409 ** any cursors associated with subsequent tables are uninitialized.
4410 */
drhce943bc2016-05-19 18:56:33 +00004411 nTabList = (wctrlFlags & WHERE_OR_SUBCLAUSE) ? 1 : pTabList->nSrc;
drhc01a3c12009-12-16 22:10:49 +00004412
drh75897232000-05-29 14:26:00 +00004413 /* Allocate and initialize the WhereInfo structure that will become the
danielk1977be229652009-03-20 14:18:51 +00004414 ** return value. A single allocation is used to store the WhereInfo
4415 ** struct, the contents of WhereInfo.a[], the WhereClause structure
4416 ** and the WhereMaskSet structure. Since WhereClause contains an 8-byte
4417 ** field (type Bitmask) it must be aligned on an 8-byte boundary on
4418 ** some architectures. Hence the ROUND8() below.
drh75897232000-05-29 14:26:00 +00004419 */
drhc01a3c12009-12-16 22:10:49 +00004420 nByteWInfo = ROUND8(sizeof(WhereInfo)+(nTabList-1)*sizeof(WhereLevel));
drh87c05f02016-10-03 14:44:47 +00004421 pWInfo = sqlite3DbMallocRawNN(db, nByteWInfo + sizeof(WhereLoop));
drh17435752007-08-16 04:30:38 +00004422 if( db->mallocFailed ){
drh8b307fb2010-04-06 15:57:05 +00004423 sqlite3DbFree(db, pWInfo);
4424 pWInfo = 0;
danielk197785574e32008-10-06 05:32:18 +00004425 goto whereBeginError;
drh75897232000-05-29 14:26:00 +00004426 }
4427 pWInfo->pParse = pParse;
4428 pWInfo->pTabList = pTabList;
drh6b7157b2013-05-10 02:00:35 +00004429 pWInfo->pOrderBy = pOrderBy;
drhf1b5ff72016-04-14 13:35:26 +00004430 pWInfo->pDistinctSet = pDistinctSet;
drh87c05f02016-10-03 14:44:47 +00004431 pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1;
4432 pWInfo->nLevel = nTabList;
drha22a75e2014-03-21 18:16:23 +00004433 pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(v);
drh6df2acd2008-12-28 16:55:25 +00004434 pWInfo->wctrlFlags = wctrlFlags;
drhc3489bb2016-02-25 16:04:59 +00004435 pWInfo->iLimit = iAuxArg;
drh8b307fb2010-04-06 15:57:05 +00004436 pWInfo->savedNQueryLoop = pParse->nQueryLoop;
drh87c05f02016-10-03 14:44:47 +00004437 memset(&pWInfo->nOBSat, 0,
4438 offsetof(WhereInfo,sWC) - offsetof(WhereInfo,nOBSat));
4439 memset(&pWInfo->a[0], 0, sizeof(WhereLoop)+nTabList*sizeof(WhereLevel));
drhb0264ee2015-09-14 14:45:50 +00004440 assert( pWInfo->eOnePass==ONEPASS_OFF ); /* ONEPASS defaults to OFF */
drh70d18342013-06-06 19:16:33 +00004441 pMaskSet = &pWInfo->sMaskSet;
drh1c8148f2013-05-04 20:25:23 +00004442 sWLB.pWInfo = pWInfo;
drh70d18342013-06-06 19:16:33 +00004443 sWLB.pWC = &pWInfo->sWC;
drh1ac87e12013-07-18 14:50:56 +00004444 sWLB.pNew = (WhereLoop*)(((char*)pWInfo)+nByteWInfo);
4445 assert( EIGHT_BYTE_ALIGNMENT(sWLB.pNew) );
drh60c96cd2013-06-09 17:21:25 +00004446 whereLoopInit(sWLB.pNew);
drhb8a8e8a2013-06-10 19:12:39 +00004447#ifdef SQLITE_DEBUG
4448 sWLB.pNew->cId = '*';
4449#endif
drh08192d52002-04-30 19:20:28 +00004450
drh111a6a72008-12-21 03:51:16 +00004451 /* Split the WHERE clause into separate subexpressions where each
4452 ** subexpression is separated by an AND operator.
4453 */
4454 initMaskSet(pMaskSet);
drh6c1f4ef2015-06-08 14:23:15 +00004455 sqlite3WhereClauseInit(&pWInfo->sWC, pWInfo);
4456 sqlite3WhereSplit(&pWInfo->sWC, pWhere, TK_AND);
drh111a6a72008-12-21 03:51:16 +00004457
drh08192d52002-04-30 19:20:28 +00004458 /* Special case: a WHERE clause that is constant. Evaluate the
4459 ** expression and either jump over all of the code or fall thru.
4460 */
drh759e8582014-01-02 21:05:10 +00004461 for(ii=0; ii<sWLB.pWC->nTerm; ii++){
4462 if( nTabList==0 || sqlite3ExprIsConstantNotJoin(sWLB.pWC->a[ii].pExpr) ){
4463 sqlite3ExprIfFalse(pParse, sWLB.pWC->a[ii].pExpr, pWInfo->iBreak,
4464 SQLITE_JUMPIFNULL);
4465 sWLB.pWC->a[ii].wtFlags |= TERM_CODED;
4466 }
drh08192d52002-04-30 19:20:28 +00004467 }
drh75897232000-05-29 14:26:00 +00004468
drh4fe425a2013-06-12 17:08:06 +00004469 /* Special case: No FROM clause
4470 */
4471 if( nTabList==0 ){
drhddba0c22014-03-18 20:33:42 +00004472 if( pOrderBy ) pWInfo->nOBSat = pOrderBy->nExpr;
drh6457a352013-06-21 00:35:37 +00004473 if( wctrlFlags & WHERE_WANT_DISTINCT ){
4474 pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE;
4475 }
drh4fe425a2013-06-12 17:08:06 +00004476 }
4477
drh42165be2008-03-26 14:56:34 +00004478 /* Assign a bit from the bitmask to every term in the FROM clause.
4479 **
drh47991422015-08-31 15:58:06 +00004480 ** The N-th term of the FROM clause is assigned a bitmask of 1<<N.
4481 **
4482 ** The rule of the previous sentence ensures thta if X is the bitmask for
4483 ** a table T, then X-1 is the bitmask for all other tables to the left of T.
4484 ** Knowing the bitmask for all tables to the left of a left join is
4485 ** important. Ticket #3015.
danielk1977e672c8e2009-05-22 15:43:26 +00004486 **
drhc01a3c12009-12-16 22:10:49 +00004487 ** Note that bitmasks are created for all pTabList->nSrc tables in
4488 ** pTabList, not just the first nTabList tables. nTabList is normally
4489 ** equal to pTabList->nSrc but might be shortened to 1 if the
drhce943bc2016-05-19 18:56:33 +00004490 ** WHERE_OR_SUBCLAUSE flag is set.
drh42165be2008-03-26 14:56:34 +00004491 */
drh9cd1c992012-09-25 20:43:35 +00004492 for(ii=0; ii<pTabList->nSrc; ii++){
4493 createMask(pMaskSet, pTabList->a[ii].iCursor);
drh01d230c2015-08-19 17:11:37 +00004494 sqlite3WhereTabFuncArgs(pParse, &pTabList->a[ii], &pWInfo->sWC);
drh42165be2008-03-26 14:56:34 +00004495 }
drh47991422015-08-31 15:58:06 +00004496#ifdef SQLITE_DEBUG
4497 for(ii=0; ii<pTabList->nSrc; ii++){
4498 Bitmask m = sqlite3WhereGetMask(pMaskSet, pTabList->a[ii].iCursor);
4499 assert( m==MASKBIT(ii) );
drh42165be2008-03-26 14:56:34 +00004500 }
4501#endif
4502
drhb121dd12015-06-06 18:30:17 +00004503 /* Analyze all of the subexpressions. */
drh6c1f4ef2015-06-08 14:23:15 +00004504 sqlite3WhereExprAnalyze(pTabList, &pWInfo->sWC);
drhb121dd12015-06-06 18:30:17 +00004505 if( db->mallocFailed ) goto whereBeginError;
drh75897232000-05-29 14:26:00 +00004506
drh6457a352013-06-21 00:35:37 +00004507 if( wctrlFlags & WHERE_WANT_DISTINCT ){
drhf1b5ff72016-04-14 13:35:26 +00004508 if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pDistinctSet) ){
drh6457a352013-06-21 00:35:37 +00004509 /* The DISTINCT marking is pointless. Ignore it. */
drh4f402f22013-06-11 18:59:38 +00004510 pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE;
4511 }else if( pOrderBy==0 ){
drh6457a352013-06-21 00:35:37 +00004512 /* Try to ORDER BY the result set to make distinct processing easier */
drh4f402f22013-06-11 18:59:38 +00004513 pWInfo->wctrlFlags |= WHERE_DISTINCTBY;
drhf1b5ff72016-04-14 13:35:26 +00004514 pWInfo->pOrderBy = pDistinctSet;
drh4f402f22013-06-11 18:59:38 +00004515 }
dan38cc40c2011-06-30 20:17:15 +00004516 }
4517
drhf1b5f5b2013-05-02 00:15:01 +00004518 /* Construct the WhereLoop objects */
drhc90713d2014-09-30 13:46:49 +00004519#if defined(WHERETRACE_ENABLED)
drhc3489bb2016-02-25 16:04:59 +00004520 if( sqlite3WhereTrace & 0xffff ){
4521 sqlite3DebugPrintf("*** Optimizer Start *** (wctrlFlags: 0x%x",wctrlFlags);
4522 if( wctrlFlags & WHERE_USE_LIMIT ){
4523 sqlite3DebugPrintf(", limit: %d", iAuxArg);
4524 }
4525 sqlite3DebugPrintf(")\n");
4526 }
drhb121dd12015-06-06 18:30:17 +00004527 if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */
drhc84a4022016-05-27 12:30:20 +00004528 sqlite3WhereClausePrint(sWLB.pWC);
drhc90713d2014-09-30 13:46:49 +00004529 }
4530#endif
4531
drhb8a8e8a2013-06-10 19:12:39 +00004532 if( nTabList!=1 || whereShortCut(&sWLB)==0 ){
drh60c96cd2013-06-09 17:21:25 +00004533 rc = whereLoopAddAll(&sWLB);
4534 if( rc ) goto whereBeginError;
4535
drhb121dd12015-06-06 18:30:17 +00004536#ifdef WHERETRACE_ENABLED
4537 if( sqlite3WhereTrace ){ /* Display all of the WhereLoop objects */
drh60c96cd2013-06-09 17:21:25 +00004538 WhereLoop *p;
drhfd636c72013-06-21 02:05:06 +00004539 int i;
drhb121dd12015-06-06 18:30:17 +00004540 static const char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz"
4541 "ABCDEFGHIJKLMNOPQRSTUVWYXZ";
drhfd636c72013-06-21 02:05:06 +00004542 for(p=pWInfo->pLoops, i=0; p; p=p->pNextLoop, i++){
4543 p->cId = zLabel[i%sizeof(zLabel)];
drhc1ba2e72013-10-28 19:03:21 +00004544 whereLoopPrint(p, sWLB.pWC);
drh60c96cd2013-06-09 17:21:25 +00004545 }
4546 }
4547#endif
4548
drh4f402f22013-06-11 18:59:38 +00004549 wherePathSolver(pWInfo, 0);
drh60c96cd2013-06-09 17:21:25 +00004550 if( db->mallocFailed ) goto whereBeginError;
4551 if( pWInfo->pOrderBy ){
drhc7f0d222013-06-19 03:27:12 +00004552 wherePathSolver(pWInfo, pWInfo->nRowOut+1);
drh60c96cd2013-06-09 17:21:25 +00004553 if( db->mallocFailed ) goto whereBeginError;
drha18f3d22013-05-08 03:05:41 +00004554 }
4555 }
drh60c96cd2013-06-09 17:21:25 +00004556 if( pWInfo->pOrderBy==0 && (db->flags & SQLITE_ReverseOrder)!=0 ){
drh8426e362016-03-08 01:32:30 +00004557 pWInfo->revMask = ALLBITS;
drha50ef112013-05-22 02:06:59 +00004558 }
drh81186b42013-06-18 01:52:41 +00004559 if( pParse->nErr || NEVER(db->mallocFailed) ){
drh75b93402013-05-31 20:43:57 +00004560 goto whereBeginError;
4561 }
drhb121dd12015-06-06 18:30:17 +00004562#ifdef WHERETRACE_ENABLED
drha18f3d22013-05-08 03:05:41 +00004563 if( sqlite3WhereTrace ){
drh4f402f22013-06-11 18:59:38 +00004564 sqlite3DebugPrintf("---- Solution nRow=%d", pWInfo->nRowOut);
drhddba0c22014-03-18 20:33:42 +00004565 if( pWInfo->nOBSat>0 ){
4566 sqlite3DebugPrintf(" ORDERBY=%d,0x%llx", pWInfo->nOBSat, pWInfo->revMask);
drh319f6772013-05-14 15:31:07 +00004567 }
drh4f402f22013-06-11 18:59:38 +00004568 switch( pWInfo->eDistinct ){
4569 case WHERE_DISTINCT_UNIQUE: {
4570 sqlite3DebugPrintf(" DISTINCT=unique");
4571 break;
4572 }
4573 case WHERE_DISTINCT_ORDERED: {
4574 sqlite3DebugPrintf(" DISTINCT=ordered");
4575 break;
4576 }
4577 case WHERE_DISTINCT_UNORDERED: {
4578 sqlite3DebugPrintf(" DISTINCT=unordered");
4579 break;
4580 }
4581 }
4582 sqlite3DebugPrintf("\n");
drhfd636c72013-06-21 02:05:06 +00004583 for(ii=0; ii<pWInfo->nLevel; ii++){
drhc1ba2e72013-10-28 19:03:21 +00004584 whereLoopPrint(pWInfo->a[ii].pWLoop, sWLB.pWC);
drhf1b5f5b2013-05-02 00:15:01 +00004585 }
4586 }
4587#endif
drhfd636c72013-06-21 02:05:06 +00004588 /* Attempt to omit tables from the join that do not effect the result */
drh1031bd92013-06-22 15:44:26 +00004589 if( pWInfo->nLevel>=2
drhf1b5ff72016-04-14 13:35:26 +00004590 && pDistinctSet!=0
drh1031bd92013-06-22 15:44:26 +00004591 && OptimizationEnabled(db, SQLITE_OmitNoopJoin)
4592 ){
drhf1b5ff72016-04-14 13:35:26 +00004593 Bitmask tabUsed = sqlite3WhereExprListUsage(pMaskSet, pDistinctSet);
drh6c1f4ef2015-06-08 14:23:15 +00004594 if( sWLB.pOrderBy ){
4595 tabUsed |= sqlite3WhereExprListUsage(pMaskSet, sWLB.pOrderBy);
4596 }
drhfd636c72013-06-21 02:05:06 +00004597 while( pWInfo->nLevel>=2 ){
drh9d5a5792013-06-28 13:43:33 +00004598 WhereTerm *pTerm, *pEnd;
drhfd636c72013-06-21 02:05:06 +00004599 pLoop = pWInfo->a[pWInfo->nLevel-1].pWLoop;
drh8a48b9c2015-08-19 15:20:00 +00004600 if( (pWInfo->pTabList->a[pLoop->iTab].fg.jointype & JT_LEFT)==0 ) break;
drhbc71b1d2013-06-21 02:15:48 +00004601 if( (wctrlFlags & WHERE_WANT_DISTINCT)==0
4602 && (pLoop->wsFlags & WHERE_ONEROW)==0
drhfd636c72013-06-21 02:05:06 +00004603 ){
drhfd636c72013-06-21 02:05:06 +00004604 break;
4605 }
drhbc71b1d2013-06-21 02:15:48 +00004606 if( (tabUsed & pLoop->maskSelf)!=0 ) break;
drh9d5a5792013-06-28 13:43:33 +00004607 pEnd = sWLB.pWC->a + sWLB.pWC->nTerm;
4608 for(pTerm=sWLB.pWC->a; pTerm<pEnd; pTerm++){
4609 if( (pTerm->prereqAll & pLoop->maskSelf)!=0
4610 && !ExprHasProperty(pTerm->pExpr, EP_FromJoin)
4611 ){
4612 break;
4613 }
4614 }
4615 if( pTerm<pEnd ) break;
drhbc71b1d2013-06-21 02:15:48 +00004616 WHERETRACE(0xffff, ("-> drop loop %c not used\n", pLoop->cId));
4617 pWInfo->nLevel--;
4618 nTabList--;
drhfd636c72013-06-21 02:05:06 +00004619 }
4620 }
drh3b48e8c2013-06-12 20:18:16 +00004621 WHERETRACE(0xffff,("*** Optimizer Finished ***\n"));
drh8e23daf2013-06-11 13:30:04 +00004622 pWInfo->pParse->nQueryLoop += pWInfo->nRowOut;
drhf1b5f5b2013-05-02 00:15:01 +00004623
drh08c88eb2008-04-10 13:33:18 +00004624 /* If the caller is an UPDATE or DELETE statement that is requesting
4625 ** to use a one-pass algorithm, determine if this is appropriate.
drh08c88eb2008-04-10 13:33:18 +00004626 */
drh165be382008-12-05 02:36:33 +00004627 assert( (wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || pWInfo->nLevel==1 );
danf0ee1d32015-09-12 19:26:11 +00004628 if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 ){
4629 int wsFlags = pWInfo->a[0].pWLoop->wsFlags;
4630 int bOnerow = (wsFlags & WHERE_ONEROW)!=0;
drh9c0c57a2016-01-21 15:55:37 +00004631 if( bOnerow
4632 || ((wctrlFlags & WHERE_ONEPASS_MULTIROW)!=0
4633 && 0==(wsFlags & WHERE_VIRTUALTABLE))
4634 ){
drhb0264ee2015-09-14 14:45:50 +00004635 pWInfo->eOnePass = bOnerow ? ONEPASS_SINGLE : ONEPASS_MULTI;
danfd261ec2015-10-22 20:54:33 +00004636 if( HasRowid(pTabList->a[0].pTab) && (wsFlags & WHERE_IDX_ONLY) ){
4637 if( wctrlFlags & WHERE_ONEPASS_MULTIROW ){
4638 bFordelete = OPFLAG_FORDELETE;
4639 }
4640 pWInfo->a[0].pWLoop->wsFlags = (wsFlags & ~WHERE_IDX_ONLY);
danf0ee1d32015-09-12 19:26:11 +00004641 }
drh702ba9f2013-11-07 21:25:13 +00004642 }
drh08c88eb2008-04-10 13:33:18 +00004643 }
drheb04de32013-05-10 15:16:30 +00004644
drh9012bcb2004-12-19 00:11:35 +00004645 /* Open all tables in the pTabList and any indices selected for
4646 ** searching those tables.
4647 */
drh9cd1c992012-09-25 20:43:35 +00004648 for(ii=0, pLevel=pWInfo->a; ii<nTabList; ii++, pLevel++){
danielk1977da184232006-01-05 11:34:32 +00004649 Table *pTab; /* Table to open */
danielk1977da184232006-01-05 11:34:32 +00004650 int iDb; /* Index of database containing table/index */
drh56f1b992012-09-25 14:29:39 +00004651 struct SrcList_item *pTabItem;
drh9012bcb2004-12-19 00:11:35 +00004652
drh29dda4a2005-07-21 18:23:20 +00004653 pTabItem = &pTabList->a[pLevel->iFrom];
drh9012bcb2004-12-19 00:11:35 +00004654 pTab = pTabItem->pTab;
danielk1977595a5232009-07-24 17:58:53 +00004655 iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
drh7ba39a92013-05-30 17:43:19 +00004656 pLoop = pLevel->pWLoop;
drh424aab82010-04-06 18:28:20 +00004657 if( (pTab->tabFlags & TF_Ephemeral)!=0 || pTab->pSelect ){
drh75bb9f52010-04-06 18:51:42 +00004658 /* Do nothing */
4659 }else
drh9eff6162006-06-12 21:59:13 +00004660#ifndef SQLITE_OMIT_VIRTUALTABLE
drh7ba39a92013-05-30 17:43:19 +00004661 if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){
danielk1977595a5232009-07-24 17:58:53 +00004662 const char *pVTab = (const char *)sqlite3GetVTable(db, pTab);
danielk197793626f42006-06-20 13:07:27 +00004663 int iCur = pTabItem->iCursor;
danielk1977595a5232009-07-24 17:58:53 +00004664 sqlite3VdbeAddOp4(v, OP_VOpen, iCur, 0, 0, pVTab, P4_VTAB);
drhfc5e5462012-12-03 17:04:40 +00004665 }else if( IsVirtual(pTab) ){
4666 /* noop */
drh9eff6162006-06-12 21:59:13 +00004667 }else
4668#endif
drh7ba39a92013-05-30 17:43:19 +00004669 if( (pLoop->wsFlags & WHERE_IDX_ONLY)==0
drhce943bc2016-05-19 18:56:33 +00004670 && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0 ){
drhfc8d4f92013-11-08 15:19:46 +00004671 int op = OP_OpenRead;
drhb0264ee2015-09-14 14:45:50 +00004672 if( pWInfo->eOnePass!=ONEPASS_OFF ){
drhfc8d4f92013-11-08 15:19:46 +00004673 op = OP_OpenWrite;
4674 pWInfo->aiCurOnePass[0] = pTabItem->iCursor;
4675 };
drh08c88eb2008-04-10 13:33:18 +00004676 sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op);
drhfc8d4f92013-11-08 15:19:46 +00004677 assert( pTabItem->iCursor==pLevel->iTabCur );
drhb0264ee2015-09-14 14:45:50 +00004678 testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS-1 );
4679 testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS );
4680 if( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol<BMS && HasRowid(pTab) ){
danielk19779792eef2006-01-13 15:58:43 +00004681 Bitmask b = pTabItem->colUsed;
4682 int n = 0;
drh74161702006-02-24 02:53:49 +00004683 for(; b; b=b>>1, n++){}
drh00dceca2016-01-11 22:58:50 +00004684 sqlite3VdbeChangeP4(v, -1, SQLITE_INT_TO_PTR(n), P4_INT32);
danielk19779792eef2006-01-13 15:58:43 +00004685 assert( n<=pTab->nCol );
4686 }
drh2f2b0272015-08-14 18:50:04 +00004687#ifdef SQLITE_ENABLE_CURSOR_HINTS
danc5dc3dc2015-10-26 20:11:24 +00004688 if( pLoop->u.btree.pIndex!=0 ){
4689 sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ|bFordelete);
4690 }else
drh2f2b0272015-08-14 18:50:04 +00004691#endif
danc5dc3dc2015-10-26 20:11:24 +00004692 {
4693 sqlite3VdbeChangeP5(v, bFordelete);
4694 }
drh97bae792015-06-05 15:59:57 +00004695#ifdef SQLITE_ENABLE_COLUMN_USED_MASK
4696 sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, pTabItem->iCursor, 0, 0,
4697 (const u8*)&pTabItem->colUsed, P4_INT64);
4698#endif
danielk1977c00da102006-01-07 13:21:04 +00004699 }else{
4700 sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName);
drh9012bcb2004-12-19 00:11:35 +00004701 }
drh7e47cb82013-05-31 17:55:27 +00004702 if( pLoop->wsFlags & WHERE_INDEXED ){
drh7ba39a92013-05-30 17:43:19 +00004703 Index *pIx = pLoop->u.btree.pIndex;
drhfc8d4f92013-11-08 15:19:46 +00004704 int iIndexCur;
4705 int op = OP_OpenRead;
drhc3489bb2016-02-25 16:04:59 +00004706 /* iAuxArg is always set if to a positive value if ONEPASS is possible */
4707 assert( iAuxArg!=0 || (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 );
drh48dd1d82014-05-27 18:18:58 +00004708 if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIx)
drhce943bc2016-05-19 18:56:33 +00004709 && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0
drha3bc66a2014-05-27 17:57:32 +00004710 ){
4711 /* This is one term of an OR-optimization using the PRIMARY KEY of a
4712 ** WITHOUT ROWID table. No need for a separate index */
4713 iIndexCur = pLevel->iTabCur;
4714 op = 0;
drhb0264ee2015-09-14 14:45:50 +00004715 }else if( pWInfo->eOnePass!=ONEPASS_OFF ){
drhfc8d4f92013-11-08 15:19:46 +00004716 Index *pJ = pTabItem->pTab->pIndex;
drhc3489bb2016-02-25 16:04:59 +00004717 iIndexCur = iAuxArg;
drhfc8d4f92013-11-08 15:19:46 +00004718 assert( wctrlFlags & WHERE_ONEPASS_DESIRED );
4719 while( ALWAYS(pJ) && pJ!=pIx ){
4720 iIndexCur++;
4721 pJ = pJ->pNext;
4722 }
4723 op = OP_OpenWrite;
4724 pWInfo->aiCurOnePass[1] = iIndexCur;
drhce943bc2016-05-19 18:56:33 +00004725 }else if( iAuxArg && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 ){
drhc3489bb2016-02-25 16:04:59 +00004726 iIndexCur = iAuxArg;
drha72a15e2016-05-09 19:58:56 +00004727 op = OP_ReopenIdx;
drhfc8d4f92013-11-08 15:19:46 +00004728 }else{
4729 iIndexCur = pParse->nTab++;
4730 }
4731 pLevel->iIdxCur = iIndexCur;
danielk1977da184232006-01-05 11:34:32 +00004732 assert( pIx->pSchema==pTab->pSchema );
drhb0367fb2012-08-25 02:11:13 +00004733 assert( iIndexCur>=0 );
drha3bc66a2014-05-27 17:57:32 +00004734 if( op ){
4735 sqlite3VdbeAddOp3(v, op, iIndexCur, pIx->tnum, iDb);
4736 sqlite3VdbeSetP4KeyInfo(pParse, pIx);
drhe0997b32015-03-20 14:57:50 +00004737 if( (pLoop->wsFlags & WHERE_CONSTRAINT)!=0
4738 && (pLoop->wsFlags & (WHERE_COLUMN_RANGE|WHERE_SKIPSCAN))==0
4739 && (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0
4740 ){
4741 sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ); /* Hint to COMDB2 */
4742 }
drha3bc66a2014-05-27 17:57:32 +00004743 VdbeComment((v, "%s", pIx->zName));
drh97bae792015-06-05 15:59:57 +00004744#ifdef SQLITE_ENABLE_COLUMN_USED_MASK
4745 {
4746 u64 colUsed = 0;
4747 int ii, jj;
4748 for(ii=0; ii<pIx->nColumn; ii++){
4749 jj = pIx->aiColumn[ii];
4750 if( jj<0 ) continue;
4751 if( jj>63 ) jj = 63;
4752 if( (pTabItem->colUsed & MASKBIT(jj))==0 ) continue;
4753 colUsed |= ((u64)1)<<(ii<63 ? ii : 63);
4754 }
4755 sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, iIndexCur, 0, 0,
4756 (u8*)&colUsed, P4_INT64);
4757 }
4758#endif /* SQLITE_ENABLE_COLUMN_USED_MASK */
drha3bc66a2014-05-27 17:57:32 +00004759 }
drh9012bcb2004-12-19 00:11:35 +00004760 }
drhaceb31b2014-02-08 01:40:27 +00004761 if( iDb>=0 ) sqlite3CodeVerifySchema(pParse, iDb);
drh9012bcb2004-12-19 00:11:35 +00004762 }
4763 pWInfo->iTop = sqlite3VdbeCurrentAddr(v);
drha21a64d2010-04-06 22:33:55 +00004764 if( db->mallocFailed ) goto whereBeginError;
drh9012bcb2004-12-19 00:11:35 +00004765
drh29dda4a2005-07-21 18:23:20 +00004766 /* Generate the code to do the search. Each iteration of the for
4767 ** loop below generates code for a single nested loop of the VM
4768 ** program.
drh75897232000-05-29 14:26:00 +00004769 */
drhfe05af82005-07-21 03:14:59 +00004770 notReady = ~(Bitmask)0;
drh9cd1c992012-09-25 20:43:35 +00004771 for(ii=0; ii<nTabList; ii++){
dan6f9702e2014-11-01 20:38:06 +00004772 int addrExplain;
4773 int wsFlags;
drh9cd1c992012-09-25 20:43:35 +00004774 pLevel = &pWInfo->a[ii];
dan6f9702e2014-11-01 20:38:06 +00004775 wsFlags = pLevel->pWLoop->wsFlags;
drhcc04afd2013-08-22 02:56:28 +00004776#ifndef SQLITE_OMIT_AUTOMATIC_INDEX
4777 if( (pLevel->pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 ){
4778 constructAutomaticIndex(pParse, &pWInfo->sWC,
4779 &pTabList->a[pLevel->iFrom], notReady, pLevel);
4780 if( db->mallocFailed ) goto whereBeginError;
4781 }
4782#endif
drh6f82e852015-06-06 20:12:09 +00004783 addrExplain = sqlite3WhereExplainOneScan(
dan6f9702e2014-11-01 20:38:06 +00004784 pParse, pTabList, pLevel, ii, pLevel->iFrom, wctrlFlags
4785 );
drhcc04afd2013-08-22 02:56:28 +00004786 pLevel->addrBody = sqlite3VdbeCurrentAddr(v);
drh6f82e852015-06-06 20:12:09 +00004787 notReady = sqlite3WhereCodeOneLoopStart(pWInfo, ii, notReady);
dan4a07e3d2010-11-09 14:48:59 +00004788 pWInfo->iContinue = pLevel->addrCont;
drhce943bc2016-05-19 18:56:33 +00004789 if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_OR_SUBCLAUSE)==0 ){
drh6f82e852015-06-06 20:12:09 +00004790 sqlite3WhereAddScanStatus(v, pTabList, pLevel, addrExplain);
dan6f9702e2014-11-01 20:38:06 +00004791 }
drh75897232000-05-29 14:26:00 +00004792 }
drh7ec764a2005-07-21 03:48:20 +00004793
drh6fa978d2013-05-30 19:29:19 +00004794 /* Done. */
drh6bc69a22013-11-19 12:33:23 +00004795 VdbeModuleComment((v, "Begin WHERE-core"));
drh75897232000-05-29 14:26:00 +00004796 return pWInfo;
drhe23399f2005-07-22 00:31:39 +00004797
4798 /* Jump here if malloc fails */
danielk197785574e32008-10-06 05:32:18 +00004799whereBeginError:
drh8b307fb2010-04-06 15:57:05 +00004800 if( pWInfo ){
4801 pParse->nQueryLoop = pWInfo->savedNQueryLoop;
4802 whereInfoFree(db, pWInfo);
4803 }
drhe23399f2005-07-22 00:31:39 +00004804 return 0;
drh75897232000-05-29 14:26:00 +00004805}
4806
4807/*
drhc27a1ce2002-06-14 20:58:45 +00004808** Generate the end of the WHERE loop. See comments on
danielk19774adee202004-05-08 08:23:19 +00004809** sqlite3WhereBegin() for additional information.
drh75897232000-05-29 14:26:00 +00004810*/
danielk19774adee202004-05-08 08:23:19 +00004811void sqlite3WhereEnd(WhereInfo *pWInfo){
drh633e6d52008-07-28 19:34:53 +00004812 Parse *pParse = pWInfo->pParse;
4813 Vdbe *v = pParse->pVdbe;
drh19a775c2000-06-05 18:54:46 +00004814 int i;
drh6b563442001-11-07 16:48:26 +00004815 WhereLevel *pLevel;
drh7ba39a92013-05-30 17:43:19 +00004816 WhereLoop *pLoop;
drhad3cab52002-05-24 02:04:32 +00004817 SrcList *pTabList = pWInfo->pTabList;
drh633e6d52008-07-28 19:34:53 +00004818 sqlite3 *db = pParse->db;
drh19a775c2000-06-05 18:54:46 +00004819
drh9012bcb2004-12-19 00:11:35 +00004820 /* Generate loop termination code.
4821 */
drh6bc69a22013-11-19 12:33:23 +00004822 VdbeModuleComment((v, "End WHERE-core"));
drhceea3322009-04-23 13:22:42 +00004823 sqlite3ExprCacheClear(pParse);
drhc01a3c12009-12-16 22:10:49 +00004824 for(i=pWInfo->nLevel-1; i>=0; i--){
drhcd8629e2013-11-13 12:27:25 +00004825 int addr;
drh6b563442001-11-07 16:48:26 +00004826 pLevel = &pWInfo->a[i];
drh7ba39a92013-05-30 17:43:19 +00004827 pLoop = pLevel->pWLoop;
drhb3190c12008-12-08 21:37:14 +00004828 sqlite3VdbeResolveLabel(v, pLevel->addrCont);
drh6b563442001-11-07 16:48:26 +00004829 if( pLevel->op!=OP_Noop ){
drhe39a7322014-02-03 14:04:11 +00004830 sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3);
drhd1d38482008-10-07 23:46:38 +00004831 sqlite3VdbeChangeP5(v, pLevel->p5);
drh688852a2014-02-17 22:40:43 +00004832 VdbeCoverage(v);
drh7d176102014-02-18 03:07:12 +00004833 VdbeCoverageIf(v, pLevel->op==OP_Next);
4834 VdbeCoverageIf(v, pLevel->op==OP_Prev);
4835 VdbeCoverageIf(v, pLevel->op==OP_VNext);
drh19a775c2000-06-05 18:54:46 +00004836 }
drh7ba39a92013-05-30 17:43:19 +00004837 if( pLoop->wsFlags & WHERE_IN_ABLE && pLevel->u.in.nIn>0 ){
drh72e8fa42007-03-28 14:30:06 +00004838 struct InLoop *pIn;
drhe23399f2005-07-22 00:31:39 +00004839 int j;
drhb3190c12008-12-08 21:37:14 +00004840 sqlite3VdbeResolveLabel(v, pLevel->addrNxt);
drh111a6a72008-12-21 03:51:16 +00004841 for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){
drhb3190c12008-12-08 21:37:14 +00004842 sqlite3VdbeJumpHere(v, pIn->addrInTop+1);
dan8da209b2016-07-26 18:06:08 +00004843 if( pIn->eEndLoopOp!=OP_Noop ){
4844 sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop);
4845 VdbeCoverage(v);
4846 VdbeCoverageIf(v, pIn->eEndLoopOp==OP_PrevIfOpen);
4847 VdbeCoverageIf(v, pIn->eEndLoopOp==OP_NextIfOpen);
4848 }
drhb3190c12008-12-08 21:37:14 +00004849 sqlite3VdbeJumpHere(v, pIn->addrInTop-1);
drhe23399f2005-07-22 00:31:39 +00004850 }
drhd99f7062002-06-08 23:25:08 +00004851 }
drhb3190c12008-12-08 21:37:14 +00004852 sqlite3VdbeResolveLabel(v, pLevel->addrBrk);
drhcd8629e2013-11-13 12:27:25 +00004853 if( pLevel->addrSkip ){
drh076e85f2015-09-03 13:46:12 +00004854 sqlite3VdbeGoto(v, pLevel->addrSkip);
drhe084f402013-11-13 17:24:38 +00004855 VdbeComment((v, "next skip-scan on %s", pLoop->u.btree.pIndex->zName));
drh2e5ef4e2013-11-13 16:58:54 +00004856 sqlite3VdbeJumpHere(v, pLevel->addrSkip);
4857 sqlite3VdbeJumpHere(v, pLevel->addrSkip-2);
drhcd8629e2013-11-13 12:27:25 +00004858 }
drh41d2e662015-12-01 21:23:07 +00004859#ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS
drhf07cf6e2015-03-06 16:45:16 +00004860 if( pLevel->addrLikeRep ){
drh44aebff2016-05-02 10:25:42 +00004861 sqlite3VdbeAddOp2(v, OP_DecrJumpZero, (int)(pLevel->iLikeRepCntr>>1),
4862 pLevel->addrLikeRep);
drhf07cf6e2015-03-06 16:45:16 +00004863 VdbeCoverage(v);
drhf07cf6e2015-03-06 16:45:16 +00004864 }
drh41d2e662015-12-01 21:23:07 +00004865#endif
drhad2d8302002-05-24 20:31:36 +00004866 if( pLevel->iLeftJoin ){
danb40897a2016-10-26 15:46:09 +00004867 int ws = pLoop->wsFlags;
drh688852a2014-02-17 22:40:43 +00004868 addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v);
danb40897a2016-10-26 15:46:09 +00004869 assert( (ws & WHERE_IDX_ONLY)==0 || (ws & WHERE_INDEXED)!=0 );
4870 if( (ws & WHERE_IDX_ONLY)==0 ){
drh35451c62009-11-12 04:26:39 +00004871 sqlite3VdbeAddOp1(v, OP_NullRow, pTabList->a[i].iCursor);
4872 }
danb40897a2016-10-26 15:46:09 +00004873 if( (ws & WHERE_INDEXED)
4874 || ((ws & WHERE_MULTI_OR) && pLevel->u.pCovidx)
4875 ){
drh3c84ddf2008-01-09 02:15:38 +00004876 sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur);
drh7f09b3e2002-08-13 13:15:49 +00004877 }
drh336a5302009-04-24 15:46:21 +00004878 if( pLevel->op==OP_Return ){
4879 sqlite3VdbeAddOp2(v, OP_Gosub, pLevel->p1, pLevel->addrFirst);
4880 }else{
drh076e85f2015-09-03 13:46:12 +00004881 sqlite3VdbeGoto(v, pLevel->addrFirst);
drh336a5302009-04-24 15:46:21 +00004882 }
drhd654be82005-09-20 17:42:23 +00004883 sqlite3VdbeJumpHere(v, addr);
drhad2d8302002-05-24 20:31:36 +00004884 }
drh6bc69a22013-11-19 12:33:23 +00004885 VdbeModuleComment((v, "End WHERE-loop%d: %s", i,
drhfc8d4f92013-11-08 15:19:46 +00004886 pWInfo->pTabList->a[pLevel->iFrom].pTab->zName));
drh19a775c2000-06-05 18:54:46 +00004887 }
drh9012bcb2004-12-19 00:11:35 +00004888
4889 /* The "break" point is here, just past the end of the outer loop.
4890 ** Set it.
4891 */
danielk19774adee202004-05-08 08:23:19 +00004892 sqlite3VdbeResolveLabel(v, pWInfo->iBreak);
drh9012bcb2004-12-19 00:11:35 +00004893
drhfd636c72013-06-21 02:05:06 +00004894 assert( pWInfo->nLevel<=pTabList->nSrc );
drhc01a3c12009-12-16 22:10:49 +00004895 for(i=0, pLevel=pWInfo->a; i<pWInfo->nLevel; i++, pLevel++){
drh5f612292014-02-08 23:20:32 +00004896 int k, last;
4897 VdbeOp *pOp;
danbfca6a42012-08-24 10:52:35 +00004898 Index *pIdx = 0;
drh29dda4a2005-07-21 18:23:20 +00004899 struct SrcList_item *pTabItem = &pTabList->a[pLevel->iFrom];
drh9012bcb2004-12-19 00:11:35 +00004900 Table *pTab = pTabItem->pTab;
drh5cf590c2003-04-24 01:45:04 +00004901 assert( pTab!=0 );
drh7ba39a92013-05-30 17:43:19 +00004902 pLoop = pLevel->pWLoop;
drhfc8d4f92013-11-08 15:19:46 +00004903
drh5f612292014-02-08 23:20:32 +00004904 /* For a co-routine, change all OP_Column references to the table of
drh7b3aa082015-05-29 13:55:33 +00004905 ** the co-routine into OP_Copy of result contained in a register.
drh5f612292014-02-08 23:20:32 +00004906 ** OP_Rowid becomes OP_Null.
4907 */
drh8a48b9c2015-08-19 15:20:00 +00004908 if( pTabItem->fg.viaCoroutine && !db->mallocFailed ){
drh7b3aa082015-05-29 13:55:33 +00004909 translateColumnToCopy(v, pLevel->addrBody, pLevel->iTabCur,
danfb785b22015-10-24 20:31:22 +00004910 pTabItem->regResult, 0);
drh5f612292014-02-08 23:20:32 +00004911 continue;
4912 }
4913
drhf0030762013-06-14 13:27:01 +00004914 /* If this scan uses an index, make VDBE code substitutions to read data
4915 ** from the index instead of from the table where possible. In some cases
4916 ** this optimization prevents the table from ever being read, which can
4917 ** yield a significant performance boost.
drh9012bcb2004-12-19 00:11:35 +00004918 **
4919 ** Calls to the code generator in between sqlite3WhereBegin and
4920 ** sqlite3WhereEnd will have created code that references the table
4921 ** directly. This loop scans all that code looking for opcodes
4922 ** that reference the table and converts them into opcodes that
4923 ** reference the index.
4924 */
drh7ba39a92013-05-30 17:43:19 +00004925 if( pLoop->wsFlags & (WHERE_INDEXED|WHERE_IDX_ONLY) ){
4926 pIdx = pLoop->u.btree.pIndex;
4927 }else if( pLoop->wsFlags & WHERE_MULTI_OR ){
drhd40e2082012-08-24 23:24:15 +00004928 pIdx = pLevel->u.pCovidx;
danbfca6a42012-08-24 10:52:35 +00004929 }
drh63c85a72015-09-28 14:40:20 +00004930 if( pIdx
4931 && (pWInfo->eOnePass==ONEPASS_OFF || !HasRowid(pIdx->pTable))
4932 && !db->mallocFailed
4933 ){
drh9012bcb2004-12-19 00:11:35 +00004934 last = sqlite3VdbeCurrentAddr(v);
drhcc04afd2013-08-22 02:56:28 +00004935 k = pLevel->addrBody;
4936 pOp = sqlite3VdbeGetOp(v, k);
4937 for(; k<last; k++, pOp++){
drh9012bcb2004-12-19 00:11:35 +00004938 if( pOp->p1!=pLevel->iTabCur ) continue;
4939 if( pOp->opcode==OP_Column ){
drhee0ec8e2013-10-31 17:38:01 +00004940 int x = pOp->p2;
drh511717c2013-11-08 17:13:23 +00004941 assert( pIdx->pTable==pTab );
drhee0ec8e2013-10-31 17:38:01 +00004942 if( !HasRowid(pTab) ){
4943 Index *pPk = sqlite3PrimaryKeyIndex(pTab);
4944 x = pPk->aiColumn[x];
drh4b92f982015-09-29 17:20:14 +00004945 assert( x>=0 );
drhee0ec8e2013-10-31 17:38:01 +00004946 }
4947 x = sqlite3ColumnOfIndex(pIdx, x);
drh44156282013-10-23 22:23:03 +00004948 if( x>=0 ){
4949 pOp->p2 = x;
4950 pOp->p1 = pLevel->iIdxCur;
drh9012bcb2004-12-19 00:11:35 +00004951 }
drh44156282013-10-23 22:23:03 +00004952 assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 || x>=0 );
drhf0863fe2005-06-12 21:35:51 +00004953 }else if( pOp->opcode==OP_Rowid ){
drh9012bcb2004-12-19 00:11:35 +00004954 pOp->p1 = pLevel->iIdxCur;
drhf0863fe2005-06-12 21:35:51 +00004955 pOp->opcode = OP_IdxRowid;
drh9012bcb2004-12-19 00:11:35 +00004956 }
4957 }
drh6b563442001-11-07 16:48:26 +00004958 }
drh19a775c2000-06-05 18:54:46 +00004959 }
drh9012bcb2004-12-19 00:11:35 +00004960
4961 /* Final cleanup
4962 */
drhf12cde52010-04-08 17:28:00 +00004963 pParse->nQueryLoop = pWInfo->savedNQueryLoop;
4964 whereInfoFree(db, pWInfo);
drh75897232000-05-29 14:26:00 +00004965 return;
4966}