blob: 950d0db62b72c32748617387d7757740ac4fbc18 [file] [log] [blame]
drhcce7d172000-05-31 15:34:51 +00001/*
drhb19a2bc2001-09-16 00:13:26 +00002** 2001 September 15
drhcce7d172000-05-31 15:34:51 +00003**
drhb19a2bc2001-09-16 00:13:26 +00004** The author disclaims copyright to this source code. In place of
5** a legal notice, here is a blessing:
drhcce7d172000-05-31 15:34:51 +00006**
drhb19a2bc2001-09-16 00:13:26 +00007** May you do good and not evil.
8** May you find forgiveness for yourself and forgive others.
9** May you share freely, never taking more than you give.
drhcce7d172000-05-31 15:34:51 +000010**
11*************************************************************************
12** This file contains C code routines that are called by the parser
drhb19a2bc2001-09-16 00:13:26 +000013** to handle SELECT statements in SQLite.
drhcce7d172000-05-31 15:34:51 +000014*/
15#include "sqliteInt.h"
16
drh079a3072014-03-19 14:10:55 +000017/*
18** An instance of the following object is used to record information about
19** how to process the DISTINCT keyword, to simplify passing that information
20** into the selectInnerLoop() routine.
drheda639e2006-01-22 00:42:09 +000021*/
drh079a3072014-03-19 14:10:55 +000022typedef struct DistinctCtx DistinctCtx;
23struct DistinctCtx {
drhd6df8552022-03-17 18:03:08 +000024 u8 isTnct; /* 0: Not distinct. 1: DISTICT 2: DISTINCT and ORDER BY */
drh079a3072014-03-19 14:10:55 +000025 u8 eTnctType; /* One of the WHERE_DISTINCT_* operators */
26 int tabTnct; /* Ephemeral table used for DISTINCT processing */
27 int addrTnct; /* Address of OP_OpenEphemeral opcode for tabTnct */
28};
29
30/*
31** An instance of the following object is used to record information about
32** the ORDER BY (or GROUP BY) clause of query is being coded.
dan24e25d32018-04-14 18:46:20 +000033**
34** The aDefer[] array is used by the sorter-references optimization. For
35** example, assuming there is no index that can be used for the ORDER BY,
36** for the query:
37**
38** SELECT a, bigblob FROM t1 ORDER BY a LIMIT 10;
39**
40** it may be more efficient to add just the "a" values to the sorter, and
41** retrieve the associated "bigblob" values directly from table t1 as the
42** 10 smallest "a" values are extracted from the sorter.
43**
44** When the sorter-reference optimization is used, there is one entry in the
45** aDefer[] array for each database table that may be read as values are
46** extracted from the sorter.
drh079a3072014-03-19 14:10:55 +000047*/
48typedef struct SortCtx SortCtx;
49struct SortCtx {
50 ExprList *pOrderBy; /* The ORDER BY (or GROUP BY clause) */
51 int nOBSat; /* Number of ORDER BY terms satisfied by indices */
52 int iECursor; /* Cursor number for the sorter */
53 int regReturn; /* Register holding block-output return address */
54 int labelBkOut; /* Start label for the block-output subroutine */
55 int addrSortIndex; /* Address of the OP_SorterOpen or OP_OpenEphemeral */
drha04a8be2016-01-13 17:50:10 +000056 int labelDone; /* Jump here when done, ex: LIMIT reached */
drh6ee5a7b2018-09-08 20:09:46 +000057 int labelOBLopt; /* Jump here when sorter is full */
drh079a3072014-03-19 14:10:55 +000058 u8 sortFlags; /* Zero or more SORTFLAG_* bits */
dan24e25d32018-04-14 18:46:20 +000059#ifdef SQLITE_ENABLE_SORTER_REFERENCES
60 u8 nDefer; /* Number of valid entries in aDefer[] */
61 struct DeferredCsr {
62 Table *pTab; /* Table definition */
63 int iCsr; /* Cursor number for table */
64 int nKey; /* Number of PK columns for table pTab (>=1) */
65 } aDefer[4];
66#endif
drhbbd4ae52018-04-30 19:32:49 +000067 struct RowLoadInfo *pDeferredRowLoad; /* Deferred row loading info or NULL */
drh079a3072014-03-19 14:10:55 +000068};
69#define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */
drh315555c2002-10-20 15:53:03 +000070
drhcce7d172000-05-31 15:34:51 +000071/*
drhb87fbed2015-01-05 15:48:45 +000072** Delete all the content of a Select structure. Deallocate the structure
drha9ebfe22019-12-25 23:54:21 +000073** itself depending on the value of bFree
74**
75** If bFree==1, call sqlite3DbFree() on the p object.
76** If bFree==0, Leave the first Select object unfreed
drheda639e2006-01-22 00:42:09 +000077*/
drhb87fbed2015-01-05 15:48:45 +000078static void clearSelect(sqlite3 *db, Select *p, int bFree){
drh41ce47c2022-08-22 02:00:26 +000079 assert( db!=0 );
drhb87fbed2015-01-05 15:48:45 +000080 while( p ){
81 Select *pPrior = p->pPrior;
82 sqlite3ExprListDelete(db, p->pEList);
83 sqlite3SrcListDelete(db, p->pSrc);
84 sqlite3ExprDelete(db, p->pWhere);
85 sqlite3ExprListDelete(db, p->pGroupBy);
86 sqlite3ExprDelete(db, p->pHaving);
87 sqlite3ExprListDelete(db, p->pOrderBy);
88 sqlite3ExprDelete(db, p->pLimit);
dan50f9f6c2021-03-04 14:18:22 +000089 if( OK_IF_ALWAYS_TRUE(p->pWith) ) sqlite3WithDelete(db, p->pWith);
dan67a9b8e2018-06-22 20:51:35 +000090#ifndef SQLITE_OMIT_WINDOWFUNC
dane3bf6322018-06-08 20:58:27 +000091 if( OK_IF_ALWAYS_TRUE(p->pWinDefn) ){
92 sqlite3WindowListDelete(db, p->pWinDefn);
93 }
dan50f9f6c2021-03-04 14:18:22 +000094 while( p->pWin ){
95 assert( p->pWin->ppThis==&p->pWin );
96 sqlite3WindowUnlinkFromSelect(p->pWin);
97 }
dan67a9b8e2018-06-22 20:51:35 +000098#endif
drh41ce47c2022-08-22 02:00:26 +000099 if( bFree ) sqlite3DbNNFreeNN(db, p);
drhb87fbed2015-01-05 15:48:45 +0000100 p = pPrior;
101 bFree = 1;
102 }
drheda639e2006-01-22 00:42:09 +0000103}
104
drh1013c932008-01-06 00:25:21 +0000105/*
106** Initialize a SelectDest structure.
107*/
108void sqlite3SelectDestInit(SelectDest *pDest, int eDest, int iParm){
drhea678832008-12-10 19:26:22 +0000109 pDest->eDest = (u8)eDest;
drh2b596da2012-07-23 21:43:19 +0000110 pDest->iSDParm = iParm;
dan9ed322d2020-04-29 17:41:29 +0000111 pDest->iSDParm2 = 0;
dan71c57db2016-07-09 20:23:55 +0000112 pDest->zAffSdst = 0;
drh2b596da2012-07-23 21:43:19 +0000113 pDest->iSdst = 0;
114 pDest->nSdst = 0;
drh1013c932008-01-06 00:25:21 +0000115}
116
drheda639e2006-01-22 00:42:09 +0000117
118/*
drh9bb61fe2000-06-05 16:01:39 +0000119** Allocate a new Select structure and return a pointer to that
120** structure.
drhcce7d172000-05-31 15:34:51 +0000121*/
danielk19774adee202004-05-08 08:23:19 +0000122Select *sqlite3SelectNew(
drh17435752007-08-16 04:30:38 +0000123 Parse *pParse, /* Parsing context */
drhdaffd0e2001-04-11 14:28:42 +0000124 ExprList *pEList, /* which columns to include in the result */
drhad3cab52002-05-24 02:04:32 +0000125 SrcList *pSrc, /* the FROM clause -- which tables to scan */
drhdaffd0e2001-04-11 14:28:42 +0000126 Expr *pWhere, /* the WHERE clause */
127 ExprList *pGroupBy, /* the GROUP BY clause */
128 Expr *pHaving, /* the HAVING clause */
129 ExprList *pOrderBy, /* the ORDER BY clause */
drhc3489bb2016-02-25 16:04:59 +0000130 u32 selFlags, /* Flag parameters, such as SF_Distinct */
drh8c0833f2017-11-14 23:48:23 +0000131 Expr *pLimit /* LIMIT value. NULL means not used */
drh9bb61fe2000-06-05 16:01:39 +0000132){
drhd3bf7662020-05-25 01:31:09 +0000133 Select *pNew, *pAllocated;
drheda639e2006-01-22 00:42:09 +0000134 Select standin;
drhd3bf7662020-05-25 01:31:09 +0000135 pAllocated = pNew = sqlite3DbMallocRawNN(pParse->db, sizeof(*pNew) );
drhdaffd0e2001-04-11 14:28:42 +0000136 if( pNew==0 ){
drhef90a6b2017-04-11 16:44:39 +0000137 assert( pParse->db->mallocFailed );
drheda639e2006-01-22 00:42:09 +0000138 pNew = &standin;
drheda639e2006-01-22 00:42:09 +0000139 }
140 if( pEList==0 ){
drh3d240d22017-09-28 16:56:55 +0000141 pEList = sqlite3ExprListAppend(pParse, 0,
142 sqlite3Expr(pParse->db,TK_ASTERISK,0));
drheda639e2006-01-22 00:42:09 +0000143 }
144 pNew->pEList = pEList;
drhca3862d2016-01-08 12:46:39 +0000145 pNew->op = TK_SELECT;
146 pNew->selFlags = selFlags;
147 pNew->iLimit = 0;
148 pNew->iOffset = 0;
drhfef37762018-07-10 19:48:35 +0000149 pNew->selId = ++pParse->nSelect;
drhca3862d2016-01-08 12:46:39 +0000150 pNew->addrOpenEphm[0] = -1;
151 pNew->addrOpenEphm[1] = -1;
152 pNew->nSelectRow = 0;
drhef90a6b2017-04-11 16:44:39 +0000153 if( pSrc==0 ) pSrc = sqlite3DbMallocZero(pParse->db, sizeof(*pSrc));
drheda639e2006-01-22 00:42:09 +0000154 pNew->pSrc = pSrc;
155 pNew->pWhere = pWhere;
156 pNew->pGroupBy = pGroupBy;
157 pNew->pHaving = pHaving;
158 pNew->pOrderBy = pOrderBy;
drhca3862d2016-01-08 12:46:39 +0000159 pNew->pPrior = 0;
160 pNew->pNext = 0;
drheda639e2006-01-22 00:42:09 +0000161 pNew->pLimit = pLimit;
drhca3862d2016-01-08 12:46:39 +0000162 pNew->pWith = 0;
dan67a9b8e2018-06-22 20:51:35 +0000163#ifndef SQLITE_OMIT_WINDOWFUNC
dan86fb6e12018-05-16 20:58:07 +0000164 pNew->pWin = 0;
dane3bf6322018-06-08 20:58:27 +0000165 pNew->pWinDefn = 0;
dan67a9b8e2018-06-22 20:51:35 +0000166#endif
drhef90a6b2017-04-11 16:44:39 +0000167 if( pParse->db->mallocFailed ) {
168 clearSelect(pParse->db, pNew, pNew!=&standin);
drhd3bf7662020-05-25 01:31:09 +0000169 pAllocated = 0;
drha464c232011-09-16 19:04:03 +0000170 }else{
171 assert( pNew->pSrc!=0 || pParse->nErr>0 );
drhdaffd0e2001-04-11 14:28:42 +0000172 }
drhd3bf7662020-05-25 01:31:09 +0000173 return pAllocated;
drh9bb61fe2000-06-05 16:01:39 +0000174}
175
drheb9b8842014-09-21 00:27:26 +0000176
drh9bb61fe2000-06-05 16:01:39 +0000177/*
drheda639e2006-01-22 00:42:09 +0000178** Delete the given Select structure and all of its substructures.
179*/
drh633e6d52008-07-28 19:34:53 +0000180void sqlite3SelectDelete(sqlite3 *db, Select *p){
drh8906a4b2017-10-03 17:29:40 +0000181 if( OK_IF_ALWAYS_TRUE(p) ) clearSelect(db, p, 1);
drheda639e2006-01-22 00:42:09 +0000182}
183
184/*
drhd227a292014-02-09 18:02:09 +0000185** Return a pointer to the right-most SELECT statement in a compound.
186*/
187static Select *findRightmost(Select *p){
188 while( p->pNext ) p = p->pNext;
189 return p;
drh9bb61fe2000-06-05 16:01:39 +0000190}
191
192/*
drhf7b54962013-05-28 12:11:54 +0000193** Given 1 to 3 identifiers preceding the JOIN keyword, determine the
drh01f3f252002-05-24 16:14:15 +0000194** type of join. Return an integer constant that expresses that type
195** in terms of the following bit values:
196**
197** JT_INNER
drh3dec2232005-09-10 15:28:09 +0000198** JT_CROSS
drh01f3f252002-05-24 16:14:15 +0000199** JT_OUTER
200** JT_NATURAL
201** JT_LEFT
202** JT_RIGHT
203**
204** A full outer join is the combination of JT_LEFT and JT_RIGHT.
205**
206** If an illegal or unsupported join type is seen, then still return
207** a join type, but put an error in the pParse structure.
drh81a23622022-04-08 15:11:10 +0000208**
209** These are the valid join types:
210**
211**
212** pA pB pC Return Value
213** ------- ----- ----- ------------
214** CROSS - - JT_CROSS
215** INNER - - JT_INNER
216** LEFT - - JT_LEFT|JT_OUTER
217** LEFT OUTER - JT_LEFT|JT_OUTER
218** RIGHT - - JT_RIGHT|JT_OUTER
219** RIGHT OUTER - JT_RIGHT|JT_OUTER
220** FULL - - JT_LEFT|JT_RIGHT|JT_OUTER
221** FULL OUTER - JT_LEFT|JT_RIGHT|JT_OUTER
222** NATURAL INNER - JT_NATURAL|JT_INNER
223** NATURAL LEFT - JT_NATURAL|JT_LEFT|JT_OUTER
224** NATURAL LEFT OUTER JT_NATURAL|JT_LEFT|JT_OUTER
225** NATURAL RIGHT - JT_NATURAL|JT_RIGHT|JT_OUTER
226** NATURAL RIGHT OUTER JT_NATURAL|JT_RIGHT|JT_OUTER
227** NATURAL FULL - JT_NATURAL|JT_LEFT|JT_RIGHT
228** NATURAL FULL OUTER JT_NATRUAL|JT_LEFT|JT_RIGHT
229**
230** To preserve historical compatibly, SQLite also accepts a variety
231** of other non-standard and in many cases non-sensical join types.
232** This routine makes as much sense at it can from the nonsense join
233** type and returns a result. Examples of accepted nonsense join types
234** include but are not limited to:
235**
236** INNER CROSS JOIN -> same as JOIN
237** NATURAL CROSS JOIN -> same as NATURAL JOIN
238** OUTER LEFT JOIN -> same as LEFT JOIN
239** LEFT NATURAL JOIN -> same as NATURAL LEFT JOIN
240** LEFT RIGHT JOIN -> same as FULL JOIN
241** RIGHT OUTER FULL JOIN -> same as FULL JOIN
242** CROSS CROSS CROSS JOIN -> same as JOIN
243**
244** The only restrictions on the join type name are:
245**
246** * "INNER" cannot appear together with "OUTER", "LEFT", "RIGHT",
247** or "FULL".
248**
249** * "CROSS" cannot appear together with "OUTER", "LEFT", "RIGHT,
250** or "FULL".
251**
252** * If "OUTER" is present then there must also be one of
253** "LEFT", "RIGHT", or "FULL"
drh01f3f252002-05-24 16:14:15 +0000254*/
danielk19774adee202004-05-08 08:23:19 +0000255int sqlite3JoinType(Parse *pParse, Token *pA, Token *pB, Token *pC){
drh01f3f252002-05-24 16:14:15 +0000256 int jointype = 0;
257 Token *apAll[3];
258 Token *p;
drh373cc2d2009-05-17 02:06:14 +0000259 /* 0123456789 123456789 123456789 123 */
260 static const char zKeyText[] = "naturaleftouterightfullinnercross";
drh57196282004-10-06 15:41:16 +0000261 static const struct {
drh373cc2d2009-05-17 02:06:14 +0000262 u8 i; /* Beginning of keyword text in zKeyText[] */
263 u8 nChar; /* Length of the keyword in characters */
264 u8 code; /* Join type mask */
265 } aKeyword[] = {
drh81a23622022-04-08 15:11:10 +0000266 /* (0) natural */ { 0, 7, JT_NATURAL },
267 /* (1) left */ { 6, 4, JT_LEFT|JT_OUTER },
268 /* (2) outer */ { 10, 5, JT_OUTER },
269 /* (3) right */ { 14, 5, JT_RIGHT|JT_OUTER },
270 /* (4) full */ { 19, 4, JT_LEFT|JT_RIGHT|JT_OUTER },
271 /* (5) inner */ { 23, 5, JT_INNER },
272 /* (6) cross */ { 28, 5, JT_INNER|JT_CROSS },
drh01f3f252002-05-24 16:14:15 +0000273 };
274 int i, j;
275 apAll[0] = pA;
276 apAll[1] = pB;
277 apAll[2] = pC;
drh195e6962002-05-25 00:18:20 +0000278 for(i=0; i<3 && apAll[i]; i++){
drh01f3f252002-05-24 16:14:15 +0000279 p = apAll[i];
drh373cc2d2009-05-17 02:06:14 +0000280 for(j=0; j<ArraySize(aKeyword); j++){
281 if( p->n==aKeyword[j].nChar
282 && sqlite3StrNICmp((char*)p->z, &zKeyText[aKeyword[j].i], p->n)==0 ){
283 jointype |= aKeyword[j].code;
drh01f3f252002-05-24 16:14:15 +0000284 break;
285 }
286 }
drh373cc2d2009-05-17 02:06:14 +0000287 testcase( j==0 || j==1 || j==2 || j==3 || j==4 || j==5 || j==6 );
288 if( j>=ArraySize(aKeyword) ){
drh01f3f252002-05-24 16:14:15 +0000289 jointype |= JT_ERROR;
290 break;
291 }
292 }
drhad2d8302002-05-24 20:31:36 +0000293 if(
294 (jointype & (JT_INNER|JT_OUTER))==(JT_INNER|JT_OUTER) ||
drh81a23622022-04-08 15:11:10 +0000295 (jointype & JT_ERROR)!=0 ||
296 (jointype & (JT_OUTER|JT_LEFT|JT_RIGHT))==JT_OUTER
drhad2d8302002-05-24 20:31:36 +0000297 ){
drh81a23622022-04-08 15:11:10 +0000298 const char *zSp1 = " ";
299 const char *zSp2 = " ";
300 if( pB==0 ){ zSp1++; }
301 if( pC==0 ){ zSp2++; }
drh0879d5f2022-04-12 18:40:14 +0000302 sqlite3ErrorMsg(pParse, "unknown join type: "
drh81a23622022-04-08 15:11:10 +0000303 "%T%s%T%s%T", pA, zSp1, pB, zSp2, pC);
drh01f3f252002-05-24 16:14:15 +0000304 jointype = JT_INNER;
305 }
306 return jointype;
307}
308
309/*
drhad2d8302002-05-24 20:31:36 +0000310** Return the index of a column in a table. Return -1 if the column
311** is not contained in the table.
312*/
dan6e6d9832021-02-16 20:43:36 +0000313int sqlite3ColumnIndex(Table *pTab, const char *zCol){
drhad2d8302002-05-24 20:31:36 +0000314 int i;
drha1928072020-07-20 13:11:19 +0000315 u8 h = sqlite3StrIHash(zCol);
316 Column *pCol;
317 for(pCol=pTab->aCol, i=0; i<pTab->nCol; pCol++, i++){
drhcf9d36d2021-08-02 18:03:43 +0000318 if( pCol->hName==h && sqlite3StrICmp(pCol->zCnName, zCol)==0 ) return i;
drhad2d8302002-05-24 20:31:36 +0000319 }
320 return -1;
321}
322
323/*
drh815b7822022-04-20 15:07:39 +0000324** Mark a subquery result column as having been used.
325*/
326void sqlite3SrcItemColumnUsed(SrcItem *pItem, int iCol){
327 assert( pItem!=0 );
mistachkin07fae322022-07-06 23:50:01 +0000328 assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) );
drh815b7822022-04-20 15:07:39 +0000329 if( pItem->fg.isNestedFrom ){
330 ExprList *pResults;
331 assert( pItem->pSelect!=0 );
332 pResults = pItem->pSelect->pEList;
333 assert( pResults!=0 );
334 assert( iCol>=0 && iCol<pResults->nExpr );
drhd88fd532022-05-02 20:49:30 +0000335 pResults->a[iCol].fg.bUsed = 1;
drh815b7822022-04-20 15:07:39 +0000336 }
337}
338
339/*
drhfe146992022-04-17 18:46:17 +0000340** Search the tables iStart..iEnd (inclusive) in pSrc, looking for a
341** table that has a column named zCol. The search is left-to-right.
342** The first match found is returned.
drh2179b432009-12-09 17:36:39 +0000343**
344** When found, set *piTab and *piCol to the table index and column index
345** of the matching column and return TRUE.
346**
347** If not found, return FALSE.
348*/
349static int tableAndColumnIndex(
350 SrcList *pSrc, /* Array of tables to search */
drhfe146992022-04-17 18:46:17 +0000351 int iStart, /* First member of pSrc->a[] to check */
352 int iEnd, /* Last member of pSrc->a[] to check */
drh2179b432009-12-09 17:36:39 +0000353 const char *zCol, /* Name of the column we are looking for */
354 int *piTab, /* Write index of pSrc->a[] here */
dan9d41af22019-12-30 14:32:27 +0000355 int *piCol, /* Write index of pSrc->a[*piTab].pTab->aCol[] here */
drhfe146992022-04-17 18:46:17 +0000356 int bIgnoreHidden /* Ignore hidden columns */
drh2179b432009-12-09 17:36:39 +0000357){
358 int i; /* For looping over tables in pSrc */
359 int iCol; /* Index of column matching zCol */
360
drhfe146992022-04-17 18:46:17 +0000361 assert( iEnd<pSrc->nSrc );
362 assert( iStart>=0 );
drh2179b432009-12-09 17:36:39 +0000363 assert( (piTab==0)==(piCol==0) ); /* Both or neither are NULL */
drhfe146992022-04-17 18:46:17 +0000364
365 for(i=iStart; i<=iEnd; i++){
dan6e6d9832021-02-16 20:43:36 +0000366 iCol = sqlite3ColumnIndex(pSrc->a[i].pTab, zCol);
dan9d41af22019-12-30 14:32:27 +0000367 if( iCol>=0
368 && (bIgnoreHidden==0 || IsHiddenColumn(&pSrc->a[i].pTab->aCol[iCol])==0)
369 ){
drh2179b432009-12-09 17:36:39 +0000370 if( piTab ){
drh815b7822022-04-20 15:07:39 +0000371 sqlite3SrcItemColumnUsed(&pSrc->a[i], iCol);
drh2179b432009-12-09 17:36:39 +0000372 *piTab = i;
373 *piCol = iCol;
374 }
drhfe146992022-04-17 18:46:17 +0000375 return 1;
drh2179b432009-12-09 17:36:39 +0000376 }
377 }
drhfe146992022-04-17 18:46:17 +0000378 return 0;
drhad2d8302002-05-24 20:31:36 +0000379}
380
381/*
drh67a99db2022-05-13 14:52:04 +0000382** Set the EP_OuterON property on all terms of the given expression.
drhd1985262022-04-11 11:25:28 +0000383** And set the Expr.w.iJoin to iTable for every term in the
drh22d6a532005-09-19 21:05:48 +0000384** expression.
drh1cc093c2002-06-24 22:01:57 +0000385**
drh67a99db2022-05-13 14:52:04 +0000386** The EP_OuterON property is used on terms of an expression to tell
drhb77c07a2022-04-11 11:59:25 +0000387** the OUTER JOIN processing logic that this term is part of the
drh1f162302002-10-27 19:35:33 +0000388** join restriction specified in the ON or USING clause and not a part
389** of the more general WHERE clause. These terms are moved over to the
390** WHERE clause during join processing but we need to remember that they
391** originated in the ON or USING clause.
drh22d6a532005-09-19 21:05:48 +0000392**
drhd1985262022-04-11 11:25:28 +0000393** The Expr.w.iJoin tells the WHERE clause processing that the
394** expression depends on table w.iJoin even if that table is not
drh22d6a532005-09-19 21:05:48 +0000395** explicitly mentioned in the expression. That information is needed
396** for cases like this:
397**
398** SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.b AND t1.x=5
399**
400** The where clause needs to defer the handling of the t1.x=5
401** term until after the t2 loop of the join. In that way, a
402** NULL t2 row will be inserted whenever t1.x!=5. If we do not
403** defer the handling of t1.x=5, it will be processed immediately
404** after the t1 loop and rows with t1.x!=5 will never appear in
405** the output, which is incorrect.
drh1cc093c2002-06-24 22:01:57 +0000406*/
drh3a6e4c52022-04-11 12:38:06 +0000407void sqlite3SetJoinExpr(Expr *p, int iTable, u32 joinFlag){
drh67a99db2022-05-13 14:52:04 +0000408 assert( joinFlag==EP_OuterON || joinFlag==EP_InnerON );
drh1cc093c2002-06-24 22:01:57 +0000409 while( p ){
drh3a6e4c52022-04-11 12:38:06 +0000410 ExprSetProperty(p, joinFlag);
drhc5cd1242013-09-12 16:50:49 +0000411 assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) );
drhebb6a652013-09-12 23:42:22 +0000412 ExprSetVVAProperty(p, EP_NoReduce);
drhd1985262022-04-11 11:25:28 +0000413 p->w.iJoin = iTable;
drha4eeccd2021-10-07 17:43:30 +0000414 if( p->op==TK_FUNCTION ){
415 assert( ExprUseXList(p) );
416 if( p->x.pList ){
417 int i;
418 for(i=0; i<p->x.pList->nExpr; i++){
drh3a6e4c52022-04-11 12:38:06 +0000419 sqlite3SetJoinExpr(p->x.pList->a[i].pExpr, iTable, joinFlag);
drha4eeccd2021-10-07 17:43:30 +0000420 }
drh606f2342015-06-18 14:32:51 +0000421 }
422 }
drh3a6e4c52022-04-11 12:38:06 +0000423 sqlite3SetJoinExpr(p->pLeft, iTable, joinFlag);
drh1cc093c2002-06-24 22:01:57 +0000424 p = p->pRight;
425 }
426}
427
drhd7480402022-06-20 17:04:44 +0000428/* Undo the work of sqlite3SetJoinExpr(). This is used when a LEFT JOIN
429** is simplified into an ordinary JOIN, and when an ON expression is
430** "pushed down" into the WHERE clause of a subquery.
drh25897872018-03-20 21:16:15 +0000431**
drhd7480402022-06-20 17:04:44 +0000432** Convert every term that is marked with EP_OuterON and w.iJoin==iTable into
433** an ordinary term that omits the EP_OuterON mark. Or if iTable<0, then
434** just clear every EP_OuterON and EP_InnerON mark from the expression tree.
drh92d1afb2022-06-13 12:42:24 +0000435**
436** If nullable is true, that means that Expr p might evaluate to NULL even
437** if it is a reference to a NOT NULL column. This can happen, for example,
438** if the table that p references is on the left side of a RIGHT JOIN.
439** If nullable is true, then take care to not remove the EP_CanBeNull bit.
440** See forum thread https://sqlite.org/forum/forumpost/b40696f50145d21c
drh25897872018-03-20 21:16:15 +0000441*/
drh92d1afb2022-06-13 12:42:24 +0000442static void unsetJoinExpr(Expr *p, int iTable, int nullable){
drh25897872018-03-20 21:16:15 +0000443 while( p ){
drhd7480402022-06-20 17:04:44 +0000444 if( iTable<0 || (ExprHasProperty(p, EP_OuterON) && p->w.iJoin==iTable) ){
445 ExprClearProperty(p, EP_OuterON|EP_InnerON);
446 if( iTable>=0 ) ExprSetProperty(p, EP_InnerON);
drh25897872018-03-20 21:16:15 +0000447 }
drh92d1afb2022-06-13 12:42:24 +0000448 if( p->op==TK_COLUMN && p->iTable==iTable && !nullable ){
dan8ddf6862021-02-26 20:14:32 +0000449 ExprClearProperty(p, EP_CanBeNull);
450 }
drha4eeccd2021-10-07 17:43:30 +0000451 if( p->op==TK_FUNCTION ){
452 assert( ExprUseXList(p) );
453 if( p->x.pList ){
454 int i;
455 for(i=0; i<p->x.pList->nExpr; i++){
drh92d1afb2022-06-13 12:42:24 +0000456 unsetJoinExpr(p->x.pList->a[i].pExpr, iTable, nullable);
drha4eeccd2021-10-07 17:43:30 +0000457 }
drh25897872018-03-20 21:16:15 +0000458 }
459 }
drh92d1afb2022-06-13 12:42:24 +0000460 unsetJoinExpr(p->pLeft, iTable, nullable);
drh25897872018-03-20 21:16:15 +0000461 p = p->pRight;
462 }
463}
464
drh1cc093c2002-06-24 22:01:57 +0000465/*
drhad2d8302002-05-24 20:31:36 +0000466** This routine processes the join information for a SELECT statement.
drhbc656e22022-04-20 16:26:22 +0000467**
468** * A NATURAL join is converted into a USING join. After that, we
469** do not need to be concerned with NATURAL joins and we only have
470** think about USING joins.
471**
472** * ON and USING clauses result in extra terms being added to the
473** WHERE clause to enforce the specified constraints. The extra
drh67a99db2022-05-13 14:52:04 +0000474** WHERE clause terms will be tagged with EP_OuterON or
475** EP_InnerON so that we know that they originated in ON/USING.
drhad2d8302002-05-24 20:31:36 +0000476**
drh91bb0ee2004-09-01 03:06:34 +0000477** The terms of a FROM clause are contained in the Select.pSrc structure.
478** The left most table is the first entry in Select.pSrc. The right-most
479** table is the last entry. The join operator is held in the entry to
drhbc656e22022-04-20 16:26:22 +0000480** the right. Thus entry 1 contains the join operator for the join between
drh91bb0ee2004-09-01 03:06:34 +0000481** entries 0 and 1. Any ON or USING clauses associated with the join are
drhbc656e22022-04-20 16:26:22 +0000482** also attached to the right entry.
drh91bb0ee2004-09-01 03:06:34 +0000483**
drhad2d8302002-05-24 20:31:36 +0000484** This routine returns the number of errors encountered.
485*/
drh358424a2022-04-15 15:15:01 +0000486static int sqlite3ProcessJoin(Parse *pParse, Select *p){
drh91bb0ee2004-09-01 03:06:34 +0000487 SrcList *pSrc; /* All tables in the FROM clause */
488 int i, j; /* Loop counters */
drh76012942021-02-21 21:04:54 +0000489 SrcItem *pLeft; /* Left table being joined */
490 SrcItem *pRight; /* Right table being joined */
drhad2d8302002-05-24 20:31:36 +0000491
drh91bb0ee2004-09-01 03:06:34 +0000492 pSrc = p->pSrc;
493 pLeft = &pSrc->a[0];
494 pRight = &pLeft[1];
495 for(i=0; i<pSrc->nSrc-1; i++, pRight++, pLeft++){
drh91bb0ee2004-09-01 03:06:34 +0000496 Table *pRightTab = pRight->pTab;
drh3a6e4c52022-04-11 12:38:06 +0000497 u32 joinType;
drh91bb0ee2004-09-01 03:06:34 +0000498
drhce2c4822017-10-03 17:17:34 +0000499 if( NEVER(pLeft->pTab==0 || pRightTab==0) ) continue;
drh67a99db2022-05-13 14:52:04 +0000500 joinType = (pRight->fg.jointype & JT_OUTER)!=0 ? EP_OuterON : EP_InnerON;
drhad2d8302002-05-24 20:31:36 +0000501
drh22c4bc82022-04-15 17:08:40 +0000502 /* If this is a NATURAL join, synthesize an approprate USING clause
503 ** to specify which columns should be joined.
drhad2d8302002-05-24 20:31:36 +0000504 */
drh8a48b9c2015-08-19 15:20:00 +0000505 if( pRight->fg.jointype & JT_NATURAL ){
drh22c4bc82022-04-15 17:08:40 +0000506 IdList *pUsing = 0;
drhd44f8b22022-04-07 01:11:13 +0000507 if( pRight->fg.isUsing || pRight->u3.pOn ){
danielk19774adee202004-05-08 08:23:19 +0000508 sqlite3ErrorMsg(pParse, "a NATURAL join may not have "
drhad2d8302002-05-24 20:31:36 +0000509 "an ON or USING clause", 0);
drhad2d8302002-05-24 20:31:36 +0000510 return 1;
511 }
drh2179b432009-12-09 17:36:39 +0000512 for(j=0; j<pRightTab->nCol; j++){
513 char *zName; /* Name of column in the right table */
drh2179b432009-12-09 17:36:39 +0000514
dan9d41af22019-12-30 14:32:27 +0000515 if( IsHiddenColumn(&pRightTab->aCol[j]) ) continue;
drhcf9d36d2021-08-02 18:03:43 +0000516 zName = pRightTab->aCol[j].zCnName;
drhfe146992022-04-17 18:46:17 +0000517 if( tableAndColumnIndex(pSrc, 0, i, zName, 0, 0, 1) ){
drhd0453f72022-04-18 00:04:15 +0000518 pUsing = sqlite3IdListAppend(pParse, pUsing, 0);
519 if( pUsing ){
520 assert( pUsing->nId>0 );
521 assert( pUsing->a[pUsing->nId-1].zName==0 );
522 pUsing->a[pUsing->nId-1].zName = sqlite3DbStrDup(pParse->db, zName);
523 }
drhad2d8302002-05-24 20:31:36 +0000524 }
525 }
drh22c4bc82022-04-15 17:08:40 +0000526 if( pUsing ){
527 pRight->fg.isUsing = 1;
drh4ce7bf92022-04-15 18:30:48 +0000528 pRight->fg.isSynthUsing = 1;
drh22c4bc82022-04-15 17:08:40 +0000529 pRight->u3.pUsing = pUsing;
530 }
drhabf86bd2022-04-18 09:59:33 +0000531 if( pParse->nErr ) return 1;
drhad2d8302002-05-24 20:31:36 +0000532 }
533
drhad2d8302002-05-24 20:31:36 +0000534 /* Create extra terms on the WHERE clause for each column named
535 ** in the USING clause. Example: If the two tables to be joined are
536 ** A and B and the USING clause names X, Y, and Z, then add this
537 ** to the WHERE clause: A.X=B.X AND A.Y=B.Y AND A.Z=B.Z
538 ** Report an error if any column mentioned in the USING clause is
539 ** not contained in both tables to be joined.
540 */
drhd44f8b22022-04-07 01:11:13 +0000541 if( pRight->fg.isUsing ){
542 IdList *pList = pRight->u3.pUsing;
drhfe146992022-04-17 18:46:17 +0000543 sqlite3 *db = pParse->db;
drhd44f8b22022-04-07 01:11:13 +0000544 assert( pList!=0 );
drhad2d8302002-05-24 20:31:36 +0000545 for(j=0; j<pList->nId; j++){
drh2179b432009-12-09 17:36:39 +0000546 char *zName; /* Name of the term in the USING clause */
547 int iLeft; /* Table on the left with matching column name */
548 int iLeftCol; /* Column number of matching column on the left */
549 int iRightCol; /* Column number of matching column on the right */
drhfe146992022-04-17 18:46:17 +0000550 Expr *pE1; /* Reference to the column on the LEFT of the join */
551 Expr *pE2; /* Reference to the column on the RIGHT of the join */
552 Expr *pEq; /* Equality constraint. pE1 == pE2 */
drh2179b432009-12-09 17:36:39 +0000553
554 zName = pList->a[j].zName;
dan6e6d9832021-02-16 20:43:36 +0000555 iRightCol = sqlite3ColumnIndex(pRightTab, zName);
drh2179b432009-12-09 17:36:39 +0000556 if( iRightCol<0
drhfe146992022-04-17 18:46:17 +0000557 || tableAndColumnIndex(pSrc, 0, i, zName, &iLeft, &iLeftCol,
558 pRight->fg.isSynthUsing)==0
drh2179b432009-12-09 17:36:39 +0000559 ){
danielk19774adee202004-05-08 08:23:19 +0000560 sqlite3ErrorMsg(pParse, "cannot join using column %s - column "
drh91bb0ee2004-09-01 03:06:34 +0000561 "not present in both tables", zName);
drhad2d8302002-05-24 20:31:36 +0000562 return 1;
563 }
drhfe146992022-04-17 18:46:17 +0000564 pE1 = sqlite3CreateColumnExpr(db, pSrc, iLeft, iLeftCol);
drh22b410d2022-04-20 18:12:42 +0000565 sqlite3SrcItemColumnUsed(&pSrc->a[iLeft], iLeftCol);
drhfe146992022-04-17 18:46:17 +0000566 if( (pSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){
567 /* This branch runs if the query contains one or more RIGHT or FULL
568 ** JOINs. If only a single table on the left side of this join
drhbc656e22022-04-20 16:26:22 +0000569 ** contains the zName column, then this branch is a no-op.
570 ** But if there are two or more tables on the left side
drhfe146992022-04-17 18:46:17 +0000571 ** of the join, construct a coalesce() function that gathers all
572 ** such tables. Raise an error if more than one of those references
573 ** to zName is not also within a prior USING clause.
574 **
575 ** We really ought to raise an error if there are two or more
576 ** non-USING references to zName on the left of an INNER or LEFT
577 ** JOIN. But older versions of SQLite do not do that, so we avoid
578 ** adding a new error so as to not break legacy applications.
drh052953a2022-04-16 22:57:41 +0000579 */
drhfe146992022-04-17 18:46:17 +0000580 ExprList *pFuncArgs = 0; /* Arguments to the coalesce() */
drh052953a2022-04-16 22:57:41 +0000581 static const Token tkCoalesce = { "coalesce", 8 };
drhfe146992022-04-17 18:46:17 +0000582 while( tableAndColumnIndex(pSrc, iLeft+1, i, zName, &iLeft, &iLeftCol,
583 pRight->fg.isSynthUsing)!=0 ){
584 if( pSrc->a[iLeft].fg.isUsing==0
585 || sqlite3IdListIndex(pSrc->a[iLeft].u3.pUsing, zName)<0
586 ){
587 sqlite3ErrorMsg(pParse, "ambiguous reference to %s in USING()",
588 zName);
589 break;
590 }
591 pFuncArgs = sqlite3ExprListAppend(pParse, pFuncArgs, pE1);
592 pE1 = sqlite3CreateColumnExpr(db, pSrc, iLeft, iLeftCol);
drh22b410d2022-04-20 18:12:42 +0000593 sqlite3SrcItemColumnUsed(&pSrc->a[iLeft], iLeftCol);
drh052953a2022-04-16 22:57:41 +0000594 }
drhfe146992022-04-17 18:46:17 +0000595 if( pFuncArgs ){
596 pFuncArgs = sqlite3ExprListAppend(pParse, pFuncArgs, pE1);
597 pE1 = sqlite3ExprFunction(pParse, pFuncArgs, &tkCoalesce, 0);
drh052953a2022-04-16 22:57:41 +0000598 }
drh052953a2022-04-16 22:57:41 +0000599 }
drhfe146992022-04-17 18:46:17 +0000600 pE2 = sqlite3CreateColumnExpr(db, pSrc, i+1, iRightCol);
drh815b7822022-04-20 15:07:39 +0000601 sqlite3SrcItemColumnUsed(pRight, iRightCol);
drhfe146992022-04-17 18:46:17 +0000602 pEq = sqlite3PExpr(pParse, TK_EQ, pE1, pE2);
603 assert( pE2!=0 || pEq==0 );
604 if( pEq ){
605 ExprSetProperty(pEq, joinType);
606 assert( !ExprHasProperty(pEq, EP_TokenOnly|EP_Reduced) );
607 ExprSetVVAProperty(pEq, EP_NoReduce);
608 pEq->w.iJoin = pE2->iTable;
609 }
610 p->pWhere = sqlite3ExprAnd(pParse, p->pWhere, pEq);
drhad2d8302002-05-24 20:31:36 +0000611 }
612 }
drhd44f8b22022-04-07 01:11:13 +0000613
614 /* Add the ON clause to the end of the WHERE clause, connected by
615 ** an AND operator.
616 */
617 else if( pRight->u3.pOn ){
drh3a6e4c52022-04-11 12:38:06 +0000618 sqlite3SetJoinExpr(pRight->u3.pOn, pRight->iCursor, joinType);
drhd44f8b22022-04-07 01:11:13 +0000619 p->pWhere = sqlite3ExprAnd(pParse, p->pWhere, pRight->u3.pOn);
620 pRight->u3.pOn = 0;
drh5c118e32022-06-08 15:30:39 +0000621 pRight->fg.isOn = 1;
drhd44f8b22022-04-07 01:11:13 +0000622 }
drhad2d8302002-05-24 20:31:36 +0000623 }
624 return 0;
625}
626
627/*
drhbbd4ae52018-04-30 19:32:49 +0000628** An instance of this object holds information (beyond pParse and pSelect)
629** needed to load the next result row that is to be added to the sorter.
630*/
631typedef struct RowLoadInfo RowLoadInfo;
632struct RowLoadInfo {
633 int regResult; /* Store results in array of registers here */
634 u8 ecelFlags; /* Flag argument to ExprCodeExprList() */
635#ifdef SQLITE_ENABLE_SORTER_REFERENCES
636 ExprList *pExtra; /* Extra columns needed by sorter refs */
637 int regExtraResult; /* Where to load the extra columns */
638#endif
639};
640
641/*
642** This routine does the work of loading query data into an array of
643** registers so that it can be added to the sorter.
644*/
645static void innerLoopLoadRow(
646 Parse *pParse, /* Statement under construction */
647 Select *pSelect, /* The query being coded */
648 RowLoadInfo *pInfo /* Info needed to complete the row load */
649){
650 sqlite3ExprCodeExprList(pParse, pSelect->pEList, pInfo->regResult,
651 0, pInfo->ecelFlags);
652#ifdef SQLITE_ENABLE_SORTER_REFERENCES
653 if( pInfo->pExtra ){
654 sqlite3ExprCodeExprList(pParse, pInfo->pExtra, pInfo->regExtraResult, 0, 0);
655 sqlite3ExprListDelete(pParse->db, pInfo->pExtra);
656 }
657#endif
658}
659
660/*
661** Code the OP_MakeRecord instruction that generates the entry to be
662** added into the sorter.
663**
664** Return the register in which the result is stored.
665*/
666static int makeSorterRecord(
667 Parse *pParse,
668 SortCtx *pSort,
669 Select *pSelect,
670 int regBase,
671 int nBase
672){
673 int nOBSat = pSort->nOBSat;
674 Vdbe *v = pParse->pVdbe;
675 int regOut = ++pParse->nMem;
676 if( pSort->pDeferredRowLoad ){
677 innerLoopLoadRow(pParse, pSelect, pSort->pDeferredRowLoad);
678 }
679 sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regOut);
680 return regOut;
681}
682
683/*
drhf45f2322014-03-23 17:45:03 +0000684** Generate code that will push the record in registers regData
685** through regData+nData-1 onto the sorter.
drhc926afb2002-06-20 03:38:26 +0000686*/
drhd59ba6c2006-01-08 05:02:54 +0000687static void pushOntoSorter(
688 Parse *pParse, /* Parser context */
drh079a3072014-03-19 14:10:55 +0000689 SortCtx *pSort, /* Information about the ORDER BY clause */
drhb7654112008-01-12 12:48:07 +0000690 Select *pSelect, /* The whole SELECT statement */
drhf45f2322014-03-23 17:45:03 +0000691 int regData, /* First register holding data to be sorted */
drh5579d592015-08-26 14:01:41 +0000692 int regOrigData, /* First register holding data before packing */
drhbbd4ae52018-04-30 19:32:49 +0000693 int nData, /* Number of elements in the regData data array */
drhfd0a2f92014-03-24 18:08:15 +0000694 int nPrefixReg /* No. of reg prior to regData available for use */
drhd59ba6c2006-01-08 05:02:54 +0000695){
drhf45f2322014-03-23 17:45:03 +0000696 Vdbe *v = pParse->pVdbe; /* Stmt under construction */
dan78d58432014-03-25 15:04:07 +0000697 int bSeq = ((pSort->sortFlags & SORTFLAG_UseSorter)==0);
drhf45f2322014-03-23 17:45:03 +0000698 int nExpr = pSort->pOrderBy->nExpr; /* No. of ORDER BY terms */
dan78d58432014-03-25 15:04:07 +0000699 int nBase = nExpr + bSeq + nData; /* Fields in sorter record */
drhfd0a2f92014-03-24 18:08:15 +0000700 int regBase; /* Regs for sorter record */
drhbbd4ae52018-04-30 19:32:49 +0000701 int regRecord = 0; /* Assembled sorter record */
dan78d58432014-03-25 15:04:07 +0000702 int nOBSat = pSort->nOBSat; /* ORDER BY terms to skip */
703 int op; /* Opcode to add sorter record to sorter */
drha04a8be2016-01-13 17:50:10 +0000704 int iLimit; /* LIMIT counter */
drhbbd4ae52018-04-30 19:32:49 +0000705 int iSkip = 0; /* End of the sorter insert loop */
drhf45f2322014-03-23 17:45:03 +0000706
dan78d58432014-03-25 15:04:07 +0000707 assert( bSeq==0 || bSeq==1 );
drhbbd4ae52018-04-30 19:32:49 +0000708
709 /* Three cases:
710 ** (1) The data to be sorted has already been packed into a Record
711 ** by a prior OP_MakeRecord. In this case nData==1 and regData
712 ** will be completely unrelated to regOrigData.
713 ** (2) All output columns are included in the sort record. In that
714 ** case regData==regOrigData.
715 ** (3) Some output columns are omitted from the sort record due to
716 ** the SQLITE_ENABLE_SORTER_REFERENCE optimization, or due to the
drhc6f36fa2018-05-05 16:50:35 +0000717 ** SQLITE_ECEL_OMITREF optimization, or due to the
718 ** SortCtx.pDeferredRowLoad optimiation. In any of these cases
719 ** regOrigData is 0 to prevent this routine from trying to copy
720 ** values that might not yet exist.
drhbbd4ae52018-04-30 19:32:49 +0000721 */
dan9af90b72016-11-11 18:08:59 +0000722 assert( nData==1 || regData==regOrigData || regOrigData==0 );
drhbbd4ae52018-04-30 19:32:49 +0000723
drhfd0a2f92014-03-24 18:08:15 +0000724 if( nPrefixReg ){
dan78d58432014-03-25 15:04:07 +0000725 assert( nPrefixReg==nExpr+bSeq );
drhbbd4ae52018-04-30 19:32:49 +0000726 regBase = regData - nPrefixReg;
drhfd0a2f92014-03-24 18:08:15 +0000727 }else{
drhfb0d6e52014-05-02 13:09:06 +0000728 regBase = pParse->nMem + 1;
729 pParse->nMem += nBase;
drhfd0a2f92014-03-24 18:08:15 +0000730 }
drha04a8be2016-01-13 17:50:10 +0000731 assert( pSelect->iOffset==0 || pSelect->iLimit!=0 );
732 iLimit = pSelect->iOffset ? pSelect->iOffset+1 : pSelect->iLimit;
drhec4ccdb2018-12-29 02:26:59 +0000733 pSort->labelDone = sqlite3VdbeMakeLabel(pParse);
drh5579d592015-08-26 14:01:41 +0000734 sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, regOrigData,
dan9af90b72016-11-11 18:08:59 +0000735 SQLITE_ECEL_DUP | (regOrigData? SQLITE_ECEL_REF : 0));
dan78d58432014-03-25 15:04:07 +0000736 if( bSeq ){
737 sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr);
drhfd0a2f92014-03-24 18:08:15 +0000738 }
dan257c13f2016-11-10 20:14:06 +0000739 if( nPrefixReg==0 && nData>0 ){
drh236241a2014-09-12 17:41:30 +0000740 sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+bSeq, nData);
dan78d58432014-03-25 15:04:07 +0000741 }
drh079a3072014-03-19 14:10:55 +0000742 if( nOBSat>0 ){
743 int regPrevKey; /* The first nOBSat columns of the previous row */
744 int addrFirst; /* Address of the OP_IfNot opcode */
745 int addrJmp; /* Address of the OP_Jump opcode */
746 VdbeOp *pOp; /* Opcode that opens the sorter */
747 int nKey; /* Number of sorting key columns, including OP_Sequence */
drhdbfca2b2014-03-22 02:19:53 +0000748 KeyInfo *pKI; /* Original KeyInfo on the sorter table */
drh079a3072014-03-19 14:10:55 +0000749
drhbbd4ae52018-04-30 19:32:49 +0000750 regRecord = makeSorterRecord(pParse, pSort, pSelect, regBase, nBase);
drh26d7e7c2014-03-19 16:56:58 +0000751 regPrevKey = pParse->nMem+1;
752 pParse->nMem += pSort->nOBSat;
dan78d58432014-03-25 15:04:07 +0000753 nKey = nExpr - pSort->nOBSat + bSeq;
754 if( bSeq ){
755 addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr);
756 }else{
757 addrFirst = sqlite3VdbeAddOp1(v, OP_SequenceTest, pSort->iECursor);
758 }
759 VdbeCoverage(v);
drh26d7e7c2014-03-19 16:56:58 +0000760 sqlite3VdbeAddOp3(v, OP_Compare, regPrevKey, regBase, pSort->nOBSat);
drh079a3072014-03-19 14:10:55 +0000761 pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex);
drh59b8f2e2014-03-22 00:27:14 +0000762 if( pParse->db->mallocFailed ) return;
drhfb0d6e52014-05-02 13:09:06 +0000763 pOp->p2 = nKey + nData;
drhdbfca2b2014-03-22 02:19:53 +0000764 pKI = pOp->p4.pKeyInfo;
dan6e118922019-08-12 16:36:38 +0000765 memset(pKI->aSortFlags, 0, pKI->nKeyField); /* Makes OP_Jump testable */
drhdbfca2b2014-03-22 02:19:53 +0000766 sqlite3VdbeChangeP4(v, -1, (char*)pKI, P4_KEYINFO);
drha485ad12017-08-02 22:43:14 +0000767 testcase( pKI->nAllField > pKI->nKeyField+2 );
danf9eae182018-05-21 19:45:11 +0000768 pOp->p4.pKeyInfo = sqlite3KeyInfoFromExprList(pParse,pSort->pOrderBy,nOBSat,
drha485ad12017-08-02 22:43:14 +0000769 pKI->nAllField-pKI->nKeyField-1);
drh166bc382019-10-26 15:40:17 +0000770 pOp = 0; /* Ensure pOp not used after sqltie3VdbeAddOp3() */
drh079a3072014-03-19 14:10:55 +0000771 addrJmp = sqlite3VdbeCurrentAddr(v);
772 sqlite3VdbeAddOp3(v, OP_Jump, addrJmp+1, 0, addrJmp+1); VdbeCoverage(v);
drhec4ccdb2018-12-29 02:26:59 +0000773 pSort->labelBkOut = sqlite3VdbeMakeLabel(pParse);
drh079a3072014-03-19 14:10:55 +0000774 pSort->regReturn = ++pParse->nMem;
775 sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut);
drh65ea12c2014-03-19 17:41:36 +0000776 sqlite3VdbeAddOp1(v, OP_ResetSorter, pSort->iECursor);
drha04a8be2016-01-13 17:50:10 +0000777 if( iLimit ){
778 sqlite3VdbeAddOp2(v, OP_IfNot, iLimit, pSort->labelDone);
779 VdbeCoverage(v);
780 }
drh079a3072014-03-19 14:10:55 +0000781 sqlite3VdbeJumpHere(v, addrFirst);
drh236241a2014-09-12 17:41:30 +0000782 sqlite3ExprCodeMove(pParse, regBase, regPrevKey, pSort->nOBSat);
drh079a3072014-03-19 14:10:55 +0000783 sqlite3VdbeJumpHere(v, addrJmp);
784 }
danf226f032018-04-26 16:13:47 +0000785 if( iLimit ){
786 /* At this point the values for the new sorter entry are stored
787 ** in an array of registers. They need to be composed into a record
788 ** and inserted into the sorter if either (a) there are currently
789 ** less than LIMIT+OFFSET items or (b) the new record is smaller than
790 ** the largest record currently in the sorter. If (b) is true and there
791 ** are already LIMIT+OFFSET items in the sorter, delete the largest
792 ** entry before inserting the new one. This way there are never more
793 ** than LIMIT+OFFSET items in the sorter.
794 **
795 ** If the new record does not need to be inserted into the sorter,
drh6ee5a7b2018-09-08 20:09:46 +0000796 ** jump to the next iteration of the loop. If the pSort->labelOBLopt
797 ** value is not zero, then it is a label of where to jump. Otherwise,
798 ** just bypass the row insert logic. See the header comment on the
799 ** sqlite3WhereOrderByLimitOptLabel() function for additional info.
danf226f032018-04-26 16:13:47 +0000800 */
801 int iCsr = pSort->iECursor;
danf226f032018-04-26 16:13:47 +0000802 sqlite3VdbeAddOp2(v, OP_IfNotZero, iLimit, sqlite3VdbeCurrentAddr(v)+4);
803 VdbeCoverage(v);
804 sqlite3VdbeAddOp2(v, OP_Last, iCsr, 0);
drhbbd4ae52018-04-30 19:32:49 +0000805 iSkip = sqlite3VdbeAddOp4Int(v, OP_IdxLE,
806 iCsr, 0, regBase+nOBSat, nExpr-nOBSat);
danf226f032018-04-26 16:13:47 +0000807 VdbeCoverage(v);
808 sqlite3VdbeAddOp1(v, OP_Delete, iCsr);
809 }
drhbbd4ae52018-04-30 19:32:49 +0000810 if( regRecord==0 ){
811 regRecord = makeSorterRecord(pParse, pSort, pSelect, regBase, nBase);
danf226f032018-04-26 16:13:47 +0000812 }
drh079a3072014-03-19 14:10:55 +0000813 if( pSort->sortFlags & SORTFLAG_UseSorter ){
drhc6aff302011-09-01 15:32:47 +0000814 op = OP_SorterInsert;
815 }else{
816 op = OP_IdxInsert;
817 }
drh4a8b0132016-11-09 01:38:56 +0000818 sqlite3VdbeAddOp4Int(v, op, pSort->iECursor, regRecord,
819 regBase+nOBSat, nBase-nOBSat);
drhbbd4ae52018-04-30 19:32:49 +0000820 if( iSkip ){
drhbbd4ae52018-04-30 19:32:49 +0000821 sqlite3VdbeChangeP2(v, iSkip,
drh6ee5a7b2018-09-08 20:09:46 +0000822 pSort->labelOBLopt ? pSort->labelOBLopt : sqlite3VdbeCurrentAddr(v));
drhbbd4ae52018-04-30 19:32:49 +0000823 }
drhc926afb2002-06-20 03:38:26 +0000824}
825
826/*
drhec7429a2005-10-06 16:53:14 +0000827** Add code to implement the OFFSET
drhea48eb22004-07-19 23:16:38 +0000828*/
drhec7429a2005-10-06 16:53:14 +0000829static void codeOffset(
drhbab39e12004-07-19 23:38:11 +0000830 Vdbe *v, /* Generate code into this VM */
drhaa9ce702014-01-22 18:07:04 +0000831 int iOffset, /* Register holding the offset counter */
drhb7654112008-01-12 12:48:07 +0000832 int iContinue /* Jump here to skip the current record */
drhea48eb22004-07-19 23:16:38 +0000833){
drha22a75e2014-03-21 18:16:23 +0000834 if( iOffset>0 ){
drh8b0cf382015-10-06 21:07:06 +0000835 sqlite3VdbeAddOp3(v, OP_IfPos, iOffset, iContinue, 1); VdbeCoverage(v);
836 VdbeComment((v, "OFFSET"));
drhea48eb22004-07-19 23:16:38 +0000837 }
drhea48eb22004-07-19 23:16:38 +0000838}
839
840/*
dan96ecb7b2021-03-09 16:47:33 +0000841** Add code that will check to make sure the array of registers starting at
842** iMem form a distinct entry. This is used by both "SELECT DISTINCT ..." and
dan76075802021-03-12 21:09:20 +0000843** distinct aggregates ("SELECT count(DISTINCT <expr>) ..."). Three strategies
844** are available. Which is used depends on the value of parameter eTnctType,
845** as follows:
drha2a49dc2008-01-02 14:28:13 +0000846**
dan96ecb7b2021-03-09 16:47:33 +0000847** WHERE_DISTINCT_UNORDERED/WHERE_DISTINCT_NOOP:
drh9be13392021-03-24 19:44:01 +0000848** Build an ephemeral table that contains all entries seen before and
849** skip entries which have been seen before.
850**
dan76075802021-03-12 21:09:20 +0000851** Parameter iTab is the cursor number of an ephemeral table that must
852** be opened before the VM code generated by this routine is executed.
dan5383db72021-03-12 18:24:31 +0000853** The ephemeral cursor table is queried for a record identical to the
dan96ecb7b2021-03-09 16:47:33 +0000854** record formed by the current array of registers. If one is found,
855** jump to VM address addrRepeat. Otherwise, insert a new record into
856** the ephemeral cursor and proceed.
857**
dan76075802021-03-12 21:09:20 +0000858** The returned value in this case is a copy of parameter iTab.
859**
dan96ecb7b2021-03-09 16:47:33 +0000860** WHERE_DISTINCT_ORDERED:
drh9be13392021-03-24 19:44:01 +0000861** In this case rows are being delivered sorted order. The ephermal
862** table is not required. Instead, the current set of values
863** is compared against previous row. If they match, the new row
dan96ecb7b2021-03-09 16:47:33 +0000864** is not distinct and control jumps to VM address addrRepeat. Otherwise,
865** the VM program proceeds with processing the new row.
866**
dan76075802021-03-12 21:09:20 +0000867** The returned value in this case is the register number of the first
868** in an array of registers used to store the previous result row so that
drh9be13392021-03-24 19:44:01 +0000869** it can be compared to the next. The caller must ensure that this
870** register is initialized to NULL. (The fixDistinctOpenEph() routine
871** will take care of this initialization.)
dan76075802021-03-12 21:09:20 +0000872**
dan96ecb7b2021-03-09 16:47:33 +0000873** WHERE_DISTINCT_UNIQUE:
874** In this case it has already been determined that the rows are distinct.
drh9be13392021-03-24 19:44:01 +0000875** No special action is required. The return value is zero.
dan96ecb7b2021-03-09 16:47:33 +0000876**
877** Parameter pEList is the list of expressions used to generated the
878** contents of each row. It is used by this routine to determine (a)
879** how many elements there are in the array of registers and (b) the
880** collation sequences that should be used for the comparisons if
881** eTnctType is WHERE_DISTINCT_ORDERED.
drha2a49dc2008-01-02 14:28:13 +0000882*/
dan5383db72021-03-12 18:24:31 +0000883static int codeDistinct(
drh2dcef112008-01-12 19:03:48 +0000884 Parse *pParse, /* Parsing and code generating context */
dan4fcb30b2021-03-09 16:06:25 +0000885 int eTnctType, /* WHERE_DISTINCT_* value */
drha2a49dc2008-01-02 14:28:13 +0000886 int iTab, /* A sorting index used to test for distinctness */
887 int addrRepeat, /* Jump to here if not distinct */
dan4fcb30b2021-03-09 16:06:25 +0000888 ExprList *pEList, /* Expression for each element */
889 int regElem /* First element */
drha2a49dc2008-01-02 14:28:13 +0000890){
dan5383db72021-03-12 18:24:31 +0000891 int iRet = 0;
dan4fcb30b2021-03-09 16:06:25 +0000892 int nResultCol = pEList->nExpr;
893 Vdbe *v = pParse->pVdbe;
drh2dcef112008-01-12 19:03:48 +0000894
dan4fcb30b2021-03-09 16:06:25 +0000895 switch( eTnctType ){
896 case WHERE_DISTINCT_ORDERED: {
897 int i;
dan4fcb30b2021-03-09 16:06:25 +0000898 int iJump; /* Jump destination */
899 int regPrev; /* Previous row content */
900
901 /* Allocate space for the previous row */
dan5383db72021-03-12 18:24:31 +0000902 iRet = regPrev = pParse->nMem+1;
dan4fcb30b2021-03-09 16:06:25 +0000903 pParse->nMem += nResultCol;
904
dan4fcb30b2021-03-09 16:06:25 +0000905 iJump = sqlite3VdbeCurrentAddr(v) + nResultCol;
906 for(i=0; i<nResultCol; i++){
907 CollSeq *pColl = sqlite3ExprCollSeq(pParse, pEList->a[i].pExpr);
908 if( i<nResultCol-1 ){
909 sqlite3VdbeAddOp3(v, OP_Ne, regElem+i, iJump, regPrev+i);
910 VdbeCoverage(v);
911 }else{
912 sqlite3VdbeAddOp3(v, OP_Eq, regElem+i, addrRepeat, regPrev+i);
913 VdbeCoverage(v);
914 }
915 sqlite3VdbeChangeP4(v, -1, (const char *)pColl, P4_COLLSEQ);
916 sqlite3VdbeChangeP5(v, SQLITE_NULLEQ);
917 }
918 assert( sqlite3VdbeCurrentAddr(v)==iJump || pParse->db->mallocFailed );
919 sqlite3VdbeAddOp3(v, OP_Copy, regElem, regPrev, nResultCol-1);
920 break;
921 }
922
923 case WHERE_DISTINCT_UNIQUE: {
dan5383db72021-03-12 18:24:31 +0000924 /* nothing to do */
dan4fcb30b2021-03-09 16:06:25 +0000925 break;
926 }
927
928 default: {
929 int r1 = sqlite3GetTempReg(pParse);
930 sqlite3VdbeAddOp4Int(v, OP_Found, iTab, addrRepeat, regElem, nResultCol);
931 VdbeCoverage(v);
932 sqlite3VdbeAddOp3(v, OP_MakeRecord, regElem, nResultCol, r1);
933 sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iTab, r1, regElem, nResultCol);
934 sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
935 sqlite3ReleaseTempReg(pParse, r1);
dan5383db72021-03-12 18:24:31 +0000936 iRet = iTab;
dan4fcb30b2021-03-09 16:06:25 +0000937 break;
938 }
939 }
dan5383db72021-03-12 18:24:31 +0000940
941 return iRet;
942}
943
dan76075802021-03-12 21:09:20 +0000944/*
drh9be13392021-03-24 19:44:01 +0000945** This routine runs after codeDistinct(). It makes necessary
946** adjustments to the OP_OpenEphemeral opcode that the codeDistinct()
947** routine made use of. This processing must be done separately since
948** sometimes codeDistinct is called before the OP_OpenEphemeral is actually
949** laid down.
dan76075802021-03-12 21:09:20 +0000950**
drh9be13392021-03-24 19:44:01 +0000951** WHERE_DISTINCT_NOOP:
952** WHERE_DISTINCT_UNORDERED:
dan76075802021-03-12 21:09:20 +0000953**
drh9be13392021-03-24 19:44:01 +0000954** No adjustments necessary. This function is a no-op.
dan76075802021-03-12 21:09:20 +0000955**
drh9be13392021-03-24 19:44:01 +0000956** WHERE_DISTINCT_UNIQUE:
957**
958** The ephemeral table is not needed. So change the
959** OP_OpenEphemeral opcode into an OP_Noop.
960**
961** WHERE_DISTINCT_ORDERED:
962**
963** The ephemeral table is not needed. But we do need register
964** iVal to be initialized to NULL. So change the OP_OpenEphemeral
965** into an OP_Null on the iVal register.
dan76075802021-03-12 21:09:20 +0000966*/
dan5383db72021-03-12 18:24:31 +0000967static void fixDistinctOpenEph(
968 Parse *pParse, /* Parsing and code generating context */
969 int eTnctType, /* WHERE_DISTINCT_* value */
970 int iVal, /* Value returned by codeDistinct() */
dan76075802021-03-12 21:09:20 +0000971 int iOpenEphAddr /* Address of OP_OpenEphemeral instruction for iTab */
dan5383db72021-03-12 18:24:31 +0000972){
drhc6da6db2021-07-28 02:04:58 +0000973 if( pParse->nErr==0
974 && (eTnctType==WHERE_DISTINCT_UNIQUE || eTnctType==WHERE_DISTINCT_ORDERED)
975 ){
dan5383db72021-03-12 18:24:31 +0000976 Vdbe *v = pParse->pVdbe;
dan76075802021-03-12 21:09:20 +0000977 sqlite3VdbeChangeToNoop(v, iOpenEphAddr);
drh9be13392021-03-24 19:44:01 +0000978 if( sqlite3VdbeGetOp(v, iOpenEphAddr+1)->opcode==OP_Explain ){
979 sqlite3VdbeChangeToNoop(v, iOpenEphAddr+1);
980 }
dan5383db72021-03-12 18:24:31 +0000981 if( eTnctType==WHERE_DISTINCT_ORDERED ){
982 /* Change the OP_OpenEphemeral to an OP_Null that sets the MEM_Cleared
983 ** bit on the first register of the previous value. This will cause the
984 ** OP_Ne added in codeDistinct() to always fail on the first iteration of
985 ** the loop even if the first row is all NULLs. */
dan76075802021-03-12 21:09:20 +0000986 VdbeOp *pOp = sqlite3VdbeGetOp(v, iOpenEphAddr);
dan5383db72021-03-12 18:24:31 +0000987 pOp->opcode = OP_Null;
988 pOp->p1 = 1;
989 pOp->p2 = iVal;
990 }
991 }
drha2a49dc2008-01-02 14:28:13 +0000992}
993
dan24e25d32018-04-14 18:46:20 +0000994#ifdef SQLITE_ENABLE_SORTER_REFERENCES
995/*
996** This function is called as part of inner-loop generation for a SELECT
997** statement with an ORDER BY that is not optimized by an index. It
998** determines the expressions, if any, that the sorter-reference
999** optimization should be used for. The sorter-reference optimization
1000** is used for SELECT queries like:
1001**
1002** SELECT a, bigblob FROM t1 ORDER BY a LIMIT 10
1003**
1004** If the optimization is used for expression "bigblob", then instead of
1005** storing values read from that column in the sorter records, the PK of
1006** the row from table t1 is stored instead. Then, as records are extracted from
1007** the sorter to return to the user, the required value of bigblob is
1008** retrieved directly from table t1. If the values are very large, this
1009** can be more efficient than storing them directly in the sorter records.
1010**
drh00e5d3d2022-05-03 16:18:25 +00001011** The ExprList_item.fg.bSorterRef flag is set for each expression in pEList
dan24e25d32018-04-14 18:46:20 +00001012** for which the sorter-reference optimization should be enabled.
1013** Additionally, the pSort->aDefer[] array is populated with entries
1014** for all cursors required to evaluate all selected expressions. Finally.
1015** output variable (*ppExtra) is set to an expression list containing
1016** expressions for all extra PK values that should be stored in the
1017** sorter records.
1018*/
1019static void selectExprDefer(
1020 Parse *pParse, /* Leave any error here */
1021 SortCtx *pSort, /* Sorter context */
1022 ExprList *pEList, /* Expressions destined for sorter */
1023 ExprList **ppExtra /* Expressions to append to sorter record */
1024){
1025 int i;
1026 int nDefer = 0;
1027 ExprList *pExtra = 0;
1028 for(i=0; i<pEList->nExpr; i++){
1029 struct ExprList_item *pItem = &pEList->a[i];
1030 if( pItem->u.x.iOrderByCol==0 ){
1031 Expr *pExpr = pItem->pExpr;
drh477572b2021-10-07 20:46:29 +00001032 Table *pTab;
drhffa5b052021-10-12 18:05:55 +00001033 if( pExpr->op==TK_COLUMN
1034 && pExpr->iColumn>=0
1035 && ALWAYS( ExprUseYTab(pExpr) )
1036 && (pTab = pExpr->y.pTab)!=0
1037 && IsOrdinaryTable(pTab)
1038 && (pTab->aCol[pExpr->iColumn].colFlags & COLFLAG_SORTERREF)!=0
dan24e25d32018-04-14 18:46:20 +00001039 ){
1040 int j;
1041 for(j=0; j<nDefer; j++){
1042 if( pSort->aDefer[j].iCsr==pExpr->iTable ) break;
1043 }
1044 if( j==nDefer ){
1045 if( nDefer==ArraySize(pSort->aDefer) ){
1046 continue;
1047 }else{
1048 int nKey = 1;
1049 int k;
1050 Index *pPk = 0;
1051 if( !HasRowid(pTab) ){
1052 pPk = sqlite3PrimaryKeyIndex(pTab);
1053 nKey = pPk->nKeyCol;
1054 }
1055 for(k=0; k<nKey; k++){
1056 Expr *pNew = sqlite3PExpr(pParse, TK_COLUMN, 0, 0);
1057 if( pNew ){
1058 pNew->iTable = pExpr->iTable;
drh477572b2021-10-07 20:46:29 +00001059 assert( ExprUseYTab(pNew) );
drheda079c2018-09-20 19:02:15 +00001060 pNew->y.pTab = pExpr->y.pTab;
dan24e25d32018-04-14 18:46:20 +00001061 pNew->iColumn = pPk ? pPk->aiColumn[k] : -1;
1062 pExtra = sqlite3ExprListAppend(pParse, pExtra, pNew);
1063 }
1064 }
drheda079c2018-09-20 19:02:15 +00001065 pSort->aDefer[nDefer].pTab = pExpr->y.pTab;
dan24e25d32018-04-14 18:46:20 +00001066 pSort->aDefer[nDefer].iCsr = pExpr->iTable;
1067 pSort->aDefer[nDefer].nKey = nKey;
1068 nDefer++;
1069 }
1070 }
drhd88fd532022-05-02 20:49:30 +00001071 pItem->fg.bSorterRef = 1;
dan24e25d32018-04-14 18:46:20 +00001072 }
1073 }
1074 }
1075 pSort->nDefer = (u8)nDefer;
1076 *ppExtra = pExtra;
1077}
1078#endif
1079
drhc99130f2005-09-11 11:56:27 +00001080/*
drh22827922000-06-06 17:27:05 +00001081** This routine generates the code for the inside of the inner loop
1082** of a SELECT.
drh82c3d632000-06-06 21:56:07 +00001083**
drh2def2f72017-09-15 17:40:34 +00001084** If srcTab is negative, then the p->pEList expressions
drh340309f2014-01-22 00:23:49 +00001085** are evaluated in order to get the data for this row. If srcTab is
drh2def2f72017-09-15 17:40:34 +00001086** zero or more, then data is pulled from srcTab and p->pEList is used only
dan257c13f2016-11-10 20:14:06 +00001087** to get the number of columns and the collation sequence for each column.
drh22827922000-06-06 17:27:05 +00001088*/
drhd2b3e232008-01-23 14:51:49 +00001089static void selectInnerLoop(
drh22827922000-06-06 17:27:05 +00001090 Parse *pParse, /* The parser context */
drhdf199a22002-06-14 22:38:41 +00001091 Select *p, /* The complete select statement being coded */
drh2def2f72017-09-15 17:40:34 +00001092 int srcTab, /* Pull data from this table if non-negative */
drh079a3072014-03-19 14:10:55 +00001093 SortCtx *pSort, /* If not NULL, info on how to process ORDER BY */
drhe8e4af72012-09-21 00:04:28 +00001094 DistinctCtx *pDistinct, /* If not NULL, info on how to process DISTINCT */
danielk19776c8c8ce2008-01-02 16:27:09 +00001095 SelectDest *pDest, /* How to dispose of the results */
drh22827922000-06-06 17:27:05 +00001096 int iContinue, /* Jump here to continue with next row */
drha9671a22008-07-08 23:40:20 +00001097 int iBreak /* Jump here to break out of the inner loop */
drh22827922000-06-06 17:27:05 +00001098){
1099 Vdbe *v = pParse->pVdbe;
drhd847eaa2008-01-17 17:15:56 +00001100 int i;
dan9af90b72016-11-11 18:08:59 +00001101 int hasDistinct; /* True if the DISTINCT keyword is present */
drhd847eaa2008-01-17 17:15:56 +00001102 int eDest = pDest->eDest; /* How to dispose of results */
drh2b596da2012-07-23 21:43:19 +00001103 int iParm = pDest->iSDParm; /* First argument to disposal method */
drhd847eaa2008-01-17 17:15:56 +00001104 int nResultCol; /* Number of result columns */
drhfd0a2f92014-03-24 18:08:15 +00001105 int nPrefixReg = 0; /* Number of extra registers before regResult */
drhbbd4ae52018-04-30 19:32:49 +00001106 RowLoadInfo sRowLoadInfo; /* Info for deferred row loading */
drh38640e12002-07-05 21:42:36 +00001107
dan9af90b72016-11-11 18:08:59 +00001108 /* Usually, regResult is the first cell in an array of memory cells
1109 ** containing the current result row. In this case regOrig is set to the
1110 ** same value. However, if the results are being sent to the sorter, the
1111 ** values for any expressions that are also part of the sort-key are omitted
1112 ** from this array. In this case regOrig is set to zero. */
1113 int regResult; /* Start of memory holding current results */
1114 int regOrig; /* Start of memory holding full result (or 0) */
1115
drh1c767f02009-01-09 02:49:31 +00001116 assert( v );
drh2def2f72017-09-15 17:40:34 +00001117 assert( p->pEList!=0 );
drhe8e4af72012-09-21 00:04:28 +00001118 hasDistinct = pDistinct ? pDistinct->eTnctType : WHERE_DISTINCT_NOOP;
drh079a3072014-03-19 14:10:55 +00001119 if( pSort && pSort->pOrderBy==0 ) pSort = 0;
1120 if( pSort==0 && !hasDistinct ){
drha22a75e2014-03-21 18:16:23 +00001121 assert( iContinue!=0 );
drhaa9ce702014-01-22 18:07:04 +00001122 codeOffset(v, p->iOffset, iContinue);
drhdf199a22002-06-14 22:38:41 +00001123 }
1124
drh967e8b72000-06-21 13:59:10 +00001125 /* Pull the requested columns.
drh22827922000-06-06 17:27:05 +00001126 */
drh2def2f72017-09-15 17:40:34 +00001127 nResultCol = p->pEList->nExpr;
drh05a86c52014-02-16 01:55:49 +00001128
drh2b596da2012-07-23 21:43:19 +00001129 if( pDest->iSdst==0 ){
drhfd0a2f92014-03-24 18:08:15 +00001130 if( pSort ){
dan78d58432014-03-25 15:04:07 +00001131 nPrefixReg = pSort->pOrderBy->nExpr;
1132 if( !(pSort->sortFlags & SORTFLAG_UseSorter) ) nPrefixReg++;
drhfd0a2f92014-03-24 18:08:15 +00001133 pParse->nMem += nPrefixReg;
1134 }
drh2b596da2012-07-23 21:43:19 +00001135 pDest->iSdst = pParse->nMem+1;
drh0acb7e42008-06-25 00:12:41 +00001136 pParse->nMem += nResultCol;
drh05a86c52014-02-16 01:55:49 +00001137 }else if( pDest->iSdst+nResultCol > pParse->nMem ){
1138 /* This is an error condition that can result, for example, when a SELECT
1139 ** on the right-hand side of an INSERT contains more result columns than
1140 ** there are columns in the table on the left. The error will be caught
1141 ** and reported later. But we need to make sure enough memory is allocated
1142 ** to avoid other spurious errors in the meantime. */
1143 pParse->nMem += nResultCol;
drh1013c932008-01-06 00:25:21 +00001144 }
drh05a86c52014-02-16 01:55:49 +00001145 pDest->nSdst = nResultCol;
dan9af90b72016-11-11 18:08:59 +00001146 regOrig = regResult = pDest->iSdst;
drh340309f2014-01-22 00:23:49 +00001147 if( srcTab>=0 ){
1148 for(i=0; i<nResultCol; i++){
drhd847eaa2008-01-17 17:15:56 +00001149 sqlite3VdbeAddOp3(v, OP_Column, srcTab, i, regResult+i);
drh41cee662019-12-12 20:22:34 +00001150 VdbeComment((v, "%s", p->pEList->a[i].zEName));
drh82c3d632000-06-06 21:56:07 +00001151 }
danielk19779ed1dfa2008-01-02 17:11:14 +00001152 }else if( eDest!=SRT_Exists ){
dan24e25d32018-04-14 18:46:20 +00001153#ifdef SQLITE_ENABLE_SORTER_REFERENCES
1154 ExprList *pExtra = 0;
1155#endif
danielk19779ed1dfa2008-01-02 17:11:14 +00001156 /* If the destination is an EXISTS(...) expression, the actual
1157 ** values returned by the SELECT are not required.
1158 */
drhbbd4ae52018-04-30 19:32:49 +00001159 u8 ecelFlags; /* "ecel" is an abbreviation of "ExprCodeExprList" */
1160 ExprList *pEList;
drhdf553652015-05-18 04:24:27 +00001161 if( eDest==SRT_Mem || eDest==SRT_Output || eDest==SRT_Coroutine ){
1162 ecelFlags = SQLITE_ECEL_DUP;
1163 }else{
1164 ecelFlags = 0;
1165 }
danac56ab72017-01-05 17:23:11 +00001166 if( pSort && hasDistinct==0 && eDest!=SRT_EphemTab && eDest!=SRT_Table ){
drh2def2f72017-09-15 17:40:34 +00001167 /* For each expression in p->pEList that is a copy of an expression in
dan257c13f2016-11-10 20:14:06 +00001168 ** the ORDER BY clause (pSort->pOrderBy), set the associated
1169 ** iOrderByCol value to one more than the index of the ORDER BY
1170 ** expression within the sort-key that pushOntoSorter() will generate.
drh2def2f72017-09-15 17:40:34 +00001171 ** This allows the p->pEList field to be omitted from the sorted record,
dan257c13f2016-11-10 20:14:06 +00001172 ** saving space and CPU cycles. */
1173 ecelFlags |= (SQLITE_ECEL_OMITREF|SQLITE_ECEL_REF);
drhbbd4ae52018-04-30 19:32:49 +00001174
dan257c13f2016-11-10 20:14:06 +00001175 for(i=pSort->nOBSat; i<pSort->pOrderBy->nExpr; i++){
1176 int j;
1177 if( (j = pSort->pOrderBy->a[i].u.x.iOrderByCol)>0 ){
drh2def2f72017-09-15 17:40:34 +00001178 p->pEList->a[j-1].u.x.iOrderByCol = i+1-pSort->nOBSat;
dan257c13f2016-11-10 20:14:06 +00001179 }
1180 }
dan24e25d32018-04-14 18:46:20 +00001181#ifdef SQLITE_ENABLE_SORTER_REFERENCES
1182 selectExprDefer(pParse, pSort, p->pEList, &pExtra);
drh5a2e65e2018-04-18 19:08:44 +00001183 if( pExtra && pParse->db->mallocFailed==0 ){
dan24e25d32018-04-14 18:46:20 +00001184 /* If there are any extra PK columns to add to the sorter records,
1185 ** allocate extra memory cells and adjust the OpenEphemeral
1186 ** instruction to account for the larger records. This is only
1187 ** required if there are one or more WITHOUT ROWID tables with
1188 ** composite primary keys in the SortCtx.aDefer[] array. */
1189 VdbeOp *pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex);
1190 pOp->p2 += (pExtra->nExpr - pSort->nDefer);
1191 pOp->p4.pKeyInfo->nAllField += (pExtra->nExpr - pSort->nDefer);
1192 pParse->nMem += pExtra->nExpr;
1193 }
1194#endif
drhbbd4ae52018-04-30 19:32:49 +00001195
1196 /* Adjust nResultCol to account for columns that are omitted
1197 ** from the sorter by the optimizations in this branch */
1198 pEList = p->pEList;
1199 for(i=0; i<pEList->nExpr; i++){
1200 if( pEList->a[i].u.x.iOrderByCol>0
1201#ifdef SQLITE_ENABLE_SORTER_REFERENCES
drhd88fd532022-05-02 20:49:30 +00001202 || pEList->a[i].fg.bSorterRef
drhbbd4ae52018-04-30 19:32:49 +00001203#endif
1204 ){
1205 nResultCol--;
1206 regOrig = 0;
1207 }
1208 }
1209
1210 testcase( regOrig );
1211 testcase( eDest==SRT_Set );
1212 testcase( eDest==SRT_Mem );
1213 testcase( eDest==SRT_Coroutine );
1214 testcase( eDest==SRT_Output );
dan257c13f2016-11-10 20:14:06 +00001215 assert( eDest==SRT_Set || eDest==SRT_Mem
danf2972b62020-04-29 20:11:01 +00001216 || eDest==SRT_Coroutine || eDest==SRT_Output
1217 || eDest==SRT_Upfrom );
dan257c13f2016-11-10 20:14:06 +00001218 }
drhbbd4ae52018-04-30 19:32:49 +00001219 sRowLoadInfo.regResult = regResult;
1220 sRowLoadInfo.ecelFlags = ecelFlags;
dan24e25d32018-04-14 18:46:20 +00001221#ifdef SQLITE_ENABLE_SORTER_REFERENCES
drhbbd4ae52018-04-30 19:32:49 +00001222 sRowLoadInfo.pExtra = pExtra;
1223 sRowLoadInfo.regExtraResult = regResult + nResultCol;
1224 if( pExtra ) nResultCol += pExtra->nExpr;
dan24e25d32018-04-14 18:46:20 +00001225#endif
drhbbd4ae52018-04-30 19:32:49 +00001226 if( p->iLimit
1227 && (ecelFlags & SQLITE_ECEL_OMITREF)!=0
1228 && nPrefixReg>0
1229 ){
1230 assert( pSort!=0 );
1231 assert( hasDistinct==0 );
1232 pSort->pDeferredRowLoad = &sRowLoadInfo;
drhc6f36fa2018-05-05 16:50:35 +00001233 regOrig = 0;
drhbbd4ae52018-04-30 19:32:49 +00001234 }else{
1235 innerLoopLoadRow(pParse, p, &sRowLoadInfo);
1236 }
drh22827922000-06-06 17:27:05 +00001237 }
1238
drhdaffd0e2001-04-11 14:28:42 +00001239 /* If the DISTINCT keyword was present on the SELECT statement
1240 ** and this row has been seen before, then do not make this row
1241 ** part of the result.
drh22827922000-06-06 17:27:05 +00001242 */
drhea48eb22004-07-19 23:16:38 +00001243 if( hasDistinct ){
dan5383db72021-03-12 18:24:31 +00001244 int eType = pDistinct->eTnctType;
1245 int iTab = pDistinct->tabTnct;
dan4fcb30b2021-03-09 16:06:25 +00001246 assert( nResultCol==p->pEList->nExpr );
dan5383db72021-03-12 18:24:31 +00001247 iTab = codeDistinct(pParse, eType, iTab, iContinue, p->pEList, regResult);
1248 fixDistinctOpenEph(pParse, eType, iTab, pDistinct->addrTnct);
drh079a3072014-03-19 14:10:55 +00001249 if( pSort==0 ){
drhaa9ce702014-01-22 18:07:04 +00001250 codeOffset(v, p->iOffset, iContinue);
drhea48eb22004-07-19 23:16:38 +00001251 }
drh22827922000-06-06 17:27:05 +00001252 }
drh82c3d632000-06-06 21:56:07 +00001253
drhc926afb2002-06-20 03:38:26 +00001254 switch( eDest ){
1255 /* In this mode, write each query result to the key of the temporary
1256 ** table iParm.
1257 */
drh13449892005-09-07 21:22:45 +00001258#ifndef SQLITE_OMIT_COMPOUND_SELECT
drhc926afb2002-06-20 03:38:26 +00001259 case SRT_Union: {
drh9cbf3422008-01-17 16:22:13 +00001260 int r1;
1261 r1 = sqlite3GetTempReg(pParse);
drh340309f2014-01-22 00:23:49 +00001262 sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1);
drh9b4eaeb2016-11-09 00:10:33 +00001263 sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regResult, nResultCol);
drh9cbf3422008-01-17 16:22:13 +00001264 sqlite3ReleaseTempReg(pParse, r1);
drhc926afb2002-06-20 03:38:26 +00001265 break;
drh22827922000-06-06 17:27:05 +00001266 }
drh22827922000-06-06 17:27:05 +00001267
drhc926afb2002-06-20 03:38:26 +00001268 /* Construct a record from the query result, but instead of
1269 ** saving that record, use it as a key to delete elements from
1270 ** the temporary table iParm.
1271 */
1272 case SRT_Except: {
drh340309f2014-01-22 00:23:49 +00001273 sqlite3VdbeAddOp3(v, OP_IdxDelete, iParm, regResult, nResultCol);
drhc926afb2002-06-20 03:38:26 +00001274 break;
1275 }
drh781def22014-01-22 13:35:53 +00001276#endif /* SQLITE_OMIT_COMPOUND_SELECT */
danielk19775338a5f2005-01-20 13:03:10 +00001277
1278 /* Store the result as data using a unique key.
1279 */
drh8e1ee882014-03-21 19:56:09 +00001280 case SRT_Fifo:
1281 case SRT_DistFifo:
danielk19775338a5f2005-01-20 13:03:10 +00001282 case SRT_Table:
drhb9bb7c12006-06-11 23:41:55 +00001283 case SRT_EphemTab: {
drhfd0a2f92014-03-24 18:08:15 +00001284 int r1 = sqlite3GetTempRange(pParse, nPrefixReg+1);
drh373cc2d2009-05-17 02:06:14 +00001285 testcase( eDest==SRT_Table );
1286 testcase( eDest==SRT_EphemTab );
drhe2248cf2015-05-22 17:29:27 +00001287 testcase( eDest==SRT_Fifo );
1288 testcase( eDest==SRT_DistFifo );
drhfd0a2f92014-03-24 18:08:15 +00001289 sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1+nPrefixReg);
dan8ce71842014-01-14 20:14:09 +00001290#ifndef SQLITE_OMIT_CTE
drh8e1ee882014-03-21 19:56:09 +00001291 if( eDest==SRT_DistFifo ){
1292 /* If the destination is DistFifo, then cursor (iParm+1) is open
dan8ce71842014-01-14 20:14:09 +00001293 ** on an ephemeral index. If the current row is already present
1294 ** in the index, do not write it to the output. If not, add the
1295 ** current row to the index and proceed with writing it to the
1296 ** output table as well. */
1297 int addr = sqlite3VdbeCurrentAddr(v) + 4;
drh38b41492015-06-08 15:08:15 +00001298 sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, addr, r1, 0);
1299 VdbeCoverage(v);
drh9b4eaeb2016-11-09 00:10:33 +00001300 sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm+1, r1,regResult,nResultCol);
drh079a3072014-03-19 14:10:55 +00001301 assert( pSort==0 );
dan8ce71842014-01-14 20:14:09 +00001302 }
1303#endif
drh079a3072014-03-19 14:10:55 +00001304 if( pSort ){
drhbbd4ae52018-04-30 19:32:49 +00001305 assert( regResult==regOrig );
1306 pushOntoSorter(pParse, pSort, p, r1+nPrefixReg, regOrig, 1, nPrefixReg);
danielk19775338a5f2005-01-20 13:03:10 +00001307 }else{
drhb7654112008-01-12 12:48:07 +00001308 int r2 = sqlite3GetTempReg(pParse);
1309 sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2);
1310 sqlite3VdbeAddOp3(v, OP_Insert, iParm, r1, r2);
1311 sqlite3VdbeChangeP5(v, OPFLAG_APPEND);
1312 sqlite3ReleaseTempReg(pParse, r2);
danielk19775338a5f2005-01-20 13:03:10 +00001313 }
drhfd0a2f92014-03-24 18:08:15 +00001314 sqlite3ReleaseTempRange(pParse, r1, nPrefixReg+1);
danielk19775338a5f2005-01-20 13:03:10 +00001315 break;
1316 }
drh5974a302000-06-07 14:42:26 +00001317
dan9ed322d2020-04-29 17:41:29 +00001318 case SRT_Upfrom: {
danf2972b62020-04-29 20:11:01 +00001319 if( pSort ){
1320 pushOntoSorter(
1321 pParse, pSort, p, regResult, regOrig, nResultCol, nPrefixReg);
drh2add24c2020-07-23 14:12:47 +00001322 }else{
danf2972b62020-04-29 20:11:01 +00001323 int i2 = pDest->iSDParm2;
1324 int r1 = sqlite3GetTempReg(pParse);
drh09cf5692020-07-20 18:07:35 +00001325
1326 /* If the UPDATE FROM join is an aggregate that matches no rows, it
1327 ** might still be trying to return one row, because that is what
1328 ** aggregates do. Don't record that empty row in the output table. */
1329 sqlite3VdbeAddOp2(v, OP_IsNull, regResult, iBreak); VdbeCoverage(v);
1330
1331 sqlite3VdbeAddOp3(v, OP_MakeRecord,
1332 regResult+(i2<0), nResultCol-(i2<0), r1);
danf2972b62020-04-29 20:11:01 +00001333 if( i2<0 ){
1334 sqlite3VdbeAddOp3(v, OP_Insert, iParm, r1, regResult);
1335 }else{
1336 sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regResult, i2);
1337 }
dan9ed322d2020-04-29 17:41:29 +00001338 }
1339 break;
1340 }
1341
1342#ifndef SQLITE_OMIT_SUBQUERY
drhc926afb2002-06-20 03:38:26 +00001343 /* If we are creating a set for an "expr IN (SELECT ...)" construct,
1344 ** then there should be a single item on the stack. Write this
1345 ** item into the set table with bogus data.
1346 */
1347 case SRT_Set: {
drh079a3072014-03-19 14:10:55 +00001348 if( pSort ){
drhde941c62005-08-28 01:34:21 +00001349 /* At first glance you would think we could optimize out the
1350 ** ORDER BY in this case since the order of entries in the set
1351 ** does not matter. But there might be a LIMIT clause, in which
1352 ** case the order does matter */
dan51d82d12016-08-02 18:50:15 +00001353 pushOntoSorter(
dan9af90b72016-11-11 18:08:59 +00001354 pParse, pSort, p, regResult, regOrig, nResultCol, nPrefixReg);
drhc926afb2002-06-20 03:38:26 +00001355 }else{
drhb7654112008-01-12 12:48:07 +00001356 int r1 = sqlite3GetTempReg(pParse);
dan9ed322d2020-04-29 17:41:29 +00001357 assert( sqlite3Strlen30(pDest->zAffSdst)==nResultCol );
1358 sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult, nResultCol,
1359 r1, pDest->zAffSdst, nResultCol);
1360 sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regResult, nResultCol);
drhb7654112008-01-12 12:48:07 +00001361 sqlite3ReleaseTempReg(pParse, r1);
drhc926afb2002-06-20 03:38:26 +00001362 }
1363 break;
1364 }
drh22827922000-06-06 17:27:05 +00001365
dan9ed322d2020-04-29 17:41:29 +00001366
drh504b6982006-01-22 21:52:56 +00001367 /* If any row exist in the result set, record that fact and abort.
drhec7429a2005-10-06 16:53:14 +00001368 */
1369 case SRT_Exists: {
drh4c583122008-01-04 22:01:03 +00001370 sqlite3VdbeAddOp2(v, OP_Integer, 1, iParm);
drhec7429a2005-10-06 16:53:14 +00001371 /* The LIMIT clause will terminate the loop for us */
1372 break;
1373 }
1374
drhc926afb2002-06-20 03:38:26 +00001375 /* If this is a scalar select that is part of an expression, then
dan870a0702016-08-01 16:37:43 +00001376 ** store the results in the appropriate memory cell or array of
1377 ** memory cells and break out of the scan loop.
drhc926afb2002-06-20 03:38:26 +00001378 */
1379 case SRT_Mem: {
drh079a3072014-03-19 14:10:55 +00001380 if( pSort ){
dan257c13f2016-11-10 20:14:06 +00001381 assert( nResultCol<=pDest->nSdst );
dan870a0702016-08-01 16:37:43 +00001382 pushOntoSorter(
dan9af90b72016-11-11 18:08:59 +00001383 pParse, pSort, p, regResult, regOrig, nResultCol, nPrefixReg);
drhc926afb2002-06-20 03:38:26 +00001384 }else{
dan257c13f2016-11-10 20:14:06 +00001385 assert( nResultCol==pDest->nSdst );
drh53932ce2014-08-29 12:29:39 +00001386 assert( regResult==iParm );
drhec7429a2005-10-06 16:53:14 +00001387 /* The LIMIT clause will jump out of the loop for us */
drhc926afb2002-06-20 03:38:26 +00001388 }
1389 break;
1390 }
danielk197793758c82005-01-21 08:13:14 +00001391#endif /* #ifndef SQLITE_OMIT_SUBQUERY */
drh22827922000-06-06 17:27:05 +00001392
drh81cf13e2014-02-07 18:27:53 +00001393 case SRT_Coroutine: /* Send data to a co-routine */
1394 case SRT_Output: { /* Return the results */
drh373cc2d2009-05-17 02:06:14 +00001395 testcase( eDest==SRT_Coroutine );
1396 testcase( eDest==SRT_Output );
drh079a3072014-03-19 14:10:55 +00001397 if( pSort ){
dan9af90b72016-11-11 18:08:59 +00001398 pushOntoSorter(pParse, pSort, p, regResult, regOrig, nResultCol,
drh5579d592015-08-26 14:01:41 +00001399 nPrefixReg);
drhe00ee6e2008-06-20 15:24:01 +00001400 }else if( eDest==SRT_Coroutine ){
drh2b596da2012-07-23 21:43:19 +00001401 sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm);
drhc182d162005-08-14 20:47:16 +00001402 }else{
drh340309f2014-01-22 00:23:49 +00001403 sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, nResultCol);
drhac82fcf2002-09-08 17:23:41 +00001404 }
drh142e30d2002-08-28 03:00:58 +00001405 break;
1406 }
1407
drhfe1c6bb2014-01-22 17:28:35 +00001408#ifndef SQLITE_OMIT_CTE
1409 /* Write the results into a priority queue that is order according to
1410 ** pDest->pOrderBy (in pSO). pDest->iSDParm (in iParm) is the cursor for an
1411 ** index with pSO->nExpr+2 columns. Build a key using pSO for the first
1412 ** pSO->nExpr columns, then make sure all keys are unique by adding a
1413 ** final OP_Sequence column. The last column is the record as a blob.
1414 */
1415 case SRT_DistQueue:
1416 case SRT_Queue: {
1417 int nKey;
1418 int r1, r2, r3;
1419 int addrTest = 0;
1420 ExprList *pSO;
1421 pSO = pDest->pOrderBy;
1422 assert( pSO );
1423 nKey = pSO->nExpr;
1424 r1 = sqlite3GetTempReg(pParse);
1425 r2 = sqlite3GetTempRange(pParse, nKey+2);
1426 r3 = r2+nKey+1;
drhfe1c6bb2014-01-22 17:28:35 +00001427 if( eDest==SRT_DistQueue ){
1428 /* If the destination is DistQueue, then cursor (iParm+1) is open
1429 ** on a second ephemeral index that holds all values every previously
drh7e4efae2014-02-26 21:35:31 +00001430 ** added to the queue. */
1431 addrTest = sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, 0,
1432 regResult, nResultCol);
drh688852a2014-02-17 22:40:43 +00001433 VdbeCoverage(v);
drh7e4efae2014-02-26 21:35:31 +00001434 }
1435 sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r3);
1436 if( eDest==SRT_DistQueue ){
drhfe1c6bb2014-01-22 17:28:35 +00001437 sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r3);
dancfe24582014-01-22 19:23:30 +00001438 sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
drhfe1c6bb2014-01-22 17:28:35 +00001439 }
1440 for(i=0; i<nKey; i++){
1441 sqlite3VdbeAddOp2(v, OP_SCopy,
1442 regResult + pSO->a[i].u.x.iOrderByCol - 1,
1443 r2+i);
1444 }
1445 sqlite3VdbeAddOp2(v, OP_Sequence, iParm, r2+nKey);
1446 sqlite3VdbeAddOp3(v, OP_MakeRecord, r2, nKey+2, r1);
drh9b4eaeb2016-11-09 00:10:33 +00001447 sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, r2, nKey+2);
drhfe1c6bb2014-01-22 17:28:35 +00001448 if( addrTest ) sqlite3VdbeJumpHere(v, addrTest);
1449 sqlite3ReleaseTempReg(pParse, r1);
1450 sqlite3ReleaseTempRange(pParse, r2, nKey+2);
1451 break;
1452 }
1453#endif /* SQLITE_OMIT_CTE */
1454
1455
1456
danielk19776a67fe82005-02-04 04:07:16 +00001457#if !defined(SQLITE_OMIT_TRIGGER)
drhc926afb2002-06-20 03:38:26 +00001458 /* Discard the results. This is used for SELECT statements inside
1459 ** the body of a TRIGGER. The purpose of such selects is to call
1460 ** user-defined functions that have side effects. We do not care
1461 ** about the actual results of the select.
1462 */
drhc926afb2002-06-20 03:38:26 +00001463 default: {
drhf46f9052002-06-22 02:33:38 +00001464 assert( eDest==SRT_Discard );
drhc926afb2002-06-20 03:38:26 +00001465 break;
1466 }
danielk197793758c82005-01-21 08:13:14 +00001467#endif
drh82c3d632000-06-06 21:56:07 +00001468 }
drhec7429a2005-10-06 16:53:14 +00001469
drh5e87be82010-10-06 18:55:37 +00001470 /* Jump to the end of the loop if the LIMIT is reached. Except, if
1471 ** there is a sorter, in which case the sorter has already limited
1472 ** the output for us.
drhec7429a2005-10-06 16:53:14 +00001473 */
drh079a3072014-03-19 14:10:55 +00001474 if( pSort==0 && p->iLimit ){
drh16897072015-03-07 00:57:37 +00001475 sqlite3VdbeAddOp2(v, OP_DecrJumpZero, p->iLimit, iBreak); VdbeCoverage(v);
drhec7429a2005-10-06 16:53:14 +00001476 }
drh82c3d632000-06-06 21:56:07 +00001477}
1478
1479/*
drhad124322013-10-23 13:30:58 +00001480** Allocate a KeyInfo object sufficient for an index of N key columns and
1481** X extra columns.
drh323df792013-08-05 19:11:29 +00001482*/
drhad124322013-10-23 13:30:58 +00001483KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){
drhd4ab0032017-05-31 13:45:59 +00001484 int nExtra = (N+X)*(sizeof(CollSeq*)+1) - sizeof(CollSeq*);
drhd8e4b132016-10-01 19:21:56 +00001485 KeyInfo *p = sqlite3DbMallocRawNN(db, sizeof(KeyInfo) + nExtra);
drh323df792013-08-05 19:11:29 +00001486 if( p ){
dan6e118922019-08-12 16:36:38 +00001487 p->aSortFlags = (u8*)&p->aColl[N+X];
drha485ad12017-08-02 22:43:14 +00001488 p->nKeyField = (u16)N;
1489 p->nAllField = (u16)(N+X);
drh323df792013-08-05 19:11:29 +00001490 p->enc = ENC(db);
1491 p->db = db;
drh2ec2fb22013-11-06 19:59:23 +00001492 p->nRef = 1;
drhc263f7c2016-01-18 13:18:54 +00001493 memset(&p[1], 0, nExtra);
drh2ec2fb22013-11-06 19:59:23 +00001494 }else{
drh3cdb1392022-01-24 12:48:54 +00001495 return (KeyInfo*)sqlite3OomFault(db);
drh323df792013-08-05 19:11:29 +00001496 }
1497 return p;
1498}
1499
1500/*
drh2ec2fb22013-11-06 19:59:23 +00001501** Deallocate a KeyInfo object
1502*/
1503void sqlite3KeyInfoUnref(KeyInfo *p){
1504 if( p ){
drh41ce47c2022-08-22 02:00:26 +00001505 assert( p->db!=0 );
drh2ec2fb22013-11-06 19:59:23 +00001506 assert( p->nRef>0 );
drh2ec2fb22013-11-06 19:59:23 +00001507 p->nRef--;
drh41ce47c2022-08-22 02:00:26 +00001508 if( p->nRef==0 ) sqlite3DbNNFreeNN(p->db, p);
drh2ec2fb22013-11-06 19:59:23 +00001509 }
1510}
1511
1512/*
1513** Make a new pointer to a KeyInfo object
1514*/
1515KeyInfo *sqlite3KeyInfoRef(KeyInfo *p){
1516 if( p ){
1517 assert( p->nRef>0 );
1518 p->nRef++;
1519 }
1520 return p;
1521}
1522
1523#ifdef SQLITE_DEBUG
1524/*
1525** Return TRUE if a KeyInfo object can be change. The KeyInfo object
1526** can only be changed if this is just a single reference to the object.
1527**
1528** This routine is used only inside of assert() statements.
1529*/
1530int sqlite3KeyInfoIsWriteable(KeyInfo *p){ return p->nRef==1; }
1531#endif /* SQLITE_DEBUG */
1532
1533/*
drhdece1a82005-08-31 18:20:00 +00001534** Given an expression list, generate a KeyInfo structure that records
1535** the collating sequence for each expression in that expression list.
1536**
drh0342b1f2005-09-01 03:07:44 +00001537** If the ExprList is an ORDER BY or GROUP BY clause then the resulting
1538** KeyInfo structure is appropriate for initializing a virtual index to
1539** implement that clause. If the ExprList is the result set of a SELECT
1540** then the KeyInfo structure is appropriate for initializing a virtual
1541** index to implement a DISTINCT test.
1542**
peter.d.reid60ec9142014-09-06 16:39:46 +00001543** Space to hold the KeyInfo structure is obtained from malloc. The calling
drhdece1a82005-08-31 18:20:00 +00001544** function is responsible for seeing that this structure is eventually
drh2ec2fb22013-11-06 19:59:23 +00001545** freed.
drhdece1a82005-08-31 18:20:00 +00001546*/
danf9eae182018-05-21 19:45:11 +00001547KeyInfo *sqlite3KeyInfoFromExprList(
drh079a3072014-03-19 14:10:55 +00001548 Parse *pParse, /* Parsing context */
1549 ExprList *pList, /* Form the KeyInfo object from this ExprList */
1550 int iStart, /* Begin with this column of pList */
1551 int nExtra /* Add this many extra columns to the end */
1552){
drhdece1a82005-08-31 18:20:00 +00001553 int nExpr;
1554 KeyInfo *pInfo;
1555 struct ExprList_item *pItem;
drh323df792013-08-05 19:11:29 +00001556 sqlite3 *db = pParse->db;
drhdece1a82005-08-31 18:20:00 +00001557 int i;
1558
1559 nExpr = pList->nExpr;
drh3f39bcf2015-01-19 20:59:34 +00001560 pInfo = sqlite3KeyInfoAlloc(db, nExpr-iStart, nExtra+1);
drhdece1a82005-08-31 18:20:00 +00001561 if( pInfo ){
drh2ec2fb22013-11-06 19:59:23 +00001562 assert( sqlite3KeyInfoIsWriteable(pInfo) );
drh6284db92014-03-19 23:24:49 +00001563 for(i=iStart, pItem=pList->a+iStart; i<nExpr; i++, pItem++){
drh70efa842017-09-28 01:58:23 +00001564 pInfo->aColl[i-iStart] = sqlite3ExprNNCollSeq(pParse, pItem->pExpr);
drhd88fd532022-05-02 20:49:30 +00001565 pInfo->aSortFlags[i-iStart] = pItem->fg.sortFlags;
drhdece1a82005-08-31 18:20:00 +00001566 }
1567 }
1568 return pInfo;
1569}
1570
dan7f61e922010-11-11 16:46:40 +00001571/*
1572** Name of the connection operator, used for error messages.
1573*/
drhaae0f742021-03-04 16:03:32 +00001574const char *sqlite3SelectOpName(int id){
dan7f61e922010-11-11 16:46:40 +00001575 char *z;
1576 switch( id ){
1577 case TK_ALL: z = "UNION ALL"; break;
1578 case TK_INTERSECT: z = "INTERSECT"; break;
1579 case TK_EXCEPT: z = "EXCEPT"; break;
1580 default: z = "UNION"; break;
1581 }
1582 return z;
1583}
dan7f61e922010-11-11 16:46:40 +00001584
dan2ce22452010-11-08 19:01:16 +00001585#ifndef SQLITE_OMIT_EXPLAIN
dan17c0bc02010-11-09 17:35:19 +00001586/*
1587** Unless an "EXPLAIN QUERY PLAN" command is being processed, this function
1588** is a no-op. Otherwise, it adds a single row of output to the EQP result,
1589** where the caption is of the form:
1590**
1591** "USE TEMP B-TREE FOR xxx"
1592**
1593** where xxx is one of "DISTINCT", "ORDER BY" or "GROUP BY". Exactly which
1594** is determined by the zUsage argument.
1595*/
dan2ce22452010-11-08 19:01:16 +00001596static void explainTempTable(Parse *pParse, const char *zUsage){
drhe2ca99c2018-05-02 00:33:43 +00001597 ExplainQueryPlan((pParse, 0, "USE TEMP B-TREE FOR %s", zUsage));
dan2ce22452010-11-08 19:01:16 +00001598}
dan17c0bc02010-11-09 17:35:19 +00001599
1600/*
danbb2b4412011-04-06 17:54:31 +00001601** Assign expression b to lvalue a. A second, no-op, version of this macro
1602** is provided when SQLITE_OMIT_EXPLAIN is defined. This allows the code
1603** in sqlite3Select() to assign values to structure member variables that
1604** only exist if SQLITE_OMIT_EXPLAIN is not defined without polluting the
1605** code with #ifndef directives.
1606*/
1607# define explainSetInteger(a, b) a = b
1608
1609#else
1610/* No-op versions of the explainXXX() functions and macros. */
1611# define explainTempTable(y,z)
1612# define explainSetInteger(y,z)
1613#endif
1614
drhdece1a82005-08-31 18:20:00 +00001615
1616/*
drhd8bc7082000-06-07 23:51:50 +00001617** If the inner loop was generated using a non-null pOrderBy argument,
1618** then the results were placed in a sorter. After the loop is terminated
1619** we need to run the sorter and output the results. The following
1620** routine generates the code needed to do that.
1621*/
drhc926afb2002-06-20 03:38:26 +00001622static void generateSortTail(
danielk19776c8c8ce2008-01-02 16:27:09 +00001623 Parse *pParse, /* Parsing context */
1624 Select *p, /* The SELECT statement */
drh079a3072014-03-19 14:10:55 +00001625 SortCtx *pSort, /* Information on the ORDER BY clause */
danielk19776c8c8ce2008-01-02 16:27:09 +00001626 int nColumn, /* Number of columns of data */
1627 SelectDest *pDest /* Write the sorted results here */
drhc926afb2002-06-20 03:38:26 +00001628){
drhddba0c22014-03-18 20:33:42 +00001629 Vdbe *v = pParse->pVdbe; /* The prepared statement */
drha04a8be2016-01-13 17:50:10 +00001630 int addrBreak = pSort->labelDone; /* Jump here to exit loop */
drhec4ccdb2018-12-29 02:26:59 +00001631 int addrContinue = sqlite3VdbeMakeLabel(pParse);/* Jump here for next cycle */
dan24e25d32018-04-14 18:46:20 +00001632 int addr; /* Top of output loop. Jump for Next. */
drh079a3072014-03-19 14:10:55 +00001633 int addrOnce = 0;
drh0342b1f2005-09-01 03:07:44 +00001634 int iTab;
drh079a3072014-03-19 14:10:55 +00001635 ExprList *pOrderBy = pSort->pOrderBy;
danielk19776c8c8ce2008-01-02 16:27:09 +00001636 int eDest = pDest->eDest;
drh2b596da2012-07-23 21:43:19 +00001637 int iParm = pDest->iSDParm;
drh2d401ab2008-01-10 23:50:11 +00001638 int regRow;
1639 int regRowid;
dan257c13f2016-11-10 20:14:06 +00001640 int iCol;
dan24e25d32018-04-14 18:46:20 +00001641 int nKey; /* Number of key columns in sorter record */
drhf45f2322014-03-23 17:45:03 +00001642 int iSortTab; /* Sorter cursor to read from */
drhf45f2322014-03-23 17:45:03 +00001643 int i;
dan78d58432014-03-25 15:04:07 +00001644 int bSeq; /* True if sorter record includes seq. no. */
dan24e25d32018-04-14 18:46:20 +00001645 int nRefKey = 0;
drh70f624c2014-03-24 01:43:50 +00001646 struct ExprList_item *aOutEx = p->pEList->a;
drh2d401ab2008-01-10 23:50:11 +00001647
drha04a8be2016-01-13 17:50:10 +00001648 assert( addrBreak<0 );
drh079a3072014-03-19 14:10:55 +00001649 if( pSort->labelBkOut ){
1650 sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut);
drh076e85f2015-09-03 13:46:12 +00001651 sqlite3VdbeGoto(v, addrBreak);
drh079a3072014-03-19 14:10:55 +00001652 sqlite3VdbeResolveLabel(v, pSort->labelBkOut);
drh079a3072014-03-19 14:10:55 +00001653 }
dan24e25d32018-04-14 18:46:20 +00001654
1655#ifdef SQLITE_ENABLE_SORTER_REFERENCES
1656 /* Open any cursors needed for sorter-reference expressions */
1657 for(i=0; i<pSort->nDefer; i++){
1658 Table *pTab = pSort->aDefer[i].pTab;
1659 int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema);
1660 sqlite3OpenTable(pParse, pSort->aDefer[i].iCsr, iDb, pTab, OP_OpenRead);
1661 nRefKey = MAX(nRefKey, pSort->aDefer[i].nKey);
1662 }
1663#endif
1664
drh079a3072014-03-19 14:10:55 +00001665 iTab = pSort->iECursor;
dan71c57db2016-07-09 20:23:55 +00001666 if( eDest==SRT_Output || eDest==SRT_Coroutine || eDest==SRT_Mem ){
drh8efc6a82022-01-12 20:31:14 +00001667 if( eDest==SRT_Mem && p->iOffset ){
1668 sqlite3VdbeAddOp2(v, OP_Null, 0, pDest->iSdst);
1669 }
drh3e9ca092009-09-08 01:14:48 +00001670 regRowid = 0;
drhf45f2322014-03-23 17:45:03 +00001671 regRow = pDest->iSdst;
drhed24da42016-09-06 14:37:05 +00001672 }else{
dan51d82d12016-08-02 18:50:15 +00001673 regRowid = sqlite3GetTempReg(pParse);
drh375afb82019-01-16 19:26:31 +00001674 if( eDest==SRT_EphemTab || eDest==SRT_Table ){
1675 regRow = sqlite3GetTempReg(pParse);
1676 nColumn = 0;
1677 }else{
1678 regRow = sqlite3GetTempRange(pParse, nColumn);
1679 }
drhcdd536f2006-03-17 00:04:03 +00001680 }
drh079a3072014-03-19 14:10:55 +00001681 nKey = pOrderBy->nExpr - pSort->nOBSat;
1682 if( pSort->sortFlags & SORTFLAG_UseSorter ){
drhc2bb3282011-09-04 01:11:46 +00001683 int regSortOut = ++pParse->nMem;
drhf45f2322014-03-23 17:45:03 +00001684 iSortTab = pParse->nTab++;
drh83553ee2014-03-24 02:20:53 +00001685 if( pSort->labelBkOut ){
drh511f9e82016-09-22 18:53:13 +00001686 addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
drh83553ee2014-03-24 02:20:53 +00001687 }
dan24e25d32018-04-14 18:46:20 +00001688 sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut,
1689 nKey+1+nColumn+nRefKey);
drh079a3072014-03-19 14:10:55 +00001690 if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce);
drhc6aff302011-09-01 15:32:47 +00001691 addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak);
drh688852a2014-02-17 22:40:43 +00001692 VdbeCoverage(v);
drhbffd5c12022-08-04 17:15:00 +00001693 assert( p->iLimit==0 && p->iOffset==0 );
drh6cf4a7d2014-10-13 13:00:58 +00001694 sqlite3VdbeAddOp3(v, OP_SorterData, iTab, regSortOut, iSortTab);
dan78d58432014-03-25 15:04:07 +00001695 bSeq = 0;
drhc6aff302011-09-01 15:32:47 +00001696 }else{
drh688852a2014-02-17 22:40:43 +00001697 addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); VdbeCoverage(v);
drhaa9ce702014-01-22 18:07:04 +00001698 codeOffset(v, p->iOffset, addrContinue);
drhf45f2322014-03-23 17:45:03 +00001699 iSortTab = iTab;
dan78d58432014-03-25 15:04:07 +00001700 bSeq = 1;
drhbffd5c12022-08-04 17:15:00 +00001701 if( p->iOffset>0 ){
1702 sqlite3VdbeAddOp2(v, OP_AddImm, p->iLimit, -1);
1703 }
drhf45f2322014-03-23 17:45:03 +00001704 }
dand6189ea2018-04-11 14:11:53 +00001705 for(i=0, iCol=nKey+bSeq-1; i<nColumn; i++){
dan24e25d32018-04-14 18:46:20 +00001706#ifdef SQLITE_ENABLE_SORTER_REFERENCES
drhd88fd532022-05-02 20:49:30 +00001707 if( aOutEx[i].fg.bSorterRef ) continue;
dan24e25d32018-04-14 18:46:20 +00001708#endif
drh9f895232018-01-24 20:42:42 +00001709 if( aOutEx[i].u.x.iOrderByCol==0 ) iCol++;
1710 }
dan24e25d32018-04-14 18:46:20 +00001711#ifdef SQLITE_ENABLE_SORTER_REFERENCES
1712 if( pSort->nDefer ){
1713 int iKey = iCol+1;
1714 int regKey = sqlite3GetTempRange(pParse, nRefKey);
1715
1716 for(i=0; i<pSort->nDefer; i++){
1717 int iCsr = pSort->aDefer[i].iCsr;
1718 Table *pTab = pSort->aDefer[i].pTab;
1719 int nKey = pSort->aDefer[i].nKey;
1720
1721 sqlite3VdbeAddOp1(v, OP_NullRow, iCsr);
1722 if( HasRowid(pTab) ){
1723 sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iKey++, regKey);
1724 sqlite3VdbeAddOp3(v, OP_SeekRowid, iCsr,
1725 sqlite3VdbeCurrentAddr(v)+1, regKey);
1726 }else{
dan24e25d32018-04-14 18:46:20 +00001727 int k;
1728 int iJmp;
drh7590d092018-04-18 14:04:19 +00001729 assert( sqlite3PrimaryKeyIndex(pTab)->nKeyCol==nKey );
dan24e25d32018-04-14 18:46:20 +00001730 for(k=0; k<nKey; k++){
1731 sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iKey++, regKey+k);
1732 }
1733 iJmp = sqlite3VdbeCurrentAddr(v);
1734 sqlite3VdbeAddOp4Int(v, OP_SeekGE, iCsr, iJmp+2, regKey, nKey);
1735 sqlite3VdbeAddOp4Int(v, OP_IdxLE, iCsr, iJmp+3, regKey, nKey);
1736 sqlite3VdbeAddOp1(v, OP_NullRow, iCsr);
1737 }
dan257c13f2016-11-10 20:14:06 +00001738 }
dan24e25d32018-04-14 18:46:20 +00001739 sqlite3ReleaseTempRange(pParse, regKey, nRefKey);
1740 }
1741#endif
1742 for(i=nColumn-1; i>=0; i--){
1743#ifdef SQLITE_ENABLE_SORTER_REFERENCES
drhd88fd532022-05-02 20:49:30 +00001744 if( aOutEx[i].fg.bSorterRef ){
dan24e25d32018-04-14 18:46:20 +00001745 sqlite3ExprCode(pParse, aOutEx[i].pExpr, regRow+i);
1746 }else
1747#endif
1748 {
1749 int iRead;
1750 if( aOutEx[i].u.x.iOrderByCol ){
1751 iRead = aOutEx[i].u.x.iOrderByCol-1;
1752 }else{
1753 iRead = iCol--;
1754 }
1755 sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iRead, regRow+i);
drhcbb9da32019-12-12 22:11:33 +00001756 VdbeComment((v, "%s", aOutEx[i].zEName));
dan24e25d32018-04-14 18:46:20 +00001757 }
drhc6aff302011-09-01 15:32:47 +00001758 }
drhc926afb2002-06-20 03:38:26 +00001759 switch( eDest ){
danac56ab72017-01-05 17:23:11 +00001760 case SRT_Table:
drhb9bb7c12006-06-11 23:41:55 +00001761 case SRT_EphemTab: {
drh375afb82019-01-16 19:26:31 +00001762 sqlite3VdbeAddOp3(v, OP_Column, iSortTab, nKey+bSeq, regRow);
drh2d401ab2008-01-10 23:50:11 +00001763 sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, regRowid);
1764 sqlite3VdbeAddOp3(v, OP_Insert, iParm, regRow, regRowid);
1765 sqlite3VdbeChangeP5(v, OPFLAG_APPEND);
drhc926afb2002-06-20 03:38:26 +00001766 break;
1767 }
danielk197793758c82005-01-21 08:13:14 +00001768#ifndef SQLITE_OMIT_SUBQUERY
drhc926afb2002-06-20 03:38:26 +00001769 case SRT_Set: {
dan71c57db2016-07-09 20:23:55 +00001770 assert( nColumn==sqlite3Strlen30(pDest->zAffSdst) );
1771 sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, nColumn, regRowid,
dan553168c2016-08-01 20:14:31 +00001772 pDest->zAffSdst, nColumn);
drh9b4eaeb2016-11-09 00:10:33 +00001773 sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, regRowid, regRow, nColumn);
drhc926afb2002-06-20 03:38:26 +00001774 break;
1775 }
1776 case SRT_Mem: {
drhec7429a2005-10-06 16:53:14 +00001777 /* The LIMIT clause will terminate the loop for us */
drhc926afb2002-06-20 03:38:26 +00001778 break;
1779 }
danielk197793758c82005-01-21 08:13:14 +00001780#endif
danf2972b62020-04-29 20:11:01 +00001781 case SRT_Upfrom: {
1782 int i2 = pDest->iSDParm2;
1783 int r1 = sqlite3GetTempReg(pParse);
1784 sqlite3VdbeAddOp3(v, OP_MakeRecord,regRow+(i2<0),nColumn-(i2<0),r1);
1785 if( i2<0 ){
1786 sqlite3VdbeAddOp3(v, OP_Insert, iParm, r1, regRow);
1787 }else{
1788 sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regRow, i2);
1789 }
1790 break;
1791 }
drh373cc2d2009-05-17 02:06:14 +00001792 default: {
drh373cc2d2009-05-17 02:06:14 +00001793 assert( eDest==SRT_Output || eDest==SRT_Coroutine );
drh1c767f02009-01-09 02:49:31 +00001794 testcase( eDest==SRT_Output );
1795 testcase( eDest==SRT_Coroutine );
drh7d10d5a2008-08-20 16:35:10 +00001796 if( eDest==SRT_Output ){
drh2b596da2012-07-23 21:43:19 +00001797 sqlite3VdbeAddOp2(v, OP_ResultRow, pDest->iSdst, nColumn);
drha9671a22008-07-08 23:40:20 +00001798 }else{
drh2b596da2012-07-23 21:43:19 +00001799 sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm);
drhce665cf2004-05-21 03:01:58 +00001800 }
drhac82fcf2002-09-08 17:23:41 +00001801 break;
1802 }
drhc926afb2002-06-20 03:38:26 +00001803 }
drhf45f2322014-03-23 17:45:03 +00001804 if( regRowid ){
dan51d82d12016-08-02 18:50:15 +00001805 if( eDest==SRT_Set ){
1806 sqlite3ReleaseTempRange(pParse, regRow, nColumn);
1807 }else{
1808 sqlite3ReleaseTempReg(pParse, regRow);
1809 }
drhf45f2322014-03-23 17:45:03 +00001810 sqlite3ReleaseTempReg(pParse, regRowid);
1811 }
drhec7429a2005-10-06 16:53:14 +00001812 /* The bottom of the loop
1813 */
drhdc5ea5c2008-12-10 17:19:59 +00001814 sqlite3VdbeResolveLabel(v, addrContinue);
drh079a3072014-03-19 14:10:55 +00001815 if( pSort->sortFlags & SORTFLAG_UseSorter ){
drh688852a2014-02-17 22:40:43 +00001816 sqlite3VdbeAddOp2(v, OP_SorterNext, iTab, addr); VdbeCoverage(v);
drhc6aff302011-09-01 15:32:47 +00001817 }else{
drh688852a2014-02-17 22:40:43 +00001818 sqlite3VdbeAddOp2(v, OP_Next, iTab, addr); VdbeCoverage(v);
drhc6aff302011-09-01 15:32:47 +00001819 }
drh079a3072014-03-19 14:10:55 +00001820 if( pSort->regReturn ) sqlite3VdbeAddOp1(v, OP_Return, pSort->regReturn);
drhdc5ea5c2008-12-10 17:19:59 +00001821 sqlite3VdbeResolveLabel(v, addrBreak);
drhd8bc7082000-06-07 23:51:50 +00001822}
1823
1824/*
danielk1977517eb642004-06-07 10:00:31 +00001825** Return a pointer to a string containing the 'declaration type' of the
1826** expression pExpr. The string may be treated as static by the caller.
drhe78e8282003-01-19 03:59:45 +00001827**
danielk1977955de522006-02-10 02:27:42 +00001828** The declaration type is the exact datatype definition extracted from the
1829** original CREATE TABLE statement if the expression is a column. The
1830** declaration type for a ROWID field is INTEGER. Exactly when an expression
1831** is considered a column can be complex in the presence of subqueries. The
1832** result-set expression in all of the following SELECT statements is
1833** considered a column by this function.
1834**
1835** SELECT col FROM tbl;
1836** SELECT (SELECT col FROM tbl;
1837** SELECT (SELECT col FROM tbl);
1838** SELECT abc FROM (SELECT col AS abc FROM tbl);
danielk1977517eb642004-06-07 10:00:31 +00001839**
danielk1977955de522006-02-10 02:27:42 +00001840** The declaration type for any expression other than a column is NULL.
drh5f3e5e72013-10-08 20:01:35 +00001841**
1842** This routine has either 3 or 6 parameters depending on whether or not
1843** the SQLITE_ENABLE_COLUMN_METADATA compile-time option is used.
danielk1977517eb642004-06-07 10:00:31 +00001844*/
drh5f3e5e72013-10-08 20:01:35 +00001845#ifdef SQLITE_ENABLE_COLUMN_METADATA
drhcafc2f72017-10-03 03:01:09 +00001846# define columnType(A,B,C,D,E) columnTypeImpl(A,B,C,D,E)
drhb121dd12015-06-06 18:30:17 +00001847#else /* if !defined(SQLITE_ENABLE_COLUMN_METADATA) */
drhcafc2f72017-10-03 03:01:09 +00001848# define columnType(A,B,C,D,E) columnTypeImpl(A,B)
drhb121dd12015-06-06 18:30:17 +00001849#endif
drh5f3e5e72013-10-08 20:01:35 +00001850static const char *columnTypeImpl(
danielk1977955de522006-02-10 02:27:42 +00001851 NameContext *pNC,
drhcafc2f72017-10-03 03:01:09 +00001852#ifndef SQLITE_ENABLE_COLUMN_METADATA
1853 Expr *pExpr
1854#else
danielk1977955de522006-02-10 02:27:42 +00001855 Expr *pExpr,
drh5f3e5e72013-10-08 20:01:35 +00001856 const char **pzOrigDb,
1857 const char **pzOrigTab,
drhcafc2f72017-10-03 03:01:09 +00001858 const char **pzOrigCol
drhb121dd12015-06-06 18:30:17 +00001859#endif
danielk1977955de522006-02-10 02:27:42 +00001860){
1861 char const *zType = 0;
danielk1977517eb642004-06-07 10:00:31 +00001862 int j;
drhb121dd12015-06-06 18:30:17 +00001863#ifdef SQLITE_ENABLE_COLUMN_METADATA
1864 char const *zOrigDb = 0;
1865 char const *zOrigTab = 0;
1866 char const *zOrigCol = 0;
1867#endif
danielk19775338a5f2005-01-20 13:03:10 +00001868
drhf7ce4292015-12-02 19:46:12 +00001869 assert( pExpr!=0 );
1870 assert( pNC->pSrcList!=0 );
danielk197700e279d2004-06-21 07:36:32 +00001871 switch( pExpr->op ){
1872 case TK_COLUMN: {
danielk1977955de522006-02-10 02:27:42 +00001873 /* The expression is a column. Locate the table the column is being
1874 ** extracted from in NameContext.pSrcList. This table may be real
1875 ** database table or a subquery.
1876 */
1877 Table *pTab = 0; /* Table structure column is extracted from */
1878 Select *pS = 0; /* Select the column is extracted from */
1879 int iCol = pExpr->iColumn; /* Index of column in pTab */
dan43bc88b2009-09-10 10:15:59 +00001880 while( pNC && !pTab ){
danielk1977b3bce662005-01-29 08:32:43 +00001881 SrcList *pTabList = pNC->pSrcList;
1882 for(j=0;j<pTabList->nSrc && pTabList->a[j].iCursor!=pExpr->iTable;j++);
1883 if( j<pTabList->nSrc ){
1884 pTab = pTabList->a[j].pTab;
danielk1977955de522006-02-10 02:27:42 +00001885 pS = pTabList->a[j].pSelect;
danielk1977b3bce662005-01-29 08:32:43 +00001886 }else{
1887 pNC = pNC->pNext;
1888 }
1889 }
danielk1977955de522006-02-10 02:27:42 +00001890
dan43bc88b2009-09-10 10:15:59 +00001891 if( pTab==0 ){
drh417168a2009-09-07 18:14:02 +00001892 /* At one time, code such as "SELECT new.x" within a trigger would
1893 ** cause this condition to run. Since then, we have restructured how
1894 ** trigger code is generated and so this condition is no longer
dan43bc88b2009-09-10 10:15:59 +00001895 ** possible. However, it can still be true for statements like
1896 ** the following:
1897 **
1898 ** CREATE TABLE t1(col INTEGER);
1899 ** SELECT (SELECT t1.col) FROM FROM t1;
1900 **
1901 ** when columnType() is called on the expression "t1.col" in the
1902 ** sub-select. In this case, set the column type to NULL, even
1903 ** though it should really be "INTEGER".
1904 **
1905 ** This is not a problem, as the column type of "t1.col" is never
1906 ** used. When columnType() is called on the expression
1907 ** "(SELECT t1.col)", the correct type is returned (see the TK_SELECT
1908 ** branch below. */
drh7e627792005-04-29 02:10:00 +00001909 break;
1910 }
danielk1977955de522006-02-10 02:27:42 +00001911
drh477572b2021-10-07 20:46:29 +00001912 assert( pTab && ExprUseYTab(pExpr) && pExpr->y.pTab==pTab );
danielk1977955de522006-02-10 02:27:42 +00001913 if( pS ){
1914 /* The "table" is actually a sub-select or a view in the FROM clause
1915 ** of the SELECT statement. Return the declaration type and origin
1916 ** data for the result-set column of the sub-select.
1917 */
drh6e5020e2021-04-07 15:45:01 +00001918 if( iCol<pS->pEList->nExpr
1919#ifdef SQLITE_ALLOW_ROWID_IN_VIEW
1920 && iCol>=0
1921#else
1922 && ALWAYS(iCol>=0)
1923#endif
1924 ){
danielk1977955de522006-02-10 02:27:42 +00001925 /* If iCol is less than zero, then the expression requests the
1926 ** rowid of the sub-select or view. This expression is legal (see
1927 ** test case misc2.2.2) - it always evaluates to NULL.
1928 */
1929 NameContext sNC;
1930 Expr *p = pS->pEList->a[iCol].pExpr;
1931 sNC.pSrcList = pS->pSrc;
dan43bc88b2009-09-10 10:15:59 +00001932 sNC.pNext = pNC;
danielk1977955de522006-02-10 02:27:42 +00001933 sNC.pParse = pNC->pParse;
drhcafc2f72017-10-03 03:01:09 +00001934 zType = columnType(&sNC, p,&zOrigDb,&zOrigTab,&zOrigCol);
danielk1977955de522006-02-10 02:27:42 +00001935 }
drha78d7572017-10-03 16:57:33 +00001936 }else{
1937 /* A real table or a CTE table */
danielk1977955de522006-02-10 02:27:42 +00001938 assert( !pS );
drh5f3e5e72013-10-08 20:01:35 +00001939#ifdef SQLITE_ENABLE_COLUMN_METADATA
drha78d7572017-10-03 16:57:33 +00001940 if( iCol<0 ) iCol = pTab->iPKey;
1941 assert( iCol==XN_ROWID || (iCol>=0 && iCol<pTab->nCol) );
danielk1977955de522006-02-10 02:27:42 +00001942 if( iCol<0 ){
1943 zType = "INTEGER";
drh5f3e5e72013-10-08 20:01:35 +00001944 zOrigCol = "rowid";
danielk1977955de522006-02-10 02:27:42 +00001945 }else{
drhcf9d36d2021-08-02 18:03:43 +00001946 zOrigCol = pTab->aCol[iCol].zCnName;
drhd7564862016-03-22 20:05:09 +00001947 zType = sqlite3ColumnType(&pTab->aCol[iCol],0);
danielk1977955de522006-02-10 02:27:42 +00001948 }
drh5f3e5e72013-10-08 20:01:35 +00001949 zOrigTab = pTab->zName;
drha78d7572017-10-03 16:57:33 +00001950 if( pNC->pParse && pTab->pSchema ){
danielk1977955de522006-02-10 02:27:42 +00001951 int iDb = sqlite3SchemaToIndex(pNC->pParse->db, pTab->pSchema);
drhe59be012016-08-18 20:56:39 +00001952 zOrigDb = pNC->pParse->db->aDb[iDb].zDbSName;
danielk1977955de522006-02-10 02:27:42 +00001953 }
drh5f3e5e72013-10-08 20:01:35 +00001954#else
drha78d7572017-10-03 16:57:33 +00001955 assert( iCol==XN_ROWID || (iCol>=0 && iCol<pTab->nCol) );
drh5f3e5e72013-10-08 20:01:35 +00001956 if( iCol<0 ){
1957 zType = "INTEGER";
1958 }else{
drhd7564862016-03-22 20:05:09 +00001959 zType = sqlite3ColumnType(&pTab->aCol[iCol],0);
drh5f3e5e72013-10-08 20:01:35 +00001960 }
1961#endif
danielk197700e279d2004-06-21 07:36:32 +00001962 }
1963 break;
danielk1977517eb642004-06-07 10:00:31 +00001964 }
danielk197793758c82005-01-21 08:13:14 +00001965#ifndef SQLITE_OMIT_SUBQUERY
danielk197700e279d2004-06-21 07:36:32 +00001966 case TK_SELECT: {
danielk1977955de522006-02-10 02:27:42 +00001967 /* The expression is a sub-select. Return the declaration type and
1968 ** origin info for the single column in the result set of the SELECT
1969 ** statement.
1970 */
danielk1977b3bce662005-01-29 08:32:43 +00001971 NameContext sNC;
drha4eeccd2021-10-07 17:43:30 +00001972 Select *pS;
1973 Expr *p;
1974 assert( ExprUseXSelect(pExpr) );
1975 pS = pExpr->x.pSelect;
1976 p = pS->pEList->a[0].pExpr;
danielk1977955de522006-02-10 02:27:42 +00001977 sNC.pSrcList = pS->pSrc;
danielk1977b3bce662005-01-29 08:32:43 +00001978 sNC.pNext = pNC;
danielk1977955de522006-02-10 02:27:42 +00001979 sNC.pParse = pNC->pParse;
drhcafc2f72017-10-03 03:01:09 +00001980 zType = columnType(&sNC, p, &zOrigDb, &zOrigTab, &zOrigCol);
danielk197700e279d2004-06-21 07:36:32 +00001981 break;
danielk1977517eb642004-06-07 10:00:31 +00001982 }
danielk197793758c82005-01-21 08:13:14 +00001983#endif
danielk1977517eb642004-06-07 10:00:31 +00001984 }
drh5f3e5e72013-10-08 20:01:35 +00001985
1986#ifdef SQLITE_ENABLE_COLUMN_METADATA
1987 if( pzOrigDb ){
1988 assert( pzOrigTab && pzOrigCol );
1989 *pzOrigDb = zOrigDb;
1990 *pzOrigTab = zOrigTab;
1991 *pzOrigCol = zOrigCol;
danielk1977955de522006-02-10 02:27:42 +00001992 }
drh5f3e5e72013-10-08 20:01:35 +00001993#endif
danielk1977517eb642004-06-07 10:00:31 +00001994 return zType;
1995}
1996
1997/*
1998** Generate code that will tell the VDBE the declaration types of columns
1999** in the result set.
drhfcb78a42003-01-18 20:11:05 +00002000*/
2001static void generateColumnTypes(
2002 Parse *pParse, /* Parser context */
drhfcb78a42003-01-18 20:11:05 +00002003 SrcList *pTabList, /* List of tables */
2004 ExprList *pEList /* Expressions defining the result set */
2005){
drh3f913572008-03-22 01:07:17 +00002006#ifndef SQLITE_OMIT_DECLTYPE
drhfcb78a42003-01-18 20:11:05 +00002007 Vdbe *v = pParse->pVdbe;
danielk1977517eb642004-06-07 10:00:31 +00002008 int i;
danielk1977b3bce662005-01-29 08:32:43 +00002009 NameContext sNC;
2010 sNC.pSrcList = pTabList;
danielk1977955de522006-02-10 02:27:42 +00002011 sNC.pParse = pParse;
drheac5fc02017-04-11 01:01:27 +00002012 sNC.pNext = 0;
drhfcb78a42003-01-18 20:11:05 +00002013 for(i=0; i<pEList->nExpr; i++){
2014 Expr *p = pEList->a[i].pExpr;
drh3f913572008-03-22 01:07:17 +00002015 const char *zType;
2016#ifdef SQLITE_ENABLE_COLUMN_METADATA
danielk1977955de522006-02-10 02:27:42 +00002017 const char *zOrigDb = 0;
2018 const char *zOrigTab = 0;
2019 const char *zOrigCol = 0;
drhcafc2f72017-10-03 03:01:09 +00002020 zType = columnType(&sNC, p, &zOrigDb, &zOrigTab, &zOrigCol);
danielk1977955de522006-02-10 02:27:42 +00002021
drh85b623f2007-12-13 21:54:09 +00002022 /* The vdbe must make its own copy of the column-type and other
danielk19774b1ae992006-02-10 03:06:10 +00002023 ** column specific strings, in case the schema is reset before this
2024 ** virtual machine is deleted.
danielk1977fbcd5852004-06-15 02:44:18 +00002025 */
danielk197710fb7492008-10-31 10:53:22 +00002026 sqlite3VdbeSetColName(v, i, COLNAME_DATABASE, zOrigDb, SQLITE_TRANSIENT);
2027 sqlite3VdbeSetColName(v, i, COLNAME_TABLE, zOrigTab, SQLITE_TRANSIENT);
2028 sqlite3VdbeSetColName(v, i, COLNAME_COLUMN, zOrigCol, SQLITE_TRANSIENT);
drh3f913572008-03-22 01:07:17 +00002029#else
drhcafc2f72017-10-03 03:01:09 +00002030 zType = columnType(&sNC, p, 0, 0, 0);
drh3f913572008-03-22 01:07:17 +00002031#endif
danielk197710fb7492008-10-31 10:53:22 +00002032 sqlite3VdbeSetColName(v, i, COLNAME_DECLTYPE, zType, SQLITE_TRANSIENT);
drhfcb78a42003-01-18 20:11:05 +00002033 }
drh5f3e5e72013-10-08 20:01:35 +00002034#endif /* !defined(SQLITE_OMIT_DECLTYPE) */
drhfcb78a42003-01-18 20:11:05 +00002035}
2036
drheac5fc02017-04-11 01:01:27 +00002037
2038/*
drhec360a82017-07-12 14:10:19 +00002039** Compute the column names for a SELECT statement.
2040**
2041** The only guarantee that SQLite makes about column names is that if the
2042** column has an AS clause assigning it a name, that will be the name used.
2043** That is the only documented guarantee. However, countless applications
2044** developed over the years have made baseless assumptions about column names
2045** and will break if those assumptions changes. Hence, use extreme caution
2046** when modifying this routine to avoid breaking legacy.
2047**
2048** See Also: sqlite3ColumnsFromExprList()
2049**
2050** The PRAGMA short_column_names and PRAGMA full_column_names settings are
2051** deprecated. The default setting is short=ON, full=OFF. 99.9% of all
2052** applications should operate this way. Nevertheless, we need to support the
2053** other modes for legacy:
2054**
2055** short=OFF, full=OFF: Column name is the text of the expression has it
2056** originally appears in the SELECT statement. In
2057** other words, the zSpan of the result expression.
2058**
2059** short=ON, full=OFF: (This is the default setting). If the result
drh3d240d22017-09-28 16:56:55 +00002060** refers directly to a table column, then the
2061** result column name is just the table column
2062** name: COLUMN. Otherwise use zSpan.
drhec360a82017-07-12 14:10:19 +00002063**
2064** full=ON, short=ANY: If the result refers directly to a table column,
2065** then the result column name with the table name
2066** prefix, ex: TABLE.COLUMN. Otherwise use zSpan.
drh82c3d632000-06-06 21:56:07 +00002067*/
drh90881862021-05-19 12:17:03 +00002068void sqlite3GenerateColumnNames(
drh832508b2002-03-02 17:04:07 +00002069 Parse *pParse, /* Parser context */
drhf35f2f92017-07-29 16:01:55 +00002070 Select *pSelect /* Generate column names for this SELECT statement */
drh832508b2002-03-02 17:04:07 +00002071){
drhd8bc7082000-06-07 23:51:50 +00002072 Vdbe *v = pParse->pVdbe;
drheac5fc02017-04-11 01:01:27 +00002073 int i;
2074 Table *pTab;
drhf35f2f92017-07-29 16:01:55 +00002075 SrcList *pTabList;
2076 ExprList *pEList;
drh9bb575f2004-09-06 17:24:11 +00002077 sqlite3 *db = pParse->db;
drhf35f2f92017-07-29 16:01:55 +00002078 int fullName; /* TABLE.COLUMN if no AS clause and is a direct table ref */
2079 int srcName; /* COLUMN or TABLE.COLUMN if no AS clause and is direct */
drhfcabd462004-02-20 14:50:58 +00002080
drhfe2093d2005-01-20 22:48:47 +00002081#ifndef SQLITE_OMIT_EXPLAIN
danielk19773cf86062004-05-26 10:11:05 +00002082 /* If this is an EXPLAIN, skip this step */
2083 if( pParse->explain ){
danielk197761de0d12004-05-27 23:56:16 +00002084 return;
danielk19773cf86062004-05-26 10:11:05 +00002085 }
danielk19775338a5f2005-01-20 13:03:10 +00002086#endif
danielk19773cf86062004-05-26 10:11:05 +00002087
drh5e8b9852018-05-03 22:52:56 +00002088 if( pParse->colNamesSet ) return;
drhf35f2f92017-07-29 16:01:55 +00002089 /* Column names are determined by the left-most term of a compound select */
2090 while( pSelect->pPrior ) pSelect = pSelect->pPrior;
drh07859482017-12-23 11:51:40 +00002091 SELECTTRACE(1,pParse,pSelect,("generating column names\n"));
drhf35f2f92017-07-29 16:01:55 +00002092 pTabList = pSelect->pSrc;
2093 pEList = pSelect->pEList;
drh98029472015-12-03 21:47:30 +00002094 assert( v!=0 );
drhf7ce4292015-12-02 19:46:12 +00002095 assert( pTabList!=0 );
drhd8bc7082000-06-07 23:51:50 +00002096 pParse->colNamesSet = 1;
drhec360a82017-07-12 14:10:19 +00002097 fullName = (db->flags & SQLITE_FullColNames)!=0;
2098 srcName = (db->flags & SQLITE_ShortColNames)!=0 || fullName;
danielk197722322fd2004-05-25 23:35:17 +00002099 sqlite3VdbeSetNumCols(v, pEList->nExpr);
drh82c3d632000-06-06 21:56:07 +00002100 for(i=0; i<pEList->nExpr; i++){
drhec360a82017-07-12 14:10:19 +00002101 Expr *p = pEList->a[i].pExpr;
2102
2103 assert( p!=0 );
drh4dd89d52017-08-14 14:53:24 +00002104 assert( p->op!=TK_AGG_COLUMN ); /* Agg processing has not run yet */
drh477572b2021-10-07 20:46:29 +00002105 assert( p->op!=TK_COLUMN
2106 || (ExprUseYTab(p) && p->y.pTab!=0) ); /* Covering idx not yet coded */
drhd88fd532022-05-02 20:49:30 +00002107 if( pEList->a[i].zEName && pEList->a[i].fg.eEName==ENAME_NAME ){
drhec360a82017-07-12 14:10:19 +00002108 /* An AS clause always takes first priority */
drh41cee662019-12-12 20:22:34 +00002109 char *zName = pEList->a[i].zEName;
danielk197710fb7492008-10-31 10:53:22 +00002110 sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, SQLITE_TRANSIENT);
drhf35f2f92017-07-29 16:01:55 +00002111 }else if( srcName && p->op==TK_COLUMN ){
drh97665872002-02-13 23:22:53 +00002112 char *zCol;
drh8aff1012001-12-22 14:49:24 +00002113 int iCol = p->iColumn;
drheda079c2018-09-20 19:02:15 +00002114 pTab = p->y.pTab;
drhf35f2f92017-07-29 16:01:55 +00002115 assert( pTab!=0 );
drh8aff1012001-12-22 14:49:24 +00002116 if( iCol<0 ) iCol = pTab->iPKey;
drh97665872002-02-13 23:22:53 +00002117 assert( iCol==-1 || (iCol>=0 && iCol<pTab->nCol) );
drhb1363202002-06-26 02:45:03 +00002118 if( iCol<0 ){
drh47a6db22005-01-18 16:02:40 +00002119 zCol = "rowid";
drhb1363202002-06-26 02:45:03 +00002120 }else{
drhcf9d36d2021-08-02 18:03:43 +00002121 zCol = pTab->aCol[iCol].zCnName;
drhb1363202002-06-26 02:45:03 +00002122 }
drhec360a82017-07-12 14:10:19 +00002123 if( fullName ){
drh82c3d632000-06-06 21:56:07 +00002124 char *zName = 0;
drh1c767f02009-01-09 02:49:31 +00002125 zName = sqlite3MPrintf(db, "%s.%s", pTab->zName, zCol);
danielk197710fb7492008-10-31 10:53:22 +00002126 sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, SQLITE_DYNAMIC);
drh82c3d632000-06-06 21:56:07 +00002127 }else{
danielk197710fb7492008-10-31 10:53:22 +00002128 sqlite3VdbeSetColName(v, i, COLNAME_NAME, zCol, SQLITE_TRANSIENT);
drh82c3d632000-06-06 21:56:07 +00002129 }
drh1bee3d72001-10-15 00:44:35 +00002130 }else{
drhcbb9da32019-12-12 22:11:33 +00002131 const char *z = pEList->a[i].zEName;
drh859bc542014-01-13 20:32:18 +00002132 z = z==0 ? sqlite3MPrintf(db, "column%d", i+1) : sqlite3DbStrDup(db, z);
2133 sqlite3VdbeSetColName(v, i, COLNAME_NAME, z, SQLITE_DYNAMIC);
drh82c3d632000-06-06 21:56:07 +00002134 }
2135 }
danielk197776d505b2004-05-28 13:13:02 +00002136 generateColumnTypes(pParse, pTabList, pEList);
drh82c3d632000-06-06 21:56:07 +00002137}
2138
drhd8bc7082000-06-07 23:51:50 +00002139/*
peter.d.reid60ec9142014-09-06 16:39:46 +00002140** Given an expression list (which is really the list of expressions
drh7d10d5a2008-08-20 16:35:10 +00002141** that form the result set of a SELECT statement) compute appropriate
2142** column names for a table that would hold the expression list.
2143**
2144** All column names will be unique.
2145**
2146** Only the column names are computed. Column.zType, Column.zColl,
2147** and other fields of Column are zeroed.
2148**
2149** Return SQLITE_OK on success. If a memory allocation error occurs,
2150** store NULL in *paCol and 0 in *pnCol and return SQLITE_NOMEM.
drhec360a82017-07-12 14:10:19 +00002151**
2152** The only guarantee that SQLite makes about column names is that if the
2153** column has an AS clause assigning it a name, that will be the name used.
2154** That is the only documented guarantee. However, countless applications
2155** developed over the years have made baseless assumptions about column names
2156** and will break if those assumptions changes. Hence, use extreme caution
2157** when modifying this routine to avoid breaking legacy.
2158**
drh90881862021-05-19 12:17:03 +00002159** See Also: sqlite3GenerateColumnNames()
drh315555c2002-10-20 15:53:03 +00002160*/
drh8981b902015-08-24 17:42:49 +00002161int sqlite3ColumnsFromExprList(
drh7d10d5a2008-08-20 16:35:10 +00002162 Parse *pParse, /* Parsing context */
2163 ExprList *pEList, /* Expr list from which to derive column names */
drhd815f172012-09-13 14:42:43 +00002164 i16 *pnCol, /* Write the number of columns here */
drh7d10d5a2008-08-20 16:35:10 +00002165 Column **paCol /* Write the new column list here */
2166){
drhdc5ea5c2008-12-10 17:19:59 +00002167 sqlite3 *db = pParse->db; /* Database connection */
2168 int i, j; /* Loop counters */
drhebed3fa2015-11-14 16:47:23 +00002169 u32 cnt; /* Index added to make the name unique */
drhdc5ea5c2008-12-10 17:19:59 +00002170 Column *aCol, *pCol; /* For looping over result columns */
2171 int nCol; /* Number of columns in the result set */
drhdc5ea5c2008-12-10 17:19:59 +00002172 char *zName; /* Column name */
2173 int nName; /* Size of name in zName[] */
drh0315e3c2015-11-14 20:52:43 +00002174 Hash ht; /* Hash table of column names */
drh6d5ab2a2020-11-02 00:40:05 +00002175 Table *pTab;
drh22f70c32002-02-18 01:17:00 +00002176
drh0315e3c2015-11-14 20:52:43 +00002177 sqlite3HashInit(&ht);
dan8c2e0f02012-03-31 15:08:56 +00002178 if( pEList ){
2179 nCol = pEList->nExpr;
2180 aCol = sqlite3DbMallocZero(db, sizeof(aCol[0])*nCol);
2181 testcase( aCol==0 );
drh080fe6d2021-03-18 20:04:46 +00002182 if( NEVER(nCol>32767) ) nCol = 32767;
dan8c2e0f02012-03-31 15:08:56 +00002183 }else{
2184 nCol = 0;
2185 aCol = 0;
2186 }
dan8836cbb2015-11-21 19:43:29 +00002187 assert( nCol==(i16)nCol );
dan8c2e0f02012-03-31 15:08:56 +00002188 *pnCol = nCol;
2189 *paCol = aCol;
2190
drh0315e3c2015-11-14 20:52:43 +00002191 for(i=0, pCol=aCol; i<nCol && !db->mallocFailed; i++, pCol++){
drh18f86002022-04-21 13:11:26 +00002192 struct ExprList_item *pX = &pEList->a[i];
drh45e41b72022-04-22 23:18:21 +00002193 struct ExprList_item *pCollide;
drh79d5f632005-01-18 17:20:10 +00002194 /* Get an appropriate name for the column
2195 */
drhd88fd532022-05-02 20:49:30 +00002196 if( (zName = pX->zEName)!=0 && pX->fg.eEName==ENAME_NAME ){
drh79d5f632005-01-18 17:20:10 +00002197 /* If the column contains an "AS <name>" phrase, use <name> as the name */
drh93a960a2008-07-10 00:32:42 +00002198 }else{
drh18f86002022-04-21 13:11:26 +00002199 Expr *pColExpr = sqlite3ExprSkipCollateAndLikely(pX->pExpr);
drh235667a2020-11-08 20:44:30 +00002200 while( ALWAYS(pColExpr!=0) && pColExpr->op==TK_DOT ){
drhb07028f2011-10-14 21:49:18 +00002201 pColExpr = pColExpr->pRight;
2202 assert( pColExpr!=0 );
2203 }
drh477572b2021-10-07 20:46:29 +00002204 if( pColExpr->op==TK_COLUMN
2205 && ALWAYS( ExprUseYTab(pColExpr) )
drh5af8a862022-05-27 18:06:49 +00002206 && ALWAYS( pColExpr->y.pTab!=0 )
drh477572b2021-10-07 20:46:29 +00002207 ){
drh7d10d5a2008-08-20 16:35:10 +00002208 /* For columns use the column name name */
drhdc5ea5c2008-12-10 17:19:59 +00002209 int iCol = pColExpr->iColumn;
drh5af8a862022-05-27 18:06:49 +00002210 pTab = pColExpr->y.pTab;
drhf0209f72008-08-21 14:54:28 +00002211 if( iCol<0 ) iCol = pTab->iPKey;
drhcf9d36d2021-08-02 18:03:43 +00002212 zName = iCol>=0 ? pTab->aCol[iCol].zCnName : "rowid";
drhb7916a72009-05-27 10:31:29 +00002213 }else if( pColExpr->op==TK_ID ){
drh33e619f2009-05-28 01:00:55 +00002214 assert( !ExprHasProperty(pColExpr, EP_IntValue) );
drh96ceaf82015-11-14 22:04:22 +00002215 zName = pColExpr->u.zToken;
drh7d10d5a2008-08-20 16:35:10 +00002216 }else{
2217 /* Use the original text of the column expression as its name */
drh18f86002022-04-21 13:11:26 +00002218 assert( zName==pX->zEName ); /* pointer comparison intended */
drh7d10d5a2008-08-20 16:35:10 +00002219 }
drh22f70c32002-02-18 01:17:00 +00002220 }
drh0cbec592020-01-03 02:20:37 +00002221 if( zName && !sqlite3IsTrueOrFalse(zName) ){
drhd7ca6002017-07-09 00:30:58 +00002222 zName = sqlite3DbStrDup(db, zName);
2223 }else{
drh155507b2017-07-09 18:55:29 +00002224 zName = sqlite3MPrintf(db,"column%d",i+1);
drhd7ca6002017-07-09 00:30:58 +00002225 }
drh79d5f632005-01-18 17:20:10 +00002226
2227 /* Make sure the column name is unique. If the name is not unique,
peter.d.reid60ec9142014-09-06 16:39:46 +00002228 ** append an integer to the name so that it becomes unique.
drh79d5f632005-01-18 17:20:10 +00002229 */
drh0315e3c2015-11-14 20:52:43 +00002230 cnt = 0;
drh45e41b72022-04-22 23:18:21 +00002231 while( zName && (pCollide = sqlite3HashFind(&ht, zName))!=0 ){
drhd88fd532022-05-02 20:49:30 +00002232 if( pCollide->fg.bUsingTerm ){
drhab843a52022-04-23 07:29:34 +00002233 pCol->colFlags |= COLFLAG_NOEXPAND;
drh45e41b72022-04-22 23:18:21 +00002234 }
drh0315e3c2015-11-14 20:52:43 +00002235 nName = sqlite3Strlen30(zName);
drhf7ee8962015-11-15 11:13:49 +00002236 if( nName>0 ){
2237 for(j=nName-1; j>0 && sqlite3Isdigit(zName[j]); j--){}
2238 if( zName[j]==':' ) nName = j;
2239 }
drh96ceaf82015-11-14 22:04:22 +00002240 zName = sqlite3MPrintf(db, "%.*z:%u", nName, zName, ++cnt);
drh0315e3c2015-11-14 20:52:43 +00002241 if( cnt>3 ) sqlite3_randomness(sizeof(cnt), &cnt);
drh79d5f632005-01-18 17:20:10 +00002242 }
drhcf9d36d2021-08-02 18:03:43 +00002243 pCol->zCnName = zName;
drhd44390c2020-04-06 18:16:31 +00002244 pCol->hName = sqlite3StrIHash(zName);
drhd88fd532022-05-02 20:49:30 +00002245 if( pX->fg.bNoExpand ){
drh72d620b2022-05-02 19:59:03 +00002246 pCol->colFlags |= COLFLAG_NOEXPAND;
2247 }
danba68f8f2015-11-19 16:46:46 +00002248 sqlite3ColumnPropertiesFromName(0, pCol);
drh45e41b72022-04-22 23:18:21 +00002249 if( zName && sqlite3HashInsert(&ht, zName, pX)==pX ){
drh4a642b62016-02-05 01:55:27 +00002250 sqlite3OomFault(db);
drh0315e3c2015-11-14 20:52:43 +00002251 }
drh7d10d5a2008-08-20 16:35:10 +00002252 }
drh0315e3c2015-11-14 20:52:43 +00002253 sqlite3HashClear(&ht);
drh7d10d5a2008-08-20 16:35:10 +00002254 if( db->mallocFailed ){
drh7d10d5a2008-08-20 16:35:10 +00002255 for(j=0; j<i; j++){
drhcf9d36d2021-08-02 18:03:43 +00002256 sqlite3DbFree(db, aCol[j].zCnName);
drh7d10d5a2008-08-20 16:35:10 +00002257 }
2258 sqlite3DbFree(db, aCol);
2259 *paCol = 0;
2260 *pnCol = 0;
mistachkinfad30392016-02-13 23:43:46 +00002261 return SQLITE_NOMEM_BKPT;
drh7d10d5a2008-08-20 16:35:10 +00002262 }
2263 return SQLITE_OK;
2264}
danielk1977517eb642004-06-07 10:00:31 +00002265
drh7d10d5a2008-08-20 16:35:10 +00002266/*
2267** Add type and collation information to a column list based on
2268** a SELECT statement.
2269**
2270** The column list presumably came from selectColumnNamesFromExprList().
2271** The column list has only names, not types or collations. This
2272** routine goes through and adds the types and collations.
2273**
shaneb08a67a2009-03-31 03:41:56 +00002274** This routine requires that all identifiers in the SELECT
drh7d10d5a2008-08-20 16:35:10 +00002275** statement be resolved.
2276*/
drhed06a132016-04-05 20:59:12 +00002277void sqlite3SelectAddColumnTypeAndCollation(
drh7d10d5a2008-08-20 16:35:10 +00002278 Parse *pParse, /* Parsing contexts */
drh186ad8c2013-10-08 18:40:37 +00002279 Table *pTab, /* Add column type information to this table */
drh81506b82019-08-05 19:32:06 +00002280 Select *pSelect, /* SELECT used to determine types and collations */
2281 char aff /* Default affinity for columns */
drh7d10d5a2008-08-20 16:35:10 +00002282){
2283 sqlite3 *db = pParse->db;
2284 NameContext sNC;
2285 Column *pCol;
2286 CollSeq *pColl;
2287 int i;
2288 Expr *p;
2289 struct ExprList_item *a;
2290
2291 assert( pSelect!=0 );
2292 assert( (pSelect->selFlags & SF_Resolved)!=0 );
drh186ad8c2013-10-08 18:40:37 +00002293 assert( pTab->nCol==pSelect->pEList->nExpr || db->mallocFailed );
drh7d10d5a2008-08-20 16:35:10 +00002294 if( db->mallocFailed ) return;
2295 memset(&sNC, 0, sizeof(sNC));
2296 sNC.pSrcList = pSelect->pSrc;
2297 a = pSelect->pEList->a;
drh186ad8c2013-10-08 18:40:37 +00002298 for(i=0, pCol=pTab->aCol; i<pTab->nCol; i++, pCol++){
drhed06a132016-04-05 20:59:12 +00002299 const char *zType;
drh913306a2021-11-26 17:10:18 +00002300 i64 n, m;
drh6f6e60d2021-02-18 15:45:34 +00002301 pTab->tabFlags |= (pCol->colFlags & COLFLAG_NOINSERT);
drh7d10d5a2008-08-20 16:35:10 +00002302 p = a[i].pExpr;
drhcafc2f72017-10-03 03:01:09 +00002303 zType = columnType(&sNC, p, 0, 0, 0);
2304 /* pCol->szEst = ... // Column size est for SELECT tables never used */
danielk1977c60e9b82005-01-31 12:42:29 +00002305 pCol->affinity = sqlite3ExprAffinity(p);
drh0c4db032017-10-03 19:53:12 +00002306 if( zType ){
2307 m = sqlite3Strlen30(zType);
drhcf9d36d2021-08-02 18:03:43 +00002308 n = sqlite3Strlen30(pCol->zCnName);
2309 pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+m+2);
2310 if( pCol->zCnName ){
2311 memcpy(&pCol->zCnName[n+1], zType, m+1);
drhed06a132016-04-05 20:59:12 +00002312 pCol->colFlags |= COLFLAG_HASTYPE;
drh146121f2021-11-12 14:39:49 +00002313 }else{
2314 testcase( pCol->colFlags & COLFLAG_HASTYPE );
2315 pCol->colFlags &= ~(COLFLAG_HASTYPE|COLFLAG_HASCOLL);
drhed06a132016-04-05 20:59:12 +00002316 }
2317 }
drh96fb16e2019-08-06 14:37:24 +00002318 if( pCol->affinity<=SQLITE_AFF_NONE ) pCol->affinity = aff;
danielk1977b3bf5562006-01-10 17:58:23 +00002319 pColl = sqlite3ExprCollSeq(pParse, p);
drh5ced0a92021-08-24 17:07:44 +00002320 if( pColl ){
drh65b40092021-08-05 15:27:19 +00002321 assert( pTab->pIndex==0 );
2322 sqlite3ColumnSetColl(db, pCol, pColl->zName);
danielk19770202b292004-06-09 09:55:16 +00002323 }
drh22f70c32002-02-18 01:17:00 +00002324 }
drhcafc2f72017-10-03 03:01:09 +00002325 pTab->szTabRow = 1; /* Any non-zero value works */
drh7d10d5a2008-08-20 16:35:10 +00002326}
2327
2328/*
2329** Given a SELECT statement, generate a Table structure that describes
2330** the result set of that SELECT.
2331*/
drh81506b82019-08-05 19:32:06 +00002332Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect, char aff){
drh7d10d5a2008-08-20 16:35:10 +00002333 Table *pTab;
2334 sqlite3 *db = pParse->db;
drh70d5dfb2018-12-06 16:50:55 +00002335 u64 savedFlags;
drh7d10d5a2008-08-20 16:35:10 +00002336
2337 savedFlags = db->flags;
drhd5b44d62018-12-06 17:06:02 +00002338 db->flags &= ~(u64)SQLITE_FullColNames;
drh7d10d5a2008-08-20 16:35:10 +00002339 db->flags |= SQLITE_ShortColNames;
2340 sqlite3SelectPrep(pParse, pSelect, 0);
drh491b6d82019-01-24 15:51:03 +00002341 db->flags = savedFlags;
drh7d10d5a2008-08-20 16:35:10 +00002342 if( pParse->nErr ) return 0;
2343 while( pSelect->pPrior ) pSelect = pSelect->pPrior;
drh7d10d5a2008-08-20 16:35:10 +00002344 pTab = sqlite3DbMallocZero(db, sizeof(Table) );
2345 if( pTab==0 ){
2346 return 0;
2347 }
drh79df7782016-12-14 14:07:35 +00002348 pTab->nTabRef = 1;
drh7d10d5a2008-08-20 16:35:10 +00002349 pTab->zName = 0;
dancfc9df72014-04-25 15:01:01 +00002350 pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) );
drh8981b902015-08-24 17:42:49 +00002351 sqlite3ColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol);
drh81506b82019-08-05 19:32:06 +00002352 sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSelect, aff);
drh22f70c32002-02-18 01:17:00 +00002353 pTab->iPKey = -1;
drh7ce72f62008-07-24 15:50:41 +00002354 if( db->mallocFailed ){
dan1feeaed2010-07-23 15:41:47 +00002355 sqlite3DeleteTable(db, pTab);
drh7ce72f62008-07-24 15:50:41 +00002356 return 0;
2357 }
drh22f70c32002-02-18 01:17:00 +00002358 return pTab;
2359}
2360
2361/*
drhd8bc7082000-06-07 23:51:50 +00002362** Get a VDBE for the given parser context. Create a new one if necessary.
2363** If an error occurs, return NULL and leave a message in pParse.
2364*/
drh55965612017-09-16 20:58:41 +00002365Vdbe *sqlite3GetVdbe(Parse *pParse){
2366 if( pParse->pVdbe ){
2367 return pParse->pVdbe;
2368 }
drh6f077342016-04-12 00:26:59 +00002369 if( pParse->pToplevel==0
2370 && OptimizationEnabled(pParse->db,SQLITE_FactorOutConst)
2371 ){
2372 pParse->okConstFactor = 1;
drhd8bc7082000-06-07 23:51:50 +00002373 }
drh55965612017-09-16 20:58:41 +00002374 return sqlite3VdbeCreate(pParse);
drh6f077342016-04-12 00:26:59 +00002375}
drhfcb78a42003-01-18 20:11:05 +00002376
drh15007a92006-01-08 18:10:17 +00002377
drhd8bc7082000-06-07 23:51:50 +00002378/*
drh7b58dae2003-07-20 01:16:46 +00002379** Compute the iLimit and iOffset fields of the SELECT based on the
drh8c0833f2017-11-14 23:48:23 +00002380** pLimit expressions. pLimit->pLeft and pLimit->pRight hold the expressions
drh7b58dae2003-07-20 01:16:46 +00002381** that appear in the original SQL statement after the LIMIT and OFFSET
danielk1977a2dc3b12005-02-05 12:48:48 +00002382** keywords. Or NULL if those keywords are omitted. iLimit and iOffset
2383** are the integer memory register numbers for counters used to compute
2384** the limit and offset. If there is no limit and/or offset, then
2385** iLimit and iOffset are negative.
drh7b58dae2003-07-20 01:16:46 +00002386**
drhd59ba6c2006-01-08 05:02:54 +00002387** This routine changes the values of iLimit and iOffset only if
drh8c0833f2017-11-14 23:48:23 +00002388** a limit or offset is defined by pLimit->pLeft and pLimit->pRight. iLimit
2389** and iOffset should have been preset to appropriate default values (zero)
drhaa9ce702014-01-22 18:07:04 +00002390** prior to calling this routine.
2391**
2392** The iOffset register (if it exists) is initialized to the value
2393** of the OFFSET. The iLimit register is initialized to LIMIT. Register
2394** iOffset+1 is initialized to LIMIT+OFFSET.
2395**
drh8c0833f2017-11-14 23:48:23 +00002396** Only if pLimit->pLeft!=0 do the limit registers get
drh7b58dae2003-07-20 01:16:46 +00002397** redefined. The UNION ALL operator uses this property to force
2398** the reuse of the same limit and offset registers across multiple
2399** SELECT statements.
2400*/
drhec7429a2005-10-06 16:53:14 +00002401static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){
drh02afc862006-01-20 18:10:57 +00002402 Vdbe *v = 0;
2403 int iLimit = 0;
drh15007a92006-01-08 18:10:17 +00002404 int iOffset;
drh8b0cf382015-10-06 21:07:06 +00002405 int n;
drh8c0833f2017-11-14 23:48:23 +00002406 Expr *pLimit = p->pLimit;
2407
drh0acb7e42008-06-25 00:12:41 +00002408 if( p->iLimit ) return;
drh15007a92006-01-08 18:10:17 +00002409
drh7b58dae2003-07-20 01:16:46 +00002410 /*
drh7b58dae2003-07-20 01:16:46 +00002411 ** "LIMIT -1" always shows all rows. There is some
drhf7b54962013-05-28 12:11:54 +00002412 ** controversy about what the correct behavior should be.
drh7b58dae2003-07-20 01:16:46 +00002413 ** The current implementation interprets "LIMIT 0" to mean
2414 ** no rows.
2415 */
drh8c0833f2017-11-14 23:48:23 +00002416 if( pLimit ){
2417 assert( pLimit->op==TK_LIMIT );
2418 assert( pLimit->pLeft!=0 );
drh0a07c102008-01-03 18:03:08 +00002419 p->iLimit = iLimit = ++pParse->nMem;
drh15007a92006-01-08 18:10:17 +00002420 v = sqlite3GetVdbe(pParse);
drhaa9ce702014-01-22 18:07:04 +00002421 assert( v!=0 );
drh8c0833f2017-11-14 23:48:23 +00002422 if( sqlite3ExprIsInteger(pLimit->pLeft, &n) ){
drh456e4e42009-11-20 16:13:15 +00002423 sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit);
2424 VdbeComment((v, "LIMIT counter"));
drh9b918ed2009-11-12 03:13:26 +00002425 if( n==0 ){
drh076e85f2015-09-03 13:46:12 +00002426 sqlite3VdbeGoto(v, iBreak);
drhc3489bb2016-02-25 16:04:59 +00002427 }else if( n>=0 && p->nSelectRow>sqlite3LogEst((u64)n) ){
2428 p->nSelectRow = sqlite3LogEst((u64)n);
2429 p->selFlags |= SF_FixedLimit;
drh9b918ed2009-11-12 03:13:26 +00002430 }
2431 }else{
drh8c0833f2017-11-14 23:48:23 +00002432 sqlite3ExprCode(pParse, pLimit->pLeft, iLimit);
drh688852a2014-02-17 22:40:43 +00002433 sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit); VdbeCoverage(v);
drh9b918ed2009-11-12 03:13:26 +00002434 VdbeComment((v, "LIMIT counter"));
drh16897072015-03-07 00:57:37 +00002435 sqlite3VdbeAddOp2(v, OP_IfNot, iLimit, iBreak); VdbeCoverage(v);
drh9b918ed2009-11-12 03:13:26 +00002436 }
drh8c0833f2017-11-14 23:48:23 +00002437 if( pLimit->pRight ){
drh373cc2d2009-05-17 02:06:14 +00002438 p->iOffset = iOffset = ++pParse->nMem;
drhb7654112008-01-12 12:48:07 +00002439 pParse->nMem++; /* Allocate an extra register for limit+offset */
drh8c0833f2017-11-14 23:48:23 +00002440 sqlite3ExprCode(pParse, pLimit->pRight, iOffset);
drh688852a2014-02-17 22:40:43 +00002441 sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset); VdbeCoverage(v);
drh373cc2d2009-05-17 02:06:14 +00002442 VdbeComment((v, "OFFSET counter"));
drhcc2fa4c2016-01-25 15:57:29 +00002443 sqlite3VdbeAddOp3(v, OP_OffsetLimit, iLimit, iOffset+1, iOffset);
drhb7654112008-01-12 12:48:07 +00002444 VdbeComment((v, "LIMIT+OFFSET"));
drhd59ba6c2006-01-08 05:02:54 +00002445 }
drh7b58dae2003-07-20 01:16:46 +00002446 }
2447}
2448
drhb7f91642004-10-31 02:22:47 +00002449#ifndef SQLITE_OMIT_COMPOUND_SELECT
drhfbc4ee72004-08-29 01:31:05 +00002450/*
drhfbc4ee72004-08-29 01:31:05 +00002451** Return the appropriate collating sequence for the iCol-th column of
2452** the result set for the compound-select statement "p". Return NULL if
2453** the column has no default collating sequence.
2454**
2455** The collating sequence for the compound select is taken from the
2456** left-most term of the select that has a collating sequence.
2457*/
danielk1977dc1bdc42004-06-11 10:51:27 +00002458static CollSeq *multiSelectCollSeq(Parse *pParse, Select *p, int iCol){
drhfbc4ee72004-08-29 01:31:05 +00002459 CollSeq *pRet;
danielk1977dc1bdc42004-06-11 10:51:27 +00002460 if( p->pPrior ){
2461 pRet = multiSelectCollSeq(pParse, p->pPrior, iCol);
drhfbc4ee72004-08-29 01:31:05 +00002462 }else{
2463 pRet = 0;
danielk1977dc1bdc42004-06-11 10:51:27 +00002464 }
drh10c081a2009-04-16 00:24:23 +00002465 assert( iCol>=0 );
drh2ec18a32015-06-23 23:31:52 +00002466 /* iCol must be less than p->pEList->nExpr. Otherwise an error would
2467 ** have been thrown during name resolution and we would not have gotten
2468 ** this far */
2469 if( pRet==0 && ALWAYS(iCol<p->pEList->nExpr) ){
danielk1977dc1bdc42004-06-11 10:51:27 +00002470 pRet = sqlite3ExprCollSeq(pParse, p->pEList->a[iCol].pExpr);
2471 }
2472 return pRet;
drhd3d39e92004-05-20 22:16:29 +00002473}
2474
dan53bed452014-01-24 20:37:18 +00002475/*
2476** The select statement passed as the second parameter is a compound SELECT
2477** with an ORDER BY clause. This function allocates and returns a KeyInfo
2478** structure suitable for implementing the ORDER BY.
2479**
2480** Space to hold the KeyInfo structure is obtained from malloc. The calling
2481** function is responsible for ensuring that this structure is eventually
2482** freed.
2483*/
2484static KeyInfo *multiSelectOrderByKeyInfo(Parse *pParse, Select *p, int nExtra){
2485 ExprList *pOrderBy = p->pOrderBy;
drh7d4c94b2021-10-04 22:34:38 +00002486 int nOrderBy = ALWAYS(pOrderBy!=0) ? pOrderBy->nExpr : 0;
dan53bed452014-01-24 20:37:18 +00002487 sqlite3 *db = pParse->db;
2488 KeyInfo *pRet = sqlite3KeyInfoAlloc(db, nOrderBy+nExtra, 1);
2489 if( pRet ){
2490 int i;
2491 for(i=0; i<nOrderBy; i++){
2492 struct ExprList_item *pItem = &pOrderBy->a[i];
2493 Expr *pTerm = pItem->pExpr;
2494 CollSeq *pColl;
2495
2496 if( pTerm->flags & EP_Collate ){
2497 pColl = sqlite3ExprCollSeq(pParse, pTerm);
2498 }else{
2499 pColl = multiSelectCollSeq(pParse, p, pItem->u.x.iOrderByCol-1);
2500 if( pColl==0 ) pColl = db->pDfltColl;
2501 pOrderBy->a[i].pExpr =
2502 sqlite3ExprAddCollateString(pParse, pTerm, pColl->zName);
2503 }
2504 assert( sqlite3KeyInfoIsWriteable(pRet) );
2505 pRet->aColl[i] = pColl;
drhd88fd532022-05-02 20:49:30 +00002506 pRet->aSortFlags[i] = pOrderBy->a[i].fg.sortFlags;
dan53bed452014-01-24 20:37:18 +00002507 }
2508 }
2509
2510 return pRet;
2511}
drhd3d39e92004-05-20 22:16:29 +00002512
drh781def22014-01-22 13:35:53 +00002513#ifndef SQLITE_OMIT_CTE
2514/*
2515** This routine generates VDBE code to compute the content of a WITH RECURSIVE
2516** query of the form:
2517**
2518** <recursive-table> AS (<setup-query> UNION [ALL] <recursive-query>)
2519** \___________/ \_______________/
2520** p->pPrior p
2521**
2522**
2523** There is exactly one reference to the recursive-table in the FROM clause
drh8a48b9c2015-08-19 15:20:00 +00002524** of recursive-query, marked with the SrcList->a[].fg.isRecursive flag.
drh781def22014-01-22 13:35:53 +00002525**
2526** The setup-query runs once to generate an initial set of rows that go
2527** into a Queue table. Rows are extracted from the Queue table one by
drhfe1c6bb2014-01-22 17:28:35 +00002528** one. Each row extracted from Queue is output to pDest. Then the single
2529** extracted row (now in the iCurrent table) becomes the content of the
2530** recursive-table for a recursive-query run. The output of the recursive-query
drh781def22014-01-22 13:35:53 +00002531** is added back into the Queue table. Then another row is extracted from Queue
2532** and the iteration continues until the Queue table is empty.
2533**
2534** If the compound query operator is UNION then no duplicate rows are ever
2535** inserted into the Queue table. The iDistinct table keeps a copy of all rows
2536** that have ever been inserted into Queue and causes duplicates to be
2537** discarded. If the operator is UNION ALL, then duplicates are allowed.
2538**
2539** If the query has an ORDER BY, then entries in the Queue table are kept in
2540** ORDER BY order and the first entry is extracted for each cycle. Without
2541** an ORDER BY, the Queue table is just a FIFO.
2542**
2543** If a LIMIT clause is provided, then the iteration stops after LIMIT rows
2544** have been output to pDest. A LIMIT of zero means to output no rows and a
2545** negative LIMIT means to output all rows. If there is also an OFFSET clause
2546** with a positive value, then the first OFFSET outputs are discarded rather
2547** than being sent to pDest. The LIMIT count does not begin until after OFFSET
2548** rows have been skipped.
2549*/
2550static void generateWithRecursiveQuery(
2551 Parse *pParse, /* Parsing context */
2552 Select *p, /* The recursive SELECT to be coded */
2553 SelectDest *pDest /* What to do with query results */
2554){
2555 SrcList *pSrc = p->pSrc; /* The FROM clause of the recursive query */
2556 int nCol = p->pEList->nExpr; /* Number of columns in the recursive table */
2557 Vdbe *v = pParse->pVdbe; /* The prepared statement under construction */
drhe85e1da2021-10-01 21:01:07 +00002558 Select *pSetup; /* The setup query */
drh34055852020-10-19 01:23:48 +00002559 Select *pFirstRec; /* Left-most recursive term */
drh781def22014-01-22 13:35:53 +00002560 int addrTop; /* Top of the loop */
2561 int addrCont, addrBreak; /* CONTINUE and BREAK addresses */
drhedf83d12014-01-22 18:31:27 +00002562 int iCurrent = 0; /* The Current table */
drh781def22014-01-22 13:35:53 +00002563 int regCurrent; /* Register holding Current table */
2564 int iQueue; /* The Queue table */
2565 int iDistinct = 0; /* To ensure unique results if UNION */
drh8e1ee882014-03-21 19:56:09 +00002566 int eDest = SRT_Fifo; /* How to write to Queue */
drh781def22014-01-22 13:35:53 +00002567 SelectDest destQueue; /* SelectDest targetting the Queue table */
2568 int i; /* Loop counter */
2569 int rc; /* Result code */
drhfe1c6bb2014-01-22 17:28:35 +00002570 ExprList *pOrderBy; /* The ORDER BY clause */
drh8c0833f2017-11-14 23:48:23 +00002571 Expr *pLimit; /* Saved LIMIT and OFFSET */
drhaa9ce702014-01-22 18:07:04 +00002572 int regLimit, regOffset; /* Registers used by LIMIT and OFFSET */
drh781def22014-01-22 13:35:53 +00002573
dan6afa35c2018-09-27 12:14:15 +00002574#ifndef SQLITE_OMIT_WINDOWFUNC
2575 if( p->pWin ){
2576 sqlite3ErrorMsg(pParse, "cannot use window functions in recursive queries");
2577 return;
2578 }
2579#endif
2580
drh781def22014-01-22 13:35:53 +00002581 /* Obtain authorization to do a recursive query */
2582 if( sqlite3AuthCheck(pParse, SQLITE_RECURSIVE, 0, 0, 0) ) return;
drh781def22014-01-22 13:35:53 +00002583
drhaa9ce702014-01-22 18:07:04 +00002584 /* Process the LIMIT and OFFSET clauses, if they exist */
drhec4ccdb2018-12-29 02:26:59 +00002585 addrBreak = sqlite3VdbeMakeLabel(pParse);
dan69b93832016-12-16 15:05:40 +00002586 p->nSelectRow = 320; /* 4 billion rows */
drhaa9ce702014-01-22 18:07:04 +00002587 computeLimitRegisters(pParse, p, addrBreak);
2588 pLimit = p->pLimit;
drhaa9ce702014-01-22 18:07:04 +00002589 regLimit = p->iLimit;
2590 regOffset = p->iOffset;
drh8c0833f2017-11-14 23:48:23 +00002591 p->pLimit = 0;
drhaa9ce702014-01-22 18:07:04 +00002592 p->iLimit = p->iOffset = 0;
dan53bed452014-01-24 20:37:18 +00002593 pOrderBy = p->pOrderBy;
drh781def22014-01-22 13:35:53 +00002594
2595 /* Locate the cursor number of the Current table */
2596 for(i=0; ALWAYS(i<pSrc->nSrc); i++){
drh8a48b9c2015-08-19 15:20:00 +00002597 if( pSrc->a[i].fg.isRecursive ){
drh781def22014-01-22 13:35:53 +00002598 iCurrent = pSrc->a[i].iCursor;
2599 break;
2600 }
2601 }
2602
drhfe1c6bb2014-01-22 17:28:35 +00002603 /* Allocate cursors numbers for Queue and Distinct. The cursor number for
drh781def22014-01-22 13:35:53 +00002604 ** the Distinct table must be exactly one greater than Queue in order
drh8e1ee882014-03-21 19:56:09 +00002605 ** for the SRT_DistFifo and SRT_DistQueue destinations to work. */
drh781def22014-01-22 13:35:53 +00002606 iQueue = pParse->nTab++;
2607 if( p->op==TK_UNION ){
drh8e1ee882014-03-21 19:56:09 +00002608 eDest = pOrderBy ? SRT_DistQueue : SRT_DistFifo;
drh781def22014-01-22 13:35:53 +00002609 iDistinct = pParse->nTab++;
drhfe1c6bb2014-01-22 17:28:35 +00002610 }else{
drh8e1ee882014-03-21 19:56:09 +00002611 eDest = pOrderBy ? SRT_Queue : SRT_Fifo;
drh781def22014-01-22 13:35:53 +00002612 }
2613 sqlite3SelectDestInit(&destQueue, eDest, iQueue);
2614
2615 /* Allocate cursors for Current, Queue, and Distinct. */
2616 regCurrent = ++pParse->nMem;
2617 sqlite3VdbeAddOp3(v, OP_OpenPseudo, iCurrent, regCurrent, nCol);
drhfe1c6bb2014-01-22 17:28:35 +00002618 if( pOrderBy ){
dan53bed452014-01-24 20:37:18 +00002619 KeyInfo *pKeyInfo = multiSelectOrderByKeyInfo(pParse, p, 1);
drhfe1c6bb2014-01-22 17:28:35 +00002620 sqlite3VdbeAddOp4(v, OP_OpenEphemeral, iQueue, pOrderBy->nExpr+2, 0,
2621 (char*)pKeyInfo, P4_KEYINFO);
2622 destQueue.pOrderBy = pOrderBy;
2623 }else{
2624 sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iQueue, nCol);
2625 }
2626 VdbeComment((v, "Queue table"));
drh781def22014-01-22 13:35:53 +00002627 if( iDistinct ){
2628 p->addrOpenEphm[0] = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iDistinct, 0);
2629 p->selFlags |= SF_UsesEphemeral;
2630 }
2631
dan53bed452014-01-24 20:37:18 +00002632 /* Detach the ORDER BY clause from the compound SELECT */
2633 p->pOrderBy = 0;
2634
drh34055852020-10-19 01:23:48 +00002635 /* Figure out how many elements of the compound SELECT are part of the
2636 ** recursive query. Make sure no recursive elements use aggregate
2637 ** functions. Mark the recursive elements as UNION ALL even if they
2638 ** are really UNION because the distinctness will be enforced by the
2639 ** iDistinct table. pFirstRec is left pointing to the left-most
2640 ** recursive term of the CTE.
2641 */
drh34055852020-10-19 01:23:48 +00002642 for(pFirstRec=p; ALWAYS(pFirstRec!=0); pFirstRec=pFirstRec->pPrior){
2643 if( pFirstRec->selFlags & SF_Aggregate ){
2644 sqlite3ErrorMsg(pParse, "recursive aggregate queries not supported");
2645 goto end_of_recursive_query;
2646 }
2647 pFirstRec->op = TK_ALL;
2648 if( (pFirstRec->pPrior->selFlags & SF_Recursive)==0 ) break;
2649 }
2650
drh781def22014-01-22 13:35:53 +00002651 /* Store the results of the setup-query in Queue. */
drh34055852020-10-19 01:23:48 +00002652 pSetup = pFirstRec->pPrior;
drhd227a292014-02-09 18:02:09 +00002653 pSetup->pNext = 0;
drh84a01de2018-05-02 16:13:48 +00002654 ExplainQueryPlan((pParse, 1, "SETUP"));
drh781def22014-01-22 13:35:53 +00002655 rc = sqlite3Select(pParse, pSetup, &destQueue);
drhd227a292014-02-09 18:02:09 +00002656 pSetup->pNext = p;
drhfe1c6bb2014-01-22 17:28:35 +00002657 if( rc ) goto end_of_recursive_query;
drh781def22014-01-22 13:35:53 +00002658
2659 /* Find the next row in the Queue and output that row */
drh688852a2014-02-17 22:40:43 +00002660 addrTop = sqlite3VdbeAddOp2(v, OP_Rewind, iQueue, addrBreak); VdbeCoverage(v);
drh781def22014-01-22 13:35:53 +00002661
2662 /* Transfer the next row in Queue over to Current */
2663 sqlite3VdbeAddOp1(v, OP_NullRow, iCurrent); /* To reset column cache */
drhfe1c6bb2014-01-22 17:28:35 +00002664 if( pOrderBy ){
2665 sqlite3VdbeAddOp3(v, OP_Column, iQueue, pOrderBy->nExpr+1, regCurrent);
2666 }else{
2667 sqlite3VdbeAddOp2(v, OP_RowData, iQueue, regCurrent);
2668 }
drh781def22014-01-22 13:35:53 +00002669 sqlite3VdbeAddOp1(v, OP_Delete, iQueue);
2670
drhfe1c6bb2014-01-22 17:28:35 +00002671 /* Output the single row in Current */
drhec4ccdb2018-12-29 02:26:59 +00002672 addrCont = sqlite3VdbeMakeLabel(pParse);
drhaa9ce702014-01-22 18:07:04 +00002673 codeOffset(v, regOffset, addrCont);
drh2def2f72017-09-15 17:40:34 +00002674 selectInnerLoop(pParse, p, iCurrent,
drh079a3072014-03-19 14:10:55 +00002675 0, 0, pDest, addrCont, addrBreak);
drh688852a2014-02-17 22:40:43 +00002676 if( regLimit ){
drh16897072015-03-07 00:57:37 +00002677 sqlite3VdbeAddOp2(v, OP_DecrJumpZero, regLimit, addrBreak);
drh688852a2014-02-17 22:40:43 +00002678 VdbeCoverage(v);
2679 }
drhfe1c6bb2014-01-22 17:28:35 +00002680 sqlite3VdbeResolveLabel(v, addrCont);
2681
drh781def22014-01-22 13:35:53 +00002682 /* Execute the recursive SELECT taking the single row in Current as
2683 ** the value for the recursive-table. Store the results in the Queue.
2684 */
drh34055852020-10-19 01:23:48 +00002685 pFirstRec->pPrior = 0;
2686 ExplainQueryPlan((pParse, 1, "RECURSIVE STEP"));
2687 sqlite3Select(pParse, p, &destQueue);
2688 assert( pFirstRec->pPrior==0 );
2689 pFirstRec->pPrior = pSetup;
drh781def22014-01-22 13:35:53 +00002690
2691 /* Keep running the loop until the Queue is empty */
drh076e85f2015-09-03 13:46:12 +00002692 sqlite3VdbeGoto(v, addrTop);
drh781def22014-01-22 13:35:53 +00002693 sqlite3VdbeResolveLabel(v, addrBreak);
drhfe1c6bb2014-01-22 17:28:35 +00002694
2695end_of_recursive_query:
dan9afccba2014-03-21 19:27:54 +00002696 sqlite3ExprListDelete(pParse->db, p->pOrderBy);
drhfe1c6bb2014-01-22 17:28:35 +00002697 p->pOrderBy = pOrderBy;
drhaa9ce702014-01-22 18:07:04 +00002698 p->pLimit = pLimit;
drhfe1c6bb2014-01-22 17:28:35 +00002699 return;
drh781def22014-01-22 13:35:53 +00002700}
danb68b9772014-01-25 12:16:53 +00002701#endif /* SQLITE_OMIT_CTE */
drh781def22014-01-22 13:35:53 +00002702
2703/* Forward references */
drhb21e7c72008-06-22 12:37:57 +00002704static int multiSelectOrderBy(
2705 Parse *pParse, /* Parsing context */
2706 Select *p, /* The right-most of SELECTs to be coded */
drha9671a22008-07-08 23:40:20 +00002707 SelectDest *pDest /* What to do with query results */
drhb21e7c72008-06-22 12:37:57 +00002708);
2709
drh45f54a52015-01-05 19:16:42 +00002710/*
drh45f54a52015-01-05 19:16:42 +00002711** Handle the special case of a compound-select that originates from a
2712** VALUES clause. By handling this as a special case, we avoid deep
2713** recursion, and thus do not need to enforce the SQLITE_LIMIT_COMPOUND_SELECT
2714** on a VALUES clause.
2715**
2716** Because the Select object originates from a VALUES clause:
drhb058d052018-01-14 20:12:23 +00002717** (1) There is no LIMIT or OFFSET or else there is a LIMIT of exactly 1
drh45f54a52015-01-05 19:16:42 +00002718** (2) All terms are UNION ALL
2719** (3) There is no ORDER BY clause
drhb058d052018-01-14 20:12:23 +00002720**
2721** The "LIMIT of exactly 1" case of condition (1) comes about when a VALUES
2722** clause occurs within scalar expression (ex: "SELECT (VALUES(1),(2),(3))").
2723** The sqlite3CodeSubselect will have added the LIMIT 1 clause in tht case.
drhd1f0a862022-02-25 11:30:29 +00002724** Since the limit is exactly 1, we only need to evaluate the left-most VALUES.
drh45f54a52015-01-05 19:16:42 +00002725*/
2726static int multiSelectValues(
2727 Parse *pParse, /* Parsing context */
2728 Select *p, /* The right-most of SELECTs to be coded */
2729 SelectDest *pDest /* What to do with query results */
2730){
drh45f54a52015-01-05 19:16:42 +00002731 int nRow = 1;
2732 int rc = 0;
drhfa16f5d2018-05-03 01:37:13 +00002733 int bShowAll = p->pLimit==0;
drh772460f2015-04-16 14:13:12 +00002734 assert( p->selFlags & SF_MultiValue );
drh45f54a52015-01-05 19:16:42 +00002735 do{
2736 assert( p->selFlags & SF_Values );
2737 assert( p->op==TK_ALL || (p->op==TK_SELECT && p->pPrior==0) );
dan923cadb2015-06-23 12:19:55 +00002738 assert( p->pNext==0 || p->pEList->nExpr==p->pNext->pEList->nExpr );
drhef9f7192020-01-17 19:14:08 +00002739#ifndef SQLITE_OMIT_WINDOWFUNC
drh29cdbad2019-12-07 13:42:47 +00002740 if( p->pWin ) return -1;
drhef9f7192020-01-17 19:14:08 +00002741#endif
drh45f54a52015-01-05 19:16:42 +00002742 if( p->pPrior==0 ) break;
2743 assert( p->pPrior->pNext==p );
2744 p = p->pPrior;
drhfa16f5d2018-05-03 01:37:13 +00002745 nRow += bShowAll;
drh45f54a52015-01-05 19:16:42 +00002746 }while(1);
drhfa16f5d2018-05-03 01:37:13 +00002747 ExplainQueryPlan((pParse, 0, "SCAN %d CONSTANT ROW%s", nRow,
2748 nRow==1 ? "" : "S"));
drh45f54a52015-01-05 19:16:42 +00002749 while( p ){
drhfa16f5d2018-05-03 01:37:13 +00002750 selectInnerLoop(pParse, p, -1, 0, 0, pDest, 1, 1);
2751 if( !bShowAll ) break;
drh45f54a52015-01-05 19:16:42 +00002752 p->nSelectRow = nRow;
2753 p = p->pNext;
2754 }
2755 return rc;
2756}
drhb21e7c72008-06-22 12:37:57 +00002757
drhd3d39e92004-05-20 22:16:29 +00002758/*
drh34055852020-10-19 01:23:48 +00002759** Return true if the SELECT statement which is known to be the recursive
2760** part of a recursive CTE still has its anchor terms attached. If the
2761** anchor terms have already been removed, then return false.
2762*/
2763static int hasAnchor(Select *p){
2764 while( p && (p->selFlags & SF_Recursive)!=0 ){ p = p->pPrior; }
2765 return p!=0;
2766}
2767
2768/*
drh16ee60f2008-06-20 18:13:25 +00002769** This routine is called to process a compound query form from
2770** two or more separate queries using UNION, UNION ALL, EXCEPT, or
2771** INTERSECT
drhc926afb2002-06-20 03:38:26 +00002772**
drhe78e8282003-01-19 03:59:45 +00002773** "p" points to the right-most of the two queries. the query on the
2774** left is p->pPrior. The left query could also be a compound query
2775** in which case this routine will be called recursively.
2776**
2777** The results of the total query are to be written into a destination
2778** of type eDest with parameter iParm.
2779**
2780** Example 1: Consider a three-way compound SQL statement.
2781**
2782** SELECT a FROM t1 UNION SELECT b FROM t2 UNION SELECT c FROM t3
2783**
2784** This statement is parsed up as follows:
2785**
2786** SELECT c FROM t3
2787** |
2788** `-----> SELECT b FROM t2
2789** |
jplyon4b11c6d2004-01-19 04:57:53 +00002790** `------> SELECT a FROM t1
drhe78e8282003-01-19 03:59:45 +00002791**
2792** The arrows in the diagram above represent the Select.pPrior pointer.
2793** So if this routine is called with p equal to the t3 query, then
2794** pPrior will be the t2 query. p->op will be TK_UNION in this case.
2795**
2796** Notice that because of the way SQLite parses compound SELECTs, the
2797** individual selects always group from left to right.
drh82c3d632000-06-06 21:56:07 +00002798*/
danielk197784ac9d02004-05-18 09:58:06 +00002799static int multiSelect(
drhfbc4ee72004-08-29 01:31:05 +00002800 Parse *pParse, /* Parsing context */
2801 Select *p, /* The right-most of SELECTs to be coded */
drha9671a22008-07-08 23:40:20 +00002802 SelectDest *pDest /* What to do with query results */
danielk197784ac9d02004-05-18 09:58:06 +00002803){
drhfbc4ee72004-08-29 01:31:05 +00002804 int rc = SQLITE_OK; /* Success code from a subroutine */
2805 Select *pPrior; /* Another SELECT immediately to our left */
2806 Vdbe *v; /* Generate code to this VDBE */
drh1013c932008-01-06 00:25:21 +00002807 SelectDest dest; /* Alternative data destination */
danielk1977eca7e012008-07-01 16:05:25 +00002808 Select *pDelete = 0; /* Chain of simple selects to delete */
drh633e6d52008-07-28 19:34:53 +00002809 sqlite3 *db; /* Database connection */
drh82c3d632000-06-06 21:56:07 +00002810
drh7b58dae2003-07-20 01:16:46 +00002811 /* Make sure there is no ORDER BY or LIMIT clause on prior SELECTs. Only
drhfbc4ee72004-08-29 01:31:05 +00002812 ** the last (right-most) SELECT in the series may have an ORDER BY or LIMIT.
drh82c3d632000-06-06 21:56:07 +00002813 */
drh701bb3b2008-08-02 03:50:39 +00002814 assert( p && p->pPrior ); /* Calling function guarantees this much */
daneae73fb2014-01-16 18:34:33 +00002815 assert( (p->selFlags & SF_Recursive)==0 || p->op==TK_ALL || p->op==TK_UNION );
drhb0968b62019-05-29 18:33:59 +00002816 assert( p->selFlags & SF_Compound );
drh633e6d52008-07-28 19:34:53 +00002817 db = pParse->db;
drhd8bc7082000-06-07 23:51:50 +00002818 pPrior = p->pPrior;
drhbc103772008-08-08 18:06:25 +00002819 dest = *pDest;
drhaae0f742021-03-04 16:03:32 +00002820 assert( pPrior->pOrderBy==0 );
2821 assert( pPrior->pLimit==0 );
drh82c3d632000-06-06 21:56:07 +00002822
danielk19774adee202004-05-08 08:23:19 +00002823 v = sqlite3GetVdbe(pParse);
drh701bb3b2008-08-02 03:50:39 +00002824 assert( v!=0 ); /* The VDBE already created by calling function */
drhd8bc7082000-06-07 23:51:50 +00002825
drh1cc3d752002-03-23 00:31:29 +00002826 /* Create the destination temporary table if necessary
2827 */
danielk19776c8c8ce2008-01-02 16:27:09 +00002828 if( dest.eDest==SRT_EphemTab ){
danielk1977b4964b72004-05-18 01:23:38 +00002829 assert( p->pEList );
drh2b596da2012-07-23 21:43:19 +00002830 sqlite3VdbeAddOp2(v, OP_OpenEphemeral, dest.iSDParm, p->pEList->nExpr);
danielk19776c8c8ce2008-01-02 16:27:09 +00002831 dest.eDest = SRT_Table;
drh1cc3d752002-03-23 00:31:29 +00002832 }
2833
drh45f54a52015-01-05 19:16:42 +00002834 /* Special handling for a compound-select that originates as a VALUES clause.
2835 */
drh772460f2015-04-16 14:13:12 +00002836 if( p->selFlags & SF_MultiValue ){
drh45f54a52015-01-05 19:16:42 +00002837 rc = multiSelectValues(pParse, p, &dest);
drh29cdbad2019-12-07 13:42:47 +00002838 if( rc>=0 ) goto multi_select_end;
2839 rc = SQLITE_OK;
drh45f54a52015-01-05 19:16:42 +00002840 }
2841
drhf6e369a2008-06-24 12:46:30 +00002842 /* Make sure all SELECTs in the statement have the same number of elements
2843 ** in their result sets.
2844 */
2845 assert( p->pEList && pPrior->pEList );
dan923cadb2015-06-23 12:19:55 +00002846 assert( p->pEList->nExpr==pPrior->pEList->nExpr );
drhf6e369a2008-06-24 12:46:30 +00002847
daneede6a52014-01-15 19:42:23 +00002848#ifndef SQLITE_OMIT_CTE
drh34055852020-10-19 01:23:48 +00002849 if( (p->selFlags & SF_Recursive)!=0 && hasAnchor(p) ){
drh781def22014-01-22 13:35:53 +00002850 generateWithRecursiveQuery(pParse, p, &dest);
dan8ce71842014-01-14 20:14:09 +00002851 }else
2852#endif
drhf6e369a2008-06-24 12:46:30 +00002853
drha9671a22008-07-08 23:40:20 +00002854 /* Compound SELECTs that have an ORDER BY clause are handled separately.
2855 */
drhf6e369a2008-06-24 12:46:30 +00002856 if( p->pOrderBy ){
drha9671a22008-07-08 23:40:20 +00002857 return multiSelectOrderBy(pParse, p, pDest);
drh03c39052018-05-02 14:24:34 +00002858 }else{
drhf6e369a2008-06-24 12:46:30 +00002859
drhc631ded2018-05-02 02:22:22 +00002860#ifndef SQLITE_OMIT_EXPLAIN
drh03c39052018-05-02 14:24:34 +00002861 if( pPrior->pPrior==0 ){
drh4d799832018-05-03 19:47:14 +00002862 ExplainQueryPlan((pParse, 1, "COMPOUND QUERY"));
drh03c39052018-05-02 14:24:34 +00002863 ExplainQueryPlan((pParse, 1, "LEFT-MOST SUBQUERY"));
drh03c39052018-05-02 14:24:34 +00002864 }
drhc631ded2018-05-02 02:22:22 +00002865#endif
2866
drh03c39052018-05-02 14:24:34 +00002867 /* Generate code for the left and right SELECT statements.
2868 */
2869 switch( p->op ){
2870 case TK_ALL: {
2871 int addr = 0;
drh252d5822021-04-07 13:20:34 +00002872 int nLimit = 0; /* Initialize to suppress harmless compiler warning */
drh03c39052018-05-02 14:24:34 +00002873 assert( !pPrior->pLimit );
2874 pPrior->iLimit = p->iLimit;
2875 pPrior->iOffset = p->iOffset;
2876 pPrior->pLimit = p->pLimit;
drh93ffb502021-05-18 19:10:10 +00002877 SELECTTRACE(1, pParse, p, ("multiSelect UNION ALL left...\n"));
drh03c39052018-05-02 14:24:34 +00002878 rc = sqlite3Select(pParse, pPrior, &dest);
drhbc7819d2021-03-05 14:08:45 +00002879 pPrior->pLimit = 0;
drh03c39052018-05-02 14:24:34 +00002880 if( rc ){
2881 goto multi_select_end;
drh9f1ef452015-10-06 17:27:18 +00002882 }
drh03c39052018-05-02 14:24:34 +00002883 p->pPrior = 0;
2884 p->iLimit = pPrior->iLimit;
2885 p->iOffset = pPrior->iOffset;
2886 if( p->iLimit ){
2887 addr = sqlite3VdbeAddOp1(v, OP_IfNot, p->iLimit); VdbeCoverage(v);
2888 VdbeComment((v, "Jump ahead if LIMIT reached"));
2889 if( p->iOffset ){
2890 sqlite3VdbeAddOp3(v, OP_OffsetLimit,
2891 p->iLimit, p->iOffset+1, p->iOffset);
2892 }
2893 }
2894 ExplainQueryPlan((pParse, 1, "UNION ALL"));
drh93ffb502021-05-18 19:10:10 +00002895 SELECTTRACE(1, pParse, p, ("multiSelect UNION ALL right...\n"));
drh03c39052018-05-02 14:24:34 +00002896 rc = sqlite3Select(pParse, p, &dest);
2897 testcase( rc!=SQLITE_OK );
2898 pDelete = p->pPrior;
2899 p->pPrior = pPrior;
2900 p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow);
drhbc7819d2021-03-05 14:08:45 +00002901 if( p->pLimit
2902 && sqlite3ExprIsInteger(p->pLimit->pLeft, &nLimit)
drh03c39052018-05-02 14:24:34 +00002903 && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit)
2904 ){
2905 p->nSelectRow = sqlite3LogEst((u64)nLimit);
2906 }
2907 if( addr ){
2908 sqlite3VdbeJumpHere(v, addr);
2909 }
2910 break;
drha9671a22008-07-08 23:40:20 +00002911 }
drh03c39052018-05-02 14:24:34 +00002912 case TK_EXCEPT:
2913 case TK_UNION: {
2914 int unionTab; /* Cursor number of the temp table holding result */
2915 u8 op = 0; /* One of the SRT_ operations to apply to self */
2916 int priorOp; /* The SRT_ operation to apply to prior selects */
2917 Expr *pLimit; /* Saved values of p->nLimit */
2918 int addr;
2919 SelectDest uniondest;
2920
2921 testcase( p->op==TK_EXCEPT );
2922 testcase( p->op==TK_UNION );
2923 priorOp = SRT_Union;
2924 if( dest.eDest==priorOp ){
2925 /* We can reuse a temporary table generated by a SELECT to our
2926 ** right.
2927 */
2928 assert( p->pLimit==0 ); /* Not allowed on leftward elements */
2929 unionTab = dest.iSDParm;
2930 }else{
2931 /* We will need to create our own temporary table to hold the
2932 ** intermediate results.
2933 */
2934 unionTab = pParse->nTab++;
2935 assert( p->pOrderBy==0 );
2936 addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, unionTab, 0);
2937 assert( p->addrOpenEphm[0] == -1 );
2938 p->addrOpenEphm[0] = addr;
2939 findRightmost(p)->selFlags |= SF_UsesEphemeral;
2940 assert( p->pEList );
2941 }
drh34055852020-10-19 01:23:48 +00002942
drh03c39052018-05-02 14:24:34 +00002943
2944 /* Code the SELECT statements to our left
drhd8bc7082000-06-07 23:51:50 +00002945 */
drh03c39052018-05-02 14:24:34 +00002946 assert( !pPrior->pOrderBy );
2947 sqlite3SelectDestInit(&uniondest, priorOp, unionTab);
drh93ffb502021-05-18 19:10:10 +00002948 SELECTTRACE(1, pParse, p, ("multiSelect EXCEPT/UNION left...\n"));
drh03c39052018-05-02 14:24:34 +00002949 rc = sqlite3Select(pParse, pPrior, &uniondest);
2950 if( rc ){
2951 goto multi_select_end;
2952 }
2953
2954 /* Code the current SELECT statement
drhd8bc7082000-06-07 23:51:50 +00002955 */
drh03c39052018-05-02 14:24:34 +00002956 if( p->op==TK_EXCEPT ){
2957 op = SRT_Except;
2958 }else{
2959 assert( p->op==TK_UNION );
2960 op = SRT_Union;
2961 }
2962 p->pPrior = 0;
2963 pLimit = p->pLimit;
2964 p->pLimit = 0;
2965 uniondest.eDest = op;
2966 ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE",
drhaae0f742021-03-04 16:03:32 +00002967 sqlite3SelectOpName(p->op)));
drh93ffb502021-05-18 19:10:10 +00002968 SELECTTRACE(1, pParse, p, ("multiSelect EXCEPT/UNION right...\n"));
drh03c39052018-05-02 14:24:34 +00002969 rc = sqlite3Select(pParse, p, &uniondest);
2970 testcase( rc!=SQLITE_OK );
drhb7cbf5c2020-06-15 13:51:34 +00002971 assert( p->pOrderBy==0 );
drh03c39052018-05-02 14:24:34 +00002972 pDelete = p->pPrior;
2973 p->pPrior = pPrior;
2974 p->pOrderBy = 0;
2975 if( p->op==TK_UNION ){
2976 p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow);
2977 }
2978 sqlite3ExprDelete(db, p->pLimit);
2979 p->pLimit = pLimit;
2980 p->iLimit = 0;
2981 p->iOffset = 0;
2982
2983 /* Convert the data in the temporary table into whatever form
2984 ** it is that we currently need.
2985 */
2986 assert( unionTab==dest.iSDParm || dest.eDest!=priorOp );
drha9ebfe22019-12-25 23:54:21 +00002987 assert( p->pEList || db->mallocFailed );
2988 if( dest.eDest!=priorOp && db->mallocFailed==0 ){
drh03c39052018-05-02 14:24:34 +00002989 int iCont, iBreak, iStart;
drhec4ccdb2018-12-29 02:26:59 +00002990 iBreak = sqlite3VdbeMakeLabel(pParse);
2991 iCont = sqlite3VdbeMakeLabel(pParse);
drh03c39052018-05-02 14:24:34 +00002992 computeLimitRegisters(pParse, p, iBreak);
2993 sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); VdbeCoverage(v);
2994 iStart = sqlite3VdbeCurrentAddr(v);
2995 selectInnerLoop(pParse, p, unionTab,
2996 0, 0, &dest, iCont, iBreak);
2997 sqlite3VdbeResolveLabel(v, iCont);
2998 sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); VdbeCoverage(v);
2999 sqlite3VdbeResolveLabel(v, iBreak);
3000 sqlite3VdbeAddOp2(v, OP_Close, unionTab, 0);
3001 }
3002 break;
3003 }
3004 default: assert( p->op==TK_INTERSECT ); {
3005 int tab1, tab2;
3006 int iCont, iBreak, iStart;
3007 Expr *pLimit;
3008 int addr;
3009 SelectDest intersectdest;
3010 int r1;
3011
3012 /* INTERSECT is different from the others since it requires
3013 ** two temporary tables. Hence it has its own case. Begin
3014 ** by allocating the tables we will need.
3015 */
3016 tab1 = pParse->nTab++;
3017 tab2 = pParse->nTab++;
drh93a960a2008-07-10 00:32:42 +00003018 assert( p->pOrderBy==0 );
drh03c39052018-05-02 14:24:34 +00003019
3020 addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab1, 0);
drh93a960a2008-07-10 00:32:42 +00003021 assert( p->addrOpenEphm[0] == -1 );
3022 p->addrOpenEphm[0] = addr;
drhd227a292014-02-09 18:02:09 +00003023 findRightmost(p)->selFlags |= SF_UsesEphemeral;
danielk197784ac9d02004-05-18 09:58:06 +00003024 assert( p->pEList );
drh03c39052018-05-02 14:24:34 +00003025
3026 /* Code the SELECTs to our left into temporary table "tab1".
3027 */
3028 sqlite3SelectDestInit(&intersectdest, SRT_Union, tab1);
drh93ffb502021-05-18 19:10:10 +00003029 SELECTTRACE(1, pParse, p, ("multiSelect INTERSECT left...\n"));
drh03c39052018-05-02 14:24:34 +00003030 rc = sqlite3Select(pParse, pPrior, &intersectdest);
3031 if( rc ){
3032 goto multi_select_end;
3033 }
3034
3035 /* Code the current SELECT into temporary table "tab2"
3036 */
3037 addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab2, 0);
3038 assert( p->addrOpenEphm[1] == -1 );
3039 p->addrOpenEphm[1] = addr;
3040 p->pPrior = 0;
3041 pLimit = p->pLimit;
3042 p->pLimit = 0;
3043 intersectdest.iSDParm = tab2;
3044 ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE",
drhaae0f742021-03-04 16:03:32 +00003045 sqlite3SelectOpName(p->op)));
drh93ffb502021-05-18 19:10:10 +00003046 SELECTTRACE(1, pParse, p, ("multiSelect INTERSECT right...\n"));
drh03c39052018-05-02 14:24:34 +00003047 rc = sqlite3Select(pParse, p, &intersectdest);
3048 testcase( rc!=SQLITE_OK );
3049 pDelete = p->pPrior;
3050 p->pPrior = pPrior;
3051 if( p->nSelectRow>pPrior->nSelectRow ){
3052 p->nSelectRow = pPrior->nSelectRow;
3053 }
3054 sqlite3ExprDelete(db, p->pLimit);
3055 p->pLimit = pLimit;
3056
3057 /* Generate code to take the intersection of the two temporary
3058 ** tables.
3059 */
drh5f695122020-02-20 14:08:51 +00003060 if( rc ) break;
drh82c3d632000-06-06 21:56:07 +00003061 assert( p->pEList );
drhec4ccdb2018-12-29 02:26:59 +00003062 iBreak = sqlite3VdbeMakeLabel(pParse);
3063 iCont = sqlite3VdbeMakeLabel(pParse);
drhec7429a2005-10-06 16:53:14 +00003064 computeLimitRegisters(pParse, p, iBreak);
drh03c39052018-05-02 14:24:34 +00003065 sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak); VdbeCoverage(v);
3066 r1 = sqlite3GetTempReg(pParse);
3067 iStart = sqlite3VdbeAddOp2(v, OP_RowData, tab1, r1);
3068 sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0);
3069 VdbeCoverage(v);
3070 sqlite3ReleaseTempReg(pParse, r1);
3071 selectInnerLoop(pParse, p, tab1,
drhe8e4af72012-09-21 00:04:28 +00003072 0, 0, &dest, iCont, iBreak);
danielk19774adee202004-05-08 08:23:19 +00003073 sqlite3VdbeResolveLabel(v, iCont);
drh03c39052018-05-02 14:24:34 +00003074 sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); VdbeCoverage(v);
danielk19774adee202004-05-08 08:23:19 +00003075 sqlite3VdbeResolveLabel(v, iBreak);
drh03c39052018-05-02 14:24:34 +00003076 sqlite3VdbeAddOp2(v, OP_Close, tab2, 0);
3077 sqlite3VdbeAddOp2(v, OP_Close, tab1, 0);
3078 break;
drh82c3d632000-06-06 21:56:07 +00003079 }
drh82c3d632000-06-06 21:56:07 +00003080 }
drh03c39052018-05-02 14:24:34 +00003081
3082 #ifndef SQLITE_OMIT_EXPLAIN
3083 if( p->pNext==0 ){
3084 ExplainQueryPlanPop(pParse);
drh82c3d632000-06-06 21:56:07 +00003085 }
drh03c39052018-05-02 14:24:34 +00003086 #endif
drh82c3d632000-06-06 21:56:07 +00003087 }
drh8428b3b2019-12-19 22:08:19 +00003088 if( pParse->nErr ) goto multi_select_end;
drh03c39052018-05-02 14:24:34 +00003089
drha9671a22008-07-08 23:40:20 +00003090 /* Compute collating sequences used by
3091 ** temporary tables needed to implement the compound select.
3092 ** Attach the KeyInfo structure to all temporary tables.
drh8cdbf832004-08-29 16:25:03 +00003093 **
3094 ** This section is run by the right-most SELECT statement only.
3095 ** SELECT statements to the left always skip this part. The right-most
3096 ** SELECT might also skip this part if it has no ORDER BY clause and
3097 ** no temp tables are required.
drhfbc4ee72004-08-29 01:31:05 +00003098 */
drh7d10d5a2008-08-20 16:35:10 +00003099 if( p->selFlags & SF_UsesEphemeral ){
drhfbc4ee72004-08-29 01:31:05 +00003100 int i; /* Loop counter */
3101 KeyInfo *pKeyInfo; /* Collating sequence for the result set */
drh0342b1f2005-09-01 03:07:44 +00003102 Select *pLoop; /* For looping through SELECT statements */
drhf68d7d12007-05-03 13:02:26 +00003103 CollSeq **apColl; /* For looping through pKeyInfo->aColl[] */
drh93a960a2008-07-10 00:32:42 +00003104 int nCol; /* Number of columns in result set */
drhfbc4ee72004-08-29 01:31:05 +00003105
drhd227a292014-02-09 18:02:09 +00003106 assert( p->pNext==0 );
drhaa6fe5b2021-10-04 13:18:44 +00003107 assert( p->pEList!=0 );
drh93a960a2008-07-10 00:32:42 +00003108 nCol = p->pEList->nExpr;
drhad124322013-10-23 13:30:58 +00003109 pKeyInfo = sqlite3KeyInfoAlloc(db, nCol, 1);
danielk1977dc1bdc42004-06-11 10:51:27 +00003110 if( !pKeyInfo ){
mistachkinfad30392016-02-13 23:43:46 +00003111 rc = SQLITE_NOMEM_BKPT;
danielk1977dc1bdc42004-06-11 10:51:27 +00003112 goto multi_select_end;
3113 }
drh0342b1f2005-09-01 03:07:44 +00003114 for(i=0, apColl=pKeyInfo->aColl; i<nCol; i++, apColl++){
3115 *apColl = multiSelectCollSeq(pParse, p, i);
3116 if( 0==*apColl ){
drh633e6d52008-07-28 19:34:53 +00003117 *apColl = db->pDfltColl;
danielk1977dc1bdc42004-06-11 10:51:27 +00003118 }
3119 }
3120
drh0342b1f2005-09-01 03:07:44 +00003121 for(pLoop=p; pLoop; pLoop=pLoop->pPrior){
3122 for(i=0; i<2; i++){
drhb9bb7c12006-06-11 23:41:55 +00003123 int addr = pLoop->addrOpenEphm[i];
drh0342b1f2005-09-01 03:07:44 +00003124 if( addr<0 ){
3125 /* If [0] is unused then [1] is also unused. So we can
3126 ** always safely abort as soon as the first unused slot is found */
drhb9bb7c12006-06-11 23:41:55 +00003127 assert( pLoop->addrOpenEphm[1]<0 );
drh0342b1f2005-09-01 03:07:44 +00003128 break;
3129 }
3130 sqlite3VdbeChangeP2(v, addr, nCol);
drh2ec2fb22013-11-06 19:59:23 +00003131 sqlite3VdbeChangeP4(v, addr, (char*)sqlite3KeyInfoRef(pKeyInfo),
3132 P4_KEYINFO);
drh0ee5a1e2007-01-26 19:04:00 +00003133 pLoop->addrOpenEphm[i] = -1;
drh0342b1f2005-09-01 03:07:44 +00003134 }
danielk1977dc1bdc42004-06-11 10:51:27 +00003135 }
drh2ec2fb22013-11-06 19:59:23 +00003136 sqlite3KeyInfoUnref(pKeyInfo);
danielk197784ac9d02004-05-18 09:58:06 +00003137 }
danielk1977dc1bdc42004-06-11 10:51:27 +00003138
3139multi_select_end:
drh2b596da2012-07-23 21:43:19 +00003140 pDest->iSdst = dest.iSdst;
3141 pDest->nSdst = dest.nSdst;
drhce68b6b2021-08-21 16:42:58 +00003142 if( pDelete ){
3143 sqlite3ParserAddCleanup(pParse,
3144 (void(*)(sqlite3*,void*))sqlite3SelectDelete,
3145 pDelete);
3146 }
danielk197784ac9d02004-05-18 09:58:06 +00003147 return rc;
drh22827922000-06-06 17:27:05 +00003148}
drhb7f91642004-10-31 02:22:47 +00003149#endif /* SQLITE_OMIT_COMPOUND_SELECT */
drh22827922000-06-06 17:27:05 +00003150
drhb21e7c72008-06-22 12:37:57 +00003151/*
mistachkin89b31d72015-07-16 17:29:27 +00003152** Error message for when two or more terms of a compound select have different
3153** size result sets.
3154*/
3155void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p){
3156 if( p->selFlags & SF_Values ){
3157 sqlite3ErrorMsg(pParse, "all VALUES must have the same number of terms");
3158 }else{
3159 sqlite3ErrorMsg(pParse, "SELECTs to the left and right of %s"
drhaae0f742021-03-04 16:03:32 +00003160 " do not have the same number of result columns",
3161 sqlite3SelectOpName(p->op));
mistachkin89b31d72015-07-16 17:29:27 +00003162 }
3163}
3164
3165/*
drhb21e7c72008-06-22 12:37:57 +00003166** Code an output subroutine for a coroutine implementation of a
3167** SELECT statment.
drh0acb7e42008-06-25 00:12:41 +00003168**
drh2b596da2012-07-23 21:43:19 +00003169** The data to be output is contained in pIn->iSdst. There are
3170** pIn->nSdst columns to be output. pDest is where the output should
drh0acb7e42008-06-25 00:12:41 +00003171** be sent.
3172**
3173** regReturn is the number of the register holding the subroutine
3174** return address.
3175**
drhf053d5b2010-08-09 14:26:32 +00003176** If regPrev>0 then it is the first register in a vector that
drh0acb7e42008-06-25 00:12:41 +00003177** records the previous output. mem[regPrev] is a flag that is false
3178** if there has been no previous output. If regPrev>0 then code is
3179** generated to suppress duplicates. pKeyInfo is used for comparing
3180** keys.
3181**
3182** If the LIMIT found in p->iLimit is reached, jump immediately to
3183** iBreak.
drhb21e7c72008-06-22 12:37:57 +00003184*/
drh0acb7e42008-06-25 00:12:41 +00003185static int generateOutputSubroutine(
drh92b01d52008-06-24 00:32:35 +00003186 Parse *pParse, /* Parsing context */
3187 Select *p, /* The SELECT statement */
3188 SelectDest *pIn, /* Coroutine supplying data */
3189 SelectDest *pDest, /* Where to send the data */
3190 int regReturn, /* The return address register */
drh0acb7e42008-06-25 00:12:41 +00003191 int regPrev, /* Previous result register. No uniqueness if 0 */
3192 KeyInfo *pKeyInfo, /* For comparing with previous entry */
drh92b01d52008-06-24 00:32:35 +00003193 int iBreak /* Jump here if we hit the LIMIT */
drhb21e7c72008-06-22 12:37:57 +00003194){
3195 Vdbe *v = pParse->pVdbe;
drh92b01d52008-06-24 00:32:35 +00003196 int iContinue;
3197 int addr;
drhb21e7c72008-06-22 12:37:57 +00003198
drh92b01d52008-06-24 00:32:35 +00003199 addr = sqlite3VdbeCurrentAddr(v);
drhec4ccdb2018-12-29 02:26:59 +00003200 iContinue = sqlite3VdbeMakeLabel(pParse);
drh0acb7e42008-06-25 00:12:41 +00003201
3202 /* Suppress duplicates for UNION, EXCEPT, and INTERSECT
3203 */
3204 if( regPrev ){
drh728e0f92015-10-10 14:41:28 +00003205 int addr1, addr2;
3206 addr1 = sqlite3VdbeAddOp1(v, OP_IfNot, regPrev); VdbeCoverage(v);
3207 addr2 = sqlite3VdbeAddOp4(v, OP_Compare, pIn->iSdst, regPrev+1, pIn->nSdst,
drh2ec2fb22013-11-06 19:59:23 +00003208 (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO);
drh728e0f92015-10-10 14:41:28 +00003209 sqlite3VdbeAddOp3(v, OP_Jump, addr2+2, iContinue, addr2+2); VdbeCoverage(v);
3210 sqlite3VdbeJumpHere(v, addr1);
drhe8e4af72012-09-21 00:04:28 +00003211 sqlite3VdbeAddOp3(v, OP_Copy, pIn->iSdst, regPrev+1, pIn->nSdst-1);
drhec86c722011-12-09 17:27:51 +00003212 sqlite3VdbeAddOp2(v, OP_Integer, 1, regPrev);
drh0acb7e42008-06-25 00:12:41 +00003213 }
danielk19771f9caa42008-07-02 16:10:45 +00003214 if( pParse->db->mallocFailed ) return 0;
drh0acb7e42008-06-25 00:12:41 +00003215
mistachkind5578432012-08-25 10:01:29 +00003216 /* Suppress the first OFFSET entries if there is an OFFSET clause
drh0acb7e42008-06-25 00:12:41 +00003217 */
drhaa9ce702014-01-22 18:07:04 +00003218 codeOffset(v, p->iOffset, iContinue);
drhb21e7c72008-06-22 12:37:57 +00003219
drhe2248cf2015-05-22 17:29:27 +00003220 assert( pDest->eDest!=SRT_Exists );
3221 assert( pDest->eDest!=SRT_Table );
drhb21e7c72008-06-22 12:37:57 +00003222 switch( pDest->eDest ){
3223 /* Store the result as data using a unique key.
3224 */
drhb21e7c72008-06-22 12:37:57 +00003225 case SRT_EphemTab: {
3226 int r1 = sqlite3GetTempReg(pParse);
3227 int r2 = sqlite3GetTempReg(pParse);
drh2b596da2012-07-23 21:43:19 +00003228 sqlite3VdbeAddOp3(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, r1);
3229 sqlite3VdbeAddOp2(v, OP_NewRowid, pDest->iSDParm, r2);
3230 sqlite3VdbeAddOp3(v, OP_Insert, pDest->iSDParm, r1, r2);
drhb21e7c72008-06-22 12:37:57 +00003231 sqlite3VdbeChangeP5(v, OPFLAG_APPEND);
3232 sqlite3ReleaseTempReg(pParse, r2);
3233 sqlite3ReleaseTempReg(pParse, r1);
3234 break;
3235 }
3236
3237#ifndef SQLITE_OMIT_SUBQUERY
drh14318072016-09-06 18:51:25 +00003238 /* If we are creating a set for an "expr IN (SELECT ...)".
drhb21e7c72008-06-22 12:37:57 +00003239 */
3240 case SRT_Set: {
drh6fccc352008-06-27 00:52:45 +00003241 int r1;
drh63cecc42016-09-06 19:08:21 +00003242 testcase( pIn->nSdst>1 );
drhb21e7c72008-06-22 12:37:57 +00003243 r1 = sqlite3GetTempReg(pParse);
dan71c57db2016-07-09 20:23:55 +00003244 sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst,
drh14318072016-09-06 18:51:25 +00003245 r1, pDest->zAffSdst, pIn->nSdst);
drh9b4eaeb2016-11-09 00:10:33 +00003246 sqlite3VdbeAddOp4Int(v, OP_IdxInsert, pDest->iSDParm, r1,
3247 pIn->iSdst, pIn->nSdst);
drhb21e7c72008-06-22 12:37:57 +00003248 sqlite3ReleaseTempReg(pParse, r1);
drhb21e7c72008-06-22 12:37:57 +00003249 break;
3250 }
3251
drhb21e7c72008-06-22 12:37:57 +00003252 /* If this is a scalar select that is part of an expression, then
3253 ** store the results in the appropriate memory cell and break out
drhcb99c572019-08-09 20:26:01 +00003254 ** of the scan loop. Note that the select might return multiple columns
3255 ** if it is the RHS of a row-value IN operator.
drhb21e7c72008-06-22 12:37:57 +00003256 */
3257 case SRT_Mem: {
drh5f0922b2021-03-04 16:32:28 +00003258 testcase( pIn->nSdst>1 );
3259 sqlite3ExprCodeMove(pParse, pIn->iSdst, pDest->iSDParm, pIn->nSdst);
drhb21e7c72008-06-22 12:37:57 +00003260 /* The LIMIT clause will jump out of the loop for us */
3261 break;
3262 }
3263#endif /* #ifndef SQLITE_OMIT_SUBQUERY */
3264
drh7d10d5a2008-08-20 16:35:10 +00003265 /* The results are stored in a sequence of registers
drh2b596da2012-07-23 21:43:19 +00003266 ** starting at pDest->iSdst. Then the co-routine yields.
drhb21e7c72008-06-22 12:37:57 +00003267 */
drh92b01d52008-06-24 00:32:35 +00003268 case SRT_Coroutine: {
drh2b596da2012-07-23 21:43:19 +00003269 if( pDest->iSdst==0 ){
3270 pDest->iSdst = sqlite3GetTempRange(pParse, pIn->nSdst);
3271 pDest->nSdst = pIn->nSdst;
drhb21e7c72008-06-22 12:37:57 +00003272 }
dan4b79bde2015-04-21 15:49:04 +00003273 sqlite3ExprCodeMove(pParse, pIn->iSdst, pDest->iSdst, pIn->nSdst);
drh2b596da2012-07-23 21:43:19 +00003274 sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm);
drh92b01d52008-06-24 00:32:35 +00003275 break;
3276 }
3277
drhccfcbce2009-05-18 15:46:07 +00003278 /* If none of the above, then the result destination must be
3279 ** SRT_Output. This routine is never called with any other
3280 ** destination other than the ones handled above or SRT_Output.
3281 **
3282 ** For SRT_Output, results are stored in a sequence of registers.
3283 ** Then the OP_ResultRow opcode is used to cause sqlite3_step() to
3284 ** return the next row of result.
drh7d10d5a2008-08-20 16:35:10 +00003285 */
drhccfcbce2009-05-18 15:46:07 +00003286 default: {
3287 assert( pDest->eDest==SRT_Output );
drh2b596da2012-07-23 21:43:19 +00003288 sqlite3VdbeAddOp2(v, OP_ResultRow, pIn->iSdst, pIn->nSdst);
drhb21e7c72008-06-22 12:37:57 +00003289 break;
3290 }
drhb21e7c72008-06-22 12:37:57 +00003291 }
drh92b01d52008-06-24 00:32:35 +00003292
3293 /* Jump to the end of the loop if the LIMIT is reached.
3294 */
3295 if( p->iLimit ){
drh16897072015-03-07 00:57:37 +00003296 sqlite3VdbeAddOp2(v, OP_DecrJumpZero, p->iLimit, iBreak); VdbeCoverage(v);
drh92b01d52008-06-24 00:32:35 +00003297 }
3298
drh92b01d52008-06-24 00:32:35 +00003299 /* Generate the subroutine return
3300 */
drh0acb7e42008-06-25 00:12:41 +00003301 sqlite3VdbeResolveLabel(v, iContinue);
drh92b01d52008-06-24 00:32:35 +00003302 sqlite3VdbeAddOp1(v, OP_Return, regReturn);
3303
3304 return addr;
drhb21e7c72008-06-22 12:37:57 +00003305}
3306
3307/*
3308** Alternative compound select code generator for cases when there
3309** is an ORDER BY clause.
3310**
3311** We assume a query of the following form:
3312**
3313** <selectA> <operator> <selectB> ORDER BY <orderbylist>
3314**
3315** <operator> is one of UNION ALL, UNION, EXCEPT, or INTERSECT. The idea
3316** is to code both <selectA> and <selectB> with the ORDER BY clause as
3317** co-routines. Then run the co-routines in parallel and merge the results
3318** into the output. In addition to the two coroutines (called selectA and
3319** selectB) there are 7 subroutines:
3320**
3321** outA: Move the output of the selectA coroutine into the output
3322** of the compound query.
3323**
3324** outB: Move the output of the selectB coroutine into the output
3325** of the compound query. (Only generated for UNION and
3326** UNION ALL. EXCEPT and INSERTSECT never output a row that
3327** appears only in B.)
3328**
3329** AltB: Called when there is data from both coroutines and A<B.
3330**
3331** AeqB: Called when there is data from both coroutines and A==B.
3332**
3333** AgtB: Called when there is data from both coroutines and A>B.
3334**
3335** EofA: Called when data is exhausted from selectA.
3336**
3337** EofB: Called when data is exhausted from selectB.
3338**
3339** The implementation of the latter five subroutines depend on which
3340** <operator> is used:
3341**
3342**
3343** UNION ALL UNION EXCEPT INTERSECT
3344** ------------- ----------------- -------------- -----------------
drh0acb7e42008-06-25 00:12:41 +00003345** AltB: outA, nextA outA, nextA outA, nextA nextA
drhb21e7c72008-06-22 12:37:57 +00003346**
drh0acb7e42008-06-25 00:12:41 +00003347** AeqB: outA, nextA nextA nextA outA, nextA
drhb21e7c72008-06-22 12:37:57 +00003348**
drh0acb7e42008-06-25 00:12:41 +00003349** AgtB: outB, nextB outB, nextB nextB nextB
drhb21e7c72008-06-22 12:37:57 +00003350**
drh0acb7e42008-06-25 00:12:41 +00003351** EofA: outB, nextB outB, nextB halt halt
drhb21e7c72008-06-22 12:37:57 +00003352**
drh0acb7e42008-06-25 00:12:41 +00003353** EofB: outA, nextA outA, nextA outA, nextA halt
3354**
3355** In the AltB, AeqB, and AgtB subroutines, an EOF on A following nextA
3356** causes an immediate jump to EofA and an EOF on B following nextB causes
3357** an immediate jump to EofB. Within EofA and EofB, and EOF on entry or
3358** following nextX causes a jump to the end of the select processing.
3359**
3360** Duplicate removal in the UNION, EXCEPT, and INTERSECT cases is handled
3361** within the output subroutine. The regPrev register set holds the previously
3362** output value. A comparison is made against this value and the output
3363** is skipped if the next results would be the same as the previous.
drhb21e7c72008-06-22 12:37:57 +00003364**
3365** The implementation plan is to implement the two coroutines and seven
3366** subroutines first, then put the control logic at the bottom. Like this:
3367**
3368** goto Init
3369** coA: coroutine for left query (A)
3370** coB: coroutine for right query (B)
3371** outA: output one row of A
3372** outB: output one row of B (UNION and UNION ALL only)
3373** EofA: ...
3374** EofB: ...
3375** AltB: ...
3376** AeqB: ...
3377** AgtB: ...
3378** Init: initialize coroutine registers
3379** yield coA
3380** if eof(A) goto EofA
3381** yield coB
3382** if eof(B) goto EofB
3383** Cmpr: Compare A, B
3384** Jump AltB, AeqB, AgtB
3385** End: ...
3386**
3387** We call AltB, AeqB, AgtB, EofA, and EofB "subroutines" but they are not
3388** actually called using Gosub and they do not Return. EofA and EofB loop
3389** until all data is exhausted then jump to the "end" labe. AltB, AeqB,
3390** and AgtB jump to either L2 or to one of EofA or EofB.
3391*/
danielk1977de3e41e2008-08-04 03:51:24 +00003392#ifndef SQLITE_OMIT_COMPOUND_SELECT
drhb21e7c72008-06-22 12:37:57 +00003393static int multiSelectOrderBy(
3394 Parse *pParse, /* Parsing context */
3395 Select *p, /* The right-most of SELECTs to be coded */
drha9671a22008-07-08 23:40:20 +00003396 SelectDest *pDest /* What to do with query results */
drhb21e7c72008-06-22 12:37:57 +00003397){
drh0acb7e42008-06-25 00:12:41 +00003398 int i, j; /* Loop counters */
drhb21e7c72008-06-22 12:37:57 +00003399 Select *pPrior; /* Another SELECT immediately to our left */
drh38cebe02021-12-30 00:37:11 +00003400 Select *pSplit; /* Left-most SELECT in the right-hand group */
3401 int nSelect; /* Number of SELECT statements in the compound */
drhb21e7c72008-06-22 12:37:57 +00003402 Vdbe *v; /* Generate code to this VDBE */
drhb21e7c72008-06-22 12:37:57 +00003403 SelectDest destA; /* Destination for coroutine A */
3404 SelectDest destB; /* Destination for coroutine B */
drh92b01d52008-06-24 00:32:35 +00003405 int regAddrA; /* Address register for select-A coroutine */
drh92b01d52008-06-24 00:32:35 +00003406 int regAddrB; /* Address register for select-B coroutine */
drh92b01d52008-06-24 00:32:35 +00003407 int addrSelectA; /* Address of the select-A coroutine */
3408 int addrSelectB; /* Address of the select-B coroutine */
3409 int regOutA; /* Address register for the output-A subroutine */
3410 int regOutB; /* Address register for the output-B subroutine */
3411 int addrOutA; /* Address of the output-A subroutine */
drhb27b7f52008-12-10 18:03:45 +00003412 int addrOutB = 0; /* Address of the output-B subroutine */
drh92b01d52008-06-24 00:32:35 +00003413 int addrEofA; /* Address of the select-A-exhausted subroutine */
drh81cf13e2014-02-07 18:27:53 +00003414 int addrEofA_noB; /* Alternate addrEofA if B is uninitialized */
drh92b01d52008-06-24 00:32:35 +00003415 int addrEofB; /* Address of the select-B-exhausted subroutine */
3416 int addrAltB; /* Address of the A<B subroutine */
3417 int addrAeqB; /* Address of the A==B subroutine */
3418 int addrAgtB; /* Address of the A>B subroutine */
3419 int regLimitA; /* Limit register for select-A */
3420 int regLimitB; /* Limit register for select-A */
drh0acb7e42008-06-25 00:12:41 +00003421 int regPrev; /* A range of registers to hold previous output */
drh92b01d52008-06-24 00:32:35 +00003422 int savedLimit; /* Saved value of p->iLimit */
3423 int savedOffset; /* Saved value of p->iOffset */
3424 int labelCmpr; /* Label for the start of the merge algorithm */
3425 int labelEnd; /* Label for the end of the overall SELECT stmt */
drh728e0f92015-10-10 14:41:28 +00003426 int addr1; /* Jump instructions that get retargetted */
drh92b01d52008-06-24 00:32:35 +00003427 int op; /* One of TK_ALL, TK_UNION, TK_EXCEPT, TK_INTERSECT */
drh96067812008-09-23 09:36:10 +00003428 KeyInfo *pKeyDup = 0; /* Comparison information for duplicate removal */
drh0acb7e42008-06-25 00:12:41 +00003429 KeyInfo *pKeyMerge; /* Comparison information for merging rows */
3430 sqlite3 *db; /* Database connection */
3431 ExprList *pOrderBy; /* The ORDER BY clause */
3432 int nOrderBy; /* Number of terms in the ORDER BY clause */
drhabc38152020-07-22 13:38:04 +00003433 u32 *aPermute; /* Mapping from ORDER BY terms to result set columns */
drhb21e7c72008-06-22 12:37:57 +00003434
drh92b01d52008-06-24 00:32:35 +00003435 assert( p->pOrderBy!=0 );
drh96067812008-09-23 09:36:10 +00003436 assert( pKeyDup==0 ); /* "Managed" code needs this. Ticket #3382. */
drh0acb7e42008-06-25 00:12:41 +00003437 db = pParse->db;
drh92b01d52008-06-24 00:32:35 +00003438 v = pParse->pVdbe;
drhccfcbce2009-05-18 15:46:07 +00003439 assert( v!=0 ); /* Already thrown the error if VDBE alloc failed */
drhec4ccdb2018-12-29 02:26:59 +00003440 labelEnd = sqlite3VdbeMakeLabel(pParse);
3441 labelCmpr = sqlite3VdbeMakeLabel(pParse);
drh0acb7e42008-06-25 00:12:41 +00003442
drh92b01d52008-06-24 00:32:35 +00003443
3444 /* Patch up the ORDER BY clause
3445 */
3446 op = p->op;
drh38cebe02021-12-30 00:37:11 +00003447 assert( p->pPrior->pOrderBy==0 );
drh0acb7e42008-06-25 00:12:41 +00003448 pOrderBy = p->pOrderBy;
drh93a960a2008-07-10 00:32:42 +00003449 assert( pOrderBy );
drh93a960a2008-07-10 00:32:42 +00003450 nOrderBy = pOrderBy->nExpr;
3451
drh0acb7e42008-06-25 00:12:41 +00003452 /* For operators other than UNION ALL we have to make sure that
3453 ** the ORDER BY clause covers every term of the result set. Add
3454 ** terms to the ORDER BY clause as necessary.
3455 */
3456 if( op!=TK_ALL ){
3457 for(i=1; db->mallocFailed==0 && i<=p->pEList->nExpr; i++){
drh7d10d5a2008-08-20 16:35:10 +00003458 struct ExprList_item *pItem;
3459 for(j=0, pItem=pOrderBy->a; j<nOrderBy; j++, pItem++){
drh7d4c94b2021-10-04 22:34:38 +00003460 assert( pItem!=0 );
drhc2acc4e2013-11-15 18:15:19 +00003461 assert( pItem->u.x.iOrderByCol>0 );
3462 if( pItem->u.x.iOrderByCol==i ) break;
drh0acb7e42008-06-25 00:12:41 +00003463 }
3464 if( j==nOrderBy ){
drhb7916a72009-05-27 10:31:29 +00003465 Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0);
mistachkinfad30392016-02-13 23:43:46 +00003466 if( pNew==0 ) return SQLITE_NOMEM_BKPT;
drh0acb7e42008-06-25 00:12:41 +00003467 pNew->flags |= EP_IntValue;
drh33e619f2009-05-28 01:00:55 +00003468 pNew->u.iValue = i;
drh43606172017-04-05 11:32:13 +00003469 p->pOrderBy = pOrderBy = sqlite3ExprListAppend(pParse, pOrderBy, pNew);
drhc2acc4e2013-11-15 18:15:19 +00003470 if( pOrderBy ) pOrderBy->a[nOrderBy++].u.x.iOrderByCol = (u16)i;
drh0acb7e42008-06-25 00:12:41 +00003471 }
3472 }
3473 }
3474
3475 /* Compute the comparison permutation and keyinfo that is used with
drh10c081a2009-04-16 00:24:23 +00003476 ** the permutation used to determine if the next
drh0acb7e42008-06-25 00:12:41 +00003477 ** row of results comes from selectA or selectB. Also add explicit
3478 ** collations to the ORDER BY clause terms so that when the subqueries
3479 ** to the right and the left are evaluated, they use the correct
3480 ** collation.
3481 */
drhabc38152020-07-22 13:38:04 +00003482 aPermute = sqlite3DbMallocRawNN(db, sizeof(u32)*(nOrderBy + 1));
drh0acb7e42008-06-25 00:12:41 +00003483 if( aPermute ){
drh7d10d5a2008-08-20 16:35:10 +00003484 struct ExprList_item *pItem;
drhb1702022016-01-30 00:45:18 +00003485 aPermute[0] = nOrderBy;
3486 for(i=1, pItem=pOrderBy->a; i<=nOrderBy; i++, pItem++){
drh7d4c94b2021-10-04 22:34:38 +00003487 assert( pItem!=0 );
drh67366182015-04-16 14:33:35 +00003488 assert( pItem->u.x.iOrderByCol>0 );
drh2ec18a32015-06-23 23:31:52 +00003489 assert( pItem->u.x.iOrderByCol<=p->pEList->nExpr );
drhc2acc4e2013-11-15 18:15:19 +00003490 aPermute[i] = pItem->u.x.iOrderByCol - 1;
drh0acb7e42008-06-25 00:12:41 +00003491 }
dan53bed452014-01-24 20:37:18 +00003492 pKeyMerge = multiSelectOrderByKeyInfo(pParse, p, 1);
drh0acb7e42008-06-25 00:12:41 +00003493 }else{
3494 pKeyMerge = 0;
3495 }
3496
drh0acb7e42008-06-25 00:12:41 +00003497 /* Allocate a range of temporary registers and the KeyInfo needed
3498 ** for the logic that removes duplicate result rows when the
3499 ** operator is UNION, EXCEPT, or INTERSECT (but not UNION ALL).
3500 */
3501 if( op==TK_ALL ){
3502 regPrev = 0;
3503 }else{
3504 int nExpr = p->pEList->nExpr;
drh1c0dc822008-10-30 22:13:23 +00003505 assert( nOrderBy>=nExpr || db->mallocFailed );
drhc8ac0d12013-02-16 02:41:01 +00003506 regPrev = pParse->nMem+1;
3507 pParse->nMem += nExpr+1;
drh0acb7e42008-06-25 00:12:41 +00003508 sqlite3VdbeAddOp2(v, OP_Integer, 0, regPrev);
drhad124322013-10-23 13:30:58 +00003509 pKeyDup = sqlite3KeyInfoAlloc(db, nExpr, 1);
drh0acb7e42008-06-25 00:12:41 +00003510 if( pKeyDup ){
drh2ec2fb22013-11-06 19:59:23 +00003511 assert( sqlite3KeyInfoIsWriteable(pKeyDup) );
drh0acb7e42008-06-25 00:12:41 +00003512 for(i=0; i<nExpr; i++){
3513 pKeyDup->aColl[i] = multiSelectCollSeq(pParse, p, i);
dan6e118922019-08-12 16:36:38 +00003514 pKeyDup->aSortFlags[i] = 0;
drh0acb7e42008-06-25 00:12:41 +00003515 }
3516 }
3517 }
drh92b01d52008-06-24 00:32:35 +00003518
3519 /* Separate the left and the right query from one another
3520 */
drh38cebe02021-12-30 00:37:11 +00003521 nSelect = 1;
3522 if( (op==TK_ALL || op==TK_UNION)
3523 && OptimizationEnabled(db, SQLITE_BalancedMerge)
3524 ){
3525 for(pSplit=p; pSplit->pPrior!=0 && pSplit->op==op; pSplit=pSplit->pPrior){
3526 nSelect++;
3527 assert( pSplit->pPrior->pNext==pSplit );
3528 }
3529 }
3530 if( nSelect<=3 ){
3531 pSplit = p;
3532 }else{
3533 pSplit = p;
3534 for(i=2; i<nSelect; i+=2){ pSplit = pSplit->pPrior; }
3535 }
3536 pPrior = pSplit->pPrior;
drh98bb34c2022-01-20 12:58:15 +00003537 assert( pPrior!=0 );
drh38cebe02021-12-30 00:37:11 +00003538 pSplit->pPrior = 0;
drhd227a292014-02-09 18:02:09 +00003539 pPrior->pNext = 0;
drh38cebe02021-12-30 00:37:11 +00003540 assert( p->pOrderBy == pOrderBy );
3541 assert( pOrderBy!=0 || db->mallocFailed );
3542 pPrior->pOrderBy = sqlite3ExprListDup(pParse->db, pOrderBy, 0);
drh9aff8242021-12-30 17:46:15 +00003543 sqlite3ResolveOrderGroupBy(pParse, p, p->pOrderBy, "ORDER");
3544 sqlite3ResolveOrderGroupBy(pParse, pPrior, pPrior->pOrderBy, "ORDER");
drh0acb7e42008-06-25 00:12:41 +00003545
drh92b01d52008-06-24 00:32:35 +00003546 /* Compute the limit registers */
3547 computeLimitRegisters(pParse, p, labelEnd);
drh0acb7e42008-06-25 00:12:41 +00003548 if( p->iLimit && op==TK_ALL ){
drh92b01d52008-06-24 00:32:35 +00003549 regLimitA = ++pParse->nMem;
3550 regLimitB = ++pParse->nMem;
3551 sqlite3VdbeAddOp2(v, OP_Copy, p->iOffset ? p->iOffset+1 : p->iLimit,
3552 regLimitA);
3553 sqlite3VdbeAddOp2(v, OP_Copy, regLimitA, regLimitB);
3554 }else{
3555 regLimitA = regLimitB = 0;
3556 }
drh633e6d52008-07-28 19:34:53 +00003557 sqlite3ExprDelete(db, p->pLimit);
drh0acb7e42008-06-25 00:12:41 +00003558 p->pLimit = 0;
drh92b01d52008-06-24 00:32:35 +00003559
drhb21e7c72008-06-22 12:37:57 +00003560 regAddrA = ++pParse->nMem;
drhb21e7c72008-06-22 12:37:57 +00003561 regAddrB = ++pParse->nMem;
drhb21e7c72008-06-22 12:37:57 +00003562 regOutA = ++pParse->nMem;
3563 regOutB = ++pParse->nMem;
3564 sqlite3SelectDestInit(&destA, SRT_Coroutine, regAddrA);
3565 sqlite3SelectDestInit(&destB, SRT_Coroutine, regAddrB);
drhe2ca99c2018-05-02 00:33:43 +00003566
drhaae0f742021-03-04 16:03:32 +00003567 ExplainQueryPlan((pParse, 1, "MERGE (%s)", sqlite3SelectOpName(p->op)));
drhb21e7c72008-06-22 12:37:57 +00003568
drh92b01d52008-06-24 00:32:35 +00003569 /* Generate a coroutine to evaluate the SELECT statement to the
drh0acb7e42008-06-25 00:12:41 +00003570 ** left of the compound operator - the "A" select.
3571 */
drhed71a832014-02-07 19:18:10 +00003572 addrSelectA = sqlite3VdbeCurrentAddr(v) + 1;
drh728e0f92015-10-10 14:41:28 +00003573 addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrA, 0, addrSelectA);
drhed71a832014-02-07 19:18:10 +00003574 VdbeComment((v, "left SELECT"));
drh92b01d52008-06-24 00:32:35 +00003575 pPrior->iLimit = regLimitA;
drhc631ded2018-05-02 02:22:22 +00003576 ExplainQueryPlan((pParse, 1, "LEFT"));
drh7d10d5a2008-08-20 16:35:10 +00003577 sqlite3Select(pParse, pPrior, &destA);
drh2fade2f2016-02-09 02:12:20 +00003578 sqlite3VdbeEndCoroutine(v, regAddrA);
drh728e0f92015-10-10 14:41:28 +00003579 sqlite3VdbeJumpHere(v, addr1);
drhb21e7c72008-06-22 12:37:57 +00003580
drh92b01d52008-06-24 00:32:35 +00003581 /* Generate a coroutine to evaluate the SELECT statement on
3582 ** the right - the "B" select
3583 */
drhed71a832014-02-07 19:18:10 +00003584 addrSelectB = sqlite3VdbeCurrentAddr(v) + 1;
drh728e0f92015-10-10 14:41:28 +00003585 addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrB, 0, addrSelectB);
drhed71a832014-02-07 19:18:10 +00003586 VdbeComment((v, "right SELECT"));
drh92b01d52008-06-24 00:32:35 +00003587 savedLimit = p->iLimit;
3588 savedOffset = p->iOffset;
3589 p->iLimit = regLimitB;
3590 p->iOffset = 0;
drhc631ded2018-05-02 02:22:22 +00003591 ExplainQueryPlan((pParse, 1, "RIGHT"));
drh7d10d5a2008-08-20 16:35:10 +00003592 sqlite3Select(pParse, p, &destB);
drh92b01d52008-06-24 00:32:35 +00003593 p->iLimit = savedLimit;
3594 p->iOffset = savedOffset;
drh2fade2f2016-02-09 02:12:20 +00003595 sqlite3VdbeEndCoroutine(v, regAddrB);
drhb21e7c72008-06-22 12:37:57 +00003596
drh92b01d52008-06-24 00:32:35 +00003597 /* Generate a subroutine that outputs the current row of the A
drh0acb7e42008-06-25 00:12:41 +00003598 ** select as the next output row of the compound select.
drh92b01d52008-06-24 00:32:35 +00003599 */
drhb21e7c72008-06-22 12:37:57 +00003600 VdbeNoopComment((v, "Output routine for A"));
drh0acb7e42008-06-25 00:12:41 +00003601 addrOutA = generateOutputSubroutine(pParse,
3602 p, &destA, pDest, regOutA,
drh2ec2fb22013-11-06 19:59:23 +00003603 regPrev, pKeyDup, labelEnd);
drhb21e7c72008-06-22 12:37:57 +00003604
drh92b01d52008-06-24 00:32:35 +00003605 /* Generate a subroutine that outputs the current row of the B
drh0acb7e42008-06-25 00:12:41 +00003606 ** select as the next output row of the compound select.
drh92b01d52008-06-24 00:32:35 +00003607 */
drh0acb7e42008-06-25 00:12:41 +00003608 if( op==TK_ALL || op==TK_UNION ){
3609 VdbeNoopComment((v, "Output routine for B"));
3610 addrOutB = generateOutputSubroutine(pParse,
3611 p, &destB, pDest, regOutB,
drh2ec2fb22013-11-06 19:59:23 +00003612 regPrev, pKeyDup, labelEnd);
drh0acb7e42008-06-25 00:12:41 +00003613 }
drh2ec2fb22013-11-06 19:59:23 +00003614 sqlite3KeyInfoUnref(pKeyDup);
drhb21e7c72008-06-22 12:37:57 +00003615
drh92b01d52008-06-24 00:32:35 +00003616 /* Generate a subroutine to run when the results from select A
3617 ** are exhausted and only data in select B remains.
3618 */
drhb21e7c72008-06-22 12:37:57 +00003619 if( op==TK_EXCEPT || op==TK_INTERSECT ){
drh81cf13e2014-02-07 18:27:53 +00003620 addrEofA_noB = addrEofA = labelEnd;
drhb21e7c72008-06-22 12:37:57 +00003621 }else{
drh81cf13e2014-02-07 18:27:53 +00003622 VdbeNoopComment((v, "eof-A subroutine"));
3623 addrEofA = sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB);
3624 addrEofA_noB = sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, labelEnd);
drh688852a2014-02-17 22:40:43 +00003625 VdbeCoverage(v);
drh076e85f2015-09-03 13:46:12 +00003626 sqlite3VdbeGoto(v, addrEofA);
drhc3489bb2016-02-25 16:04:59 +00003627 p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow);
drhb21e7c72008-06-22 12:37:57 +00003628 }
3629
drh92b01d52008-06-24 00:32:35 +00003630 /* Generate a subroutine to run when the results from select B
3631 ** are exhausted and only data in select A remains.
3632 */
drhb21e7c72008-06-22 12:37:57 +00003633 if( op==TK_INTERSECT ){
drh92b01d52008-06-24 00:32:35 +00003634 addrEofB = addrEofA;
drh95aa47b2010-11-16 02:49:15 +00003635 if( p->nSelectRow > pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow;
drhb21e7c72008-06-22 12:37:57 +00003636 }else{
drh92b01d52008-06-24 00:32:35 +00003637 VdbeNoopComment((v, "eof-B subroutine"));
drh81cf13e2014-02-07 18:27:53 +00003638 addrEofB = sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA);
drh688852a2014-02-17 22:40:43 +00003639 sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, labelEnd); VdbeCoverage(v);
drh076e85f2015-09-03 13:46:12 +00003640 sqlite3VdbeGoto(v, addrEofB);
drhb21e7c72008-06-22 12:37:57 +00003641 }
3642
drh92b01d52008-06-24 00:32:35 +00003643 /* Generate code to handle the case of A<B
3644 */
drhb21e7c72008-06-22 12:37:57 +00003645 VdbeNoopComment((v, "A-lt-B subroutine"));
drh0acb7e42008-06-25 00:12:41 +00003646 addrAltB = sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA);
drh688852a2014-02-17 22:40:43 +00003647 sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA); VdbeCoverage(v);
drh076e85f2015-09-03 13:46:12 +00003648 sqlite3VdbeGoto(v, labelCmpr);
drhb21e7c72008-06-22 12:37:57 +00003649
drh92b01d52008-06-24 00:32:35 +00003650 /* Generate code to handle the case of A==B
3651 */
drhb21e7c72008-06-22 12:37:57 +00003652 if( op==TK_ALL ){
3653 addrAeqB = addrAltB;
drh0acb7e42008-06-25 00:12:41 +00003654 }else if( op==TK_INTERSECT ){
3655 addrAeqB = addrAltB;
3656 addrAltB++;
drh92b01d52008-06-24 00:32:35 +00003657 }else{
drhb21e7c72008-06-22 12:37:57 +00003658 VdbeNoopComment((v, "A-eq-B subroutine"));
drh0acb7e42008-06-25 00:12:41 +00003659 addrAeqB =
drh688852a2014-02-17 22:40:43 +00003660 sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA); VdbeCoverage(v);
drh076e85f2015-09-03 13:46:12 +00003661 sqlite3VdbeGoto(v, labelCmpr);
drhb21e7c72008-06-22 12:37:57 +00003662 }
3663
drh92b01d52008-06-24 00:32:35 +00003664 /* Generate code to handle the case of A>B
3665 */
drhb21e7c72008-06-22 12:37:57 +00003666 VdbeNoopComment((v, "A-gt-B subroutine"));
3667 addrAgtB = sqlite3VdbeCurrentAddr(v);
3668 if( op==TK_ALL || op==TK_UNION ){
3669 sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB);
3670 }
drh688852a2014-02-17 22:40:43 +00003671 sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v);
drh076e85f2015-09-03 13:46:12 +00003672 sqlite3VdbeGoto(v, labelCmpr);
drhb21e7c72008-06-22 12:37:57 +00003673
drh92b01d52008-06-24 00:32:35 +00003674 /* This code runs once to initialize everything.
3675 */
drh728e0f92015-10-10 14:41:28 +00003676 sqlite3VdbeJumpHere(v, addr1);
drh688852a2014-02-17 22:40:43 +00003677 sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA_noB); VdbeCoverage(v);
3678 sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v);
drh92b01d52008-06-24 00:32:35 +00003679
3680 /* Implement the main merge loop
3681 */
3682 sqlite3VdbeResolveLabel(v, labelCmpr);
drh0acb7e42008-06-25 00:12:41 +00003683 sqlite3VdbeAddOp4(v, OP_Permutation, 0, 0, 0, (char*)aPermute, P4_INTARRAY);
drh2b596da2012-07-23 21:43:19 +00003684 sqlite3VdbeAddOp4(v, OP_Compare, destA.iSdst, destB.iSdst, nOrderBy,
drh2ec2fb22013-11-06 19:59:23 +00003685 (char*)pKeyMerge, P4_KEYINFO);
drh953f7612012-12-07 22:18:54 +00003686 sqlite3VdbeChangeP5(v, OPFLAG_PERMUTE);
drh688852a2014-02-17 22:40:43 +00003687 sqlite3VdbeAddOp3(v, OP_Jump, addrAltB, addrAeqB, addrAgtB); VdbeCoverage(v);
drh92b01d52008-06-24 00:32:35 +00003688
3689 /* Jump to the this point in order to terminate the query.
3690 */
drhb21e7c72008-06-22 12:37:57 +00003691 sqlite3VdbeResolveLabel(v, labelEnd);
drh92b01d52008-06-24 00:32:35 +00003692
drh44132242022-09-01 13:51:09 +00003693 /* Reassemble the compound query so that it will be freed correctly
drh0acb7e42008-06-25 00:12:41 +00003694 ** by the calling function */
drh38cebe02021-12-30 00:37:11 +00003695 if( pSplit->pPrior ){
drh44132242022-09-01 13:51:09 +00003696 sqlite3ParserAddCleanup(pParse,
3697 (void(*)(sqlite3*,void*))sqlite3SelectDelete, pSplit->pPrior);
danielk19775e7ad502008-07-01 17:39:27 +00003698 }
drh38cebe02021-12-30 00:37:11 +00003699 pSplit->pPrior = pPrior;
3700 pPrior->pNext = pSplit;
dancd0b2452021-04-12 12:02:49 +00003701 sqlite3ExprListDelete(db, pPrior->pOrderBy);
3702 pPrior->pOrderBy = 0;
3703
drh92b01d52008-06-24 00:32:35 +00003704 /*** TBD: Insert subroutine calls to close cursors on incomplete
3705 **** subqueries ****/
drhe2ca99c2018-05-02 00:33:43 +00003706 ExplainQueryPlanPop(pParse);
drh3dc4cc62015-04-15 07:10:25 +00003707 return pParse->nErr!=0;
drhb21e7c72008-06-22 12:37:57 +00003708}
danielk1977de3e41e2008-08-04 03:51:24 +00003709#endif
drhb21e7c72008-06-22 12:37:57 +00003710
shane3514b6f2008-07-22 05:00:55 +00003711#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
drh46967de2017-04-14 12:39:37 +00003712
3713/* An instance of the SubstContext object describes an substitution edit
3714** to be performed on a parse tree.
3715**
3716** All references to columns in table iTable are to be replaced by corresponding
3717** expressions in pEList.
drhc133bab2022-04-11 20:15:52 +00003718**
3719** ## About "isOuterJoin":
3720**
3721** The isOuterJoin column indicates that the replacement will occur into a
3722** position in the parent that NULL-able due to an OUTER JOIN. Either the
3723** target slot in the parent is the right operand of a LEFT JOIN, or one of
3724** the left operands of a RIGHT JOIN. In either case, we need to potentially
3725** bypass the substituted expression with OP_IfNullRow.
3726**
3727** Suppose the original expression integer constant. Even though the table
3728** has the nullRow flag set, because the expression is an integer constant,
3729** it will not be NULLed out. So instead, we insert an OP_IfNullRow opcode
3730** that checks to see if the nullRow flag is set on the table. If the nullRow
3731** flag is set, then the value in the register is set to NULL and the original
3732** expression is bypassed. If the nullRow flag is not set, then the original
3733** expression runs to populate the register.
3734**
3735** Example where this is needed:
3736**
3737** CREATE TABLE t1(a INTEGER PRIMARY KEY, b INT);
3738** CREATE TABLE t2(x INT UNIQUE);
3739**
3740** SELECT a,b,m,x FROM t1 LEFT JOIN (SELECT 59 AS m,x FROM t2) ON b=x;
3741**
3742** When the subquery on the right side of the LEFT JOIN is flattened, we
3743** have to add OP_IfNullRow in front of the OP_Integer that implements the
3744** "m" value of the subquery so that a NULL will be loaded instead of 59
3745** when processing a non-matched row of the left.
drh46967de2017-04-14 12:39:37 +00003746*/
3747typedef struct SubstContext {
3748 Parse *pParse; /* The parsing context */
3749 int iTable; /* Replace references to this table */
drh31d6fd52017-04-14 19:03:10 +00003750 int iNewTable; /* New table number */
drhc133bab2022-04-11 20:15:52 +00003751 int isOuterJoin; /* Add TK_IF_NULL_ROW opcodes on each replacement */
drh46967de2017-04-14 12:39:37 +00003752 ExprList *pEList; /* Replacement expressions */
3753} SubstContext;
3754
drh17435752007-08-16 04:30:38 +00003755/* Forward Declarations */
drh46967de2017-04-14 12:39:37 +00003756static void substExprList(SubstContext*, ExprList*);
3757static void substSelect(SubstContext*, Select*, int);
drh17435752007-08-16 04:30:38 +00003758
drh22827922000-06-06 17:27:05 +00003759/*
drh832508b2002-03-02 17:04:07 +00003760** Scan through the expression pExpr. Replace every reference to
drh6a3ea0e2003-05-02 14:32:12 +00003761** a column in table number iTable with a copy of the iColumn-th
drh84e59202002-03-14 14:33:31 +00003762** entry in pEList. (But leave references to the ROWID column
drh6a3ea0e2003-05-02 14:32:12 +00003763** unchanged.)
drh832508b2002-03-02 17:04:07 +00003764**
3765** This routine is part of the flattening procedure. A subquery
3766** whose result set is defined by pEList appears as entry in the
3767** FROM clause of a SELECT such that the VDBE cursor assigned to that
drhaca19e12017-04-07 19:41:31 +00003768** FORM clause entry is iTable. This routine makes the necessary
drh832508b2002-03-02 17:04:07 +00003769** changes to pExpr so that it refers directly to the source table
3770** of the subquery rather the result set of the subquery.
3771*/
drhb7916a72009-05-27 10:31:29 +00003772static Expr *substExpr(
drh46967de2017-04-14 12:39:37 +00003773 SubstContext *pSubst, /* Description of the substitution */
3774 Expr *pExpr /* Expr in which substitution occurs */
drh17435752007-08-16 04:30:38 +00003775){
drhb7916a72009-05-27 10:31:29 +00003776 if( pExpr==0 ) return 0;
drh33b2cb92022-06-07 13:09:58 +00003777 if( ExprHasProperty(pExpr, EP_OuterON|EP_InnerON)
drhd1985262022-04-11 11:25:28 +00003778 && pExpr->w.iJoin==pSubst->iTable
drh3d240d22017-09-28 16:56:55 +00003779 ){
drh33b2cb92022-06-07 13:09:58 +00003780 testcase( ExprHasProperty(pExpr, EP_InnerON) );
drhd1985262022-04-11 11:25:28 +00003781 pExpr->w.iJoin = pSubst->iNewTable;
drh399c7e22017-04-14 17:18:45 +00003782 }
drh2e52a9c2020-04-04 00:15:54 +00003783 if( pExpr->op==TK_COLUMN
3784 && pExpr->iTable==pSubst->iTable
3785 && !ExprHasProperty(pExpr, EP_FixedCol)
3786 ){
drh6e5020e2021-04-07 15:45:01 +00003787#ifdef SQLITE_ALLOW_ROWID_IN_VIEW
drh50350a12004-02-13 16:22:22 +00003788 if( pExpr->iColumn<0 ){
3789 pExpr->op = TK_NULL;
drh6e5020e2021-04-07 15:45:01 +00003790 }else
3791#endif
3792 {
drh50350a12004-02-13 16:22:22 +00003793 Expr *pNew;
drh46967de2017-04-14 12:39:37 +00003794 Expr *pCopy = pSubst->pEList->a[pExpr->iColumn].pExpr;
drh31d6fd52017-04-14 19:03:10 +00003795 Expr ifNullRow;
drh46967de2017-04-14 12:39:37 +00003796 assert( pSubst->pEList!=0 && pExpr->iColumn<pSubst->pEList->nExpr );
drh1fd4e7b2018-07-28 14:56:56 +00003797 assert( pExpr->pRight==0 );
dan44c56042016-12-07 15:38:37 +00003798 if( sqlite3ExprIsVector(pCopy) ){
drh46967de2017-04-14 12:39:37 +00003799 sqlite3VectorErrorMsg(pSubst->pParse, pCopy);
dan44c56042016-12-07 15:38:37 +00003800 }else{
drh46967de2017-04-14 12:39:37 +00003801 sqlite3 *db = pSubst->pParse->db;
drhc133bab2022-04-11 20:15:52 +00003802 if( pSubst->isOuterJoin && pCopy->op!=TK_COLUMN ){
drh31d6fd52017-04-14 19:03:10 +00003803 memset(&ifNullRow, 0, sizeof(ifNullRow));
3804 ifNullRow.op = TK_IF_NULL_ROW;
3805 ifNullRow.pLeft = pCopy;
3806 ifNullRow.iTable = pSubst->iNewTable;
drh4b1b65c2022-07-26 15:32:02 +00003807 ifNullRow.iColumn = -99;
drh46fe1382020-08-19 23:32:06 +00003808 ifNullRow.flags = EP_IfNullRow;
drh31d6fd52017-04-14 19:03:10 +00003809 pCopy = &ifNullRow;
3810 }
drh11df7d22018-12-06 19:15:36 +00003811 testcase( ExprHasProperty(pCopy, EP_Subquery) );
dan44c56042016-12-07 15:38:37 +00003812 pNew = sqlite3ExprDup(db, pCopy, 0);
drh8c6cb1b2021-04-19 20:36:13 +00003813 if( db->mallocFailed ){
3814 sqlite3ExprDelete(db, pNew);
3815 return pExpr;
3816 }
drhc133bab2022-04-11 20:15:52 +00003817 if( pSubst->isOuterJoin ){
danbd11a2a2017-06-20 17:43:26 +00003818 ExprSetProperty(pNew, EP_CanBeNull);
3819 }
drh67a99db2022-05-13 14:52:04 +00003820 if( ExprHasProperty(pExpr,EP_OuterON|EP_InnerON) ){
drh3a6e4c52022-04-11 12:38:06 +00003821 sqlite3SetJoinExpr(pNew, pExpr->w.iJoin,
drh67a99db2022-05-13 14:52:04 +00003822 pExpr->flags & (EP_OuterON|EP_InnerON));
dan44c56042016-12-07 15:38:37 +00003823 }
3824 sqlite3ExprDelete(db, pExpr);
3825 pExpr = pNew;
dan410fac32022-06-02 16:26:21 +00003826 if( pExpr->op==TK_TRUEFALSE ){
3827 pExpr->u.iValue = sqlite3ExprTruthValue(pExpr);
3828 pExpr->op = TK_INTEGER;
3829 ExprSetProperty(pExpr, EP_IntValue);
3830 }
dane0866392019-09-09 19:49:42 +00003831
danfa508342019-09-10 15:33:52 +00003832 /* Ensure that the expression now has an implicit collation sequence,
3833 ** just as it did when it was a column of a view or sub-query. */
drh60b95332021-04-19 15:05:27 +00003834 if( pExpr->op!=TK_COLUMN && pExpr->op!=TK_COLLATE ){
3835 CollSeq *pColl = sqlite3ExprCollSeq(pSubst->pParse, pExpr);
3836 pExpr = sqlite3ExprAddCollateString(pSubst->pParse, pExpr,
3837 (pColl ? pColl->zName : "BINARY")
3838 );
dane0866392019-09-09 19:49:42 +00003839 }
drh60b95332021-04-19 15:05:27 +00003840 ExprClearProperty(pExpr, EP_Collate);
dan92ddb3b2016-12-01 19:38:05 +00003841 }
drh50350a12004-02-13 16:22:22 +00003842 }
drh832508b2002-03-02 17:04:07 +00003843 }else{
drh7c1544e2017-05-23 12:44:57 +00003844 if( pExpr->op==TK_IF_NULL_ROW && pExpr->iTable==pSubst->iTable ){
3845 pExpr->iTable = pSubst->iNewTable;
3846 }
drh46967de2017-04-14 12:39:37 +00003847 pExpr->pLeft = substExpr(pSubst, pExpr->pLeft);
3848 pExpr->pRight = substExpr(pSubst, pExpr->pRight);
drha4eeccd2021-10-07 17:43:30 +00003849 if( ExprUseXSelect(pExpr) ){
drh46967de2017-04-14 12:39:37 +00003850 substSelect(pSubst, pExpr->x.pSelect, 1);
danielk19776ab3a2e2009-02-19 14:39:25 +00003851 }else{
drh46967de2017-04-14 12:39:37 +00003852 substExprList(pSubst, pExpr->x.pList);
danielk19776ab3a2e2009-02-19 14:39:25 +00003853 }
dan3703edf2019-10-10 15:17:09 +00003854#ifndef SQLITE_OMIT_WINDOWFUNC
3855 if( ExprHasProperty(pExpr, EP_WinFunc) ){
3856 Window *pWin = pExpr->y.pWin;
3857 pWin->pFilter = substExpr(pSubst, pWin->pFilter);
3858 substExprList(pSubst, pWin->pPartition);
3859 substExprList(pSubst, pWin->pOrderBy);
3860 }
3861#endif
drh832508b2002-03-02 17:04:07 +00003862 }
drhb7916a72009-05-27 10:31:29 +00003863 return pExpr;
drh832508b2002-03-02 17:04:07 +00003864}
drh17435752007-08-16 04:30:38 +00003865static void substExprList(
drh46967de2017-04-14 12:39:37 +00003866 SubstContext *pSubst, /* Description of the substitution */
3867 ExprList *pList /* List to scan and in which to make substitutes */
drh17435752007-08-16 04:30:38 +00003868){
drh832508b2002-03-02 17:04:07 +00003869 int i;
3870 if( pList==0 ) return;
3871 for(i=0; i<pList->nExpr; i++){
drh46967de2017-04-14 12:39:37 +00003872 pList->a[i].pExpr = substExpr(pSubst, pList->a[i].pExpr);
drh832508b2002-03-02 17:04:07 +00003873 }
3874}
drh17435752007-08-16 04:30:38 +00003875static void substSelect(
drh46967de2017-04-14 12:39:37 +00003876 SubstContext *pSubst, /* Description of the substitution */
3877 Select *p, /* SELECT statement in which to make substitutions */
3878 int doPrior /* Do substitutes on p->pPrior too */
drh17435752007-08-16 04:30:38 +00003879){
drh588a9a12008-09-01 15:52:10 +00003880 SrcList *pSrc;
drh76012942021-02-21 21:04:54 +00003881 SrcItem *pItem;
drh588a9a12008-09-01 15:52:10 +00003882 int i;
danielk1977b3bce662005-01-29 08:32:43 +00003883 if( !p ) return;
drhd12b6362015-10-11 19:46:59 +00003884 do{
drh46967de2017-04-14 12:39:37 +00003885 substExprList(pSubst, p->pEList);
3886 substExprList(pSubst, p->pGroupBy);
3887 substExprList(pSubst, p->pOrderBy);
3888 p->pHaving = substExpr(pSubst, p->pHaving);
3889 p->pWhere = substExpr(pSubst, p->pWhere);
drhd12b6362015-10-11 19:46:59 +00003890 pSrc = p->pSrc;
drh29064902015-10-11 20:08:31 +00003891 assert( pSrc!=0 );
drh588a9a12008-09-01 15:52:10 +00003892 for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){
drh46967de2017-04-14 12:39:37 +00003893 substSelect(pSubst, pItem->pSelect, 1);
drh29064902015-10-11 20:08:31 +00003894 if( pItem->fg.isTabFunc ){
drh46967de2017-04-14 12:39:37 +00003895 substExprList(pSubst, pItem->u1.pFuncArg);
drhd12b6362015-10-11 19:46:59 +00003896 }
drh588a9a12008-09-01 15:52:10 +00003897 }
drhd12b6362015-10-11 19:46:59 +00003898 }while( doPrior && (p = p->pPrior)!=0 );
danielk1977b3bce662005-01-29 08:32:43 +00003899}
shane3514b6f2008-07-22 05:00:55 +00003900#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */
drh832508b2002-03-02 17:04:07 +00003901
shane3514b6f2008-07-22 05:00:55 +00003902#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
drh832508b2002-03-02 17:04:07 +00003903/*
drh2aee5142020-03-21 00:05:53 +00003904** pSelect is a SELECT statement and pSrcItem is one item in the FROM
3905** clause of that SELECT.
3906**
3907** This routine scans the entire SELECT statement and recomputes the
3908** pSrcItem->colUsed mask.
3909*/
3910static int recomputeColumnsUsedExpr(Walker *pWalker, Expr *pExpr){
drh76012942021-02-21 21:04:54 +00003911 SrcItem *pItem;
drh2aee5142020-03-21 00:05:53 +00003912 if( pExpr->op!=TK_COLUMN ) return WRC_Continue;
3913 pItem = pWalker->u.pSrcItem;
3914 if( pItem->iCursor!=pExpr->iTable ) return WRC_Continue;
drh74a07982020-03-21 23:10:38 +00003915 if( pExpr->iColumn<0 ) return WRC_Continue;
3916 pItem->colUsed |= sqlite3ExprColUsed(pExpr);
drh2aee5142020-03-21 00:05:53 +00003917 return WRC_Continue;
3918}
3919static void recomputeColumnsUsed(
3920 Select *pSelect, /* The complete SELECT statement */
drh76012942021-02-21 21:04:54 +00003921 SrcItem *pSrcItem /* Which FROM clause item to recompute */
drh2aee5142020-03-21 00:05:53 +00003922){
3923 Walker w;
3924 if( NEVER(pSrcItem->pTab==0) ) return;
3925 memset(&w, 0, sizeof(w));
3926 w.xExprCallback = recomputeColumnsUsedExpr;
3927 w.xSelectCallback = sqlite3SelectWalkNoop;
3928 w.u.pSrcItem = pSrcItem;
3929 pSrcItem->colUsed = 0;
3930 sqlite3WalkSelect(&w, pSelect);
3931}
3932#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */
3933
3934#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
3935/*
dan964fa262020-12-18 16:13:39 +00003936** Assign new cursor numbers to each of the items in pSrc. For each
3937** new cursor number assigned, set an entry in the aCsrMap[] array
3938** to map the old cursor number to the new:
3939**
drhc8d21472021-07-21 15:42:05 +00003940** aCsrMap[iOld+1] = iNew;
dan964fa262020-12-18 16:13:39 +00003941**
3942** The array is guaranteed by the caller to be large enough for all
drhc8d21472021-07-21 15:42:05 +00003943** existing cursor numbers in pSrc. aCsrMap[0] is the array size.
dan964fa262020-12-18 16:13:39 +00003944**
3945** If pSrc contains any sub-selects, call this routine recursively
3946** on the FROM clause of each such sub-select, with iExcept set to -1.
3947*/
3948static void srclistRenumberCursors(
3949 Parse *pParse, /* Parse context */
3950 int *aCsrMap, /* Array to store cursor mappings in */
3951 SrcList *pSrc, /* FROM clause to renumber */
3952 int iExcept /* FROM clause item to skip */
3953){
3954 int i;
drh76012942021-02-21 21:04:54 +00003955 SrcItem *pItem;
dan964fa262020-12-18 16:13:39 +00003956 for(i=0, pItem=pSrc->a; i<pSrc->nSrc; i++, pItem++){
3957 if( i!=iExcept ){
dand131b512020-12-18 18:04:44 +00003958 Select *p;
drhc8d21472021-07-21 15:42:05 +00003959 assert( pItem->iCursor < aCsrMap[0] );
3960 if( !pItem->fg.isRecursive || aCsrMap[pItem->iCursor+1]==0 ){
3961 aCsrMap[pItem->iCursor+1] = pParse->nTab++;
drhc7f50772021-04-26 21:23:01 +00003962 }
drhc8d21472021-07-21 15:42:05 +00003963 pItem->iCursor = aCsrMap[pItem->iCursor+1];
dand131b512020-12-18 18:04:44 +00003964 for(p=pItem->pSelect; p; p=p->pPrior){
3965 srclistRenumberCursors(pParse, aCsrMap, p->pSrc, -1);
dan964fa262020-12-18 16:13:39 +00003966 }
3967 }
3968 }
3969}
3970
3971/*
drhc8d21472021-07-21 15:42:05 +00003972** *piCursor is a cursor number. Change it if it needs to be mapped.
3973*/
3974static void renumberCursorDoMapping(Walker *pWalker, int *piCursor){
3975 int *aCsrMap = pWalker->u.aiCol;
3976 int iCsr = *piCursor;
3977 if( iCsr < aCsrMap[0] && aCsrMap[iCsr+1]>0 ){
3978 *piCursor = aCsrMap[iCsr+1];
3979 }
3980}
3981
3982/*
dan964fa262020-12-18 16:13:39 +00003983** Expression walker callback used by renumberCursors() to update
3984** Expr objects to match newly assigned cursor numbers.
3985*/
3986static int renumberCursorsCb(Walker *pWalker, Expr *pExpr){
dane8f14902021-02-13 14:26:25 +00003987 int op = pExpr->op;
drhc8d21472021-07-21 15:42:05 +00003988 if( op==TK_COLUMN || op==TK_IF_NULL_ROW ){
3989 renumberCursorDoMapping(pWalker, &pExpr->iTable);
dan964fa262020-12-18 16:13:39 +00003990 }
drh67a99db2022-05-13 14:52:04 +00003991 if( ExprHasProperty(pExpr, EP_OuterON) ){
drhd1985262022-04-11 11:25:28 +00003992 renumberCursorDoMapping(pWalker, &pExpr->w.iJoin);
dan961a7262020-12-21 19:50:10 +00003993 }
dan964fa262020-12-18 16:13:39 +00003994 return WRC_Continue;
3995}
3996
3997/*
3998** Assign a new cursor number to each cursor in the FROM clause (Select.pSrc)
3999** of the SELECT statement passed as the second argument, and to each
4000** cursor in the FROM clause of any FROM clause sub-selects, recursively.
4001** Except, do not assign a new cursor number to the iExcept'th element in
4002** the FROM clause of (*p). Update all expressions and other references
4003** to refer to the new cursor numbers.
4004**
4005** Argument aCsrMap is an array that may be used for temporary working
4006** space. Two guarantees are made by the caller:
4007**
4008** * the array is larger than the largest cursor number used within the
4009** select statement passed as an argument, and
4010**
4011** * the array entries for all cursor numbers that do *not* appear in
4012** FROM clauses of the select statement as described above are
4013** initialized to zero.
4014*/
4015static void renumberCursors(
4016 Parse *pParse, /* Parse context */
4017 Select *p, /* Select to renumber cursors within */
4018 int iExcept, /* FROM clause item to skip */
4019 int *aCsrMap /* Working space */
4020){
4021 Walker w;
4022 srclistRenumberCursors(pParse, aCsrMap, p->pSrc, iExcept);
4023 memset(&w, 0, sizeof(w));
4024 w.u.aiCol = aCsrMap;
4025 w.xExprCallback = renumberCursorsCb;
4026 w.xSelectCallback = sqlite3SelectWalkNoop;
4027 sqlite3WalkSelect(&w, p);
4028}
4029#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */
4030
4031#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
4032/*
drh630d2962011-12-11 21:51:04 +00004033** This routine attempts to flatten subqueries as a performance optimization.
4034** This routine returns 1 if it makes changes and 0 if no flattening occurs.
drh1350b032002-02-27 19:00:20 +00004035**
4036** To understand the concept of flattening, consider the following
4037** query:
4038**
4039** SELECT a FROM (SELECT x+y AS a FROM t1 WHERE z<100) WHERE a>5
4040**
4041** The default way of implementing this query is to execute the
4042** subquery first and store the results in a temporary table, then
4043** run the outer query on that temporary table. This requires two
4044** passes over the data. Furthermore, because the temporary table
4045** has no indices, the WHERE clause on the outer query cannot be
drh832508b2002-03-02 17:04:07 +00004046** optimized.
drh1350b032002-02-27 19:00:20 +00004047**
drh832508b2002-03-02 17:04:07 +00004048** This routine attempts to rewrite queries such as the above into
drh1350b032002-02-27 19:00:20 +00004049** a single flat select, like this:
4050**
4051** SELECT x+y AS a FROM t1 WHERE z<100 AND a>5
4052**
peter.d.reid60ec9142014-09-06 16:39:46 +00004053** The code generated for this simplification gives the same result
drh832508b2002-03-02 17:04:07 +00004054** but only has to scan the data once. And because indices might
4055** exist on the table t1, a complete scan of the data might be
4056** avoided.
drh1350b032002-02-27 19:00:20 +00004057**
drhd981e822017-09-29 16:07:56 +00004058** Flattening is subject to the following constraints:
drh1350b032002-02-27 19:00:20 +00004059**
drh25c221e2017-09-29 22:13:24 +00004060** (**) We no longer attempt to flatten aggregate subqueries. Was:
4061** The subquery and the outer query cannot both be aggregates.
drh1350b032002-02-27 19:00:20 +00004062**
drh25c221e2017-09-29 22:13:24 +00004063** (**) We no longer attempt to flatten aggregate subqueries. Was:
4064** (2) If the subquery is an aggregate then
drhd981e822017-09-29 16:07:56 +00004065** (2a) the outer query must not be a join and
4066** (2b) the outer query must not use subqueries
4067** other than the one FROM-clause subquery that is a candidate
4068** for flattening. (This is due to ticket [2f7170d73bf9abf80]
4069** from 2015-02-09.)
drh832508b2002-03-02 17:04:07 +00004070**
drhd981e822017-09-29 16:07:56 +00004071** (3) If the subquery is the right operand of a LEFT JOIN then
4072** (3a) the subquery may not be a join and
4073** (3b) the FROM clause of the subquery may not contain a virtual
4074** table and
drhee373022022-07-25 15:54:23 +00004075** (**) Was: "The outer query may not have a GROUP BY." This case
4076** is now managed correctly
drh396afe62019-12-18 20:51:58 +00004077** (3d) the outer query may not be DISTINCT.
drh41798d52022-04-11 00:21:53 +00004078** See also (26) for restrictions on RIGHT JOIN.
drh832508b2002-03-02 17:04:07 +00004079**
drhd981e822017-09-29 16:07:56 +00004080** (4) The subquery can not be DISTINCT.
drh832508b2002-03-02 17:04:07 +00004081**
dan49ad3302010-08-13 16:38:48 +00004082** (**) At one point restrictions (4) and (5) defined a subset of DISTINCT
4083** sub-queries that were excluded from this optimization. Restriction
4084** (4) has since been expanded to exclude all DISTINCT subqueries.
drh832508b2002-03-02 17:04:07 +00004085**
drh25c221e2017-09-29 22:13:24 +00004086** (**) We no longer attempt to flatten aggregate subqueries. Was:
4087** If the subquery is aggregate, the outer query may not be DISTINCT.
drh832508b2002-03-02 17:04:07 +00004088**
drhd981e822017-09-29 16:07:56 +00004089** (7) The subquery must have a FROM clause. TODO: For subqueries without
drh31d6fd52017-04-14 19:03:10 +00004090** A FROM clause, consider adding a FROM clause with the special
drh630d2962011-12-11 21:51:04 +00004091** table sqlite_once that consists of a single row containing a
4092** single NULL.
drh08192d52002-04-30 19:20:28 +00004093**
drhd981e822017-09-29 16:07:56 +00004094** (8) If the subquery uses LIMIT then the outer query may not be a join.
drhdf199a22002-06-14 22:38:41 +00004095**
drhd981e822017-09-29 16:07:56 +00004096** (9) If the subquery uses LIMIT then the outer query may not be aggregate.
drhdf199a22002-06-14 22:38:41 +00004097**
drh6092d2b2014-09-15 11:14:50 +00004098** (**) Restriction (10) was removed from the code on 2005-02-05 but we
4099** accidently carried the comment forward until 2014-09-15. Original
drhd981e822017-09-29 16:07:56 +00004100** constraint: "If the subquery is aggregate then the outer query
4101** may not use LIMIT."
drhdf199a22002-06-14 22:38:41 +00004102**
drhd981e822017-09-29 16:07:56 +00004103** (11) The subquery and the outer query may not both have ORDER BY clauses.
drh174b6192002-12-03 02:22:52 +00004104**
drh7b688ed2009-12-22 00:29:53 +00004105** (**) Not implemented. Subsumed into restriction (3). Was previously
drh2b300d52008-08-14 00:19:48 +00004106** a separate restriction deriving from ticket #350.
drh3fc673e2003-06-16 00:40:34 +00004107**
drhd981e822017-09-29 16:07:56 +00004108** (13) The subquery and outer query may not both use LIMIT.
drhac839632006-01-21 22:19:54 +00004109**
drhd981e822017-09-29 16:07:56 +00004110** (14) The subquery may not use OFFSET.
drhac839632006-01-21 22:19:54 +00004111**
drhd981e822017-09-29 16:07:56 +00004112** (15) If the outer query is part of a compound select, then the
4113** subquery may not use LIMIT.
drhf3913272010-04-15 23:24:29 +00004114** (See ticket #2339 and ticket [02a8e81d44]).
drhad91c6c2007-05-06 20:04:24 +00004115**
drhd981e822017-09-29 16:07:56 +00004116** (16) If the outer query is aggregate, then the subquery may not
4117** use ORDER BY. (Ticket #2942) This used to not matter
drhc52e3552008-02-15 14:33:03 +00004118** until we introduced the group_concat() function.
4119**
drhd981e822017-09-29 16:07:56 +00004120** (17) If the subquery is a compound select, then
4121** (17a) all compound operators must be a UNION ALL, and
4122** (17b) no terms within the subquery compound may be aggregate
drhe76acc62017-10-04 02:30:45 +00004123** or DISTINCT, and
drhd981e822017-09-29 16:07:56 +00004124** (17c) every term within the subquery compound must have a FROM clause
4125** (17d) the outer query may not be
4126** (17d1) aggregate, or
dan8daf5ae2020-12-17 16:48:04 +00004127** (17d2) DISTINCT
4128** (17e) the subquery may not contain window functions, and
4129** (17f) the subquery must not be the RHS of a LEFT JOIN.
drhb88bf862022-07-13 15:52:15 +00004130** (17g) either the subquery is the first element of the outer
4131** query or there are no RIGHT or FULL JOINs in any arm
4132** of the subquery. (This is a duplicate of condition (27b).)
danielk1977f23329a2008-07-01 14:09:13 +00004133**
danielk19774914cf92008-07-01 18:26:49 +00004134** The parent and sub-query may contain WHERE clauses. Subject to
4135** rules (11), (13) and (14), they may also contain ORDER BY,
drh630d2962011-12-11 21:51:04 +00004136** LIMIT and OFFSET clauses. The subquery cannot use any compound
4137** operator other than UNION ALL because all the other compound
4138** operators have an implied DISTINCT which is disallowed by
4139** restriction (4).
danielk1977f23329a2008-07-01 14:09:13 +00004140**
dan67c70142012-08-28 14:45:50 +00004141** Also, each component of the sub-query must return the same number
4142** of result columns. This is actually a requirement for any compound
4143** SELECT statement, but all the code here does is make sure that no
4144** such (illegal) sub-query is flattened. The caller will detect the
4145** syntax error and return a detailed message.
4146**
danielk197749fc1f62008-07-08 17:43:56 +00004147** (18) If the sub-query is a compound select, then all terms of the
dan9353bea2020-12-17 17:17:12 +00004148** ORDER BY clause of the parent must be copies of a term returned
4149** by the parent query.
danielk197749fc1f62008-07-08 17:43:56 +00004150**
drhd981e822017-09-29 16:07:56 +00004151** (19) If the subquery uses LIMIT then the outer query may not
drh229cf702008-08-26 12:56:14 +00004152** have a WHERE clause.
4153**
drhfca23552017-10-28 20:51:54 +00004154** (20) If the sub-query is a compound select, then it must not use
4155** an ORDER BY clause. Ticket #3773. We could relax this constraint
4156** somewhat by saying that the terms of the ORDER BY clause must
4157** appear as unmodified result columns in the outer query. But we
4158** have other optimizations in mind to deal with that case.
drhe8902a72009-04-02 16:59:47 +00004159**
drhd981e822017-09-29 16:07:56 +00004160** (21) If the subquery uses LIMIT then the outer query may not be
shaneha91491e2011-02-11 20:52:20 +00004161** DISTINCT. (See ticket [752e1646fc]).
4162**
drhd981e822017-09-29 16:07:56 +00004163** (22) The subquery may not be a recursive CTE.
dan8290c2a2014-01-16 10:58:39 +00004164**
dan8daf5ae2020-12-17 16:48:04 +00004165** (23) If the outer query is a recursive CTE, then the sub-query may not be
4166** a compound query. This restriction is because transforming the
dan8290c2a2014-01-16 10:58:39 +00004167** parent to a compound query confuses the code that handles
4168** recursive queries in multiSelect().
4169**
drh508e2d02017-09-30 01:25:04 +00004170** (**) We no longer attempt to flatten aggregate subqueries. Was:
4171** The subquery may not be an aggregate that uses the built-in min() or
drh9588ad92014-09-15 14:46:02 +00004172** or max() functions. (Without this restriction, a query like:
4173** "SELECT x FROM (SELECT max(y), x FROM t1)" would not necessarily
4174** return the value X for which Y was maximal.)
4175**
dan9a947222018-06-14 19:06:36 +00004176** (25) If either the subquery or the parent query contains a window
4177** function in the select list or ORDER BY clause, flattening
4178** is not attempted.
4179**
drh41798d52022-04-11 00:21:53 +00004180** (26) The subquery may not be the right operand of a RIGHT JOIN.
4181** See also (3) for restrictions on LEFT JOIN.
4182**
drh1c2bf412022-04-18 23:20:02 +00004183** (27) The subquery may not contain a FULL or RIGHT JOIN unless it
drhb88bf862022-07-13 15:52:15 +00004184** is the first element of the parent query. This must be the
4185** the case if:
4186** (27a) the subquery is not compound query, and
4187** (27b) the subquery is a compound query and the RIGHT JOIN occurs
4188** in any arm of the compound query. (See also (17g).)
drh1c2bf412022-04-18 23:20:02 +00004189**
drh67f70be2022-04-22 16:15:48 +00004190** (28) The subquery is not a MATERIALIZED CTE.
4191**
drh5c118e32022-06-08 15:30:39 +00004192** (29) Either the subquery is not the right-hand operand of a join with an
4193** ON or USING clause nor the right-hand operand of a NATURAL JOIN, or
4194** the right-most table within the FROM clause of the subquery
4195** is not part of an outer join.
4196**
dan8290c2a2014-01-16 10:58:39 +00004197**
drh832508b2002-03-02 17:04:07 +00004198** In this routine, the "p" parameter is a pointer to the outer query.
4199** The subquery is p->pSrc->a[iFrom]. isAgg is true if the outer query
drh25c221e2017-09-29 22:13:24 +00004200** uses aggregates.
drh832508b2002-03-02 17:04:07 +00004201**
drh665de472003-03-31 13:36:09 +00004202** If flattening is not attempted, this routine is a no-op and returns 0.
drh832508b2002-03-02 17:04:07 +00004203** If flattening is attempted this routine returns 1.
4204**
4205** All of the expression analysis must occur on both the outer query and
4206** the subquery before this routine runs.
drh1350b032002-02-27 19:00:20 +00004207*/
drh8c74a8c2002-08-25 19:20:40 +00004208static int flattenSubquery(
danielk1977524cc212008-07-02 13:13:51 +00004209 Parse *pParse, /* Parsing context */
drh8c74a8c2002-08-25 19:20:40 +00004210 Select *p, /* The parent or outer SELECT statement */
4211 int iFrom, /* Index in p->pSrc->a[] of the inner subquery */
drh25c221e2017-09-29 22:13:24 +00004212 int isAgg /* True if outer SELECT uses aggregate functions */
drh8c74a8c2002-08-25 19:20:40 +00004213){
danielk1977524cc212008-07-02 13:13:51 +00004214 const char *zSavedAuthContext = pParse->zAuthContext;
drhd12b6362015-10-11 19:46:59 +00004215 Select *pParent; /* Current UNION ALL term of the other query */
drh0bb28102002-05-08 11:54:14 +00004216 Select *pSub; /* The inner query or "subquery" */
danielk1977f23329a2008-07-01 14:09:13 +00004217 Select *pSub1; /* Pointer to the rightmost select in sub-query */
drhad3cab52002-05-24 02:04:32 +00004218 SrcList *pSrc; /* The FROM clause of the outer query */
4219 SrcList *pSubSrc; /* The FROM clause of the subquery */
drh6a3ea0e2003-05-02 14:32:12 +00004220 int iParent; /* VDBE cursor number of the pSub result set temp table */
drh399c7e22017-04-14 17:18:45 +00004221 int iNewParent = -1;/* Replacement table for iParent */
drhc133bab2022-04-11 20:15:52 +00004222 int isOuterJoin = 0; /* True if pSub is the right side of a LEFT JOIN */
drh91bb0ee2004-09-01 03:06:34 +00004223 int i; /* Loop counter */
4224 Expr *pWhere; /* The WHERE clause */
drh76012942021-02-21 21:04:54 +00004225 SrcItem *pSubitem; /* The subquery */
danielk1977524cc212008-07-02 13:13:51 +00004226 sqlite3 *db = pParse->db;
drh89636622020-06-07 17:33:18 +00004227 Walker w; /* Walker to persist agginfo data */
dan964fa262020-12-18 16:13:39 +00004228 int *aCsrMap = 0;
drh1350b032002-02-27 19:00:20 +00004229
drh832508b2002-03-02 17:04:07 +00004230 /* Check to see if flattening is permitted. Return 0 if not.
4231 */
drha78c22c2008-11-11 18:28:58 +00004232 assert( p!=0 );
drhd981e822017-09-29 16:07:56 +00004233 assert( p->pPrior==0 );
drh7e5418e2012-09-27 15:05:54 +00004234 if( OptimizationDisabled(db, SQLITE_QueryFlattener) ) return 0;
drh832508b2002-03-02 17:04:07 +00004235 pSrc = p->pSrc;
drhad3cab52002-05-24 02:04:32 +00004236 assert( pSrc && iFrom>=0 && iFrom<pSrc->nSrc );
drh91bb0ee2004-09-01 03:06:34 +00004237 pSubitem = &pSrc->a[iFrom];
danielk197749fc1f62008-07-08 17:43:56 +00004238 iParent = pSubitem->iCursor;
drh91bb0ee2004-09-01 03:06:34 +00004239 pSub = pSubitem->pSelect;
drh832508b2002-03-02 17:04:07 +00004240 assert( pSub!=0 );
drh399c7e22017-04-14 17:18:45 +00004241
dan67a9b8e2018-06-22 20:51:35 +00004242#ifndef SQLITE_OMIT_WINDOWFUNC
dan9a947222018-06-14 19:06:36 +00004243 if( p->pWin || pSub->pWin ) return 0; /* Restriction (25) */
dan67a9b8e2018-06-22 20:51:35 +00004244#endif
dan86fb6e12018-05-16 20:58:07 +00004245
drh832508b2002-03-02 17:04:07 +00004246 pSubSrc = pSub->pSrc;
4247 assert( pSubSrc );
drhac839632006-01-21 22:19:54 +00004248 /* Prior to version 3.1.2, when LIMIT and OFFSET had to be simple constants,
peter.d.reid60ec9142014-09-06 16:39:46 +00004249 ** not arbitrary expressions, we allowed some combining of LIMIT and OFFSET
drhac839632006-01-21 22:19:54 +00004250 ** because they could be computed at compile-time. But when LIMIT and OFFSET
4251 ** became arbitrary expressions, we were forced to add restrictions (13)
4252 ** and (14). */
4253 if( pSub->pLimit && p->pLimit ) return 0; /* Restriction (13) */
drh8c0833f2017-11-14 23:48:23 +00004254 if( pSub->pLimit && pSub->pLimit->pRight ) return 0; /* Restriction (14) */
drhd227a292014-02-09 18:02:09 +00004255 if( (p->selFlags & SF_Compound)!=0 && pSub->pLimit ){
drhad91c6c2007-05-06 20:04:24 +00004256 return 0; /* Restriction (15) */
4257 }
drhac839632006-01-21 22:19:54 +00004258 if( pSubSrc->nSrc==0 ) return 0; /* Restriction (7) */
drhd981e822017-09-29 16:07:56 +00004259 if( pSub->selFlags & SF_Distinct ) return 0; /* Restriction (4) */
dan49ad3302010-08-13 16:38:48 +00004260 if( pSub->pLimit && (pSrc->nSrc>1 || isAgg) ){
4261 return 0; /* Restrictions (8)(9) */
drhdf199a22002-06-14 22:38:41 +00004262 }
drh7d10d5a2008-08-20 16:35:10 +00004263 if( p->pOrderBy && pSub->pOrderBy ){
drhac839632006-01-21 22:19:54 +00004264 return 0; /* Restriction (11) */
4265 }
drhc52e3552008-02-15 14:33:03 +00004266 if( isAgg && pSub->pOrderBy ) return 0; /* Restriction (16) */
drh229cf702008-08-26 12:56:14 +00004267 if( pSub->pLimit && p->pWhere ) return 0; /* Restriction (19) */
shaneha91491e2011-02-11 20:52:20 +00004268 if( pSub->pLimit && (p->selFlags & SF_Distinct)!=0 ){
4269 return 0; /* Restriction (21) */
4270 }
drh508e2d02017-09-30 01:25:04 +00004271 if( pSub->selFlags & (SF_Recursive) ){
4272 return 0; /* Restrictions (22) */
drh9588ad92014-09-15 14:46:02 +00004273 }
drh832508b2002-03-02 17:04:07 +00004274
drh399c7e22017-04-14 17:18:45 +00004275 /*
4276 ** If the subquery is the right operand of a LEFT JOIN, then the
drhd981e822017-09-29 16:07:56 +00004277 ** subquery may not be a join itself (3a). Example of why this is not
4278 ** allowed:
drh8af4d3a2003-05-06 20:35:16 +00004279 **
4280 ** t1 LEFT OUTER JOIN (t2 JOIN t3)
4281 **
4282 ** If we flatten the above, we would get
4283 **
4284 ** (t1 LEFT OUTER JOIN t2) JOIN t3
4285 **
4286 ** which is not at all the same thing.
drh2b300d52008-08-14 00:19:48 +00004287 **
drh31d6fd52017-04-14 19:03:10 +00004288 ** See also tickets #306, #350, and #3300.
drh3fc673e2003-06-16 00:40:34 +00004289 */
drhc133bab2022-04-11 20:15:52 +00004290 if( (pSubitem->fg.jointype & (JT_OUTER|JT_LTORJ))!=0 ){
drh41798d52022-04-11 00:21:53 +00004291 if( pSubSrc->nSrc>1 /* (3a) */
drh9efc6182022-06-23 12:36:56 +00004292 || IsVirtual(pSubSrc->a[0].pTab) /* (3b) */
drh41798d52022-04-11 00:21:53 +00004293 || (p->selFlags & SF_Distinct)!=0 /* (3d) */
4294 || (pSubitem->fg.jointype & JT_RIGHT)!=0 /* (26) */
drh396afe62019-12-18 20:51:58 +00004295 ){
drhd981e822017-09-29 16:07:56 +00004296 return 0;
drh399c7e22017-04-14 17:18:45 +00004297 }
drhc133bab2022-04-11 20:15:52 +00004298 isOuterJoin = 1;
drh3fc673e2003-06-16 00:40:34 +00004299 }
drhdc6de472017-05-25 11:39:50 +00004300#ifdef SQLITE_EXTRA_IFNULLROW
4301 else if( iFrom>0 && !isAgg ){
drhc133bab2022-04-11 20:15:52 +00004302 /* Setting isOuterJoin to -1 causes OP_IfNullRow opcodes to be generated for
drh3d240d22017-09-28 16:56:55 +00004303 ** every reference to any result column from subquery in a join, even
4304 ** though they are not necessary. This will stress-test the OP_IfNullRow
4305 ** opcode. */
drhc133bab2022-04-11 20:15:52 +00004306 isOuterJoin = -1;
drhdc6de472017-05-25 11:39:50 +00004307 }
4308#endif
drh3fc673e2003-06-16 00:40:34 +00004309
drh1c2bf412022-04-18 23:20:02 +00004310 assert( pSubSrc->nSrc>0 ); /* True by restriction (7) */
4311 if( iFrom>0 && (pSubSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){
drhb88bf862022-07-13 15:52:15 +00004312 return 0; /* Restriction (27a) */
drh1c2bf412022-04-18 23:20:02 +00004313 }
drh67f70be2022-04-22 16:15:48 +00004314 if( pSubitem->fg.isCte && pSubitem->u2.pCteUse->eM10d==M10d_Yes ){
4315 return 0; /* (28) */
4316 }
drh1c2bf412022-04-18 23:20:02 +00004317
drh8a5469b2022-06-08 17:48:14 +00004318 /* Restriction (29):
4319 **
4320 ** We do not want two constraints on the same term of the flattened
4321 ** query where one constraint has EP_InnerON and the other is EP_OuterON.
4322 ** To prevent this, one or the other of the following conditions must be
4323 ** false:
4324 **
4325 ** (29a) The right-most entry in the FROM clause of the subquery
4326 ** must not be part of an outer join.
4327 **
4328 ** (29b) The subquery itself must not be the right operand of a
4329 ** NATURAL join or a join that as an ON or USING clause.
4330 **
4331 ** These conditions are sufficient to keep an EP_OuterON from being
drhb88bf862022-07-13 15:52:15 +00004332 ** flattened into an EP_InnerON. Restrictions (3a) and (27a) prevent
drh8a5469b2022-06-08 17:48:14 +00004333 ** an EP_InnerON from being flattened into an EP_OuterON.
4334 */
drh5c118e32022-06-08 15:30:39 +00004335 if( pSubSrc->nSrc>=2
4336 && (pSubSrc->a[pSubSrc->nSrc-1].fg.jointype & JT_OUTER)!=0
4337 ){
drh8a5469b2022-06-08 17:48:14 +00004338 if( (pSubitem->fg.jointype & JT_NATURAL)!=0
drh5c118e32022-06-08 15:30:39 +00004339 || pSubitem->fg.isUsing
drh8a5469b2022-06-08 17:48:14 +00004340 || NEVER(pSubitem->u3.pOn!=0) /* ON clause already shifted into WHERE */
drh5c118e32022-06-08 15:30:39 +00004341 || pSubitem->fg.isOn
4342 ){
4343 return 0;
4344 }
4345 }
4346
drhd981e822017-09-29 16:07:56 +00004347 /* Restriction (17): If the sub-query is a compound SELECT, then it must
danielk1977f23329a2008-07-01 14:09:13 +00004348 ** use only the UNION ALL operator. And none of the simple select queries
4349 ** that make up the compound SELECT are allowed to be aggregate or distinct
4350 ** queries.
4351 */
4352 if( pSub->pPrior ){
drhfca23552017-10-28 20:51:54 +00004353 if( pSub->pOrderBy ){
4354 return 0; /* Restriction (20) */
4355 }
drhc133bab2022-04-11 20:15:52 +00004356 if( isAgg || (p->selFlags & SF_Distinct)!=0 || isOuterJoin>0 ){
dan8daf5ae2020-12-17 16:48:04 +00004357 return 0; /* (17d1), (17d2), or (17f) */
danielk1977f23329a2008-07-01 14:09:13 +00004358 }
4359 for(pSub1=pSub; pSub1; pSub1=pSub1->pPrior){
drhccfcbce2009-05-18 15:46:07 +00004360 testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct );
4361 testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Aggregate );
drh4b3ac732011-12-10 23:18:32 +00004362 assert( pSub->pSrc!=0 );
dan8daf5ae2020-12-17 16:48:04 +00004363 assert( (pSub->selFlags & SF_Recursive)==0 );
drh2ec18a32015-06-23 23:31:52 +00004364 assert( pSub->pEList->nExpr==pSub1->pEList->nExpr );
drhd981e822017-09-29 16:07:56 +00004365 if( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))!=0 /* (17b) */
4366 || (pSub1->pPrior && pSub1->op!=TK_ALL) /* (17a) */
4367 || pSub1->pSrc->nSrc<1 /* (17c) */
drhef9f7192020-01-17 19:14:08 +00004368#ifndef SQLITE_OMIT_WINDOWFUNC
drh8d95ed72019-12-30 20:42:17 +00004369 || pSub1->pWin /* (17e) */
drhef9f7192020-01-17 19:14:08 +00004370#endif
danielk197780b3c542008-07-10 17:59:12 +00004371 ){
danielk1977f23329a2008-07-01 14:09:13 +00004372 return 0;
4373 }
drhb88bf862022-07-13 15:52:15 +00004374 if( iFrom>0 && (pSub1->pSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){
4375 /* Without this restriction, the JT_LTORJ flag would end up being
4376 ** omitted on left-hand tables of the right join that is being
4377 ** flattened. */
4378 return 0; /* Restrictions (17g), (27b) */
4379 }
drh4b3ac732011-12-10 23:18:32 +00004380 testcase( pSub1->pSrc->nSrc>1 );
danielk1977f23329a2008-07-01 14:09:13 +00004381 }
danielk197749fc1f62008-07-08 17:43:56 +00004382
drhd981e822017-09-29 16:07:56 +00004383 /* Restriction (18). */
danielk197749fc1f62008-07-08 17:43:56 +00004384 if( p->pOrderBy ){
4385 int ii;
4386 for(ii=0; ii<p->pOrderBy->nExpr; ii++){
drhc2acc4e2013-11-15 18:15:19 +00004387 if( p->pOrderBy->a[ii].u.x.iOrderByCol==0 ) return 0;
danielk197749fc1f62008-07-08 17:43:56 +00004388 }
4389 }
danielk1977f23329a2008-07-01 14:09:13 +00004390
dande9ed622020-12-16 20:00:46 +00004391 /* Restriction (23) */
4392 if( (p->selFlags & SF_Recursive) ) return 0;
dan964fa262020-12-18 16:13:39 +00004393
4394 if( pSrc->nSrc>1 ){
dan51ddfef2021-03-17 14:29:37 +00004395 if( pParse->nSelect>500 ) return 0;
drh95fe38f2022-04-25 14:59:59 +00004396 if( OptimizationDisabled(db, SQLITE_FlttnUnionAll) ) return 0;
drh913306a2021-11-26 17:10:18 +00004397 aCsrMap = sqlite3DbMallocZero(db, ((i64)pParse->nTab+1)*sizeof(int));
drhc8d21472021-07-21 15:42:05 +00004398 if( aCsrMap ) aCsrMap[0] = pParse->nTab;
dan964fa262020-12-18 16:13:39 +00004399 }
dande9ed622020-12-16 20:00:46 +00004400 }
drhcdb2f602017-10-04 05:59:54 +00004401
drh7d10d5a2008-08-20 16:35:10 +00004402 /***** If we reach this point, flattening is permitted. *****/
drhfef37762018-07-10 19:48:35 +00004403 SELECTTRACE(1,pParse,p,("flatten %u.%p from term %d\n",
4404 pSub->selId, pSub, iFrom));
drh7d10d5a2008-08-20 16:35:10 +00004405
4406 /* Authorize the subquery */
danielk1977524cc212008-07-02 13:13:51 +00004407 pParse->zAuthContext = pSubitem->zName;
drha2acb0d2012-04-11 23:22:37 +00004408 TESTONLY(i =) sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0);
4409 testcase( i==SQLITE_DENY );
danielk1977524cc212008-07-02 13:13:51 +00004410 pParse->zAuthContext = zSavedAuthContext;
4411
dande9ed622020-12-16 20:00:46 +00004412 /* Delete the transient structures associated with thesubquery */
4413 pSub1 = pSubitem->pSelect;
4414 sqlite3DbFree(db, pSubitem->zDatabase);
4415 sqlite3DbFree(db, pSubitem->zName);
4416 sqlite3DbFree(db, pSubitem->zAlias);
4417 pSubitem->zDatabase = 0;
4418 pSubitem->zName = 0;
4419 pSubitem->zAlias = 0;
4420 pSubitem->pSelect = 0;
drhd44f8b22022-04-07 01:11:13 +00004421 assert( pSubitem->fg.isUsing!=0 || pSubitem->u3.pOn==0 );
dande9ed622020-12-16 20:00:46 +00004422
drh7d10d5a2008-08-20 16:35:10 +00004423 /* If the sub-query is a compound SELECT statement, then (by restrictions
4424 ** 17 and 18 above) it must be a UNION ALL and the parent query must
4425 ** be of the form:
danielk1977f23329a2008-07-01 14:09:13 +00004426 **
4427 ** SELECT <expr-list> FROM (<sub-query>) <where-clause>
4428 **
4429 ** followed by any ORDER BY, LIMIT and/or OFFSET clauses. This block
drha78c22c2008-11-11 18:28:58 +00004430 ** creates N-1 copies of the parent query without any ORDER BY, LIMIT or
danielk1977f23329a2008-07-01 14:09:13 +00004431 ** OFFSET clauses and joins them to the left-hand-side of the original
4432 ** using UNION ALL operators. In this case N is the number of simple
4433 ** select statements in the compound sub-query.
drha78c22c2008-11-11 18:28:58 +00004434 **
4435 ** Example:
4436 **
4437 ** SELECT a+1 FROM (
4438 ** SELECT x FROM tab
4439 ** UNION ALL
4440 ** SELECT y FROM tab
4441 ** UNION ALL
4442 ** SELECT abs(z*2) FROM tab2
4443 ** ) WHERE a!=5 ORDER BY 1
4444 **
4445 ** Transformed into:
4446 **
4447 ** SELECT x+1 FROM tab WHERE x+1!=5
4448 ** UNION ALL
4449 ** SELECT y+1 FROM tab WHERE y+1!=5
4450 ** UNION ALL
4451 ** SELECT abs(z*2)+1 FROM tab2 WHERE abs(z*2)+1!=5
4452 ** ORDER BY 1
4453 **
4454 ** We call this the "compound-subquery flattening".
danielk1977f23329a2008-07-01 14:09:13 +00004455 */
4456 for(pSub=pSub->pPrior; pSub; pSub=pSub->pPrior){
4457 Select *pNew;
4458 ExprList *pOrderBy = p->pOrderBy;
danielk19774b86ef12008-07-01 14:39:35 +00004459 Expr *pLimit = p->pLimit;
danielk1977f23329a2008-07-01 14:09:13 +00004460 Select *pPrior = p->pPrior;
dande9ed622020-12-16 20:00:46 +00004461 Table *pItemTab = pSubitem->pTab;
4462 pSubitem->pTab = 0;
danielk1977f23329a2008-07-01 14:09:13 +00004463 p->pOrderBy = 0;
danielk1977f23329a2008-07-01 14:09:13 +00004464 p->pPrior = 0;
danielk19774b86ef12008-07-01 14:39:35 +00004465 p->pLimit = 0;
danielk19776ab3a2e2009-02-19 14:39:25 +00004466 pNew = sqlite3SelectDup(db, p, 0);
danielk19774b86ef12008-07-01 14:39:35 +00004467 p->pLimit = pLimit;
drha78c22c2008-11-11 18:28:58 +00004468 p->pOrderBy = pOrderBy;
drha78c22c2008-11-11 18:28:58 +00004469 p->op = TK_ALL;
dande9ed622020-12-16 20:00:46 +00004470 pSubitem->pTab = pItemTab;
drha78c22c2008-11-11 18:28:58 +00004471 if( pNew==0 ){
drhd227a292014-02-09 18:02:09 +00004472 p->pPrior = pPrior;
drha78c22c2008-11-11 18:28:58 +00004473 }else{
dan51ddfef2021-03-17 14:29:37 +00004474 pNew->selId = ++pParse->nSelect;
drh9da977f2021-04-20 12:14:12 +00004475 if( aCsrMap && ALWAYS(db->mallocFailed==0) ){
dan964fa262020-12-18 16:13:39 +00004476 renumberCursors(pParse, pNew, iFrom, aCsrMap);
4477 }
drha78c22c2008-11-11 18:28:58 +00004478 pNew->pPrior = pPrior;
drhd227a292014-02-09 18:02:09 +00004479 if( pPrior ) pPrior->pNext = pNew;
4480 pNew->pNext = p;
4481 p->pPrior = pNew;
drhe2243d22018-04-23 17:18:03 +00004482 SELECTTRACE(2,pParse,p,("compound-subquery flattener"
drhfef37762018-07-10 19:48:35 +00004483 " creates %u as peer\n",pNew->selId));
drha78c22c2008-11-11 18:28:58 +00004484 }
dande9ed622020-12-16 20:00:46 +00004485 assert( pSubitem->pSelect==0 );
dan964fa262020-12-18 16:13:39 +00004486 }
4487 sqlite3DbFree(db, aCsrMap);
4488 if( db->mallocFailed ){
4489 pSubitem->pSelect = pSub1;
4490 return 1;
danielk1977f23329a2008-07-01 14:09:13 +00004491 }
4492
drha78c22c2008-11-11 18:28:58 +00004493 /* Defer deleting the Table object associated with the
4494 ** subquery until code generation is
4495 ** complete, since there may still exist Expr.pTab entries that
4496 ** refer to the subquery even after flattening. Ticket #3346.
drhccfcbce2009-05-18 15:46:07 +00004497 **
4498 ** pSubitem->pTab is always non-NULL by test restrictions and tests above.
drha78c22c2008-11-11 18:28:58 +00004499 */
drhccfcbce2009-05-18 15:46:07 +00004500 if( ALWAYS(pSubitem->pTab!=0) ){
drha78c22c2008-11-11 18:28:58 +00004501 Table *pTabToDel = pSubitem->pTab;
drh79df7782016-12-14 14:07:35 +00004502 if( pTabToDel->nTabRef==1 ){
dan65a7cd12009-09-01 12:16:01 +00004503 Parse *pToplevel = sqlite3ParseToplevel(pParse);
drhcf3c0782021-01-11 20:37:02 +00004504 sqlite3ParserAddCleanup(pToplevel,
4505 (void(*)(sqlite3*,void*))sqlite3DeleteTable,
4506 pTabToDel);
drh21d4f5b2021-01-12 15:30:01 +00004507 testcase( pToplevel->earlyCleanup );
drha78c22c2008-11-11 18:28:58 +00004508 }else{
drh79df7782016-12-14 14:07:35 +00004509 pTabToDel->nTabRef--;
drha78c22c2008-11-11 18:28:58 +00004510 }
4511 pSubitem->pTab = 0;
4512 }
4513
4514 /* The following loop runs once for each term in a compound-subquery
4515 ** flattening (as described above). If we are doing a different kind
4516 ** of flattening - a flattening other than a compound-subquery flattening -
4517 ** then this loop only runs once.
4518 **
4519 ** This loop moves all of the FROM elements of the subquery into the
4520 ** the FROM clause of the outer query. Before doing this, remember
4521 ** the cursor number for the original outer query FROM element in
4522 ** iParent. The iParent cursor will never be used. Subsequent code
4523 ** will scan expressions looking for iParent references and replace
4524 ** those references with expressions that resolve to the subquery FROM
4525 ** elements we are now copying in.
4526 */
dande9ed622020-12-16 20:00:46 +00004527 pSub = pSub1;
danielk1977f23329a2008-07-01 14:09:13 +00004528 for(pParent=p; pParent; pParent=pParent->pPrior, pSub=pSub->pPrior){
drha78c22c2008-11-11 18:28:58 +00004529 int nSubSrc;
drhea678832008-12-10 19:26:22 +00004530 u8 jointype = 0;
drh79d26582022-04-11 10:38:28 +00004531 u8 ltorj = pSrc->a[iFrom].fg.jointype & JT_LTORJ;
drh55f66b32019-07-16 19:44:32 +00004532 assert( pSub!=0 );
drha78c22c2008-11-11 18:28:58 +00004533 pSubSrc = pSub->pSrc; /* FROM clause of subquery */
4534 nSubSrc = pSubSrc->nSrc; /* Number of terms in subquery FROM clause */
4535 pSrc = pParent->pSrc; /* FROM clause of the outer query */
drhc31c2eb2003-05-02 16:04:17 +00004536
dande9ed622020-12-16 20:00:46 +00004537 if( pParent==p ){
4538 jointype = pSubitem->fg.jointype; /* First time through the loop */
drha78c22c2008-11-11 18:28:58 +00004539 }
dande9ed622020-12-16 20:00:46 +00004540
drha78c22c2008-11-11 18:28:58 +00004541 /* The subquery uses a single slot of the FROM clause of the outer
4542 ** query. If the subquery has more than one element in its FROM clause,
4543 ** then expand the outer query to make space for it to hold all elements
4544 ** of the subquery.
4545 **
4546 ** Example:
4547 **
4548 ** SELECT * FROM tabA, (SELECT * FROM sub1, sub2), tabB;
4549 **
4550 ** The outer query has 3 slots in its FROM clause. One slot of the
4551 ** outer query (the middle slot) is used by the subquery. The next
drhd12b6362015-10-11 19:46:59 +00004552 ** block of code will expand the outer query FROM clause to 4 slots.
4553 ** The middle slot is expanded to two slots in order to make space
4554 ** for the two elements in the FROM clause of the subquery.
drha78c22c2008-11-11 18:28:58 +00004555 */
4556 if( nSubSrc>1 ){
drh29c992c2019-01-17 15:40:41 +00004557 pSrc = sqlite3SrcListEnlarge(pParse, pSrc, nSubSrc-1,iFrom+1);
4558 if( pSrc==0 ) break;
4559 pParent->pSrc = pSrc;
drhc31c2eb2003-05-02 16:04:17 +00004560 }
drha78c22c2008-11-11 18:28:58 +00004561
4562 /* Transfer the FROM clause terms from the subquery into the
4563 ** outer query.
4564 */
drhc31c2eb2003-05-02 16:04:17 +00004565 for(i=0; i<nSubSrc; i++){
drhd44f8b22022-04-07 01:11:13 +00004566 SrcItem *pItem = &pSrc->a[i+iFrom];
4567 if( pItem->fg.isUsing ) sqlite3IdListDelete(db, pItem->u3.pUsing);
4568 assert( pItem->fg.isTabFunc==0 );
4569 *pItem = pSubSrc->a[i];
drh79d26582022-04-11 10:38:28 +00004570 pItem->fg.jointype |= ltorj;
drh399c7e22017-04-14 17:18:45 +00004571 iNewParent = pSubSrc->a[i].iCursor;
drhc31c2eb2003-05-02 16:04:17 +00004572 memset(&pSubSrc->a[i], 0, sizeof(pSubSrc->a[i]));
4573 }
drhf7309bc2022-04-12 20:20:54 +00004574 pSrc->a[iFrom].fg.jointype &= JT_LTORJ;
4575 pSrc->a[iFrom].fg.jointype |= jointype | ltorj;
danielk1977f23329a2008-07-01 14:09:13 +00004576
4577 /* Now begin substituting subquery result set expressions for
4578 ** references to the iParent in the outer query.
4579 **
4580 ** Example:
4581 **
4582 ** SELECT a+5, b*10 FROM (SELECT x*3 AS a, y+10 AS b FROM t1) WHERE a>b;
4583 ** \ \_____________ subquery __________/ /
4584 ** \_____________________ outer query ______________________________/
4585 **
4586 ** We look at every expression in the outer query and every place we see
4587 ** "a" we substitute "x*3" and every place we see "b" we substitute "y+10".
4588 */
drhb7cbf5c2020-06-15 13:51:34 +00004589 if( pSub->pOrderBy && (pParent->selFlags & SF_NoopOrderBy)==0 ){
dan7c0a4722014-09-20 20:38:48 +00004590 /* At this point, any non-zero iOrderByCol values indicate that the
4591 ** ORDER BY column expression is identical to the iOrderByCol'th
4592 ** expression returned by SELECT statement pSub. Since these values
4593 ** do not necessarily correspond to columns in SELECT statement pParent,
4594 ** zero them before transfering the ORDER BY clause.
4595 **
4596 ** Not doing this may cause an error if a subsequent call to this
4597 ** function attempts to flatten a compound sub-query into pParent
4598 ** (the only way this can happen is if the compound sub-query is
4599 ** currently part of pSub->pSrc). See ticket [d11a6e908f]. */
4600 ExprList *pOrderBy = pSub->pOrderBy;
4601 for(i=0; i<pOrderBy->nExpr; i++){
4602 pOrderBy->a[i].u.x.iOrderByCol = 0;
4603 }
danielk1977f23329a2008-07-01 14:09:13 +00004604 assert( pParent->pOrderBy==0 );
dan7c0a4722014-09-20 20:38:48 +00004605 pParent->pOrderBy = pOrderBy;
danielk1977f23329a2008-07-01 14:09:13 +00004606 pSub->pOrderBy = 0;
danielk1977f23329a2008-07-01 14:09:13 +00004607 }
drh11df7d22018-12-06 19:15:36 +00004608 pWhere = pSub->pWhere;
4609 pSub->pWhere = 0;
drhc133bab2022-04-11 20:15:52 +00004610 if( isOuterJoin>0 ){
drh67a99db2022-05-13 14:52:04 +00004611 sqlite3SetJoinExpr(pWhere, iNewParent, EP_OuterON);
drh399c7e22017-04-14 17:18:45 +00004612 }
dancd653a32020-06-13 21:24:40 +00004613 if( pWhere ){
4614 if( pParent->pWhere ){
4615 pParent->pWhere = sqlite3PExpr(pParse, TK_AND, pWhere, pParent->pWhere);
4616 }else{
4617 pParent->pWhere = pWhere;
4618 }
4619 }
danc3becdd2017-03-13 14:30:40 +00004620 if( db->mallocFailed==0 ){
drh46967de2017-04-14 12:39:37 +00004621 SubstContext x;
4622 x.pParse = pParse;
4623 x.iTable = iParent;
drh399c7e22017-04-14 17:18:45 +00004624 x.iNewTable = iNewParent;
drhc133bab2022-04-11 20:15:52 +00004625 x.isOuterJoin = isOuterJoin;
drh46967de2017-04-14 12:39:37 +00004626 x.pEList = pSub->pEList;
4627 substSelect(&x, pParent, 0);
danc3becdd2017-03-13 14:30:40 +00004628 }
danielk1977f23329a2008-07-01 14:09:13 +00004629
drh7cd5e852019-05-29 17:22:38 +00004630 /* The flattened query is a compound if either the inner or the
4631 ** outer query is a compound. */
4632 pParent->selFlags |= pSub->selFlags & SF_Compound;
4633 assert( (pSub->selFlags & SF_Distinct)==0 ); /* restriction (17b) */
danielk1977f23329a2008-07-01 14:09:13 +00004634
4635 /*
4636 ** SELECT ... FROM (SELECT ... LIMIT a OFFSET b) LIMIT x OFFSET y;
4637 **
4638 ** One is tempted to try to add a and b to combine the limits. But this
4639 ** does not work if either limit is negative.
4640 */
4641 if( pSub->pLimit ){
4642 pParent->pLimit = pSub->pLimit;
4643 pSub->pLimit = 0;
4644 }
drh2aee5142020-03-21 00:05:53 +00004645
4646 /* Recompute the SrcList_item.colUsed masks for the flattened
4647 ** tables. */
4648 for(i=0; i<nSubSrc; i++){
4649 recomputeColumnsUsed(pParent, &pSrc->a[i+iFrom]);
4650 }
drhdf199a22002-06-14 22:38:41 +00004651 }
drh8c74a8c2002-08-25 19:20:40 +00004652
drhc31c2eb2003-05-02 16:04:17 +00004653 /* Finially, delete what is left of the subquery and return
4654 ** success.
4655 */
drh89636622020-06-07 17:33:18 +00004656 sqlite3AggInfoPersistWalkerInit(&w, pParse);
4657 sqlite3WalkSelect(&w,pSub1);
drh633e6d52008-07-28 19:34:53 +00004658 sqlite3SelectDelete(db, pSub1);
danielk1977f23329a2008-07-01 14:09:13 +00004659
drh5e431be2022-04-06 11:08:38 +00004660#if TREETRACE_ENABLED
4661 if( sqlite3TreeTrace & 0x100 ){
drhbc8edba2015-06-05 20:27:26 +00004662 SELECTTRACE(0x100,pParse,p,("After flattening:\n"));
drhc90713d2014-09-30 13:46:49 +00004663 sqlite3TreeViewSelect(0, p, 0);
4664 }
4665#endif
4666
drh832508b2002-03-02 17:04:07 +00004667 return 1;
4668}
shane3514b6f2008-07-22 05:00:55 +00004669#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */
drh1350b032002-02-27 19:00:20 +00004670
drh660ee552018-07-26 21:16:53 +00004671/*
drh8e5bfed2018-10-25 14:15:37 +00004672** A structure to keep track of all of the column values that are fixed to
drhefad2e22018-07-27 16:57:11 +00004673** a known value due to WHERE clause constraints of the form COLUMN=VALUE.
drh660ee552018-07-26 21:16:53 +00004674*/
4675typedef struct WhereConst WhereConst;
4676struct WhereConst {
drhefad2e22018-07-27 16:57:11 +00004677 Parse *pParse; /* Parsing context */
drh75016052021-06-10 14:36:23 +00004678 u8 *pOomFault; /* Pointer to pParse->db->mallocFailed */
drh660ee552018-07-26 21:16:53 +00004679 int nConst; /* Number for COLUMN=CONSTANT terms */
4680 int nChng; /* Number of times a constant is propagated */
dan40b82f92021-05-26 18:51:57 +00004681 int bHasAffBlob; /* At least one column in apExpr[] as affinity BLOB */
drhae8776e2022-06-20 12:42:28 +00004682 u32 mExcludeOn; /* Which ON expressions to exclude from considertion.
4683 ** Either EP_OuterON or EP_InnerON|EP_OuterON */
drhefad2e22018-07-27 16:57:11 +00004684 Expr **apExpr; /* [i*2] is COLUMN and [i*2+1] is VALUE */
drh660ee552018-07-26 21:16:53 +00004685};
drh69b72d52015-06-01 20:28:03 +00004686
drh660ee552018-07-26 21:16:53 +00004687/*
drh8e5bfed2018-10-25 14:15:37 +00004688** Add a new entry to the pConst object. Except, do not add duplicate
drhf8f76d62020-01-08 04:36:01 +00004689** pColumn entires. Also, do not add if doing so would not be appropriate.
4690**
4691** The caller guarantees the pColumn is a column and pValue is a constant.
4692** This routine has to do some additional checks before completing the
4693** insert.
drh660ee552018-07-26 21:16:53 +00004694*/
4695static void constInsert(
drhf8f76d62020-01-08 04:36:01 +00004696 WhereConst *pConst, /* The WhereConst into which we are inserting */
4697 Expr *pColumn, /* The COLUMN part of the constraint */
4698 Expr *pValue, /* The VALUE part of the constraint */
4699 Expr *pExpr /* Overall expression: COLUMN=VALUE or VALUE=COLUMN */
drh660ee552018-07-26 21:16:53 +00004700){
drh8e5bfed2018-10-25 14:15:37 +00004701 int i;
4702 assert( pColumn->op==TK_COLUMN );
drhf8f76d62020-01-08 04:36:01 +00004703 assert( sqlite3ExprIsConstant(pValue) );
4704
drhfdfd45a2020-02-13 22:12:35 +00004705 if( ExprHasProperty(pColumn, EP_FixedCol) ) return;
4706 if( sqlite3ExprAffinity(pValue)!=0 ) return;
drhf8f76d62020-01-08 04:36:01 +00004707 if( !sqlite3IsBinary(sqlite3ExprCompareCollSeq(pConst->pParse,pExpr)) ){
4708 return;
4709 }
drh8e5bfed2018-10-25 14:15:37 +00004710
4711 /* 2018-10-25 ticket [cf5ed20f]
4712 ** Make sure the same pColumn is not inserted more than once */
4713 for(i=0; i<pConst->nConst; i++){
drh7be5e3d2020-01-09 20:33:13 +00004714 const Expr *pE2 = pConst->apExpr[i*2];
4715 assert( pE2->op==TK_COLUMN );
4716 if( pE2->iTable==pColumn->iTable
4717 && pE2->iColumn==pColumn->iColumn
drh8e5bfed2018-10-25 14:15:37 +00004718 ){
4719 return; /* Already present. Return without doing anything. */
4720 }
4721 }
dan40b82f92021-05-26 18:51:57 +00004722 if( sqlite3ExprAffinity(pColumn)==SQLITE_AFF_BLOB ){
4723 pConst->bHasAffBlob = 1;
4724 }
drh9cbf4f32018-07-27 20:01:00 +00004725
drh660ee552018-07-26 21:16:53 +00004726 pConst->nConst++;
drhefad2e22018-07-27 16:57:11 +00004727 pConst->apExpr = sqlite3DbReallocOrFree(pConst->pParse->db, pConst->apExpr,
drh660ee552018-07-26 21:16:53 +00004728 pConst->nConst*2*sizeof(Expr*));
4729 if( pConst->apExpr==0 ){
4730 pConst->nConst = 0;
4731 }else{
4732 pConst->apExpr[pConst->nConst*2-2] = pColumn;
4733 pConst->apExpr[pConst->nConst*2-1] = pValue;
4734 }
4735}
4736
4737/*
drhefad2e22018-07-27 16:57:11 +00004738** Find all terms of COLUMN=VALUE or VALUE=COLUMN in pExpr where VALUE
4739** is a constant expression and where the term must be true because it
4740** is part of the AND-connected terms of the expression. For each term
4741** found, add it to the pConst structure.
drh660ee552018-07-26 21:16:53 +00004742*/
4743static void findConstInWhere(WhereConst *pConst, Expr *pExpr){
drhefad2e22018-07-27 16:57:11 +00004744 Expr *pRight, *pLeft;
drhb775c972021-05-14 14:26:57 +00004745 if( NEVER(pExpr==0) ) return;
drhae8776e2022-06-20 12:42:28 +00004746 if( ExprHasProperty(pExpr, pConst->mExcludeOn) ){
drh958fcd42022-06-10 11:28:52 +00004747 testcase( ExprHasProperty(pExpr, EP_OuterON) );
4748 testcase( ExprHasProperty(pExpr, EP_InnerON) );
4749 return;
4750 }
drh660ee552018-07-26 21:16:53 +00004751 if( pExpr->op==TK_AND ){
4752 findConstInWhere(pConst, pExpr->pRight);
4753 findConstInWhere(pConst, pExpr->pLeft);
4754 return;
4755 }
4756 if( pExpr->op!=TK_EQ ) return;
drhefad2e22018-07-27 16:57:11 +00004757 pRight = pExpr->pRight;
4758 pLeft = pExpr->pLeft;
4759 assert( pRight!=0 );
4760 assert( pLeft!=0 );
drhf8f76d62020-01-08 04:36:01 +00004761 if( pRight->op==TK_COLUMN && sqlite3ExprIsConstant(pLeft) ){
4762 constInsert(pConst,pRight,pLeft,pExpr);
4763 }
4764 if( pLeft->op==TK_COLUMN && sqlite3ExprIsConstant(pRight) ){
4765 constInsert(pConst,pLeft,pRight,pExpr);
drh660ee552018-07-26 21:16:53 +00004766 }
4767}
4768
4769/*
dan40b82f92021-05-26 18:51:57 +00004770** This is a helper function for Walker callback propagateConstantExprRewrite().
4771**
4772** Argument pExpr is a candidate expression to be replaced by a value. If
4773** pExpr is equivalent to one of the columns named in pWalker->u.pConst,
4774** then overwrite it with the corresponding value. Except, do not do so
4775** if argument bIgnoreAffBlob is non-zero and the affinity of pExpr
4776** is SQLITE_AFF_BLOB.
drh660ee552018-07-26 21:16:53 +00004777*/
dan40b82f92021-05-26 18:51:57 +00004778static int propagateConstantExprRewriteOne(
4779 WhereConst *pConst,
4780 Expr *pExpr,
4781 int bIgnoreAffBlob
4782){
drh660ee552018-07-26 21:16:53 +00004783 int i;
drh75016052021-06-10 14:36:23 +00004784 if( pConst->pOomFault[0] ) return WRC_Prune;
drh660ee552018-07-26 21:16:53 +00004785 if( pExpr->op!=TK_COLUMN ) return WRC_Continue;
drhae8776e2022-06-20 12:42:28 +00004786 if( ExprHasProperty(pExpr, EP_FixedCol|pConst->mExcludeOn) ){
drhbe0330e2020-01-16 17:53:24 +00004787 testcase( ExprHasProperty(pExpr, EP_FixedCol) );
drh67a99db2022-05-13 14:52:04 +00004788 testcase( ExprHasProperty(pExpr, EP_OuterON) );
drhae8776e2022-06-20 12:42:28 +00004789 testcase( ExprHasProperty(pExpr, EP_InnerON) );
drhbe0330e2020-01-16 17:53:24 +00004790 return WRC_Continue;
4791 }
drh660ee552018-07-26 21:16:53 +00004792 for(i=0; i<pConst->nConst; i++){
4793 Expr *pColumn = pConst->apExpr[i*2];
4794 if( pColumn==pExpr ) continue;
4795 if( pColumn->iTable!=pExpr->iTable ) continue;
4796 if( pColumn->iColumn!=pExpr->iColumn ) continue;
dan40b82f92021-05-26 18:51:57 +00004797 if( bIgnoreAffBlob && sqlite3ExprAffinity(pColumn)==SQLITE_AFF_BLOB ){
4798 break;
4799 }
drhefad2e22018-07-27 16:57:11 +00004800 /* A match is found. Add the EP_FixedCol property */
drh660ee552018-07-26 21:16:53 +00004801 pConst->nChng++;
4802 ExprClearProperty(pExpr, EP_Leaf);
drhefad2e22018-07-27 16:57:11 +00004803 ExprSetProperty(pExpr, EP_FixedCol);
4804 assert( pExpr->pLeft==0 );
4805 pExpr->pLeft = sqlite3ExprDup(pConst->pParse->db, pConst->apExpr[i*2+1], 0);
drh75016052021-06-10 14:36:23 +00004806 if( pConst->pParse->db->mallocFailed ) return WRC_Prune;
drh660ee552018-07-26 21:16:53 +00004807 break;
4808 }
4809 return WRC_Prune;
4810}
4811
4812/*
dan40b82f92021-05-26 18:51:57 +00004813** This is a Walker expression callback. pExpr is a node from the WHERE
4814** clause of a SELECT statement. This function examines pExpr to see if
4815** any substitutions based on the contents of pWalker->u.pConst should
4816** be made to pExpr or its immediate children.
4817**
4818** A substitution is made if:
4819**
4820** + pExpr is a column with an affinity other than BLOB that matches
4821** one of the columns in pWalker->u.pConst, or
4822**
4823** + pExpr is a binary comparison operator (=, <=, >=, <, >) that
4824** uses an affinity other than TEXT and one of its immediate
4825** children is a column that matches one of the columns in
4826** pWalker->u.pConst.
4827*/
4828static int propagateConstantExprRewrite(Walker *pWalker, Expr *pExpr){
4829 WhereConst *pConst = pWalker->u.pConst;
drha8ed5152021-05-26 19:52:21 +00004830 assert( TK_GT==TK_EQ+1 );
4831 assert( TK_LE==TK_EQ+2 );
4832 assert( TK_LT==TK_EQ+3 );
4833 assert( TK_GE==TK_EQ+4 );
dan40b82f92021-05-26 18:51:57 +00004834 if( pConst->bHasAffBlob ){
drha8ed5152021-05-26 19:52:21 +00004835 if( (pExpr->op>=TK_EQ && pExpr->op<=TK_GE)
dan40b82f92021-05-26 18:51:57 +00004836 || pExpr->op==TK_IS
4837 ){
4838 propagateConstantExprRewriteOne(pConst, pExpr->pLeft, 0);
drh75016052021-06-10 14:36:23 +00004839 if( pConst->pOomFault[0] ) return WRC_Prune;
dan40b82f92021-05-26 18:51:57 +00004840 if( sqlite3ExprAffinity(pExpr->pLeft)!=SQLITE_AFF_TEXT ){
4841 propagateConstantExprRewriteOne(pConst, pExpr->pRight, 0);
4842 }
4843 }
4844 }
4845 return propagateConstantExprRewriteOne(pConst, pExpr, pConst->bHasAffBlob);
4846}
4847
4848/*
drh660ee552018-07-26 21:16:53 +00004849** The WHERE-clause constant propagation optimization.
4850**
4851** If the WHERE clause contains terms of the form COLUMN=CONSTANT or
drh97bffe62020-01-08 00:39:37 +00004852** CONSTANT=COLUMN that are top-level AND-connected terms that are not
4853** part of a ON clause from a LEFT JOIN, then throughout the query
4854** replace all other occurrences of COLUMN with CONSTANT.
drh660ee552018-07-26 21:16:53 +00004855**
4856** For example, the query:
4857**
4858** SELECT * FROM t1, t2, t3 WHERE t1.a=39 AND t2.b=t1.a AND t3.c=t2.b
4859**
4860** Is transformed into
4861**
4862** SELECT * FROM t1, t2, t3 WHERE t1.a=39 AND t2.b=39 AND t3.c=39
4863**
4864** Return true if any transformations where made and false if not.
drhefad2e22018-07-27 16:57:11 +00004865**
4866** Implementation note: Constant propagation is tricky due to affinity
4867** and collating sequence interactions. Consider this example:
4868**
4869** CREATE TABLE t1(a INT,b TEXT);
4870** INSERT INTO t1 VALUES(123,'0123');
4871** SELECT * FROM t1 WHERE a=123 AND b=a;
4872** SELECT * FROM t1 WHERE a=123 AND b=123;
4873**
4874** The two SELECT statements above should return different answers. b=a
4875** is alway true because the comparison uses numeric affinity, but b=123
4876** is false because it uses text affinity and '0123' is not the same as '123'.
4877** To work around this, the expression tree is not actually changed from
4878** "b=a" to "b=123" but rather the "a" in "b=a" is tagged with EP_FixedCol
4879** and the "123" value is hung off of the pLeft pointer. Code generator
4880** routines know to generate the constant "123" instead of looking up the
4881** column value. Also, to avoid collation problems, this optimization is
4882** only attempted if the "a=123" term uses the default BINARY collation.
drh342cb332021-05-27 14:09:01 +00004883**
4884** 2021-05-25 forum post 6a06202608: Another troublesome case is...
4885**
4886** CREATE TABLE t1(x);
4887** INSERT INTO t1 VALUES(10.0);
4888** SELECT 1 FROM t1 WHERE x=10 AND x LIKE 10;
4889**
4890** The query should return no rows, because the t1.x value is '10.0' not '10'
4891** and '10.0' is not LIKE '10'. But if we are not careful, the first WHERE
4892** term "x=10" will cause the second WHERE term to become "10 LIKE 10",
4893** resulting in a false positive. To avoid this, constant propagation for
4894** columns with BLOB affinity is only allowed if the constant is used with
4895** operators ==, <=, <, >=, >, or IS in a way that will cause the correct
4896** type conversions to occur. See logic associated with the bHasAffBlob flag
4897** for details.
drh660ee552018-07-26 21:16:53 +00004898*/
4899static int propagateConstants(
4900 Parse *pParse, /* The parsing context */
4901 Select *p /* The query in which to propagate constants */
4902){
4903 WhereConst x;
4904 Walker w;
4905 int nChng = 0;
drhefad2e22018-07-27 16:57:11 +00004906 x.pParse = pParse;
drh75016052021-06-10 14:36:23 +00004907 x.pOomFault = &pParse->db->mallocFailed;
drh660ee552018-07-26 21:16:53 +00004908 do{
4909 x.nConst = 0;
4910 x.nChng = 0;
4911 x.apExpr = 0;
dan40b82f92021-05-26 18:51:57 +00004912 x.bHasAffBlob = 0;
drhae8776e2022-06-20 12:42:28 +00004913 if( ALWAYS(p->pSrc!=0)
4914 && p->pSrc->nSrc>0
4915 && (p->pSrc->a[0].fg.jointype & JT_LTORJ)!=0
4916 ){
4917 /* Do not propagate constants on any ON clause if there is a
4918 ** RIGHT JOIN anywhere in the query */
4919 x.mExcludeOn = EP_InnerON | EP_OuterON;
4920 }else{
4921 /* Do not propagate constants through the ON clause of a LEFT JOIN */
4922 x.mExcludeOn = EP_OuterON;
4923 }
drh660ee552018-07-26 21:16:53 +00004924 findConstInWhere(&x, p->pWhere);
4925 if( x.nConst ){
4926 memset(&w, 0, sizeof(w));
4927 w.pParse = pParse;
4928 w.xExprCallback = propagateConstantExprRewrite;
4929 w.xSelectCallback = sqlite3SelectWalkNoop;
4930 w.xSelectCallback2 = 0;
4931 w.walkerDepth = 0;
4932 w.u.pConst = &x;
drhefad2e22018-07-27 16:57:11 +00004933 sqlite3WalkExpr(&w, p->pWhere);
4934 sqlite3DbFree(x.pParse->db, x.apExpr);
drh660ee552018-07-26 21:16:53 +00004935 nChng += x.nChng;
4936 }
4937 }while( x.nChng );
4938 return nChng;
4939}
drh69b72d52015-06-01 20:28:03 +00004940
4941#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
dan903fdd42021-02-22 20:56:13 +00004942# if !defined(SQLITE_OMIT_WINDOWFUNC)
4943/*
4944** This function is called to determine whether or not it is safe to
4945** push WHERE clause expression pExpr down to FROM clause sub-query
4946** pSubq, which contains at least one window function. Return 1
4947** if it is safe and the expression should be pushed down, or 0
4948** otherwise.
4949**
4950** It is only safe to push the expression down if it consists only
4951** of constants and copies of expressions that appear in the PARTITION
4952** BY clause of all window function used by the sub-query. It is safe
4953** to filter out entire partitions, but not rows within partitions, as
4954** this may change the results of the window functions.
4955**
4956** At the time this function is called it is guaranteed that
4957**
4958** * the sub-query uses only one distinct window frame, and
4959** * that the window frame has a PARTITION BY clase.
4960*/
4961static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){
4962 assert( pSubq->pWin->pPartition );
4963 assert( (pSubq->selFlags & SF_MultiPart)==0 );
4964 assert( pSubq->pPrior==0 );
4965 return sqlite3ExprIsConstantOrGroupBy(pParse, pExpr, pSubq->pWin->pPartition);
4966}
4967# endif /* SQLITE_OMIT_WINDOWFUNC */
4968#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */
4969
4970#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
drh69b72d52015-06-01 20:28:03 +00004971/*
4972** Make copies of relevant WHERE clause terms of the outer query into
4973** the WHERE clause of subquery. Example:
4974**
4975** SELECT * FROM (SELECT a AS x, c-d AS y FROM t1) WHERE x=5 AND y=10;
4976**
4977** Transformed into:
4978**
4979** SELECT * FROM (SELECT a AS x, c-d AS y FROM t1 WHERE a=5 AND c-d=10)
4980** WHERE x=5 AND y=10;
4981**
4982** The hope is that the terms added to the inner query will make it more
4983** efficient.
4984**
4985** Do not attempt this optimization if:
4986**
drh25c221e2017-09-29 22:13:24 +00004987** (1) (** This restriction was removed on 2017-09-29. We used to
4988** disallow this optimization for aggregate subqueries, but now
drh67cc51a2017-09-30 11:47:06 +00004989** it is allowed by putting the extra terms on the HAVING clause.
4990** The added HAVING clause is pointless if the subquery lacks
4991** a GROUP BY clause. But such a HAVING clause is also harmless
4992** so there does not appear to be any reason to add extra logic
4993** to suppress it. **)
drh69b72d52015-06-01 20:28:03 +00004994**
4995** (2) The inner query is the recursive part of a common table expression.
4996**
4997** (3) The inner query has a LIMIT clause (since the changes to the WHERE
dance103732018-06-23 07:59:39 +00004998** clause would change the meaning of the LIMIT).
drh69b72d52015-06-01 20:28:03 +00004999**
drh6a9b9522018-03-27 15:13:43 +00005000** (4) The inner query is the right operand of a LEFT JOIN and the
5001** expression to be pushed down does not come from the ON clause
5002** on that LEFT JOIN.
drh69b72d52015-06-01 20:28:03 +00005003**
drh38978dd2015-08-22 01:32:29 +00005004** (5) The WHERE clause expression originates in the ON or USING clause
drh7fbb1012018-03-21 01:59:46 +00005005** of a LEFT JOIN where iCursor is not the right-hand table of that
5006** left join. An example:
5007**
5008** SELECT *
5009** FROM (SELECT 1 AS a1 UNION ALL SELECT 2) AS aa
5010** JOIN (SELECT 1 AS b2 UNION ALL SELECT 2) AS bb ON (a1=b2)
5011** LEFT JOIN (SELECT 8 AS c3 UNION ALL SELECT 9) AS cc ON (b2=2);
5012**
5013** The correct answer is three rows: (1,1,NULL),(2,2,8),(2,2,9).
5014** But if the (b2=2) term were to be pushed down into the bb subquery,
5015** then the (1,1,NULL) row would be suppressed.
drh38978dd2015-08-22 01:32:29 +00005016**
dan903fdd42021-02-22 20:56:13 +00005017** (6) Window functions make things tricky as changes to the WHERE clause
5018** of the inner query could change the window over which window
5019** functions are calculated. Therefore, do not attempt the optimization
5020** if:
5021**
5022** (6a) The inner query uses multiple incompatible window partitions.
5023**
5024** (6b) The inner query is a compound and uses window-functions.
5025**
5026** (6c) The WHERE clause does not consist entirely of constants and
5027** copies of expressions found in the PARTITION BY clause of
5028** all window-functions used by the sub-query. It is safe to
5029** filter out entire partitions, as this does not change the
5030** window over which any window-function is calculated.
dance103732018-06-23 07:59:39 +00005031**
drh745912e2021-02-22 03:04:25 +00005032** (7) The inner query is a Common Table Expression (CTE) that should
5033** be materialized. (This restriction is implemented in the calling
5034** routine.)
5035**
drh69b72d52015-06-01 20:28:03 +00005036** Return 0 if no changes are made and non-zero if one or more WHERE clause
5037** terms are duplicated into the subquery.
5038*/
5039static int pushDownWhereTerms(
dan44c56042016-12-07 15:38:37 +00005040 Parse *pParse, /* Parse context (for malloc() and error reporting) */
drh69b72d52015-06-01 20:28:03 +00005041 Select *pSubq, /* The subquery whose WHERE clause is to be augmented */
5042 Expr *pWhere, /* The WHERE clause of the outer query */
drha9cdb902022-04-25 19:40:33 +00005043 SrcItem *pSrc /* The subquery term of the outer FROM clause */
drh69b72d52015-06-01 20:28:03 +00005044){
5045 Expr *pNew;
5046 int nChng = 0;
5047 if( pWhere==0 ) return 0;
dan903fdd42021-02-22 20:56:13 +00005048 if( pSubq->selFlags & (SF_Recursive|SF_MultiPart) ) return 0;
drh0e464b52022-04-25 20:47:58 +00005049 if( pSrc->fg.jointype & (JT_LTORJ|JT_RIGHT) ) return 0;
drh508e2d02017-09-30 01:25:04 +00005050
dance103732018-06-23 07:59:39 +00005051#ifndef SQLITE_OMIT_WINDOWFUNC
dan903fdd42021-02-22 20:56:13 +00005052 if( pSubq->pPrior ){
5053 Select *pSel;
5054 for(pSel=pSubq; pSel; pSel=pSel->pPrior){
5055 if( pSel->pWin ) return 0; /* restriction (6b) */
5056 }
5057 }else{
5058 if( pSubq->pWin && pSubq->pWin->pPartition==0 ) return 0;
danf65e3792020-06-10 10:58:15 +00005059 }
dance103732018-06-23 07:59:39 +00005060#endif
5061
drh508e2d02017-09-30 01:25:04 +00005062#ifdef SQLITE_DEBUG
5063 /* Only the first term of a compound can have a WITH clause. But make
5064 ** sure no other terms are marked SF_Recursive in case something changes
5065 ** in the future.
5066 */
5067 {
5068 Select *pX;
5069 for(pX=pSubq; pX; pX=pX->pPrior){
5070 assert( (pX->selFlags & (SF_Recursive))==0 );
drhb1ec87a2016-04-25 02:20:10 +00005071 }
drh69b72d52015-06-01 20:28:03 +00005072 }
drh508e2d02017-09-30 01:25:04 +00005073#endif
5074
drh69b72d52015-06-01 20:28:03 +00005075 if( pSubq->pLimit!=0 ){
drhb1ec87a2016-04-25 02:20:10 +00005076 return 0; /* restriction (3) */
drh69b72d52015-06-01 20:28:03 +00005077 }
5078 while( pWhere->op==TK_AND ){
drha9cdb902022-04-25 19:40:33 +00005079 nChng += pushDownWhereTerms(pParse, pSubq, pWhere->pRight, pSrc);
drh69b72d52015-06-01 20:28:03 +00005080 pWhere = pWhere->pLeft;
5081 }
drha9cdb902022-04-25 19:40:33 +00005082
5083#if 0 /* Legacy code. Checks now done by sqlite3ExprIsTableConstraint() */
drh6a9b9522018-03-27 15:13:43 +00005084 if( isLeftJoin
drh67a99db2022-05-13 14:52:04 +00005085 && (ExprHasProperty(pWhere,EP_OuterON)==0
drhd1985262022-04-11 11:25:28 +00005086 || pWhere->w.iJoin!=iCursor)
drh6a9b9522018-03-27 15:13:43 +00005087 ){
5088 return 0; /* restriction (4) */
5089 }
drh67a99db2022-05-13 14:52:04 +00005090 if( ExprHasProperty(pWhere,EP_OuterON)
drhd1985262022-04-11 11:25:28 +00005091 && pWhere->w.iJoin!=iCursor
drh796588a2022-02-05 21:49:47 +00005092 ){
drh7fbb1012018-03-21 01:59:46 +00005093 return 0; /* restriction (5) */
5094 }
drha9cdb902022-04-25 19:40:33 +00005095#endif
5096
drh5eb6e092022-04-25 20:38:42 +00005097 if( sqlite3ExprIsTableConstraint(pWhere, pSrc) ){
drh69b72d52015-06-01 20:28:03 +00005098 nChng++;
drh8794c682021-02-13 16:39:24 +00005099 pSubq->selFlags |= SF_PushDown;
drh69b72d52015-06-01 20:28:03 +00005100 while( pSubq ){
drh46967de2017-04-14 12:39:37 +00005101 SubstContext x;
dan44c56042016-12-07 15:38:37 +00005102 pNew = sqlite3ExprDup(pParse->db, pWhere, 0);
drh92d1afb2022-06-13 12:42:24 +00005103 unsetJoinExpr(pNew, -1, 1);
drh46967de2017-04-14 12:39:37 +00005104 x.pParse = pParse;
drha9cdb902022-04-25 19:40:33 +00005105 x.iTable = pSrc->iCursor;
5106 x.iNewTable = pSrc->iCursor;
drhc133bab2022-04-11 20:15:52 +00005107 x.isOuterJoin = 0;
drh46967de2017-04-14 12:39:37 +00005108 x.pEList = pSubq->pEList;
5109 pNew = substExpr(&x, pNew);
dan903fdd42021-02-22 20:56:13 +00005110#ifndef SQLITE_OMIT_WINDOWFUNC
5111 if( pSubq->pWin && 0==pushDownWindowCheck(pParse, pSubq, pNew) ){
5112 /* Restriction 6c has prevented push-down in this case */
5113 sqlite3ExprDelete(pParse->db, pNew);
5114 nChng--;
5115 break;
5116 }
5117#endif
drh25c221e2017-09-29 22:13:24 +00005118 if( pSubq->selFlags & SF_Aggregate ){
drhd5c851c2019-04-19 13:38:34 +00005119 pSubq->pHaving = sqlite3ExprAnd(pParse, pSubq->pHaving, pNew);
drh25c221e2017-09-29 22:13:24 +00005120 }else{
drhd5c851c2019-04-19 13:38:34 +00005121 pSubq->pWhere = sqlite3ExprAnd(pParse, pSubq->pWhere, pNew);
drh25c221e2017-09-29 22:13:24 +00005122 }
drh69b72d52015-06-01 20:28:03 +00005123 pSubq = pSubq->pPrior;
5124 }
5125 }
5126 return nChng;
5127}
5128#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */
5129
drh1350b032002-02-27 19:00:20 +00005130/*
drh47d9f832017-10-26 20:04:28 +00005131** The pFunc is the only aggregate function in the query. Check to see
5132** if the query is a candidate for the min/max optimization.
danielk1977a9d1ccb2008-01-05 17:39:29 +00005133**
drh47d9f832017-10-26 20:04:28 +00005134** If the query is a candidate for the min/max optimization, then set
5135** *ppMinMax to be an ORDER BY clause to be used for the optimization
5136** and return either WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX depending on
5137** whether pFunc is a min() or max() function.
danielk1977738bdcf2008-01-07 10:16:40 +00005138**
drh47d9f832017-10-26 20:04:28 +00005139** If the query is not a candidate for the min/max optimization, return
5140** WHERE_ORDERBY_NORMAL (which must be zero).
dan4ac391f2012-12-13 16:37:10 +00005141**
drh47d9f832017-10-26 20:04:28 +00005142** This routine must be called after aggregate functions have been
5143** located but before their arguments have been subjected to aggregate
5144** analysis.
danielk1977a9d1ccb2008-01-05 17:39:29 +00005145*/
drh47d9f832017-10-26 20:04:28 +00005146static u8 minMaxQuery(sqlite3 *db, Expr *pFunc, ExprList **ppMinMax){
5147 int eRet = WHERE_ORDERBY_NORMAL; /* Return value */
drha4eeccd2021-10-07 17:43:30 +00005148 ExprList *pEList; /* Arguments to agg function */
drh47d9f832017-10-26 20:04:28 +00005149 const char *zFunc; /* Name of aggregate function pFunc */
5150 ExprList *pOrderBy;
drhbe284e42020-02-27 16:21:39 +00005151 u8 sortFlags = 0;
danielk1977a9d1ccb2008-01-05 17:39:29 +00005152
drh47d9f832017-10-26 20:04:28 +00005153 assert( *ppMinMax==0 );
5154 assert( pFunc->op==TK_AGG_FUNCTION );
dan4f9adee2019-07-13 16:22:50 +00005155 assert( !IsWindowFunc(pFunc) );
drha4eeccd2021-10-07 17:43:30 +00005156 assert( ExprUseXList(pFunc) );
5157 pEList = pFunc->x.pList;
drhaf7b7652021-01-13 19:28:17 +00005158 if( pEList==0
5159 || pEList->nExpr!=1
5160 || ExprHasProperty(pFunc, EP_WinFunc)
5161 || OptimizationDisabled(db, SQLITE_MinMaxOpt)
5162 ){
dan6ba7ab02019-07-02 11:56:47 +00005163 return eRet;
5164 }
drhf9751072021-10-07 13:40:29 +00005165 assert( !ExprHasProperty(pFunc, EP_IntValue) );
drh47d9f832017-10-26 20:04:28 +00005166 zFunc = pFunc->u.zToken;
5167 if( sqlite3StrICmp(zFunc, "min")==0 ){
5168 eRet = WHERE_ORDERBY_MIN;
dan67e2bb92020-02-27 15:07:16 +00005169 if( sqlite3ExprCanBeNull(pEList->a[0].pExpr) ){
5170 sortFlags = KEYINFO_ORDER_BIGNULL;
5171 }
drh47d9f832017-10-26 20:04:28 +00005172 }else if( sqlite3StrICmp(zFunc, "max")==0 ){
5173 eRet = WHERE_ORDERBY_MAX;
dan5b32bdf2019-08-17 15:47:32 +00005174 sortFlags = KEYINFO_ORDER_DESC;
drh47d9f832017-10-26 20:04:28 +00005175 }else{
5176 return eRet;
danielk1977a9d1ccb2008-01-05 17:39:29 +00005177 }
drh47d9f832017-10-26 20:04:28 +00005178 *ppMinMax = pOrderBy = sqlite3ExprListDup(db, pEList, 0);
5179 assert( pOrderBy!=0 || db->mallocFailed );
drhd88fd532022-05-02 20:49:30 +00005180 if( pOrderBy ) pOrderBy->a[0].fg.sortFlags = sortFlags;
dan4ac391f2012-12-13 16:37:10 +00005181 return eRet;
danielk1977a9d1ccb2008-01-05 17:39:29 +00005182}
5183
5184/*
danielk1977a5533162009-02-24 10:01:51 +00005185** The select statement passed as the first argument is an aggregate query.
peter.d.reid60ec9142014-09-06 16:39:46 +00005186** The second argument is the associated aggregate-info object. This
danielk1977a5533162009-02-24 10:01:51 +00005187** function tests if the SELECT is of the form:
5188**
5189** SELECT count(*) FROM <tbl>
5190**
5191** where table is a database table, not a sub-select or view. If the query
5192** does match this pattern, then a pointer to the Table object representing
drha1c4c3c2021-11-05 11:52:33 +00005193** <tbl> is returned. Otherwise, NULL is returned.
5194**
drhe574a922021-11-26 15:08:55 +00005195** This routine checks to see if it is safe to use the count optimization.
5196** A correct answer is still obtained (though perhaps more slowly) if
5197** this routine returns NULL when it could have returned a table pointer.
5198** But returning the pointer when NULL should have been returned can
5199** result in incorrect answers and/or crashes. So, when in doubt, return NULL.
danielk1977a5533162009-02-24 10:01:51 +00005200*/
5201static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){
5202 Table *pTab;
5203 Expr *pExpr;
5204
5205 assert( !p->pGroupBy );
5206
drha1c4c3c2021-11-05 11:52:33 +00005207 if( p->pWhere
5208 || p->pEList->nExpr!=1
5209 || p->pSrc->nSrc!=1
5210 || p->pSrc->a[0].pSelect
5211 || pAggInfo->nFunc!=1
dan2c1b1dd2022-07-01 21:03:19 +00005212 || p->pHaving
danielk1977a5533162009-02-24 10:01:51 +00005213 ){
5214 return 0;
5215 }
danielk1977a5533162009-02-24 10:01:51 +00005216 pTab = p->pSrc->a[0].pTab;
drha1c4c3c2021-11-05 11:52:33 +00005217 assert( pTab!=0 );
5218 assert( !IsView(pTab) );
5219 if( !IsOrdinaryTable(pTab) ) return 0;
danielk1977a5533162009-02-24 10:01:51 +00005220 pExpr = p->pEList->a[0].pExpr;
drha1c4c3c2021-11-05 11:52:33 +00005221 assert( pExpr!=0 );
danielk1977a5533162009-02-24 10:01:51 +00005222 if( pExpr->op!=TK_AGG_FUNCTION ) return 0;
dan8745f8a2021-11-15 14:11:23 +00005223 if( pExpr->pAggInfo!=pAggInfo ) return 0;
drhd36e1042013-09-06 13:10:12 +00005224 if( (pAggInfo->aFunc[0].pFunc->funcFlags&SQLITE_FUNC_COUNT)==0 ) return 0;
drha1c4c3c2021-11-05 11:52:33 +00005225 assert( pAggInfo->aFunc[0].pFExpr==pExpr );
5226 testcase( ExprHasProperty(pExpr, EP_Distinct) );
5227 testcase( ExprHasProperty(pExpr, EP_WinFunc) );
dan4f9adee2019-07-13 16:22:50 +00005228 if( ExprHasProperty(pExpr, EP_Distinct|EP_WinFunc) ) return 0;
danielk1977a5533162009-02-24 10:01:51 +00005229
5230 return pTab;
5231}
5232
5233/*
danielk1977b1c685b2008-10-06 16:18:39 +00005234** If the source-list item passed as an argument was augmented with an
5235** INDEXED BY clause, then try to locate the specified index. If there
5236** was such a clause and the named index cannot be found, return
5237** SQLITE_ERROR and leave an error in pParse. Otherwise, populate
5238** pFrom->pIndex and return SQLITE_OK.
5239*/
drh76012942021-02-21 21:04:54 +00005240int sqlite3IndexedByLookup(Parse *pParse, SrcItem *pFrom){
drh18622712021-02-20 21:20:54 +00005241 Table *pTab = pFrom->pTab;
5242 char *zIndexedBy = pFrom->u1.zIndexedBy;
5243 Index *pIdx;
5244 assert( pTab!=0 );
5245 assert( pFrom->fg.isIndexedBy!=0 );
5246
5247 for(pIdx=pTab->pIndex;
5248 pIdx && sqlite3StrICmp(pIdx->zName, zIndexedBy);
5249 pIdx=pIdx->pNext
5250 );
5251 if( !pIdx ){
5252 sqlite3ErrorMsg(pParse, "no such index: %s", zIndexedBy, 0);
5253 pParse->checkSchema = 1;
5254 return SQLITE_ERROR;
danielk1977b1c685b2008-10-06 16:18:39 +00005255 }
drhdbfbb5a2021-10-07 23:04:50 +00005256 assert( pFrom->fg.isCte==0 );
drha79e2a22021-02-21 23:44:14 +00005257 pFrom->u2.pIBIndex = pIdx;
danielk1977b1c685b2008-10-06 16:18:39 +00005258 return SQLITE_OK;
5259}
drh18622712021-02-20 21:20:54 +00005260
drhc01b7302013-05-07 17:49:08 +00005261/*
5262** Detect compound SELECT statements that use an ORDER BY clause with
5263** an alternative collating sequence.
5264**
5265** SELECT ... FROM t1 EXCEPT SELECT ... FROM t2 ORDER BY .. COLLATE ...
5266**
5267** These are rewritten as a subquery:
5268**
5269** SELECT * FROM (SELECT ... FROM t1 EXCEPT SELECT ... FROM t2)
5270** ORDER BY ... COLLATE ...
5271**
5272** This transformation is necessary because the multiSelectOrderBy() routine
5273** above that generates the code for a compound SELECT with an ORDER BY clause
5274** uses a merge algorithm that requires the same collating sequence on the
5275** result columns as on the ORDER BY clause. See ticket
5276** http://www.sqlite.org/src/info/6709574d2a
5277**
5278** This transformation is only needed for EXCEPT, INTERSECT, and UNION.
5279** The UNION ALL operator works fine with multiSelectOrderBy() even when
5280** there are COLLATE terms in the ORDER BY.
5281*/
5282static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){
5283 int i;
5284 Select *pNew;
5285 Select *pX;
5286 sqlite3 *db;
5287 struct ExprList_item *a;
5288 SrcList *pNewSrc;
5289 Parse *pParse;
5290 Token dummy;
5291
5292 if( p->pPrior==0 ) return WRC_Continue;
5293 if( p->pOrderBy==0 ) return WRC_Continue;
5294 for(pX=p; pX && (pX->op==TK_ALL || pX->op==TK_SELECT); pX=pX->pPrior){}
5295 if( pX==0 ) return WRC_Continue;
5296 a = p->pOrderBy->a;
dan46daa992020-06-11 15:53:54 +00005297#ifndef SQLITE_OMIT_WINDOWFUNC
5298 /* If iOrderByCol is already non-zero, then it has already been matched
5299 ** to a result column of the SELECT statement. This occurs when the
5300 ** SELECT is rewritten for window-functions processing and then passed
5301 ** to sqlite3SelectPrep() and similar a second time. The rewriting done
5302 ** by this function is not required in this case. */
5303 if( a[0].u.x.iOrderByCol ) return WRC_Continue;
5304#endif
drhc01b7302013-05-07 17:49:08 +00005305 for(i=p->pOrderBy->nExpr-1; i>=0; i--){
5306 if( a[i].pExpr->flags & EP_Collate ) break;
5307 }
5308 if( i<0 ) return WRC_Continue;
5309
5310 /* If we reach this point, that means the transformation is required. */
5311
5312 pParse = pWalker->pParse;
5313 db = pParse->db;
5314 pNew = sqlite3DbMallocZero(db, sizeof(*pNew) );
5315 if( pNew==0 ) return WRC_Abort;
5316 memset(&dummy, 0, sizeof(dummy));
drhd44f8b22022-04-07 01:11:13 +00005317 pNewSrc = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&dummy,pNew,0);
drhc01b7302013-05-07 17:49:08 +00005318 if( pNewSrc==0 ) return WRC_Abort;
5319 *pNew = *p;
5320 p->pSrc = pNewSrc;
drh1a1d3cd2015-11-19 16:33:31 +00005321 p->pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ASTERISK, 0));
drhc01b7302013-05-07 17:49:08 +00005322 p->op = TK_SELECT;
5323 p->pWhere = 0;
5324 pNew->pGroupBy = 0;
5325 pNew->pHaving = 0;
5326 pNew->pOrderBy = 0;
5327 p->pPrior = 0;
drh8af9ad92014-02-10 18:56:05 +00005328 p->pNext = 0;
drhf932f712015-04-12 17:35:27 +00005329 p->pWith = 0;
danfcc057d2019-12-04 01:42:07 +00005330#ifndef SQLITE_OMIT_WINDOWFUNC
5331 p->pWinDefn = 0;
5332#endif
drh8af9ad92014-02-10 18:56:05 +00005333 p->selFlags &= ~SF_Compound;
danb33c50f2015-04-04 16:43:16 +00005334 assert( (p->selFlags & SF_Converted)==0 );
5335 p->selFlags |= SF_Converted;
drha6e3a8c2014-02-10 21:07:51 +00005336 assert( pNew->pPrior!=0 );
5337 pNew->pPrior->pNext = pNew;
drhc01b7302013-05-07 17:49:08 +00005338 pNew->pLimit = 0;
drhc01b7302013-05-07 17:49:08 +00005339 return WRC_Continue;
5340}
danielk1977b1c685b2008-10-06 16:18:39 +00005341
drh20292312015-11-21 13:24:46 +00005342/*
5343** Check to see if the FROM clause term pFrom has table-valued function
5344** arguments. If it does, leave an error message in pParse and return
5345** non-zero, since pFrom is not allowed to be a table-valued function.
5346*/
drh76012942021-02-21 21:04:54 +00005347static int cannotBeFunction(Parse *pParse, SrcItem *pFrom){
drh20292312015-11-21 13:24:46 +00005348 if( pFrom->fg.isTabFunc ){
5349 sqlite3ErrorMsg(pParse, "'%s' is not a function", pFrom->zName);
5350 return 1;
5351 }
5352 return 0;
5353}
5354
daneede6a52014-01-15 19:42:23 +00005355#ifndef SQLITE_OMIT_CTE
5356/*
5357** Argument pWith (which may be NULL) points to a linked list of nested
5358** WITH contexts, from inner to outermost. If the table identified by
5359** FROM clause element pItem is really a common-table-expression (CTE)
5360** then return a pointer to the CTE definition for that table. Otherwise
5361** return NULL.
dan98f45e52014-01-17 17:40:46 +00005362**
5363** If a non-NULL value is returned, set *ppContext to point to the With
5364** object that the returned CTE belongs to.
drh60c1a2f2014-01-15 18:23:00 +00005365*/
dan98f45e52014-01-17 17:40:46 +00005366static struct Cte *searchWith(
drh2476a6f2015-11-07 15:19:59 +00005367 With *pWith, /* Current innermost WITH clause */
drh76012942021-02-21 21:04:54 +00005368 SrcItem *pItem, /* FROM clause element to resolve */
dan98f45e52014-01-17 17:40:46 +00005369 With **ppContext /* OUT: WITH clause return value belongs to */
5370){
drh5abe1d32021-02-21 01:19:42 +00005371 const char *zName = pItem->zName;
5372 With *p;
5373 assert( pItem->zDatabase==0 );
5374 assert( zName!=0 );
5375 for(p=pWith; p; p=p->pOuter){
5376 int i;
5377 for(i=0; i<p->nCte; i++){
5378 if( sqlite3StrICmp(zName, p->a[i].zName)==0 ){
5379 *ppContext = p;
5380 return &p->a[i];
dan4e9119d2014-01-13 15:12:23 +00005381 }
5382 }
dan90bc36f2021-05-20 17:15:06 +00005383 if( p->bView ) break;
dan4e9119d2014-01-13 15:12:23 +00005384 }
5385 return 0;
5386}
5387
drhc49832c2014-01-15 18:35:52 +00005388/* The code generator maintains a stack of active WITH clauses
5389** with the inner-most WITH clause being at the top of the stack.
5390**
danb290f112014-01-17 14:59:27 +00005391** This routine pushes the WITH clause passed as the second argument
5392** onto the top of the stack. If argument bFree is true, then this
drh7cc73b32021-05-23 17:47:04 +00005393** WITH clause will never be popped from the stack but should instead
5394** be freed along with the Parse object. In other cases, when
danb290f112014-01-17 14:59:27 +00005395** bFree==0, the With object will be freed along with the SELECT
5396** statement with which it is associated.
drh24ce9442021-06-12 17:45:32 +00005397**
5398** This routine returns a copy of pWith. Or, if bFree is true and
5399** the pWith object is destroyed immediately due to an OOM condition,
5400** then this routine return NULL.
5401**
5402** If bFree is true, do not continue to use the pWith pointer after
5403** calling this routine, Instead, use only the return value.
drhc49832c2014-01-15 18:35:52 +00005404*/
drh24ce9442021-06-12 17:45:32 +00005405With *sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){
dan4e9119d2014-01-13 15:12:23 +00005406 if( pWith ){
drh24ce9442021-06-12 17:45:32 +00005407 if( bFree ){
5408 pWith = (With*)sqlite3ParserAddCleanup(pParse,
5409 (void(*)(sqlite3*,void*))sqlite3WithDelete,
5410 pWith);
5411 if( pWith==0 ) return 0;
5412 }
drh7cc73b32021-05-23 17:47:04 +00005413 if( pParse->nErr==0 ){
5414 assert( pParse->pWith!=pWith );
5415 pWith->pOuter = pParse->pWith;
5416 pParse->pWith = pWith;
5417 }
dan4e9119d2014-01-13 15:12:23 +00005418 }
drh24ce9442021-06-12 17:45:32 +00005419 return pWith;
dan4e9119d2014-01-13 15:12:23 +00005420}
dan4e9119d2014-01-13 15:12:23 +00005421
daneede6a52014-01-15 19:42:23 +00005422/*
daneede6a52014-01-15 19:42:23 +00005423** This function checks if argument pFrom refers to a CTE declared by
drh5abe1d32021-02-21 01:19:42 +00005424** a WITH clause on the stack currently maintained by the parser (on the
5425** pParse->pWith linked list). And if currently processing a CTE
5426** CTE expression, through routine checks to see if the reference is
5427** a recursive reference to the CTE.
daneede6a52014-01-15 19:42:23 +00005428**
drh5abe1d32021-02-21 01:19:42 +00005429** If pFrom matches a CTE according to either of these two above, pFrom->pTab
5430** and other fields are populated accordingly.
daneede6a52014-01-15 19:42:23 +00005431**
drh5abe1d32021-02-21 01:19:42 +00005432** Return 0 if no match is found.
5433** Return 1 if a match is found.
5434** Return 2 if an error condition is detected.
daneede6a52014-01-15 19:42:23 +00005435*/
drh5abe1d32021-02-21 01:19:42 +00005436static int resolveFromTermToCte(
5437 Parse *pParse, /* The parsing context */
5438 Walker *pWalker, /* Current tree walker */
drh76012942021-02-21 21:04:54 +00005439 SrcItem *pFrom /* The FROM clause term to check */
dan8ce71842014-01-14 20:14:09 +00005440){
drh5abe1d32021-02-21 01:19:42 +00005441 Cte *pCte; /* Matched CTE (or NULL if no match) */
5442 With *pWith; /* The matching WITH */
dan8ce71842014-01-14 20:14:09 +00005443
dan8ce71842014-01-14 20:14:09 +00005444 assert( pFrom->pTab==0 );
drh5abe1d32021-02-21 01:19:42 +00005445 if( pParse->pWith==0 ){
5446 /* There are no WITH clauses in the stack. No match is possible */
5447 return 0;
drh46a31cd2019-11-09 14:38:58 +00005448 }
drh93c81392021-05-21 21:49:07 +00005449 if( pParse->nErr ){
5450 /* Prior errors might have left pParse->pWith in a goofy state, so
5451 ** go no further. */
5452 return 0;
5453 }
drh5abe1d32021-02-21 01:19:42 +00005454 if( pFrom->zDatabase!=0 ){
5455 /* The FROM term contains a schema qualifier (ex: main.t1) and so
5456 ** it cannot possibly be a CTE reference. */
5457 return 0;
5458 }
drhcd1499f2021-05-20 00:44:04 +00005459 if( pFrom->fg.notCte ){
5460 /* The FROM term is specifically excluded from matching a CTE.
5461 ** (1) It is part of a trigger that used to have zDatabase but had
5462 ** zDatabase removed by sqlite3FixTriggerStep().
5463 ** (2) This is the first term in the FROM clause of an UPDATE.
5464 */
5465 return 0;
5466 }
dan98f45e52014-01-17 17:40:46 +00005467 pCte = searchWith(pParse->pWith, pFrom, &pWith);
daneae73fb2014-01-16 18:34:33 +00005468 if( pCte ){
drh5abe1d32021-02-21 01:19:42 +00005469 sqlite3 *db = pParse->db;
dan98f45e52014-01-17 17:40:46 +00005470 Table *pTab;
dan8ce71842014-01-14 20:14:09 +00005471 ExprList *pEList;
5472 Select *pSel;
dan60e70682014-01-15 15:27:51 +00005473 Select *pLeft; /* Left-most SELECT statement */
drh34055852020-10-19 01:23:48 +00005474 Select *pRecTerm; /* Left-most recursive term */
danf2655fe2014-01-16 21:02:02 +00005475 int bMayRecursive; /* True if compound joined by UNION [ALL] */
dan98f45e52014-01-17 17:40:46 +00005476 With *pSavedWith; /* Initial value of pParse->pWith */
drh34055852020-10-19 01:23:48 +00005477 int iRecTab = -1; /* Cursor for recursive table */
drh745912e2021-02-22 03:04:25 +00005478 CteUse *pCteUse;
danf2655fe2014-01-16 21:02:02 +00005479
drh0576bc52015-08-26 11:40:11 +00005480 /* If pCte->zCteErr is non-NULL at this point, then this is an illegal
danf2655fe2014-01-16 21:02:02 +00005481 ** recursive reference to CTE pCte. Leave an error in pParse and return
drh0576bc52015-08-26 11:40:11 +00005482 ** early. If pCte->zCteErr is NULL, then this is not a recursive reference.
danf2655fe2014-01-16 21:02:02 +00005483 ** In this case, proceed. */
drh0576bc52015-08-26 11:40:11 +00005484 if( pCte->zCteErr ){
5485 sqlite3ErrorMsg(pParse, pCte->zCteErr, pCte->zName);
drh5abe1d32021-02-21 01:19:42 +00005486 return 2;
danf2655fe2014-01-16 21:02:02 +00005487 }
drh5abe1d32021-02-21 01:19:42 +00005488 if( cannotBeFunction(pParse, pFrom) ) return 2;
dan8ce71842014-01-14 20:14:09 +00005489
drhc25e2eb2014-01-20 14:58:55 +00005490 assert( pFrom->pTab==0 );
drha79e2a22021-02-21 23:44:14 +00005491 pTab = sqlite3DbMallocZero(db, sizeof(Table));
drh5abe1d32021-02-21 01:19:42 +00005492 if( pTab==0 ) return 2;
drh745912e2021-02-22 03:04:25 +00005493 pCteUse = pCte->pUse;
5494 if( pCteUse==0 ){
5495 pCte->pUse = pCteUse = sqlite3DbMallocZero(db, sizeof(pCteUse[0]));
5496 if( pCteUse==0
5497 || sqlite3ParserAddCleanup(pParse,sqlite3DbFree,pCteUse)==0
drha79e2a22021-02-21 23:44:14 +00005498 ){
5499 sqlite3DbFree(db, pTab);
5500 return 2;
5501 }
drh745912e2021-02-22 03:04:25 +00005502 pCteUse->eM10d = pCte->eM10d;
drha79e2a22021-02-21 23:44:14 +00005503 }
5504 pFrom->pTab = pTab;
drh79df7782016-12-14 14:07:35 +00005505 pTab->nTabRef = 1;
dan2d4dc5f2014-01-17 11:48:24 +00005506 pTab->zName = sqlite3DbStrDup(db, pCte->zName);
dan8ce71842014-01-14 20:14:09 +00005507 pTab->iPKey = -1;
dancfc9df72014-04-25 15:01:01 +00005508 pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) );
drhfccda8a2015-05-27 13:06:55 +00005509 pTab->tabFlags |= TF_Ephemeral | TF_NoVisibleRowid;
dan8ce71842014-01-14 20:14:09 +00005510 pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0);
drh5abe1d32021-02-21 01:19:42 +00005511 if( db->mallocFailed ) return 2;
danac67f562021-06-14 20:08:48 +00005512 pFrom->pSelect->selFlags |= SF_CopyCte;
dan8ce71842014-01-14 20:14:09 +00005513 assert( pFrom->pSelect );
drh0c1da682021-10-09 16:00:56 +00005514 if( pFrom->fg.isIndexedBy ){
5515 sqlite3ErrorMsg(pParse, "no such index: \"%s\"", pFrom->u1.zIndexedBy);
5516 return 2;
5517 }
drha79e2a22021-02-21 23:44:14 +00005518 pFrom->fg.isCte = 1;
drh745912e2021-02-22 03:04:25 +00005519 pFrom->u2.pCteUse = pCteUse;
5520 pCteUse->nUse++;
5521 if( pCteUse->nUse>=2 && pCteUse->eM10d==M10d_Any ){
5522 pCteUse->eM10d = M10d_Yes;
5523 }
dan8ce71842014-01-14 20:14:09 +00005524
daneae73fb2014-01-16 18:34:33 +00005525 /* Check if this is a recursive CTE. */
drh34055852020-10-19 01:23:48 +00005526 pRecTerm = pSel = pFrom->pSelect;
danf2655fe2014-01-16 21:02:02 +00005527 bMayRecursive = ( pSel->op==TK_ALL || pSel->op==TK_UNION );
drh34055852020-10-19 01:23:48 +00005528 while( bMayRecursive && pRecTerm->op==pSel->op ){
daneae73fb2014-01-16 18:34:33 +00005529 int i;
drh34055852020-10-19 01:23:48 +00005530 SrcList *pSrc = pRecTerm->pSrc;
5531 assert( pRecTerm->pPrior!=0 );
daneae73fb2014-01-16 18:34:33 +00005532 for(i=0; i<pSrc->nSrc; i++){
drh76012942021-02-21 21:04:54 +00005533 SrcItem *pItem = &pSrc->a[i];
daneae73fb2014-01-16 18:34:33 +00005534 if( pItem->zDatabase==0
5535 && pItem->zName!=0
5536 && 0==sqlite3StrICmp(pItem->zName, pCte->zName)
drh34055852020-10-19 01:23:48 +00005537 ){
daneae73fb2014-01-16 18:34:33 +00005538 pItem->pTab = pTab;
drh94d6f3f2020-10-19 20:49:54 +00005539 pTab->nTabRef++;
drh8a48b9c2015-08-19 15:20:00 +00005540 pItem->fg.isRecursive = 1;
drh34055852020-10-19 01:23:48 +00005541 if( pRecTerm->selFlags & SF_Recursive ){
5542 sqlite3ErrorMsg(pParse,
5543 "multiple references to recursive table: %s", pCte->zName
5544 );
drh5abe1d32021-02-21 01:19:42 +00005545 return 2;
drh34055852020-10-19 01:23:48 +00005546 }
drh34055852020-10-19 01:23:48 +00005547 pRecTerm->selFlags |= SF_Recursive;
5548 if( iRecTab<0 ) iRecTab = pParse->nTab++;
5549 pItem->iCursor = iRecTab;
daneae73fb2014-01-16 18:34:33 +00005550 }
5551 }
drh34055852020-10-19 01:23:48 +00005552 if( (pRecTerm->selFlags & SF_Recursive)==0 ) break;
5553 pRecTerm = pRecTerm->pPrior;
dan8ce71842014-01-14 20:14:09 +00005554 }
5555
drh0576bc52015-08-26 11:40:11 +00005556 pCte->zCteErr = "circular reference: %s";
dan98f45e52014-01-17 17:40:46 +00005557 pSavedWith = pParse->pWith;
5558 pParse->pWith = pWith;
drh34055852020-10-19 01:23:48 +00005559 if( pSel->selFlags & SF_Recursive ){
danca237a82021-03-18 18:25:43 +00005560 int rc;
drh34055852020-10-19 01:23:48 +00005561 assert( pRecTerm!=0 );
5562 assert( (pRecTerm->selFlags & SF_Recursive)==0 );
5563 assert( pRecTerm->pNext!=0 );
5564 assert( (pRecTerm->pNext->selFlags & SF_Recursive)!=0 );
5565 assert( pRecTerm->pWith==0 );
5566 pRecTerm->pWith = pSel->pWith;
danca237a82021-03-18 18:25:43 +00005567 rc = sqlite3WalkSelect(pWalker, pRecTerm);
drh34055852020-10-19 01:23:48 +00005568 pRecTerm->pWith = 0;
danca237a82021-03-18 18:25:43 +00005569 if( rc ){
5570 pParse->pWith = pSavedWith;
5571 return 2;
5572 }
dan067cd832017-02-03 19:16:39 +00005573 }else{
danca237a82021-03-18 18:25:43 +00005574 if( sqlite3WalkSelect(pWalker, pSel) ){
5575 pParse->pWith = pSavedWith;
5576 return 2;
5577 }
dan067cd832017-02-03 19:16:39 +00005578 }
drh6e772262015-11-07 17:48:21 +00005579 pParse->pWith = pWith;
daneae73fb2014-01-16 18:34:33 +00005580
dan60e70682014-01-15 15:27:51 +00005581 for(pLeft=pSel; pLeft->pPrior; pLeft=pLeft->pPrior);
5582 pEList = pLeft->pEList;
dan8ce71842014-01-14 20:14:09 +00005583 if( pCte->pCols ){
drh8f9d0b22015-03-21 03:18:22 +00005584 if( pEList && pEList->nExpr!=pCte->pCols->nExpr ){
drh727a99f2014-01-16 21:59:51 +00005585 sqlite3ErrorMsg(pParse, "table %s has %d values for %d columns",
dan60e70682014-01-15 15:27:51 +00005586 pCte->zName, pEList->nExpr, pCte->pCols->nExpr
5587 );
dan98f45e52014-01-17 17:40:46 +00005588 pParse->pWith = pSavedWith;
drh5abe1d32021-02-21 01:19:42 +00005589 return 2;
dan60e70682014-01-15 15:27:51 +00005590 }
dan8ce71842014-01-14 20:14:09 +00005591 pEList = pCte->pCols;
dan8ce71842014-01-14 20:14:09 +00005592 }
dan8ce71842014-01-14 20:14:09 +00005593
drh8981b902015-08-24 17:42:49 +00005594 sqlite3ColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol);
danf2655fe2014-01-16 21:02:02 +00005595 if( bMayRecursive ){
5596 if( pSel->selFlags & SF_Recursive ){
drh0576bc52015-08-26 11:40:11 +00005597 pCte->zCteErr = "multiple recursive references: %s";
danf2655fe2014-01-16 21:02:02 +00005598 }else{
drh0576bc52015-08-26 11:40:11 +00005599 pCte->zCteErr = "recursive reference in a subquery: %s";
danf2655fe2014-01-16 21:02:02 +00005600 }
5601 sqlite3WalkSelect(pWalker, pSel);
5602 }
drh0576bc52015-08-26 11:40:11 +00005603 pCte->zCteErr = 0;
dan98f45e52014-01-17 17:40:46 +00005604 pParse->pWith = pSavedWith;
drh5abe1d32021-02-21 01:19:42 +00005605 return 1; /* Success */
dan8ce71842014-01-14 20:14:09 +00005606 }
drh5abe1d32021-02-21 01:19:42 +00005607 return 0; /* No match */
dan8ce71842014-01-14 20:14:09 +00005608}
daneede6a52014-01-15 19:42:23 +00005609#endif
dan4e9119d2014-01-13 15:12:23 +00005610
danb290f112014-01-17 14:59:27 +00005611#ifndef SQLITE_OMIT_CTE
dan71856942014-01-17 15:15:10 +00005612/*
5613** If the SELECT passed as the second argument has an associated WITH
5614** clause, pop it from the stack stored as part of the Parse object.
5615**
5616** This function is used as the xSelectCallback2() callback by
5617** sqlite3SelectExpand() when walking a SELECT tree to resolve table
5618** names and other FROM clause elements.
5619*/
danbe120832021-05-17 16:20:41 +00005620void sqlite3SelectPopWith(Walker *pWalker, Select *p){
danb290f112014-01-17 14:59:27 +00005621 Parse *pParse = pWalker->pParse;
drh2f65b2f2017-10-02 21:29:51 +00005622 if( OK_IF_ALWAYS_TRUE(pParse->pWith) && p->pPrior==0 ){
dan067cd832017-02-03 19:16:39 +00005623 With *pWith = findRightmost(p)->pWith;
5624 if( pWith!=0 ){
dan70a32702020-01-21 14:42:48 +00005625 assert( pParse->pWith==pWith || pParse->nErr );
dan067cd832017-02-03 19:16:39 +00005626 pParse->pWith = pWith->pOuter;
5627 }
danb290f112014-01-17 14:59:27 +00005628 }
danb290f112014-01-17 14:59:27 +00005629}
danb290f112014-01-17 14:59:27 +00005630#endif
5631
dan9a947222018-06-14 19:06:36 +00005632/*
5633** The SrcList_item structure passed as the second argument represents a
5634** sub-query in the FROM clause of a SELECT statement. This function
5635** allocates and populates the SrcList_item.pTab object. If successful,
5636** SQLITE_OK is returned. Otherwise, if an OOM error is encountered,
5637** SQLITE_NOMEM.
5638*/
drh76012942021-02-21 21:04:54 +00005639int sqlite3ExpandSubquery(Parse *pParse, SrcItem *pFrom){
dan86fb6e12018-05-16 20:58:07 +00005640 Select *pSel = pFrom->pSelect;
5641 Table *pTab;
5642
dan9a947222018-06-14 19:06:36 +00005643 assert( pSel );
dan86fb6e12018-05-16 20:58:07 +00005644 pFrom->pTab = pTab = sqlite3DbMallocZero(pParse->db, sizeof(Table));
dan9a947222018-06-14 19:06:36 +00005645 if( pTab==0 ) return SQLITE_NOMEM;
dan86fb6e12018-05-16 20:58:07 +00005646 pTab->nTabRef = 1;
5647 if( pFrom->zAlias ){
5648 pTab->zName = sqlite3DbStrDup(pParse->db, pFrom->zAlias);
5649 }else{
drhd6bb6002022-04-22 18:07:38 +00005650 pTab->zName = sqlite3MPrintf(pParse->db, "%!S", pFrom);
dan86fb6e12018-05-16 20:58:07 +00005651 }
5652 while( pSel->pPrior ){ pSel = pSel->pPrior; }
5653 sqlite3ColumnsFromExprList(pParse, pSel->pEList,&pTab->nCol,&pTab->aCol);
5654 pTab->iPKey = -1;
5655 pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) );
drh6e5020e2021-04-07 15:45:01 +00005656#ifndef SQLITE_ALLOW_ROWID_IN_VIEW
5657 /* The usual case - do not allow ROWID on a subquery */
5658 pTab->tabFlags |= TF_Ephemeral | TF_NoVisibleRowid;
5659#else
5660 pTab->tabFlags |= TF_Ephemeral; /* Legacy compatibility mode */
5661#endif
drhf4b33152019-04-23 12:30:15 +00005662 return pParse->nErr ? SQLITE_ERROR : SQLITE_OK;
dan86fb6e12018-05-16 20:58:07 +00005663}
5664
drh4ce7bf92022-04-15 18:30:48 +00005665
5666/*
5667** Check the N SrcItem objects to the right of pBase. (N might be zero!)
5668** If any of those SrcItem objects have a USING clause containing zName
5669** then return true.
5670**
5671** If N is zero, or none of the N SrcItem objects to the right of pBase
5672** contains a USING clause, or if none of the USING clauses contain zName,
5673** then return false.
5674*/
5675static int inAnyUsingClause(
5676 const char *zName, /* Name we are looking for */
5677 SrcItem *pBase, /* The base SrcItem. Looking at pBase[1] and following */
5678 int N /* How many SrcItems to check */
5679){
5680 while( N>0 ){
5681 N--;
5682 pBase++;
5683 if( pBase->fg.isUsing==0 ) continue;
drhe32d6a02022-04-19 15:56:03 +00005684 if( NEVER(pBase->u3.pUsing==0) ) continue;
drh4ce7bf92022-04-15 18:30:48 +00005685 if( sqlite3IdListIndex(pBase->u3.pUsing, zName)>=0 ) return 1;
5686 }
5687 return 0;
5688}
5689
5690
danielk1977b1c685b2008-10-06 16:18:39 +00005691/*
drh7d10d5a2008-08-20 16:35:10 +00005692** This routine is a Walker callback for "expanding" a SELECT statement.
5693** "Expanding" means to do the following:
5694**
5695** (1) Make sure VDBE cursor numbers have been assigned to every
5696** element of the FROM clause.
5697**
5698** (2) Fill in the pTabList->a[].pTab fields in the SrcList that
5699** defines FROM clause. When views appear in the FROM clause,
5700** fill pTabList->a[].pSelect with a copy of the SELECT statement
5701** that implements the view. A copy is made of the view's SELECT
5702** statement so that we can freely modify or delete that statement
peter.d.reid60ec9142014-09-06 16:39:46 +00005703** without worrying about messing up the persistent representation
drh7d10d5a2008-08-20 16:35:10 +00005704** of the view.
5705**
peter.d.reid60ec9142014-09-06 16:39:46 +00005706** (3) Add terms to the WHERE clause to accommodate the NATURAL keyword
drh7d10d5a2008-08-20 16:35:10 +00005707** on joins and the ON and USING clause of joins.
5708**
5709** (4) Scan the list of columns in the result set (pEList) looking
5710** for instances of the "*" operator or the TABLE.* operator.
5711** If found, expand each "*" to be every column in every table
5712** and TABLE.* to be every column in TABLE.
5713**
danielk1977b3bce662005-01-29 08:32:43 +00005714*/
drh7d10d5a2008-08-20 16:35:10 +00005715static int selectExpander(Walker *pWalker, Select *p){
5716 Parse *pParse = pWalker->pParse;
drh5abe1d32021-02-21 01:19:42 +00005717 int i, j, k, rc;
drh7d10d5a2008-08-20 16:35:10 +00005718 SrcList *pTabList;
5719 ExprList *pEList;
drh76012942021-02-21 21:04:54 +00005720 SrcItem *pFrom;
drh7d10d5a2008-08-20 16:35:10 +00005721 sqlite3 *db = pParse->db;
drh3e3f1a52013-01-03 00:45:56 +00005722 Expr *pE, *pRight, *pExpr;
drh785097d2013-02-12 22:09:48 +00005723 u16 selFlags = p->selFlags;
drhfca23552017-10-28 20:51:54 +00005724 u32 elistFlags = 0;
danielk1977b3bce662005-01-29 08:32:43 +00005725
drh785097d2013-02-12 22:09:48 +00005726 p->selFlags |= SF_Expanded;
drh7d10d5a2008-08-20 16:35:10 +00005727 if( db->mallocFailed ){
5728 return WRC_Abort;
danielk1977b3bce662005-01-29 08:32:43 +00005729 }
drh9d9c41e2017-10-31 03:40:15 +00005730 assert( p->pSrc!=0 );
5731 if( (selFlags & SF_Expanded)!=0 ){
drh7d10d5a2008-08-20 16:35:10 +00005732 return WRC_Prune;
danielk1977b3bce662005-01-29 08:32:43 +00005733 }
drh59145812019-05-22 22:49:23 +00005734 if( pWalker->eCode ){
5735 /* Renumber selId because it has been copied from a view */
5736 p->selId = ++pParse->nSelect;
5737 }
drh7d10d5a2008-08-20 16:35:10 +00005738 pTabList = p->pSrc;
danielk1977b3bce662005-01-29 08:32:43 +00005739 pEList = p->pEList;
dan90bc36f2021-05-20 17:15:06 +00005740 if( pParse->pWith && (p->selFlags & SF_View) ){
5741 if( p->pWith==0 ){
5742 p->pWith = (With*)sqlite3DbMallocZero(db, sizeof(With));
5743 if( p->pWith==0 ){
5744 return WRC_Abort;
5745 }
5746 }
5747 p->pWith->bView = 1;
5748 }
drha5746e02018-04-09 20:36:09 +00005749 sqlite3WithPush(pParse, p->pWith, 0);
drh7d10d5a2008-08-20 16:35:10 +00005750
5751 /* Make sure cursor numbers have been assigned to all entries in
5752 ** the FROM clause of the SELECT statement.
5753 */
5754 sqlite3SrcListAssignCursors(pParse, pTabList);
5755
5756 /* Look up every table named in the FROM clause of the select. If
5757 ** an entry of the FROM clause is a subquery instead of a table or view,
5758 ** then create a transient table structure to describe the subquery.
5759 */
5760 for(i=0, pFrom=pTabList->a; i<pTabList->nSrc; i++, pFrom++){
5761 Table *pTab;
drhe2b7d7a2015-09-29 15:50:04 +00005762 assert( pFrom->fg.isRecursive==0 || pFrom->pTab!=0 );
dan69887c92020-04-27 20:55:33 +00005763 if( pFrom->pTab ) continue;
5764 assert( pFrom->fg.isRecursive==0 );
drh7d10d5a2008-08-20 16:35:10 +00005765 if( pFrom->zName==0 ){
5766#ifndef SQLITE_OMIT_SUBQUERY
5767 Select *pSel = pFrom->pSelect;
5768 /* A sub-query in the FROM clause of a SELECT */
5769 assert( pSel!=0 );
5770 assert( pFrom->pTab==0 );
drh2b8c5a02015-01-22 12:00:17 +00005771 if( sqlite3WalkSelect(pWalker, pSel) ) return WRC_Abort;
dandfa552f2018-06-02 21:04:28 +00005772 if( sqlite3ExpandSubquery(pParse, pFrom) ) return WRC_Abort;
dan86fb6e12018-05-16 20:58:07 +00005773#endif
drh5abe1d32021-02-21 01:19:42 +00005774#ifndef SQLITE_OMIT_CTE
5775 }else if( (rc = resolveFromTermToCte(pParse, pWalker, pFrom))!=0 ){
5776 if( rc>1 ) return WRC_Abort;
5777 pTab = pFrom->pTab;
5778 assert( pTab!=0 );
5779#endif
drh7d10d5a2008-08-20 16:35:10 +00005780 }else{
5781 /* An ordinary table or view name in the FROM clause */
5782 assert( pFrom->pTab==0 );
dan41fb5cd2012-10-04 19:33:00 +00005783 pFrom->pTab = pTab = sqlite3LocateTableItem(pParse, 0, pFrom);
drh7d10d5a2008-08-20 16:35:10 +00005784 if( pTab==0 ) return WRC_Abort;
drh79df7782016-12-14 14:07:35 +00005785 if( pTab->nTabRef>=0xffff ){
drhd2a56232013-01-28 19:00:20 +00005786 sqlite3ErrorMsg(pParse, "too many references to \"%s\": max 65535",
5787 pTab->zName);
5788 pFrom->pTab = 0;
5789 return WRC_Abort;
5790 }
drh79df7782016-12-14 14:07:35 +00005791 pTab->nTabRef++;
drh20292312015-11-21 13:24:46 +00005792 if( !IsVirtual(pTab) && cannotBeFunction(pParse, pFrom) ){
5793 return WRC_Abort;
5794 }
dan8c812f92020-01-21 16:23:17 +00005795#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE)
drhf38524d2021-08-02 16:41:57 +00005796 if( !IsOrdinaryTable(pTab) ){
drhbfad7be2015-10-11 20:39:46 +00005797 i16 nCol;
drh59145812019-05-22 22:49:23 +00005798 u8 eCodeOrig = pWalker->eCode;
drh7d10d5a2008-08-20 16:35:10 +00005799 if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort;
drh43152cf2009-05-19 19:04:58 +00005800 assert( pFrom->pSelect==0 );
drhf38524d2021-08-02 16:41:57 +00005801 if( IsView(pTab) ){
5802 if( (db->flags & SQLITE_EnableView)==0
5803 && pTab->pSchema!=db->aDb[1].pSchema
5804 ){
5805 sqlite3ErrorMsg(pParse, "access to view \"%s\" prohibited",
5806 pTab->zName);
5807 }
5808 pFrom->pSelect = sqlite3SelectDup(db, pTab->u.view.pSelect, 0);
danafaa6602021-09-30 18:42:52 +00005809 }
dan8c812f92020-01-21 16:23:17 +00005810#ifndef SQLITE_OMIT_VIRTUALTABLE
danafaa6602021-09-30 18:42:52 +00005811 else if( ALWAYS(IsVirtual(pTab))
drh3f681422020-01-07 18:10:56 +00005812 && pFrom->fg.fromDDL
drhf38524d2021-08-02 16:41:57 +00005813 && ALWAYS(pTab->u.vtab.p!=0)
5814 && pTab->u.vtab.p->eVtabRisk > ((db->flags & SQLITE_TrustedSchema)!=0)
drh3f681422020-01-07 18:10:56 +00005815 ){
drh32266a12020-01-09 13:08:28 +00005816 sqlite3ErrorMsg(pParse, "unsafe use of virtual table \"%s\"",
5817 pTab->zName);
drh11d88e62019-08-15 21:27:20 +00005818 }
drhf38524d2021-08-02 16:41:57 +00005819 assert( SQLITE_VTABRISK_Normal==1 && SQLITE_VTABRISK_High==2 );
dan8c812f92020-01-21 16:23:17 +00005820#endif
drhbfad7be2015-10-11 20:39:46 +00005821 nCol = pTab->nCol;
5822 pTab->nCol = -1;
drh59145812019-05-22 22:49:23 +00005823 pWalker->eCode = 1; /* Turn on Select.selId renumbering */
drh43152cf2009-05-19 19:04:58 +00005824 sqlite3WalkSelect(pWalker, pFrom->pSelect);
drh59145812019-05-22 22:49:23 +00005825 pWalker->eCode = eCodeOrig;
drhbfad7be2015-10-11 20:39:46 +00005826 pTab->nCol = nCol;
drh7d10d5a2008-08-20 16:35:10 +00005827 }
5828#endif
danielk1977b3bce662005-01-29 08:32:43 +00005829 }
danielk197785574e32008-10-06 05:32:18 +00005830
5831 /* Locate the index named by the INDEXED BY clause, if any. */
drh18622712021-02-20 21:20:54 +00005832 if( pFrom->fg.isIndexedBy && sqlite3IndexedByLookup(pParse, pFrom) ){
danielk1977b1c685b2008-10-06 16:18:39 +00005833 return WRC_Abort;
danielk197785574e32008-10-06 05:32:18 +00005834 }
danielk1977b3bce662005-01-29 08:32:43 +00005835 }
5836
drh7d10d5a2008-08-20 16:35:10 +00005837 /* Process NATURAL keywords, and ON and USING clauses of joins.
danielk1977b3bce662005-01-29 08:32:43 +00005838 */
drh0c7d3d32022-01-24 16:47:12 +00005839 assert( db->mallocFailed==0 || pParse->nErr!=0 );
drh358424a2022-04-15 15:15:01 +00005840 if( pParse->nErr || sqlite3ProcessJoin(pParse, p) ){
drh7d10d5a2008-08-20 16:35:10 +00005841 return WRC_Abort;
danielk1977b3bce662005-01-29 08:32:43 +00005842 }
5843
drh7d10d5a2008-08-20 16:35:10 +00005844 /* For every "*" that occurs in the column list, insert the names of
5845 ** all columns in all tables. And for every TABLE.* insert the names
5846 ** of all columns in TABLE. The parser inserted a special expression
drh1a1d3cd2015-11-19 16:33:31 +00005847 ** with the TK_ASTERISK operator for each "*" that it found in the column
5848 ** list. The following code just has to locate the TK_ASTERISK
5849 ** expressions and expand each one to the list of all columns in
5850 ** all tables.
danielk1977b3bce662005-01-29 08:32:43 +00005851 **
drh7d10d5a2008-08-20 16:35:10 +00005852 ** The first loop just checks to see if there are any "*" operators
5853 ** that need expanding.
danielk1977b3bce662005-01-29 08:32:43 +00005854 */
drh7d10d5a2008-08-20 16:35:10 +00005855 for(k=0; k<pEList->nExpr; k++){
drh3e3f1a52013-01-03 00:45:56 +00005856 pE = pEList->a[k].pExpr;
drh1a1d3cd2015-11-19 16:33:31 +00005857 if( pE->op==TK_ASTERISK ) break;
drh43152cf2009-05-19 19:04:58 +00005858 assert( pE->op!=TK_DOT || pE->pRight!=0 );
5859 assert( pE->op!=TK_DOT || (pE->pLeft!=0 && pE->pLeft->op==TK_ID) );
drh1a1d3cd2015-11-19 16:33:31 +00005860 if( pE->op==TK_DOT && pE->pRight->op==TK_ASTERISK ) break;
drhfca23552017-10-28 20:51:54 +00005861 elistFlags |= pE->flags;
danielk1977b3bce662005-01-29 08:32:43 +00005862 }
drh7d10d5a2008-08-20 16:35:10 +00005863 if( k<pEList->nExpr ){
5864 /*
5865 ** If we get here it means the result set contains one or more "*"
5866 ** operators that need to be expanded. Loop through each expression
5867 ** in the result set and expand them one by one.
5868 */
5869 struct ExprList_item *a = pEList->a;
5870 ExprList *pNew = 0;
5871 int flags = pParse->db->flags;
5872 int longNames = (flags & SQLITE_FullColNames)!=0
drh38b384a2013-01-03 17:34:28 +00005873 && (flags & SQLITE_ShortColNames)==0;
5874
drh7d10d5a2008-08-20 16:35:10 +00005875 for(k=0; k<pEList->nExpr; k++){
drh3e3f1a52013-01-03 00:45:56 +00005876 pE = a[k].pExpr;
drhfca23552017-10-28 20:51:54 +00005877 elistFlags |= pE->flags;
drh3e3f1a52013-01-03 00:45:56 +00005878 pRight = pE->pRight;
5879 assert( pE->op!=TK_DOT || pRight!=0 );
drh1a1d3cd2015-11-19 16:33:31 +00005880 if( pE->op!=TK_ASTERISK
5881 && (pE->op!=TK_DOT || pRight->op!=TK_ASTERISK)
5882 ){
drh7d10d5a2008-08-20 16:35:10 +00005883 /* This particular expression does not need to be expanded.
5884 */
drhb7916a72009-05-27 10:31:29 +00005885 pNew = sqlite3ExprListAppend(pParse, pNew, a[k].pExpr);
drh7d10d5a2008-08-20 16:35:10 +00005886 if( pNew ){
drh41cee662019-12-12 20:22:34 +00005887 pNew->a[pNew->nExpr-1].zEName = a[k].zEName;
drhd88fd532022-05-02 20:49:30 +00005888 pNew->a[pNew->nExpr-1].fg.eEName = a[k].fg.eEName;
drh41cee662019-12-12 20:22:34 +00005889 a[k].zEName = 0;
drh7d10d5a2008-08-20 16:35:10 +00005890 }
5891 a[k].pExpr = 0;
drh7d10d5a2008-08-20 16:35:10 +00005892 }else{
5893 /* This expression is a "*" or a "TABLE.*" and needs to be
5894 ** expanded. */
5895 int tableSeen = 0; /* Set to 1 when TABLE matches */
drh3e3f1a52013-01-03 00:45:56 +00005896 char *zTName = 0; /* text of name of TABLE */
drh43152cf2009-05-19 19:04:58 +00005897 if( pE->op==TK_DOT ){
5898 assert( pE->pLeft!=0 );
drh33e619f2009-05-28 01:00:55 +00005899 assert( !ExprHasProperty(pE->pLeft, EP_IntValue) );
5900 zTName = pE->pLeft->u.zToken;
drh7d10d5a2008-08-20 16:35:10 +00005901 }
5902 for(i=0, pFrom=pTabList->a; i<pTabList->nSrc; i++, pFrom++){
drh9c2d7aa2022-04-22 12:46:46 +00005903 Table *pTab = pFrom->pTab; /* Table for this data source */
5904 ExprList *pNestedFrom; /* Result-set of a nested FROM clause */
5905 char *zTabName; /* AS name for this data source */
5906 const char *zSchemaName = 0; /* Schema name for this data source */
5907 int iDb; /* Schema index for this data src */
drh63879a22022-05-02 20:04:34 +00005908 IdList *pUsing; /* USING clause for pFrom[1] */
drh9c2d7aa2022-04-22 12:46:46 +00005909
5910 if( (zTabName = pFrom->zAlias)==0 ){
drh7d10d5a2008-08-20 16:35:10 +00005911 zTabName = pTab->zName;
5912 }
5913 if( db->mallocFailed ) break;
mistachkin07fae322022-07-06 23:50:01 +00005914 assert( (int)pFrom->fg.isNestedFrom == IsNestedFrom(pFrom->pSelect) );
drh815b7822022-04-20 15:07:39 +00005915 if( pFrom->fg.isNestedFrom ){
drh9c2d7aa2022-04-22 12:46:46 +00005916 assert( pFrom->pSelect!=0 );
5917 pNestedFrom = pFrom->pSelect->pEList;
5918 assert( pNestedFrom!=0 );
5919 assert( pNestedFrom->nExpr==pTab->nCol );
drh815b7822022-04-20 15:07:39 +00005920 }else{
drh3e3f1a52013-01-03 00:45:56 +00005921 if( zTName && sqlite3StrICmp(zTName, zTabName)!=0 ){
5922 continue;
5923 }
drh9c2d7aa2022-04-22 12:46:46 +00005924 pNestedFrom = 0;
drh3e3f1a52013-01-03 00:45:56 +00005925 iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
drh69c33822016-08-18 14:33:11 +00005926 zSchemaName = iDb>=0 ? db->aDb[iDb].zDbSName : "*";
drh7d10d5a2008-08-20 16:35:10 +00005927 }
drh45e41b72022-04-22 23:18:21 +00005928 if( i+1<pTabList->nSrc
5929 && pFrom[1].fg.isUsing
5930 && (selFlags & SF_NestedFrom)!=0
5931 ){
5932 int ii;
drh72d620b2022-05-02 19:59:03 +00005933 pUsing = pFrom[1].u3.pUsing;
drh45e41b72022-04-22 23:18:21 +00005934 for(ii=0; ii<pUsing->nId; ii++){
5935 const char *zUName = pUsing->a[ii].zName;
5936 pRight = sqlite3Expr(db, TK_ID, zUName);
5937 pNew = sqlite3ExprListAppend(pParse, pNew, pRight);
5938 if( pNew ){
5939 struct ExprList_item *pX = &pNew->a[pNew->nExpr-1];
5940 assert( pX->zEName==0 );
5941 pX->zEName = sqlite3MPrintf(db,"..%s", zUName);
drhd88fd532022-05-02 20:49:30 +00005942 pX->fg.eEName = ENAME_TAB;
5943 pX->fg.bUsingTerm = 1;
drh45e41b72022-04-22 23:18:21 +00005944 }
5945 }
drh72d620b2022-05-02 19:59:03 +00005946 }else{
5947 pUsing = 0;
drh45e41b72022-04-22 23:18:21 +00005948 }
drh7d10d5a2008-08-20 16:35:10 +00005949 for(j=0; j<pTab->nCol; j++){
drhcf9d36d2021-08-02 18:03:43 +00005950 char *zName = pTab->aCol[j].zCnName;
drh18f86002022-04-21 13:11:26 +00005951 struct ExprList_item *pX; /* Newly added ExprList term */
drh7d10d5a2008-08-20 16:35:10 +00005952
drhc75e09c2013-01-03 16:54:20 +00005953 assert( zName );
drh9c2d7aa2022-04-22 12:46:46 +00005954 if( zTName
5955 && pNestedFrom
5956 && sqlite3MatchEName(&pNestedFrom->a[j], 0, zTName, 0)==0
drh3e3f1a52013-01-03 00:45:56 +00005957 ){
5958 continue;
5959 }
5960
drh80090f92015-11-19 17:55:11 +00005961 /* If a column is marked as 'hidden', omit it from the expanded
5962 ** result-set list unless the SELECT has the SF_IncludeHidden
5963 ** bit set.
drh7d10d5a2008-08-20 16:35:10 +00005964 */
drhadef1592022-05-10 00:24:01 +00005965 if( (p->selFlags & SF_IncludeHidden)==0
5966 && IsHiddenColumn(&pTab->aCol[j])
5967 ){
5968 continue;
5969 }
5970 if( (pTab->aCol[j].colFlags & COLFLAG_NOEXPAND)!=0
5971 && zTName==0
5972 && (selFlags & (SF_NestedFrom))==0
5973 ){
5974 continue;
drh7d10d5a2008-08-20 16:35:10 +00005975 }
drh3e3f1a52013-01-03 00:45:56 +00005976 tableSeen = 1;
drh7d10d5a2008-08-20 16:35:10 +00005977
drh45e41b72022-04-22 23:18:21 +00005978 if( i>0 && zTName==0 && (selFlags & SF_NestedFrom)==0 ){
drhd44f8b22022-04-07 01:11:13 +00005979 if( pFrom->fg.isUsing
5980 && sqlite3IdListIndex(pFrom->u3.pUsing, zName)>=0
5981 ){
drh7d10d5a2008-08-20 16:35:10 +00005982 /* In a join with a USING clause, omit columns in the
5983 ** using clause from the table on the right. */
5984 continue;
5985 }
5986 }
drhb7916a72009-05-27 10:31:29 +00005987 pRight = sqlite3Expr(db, TK_ID, zName);
drhbefbb402022-04-19 23:00:32 +00005988 if( (pTabList->nSrc>1
5989 && ( (pFrom->fg.jointype & JT_LTORJ)==0
drhd63c07e2022-04-23 14:30:22 +00005990 || (selFlags & SF_NestedFrom)!=0
drhbefbb402022-04-19 23:00:32 +00005991 || !inAnyUsingClause(zName,pFrom,pTabList->nSrc-i-1)
5992 )
5993 )
5994 || IN_RENAME_OBJECT
drh4ce7bf92022-04-15 18:30:48 +00005995 ){
drhb7916a72009-05-27 10:31:29 +00005996 Expr *pLeft;
5997 pLeft = sqlite3Expr(db, TK_ID, zTabName);
drhabfd35e2016-12-06 22:47:23 +00005998 pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight);
dancbde37d2022-04-19 20:47:18 +00005999 if( IN_RENAME_OBJECT && pE->pLeft ){
6000 sqlite3RenameTokenRemap(pParse, pLeft, pE->pLeft);
6001 }
drh38b384a2013-01-03 17:34:28 +00006002 if( zSchemaName ){
drhc75e09c2013-01-03 16:54:20 +00006003 pLeft = sqlite3Expr(db, TK_ID, zSchemaName);
drhabfd35e2016-12-06 22:47:23 +00006004 pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pExpr);
drhc75e09c2013-01-03 16:54:20 +00006005 }
drh7d10d5a2008-08-20 16:35:10 +00006006 }else{
6007 pExpr = pRight;
drh7d10d5a2008-08-20 16:35:10 +00006008 }
drhb7916a72009-05-27 10:31:29 +00006009 pNew = sqlite3ExprListAppend(pParse, pNew, pExpr);
drh18f86002022-04-21 13:11:26 +00006010 if( pNew==0 ){
6011 break; /* OOM */
6012 }
6013 pX = &pNew->a[pNew->nExpr-1];
6014 assert( pX->zEName==0 );
6015 if( (selFlags & SF_NestedFrom)!=0 && !IN_RENAME_OBJECT ){
drh9c2d7aa2022-04-22 12:46:46 +00006016 if( pNestedFrom ){
6017 pX->zEName = sqlite3DbStrDup(db, pNestedFrom->a[j].zEName);
drhcbb9da32019-12-12 22:11:33 +00006018 testcase( pX->zEName==0 );
drh3e3f1a52013-01-03 00:45:56 +00006019 }else{
drhcbb9da32019-12-12 22:11:33 +00006020 pX->zEName = sqlite3MPrintf(db, "%s.%s.%s",
drh18f86002022-04-21 13:11:26 +00006021 zSchemaName, zTabName, zName);
drhcbb9da32019-12-12 22:11:33 +00006022 testcase( pX->zEName==0 );
drh3e3f1a52013-01-03 00:45:56 +00006023 }
drhd88fd532022-05-02 20:49:30 +00006024 pX->fg.eEName = ENAME_TAB;
drh72d620b2022-05-02 19:59:03 +00006025 if( (pFrom->fg.isUsing
6026 && sqlite3IdListIndex(pFrom->u3.pUsing, zName)>=0)
6027 || (pUsing && sqlite3IdListIndex(pUsing, zName)>=0)
6028 || (pTab->aCol[j].colFlags & COLFLAG_NOEXPAND)!=0
6029 ){
drhd88fd532022-05-02 20:49:30 +00006030 pX->fg.bNoExpand = 1;
drh72d620b2022-05-02 19:59:03 +00006031 }
drh18f86002022-04-21 13:11:26 +00006032 }else if( longNames ){
6033 pX->zEName = sqlite3MPrintf(db, "%s.%s", zTabName, zName);
drhd88fd532022-05-02 20:49:30 +00006034 pX->fg.eEName = ENAME_NAME;
drh18f86002022-04-21 13:11:26 +00006035 }else{
6036 pX->zEName = sqlite3DbStrDup(db, zName);
drhd88fd532022-05-02 20:49:30 +00006037 pX->fg.eEName = ENAME_NAME;
drh8f25d182012-12-19 02:36:45 +00006038 }
drh7d10d5a2008-08-20 16:35:10 +00006039 }
6040 }
6041 if( !tableSeen ){
6042 if( zTName ){
6043 sqlite3ErrorMsg(pParse, "no such table: %s", zTName);
6044 }else{
6045 sqlite3ErrorMsg(pParse, "no tables specified");
6046 }
6047 }
drh7d10d5a2008-08-20 16:35:10 +00006048 }
drh9a993342007-12-13 02:45:31 +00006049 }
drh7d10d5a2008-08-20 16:35:10 +00006050 sqlite3ExprListDelete(db, pEList);
6051 p->pEList = pNew;
drh4c774312007-12-08 21:10:20 +00006052 }
drhfca23552017-10-28 20:51:54 +00006053 if( p->pEList ){
6054 if( p->pEList->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){
6055 sqlite3ErrorMsg(pParse, "too many columns in result set");
6056 return WRC_Abort;
6057 }
6058 if( (elistFlags & (EP_HasFunc|EP_Subquery))!=0 ){
6059 p->selFlags |= SF_ComplexResult;
6060 }
drh994c80a2007-04-12 21:25:01 +00006061 }
drhcebc8002022-04-16 18:33:22 +00006062#if TREETRACE_ENABLED
6063 if( sqlite3TreeTrace & 0x100 ){
6064 SELECTTRACE(0x100,pParse,p,("After result-set wildcard expansion:\n"));
6065 sqlite3TreeViewSelect(0, p, 0);
6066 }
6067#endif
drh7d10d5a2008-08-20 16:35:10 +00006068 return WRC_Continue;
6069}
danielk1977b3bce662005-01-29 08:32:43 +00006070
drh979dd1b2017-05-29 14:26:07 +00006071#if SQLITE_DEBUG
6072/*
6073** Always assert. This xSelectCallback2 implementation proves that the
6074** xSelectCallback2 is never invoked.
6075*/
6076void sqlite3SelectWalkAssert2(Walker *NotUsed, Select *NotUsed2){
6077 UNUSED_PARAMETER2(NotUsed, NotUsed2);
6078 assert( 0 );
6079}
6080#endif
6081/*
drh7d10d5a2008-08-20 16:35:10 +00006082** This routine "expands" a SELECT statement and all of its subqueries.
6083** For additional information on what it means to "expand" a SELECT
6084** statement, see the comment on the selectExpand worker callback above.
6085**
6086** Expanding a SELECT statement is the first step in processing a
6087** SELECT statement. The SELECT statement must be expanded before
6088** name resolution is performed.
6089**
6090** If anything goes wrong, an error message is written into pParse.
6091** The calling function can detect the problem by looking at pParse->nErr
6092** and/or pParse->db->mallocFailed.
6093*/
6094static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){
6095 Walker w;
drh5b88bc42013-12-07 23:35:21 +00006096 w.xExprCallback = sqlite3ExprWalkNoop;
drh7d10d5a2008-08-20 16:35:10 +00006097 w.pParse = pParse;
drh878fcf92017-10-02 23:50:08 +00006098 if( OK_IF_ALWAYS_TRUE(pParse->hasCompound) ){
drhd58d3272013-08-05 22:05:02 +00006099 w.xSelectCallback = convertCompoundSelectToSubquery;
drh979dd1b2017-05-29 14:26:07 +00006100 w.xSelectCallback2 = 0;
drhd58d3272013-08-05 22:05:02 +00006101 sqlite3WalkSelect(&w, pSelect);
6102 }
drhc01b7302013-05-07 17:49:08 +00006103 w.xSelectCallback = selectExpander;
danbe120832021-05-17 16:20:41 +00006104 w.xSelectCallback2 = sqlite3SelectPopWith;
drh59145812019-05-22 22:49:23 +00006105 w.eCode = 0;
drhc01b7302013-05-07 17:49:08 +00006106 sqlite3WalkSelect(&w, pSelect);
drh7d10d5a2008-08-20 16:35:10 +00006107}
6108
6109
6110#ifndef SQLITE_OMIT_SUBQUERY
6111/*
6112** This is a Walker.xSelectCallback callback for the sqlite3SelectTypeInfo()
6113** interface.
6114**
6115** For each FROM-clause subquery, add Column.zType and Column.zColl
6116** information to the Table structure that represents the result set
6117** of that subquery.
6118**
6119** The Table structure that represents the result set was constructed
6120** by selectExpander() but the type and collation information was omitted
6121** at that point because identifiers had not yet been resolved. This
6122** routine is called after identifier resolution.
6123*/
danb290f112014-01-17 14:59:27 +00006124static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){
drh7d10d5a2008-08-20 16:35:10 +00006125 Parse *pParse;
6126 int i;
6127 SrcList *pTabList;
drh76012942021-02-21 21:04:54 +00006128 SrcItem *pFrom;
drh7d10d5a2008-08-20 16:35:10 +00006129
drh9d8b3072008-08-22 16:29:51 +00006130 assert( p->selFlags & SF_Resolved );
dancc464412018-06-19 18:11:05 +00006131 if( p->selFlags & SF_HasTypeInfo ) return;
drhe2b7d7a2015-09-29 15:50:04 +00006132 p->selFlags |= SF_HasTypeInfo;
6133 pParse = pWalker->pParse;
6134 pTabList = p->pSrc;
6135 for(i=0, pFrom=pTabList->a; i<pTabList->nSrc; i++, pFrom++){
6136 Table *pTab = pFrom->pTab;
6137 assert( pTab!=0 );
6138 if( (pTab->tabFlags & TF_Ephemeral)!=0 ){
6139 /* A sub-query in the FROM clause of a SELECT */
6140 Select *pSel = pFrom->pSelect;
6141 if( pSel ){
6142 while( pSel->pPrior ) pSel = pSel->pPrior;
drh96fb16e2019-08-06 14:37:24 +00006143 sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSel,
6144 SQLITE_AFF_NONE);
drh5a29d9c2010-02-24 15:10:14 +00006145 }
drh13449892005-09-07 21:22:45 +00006146 }
6147 }
drh7d10d5a2008-08-20 16:35:10 +00006148}
6149#endif
drh13449892005-09-07 21:22:45 +00006150
drh7d10d5a2008-08-20 16:35:10 +00006151
6152/*
6153** This routine adds datatype and collating sequence information to
6154** the Table structures of all FROM-clause subqueries in a
6155** SELECT statement.
6156**
6157** Use this routine after name resolution.
6158*/
6159static void sqlite3SelectAddTypeInfo(Parse *pParse, Select *pSelect){
6160#ifndef SQLITE_OMIT_SUBQUERY
6161 Walker w;
drh979dd1b2017-05-29 14:26:07 +00006162 w.xSelectCallback = sqlite3SelectWalkNoop;
danb290f112014-01-17 14:59:27 +00006163 w.xSelectCallback2 = selectAddSubqueryTypeInfo;
drh5b88bc42013-12-07 23:35:21 +00006164 w.xExprCallback = sqlite3ExprWalkNoop;
drh7d10d5a2008-08-20 16:35:10 +00006165 w.pParse = pParse;
6166 sqlite3WalkSelect(&w, pSelect);
6167#endif
6168}
6169
6170
6171/*
drh030796d2012-08-23 16:18:10 +00006172** This routine sets up a SELECT statement for processing. The
drh7d10d5a2008-08-20 16:35:10 +00006173** following is accomplished:
6174**
6175** * VDBE Cursor numbers are assigned to all FROM-clause terms.
6176** * Ephemeral Table objects are created for all FROM-clause subqueries.
6177** * ON and USING clauses are shifted into WHERE statements
6178** * Wildcards "*" and "TABLE.*" in result sets are expanded.
6179** * Identifiers in expression are matched to tables.
6180**
6181** This routine acts recursively on all subqueries within the SELECT.
6182*/
6183void sqlite3SelectPrep(
6184 Parse *pParse, /* The parser context */
6185 Select *p, /* The SELECT statement being coded. */
6186 NameContext *pOuterNC /* Name context for container */
6187){
drhe2463392017-10-03 14:24:24 +00006188 assert( p!=0 || pParse->db->mallocFailed );
drh0c7d3d32022-01-24 16:47:12 +00006189 assert( pParse->db->pParse==pParse );
drhe2463392017-10-03 14:24:24 +00006190 if( pParse->db->mallocFailed ) return;
drh7d10d5a2008-08-20 16:35:10 +00006191 if( p->selFlags & SF_HasTypeInfo ) return;
drh7d10d5a2008-08-20 16:35:10 +00006192 sqlite3SelectExpand(pParse, p);
drh0c7d3d32022-01-24 16:47:12 +00006193 if( pParse->nErr ) return;
drh7d10d5a2008-08-20 16:35:10 +00006194 sqlite3ResolveSelectNames(pParse, p, pOuterNC);
drh0c7d3d32022-01-24 16:47:12 +00006195 if( pParse->nErr ) return;
drh7d10d5a2008-08-20 16:35:10 +00006196 sqlite3SelectAddTypeInfo(pParse, p);
danielk1977b3bce662005-01-29 08:32:43 +00006197}
6198
6199/*
drh13449892005-09-07 21:22:45 +00006200** Reset the aggregate accumulator.
6201**
6202** The aggregate accumulator is a set of memory cells that hold
6203** intermediate results while calculating an aggregate. This
drh030796d2012-08-23 16:18:10 +00006204** routine generates code that stores NULLs in all of those memory
6205** cells.
danielk1977b3bce662005-01-29 08:32:43 +00006206*/
drh13449892005-09-07 21:22:45 +00006207static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){
6208 Vdbe *v = pParse->pVdbe;
6209 int i;
drhc99130f2005-09-11 11:56:27 +00006210 struct AggInfo_func *pFunc;
drh7e61d182013-12-20 13:11:45 +00006211 int nReg = pAggInfo->nFunc + pAggInfo->nColumn;
drh0c7d3d32022-01-24 16:47:12 +00006212 assert( pParse->db->pParse==pParse );
6213 assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 );
drh7e61d182013-12-20 13:11:45 +00006214 if( nReg==0 ) return;
drh0c7d3d32022-01-24 16:47:12 +00006215 if( pParse->nErr ) return;
drh7e61d182013-12-20 13:11:45 +00006216#ifdef SQLITE_DEBUG
6217 /* Verify that all AggInfo registers are within the range specified by
6218 ** AggInfo.mnReg..AggInfo.mxReg */
6219 assert( nReg==pAggInfo->mxReg-pAggInfo->mnReg+1 );
drh13449892005-09-07 21:22:45 +00006220 for(i=0; i<pAggInfo->nColumn; i++){
drh7e61d182013-12-20 13:11:45 +00006221 assert( pAggInfo->aCol[i].iMem>=pAggInfo->mnReg
6222 && pAggInfo->aCol[i].iMem<=pAggInfo->mxReg );
drh13449892005-09-07 21:22:45 +00006223 }
drh7e61d182013-12-20 13:11:45 +00006224 for(i=0; i<pAggInfo->nFunc; i++){
6225 assert( pAggInfo->aFunc[i].iMem>=pAggInfo->mnReg
6226 && pAggInfo->aFunc[i].iMem<=pAggInfo->mxReg );
6227 }
6228#endif
6229 sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->mnReg, pAggInfo->mxReg);
drhc99130f2005-09-11 11:56:27 +00006230 for(pFunc=pAggInfo->aFunc, i=0; i<pAggInfo->nFunc; i++, pFunc++){
drhc99130f2005-09-11 11:56:27 +00006231 if( pFunc->iDistinct>=0 ){
drh81185a52020-06-09 13:38:12 +00006232 Expr *pE = pFunc->pFExpr;
drha4eeccd2021-10-07 17:43:30 +00006233 assert( ExprUseXList(pE) );
danielk19776ab3a2e2009-02-19 14:39:25 +00006234 if( pE->x.pList==0 || pE->x.pList->nExpr!=1 ){
drh0daa0022009-02-09 13:19:28 +00006235 sqlite3ErrorMsg(pParse, "DISTINCT aggregates must have exactly one "
6236 "argument");
drhc99130f2005-09-11 11:56:27 +00006237 pFunc->iDistinct = -1;
6238 }else{
danf9eae182018-05-21 19:45:11 +00006239 KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pE->x.pList,0,0);
dan4fcb30b2021-03-09 16:06:25 +00006240 pFunc->iDistAddr = sqlite3VdbeAddOp4(v, OP_OpenEphemeral,
6241 pFunc->iDistinct, 0, 0, (char*)pKeyInfo, P4_KEYINFO);
drh9be13392021-03-24 19:44:01 +00006242 ExplainQueryPlan((pParse, 0, "USE TEMP B-TREE FOR %s(DISTINCT)",
6243 pFunc->pFunc->zName));
drhc99130f2005-09-11 11:56:27 +00006244 }
6245 }
drh13449892005-09-07 21:22:45 +00006246 }
danielk1977b3bce662005-01-29 08:32:43 +00006247}
6248
6249/*
drh13449892005-09-07 21:22:45 +00006250** Invoke the OP_AggFinalize opcode for every aggregate function
6251** in the AggInfo structure.
danielk1977b3bce662005-01-29 08:32:43 +00006252*/
drh13449892005-09-07 21:22:45 +00006253static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){
6254 Vdbe *v = pParse->pVdbe;
6255 int i;
6256 struct AggInfo_func *pF;
6257 for(i=0, pF=pAggInfo->aFunc; i<pAggInfo->nFunc; i++, pF++){
drha4eeccd2021-10-07 17:43:30 +00006258 ExprList *pList;
6259 assert( ExprUseXList(pF->pFExpr) );
6260 pList = pF->pFExpr->x.pList;
drh2700aca2016-12-08 01:38:24 +00006261 sqlite3VdbeAddOp2(v, OP_AggFinal, pF->iMem, pList ? pList->nExpr : 0);
6262 sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF);
drh13449892005-09-07 21:22:45 +00006263 }
danielk1977b3bce662005-01-29 08:32:43 +00006264}
drh13449892005-09-07 21:22:45 +00006265
dan280c8942018-06-05 20:04:28 +00006266
drh13449892005-09-07 21:22:45 +00006267/*
6268** Update the accumulator memory cells for an aggregate based on
6269** the current cursor position.
dan280c8942018-06-05 20:04:28 +00006270**
6271** If regAcc is non-zero and there are no min() or max() aggregates
6272** in pAggInfo, then only populate the pAggInfo->nAccumulator accumulator
drh10cc16c2019-03-01 21:12:40 +00006273** registers if register regAcc contains 0. The caller will take care
dan280c8942018-06-05 20:04:28 +00006274** of setting and clearing regAcc.
drh13449892005-09-07 21:22:45 +00006275*/
dan4fcb30b2021-03-09 16:06:25 +00006276static void updateAccumulator(
6277 Parse *pParse,
6278 int regAcc,
6279 AggInfo *pAggInfo,
6280 int eDistinctType
6281){
drh13449892005-09-07 21:22:45 +00006282 Vdbe *v = pParse->pVdbe;
6283 int i;
drh7a957892012-02-02 17:35:43 +00006284 int regHit = 0;
6285 int addrHitTest = 0;
drh13449892005-09-07 21:22:45 +00006286 struct AggInfo_func *pF;
6287 struct AggInfo_col *pC;
drh13449892005-09-07 21:22:45 +00006288
6289 pAggInfo->directMode = 1;
6290 for(i=0, pF=pAggInfo->aFunc; i<pAggInfo->nFunc; i++, pF++){
6291 int nArg;
drhc99130f2005-09-11 11:56:27 +00006292 int addrNext = 0;
drh98757152008-01-09 23:04:12 +00006293 int regAgg;
drha4eeccd2021-10-07 17:43:30 +00006294 ExprList *pList;
6295 assert( ExprUseXList(pF->pFExpr) );
drh81185a52020-06-09 13:38:12 +00006296 assert( !IsWindowFunc(pF->pFExpr) );
drha4eeccd2021-10-07 17:43:30 +00006297 pList = pF->pFExpr->x.pList;
drh81185a52020-06-09 13:38:12 +00006298 if( ExprHasProperty(pF->pFExpr, EP_WinFunc) ){
6299 Expr *pFilter = pF->pFExpr->y.pWin->pFilter;
daned09ddd2019-09-20 20:52:16 +00006300 if( pAggInfo->nAccumulator
6301 && (pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL)
danfa4b0d42020-07-01 14:07:45 +00006302 && regAcc
daned09ddd2019-09-20 20:52:16 +00006303 ){
danfa4b0d42020-07-01 14:07:45 +00006304 /* If regAcc==0, there there exists some min() or max() function
6305 ** without a FILTER clause that will ensure the magnet registers
6306 ** are populated. */
daned09ddd2019-09-20 20:52:16 +00006307 if( regHit==0 ) regHit = ++pParse->nMem;
danfa4b0d42020-07-01 14:07:45 +00006308 /* If this is the first row of the group (regAcc contains 0), clear the
daned09ddd2019-09-20 20:52:16 +00006309 ** "magnet" register regHit so that the accumulator registers
danaf9b58b2019-09-20 21:12:40 +00006310 ** are populated if the FILTER clause jumps over the the
6311 ** invocation of min() or max() altogether. Or, if this is not
danfa4b0d42020-07-01 14:07:45 +00006312 ** the first row (regAcc contains 1), set the magnet register so that
6313 ** the accumulators are not populated unless the min()/max() is invoked
6314 ** and indicates that they should be. */
daned09ddd2019-09-20 20:52:16 +00006315 sqlite3VdbeAddOp2(v, OP_Copy, regAcc, regHit);
6316 }
dan6ba7ab02019-07-02 11:56:47 +00006317 addrNext = sqlite3VdbeMakeLabel(pParse);
6318 sqlite3ExprIfFalse(pParse, pFilter, addrNext, SQLITE_JUMPIFNULL);
6319 }
drh13449892005-09-07 21:22:45 +00006320 if( pList ){
6321 nArg = pList->nExpr;
drh892d3172008-01-10 03:46:36 +00006322 regAgg = sqlite3GetTempRange(pParse, nArg);
drh5579d592015-08-26 14:01:41 +00006323 sqlite3ExprCodeExprList(pParse, pList, regAgg, 0, SQLITE_ECEL_DUP);
drh13449892005-09-07 21:22:45 +00006324 }else{
6325 nArg = 0;
drh98757152008-01-09 23:04:12 +00006326 regAgg = 0;
drh13449892005-09-07 21:22:45 +00006327 }
dan4fcb30b2021-03-09 16:06:25 +00006328 if( pF->iDistinct>=0 && pList ){
dan6ba7ab02019-07-02 11:56:47 +00006329 if( addrNext==0 ){
6330 addrNext = sqlite3VdbeMakeLabel(pParse);
6331 }
dan5383db72021-03-12 18:24:31 +00006332 pF->iDistinct = codeDistinct(pParse, eDistinctType,
6333 pF->iDistinct, addrNext, pList, regAgg);
drhc99130f2005-09-11 11:56:27 +00006334 }
drhd36e1042013-09-06 13:10:12 +00006335 if( pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){
drh13449892005-09-07 21:22:45 +00006336 CollSeq *pColl = 0;
6337 struct ExprList_item *pItem;
6338 int j;
drhe82f5d02008-10-07 19:53:14 +00006339 assert( pList!=0 ); /* pList!=0 if pF->pFunc has NEEDCOLL */
drh43617e92006-03-06 20:55:46 +00006340 for(j=0, pItem=pList->a; !pColl && j<nArg; j++, pItem++){
drh13449892005-09-07 21:22:45 +00006341 pColl = sqlite3ExprCollSeq(pParse, pItem->pExpr);
6342 }
6343 if( !pColl ){
6344 pColl = pParse->db->pDfltColl;
6345 }
drh7a957892012-02-02 17:35:43 +00006346 if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem;
6347 sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, (char *)pColl, P4_COLLSEQ);
drh13449892005-09-07 21:22:45 +00006348 }
drh8f26da62018-07-05 21:22:57 +00006349 sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, pF->iMem);
drh2700aca2016-12-08 01:38:24 +00006350 sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF);
drhea678832008-12-10 19:26:22 +00006351 sqlite3VdbeChangeP5(v, (u8)nArg);
drhf49f3522009-12-30 14:12:38 +00006352 sqlite3ReleaseTempRange(pParse, regAgg, nArg);
drhc99130f2005-09-11 11:56:27 +00006353 if( addrNext ){
6354 sqlite3VdbeResolveLabel(v, addrNext);
6355 }
drh13449892005-09-07 21:22:45 +00006356 }
dan280c8942018-06-05 20:04:28 +00006357 if( regHit==0 && pAggInfo->nAccumulator ){
6358 regHit = regAcc;
6359 }
drh7a957892012-02-02 17:35:43 +00006360 if( regHit ){
drh688852a2014-02-17 22:40:43 +00006361 addrHitTest = sqlite3VdbeAddOp1(v, OP_If, regHit); VdbeCoverage(v);
drh7a957892012-02-02 17:35:43 +00006362 }
drh13449892005-09-07 21:22:45 +00006363 for(i=0, pC=pAggInfo->aCol; i<pAggInfo->nAccumulator; i++, pC++){
drh81185a52020-06-09 13:38:12 +00006364 sqlite3ExprCode(pParse, pC->pCExpr, pC->iMem);
drh13449892005-09-07 21:22:45 +00006365 }
daned09ddd2019-09-20 20:52:16 +00006366
drh13449892005-09-07 21:22:45 +00006367 pAggInfo->directMode = 0;
drh7a957892012-02-02 17:35:43 +00006368 if( addrHitTest ){
drhdc4f6fc2020-02-07 19:44:13 +00006369 sqlite3VdbeJumpHereOrPopInst(v, addrHitTest);
drh7a957892012-02-02 17:35:43 +00006370 }
drh13449892005-09-07 21:22:45 +00006371}
6372
danielk1977b3bce662005-01-29 08:32:43 +00006373/*
danef7075d2011-02-21 17:49:49 +00006374** Add a single OP_Explain instruction to the VDBE to explain a simple
6375** count(*) query ("SELECT count(*) FROM pTab").
6376*/
6377#ifndef SQLITE_OMIT_EXPLAIN
6378static void explainSimpleCount(
6379 Parse *pParse, /* Parse context */
6380 Table *pTab, /* Table being queried */
6381 Index *pIdx /* Index used to optimize scan, or NULL */
6382){
6383 if( pParse->explain==2 ){
drh48dd1d82014-05-27 18:18:58 +00006384 int bCover = (pIdx!=0 && (HasRowid(pTab) || !IsPrimaryKeyIndex(pIdx)));
drh82102332021-03-20 15:11:29 +00006385 sqlite3VdbeExplain(pParse, 0, "SCAN %s%s%s",
dane96f2df2014-05-23 17:17:06 +00006386 pTab->zName,
6387 bCover ? " USING COVERING INDEX " : "",
6388 bCover ? pIdx->zName : ""
danef7075d2011-02-21 17:49:49 +00006389 );
danef7075d2011-02-21 17:49:49 +00006390 }
6391}
6392#else
6393# define explainSimpleCount(a,b,c)
6394#endif
6395
6396/*
danab31a842017-04-29 20:53:09 +00006397** sqlite3WalkExpr() callback used by havingToWhere().
6398**
6399** If the node passed to the callback is a TK_AND node, return
6400** WRC_Continue to tell sqlite3WalkExpr() to iterate through child nodes.
6401**
6402** Otherwise, return WRC_Prune. In this case, also check if the
6403** sub-expression matches the criteria for being moved to the WHERE
6404** clause. If so, add it to the WHERE clause and replace the sub-expression
6405** within the HAVING expression with a constant "1".
6406*/
6407static int havingToWhereExprCb(Walker *pWalker, Expr *pExpr){
6408 if( pExpr->op!=TK_AND ){
drhcd0abc22018-03-20 18:08:33 +00006409 Select *pS = pWalker->u.pSelect;
dand59f9832021-07-20 14:57:49 +00006410 /* This routine is called before the HAVING clause of the current
6411 ** SELECT is analyzed for aggregates. So if pExpr->pAggInfo is set
6412 ** here, it indicates that the expression is a correlated reference to a
6413 ** column from an outer aggregate query, or an aggregate function that
6414 ** belongs to an outer query. Do not move the expression to the WHERE
6415 ** clause in this obscure case, as doing so may corrupt the outer Select
6416 ** statements AggInfo structure. */
danf39168e2020-12-22 16:23:29 +00006417 if( sqlite3ExprIsConstantOrGroupBy(pWalker->pParse, pExpr, pS->pGroupBy)
6418 && ExprAlwaysFalse(pExpr)==0
dand59f9832021-07-20 14:57:49 +00006419 && pExpr->pAggInfo==0
danf39168e2020-12-22 16:23:29 +00006420 ){
danab31a842017-04-29 20:53:09 +00006421 sqlite3 *db = pWalker->pParse->db;
drh5776ee52019-09-23 12:38:10 +00006422 Expr *pNew = sqlite3Expr(db, TK_INTEGER, "1");
danab31a842017-04-29 20:53:09 +00006423 if( pNew ){
drhcd0abc22018-03-20 18:08:33 +00006424 Expr *pWhere = pS->pWhere;
danab31a842017-04-29 20:53:09 +00006425 SWAP(Expr, *pNew, *pExpr);
drhd5c851c2019-04-19 13:38:34 +00006426 pNew = sqlite3ExprAnd(pWalker->pParse, pWhere, pNew);
drhcd0abc22018-03-20 18:08:33 +00006427 pS->pWhere = pNew;
6428 pWalker->eCode = 1;
danab31a842017-04-29 20:53:09 +00006429 }
6430 }
6431 return WRC_Prune;
6432 }
6433 return WRC_Continue;
6434}
6435
6436/*
6437** Transfer eligible terms from the HAVING clause of a query, which is
6438** processed after grouping, to the WHERE clause, which is processed before
6439** grouping. For example, the query:
6440**
6441** SELECT * FROM <tables> WHERE a=? GROUP BY b HAVING b=? AND c=?
6442**
6443** can be rewritten as:
6444**
6445** SELECT * FROM <tables> WHERE a=? AND b=? GROUP BY b HAVING c=?
6446**
6447** A term of the HAVING expression is eligible for transfer if it consists
6448** entirely of constants and expressions that are also GROUP BY terms that
6449** use the "BINARY" collation sequence.
6450*/
drhcd0abc22018-03-20 18:08:33 +00006451static void havingToWhere(Parse *pParse, Select *p){
danab31a842017-04-29 20:53:09 +00006452 Walker sWalker;
danab31a842017-04-29 20:53:09 +00006453 memset(&sWalker, 0, sizeof(sWalker));
6454 sWalker.pParse = pParse;
6455 sWalker.xExprCallback = havingToWhereExprCb;
drhcd0abc22018-03-20 18:08:33 +00006456 sWalker.u.pSelect = p;
6457 sqlite3WalkExpr(&sWalker, p->pHaving);
drh5e431be2022-04-06 11:08:38 +00006458#if TREETRACE_ENABLED
6459 if( sWalker.eCode && (sqlite3TreeTrace & 0x100)!=0 ){
drhcd0abc22018-03-20 18:08:33 +00006460 SELECTTRACE(0x100,pParse,p,("Move HAVING terms into WHERE:\n"));
6461 sqlite3TreeViewSelect(0, p, 0);
6462 }
6463#endif
danab31a842017-04-29 20:53:09 +00006464}
6465
6466/*
drhe08e8d62017-05-01 15:15:41 +00006467** Check to see if the pThis entry of pTabList is a self-join of a prior view.
6468** If it is, then return the SrcList_item for the prior view. If it is not,
6469** then return 0.
6470*/
drh76012942021-02-21 21:04:54 +00006471static SrcItem *isSelfJoinView(
drhe08e8d62017-05-01 15:15:41 +00006472 SrcList *pTabList, /* Search for self-joins in this FROM clause */
drh76012942021-02-21 21:04:54 +00006473 SrcItem *pThis /* Search for prior reference to this subquery */
drhe08e8d62017-05-01 15:15:41 +00006474){
drh76012942021-02-21 21:04:54 +00006475 SrcItem *pItem;
drh8794c682021-02-13 16:39:24 +00006476 assert( pThis->pSelect!=0 );
6477 if( pThis->pSelect->selFlags & SF_PushDown ) return 0;
drhe08e8d62017-05-01 15:15:41 +00006478 for(pItem = pTabList->a; pItem<pThis; pItem++){
drhbdefaf02018-12-27 02:16:01 +00006479 Select *pS1;
drhe08e8d62017-05-01 15:15:41 +00006480 if( pItem->pSelect==0 ) continue;
6481 if( pItem->fg.viaCoroutine ) continue;
drh33543c22017-05-01 16:37:20 +00006482 if( pItem->zName==0 ) continue;
drh30ad79a2019-05-23 16:38:12 +00006483 assert( pItem->pTab!=0 );
6484 assert( pThis->pTab!=0 );
6485 if( pItem->pTab->pSchema!=pThis->pTab->pSchema ) continue;
drhed712982017-05-01 17:04:35 +00006486 if( sqlite3_stricmp(pItem->zName, pThis->zName)!=0 ) continue;
drhbdefaf02018-12-27 02:16:01 +00006487 pS1 = pItem->pSelect;
drh30ad79a2019-05-23 16:38:12 +00006488 if( pItem->pTab->pSchema==0 && pThis->pSelect->selId!=pS1->selId ){
drhbdefaf02018-12-27 02:16:01 +00006489 /* The query flattener left two different CTE tables with identical
6490 ** names in the same FROM clause. */
6491 continue;
6492 }
drh8794c682021-02-13 16:39:24 +00006493 if( pItem->pSelect->selFlags & SF_PushDown ){
drhed712982017-05-01 17:04:35 +00006494 /* The view was modified by some other optimization such as
6495 ** pushDownWhereTerms() */
6496 continue;
6497 }
6498 return pItem;
drhe08e8d62017-05-01 15:15:41 +00006499 }
6500 return 0;
6501}
6502
drhc54246f2021-02-17 21:13:14 +00006503/*
6504** Deallocate a single AggInfo object
6505*/
6506static void agginfoFree(sqlite3 *db, AggInfo *p){
6507 sqlite3DbFree(db, p->aCol);
6508 sqlite3DbFree(db, p->aFunc);
6509 sqlite3DbFreeNN(db, p);
6510}
6511
drh269ba802017-07-04 19:34:36 +00006512#ifdef SQLITE_COUNTOFVIEW_OPTIMIZATION
6513/*
6514** Attempt to transform a query of the form
6515**
6516** SELECT count(*) FROM (SELECT x FROM t1 UNION ALL SELECT y FROM t2)
6517**
6518** Into this:
6519**
6520** SELECT (SELECT count(*) FROM t1)+(SELECT count(*) FROM t2)
6521**
6522** The transformation only works if all of the following are true:
6523**
6524** * The subquery is a UNION ALL of two or more terms
dana4b5fb52018-08-03 20:19:52 +00006525** * The subquery does not have a LIMIT clause
drh269ba802017-07-04 19:34:36 +00006526** * There is no WHERE or GROUP BY or HAVING clauses on the subqueries
drh73c53b32019-05-15 18:42:15 +00006527** * The outer query is a simple count(*) with no WHERE clause or other
6528** extraneous syntax.
drh269ba802017-07-04 19:34:36 +00006529**
6530** Return TRUE if the optimization is undertaken.
6531*/
6532static int countOfViewOptimization(Parse *pParse, Select *p){
6533 Select *pSub, *pPrior;
6534 Expr *pExpr;
6535 Expr *pCount;
6536 sqlite3 *db;
drh3d240d22017-09-28 16:56:55 +00006537 if( (p->selFlags & SF_Aggregate)==0 ) return 0; /* This is an aggregate */
drh269ba802017-07-04 19:34:36 +00006538 if( p->pEList->nExpr!=1 ) return 0; /* Single result column */
drh73c53b32019-05-15 18:42:15 +00006539 if( p->pWhere ) return 0;
6540 if( p->pGroupBy ) return 0;
drh269ba802017-07-04 19:34:36 +00006541 pExpr = p->pEList->a[0].pExpr;
6542 if( pExpr->op!=TK_AGG_FUNCTION ) return 0; /* Result is an aggregate */
drha4eeccd2021-10-07 17:43:30 +00006543 assert( ExprUseUToken(pExpr) );
drh3d240d22017-09-28 16:56:55 +00006544 if( sqlite3_stricmp(pExpr->u.zToken,"count") ) return 0; /* Is count() */
drha4eeccd2021-10-07 17:43:30 +00006545 assert( ExprUseXList(pExpr) );
drh269ba802017-07-04 19:34:36 +00006546 if( pExpr->x.pList!=0 ) return 0; /* Must be count(*) */
drh3d240d22017-09-28 16:56:55 +00006547 if( p->pSrc->nSrc!=1 ) return 0; /* One table in FROM */
drh269ba802017-07-04 19:34:36 +00006548 pSub = p->pSrc->a[0].pSelect;
6549 if( pSub==0 ) return 0; /* The FROM is a subquery */
drh3d240d22017-09-28 16:56:55 +00006550 if( pSub->pPrior==0 ) return 0; /* Must be a compound ry */
drh269ba802017-07-04 19:34:36 +00006551 do{
6552 if( pSub->op!=TK_ALL && pSub->pPrior ) return 0; /* Must be UNION ALL */
6553 if( pSub->pWhere ) return 0; /* No WHERE clause */
dana4b5fb52018-08-03 20:19:52 +00006554 if( pSub->pLimit ) return 0; /* No LIMIT clause */
drh269ba802017-07-04 19:34:36 +00006555 if( pSub->selFlags & SF_Aggregate ) return 0; /* Not an aggregate */
drh3d240d22017-09-28 16:56:55 +00006556 pSub = pSub->pPrior; /* Repeat over compound */
drh269ba802017-07-04 19:34:36 +00006557 }while( pSub );
6558
drh3d240d22017-09-28 16:56:55 +00006559 /* If we reach this point then it is OK to perform the transformation */
drh269ba802017-07-04 19:34:36 +00006560
6561 db = pParse->db;
6562 pCount = pExpr;
6563 pExpr = 0;
6564 pSub = p->pSrc->a[0].pSelect;
6565 p->pSrc->a[0].pSelect = 0;
6566 sqlite3SrcListDelete(db, p->pSrc);
6567 p->pSrc = sqlite3DbMallocZero(pParse->db, sizeof(*p->pSrc));
6568 while( pSub ){
6569 Expr *pTerm;
6570 pPrior = pSub->pPrior;
6571 pSub->pPrior = 0;
6572 pSub->pNext = 0;
6573 pSub->selFlags |= SF_Aggregate;
6574 pSub->selFlags &= ~SF_Compound;
6575 pSub->nSelectRow = 0;
6576 sqlite3ExprListDelete(db, pSub->pEList);
6577 pTerm = pPrior ? sqlite3ExprDup(db, pCount, 0) : pCount;
6578 pSub->pEList = sqlite3ExprListAppend(pParse, 0, pTerm);
6579 pTerm = sqlite3PExpr(pParse, TK_SELECT, 0, 0);
6580 sqlite3PExprAddSelect(pParse, pTerm, pSub);
6581 if( pExpr==0 ){
6582 pExpr = pTerm;
6583 }else{
6584 pExpr = sqlite3PExpr(pParse, TK_PLUS, pTerm, pExpr);
6585 }
6586 pSub = pPrior;
6587 }
6588 p->pEList->a[0].pExpr = pExpr;
6589 p->selFlags &= ~SF_Aggregate;
6590
drh5e431be2022-04-06 11:08:38 +00006591#if TREETRACE_ENABLED
6592 if( sqlite3TreeTrace & 0x400 ){
drh269ba802017-07-04 19:34:36 +00006593 SELECTTRACE(0x400,pParse,p,("After count-of-view optimization:\n"));
6594 sqlite3TreeViewSelect(0, p, 0);
6595 }
6596#endif
6597 return 1;
6598}
6599#endif /* SQLITE_COUNTOFVIEW_OPTIMIZATION */
6600
drhe08e8d62017-05-01 15:15:41 +00006601/*
drhfb98dac2022-05-25 02:32:11 +00006602** If any term of pSrc, or any SF_NestedFrom sub-query, is not the same
6603** as pSrcItem but has the same alias as p0, then return true.
6604** Otherwise return false.
6605*/
6606static int sameSrcAlias(SrcItem *p0, SrcList *pSrc){
6607 int i;
6608 for(i=0; i<pSrc->nSrc; i++){
6609 SrcItem *p1 = &pSrc->a[i];
6610 if( p1==p0 ) continue;
6611 if( p0->pTab==p1->pTab && 0==sqlite3_stricmp(p0->zAlias, p1->zAlias) ){
6612 return 1;
6613 }
6614 if( p1->pSelect
6615 && (p1->pSelect->selFlags & SF_NestedFrom)!=0
6616 && sameSrcAlias(p0, p1->pSelect->pSrc)
6617 ){
6618 return 1;
6619 }
6620 }
6621 return 0;
6622}
6623
6624/*
drh7d10d5a2008-08-20 16:35:10 +00006625** Generate code for the SELECT statement given in the p argument.
drh9bb61fe2000-06-05 16:01:39 +00006626**
drh340309f2014-01-22 00:23:49 +00006627** The results are returned according to the SelectDest structure.
6628** See comments in sqliteInt.h for further information.
drhe78e8282003-01-19 03:59:45 +00006629**
drh9bb61fe2000-06-05 16:01:39 +00006630** This routine returns the number of errors. If any errors are
6631** encountered, then an appropriate error message is left in
6632** pParse->zErrMsg.
6633**
6634** This routine does NOT free the Select structure passed in. The
6635** calling function needs to do that.
6636*/
danielk19774adee202004-05-08 08:23:19 +00006637int sqlite3Select(
drhcce7d172000-05-31 15:34:51 +00006638 Parse *pParse, /* The parser context */
drh9bb61fe2000-06-05 16:01:39 +00006639 Select *p, /* The SELECT statement being coded. */
drh7d10d5a2008-08-20 16:35:10 +00006640 SelectDest *pDest /* What to do with the query results */
drhcce7d172000-05-31 15:34:51 +00006641){
drh13449892005-09-07 21:22:45 +00006642 int i, j; /* Loop counters */
6643 WhereInfo *pWInfo; /* Return from sqlite3WhereBegin() */
6644 Vdbe *v; /* The virtual machine under construction */
danielk1977b3bce662005-01-29 08:32:43 +00006645 int isAgg; /* True for select lists like "count(*)" */
mistachkinc29cbb02015-07-02 16:52:01 +00006646 ExprList *pEList = 0; /* List of columns to extract. */
drhad3cab52002-05-24 02:04:32 +00006647 SrcList *pTabList; /* List of tables to select from */
drh9bb61fe2000-06-05 16:01:39 +00006648 Expr *pWhere; /* The WHERE clause. May be NULL */
drh22827922000-06-06 17:27:05 +00006649 ExprList *pGroupBy; /* The GROUP BY clause. May be NULL */
6650 Expr *pHaving; /* The HAVING clause. May be NULL */
drhbf790972020-06-07 20:18:07 +00006651 AggInfo *pAggInfo = 0; /* Aggregate information */
drh1d83f052002-02-17 00:30:36 +00006652 int rc = 1; /* Value to return from this function */
drhe8e4af72012-09-21 00:04:28 +00006653 DistinctCtx sDistinct; /* Info on how to code the DISTINCT keyword */
drh079a3072014-03-19 14:10:55 +00006654 SortCtx sSort; /* Info on how to code the ORDER BY clause */
drhec7429a2005-10-06 16:53:14 +00006655 int iEnd; /* Address of the end of the query */
drh17435752007-08-16 04:30:38 +00006656 sqlite3 *db; /* The database connection */
drh47d9f832017-10-26 20:04:28 +00006657 ExprList *pMinMaxOrderBy = 0; /* Added ORDER BY for min/max queries */
6658 u8 minMaxFlag; /* Flag for min/max queries */
drh9bb61fe2000-06-05 16:01:39 +00006659
drh17435752007-08-16 04:30:38 +00006660 db = pParse->db;
drh0c7d3d32022-01-24 16:47:12 +00006661 assert( pParse==db->pParse );
drhe2ca99c2018-05-02 00:33:43 +00006662 v = sqlite3GetVdbe(pParse);
drh0c7d3d32022-01-24 16:47:12 +00006663 if( p==0 || pParse->nErr ){
drh6f7adc82006-01-11 21:41:20 +00006664 return 1;
6665 }
drh0c7d3d32022-01-24 16:47:12 +00006666 assert( db->mallocFailed==0 );
danielk19774adee202004-05-08 08:23:19 +00006667 if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1;
drh5e431be2022-04-06 11:08:38 +00006668#if TREETRACE_ENABLED
drhe2ca99c2018-05-02 00:33:43 +00006669 SELECTTRACE(1,pParse,p, ("begin processing:\n", pParse->addrExplain));
drhc2d0df92022-04-06 18:30:17 +00006670 if( sqlite3TreeTrace & 0x10100 ){
6671 if( (sqlite3TreeTrace & 0x10001)==0x10000 ){
6672 sqlite3TreeViewLine(0, "In sqlite3Select() at %s:%d",
6673 __FILE__, __LINE__);
6674 }
6675 sqlite3ShowSelect(p);
drhc90713d2014-09-30 13:46:49 +00006676 }
drheb9b8842014-09-21 00:27:26 +00006677#endif
drhdaffd0e2001-04-11 14:28:42 +00006678
drh8e1ee882014-03-21 19:56:09 +00006679 assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistFifo );
6680 assert( p->pOrderBy==0 || pDest->eDest!=SRT_Fifo );
dan9afccba2014-03-21 19:27:54 +00006681 assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistQueue );
6682 assert( p->pOrderBy==0 || pDest->eDest!=SRT_Queue );
drhf1ea4252020-09-17 00:46:09 +00006683 if( IgnorableDistinct(pDest) ){
6684 assert(pDest->eDest==SRT_Exists || pDest->eDest==SRT_Union ||
6685 pDest->eDest==SRT_Except || pDest->eDest==SRT_Discard ||
6686 pDest->eDest==SRT_DistQueue || pDest->eDest==SRT_DistFifo );
6687 /* All of these destinations are also able to ignore the ORDER BY clause */
drhfad1ad02021-03-03 14:07:52 +00006688 if( p->pOrderBy ){
drh5e431be2022-04-06 11:08:38 +00006689#if TREETRACE_ENABLED
drhfad1ad02021-03-03 14:07:52 +00006690 SELECTTRACE(1,pParse,p, ("dropping superfluous ORDER BY:\n"));
drh5e431be2022-04-06 11:08:38 +00006691 if( sqlite3TreeTrace & 0x100 ){
drhfad1ad02021-03-03 14:07:52 +00006692 sqlite3TreeViewExprList(0, p->pOrderBy, 0, "ORDERBY");
6693 }
6694#endif
6695 sqlite3ParserAddCleanup(pParse,
6696 (void(*)(sqlite3*,void*))sqlite3ExprListDelete,
6697 p->pOrderBy);
drh6d0053c2021-03-09 19:32:37 +00006698 testcase( pParse->earlyCleanup );
drhfad1ad02021-03-03 14:07:52 +00006699 p->pOrderBy = 0;
6700 }
drh7d10d5a2008-08-20 16:35:10 +00006701 p->selFlags &= ~SF_Distinct;
drhb7cbf5c2020-06-15 13:51:34 +00006702 p->selFlags |= SF_NoopOrderBy;
drh9a993342007-12-13 02:45:31 +00006703 }
drh7d10d5a2008-08-20 16:35:10 +00006704 sqlite3SelectPrep(pParse, p, 0);
drh0c7d3d32022-01-24 16:47:12 +00006705 if( pParse->nErr ){
drh9a993342007-12-13 02:45:31 +00006706 goto select_end;
6707 }
drh0c7d3d32022-01-24 16:47:12 +00006708 assert( db->mallocFailed==0 );
drhadc57f62015-06-06 00:18:01 +00006709 assert( p->pEList!=0 );
drh5e431be2022-04-06 11:08:38 +00006710#if TREETRACE_ENABLED
6711 if( sqlite3TreeTrace & 0x104 ){
drhe2243d22018-04-23 17:18:03 +00006712 SELECTTRACE(0x104,pParse,p, ("after name resolution:\n"));
drh17645f52015-02-09 13:42:59 +00006713 sqlite3TreeViewSelect(0, p, 0);
6714 }
6715#endif
drhcce7d172000-05-31 15:34:51 +00006716
dan5daf69e2021-07-05 11:27:13 +00006717 /* If the SF_UFSrcCheck flag is set, then this function is being called
dan07ca7d62020-07-17 16:31:37 +00006718 ** as part of populating the temp table for an UPDATE...FROM statement.
6719 ** In this case, it is an error if the target object (pSrc->a[0]) name
dan5daf69e2021-07-05 11:27:13 +00006720 ** or alias is duplicated within FROM clause (pSrc->a[1..n]).
6721 **
6722 ** Postgres disallows this case too. The reason is that some other
6723 ** systems handle this case differently, and not all the same way,
6724 ** which is just confusing. To avoid this, we follow PG's lead and
6725 ** disallow it altogether. */
6726 if( p->selFlags & SF_UFSrcCheck ){
drh76012942021-02-21 21:04:54 +00006727 SrcItem *p0 = &p->pSrc->a[0];
drhfb98dac2022-05-25 02:32:11 +00006728 if( sameSrcAlias(p0, p->pSrc) ){
6729 sqlite3ErrorMsg(pParse,
6730 "target object/alias may not appear in FROM clause: %s",
6731 p0->zAlias ? p0->zAlias : p0->pTab->zName
6732 );
6733 goto select_end;
dan07ca7d62020-07-17 16:31:37 +00006734 }
dan5daf69e2021-07-05 11:27:13 +00006735
6736 /* Clear the SF_UFSrcCheck flag. The check has already been performed,
6737 ** and leaving this flag set can cause errors if a compound sub-query
6738 ** in p->pSrc is flattened into this query and this function called
6739 ** again as part of compound SELECT processing. */
6740 p->selFlags &= ~SF_UFSrcCheck;
dan07ca7d62020-07-17 16:31:37 +00006741 }
6742
drhf35f2f92017-07-29 16:01:55 +00006743 if( pDest->eDest==SRT_Output ){
drh90881862021-05-19 12:17:03 +00006744 sqlite3GenerateColumnNames(pParse, p);
drhf35f2f92017-07-29 16:01:55 +00006745 }
6746
dan67a9b8e2018-06-22 20:51:35 +00006747#ifndef SQLITE_OMIT_WINDOWFUNC
dan44918c72021-04-17 14:42:37 +00006748 if( sqlite3WindowRewrite(pParse, p) ){
drh0c7d3d32022-01-24 16:47:12 +00006749 assert( pParse->nErr );
dan86fb6e12018-05-16 20:58:07 +00006750 goto select_end;
6751 }
drh5e431be2022-04-06 11:08:38 +00006752#if TREETRACE_ENABLED
6753 if( p->pWin && (sqlite3TreeTrace & 0x108)!=0 ){
dandfa552f2018-06-02 21:04:28 +00006754 SELECTTRACE(0x104,pParse,p, ("after window rewrite:\n"));
6755 sqlite3TreeViewSelect(0, p, 0);
6756 }
6757#endif
dan67a9b8e2018-06-22 20:51:35 +00006758#endif /* SQLITE_OMIT_WINDOWFUNC */
dan86fb6e12018-05-16 20:58:07 +00006759 pTabList = p->pSrc;
dan73925692018-06-12 18:40:17 +00006760 isAgg = (p->selFlags & SF_Aggregate)!=0;
danf02cdd32018-06-27 19:48:50 +00006761 memset(&sSort, 0, sizeof(sSort));
6762 sSort.pOrderBy = p->pOrderBy;
dan86fb6e12018-05-16 20:58:07 +00006763
drh9216de82020-06-11 00:57:09 +00006764 /* Try to do various optimizations (flattening subqueries, and strength
drh25897872018-03-20 21:16:15 +00006765 ** reduction of join operators) in the FROM clause up into the main query
drhd820cb12002-02-18 03:21:45 +00006766 */
drh51522cd2005-01-20 13:36:19 +00006767#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
danielk1977f23329a2008-07-01 14:09:13 +00006768 for(i=0; !p->pPrior && i<pTabList->nSrc; i++){
drh76012942021-02-21 21:04:54 +00006769 SrcItem *pItem = &pTabList->a[i];
danielk1977daf79ac2008-06-30 18:12:28 +00006770 Select *pSub = pItem->pSelect;
drh2679f142015-09-25 13:42:55 +00006771 Table *pTab = pItem->pTab;
drh25897872018-03-20 21:16:15 +00006772
drhb5aaee52020-06-11 16:04:10 +00006773 /* The expander should have already created transient Table objects
6774 ** even for FROM clause elements such as subqueries that do not correspond
6775 ** to a real table */
6776 assert( pTab!=0 );
6777
drh25897872018-03-20 21:16:15 +00006778 /* Convert LEFT JOIN into JOIN if there are terms of the right table
6779 ** of the LEFT JOIN used in the WHERE clause.
6780 */
drhff02ac72022-04-11 14:43:11 +00006781 if( (pItem->fg.jointype & (JT_LEFT|JT_RIGHT))==JT_LEFT
drh25897872018-03-20 21:16:15 +00006782 && sqlite3ExprImpliesNonNullRow(p->pWhere, pItem->iCursor)
6783 && OptimizationEnabled(db, SQLITE_SimplifyJoin)
6784 ){
6785 SELECTTRACE(0x100,pParse,p,
6786 ("LEFT-JOIN simplifies to JOIN on term %d\n",i));
drhefce69d2018-03-20 22:52:27 +00006787 pItem->fg.jointype &= ~(JT_LEFT|JT_OUTER);
drhd7480402022-06-20 17:04:44 +00006788 assert( pItem->iCursor>=0 );
drh92d1afb2022-06-13 12:42:24 +00006789 unsetJoinExpr(p->pWhere, pItem->iCursor,
6790 pTabList->a[0].fg.jointype & JT_LTORJ);
drh25897872018-03-20 21:16:15 +00006791 }
6792
6793 /* No futher action if this term of the FROM clause is no a subquery */
drh4490c402015-06-05 22:33:39 +00006794 if( pSub==0 ) continue;
drh2679f142015-09-25 13:42:55 +00006795
6796 /* Catch mismatch in the declared columns of a view and the number of
6797 ** columns in the SELECT on the RHS */
6798 if( pTab->nCol!=pSub->pEList->nExpr ){
6799 sqlite3ErrorMsg(pParse, "expected %d columns for '%s' but got %d",
6800 pTab->nCol, pTab->zName, pSub->pEList->nExpr);
6801 goto select_end;
6802 }
6803
drh25c221e2017-09-29 22:13:24 +00006804 /* Do not try to flatten an aggregate subquery.
6805 **
6806 ** Flattening an aggregate subquery is only possible if the outer query
6807 ** is not a join. But if the outer query is not a join, then the subquery
6808 ** will be implemented as a co-routine and there is no advantage to
6809 ** flattening in that case.
6810 */
6811 if( (pSub->selFlags & SF_Aggregate)!=0 ) continue;
6812 assert( pSub->pGroupBy==0 );
6813
drhbb301232021-07-15 19:29:43 +00006814 /* If a FROM-clause subquery has an ORDER BY clause that is not
6815 ** really doing anything, then delete it now so that it does not
drh0fb78f02021-07-16 01:19:19 +00006816 ** interfere with query flattening. See the discussion at
6817 ** https://sqlite.org/forum/forumpost/2d76f2bcf65d256a
drhbb301232021-07-15 19:29:43 +00006818 **
6819 ** Beware of these cases where the ORDER BY clause may not be safely
6820 ** omitted:
6821 **
6822 ** (1) There is also a LIMIT clause
6823 ** (2) The subquery was added to help with window-function
6824 ** processing
drh0fb78f02021-07-16 01:19:19 +00006825 ** (3) The subquery is in the FROM clause of an UPDATE
6826 ** (4) The outer query uses an aggregate function other than
drhbb301232021-07-15 19:29:43 +00006827 ** the built-in count(), min(), or max().
drh0fb78f02021-07-16 01:19:19 +00006828 ** (5) The ORDER BY isn't going to accomplish anything because
6829 ** one of:
6830 ** (a) The outer query has a different ORDER BY clause
6831 ** (b) The subquery is part of a join
6832 ** See forum post 062d576715d277c8
drhbb301232021-07-15 19:29:43 +00006833 */
6834 if( pSub->pOrderBy!=0
drh0fb78f02021-07-16 01:19:19 +00006835 && (p->pOrderBy!=0 || pTabList->nSrc>1) /* Condition (5) */
drhbb301232021-07-15 19:29:43 +00006836 && pSub->pLimit==0 /* Condition (1) */
6837 && (pSub->selFlags & SF_OrderByReqd)==0 /* Condition (2) */
drh0fb78f02021-07-16 01:19:19 +00006838 && (p->selFlags & SF_OrderByReqd)==0 /* Condition (3) and (4) */
drhbb301232021-07-15 19:29:43 +00006839 && OptimizationEnabled(db, SQLITE_OmitOrderBy)
6840 ){
6841 SELECTTRACE(0x100,pParse,p,
6842 ("omit superfluous ORDER BY on %r FROM-clause subquery\n",i+1));
drhe8344842022-07-14 01:48:27 +00006843 sqlite3ParserAddCleanup(pParse,
6844 (void(*)(sqlite3*,void*))sqlite3ExprListDelete,
6845 pSub->pOrderBy);
drhbb301232021-07-15 19:29:43 +00006846 pSub->pOrderBy = 0;
6847 }
6848
drhfca23552017-10-28 20:51:54 +00006849 /* If the outer query contains a "complex" result set (that is,
6850 ** if the result set of the outer query uses functions or subqueries)
6851 ** and if the subquery contains an ORDER BY clause and if
drh648fe492017-09-28 20:06:53 +00006852 ** it will be implemented as a co-routine, then do not flatten. This
6853 ** restriction allows SQL constructs like this:
6854 **
6855 ** SELECT expensive_function(x)
6856 ** FROM (SELECT x FROM tab ORDER BY y LIMIT 10);
6857 **
6858 ** The expensive_function() is only computed on the 10 rows that
6859 ** are output, rather than every row of the table.
drhfca23552017-10-28 20:51:54 +00006860 **
6861 ** The requirement that the outer query have a complex result set
6862 ** means that flattening does occur on simpler SQL constraints without
6863 ** the expensive_function() like:
6864 **
6865 ** SELECT x FROM (SELECT x FROM tab ORDER BY y LIMIT 10);
drh648fe492017-09-28 20:06:53 +00006866 */
drh25c221e2017-09-29 22:13:24 +00006867 if( pSub->pOrderBy!=0
drh648fe492017-09-28 20:06:53 +00006868 && i==0
drhfca23552017-10-28 20:51:54 +00006869 && (p->selFlags & SF_ComplexResult)!=0
drh648fe492017-09-28 20:06:53 +00006870 && (pTabList->nSrc==1
drha76ac882022-04-08 19:20:12 +00006871 || (pTabList->a[1].fg.jointype&(JT_OUTER|JT_CROSS))!=0)
drh648fe492017-09-28 20:06:53 +00006872 ){
6873 continue;
6874 }
6875
drh25c221e2017-09-29 22:13:24 +00006876 if( flattenSubquery(pParse, p, i, isAgg) ){
drh4acd7542019-02-06 00:55:47 +00006877 if( pParse->nErr ) goto select_end;
drh4490c402015-06-05 22:33:39 +00006878 /* This subquery can be absorbed into its parent. */
drh4490c402015-06-05 22:33:39 +00006879 i = -1;
6880 }
6881 pTabList = p->pSrc;
6882 if( db->mallocFailed ) goto select_end;
6883 if( !IgnorableOrderby(pDest) ){
6884 sSort.pOrderBy = p->pOrderBy;
6885 }
6886 }
6887#endif
6888
drh4490c402015-06-05 22:33:39 +00006889#ifndef SQLITE_OMIT_COMPOUND_SELECT
6890 /* Handle compound SELECT statements using the separate multiSelect()
6891 ** procedure.
6892 */
6893 if( p->pPrior ){
6894 rc = multiSelect(pParse, p, pDest);
drh5e431be2022-04-06 11:08:38 +00006895#if TREETRACE_ENABLED
drhf20609d2018-04-23 17:43:35 +00006896 SELECTTRACE(0x1,pParse,p,("end compound-select processing\n"));
drh5e431be2022-04-06 11:08:38 +00006897 if( (sqlite3TreeTrace & 0x2000)!=0 && ExplainQueryPlanParent(pParse)==0 ){
drhf20609d2018-04-23 17:43:35 +00006898 sqlite3TreeViewSelect(0, p, 0);
6899 }
drh4490c402015-06-05 22:33:39 +00006900#endif
drhc631ded2018-05-02 02:22:22 +00006901 if( p->pNext==0 ) ExplainQueryPlanPop(pParse);
drh4490c402015-06-05 22:33:39 +00006902 return rc;
6903 }
6904#endif
6905
drh7810ab62018-07-27 17:51:20 +00006906 /* Do the WHERE-clause constant propagation optimization if this is
6907 ** a join. No need to speed time on this operation for non-join queries
6908 ** as the equivalent optimization will be handled by query planner in
6909 ** sqlite3WhereBegin().
6910 */
drhb775c972021-05-14 14:26:57 +00006911 if( p->pWhere!=0
6912 && p->pWhere->op==TK_AND
drh7810ab62018-07-27 17:51:20 +00006913 && OptimizationEnabled(db, SQLITE_PropagateConst)
drh24e11162018-07-26 23:47:11 +00006914 && propagateConstants(pParse, p)
6915 ){
drh5e431be2022-04-06 11:08:38 +00006916#if TREETRACE_ENABLED
6917 if( sqlite3TreeTrace & 0x100 ){
drh24e11162018-07-26 23:47:11 +00006918 SELECTTRACE(0x100,pParse,p,("After constant propagation:\n"));
6919 sqlite3TreeViewSelect(0, p, 0);
6920 }
6921#endif
6922 }else{
drh7810ab62018-07-27 17:51:20 +00006923 SELECTTRACE(0x100,pParse,p,("Constant propagation not helpful\n"));
drh24e11162018-07-26 23:47:11 +00006924 }
6925
dana4b5fb52018-08-03 20:19:52 +00006926#ifdef SQLITE_COUNTOFVIEW_OPTIMIZATION
6927 if( OptimizationEnabled(db, SQLITE_QueryFlattener|SQLITE_CountOfView)
6928 && countOfViewOptimization(pParse, p)
6929 ){
6930 if( db->mallocFailed ) goto select_end;
6931 pEList = p->pEList;
6932 pTabList = p->pSrc;
6933 }
6934#endif
6935
drh701caf12017-05-10 16:12:00 +00006936 /* For each term in the FROM clause, do two things:
6937 ** (1) Authorized unreferenced tables
6938 ** (2) Generate code for all sub-queries
drhd820cb12002-02-18 03:21:45 +00006939 */
drh4490c402015-06-05 22:33:39 +00006940 for(i=0; i<pTabList->nSrc; i++){
drh76012942021-02-21 21:04:54 +00006941 SrcItem *pItem = &pTabList->a[i];
drha79e2a22021-02-21 23:44:14 +00006942 SrcItem *pPrior;
drh13449892005-09-07 21:22:45 +00006943 SelectDest dest;
drh701caf12017-05-10 16:12:00 +00006944 Select *pSub;
drh824d21a2017-09-29 12:44:52 +00006945#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
6946 const char *zSavedAuthContext;
6947#endif
drh701caf12017-05-10 16:12:00 +00006948
drh3d240d22017-09-28 16:56:55 +00006949 /* Issue SQLITE_READ authorizations with a fake column name for any
6950 ** tables that are referenced but from which no values are extracted.
6951 ** Examples of where these kinds of null SQLITE_READ authorizations
6952 ** would occur:
drh701caf12017-05-10 16:12:00 +00006953 **
drh2336c932017-05-11 12:05:23 +00006954 ** SELECT count(*) FROM t1; -- SQLITE_READ t1.""
6955 ** SELECT t1.* FROM t1, t2; -- SQLITE_READ t2.""
6956 **
6957 ** The fake column name is an empty string. It is possible for a table to
6958 ** have a column named by the empty string, in which case there is no way to
6959 ** distinguish between an unreferenced table and an actual reference to the
drh3d240d22017-09-28 16:56:55 +00006960 ** "" column. The original design was for the fake column name to be a NULL,
drh2336c932017-05-11 12:05:23 +00006961 ** which would be unambiguous. But legacy authorization callbacks might
drh3d240d22017-09-28 16:56:55 +00006962 ** assume the column name is non-NULL and segfault. The use of an empty
6963 ** string for the fake column name seems safer.
drh701caf12017-05-10 16:12:00 +00006964 */
drh74c490e2019-08-07 13:25:21 +00006965 if( pItem->colUsed==0 && pItem->zName!=0 ){
drh2336c932017-05-11 12:05:23 +00006966 sqlite3AuthCheck(pParse, SQLITE_READ, pItem->zName, "", pItem->zDatabase);
drh701caf12017-05-10 16:12:00 +00006967 }
6968
6969#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
6970 /* Generate code for all sub-queries in the FROM clause
6971 */
6972 pSub = pItem->pSelect;
drh5b6a9ed2011-09-15 23:58:14 +00006973 if( pSub==0 ) continue;
drh21172c42012-10-30 00:29:07 +00006974
drh14c4d422021-05-26 18:46:51 +00006975 /* The code for a subquery should only be generated once. */
6976 assert( pItem->addrFillSub==0 );
danielk1977daf79ac2008-06-30 18:12:28 +00006977
danielk1977fc976062007-05-10 10:46:56 +00006978 /* Increment Parse.nHeight by the height of the largest expression
drhf7b54962013-05-28 12:11:54 +00006979 ** tree referred to by this, the parent select. The child select
danielk1977fc976062007-05-10 10:46:56 +00006980 ** may contain expression trees of at most
6981 ** (SQLITE_MAX_EXPR_DEPTH-Parse.nHeight) height. This is a bit
6982 ** more conservative than necessary, but much easier than enforcing
6983 ** an exact limit.
6984 */
6985 pParse->nHeight += sqlite3SelectExprHeight(p);
danielk1977daf79ac2008-06-30 18:12:28 +00006986
drhadc57f62015-06-06 00:18:01 +00006987 /* Make copies of constant WHERE-clause terms in the outer query down
6988 ** inside the subquery. This can help the subquery to run more efficiently.
6989 */
drh7fbb1012018-03-21 01:59:46 +00006990 if( OptimizationEnabled(db, SQLITE_PushDown)
drhdf67ec02021-08-11 13:48:56 +00006991 && (pItem->fg.isCte==0
6992 || (pItem->u2.pCteUse->eM10d!=M10d_Yes && pItem->u2.pCteUse->nUse<2))
drha9cdb902022-04-25 19:40:33 +00006993 && pushDownWhereTerms(pParse, pSub, p->pWhere, pItem)
drh4490c402015-06-05 22:33:39 +00006994 ){
drh5e431be2022-04-06 11:08:38 +00006995#if TREETRACE_ENABLED
6996 if( sqlite3TreeTrace & 0x100 ){
drhd2a44012018-07-28 14:34:22 +00006997 SELECTTRACE(0x100,pParse,p,
6998 ("After WHERE-clause push-down into subquery %d:\n", pSub->selId));
drh4490c402015-06-05 22:33:39 +00006999 sqlite3TreeViewSelect(0, p, 0);
danielk1977daf79ac2008-06-30 18:12:28 +00007000 }
drh69b72d52015-06-01 20:28:03 +00007001#endif
drh8794c682021-02-13 16:39:24 +00007002 assert( pItem->pSelect && (pItem->pSelect->selFlags & SF_PushDown)!=0 );
drh2d277bb2018-03-20 11:24:30 +00007003 }else{
7004 SELECTTRACE(0x100,pParse,p,("Push-down not possible\n"));
drh4490c402015-06-05 22:33:39 +00007005 }
drhadc57f62015-06-06 00:18:01 +00007006
drh824d21a2017-09-29 12:44:52 +00007007 zSavedAuthContext = pParse->zAuthContext;
7008 pParse->zAuthContext = pItem->zName;
7009
drhadc57f62015-06-06 00:18:01 +00007010 /* Generate code to implement the subquery
drh0ff47e92016-03-15 17:52:12 +00007011 **
drhcf5cab02022-06-22 15:55:28 +00007012 ** The subquery is implemented as a co-routine if all of the following are
drh9debb582022-04-10 23:01:20 +00007013 ** true:
7014 **
drha79e2a22021-02-21 23:44:14 +00007015 ** (1) the subquery is guaranteed to be the outer loop (so that
7016 ** it does not need to be computed more than once), and
drh745912e2021-02-22 03:04:25 +00007017 ** (2) the subquery is not a CTE that should be materialized
drh9debb582022-04-10 23:01:20 +00007018 ** (3) the subquery is not part of a left operand for a RIGHT JOIN
drhadc57f62015-06-06 00:18:01 +00007019 */
drh0ff47e92016-03-15 17:52:12 +00007020 if( i==0
7021 && (pTabList->nSrc==1
drha76ac882022-04-08 19:20:12 +00007022 || (pTabList->a[1].fg.jointype&(JT_OUTER|JT_CROSS))!=0) /* (1) */
7023 && (pItem->fg.isCte==0 || pItem->u2.pCteUse->eM10d!=M10d_Yes) /* (2) */
drh9debb582022-04-10 23:01:20 +00007024 && (pTabList->a[0].fg.jointype & JT_LTORJ)==0 /* (3) */
drha5759672012-10-30 14:39:12 +00007025 ){
drh21172c42012-10-30 00:29:07 +00007026 /* Implement a co-routine that will return a single row of the result
7027 ** set on each invocation.
7028 */
drh4490c402015-06-05 22:33:39 +00007029 int addrTop = sqlite3VdbeCurrentAddr(v)+1;
drh824d21a2017-09-29 12:44:52 +00007030
drh21172c42012-10-30 00:29:07 +00007031 pItem->regReturn = ++pParse->nMem;
drh4490c402015-06-05 22:33:39 +00007032 sqlite3VdbeAddOp3(v, OP_InitCoroutine, pItem->regReturn, 0, addrTop);
drha9799932021-03-19 13:00:28 +00007033 VdbeComment((v, "%!S", pItem));
drh21172c42012-10-30 00:29:07 +00007034 pItem->addrFillSub = addrTop;
drh21172c42012-10-30 00:29:07 +00007035 sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn);
drha9799932021-03-19 13:00:28 +00007036 ExplainQueryPlan((pParse, 1, "CO-ROUTINE %!S", pItem));
drh21172c42012-10-30 00:29:07 +00007037 sqlite3Select(pParse, pSub, &dest);
drhc3489bb2016-02-25 16:04:59 +00007038 pItem->pTab->nRowLogEst = pSub->nSelectRow;
drh8a48b9c2015-08-19 15:20:00 +00007039 pItem->fg.viaCoroutine = 1;
drh4490c402015-06-05 22:33:39 +00007040 pItem->regResult = dest.iSdst;
drh2fade2f2016-02-09 02:12:20 +00007041 sqlite3VdbeEndCoroutine(v, pItem->regReturn);
drh21172c42012-10-30 00:29:07 +00007042 sqlite3VdbeJumpHere(v, addrTop-1);
7043 sqlite3ClearTempRegCache(pParse);
drha79e2a22021-02-21 23:44:14 +00007044 }else if( pItem->fg.isCte && pItem->u2.pCteUse->addrM9e>0 ){
7045 /* This is a CTE for which materialization code has already been
7046 ** generated. Invoke the subroutine to compute the materialization,
7047 ** the make the pItem->iCursor be a copy of the ephemerial table that
7048 ** holds the result of the materialization. */
7049 CteUse *pCteUse = pItem->u2.pCteUse;
7050 sqlite3VdbeAddOp2(v, OP_Gosub, pCteUse->regRtn, pCteUse->addrM9e);
7051 if( pItem->iCursor!=pCteUse->iCur ){
7052 sqlite3VdbeAddOp2(v, OP_OpenDup, pItem->iCursor, pCteUse->iCur);
drh834c6882021-08-11 13:19:13 +00007053 VdbeComment((v, "%!S", pItem));
drha79e2a22021-02-21 23:44:14 +00007054 }
7055 pSub->nSelectRow = pCteUse->nRowEst;
7056 }else if( (pPrior = isSelfJoinView(pTabList, pItem))!=0 ){
7057 /* This view has already been materialized by a prior entry in
7058 ** this same FROM clause. Reuse it. */
7059 if( pPrior->addrFillSub ){
7060 sqlite3VdbeAddOp2(v, OP_Gosub, pPrior->regReturn, pPrior->addrFillSub);
7061 }
7062 sqlite3VdbeAddOp2(v, OP_OpenDup, pItem->iCursor, pPrior->iCursor);
7063 pSub->nSelectRow = pPrior->pSelect->nSelectRow;
danielk1977daf79ac2008-06-30 18:12:28 +00007064 }else{
drh14c4d422021-05-26 18:46:51 +00007065 /* Materialize the view. If the view is not correlated, generate a
drhd685dd62021-03-19 13:53:34 +00007066 ** subroutine to do the materialization so that subsequent uses of
7067 ** the same view can reuse the materialization. */
drh7157e8e2011-09-16 16:00:51 +00007068 int topAddr;
drh48f2d3b2011-09-16 01:34:43 +00007069 int onceAddr = 0;
drhe08e8d62017-05-01 15:15:41 +00007070
drh5b6a9ed2011-09-15 23:58:14 +00007071 pItem->regReturn = ++pParse->nMem;
drh40822eb2022-05-21 18:03:33 +00007072 topAddr = sqlite3VdbeAddOp0(v, OP_Goto);
drh7157e8e2011-09-16 16:00:51 +00007073 pItem->addrFillSub = topAddr+1;
drh40822eb2022-05-21 18:03:33 +00007074 pItem->fg.isMaterialized = 1;
drh8a48b9c2015-08-19 15:20:00 +00007075 if( pItem->fg.isCorrelated==0 ){
drhed171672013-04-24 13:50:09 +00007076 /* If the subquery is not correlated and if we are not inside of
drh5b6a9ed2011-09-15 23:58:14 +00007077 ** a trigger, then we only need to compute the value of the subquery
7078 ** once. */
drh511f9e82016-09-22 18:53:13 +00007079 onceAddr = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
drha9799932021-03-19 13:00:28 +00007080 VdbeComment((v, "materialize %!S", pItem));
drh69b72d52015-06-01 20:28:03 +00007081 }else{
drha9799932021-03-19 13:00:28 +00007082 VdbeNoopComment((v, "materialize %!S", pItem));
drh5b6a9ed2011-09-15 23:58:14 +00007083 }
drha79e2a22021-02-21 23:44:14 +00007084 sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor);
drha9799932021-03-19 13:00:28 +00007085 ExplainQueryPlan((pParse, 1, "MATERIALIZE %!S", pItem));
drha79e2a22021-02-21 23:44:14 +00007086 sqlite3Select(pParse, pSub, &dest);
drhc3489bb2016-02-25 16:04:59 +00007087 pItem->pTab->nRowLogEst = pSub->nSelectRow;
drh48f2d3b2011-09-16 01:34:43 +00007088 if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr);
drh40822eb2022-05-21 18:03:33 +00007089 sqlite3VdbeAddOp2(v, OP_Return, pItem->regReturn, topAddr+1);
drha9799932021-03-19 13:00:28 +00007090 VdbeComment((v, "end %!S", pItem));
drh40822eb2022-05-21 18:03:33 +00007091 sqlite3VdbeJumpHere(v, topAddr);
drhcdc69552011-12-06 13:24:59 +00007092 sqlite3ClearTempRegCache(pParse);
drhd685dd62021-03-19 13:53:34 +00007093 if( pItem->fg.isCte && pItem->fg.isCorrelated==0 ){
drha79e2a22021-02-21 23:44:14 +00007094 CteUse *pCteUse = pItem->u2.pCteUse;
7095 pCteUse->addrM9e = pItem->addrFillSub;
7096 pCteUse->regRtn = pItem->regReturn;
7097 pCteUse->iCur = pItem->iCursor;
7098 pCteUse->nRowEst = pSub->nSelectRow;
7099 }
danielk1977daf79ac2008-06-30 18:12:28 +00007100 }
drhadc57f62015-06-06 00:18:01 +00007101 if( db->mallocFailed ) goto select_end;
danielk1977fc976062007-05-10 10:46:56 +00007102 pParse->nHeight -= sqlite3SelectExprHeight(p);
drh824d21a2017-09-29 12:44:52 +00007103 pParse->zAuthContext = zSavedAuthContext;
drh51522cd2005-01-20 13:36:19 +00007104#endif
drh701caf12017-05-10 16:12:00 +00007105 }
drhadc57f62015-06-06 00:18:01 +00007106
drh38b41492015-06-08 15:08:15 +00007107 /* Various elements of the SELECT copied into local variables for
7108 ** convenience */
drhadc57f62015-06-06 00:18:01 +00007109 pEList = p->pEList;
danielk1977daf79ac2008-06-30 18:12:28 +00007110 pWhere = p->pWhere;
7111 pGroupBy = p->pGroupBy;
7112 pHaving = p->pHaving;
drhe8e4af72012-09-21 00:04:28 +00007113 sDistinct.isTnct = (p->selFlags & SF_Distinct)!=0;
drh832508b2002-03-02 17:04:07 +00007114
drh5e431be2022-04-06 11:08:38 +00007115#if TREETRACE_ENABLED
7116 if( sqlite3TreeTrace & 0x400 ){
drhbc8edba2015-06-05 20:27:26 +00007117 SELECTTRACE(0x400,pParse,p,("After all FROM-clause analysis:\n"));
7118 sqlite3TreeViewSelect(0, p, 0);
danielk1977f23329a2008-07-01 14:09:13 +00007119 }
7120#endif
7121
dan50118cd2011-07-01 14:21:38 +00007122 /* If the query is DISTINCT with an ORDER BY but is not an aggregate, and
7123 ** if the select-list is the same as the ORDER BY list, then this query
7124 ** can be rewritten as a GROUP BY. In other words, this:
7125 **
7126 ** SELECT DISTINCT xyz FROM ... ORDER BY xyz
7127 **
7128 ** is transformed to:
7129 **
drhdea7d702014-12-04 21:54:58 +00007130 ** SELECT xyz FROM ... GROUP BY xyz ORDER BY xyz
dan50118cd2011-07-01 14:21:38 +00007131 **
7132 ** The second form is preferred as a single index (or temp-table) may be
7133 ** used for both the ORDER BY and DISTINCT processing. As originally
7134 ** written the query must use a temp-table for at least one of the ORDER
7135 ** BY and DISTINCT, and an index or separate temp-table for the other.
7136 */
7137 if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct
drhadc57f62015-06-06 00:18:01 +00007138 && sqlite3ExprListCompare(sSort.pOrderBy, pEList, -1)==0
drhef9f7192020-01-17 19:14:08 +00007139#ifndef SQLITE_OMIT_WINDOWFUNC
dane59c5622019-11-22 10:14:01 +00007140 && p->pWin==0
drhef9f7192020-01-17 19:14:08 +00007141#endif
dan50118cd2011-07-01 14:21:38 +00007142 ){
7143 p->selFlags &= ~SF_Distinct;
drhadc57f62015-06-06 00:18:01 +00007144 pGroupBy = p->pGroupBy = sqlite3ExprListDup(db, pEList, 0);
dan9e10f9a2020-01-06 17:06:12 +00007145 p->selFlags |= SF_Aggregate;
drhe8e4af72012-09-21 00:04:28 +00007146 /* Notice that even thought SF_Distinct has been cleared from p->selFlags,
7147 ** the sDistinct.isTnct is still set. Hence, isTnct represents the
7148 ** original setting of the SF_Distinct flag, not the current setting */
7149 assert( sDistinct.isTnct );
danece092e2022-03-16 12:06:00 +00007150 sDistinct.isTnct = 2;
drh7512cb42016-04-14 13:06:49 +00007151
drh5e431be2022-04-06 11:08:38 +00007152#if TREETRACE_ENABLED
7153 if( sqlite3TreeTrace & 0x400 ){
drh7512cb42016-04-14 13:06:49 +00007154 SELECTTRACE(0x400,pParse,p,("Transform DISTINCT into GROUP BY:\n"));
7155 sqlite3TreeViewSelect(0, p, 0);
7156 }
7157#endif
dan50118cd2011-07-01 14:21:38 +00007158 }
7159
drhadc57f62015-06-06 00:18:01 +00007160 /* If there is an ORDER BY clause, then create an ephemeral index to
7161 ** do the sorting. But this sorting ephemeral index might end up
7162 ** being unused if the data can be extracted in pre-sorted order.
7163 ** If that is the case, then the OP_OpenEphemeral instruction will be
7164 ** changed to an OP_Noop once we figure out that the sorting index is
7165 ** not needed. The sSort.addrSortIndex variable is used to facilitate
7166 ** that change.
danielk19777cedc8d2004-06-10 10:50:08 +00007167 */
drh079a3072014-03-19 14:10:55 +00007168 if( sSort.pOrderBy ){
drh0342b1f2005-09-01 03:07:44 +00007169 KeyInfo *pKeyInfo;
danf9eae182018-05-21 19:45:11 +00007170 pKeyInfo = sqlite3KeyInfoFromExprList(
7171 pParse, sSort.pOrderBy, 0, pEList->nExpr);
drh079a3072014-03-19 14:10:55 +00007172 sSort.iECursor = pParse->nTab++;
7173 sSort.addrSortIndex =
drh66a51672008-01-03 00:01:23 +00007174 sqlite3VdbeAddOp4(v, OP_OpenEphemeral,
drhf45f2322014-03-23 17:45:03 +00007175 sSort.iECursor, sSort.pOrderBy->nExpr+1+pEList->nExpr, 0,
7176 (char*)pKeyInfo, P4_KEYINFO
7177 );
drh9d2985c2005-09-08 01:58:42 +00007178 }else{
drh079a3072014-03-19 14:10:55 +00007179 sSort.addrSortIndex = -1;
danielk19777cedc8d2004-06-10 10:50:08 +00007180 }
7181
drh2d0794e2002-03-03 03:03:52 +00007182 /* If the output is destined for a temporary table, open that table.
7183 */
danielk19776c8c8ce2008-01-02 16:27:09 +00007184 if( pDest->eDest==SRT_EphemTab ){
drh2b596da2012-07-23 21:43:19 +00007185 sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pDest->iSDParm, pEList->nExpr);
drh18f86002022-04-21 13:11:26 +00007186 if( p->selFlags & SF_NestedFrom ){
7187 /* Delete or NULL-out result columns that will never be used */
7188 int ii;
drhd88fd532022-05-02 20:49:30 +00007189 for(ii=pEList->nExpr-1; ii>0 && pEList->a[ii].fg.bUsed==0; ii--){
drh18f86002022-04-21 13:11:26 +00007190 sqlite3ExprDelete(db, pEList->a[ii].pExpr);
7191 sqlite3DbFree(db, pEList->a[ii].zEName);
7192 pEList->nExpr--;
7193 }
7194 for(ii=0; ii<pEList->nExpr; ii++){
drhd88fd532022-05-02 20:49:30 +00007195 if( pEList->a[ii].fg.bUsed==0 ) pEList->a[ii].pExpr->op = TK_NULL;
drh18f86002022-04-21 13:11:26 +00007196 }
7197 }
drh2d0794e2002-03-03 03:03:52 +00007198 }
7199
drhf42bacc2006-04-26 17:39:34 +00007200 /* Set the limiter.
7201 */
drhec4ccdb2018-12-29 02:26:59 +00007202 iEnd = sqlite3VdbeMakeLabel(pParse);
dan69b93832016-12-16 15:05:40 +00007203 if( (p->selFlags & SF_FixedLimit)==0 ){
7204 p->nSelectRow = 320; /* 4 billion rows */
7205 }
drh3c8fb6f2022-07-26 01:20:18 +00007206 if( p->pLimit ) computeLimitRegisters(pParse, p, iEnd);
drh079a3072014-03-19 14:10:55 +00007207 if( p->iLimit==0 && sSort.addrSortIndex>=0 ){
drh0ff287f2015-09-02 18:40:33 +00007208 sqlite3VdbeChangeOpcode(v, sSort.addrSortIndex, OP_SorterOpen);
drh079a3072014-03-19 14:10:55 +00007209 sSort.sortFlags |= SORTFLAG_UseSorter;
drhc6aff302011-09-01 15:32:47 +00007210 }
drhf42bacc2006-04-26 17:39:34 +00007211
drhadc57f62015-06-06 00:18:01 +00007212 /* Open an ephemeral index to use for the distinct set.
drhefb72512000-05-31 20:00:52 +00007213 */
dan2ce22452010-11-08 19:01:16 +00007214 if( p->selFlags & SF_Distinct ){
drhe8e4af72012-09-21 00:04:28 +00007215 sDistinct.tabTnct = pParse->nTab++;
7216 sDistinct.addrTnct = sqlite3VdbeAddOp4(v, OP_OpenEphemeral,
danf9eae182018-05-21 19:45:11 +00007217 sDistinct.tabTnct, 0, 0,
7218 (char*)sqlite3KeyInfoFromExprList(pParse, p->pEList,0,0),
7219 P4_KEYINFO);
drhd4187c72010-08-30 22:15:45 +00007220 sqlite3VdbeChangeP5(v, BTREE_UNORDERED);
drhe8e4af72012-09-21 00:04:28 +00007221 sDistinct.eTnctType = WHERE_DISTINCT_UNORDERED;
drh832508b2002-03-02 17:04:07 +00007222 }else{
drhe8e4af72012-09-21 00:04:28 +00007223 sDistinct.eTnctType = WHERE_DISTINCT_NOOP;
drhefb72512000-05-31 20:00:52 +00007224 }
drh832508b2002-03-02 17:04:07 +00007225
drh13449892005-09-07 21:22:45 +00007226 if( !isAgg && pGroupBy==0 ){
drhe8e4af72012-09-21 00:04:28 +00007227 /* No aggregate functions and no GROUP BY clause */
dan67a9b8e2018-06-22 20:51:35 +00007228 u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0)
7229 | (p->selFlags & SF_FixedLimit);
7230#ifndef SQLITE_OMIT_WINDOWFUNC
drh067b92b2020-06-19 15:24:12 +00007231 Window *pWin = p->pWin; /* Main window object (or NULL) */
danf9eae182018-05-21 19:45:11 +00007232 if( pWin ){
dan4ea562e2020-01-01 20:17:15 +00007233 sqlite3WindowCodeInit(pParse, p);
dan86fb6e12018-05-16 20:58:07 +00007234 }
dan67a9b8e2018-06-22 20:51:35 +00007235#endif
7236 assert( WHERE_USE_LIMIT==SF_FixedLimit );
7237
dan86fb6e12018-05-16 20:58:07 +00007238
dan38cc40c2011-06-30 20:17:15 +00007239 /* Begin the database scan. */
drhcfd74702018-03-19 22:28:34 +00007240 SELECTTRACE(1,pParse,p,("WhereBegin\n"));
drh079a3072014-03-19 14:10:55 +00007241 pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, sSort.pOrderBy,
drh895bab32022-01-27 16:14:50 +00007242 p->pEList, p, wctrlFlags, p->nSelectRow);
drh13449892005-09-07 21:22:45 +00007243 if( pWInfo==0 ) goto select_end;
drh6f328482013-06-05 23:39:34 +00007244 if( sqlite3WhereOutputRowCount(pWInfo) < p->nSelectRow ){
7245 p->nSelectRow = sqlite3WhereOutputRowCount(pWInfo);
7246 }
drh6457a352013-06-21 00:35:37 +00007247 if( sDistinct.isTnct && sqlite3WhereIsDistinct(pWInfo) ){
drh6f328482013-06-05 23:39:34 +00007248 sDistinct.eTnctType = sqlite3WhereIsDistinct(pWInfo);
7249 }
drh079a3072014-03-19 14:10:55 +00007250 if( sSort.pOrderBy ){
7251 sSort.nOBSat = sqlite3WhereIsOrdered(pWInfo);
drh6ee5a7b2018-09-08 20:09:46 +00007252 sSort.labelOBLopt = sqlite3WhereOrderByLimitOptLabel(pWInfo);
drh079a3072014-03-19 14:10:55 +00007253 if( sSort.nOBSat==sSort.pOrderBy->nExpr ){
7254 sSort.pOrderBy = 0;
7255 }
7256 }
drh13162ef2021-02-16 20:01:08 +00007257 SELECTTRACE(1,pParse,p,("WhereBegin returns\n"));
drhcce7d172000-05-31 15:34:51 +00007258
drhb9bb7c12006-06-11 23:41:55 +00007259 /* If sorting index that was created by a prior OP_OpenEphemeral
7260 ** instruction ended up not being needed, then change the OP_OpenEphemeral
drh9d2985c2005-09-08 01:58:42 +00007261 ** into an OP_Noop.
7262 */
drh079a3072014-03-19 14:10:55 +00007263 if( sSort.addrSortIndex>=0 && sSort.pOrderBy==0 ){
7264 sqlite3VdbeChangeToNoop(v, sSort.addrSortIndex);
drh9d2985c2005-09-08 01:58:42 +00007265 }
7266
drh2def2f72017-09-15 17:40:34 +00007267 assert( p->pEList==pEList );
dan67a9b8e2018-06-22 20:51:35 +00007268#ifndef SQLITE_OMIT_WINDOWFUNC
danf9eae182018-05-21 19:45:11 +00007269 if( pWin ){
drhec4ccdb2018-12-29 02:26:59 +00007270 int addrGosub = sqlite3VdbeMakeLabel(pParse);
7271 int iCont = sqlite3VdbeMakeLabel(pParse);
7272 int iBreak = sqlite3VdbeMakeLabel(pParse);
danf9eae182018-05-21 19:45:11 +00007273 int regGosub = ++pParse->nMem;
drhefb72512000-05-31 20:00:52 +00007274
dandacf1de2018-06-08 16:11:55 +00007275 sqlite3WindowCodeStep(pParse, p, pWInfo, regGosub, addrGosub);
dan86fb6e12018-05-16 20:58:07 +00007276
danefa3a3c2018-06-23 16:26:20 +00007277 sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak);
danf9eae182018-05-21 19:45:11 +00007278 sqlite3VdbeResolveLabel(v, addrGosub);
drhb0225bc2018-07-10 20:50:27 +00007279 VdbeNoopComment((v, "inner-loop subroutine"));
drhd4cb09e2018-09-17 15:19:13 +00007280 sSort.labelOBLopt = 0;
danefa3a3c2018-06-23 16:26:20 +00007281 selectInnerLoop(pParse, p, -1, &sSort, &sDistinct, pDest, iCont, iBreak);
dandacf1de2018-06-08 16:11:55 +00007282 sqlite3VdbeResolveLabel(v, iCont);
dan86fb6e12018-05-16 20:58:07 +00007283 sqlite3VdbeAddOp1(v, OP_Return, regGosub);
drh0b3b0dd2018-07-10 05:11:03 +00007284 VdbeComment((v, "end inner-loop subroutine"));
danefa3a3c2018-06-23 16:26:20 +00007285 sqlite3VdbeResolveLabel(v, iBreak);
dan67a9b8e2018-06-22 20:51:35 +00007286 }else
7287#endif /* SQLITE_OMIT_WINDOWFUNC */
7288 {
dan86fb6e12018-05-16 20:58:07 +00007289 /* Use the standard inner loop. */
7290 selectInnerLoop(pParse, p, -1, &sSort, &sDistinct, pDest,
7291 sqlite3WhereContinueLabel(pWInfo),
7292 sqlite3WhereBreakLabel(pWInfo));
7293
7294 /* End the database scan loop.
7295 */
drh13162ef2021-02-16 20:01:08 +00007296 SELECTTRACE(1,pParse,p,("WhereEnd\n"));
dan86fb6e12018-05-16 20:58:07 +00007297 sqlite3WhereEnd(pWInfo);
7298 }
drh13449892005-09-07 21:22:45 +00007299 }else{
drhe8e4af72012-09-21 00:04:28 +00007300 /* This case when there exist aggregate functions or a GROUP BY clause
7301 ** or both */
drh13449892005-09-07 21:22:45 +00007302 NameContext sNC; /* Name context for processing aggregate information */
7303 int iAMem; /* First Mem address for storing current GROUP BY */
7304 int iBMem; /* First Mem address for previous GROUP BY */
7305 int iUseFlag; /* Mem address holding flag indicating that at least
7306 ** one row of the input to the aggregator has been
7307 ** processed */
7308 int iAbortFlag; /* Mem address which causes query abort if positive */
7309 int groupBySort; /* Rows come from source in GROUP BY order */
drhd1766112008-09-17 00:13:12 +00007310 int addrEnd; /* End of processing for this SELECT */
drh1c9d8352011-09-01 16:01:27 +00007311 int sortPTab = 0; /* Pseudotable used to decode sorting results */
7312 int sortOut = 0; /* Output register from the sorter */
dan374cd782014-04-21 13:21:56 +00007313 int orderByGrp = 0; /* True if the GROUP BY and ORDER BY are the same */
drhcce7d172000-05-31 15:34:51 +00007314
drhd1766112008-09-17 00:13:12 +00007315 /* Remove any and all aliases between the result set and the
7316 ** GROUP BY clause.
7317 */
7318 if( pGroupBy ){
drhdc5ea5c2008-12-10 17:19:59 +00007319 int k; /* Loop counter */
drhd1766112008-09-17 00:13:12 +00007320 struct ExprList_item *pItem; /* For looping over expression in a list */
drhcce7d172000-05-31 15:34:51 +00007321
drhdc5ea5c2008-12-10 17:19:59 +00007322 for(k=p->pEList->nExpr, pItem=p->pEList->a; k>0; k--, pItem++){
drhc2acc4e2013-11-15 18:15:19 +00007323 pItem->u.x.iAlias = 0;
drhd1766112008-09-17 00:13:12 +00007324 }
drhdc5ea5c2008-12-10 17:19:59 +00007325 for(k=pGroupBy->nExpr, pItem=pGroupBy->a; k>0; k--, pItem++){
drhc2acc4e2013-11-15 18:15:19 +00007326 pItem->u.x.iAlias = 0;
drhd1766112008-09-17 00:13:12 +00007327 }
drhc3489bb2016-02-25 16:04:59 +00007328 assert( 66==sqlite3LogEst(100) );
7329 if( p->nSelectRow>66 ) p->nSelectRow = 66;
dan8c9bcb22019-09-21 15:44:12 +00007330
7331 /* If there is both a GROUP BY and an ORDER BY clause and they are
7332 ** identical, then it may be possible to disable the ORDER BY clause
7333 ** on the grounds that the GROUP BY will cause elements to come out
7334 ** in the correct order. It also may not - the GROUP BY might use a
7335 ** database index that causes rows to be grouped together as required
7336 ** but not actually sorted. Either way, record the fact that the
7337 ** ORDER BY and GROUP BY clauses are the same by setting the orderByGrp
7338 ** variable. */
7339 if( sSort.pOrderBy && pGroupBy->nExpr==sSort.pOrderBy->nExpr ){
drhe39f3882019-09-21 17:31:03 +00007340 int ii;
dan8c9bcb22019-09-21 15:44:12 +00007341 /* The GROUP BY processing doesn't care whether rows are delivered in
7342 ** ASC or DESC order - only that each group is returned contiguously.
7343 ** So set the ASC/DESC flags in the GROUP BY to match those in the
7344 ** ORDER BY to maximize the chances of rows being delivered in an
7345 ** order that makes the ORDER BY redundant. */
drhe39f3882019-09-21 17:31:03 +00007346 for(ii=0; ii<pGroupBy->nExpr; ii++){
drhd88fd532022-05-02 20:49:30 +00007347 u8 sortFlags;
7348 sortFlags = sSort.pOrderBy->a[ii].fg.sortFlags & KEYINFO_ORDER_DESC;
7349 pGroupBy->a[ii].fg.sortFlags = sortFlags;
dan8c9bcb22019-09-21 15:44:12 +00007350 }
7351 if( sqlite3ExprListCompare(pGroupBy, sSort.pOrderBy, -1)==0 ){
7352 orderByGrp = 1;
7353 }
7354 }
drh95aa47b2010-11-16 02:49:15 +00007355 }else{
drhc3489bb2016-02-25 16:04:59 +00007356 assert( 0==sqlite3LogEst(1) );
7357 p->nSelectRow = 0;
drhd1766112008-09-17 00:13:12 +00007358 }
drh13449892005-09-07 21:22:45 +00007359
drhd1766112008-09-17 00:13:12 +00007360 /* Create a label to jump to when we want to abort the query */
drhec4ccdb2018-12-29 02:26:59 +00007361 addrEnd = sqlite3VdbeMakeLabel(pParse);
drh13449892005-09-07 21:22:45 +00007362
7363 /* Convert TK_COLUMN nodes into TK_AGG_COLUMN and make entries in
7364 ** sAggInfo for all TK_AGG_FUNCTION nodes in expressions of the
7365 ** SELECT statement.
7366 */
drhbf790972020-06-07 20:18:07 +00007367 pAggInfo = sqlite3DbMallocZero(db, sizeof(*pAggInfo) );
drhc54246f2021-02-17 21:13:14 +00007368 if( pAggInfo ){
7369 sqlite3ParserAddCleanup(pParse,
7370 (void(*)(sqlite3*,void*))agginfoFree, pAggInfo);
drh6d0053c2021-03-09 19:32:37 +00007371 testcase( pParse->earlyCleanup );
drhc54246f2021-02-17 21:13:14 +00007372 }
7373 if( db->mallocFailed ){
drhbf790972020-06-07 20:18:07 +00007374 goto select_end;
7375 }
drhe26d4282020-06-09 11:59:15 +00007376 pAggInfo->selId = p->selId;
drh13449892005-09-07 21:22:45 +00007377 memset(&sNC, 0, sizeof(sNC));
7378 sNC.pParse = pParse;
7379 sNC.pSrcList = pTabList;
drhbf790972020-06-07 20:18:07 +00007380 sNC.uNC.pAggInfo = pAggInfo;
drh25c3b8c2018-04-16 10:34:13 +00007381 VVA_ONLY( sNC.ncFlags = NC_UAggInfo; )
drhbf790972020-06-07 20:18:07 +00007382 pAggInfo->mnReg = pParse->nMem+1;
7383 pAggInfo->nSortingColumn = pGroupBy ? pGroupBy->nExpr : 0;
7384 pAggInfo->pGroupBy = pGroupBy;
drhd2b3e232008-01-23 14:51:49 +00007385 sqlite3ExprAnalyzeAggList(&sNC, pEList);
drh079a3072014-03-19 14:10:55 +00007386 sqlite3ExprAnalyzeAggList(&sNC, sSort.pOrderBy);
drhd2b3e232008-01-23 14:51:49 +00007387 if( pHaving ){
danab31a842017-04-29 20:53:09 +00007388 if( pGroupBy ){
7389 assert( pWhere==p->pWhere );
drhcd0abc22018-03-20 18:08:33 +00007390 assert( pHaving==p->pHaving );
7391 assert( pGroupBy==p->pGroupBy );
7392 havingToWhere(pParse, p);
danab31a842017-04-29 20:53:09 +00007393 pWhere = p->pWhere;
7394 }
drhd2b3e232008-01-23 14:51:49 +00007395 sqlite3ExprAnalyzeAggregates(&sNC, pHaving);
drh13449892005-09-07 21:22:45 +00007396 }
drhbf790972020-06-07 20:18:07 +00007397 pAggInfo->nAccumulator = pAggInfo->nColumn;
7398 if( p->pGroupBy==0 && p->pHaving==0 && pAggInfo->nFunc==1 ){
drh81185a52020-06-09 13:38:12 +00007399 minMaxFlag = minMaxQuery(db, pAggInfo->aFunc[0].pFExpr, &pMinMaxOrderBy);
drh47d9f832017-10-26 20:04:28 +00007400 }else{
7401 minMaxFlag = WHERE_ORDERBY_NORMAL;
7402 }
drhbf790972020-06-07 20:18:07 +00007403 for(i=0; i<pAggInfo->nFunc; i++){
drh81185a52020-06-09 13:38:12 +00007404 Expr *pExpr = pAggInfo->aFunc[i].pFExpr;
drha4eeccd2021-10-07 17:43:30 +00007405 assert( ExprUseXList(pExpr) );
drh3a8c4be2012-05-21 20:13:39 +00007406 sNC.ncFlags |= NC_InAggFunc;
dan6ba7ab02019-07-02 11:56:47 +00007407 sqlite3ExprAnalyzeAggList(&sNC, pExpr->x.pList);
7408#ifndef SQLITE_OMIT_WINDOWFUNC
dan4f9adee2019-07-13 16:22:50 +00007409 assert( !IsWindowFunc(pExpr) );
7410 if( ExprHasProperty(pExpr, EP_WinFunc) ){
7411 sqlite3ExprAnalyzeAggregates(&sNC, pExpr->y.pWin->pFilter);
danb28c4e52019-07-05 17:38:55 +00007412 }
dan6ba7ab02019-07-02 11:56:47 +00007413#endif
drh3a8c4be2012-05-21 20:13:39 +00007414 sNC.ncFlags &= ~NC_InAggFunc;
drh13449892005-09-07 21:22:45 +00007415 }
drhbf790972020-06-07 20:18:07 +00007416 pAggInfo->mxReg = pParse->nMem;
drh17435752007-08-16 04:30:38 +00007417 if( db->mallocFailed ) goto select_end;
drh5e431be2022-04-06 11:08:38 +00007418#if TREETRACE_ENABLED
7419 if( sqlite3TreeTrace & 0x400 ){
drh7ea11062017-10-26 18:43:19 +00007420 int ii;
drhbf790972020-06-07 20:18:07 +00007421 SELECTTRACE(0x400,pParse,p,("After aggregate analysis %p:\n", pAggInfo));
drh7ea11062017-10-26 18:43:19 +00007422 sqlite3TreeViewSelect(0, p, 0);
drhb9366f82021-01-13 11:44:51 +00007423 if( minMaxFlag ){
7424 sqlite3DebugPrintf("MIN/MAX Optimization (0x%02x) adds:\n", minMaxFlag);
7425 sqlite3TreeViewExprList(0, pMinMaxOrderBy, 0, "ORDERBY");
7426 }
drhbf790972020-06-07 20:18:07 +00007427 for(ii=0; ii<pAggInfo->nColumn; ii++){
drhf4c29122022-07-26 14:37:04 +00007428 struct AggInfo_col *pCol = &pAggInfo->aCol[ii];
7429 sqlite3DebugPrintf(
7430 "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d"
7431 " iSorterColumn=%d\n",
7432 ii, pCol->pTab ? pCol->pTab->zName : "NULL",
7433 pCol->iTable, pCol->iColumn, pCol->iMem,
7434 pCol->iSorterColumn);
drh81185a52020-06-09 13:38:12 +00007435 sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0);
drh7ea11062017-10-26 18:43:19 +00007436 }
drhbf790972020-06-07 20:18:07 +00007437 for(ii=0; ii<pAggInfo->nFunc; ii++){
drh7ea11062017-10-26 18:43:19 +00007438 sqlite3DebugPrintf("agg-func[%d]: iMem=%d\n",
drhbf790972020-06-07 20:18:07 +00007439 ii, pAggInfo->aFunc[ii].iMem);
drh81185a52020-06-09 13:38:12 +00007440 sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0);
drh7ea11062017-10-26 18:43:19 +00007441 }
7442 }
7443#endif
7444
drh13449892005-09-07 21:22:45 +00007445
7446 /* Processing for aggregates with GROUP BY is very different and
danielk19773c4809a2007-11-12 15:29:18 +00007447 ** much more complex than aggregates without a GROUP BY.
drh13449892005-09-07 21:22:45 +00007448 */
7449 if( pGroupBy ){
7450 KeyInfo *pKeyInfo; /* Keying information for the group by clause */
drh728e0f92015-10-10 14:41:28 +00007451 int addr1; /* A-vs-B comparision jump */
drhd1766112008-09-17 00:13:12 +00007452 int addrOutputRow; /* Start of subroutine that outputs a result row */
7453 int regOutputRow; /* Return address register for output subroutine */
7454 int addrSetAbort; /* Set the abort flag and return */
7455 int addrTopOfLoop; /* Top of the input loop */
7456 int addrSortingIdx; /* The OP_OpenEphemeral for the sorting index */
7457 int addrReset; /* Subroutine for resetting the accumulator */
7458 int regReset; /* Return address register for reset subroutine */
dan5383db72021-03-12 18:24:31 +00007459 ExprList *pDistinct = 0;
7460 u16 distFlag = 0;
7461 int eDist = WHERE_DISTINCT_NOOP;
7462
7463 if( pAggInfo->nFunc==1
7464 && pAggInfo->aFunc[0].iDistinct>=0
drha4eeccd2021-10-07 17:43:30 +00007465 && ALWAYS(pAggInfo->aFunc[0].pFExpr!=0)
7466 && ALWAYS(ExprUseXList(pAggInfo->aFunc[0].pFExpr))
7467 && pAggInfo->aFunc[0].pFExpr->x.pList!=0
dan5383db72021-03-12 18:24:31 +00007468 ){
7469 Expr *pExpr = pAggInfo->aFunc[0].pFExpr->x.pList->a[0].pExpr;
7470 pExpr = sqlite3ExprDup(db, pExpr, 0);
7471 pDistinct = sqlite3ExprListDup(db, pGroupBy, 0);
7472 pDistinct = sqlite3ExprListAppend(pParse, pDistinct, pExpr);
danf330d532021-04-03 19:23:59 +00007473 distFlag = pDistinct ? (WHERE_WANT_DISTINCT|WHERE_AGG_DISTINCT) : 0;
dan5383db72021-03-12 18:24:31 +00007474 }
drh13449892005-09-07 21:22:45 +00007475
7476 /* If there is a GROUP BY clause we might need a sorting index to
7477 ** implement it. Allocate that sorting index now. If it turns out
drh1c9d8352011-09-01 16:01:27 +00007478 ** that we do not need it after all, the OP_SorterOpen instruction
drh13449892005-09-07 21:22:45 +00007479 ** will be converted into a Noop.
7480 */
drhbf790972020-06-07 20:18:07 +00007481 pAggInfo->sortingIdx = pParse->nTab++;
7482 pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pGroupBy,
7483 0, pAggInfo->nColumn);
drh1c9d8352011-09-01 16:01:27 +00007484 addrSortingIdx = sqlite3VdbeAddOp4(v, OP_SorterOpen,
drhbf790972020-06-07 20:18:07 +00007485 pAggInfo->sortingIdx, pAggInfo->nSortingColumn,
drh2ec2fb22013-11-06 19:59:23 +00007486 0, (char*)pKeyInfo, P4_KEYINFO);
drh13449892005-09-07 21:22:45 +00007487
7488 /* Initialize memory locations used by GROUP BY aggregate processing
7489 */
drh0a07c102008-01-03 18:03:08 +00007490 iUseFlag = ++pParse->nMem;
7491 iAbortFlag = ++pParse->nMem;
drhd1766112008-09-17 00:13:12 +00007492 regOutputRow = ++pParse->nMem;
drhec4ccdb2018-12-29 02:26:59 +00007493 addrOutputRow = sqlite3VdbeMakeLabel(pParse);
drhd1766112008-09-17 00:13:12 +00007494 regReset = ++pParse->nMem;
drhec4ccdb2018-12-29 02:26:59 +00007495 addrReset = sqlite3VdbeMakeLabel(pParse);
drh0a07c102008-01-03 18:03:08 +00007496 iAMem = pParse->nMem + 1;
drh13449892005-09-07 21:22:45 +00007497 pParse->nMem += pGroupBy->nExpr;
drh0a07c102008-01-03 18:03:08 +00007498 iBMem = pParse->nMem + 1;
drh13449892005-09-07 21:22:45 +00007499 pParse->nMem += pGroupBy->nExpr;
drh4c583122008-01-04 22:01:03 +00007500 sqlite3VdbeAddOp2(v, OP_Integer, 0, iAbortFlag);
drhd4e70eb2008-01-02 00:34:36 +00007501 VdbeComment((v, "clear abort flag"));
drhb8475df2011-12-09 16:21:19 +00007502 sqlite3VdbeAddOp3(v, OP_Null, 0, iAMem, iAMem+pGroupBy->nExpr-1);
drhe3133822005-09-20 13:11:59 +00007503
drh13449892005-09-07 21:22:45 +00007504 /* Begin a loop that will extract all source rows in GROUP BY order.
7505 ** This might involve two separate loops with an OP_Sort in between, or
7506 ** it might be a single loop that uses an index to extract information
7507 ** in the right order to begin with.
7508 */
drh2eb95372008-06-06 15:04:36 +00007509 sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset);
drhcfd74702018-03-19 22:28:34 +00007510 SELECTTRACE(1,pParse,p,("WhereBegin\n"));
dan5383db72021-03-12 18:24:31 +00007511 pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, pDistinct,
danece092e2022-03-16 12:06:00 +00007512 0, (sDistinct.isTnct==2 ? WHERE_DISTINCTBY : WHERE_GROUPBY)
7513 | (orderByGrp ? WHERE_SORTBYGROUP : 0) | distFlag, 0
dan374cd782014-04-21 13:21:56 +00007514 );
danbfd6f1b2021-04-08 20:29:12 +00007515 if( pWInfo==0 ){
7516 sqlite3ExprListDelete(db, pDistinct);
7517 goto select_end;
7518 }
dan76075802021-03-12 21:09:20 +00007519 eDist = sqlite3WhereIsDistinct(pWInfo);
drh13162ef2021-02-16 20:01:08 +00007520 SELECTTRACE(1,pParse,p,("WhereBegin returns\n"));
drhddba0c22014-03-18 20:33:42 +00007521 if( sqlite3WhereIsOrdered(pWInfo)==pGroupBy->nExpr ){
drh13449892005-09-07 21:22:45 +00007522 /* The optimizer is able to deliver rows in group by order so
drhb9bb7c12006-06-11 23:41:55 +00007523 ** we do not have to sort. The OP_OpenEphemeral table will be
drh13449892005-09-07 21:22:45 +00007524 ** cancelled later because we still need to use the pKeyInfo
7525 */
drh13449892005-09-07 21:22:45 +00007526 groupBySort = 0;
7527 }else{
7528 /* Rows are coming out in undetermined order. We have to push
7529 ** each row into a sorting index, terminate the first loop,
7530 ** then loop over the sorting index in order to get the output
7531 ** in sorted order
7532 */
drh892d3172008-01-10 03:46:36 +00007533 int regBase;
7534 int regRecord;
7535 int nCol;
7536 int nGroupBy;
7537
dan2ce22452010-11-08 19:01:16 +00007538 explainTempTable(pParse,
drhe8e4af72012-09-21 00:04:28 +00007539 (sDistinct.isTnct && (p->selFlags&SF_Distinct)==0) ?
7540 "DISTINCT" : "GROUP BY");
dan2ce22452010-11-08 19:01:16 +00007541
drh13449892005-09-07 21:22:45 +00007542 groupBySort = 1;
drh892d3172008-01-10 03:46:36 +00007543 nGroupBy = pGroupBy->nExpr;
dandd23c6b2014-03-24 20:19:07 +00007544 nCol = nGroupBy;
7545 j = nGroupBy;
drhbf790972020-06-07 20:18:07 +00007546 for(i=0; i<pAggInfo->nColumn; i++){
7547 if( pAggInfo->aCol[i].iSorterColumn>=j ){
drh892d3172008-01-10 03:46:36 +00007548 nCol++;
7549 j++;
7550 }
7551 }
7552 regBase = sqlite3GetTempRange(pParse, nCol);
drh5579d592015-08-26 14:01:41 +00007553 sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0, 0);
dandd23c6b2014-03-24 20:19:07 +00007554 j = nGroupBy;
drhee373022022-07-25 15:54:23 +00007555 pAggInfo->directMode = 1;
drhbf790972020-06-07 20:18:07 +00007556 for(i=0; i<pAggInfo->nColumn; i++){
7557 struct AggInfo_col *pCol = &pAggInfo->aCol[i];
drh892d3172008-01-10 03:46:36 +00007558 if( pCol->iSorterColumn>=j ){
drhee373022022-07-25 15:54:23 +00007559 sqlite3ExprCode(pParse, pCol->pCExpr, j + regBase);
drh6a012f02008-08-21 14:15:59 +00007560 j++;
drh892d3172008-01-10 03:46:36 +00007561 }
drh13449892005-09-07 21:22:45 +00007562 }
drhee373022022-07-25 15:54:23 +00007563 pAggInfo->directMode = 0;
drh892d3172008-01-10 03:46:36 +00007564 regRecord = sqlite3GetTempReg(pParse);
drh1db639c2008-01-17 02:36:28 +00007565 sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regRecord);
drhbf790972020-06-07 20:18:07 +00007566 sqlite3VdbeAddOp2(v, OP_SorterInsert, pAggInfo->sortingIdx, regRecord);
drh892d3172008-01-10 03:46:36 +00007567 sqlite3ReleaseTempReg(pParse, regRecord);
7568 sqlite3ReleaseTempRange(pParse, regBase, nCol);
drh13162ef2021-02-16 20:01:08 +00007569 SELECTTRACE(1,pParse,p,("WhereEnd\n"));
drh13449892005-09-07 21:22:45 +00007570 sqlite3WhereEnd(pWInfo);
drhbf790972020-06-07 20:18:07 +00007571 pAggInfo->sortingIdxPTab = sortPTab = pParse->nTab++;
drh1c9d8352011-09-01 16:01:27 +00007572 sortOut = sqlite3GetTempReg(pParse);
7573 sqlite3VdbeAddOp3(v, OP_OpenPseudo, sortPTab, sortOut, nCol);
drhbf790972020-06-07 20:18:07 +00007574 sqlite3VdbeAddOp2(v, OP_SorterSort, pAggInfo->sortingIdx, addrEnd);
drh688852a2014-02-17 22:40:43 +00007575 VdbeComment((v, "GROUP BY sort")); VdbeCoverage(v);
drhbf790972020-06-07 20:18:07 +00007576 pAggInfo->useSortingIdx = 1;
dan374cd782014-04-21 13:21:56 +00007577 }
7578
7579 /* If the index or temporary table used by the GROUP BY sort
7580 ** will naturally deliver rows in the order required by the ORDER BY
7581 ** clause, cancel the ephemeral table open coded earlier.
7582 **
7583 ** This is an optimization - the correct answer should result regardless.
7584 ** Use the SQLITE_GroupByOrder flag with SQLITE_TESTCTRL_OPTIMIZER to
7585 ** disable this optimization for testing purposes. */
7586 if( orderByGrp && OptimizationEnabled(db, SQLITE_GroupByOrder)
7587 && (groupBySort || sqlite3WhereIsSorted(pWInfo))
7588 ){
7589 sSort.pOrderBy = 0;
7590 sqlite3VdbeChangeToNoop(v, sSort.addrSortIndex);
drh13449892005-09-07 21:22:45 +00007591 }
7592
7593 /* Evaluate the current GROUP BY terms and store in b0, b1, b2...
7594 ** (b0 is memory location iBMem+0, b1 is iBMem+1, and so forth)
7595 ** Then compare the current GROUP BY terms against the GROUP BY terms
7596 ** from the previous row currently stored in a0, a1, a2...
7597 */
7598 addrTopOfLoop = sqlite3VdbeCurrentAddr(v);
drh1c9d8352011-09-01 16:01:27 +00007599 if( groupBySort ){
drhbf790972020-06-07 20:18:07 +00007600 sqlite3VdbeAddOp3(v, OP_SorterData, pAggInfo->sortingIdx,
drh38b41492015-06-08 15:08:15 +00007601 sortOut, sortPTab);
drh1c9d8352011-09-01 16:01:27 +00007602 }
drh13449892005-09-07 21:22:45 +00007603 for(j=0; j<pGroupBy->nExpr; j++){
7604 if( groupBySort ){
drh1c9d8352011-09-01 16:01:27 +00007605 sqlite3VdbeAddOp3(v, OP_Column, sortPTab, j, iBMem+j);
drh13449892005-09-07 21:22:45 +00007606 }else{
drhbf790972020-06-07 20:18:07 +00007607 pAggInfo->directMode = 1;
drh2dcef112008-01-12 19:03:48 +00007608 sqlite3ExprCode(pParse, pGroupBy->a[j].pExpr, iBMem+j);
drh13449892005-09-07 21:22:45 +00007609 }
drh13449892005-09-07 21:22:45 +00007610 }
drh16ee60f2008-06-20 18:13:25 +00007611 sqlite3VdbeAddOp4(v, OP_Compare, iAMem, iBMem, pGroupBy->nExpr,
drh2ec2fb22013-11-06 19:59:23 +00007612 (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO);
drh728e0f92015-10-10 14:41:28 +00007613 addr1 = sqlite3VdbeCurrentAddr(v);
7614 sqlite3VdbeAddOp3(v, OP_Jump, addr1+1, 0, addr1+1); VdbeCoverage(v);
drh13449892005-09-07 21:22:45 +00007615
7616 /* Generate code that runs whenever the GROUP BY changes.
drhe00ee6e2008-06-20 15:24:01 +00007617 ** Changes in the GROUP BY are detected by the previous code
drh13449892005-09-07 21:22:45 +00007618 ** block. If there were no changes, this block is skipped.
7619 **
7620 ** This code copies current group by terms in b0,b1,b2,...
7621 ** over to a0,a1,a2. It then calls the output subroutine
7622 ** and resets the aggregate accumulator registers in preparation
7623 ** for the next GROUP BY batch.
7624 */
drhb21e7c72008-06-22 12:37:57 +00007625 sqlite3ExprCodeMove(pParse, iBMem, iAMem, pGroupBy->nExpr);
drh2eb95372008-06-06 15:04:36 +00007626 sqlite3VdbeAddOp2(v, OP_Gosub, regOutputRow, addrOutputRow);
drhd4e70eb2008-01-02 00:34:36 +00007627 VdbeComment((v, "output one row"));
drh688852a2014-02-17 22:40:43 +00007628 sqlite3VdbeAddOp2(v, OP_IfPos, iAbortFlag, addrEnd); VdbeCoverage(v);
drhd4e70eb2008-01-02 00:34:36 +00007629 VdbeComment((v, "check abort flag"));
drh2eb95372008-06-06 15:04:36 +00007630 sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset);
drhd4e70eb2008-01-02 00:34:36 +00007631 VdbeComment((v, "reset accumulator"));
drh13449892005-09-07 21:22:45 +00007632
7633 /* Update the aggregate accumulators based on the content of
7634 ** the current row
7635 */
drh728e0f92015-10-10 14:41:28 +00007636 sqlite3VdbeJumpHere(v, addr1);
dan5383db72021-03-12 18:24:31 +00007637 updateAccumulator(pParse, iUseFlag, pAggInfo, eDist);
drh4c583122008-01-04 22:01:03 +00007638 sqlite3VdbeAddOp2(v, OP_Integer, 1, iUseFlag);
drhd4e70eb2008-01-02 00:34:36 +00007639 VdbeComment((v, "indicate data in accumulator"));
drh13449892005-09-07 21:22:45 +00007640
7641 /* End of the loop
7642 */
7643 if( groupBySort ){
drhd1930572021-01-13 15:23:17 +00007644 sqlite3VdbeAddOp2(v, OP_SorterNext, pAggInfo->sortingIdx,addrTopOfLoop);
drh688852a2014-02-17 22:40:43 +00007645 VdbeCoverage(v);
drh13449892005-09-07 21:22:45 +00007646 }else{
drh13162ef2021-02-16 20:01:08 +00007647 SELECTTRACE(1,pParse,p,("WhereEnd\n"));
drh13449892005-09-07 21:22:45 +00007648 sqlite3WhereEnd(pWInfo);
drh48f2d3b2011-09-16 01:34:43 +00007649 sqlite3VdbeChangeToNoop(v, addrSortingIdx);
drh13449892005-09-07 21:22:45 +00007650 }
danbfd6f1b2021-04-08 20:29:12 +00007651 sqlite3ExprListDelete(db, pDistinct);
drh13449892005-09-07 21:22:45 +00007652
7653 /* Output the final row of result
7654 */
drh2eb95372008-06-06 15:04:36 +00007655 sqlite3VdbeAddOp2(v, OP_Gosub, regOutputRow, addrOutputRow);
drhd4e70eb2008-01-02 00:34:36 +00007656 VdbeComment((v, "output final row"));
drhd1766112008-09-17 00:13:12 +00007657
7658 /* Jump over the subroutines
7659 */
drh076e85f2015-09-03 13:46:12 +00007660 sqlite3VdbeGoto(v, addrEnd);
drhd1766112008-09-17 00:13:12 +00007661
7662 /* Generate a subroutine that outputs a single row of the result
7663 ** set. This subroutine first looks at the iUseFlag. If iUseFlag
7664 ** is less than or equal to zero, the subroutine is a no-op. If
7665 ** the processing calls for the query to abort, this subroutine
7666 ** increments the iAbortFlag memory location before returning in
7667 ** order to signal the caller to abort.
7668 */
7669 addrSetAbort = sqlite3VdbeCurrentAddr(v);
7670 sqlite3VdbeAddOp2(v, OP_Integer, 1, iAbortFlag);
7671 VdbeComment((v, "set abort flag"));
7672 sqlite3VdbeAddOp1(v, OP_Return, regOutputRow);
7673 sqlite3VdbeResolveLabel(v, addrOutputRow);
7674 addrOutputRow = sqlite3VdbeCurrentAddr(v);
7675 sqlite3VdbeAddOp2(v, OP_IfPos, iUseFlag, addrOutputRow+2);
drh38b41492015-06-08 15:08:15 +00007676 VdbeCoverage(v);
drhd1766112008-09-17 00:13:12 +00007677 VdbeComment((v, "Groupby result generator entry point"));
7678 sqlite3VdbeAddOp1(v, OP_Return, regOutputRow);
drhbf790972020-06-07 20:18:07 +00007679 finalizeAggFunctions(pParse, pAggInfo);
drh33cd4902009-05-30 20:49:20 +00007680 sqlite3ExprIfFalse(pParse, pHaving, addrOutputRow+1, SQLITE_JUMPIFNULL);
drh2def2f72017-09-15 17:40:34 +00007681 selectInnerLoop(pParse, p, -1, &sSort,
drhe8e4af72012-09-21 00:04:28 +00007682 &sDistinct, pDest,
drhd1766112008-09-17 00:13:12 +00007683 addrOutputRow+1, addrSetAbort);
7684 sqlite3VdbeAddOp1(v, OP_Return, regOutputRow);
7685 VdbeComment((v, "end groupby result generator"));
7686
7687 /* Generate a subroutine that will reset the group-by accumulator
7688 */
7689 sqlite3VdbeResolveLabel(v, addrReset);
drhbf790972020-06-07 20:18:07 +00007690 resetAccumulator(pParse, pAggInfo);
dan280c8942018-06-05 20:04:28 +00007691 sqlite3VdbeAddOp2(v, OP_Integer, 0, iUseFlag);
7692 VdbeComment((v, "indicate accumulator empty"));
drhd1766112008-09-17 00:13:12 +00007693 sqlite3VdbeAddOp1(v, OP_Return, regReset);
dan5383db72021-03-12 18:24:31 +00007694
danece092e2022-03-16 12:06:00 +00007695 if( distFlag!=0 && eDist!=WHERE_DISTINCT_NOOP ){
dan5383db72021-03-12 18:24:31 +00007696 struct AggInfo_func *pF = &pAggInfo->aFunc[0];
7697 fixDistinctOpenEph(pParse, eDist, pF->iDistinct, pF->iDistAddr);
7698 }
drh43152cf2009-05-19 19:04:58 +00007699 } /* endif pGroupBy. Begin aggregate queries without GROUP BY: */
drh13449892005-09-07 21:22:45 +00007700 else {
danielk1977a5533162009-02-24 10:01:51 +00007701 Table *pTab;
drhbf790972020-06-07 20:18:07 +00007702 if( (pTab = isSimpleCount(p, pAggInfo))!=0 ){
danielk1977a5533162009-02-24 10:01:51 +00007703 /* If isSimpleCount() returns a pointer to a Table structure, then
7704 ** the SQL statement is of the form:
7705 **
7706 ** SELECT count(*) FROM <tbl>
7707 **
7708 ** where the Table structure returned represents table <tbl>.
7709 **
7710 ** This statement is so common that it is optimized specially. The
7711 ** OP_Count instruction is executed either on the intkey table that
7712 ** contains the data for table <tbl> or on one of its indexes. It
7713 ** is better to execute the op on an index, as indexes are almost
7714 ** always spread across less pages than their corresponding tables.
7715 */
7716 const int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema);
7717 const int iCsr = pParse->nTab++; /* Cursor to scan b-tree */
7718 Index *pIdx; /* Iterator variable */
7719 KeyInfo *pKeyInfo = 0; /* Keyinfo for scanned index */
7720 Index *pBest = 0; /* Best index found so far */
drh013e7bb2020-07-30 17:37:49 +00007721 Pgno iRoot = pTab->tnum; /* Root page of scanned b-tree */
danielk1977a9d1ccb2008-01-05 17:39:29 +00007722
danielk1977a5533162009-02-24 10:01:51 +00007723 sqlite3CodeVerifySchema(pParse, iDb);
7724 sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName);
7725
drhd9e3cad2013-10-04 02:36:19 +00007726 /* Search for the index that has the lowest scan cost.
danielk1977a5533162009-02-24 10:01:51 +00007727 **
drh3e9548b2011-04-15 14:46:27 +00007728 ** (2011-04-15) Do not do a full scan of an unordered index.
7729 **
drhabcc1942013-11-12 14:55:40 +00007730 ** (2013-10-03) Do not count the entries in a partial index.
drh5f33f372013-10-04 00:00:12 +00007731 **
danielk1977a5533162009-02-24 10:01:51 +00007732 ** In practice the KeyInfo structure will not be used. It is only
7733 ** passed to keep OP_OpenRead happy.
7734 */
drh5c7917e2013-11-12 15:33:40 +00007735 if( !HasRowid(pTab) ) pBest = sqlite3PrimaryKeyIndex(pTab);
drh98aa1d72020-05-08 18:22:00 +00007736 if( !p->pSrc->a[0].fg.notIndexed ){
7737 for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
7738 if( pIdx->bUnordered==0
7739 && pIdx->szIdxRow<pTab->szTabRow
7740 && pIdx->pPartIdxWhere==0
7741 && (!pBest || pIdx->szIdxRow<pBest->szIdxRow)
7742 ){
7743 pBest = pIdx;
7744 }
danielk1977a5533162009-02-24 10:01:51 +00007745 }
7746 }
drhd9e3cad2013-10-04 02:36:19 +00007747 if( pBest ){
danielk1977a5533162009-02-24 10:01:51 +00007748 iRoot = pBest->tnum;
drh2ec2fb22013-11-06 19:59:23 +00007749 pKeyInfo = sqlite3KeyInfoOfIndex(pParse, pBest);
danielk1977a5533162009-02-24 10:01:51 +00007750 }
7751
7752 /* Open a read-only cursor, execute the OP_Count, close the cursor. */
drh013e7bb2020-07-30 17:37:49 +00007753 sqlite3VdbeAddOp4Int(v, OP_OpenRead, iCsr, (int)iRoot, iDb, 1);
danielk1977a5533162009-02-24 10:01:51 +00007754 if( pKeyInfo ){
drh2ec2fb22013-11-06 19:59:23 +00007755 sqlite3VdbeChangeP4(v, -1, (char *)pKeyInfo, P4_KEYINFO);
danielk1977a5533162009-02-24 10:01:51 +00007756 }
drhbf790972020-06-07 20:18:07 +00007757 sqlite3VdbeAddOp2(v, OP_Count, iCsr, pAggInfo->aFunc[0].iMem);
danielk1977a5533162009-02-24 10:01:51 +00007758 sqlite3VdbeAddOp1(v, OP_Close, iCsr);
danef7075d2011-02-21 17:49:49 +00007759 explainSimpleCount(pParse, pTab, pBest);
drhe50478d2020-03-17 13:41:51 +00007760 }else{
dan280c8942018-06-05 20:04:28 +00007761 int regAcc = 0; /* "populate accumulators" flag */
dan4fcb30b2021-03-09 16:06:25 +00007762 ExprList *pDistinct = 0;
7763 u16 distFlag = 0;
dan5383db72021-03-12 18:24:31 +00007764 int eDist;
dan280c8942018-06-05 20:04:28 +00007765
daned09ddd2019-09-20 20:52:16 +00007766 /* If there are accumulator registers but no min() or max() functions
7767 ** without FILTER clauses, allocate register regAcc. Register regAcc
7768 ** will contain 0 the first time the inner loop runs, and 1 thereafter.
7769 ** The code generated by updateAccumulator() uses this to ensure
7770 ** that the accumulator registers are (a) updated only once if
7771 ** there are no min() or max functions or (b) always updated for the
7772 ** first row visited by the aggregate, so that they are updated at
7773 ** least once even if the FILTER clause means the min() or max()
7774 ** function visits zero rows. */
drhbf790972020-06-07 20:18:07 +00007775 if( pAggInfo->nAccumulator ){
7776 for(i=0; i<pAggInfo->nFunc; i++){
drh81185a52020-06-09 13:38:12 +00007777 if( ExprHasProperty(pAggInfo->aFunc[i].pFExpr, EP_WinFunc) ){
drhbf790972020-06-07 20:18:07 +00007778 continue;
7779 }
7780 if( pAggInfo->aFunc[i].pFunc->funcFlags&SQLITE_FUNC_NEEDCOLL ){
7781 break;
7782 }
dan280c8942018-06-05 20:04:28 +00007783 }
drhbf790972020-06-07 20:18:07 +00007784 if( i==pAggInfo->nFunc ){
dan280c8942018-06-05 20:04:28 +00007785 regAcc = ++pParse->nMem;
7786 sqlite3VdbeAddOp2(v, OP_Integer, 0, regAcc);
7787 }
dan4fcb30b2021-03-09 16:06:25 +00007788 }else if( pAggInfo->nFunc==1 && pAggInfo->aFunc[0].iDistinct>=0 ){
drha4eeccd2021-10-07 17:43:30 +00007789 assert( ExprUseXList(pAggInfo->aFunc[0].pFExpr) );
dan4fcb30b2021-03-09 16:06:25 +00007790 pDistinct = pAggInfo->aFunc[0].pFExpr->x.pList;
danf330d532021-04-03 19:23:59 +00007791 distFlag = pDistinct ? (WHERE_WANT_DISTINCT|WHERE_AGG_DISTINCT) : 0;
dan280c8942018-06-05 20:04:28 +00007792 }
7793
danielk1977a5533162009-02-24 10:01:51 +00007794 /* This case runs if the aggregate has no GROUP BY clause. The
7795 ** processing is much simpler since there is only a single row
7796 ** of output.
7797 */
drh47d9f832017-10-26 20:04:28 +00007798 assert( p->pGroupBy==0 );
drhbf790972020-06-07 20:18:07 +00007799 resetAccumulator(pParse, pAggInfo);
drh47d9f832017-10-26 20:04:28 +00007800
7801 /* If this query is a candidate for the min/max optimization, then
7802 ** minMaxFlag will have been previously set to either
7803 ** WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX and pMinMaxOrderBy will
7804 ** be an appropriate ORDER BY expression for the optimization.
7805 */
7806 assert( minMaxFlag==WHERE_ORDERBY_NORMAL || pMinMaxOrderBy!=0 );
7807 assert( pMinMaxOrderBy==0 || pMinMaxOrderBy->nExpr==1 );
7808
drhcfd74702018-03-19 22:28:34 +00007809 SELECTTRACE(1,pParse,p,("WhereBegin\n"));
drh47d9f832017-10-26 20:04:28 +00007810 pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMaxOrderBy,
drh895bab32022-01-27 16:14:50 +00007811 pDistinct, 0, minMaxFlag|distFlag, 0);
danielk1977a5533162009-02-24 10:01:51 +00007812 if( pWInfo==0 ){
danielk1977a5533162009-02-24 10:01:51 +00007813 goto select_end;
7814 }
drh13162ef2021-02-16 20:01:08 +00007815 SELECTTRACE(1,pParse,p,("WhereBegin returns\n"));
dan5383db72021-03-12 18:24:31 +00007816 eDist = sqlite3WhereIsDistinct(pWInfo);
7817 updateAccumulator(pParse, regAcc, pAggInfo, eDist);
7818 if( eDist!=WHERE_DISTINCT_NOOP ){
drhde695ea2022-05-10 15:55:10 +00007819 struct AggInfo_func *pF = pAggInfo->aFunc;
drh3117b7b2022-04-19 18:23:01 +00007820 if( pF ){
7821 fixDistinctOpenEph(pParse, eDist, pF->iDistinct, pF->iDistAddr);
7822 }
dan5383db72021-03-12 18:24:31 +00007823 }
7824
dan280c8942018-06-05 20:04:28 +00007825 if( regAcc ) sqlite3VdbeAddOp2(v, OP_Integer, 1, regAcc);
drhd1930572021-01-13 15:23:17 +00007826 if( minMaxFlag ){
7827 sqlite3WhereMinMaxOptEarlyOut(v, pWInfo);
danielk1977a5533162009-02-24 10:01:51 +00007828 }
drh13162ef2021-02-16 20:01:08 +00007829 SELECTTRACE(1,pParse,p,("WhereEnd\n"));
danielk1977a5533162009-02-24 10:01:51 +00007830 sqlite3WhereEnd(pWInfo);
drhbf790972020-06-07 20:18:07 +00007831 finalizeAggFunctions(pParse, pAggInfo);
danielk1977a9d1ccb2008-01-05 17:39:29 +00007832 }
7833
drh079a3072014-03-19 14:10:55 +00007834 sSort.pOrderBy = 0;
drh33cd4902009-05-30 20:49:20 +00007835 sqlite3ExprIfFalse(pParse, pHaving, addrEnd, SQLITE_JUMPIFNULL);
drh2def2f72017-09-15 17:40:34 +00007836 selectInnerLoop(pParse, p, -1, 0, 0,
drha9671a22008-07-08 23:40:20 +00007837 pDest, addrEnd, addrEnd);
drh13449892005-09-07 21:22:45 +00007838 }
7839 sqlite3VdbeResolveLabel(v, addrEnd);
7840
7841 } /* endif aggregate query */
drh22827922000-06-06 17:27:05 +00007842
drhe8e4af72012-09-21 00:04:28 +00007843 if( sDistinct.eTnctType==WHERE_DISTINCT_UNORDERED ){
dan2ce22452010-11-08 19:01:16 +00007844 explainTempTable(pParse, "DISTINCT");
7845 }
7846
drhcce7d172000-05-31 15:34:51 +00007847 /* If there is an ORDER BY clause, then we need to sort the results
7848 ** and send them to the callback one by one.
7849 */
drh079a3072014-03-19 14:10:55 +00007850 if( sSort.pOrderBy ){
drh38b41492015-06-08 15:08:15 +00007851 explainTempTable(pParse,
7852 sSort.nOBSat>0 ? "RIGHT PART OF ORDER BY":"ORDER BY");
dan24e25d32018-04-14 18:46:20 +00007853 assert( p->pEList==pEList );
drh079a3072014-03-19 14:10:55 +00007854 generateSortTail(pParse, p, &sSort, pEList->nExpr, pDest);
drhcce7d172000-05-31 15:34:51 +00007855 }
drh6a535342001-10-19 16:44:56 +00007856
drhec7429a2005-10-06 16:53:14 +00007857 /* Jump here to skip this query
7858 */
7859 sqlite3VdbeResolveLabel(v, iEnd);
7860
dan5b1c07e2015-04-16 07:19:23 +00007861 /* The SELECT has been coded. If there is an error in the Parse structure,
7862 ** set the return code to 1. Otherwise 0. */
7863 rc = (pParse->nErr>0);
drh1d83f052002-02-17 00:30:36 +00007864
7865 /* Control jumps to here if an error is encountered above, or upon
7866 ** successful coding of the SELECT.
7867 */
7868select_end:
dan44918c72021-04-17 14:42:37 +00007869 assert( db->mallocFailed==0 || db->mallocFailed==1 );
drh0c7d3d32022-01-24 16:47:12 +00007870 assert( db->mallocFailed==0 || pParse->nErr!=0 );
drh47d9f832017-10-26 20:04:28 +00007871 sqlite3ExprListDelete(db, pMinMaxOrderBy);
drh89636622020-06-07 17:33:18 +00007872#ifdef SQLITE_DEBUG
drh7b4c4d42020-06-10 03:07:26 +00007873 if( pAggInfo && !db->mallocFailed ){
drhbf790972020-06-07 20:18:07 +00007874 for(i=0; i<pAggInfo->nColumn; i++){
drh81185a52020-06-09 13:38:12 +00007875 Expr *pExpr = pAggInfo->aCol[i].pCExpr;
drh06ddb082020-12-22 14:54:20 +00007876 assert( pExpr!=0 );
drhbf790972020-06-07 20:18:07 +00007877 assert( pExpr->pAggInfo==pAggInfo );
drh89636622020-06-07 17:33:18 +00007878 assert( pExpr->iAgg==i );
7879 }
drhbf790972020-06-07 20:18:07 +00007880 for(i=0; i<pAggInfo->nFunc; i++){
drh81185a52020-06-09 13:38:12 +00007881 Expr *pExpr = pAggInfo->aFunc[i].pFExpr;
drh06ddb082020-12-22 14:54:20 +00007882 assert( pExpr!=0 );
drhbf790972020-06-07 20:18:07 +00007883 assert( pExpr->pAggInfo==pAggInfo );
drh89636622020-06-07 17:33:18 +00007884 assert( pExpr->iAgg==i );
7885 }
drh89636622020-06-07 17:33:18 +00007886 }
drh89636622020-06-07 17:33:18 +00007887#endif
7888
drh5e431be2022-04-06 11:08:38 +00007889#if TREETRACE_ENABLED
drhf20609d2018-04-23 17:43:35 +00007890 SELECTTRACE(0x1,pParse,p,("end processing\n"));
drh5e431be2022-04-06 11:08:38 +00007891 if( (sqlite3TreeTrace & 0x2000)!=0 && ExplainQueryPlanParent(pParse)==0 ){
drhf20609d2018-04-23 17:43:35 +00007892 sqlite3TreeViewSelect(0, p, 0);
7893 }
drheb9b8842014-09-21 00:27:26 +00007894#endif
drhe2ca99c2018-05-02 00:33:43 +00007895 ExplainQueryPlanPop(pParse);
drh1d83f052002-02-17 00:30:36 +00007896 return rc;
drhcce7d172000-05-31 15:34:51 +00007897}