Record within the Token structure itself whether or not the token has
been dequoted. This steals one bit from the length of a token and
thus limits the size of tokens to 1GiB. (CVS 6589)
FossilOrigin-Name: 12bcb03d9b9e1a31c1a3c67cbb4263cc0af2f3d0
diff --git a/src/build.c b/src/build.c
index f6e325a..0122259 100644
--- a/src/build.c
+++ b/src/build.c
@@ -22,7 +22,7 @@
** COMMIT
** ROLLBACK
**
-** $Id: build.c,v 1.532 2009/04/28 13:01:09 drh Exp $
+** $Id: build.c,v 1.533 2009/05/01 21:13:37 drh Exp $
*/
#include "sqliteInt.h"
@@ -579,10 +579,13 @@
/*
** Given a token, return a string that consists of the text of that
-** token with any quotations removed. Space to hold the returned string
+** token. Space to hold the returned string
** is obtained from sqliteMalloc() and must be freed by the calling
** function.
**
+** Any quotation marks (ex: "name", 'name', [name], or `name`) that
+** surround the body of the token are removed.
+**
** Tokens are often just pointers into the original SQL text and so
** are not \000 terminated and are not persistent. The returned string
** is \000 terminated and is persistent.
@@ -591,7 +594,7 @@
char *zName;
if( pName ){
zName = sqlite3DbStrNDup(db, (char*)pName->z, pName->n);
- sqlite3Dequote(zName);
+ if( pName->quoted ) sqlite3Dequote(zName);
}else{
zName = 0;
}
diff --git a/src/delete.c b/src/delete.c
index b0a05a4..cf0a36e 100644
--- a/src/delete.c
+++ b/src/delete.c
@@ -12,7 +12,7 @@
** This file contains C code routines that are called by the parser
** in order to generate code for DELETE FROM statements.
**
-** $Id: delete.c,v 1.200 2009/04/30 00:11:10 drh Exp $
+** $Id: delete.c,v 1.201 2009/05/01 21:13:37 drh Exp $
*/
#include "sqliteInt.h"
@@ -87,6 +87,7 @@
pWhere = sqlite3ExprDup(db, pWhere, 0);
viewName.z = (u8*)pView->zName;
viewName.n = (unsigned int)sqlite3Strlen30((const char*)viewName.z);
+ viewName.quoted = 0;
pFrom = sqlite3SrcListAppendFromTerm(pParse, 0, 0, 0, &viewName, pDup, 0,0);
pDup = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, 0, 0, 0);
}
diff --git a/src/expr.c b/src/expr.c
index ff5c4bd..fa15cad 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -12,7 +12,7 @@
** This file contains routines used for analyzing expressions and
** for generating VDBE code that evaluates expressions in SQLite.
**
-** $Id: expr.c,v 1.430 2009/04/28 12:08:15 danielk1977 Exp $
+** $Id: expr.c,v 1.431 2009/05/01 21:13:37 drh Exp $
*/
#include "sqliteInt.h"
@@ -407,21 +407,18 @@
int c;
assert( pToken->dyn==0 );
pNew->span = *pToken;
-
- /* The pToken->z value is read-only. But the new expression
- ** node created here might be passed to sqlite3DequoteExpr() which
- ** will attempt to modify pNew->token.z. Hence, if the token
- ** is quoted, make a copy now so that DequoteExpr() will change
- ** the copy rather than the original text.
- */
if( pToken->n>=2
&& ((c = pToken->z[0])=='\'' || c=='"' || c=='[' || c=='`') ){
sqlite3TokenCopy(db, &pNew->token, pToken);
+ if( pNew->token.z ){
+ pNew->token.n = sqlite3Dequote((char*)pNew->token.z);
+ assert( pNew->token.n==sqlite3Strlen30((char*)pNew->token.z) );
+ }
+ if( c=='"' ) pNew->flags |= EP_DblQuoted;
}else{
pNew->token = *pToken;
- pNew->flags |= EP_Dequoted;
- VVA_ONLY( pNew->vvaFlags |= EVVA_ReadOnlyToken; )
}
+ pNew->token.quoted = 0;
}else if( pLeft ){
if( pRight ){
if( pRight->span.dyn==0 && pLeft->span.dyn==0 ){
@@ -659,18 +656,6 @@
}
/*
-** The Expr.token field might be a string literal that is quoted.
-** If so, remove the quotation marks.
-*/
-void sqlite3DequoteExpr(Expr *p){
- if( !ExprHasAnyProperty(p, EP_Dequoted) ){
- ExprSetProperty(p, EP_Dequoted);
- assert( (p->vvaFlags & EVVA_ReadOnlyToken)==0 );
- sqlite3Dequote((char*)p->token.z);
- }
-}
-
-/*
** Return the number of bytes allocated for the expression structure
** passed as the first argument. This is always one of EXPR_FULLSIZE,
** EXPR_REDUCEDSIZE or EXPR_TOKENONLYSIZE.
@@ -1595,7 +1580,7 @@
** value of this select in a memory cell and record the number
** of the memory cell in iColumn.
*/
- static const Token one = { (u8*)"1", 0, 1 };
+ static const Token one = { (u8*)"1", 0, 0, 1 };
Select *pSel;
SelectDest dest;
@@ -2082,8 +2067,7 @@
break;
}
case TK_STRING: {
- sqlite3DequoteExpr(pExpr);
- sqlite3VdbeAddOp4(v,OP_String8, 0, target, 0,
+ sqlite3VdbeAddOp4(v, OP_String8, 0, target, 0,
(char*)pExpr->token.z, pExpr->token.n);
break;
}
@@ -2591,7 +2575,6 @@
assert( pExpr->affinity==OE_Rollback ||
pExpr->affinity == OE_Abort ||
pExpr->affinity == OE_Fail );
- sqlite3DequoteExpr(pExpr);
sqlite3VdbeAddOp4(v, OP_Halt, SQLITE_CONSTRAINT, pExpr->affinity, 0,
(char*)pExpr->token.z, pExpr->token.n);
} else {
diff --git a/src/parse.y b/src/parse.y
index 7024814..26017bd 100644
--- a/src/parse.y
+++ b/src/parse.y
@@ -14,7 +14,7 @@
** the parser. Lemon will also generate a header file containing
** numeric codes for all of the tokens.
**
-** @(#) $Id: parse.y,v 1.274 2009/04/06 14:16:43 drh Exp $
+** @(#) $Id: parse.y,v 1.275 2009/05/01 21:13:37 drh Exp $
*/
// All token codes are small integers with #defines that begin with "TK_"
@@ -166,6 +166,8 @@
column(A) ::= columnid(X) type carglist. {
A.z = X.z;
A.n = (int)(pParse->sLastToken.z-X.z) + pParse->sLastToken.n;
+ A.quoted = 0;
+ A.dyn = 0;
}
columnid(A) ::= nm(X). {
sqlite3AddColumn(pParse,&X);
diff --git a/src/resolve.c b/src/resolve.c
index 1f2641d..ac6ada0 100644
--- a/src/resolve.c
+++ b/src/resolve.c
@@ -14,7 +14,7 @@
** resolve all identifiers by associating them with a particular
** table and column.
**
-** $Id: resolve.c,v 1.20 2009/03/05 04:23:47 shane Exp $
+** $Id: resolve.c,v 1.21 2009/05/01 21:13:37 drh Exp $
*/
#include "sqliteInt.h"
#include <stdlib.h>
@@ -319,7 +319,7 @@
** Because no reference was made to outer contexts, the pNC->nRef
** fields are not changed in any context.
*/
- if( cnt==0 && zTab==0 && pColumnToken->z[0]=='"' ){
+ if( cnt==0 && zTab==0 && ExprHasProperty(pExpr,EP_DblQuoted) ){
sqlite3DbFree(db, zCol);
pExpr->op = TK_STRING;
pExpr->pTab = 0;
diff --git a/src/select.c b/src/select.c
index 3c806b1..54600e5 100644
--- a/src/select.c
+++ b/src/select.c
@@ -12,7 +12,7 @@
** This file contains C code routines that are called by the parser
** to handle SELECT statements in SQLite.
**
-** $Id: select.c,v 1.510 2009/04/24 15:46:22 drh Exp $
+** $Id: select.c,v 1.511 2009/05/01 21:13:37 drh Exp $
*/
#include "sqliteInt.h"
@@ -193,42 +193,7 @@
p->z = (u8*)z;
p->n = z ? sqlite3Strlen30(z) : 0;
p->dyn = 0;
-}
-
-/*
-** Set the token to the double-quoted and escaped version of the string pointed
-** to by z. For example;
-**
-** {a"bc} -> {"a""bc"}
-*/
-static void setQuotedToken(Parse *pParse, Token *p, const char *z){
-
- /* Check if the string appears to be quoted using "..." or `...`
- ** or [...] or '...' or if the string contains any " characters.
- ** If it does, then record a version of the string with the special
- ** characters escaped.
- */
- const char *z2 = z;
- if( *z2!='[' && *z2!='`' && *z2!='\'' ){
- while( *z2 ){
- if( *z2=='"' ) break;
- z2++;
- }
- }
-
- if( *z2 ){
- /* String contains " characters - copy and quote the string. */
- p->z = (u8 *)sqlite3MPrintf(pParse->db, "\"%w\"", z);
- if( p->z ){
- p->n = sqlite3Strlen30((char *)p->z);
- p->dyn = 1;
- }
- }else{
- /* String contains no " characters - copy the pointer. */
- p->z = (u8*)z;
- p->n = (int)(z2 - z);
- p->dyn = 0;
- }
+ p->quoted = 0;
}
/*
@@ -3228,12 +3193,12 @@
}
pRight = sqlite3PExpr(pParse, TK_ID, 0, 0, 0);
if( pRight==0 ) break;
- setQuotedToken(pParse, &pRight->token, zName);
+ setToken(&pRight->token, zName);
if( longNames || pTabList->nSrc>1 ){
Expr *pLeft = sqlite3PExpr(pParse, TK_ID, 0, 0, 0);
pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0);
if( pExpr==0 ) break;
- setQuotedToken(pParse, &pLeft->token, zTabName);
+ setToken(&pLeft->token, zTabName);
setToken(&pExpr->span,
sqlite3MPrintf(db, "%s.%s", zTabName, zName));
pExpr->span.dyn = 1;
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index afdb4b3..488a068 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -11,7 +11,7 @@
*************************************************************************
** Internal interface definitions for SQLite.
**
-** @(#) $Id: sqliteInt.h,v 1.864 2009/04/30 12:25:10 drh Exp $
+** @(#) $Id: sqliteInt.h,v 1.865 2009/05/01 21:13:37 drh Exp $
*/
#ifndef _SQLITEINT_H_
#define _SQLITEINT_H_
@@ -1354,8 +1354,9 @@
*/
struct Token {
const unsigned char *z; /* Text of the token. Not NULL-terminated! */
- unsigned dyn : 1; /* True for malloced memory, false for static */
- unsigned n : 31; /* Number of characters in this token */
+ unsigned dyn : 1; /* True for malloced memory, false for static */
+ unsigned quoted : 1; /* True if token still has its quotes */
+ unsigned n : 30; /* Number of characters in this token */
};
/*
@@ -1517,7 +1518,7 @@
#define EP_Error 0x0008 /* Expression contains one or more errors */
#define EP_Distinct 0x0010 /* Aggregate function with DISTINCT keyword */
#define EP_VarSelect 0x0020 /* pSelect is correlated, not constant */
-#define EP_Dequoted 0x0040 /* True if the string has been dequoted */
+#define EP_DblQuoted 0x0040 /* token.z was originally in "..." */
#define EP_InfixFunc 0x0080 /* True for an infix function: LIKE, GLOB, etc */
#define EP_ExpCollate 0x0100 /* Collating sequence specified explicitly */
#define EP_AnyAff 0x0200 /* Can take a cached column of any affinity */
@@ -2360,8 +2361,7 @@
void sqlite3SetString(char **, sqlite3*, const char*, ...);
void sqlite3ErrorMsg(Parse*, const char*, ...);
void sqlite3ErrorClear(Parse*);
-void sqlite3Dequote(char*);
-void sqlite3DequoteExpr(Expr*);
+int sqlite3Dequote(char*);
int sqlite3KeywordCode(const unsigned char*, int);
int sqlite3RunParser(Parse*, const char*, char **);
void sqlite3FinishCoding(Parse*);
diff --git a/src/tokenize.c b/src/tokenize.c
index 4677811..a933de1 100644
--- a/src/tokenize.c
+++ b/src/tokenize.c
@@ -15,7 +15,7 @@
** individual tokens and sends those tokens one-by-one over to the
** parser for analysis.
**
-** $Id: tokenize.c,v 1.155 2009/03/31 03:41:57 shane Exp $
+** $Id: tokenize.c,v 1.156 2009/05/01 21:13:37 drh Exp $
*/
#include "sqliteInt.h"
#include <stdlib.h>
@@ -413,10 +413,12 @@
assert( pParse->apVarExpr==0 );
enableLookaside = db->lookaside.bEnabled;
if( db->lookaside.pStart ) db->lookaside.bEnabled = 1;
+ pParse->sLastToken.quoted = 1;
while( !db->mallocFailed && zSql[i]!=0 ){
assert( i>=0 );
pParse->sLastToken.z = (u8*)&zSql[i];
assert( pParse->sLastToken.dyn==0 );
+ assert( pParse->sLastToken.quoted );
pParse->sLastToken.n = sqlite3GetToken((unsigned char*)&zSql[i],&tokenType);
i += pParse->sLastToken.n;
if( i>mxSqlLen ){
diff --git a/src/util.c b/src/util.c
index 6b18cf2..b2ab111 100644
--- a/src/util.c
+++ b/src/util.c
@@ -14,7 +14,7 @@
** This file contains functions for allocating memory, comparing
** strings, and stuff like that.
**
-** $Id: util.c,v 1.251 2009/04/17 15:18:48 drh Exp $
+** $Id: util.c,v 1.252 2009/05/01 21:13:37 drh Exp $
*/
#include "sqliteInt.h"
#include <stdarg.h>
@@ -203,21 +203,28 @@
** input does not begin with a quote character, then this routine
** is a no-op.
**
+** The input string must be zero-terminated. A new zero-terminator
+** is added to the dequoted string.
+**
+** The return value is -1 if no dequoting occurs or the length of the
+** dequoted string, exclusive of the zero terminator, if dequoting does
+** occur.
+**
** 2002-Feb-14: This routine is extended to remove MS-Access style
** brackets from around identifers. For example: "[a-b-c]" becomes
** "a-b-c".
*/
-void sqlite3Dequote(char *z){
+int sqlite3Dequote(char *z){
char quote;
int i, j;
- if( z==0 ) return;
+ if( z==0 ) return -1;
quote = z[0];
switch( quote ){
case '\'': break;
case '"': break;
case '`': break; /* For MySQL compatibility */
case '[': quote = ']'; break; /* For MS SqlServer compatibility */
- default: return;
+ default: return -1;
}
for(i=1, j=0; z[i]; i++){
if( z[i]==quote ){
@@ -225,13 +232,14 @@
z[j++] = quote;
i++;
}else{
- z[j++] = 0;
break;
}
}else{
z[j++] = z[i];
}
}
+ z[j] = 0;
+ return j;
}
/* Convenient short-hand */
diff --git a/src/vdbemem.c b/src/vdbemem.c
index 6bc0e1f..9ad47f5 100644
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -15,7 +15,7 @@
** only within the VDBE. Interface routines refer to a Mem using the
** name sqlite_value
**
-** $Id: vdbemem.c,v 1.142 2009/04/22 02:15:49 drh Exp $
+** $Id: vdbemem.c,v 1.143 2009/05/01 21:13:37 drh Exp $
*/
#include "sqliteInt.h"
#include "vdbeInt.h"
@@ -977,7 +977,6 @@
zVal = sqlite3DbStrNDup(db, (char*)pExpr->token.z, pExpr->token.n);
pVal = sqlite3ValueNew(db);
if( !zVal || !pVal ) goto no_mem;
- sqlite3Dequote(zVal);
sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC);
if( (op==TK_INTEGER || op==TK_FLOAT ) && affinity==SQLITE_AFF_NONE ){
sqlite3ValueApplyAffinity(pVal, SQLITE_AFF_NUMERIC, enc);
diff --git a/src/where.c b/src/where.c
index 3f850d6..436c246 100644
--- a/src/where.c
+++ b/src/where.c
@@ -16,7 +16,7 @@
** so is applicable. Because this module is responsible for selecting
** indices, you might also think of this module as the "query optimizer".
**
-** $Id: where.c,v 1.391 2009/04/29 11:50:54 danielk1977 Exp $
+** $Id: where.c,v 1.392 2009/05/01 21:13:37 drh Exp $
*/
#include "sqliteInt.h"
@@ -625,6 +625,7 @@
Expr *pRight, *pLeft; /* Right and left size of LIKE operator */
ExprList *pList; /* List of operands to the LIKE operator */
int c; /* One character in z[] */
+ int n; /* Length of string z[] */
int cnt; /* Number of non-wildcard prefix characters */
char wc[3]; /* Wildcard characters */
CollSeq *pColl; /* Collating sequence for LHS */
@@ -655,11 +656,13 @@
(pColl->type!=SQLITE_COLL_NOCASE || !*pnoCase) ){
return 0;
}
- sqlite3DequoteExpr(pRight);
- z = (char *)pRight->token.z;
+ z = (const char*)pRight->token.z;
cnt = 0;
if( z ){
- while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){ cnt++; }
+ n = pRight->token.n;
+ while( cnt<n && (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){
+ cnt++;
+ }
}
if( cnt==0 || 255==(u8)z[cnt-1] ){
return 0;
@@ -1160,7 +1163,6 @@
if( pStr1 ){
sqlite3TokenCopy(db, &pStr1->token, &pRight->token);
pStr1->token.n = nPattern;
- pStr1->flags = EP_Dequoted;
}
pStr2 = sqlite3ExprDup(db, pStr1, 0);
if( !db->mallocFailed ){