Add coverage tests for fts3_snippet.c. Also fixes related to the same.
FossilOrigin-Name: 5e9d8ccae9731f380527463ef87ddcd216b4b721
diff --git a/test/fts3snippet.test b/test/fts3snippet.test
index 0f84898..179c16b 100644
--- a/test/fts3snippet.test
+++ b/test/fts3snippet.test
@@ -9,12 +9,21 @@
#
#*************************************************************************
#
+# The tests in this file test the FTS3 auxillary functions offsets(),
+# snippet() and matchinfo() work. At time of writing, running this file
+# provides full coverage of fts3_snippet.c.
+#
set testdir [file dirname $argv0]
source $testdir/tester.tcl
# If SQLITE_ENABLE_FTS3 is not defined, omit this file.
ifcapable !fts3 { finish_test ; return }
+source $testdir/fts3_common.tcl
+source $testdir/malloc_common.tcl
+
+set sqlite_fts3_enable_parentheses 1
+set DO_MALLOC_TEST 0
# Transform the list $L to its "normal" form. So that it can be compared to
# another list with the same set of elements using [string compare].
@@ -25,84 +34,404 @@
return $ret
}
-do_test fts3snippet-1.1 {
- execsql {
- CREATE VIRTUAL TABLE ft USING fts3;
- INSERT INTO ft VALUES('xxx xxx xxx xxx');
+proc do_offsets_test {name expr args} {
+ set result [list]
+ foreach a $args {
+ lappend result [normalize $a]
}
-} {}
+ do_select_test $name {
+ SELECT offsets(ft) FROM ft WHERE ft MATCH $expr
+ } $result
+}
+
+# Document text used by a few tests. Contains the English names of all
+# integers between 1 and 300.
+#
+set numbers [normalize {
+ one two three four five six seven eight nine ten eleven twelve thirteen
+ fourteen fifteen sixteen seventeen eighteen nineteen twenty twentyone
+ twentytwo twentythree twentyfour twentyfive twentysix twentyseven
+ twentyeight twentynine thirty thirtyone thirtytwo thirtythree thirtyfour
+ thirtyfive thirtysix thirtyseven thirtyeight thirtynine forty fortyone
+ fortytwo fortythree fortyfour fortyfive fortysix fortyseven fortyeight
+ fortynine fifty fiftyone fiftytwo fiftythree fiftyfour fiftyfive fiftysix
+ fiftyseven fiftyeight fiftynine sixty sixtyone sixtytwo sixtythree sixtyfour
+ sixtyfive sixtysix sixtyseven sixtyeight sixtynine seventy seventyone
+ seventytwo seventythree seventyfour seventyfive seventysix seventyseven
+ seventyeight seventynine eighty eightyone eightytwo eightythree eightyfour
+ eightyfive eightysix eightyseven eightyeight eightynine ninety ninetyone
+ ninetytwo ninetythree ninetyfour ninetyfive ninetysix ninetyseven
+ ninetyeight ninetynine onehundred onehundredone onehundredtwo
+ onehundredthree onehundredfour onehundredfive onehundredsix onehundredseven
+ onehundredeight onehundrednine onehundredten onehundredeleven
+ onehundredtwelve onehundredthirteen onehundredfourteen onehundredfifteen
+ onehundredsixteen onehundredseventeen onehundredeighteen onehundrednineteen
+ onehundredtwenty onehundredtwentyone onehundredtwentytwo
+ onehundredtwentythree onehundredtwentyfour onehundredtwentyfive
+ onehundredtwentysix onehundredtwentyseven onehundredtwentyeight
+ onehundredtwentynine onehundredthirty onehundredthirtyone
+ onehundredthirtytwo onehundredthirtythree onehundredthirtyfour
+ onehundredthirtyfive onehundredthirtysix onehundredthirtyseven
+ onehundredthirtyeight onehundredthirtynine onehundredforty
+ onehundredfortyone onehundredfortytwo onehundredfortythree
+ onehundredfortyfour onehundredfortyfive onehundredfortysix
+ onehundredfortyseven onehundredfortyeight onehundredfortynine
+ onehundredfifty onehundredfiftyone onehundredfiftytwo onehundredfiftythree
+ onehundredfiftyfour onehundredfiftyfive onehundredfiftysix
+ onehundredfiftyseven onehundredfiftyeight onehundredfiftynine
+ onehundredsixty onehundredsixtyone onehundredsixtytwo onehundredsixtythree
+ onehundredsixtyfour onehundredsixtyfive onehundredsixtysix
+ onehundredsixtyseven onehundredsixtyeight onehundredsixtynine
+ onehundredseventy onehundredseventyone onehundredseventytwo
+ onehundredseventythree onehundredseventyfour onehundredseventyfive
+ onehundredseventysix onehundredseventyseven onehundredseventyeight
+ onehundredseventynine onehundredeighty onehundredeightyone
+ onehundredeightytwo onehundredeightythree onehundredeightyfour
+ onehundredeightyfive onehundredeightysix onehundredeightyseven
+ onehundredeightyeight onehundredeightynine onehundredninety
+ onehundredninetyone onehundredninetytwo onehundredninetythree
+ onehundredninetyfour onehundredninetyfive onehundredninetysix
+ onehundredninetyseven onehundredninetyeight onehundredninetynine twohundred
+ twohundredone twohundredtwo twohundredthree twohundredfour twohundredfive
+ twohundredsix twohundredseven twohundredeight twohundrednine twohundredten
+ twohundredeleven twohundredtwelve twohundredthirteen twohundredfourteen
+ twohundredfifteen twohundredsixteen twohundredseventeen twohundredeighteen
+ twohundrednineteen twohundredtwenty twohundredtwentyone twohundredtwentytwo
+ twohundredtwentythree twohundredtwentyfour twohundredtwentyfive
+ twohundredtwentysix twohundredtwentyseven twohundredtwentyeight
+ twohundredtwentynine twohundredthirty twohundredthirtyone
+ twohundredthirtytwo twohundredthirtythree twohundredthirtyfour
+ twohundredthirtyfive twohundredthirtysix twohundredthirtyseven
+ twohundredthirtyeight twohundredthirtynine twohundredforty
+ twohundredfortyone twohundredfortytwo twohundredfortythree
+ twohundredfortyfour twohundredfortyfive twohundredfortysix
+ twohundredfortyseven twohundredfortyeight twohundredfortynine
+ twohundredfifty twohundredfiftyone twohundredfiftytwo twohundredfiftythree
+ twohundredfiftyfour twohundredfiftyfive twohundredfiftysix
+ twohundredfiftyseven twohundredfiftyeight twohundredfiftynine
+ twohundredsixty twohundredsixtyone twohundredsixtytwo twohundredsixtythree
+ twohundredsixtyfour twohundredsixtyfive twohundredsixtysix
+ twohundredsixtyseven twohundredsixtyeight twohundredsixtynine
+ twohundredseventy twohundredseventyone twohundredseventytwo
+ twohundredseventythree twohundredseventyfour twohundredseventyfive
+ twohundredseventysix twohundredseventyseven twohundredseventyeight
+ twohundredseventynine twohundredeighty twohundredeightyone
+ twohundredeightytwo twohundredeightythree twohundredeightyfour
+ twohundredeightyfive twohundredeightysix twohundredeightyseven
+ twohundredeightyeight twohundredeightynine twohundredninety
+ twohundredninetyone twohundredninetytwo twohundredninetythree
+ twohundredninetyfour twohundredninetyfive twohundredninetysix
+ twohundredninetyseven twohundredninetyeight twohundredninetynine
+ threehundred
+}]
-do_test fts3snippet-1.2 {
- execsql { SELECT offsets(ft) FROM ft WHERE ft MATCH 'xxx' }
-} {{0 0 0 3 0 0 4 3 0 0 8 3 0 0 12 3}}
-
-do_test fts3snippet-1.3 {
- execsql { SELECT offsets(ft) FROM ft WHERE ft MATCH '"xxx xxx"' }
-} [list [normalize {
- 0 0 0 3
- 0 0 4 3
- 0 1 4 3
- 0 0 8 3
- 0 1 8 3
- 0 1 12 3
-}]]
-
-
-do_test fts3snippet-1.4 {
- execsql { SELECT offsets(ft) FROM ft WHERE ft MATCH '"xxx xxx" xxx' }
-} [list [normalize {
- 0 0 0 3
- 0 2 0 3
- 0 0 4 3
- 0 1 4 3
- 0 2 4 3
- 0 0 8 3
- 0 1 8 3
- 0 2 8 3
- 0 1 12 3
- 0 2 12 3
-}]]
-
-do_test fts3snippet-1.5 {
- execsql { SELECT offsets(ft) FROM ft WHERE ft MATCH 'xxx "xxx xxx"' }
-} [list [normalize {
- 0 0 0 3
- 0 1 0 3
- 0 0 4 3
- 0 1 4 3
- 0 2 4 3
- 0 0 8 3
- 0 1 8 3
- 0 2 8 3
- 0 0 12 3
- 0 2 12 3
-}]]
-
-do_test fts3snippet-2.1 {
- execsql {
- DROP TABLE IF EXISTS ft;
- CREATE VIRTUAL TABLE ft USING fts3;
- INSERT INTO ft VALUES('one two three four five six seven eight nine ten');
- }
-} {}
-foreach {tn expr res} {
- 1 one "[one] two three four five..."
- 2 two "one [two] three four five..."
- 3 three "one two [three] four five..."
- 4 four "...two three [four] five six..."
- 5 five "...three four [five] six seven..."
- 6 six "...four five [six] seven eight..."
- 7 seven "...five six [seven] eight nine..."
- 8 eight "...six seven [eight] nine ten"
- 9 nine "...six seven eight [nine] ten"
- 10 ten "...six seven eight nine [ten]"
+foreach {DO_MALLOC_TEST enc} {
+ 0 utf8
+ 1 utf8
+ 1 utf16
} {
- do_test fts3snippet-2.2.$tn {
+
+ db close
+ file delete -force test.db
+ sqlite3 db test.db
+ db eval "PRAGMA encoding = \"$enc\""
+
+ # Set variable $T to the test name prefix for this iteration of the loop.
+ #
+ set T "fts3snippet-$enc"
+
+ ##########################################################################
+ # Test the offset function.
+ #
+ do_test $T.1.1 {
execsql {
- SELECT snippet(ft, '[', ']', '...', 0, 5) FROM ft WHERE ft MATCH $expr
+ CREATE VIRTUAL TABLE ft USING fts3;
+ INSERT INTO ft VALUES('xxx xxx xxx xxx');
}
- } [list $res]
+ } {}
+ do_offsets_test $T.1.2 {xxx} {0 0 0 3 0 0 4 3 0 0 8 3 0 0 12 3}
+ do_offsets_test $T.1.3 {"xxx xxx"} {
+ 0 0 0 3 0 0 4 3 0 1 4 3 0 0 8 3
+ 0 1 8 3 0 1 12 3
+ }
+ do_offsets_test $T.1.4 {"xxx xxx" xxx} {
+ 0 0 0 3 0 2 0 3 0 0 4 3 0 1 4 3
+ 0 2 4 3 0 0 8 3 0 1 8 3 0 2 8 3
+ 0 1 12 3 0 2 12 3
+ }
+ do_offsets_test $T.1.5 {xxx "xxx xxx"} {
+ 0 0 0 3 0 1 0 3 0 0 4 3 0 1 4 3
+ 0 2 4 3 0 0 8 3 0 1 8 3 0 2 8 3
+ 0 0 12 3 0 2 12 3
+ }
+
+ do_test $T.9.1 {
+ set v1 [lrange $numbers 0 99]
+ execsql {
+ DROP TABLE IF EXISTS ft;
+ CREATE VIRTUAL TABLE ft USING fts3(a, b);
+ INSERT INTO ft VALUES($v1, $numbers);
+ INSERT INTO ft VALUES($v1, NULL);
+ }
+ } {}
+
+ set off [string first "twohundred " $numbers]
+ do_offsets_test $T.9.1 {twohundred} [list 1 0 $off 10]
+
+ set off [string first "onehundred " $numbers]
+ do_offsets_test $T.9.2 {onehundred} \
+ [list 0 0 $off 10 1 0 $off 10] [list 0 0 $off 10]
+
+ # Test a corruption case:
+ execsql { UPDATE ft_content SET c1b = 'hello world' WHERE c1b = $numbers }
+ do_error_test $T.9.3 {
+ SELECT offsets(ft) FROM ft WHERE ft MATCH 'onehundred'
+ } {database disk image is malformed}
+
+ ##########################################################################
+ # Test the snippet function.
+ #
+ proc do_snippet_test {name expr iCol nTok args} {
+ set res [list]
+ foreach a $args { lappend res [string trim $a] }
+ do_select_test $name {
+ SELECT snippet(ft,'{','}','...',$iCol,$nTok) FROM ft WHERE ft MATCH $expr
+ } $res
+ }
+ do_test $T.2.1 {
+ execsql {
+ DROP TABLE IF EXISTS ft;
+ CREATE VIRTUAL TABLE ft USING fts3;
+ INSERT INTO ft VALUES('one two three four five six seven eight nine ten');
+ }
+ } {}
+ do_snippet_test $T.2.2 one 0 5 "{one} two three four five..."
+ do_snippet_test $T.2.3 two 0 5 "one {two} three four five..."
+ do_snippet_test $T.2.4 three 0 5 "one two {three} four five..."
+ do_snippet_test $T.2.5 four 0 5 "...two three {four} five six..."
+ do_snippet_test $T.2.6 five 0 5 "...three four {five} six seven..."
+ do_snippet_test $T.2.7 six 0 5 "...four five {six} seven eight..."
+ do_snippet_test $T.2.8 seven 0 5 "...five six {seven} eight nine..."
+ do_snippet_test $T.2.9 eight 0 5 "...six seven {eight} nine ten"
+ do_snippet_test $T.2.10 nine 0 5 "...six seven eight {nine} ten"
+ do_snippet_test $T.2.11 ten 0 5 "...six seven eight nine {ten}"
+
+ do_test $T.3.1 {
+ execsql {
+ INSERT INTO ft VALUES(
+ 'one two three four five '
+ || 'six seven eight nine ten '
+ || 'eleven twelve thirteen fourteen fifteen '
+ || 'sixteen seventeen eighteen nineteen twenty '
+ || 'one two three four five '
+ || 'six seven eight nine ten '
+ || 'eleven twelve thirteen fourteen fifteen '
+ || 'sixteen seventeen eighteen nineteen twenty'
+ );
+ }
+ } {}
+
+ do_snippet_test $T.3.2 {one nine} 0 5 {
+ {one} two three...eight {nine} ten
+ } {
+ {one} two three...eight {nine} ten...
+ }
+
+ do_snippet_test $T.3.3 {one nine} 0 -5 {
+ {one} two three four five...six seven eight {nine} ten
+ } {
+ {one} two three four five...seven eight {nine} ten eleven...
+ }
+ do_snippet_test $T.3.3 {one nineteen} 0 -5 {
+ ...eighteen {nineteen} twenty {one} two...
+ }
+ do_snippet_test $T.3.4 {two nineteen} 0 -5 {
+ ...eighteen {nineteen} twenty one {two}...
+ }
+ do_snippet_test $T.3.5 {three nineteen} 0 -5 {
+ ...{nineteen} twenty one two {three}...
+ }
+
+ do_snippet_test $T.3.6 {four nineteen} 0 -5 {
+ ...two three {four} five six...seventeen eighteen {nineteen} twenty one...
+ }
+ do_snippet_test $T.3.7 {four NEAR nineteen} 0 -5 {
+ ...seventeen eighteen {nineteen} twenty one...two three {four} five six...
+ }
+
+ do_snippet_test $T.3.8 {four nineteen} 0 5 {
+ ...three {four} five...eighteen {nineteen} twenty...
+ }
+ do_snippet_test $T.3.9 {four NEAR nineteen} 0 5 {
+ ...eighteen {nineteen} twenty...three {four} five...
+ }
+ do_snippet_test $T.3.10 {four NEAR nineteen} 0 -5 {
+ ...seventeen eighteen {nineteen} twenty one...two three {four} five six...
+ }
+ do_snippet_test $T.3.11 {four NOT (nineteen twentyone)} 0 5 {
+ ...two three {four} five six...
+ } {
+ ...two three {four} five six...
+ }
+ do_snippet_test $T.3.12 {four OR nineteen NEAR twentyone} 0 5 {
+ ...two three {four} five six...
+ } {
+ ...two three {four} five six...
+ }
+
+ do_test $T.5.1 {
+ execsql {
+ DROP TABLE IF EXISTS ft;
+ CREATE VIRTUAL TABLE ft USING fts3(a, b, c);
+ INSERT INTO ft VALUES(
+ 'one two three four five',
+ 'four five six seven eight',
+ 'seven eight nine ten eleven'
+ );
+ }
+ } {}
+
+ do_snippet_test $T.5.2 {five} -1 3 {...three four {five}}
+ do_snippet_test $T.5.3 {five} 0 3 {...three four {five}}
+ do_snippet_test $T.5.4 {five} 1 3 {four {five} six...}
+ do_snippet_test $T.5.5 {five} 2 3 {seven eight nine...}
+
+ do_test $T.5.6 {
+ execsql { UPDATE ft SET b = NULL }
+ } {}
+
+ do_snippet_test $T.5.7 {five} -1 3 {...three four {five}}
+ do_snippet_test $T.5.8 {five} 0 3 {...three four {five}}
+ do_snippet_test $T.5.9 {five} 1 3 {}
+ do_snippet_test $T.5.10 {five} 2 3 {seven eight nine...}
+
+ do_snippet_test $T.5.11 {one "seven eight nine"} -1 -3 {
+ {one} two three...{seven} {eight} {nine}...
+ }
+
+ do_test $T.6.1 {
+ execsql {
+ DROP TABLE IF EXISTS ft;
+ CREATE VIRTUAL TABLE ft USING fts3(x);
+ INSERT INTO ft VALUES($numbers);
+ }
+ } {}
+ do_snippet_test $T.6.2 {
+ one fifty onehundred onehundredfifty twohundredfifty threehundred
+ } -1 4 {
+ {one}...{fifty}...{onehundred}...{onehundredfifty}...
+ }
+ do_snippet_test $T.6.3 {
+ one fifty onehundred onehundredfifty twohundredfifty threehundred
+ } -1 -4 {
+ {one} two three four...fortyeight fortynine {fifty} fiftyone...ninetyeight ninetynine {onehundred} onehundredone...onehundredfortyeight onehundredfortynine {onehundredfifty} onehundredfiftyone...
+ }
+
+ do_test $T.7.1 {
+ execsql {
+ BEGIN;
+ DROP TABLE IF EXISTS ft;
+ CREATE VIRTUAL TABLE ft USING fts3(x);
+ }
+ set testresults [list]
+ for {set i 1} {$i < 150} {incr i} {
+ set commas [string repeat , $i]
+ execsql {INSERT INTO ft VALUES('one' || $commas || 'two')}
+ lappend testresults "{one}$commas{two}"
+ }
+ execsql COMMIT
+ } {}
+ do_snippet_test $T.7.2 {one two} -1 3 {*}$testresults
+
+ ##########################################################################
+ # Test the matchinfo function.
+ #
+ proc mit {blob} {
+ set scan(littleEndian) i*
+ set scan(bigEndian) I*
+ binary scan $blob $scan($::tcl_platform(byteOrder)) r
+ return $r
+ }
+ db func mit mit
+ proc do_matchinfo_test {name expr args} {
+ set res [list]
+ foreach a $args { lappend res [normalize $a] }
+ do_select_test $name {
+ SELECT mit(matchinfo(ft)) FROM ft WHERE ft MATCH $expr
+ } $res
+ }
+ do_test $T.4.1 {
+ set ten {one two three four five six seven eight nine ten}
+ execsql {
+ DROP TABLE IF EXISTS ft;
+ CREATE VIRTUAL TABLE ft USING fts3;
+ INSERT INTO ft VALUES($ten);
+ INSERT INTO ft VALUES($ten || ' ' || $ten);
+ }
+ } {}
+
+ do_matchinfo_test $T.4.2 "one" {1 1 1 3 2} {1 1 2 3 2}
+ do_matchinfo_test $T.4.3 "one NEAR/3 ten" {2 1 1 1 1 1 1 1}
+ do_matchinfo_test $T.4.4 "five NEAR/4 ten" \
+ {2 1 1 3 2 1 3 2} {2 1 2 3 2 2 3 2}
+ do_matchinfo_test $T.4.5 "six NEAR/3 ten NEAR/3 two" \
+ {3 1 1 1 1 1 1 1 1 1 1}
+ do_matchinfo_test $T.4.6 "five NEAR/4 ten NEAR/3 two" \
+ {3 1 2 2 1 1 1 1 1 1 1}
+
+ do_test $T.8.1 {
+ execsql {
+ DROP TABLE IF EXISTS ft;
+ CREATE VIRTUAL TABLE ft USING fts3(x, y);
+ }
+ foreach n {1 2 3} {
+ set v1 [lrange $numbers 0 [expr $n*100]]
+ set v2 [string trim [string repeat "$numbers " $n]]
+ set docid [expr $n * 1000000]
+ execsql { INSERT INTO ft(docid, x, y) VALUES($docid, $v1, $v2) }
+ }
+ } {}
+ do_matchinfo_test $T.8.2 {two*} \
+ { 1 2 1 105 3 101 606 3} \
+ { 1 2 3 105 3 202 606 3} \
+ { 1 2 101 105 3 303 606 3}
+
+ do_matchinfo_test $T.8.4 {"one* two*"} \
+ { 1 2 1 5 3 2 12 3} \
+ { 1 2 2 5 3 4 12 3} \
+ { 1 2 2 5 3 6 12 3}
+
+ do_matchinfo_test $T.8.5 {twohundredfifty} \
+ { 1 2 0 1 1 1 6 3} \
+ { 1 2 0 1 1 2 6 3} \
+ { 1 2 1 1 1 3 6 3}
+
+ do_matchinfo_test $T.8.6 {"threehundred one"} \
+ { 1 2 0 0 0 1 3 2} \
+ { 1 2 0 0 0 2 3 2}
+
+ do_matchinfo_test $T.8.7 {one OR fivehundred} \
+ { 2 2 1 3 3 1 6 3 0 0 0 0 0 0 } \
+ { 2 2 1 3 3 2 6 3 0 0 0 0 0 0 } \
+ { 2 2 1 3 3 3 6 3 0 0 0 0 0 0 }
+
+ do_matchinfo_test $T.8.8 {two OR "threehundred one"} \
+ { 2 2 1 3 3 1 6 3 0 0 0 0 3 2 } \
+ { 2 2 1 3 3 2 6 3 0 0 0 1 3 2 } \
+ { 2 2 1 3 3 3 6 3 0 0 0 2 3 2 }
+
+ do_select_test $T.8.9 {
+ SELECT mit(matchinfo(ft)), mit(matchinfo(ft))
+ FROM ft WHERE ft MATCH 'two OR "threehundred one"'
+ } [normalize {
+ {2 2 1 3 3 1 6 3 0 0 0 0 3 2}
+ {2 2 1 3 3 1 6 3 0 0 0 0 3 2}
+ {2 2 1 3 3 2 6 3 0 0 0 1 3 2}
+ {2 2 1 3 3 2 6 3 0 0 0 1 3 2}
+ {2 2 1 3 3 3 6 3 0 0 0 2 3 2}
+ {2 2 1 3 3 3 6 3 0 0 0 2 3 2}
+ }]
}
+set sqlite_fts3_enable_parentheses 0
finish_test
-