| blob | 0cb1a8829d61967a51587d18f0991a9ac970c976 |
1 diff -ru ext-orig/fts2/fts2.c ext/fts2/fts2.c
2 --- ext-orig/fts2/fts2.c 2009-09-04 13:37:41.000000000 -0700
3 +++ ext/fts2/fts2.c 2009-09-30 14:48:14.000000000 -0700
4 @@ -37,6 +37,20 @@
5 ** This is an SQLite module implementing full-text search.
6 */
8 +/* TODO(shess): To make it easier to spot changes without groveling
9 +** through changelogs, I've defined GEARS_FTS2_CHANGES to call them
10 +** out, and I will document them here. On imports, these changes
11 +** should be reviewed to make sure they are still present, or are
12 +** dropped as appropriate.
13 +**
14 +** SQLite core adds the custom function fts2_tokenizer() to be used
15 +** for defining new tokenizers. The second parameter is a vtable
16 +** pointer encoded as a blob. Obviously this cannot be exposed to
17 +** Gears callers for security reasons. It could be suppressed in the
18 +** authorizer, but for now I have simply commented the definition out.
19 +*/
20 +#define GEARS_FTS2_CHANGES 1
21 +
22 /*
23 ** The code in this file is only compiled if:
24 **
25 @@ -326,8 +326,10 @@
26 #include "fts2_hash.h"
27 #include "fts2_tokenizer.h"
28 #include "sqlite3.h"
29 -#include "sqlite3ext.h"
30 -SQLITE_EXTENSION_INIT1
31 +#ifndef SQLITE_CORE
32 +# include "sqlite3ext.h"
33 + SQLITE_EXTENSION_INIT1
34 +#endif
37 /* TODO(shess) MAN, this thing needs some refactoring. At minimum, it
38 @@ -335,6 +349,16 @@
39 # define TRACE(A)
40 #endif
42 +#if 0
43 +/* Useful to set breakpoints. See main.c sqlite3Corrupt(). */
44 +static int fts2Corrupt(void){
45 + return SQLITE_CORRUPT;
46 +}
47 +# define SQLITE_CORRUPT_BKPT fts2Corrupt()
48 +#else
49 +# define SQLITE_CORRUPT_BKPT SQLITE_CORRUPT
50 +#endif
51 +
52 /* It is not safe to call isspace(), tolower(), or isalnum() on
53 ** hi-bit-set characters. This is the same solution used in the
54 ** tokenizer.
55 @@ -423,30 +447,41 @@
56 /* Read a 64-bit variable-length integer from memory starting at p[0].
57 * Return the number of bytes read, or 0 on error.
58 * The value is stored in *v. */
59 -static int getVarint(const char *p, sqlite_int64 *v){
60 +static int getVarintSafe(const char *p, sqlite_int64 *v, int max){
61 const unsigned char *q = (const unsigned char *) p;
62 sqlite_uint64 x = 0, y = 1;
63 - while( (*q & 0x80) == 0x80 ){
64 + if( max>VARINT_MAX ) max = VARINT_MAX;
65 + while( max && (*q & 0x80) == 0x80 ){
66 + max--;
67 x += y * (*q++ & 0x7f);
68 y <<= 7;
69 - if( q - (unsigned char *)p >= VARINT_MAX ){ /* bad data */
70 - assert( 0 );
71 - return 0;
72 - }
73 + }
74 + if ( !max ){
75 + assert( 0 );
76 + return 0; /* tried to read too much; bad data */
77 }
78 x += y * (*q++);
79 *v = (sqlite_int64) x;
80 return (int) (q - (unsigned char *)p);
81 }
83 -static int getVarint32(const char *p, int *pi){
84 +static int getVarint(const char *p, sqlite_int64 *v){
85 + return getVarintSafe(p, v, VARINT_MAX);
86 +}
87 +
88 +static int getVarint32Safe(const char *p, int *pi, int max){
89 sqlite_int64 i;
90 - int ret = getVarint(p, &i);
91 + int ret = getVarintSafe(p, &i, max);
92 + if( !ret ) return ret;
93 *pi = (int) i;
94 assert( *pi==i );
95 return ret;
96 }
98 +static int getVarint32(const char* p, int *pi){
99 + return getVarint32Safe(p, pi, VARINT_MAX);
100 +}
101 +
102 /*******************************************************************/
103 /* DataBuffer is used to collect data into a buffer in piecemeal
104 ** fashion. It implements the usual distinction between amount of
105 @@ -615,7 +650,7 @@
107 static int dlrAtEnd(DLReader *pReader){
108 assert( pReader->nData>=0 );
109 - return pReader->nData==0;
110 + return pReader->nData<=0;
111 }
112 static sqlite_int64 dlrDocid(DLReader *pReader){
113 assert( !dlrAtEnd(pReader) );
114 @@ -639,7 +674,8 @@
115 */
116 static const char *dlrPosData(DLReader *pReader){
117 sqlite_int64 iDummy;
118 - int n = getVarint(pReader->pData, &iDummy);
119 + int n = getVarintSafe(pReader->pData, &iDummy, pReader->nElement);
120 + if( !n ) return NULL;
121 assert( !dlrAtEnd(pReader) );
122 return pReader->pData+n;
123 }
124 @@ -649,7 +685,7 @@
125 assert( !dlrAtEnd(pReader) );
126 return pReader->nElement-n;
127 }
128 -static void dlrStep(DLReader *pReader){
129 +static int dlrStep(DLReader *pReader){
130 assert( !dlrAtEnd(pReader) );
132 /* Skip past current doclist element. */
133 @@ -658,32 +694,48 @@
134 pReader->nData -= pReader->nElement;
136 /* If there is more data, read the next doclist element. */
137 - if( pReader->nData!=0 ){
138 + if( pReader->nData>0 ){
139 sqlite_int64 iDocidDelta;
140 - int iDummy, n = getVarint(pReader->pData, &iDocidDelta);
141 + int nTotal = 0;
142 + int iDummy, n = getVarintSafe(pReader->pData, &iDocidDelta, pReader->nData);
143 + if( !n ) return SQLITE_CORRUPT_BKPT;
144 + nTotal += n;
145 pReader->iDocid += iDocidDelta;
146 if( pReader->iType>=DL_POSITIONS ){
147 - assert( n<pReader->nData );
148 while( 1 ){
149 - n += getVarint32(pReader->pData+n, &iDummy);
150 - assert( n<=pReader->nData );
151 + n = getVarint32Safe(pReader->pData+nTotal, &iDummy,
152 + pReader->nData-nTotal);
153 + if( !n ) return SQLITE_CORRUPT_BKPT;
154 + nTotal += n;
155 if( iDummy==POS_END ) break;
156 if( iDummy==POS_COLUMN ){
157 - n += getVarint32(pReader->pData+n, &iDummy);
158 - assert( n<pReader->nData );
159 + n = getVarint32Safe(pReader->pData+nTotal, &iDummy,
160 + pReader->nData-nTotal);
161 + if( !n ) return SQLITE_CORRUPT_BKPT;
162 + nTotal += n;
163 }else if( pReader->iType==DL_POSITIONS_OFFSETS ){
164 - n += getVarint32(pReader->pData+n, &iDummy);
165 - n += getVarint32(pReader->pData+n, &iDummy);
166 - assert( n<pReader->nData );
167 + n = getVarint32Safe(pReader->pData+nTotal, &iDummy,
168 + pReader->nData-nTotal);
169 + if( !n ) return SQLITE_CORRUPT_BKPT;
170 + nTotal += n;
171 + n = getVarint32Safe(pReader->pData+nTotal, &iDummy,
172 + pReader->nData-nTotal);
173 + if( !n ) return SQLITE_CORRUPT_BKPT;
174 + nTotal += n;
175 }
176 }
177 }
178 - pReader->nElement = n;
179 + pReader->nElement = nTotal;
180 assert( pReader->nElement<=pReader->nData );
181 }
182 + return SQLITE_OK;
183 }
184 -static void dlrInit(DLReader *pReader, DocListType iType,
185 - const char *pData, int nData){
186 +static void dlrDestroy(DLReader *pReader){
187 + SCRAMBLE(pReader);
188 +}
189 +static int dlrInit(DLReader *pReader, DocListType iType,
190 + const char *pData, int nData){
191 + int rc;
192 assert( pData!=NULL && nData!=0 );
193 pReader->iType = iType;
194 pReader->pData = pData;
195 @@ -692,10 +744,9 @@
196 pReader->iDocid = 0;
198 /* Load the first element's data. There must be a first element. */
199 - dlrStep(pReader);
200 -}
201 -static void dlrDestroy(DLReader *pReader){
202 - SCRAMBLE(pReader);
203 + rc = dlrStep(pReader);
204 + if( rc!=SQLITE_OK ) dlrDestroy(pReader);
205 + return rc;
206 }
208 #ifndef NDEBUG
209 @@ -782,9 +833,9 @@
210 /* TODO(shess) This has become just a helper for docListMerge.
211 ** Consider a refactor to make this cleaner.
212 */
213 -static void dlwAppend(DLWriter *pWriter,
214 - const char *pData, int nData,
215 - sqlite_int64 iFirstDocid, sqlite_int64 iLastDocid){
216 +static int dlwAppend(DLWriter *pWriter,
217 + const char *pData, int nData,
218 + sqlite_int64 iFirstDocid, sqlite_int64 iLastDocid){
219 sqlite_int64 iDocid = 0;
220 char c[VARINT_MAX];
221 int nFirstOld, nFirstNew; /* Old and new varint len of first docid. */
222 @@ -793,7 +844,8 @@
223 #endif
225 /* Recode the initial docid as delta from iPrevDocid. */
226 - nFirstOld = getVarint(pData, &iDocid);
227 + nFirstOld = getVarintSafe(pData, &iDocid, nData);
228 + if( !nFirstOld ) return SQLITE_CORRUPT_BKPT;
229 assert( nFirstOld<nData || (nFirstOld==nData && pWriter->iType==DL_DOCIDS) );
230 nFirstNew = putVarint(c, iFirstDocid-pWriter->iPrevDocid);
232 @@ -814,10 +866,11 @@
233 dataBufferAppend(pWriter->b, c, nFirstNew);
234 }
235 pWriter->iPrevDocid = iLastDocid;
236 + return SQLITE_OK;
237 }
238 -static void dlwCopy(DLWriter *pWriter, DLReader *pReader){
239 - dlwAppend(pWriter, dlrDocData(pReader), dlrDocDataBytes(pReader),
240 - dlrDocid(pReader), dlrDocid(pReader));
241 +static int dlwCopy(DLWriter *pWriter, DLReader *pReader){
242 + return dlwAppend(pWriter, dlrDocData(pReader), dlrDocDataBytes(pReader),
243 + dlrDocid(pReader), dlrDocid(pReader));
244 }
245 static void dlwAdd(DLWriter *pWriter, sqlite_int64 iDocid){
246 char c[VARINT_MAX];
247 @@ -878,45 +931,63 @@
248 assert( !plrAtEnd(pReader) );
249 return pReader->iEndOffset;
250 }
251 -static void plrStep(PLReader *pReader){
252 - int i, n;
253 +static int plrStep(PLReader *pReader){
254 + int i, n, nTotal = 0;
256 assert( !plrAtEnd(pReader) );
258 - if( pReader->nData==0 ){
259 + if( pReader->nData<=0 ){
260 pReader->pData = NULL;
261 - return;
262 + return SQLITE_OK;
263 }
265 - n = getVarint32(pReader->pData, &i);
266 + n = getVarint32Safe(pReader->pData, &i, pReader->nData);
267 + if( !n ) return SQLITE_CORRUPT_BKPT;
268 + nTotal += n;
269 if( i==POS_COLUMN ){
270 - n += getVarint32(pReader->pData+n, &pReader->iColumn);
271 + n = getVarint32Safe(pReader->pData+nTotal, &pReader->iColumn,
272 + pReader->nData-nTotal);
273 + if( !n ) return SQLITE_CORRUPT_BKPT;
274 + nTotal += n;
275 pReader->iPosition = 0;
276 pReader->iStartOffset = 0;
277 - n += getVarint32(pReader->pData+n, &i);
278 + n = getVarint32Safe(pReader->pData+nTotal, &i, pReader->nData-nTotal);
279 + if( !n ) return SQLITE_CORRUPT_BKPT;
280 + nTotal += n;
281 }
282 /* Should never see adjacent column changes. */
283 assert( i!=POS_COLUMN );
285 if( i==POS_END ){
286 + assert( nTotal<=pReader->nData );
287 pReader->nData = 0;
288 pReader->pData = NULL;
289 - return;
290 + return SQLITE_OK;
291 }
293 pReader->iPosition += i-POS_BASE;
294 if( pReader->iType==DL_POSITIONS_OFFSETS ){
295 - n += getVarint32(pReader->pData+n, &i);
296 + n = getVarint32Safe(pReader->pData+nTotal, &i, pReader->nData-nTotal);
297 + if( !n ) return SQLITE_CORRUPT_BKPT;
298 + nTotal += n;
299 pReader->iStartOffset += i;
300 - n += getVarint32(pReader->pData+n, &i);
301 + n = getVarint32Safe(pReader->pData+nTotal, &i, pReader->nData-nTotal);
302 + if( !n ) return SQLITE_CORRUPT_BKPT;
303 + nTotal += n;
304 pReader->iEndOffset = pReader->iStartOffset+i;
305 }
306 - assert( n<=pReader->nData );
307 - pReader->pData += n;
308 - pReader->nData -= n;
309 + assert( nTotal<=pReader->nData );
310 + pReader->pData += nTotal;
311 + pReader->nData -= nTotal;
312 + return SQLITE_OK;
313 }
315 -static void plrInit(PLReader *pReader, DLReader *pDLReader){
316 +static void plrDestroy(PLReader *pReader){
317 + SCRAMBLE(pReader);
318 +}
319 +
320 +static int plrInit(PLReader *pReader, DLReader *pDLReader){
321 + int rc;
322 pReader->pData = dlrPosData(pDLReader);
323 pReader->nData = dlrPosDataLen(pDLReader);
324 pReader->iType = pDLReader->iType;
325 @@ -924,10 +995,9 @@
326 pReader->iPosition = 0;
327 pReader->iStartOffset = 0;
328 pReader->iEndOffset = 0;
329 - plrStep(pReader);
330 -}
331 -static void plrDestroy(PLReader *pReader){
332 - SCRAMBLE(pReader);
333 + rc = plrStep(pReader);
334 + if( rc!=SQLITE_OK ) plrDestroy(pReader);
335 + return rc;
336 }
338 /*******************************************************************/
339 @@ -1113,14 +1183,16 @@
340 ** deletion will be trimmed, and will thus not effect a deletion
341 ** during the merge.
342 */
343 -static void docListTrim(DocListType iType, const char *pData, int nData,
344 - int iColumn, DocListType iOutType, DataBuffer *out){
345 +static int docListTrim(DocListType iType, const char *pData, int nData,
346 + int iColumn, DocListType iOutType, DataBuffer *out){
347 DLReader dlReader;
348 DLWriter dlWriter;
349 + int rc;
351 assert( iOutType<=iType );
353 - dlrInit(&dlReader, iType, pData, nData);
354 + rc = dlrInit(&dlReader, iType, pData, nData);
355 + if( rc!=SQLITE_OK ) return rc;
356 dlwInit(&dlWriter, iOutType, out);
358 while( !dlrAtEnd(&dlReader) ){
359 @@ -1128,7 +1200,8 @@
360 PLWriter plWriter;
361 int match = 0;
363 - plrInit(&plReader, &dlReader);
364 + rc = plrInit(&plReader, &dlReader);
365 + if( rc!=SQLITE_OK ) break;
367 while( !plrAtEnd(&plReader) ){
368 if( iColumn==-1 || plrColumn(&plReader)==iColumn ){
369 @@ -1139,7 +1212,11 @@
370 plwAdd(&plWriter, plrColumn(&plReader), plrPosition(&plReader),
371 plrStartOffset(&plReader), plrEndOffset(&plReader));
372 }
373 - plrStep(&plReader);
374 + rc = plrStep(&plReader);
375 + if( rc!=SQLITE_OK ){
376 + plrDestroy(&plReader);
377 + goto err;
378 + }
379 }
380 if( match ){
381 plwTerminate(&plWriter);
382 @@ -1147,10 +1224,13 @@
383 }
385 plrDestroy(&plReader);
386 - dlrStep(&dlReader);
387 + rc = dlrStep(&dlReader);
388 + if( rc!=SQLITE_OK ) break;
389 }
390 +err:
391 dlwDestroy(&dlWriter);
392 dlrDestroy(&dlReader);
393 + return rc;
394 }
396 /* Used by docListMerge() to keep doclists in the ascending order by
397 @@ -1207,19 +1287,20 @@
398 /* TODO(shess) nReaders must be <= MERGE_COUNT. This should probably
399 ** be fixed.
400 */
401 -static void docListMerge(DataBuffer *out,
402 - DLReader *pReaders, int nReaders){
403 +static int docListMerge(DataBuffer *out,
404 + DLReader *pReaders, int nReaders){
405 OrderedDLReader readers[MERGE_COUNT];
406 DLWriter writer;
407 int i, n;
408 const char *pStart = 0;
409 int nStart = 0;
410 sqlite_int64 iFirstDocid = 0, iLastDocid = 0;
411 + int rc = SQLITE_OK;
413 assert( nReaders>0 );
414 if( nReaders==1 ){
415 dataBufferAppend(out, dlrDocData(pReaders), dlrAllDataBytes(pReaders));
416 - return;
417 + return SQLITE_OK;
418 }
420 assert( nReaders<=MERGE_COUNT );
421 @@ -1252,20 +1333,23 @@
422 nStart += dlrDocDataBytes(readers[0].pReader);
423 }else{
424 if( pStart!=0 ){
425 - dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid);
426 + rc = dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid);
427 + if( rc!=SQLITE_OK ) goto err;
428 }
429 pStart = dlrDocData(readers[0].pReader);
430 nStart = dlrDocDataBytes(readers[0].pReader);
431 iFirstDocid = iDocid;
432 }
433 iLastDocid = iDocid;
434 - dlrStep(readers[0].pReader);
435 + rc = dlrStep(readers[0].pReader);
436 + if( rc!=SQLITE_OK ) goto err;
438 /* Drop all of the older elements with the same docid. */
439 for(i=1; i<nReaders &&
440 !dlrAtEnd(readers[i].pReader) &&
441 dlrDocid(readers[i].pReader)==iDocid; i++){
442 - dlrStep(readers[i].pReader);
443 + rc = dlrStep(readers[i].pReader);
444 + if( rc!=SQLITE_OK ) goto err;
445 }
447 /* Get the readers back into order. */
448 @@ -1275,8 +1359,11 @@
449 }
451 /* Copy over any remaining elements. */
452 - if( nStart>0 ) dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid);
453 + if( nStart>0 )
454 + rc = dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid);
455 +err:
456 dlwDestroy(&writer);
457 + return rc;
458 }
460 /* Helper function for posListUnion(). Compares the current position
461 @@ -1312,30 +1399,40 @@
462 ** work with any doclist type, though both inputs and the output
463 ** should be the same type.
464 */
465 -static void posListUnion(DLReader *pLeft, DLReader *pRight, DLWriter *pOut){
466 +static int posListUnion(DLReader *pLeft, DLReader *pRight, DLWriter *pOut){
467 PLReader left, right;
468 PLWriter writer;
469 + int rc;
471 assert( dlrDocid(pLeft)==dlrDocid(pRight) );
472 assert( pLeft->iType==pRight->iType );
473 assert( pLeft->iType==pOut->iType );
475 - plrInit(&left, pLeft);
476 - plrInit(&right, pRight);
477 + rc = plrInit(&left, pLeft);
478 + if( rc != SQLITE_OK ) return rc;
479 + rc = plrInit(&right, pRight);
480 + if( rc != SQLITE_OK ){
481 + plrDestroy(&left);
482 + return rc;
483 + }
484 plwInit(&writer, pOut, dlrDocid(pLeft));
486 while( !plrAtEnd(&left) || !plrAtEnd(&right) ){
487 int c = posListCmp(&left, &right);
488 if( c<0 ){
489 plwCopy(&writer, &left);
490 - plrStep(&left);
491 + rc = plrStep(&left);
492 + if( rc != SQLITE_OK ) break;
493 }else if( c>0 ){
494 plwCopy(&writer, &right);
495 - plrStep(&right);
496 + rc = plrStep(&right);
497 + if( rc != SQLITE_OK ) break;
498 }else{
499 plwCopy(&writer, &left);
500 - plrStep(&left);
501 - plrStep(&right);
502 + rc = plrStep(&left);
503 + if( rc != SQLITE_OK ) break;
504 + rc = plrStep(&right);
505 + if( rc != SQLITE_OK ) break;
506 }
507 }
509 @@ -1343,56 +1440,75 @@
510 plwDestroy(&writer);
511 plrDestroy(&left);
512 plrDestroy(&right);
513 + return rc;
514 }
516 /* Write the union of doclists in pLeft and pRight to pOut. For
517 ** docids in common between the inputs, the union of the position
518 ** lists is written. Inputs and outputs are always type DL_DEFAULT.
519 */
520 -static void docListUnion(
521 +static int docListUnion(
522 const char *pLeft, int nLeft,
523 const char *pRight, int nRight,
524 DataBuffer *pOut /* Write the combined doclist here */
525 ){
526 DLReader left, right;
527 DLWriter writer;
528 + int rc;
530 if( nLeft==0 ){
531 if( nRight!=0) dataBufferAppend(pOut, pRight, nRight);
532 - return;
533 + return SQLITE_OK;
534 }
535 if( nRight==0 ){
536 dataBufferAppend(pOut, pLeft, nLeft);
537 - return;
538 + return SQLITE_OK;
539 }
541 - dlrInit(&left, DL_DEFAULT, pLeft, nLeft);
542 - dlrInit(&right, DL_DEFAULT, pRight, nRight);
543 + rc = dlrInit(&left, DL_DEFAULT, pLeft, nLeft);
544 + if( rc!=SQLITE_OK ) return rc;
545 + rc = dlrInit(&right, DL_DEFAULT, pRight, nRight);
546 + if( rc!=SQLITE_OK ){
547 + dlrDestroy(&left);
548 + return rc;
549 + }
550 dlwInit(&writer, DL_DEFAULT, pOut);
552 while( !dlrAtEnd(&left) || !dlrAtEnd(&right) ){
553 if( dlrAtEnd(&right) ){
554 - dlwCopy(&writer, &left);
555 - dlrStep(&left);
556 + rc = dlwCopy(&writer, &left);
557 + if( rc!=SQLITE_OK ) break;
558 + rc = dlrStep(&left);
559 + if( rc!=SQLITE_OK ) break;
560 }else if( dlrAtEnd(&left) ){
561 - dlwCopy(&writer, &right);
562 - dlrStep(&right);
563 + rc = dlwCopy(&writer, &right);
564 + if( rc!=SQLITE_OK ) break;
565 + rc = dlrStep(&right);
566 + if( rc!=SQLITE_OK ) break;
567 }else if( dlrDocid(&left)<dlrDocid(&right) ){
568 - dlwCopy(&writer, &left);
569 - dlrStep(&left);
570 + rc = dlwCopy(&writer, &left);
571 + if( rc!=SQLITE_OK ) break;
572 + rc = dlrStep(&left);
573 + if( rc!=SQLITE_OK ) break;
574 }else if( dlrDocid(&left)>dlrDocid(&right) ){
575 - dlwCopy(&writer, &right);
576 - dlrStep(&right);
577 + rc = dlwCopy(&writer, &right);
578 + if( rc!=SQLITE_OK ) break;
579 + rc = dlrStep(&right);
580 + if( rc!=SQLITE_OK ) break;
581 }else{
582 - posListUnion(&left, &right, &writer);
583 - dlrStep(&left);
584 - dlrStep(&right);
585 + rc = posListUnion(&left, &right, &writer);
586 + if( rc!=SQLITE_OK ) break;
587 + rc = dlrStep(&left);
588 + if( rc!=SQLITE_OK ) break;
589 + rc = dlrStep(&right);
590 + if( rc!=SQLITE_OK ) break;
591 }
592 }
594 dlrDestroy(&left);
595 dlrDestroy(&right);
596 dlwDestroy(&writer);
597 + return rc;
598 }
600 /* pLeft and pRight are DLReaders positioned to the same docid.
601 @@ -1407,35 +1523,47 @@
602 ** include the positions from pRight that are one more than a
603 ** position in pLeft. In other words: pRight.iPos==pLeft.iPos+1.
604 */
605 -static void posListPhraseMerge(DLReader *pLeft, DLReader *pRight,
606 - DLWriter *pOut){
607 +static int posListPhraseMerge(DLReader *pLeft, DLReader *pRight,
608 + DLWriter *pOut){
609 PLReader left, right;
610 PLWriter writer;
611 int match = 0;
612 + int rc;
614 assert( dlrDocid(pLeft)==dlrDocid(pRight) );
615 assert( pOut->iType!=DL_POSITIONS_OFFSETS );
617 - plrInit(&left, pLeft);
618 - plrInit(&right, pRight);
619 + rc = plrInit(&left, pLeft);
620 + if( rc!=SQLITE_OK ) return rc;
621 + rc = plrInit(&right, pRight);
622 + if( rc!=SQLITE_OK ){
623 + plrDestroy(&left);
624 + return rc;
625 + }
627 while( !plrAtEnd(&left) && !plrAtEnd(&right) ){
628 if( plrColumn(&left)<plrColumn(&right) ){
629 - plrStep(&left);
630 + rc = plrStep(&left);
631 + if( rc!=SQLITE_OK ) break;
632 }else if( plrColumn(&left)>plrColumn(&right) ){
633 - plrStep(&right);
634 + rc = plrStep(&right);
635 + if( rc!=SQLITE_OK ) break;
636 }else if( plrPosition(&left)+1<plrPosition(&right) ){
637 - plrStep(&left);
638 + rc = plrStep(&left);
639 + if( rc!=SQLITE_OK ) break;
640 }else if( plrPosition(&left)+1>plrPosition(&right) ){
641 - plrStep(&right);
642 + rc = plrStep(&right);
643 + if( rc!=SQLITE_OK ) break;
644 }else{
645 if( !match ){
646 plwInit(&writer, pOut, dlrDocid(pLeft));
647 match = 1;
648 }
649 plwAdd(&writer, plrColumn(&right), plrPosition(&right), 0, 0);
650 - plrStep(&left);
651 - plrStep(&right);
652 + rc = plrStep(&left);
653 + if( rc!=SQLITE_OK ) break;
654 + rc = plrStep(&right);
655 + if( rc!=SQLITE_OK ) break;
656 }
657 }
659 @@ -1446,6 +1574,7 @@
661 plrDestroy(&left);
662 plrDestroy(&right);
663 + return rc;
664 }
666 /* We have two doclists with positions: pLeft and pRight.
667 @@ -1457,7 +1586,7 @@
668 ** iType controls the type of data written to pOut. If iType is
669 ** DL_POSITIONS, the positions are those from pRight.
670 */
671 -static void docListPhraseMerge(
672 +static int docListPhraseMerge(
673 const char *pLeft, int nLeft,
674 const char *pRight, int nRight,
675 DocListType iType,
676 @@ -1465,152 +1594,198 @@
677 ){
678 DLReader left, right;
679 DLWriter writer;
680 + int rc;
682 - if( nLeft==0 || nRight==0 ) return;
683 + if( nLeft==0 || nRight==0 ) return SQLITE_OK;
685 assert( iType!=DL_POSITIONS_OFFSETS );
687 - dlrInit(&left, DL_POSITIONS, pLeft, nLeft);
688 - dlrInit(&right, DL_POSITIONS, pRight, nRight);
689 + rc = dlrInit(&left, DL_POSITIONS, pLeft, nLeft);
690 + if( rc!=SQLITE_OK ) return rc;
691 + rc = dlrInit(&right, DL_POSITIONS, pRight, nRight);
692 + if( rc!=SQLITE_OK ){
693 + dlrDestroy(&left);
694 + return rc;
695 + }
696 dlwInit(&writer, iType, pOut);
698 while( !dlrAtEnd(&left) && !dlrAtEnd(&right) ){
699 if( dlrDocid(&left)<dlrDocid(&right) ){
700 - dlrStep(&left);
701 + rc = dlrStep(&left);
702 + if( rc!=SQLITE_OK ) break;
703 }else if( dlrDocid(&right)<dlrDocid(&left) ){
704 - dlrStep(&right);
705 + rc = dlrStep(&right);
706 + if( rc!=SQLITE_OK ) break;
707 }else{
708 - posListPhraseMerge(&left, &right, &writer);
709 - dlrStep(&left);
710 - dlrStep(&right);
711 + rc = posListPhraseMerge(&left, &right, &writer);
712 + if( rc!=SQLITE_OK ) break;
713 + rc = dlrStep(&left);
714 + if( rc!=SQLITE_OK ) break;
715 + rc = dlrStep(&right);
716 + if( rc!=SQLITE_OK ) break;
717 }
718 }
720 dlrDestroy(&left);
721 dlrDestroy(&right);
722 dlwDestroy(&writer);
723 + return rc;
724 }
726 /* We have two DL_DOCIDS doclists: pLeft and pRight.
727 ** Write the intersection of these two doclists into pOut as a
728 ** DL_DOCIDS doclist.
729 */
730 -static void docListAndMerge(
731 +static int docListAndMerge(
732 const char *pLeft, int nLeft,
733 const char *pRight, int nRight,
734 DataBuffer *pOut /* Write the combined doclist here */
735 ){
736 DLReader left, right;
737 DLWriter writer;
738 + int rc;
740 - if( nLeft==0 || nRight==0 ) return;
741 + if( nLeft==0 || nRight==0 ) return SQLITE_OK;
743 - dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
744 - dlrInit(&right, DL_DOCIDS, pRight, nRight);
745 + rc = dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
746 + if( rc!=SQLITE_OK ) return rc;
747 + rc = dlrInit(&right, DL_DOCIDS, pRight, nRight);
748 + if( rc!=SQLITE_OK ){
749 + dlrDestroy(&left);
750 + return rc;
751 + }
752 dlwInit(&writer, DL_DOCIDS, pOut);
754 while( !dlrAtEnd(&left) && !dlrAtEnd(&right) ){
755 if( dlrDocid(&left)<dlrDocid(&right) ){
756 - dlrStep(&left);
757 + rc = dlrStep(&left);
758 + if( rc!=SQLITE_OK ) break;
759 }else if( dlrDocid(&right)<dlrDocid(&left) ){
760 - dlrStep(&right);
761 + rc = dlrStep(&right);
762 + if( rc!=SQLITE_OK ) break;
763 }else{
764 dlwAdd(&writer, dlrDocid(&left));
765 - dlrStep(&left);
766 - dlrStep(&right);
767 + rc = dlrStep(&left);
768 + if( rc!=SQLITE_OK ) break;
769 + rc = dlrStep(&right);
770 + if( rc!=SQLITE_OK ) break;
771 }
772 }
774 dlrDestroy(&left);
775 dlrDestroy(&right);
776 dlwDestroy(&writer);
777 + return rc;
778 }
780 /* We have two DL_DOCIDS doclists: pLeft and pRight.
781 ** Write the union of these two doclists into pOut as a
782 ** DL_DOCIDS doclist.
783 */
784 -static void docListOrMerge(
785 +static int docListOrMerge(
786 const char *pLeft, int nLeft,
787 const char *pRight, int nRight,
788 DataBuffer *pOut /* Write the combined doclist here */
789 ){
790 DLReader left, right;
791 DLWriter writer;
792 + int rc;
794 if( nLeft==0 ){
795 if( nRight!=0 ) dataBufferAppend(pOut, pRight, nRight);
796 - return;
797 + return SQLITE_OK;
798 }
799 if( nRight==0 ){
800 dataBufferAppend(pOut, pLeft, nLeft);
801 - return;
802 + return SQLITE_OK;
803 }
805 - dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
806 - dlrInit(&right, DL_DOCIDS, pRight, nRight);
807 + rc = dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
808 + if( rc!=SQLITE_OK ) return rc;
809 + rc = dlrInit(&right, DL_DOCIDS, pRight, nRight);
810 + if( rc!=SQLITE_OK ){
811 + dlrDestroy(&left);
812 + return rc;
813 + }
814 dlwInit(&writer, DL_DOCIDS, pOut);
816 while( !dlrAtEnd(&left) || !dlrAtEnd(&right) ){
817 if( dlrAtEnd(&right) ){
818 dlwAdd(&writer, dlrDocid(&left));
819 - dlrStep(&left);
820 + rc = dlrStep(&left);
821 + if( rc!=SQLITE_OK ) break;
822 }else if( dlrAtEnd(&left) ){
823 dlwAdd(&writer, dlrDocid(&right));
824 - dlrStep(&right);
825 + rc = dlrStep(&right);
826 + if( rc!=SQLITE_OK ) break;
827 }else if( dlrDocid(&left)<dlrDocid(&right) ){
828 dlwAdd(&writer, dlrDocid(&left));
829 - dlrStep(&left);
830 + rc = dlrStep(&left);
831 + if( rc!=SQLITE_OK ) break;
832 }else if( dlrDocid(&right)<dlrDocid(&left) ){
833 dlwAdd(&writer, dlrDocid(&right));
834 - dlrStep(&right);
835 + rc = dlrStep(&right);
836 + if( rc!=SQLITE_OK ) break;
837 }else{
838 dlwAdd(&writer, dlrDocid(&left));
839 - dlrStep(&left);
840 - dlrStep(&right);
841 + rc = dlrStep(&left);
842 + if( rc!=SQLITE_OK ) break;
843 + rc = dlrStep(&right);
844 + if( rc!=SQLITE_OK ) break;
845 }
846 }
848 dlrDestroy(&left);
849 dlrDestroy(&right);
850 dlwDestroy(&writer);
851 + return rc;
852 }
854 /* We have two DL_DOCIDS doclists: pLeft and pRight.
855 ** Write into pOut as DL_DOCIDS doclist containing all documents that
856 ** occur in pLeft but not in pRight.
857 */
858 -static void docListExceptMerge(
859 +static int docListExceptMerge(
860 const char *pLeft, int nLeft,
861 const char *pRight, int nRight,
862 DataBuffer *pOut /* Write the combined doclist here */
863 ){
864 DLReader left, right;
865 DLWriter writer;
866 + int rc;
868 - if( nLeft==0 ) return;
869 + if( nLeft==0 ) return SQLITE_OK;
870 if( nRight==0 ){
871 dataBufferAppend(pOut, pLeft, nLeft);
872 - return;
873 + return SQLITE_OK;
874 }
876 - dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
877 - dlrInit(&right, DL_DOCIDS, pRight, nRight);
878 + rc = dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
879 + if( rc!=SQLITE_OK ) return rc;
880 + rc = dlrInit(&right, DL_DOCIDS, pRight, nRight);
881 + if( rc!=SQLITE_OK ){
882 + dlrDestroy(&left);
883 + return rc;
884 + }
885 dlwInit(&writer, DL_DOCIDS, pOut);
887 while( !dlrAtEnd(&left) ){
888 while( !dlrAtEnd(&right) && dlrDocid(&right)<dlrDocid(&left) ){
889 - dlrStep(&right);
890 + rc = dlrStep(&right);
891 + if( rc!=SQLITE_OK ) goto err;
892 }
893 if( dlrAtEnd(&right) || dlrDocid(&left)<dlrDocid(&right) ){
894 dlwAdd(&writer, dlrDocid(&left));
895 }
896 - dlrStep(&left);
897 + rc = dlrStep(&left);
898 + if( rc!=SQLITE_OK ) break;
899 }
901 +err:
902 dlrDestroy(&left);
903 dlrDestroy(&right);
904 dlwDestroy(&writer);
905 + return rc;
906 }
908 static char *string_dup_n(const char *s, int n){
909 @@ -1814,7 +1989,7 @@
910 /* SEGDIR_MAX_INDEX */ "select max(idx) from %_segdir where level = ?",
911 /* SEGDIR_SET */ "insert into %_segdir values (?, ?, ?, ?, ?, ?)",
912 /* SEGDIR_SELECT_LEVEL */
913 - "select start_block, leaves_end_block, root from %_segdir "
914 + "select start_block, leaves_end_block, root, idx from %_segdir "
915 " where level = ? order by idx",
916 /* SEGDIR_SPAN */
917 "select min(start_block), max(end_block) from %_segdir "
918 @@ -3413,7 +3588,8 @@
919 return SQLITE_OK;
920 }
921 rc = sqlite3_bind_int64(c->pStmt, 1, dlrDocid(&c->reader));
922 - dlrStep(&c->reader);
923 + if( rc!=SQLITE_OK ) return rc;
924 + rc = dlrStep(&c->reader);
925 if( rc!=SQLITE_OK ) return rc;
926 /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */
927 rc = sqlite3_step(c->pStmt);
928 @@ -3421,8 +3597,11 @@
929 c->eof = 0;
930 return SQLITE_OK;
931 }
932 - /* an error occurred; abort */
933 - return rc==SQLITE_DONE ? SQLITE_ERROR : rc;
934 +
935 + /* Corrupt if the index refers to missing document. */
936 + if( rc==SQLITE_DONE ) return SQLITE_CORRUPT_BKPT;
937 +
938 + return rc;
939 }
940 }
942 @@ -3470,14 +3649,18 @@
943 return rc;
944 }
945 dataBufferInit(&new, 0);
946 - docListPhraseMerge(left.pData, left.nData, right.pData, right.nData,
947 - i<pQTerm->nPhrase ? DL_POSITIONS : DL_DOCIDS, &new);
948 + rc = docListPhraseMerge(left.pData, left.nData, right.pData, right.nData,
949 + i<pQTerm->nPhrase ? DL_POSITIONS : DL_DOCIDS, &new);
950 dataBufferDestroy(&left);
951 dataBufferDestroy(&right);
952 + if( rc!=SQLITE_OK ){
953 + dataBufferDestroy(&new);
954 + return rc;
955 + }
956 left = new;
957 }
958 *pResult = left;
959 - return SQLITE_OK;
960 + return rc;
961 }
963 /* Add a new term pTerm[0..nTerm-1] to the query *q.
964 @@ -3544,6 +3727,7 @@
965 int firstIndex = pQuery->nTerms;
966 int iCol;
967 int nTerm = 1;
968 + int iEndLast = -1;
970 int rc = pModule->xOpen(pTokenizer, pSegment, nSegment, &pCursor);
971 if( rc!=SQLITE_OK ) return rc;
972 @@ -3568,6 +3752,20 @@
973 pQuery->nextIsOr = 1;
974 continue;
975 }
976 +
977 + /*
978 + * The ICU tokenizer considers '*' a break character, so the code below
979 + * sets isPrefix correctly, but since that code doesn't eat the '*', the
980 + * ICU tokenizer returns it as the next token. So eat it here until a
981 + * better solution presents itself.
982 + */
983 + if( pQuery->nTerms>0 && nToken==1 && pSegment[iBegin]=='*' &&
984 + iEndLast==iBegin){
985 + pQuery->pTerms[pQuery->nTerms-1].isPrefix = 1;
986 + continue;
987 + }
988 + iEndLast = iEnd;
989 +
990 queryAdd(pQuery, pToken, nToken);
991 if( !inPhrase && iBegin>0 && pSegment[iBegin-1]=='-' ){
992 pQuery->pTerms[pQuery->nTerms-1].isNot = 1;
993 @@ -3707,18 +3905,30 @@
994 return rc;
995 }
996 dataBufferInit(&new, 0);
997 - docListOrMerge(right.pData, right.nData, or.pData, or.nData, &new);
998 + rc = docListOrMerge(right.pData, right.nData, or.pData, or.nData, &new);
999 dataBufferDestroy(&right);
1000 dataBufferDestroy(&or);
1001 + if( rc!=SQLITE_OK ){
1002 + if( i!=nNot ) dataBufferDestroy(&left);
1003 + queryClear(pQuery);
1004 + dataBufferDestroy(&new);
1005 + return rc;
1006 + }
1007 right = new;
1008 }
1009 if( i==nNot ){ /* first term processed. */
1010 left = right;
1011 }else{
1012 dataBufferInit(&new, 0);
1013 - docListAndMerge(left.pData, left.nData, right.pData, right.nData, &new);
1014 + rc = docListAndMerge(left.pData, left.nData,
1015 + right.pData, right.nData, &new);
1016 dataBufferDestroy(&right);
1017 dataBufferDestroy(&left);
1018 + if( rc!=SQLITE_OK ){
1019 + queryClear(pQuery);
1020 + dataBufferDestroy(&new);
1021 + return rc;
1022 + }
1023 left = new;
1024 }
1025 }
1026 @@ -3738,9 +3948,15 @@
1027 return rc;
1028 }
1029 dataBufferInit(&new, 0);
1030 - docListExceptMerge(left.pData, left.nData, right.pData, right.nData, &new);
1031 + rc = docListExceptMerge(left.pData, left.nData,
1032 + right.pData, right.nData, &new);
1033 dataBufferDestroy(&right);
1034 dataBufferDestroy(&left);
1035 + if( rc!=SQLITE_OK ){
1036 + queryClear(pQuery);
1037 + dataBufferDestroy(&new);
1038 + return rc;
1039 + }
1040 left = new;
1041 }
1043 @@ -3834,7 +4050,8 @@
1044 rc = fulltextQuery(v, idxNum-QUERY_FULLTEXT, zQuery, -1, &c->result, &c->q);
1045 if( rc!=SQLITE_OK ) return rc;
1046 if( c->result.nData!=0 ){
1047 - dlrInit(&c->reader, DL_DOCIDS, c->result.pData, c->result.nData);
1048 + rc = dlrInit(&c->reader, DL_DOCIDS, c->result.pData, c->result.nData);
1049 + if( rc!=SQLITE_OK ) return rc;
1050 }
1051 break;
1052 }
1053 @@ -4335,22 +4552,19 @@
1054 SCRAMBLE(pReader);
1055 }
1057 -/* TODO(shess) The assertions are great, but what if we're in NDEBUG
1058 -** and the blob is empty or otherwise contains suspect data?
1059 -*/
1060 -static void interiorReaderInit(const char *pData, int nData,
1061 - InteriorReader *pReader){
1062 +static int interiorReaderInit(const char *pData, int nData,
1063 + InteriorReader *pReader){
1064 int n, nTerm;
1066 - /* Require at least the leading flag byte */
1067 + /* These conditions are checked and met by the callers. */
1068 assert( nData>0 );
1069 assert( pData[0]!='\0' );
1071 CLEAR(pReader);
1073 /* Decode the base blockid, and set the cursor to the first term. */
1074 - n = getVarint(pData+1, &pReader->iBlockid);
1075 - assert( 1+n<=nData );
1076 + n = getVarintSafe(pData+1, &pReader->iBlockid, nData-1);
1077 + if( !n ) return SQLITE_CORRUPT_BKPT;
1078 pReader->pData = pData+1+n;
1079 pReader->nData = nData-(1+n);
1081 @@ -4361,17 +4575,18 @@
1082 if( pReader->nData==0 ){
1083 dataBufferInit(&pReader->term, 0);
1084 }else{
1085 - n = getVarint32(pReader->pData, &nTerm);
1086 + n = getVarint32Safe(pReader->pData, &nTerm, pReader->nData);
1087 + if( !n || nTerm<0 || nTerm>pReader->nData-n) return SQLITE_CORRUPT_BKPT;
1088 dataBufferInit(&pReader->term, nTerm);
1089 dataBufferReplace(&pReader->term, pReader->pData+n, nTerm);
1090 - assert( n+nTerm<=pReader->nData );
1091 pReader->pData += n+nTerm;
1092 pReader->nData -= n+nTerm;
1093 }
1094 + return SQLITE_OK;
1095 }
1097 static int interiorReaderAtEnd(InteriorReader *pReader){
1098 - return pReader->term.nData==0;
1099 + return pReader->term.nData<=0;
1100 }
1102 static sqlite_int64 interiorReaderCurrentBlockid(InteriorReader *pReader){
1103 @@ -4388,7 +4603,7 @@
1104 }
1106 /* Step forward to the next term in the node. */
1107 -static void interiorReaderStep(InteriorReader *pReader){
1108 +static int interiorReaderStep(InteriorReader *pReader){
1109 assert( !interiorReaderAtEnd(pReader) );
1111 /* If the last term has been read, signal eof, else construct the
1112 @@ -4399,18 +4614,26 @@
1113 }else{
1114 int n, nPrefix, nSuffix;
1116 - n = getVarint32(pReader->pData, &nPrefix);
1117 - n += getVarint32(pReader->pData+n, &nSuffix);
1118 + n = getVarint32Safe(pReader->pData, &nPrefix, pReader->nData);
1119 + if( !n ) return SQLITE_CORRUPT_BKPT;
1120 + pReader->nData -= n;
1121 + pReader->pData += n;
1122 + n = getVarint32Safe(pReader->pData, &nSuffix, pReader->nData);
1123 + if( !n ) return SQLITE_CORRUPT_BKPT;
1124 + pReader->nData -= n;
1125 + pReader->pData += n;
1126 + if( nSuffix<0 || nSuffix>pReader->nData ) return SQLITE_CORRUPT_BKPT;
1127 + if( nPrefix<0 || nPrefix>pReader->term.nData ) return SQLITE_CORRUPT_BKPT;
1129 /* Truncate the current term and append suffix data. */
1130 pReader->term.nData = nPrefix;
1131 - dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix);
1132 + dataBufferAppend(&pReader->term, pReader->pData, nSuffix);
1134 - assert( n+nSuffix<=pReader->nData );
1135 - pReader->pData += n+nSuffix;
1136 - pReader->nData -= n+nSuffix;
1137 + pReader->pData += nSuffix;
1138 + pReader->nData -= nSuffix;
1139 }
1140 pReader->iBlockid++;
1141 + return SQLITE_OK;
1142 }
1144 /* Compare the current term to pTerm[nTerm], returning strcmp-style
1145 @@ -4782,7 +5005,8 @@
1146 n = putVarint(c, nData);
1147 dataBufferAppend(&pWriter->data, c, n);
1149 - docListMerge(&pWriter->data, pReaders, nReaders);
1150 + rc = docListMerge(&pWriter->data, pReaders, nReaders);
1151 + if( rc!= SQLITE_OK ) return rc;
1152 ASSERT_VALID_DOCLIST(DL_DEFAULT,
1153 pWriter->data.pData+iDoclistData+n,
1154 pWriter->data.nData-iDoclistData-n, NULL);
1155 @@ -4892,7 +5116,8 @@
1156 int rc;
1157 DLReader reader;
1159 - dlrInit(&reader, DL_DEFAULT, pData, nData);
1160 + rc = dlrInit(&reader, DL_DEFAULT, pData, nData);
1161 + if( rc!=SQLITE_OK ) return rc;
1162 rc = leafWriterStepMerge(v, pWriter, pTerm, nTerm, &reader, 1);
1163 dlrDestroy(&reader);
1165 @@ -4937,38 +5162,40 @@
1166 static const char *leafReaderData(LeafReader *pReader){
1167 int n, nData;
1168 assert( pReader->term.nData>0 );
1169 - n = getVarint32(pReader->pData, &nData);
1170 + n = getVarint32Safe(pReader->pData, &nData, pReader->nData);
1171 + if( !n || nData>pReader->nData-n ) return NULL;
1172 return pReader->pData+n;
1173 }
1175 -static void leafReaderInit(const char *pData, int nData,
1176 - LeafReader *pReader){
1177 +static int leafReaderInit(const char *pData, int nData, LeafReader *pReader){
1178 int nTerm, n;
1180 + /* All callers check this precondition. */
1181 assert( nData>0 );
1182 assert( pData[0]=='\0' );
1184 CLEAR(pReader);
1186 /* Read the first term, skipping the header byte. */
1187 - n = getVarint32(pData+1, &nTerm);
1188 + n = getVarint32Safe(pData+1, &nTerm, nData-1);
1189 + if( !n || nTerm<0 || nTerm>nData-1-n ) return SQLITE_CORRUPT_BKPT;
1190 dataBufferInit(&pReader->term, nTerm);
1191 dataBufferReplace(&pReader->term, pData+1+n, nTerm);
1193 /* Position after the first term. */
1194 - assert( 1+n+nTerm<nData );
1195 pReader->pData = pData+1+n+nTerm;
1196 pReader->nData = nData-1-n-nTerm;
1197 + return SQLITE_OK;
1198 }
1200 /* Step the reader forward to the next term. */
1201 -static void leafReaderStep(LeafReader *pReader){
1202 +static int leafReaderStep(LeafReader *pReader){
1203 int n, nData, nPrefix, nSuffix;
1204 assert( !leafReaderAtEnd(pReader) );
1206 /* Skip previous entry's data block. */
1207 - n = getVarint32(pReader->pData, &nData);
1208 - assert( n+nData<=pReader->nData );
1209 + n = getVarint32Safe(pReader->pData, &nData, pReader->nData);
1210 + if( !n || nData<0 || nData>pReader->nData-n ) return SQLITE_CORRUPT_BKPT;
1211 pReader->pData += n+nData;
1212 pReader->nData -= n+nData;
1214 @@ -4976,15 +5203,23 @@
1215 /* Construct the new term using a prefix from the old term plus a
1216 ** suffix from the leaf data.
1217 */
1218 - n = getVarint32(pReader->pData, &nPrefix);
1219 - n += getVarint32(pReader->pData+n, &nSuffix);
1220 - assert( n+nSuffix<pReader->nData );
1221 + n = getVarint32Safe(pReader->pData, &nPrefix, pReader->nData);
1222 + if( !n ) return SQLITE_CORRUPT_BKPT;
1223 + pReader->nData -= n;
1224 + pReader->pData += n;
1225 + n = getVarint32Safe(pReader->pData, &nSuffix, pReader->nData);
1226 + if( !n ) return SQLITE_CORRUPT_BKPT;
1227 + pReader->nData -= n;
1228 + pReader->pData += n;
1229 + if( nSuffix<0 || nSuffix>pReader->nData ) return SQLITE_CORRUPT_BKPT;
1230 + if( nPrefix<0 || nPrefix>pReader->term.nData ) return SQLITE_CORRUPT_BKPT;
1231 pReader->term.nData = nPrefix;
1232 - dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix);
1233 + dataBufferAppend(&pReader->term, pReader->pData, nSuffix);
1235 - pReader->pData += n+nSuffix;
1236 - pReader->nData -= n+nSuffix;
1237 + pReader->pData += nSuffix;
1238 + pReader->nData -= nSuffix;
1239 }
1240 + return SQLITE_OK;
1241 }
1243 /* strcmp-style comparison of pReader's current term against pTerm.
1244 @@ -5077,6 +5312,9 @@
1245 ** the leaf data was entirely contained in the root), or from the
1246 ** stream of blocks between iStartBlockid and iEndBlockid, inclusive.
1247 */
1248 +/* TODO(shess): Figure out a means of indicating how many leaves are
1249 +** expected, for purposes of detecting corruption.
1250 +*/
1251 static int leavesReaderInit(fulltext_vtab *v,
1252 int idx,
1253 sqlite_int64 iStartBlockid,
1254 @@ -5088,32 +5326,67 @@
1256 dataBufferInit(&pReader->rootData, 0);
1257 if( iStartBlockid==0 ){
1258 + int rc;
1259 + /* Corrupt if this can't be a leaf node. */
1260 + if( pRootData==NULL || nRootData<1 || pRootData[0]!='\0' ){
1261 + return SQLITE_CORRUPT_BKPT;
1262 + }
1263 /* Entire leaf level fit in root data. */
1264 dataBufferReplace(&pReader->rootData, pRootData, nRootData);
1265 - leafReaderInit(pReader->rootData.pData, pReader->rootData.nData,
1266 - &pReader->leafReader);
1267 + rc = leafReaderInit(pReader->rootData.pData, pReader->rootData.nData,
1268 + &pReader->leafReader);
1269 + if( rc!=SQLITE_OK ){
1270 + dataBufferDestroy(&pReader->rootData);
1271 + return rc;
1272 + }
1273 }else{
1274 sqlite3_stmt *s;
1275 int rc = sql_get_leaf_statement(v, idx, &s);
1276 if( rc!=SQLITE_OK ) return rc;
1278 rc = sqlite3_bind_int64(s, 1, iStartBlockid);
1279 - if( rc!=SQLITE_OK ) return rc;
1280 + if( rc!=SQLITE_OK ) goto err;
1282 rc = sqlite3_bind_int64(s, 2, iEndBlockid);
1283 - if( rc!=SQLITE_OK ) return rc;
1284 + if( rc!=SQLITE_OK ) goto err;
1286 rc = sqlite3_step(s);
1287 +
1288 + /* Corrupt if interior node referenced missing leaf node. */
1289 if( rc==SQLITE_DONE ){
1290 - pReader->eof = 1;
1291 - return SQLITE_OK;
1292 + rc = SQLITE_CORRUPT_BKPT;
1293 + goto err;
1294 + }
1295 +
1296 + if( rc!=SQLITE_ROW ) goto err;
1297 + rc = SQLITE_OK;
1298 +
1299 + /* Corrupt if leaf data isn't a blob. */
1300 + if( sqlite3_column_type(s, 0)!=SQLITE_BLOB ){
1301 + rc = SQLITE_CORRUPT_BKPT;
1302 + }else{
1303 + const char *pLeafData = sqlite3_column_blob(s, 0);
1304 + int nLeafData = sqlite3_column_bytes(s, 0);
1305 +
1306 + /* Corrupt if this can't be a leaf node. */
1307 + if( pLeafData==NULL || nLeafData<1 || pLeafData[0]!='\0' ){
1308 + rc = SQLITE_CORRUPT_BKPT;
1309 + }else{
1310 + rc = leafReaderInit(pLeafData, nLeafData, &pReader->leafReader);
1311 + }
1312 + }
1313 +
1314 + err:
1315 + if( rc!=SQLITE_OK ){
1316 + if( idx==-1 ){
1317 + sqlite3_finalize(s);
1318 + }else{
1319 + sqlite3_reset(s);
1320 + }
1321 + return rc;
1322 }
1323 - if( rc!=SQLITE_ROW ) return rc;
1325 pReader->pStmt = s;
1326 - leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0),
1327 - sqlite3_column_bytes(pReader->pStmt, 0),
1328 - &pReader->leafReader);
1329 }
1330 return SQLITE_OK;
1331 }
1332 @@ -5122,11 +5395,12 @@
1333 ** end of the current leaf, step forward to the next leaf block.
1334 */
1335 static int leavesReaderStep(fulltext_vtab *v, LeavesReader *pReader){
1336 + int rc;
1337 assert( !leavesReaderAtEnd(pReader) );
1338 - leafReaderStep(&pReader->leafReader);
1339 + rc = leafReaderStep(&pReader->leafReader);
1340 + if( rc!=SQLITE_OK ) return rc;
1342 if( leafReaderAtEnd(&pReader->leafReader) ){
1343 - int rc;
1344 if( pReader->rootData.pData ){
1345 pReader->eof = 1;
1346 return SQLITE_OK;
1347 @@ -5136,10 +5410,25 @@
1348 pReader->eof = 1;
1349 return rc==SQLITE_DONE ? SQLITE_OK : rc;
1350 }
1351 - leafReaderDestroy(&pReader->leafReader);
1352 - leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0),
1353 - sqlite3_column_bytes(pReader->pStmt, 0),
1354 - &pReader->leafReader);
1355 +
1356 + /* Corrupt if leaf data isn't a blob. */
1357 + if( sqlite3_column_type(pReader->pStmt, 0)!=SQLITE_BLOB ){
1358 + return SQLITE_CORRUPT_BKPT;
1359 + }else{
1360 + LeafReader tmp;
1361 + const char *pLeafData = sqlite3_column_blob(pReader->pStmt, 0);
1362 + int nLeafData = sqlite3_column_bytes(pReader->pStmt, 0);
1363 +
1364 + /* Corrupt if this can't be a leaf node. */
1365 + if( pLeafData==NULL || nLeafData<1 || pLeafData[0]!='\0' ){
1366 + return SQLITE_CORRUPT_BKPT;
1367 + }
1368 +
1369 + rc = leafReaderInit(pLeafData, nLeafData, &tmp);
1370 + if( rc!=SQLITE_OK ) return rc;
1371 + leafReaderDestroy(&pReader->leafReader);
1372 + pReader->leafReader = tmp;
1373 + }
1374 }
1375 return SQLITE_OK;
1376 }
1377 @@ -5200,8 +5489,19 @@
1378 sqlite_int64 iEnd = sqlite3_column_int64(s, 1);
1379 const char *pRootData = sqlite3_column_blob(s, 2);
1380 int nRootData = sqlite3_column_bytes(s, 2);
1381 + sqlite_int64 iIndex = sqlite3_column_int64(s, 3);
1382 +
1383 + /* Corrupt if we get back different types than we stored. */
1384 + /* Also corrupt if the index is not sequential starting at 0. */
1385 + if( sqlite3_column_type(s, 0)!=SQLITE_INTEGER ||
1386 + sqlite3_column_type(s, 1)!=SQLITE_INTEGER ||
1387 + sqlite3_column_type(s, 2)!=SQLITE_BLOB ||
1388 + i!=iIndex ||
1389 + i>=MERGE_COUNT ){
1390 + rc = SQLITE_CORRUPT_BKPT;
1391 + break;
1392 + }
1394 - assert( i<MERGE_COUNT );
1395 rc = leavesReaderInit(v, i, iStart, iEnd, pRootData, nRootData,
1396 &pReaders[i]);
1397 if( rc!=SQLITE_OK ) break;
1398 @@ -5212,6 +5512,7 @@
1399 while( i-->0 ){
1400 leavesReaderDestroy(&pReaders[i]);
1401 }
1402 + sqlite3_reset(s); /* So we don't leave a lock. */
1403 return rc;
1404 }
1406 @@ -5235,13 +5536,26 @@
1407 DLReader dlReaders[MERGE_COUNT];
1408 const char *pTerm = leavesReaderTerm(pReaders);
1409 int i, nTerm = leavesReaderTermBytes(pReaders);
1410 + int rc;
1412 assert( nReaders<=MERGE_COUNT );
1414 for(i=0; i<nReaders; i++){
1415 - dlrInit(&dlReaders[i], DL_DEFAULT,
1416 - leavesReaderData(pReaders+i),
1417 - leavesReaderDataBytes(pReaders+i));
1418 + const char *pData = leavesReaderData(pReaders+i);
1419 + if( pData==NULL ){
1420 + rc = SQLITE_CORRUPT_BKPT;
1421 + break;
1422 + }
1423 + rc = dlrInit(&dlReaders[i], DL_DEFAULT,
1424 + pData,
1425 + leavesReaderDataBytes(pReaders+i));
1426 + if( rc!=SQLITE_OK ) break;
1427 + }
1428 + if( rc!=SQLITE_OK ){
1429 + while( i-->0 ){
1430 + dlrDestroy(&dlReaders[i]);
1431 + }
1432 + return rc;
1433 }
1435 return leafWriterStepMerge(v, pWriter, pTerm, nTerm, dlReaders, nReaders);
1436 @@ -5295,10 +5609,14 @@
1437 memset(&lrs, '\0', sizeof(lrs));
1438 rc = leavesReadersInit(v, iLevel, lrs, &i);
1439 if( rc!=SQLITE_OK ) return rc;
1440 - assert( i==MERGE_COUNT );
1442 leafWriterInit(iLevel+1, idx, &writer);
1444 + if( i!=MERGE_COUNT ){
1445 + rc = SQLITE_CORRUPT_BKPT;
1446 + goto err;
1447 + }
1448 +
1449 /* Since leavesReaderReorder() pushes readers at eof to the end,
1450 ** when the first reader is empty, all will be empty.
1451 */
1452 @@ -5341,12 +5659,14 @@
1453 }
1455 /* Accumulate the union of *acc and *pData into *acc. */
1456 -static void docListAccumulateUnion(DataBuffer *acc,
1457 - const char *pData, int nData) {
1458 +static int docListAccumulateUnion(DataBuffer *acc,
1459 + const char *pData, int nData) {
1460 DataBuffer tmp = *acc;
1461 + int rc;
1462 dataBufferInit(acc, tmp.nData+nData);
1463 - docListUnion(tmp.pData, tmp.nData, pData, nData, acc);
1464 + rc = docListUnion(tmp.pData, tmp.nData, pData, nData, acc);
1465 dataBufferDestroy(&tmp);
1466 + return rc;
1467 }
1469 /* TODO(shess) It might be interesting to explore different merge
1470 @@ -5388,8 +5708,13 @@
1471 int c = leafReaderTermCmp(&pReader->leafReader, pTerm, nTerm, isPrefix);
1472 if( c>0 ) break; /* Past any possible matches. */
1473 if( c==0 ){
1474 + int iBuffer, nData;
1475 const char *pData = leavesReaderData(pReader);
1476 - int iBuffer, nData = leavesReaderDataBytes(pReader);
1477 + if( pData==NULL ){
1478 + rc = SQLITE_CORRUPT_BKPT;
1479 + break;
1480 + }
1481 + nData = leavesReaderDataBytes(pReader);
1483 /* Find the first empty buffer. */
1484 for(iBuffer=0; iBuffer<nBuffers; ++iBuffer){
1485 @@ -5435,11 +5760,13 @@
1486 ** with pData/nData.
1487 */
1488 dataBufferSwap(p, pAcc);
1489 - docListAccumulateUnion(pAcc, pData, nData);
1490 + rc = docListAccumulateUnion(pAcc, pData, nData);
1491 + if( rc!=SQLITE_OK ) goto err;
1493 /* Accumulate remaining doclists into pAcc. */
1494 for(++p; p<pAcc; ++p){
1495 - docListAccumulateUnion(pAcc, p->pData, p->nData);
1496 + rc = docListAccumulateUnion(pAcc, p->pData, p->nData);
1497 + if( rc!=SQLITE_OK ) goto err;
1499 /* dataBufferReset() could allow a large doclist to blow up
1500 ** our memory requirements.
1501 @@ -5464,13 +5791,15 @@
1502 if( out->nData==0 ){
1503 dataBufferSwap(out, &(pBuffers[iBuffer]));
1504 }else{
1505 - docListAccumulateUnion(out, pBuffers[iBuffer].pData,
1506 - pBuffers[iBuffer].nData);
1507 + rc = docListAccumulateUnion(out, pBuffers[iBuffer].pData,
1508 + pBuffers[iBuffer].nData);
1509 + if( rc!=SQLITE_OK ) break;
1510 }
1511 }
1512 }
1513 }
1515 +err:
1516 while( nBuffers-- ){
1517 dataBufferDestroy(&(pBuffers[nBuffers]));
1518 }
1519 @@ -5529,20 +5858,26 @@
1520 ** node. Consider whether breaking symmetry is worthwhile. I suspect
1521 ** it is not worthwhile.
1522 */
1523 -static void getChildrenContaining(const char *pData, int nData,
1524 - const char *pTerm, int nTerm, int isPrefix,
1525 - sqlite_int64 *piStartChild,
1526 - sqlite_int64 *piEndChild){
1527 +static int getChildrenContaining(const char *pData, int nData,
1528 + const char *pTerm, int nTerm, int isPrefix,
1529 + sqlite_int64 *piStartChild,
1530 + sqlite_int64 *piEndChild){
1531 InteriorReader reader;
1532 + int rc;
1534 assert( nData>1 );
1535 assert( *pData!='\0' );
1536 - interiorReaderInit(pData, nData, &reader);
1537 + rc = interiorReaderInit(pData, nData, &reader);
1538 + if( rc!=SQLITE_OK ) return rc;
1540 /* Scan for the first child which could contain pTerm/nTerm. */
1541 while( !interiorReaderAtEnd(&reader) ){
1542 if( interiorReaderTermCmp(&reader, pTerm, nTerm, 0)>0 ) break;
1543 - interiorReaderStep(&reader);
1544 + rc = interiorReaderStep(&reader);
1545 + if( rc!=SQLITE_OK ){
1546 + interiorReaderDestroy(&reader);
1547 + return rc;
1548 + }
1549 }
1550 *piStartChild = interiorReaderCurrentBlockid(&reader);
1552 @@ -5552,7 +5887,11 @@
1553 */
1554 while( !interiorReaderAtEnd(&reader) ){
1555 if( interiorReaderTermCmp(&reader, pTerm, nTerm, isPrefix)>0 ) break;
1556 - interiorReaderStep(&reader);
1557 + rc = interiorReaderStep(&reader);
1558 + if( rc!=SQLITE_OK ){
1559 + interiorReaderDestroy(&reader);
1560 + return rc;
1561 + }
1562 }
1563 *piEndChild = interiorReaderCurrentBlockid(&reader);
1565 @@ -5561,6 +5900,7 @@
1566 /* Children must ascend, and if !prefix, both must be the same. */
1567 assert( *piEndChild>=*piStartChild );
1568 assert( isPrefix || *piStartChild==*piEndChild );
1569 + return rc;
1570 }
1572 /* Read block at iBlockid and pass it with other params to
1573 @@ -5588,11 +5928,31 @@
1574 if( rc!=SQLITE_OK ) return rc;
1576 rc = sqlite3_step(s);
1577 - if( rc==SQLITE_DONE ) return SQLITE_ERROR;
1578 + /* Corrupt if interior node references missing child node. */
1579 + if( rc==SQLITE_DONE ) return SQLITE_CORRUPT_BKPT;
1580 if( rc!=SQLITE_ROW ) return rc;
1582 - getChildrenContaining(sqlite3_column_blob(s, 0), sqlite3_column_bytes(s, 0),
1583 - pTerm, nTerm, isPrefix, piStartChild, piEndChild);
1584 + /* Corrupt if child node isn't a blob. */
1585 + if( sqlite3_column_type(s, 0)!=SQLITE_BLOB ){
1586 + sqlite3_reset(s); /* So we don't leave a lock. */
1587 + return SQLITE_CORRUPT_BKPT;
1588 + }else{
1589 + const char *pData = sqlite3_column_blob(s, 0);
1590 + int nData = sqlite3_column_bytes(s, 0);
1591 +
1592 + /* Corrupt if child is not a valid interior node. */
1593 + if( pData==NULL || nData<1 || pData[0]=='\0' ){
1594 + sqlite3_reset(s); /* So we don't leave a lock. */
1595 + return SQLITE_CORRUPT_BKPT;
1596 + }
1597 +
1598 + rc = getChildrenContaining(pData, nData, pTerm, nTerm,
1599 + isPrefix, piStartChild, piEndChild);
1600 + if( rc!=SQLITE_OK ){
1601 + sqlite3_reset(s);
1602 + return rc;
1603 + }
1604 + }
1606 /* We expect only one row. We must execute another sqlite3_step()
1607 * to complete the iteration; otherwise the table will remain
1608 @@ -5622,8 +5982,9 @@
1609 /* Process pData as an interior node, then loop down the tree
1610 ** until we find the set of leaf nodes to scan for the term.
1611 */
1612 - getChildrenContaining(pData, nData, pTerm, nTerm, isPrefix,
1613 - &iStartChild, &iEndChild);
1614 + rc = getChildrenContaining(pData, nData, pTerm, nTerm, isPrefix,
1615 + &iStartChild, &iEndChild);
1616 + if( rc!=SQLITE_OK ) return rc;
1617 while( iStartChild>iLeavesEnd ){
1618 sqlite_int64 iNextStart, iNextEnd;
1619 rc = loadAndGetChildrenContaining(v, iStartChild, pTerm, nTerm, isPrefix,
1620 @@ -5675,7 +6036,8 @@
1621 DataBuffer result;
1622 int rc;
1624 - assert( nData>1 );
1625 + /* Corrupt if segment root can't be valid. */
1626 + if( pData==NULL || nData<1 ) return SQLITE_CORRUPT_BKPT;
1628 /* This code should never be called with buffered updates. */
1629 assert( v->nPendingData<0 );
1630 @@ -5692,16 +6054,21 @@
1631 DataBuffer merged;
1632 DLReader readers[2];
1634 - dlrInit(&readers[0], DL_DEFAULT, out->pData, out->nData);
1635 - dlrInit(&readers[1], DL_DEFAULT, result.pData, result.nData);
1636 - dataBufferInit(&merged, out->nData+result.nData);
1637 - docListMerge(&merged, readers, 2);
1638 - dataBufferDestroy(out);
1639 - *out = merged;
1640 - dlrDestroy(&readers[0]);
1641 - dlrDestroy(&readers[1]);
1642 + rc = dlrInit(&readers[0], DL_DEFAULT, out->pData, out->nData);
1643 + if( rc==SQLITE_OK ){
1644 + rc = dlrInit(&readers[1], DL_DEFAULT, result.pData, result.nData);
1645 + if( rc==SQLITE_OK ){
1646 + dataBufferInit(&merged, out->nData+result.nData);
1647 + rc = docListMerge(&merged, readers, 2);
1648 + dataBufferDestroy(out);
1649 + *out = merged;
1650 + dlrDestroy(&readers[1]);
1651 + }
1652 + dlrDestroy(&readers[0]);
1653 + }
1654 }
1655 }
1656 +
1657 dataBufferDestroy(&result);
1658 return rc;
1659 }
1660 @@ -5729,11 +6096,20 @@
1661 const char *pData = sqlite3_column_blob(s, 2);
1662 const int nData = sqlite3_column_bytes(s, 2);
1663 const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1);
1664 +
1665 + /* Corrupt if we get back different types than we stored. */
1666 + if( sqlite3_column_type(s, 1)!=SQLITE_INTEGER ||
1667 + sqlite3_column_type(s, 2)!=SQLITE_BLOB ){
1668 + rc = SQLITE_CORRUPT_BKPT;
1669 + goto err;
1670 + }
1671 +
1672 rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, isPrefix,
1673 &doclist);
1674 if( rc!=SQLITE_OK ) goto err;
1675 }
1676 if( rc==SQLITE_DONE ){
1677 + rc = SQLITE_OK;
1678 if( doclist.nData!=0 ){
1679 /* TODO(shess) The old term_select_all() code applied the column
1680 ** restrict as we merged segments, leading to smaller buffers.
1681 @@ -5741,13 +6117,13 @@
1682 ** system is checked in.
1683 */
1684 if( iColumn==v->nColumn) iColumn = -1;
1685 - docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
1686 - iColumn, iType, out);
1687 + rc = docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
1688 + iColumn, iType, out);
1689 }
1690 - rc = SQLITE_OK;
1691 }
1693 err:
1694 + sqlite3_reset(s); /* So we don't leave a lock. */
1695 dataBufferDestroy(&doclist);
1696 return rc;
1697 }
1698 @@ -6089,6 +6465,7 @@
1699 LeafWriter *pWriter){
1700 int i, rc = SQLITE_OK;
1701 DataBuffer doclist, merged, tmp;
1702 + const char *pData;
1704 /* Order the readers. */
1705 i = nReaders;
1706 @@ -6109,14 +6486,21 @@
1707 if( 0!=optLeavesReaderTermCmp(&readers[0], &readers[i]) ) break;
1708 }
1710 + pData = optLeavesReaderData(&readers[0]);
1711 + if( pData==NULL ){
1712 + rc = SQLITE_CORRUPT_BKPT;
1713 + break;
1714 + }
1715 +
1716 /* Special-case for no merge. */
1717 if( i==1 ){
1718 /* Trim deletions from the doclist. */
1719 dataBufferReset(&merged);
1720 - docListTrim(DL_DEFAULT,
1721 - optLeavesReaderData(&readers[0]),
1722 - optLeavesReaderDataBytes(&readers[0]),
1723 - -1, DL_DEFAULT, &merged);
1724 + rc = docListTrim(DL_DEFAULT,
1725 + pData,
1726 + optLeavesReaderDataBytes(&readers[0]),
1727 + -1, DL_DEFAULT, &merged);
1728 + if( rc!= SQLITE_OK ) break;
1729 }else{
1730 DLReader dlReaders[MERGE_COUNT];
1731 int iReader, nReaders;
1732 @@ -6124,9 +6508,10 @@
1733 /* Prime the pipeline with the first reader's doclist. After
1734 ** one pass index 0 will reference the accumulated doclist.
1735 */
1736 - dlrInit(&dlReaders[0], DL_DEFAULT,
1737 - optLeavesReaderData(&readers[0]),
1738 - optLeavesReaderDataBytes(&readers[0]));
1739 + rc = dlrInit(&dlReaders[0], DL_DEFAULT,
1740 + pData,
1741 + optLeavesReaderDataBytes(&readers[0]));
1742 + if( rc!=SQLITE_OK ) break;
1743 iReader = 1;
1745 assert( iReader<i ); /* Must execute the loop at least once. */
1746 @@ -6134,24 +6519,35 @@
1747 /* Merge 16 inputs per pass. */
1748 for( nReaders=1; iReader<i && nReaders<MERGE_COUNT;
1749 iReader++, nReaders++ ){
1750 - dlrInit(&dlReaders[nReaders], DL_DEFAULT,
1751 - optLeavesReaderData(&readers[iReader]),
1752 - optLeavesReaderDataBytes(&readers[iReader]));
1753 + pData = optLeavesReaderData(&readers[iReader]);
1754 + if( pData == NULL ){
1755 + rc = SQLITE_CORRUPT_BKPT;
1756 + break;
1757 + }
1758 + rc = dlrInit(&dlReaders[nReaders], DL_DEFAULT,
1759 + pData,
1760 + optLeavesReaderDataBytes(&readers[iReader]));
1761 + if( rc != SQLITE_OK ) break;
1762 }
1764 /* Merge doclists and swap result into accumulator. */
1765 - dataBufferReset(&merged);
1766 - docListMerge(&merged, dlReaders, nReaders);
1767 - tmp = merged;
1768 - merged = doclist;
1769 - doclist = tmp;
1770 + if( rc==SQLITE_OK ){
1771 + dataBufferReset(&merged);
1772 + rc = docListMerge(&merged, dlReaders, nReaders);
1773 + tmp = merged;
1774 + merged = doclist;
1775 + doclist = tmp;
1776 + }
1778 while( nReaders-- > 0 ){
1779 dlrDestroy(&dlReaders[nReaders]);
1780 }
1782 + if( rc!=SQLITE_OK ) goto err;
1783 +
1784 /* Accumulated doclist to reader 0 for next pass. */
1785 - dlrInit(&dlReaders[0], DL_DEFAULT, doclist.pData, doclist.nData);
1786 + rc = dlrInit(&dlReaders[0], DL_DEFAULT, doclist.pData, doclist.nData);
1787 + if( rc!=SQLITE_OK ) goto err;
1788 }
1790 /* Destroy reader that was left in the pipeline. */
1791 @@ -6159,8 +6555,9 @@
1793 /* Trim deletions from the doclist. */
1794 dataBufferReset(&merged);
1795 - docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
1796 - -1, DL_DEFAULT, &merged);
1797 + rc = docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
1798 + -1, DL_DEFAULT, &merged);
1799 + if( rc!=SQLITE_OK ) goto err;
1800 }
1802 /* Only pass doclists with hits (skip if all hits deleted). */
1803 @@ -6240,6 +6637,14 @@
1804 const char *pRootData = sqlite3_column_blob(s, 2);
1805 int nRootData = sqlite3_column_bytes(s, 2);
1807 + /* Corrupt if we get back different types than we stored. */
1808 + if( sqlite3_column_type(s, 0)!=SQLITE_INTEGER ||
1809 + sqlite3_column_type(s, 1)!=SQLITE_INTEGER ||
1810 + sqlite3_column_type(s, 2)!=SQLITE_BLOB ){
1811 + rc = SQLITE_CORRUPT_BKPT;
1812 + break;
1813 + }
1814 +
1815 assert( i<nReaders );
1816 rc = leavesReaderInit(v, -1, iStart, iEnd, pRootData, nRootData,
1817 &readers[i].reader);
1818 @@ -6253,6 +6658,8 @@
1819 if( rc==SQLITE_DONE ){
1820 assert( i==nReaders );
1821 rc = optimizeInternal(v, readers, nReaders, &writer);
1822 + }else{
1823 + sqlite3_reset(s); /* So we don't leave a lock. */
1824 }
1826 while( i-- > 0 ){
1827 @@ -6316,9 +6723,18 @@
1828 const sqlite_int64 iEndBlockid = sqlite3_column_int64(s, 1);
1829 const char *pRootData = sqlite3_column_blob(s, 2);
1830 const int nRootData = sqlite3_column_bytes(s, 2);
1831 + int rc;
1832 LeavesReader reader;
1833 - int rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid,
1834 - pRootData, nRootData, &reader);
1835 +
1836 + /* Corrupt if we get back different types than we stored. */
1837 + if( sqlite3_column_type(s, 0)!=SQLITE_INTEGER ||
1838 + sqlite3_column_type(s, 1)!=SQLITE_INTEGER ||
1839 + sqlite3_column_type(s, 2)!=SQLITE_BLOB ){
1840 + return SQLITE_CORRUPT_BKPT;
1841 + }
1842 +
1843 + rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid,
1844 + pRootData, nRootData, &reader);
1845 if( rc!=SQLITE_OK ) return rc;
1847 while( rc==SQLITE_OK && !leavesReaderAtEnd(&reader) ){
1848 @@ -6480,16 +6896,19 @@
1849 const char *pData, int nData){
1850 DataBuffer dump;
1851 DLReader dlReader;
1852 + int rc;
1854 assert( pData!=NULL && nData>0 );
1856 + rc = dlrInit(&dlReader, DL_DEFAULT, pData, nData);
1857 + if( rc!=SQLITE_OK ) return rc;
1858 dataBufferInit(&dump, 0);
1859 - dlrInit(&dlReader, DL_DEFAULT, pData, nData);
1860 - for( ; !dlrAtEnd(&dlReader); dlrStep(&dlReader) ){
1861 + for( ; rc==SQLITE_OK && !dlrAtEnd(&dlReader); rc = dlrStep(&dlReader) ){
1862 char buf[256];
1863 PLReader plReader;
1865 - plrInit(&plReader, &dlReader);
1866 + rc = plrInit(&plReader, &dlReader);
1867 + if( rc!=SQLITE_OK ) break;
1868 if( DL_DEFAULT==DL_DOCIDS || plrAtEnd(&plReader) ){
1869 sqlite3_snprintf(sizeof(buf), buf, "[%lld] ", dlrDocid(&dlReader));
1870 dataBufferAppend(&dump, buf, strlen(buf));
1871 @@ -6500,7 +6919,8 @@
1872 dlrDocid(&dlReader), iColumn);
1873 dataBufferAppend(&dump, buf, strlen(buf));
1875 - for( ; !plrAtEnd(&plReader); plrStep(&plReader) ){
1876 + for( ; !plrAtEnd(&plReader); rc = plrStep(&plReader) ){
1877 + if( rc!=SQLITE_OK ) break;
1878 if( plrColumn(&plReader)!=iColumn ){
1879 iColumn = plrColumn(&plReader);
1880 sqlite3_snprintf(sizeof(buf), buf, "] %d[", iColumn);
1881 @@ -6521,6 +6941,7 @@
1882 dataBufferAppend(&dump, buf, strlen(buf));
1883 }
1884 plrDestroy(&plReader);
1885 + if( rc!= SQLITE_OK ) break;
1887 assert( dump.nData>0 );
1888 dump.nData--; /* Overwrite trailing space. */
1889 @@ -6529,6 +6950,10 @@
1890 }
1891 }
1892 dlrDestroy(&dlReader);
1893 + if( rc!=SQLITE_OK ){
1894 + dataBufferDestroy(&dump);
1895 + return rc;
1896 + }
1898 assert( dump.nData>0 );
1899 dump.nData--; /* Overwrite trailing space. */
1900 @@ -6540,6 +6965,7 @@
1901 sqlite3_result_text(pContext, dump.pData, dump.nData, sqlite3_free);
1902 dump.pData = NULL;
1903 dump.nData = dump.nCapacity = 0;
1904 + return SQLITE_OK;
1905 }
1907 /* Implements dump_doclist() for use in inspecting the fts2 index from
1908 @@ -6822,7 +7248,11 @@
1909 ** module with sqlite.
1910 */
1911 if( SQLITE_OK==rc
1912 +#if GEARS_FTS2_CHANGES && !SQLITE_TEST
1913 + /* fts2_tokenizer() disabled for security reasons. */
1914 +#else
1915 && SQLITE_OK==(rc = sqlite3Fts2InitHashTable(db, pHash, "fts2_tokenizer"))
1916 +#endif
1917 && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
1918 && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", -1))
1919 && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", -1))
1920 diff -ru ext-orig/fts2/fts2_icu.c ext/fts2/fts2_icu.c
1921 --- ext-orig/fts2/fts2_icu.c 2009-09-03 13:32:06.000000000 -0700
1922 +++ ext/fts2/fts2_icu.c 2009-09-18 14:39:41.000000000 -0700
1923 @@ -198,7 +198,7 @@
1925 while( iStart<iEnd ){
1926 int iWhite = iStart;
1927 - U8_NEXT(pCsr->aChar, iWhite, pCsr->nChar, c);
1928 + U16_NEXT(pCsr->aChar, iWhite, pCsr->nChar, c);
1929 if( u_isspace(c) ){
1930 iStart = iWhite;
1931 }else{
1932 diff -ru ext-orig/fts2/fts2_tokenizer.c ext/fts2/fts2_tokenizer.c
1933 --- ext-orig/fts2/fts2_tokenizer.c 2009-09-03 13:32:06.000000000 -0700
1934 +++ ext/fts2/fts2_tokenizer.c 2009-09-18 14:39:41.000000000 -0700
1935 @@ -28,11 +28,14 @@
1937 #include "sqlite3.h"
1938 #include "sqlite3ext.h"
1939 -SQLITE_EXTENSION_INIT1
1940 +#ifndef SQLITE_CORE
1941 + SQLITE_EXTENSION_INIT1
1942 +#endif
1944 #include "fts2_hash.h"
1945 #include "fts2_tokenizer.h"
1946 #include <assert.h>
1947 +#include <stddef.h>
1949 /*
1950 ** Implementation of the SQL scalar function for accessing the underlying