DBD-SQLeet
view release on metacpan or search on metacpan
int nMem; /* Number of entries in aMem */
int nChildMem; /* Number of memory cells for child frame */
int nChildCsr; /* Number of cursors for child frame */
int nChange; /* Statement changes (Vdbe.nChange) */
int nDbChange; /* Value of db->nChange */
};
/* Magic number for sanity checking on VdbeFrame objects */
#define SQLITE_FRAME_MAGIC 0x879fb71e
/*
** Return a pointer to the array of registers allocated for use
** by a VdbeFrame.
*/
#define VdbeFrameMem(p) ((Mem *)&((u8 *)p)[ROUND8(sizeof(VdbeFrame))])
/*
** Internally, the vdbe manipulates nearly all SQL values as Mem
** structures. Each Mem struct may cache multiple representations (string,
** integer etc.) of the same value.
*/
struct sqlite3_value {
union MemValue {
double r; /* Real value used when MEM_Real is set in flags */
i64 i; /* Integer value used when MEM_Int is set in flags */
int nZero; /* Extra zero bytes when MEM_Zero and MEM_Blob set */
const char *zPType; /* Pointer type when MEM_Term|MEM_Subtype|MEM_Null */
FuncDef *pDef; /* Used only when flags==MEM_Agg */
} u;
u16 flags; /* Some combination of MEM_Null, MEM_Str, MEM_Dyn, etc. */
u8 enc; /* SQLITE_UTF8, SQLITE_UTF16BE, SQLITE_UTF16LE */
u8 eSubtype; /* Subtype for this value */
int n; /* Number of characters in string value, excluding '\0' */
char *z; /* String or BLOB value */
/* ShallowCopy only needs to copy the information above */
char *zMalloc; /* Space to hold MEM_Str or MEM_Blob if szMalloc>0 */
int szMalloc; /* Size of the zMalloc allocation */
u32 uTemp; /* Transient storage for serial_type in OP_MakeRecord */
sqlite3 *db; /* The associated database connection */
void (*xDel)(void*);/* Destructor for Mem.z - only valid if MEM_Dyn */
#ifdef SQLITE_DEBUG
Mem *pScopyFrom; /* This Mem is a shallow copy of pScopyFrom */
u16 mScopyFlags; /* flags value immediately after the shallow copy */
#endif
};
/*
** Size of struct Mem not including the Mem.zMalloc member or anything that
** follows.
*/
#define MEMCELLSIZE offsetof(Mem,zMalloc)
/* One or more of the following flags are set to indicate the validOK
** representations of the value stored in the Mem struct.
**
** If the MEM_Null flag is set, then the value is an SQL NULL value.
** For a pointer type created using sqlite3_bind_pointer() or
** sqlite3_result_pointer() the MEM_Term and MEM_Subtype flags are also set.
**
** If the MEM_Str flag is set then Mem.z points at a string representation.
** Usually this is encoded in the same unicode encoding as the main
** database (see below for exceptions). If the MEM_Term flag is also
** set, then the string is nul terminated. The MEM_Int and MEM_Real
** flags may coexist with the MEM_Str flag.
*/
#define MEM_Null 0x0001 /* Value is NULL (or a pointer) */
#define MEM_Str 0x0002 /* Value is a string */
#define MEM_Int 0x0004 /* Value is an integer */
#define MEM_Real 0x0008 /* Value is a real number */
#define MEM_Blob 0x0010 /* Value is a BLOB */
#define MEM_AffMask 0x001f /* Mask of affinity bits */
/* Available 0x0020 */
/* Available 0x0040 */
#define MEM_Undefined 0x0080 /* Value is undefined */
#define MEM_Cleared 0x0100 /* NULL set by OP_Null, not from data */
#define MEM_TypeMask 0xc1ff /* Mask of type bits */
/* Whenever Mem contains a valid string or blob representation, one of
** the following flags must be set to determine the memory management
** policy for Mem.z. The MEM_Term flag tells us whether or not the
** string is \000 or \u0000 terminated
*/
#define MEM_Term 0x0200 /* String in Mem.z is zero terminated */
#define MEM_Dyn 0x0400 /* Need to call Mem.xDel() on Mem.z */
#define MEM_Static 0x0800 /* Mem.z points to a static string */
#define MEM_Ephem 0x1000 /* Mem.z points to an ephemeral string */
#define MEM_Agg 0x2000 /* Mem.z points to an agg function context */
#define MEM_Zero 0x4000 /* Mem.i contains count of 0s appended to blob */
#define MEM_Subtype 0x8000 /* Mem.eSubtype is valid */
#ifdef SQLITE_OMIT_INCRBLOB
#undef MEM_Zero
#define MEM_Zero 0x0000
#endif
/* Return TRUE if Mem X contains dynamically allocated content - anything
** that needs to be deallocated to avoid a leak.
*/
#define VdbeMemDynamic(X) \
(((X)->flags&(MEM_Agg|MEM_Dyn))!=0)
/*
** Clear any existing type flags from a Mem and replace them with f
*/
#define MemSetTypeFlag(p, f) \
((p)->flags = ((p)->flags&~(MEM_TypeMask|MEM_Zero))|f)
/*
** Return true if a memory cell is not marked as invalid. This macro
** is for use inside assert() statements only.
*/
#ifdef SQLITE_DEBUG
#define memIsValid(M) ((M)->flags & MEM_Undefined)==0
#endif
/*
** Each auxiliary data pointer stored by a user defined function
** implementation calling sqlite3_set_auxdata() is stored in an instance
** of this structure. All such structures associated with a single VM
** are stored in a linked list headed at Vdbe.pAuxData. All are destroyed
** when the VM is halted (if not before).
*zOut++ = 0xC0 + (u8)((c>>6)&0x1F); \
*zOut++ = 0x80 + (u8)(c & 0x3F); \
} \
else if( c<0x10000 ){ \
*zOut++ = 0xE0 + (u8)((c>>12)&0x0F); \
*zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \
*zOut++ = 0x80 + (u8)(c & 0x3F); \
}else{ \
*zOut++ = 0xF0 + (u8)((c>>18) & 0x07); \
*zOut++ = 0x80 + (u8)((c>>12) & 0x3F); \
*zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \
*zOut++ = 0x80 + (u8)(c & 0x3F); \
} \
}
#define WRITE_UTF16LE(zOut, c) { \
if( c<=0xFFFF ){ \
*zOut++ = (u8)(c&0x00FF); \
*zOut++ = (u8)((c>>8)&0x00FF); \
}else{ \
*zOut++ = (u8)(((c>>10)&0x003F) + (((c-0x10000)>>10)&0x00C0)); \
*zOut++ = (u8)(0x00D8 + (((c-0x10000)>>18)&0x03)); \
*zOut++ = (u8)(c&0x00FF); \
*zOut++ = (u8)(0x00DC + ((c>>8)&0x03)); \
} \
}
#define WRITE_UTF16BE(zOut, c) { \
if( c<=0xFFFF ){ \
*zOut++ = (u8)((c>>8)&0x00FF); \
*zOut++ = (u8)(c&0x00FF); \
}else{ \
*zOut++ = (u8)(0x00D8 + (((c-0x10000)>>18)&0x03)); \
*zOut++ = (u8)(((c>>10)&0x003F) + (((c-0x10000)>>10)&0x00C0)); \
*zOut++ = (u8)(0x00DC + ((c>>8)&0x03)); \
*zOut++ = (u8)(c&0x00FF); \
} \
}
#define READ_UTF16LE(zIn, TERM, c){ \
c = (*zIn++); \
c += ((*zIn++)<<8); \
if( c>=0xD800 && c<0xE000 && TERM ){ \
int c2 = (*zIn++); \
c2 += ((*zIn++)<<8); \
c = (c2&0x03FF) + ((c&0x003F)<<10) + (((c&0x03C0)+0x0040)<<10); \
} \
}
#define READ_UTF16BE(zIn, TERM, c){ \
c = ((*zIn++)<<8); \
c += (*zIn++); \
if( c>=0xD800 && c<0xE000 && TERM ){ \
int c2 = ((*zIn++)<<8); \
c2 += (*zIn++); \
c = (c2&0x03FF) + ((c&0x003F)<<10) + (((c&0x03C0)+0x0040)<<10); \
} \
}
/*
** Translate a single UTF-8 character. Return the unicode value.
**
** During translation, assume that the byte that zTerm points
** is a 0x00.
**
** Write a pointer to the next unread byte back into *pzNext.
**
** Notes On Invalid UTF-8:
**
** * This routine never allows a 7-bit character (0x00 through 0x7f) to
** be encoded as a multi-byte character. Any multi-byte character that
** attempts to encode a value between 0x00 and 0x7f is rendered as 0xfffd.
**
** * This routine never allows a UTF16 surrogate value to be encoded.
** If a multi-byte character attempts to encode a value between
** 0xd800 and 0xe000 then it is rendered as 0xfffd.
**
** * Bytes in the range of 0x80 through 0xbf which occur as the first
** byte of a character are interpreted as single-byte characters
** and rendered as themselves even though they are technically
** invalid characters.
**
** * This routine accepts over-length UTF8 encodings
** for unicode values 0x80 and greater. It does not change over-length
** encodings to 0xfffd as some systems recommend.
*/
#define READ_UTF8(zIn, zTerm, c) \
c = *(zIn++); \
if( c>=0xc0 ){ \
c = sqlite3Utf8Trans1[c-0xc0]; \
while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \
c = (c<<6) + (0x3f & *(zIn++)); \
} \
if( c<0x80 \
|| (c&0xFFFFF800)==0xD800 \
|| (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \
}
SQLITE_PRIVATE u32 sqlite3Utf8Read(
const unsigned char **pz /* Pointer to string from which to read char */
){
unsigned int c;
/* Same as READ_UTF8() above but without the zTerm parameter.
** For this routine, we assume the UTF8 string is always zero-terminated.
*/
c = *((*pz)++);
if( c>=0xc0 ){
c = sqlite3Utf8Trans1[c-0xc0];
while( (*(*pz) & 0xc0)==0x80 ){
c = (c<<6) + (0x3f & *((*pz)++));
}
if( c<0x80
|| (c&0xFFFFF800)==0xD800
|| (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; }
}
return c;
}
/*
** If the TRANSLATE_TRACE macro is defined, the value of each Mem is
** printed on stderr on the way into and out of sqlite3VdbeMemTranslate().
*/
/* #define TRANSLATE_TRACE 1 */
#ifndef SQLITE_OMIT_UTF16
/*
** This routine transforms the internal text encoding used by pMem to
** desiredEnc. It is an error if the string is already of the desired
** encoding, or if *pMem does not contain a string value.
*/
SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){
int len; /* Maximum length of output string in bytes */
unsigned char *zOut; /* Output buffer */
unsigned char *zIn; /* Input iterator */
unsigned char *zTerm; /* End of input */
unsigned char *z; /* Output iterator */
unsigned int c;
assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) );
assert( pMem->flags&MEM_Str );
assert( pMem->enc!=desiredEnc );
pMem->flags = MEM_Str|MEM_Term|(c&(MEM_AffMask|MEM_Subtype));
pMem->enc = desiredEnc;
pMem->z = (char*)zOut;
pMem->zMalloc = pMem->z;
pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->z);
translate_out:
#if defined(TRANSLATE_TRACE) && defined(SQLITE_DEBUG)
{
char zBuf[100];
sqlite3VdbeMemPrettyPrint(pMem, zBuf);
fprintf(stderr, "OUTPUT: %s\n", zBuf);
}
#endif
return SQLITE_OK;
}
#endif /* SQLITE_OMIT_UTF16 */
#ifndef SQLITE_OMIT_UTF16
/*
** This routine checks for a byte-order mark at the beginning of the
** UTF-16 string stored in *pMem. If one is present, it is removed and
** the encoding of the Mem adjusted. This routine does not do any
** byte-swapping, it just sets Mem.enc appropriately.
**
** The allocation (static, dynamic etc.) and encoding of the Mem may be
** changed by this function.
*/
SQLITE_PRIVATE int sqlite3VdbeMemHandleBom(Mem *pMem){
int rc = SQLITE_OK;
u8 bom = 0;
assert( pMem->n>=0 );
if( pMem->n>1 ){
u8 b1 = *(u8 *)pMem->z;
u8 b2 = *(((u8 *)pMem->z) + 1);
if( b1==0xFE && b2==0xFF ){
bom = SQLITE_UTF16BE;
}
if( b1==0xFF && b2==0xFE ){
bom = SQLITE_UTF16LE;
}
}
if( bom ){
rc = sqlite3VdbeMemMakeWriteable(pMem);
if( rc==SQLITE_OK ){
pMem->n -= 2;
memmove(pMem->z, &pMem->z[2], pMem->n);
pMem->z[pMem->n] = '\0';
pMem->z[pMem->n+1] = '\0';
pMem->flags |= MEM_Term;
pMem->enc = bom;
}
}
return rc;
}
#endif /* SQLITE_OMIT_UTF16 */
/*
** pZ is a UTF-8 encoded unicode string. If nByte is less than zero,
** return the number of unicode characters in pZ up to (but not including)
** the first 0x00 byte. If nByte is not less than zero, return the
** number of unicode characters in the first nByte of pZ (or up to
** the first 0x00, whichever comes first).
*/
SQLITE_PRIVATE int sqlite3Utf8CharLen(const char *zIn, int nByte){
int r = 0;
const u8 *z = (const u8*)zIn;
const u8 *zTerm;
if( nByte>=0 ){
zTerm = &z[nByte];
}else{
zTerm = (const u8*)(-1);
}
assert( z<=zTerm );
while( *z!=0 && z<zTerm ){
SQLITE_SKIP_UTF8(z);
r++;
}
return r;
}
/* This test function is not currently used by the automated test-suite.
** Hence it is only available in debug builds.
*/
#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG)
/*
** Translate UTF-8 to UTF-8.
**
** This has the effect of making sure that the string is well-formed
** UTF-8. Miscoded characters are removed.
**
** The translation is done in-place and aborted if the output
** overruns the input.
*/
SQLITE_PRIVATE int sqlite3Utf8To8(unsigned char *zIn){
unsigned char *zOut = zIn;
unsigned char *zStart = zIn;
u32 c;
while( zIn[0] && zOut<=zIn ){
c = sqlite3Utf8Read((const u8**)&zIn);
if( c!=0xfffd ){
WRITE_UTF8(zOut, c);
}
}
*zOut = 0;
return (int)(zOut - zStart);
}
#endif
#ifndef SQLITE_OMIT_UTF16
/*
** Convert a UTF-16 string in the native encoding into a UTF-8 string.
** Memory to hold the UTF-8 string is obtained from sqlite3_malloc and must
** be freed by the calling function.
**
** NULL is returned if there is an allocation error.
*/
SQLITE_PRIVATE char *sqlite3Utf16to8(sqlite3 *db, const void *z, int nByte, u8 enc){
Mem m;
memset(&m, 0, sizeof(m));
m.db = db;
sqlite3VdbeMemSetStr(&m, z, nByte, enc, SQLITE_STATIC);
sqlite3VdbeChangeEncoding(&m, SQLITE_UTF8);
if( db->mallocFailed ){
sqlite3VdbeMemRelease(&m);
m.z = 0;
}
assert( (m.flags & MEM_Term)!=0 || db->mallocFailed );
assert( (m.flags & MEM_Str)!=0 || db->mallocFailed );
assert( m.z || db->mallocFailed );
return m.z;
}
/*
** zIn is a UTF-16 encoded unicode string at least nChar characters long.
** Return the number of bytes in the first nChar unicode characters
** in pZ. nChar must be non-negative.
*/
SQLITE_PRIVATE int sqlite3Utf16ByteLen(const void *zIn, int nChar){
int c;
unsigned char const *z = zIn;
int n = 0;
if( SQLITE_UTF16NATIVE==SQLITE_UTF16BE ){
while( n<nChar ){
READ_UTF16BE(z, 1, c);
n++;
}
}else{
while( n<nChar ){
READ_UTF16LE(z, 1, c);
n++;
}
}
return (int)(z-(unsigned char const *)zIn);
}
#if defined(SQLITE_TEST)
/*
** This routine is called from the TCL test function "translate_selftest".
** It checks that the primitives for serializing and deserializing
** characters in each encoding are inverses of each other.
*/
SQLITE_PRIVATE void sqlite3UtfSelfTest(void){
unsigned int i, t;
unsigned char zBuf[20];
unsigned char *z;
int n;
unsigned int c;
for(i=0; i<0x00110000; i++){
z = zBuf;
WRITE_UTF8(z, i);
n = (int)(z-zBuf);
assert( n>0 && n<=4 );
z[0] = 0;
z = zBuf;
c = sqlite3Utf8Read((const u8**)&z);
t = i;
if( i>=0xD800 && i<=0xDFFF ) t = 0xFFFD;
if( (i&0xFFFFFFFE)==0xFFFE ) t = 0xFFFD;
assert( c==t );
assert( (z-zBuf)==n );
}
for(i=0; i<0x00110000; i++){
if( i>=0xD800 && i<0xE000 ) continue;
z = zBuf;
WRITE_UTF16LE(z, i);
n = (int)(z-zBuf);
assert( n>0 && n<=4 );
z[0] = 0;
z = zBuf;
READ_UTF16LE(z, 1, c);
assert( c==i );
assert( (z-zBuf)==n );
}
int nByte;
char *zText;
int codepage = useAnsi ? CP_ACP : CP_OEMCP;
nByte = osWideCharToMultiByte(codepage, 0, zWideText, -1, 0, 0, 0, 0);
if( nByte == 0 ){
return 0;
}
zText = sqlite3MallocZero( nByte );
if( zText==0 ){
return 0;
}
nByte = osWideCharToMultiByte(codepage, 0, zWideText, -1, zText,
nByte, 0, 0);
if( nByte == 0 ){
sqlite3_free(zText);
zText = 0;
}
return zText;
}
/*
** Convert a multi-byte character string to UTF-8.
**
** Space to hold the returned string is obtained from sqlite3_malloc().
*/
static char *winMbcsToUtf8(const char *zText, int useAnsi){
char *zTextUtf8;
LPWSTR zTmpWide;
zTmpWide = winMbcsToUnicode(zText, useAnsi);
if( zTmpWide==0 ){
return 0;
}
zTextUtf8 = winUnicodeToUtf8(zTmpWide);
sqlite3_free(zTmpWide);
return zTextUtf8;
}
/*
** Convert a UTF-8 string to a multi-byte character string.
**
** Space to hold the returned string is obtained from sqlite3_malloc().
*/
static char *winUtf8ToMbcs(const char *zText, int useAnsi){
char *zTextMbcs;
LPWSTR zTmpWide;
zTmpWide = winUtf8ToUnicode(zText);
if( zTmpWide==0 ){
return 0;
}
zTextMbcs = winUnicodeToMbcs(zTmpWide, useAnsi);
sqlite3_free(zTmpWide);
return zTextMbcs;
}
/*
** This is a public wrapper for the winUtf8ToUnicode() function.
*/
SQLITE_API LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText){
#ifdef SQLITE_ENABLE_API_ARMOR
if( !zText ){
(void)SQLITE_MISUSE_BKPT;
return 0;
}
#endif
#ifndef SQLITE_OMIT_AUTOINIT
if( sqlite3_initialize() ) return 0;
#endif
return winUtf8ToUnicode(zText);
}
/*
** This is a public wrapper for the winUnicodeToUtf8() function.
*/
SQLITE_API char *sqlite3_win32_unicode_to_utf8(LPCWSTR zWideText){
#ifdef SQLITE_ENABLE_API_ARMOR
if( !zWideText ){
(void)SQLITE_MISUSE_BKPT;
return 0;
}
#endif
#ifndef SQLITE_OMIT_AUTOINIT
if( sqlite3_initialize() ) return 0;
#endif
return winUnicodeToUtf8(zWideText);
}
/*
** This is a public wrapper for the winMbcsToUtf8() function.
*/
SQLITE_API char *sqlite3_win32_mbcs_to_utf8(const char *zText){
#ifdef SQLITE_ENABLE_API_ARMOR
if( !zText ){
(void)SQLITE_MISUSE_BKPT;
return 0;
}
#endif
#ifndef SQLITE_OMIT_AUTOINIT
if( sqlite3_initialize() ) return 0;
#endif
return winMbcsToUtf8(zText, osAreFileApisANSI());
}
/*
** This is a public wrapper for the winMbcsToUtf8() function.
*/
SQLITE_API char *sqlite3_win32_mbcs_to_utf8_v2(const char *zText, int useAnsi){
#ifdef SQLITE_ENABLE_API_ARMOR
if( !zText ){
(void)SQLITE_MISUSE_BKPT;
return 0;
}
#endif
#ifndef SQLITE_OMIT_AUTOINIT
if( sqlite3_initialize() ) return 0;
#endif
return winMbcsToUtf8(zText, useAnsi);
}
/*
** This is a public wrapper for the winUtf8ToMbcs() function.
*/
SQLITE_API char *sqlite3_win32_utf8_to_mbcs(const char *zText){
#ifdef SQLITE_ENABLE_API_ARMOR
if( !zText ){
(void)SQLITE_MISUSE_BKPT;
return 0;
}
#endif
#ifndef SQLITE_OMIT_AUTOINIT
if( sqlite3_initialize() ) return 0;
#endif
return winUtf8ToMbcs(zText, osAreFileApisANSI());
}
#ifdef SQLITE_ENABLE_API_ARMOR
if( !zText ){
(void)SQLITE_MISUSE_BKPT;
return 0;
}
#endif
#ifndef SQLITE_OMIT_AUTOINIT
if( sqlite3_initialize() ) return 0;
#endif
return winUtf8ToMbcs(zText, useAnsi);
}
/*
** This function is the same as sqlite3_win32_set_directory (below); however,
** it accepts a UTF-8 string.
*/
SQLITE_API int sqlite3_win32_set_directory8(
unsigned long type, /* Identifier for directory being set or reset */
const char *zValue /* New value for directory being set or reset */
){
char **ppDirectory = 0;
#ifndef SQLITE_OMIT_AUTOINIT
int rc = sqlite3_initialize();
if( rc ) return rc;
#endif
if( type==SQLITE_WIN32_DATA_DIRECTORY_TYPE ){
ppDirectory = &sqlite3_data_directory;
}else if( type==SQLITE_WIN32_TEMP_DIRECTORY_TYPE ){
ppDirectory = &sqlite3_temp_directory;
}
assert( !ppDirectory || type==SQLITE_WIN32_DATA_DIRECTORY_TYPE
|| type==SQLITE_WIN32_TEMP_DIRECTORY_TYPE
);
assert( !ppDirectory || sqlite3MemdebugHasType(*ppDirectory, MEMTYPE_HEAP) );
if( ppDirectory ){
char *zCopy = 0;
if( zValue && zValue[0] ){
zCopy = sqlite3_mprintf("%s", zValue);
if ( zCopy==0 ){
return SQLITE_NOMEM_BKPT;
}
}
sqlite3_free(*ppDirectory);
*ppDirectory = zCopy;
return SQLITE_OK;
}
return SQLITE_ERROR;
}
/*
** This function is the same as sqlite3_win32_set_directory (below); however,
** it accepts a UTF-16 string.
*/
SQLITE_API int sqlite3_win32_set_directory16(
unsigned long type, /* Identifier for directory being set or reset */
const void *zValue /* New value for directory being set or reset */
){
int rc;
char *zUtf8 = 0;
if( zValue ){
zUtf8 = sqlite3_win32_unicode_to_utf8(zValue);
if( zUtf8==0 ) return SQLITE_NOMEM_BKPT;
}
rc = sqlite3_win32_set_directory8(type, zUtf8);
if( zUtf8 ) sqlite3_free(zUtf8);
return rc;
}
/*
** This function sets the data directory or the temporary directory based on
** the provided arguments. The type argument must be 1 in order to set the
** data directory or 2 in order to set the temporary directory. The zValue
** argument is the name of the directory to use. The return value will be
** SQLITE_OK if successful.
*/
SQLITE_API int sqlite3_win32_set_directory(
unsigned long type, /* Identifier for directory being set or reset */
void *zValue /* New value for directory being set or reset */
){
return sqlite3_win32_set_directory16(type, zValue);
}
/*
** The return value of winGetLastErrorMsg
** is zero if the error message fits in the buffer, or non-zero
** otherwise (if the message was truncated).
*/
static int winGetLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){
/* FormatMessage returns 0 on failure. Otherwise it
** returns the number of TCHARs written to the output
** buffer, excluding the terminating null char.
*/
DWORD dwLen = 0;
char *zOut = 0;
if( osIsNT() ){
#if SQLITE_OS_WINRT
WCHAR zTempWide[SQLITE_WIN32_MAX_ERRMSG_CHARS+1];
dwLen = osFormatMessageW(FORMAT_MESSAGE_FROM_SYSTEM |
FORMAT_MESSAGE_IGNORE_INSERTS,
NULL,
lastErrno,
0,
zTempWide,
SQLITE_WIN32_MAX_ERRMSG_CHARS,
0);
#else
LPWSTR zTempWide = NULL;
dwLen = osFormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER |
FORMAT_MESSAGE_FROM_SYSTEM |
FORMAT_MESSAGE_IGNORE_INSERTS,
NULL,
lastErrno,
0,
(LPWSTR) &zTempWide,
0,
0);
#endif
if( dwLen > 0 ){
/* allocate a buffer and convert to UTF8 */
sqlite3BeginBenignMalloc();
sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
break;
}
case SQLITE_INTEGER: {
sqlite3_result_value(context, argv[0]);
break;
}
case SQLITE_BLOB: {
char *zText = 0;
char const *zBlob = sqlite3_value_blob(argv[0]);
int nBlob = sqlite3_value_bytes(argv[0]);
assert( zBlob==sqlite3_value_blob(argv[0]) ); /* No encoding change */
zText = (char *)contextMalloc(context, (2*(i64)nBlob)+4);
if( zText ){
int i;
for(i=0; i<nBlob; i++){
zText[(i*2)+2] = hexdigits[(zBlob[i]>>4)&0x0F];
zText[(i*2)+3] = hexdigits[(zBlob[i])&0x0F];
}
zText[(nBlob*2)+2] = '\'';
zText[(nBlob*2)+3] = '\0';
zText[0] = 'X';
zText[1] = '\'';
sqlite3_result_text(context, zText, -1, SQLITE_TRANSIENT);
sqlite3_free(zText);
}
break;
}
case SQLITE_TEXT: {
int i,j;
u64 n;
const unsigned char *zArg = sqlite3_value_text(argv[0]);
char *z;
if( zArg==0 ) return;
for(i=0, n=0; zArg[i]; i++){ if( zArg[i]=='\'' ) n++; }
z = contextMalloc(context, ((i64)i)+((i64)n)+3);
if( z ){
z[0] = '\'';
for(i=0, j=1; zArg[i]; i++){
z[j++] = zArg[i];
if( zArg[i]=='\'' ){
z[j++] = '\'';
}
}
z[j++] = '\'';
z[j] = 0;
sqlite3_result_text(context, z, j, sqlite3_free);
}
break;
}
default: {
assert( sqlite3_value_type(argv[0])==SQLITE_NULL );
sqlite3_result_text(context, "NULL", 4, SQLITE_STATIC);
break;
}
}
}
/*
** The unicode() function. Return the integer unicode code-point value
** for the first character of the input string.
*/
static void unicodeFunc(
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
const unsigned char *z = sqlite3_value_text(argv[0]);
(void)argc;
if( z && z[0] ) sqlite3_result_int(context, sqlite3Utf8Read(&z));
}
/*
** The char() function takes zero or more arguments, each of which is
** an integer. It constructs a string where each character of the string
** is the unicode character for the corresponding integer argument.
*/
static void charFunc(
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
unsigned char *z, *zOut;
int i;
zOut = z = sqlite3_malloc64( argc*4+1 );
if( z==0 ){
sqlite3_result_error_nomem(context);
return;
}
for(i=0; i<argc; i++){
sqlite3_int64 x;
unsigned c;
x = sqlite3_value_int64(argv[i]);
if( x<0 || x>0x10ffff ) x = 0xfffd;
c = (unsigned)(x & 0x1fffff);
if( c<0x00080 ){
*zOut++ = (u8)(c&0xFF);
}else if( c<0x00800 ){
*zOut++ = 0xC0 + (u8)((c>>6)&0x1F);
*zOut++ = 0x80 + (u8)(c & 0x3F);
}else if( c<0x10000 ){
*zOut++ = 0xE0 + (u8)((c>>12)&0x0F);
*zOut++ = 0x80 + (u8)((c>>6) & 0x3F);
*zOut++ = 0x80 + (u8)(c & 0x3F);
}else{
*zOut++ = 0xF0 + (u8)((c>>18) & 0x07);
*zOut++ = 0x80 + (u8)((c>>12) & 0x3F);
*zOut++ = 0x80 + (u8)((c>>6) & 0x3F);
*zOut++ = 0x80 + (u8)(c & 0x3F);
} \
}
sqlite3_result_text64(context, (char*)z, zOut-z, sqlite3_free, SQLITE_UTF8);
}
/*
** The hex() function. Interpret the argument as a blob. Return
** a hexadecimal rendering as text.
*/
static void hexFunc(
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
int i, n;
const unsigned char *pBlob;
char *zHex, *z;
assert( argc==1 );
UNUSED_PARAMETER(argc);
pBlob = sqlite3_value_blob(argv[0]);
n = sqlite3_value_bytes(argv[0]);
assert( pBlob==sqlite3_value_blob(argv[0]) ); /* No encoding change */
z = zHex = contextMalloc(context, ((i64)n)*2 + 1);
if( zHex ){
for(i=0; i<n; i++, pBlob++){
unsigned char c = *pBlob;
*(z++) = hexdigits[(c>>4)&0xf];
** All of the FuncDef structures in the aBuiltinFunc[] array above
** to the global function hash table. This occurs at start-time (as
** a consequence of calling sqlite3_initialize()).
**
** After this routine runs
*/
SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){
/*
** The following array holds FuncDef structures for all of the functions
** defined in this file.
**
** The array cannot be constant since changes are made to the
** FuncDef.pHash elements at start-time. The elements of this array
** are read-only after initialization is complete.
**
** For peak efficiency, put the most frequently used function last.
*/
static FuncDef aBuiltinFunc[] = {
#ifdef SQLITE_SOUNDEX
FUNCTION(soundex, 1, 0, 0, soundexFunc ),
#endif
#ifndef SQLITE_OMIT_LOAD_EXTENSION
VFUNCTION(load_extension, 1, 0, 0, loadExt ),
VFUNCTION(load_extension, 2, 0, 0, loadExt ),
#endif
#if SQLITE_USER_AUTHENTICATION
FUNCTION(sqlite_crypt, 2, 0, 0, sqlite3CryptFunc ),
#endif
#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS
DFUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ),
DFUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ),
#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */
FUNCTION2(unlikely, 1, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY),
FUNCTION2(likelihood, 2, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY),
FUNCTION2(likely, 1, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY),
#ifdef SQLITE_DEBUG
FUNCTION2(affinity, 1, 0, 0, noopFunc, SQLITE_FUNC_AFFINITY),
#endif
#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
FUNCTION2(sqlite_offset, 1, 0, 0, noopFunc, SQLITE_FUNC_OFFSET|
SQLITE_FUNC_TYPEOF),
#endif
FUNCTION(ltrim, 1, 1, 0, trimFunc ),
FUNCTION(ltrim, 2, 1, 0, trimFunc ),
FUNCTION(rtrim, 1, 2, 0, trimFunc ),
FUNCTION(rtrim, 2, 2, 0, trimFunc ),
FUNCTION(trim, 1, 3, 0, trimFunc ),
FUNCTION(trim, 2, 3, 0, trimFunc ),
FUNCTION(min, -1, 0, 1, minmaxFunc ),
FUNCTION(min, 0, 0, 1, 0 ),
WAGGREGATE(min, 1, 0, 1, minmaxStep, minMaxFinalize, minMaxValue, 0,
SQLITE_FUNC_MINMAX ),
FUNCTION(max, -1, 1, 1, minmaxFunc ),
FUNCTION(max, 0, 1, 1, 0 ),
WAGGREGATE(max, 1, 1, 1, minmaxStep, minMaxFinalize, minMaxValue, 0,
SQLITE_FUNC_MINMAX ),
FUNCTION2(typeof, 1, 0, 0, typeofFunc, SQLITE_FUNC_TYPEOF),
FUNCTION2(length, 1, 0, 0, lengthFunc, SQLITE_FUNC_LENGTH),
FUNCTION(instr, 2, 0, 0, instrFunc ),
FUNCTION(printf, -1, 0, 0, printfFunc ),
FUNCTION(unicode, 1, 0, 0, unicodeFunc ),
FUNCTION(char, -1, 0, 0, charFunc ),
FUNCTION(abs, 1, 0, 0, absFunc ),
#ifndef SQLITE_OMIT_FLOATING_POINT
FUNCTION(round, 1, 0, 0, roundFunc ),
FUNCTION(round, 2, 0, 0, roundFunc ),
#endif
FUNCTION(upper, 1, 0, 0, upperFunc ),
FUNCTION(lower, 1, 0, 0, lowerFunc ),
FUNCTION(hex, 1, 0, 0, hexFunc ),
FUNCTION2(ifnull, 2, 0, 0, noopFunc, SQLITE_FUNC_COALESCE),
VFUNCTION(random, 0, 0, 0, randomFunc ),
VFUNCTION(randomblob, 1, 0, 0, randomBlob ),
FUNCTION(nullif, 2, 0, 1, nullifFunc ),
DFUNCTION(sqlite_version, 0, 0, 0, versionFunc ),
DFUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ),
FUNCTION(sqlite_log, 2, 0, 0, errlogFunc ),
FUNCTION(quote, 1, 0, 0, quoteFunc ),
VFUNCTION(last_insert_rowid, 0, 0, 0, last_insert_rowid),
VFUNCTION(changes, 0, 0, 0, changes ),
VFUNCTION(total_changes, 0, 0, 0, total_changes ),
FUNCTION(replace, 3, 0, 0, replaceFunc ),
FUNCTION(zeroblob, 1, 0, 0, zeroblobFunc ),
FUNCTION(substr, 2, 0, 0, substrFunc ),
FUNCTION(substr, 3, 0, 0, substrFunc ),
WAGGREGATE(sum, 1,0,0, sumStep, sumFinalize, sumFinalize, sumInverse, 0),
WAGGREGATE(total, 1,0,0, sumStep,totalFinalize,totalFinalize,sumInverse, 0),
WAGGREGATE(avg, 1,0,0, sumStep, avgFinalize, avgFinalize, sumInverse, 0),
WAGGREGATE(count, 0,0,0, countStep,
countFinalize, countFinalize, countInverse, SQLITE_FUNC_COUNT ),
WAGGREGATE(count, 1,0,0, countStep,
countFinalize, countFinalize, countInverse, 0 ),
WAGGREGATE(group_concat, 1, 0, 0, groupConcatStep,
groupConcatFinalize, groupConcatValue, groupConcatInverse, 0),
WAGGREGATE(group_concat, 2, 0, 0, groupConcatStep,
groupConcatFinalize, groupConcatValue, groupConcatInverse, 0),
LIKEFUNC(glob, 2, &globInfo, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE),
#ifdef SQLITE_CASE_SENSITIVE_LIKE
LIKEFUNC(like, 2, &likeInfoAlt, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE),
LIKEFUNC(like, 3, &likeInfoAlt, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE),
#else
LIKEFUNC(like, 2, &likeInfoNorm, SQLITE_FUNC_LIKE),
LIKEFUNC(like, 3, &likeInfoNorm, SQLITE_FUNC_LIKE),
#endif
#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
FUNCTION(unknown, -1, 0, 0, unknownFunc ),
#endif
FUNCTION(coalesce, 1, 0, 0, 0 ),
FUNCTION(coalesce, 0, 0, 0, 0 ),
FUNCTION2(coalesce, -1, 0, 0, noopFunc, SQLITE_FUNC_COALESCE),
};
#ifndef SQLITE_OMIT_ALTERTABLE
sqlite3AlterFunctions();
#endif
sqlite3WindowFunctions();
#if defined(SQLITE_ENABLE_STAT3) || defined(SQLITE_ENABLE_STAT4)
sqlite3AnalyzeFunctions();
#endif
sqlite3RegisterDateTimeFunctions();
sqlite3InsertBuiltinFuncs(aBuiltinFunc, ArraySize(aBuiltinFunc));
sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */
const char **pzTail /* OUT: End of parsed string */
){
int rc;
/* EVIDENCE-OF: R-56861-42673 sqlite3_prepare_v3() differs from
** sqlite3_prepare_v2() only in having the extra prepFlags parameter,
** which is a bit array consisting of zero or more of the
** SQLITE_PREPARE_* flags.
**
** Proof by comparison to the implementation of sqlite3_prepare_v2()
** directly above. */
rc = sqlite3LockAndPrepare(db,zSql,nBytes,
SQLITE_PREPARE_SAVESQL|(prepFlags&SQLITE_PREPARE_MASK),
0,ppStmt,pzTail);
assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 );
return rc;
}
#ifndef SQLITE_OMIT_UTF16
/*
** Compile the UTF-16 encoded SQL statement zSql into a statement handle.
*/
static int sqlite3Prepare16(
sqlite3 *db, /* Database handle. */
const void *zSql, /* UTF-16 encoded SQL statement. */
int nBytes, /* Length of zSql in bytes. */
u32 prepFlags, /* Zero or more SQLITE_PREPARE_* flags */
sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */
const void **pzTail /* OUT: End of parsed string */
){
/* This function currently works by first transforming the UTF-16
** encoded string to UTF-8, then invoking sqlite3_prepare(). The
** tricky bit is figuring out the pointer to return in *pzTail.
*/
char *zSql8;
const char *zTail8 = 0;
int rc = SQLITE_OK;
#ifdef SQLITE_ENABLE_API_ARMOR
if( ppStmt==0 ) return SQLITE_MISUSE_BKPT;
#endif
*ppStmt = 0;
if( !sqlite3SafetyCheckOk(db)||zSql==0 ){
return SQLITE_MISUSE_BKPT;
}
if( nBytes>=0 ){
int sz;
const char *z = (const char*)zSql;
for(sz=0; sz<nBytes && (z[sz]!=0 || z[sz+1]!=0); sz += 2){}
nBytes = sz;
}
sqlite3_mutex_enter(db->mutex);
zSql8 = sqlite3Utf16to8(db, zSql, nBytes, SQLITE_UTF16NATIVE);
if( zSql8 ){
rc = sqlite3LockAndPrepare(db, zSql8, -1, prepFlags, 0, ppStmt, &zTail8);
}
if( zTail8 && pzTail ){
/* If sqlite3_prepare returns a tail pointer, we calculate the
** equivalent pointer into the UTF-16 string by counting the unicode
** characters between zSql8 and zTail8, and then returning a pointer
** the same number of characters into the UTF-16 string.
*/
int chars_parsed = sqlite3Utf8CharLen(zSql8, (int)(zTail8-zSql8));
*pzTail = (u8 *)zSql + sqlite3Utf16ByteLen(zSql, chars_parsed);
}
sqlite3DbFree(db, zSql8);
rc = sqlite3ApiExit(db, rc);
sqlite3_mutex_leave(db->mutex);
return rc;
}
/*
** Two versions of the official API. Legacy and new use. In the legacy
** version, the original SQL text is not saved in the prepared statement
** and so if a schema change occurs, SQLITE_SCHEMA is returned by
** sqlite3_step(). In the new version, the original SQL text is retained
** and the statement is automatically recompiled if an schema change
** occurs.
*/
SQLITE_API int sqlite3_prepare16(
sqlite3 *db, /* Database handle. */
const void *zSql, /* UTF-16 encoded SQL statement. */
int nBytes, /* Length of zSql in bytes. */
sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */
const void **pzTail /* OUT: End of parsed string */
){
int rc;
rc = sqlite3Prepare16(db,zSql,nBytes,0,ppStmt,pzTail);
assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */
return rc;
}
SQLITE_API int sqlite3_prepare16_v2(
sqlite3 *db, /* Database handle. */
const void *zSql, /* UTF-16 encoded SQL statement. */
int nBytes, /* Length of zSql in bytes. */
sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */
const void **pzTail /* OUT: End of parsed string */
){
int rc;
rc = sqlite3Prepare16(db,zSql,nBytes,SQLITE_PREPARE_SAVESQL,ppStmt,pzTail);
assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */
return rc;
}
SQLITE_API int sqlite3_prepare16_v3(
sqlite3 *db, /* Database handle. */
const void *zSql, /* UTF-16 encoded SQL statement. */
int nBytes, /* Length of zSql in bytes. */
unsigned int prepFlags, /* Zero or more SQLITE_PREPARE_* flags */
sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */
const void **pzTail /* OUT: End of parsed string */
){
int rc;
rc = sqlite3Prepare16(db,zSql,nBytes,
SQLITE_PREPARE_SAVESQL|(prepFlags&SQLITE_PREPARE_MASK),
ppStmt,pzTail);
assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */
return rc;
}
yyStackEntry *i;
char cDiv = '[';
fprintf(yyTraceFILE,"%sReturn. Stack=",yyTracePrompt);
for(i=&yypParser->yystack[1]; i<=yypParser->yytos; i++){
fprintf(yyTraceFILE,"%c%s", cDiv, yyTokenName[i->major]);
cDiv = ' ';
}
fprintf(yyTraceFILE,"]\n");
}
#endif
return;
}
/*
** Return the fallback token corresponding to canonical token iToken, or
** 0 if iToken has no fallback.
*/
SQLITE_PRIVATE int sqlite3ParserFallback(int iToken){
#ifdef YYFALLBACK
if( iToken<(int)(sizeof(yyFallback)/sizeof(yyFallback[0])) ){
return yyFallback[iToken];
}
#else
(void)iToken;
#endif
return 0;
}
/************** End of parse.c ***********************************************/
/************** Begin file tokenize.c ****************************************/
/*
** 2001 September 15
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** An tokenizer for SQL
**
** This file contains C code that splits an SQL input string up into
** individual tokens and sends those tokens one-by-one over to the
** parser for analysis.
*/
/* #include "sqliteInt.h" */
/* #include <stdlib.h> */
/* Character classes for tokenizing
**
** In the sqlite3GetToken() function, a switch() on aiClass[c] is implemented
** using a lookup table, whereas a switch() directly on c uses a binary search.
** The lookup table is much faster. To maximize speed, and to ensure that
** a lookup table is used, all of the classes need to be small integers and
** all of them need to be used within the switch.
*/
#define CC_X 0 /* The letter 'x', or start of BLOB literal */
#define CC_KYWD 1 /* Alphabetics or '_'. Usable in a keyword */
#define CC_ID 2 /* unicode characters usable in IDs */
#define CC_DIGIT 3 /* Digits */
#define CC_DOLLAR 4 /* '$' */
#define CC_VARALPHA 5 /* '@', '#', ':'. Alphabetic SQL variables */
#define CC_VARNUM 6 /* '?'. Numeric SQL variables */
#define CC_SPACE 7 /* Space characters */
#define CC_QUOTE 8 /* '"', '\'', or '`'. String literals, quoted ids */
#define CC_QUOTE2 9 /* '['. [...] style quoted ids */
#define CC_PIPE 10 /* '|'. Bitwise OR or concatenate */
#define CC_MINUS 11 /* '-'. Minus or SQL-style comment */
#define CC_LT 12 /* '<'. Part of < or <= or <> */
#define CC_GT 13 /* '>'. Part of > or >= */
#define CC_EQ 14 /* '='. Part of = or == */
#define CC_BANG 15 /* '!'. Part of != */
#define CC_SLASH 16 /* '/'. / or c-style comment */
#define CC_LP 17 /* '(' */
#define CC_RP 18 /* ')' */
#define CC_SEMI 19 /* ';' */
#define CC_PLUS 20 /* '+' */
#define CC_STAR 21 /* '*' */
#define CC_PERCENT 22 /* '%' */
#define CC_COMMA 23 /* ',' */
#define CC_AND 24 /* '&' */
#define CC_TILDA 25 /* '~' */
#define CC_DOT 26 /* '.' */
#define CC_ILLEGAL 27 /* Illegal character */
#define CC_NUL 28 /* 0x00 */
static const unsigned char aiClass[] = {
#ifdef SQLITE_ASCII
/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */
/* 0x */ 28, 27, 27, 27, 27, 27, 27, 27, 27, 7, 7, 27, 7, 7, 27, 27,
/* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
/* 2x */ 7, 15, 8, 5, 4, 22, 24, 8, 17, 18, 21, 20, 23, 11, 26, 16,
/* 3x */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 19, 12, 14, 13, 6,
/* 4x */ 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
/* 5x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 9, 27, 27, 27, 1,
/* 6x */ 8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
/* 7x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 27, 10, 27, 25, 27,
/* 8x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
/* 9x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
/* Ax */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
/* Bx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
/* Cx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
/* Dx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
/* Ex */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
/* Fx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2
#endif
#ifdef SQLITE_EBCDIC
/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */
/* 0x */ 27, 27, 27, 27, 27, 7, 27, 27, 27, 27, 27, 27, 7, 7, 27, 27,
/* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
/* 2x */ 27, 27, 27, 27, 27, 7, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
/* 3x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
/* 4x */ 7, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 26, 12, 17, 20, 10,
/* 5x */ 24, 27, 27, 27, 27, 27, 27, 27, 27, 27, 15, 4, 21, 18, 19, 27,
/* 6x */ 11, 16, 27, 27, 27, 27, 27, 27, 27, 27, 27, 23, 22, 1, 13, 6,
/* 7x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 8, 5, 5, 5, 8, 14, 8,
/* 8x */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27,
/* 9x */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27,
/* Ax */ 27, 25, 1, 1, 1, 1, 1, 0, 1, 1, 27, 27, 27, 27, 27, 27,
/* fts3.c */
SQLITE_PRIVATE void sqlite3Fts3ErrMsg(char**,const char*,...);
SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *, sqlite3_int64);
SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *, int *);
SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64);
SQLITE_PRIVATE void sqlite3Fts3Dequote(char *);
SQLITE_PRIVATE void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*);
SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *);
SQLITE_PRIVATE int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *);
SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int*, Fts3Table*);
SQLITE_PRIVATE int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc);
/* fts3_tokenizer.c */
SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *);
SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *);
SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *,
sqlite3_tokenizer **, char **
);
SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char);
/* fts3_snippet.c */
SQLITE_PRIVATE void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*);
SQLITE_PRIVATE void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *,
const char *, const char *, int, int
);
SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *);
SQLITE_PRIVATE void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p);
/* fts3_expr.c */
SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int,
char **, int, int, int, const char *, int, Fts3Expr **, char **
);
SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *);
#ifdef SQLITE_TEST
SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash*);
SQLITE_PRIVATE int sqlite3Fts3InitTerm(sqlite3 *db);
#endif
SQLITE_PRIVATE int sqlite3Fts3OpenTokenizer(sqlite3_tokenizer *, int, const char *, int,
sqlite3_tokenizer_cursor **
);
/* fts3_aux.c */
SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db);
SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *);
SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart(
Fts3Table*, Fts3MultiSegReader*, int, const char*, int);
SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext(
Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *);
SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **);
SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *);
SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr);
/* fts3_tokenize_vtab.c */
SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *);
/* fts3_unicode2.c (functions generated by parsing unicode text files) */
#ifndef SQLITE_DISABLE_FTS3_UNICODE
SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int, int);
SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int);
SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int);
#endif
#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */
#endif /* _FTSINT_H */
/************** End of fts3Int.h *********************************************/
/************** Continuing where we left off in fts3.c ***********************/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE)
# define SQLITE_CORE 1
#endif
/* #include <assert.h> */
/* #include <stdlib.h> */
/* #include <stddef.h> */
/* #include <stdio.h> */
/* #include <string.h> */
/* #include <stdarg.h> */
/* #include "fts3.h" */
#ifndef SQLITE_CORE
/* # include "sqlite3ext.h" */
SQLITE_EXTENSION_INIT1
#endif
static int fts3EvalNext(Fts3Cursor *pCsr);
static int fts3EvalStart(Fts3Cursor *pCsr);
static int fts3TermSegReaderCursor(
Fts3Cursor *, const char *, int, int, Fts3MultiSegReader **);
#ifndef SQLITE_AMALGAMATION
# if defined(SQLITE_DEBUG)
SQLITE_PRIVATE int sqlite3Fts3Always(int b) { assert( b ); return b; }
SQLITE_PRIVATE int sqlite3Fts3Never(int b) { assert( !b ); return b; }
# endif
#endif
/*
** Write a 64-bit variable-length integer to memory starting at p[0].
** The length of data written will be between 1 and FTS3_VARINT_MAX bytes.
** The number of bytes written is returned.
*/
SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){
unsigned char *q = (unsigned char *) p;
sqlite_uint64 vu = v;
do{
*q++ = (unsigned char) ((vu & 0x7f) | 0x80);
vu >>= 7;
}while( vu!=0 );
q[-1] &= 0x7f; /* turn off high bit in final byte */
assert( q - (unsigned char *)p <= FTS3_VARINT_MAX );
return (int) (q - (unsigned char *)p);
}
#define GETVARINT_STEP(v, ptr, shift, mask1, mask2, var, ret) \
** And so on.
*/
SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
#ifndef SQLITE_DISABLE_FTS3_UNICODE
SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const**ppModule);
#endif
#ifdef SQLITE_ENABLE_ICU
SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule);
#endif
/*
** Initialize the fts3 extension. If this extension is built as part
** of the sqlite library, then this function is called directly by
** SQLite. If fts3 is built as a dynamically loadable extension, this
** function is called by the sqlite3_extension_init() entry point.
*/
SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){
int rc = SQLITE_OK;
Fts3Hash *pHash = 0;
const sqlite3_tokenizer_module *pSimple = 0;
const sqlite3_tokenizer_module *pPorter = 0;
#ifndef SQLITE_DISABLE_FTS3_UNICODE
const sqlite3_tokenizer_module *pUnicode = 0;
#endif
#ifdef SQLITE_ENABLE_ICU
const sqlite3_tokenizer_module *pIcu = 0;
sqlite3Fts3IcuTokenizerModule(&pIcu);
#endif
#ifndef SQLITE_DISABLE_FTS3_UNICODE
sqlite3Fts3UnicodeTokenizer(&pUnicode);
#endif
#ifdef SQLITE_TEST
rc = sqlite3Fts3InitTerm(db);
if( rc!=SQLITE_OK ) return rc;
#endif
rc = sqlite3Fts3InitAux(db);
if( rc!=SQLITE_OK ) return rc;
sqlite3Fts3SimpleTokenizerModule(&pSimple);
sqlite3Fts3PorterTokenizerModule(&pPorter);
/* Allocate and initialize the hash-table used to store tokenizers. */
pHash = sqlite3_malloc(sizeof(Fts3Hash));
if( !pHash ){
rc = SQLITE_NOMEM;
}else{
sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
}
/* Load the built-in tokenizers into the hash table */
if( rc==SQLITE_OK ){
if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple)
|| sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter)
#ifndef SQLITE_DISABLE_FTS3_UNICODE
|| sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode)
#endif
#ifdef SQLITE_ENABLE_ICU
|| (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu))
#endif
){
rc = SQLITE_NOMEM;
}
}
#ifdef SQLITE_TEST
if( rc==SQLITE_OK ){
rc = sqlite3Fts3ExprInitTestInterface(db, pHash);
}
#endif
/* Create the virtual table wrapper around the hash-table and overload
** the four scalar functions. If this is successful, register the
** module with sqlite.
*/
if( SQLITE_OK==rc
&& SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer"))
&& SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
&& SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1))
&& SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1))
&& SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 2))
&& SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1))
){
rc = sqlite3_create_module_v2(
db, "fts3", &fts3Module, (void *)pHash, hashDestroy
);
if( rc==SQLITE_OK ){
rc = sqlite3_create_module_v2(
db, "fts4", &fts3Module, (void *)pHash, 0
);
}
if( rc==SQLITE_OK ){
rc = sqlite3Fts3InitTok(db, (void *)pHash);
}
return rc;
}
/* An error has occurred. Delete the hash table and return the error code. */
assert( rc!=SQLITE_OK );
if( pHash ){
sqlite3Fts3HashClear(pHash);
sqlite3_free(pHash);
}
return rc;
}
/*
** Allocate an Fts3MultiSegReader for each token in the expression headed
** by pExpr.
**
** An Fts3SegReader object is a cursor that can seek or scan a range of
** entries within a single segment b-tree. An Fts3MultiSegReader uses multiple
** Fts3SegReader objects internally to provide an interface to seek or scan
** within the union of all segments of a b-tree. Hence the name.
**
sqlite3_snprintf(sizeof(aBuffer), aBuffer,
"%d %d %d %d ", iCol, pTerm-sCtx.aTerm, iStart, iEnd-iStart
);
rc = fts3StringAppend(&res, aBuffer, -1);
}else if( rc==SQLITE_DONE && pTab->zContentTbl==0 ){
rc = FTS_CORRUPT_VTAB;
}
}
}
if( rc==SQLITE_DONE ){
rc = SQLITE_OK;
}
pMod->xClose(pC);
if( rc!=SQLITE_OK ) goto offsets_out;
}
offsets_out:
sqlite3_free(sCtx.aTerm);
assert( rc!=SQLITE_DONE );
sqlite3Fts3SegmentsClose(pTab);
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pCtx, rc);
sqlite3_free(res.z);
}else{
sqlite3_result_text(pCtx, res.z, res.n-1, sqlite3_free);
}
return;
}
/*
** Implementation of matchinfo() function.
*/
SQLITE_PRIVATE void sqlite3Fts3Matchinfo(
sqlite3_context *pContext, /* Function call context */
Fts3Cursor *pCsr, /* FTS3 table cursor */
const char *zArg /* Second arg to matchinfo() function */
){
Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
const char *zFormat;
if( zArg ){
zFormat = zArg;
}else{
zFormat = FTS3_MATCHINFO_DEFAULT;
}
if( !pCsr->pExpr ){
sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC);
return;
}else{
/* Retrieve matchinfo() data. */
fts3GetMatchinfo(pContext, pCsr, zFormat);
sqlite3Fts3SegmentsClose(pTab);
}
}
#endif
/************** End of fts3_snippet.c ****************************************/
/************** Begin file fts3_unicode.c ************************************/
/*
** 2012 May 24
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** Implementation of the "unicode" full-text-search tokenizer.
*/
#ifndef SQLITE_DISABLE_FTS3_UNICODE
/* #include "fts3Int.h" */
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
/* #include <assert.h> */
/* #include <stdlib.h> */
/* #include <stdio.h> */
/* #include <string.h> */
/* #include "fts3_tokenizer.h" */
/*
** The following two macros - READ_UTF8 and WRITE_UTF8 - have been copied
** from the sqlite3 source file utf.c. If this file is compiled as part
** of the amalgamation, they are not required.
*/
#ifndef SQLITE_AMALGAMATION
static const unsigned char sqlite3Utf8Trans1[] = {
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00,
};
#define READ_UTF8(zIn, zTerm, c) \
c = *(zIn++); \
if( c>=0xc0 ){ \
c = sqlite3Utf8Trans1[c-0xc0]; \
while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \
c = (c<<6) + (0x3f & *(zIn++)); \
} \
if( c<0x80 \
|| (c&0xFFFFF800)==0xD800 \
|| (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \
}
#define WRITE_UTF8(zOut, c) { \
if( c<0x00080 ){ \
*zOut++ = (u8)(c&0xFF); \
} \
else if( c<0x00800 ){ \
*zOut++ = 0xC0 + (u8)((c>>6)&0x1F); \
*zOut++ = 0x80 + (u8)(c & 0x3F); \
} \
else if( c<0x10000 ){ \
*zOut++ = 0xE0 + (u8)((c>>12)&0x0F); \
*zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \
*zOut++ = 0x80 + (u8)(c & 0x3F); \
}else{ \
*zOut++ = 0xF0 + (u8)((c>>18) & 0x07); \
*zOut++ = 0x80 + (u8)((c>>12) & 0x3F); \
*zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \
*zOut++ = 0x80 + (u8)(c & 0x3F); \
} \
}
#endif /* ifndef SQLITE_AMALGAMATION */
typedef struct unicode_tokenizer unicode_tokenizer;
typedef struct unicode_cursor unicode_cursor;
struct unicode_tokenizer {
sqlite3_tokenizer base;
int bRemoveDiacritic;
int nException;
int *aiException;
};
struct unicode_cursor {
sqlite3_tokenizer_cursor base;
const unsigned char *aInput; /* Input text being tokenized */
int nInput; /* Size of aInput[] in bytes */
int iOff; /* Current offset within aInput[] */
int iToken; /* Index of next token to be returned */
char *zToken; /* storage for current token */
int nAlloc; /* space allocated at zToken */
};
/*
** Destroy a tokenizer allocated by unicodeCreate().
*/
static int unicodeDestroy(sqlite3_tokenizer *pTokenizer){
if( pTokenizer ){
unicode_tokenizer *p = (unicode_tokenizer *)pTokenizer;
sqlite3_free(p->aiException);
sqlite3_free(p);
}
return SQLITE_OK;
}
/*
** As part of a tokenchars= or separators= option, the CREATE VIRTUAL TABLE
** statement has specified that the tokenizer for this table shall consider
** all characters in string zIn/nIn to be separators (if bAlnum==0) or
** token characters (if bAlnum==1).
**
** For each codepoint in the zIn/nIn string, this function checks if the
** sqlite3FtsUnicodeIsalnum() function already returns the desired result.
** If so, no action is taken. Otherwise, the codepoint is added to the
** unicode_tokenizer.aiException[] array. For the purposes of tokenization,
** the return value of sqlite3FtsUnicodeIsalnum() is inverted for all
** codepoints in the aiException[] array.
**
** If a standalone diacritic mark (one that sqlite3FtsUnicodeIsdiacritic()
** identifies as a diacritic) occurs in the zIn/nIn string it is ignored.
** It is not possible to change the behavior of the tokenizer with respect
** to these codepoints.
*/
static int unicodeAddExceptions(
unicode_tokenizer *p, /* Tokenizer to add exceptions to */
int bAlnum, /* Replace Isalnum() return value with this */
const char *zIn, /* Array of characters to make exceptions */
int nIn /* Length of z in bytes */
){
const unsigned char *z = (const unsigned char *)zIn;
const unsigned char *zTerm = &z[nIn];
unsigned int iCode;
int nEntry = 0;
assert( bAlnum==0 || bAlnum==1 );
while( z<zTerm ){
READ_UTF8(z, zTerm, iCode);
assert( (sqlite3FtsUnicodeIsalnum((int)iCode) & 0xFFFFFFFE)==0 );
if( sqlite3FtsUnicodeIsalnum((int)iCode)!=bAlnum
&& sqlite3FtsUnicodeIsdiacritic((int)iCode)==0
){
nEntry++;
}
}
if( nEntry ){
int *aNew; /* New aiException[] array */
int nNew; /* Number of valid entries in array aNew[] */
aNew = sqlite3_realloc(p->aiException, (p->nException+nEntry)*sizeof(int));
if( aNew==0 ) return SQLITE_NOMEM;
nNew = p->nException;
z = (const unsigned char *)zIn;
while( z<zTerm ){
READ_UTF8(z, zTerm, iCode);
if( sqlite3FtsUnicodeIsalnum((int)iCode)!=bAlnum
&& sqlite3FtsUnicodeIsdiacritic((int)iCode)==0
){
int i, j;
for(i=0; i<nNew && aNew[i]<(int)iCode; i++);
for(j=nNew; j>i; j--) aNew[j] = aNew[j-1];
aNew[i] = (int)iCode;
nNew++;
}
}
p->aiException = aNew;
p->nException = nNew;
}
return SQLITE_OK;
}
/*
** Return true if the p->aiException[] array contains the value iCode.
*/
static int unicodeIsException(unicode_tokenizer *p, int iCode){
if( p->nException>0 ){
int *a = p->aiException;
int iLo = 0;
int iHi = p->nException-1;
while( iHi>=iLo ){
int iTest = (iHi + iLo) / 2;
if( iCode==a[iTest] ){
return 1;
}else if( iCode>a[iTest] ){
iLo = iTest+1;
}else{
iHi = iTest-1;
}
}
}
return 0;
}
/*
** Return true if, for the purposes of tokenization, codepoint iCode is
** considered a token character (not a separator).
*/
static int unicodeIsAlnum(unicode_tokenizer *p, int iCode){
assert( (sqlite3FtsUnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 );
return sqlite3FtsUnicodeIsalnum(iCode) ^ unicodeIsException(p, iCode);
}
/*
** Create a new tokenizer instance.
*/
static int unicodeCreate(
int nArg, /* Size of array argv[] */
const char * const *azArg, /* Tokenizer creation arguments */
sqlite3_tokenizer **pp /* OUT: New tokenizer handle */
){
unicode_tokenizer *pNew; /* New tokenizer object */
int i;
int rc = SQLITE_OK;
pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer));
if( pNew==NULL ) return SQLITE_NOMEM;
memset(pNew, 0, sizeof(unicode_tokenizer));
pNew->bRemoveDiacritic = 1;
for(i=0; rc==SQLITE_OK && i<nArg; i++){
const char *z = azArg[i];
int n = (int)strlen(z);
if( n==19 && memcmp("remove_diacritics=1", z, 19)==0 ){
pNew->bRemoveDiacritic = 1;
}
else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){
pNew->bRemoveDiacritic = 0;
}
else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){
rc = unicodeAddExceptions(pNew, 1, &z[11], n-11);
}
else if( n>=11 && memcmp("separators=", z, 11)==0 ){
rc = unicodeAddExceptions(pNew, 0, &z[11], n-11);
}
else{
/* Unrecognized argument */
rc = SQLITE_ERROR;
}
}
if( rc!=SQLITE_OK ){
unicodeDestroy((sqlite3_tokenizer *)pNew);
pNew = 0;
}
*pp = (sqlite3_tokenizer *)pNew;
return rc;
}
/*
** Prepare to begin tokenizing a particular string. The input
** string to be tokenized is pInput[0..nBytes-1]. A cursor
** used to incrementally tokenize this string is returned in
** *ppCursor.
*/
static int unicodeOpen(
sqlite3_tokenizer *p, /* The tokenizer */
const char *aInput, /* Input string */
int nInput, /* Size of string aInput in bytes */
sqlite3_tokenizer_cursor **pp /* OUT: New cursor object */
){
unicode_cursor *pCsr;
pCsr = (unicode_cursor *)sqlite3_malloc(sizeof(unicode_cursor));
if( pCsr==0 ){
return SQLITE_NOMEM;
}
memset(pCsr, 0, sizeof(unicode_cursor));
pCsr->aInput = (const unsigned char *)aInput;
if( aInput==0 ){
pCsr->nInput = 0;
}else if( nInput<0 ){
pCsr->nInput = (int)strlen(aInput);
}else{
pCsr->nInput = nInput;
}
*pp = &pCsr->base;
UNUSED_PARAMETER(p);
return SQLITE_OK;
}
/*
** Close a tokenization cursor previously opened by a call to
** simpleOpen() above.
*/
static int unicodeClose(sqlite3_tokenizer_cursor *pCursor){
unicode_cursor *pCsr = (unicode_cursor *) pCursor;
sqlite3_free(pCsr->zToken);
sqlite3_free(pCsr);
return SQLITE_OK;
}
/*
** Extract the next token from a tokenization cursor. The cursor must
** have been opened by a prior call to simpleOpen().
*/
static int unicodeNext(
sqlite3_tokenizer_cursor *pC, /* Cursor returned by simpleOpen */
const char **paToken, /* OUT: Token text */
int *pnToken, /* OUT: Number of bytes at *paToken */
int *piStart, /* OUT: Starting offset of token */
int *piEnd, /* OUT: Ending offset of token */
int *piPos /* OUT: Position integer of token */
){
unicode_cursor *pCsr = (unicode_cursor *)pC;
unicode_tokenizer *p = ((unicode_tokenizer *)pCsr->base.pTokenizer);
unsigned int iCode = 0;
char *zOut;
const unsigned char *z = &pCsr->aInput[pCsr->iOff];
const unsigned char *zStart = z;
const unsigned char *zEnd;
const unsigned char *zTerm = &pCsr->aInput[pCsr->nInput];
/* Scan past any delimiter characters before the start of the next token.
** Return SQLITE_DONE early if this takes us all the way to the end of
** the input. */
while( z<zTerm ){
READ_UTF8(z, zTerm, iCode);
if( unicodeIsAlnum(p, (int)iCode) ) break;
zStart = z;
}
if( zStart>=zTerm ) return SQLITE_DONE;
zOut = pCsr->zToken;
do {
int iOut;
/* Grow the output buffer if required. */
if( (zOut-pCsr->zToken)>=(pCsr->nAlloc-4) ){
char *zNew = sqlite3_realloc(pCsr->zToken, pCsr->nAlloc+64);
if( !zNew ) return SQLITE_NOMEM;
zOut = &zNew[zOut - pCsr->zToken];
pCsr->zToken = zNew;
pCsr->nAlloc += 64;
}
/* Write the folded case of the last character read to the output */
zEnd = z;
iOut = sqlite3FtsUnicodeFold((int)iCode, p->bRemoveDiacritic);
if( iOut ){
WRITE_UTF8(zOut, iOut);
}
/* If the cursor is not at EOF, read the next character */
if( z>=zTerm ) break;
READ_UTF8(z, zTerm, iCode);
}while( unicodeIsAlnum(p, (int)iCode)
|| sqlite3FtsUnicodeIsdiacritic((int)iCode)
);
/* Set the output variables and return. */
pCsr->iOff = (int)(z - pCsr->aInput);
*paToken = pCsr->zToken;
*pnToken = (int)(zOut - pCsr->zToken);
*piStart = (int)(zStart - pCsr->aInput);
*piEnd = (int)(zEnd - pCsr->aInput);
*piPos = pCsr->iToken++;
return SQLITE_OK;
}
/*
** Set *ppModule to a pointer to the sqlite3_tokenizer_module
** structure for the unicode tokenizer.
*/
SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const **ppModule){
static const sqlite3_tokenizer_module module = {
0,
unicodeCreate,
unicodeDestroy,
unicodeOpen,
unicodeClose,
unicodeNext,
0,
};
*ppModule = &module;
}
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */
/************** End of fts3_unicode.c ****************************************/
/************** Begin file fts3_unicode2.c ***********************************/
/*
** 2012 May 25
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
*/
/*
** DO NOT EDIT THIS MACHINE GENERATED FILE.
*/
#ifndef SQLITE_DISABLE_FTS3_UNICODE
#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)
/* #include <assert.h> */
/*
** Return true if the argument corresponds to a unicode codepoint
** classified as either a letter or a number. Otherwise false.
**
** The results are undefined if the value passed to this function
** is less than zero.
*/
SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int c){
/* Each unsigned integer in the following array corresponds to a contiguous
** range of unicode codepoints that are not either letters or numbers (i.e.
** codepoints for which this function should return 0).
**
** The most significant 22 bits in each 32-bit value contain the first
** codepoint in the range. The least significant 10 bits are used to store
** the size of the range (always at least 1). In other words, the value
** ((C<<22) + N) represents a range of N codepoints starting with codepoint
** C. It is not possible to represent a range larger than 1023 codepoints
** using this format.
*/
static const unsigned int aEntry[] = {
0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07,
0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01,
0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401,
0x000BBC81, 0x000DD401, 0x000DF801, 0x000E1002, 0x000E1C01,
0x000FD801, 0x00120808, 0x00156806, 0x00162402, 0x00163C01,
0x00164437, 0x0017CC02, 0x00180005, 0x00181816, 0x00187802,
0x00192C15, 0x0019A804, 0x0019C001, 0x001B5001, 0x001B580F,
0x001B9C07, 0x001BF402, 0x001C000E, 0x001C3C01, 0x001C4401,
0x001CC01B, 0x001E980B, 0x001FAC09, 0x001FD804, 0x00205804,
0x00206C09, 0x00209403, 0x0020A405, 0x0020C00F, 0x00216403,
0x00217801, 0x0023901B, 0x00240004, 0x0024E803, 0x0024F812,
0x00254407, 0x00258804, 0x0025C001, 0x00260403, 0x0026F001,
0x0026F807, 0x00271C02, 0x00272C03, 0x00275C01, 0x00278802,
0x0027C802, 0x0027E802, 0x00280403, 0x0028F001, 0x0028F805,
0x00291C02, 0x00292C03, 0x00294401, 0x0029C002, 0x0029D401,
0x002A0403, 0x002AF001, 0x002AF808, 0x002B1C03, 0x002B2C03,
0x002B8802, 0x002BC002, 0x002C0403, 0x002CF001, 0x002CF807,
0x002D1C02, 0x002D2C03, 0x002D5802, 0x002D8802, 0x002DC001,
0x002E0801, 0x002EF805, 0x002F1803, 0x002F2804, 0x002F5C01,
0x002FCC08, 0x00300403, 0x0030F807, 0x00311803, 0x00312804,
0x00315402, 0x00318802, 0x0031FC01, 0x00320802, 0x0032F001,
0x0032F807, 0x00331803, 0x00332804, 0x00335402, 0x00338802,
0x00340802, 0x0034F807, 0x00351803, 0x00352804, 0x00355C01,
0x00358802, 0x0035E401, 0x00360802, 0x00372801, 0x00373C06,
0x00375801, 0x00376008, 0x0037C803, 0x0038C401, 0x0038D007,
0x0038FC01, 0x00391C09, 0x00396802, 0x003AC401, 0x003AD006,
0x003AEC02, 0x003B2006, 0x003C041F, 0x003CD00C, 0x003DC417,
0x003E340B, 0x003E6424, 0x003EF80F, 0x003F380D, 0x0040AC14,
0x00412806, 0x00415804, 0x00417803, 0x00418803, 0x00419C07,
0x0041C404, 0x0042080C, 0x00423C01, 0x00426806, 0x0043EC01,
0x004D740C, 0x004E400A, 0x00500001, 0x0059B402, 0x005A0001,
0x005A6C02, 0x005BAC03, 0x005C4803, 0x005CC805, 0x005D4802,
0x005DC802, 0x005ED023, 0x005F6004, 0x005F7401, 0x0060000F,
0x0062A401, 0x0064800C, 0x0064C00C, 0x00650001, 0x00651002,
0x0066C011, 0x00672002, 0x00677822, 0x00685C05, 0x00687802,
0x0069540A, 0x0069801D, 0x0069FC01, 0x006A8007, 0x006AA006,
0x006C0005, 0x006CD011, 0x006D6823, 0x006E0003, 0x006E840D,
0x006F980E, 0x006FF004, 0x00709014, 0x0070EC05, 0x0071F802,
0x00730008, 0x00734019, 0x0073B401, 0x0073C803, 0x00770027,
0x0077F004, 0x007EF401, 0x007EFC03, 0x007F3403, 0x007F7403,
0x007FB403, 0x007FF402, 0x00800065, 0x0081A806, 0x0081E805,
0x00822805, 0x0082801A, 0x00834021, 0x00840002, 0x00840C04,
0x00842002, 0x00845001, 0x00845803, 0x00847806, 0x00849401,
0x00849C01, 0x0084A401, 0x0084B801, 0x0084E802, 0x00850005,
0x00852804, 0x00853C01, 0x00864264, 0x00900027, 0x0091000B,
0x0092704E, 0x00940200, 0x009C0475, 0x009E53B9, 0x00AD400A,
0x00B39406, 0x00B3BC03, 0x00B3E404, 0x00B3F802, 0x00B5C001,
0x00B5FC01, 0x00B7804F, 0x00B8C00C, 0x00BA001A, 0x00BA6C59,
0x00BC00D6, 0x00BFC00C, 0x00C00005, 0x00C02019, 0x00C0A807,
0x00C0D802, 0x00C0F403, 0x00C26404, 0x00C28001, 0x00C3EC01,
}
return 1;
}
/*
** If the argument is a codepoint corresponding to a lowercase letter
** in the ASCII range with a diacritic added, return the codepoint
** of the ASCII letter only. For example, if passed 235 - "LATIN
** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER
** E"). The resuls of passing a codepoint that corresponds to an
** uppercase letter are undefined.
*/
static int remove_diacritic(int c){
unsigned short aDia[] = {
0, 1797, 1848, 1859, 1891, 1928, 1940, 1995,
2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286,
2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732,
2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336,
3456, 3696, 3712, 3728, 3744, 3896, 3912, 3928,
3968, 4008, 4040, 4106, 4138, 4170, 4202, 4234,
4266, 4296, 4312, 4344, 4408, 4424, 4472, 4504,
6148, 6198, 6264, 6280, 6360, 6429, 6505, 6529,
61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726,
61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122,
62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536,
62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730,
62924, 63050, 63082, 63274, 63390,
};
char aChar[] = {
'\0', 'a', 'c', 'e', 'i', 'n', 'o', 'u', 'y', 'y', 'a', 'c',
'd', 'e', 'e', 'g', 'h', 'i', 'j', 'k', 'l', 'n', 'o', 'r',
's', 't', 'u', 'u', 'w', 'y', 'z', 'o', 'u', 'a', 'i', 'o',
'u', 'g', 'k', 'o', 'j', 'g', 'n', 'a', 'e', 'i', 'o', 'r',
'u', 's', 't', 'h', 'a', 'e', 'o', 'y', '\0', '\0', '\0', '\0',
'\0', '\0', '\0', '\0', 'a', 'b', 'd', 'd', 'e', 'f', 'g', 'h',
'h', 'i', 'k', 'l', 'l', 'm', 'n', 'p', 'r', 'r', 's', 't',
'u', 'v', 'w', 'w', 'x', 'y', 'z', 'h', 't', 'w', 'y', 'a',
'e', 'i', 'o', 'u', 'y',
};
unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
int iRes = 0;
int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1;
int iLo = 0;
while( iHi>=iLo ){
int iTest = (iHi + iLo) / 2;
if( key >= aDia[iTest] ){
iRes = iTest;
iLo = iTest+1;
}else{
iHi = iTest-1;
}
}
assert( key>=aDia[iRes] );
return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
}
/*
** Return true if the argument interpreted as a unicode codepoint
** is a diacritical modifier character.
*/
SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int c){
unsigned int mask0 = 0x08029FDF;
unsigned int mask1 = 0x000361F8;
if( c<768 || c>817 ) return 0;
return (c < 768+32) ?
(mask0 & (1 << (c-768))) :
(mask1 & (1 << (c-768-32)));
}
/*
** Interpret the argument as a unicode codepoint. If the codepoint
** is an upper case character that has a lower case equivalent,
** return the codepoint corresponding to the lower case version.
** Otherwise, return a copy of the argument.
**
** The results are undefined if the value passed to this function
** is less than zero.
*/
SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){
/* Each entry in the following array defines a rule for folding a range
** of codepoints to lower case. The rule applies to a range of nRange
** codepoints starting at codepoint iCode.
**
** If the least significant bit in flags is clear, then the rule applies
** to all nRange codepoints (i.e. all nRange codepoints are upper case and
** need to be folded). Or, if it is set, then the rule only applies to
** every second codepoint in the range, starting with codepoint C.
**
** The 7 most significant bits in flags are an index into the aiOff[]
** array. If a specific codepoint C does require folding, then its lower
** case equivalent is ((C + aiOff[flags>>1]) & 0xFFFF).
**
** The contents of this array are generated by parsing the CaseFolding.txt
** file distributed as part of the "Unicode Character Database". See
** http://www.unicode.org for details.
*/
static const struct TableEntry {
unsigned short iCode;
unsigned char flags;
unsigned char nRange;
} aEntry[] = {
{65, 14, 26}, {181, 64, 1}, {192, 14, 23},
{216, 14, 7}, {256, 1, 48}, {306, 1, 6},
{313, 1, 16}, {330, 1, 46}, {376, 116, 1},
{377, 1, 6}, {383, 104, 1}, {385, 50, 1},
{386, 1, 4}, {390, 44, 1}, {391, 0, 1},
{393, 42, 2}, {395, 0, 1}, {398, 32, 1},
{399, 38, 1}, {400, 40, 1}, {401, 0, 1},
{403, 42, 1}, {404, 46, 1}, {406, 52, 1},
{407, 48, 1}, {408, 0, 1}, {412, 52, 1},
{413, 54, 1}, {415, 56, 1}, {416, 1, 6},
{422, 60, 1}, {423, 0, 1}, {425, 60, 1},
{428, 0, 1}, {430, 60, 1}, {431, 0, 1},
{433, 58, 2}, {435, 1, 4}, {439, 62, 1},
{440, 0, 1}, {444, 0, 1}, {452, 2, 1},
{453, 0, 1}, {455, 2, 1}, {456, 0, 1},
{458, 2, 1}, {459, 1, 18}, {478, 1, 18},
{497, 2, 1}, {498, 1, 4}, {502, 122, 1},
{503, 134, 1}, {504, 1, 40}, {544, 110, 1},
{546, 1, 18}, {570, 70, 1}, {571, 0, 1},
{573, 108, 1}, {574, 68, 1}, {577, 0, 1},
{579, 106, 1}, {580, 28, 1}, {581, 30, 1},
{582, 1, 10}, {837, 36, 1}, {880, 1, 4},
{886, 0, 1}, {902, 18, 1}, {904, 16, 3},
{908, 26, 1}, {910, 24, 2}, {913, 14, 17},
{931, 14, 9}, {962, 0, 1}, {975, 4, 1},
{976, 140, 1}, {977, 142, 1}, {981, 146, 1},
{982, 144, 1}, {984, 1, 24}, {1008, 136, 1},
{1009, 138, 1}, {1012, 130, 1}, {1013, 128, 1},
{1015, 0, 1}, {1017, 152, 1}, {1018, 0, 1},
{1021, 110, 3}, {1024, 34, 16}, {1040, 14, 32},
{1120, 1, 34}, {1162, 1, 54}, {1216, 6, 1},
{1217, 1, 14}, {1232, 1, 88}, {1329, 22, 38},
{4256, 66, 38}, {4295, 66, 1}, {4301, 66, 1},
{7680, 1, 150}, {7835, 132, 1}, {7838, 96, 1},
{7840, 1, 96}, {7944, 150, 8}, {7960, 150, 6},
{7976, 150, 8}, {7992, 150, 8}, {8008, 150, 6},
{8025, 151, 8}, {8040, 150, 8}, {8072, 150, 8},
{8088, 150, 8}, {8104, 150, 8}, {8120, 150, 2},
{8122, 126, 2}, {8124, 148, 1}, {8126, 100, 1},
{8136, 124, 4}, {8140, 148, 1}, {8152, 150, 2},
{8154, 120, 2}, {8168, 150, 2}, {8170, 118, 2},
{8172, 152, 1}, {8184, 112, 2}, {8186, 114, 2},
{8188, 148, 1}, {8486, 98, 1}, {8490, 92, 1},
{8491, 94, 1}, {8498, 12, 1}, {8544, 8, 16},
{8579, 0, 1}, {9398, 10, 26}, {11264, 22, 47},
{11360, 0, 1}, {11362, 88, 1}, {11363, 102, 1},
{11364, 90, 1}, {11367, 1, 6}, {11373, 84, 1},
{11374, 86, 1}, {11375, 80, 1}, {11376, 82, 1},
{11378, 0, 1}, {11381, 0, 1}, {11390, 78, 2},
{11392, 1, 100}, {11499, 1, 4}, {11506, 0, 1},
{42560, 1, 46}, {42624, 1, 24}, {42786, 1, 14},
{42802, 1, 62}, {42873, 1, 4}, {42877, 76, 1},
{42878, 1, 10}, {42891, 0, 1}, {42893, 74, 1},
{42896, 1, 4}, {42912, 1, 10}, {42922, 72, 1},
{65313, 14, 26},
};
static const unsigned short aiOff[] = {
1, 2, 8, 15, 16, 26, 28, 32,
37, 38, 40, 48, 63, 64, 69, 71,
79, 80, 116, 202, 203, 205, 206, 207,
209, 210, 211, 213, 214, 217, 218, 219,
775, 7264, 10792, 10795, 23228, 23256, 30204, 54721,
54753, 54754, 54756, 54787, 54793, 54809, 57153, 57274,
57921, 58019, 58363, 61722, 65268, 65341, 65373, 65406,
65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462,
65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511,
65514, 65521, 65527, 65528, 65529,
};
int ret = c;
assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 );
if( c<128 ){
if( c>='A' && c<='Z' ) ret = c + ('a' - 'A');
}else if( c<65536 ){
const struct TableEntry *p;
int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
int iLo = 0;
int iRes = -1;
assert( c>aEntry[0].iCode );
while( iHi>=iLo ){
int iTest = (iHi + iLo) / 2;
int cmp = (c - aEntry[iTest].iCode);
if( cmp>=0 ){
iRes = iTest;
iLo = iTest+1;
}else{
iHi = iTest-1;
}
}
assert( iRes>=0 && c>=aEntry[iRes].iCode );
p = &aEntry[iRes];
if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
assert( ret>0 );
}
if( bRemoveDiacritic ) ret = remove_diacritic(ret);
}
else if( c>=66560 && c<66600 ){
ret = c + 40;
}
return ret;
}
#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */
#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */
/************** End of fts3_unicode2.c ***************************************/
/************** Begin file json1.c *******************************************/
/*
** 2015-08-12
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This SQLite extension implements JSON functions. The interface is
** modeled after MySQL JSON functions:
**
** https://dev.mysql.com/doc/refman/5.7/en/json.html
**
** For the time being, all JSON is stored as pure text. (We might add
** a JSONB type in the future which stores a binary encoding of JSON in
** a BLOB, but there is no support for JSONB in the current implementation.
** This implementation parses JSON text at 250 MB/s, so it is hard to see
** how JSONB might improve on that.)
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_JSON1)
#if !defined(SQLITEINT_H)
/* #include "sqlite3ext.h" */
#endif
SQLITE_EXTENSION_INIT1
/* #include <assert.h> */
/* #include <string.h> */
/* #include <stdlib.h> */
/* #include <stdarg.h> */
/* Mark a function parameter as unused, to suppress nuisance compiler
** warnings. */
#ifndef UNUSED_PARAM
# define UNUSED_PARAM(X) (void)(X)
#endif
#ifndef LARGEST_INT64
# define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32))
# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64)
#endif
/*
** Versions of isspace(), isalnum() and isdigit() to which it is safe
** to pass signed char values.
*/
#ifdef sqlite3Isdigit
/* Use the SQLite core versions if this routine is part of the
** SQLite amalgamation */
# define safe_isdigit(x) sqlite3Isdigit(x)
# define safe_isalnum(x) sqlite3Isalnum(x)
# define safe_isxdigit(x) sqlite3Isxdigit(x)
#else
/* Use the standard library for separate compilation */
#include <ctype.h> /* amalgamator: keep */
# define safe_isdigit(x) isdigit((unsigned char)(x))
# define safe_isalnum(x) isalnum((unsigned char)(x))
}
/*
** Register a new 2nd-generation geometry function for use with the
** r-tree MATCH operator.
*/
SQLITE_API int sqlite3_rtree_query_callback(
sqlite3 *db, /* Register SQL function on this connection */
const char *zQueryFunc, /* Name of new SQL function */
int (*xQueryFunc)(sqlite3_rtree_query_info*), /* Callback */
void *pContext, /* Extra data passed into the callback */
void (*xDestructor)(void*) /* Destructor for the extra data */
){
RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */
/* Allocate and populate the context object. */
pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback));
if( !pGeomCtx ) return SQLITE_NOMEM;
pGeomCtx->xGeom = 0;
pGeomCtx->xQueryFunc = xQueryFunc;
pGeomCtx->xDestructor = xDestructor;
pGeomCtx->pContext = pContext;
return sqlite3_create_function_v2(db, zQueryFunc, -1, SQLITE_ANY,
(void *)pGeomCtx, geomCallback, 0, 0, rtreeFreeCallback
);
}
#if !SQLITE_CORE
#ifdef _WIN32
__declspec(dllexport)
#endif
SQLITE_API int sqlite3_rtree_init(
sqlite3 *db,
char **pzErrMsg,
const sqlite3_api_routines *pApi
){
SQLITE_EXTENSION_INIT2(pApi)
return sqlite3RtreeInit(db);
}
#endif
#endif
/************** End of rtree.c ***********************************************/
/************** Begin file icu.c *********************************************/
/*
** 2007 May 6
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** $Id: icu.c,v 1.7 2007/12/13 21:54:11 drh Exp $
**
** This file implements an integration between the ICU library
** ("International Components for Unicode", an open-source library
** for handling unicode data) and SQLite. The integration uses
** ICU to provide the following to SQLite:
**
** * An implementation of the SQL regexp() function (and hence REGEXP
** operator) using the ICU uregex_XX() APIs.
**
** * Implementations of the SQL scalar upper() and lower() functions
** for case mapping.
**
** * Integration of ICU and SQLite collation sequences.
**
** * An implementation of the LIKE operator that uses ICU to
** provide case-independent matching.
*/
#if !defined(SQLITE_CORE) \
|| defined(SQLITE_ENABLE_ICU) \
|| defined(SQLITE_ENABLE_ICU_COLLATIONS)
/* Include ICU headers */
#include <unicode/utypes.h>
#include <unicode/uregex.h>
#include <unicode/ustring.h>
#include <unicode/ucol.h>
/* #include <assert.h> */
#ifndef SQLITE_CORE
/* #include "sqlite3ext.h" */
SQLITE_EXTENSION_INIT1
#else
/* #include "sqlite3.h" */
#endif
/*
** This function is called when an ICU function called from within
** the implementation of an SQL scalar function returns an error.
**
** The scalar function context passed as the first argument is
** loaded with an error message based on the following two args.
*/
static void icuFunctionError(
sqlite3_context *pCtx, /* SQLite scalar function context */
const char *zName, /* Name of ICU function that failed */
UErrorCode e /* Error code returned by ICU function */
){
char zBuf[128];
sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e));
zBuf[127] = '\0';
sqlite3_result_error(pCtx, zBuf, -1);
}
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU)
/*
** Maximum length (in bytes) of the pattern in a LIKE or GLOB
** operator.
*/
#ifndef SQLITE_MAX_LIKE_PATTERN_LENGTH
# define SQLITE_MAX_LIKE_PATTERN_LENGTH 50000
#endif
/*
** Version of sqlite3_free() that is always a function, never a macro.
*/
static void xFree(void *p){
sqlite3_free(p);
}
/*
** This lookup table is used to help decode the first byte of
** a multi-byte UTF8 character. It is copied here from SQLite source
** code file utf8.c.
*/
static const unsigned char icuUtf8Trans1[] = {
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00,
};
{"upper", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, 1, icuCaseFunc16},
{"upper", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 1, icuCaseFunc16},
{"like", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc},
{"like", 3, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc},
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */
};
int rc = SQLITE_OK;
int i;
for(i=0; rc==SQLITE_OK && i<(int)(sizeof(scalars)/sizeof(scalars[0])); i++){
const struct IcuScalar *p = &scalars[i];
rc = sqlite3_create_function(
db, p->zName, p->nArg, p->enc,
p->iContext ? (void*)db : (void*)0,
p->xFunc, 0, 0
);
}
return rc;
}
#if !SQLITE_CORE
#ifdef _WIN32
__declspec(dllexport)
#endif
SQLITE_API int sqlite3_icu_init(
sqlite3 *db,
char **pzErrMsg,
const sqlite3_api_routines *pApi
){
SQLITE_EXTENSION_INIT2(pApi)
return sqlite3IcuInit(db);
}
#endif
#endif
/************** End of icu.c *************************************************/
/************** Begin file fts3_icu.c ****************************************/
/*
** 2007 June 22
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file implements a tokenizer for fts3 based on the ICU library.
*/
/* #include "fts3Int.h" */
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
#ifdef SQLITE_ENABLE_ICU
/* #include <assert.h> */
/* #include <string.h> */
/* #include "fts3_tokenizer.h" */
#include <unicode/ubrk.h>
/* #include <unicode/ucol.h> */
/* #include <unicode/ustring.h> */
#include <unicode/utf16.h>
typedef struct IcuTokenizer IcuTokenizer;
typedef struct IcuCursor IcuCursor;
struct IcuTokenizer {
sqlite3_tokenizer base;
char *zLocale;
};
struct IcuCursor {
sqlite3_tokenizer_cursor base;
UBreakIterator *pIter; /* ICU break-iterator object */
int nChar; /* Number of UChar elements in pInput */
UChar *aChar; /* Copy of input using utf-16 encoding */
int *aOffset; /* Offsets of each character in utf-8 input */
int nBuffer;
char *zBuffer;
int iToken;
};
/*
** Create a new tokenizer instance.
*/
static int icuCreate(
int argc, /* Number of entries in argv[] */
const char * const *argv, /* Tokenizer creation arguments */
sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */
){
IcuTokenizer *p;
int n = 0;
if( argc>0 ){
n = strlen(argv[0])+1;
}
p = (IcuTokenizer *)sqlite3_malloc(sizeof(IcuTokenizer)+n);
if( !p ){
return SQLITE_NOMEM;
}
memset(p, 0, sizeof(IcuTokenizer));
if( n ){
p->zLocale = (char *)&p[1];
memcpy(p->zLocale, argv[0], n);
}
*ppTokenizer = (sqlite3_tokenizer *)p;
return SQLITE_OK;
}
/*
** Destroy a tokenizer
*/
static int icuDestroy(sqlite3_tokenizer *pTokenizer){
IcuTokenizer *p = (IcuTokenizer *)pTokenizer;
sqlite3_free(p);
return SQLITE_OK;
static void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase*);
static Fts5ExprNearset *sqlite3Fts5ParseNearset(
Fts5Parse*,
Fts5ExprNearset*,
Fts5ExprPhrase*
);
static Fts5Colset *sqlite3Fts5ParseColset(
Fts5Parse*,
Fts5Colset*,
Fts5Token *
);
static void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*);
static void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
static void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);
static void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
static void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNode*, Fts5Colset*);
static Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*);
static void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
static void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);
/*
** End of interface to code in fts5_expr.c.
**************************************************************************/
/**************************************************************************
** Interface to code in fts5_aux.c.
*/
static int sqlite3Fts5AuxInit(fts5_api*);
/*
** End of interface to code in fts5_aux.c.
**************************************************************************/
/**************************************************************************
** Interface to code in fts5_tokenizer.c.
*/
static int sqlite3Fts5TokenizerInit(fts5_api*);
/*
** End of interface to code in fts5_tokenizer.c.
**************************************************************************/
/**************************************************************************
** Interface to code in fts5_vocab.c.
*/
static int sqlite3Fts5VocabInit(Fts5Global*, sqlite3*);
/*
** End of interface to code in fts5_vocab.c.
**************************************************************************/
/**************************************************************************
** Interface to automatically generated code in fts5_unicode2.c.
*/
static int sqlite3Fts5UnicodeIsdiacritic(int c);
static int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic);
static int sqlite3Fts5UnicodeCatParse(const char*, u8*);
static int sqlite3Fts5UnicodeCategory(int iCode);
static void sqlite3Fts5UnicodeAscii(u8*, u8*);
/*
** End of interface to code in fts5_unicode2.c.
**************************************************************************/
#endif
#define FTS5_OR 1
#define FTS5_AND 2
#define FTS5_NOT 3
#define FTS5_TERM 4
#define FTS5_COLON 5
#define FTS5_MINUS 6
#define FTS5_LCP 7
#define FTS5_RCP 8
#define FTS5_STRING 9
#define FTS5_LP 10
#define FTS5_RP 11
#define FTS5_CARET 12
#define FTS5_COMMA 13
#define FTS5_PLUS 14
#define FTS5_STAR 15
/*
** 2000-05-29
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** Driver template for the LEMON parser generator.
**
** The "lemon" program processes an LALR(1) input grammar file, then uses
** this template to construct a parser. The "lemon" program inserts text
** at each "%%" line. Also, any "P-a-r-s-e" identifer prefix (without the
** interstitial "-" characters) contained in this template is changed into
** the value of the %name directive from the grammar. Otherwise, the content
** of this template is copied straight through into the generate parser
** source file.
**
** The following is the concatenation of all %include directives from the
** input grammar file:
*/
/* #include <stdio.h> */
/* #include <assert.h> */
/************ Begin %include sections from the grammar ************************/
/* #include "fts5Int.h" */
/* #include "fts5parse.h" */
/*
** Disable all error recovery processing in the parser push-down
** automaton.
*/
#define fts5YYNOERRORRECOVERY 1
/*
** Make fts5yytestcase() the same as testcase()
*/
}
static int sqlite3Fts5PoslistWriterAppend(
Fts5Buffer *pBuf,
Fts5PoslistWriter *pWriter,
i64 iPos
){
int rc = 0; /* Initialized only to suppress erroneous warning from Clang */
if( fts5BufferGrow(&rc, pBuf, 5+5+5) ) return rc;
sqlite3Fts5PoslistSafeAppend(pBuf, &pWriter->iPrev, iPos);
return SQLITE_OK;
}
static void *sqlite3Fts5MallocZero(int *pRc, int nByte){
void *pRet = 0;
if( *pRc==SQLITE_OK ){
pRet = sqlite3_malloc(nByte);
if( pRet==0 ){
if( nByte>0 ) *pRc = SQLITE_NOMEM;
}else{
memset(pRet, 0, nByte);
}
}
return pRet;
}
/*
** Return a nul-terminated copy of the string indicated by pIn. If nIn
** is non-negative, then it is the length of the string in bytes. Otherwise,
** the length of the string is determined using strlen().
**
** It is the responsibility of the caller to eventually free the returned
** buffer using sqlite3_free(). If an OOM error occurs, NULL is returned.
*/
static char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn){
char *zRet = 0;
if( *pRc==SQLITE_OK ){
if( nIn<0 ){
nIn = (int)strlen(pIn);
}
zRet = (char*)sqlite3_malloc(nIn+1);
if( zRet ){
memcpy(zRet, pIn, nIn);
zRet[nIn] = '\0';
}else{
*pRc = SQLITE_NOMEM;
}
}
return zRet;
}
/*
** Return true if character 't' may be part of an FTS5 bareword, or false
** otherwise. Characters that may be part of barewords:
**
** * All non-ASCII characters,
** * The 52 upper and lower case ASCII characters, and
** * The 10 integer ASCII characters.
** * The underscore character "_" (0x5F).
** * The unicode "subsitute" character (0x1A).
*/
static int sqlite3Fts5IsBareword(char t){
u8 aBareword[128] = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x00 .. 0x0F */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, /* 0x10 .. 0x1F */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x20 .. 0x2F */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 0x30 .. 0x3F */
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x40 .. 0x4F */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 0x50 .. 0x5F */
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x60 .. 0x6F */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0 /* 0x70 .. 0x7F */
};
return (t & 0x80) || aBareword[(int)t];
}
/*************************************************************************
*/
typedef struct Fts5TermsetEntry Fts5TermsetEntry;
struct Fts5TermsetEntry {
char *pTerm;
int nTerm;
int iIdx; /* Index (main or aPrefix[] entry) */
Fts5TermsetEntry *pNext;
};
struct Fts5Termset {
Fts5TermsetEntry *apHash[512];
};
static int sqlite3Fts5TermsetNew(Fts5Termset **pp){
int rc = SQLITE_OK;
*pp = sqlite3Fts5MallocZero(&rc, sizeof(Fts5Termset));
return rc;
}
static int sqlite3Fts5TermsetAdd(
Fts5Termset *p,
int iIdx,
const char *pTerm, int nTerm,
int *pbPresent
){
int rc = SQLITE_OK;
*pbPresent = 0;
if( p ){
int i;
u32 hash = 13;
Fts5TermsetEntry *pEntry;
/* Calculate a hash value for this term. This is the same hash checksum
** used by the fts5_hash.c module. This is not important for correct
** operation of the module, but is necessary to ensure that some tests
** designed to produce hash table collisions really do work. */
for(i=nTerm-1; i>=0; i--){
hash = (hash << 3) ^ hash ^ pTerm[i];
}
hash = (hash << 3) ^ hash ^ iIdx;
hash = hash % ArraySize(p->apHash);
pRet->db = db;
pRet->iCookie = -1;
nByte = nArg * (sizeof(char*) + sizeof(u8));
pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, nByte);
pRet->abUnindexed = (u8*)&pRet->azCol[nArg];
pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1);
pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1);
pRet->bColumnsize = 1;
pRet->eDetail = FTS5_DETAIL_FULL;
#ifdef SQLITE_DEBUG
pRet->bPrefixIndex = 1;
#endif
if( rc==SQLITE_OK && sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){
*pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName);
rc = SQLITE_ERROR;
}
for(i=3; rc==SQLITE_OK && i<nArg; i++){
const char *zOrig = azArg[i];
const char *z;
char *zOne = 0;
char *zTwo = 0;
int bOption = 0;
int bMustBeCol = 0;
z = fts5ConfigGobbleWord(&rc, zOrig, &zOne, &bMustBeCol);
z = fts5ConfigSkipWhitespace(z);
if( z && *z=='=' ){
bOption = 1;
z++;
if( bMustBeCol ) z = 0;
}
z = fts5ConfigSkipWhitespace(z);
if( z && z[0] ){
int bDummy;
z = fts5ConfigGobbleWord(&rc, z, &zTwo, &bDummy);
if( z && z[0] ) z = 0;
}
if( rc==SQLITE_OK ){
if( z==0 ){
*pzErr = sqlite3_mprintf("parse error in \"%s\"", zOrig);
rc = SQLITE_ERROR;
}else{
if( bOption ){
rc = fts5ConfigParseSpecial(pGlobal, pRet, zOne, zTwo?zTwo:"", pzErr);
}else{
rc = fts5ConfigParseColumn(pRet, zOne, zTwo, pzErr);
zOne = 0;
}
}
}
sqlite3_free(zOne);
sqlite3_free(zTwo);
}
/* If a tokenizer= option was successfully parsed, the tokenizer has
** already been allocated. Otherwise, allocate an instance of the default
** tokenizer (unicode61) now. */
if( rc==SQLITE_OK && pRet->pTok==0 ){
rc = fts5ConfigDefaultTokenizer(pGlobal, pRet);
}
/* If no zContent option was specified, fill in the default values. */
if( rc==SQLITE_OK && pRet->zContent==0 ){
const char *zTail = 0;
assert( pRet->eContent==FTS5_CONTENT_NORMAL
|| pRet->eContent==FTS5_CONTENT_NONE
);
if( pRet->eContent==FTS5_CONTENT_NORMAL ){
zTail = "content";
}else if( pRet->bColumnsize ){
zTail = "docsize";
}
if( zTail ){
pRet->zContent = sqlite3Fts5Mprintf(
&rc, "%Q.'%q_%s'", pRet->zDb, pRet->zName, zTail
);
}
}
if( rc==SQLITE_OK && pRet->zContentRowid==0 ){
pRet->zContentRowid = sqlite3Fts5Strndup(&rc, "rowid", -1);
}
/* Formulate the zContentExprlist text */
if( rc==SQLITE_OK ){
rc = fts5ConfigMakeExprlist(pRet);
}
if( rc!=SQLITE_OK ){
sqlite3Fts5ConfigFree(pRet);
*ppOut = 0;
}
return rc;
}
/*
** Free the configuration object passed as the only argument.
*/
static void sqlite3Fts5ConfigFree(Fts5Config *pConfig){
if( pConfig ){
int i;
if( pConfig->pTok ){
pConfig->pTokApi->xDelete(pConfig->pTok);
}
sqlite3_free(pConfig->zDb);
sqlite3_free(pConfig->zName);
for(i=0; i<pConfig->nCol; i++){
sqlite3_free(pConfig->azCol[i]);
}
sqlite3_free(pConfig->azCol);
sqlite3_free(pConfig->aPrefix);
sqlite3_free(pConfig->zRank);
sqlite3_free(pConfig->zRankArgs);
sqlite3_free(pConfig->zContent);
sqlite3_free(pConfig->zContentRowid);
sqlite3_free(pConfig->zContentExprlist);
azConfig[1] = "main";
azConfig[2] = "tbl";
for(i=3; iArg<nArg; iArg++){
azConfig[i++] = (const char*)sqlite3_value_text(apVal[iArg]);
}
zExpr = (const char*)sqlite3_value_text(apVal[0]);
rc = sqlite3Fts5ConfigParse(pGlobal, db, nConfig, azConfig, &pConfig, &zErr);
if( rc==SQLITE_OK ){
rc = sqlite3Fts5ExprNew(pConfig, pConfig->nCol, zExpr, &pExpr, &zErr);
}
if( rc==SQLITE_OK ){
char *zText;
if( pExpr->pRoot->xNext==0 ){
zText = sqlite3_mprintf("");
}else if( bTcl ){
zText = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->pRoot);
}else{
zText = fts5ExprPrint(pConfig, pExpr->pRoot);
}
if( zText==0 ){
rc = SQLITE_NOMEM;
}else{
sqlite3_result_text(pCtx, zText, -1, SQLITE_TRANSIENT);
sqlite3_free(zText);
}
}
if( rc!=SQLITE_OK ){
if( zErr ){
sqlite3_result_error(pCtx, zErr, -1);
sqlite3_free(zErr);
}else{
sqlite3_result_error_code(pCtx, rc);
}
}
sqlite3_free((void *)azConfig);
sqlite3Fts5ConfigFree(pConfig);
sqlite3Fts5ExprFree(pExpr);
}
static void fts5ExprFunctionHr(
sqlite3_context *pCtx, /* Function call context */
int nArg, /* Number of args */
sqlite3_value **apVal /* Function arguments */
){
fts5ExprFunction(pCtx, nArg, apVal, 0);
}
static void fts5ExprFunctionTcl(
sqlite3_context *pCtx, /* Function call context */
int nArg, /* Number of args */
sqlite3_value **apVal /* Function arguments */
){
fts5ExprFunction(pCtx, nArg, apVal, 1);
}
/*
** The implementation of an SQLite user-defined-function that accepts a
** single integer as an argument. If the integer is an alpha-numeric
** unicode code point, 1 is returned. Otherwise 0.
*/
static void fts5ExprIsAlnum(
sqlite3_context *pCtx, /* Function call context */
int nArg, /* Number of args */
sqlite3_value **apVal /* Function arguments */
){
int iCode;
u8 aArr[32];
if( nArg!=1 ){
sqlite3_result_error(pCtx,
"wrong number of arguments to function fts5_isalnum", -1
);
return;
}
memset(aArr, 0, sizeof(aArr));
sqlite3Fts5UnicodeCatParse("L*", aArr);
sqlite3Fts5UnicodeCatParse("N*", aArr);
sqlite3Fts5UnicodeCatParse("Co", aArr);
iCode = sqlite3_value_int(apVal[0]);
sqlite3_result_int(pCtx, aArr[sqlite3Fts5UnicodeCategory(iCode)]);
}
static void fts5ExprFold(
sqlite3_context *pCtx, /* Function call context */
int nArg, /* Number of args */
sqlite3_value **apVal /* Function arguments */
){
if( nArg!=1 && nArg!=2 ){
sqlite3_result_error(pCtx,
"wrong number of arguments to function fts5_fold", -1
);
}else{
int iCode;
int bRemoveDiacritics = 0;
iCode = sqlite3_value_int(apVal[0]);
if( nArg==2 ) bRemoveDiacritics = sqlite3_value_int(apVal[1]);
sqlite3_result_int(pCtx, sqlite3Fts5UnicodeFold(iCode, bRemoveDiacritics));
}
}
/*
** This is called during initialization to register the fts5_expr() scalar
** UDF with the SQLite handle passed as the only argument.
*/
static int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){
struct Fts5ExprFunc {
const char *z;
void (*x)(sqlite3_context*,int,sqlite3_value**);
} aFunc[] = {
{ "fts5_expr", fts5ExprFunctionHr },
{ "fts5_expr_tcl", fts5ExprFunctionTcl },
{ "fts5_isalnum", fts5ExprIsAlnum },
{ "fts5_fold", fts5ExprFold },
};
int i;
int rc = SQLITE_OK;
void *pCtx = (void*)pGlobal;
for(i=0; rc==SQLITE_OK && i<ArraySize(aFunc); i++){
struct Fts5ExprFunc *p = &aFunc[i];
rc = sqlite3Fts5IndexReinit(p);
}
}
}
assert( rc!=SQLITE_OK || p->rc==SQLITE_OK );
if( rc ){
sqlite3Fts5IndexClose(p);
*pp = 0;
}
return rc;
}
/*
** Close a handle opened by an earlier call to sqlite3Fts5IndexOpen().
*/
static int sqlite3Fts5IndexClose(Fts5Index *p){
int rc = SQLITE_OK;
if( p ){
assert( p->pReader==0 );
fts5StructureInvalidate(p);
sqlite3_finalize(p->pWriter);
sqlite3_finalize(p->pDeleter);
sqlite3_finalize(p->pIdxWriter);
sqlite3_finalize(p->pIdxDeleter);
sqlite3_finalize(p->pIdxSelect);
sqlite3_finalize(p->pDataVersion);
sqlite3Fts5HashFree(p->pHash);
sqlite3_free(p->zDataTbl);
sqlite3_free(p);
}
return rc;
}
/*
** Argument p points to a buffer containing utf-8 text that is n bytes in
** size. Return the number of bytes in the nChar character prefix of the
** buffer, or 0 if there are less than nChar characters in total.
*/
static int sqlite3Fts5IndexCharlenToBytelen(
const char *p,
int nByte,
int nChar
){
int n = 0;
int i;
for(i=0; i<nChar; i++){
if( n>=nByte ) return 0; /* Input contains fewer than nChar chars */
if( (unsigned char)p[n++]>=0xc0 ){
while( (p[n] & 0xc0)==0x80 ){
n++;
if( n>=nByte ) break;
}
}
}
return n;
}
/*
** pIn is a UTF-8 encoded string, nIn bytes in size. Return the number of
** unicode characters in the string.
*/
static int fts5IndexCharlen(const char *pIn, int nIn){
int nChar = 0;
int i = 0;
while( i<nIn ){
if( (unsigned char)pIn[i++]>=0xc0 ){
while( i<nIn && (pIn[i] & 0xc0)==0x80 ) i++;
}
nChar++;
}
return nChar;
}
/*
** Insert or remove data to or from the index. Each time a document is
** added to or removed from the index, this function is called one or more
** times.
**
** For an insert, it must be called once for each token in the new document.
** If the operation is a delete, it must be called (at least) once for each
** unique token in the document with an iCol value less than zero. The iPos
** argument is ignored for a delete.
*/
static int sqlite3Fts5IndexWrite(
Fts5Index *p, /* Index to write to */
int iCol, /* Column token appears in (-ve -> delete) */
int iPos, /* Position of token within column */
const char *pToken, int nToken /* Token to add or remove to or from index */
){
int i; /* Used to iterate through indexes */
int rc = SQLITE_OK; /* Return code */
Fts5Config *pConfig = p->pConfig;
assert( p->rc==SQLITE_OK );
assert( (iCol<0)==p->bDelete );
/* Add the entry to the main terms index. */
rc = sqlite3Fts5HashWrite(
p->pHash, p->iWriteRowid, iCol, iPos, FTS5_MAIN_PREFIX, pToken, nToken
);
for(i=0; i<pConfig->nPrefix && rc==SQLITE_OK; i++){
const int nChar = pConfig->aPrefix[i];
int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar);
if( nByte ){
rc = sqlite3Fts5HashWrite(p->pHash,
p->iWriteRowid, iCol, iPos, (char)(FTS5_MAIN_PREFIX+i+1), pToken,
nByte
);
}
}
return rc;
}
/*
** Open a new iterator to iterate though all rowid that match the
** specified token or token prefix.
*/
static int sqlite3Fts5IndexQuery(
}
sqlite3_set_last_insert_rowid(p->pConfig->db, iLastRowid);
return rc;
}
static int sqlite3Fts5StorageRollback(Fts5Storage *p){
p->bTotalsValid = 0;
return sqlite3Fts5IndexRollback(p->pIndex);
}
static int sqlite3Fts5StorageConfigValue(
Fts5Storage *p,
const char *z,
sqlite3_value *pVal,
int iVal
){
sqlite3_stmt *pReplace = 0;
int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_CONFIG, &pReplace, 0);
if( rc==SQLITE_OK ){
sqlite3_bind_text(pReplace, 1, z, -1, SQLITE_STATIC);
if( pVal ){
sqlite3_bind_value(pReplace, 2, pVal);
}else{
sqlite3_bind_int(pReplace, 2, iVal);
}
sqlite3_step(pReplace);
rc = sqlite3_reset(pReplace);
sqlite3_bind_null(pReplace, 1);
}
if( rc==SQLITE_OK && pVal ){
int iNew = p->pConfig->iCookie + 1;
rc = sqlite3Fts5IndexSetCookie(p->pIndex, iNew);
if( rc==SQLITE_OK ){
p->pConfig->iCookie = iNew;
}
}
return rc;
}
/*
** 2014 May 31
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
*/
/* #include "fts5Int.h" */
/**************************************************************************
** Start of ascii tokenizer implementation.
*/
/*
** For tokenizers with no "unicode" modifier, the set of token characters
** is the same as the set of ASCII range alphanumeric characters.
*/
static unsigned char aAsciiTokenChar[128] = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x00..0x0F */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x10..0x1F */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x20..0x2F */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 0x30..0x3F */
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x40..0x4F */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 0x50..0x5F */
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x60..0x6F */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 0x70..0x7F */
};
typedef struct AsciiTokenizer AsciiTokenizer;
struct AsciiTokenizer {
unsigned char aTokenChar[128];
};
static void fts5AsciiAddExceptions(
AsciiTokenizer *p,
const char *zArg,
int bTokenChars
){
int i;
for(i=0; zArg[i]; i++){
if( (zArg[i] & 0x80)==0 ){
p->aTokenChar[(int)zArg[i]] = (unsigned char)bTokenChars;
}
}
}
/*
** Delete a "ascii" tokenizer.
*/
static void fts5AsciiDelete(Fts5Tokenizer *p){
sqlite3_free(p);
}
/*
** Create an "ascii" tokenizer.
*/
static int fts5AsciiCreate(
void *pUnused,
const char **azArg, int nArg,
Fts5Tokenizer **ppOut
){
int rc = SQLITE_OK;
AsciiTokenizer *p = 0;
UNUSED_PARAM(pUnused);
if( nArg%2 ){
rc = SQLITE_ERROR;
}else{
p = sqlite3_malloc(sizeof(AsciiTokenizer));
if( p==0 ){
rc = SQLITE_NOMEM;
}else{
int i;
memset(p, 0, sizeof(AsciiTokenizer));
memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar));
for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
** Tokenize some text using the ascii tokenizer.
*/
static int fts5AsciiTokenize(
Fts5Tokenizer *pTokenizer,
void *pCtx,
int iUnused,
const char *pText, int nText,
int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd)
){
AsciiTokenizer *p = (AsciiTokenizer*)pTokenizer;
int rc = SQLITE_OK;
int ie;
int is = 0;
char aFold[64];
int nFold = sizeof(aFold);
char *pFold = aFold;
unsigned char *a = p->aTokenChar;
UNUSED_PARAM(iUnused);
while( is<nText && rc==SQLITE_OK ){
int nByte;
/* Skip any leading divider characters. */
while( is<nText && ((pText[is]&0x80)==0 && a[(int)pText[is]]==0) ){
is++;
}
if( is==nText ) break;
/* Count the token characters */
ie = is+1;
while( ie<nText && ((pText[ie]&0x80) || a[(int)pText[ie]] ) ){
ie++;
}
/* Fold to lower case */
nByte = ie-is;
if( nByte>nFold ){
if( pFold!=aFold ) sqlite3_free(pFold);
pFold = sqlite3_malloc(nByte*2);
if( pFold==0 ){
rc = SQLITE_NOMEM;
break;
}
nFold = nByte*2;
}
asciiFold(pFold, &pText[is], nByte);
/* Invoke the token callback */
rc = xToken(pCtx, 0, pFold, nByte, is, ie);
is = ie+1;
}
if( pFold!=aFold ) sqlite3_free(pFold);
if( rc==SQLITE_DONE ) rc = SQLITE_OK;
return rc;
}
/**************************************************************************
** Start of unicode61 tokenizer implementation.
*/
/*
** The following two macros - READ_UTF8 and WRITE_UTF8 - have been copied
** from the sqlite3 source file utf.c. If this file is compiled as part
** of the amalgamation, they are not required.
*/
#ifndef SQLITE_AMALGAMATION
static const unsigned char sqlite3Utf8Trans1[] = {
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00,
};
#define READ_UTF8(zIn, zTerm, c) \
c = *(zIn++); \
if( c>=0xc0 ){ \
c = sqlite3Utf8Trans1[c-0xc0]; \
while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \
c = (c<<6) + (0x3f & *(zIn++)); \
} \
if( c<0x80 \
|| (c&0xFFFFF800)==0xD800 \
|| (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \
}
#define WRITE_UTF8(zOut, c) { \
if( c<0x00080 ){ \
*zOut++ = (unsigned char)(c&0xFF); \
} \
else if( c<0x00800 ){ \
*zOut++ = 0xC0 + (unsigned char)((c>>6)&0x1F); \
*zOut++ = 0x80 + (unsigned char)(c & 0x3F); \
} \
else if( c<0x10000 ){ \
*zOut++ = 0xE0 + (unsigned char)((c>>12)&0x0F); \
*zOut++ = 0x80 + (unsigned char)((c>>6) & 0x3F); \
*zOut++ = 0x80 + (unsigned char)(c & 0x3F); \
}else{ \
*zOut++ = 0xF0 + (unsigned char)((c>>18) & 0x07); \
*zOut++ = 0x80 + (unsigned char)((c>>12) & 0x3F); \
*zOut++ = 0x80 + (unsigned char)((c>>6) & 0x3F); \
*zOut++ = 0x80 + (unsigned char)(c & 0x3F); \
} \
}
#endif /* ifndef SQLITE_AMALGAMATION */
typedef struct Unicode61Tokenizer Unicode61Tokenizer;
struct Unicode61Tokenizer {
unsigned char aTokenChar[128]; /* ASCII range token characters */
char *aFold; /* Buffer to fold text into */
if( aNew ){
int nNew = p->nException;
const unsigned char *zCsr = (const unsigned char*)z;
const unsigned char *zTerm = (const unsigned char*)&z[n];
while( zCsr<zTerm ){
int iCode;
int bToken;
READ_UTF8(zCsr, zTerm, iCode);
if( iCode<128 ){
p->aTokenChar[iCode] = (unsigned char)bTokenChars;
}else{
bToken = p->aCategory[sqlite3Fts5UnicodeCategory(iCode)];
assert( (bToken==0 || bToken==1) );
assert( (bTokenChars==0 || bTokenChars==1) );
if( bToken!=bTokenChars && sqlite3Fts5UnicodeIsdiacritic(iCode)==0 ){
int i;
for(i=0; i<nNew; i++){
if( aNew[i]>iCode ) break;
}
memmove(&aNew[i+1], &aNew[i], (nNew-i)*sizeof(int));
aNew[i] = iCode;
nNew++;
}
}
}
p->aiException = aNew;
p->nException = nNew;
}else{
rc = SQLITE_NOMEM;
}
}
return rc;
}
/*
** Return true if the p->aiException[] array contains the value iCode.
*/
static int fts5UnicodeIsException(Unicode61Tokenizer *p, int iCode){
if( p->nException>0 ){
int *a = p->aiException;
int iLo = 0;
int iHi = p->nException-1;
while( iHi>=iLo ){
int iTest = (iHi + iLo) / 2;
if( iCode==a[iTest] ){
return 1;
}else if( iCode>a[iTest] ){
iLo = iTest+1;
}else{
iHi = iTest-1;
}
}
}
return 0;
}
/*
** Delete a "unicode61" tokenizer.
*/
static void fts5UnicodeDelete(Fts5Tokenizer *pTok){
if( pTok ){
Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTok;
sqlite3_free(p->aiException);
sqlite3_free(p->aFold);
sqlite3_free(p);
}
return;
}
static int unicodeSetCategories(Unicode61Tokenizer *p, const char *zCat){
const char *z = zCat;
while( *z ){
while( *z==' ' || *z=='\t' ) z++;
if( *z && sqlite3Fts5UnicodeCatParse(z, p->aCategory) ){
return SQLITE_ERROR;
}
while( *z!=' ' && *z!='\t' && *z!='\0' ) z++;
}
sqlite3Fts5UnicodeAscii(p->aCategory, p->aTokenChar);
return SQLITE_OK;
}
/*
** Create a "unicode61" tokenizer.
*/
static int fts5UnicodeCreate(
void *pUnused,
const char **azArg, int nArg,
Fts5Tokenizer **ppOut
){
int rc = SQLITE_OK; /* Return code */
Unicode61Tokenizer *p = 0; /* New tokenizer object */
UNUSED_PARAM(pUnused);
if( nArg%2 ){
rc = SQLITE_ERROR;
}else{
p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer));
if( p ){
const char *zCat = "L* N* Co";
int i;
memset(p, 0, sizeof(Unicode61Tokenizer));
p->bRemoveDiacritic = 1;
p->nFold = 64;
p->aFold = sqlite3_malloc(p->nFold * sizeof(char));
if( p->aFold==0 ){
rc = SQLITE_NOMEM;
}
/* Search for a "categories" argument */
for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
if( 0==sqlite3_stricmp(azArg[i], "categories") ){
zCat = azArg[i+1];
}
}
if( rc==SQLITE_OK ){
rc = unicodeSetCategories(p, zCat);
}
for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
const char *zArg = azArg[i+1];
if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
rc = SQLITE_ERROR;
}
p->bRemoveDiacritic = (zArg[0]=='1');
}else
if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){
rc = fts5UnicodeAddExceptions(p, zArg, 1);
}else
if( 0==sqlite3_stricmp(azArg[i], "separators") ){
rc = fts5UnicodeAddExceptions(p, zArg, 0);
}else
if( 0==sqlite3_stricmp(azArg[i], "categories") ){
/* no-op */
}else{
rc = SQLITE_ERROR;
}
}
}else{
rc = SQLITE_NOMEM;
}
if( rc!=SQLITE_OK ){
fts5UnicodeDelete((Fts5Tokenizer*)p);
p = 0;
}
*ppOut = (Fts5Tokenizer*)p;
}
return rc;
}
/*
** Return true if, for the purposes of tokenizing with the tokenizer
** passed as the first argument, codepoint iCode is considered a token
** character (not a separator).
*/
static int fts5UnicodeIsAlnum(Unicode61Tokenizer *p, int iCode){
return (
p->aCategory[sqlite3Fts5UnicodeCategory(iCode)]
^ fts5UnicodeIsException(p, iCode)
);
}
static int fts5UnicodeTokenize(
Fts5Tokenizer *pTokenizer,
void *pCtx,
int iUnused,
const char *pText, int nText,
int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd)
){
Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTokenizer;
int rc = SQLITE_OK;
unsigned char *a = p->aTokenChar;
unsigned char *zTerm = (unsigned char*)&pText[nText];
unsigned char *zCsr = (unsigned char *)pText;
ascii_tokenchar:
if( *zCsr>='A' && *zCsr<='Z' ){
*zOut++ = *zCsr + 32;
}else{
*zOut++ = *zCsr;
}
zCsr++;
}
ie = zCsr - (unsigned char*)pText;
}
/* Invoke the token callback */
rc = xToken(pCtx, 0, aFold, zOut-aFold, is, ie);
}
tokenize_done:
if( rc==SQLITE_DONE ) rc = SQLITE_OK;
return rc;
}
/**************************************************************************
** Start of porter stemmer implementation.
*/
/* Any tokens larger than this (in bytes) are passed through without
** stemming. */
#define FTS5_PORTER_MAX_TOKEN 64
typedef struct PorterTokenizer PorterTokenizer;
struct PorterTokenizer {
fts5_tokenizer tokenizer; /* Parent tokenizer module */
Fts5Tokenizer *pTokenizer; /* Parent tokenizer instance */
char aBuf[FTS5_PORTER_MAX_TOKEN + 64];
};
/*
** Delete a "porter" tokenizer.
*/
static void fts5PorterDelete(Fts5Tokenizer *pTok){
if( pTok ){
PorterTokenizer *p = (PorterTokenizer*)pTok;
if( p->pTokenizer ){
p->tokenizer.xDelete(p->pTokenizer);
}
sqlite3_free(p);
}
}
/*
** Create a "porter" tokenizer.
*/
static int fts5PorterCreate(
void *pCtx,
const char **azArg, int nArg,
Fts5Tokenizer **ppOut
){
fts5_api *pApi = (fts5_api*)pCtx;
int rc = SQLITE_OK;
PorterTokenizer *pRet;
void *pUserdata = 0;
const char *zBase = "unicode61";
if( nArg>0 ){
zBase = azArg[0];
}
pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer));
if( pRet ){
memset(pRet, 0, sizeof(PorterTokenizer));
rc = pApi->xFindTokenizer(pApi, zBase, &pUserdata, &pRet->tokenizer);
}else{
rc = SQLITE_NOMEM;
}
if( rc==SQLITE_OK ){
int nArg2 = (nArg>0 ? nArg-1 : 0);
const char **azArg2 = (nArg2 ? &azArg[1] : 0);
rc = pRet->tokenizer.xCreate(pUserdata, azArg2, nArg2, &pRet->pTokenizer);
}
if( rc!=SQLITE_OK ){
fts5PorterDelete((Fts5Tokenizer*)pRet);
pRet = 0;
}
*ppOut = (Fts5Tokenizer*)pRet;
return rc;
}
typedef struct PorterContext PorterContext;
struct PorterContext {
void *pCtx;
int (*xToken)(void*, int, const char*, int, int, int);
char *aBuf;
};
typedef struct PorterRule PorterRule;
struct PorterRule {
const char *zSuffix;
int nSuffix;
int (*xCond)(char *zStem, int nStem);
const char *zOutput;
int nOutput;
};
#if 0
static int fts5PorterApply(char *aBuf, int *pnBuf, PorterRule *aRule){
int ret = -1;
int nBuf = *pnBuf;
PorterRule *p;
for(p=aRule; p->zSuffix; p++){
assert( strlen(p->zSuffix)==p->nSuffix );
assert( strlen(p->zOutput)==p->nOutput );
if( nBuf<p->nSuffix ) continue;
if( 0==memcmp(&aBuf[nBuf - p->nSuffix], p->zSuffix, p->nSuffix) ) break;
}
if( p->zSuffix ){
int nStem = nBuf - p->nSuffix;
if( p->xCond==0 || p->xCond(aBuf, nStem) ){
memcpy(&aBuf[nStem], p->zOutput, p->nOutput);
*pnBuf = nStem + p->nOutput;
if( aBuf[nBuf-1]=='y' && fts5Porter_Vowel(aBuf, nBuf-1) ){
aBuf[nBuf-1] = 'i';
}
/* Steps 2 through 4. */
fts5PorterStep2(aBuf, &nBuf);
fts5PorterStep3(aBuf, &nBuf);
fts5PorterStep4(aBuf, &nBuf);
/* Step 5a. */
assert( nBuf>0 );
if( aBuf[nBuf-1]=='e' ){
if( fts5Porter_MGt1(aBuf, nBuf-1)
|| (fts5Porter_MEq1(aBuf, nBuf-1) && !fts5Porter_Ostar(aBuf, nBuf-1))
){
nBuf--;
}
}
/* Step 5b. */
if( nBuf>1 && aBuf[nBuf-1]=='l'
&& aBuf[nBuf-2]=='l' && fts5Porter_MGt1(aBuf, nBuf-1)
){
nBuf--;
}
return p->xToken(p->pCtx, tflags, aBuf, nBuf, iStart, iEnd);
pass_through:
return p->xToken(p->pCtx, tflags, pToken, nToken, iStart, iEnd);
}
/*
** Tokenize using the porter tokenizer.
*/
static int fts5PorterTokenize(
Fts5Tokenizer *pTokenizer,
void *pCtx,
int flags,
const char *pText, int nText,
int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd)
){
PorterTokenizer *p = (PorterTokenizer*)pTokenizer;
PorterContext sCtx;
sCtx.xToken = xToken;
sCtx.pCtx = pCtx;
sCtx.aBuf = p->aBuf;
return p->tokenizer.xTokenize(
p->pTokenizer, (void*)&sCtx, flags, pText, nText, fts5PorterCb
);
}
/*
** Register all built-in tokenizers with FTS5.
*/
static int sqlite3Fts5TokenizerInit(fts5_api *pApi){
struct BuiltinTokenizer {
const char *zName;
fts5_tokenizer x;
} aBuiltin[] = {
{ "unicode61", {fts5UnicodeCreate, fts5UnicodeDelete, fts5UnicodeTokenize}},
{ "ascii", {fts5AsciiCreate, fts5AsciiDelete, fts5AsciiTokenize }},
{ "porter", {fts5PorterCreate, fts5PorterDelete, fts5PorterTokenize }},
};
int rc = SQLITE_OK; /* Return code */
int i; /* To iterate through builtin functions */
for(i=0; rc==SQLITE_OK && i<ArraySize(aBuiltin); i++){
rc = pApi->xCreateTokenizer(pApi,
aBuiltin[i].zName,
(void*)pApi,
&aBuiltin[i].x,
0
);
}
return rc;
}
/*
** 2012 May 25
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
*/
/*
** DO NOT EDIT THIS MACHINE GENERATED FILE.
*/
/* #include <assert.h> */
/*
** If the argument is a codepoint corresponding to a lowercase letter
** in the ASCII range with a diacritic added, return the codepoint
** of the ASCII letter only. For example, if passed 235 - "LATIN
** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER
** E"). The resuls of passing a codepoint that corresponds to an
** uppercase letter are undefined.
*/
static int fts5_remove_diacritic(int c){
unsigned short aDia[] = {
0, 1797, 1848, 1859, 1891, 1928, 1940, 1995,
2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286,
2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732,
2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336,
3456, 3696, 3712, 3728, 3744, 3896, 3912, 3928,
3968, 4008, 4040, 4106, 4138, 4170, 4202, 4234,
4266, 4296, 4312, 4344, 4408, 4424, 4472, 4504,
6148, 6198, 6264, 6280, 6360, 6429, 6505, 6529,
61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726,
61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122,
62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536,
62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730,
62924, 63050, 63082, 63274, 63390,
};
char aChar[] = {
'\0', 'a', 'c', 'e', 'i', 'n', 'o', 'u', 'y', 'y', 'a', 'c',
'd', 'e', 'e', 'g', 'h', 'i', 'j', 'k', 'l', 'n', 'o', 'r',
's', 't', 'u', 'u', 'w', 'y', 'z', 'o', 'u', 'a', 'i', 'o',
'u', 'g', 'k', 'o', 'j', 'g', 'n', 'a', 'e', 'i', 'o', 'r',
'u', 's', 't', 'h', 'a', 'e', 'o', 'y', '\0', '\0', '\0', '\0',
'\0', '\0', '\0', '\0', 'a', 'b', 'd', 'd', 'e', 'f', 'g', 'h',
'h', 'i', 'k', 'l', 'l', 'm', 'n', 'p', 'r', 'r', 's', 't',
'u', 'v', 'w', 'w', 'x', 'y', 'z', 'h', 't', 'w', 'y', 'a',
'e', 'i', 'o', 'u', 'y',
};
unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
int iRes = 0;
int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1;
int iLo = 0;
while( iHi>=iLo ){
int iTest = (iHi + iLo) / 2;
if( key >= aDia[iTest] ){
iRes = iTest;
iLo = iTest+1;
}else{
iHi = iTest-1;
}
}
assert( key>=aDia[iRes] );
return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
}
/*
** Return true if the argument interpreted as a unicode codepoint
** is a diacritical modifier character.
*/
static int sqlite3Fts5UnicodeIsdiacritic(int c){
unsigned int mask0 = 0x08029FDF;
unsigned int mask1 = 0x000361F8;
if( c<768 || c>817 ) return 0;
return (c < 768+32) ?
(mask0 & (1 << (c-768))) :
(mask1 & (1 << (c-768-32)));
}
/*
** Interpret the argument as a unicode codepoint. If the codepoint
** is an upper case character that has a lower case equivalent,
** return the codepoint corresponding to the lower case version.
** Otherwise, return a copy of the argument.
**
** The results are undefined if the value passed to this function
** is less than zero.
*/
static int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){
/* Each entry in the following array defines a rule for folding a range
** of codepoints to lower case. The rule applies to a range of nRange
** codepoints starting at codepoint iCode.
**
** If the least significant bit in flags is clear, then the rule applies
** to all nRange codepoints (i.e. all nRange codepoints are upper case and
** need to be folded). Or, if it is set, then the rule only applies to
** every second codepoint in the range, starting with codepoint C.
**
** The 7 most significant bits in flags are an index into the aiOff[]
** array. If a specific codepoint C does require folding, then its lower
** case equivalent is ((C + aiOff[flags>>1]) & 0xFFFF).
**
** The contents of this array are generated by parsing the CaseFolding.txt
** file distributed as part of the "Unicode Character Database". See
** http://www.unicode.org for details.
*/
static const struct TableEntry {
unsigned short iCode;
unsigned char flags;
unsigned char nRange;
} aEntry[] = {
{65, 14, 26}, {181, 64, 1}, {192, 14, 23},
{216, 14, 7}, {256, 1, 48}, {306, 1, 6},
{313, 1, 16}, {330, 1, 46}, {376, 116, 1},
{377, 1, 6}, {383, 104, 1}, {385, 50, 1},
{386, 1, 4}, {390, 44, 1}, {391, 0, 1},
{393, 42, 2}, {395, 0, 1}, {398, 32, 1},
{399, 38, 1}, {400, 40, 1}, {401, 0, 1},
{403, 42, 1}, {404, 46, 1}, {406, 52, 1},
{407, 48, 1}, {408, 0, 1}, {412, 52, 1},
{413, 54, 1}, {415, 56, 1}, {416, 1, 6},
{422, 60, 1}, {423, 0, 1}, {425, 60, 1},
{428, 0, 1}, {430, 60, 1}, {431, 0, 1},
{433, 58, 2}, {435, 1, 4}, {439, 62, 1},
{440, 0, 1}, {444, 0, 1}, {452, 2, 1},
{453, 0, 1}, {455, 2, 1}, {456, 0, 1},
{458, 2, 1}, {459, 1, 18}, {478, 1, 18},
{497, 2, 1}, {498, 1, 4}, {502, 122, 1},
{503, 134, 1}, {504, 1, 40}, {544, 110, 1},
{546, 1, 18}, {570, 70, 1}, {571, 0, 1},
{573, 108, 1}, {574, 68, 1}, {577, 0, 1},
{579, 106, 1}, {580, 28, 1}, {581, 30, 1},
{582, 1, 10}, {837, 36, 1}, {880, 1, 4},
{886, 0, 1}, {902, 18, 1}, {904, 16, 3},
{908, 26, 1}, {910, 24, 2}, {913, 14, 17},
{931, 14, 9}, {962, 0, 1}, {975, 4, 1},
{976, 140, 1}, {977, 142, 1}, {981, 146, 1},
{982, 144, 1}, {984, 1, 24}, {1008, 136, 1},
{1009, 138, 1}, {1012, 130, 1}, {1013, 128, 1},
{1015, 0, 1}, {1017, 152, 1}, {1018, 0, 1},
{1021, 110, 3}, {1024, 34, 16}, {1040, 14, 32},
{1120, 1, 34}, {1162, 1, 54}, {1216, 6, 1},
{1217, 1, 14}, {1232, 1, 88}, {1329, 22, 38},
{4256, 66, 38}, {4295, 66, 1}, {4301, 66, 1},
{7680, 1, 150}, {7835, 132, 1}, {7838, 96, 1},
{7840, 1, 96}, {7944, 150, 8}, {7960, 150, 6},
{7976, 150, 8}, {7992, 150, 8}, {8008, 150, 6},
{8025, 151, 8}, {8040, 150, 8}, {8072, 150, 8},
{8088, 150, 8}, {8104, 150, 8}, {8120, 150, 2},
{8122, 126, 2}, {8124, 148, 1}, {8126, 100, 1},
{8136, 124, 4}, {8140, 148, 1}, {8152, 150, 2},
{8154, 120, 2}, {8168, 150, 2}, {8170, 118, 2},
{8172, 152, 1}, {8184, 112, 2}, {8186, 114, 2},
{8188, 148, 1}, {8486, 98, 1}, {8490, 92, 1},
{8491, 94, 1}, {8498, 12, 1}, {8544, 8, 16},
{8579, 0, 1}, {9398, 10, 26}, {11264, 22, 47},
{11360, 0, 1}, {11362, 88, 1}, {11363, 102, 1},
{11364, 90, 1}, {11367, 1, 6}, {11373, 84, 1},
{11374, 86, 1}, {11375, 80, 1}, {11376, 82, 1},
{11378, 0, 1}, {11381, 0, 1}, {11390, 78, 2},
{11392, 1, 100}, {11499, 1, 4}, {11506, 0, 1},
{42560, 1, 46}, {42624, 1, 24}, {42786, 1, 14},
{42802, 1, 62}, {42873, 1, 4}, {42877, 76, 1},
{42878, 1, 10}, {42891, 0, 1}, {42893, 74, 1},
{42896, 1, 4}, {42912, 1, 10}, {42922, 72, 1},
( run in 2.140 seconds using v1.01-cache-2.11-cpan-39bf76dae61 )