DBD-SQLeet
view release on metacpan or search on metacpan
t/01_compile.t
t/02_logon.t
t/03_create_table.t
t/04_insert.t
t/05_select.t
t/06_tran.t
t/07_error.t
t/08_busy.t
t/09_create_function.t
t/10_create_aggregate.t
t/12_unicode.t
t/13_create_collation.t
t/14_progress_handler.t
t/15_ak_dbd.t
t/16_column_info.t
t/17_createdrop.t
t/18_insertfetch.t
t/19_bindparam.t
t/20_blobs.t
t/21_blobtext.t
t/22_listfields.t
t/61_strlike.t
t/62_regexp_multibyte_char_class.t
t/63_param_values.t
t/lib/SQLeetTest.pm
t/rt_106151_outermost_savepoint.t
t/rt_106950_extra_warnings_with_savepoints.t
t/rt_115465_column_info_with_spaces.t
t/rt_124227_index_regression.t
t/rt_15186_prepcached.t
t/rt_21406_auto_finish.t
t/rt_25371_asymmetric_unicode.t
t/rt_25460_numeric_aggregate.t
t/rt_25924_user_defined_func_unicode.t
t/rt_26775_distinct.t
t/rt_27553_prepared_cache_and_analyze.t
t/rt_29058_group_by.t
t/rt_29629_sqlite_where_length.t
t/rt_31324_full_names.t
t/rt_32889_prepare_cached_reexecute.t
t/rt_36836_duplicate_key.t
t/rt_36838_unique_and_bus_error.t
t/rt_40594_nullable.t
t/rt_48393_debug_panic_with_commit.t
t/rt_50503_fts3.t
t/rt_52573_manual_exclusive_lock.t
t/rt_53235_icu_compatibility.t
t/rt_62370_diconnected_handles_operation.t
t/rt_64177_ping_wipes_out_the_errstr.t
t/rt_67581_bind_params_mismatch.t
t/rt_71311_bind_col_and_unicode.t
t/rt_73159_fts_tokenizer_segfault.t
t/rt_73787_exponential_buffer_overflow.t
t/rt_76395_int_overflow.t
t/rt_77724_primary_key_with_a_whitespace.t
t/rt_78833_utf8_flag_for_column_names.t
t/rt_81536_multi_column_primary_key_info.t
t/rt_88228_sqlite_3_8_0_crash.t
t/rt_96050_db_filename_for_a_closed_database.t
t/rt_96877_unicode_statements.t
t/rt_96878_fts_contentless_table.t
t/rt_97598_crash_on_disconnect_with_virtual_tables.t
typemap
xt/cpp_comments.t
xt/meta.t
xt/pmv.t
xt/pod.t
META.yml Module YAML meta-data (added by MakeMaker)
META.json Module JSON meta-data (added by MakeMaker)
return SQLITE_BLOB;
default:
return SQLITE_TEXT;
}
}
void
init_cxt() {
dTHX;
MY_CXT_INIT;
MY_CXT.last_dbh_is_unicode = 0;
}
SV *
stacked_sv_from_sqlite3_value(pTHX_ sqlite3_value *value, int is_unicode)
{
STRLEN len;
sqlite_int64 iv;
int type = sqlite3_value_type(value);
SV *sv;
switch(type) {
case SQLITE_INTEGER:
iv = sqlite3_value_int64(value);
if ( iv >= IV_MIN && iv <= IV_MAX ) {
else {
/* warn("integer overflow, cast to NV"); */
return sv_2mortal(newSVnv((NV)iv));
}
case SQLITE_FLOAT:
return sv_2mortal(newSVnv(sqlite3_value_double(value)));
break;
case SQLITE_TEXT:
len = sqlite3_value_bytes(value);
sv = newSVpvn((const char *)sqlite3_value_text(value), len);
if (is_unicode) {
SvUTF8_on(sv);
}
return sv_2mortal(sv);
case SQLITE_BLOB:
len = sqlite3_value_bytes(value);
return sv_2mortal(newSVpvn(sqlite3_value_blob(value), len));
default:
return &PL_sv_undef;
}
}
int
sqlite_db_login6(SV *dbh, imp_dbh_t *imp_dbh, char *dbname, char *user, char *pass, SV *attr)
{
dTHX;
int rc;
HV *hv;
SV **val;
int extended = 0;
int flag = 0;
int unicode = 0;
sqlite_trace(dbh, imp_dbh, 3, form("login '%s' (version %s)", dbname, sqlite3_version));
if (SvROK(attr)) {
hv = (HV*)SvRV(attr);
if (hv_exists(hv, "sqlite_extended_result_codes", 28)) {
val = hv_fetch(hv, "sqlite_extended_result_codes", 28, 0);
extended = (val && SvOK(*val)) ? !(!SvTRUE(*val)) : 0;
}
if (hv_exists(hv, "ReadOnly", 8)) {
flag |= SQLITE_OPEN_READONLY;
}
}
if (hv_exists(hv, "sqlite_open_flags", 17)) {
val = hv_fetch(hv, "sqlite_open_flags", 17, 0);
flag |= (val && SvOK(*val)) ? SvIV(*val) : 0;
if (flag & SQLITE_OPEN_READONLY) {
hv_stores(hv, "ReadOnly", newSViv(1));
}
}
/* sqlite_unicode should be detected earlier, to register default functions correctly */
if (hv_exists(hv, "sqlite_unicode", 14)) {
val = hv_fetch(hv, "sqlite_unicode", 14, 0);
unicode = (val && SvOK(*val)) ? SvIV(*val) : 0;
} else if (hv_exists(hv, "unicode", 7)) {
val = hv_fetch(hv, "unicode", 7, 0);
unicode = (val && SvOK(*val)) ? SvIV(*val) : 0;
}
}
rc = sqlite_open2(dbname, &(imp_dbh->db), flag, extended);
if ( rc != SQLITE_OK ) {
return FALSE; /* -> undef in lib/DBD/SQLeet.pm */
}
DBIc_IMPSET_on(imp_dbh);
imp_dbh->unicode = unicode;
imp_dbh->functions = newAV();
imp_dbh->aggregates = newAV();
imp_dbh->collation_needed_callback = newSVsv( &PL_sv_undef );
imp_dbh->timeout = SQL_TIMEOUT;
imp_dbh->handle_binary_nulls = FALSE;
imp_dbh->allow_multiple_statements = FALSE;
imp_dbh->use_immediate_transaction = TRUE;
imp_dbh->see_if_its_a_number = FALSE;
imp_dbh->extended_result_codes = extended;
imp_dbh->stmt_list = NULL;
int rc = 0;
int i;
char *statement;
if (!DBIc_ACTIVE(imp_dbh)) {
sqlite_error(dbh, -2, "attempt to do on inactive database handle");
return -2; /* -> undef in SQLeet.xsi */
}
/* sqlite3_prepare wants an utf8-encoded SQL statement */
if (imp_dbh->unicode) {
sv_utf8_upgrade(sv_statement);
}
statement = SvPV_nolen(sv_statement);
sqlite_trace(dbh, imp_dbh, 3, form("do statement: %s", statement));
croak_if_db_is_null();
if (sqlite3_get_autocommit(imp_dbh->db)) {
}
if (strEQ(key, "sqlite_see_if_its_a_number")) {
imp_dbh->see_if_its_a_number = !(! SvTRUE(valuesv));
return TRUE;
}
if (strEQ(key, "sqlite_extended_result_codes")) {
imp_dbh->extended_result_codes = !(! SvTRUE(valuesv));
sqlite3_extended_result_codes(imp_dbh->db, imp_dbh->extended_result_codes);
return TRUE;
}
if (strEQ(key, "sqlite_unicode")) {
#if PERL_UNICODE_DOES_NOT_WORK_WELL
sqlite_trace(dbh, imp_dbh, 3, form("Unicode support is disabled for this version of perl."));
imp_dbh->unicode = 0;
#else
imp_dbh->unicode = !(! SvTRUE(valuesv));
#endif
return TRUE;
}
if (strEQ(key, "unicode")) {
if (DBIc_has(imp_dbh, DBIcf_WARN))
warn("\"unicode\" attribute will be deprecated. Use \"sqlite_unicode\" instead.");
#if PERL_UNICODE_DOES_NOT_WORK_WELL
sqlite_trace(dbh, imp_dbh, 3, form("Unicode support is disabled for this version of perl."));
imp_dbh->unicode = 0;
#else
imp_dbh->unicode = !(! SvTRUE(valuesv));
#endif
return TRUE;
}
return FALSE;
}
SV *
sqlite_db_FETCH_attrib(SV *dbh, imp_dbh_t *imp_dbh, SV *keysv)
{
dTHX;
}
if (strEQ(key, "sqlite_use_immediate_transaction")) {
return sv_2mortal(newSViv(imp_dbh->use_immediate_transaction ? 1 : 0));
}
if (strEQ(key, "sqlite_see_if_its_a_number")) {
return sv_2mortal(newSViv(imp_dbh->see_if_its_a_number ? 1 : 0));
}
if (strEQ(key, "sqlite_extended_result_codes")) {
return sv_2mortal(newSViv(imp_dbh->extended_result_codes ? 1 : 0));
}
if (strEQ(key, "sqlite_unicode")) {
#if PERL_UNICODE_DOES_NOT_WORK_WELL
sqlite_trace(dbh, imp_dbh, 3, "Unicode support is disabled for this version of perl.");
return sv_2mortal(newSViv(0));
#else
return sv_2mortal(newSViv(imp_dbh->unicode ? 1 : 0));
#endif
}
if (strEQ(key, "unicode")) {
if (DBIc_has(imp_dbh, DBIcf_WARN))
warn("\"unicode\" attribute will be deprecated. Use \"sqlite_unicode\" instead.");
#if PERL_UNICODE_DOES_NOT_WORK_WELL
sqlite_trace(dbh, imp_dbh, 3, "Unicode support is disabled for this version of perl.");
return sv_2mortal(newSViv(0));
#else
return sv_2mortal(newSViv(imp_dbh->unicode ? 1 : 0));
#endif
}
return NULL;
}
SV *
sqlite_db_last_insert_id(SV *dbh, imp_dbh_t *imp_dbh, SV *catalog, SV *schema, SV *table, SV *field, SV *attr)
{
dTHX;
sqlite_st_prepare_sv(SV *sth, imp_sth_t *imp_sth, SV *sv_statement, SV *attribs)
{
dTHX;
dMY_CXT;
int rc = 0;
const char *extra;
char *statement;
stmt_list_s * new_stmt;
D_imp_dbh_from_sth;
MY_CXT.last_dbh_is_unicode = imp_dbh->unicode;
if (!DBIc_ACTIVE(imp_dbh)) {
sqlite_error(sth, -2, "attempt to prepare on inactive database handle");
return FALSE; /* -> undef in lib/DBD/SQLeet.pm */
}
/* sqlite3_prepare wants an utf8-encoded SQL statement */
if (imp_dbh->unicode) {
sv_utf8_upgrade(sv_statement);
}
statement = SvPV_nolen(sv_statement);
#if 0
if (*statement == '\0') {
sqlite_error(sth, -2, "attempt to prepare empty statement");
return FALSE; /* -> undef in lib/DBD/SQLeet.pm */
}
else if (sql_type == SQLITE_BLOB) {
STRLEN len;
char * data = SvPVbyte(value, len);
rc = sqlite3_bind_blob(imp_sth->stmt, i+1, data, len, SQLITE_TRANSIENT);
}
else {
STRLEN len;
const char *data;
int numtype = 0;
if (imp_dbh->unicode) {
sv_utf8_upgrade(value);
}
data = SvPV(value, len);
/*
* XXX: For backward compatibility, it'd be better to
* accept a value like " 4" as an integer for an integer
* type column (see t/19_bindparam.t), at least when
* we explicitly specify its type. However, we should
* keep spaces when we just guess.
sqlite_trace(sth, imp_sth, 5, form("fetch column %d as text", i));
val = (char*)sqlite3_column_text(imp_sth->stmt, i);
len = sqlite3_column_bytes(imp_sth->stmt, i);
if (chopBlanks) {
while((len > 0) && (val[len-1] == ' ')) {
len--;
}
}
sv_setpvn(AvARRAY(av)[i], val, len);
if (imp_dbh->unicode) {
SvUTF8_on(AvARRAY(av)[i]);
} else {
SvUTF8_off(AvARRAY(av)[i]);
}
break;
case SQLITE_BLOB:
sqlite_trace(sth, imp_sth, 5, form("fetch column %d as blob", i));
len = sqlite3_column_bytes(imp_sth->stmt, i);
val = (char*)sqlite3_column_blob(imp_sth->stmt, i);
sv_setpvn(AvARRAY(av)[i], len ? val : "", len);
retsv = sv_2mortal(newRV_noinc((SV*)av));
for (n = 0; n < i; n++) {
/* warn("Fetch col name %d\n", n); */
const char *fieldname = sqlite3_column_name(imp_sth->stmt, n);
if (fieldname) {
/* warn("Name [%d]: %s\n", n, fieldname); */
/* char *dot = instr(fieldname, "."); */
/* if (dot) drop table name from field name */
/* fieldname = ++dot; */
SV *sv_fieldname = newSVpv(fieldname, 0);
if (imp_dbh->unicode)
SvUTF8_on(sv_fieldname);
av_store(av, n, sv_fieldname);
}
}
}
else if (strEQ(key, "PRECISION")) {
AV *av = newAV();
retsv = sv_2mortal(newRV_noinc((SV*)av));
}
else if (strEQ(key, "TYPE")) {
if (!DBIc_ACTIVE(imp_dbh)) {
sqlite_error(dbh, -2, "attempt to set busy timeout on inactive database handle");
return -2;
}
sqlite3_busy_timeout(imp_dbh->db, imp_dbh->timeout);
}
return imp_dbh->timeout;
}
static void
sqlite_db_func_dispatcher(int is_unicode, sqlite3_context *context, int argc, sqlite3_value **value)
{
dTHX;
dSP;
int count;
int i;
SV *func;
func = sqlite3_user_data(context);
ENTER;
SAVETMPS;
PUSHMARK(SP);
for ( i=0; i < argc; i++ ) {
XPUSHs(stacked_sv_from_sqlite3_value(aTHX_ value[i], is_unicode));
}
PUTBACK;
count = call_sv(func, G_SCALAR|G_EVAL);
SPAGAIN;
/* Check for an error */
if (SvTRUE(ERRSV) ) {
sqlite_set_result(aTHX_ context, ERRSV, 1);
} else {
sqlite_set_result(aTHX_ context, POPs, 0 );
}
PUTBACK;
FREETMPS;
LEAVE;
}
static void
sqlite_db_func_dispatcher_unicode(sqlite3_context *context, int argc, sqlite3_value **value)
{
sqlite_db_func_dispatcher(1, context, argc, value);
}
static void
sqlite_db_func_dispatcher_no_unicode(sqlite3_context *context, int argc, sqlite3_value **value)
{
sqlite_db_func_dispatcher(0, context, argc, value);
}
int
sqlite_db_create_function(pTHX_ SV *dbh, const char *name, int argc, SV *func, int flags)
{
D_imp_dbh(dbh);
int rc;
SV *func_sv;
/* Copy the function reference */
func_sv = newSVsv(func);
av_push( imp_dbh->functions, func_sv );
croak_if_db_is_null();
/* warn("create_function %s with %d args\n", name, argc); */
rc = sqlite3_create_function( imp_dbh->db, name, argc, SQLITE_UTF8|flags,
func_sv,
imp_dbh->unicode ? sqlite_db_func_dispatcher_unicode
: sqlite_db_func_dispatcher_no_unicode,
NULL, NULL );
if ( rc != SQLITE_OK ) {
sqlite_error(dbh, rc, form("sqlite_create_function failed with error %s", sqlite3_errmsg(imp_dbh->db)));
return FALSE;
}
return TRUE;
}
#ifndef SQLITE_OMIT_LOAD_EXTENSION
return;
}
static void
sqlite_db_aggr_step_dispatcher(sqlite3_context *context,
int argc, sqlite3_value **value)
{
dTHX;
dSP;
int i, is_unicode = 0; /* TODO : find out from db handle */
aggrInfo *aggr;
aggr = sqlite3_aggregate_context(context, sizeof (aggrInfo));
if ( !aggr )
return;
ENTER;
SAVETMPS;
/* initialize on first step */
sqlite_db_aggr_new_dispatcher(aTHX_ context, aggr);
}
if ( aggr->err || !aggr->aggr_inst )
goto cleanup;
PUSHMARK(SP);
XPUSHs( sv_2mortal( newSVsv( aggr->aggr_inst ) ));
for ( i=0; i < argc; i++ ) {
XPUSHs(stacked_sv_from_sqlite3_value(aTHX_ value[i], is_unicode));
}
PUTBACK;
call_method ("step", G_SCALAR|G_EVAL|G_DISCARD);
/* Check for an error */
if (SvTRUE(ERRSV) ) {
aggr->err = newSVpvf("error during aggregator's step(): %s",
SvPV_nolen(ERRSV));
POPs;
sqlite_trace(dbh, imp_dbh, 3, form("improper collation function: '%s' is not symmetric", name));
}
/* Copy the func reference so that it can be deallocated at disconnect */
av_push( imp_dbh->functions, func_sv );
/* Register the func within sqlite3 */
rv = sqlite3_create_collation(
imp_dbh->db, name, SQLITE_UTF8,
func_sv,
imp_dbh->unicode ? sqlite_db_collation_dispatcher_utf8
: sqlite_db_collation_dispatcher
);
if ( rv != SQLITE_OK ) {
sqlite_error(dbh, rv, form("sqlite_create_collation failed with error %s", sqlite3_errmsg(imp_dbh->db)));
return FALSE;
}
return TRUE;
}
#ifndef _DBDIMP_H
#define _DBDIMP_H 1
#include "SQLeetXS.h"
#include "sqlite3.h"
#define MY_CXT_KEY "DBD::SQLeet::_guts" XS_VERSION
typedef struct {
int last_dbh_is_unicode;
} my_cxt_t;
#define PERL_UNICODE_DOES_NOT_WORK_WELL \
(PERL_REVISION <= 5) && ((PERL_VERSION < 8) \
|| (PERL_VERSION == 8 && PERL_SUBVERSION < 5))
/* 30 second timeout by default */
#define SQL_TIMEOUT 30000
#ifndef sqlite3_int64
struct imp_drh_st {
dbih_drc_t com;
/* sqlite specific bits */
};
/* Database Handle */
struct imp_dbh_st {
dbih_dbc_t com;
/* sqlite specific bits */
sqlite3 *db;
bool unicode;
bool handle_binary_nulls;
int timeout;
AV *functions;
AV *aggregates;
SV *collation_needed_callback;
bool allow_multiple_statements;
bool use_immediate_transaction;
bool see_if_its_a_number;
int extended_result_codes;
stmt_list_s * stmt_list;
dbdimp_tokenizer.inc view on Meta::CPAN
/* allocate and initialize the cursor struct */
perl_tokenizer_cursor *c;
c = (perl_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
memset(c, 0, sizeof(*c));
*ppCursor = &c->base;
/* flags for creating the Perl SV containing the input string */
flags = SVs_TEMP; /* will call sv_2mortal */
/* special handling if working with utf8 strings */
if (MY_CXT.last_dbh_is_unicode) {
/* data to keep track of byte offsets */
c->lastByteOffset = c->pInput = pInput;
c->lastCharOffset = 0;
/* string passed to Perl needs to be flagged as utf8 */
flags |= SVf_UTF8;
}
ENTER;
dbdimp_virtual_table.inc view on Meta::CPAN
return SQLITE_OK;
}
static int perl_vt_Filter( sqlite3_vtab_cursor *pVtabCursor,
int idxNum, const char *idxStr,
int argc, sqlite3_value **argv ){
dTHX;
dSP;
dMY_CXT;
int i, count;
int is_unicode = MY_CXT.last_dbh_is_unicode;
ENTER;
SAVETMPS;
/* call the FILTER() method with ($idxNum, $idxStr, @args) */
PUSHMARK(SP);
XPUSHs(((perl_vtab_cursor *) pVtabCursor)->perl_cursor_obj);
XPUSHs(sv_2mortal(newSViv(idxNum)));
XPUSHs(sv_2mortal(newSVpv(idxStr, 0)));
for(i = 0; i < argc; i++) {
XPUSHs(stacked_sv_from_sqlite3_value(aTHX_ argv[i], is_unicode));
}
PUTBACK;
count = call_method("FILTER", G_VOID);
SPAGAIN;
SP -= count;
PUTBACK;
FREETMPS;
LEAVE;
dbdimp_virtual_table.inc view on Meta::CPAN
return rc;
}
static int perl_vt_Update( sqlite3_vtab *pVTab,
int argc, sqlite3_value **argv,
sqlite3_int64 *pRowid ){
dTHX;
dSP;
dMY_CXT;
int count, i;
int is_unicode = MY_CXT.last_dbh_is_unicode;
int rc = SQLITE_ERROR;
SV *rowidsv;
ENTER;
SAVETMPS;
/* call the _SQLITE_UPDATE() method */
PUSHMARK(SP);
XPUSHs(((perl_vtab *) pVTab)->perl_vtab_obj);
for(i = 0; i < argc; i++) {
XPUSHs(stacked_sv_from_sqlite3_value(aTHX_ argv[i], is_unicode));
}
PUTBACK;
count = call_method ("_SQLITE_UPDATE", G_SCALAR);
SPAGAIN;
if (count != 1) {
warn("cursor->_SQLITE_UPDATE() returned %d vals instead of 1", count);
SP -= count;
}
else {
if (argc > 1 && sqlite3_value_type(argv[0]) == SQLITE_NULL
dbdimp_virtual_table.inc view on Meta::CPAN
coderef = newSVsv(result);
}
/* store result in cache */
hv_store(functions, func_name, len, coderef ? coderef : &PL_sv_undef, 0);
}
/* return function information for sqlite3 within *pxFunc and *ppArg */
is_overloaded = coderef && SvTRUE(coderef);
if (is_overloaded) {
*pxFunc = MY_CXT.last_dbh_is_unicode ? sqlite_db_func_dispatcher_unicode
: sqlite_db_func_dispatcher_no_unicode;
*ppArg = coderef;
}
cleanup:
PUTBACK;
FREETMPS;
LEAVE;
sqlite3_free(func_name);
return is_overloaded;
}
lib/DBD/SQLeet.pm view on Meta::CPAN
}
}
}
if (my $flags = $attr->{sqlite_open_flags}) {
unless ($flags & (DBD::SQLeet::OPEN_READONLY() | DBD::SQLeet::OPEN_READWRITE())) {
$attr->{sqlite_open_flags} |= DBD::SQLeet::OPEN_READWRITE() | DBD::SQLeet::OPEN_CREATE();
}
}
# To avoid unicode and long file name problems on Windows,
# convert to the shortname if the file (or parent directory) exists.
if ($^O =~ /MSWin32/ and $real ne ':memory:' and $real ne '' and $real !~ /^file:/ and !-f $real) {
require File::Basename;
my ($file, $dir, $suffix) = File::Basename::fileparse($real);
# We are creating a new file.
# Does the directory it's in at least exist?
if (-d $dir) {
require Win32;
$real = join '', grep { defined } Win32::GetShortPathName($dir), $file, $suffix;
} else {
parse_arithexpr||5.013008|
parse_barestmt||5.013007|
parse_block||5.013007|
parse_body|||
parse_fullexpr||5.013008|
parse_fullstmt||5.013005|
parse_label||5.013007|
parse_listexpr||5.013008|
parse_stmtseq||5.013006|
parse_termexpr||5.013008|
parse_unicode_opts|||
parser_dup|||
parser_free|||
path_is_absolute|||n
peep|||
pending_Slabs_to_ro|||
perl_alloc_using|||n
perl_alloc|||n
perl_clone_using|||n
perl_clone|||n
perl_construct|||n
#endif
#ifndef PERL_PV_PRETTY_DUMP
# define PERL_PV_PRETTY_DUMP PERL_PV_PRETTY_ELLIPSES|PERL_PV_PRETTY_QUOTE
#endif
#ifndef PERL_PV_PRETTY_REGPROP
# define PERL_PV_PRETTY_REGPROP PERL_PV_PRETTY_ELLIPSES|PERL_PV_PRETTY_LTGT|PERL_PV_ESCAPE_RE
#endif
/* Hint: pv_escape
* Note that unicode functionality is only backported to
* those perl versions that support it. For older perl
* versions, the implementation will fall back to bytes.
*/
#ifndef pv_escape
#if defined(NEED_pv_escape)
static char * DPPP_(my_pv_escape)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, STRLEN * const escaped, const U32 flags);
static
#else
extern char * DPPP_(my_pv_escape)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, STRLEN * const escaped, const U32 flags);
#define MEMCELLSIZE offsetof(Mem,zMalloc)
/* One or more of the following flags are set to indicate the validOK
** representations of the value stored in the Mem struct.
**
** If the MEM_Null flag is set, then the value is an SQL NULL value.
** For a pointer type created using sqlite3_bind_pointer() or
** sqlite3_result_pointer() the MEM_Term and MEM_Subtype flags are also set.
**
** If the MEM_Str flag is set then Mem.z points at a string representation.
** Usually this is encoded in the same unicode encoding as the main
** database (see below for exceptions). If the MEM_Term flag is also
** set, then the string is nul terminated. The MEM_Int and MEM_Real
** flags may coexist with the MEM_Str flag.
*/
#define MEM_Null 0x0001 /* Value is NULL (or a pointer) */
#define MEM_Str 0x0002 /* Value is a string */
#define MEM_Int 0x0004 /* Value is an integer */
#define MEM_Real 0x0008 /* Value is a real number */
#define MEM_Blob 0x0010 /* Value is a BLOB */
#define MEM_AffMask 0x001f /* Mask of affinity bits */
c = ((*zIn++)<<8); \
c += (*zIn++); \
if( c>=0xD800 && c<0xE000 && TERM ){ \
int c2 = ((*zIn++)<<8); \
c2 += (*zIn++); \
c = (c2&0x03FF) + ((c&0x003F)<<10) + (((c&0x03C0)+0x0040)<<10); \
} \
}
/*
** Translate a single UTF-8 character. Return the unicode value.
**
** During translation, assume that the byte that zTerm points
** is a 0x00.
**
** Write a pointer to the next unread byte back into *pzNext.
**
** Notes On Invalid UTF-8:
**
** * This routine never allows a 7-bit character (0x00 through 0x7f) to
** be encoded as a multi-byte character. Any multi-byte character that
** * This routine never allows a UTF16 surrogate value to be encoded.
** If a multi-byte character attempts to encode a value between
** 0xd800 and 0xe000 then it is rendered as 0xfffd.
**
** * Bytes in the range of 0x80 through 0xbf which occur as the first
** byte of a character are interpreted as single-byte characters
** and rendered as themselves even though they are technically
** invalid characters.
**
** * This routine accepts over-length UTF8 encodings
** for unicode values 0x80 and greater. It does not change over-length
** encodings to 0xfffd as some systems recommend.
*/
#define READ_UTF8(zIn, zTerm, c) \
c = *(zIn++); \
if( c>=0xc0 ){ \
c = sqlite3Utf8Trans1[c-0xc0]; \
while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \
c = (c<<6) + (0x3f & *(zIn++)); \
} \
if( c<0x80 \
pMem->z[pMem->n+1] = '\0';
pMem->flags |= MEM_Term;
pMem->enc = bom;
}
}
return rc;
}
#endif /* SQLITE_OMIT_UTF16 */
/*
** pZ is a UTF-8 encoded unicode string. If nByte is less than zero,
** return the number of unicode characters in pZ up to (but not including)
** the first 0x00 byte. If nByte is not less than zero, return the
** number of unicode characters in the first nByte of pZ (or up to
** the first 0x00, whichever comes first).
*/
SQLITE_PRIVATE int sqlite3Utf8CharLen(const char *zIn, int nByte){
int r = 0;
const u8 *z = (const u8*)zIn;
const u8 *zTerm;
if( nByte>=0 ){
zTerm = &z[nByte];
}else{
zTerm = (const u8*)(-1);
sqlite3VdbeMemRelease(&m);
m.z = 0;
}
assert( (m.flags & MEM_Term)!=0 || db->mallocFailed );
assert( (m.flags & MEM_Str)!=0 || db->mallocFailed );
assert( m.z || db->mallocFailed );
return m.z;
}
/*
** zIn is a UTF-16 encoded unicode string at least nChar characters long.
** Return the number of bytes in the first nChar unicode characters
** in pZ. nChar must be non-negative.
*/
SQLITE_PRIVATE int sqlite3Utf16ByteLen(const void *zIn, int nChar){
int c;
unsigned char const *z = zIn;
int n = 0;
if( SQLITE_UTF16NATIVE==SQLITE_UTF16BE ){
while( n<nChar ){
READ_UTF16BE(z, 1, c);
return 0;
}
zTextMbcs = winUnicodeToMbcs(zTmpWide, useAnsi);
sqlite3_free(zTmpWide);
return zTextMbcs;
}
/*
** This is a public wrapper for the winUtf8ToUnicode() function.
*/
SQLITE_API LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText){
#ifdef SQLITE_ENABLE_API_ARMOR
if( !zText ){
(void)SQLITE_MISUSE_BKPT;
return 0;
}
#endif
#ifndef SQLITE_OMIT_AUTOINIT
if( sqlite3_initialize() ) return 0;
#endif
return winUtf8ToUnicode(zText);
}
/*
** This is a public wrapper for the winUnicodeToUtf8() function.
*/
SQLITE_API char *sqlite3_win32_unicode_to_utf8(LPCWSTR zWideText){
#ifdef SQLITE_ENABLE_API_ARMOR
if( !zWideText ){
(void)SQLITE_MISUSE_BKPT;
return 0;
}
#endif
#ifndef SQLITE_OMIT_AUTOINIT
if( sqlite3_initialize() ) return 0;
#endif
return winUnicodeToUtf8(zWideText);
** This function is the same as sqlite3_win32_set_directory (below); however,
** it accepts a UTF-16 string.
*/
SQLITE_API int sqlite3_win32_set_directory16(
unsigned long type, /* Identifier for directory being set or reset */
const void *zValue /* New value for directory being set or reset */
){
int rc;
char *zUtf8 = 0;
if( zValue ){
zUtf8 = sqlite3_win32_unicode_to_utf8(zValue);
if( zUtf8==0 ) return SQLITE_NOMEM_BKPT;
}
rc = sqlite3_win32_set_directory8(type, zUtf8);
if( zUtf8 ) sqlite3_free(zUtf8);
return rc;
}
/*
** This function sets the data directory or the temporary directory based on
** the provided arguments. The type argument must be 1 in order to set the
}
default: {
assert( sqlite3_value_type(argv[0])==SQLITE_NULL );
sqlite3_result_text(context, "NULL", 4, SQLITE_STATIC);
break;
}
}
}
/*
** The unicode() function. Return the integer unicode code-point value
** for the first character of the input string.
*/
static void unicodeFunc(
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
const unsigned char *z = sqlite3_value_text(argv[0]);
(void)argc;
if( z && z[0] ) sqlite3_result_int(context, sqlite3Utf8Read(&z));
}
/*
** The char() function takes zero or more arguments, each of which is
** an integer. It constructs a string where each character of the string
** is the unicode character for the corresponding integer argument.
*/
static void charFunc(
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
unsigned char *z, *zOut;
int i;
zOut = z = sqlite3_malloc64( argc*4+1 );
if( z==0 ){
WAGGREGATE(min, 1, 0, 1, minmaxStep, minMaxFinalize, minMaxValue, 0,
SQLITE_FUNC_MINMAX ),
FUNCTION(max, -1, 1, 1, minmaxFunc ),
FUNCTION(max, 0, 1, 1, 0 ),
WAGGREGATE(max, 1, 1, 1, minmaxStep, minMaxFinalize, minMaxValue, 0,
SQLITE_FUNC_MINMAX ),
FUNCTION2(typeof, 1, 0, 0, typeofFunc, SQLITE_FUNC_TYPEOF),
FUNCTION2(length, 1, 0, 0, lengthFunc, SQLITE_FUNC_LENGTH),
FUNCTION(instr, 2, 0, 0, instrFunc ),
FUNCTION(printf, -1, 0, 0, printfFunc ),
FUNCTION(unicode, 1, 0, 0, unicodeFunc ),
FUNCTION(char, -1, 0, 0, charFunc ),
FUNCTION(abs, 1, 0, 0, absFunc ),
#ifndef SQLITE_OMIT_FLOATING_POINT
FUNCTION(round, 1, 0, 0, roundFunc ),
FUNCTION(round, 2, 0, 0, roundFunc ),
#endif
FUNCTION(upper, 1, 0, 0, upperFunc ),
FUNCTION(lower, 1, 0, 0, lowerFunc ),
FUNCTION(hex, 1, 0, 0, hexFunc ),
FUNCTION2(ifnull, 2, 0, 0, noopFunc, SQLITE_FUNC_COALESCE),
nBytes = sz;
}
sqlite3_mutex_enter(db->mutex);
zSql8 = sqlite3Utf16to8(db, zSql, nBytes, SQLITE_UTF16NATIVE);
if( zSql8 ){
rc = sqlite3LockAndPrepare(db, zSql8, -1, prepFlags, 0, ppStmt, &zTail8);
}
if( zTail8 && pzTail ){
/* If sqlite3_prepare returns a tail pointer, we calculate the
** equivalent pointer into the UTF-16 string by counting the unicode
** characters between zSql8 and zTail8, and then returning a pointer
** the same number of characters into the UTF-16 string.
*/
int chars_parsed = sqlite3Utf8CharLen(zSql8, (int)(zTail8-zSql8));
*pzTail = (u8 *)zSql + sqlite3Utf16ByteLen(zSql, chars_parsed);
}
sqlite3DbFree(db, zSql8);
rc = sqlite3ApiExit(db, rc);
sqlite3_mutex_leave(db->mutex);
return rc;
/* Character classes for tokenizing
**
** In the sqlite3GetToken() function, a switch() on aiClass[c] is implemented
** using a lookup table, whereas a switch() directly on c uses a binary search.
** The lookup table is much faster. To maximize speed, and to ensure that
** a lookup table is used, all of the classes need to be small integers and
** all of them need to be used within the switch.
*/
#define CC_X 0 /* The letter 'x', or start of BLOB literal */
#define CC_KYWD 1 /* Alphabetics or '_'. Usable in a keyword */
#define CC_ID 2 /* unicode characters usable in IDs */
#define CC_DIGIT 3 /* Digits */
#define CC_DOLLAR 4 /* '$' */
#define CC_VARALPHA 5 /* '@', '#', ':'. Alphabetic SQL variables */
#define CC_VARNUM 6 /* '?'. Numeric SQL variables */
#define CC_SPACE 7 /* Space characters */
#define CC_QUOTE 8 /* '"', '\'', or '`'. String literals, quoted ids */
#define CC_QUOTE2 9 /* '['. [...] style quoted ids */
#define CC_PIPE 10 /* '|'. Bitwise OR or concatenate */
#define CC_MINUS 11 /* '-'. Minus or SQL-style comment */
#define CC_LT 12 /* '<'. Part of < or <= or <> */
Fts3Table*, Fts3MultiSegReader*, int, const char*, int);
SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext(
Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *);
SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **);
SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *);
SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr);
/* fts3_tokenize_vtab.c */
SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *);
/* fts3_unicode2.c (functions generated by parsing unicode text files) */
#ifndef SQLITE_DISABLE_FTS3_UNICODE
SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int, int);
SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int);
SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int);
#endif
#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */
#endif /* _FTSINT_H */
/************** End of fts3Int.h *********************************************/
}else{
sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
}
/* Load the built-in tokenizers into the hash table */
if( rc==SQLITE_OK ){
if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple)
|| sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter)
#ifndef SQLITE_DISABLE_FTS3_UNICODE
|| sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode)
#endif
#ifdef SQLITE_ENABLE_ICU
|| (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu))
#endif
){
rc = SQLITE_NOMEM;
}
}
#ifdef SQLITE_TEST
}else{
/* Retrieve matchinfo() data. */
fts3GetMatchinfo(pContext, pCsr, zFormat);
sqlite3Fts3SegmentsClose(pTab);
}
}
#endif
/************** End of fts3_snippet.c ****************************************/
/************** Begin file fts3_unicode.c ************************************/
/*
** 2012 May 24
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** Implementation of the "unicode" full-text-search tokenizer.
*/
#ifndef SQLITE_DISABLE_FTS3_UNICODE
/* #include "fts3Int.h" */
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
/* #include <assert.h> */
/* #include <stdlib.h> */
/* #include <stdio.h> */
}else{ \
*zOut++ = 0xF0 + (u8)((c>>18) & 0x07); \
*zOut++ = 0x80 + (u8)((c>>12) & 0x3F); \
*zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \
*zOut++ = 0x80 + (u8)(c & 0x3F); \
} \
}
#endif /* ifndef SQLITE_AMALGAMATION */
typedef struct unicode_tokenizer unicode_tokenizer;
typedef struct unicode_cursor unicode_cursor;
struct unicode_tokenizer {
sqlite3_tokenizer base;
int bRemoveDiacritic;
int nException;
int *aiException;
};
struct unicode_cursor {
sqlite3_tokenizer_cursor base;
const unsigned char *aInput; /* Input text being tokenized */
int nInput; /* Size of aInput[] in bytes */
int iOff; /* Current offset within aInput[] */
int iToken; /* Index of next token to be returned */
char *zToken; /* storage for current token */
int nAlloc; /* space allocated at zToken */
};
/*
** Destroy a tokenizer allocated by unicodeCreate().
*/
static int unicodeDestroy(sqlite3_tokenizer *pTokenizer){
if( pTokenizer ){
unicode_tokenizer *p = (unicode_tokenizer *)pTokenizer;
sqlite3_free(p->aiException);
sqlite3_free(p);
}
return SQLITE_OK;
}
/*
** As part of a tokenchars= or separators= option, the CREATE VIRTUAL TABLE
** statement has specified that the tokenizer for this table shall consider
** all characters in string zIn/nIn to be separators (if bAlnum==0) or
** token characters (if bAlnum==1).
**
** For each codepoint in the zIn/nIn string, this function checks if the
** sqlite3FtsUnicodeIsalnum() function already returns the desired result.
** If so, no action is taken. Otherwise, the codepoint is added to the
** unicode_tokenizer.aiException[] array. For the purposes of tokenization,
** the return value of sqlite3FtsUnicodeIsalnum() is inverted for all
** codepoints in the aiException[] array.
**
** If a standalone diacritic mark (one that sqlite3FtsUnicodeIsdiacritic()
** identifies as a diacritic) occurs in the zIn/nIn string it is ignored.
** It is not possible to change the behavior of the tokenizer with respect
** to these codepoints.
*/
static int unicodeAddExceptions(
unicode_tokenizer *p, /* Tokenizer to add exceptions to */
int bAlnum, /* Replace Isalnum() return value with this */
const char *zIn, /* Array of characters to make exceptions */
int nIn /* Length of z in bytes */
){
const unsigned char *z = (const unsigned char *)zIn;
const unsigned char *zTerm = &z[nIn];
unsigned int iCode;
int nEntry = 0;
assert( bAlnum==0 || bAlnum==1 );
p->aiException = aNew;
p->nException = nNew;
}
return SQLITE_OK;
}
/*
** Return true if the p->aiException[] array contains the value iCode.
*/
static int unicodeIsException(unicode_tokenizer *p, int iCode){
if( p->nException>0 ){
int *a = p->aiException;
int iLo = 0;
int iHi = p->nException-1;
while( iHi>=iLo ){
int iTest = (iHi + iLo) / 2;
if( iCode==a[iTest] ){
return 1;
}else if( iCode>a[iTest] ){
}
}
return 0;
}
/*
** Return true if, for the purposes of tokenization, codepoint iCode is
** considered a token character (not a separator).
*/
static int unicodeIsAlnum(unicode_tokenizer *p, int iCode){
assert( (sqlite3FtsUnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 );
return sqlite3FtsUnicodeIsalnum(iCode) ^ unicodeIsException(p, iCode);
}
/*
** Create a new tokenizer instance.
*/
static int unicodeCreate(
int nArg, /* Size of array argv[] */
const char * const *azArg, /* Tokenizer creation arguments */
sqlite3_tokenizer **pp /* OUT: New tokenizer handle */
){
unicode_tokenizer *pNew; /* New tokenizer object */
int i;
int rc = SQLITE_OK;
pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer));
if( pNew==NULL ) return SQLITE_NOMEM;
memset(pNew, 0, sizeof(unicode_tokenizer));
pNew->bRemoveDiacritic = 1;
for(i=0; rc==SQLITE_OK && i<nArg; i++){
const char *z = azArg[i];
int n = (int)strlen(z);
if( n==19 && memcmp("remove_diacritics=1", z, 19)==0 ){
pNew->bRemoveDiacritic = 1;
}
else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){
pNew->bRemoveDiacritic = 0;
}
else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){
rc = unicodeAddExceptions(pNew, 1, &z[11], n-11);
}
else if( n>=11 && memcmp("separators=", z, 11)==0 ){
rc = unicodeAddExceptions(pNew, 0, &z[11], n-11);
}
else{
/* Unrecognized argument */
rc = SQLITE_ERROR;
}
}
if( rc!=SQLITE_OK ){
unicodeDestroy((sqlite3_tokenizer *)pNew);
pNew = 0;
}
*pp = (sqlite3_tokenizer *)pNew;
return rc;
}
/*
** Prepare to begin tokenizing a particular string. The input
** string to be tokenized is pInput[0..nBytes-1]. A cursor
** used to incrementally tokenize this string is returned in
** *ppCursor.
*/
static int unicodeOpen(
sqlite3_tokenizer *p, /* The tokenizer */
const char *aInput, /* Input string */
int nInput, /* Size of string aInput in bytes */
sqlite3_tokenizer_cursor **pp /* OUT: New cursor object */
){
unicode_cursor *pCsr;
pCsr = (unicode_cursor *)sqlite3_malloc(sizeof(unicode_cursor));
if( pCsr==0 ){
return SQLITE_NOMEM;
}
memset(pCsr, 0, sizeof(unicode_cursor));
pCsr->aInput = (const unsigned char *)aInput;
if( aInput==0 ){
pCsr->nInput = 0;
}else if( nInput<0 ){
pCsr->nInput = (int)strlen(aInput);
}else{
pCsr->nInput = nInput;
}
*pp = &pCsr->base;
UNUSED_PARAMETER(p);
return SQLITE_OK;
}
/*
** Close a tokenization cursor previously opened by a call to
** simpleOpen() above.
*/
static int unicodeClose(sqlite3_tokenizer_cursor *pCursor){
unicode_cursor *pCsr = (unicode_cursor *) pCursor;
sqlite3_free(pCsr->zToken);
sqlite3_free(pCsr);
return SQLITE_OK;
}
/*
** Extract the next token from a tokenization cursor. The cursor must
** have been opened by a prior call to simpleOpen().
*/
static int unicodeNext(
sqlite3_tokenizer_cursor *pC, /* Cursor returned by simpleOpen */
const char **paToken, /* OUT: Token text */
int *pnToken, /* OUT: Number of bytes at *paToken */
int *piStart, /* OUT: Starting offset of token */
int *piEnd, /* OUT: Ending offset of token */
int *piPos /* OUT: Position integer of token */
){
unicode_cursor *pCsr = (unicode_cursor *)pC;
unicode_tokenizer *p = ((unicode_tokenizer *)pCsr->base.pTokenizer);
unsigned int iCode = 0;
char *zOut;
const unsigned char *z = &pCsr->aInput[pCsr->iOff];
const unsigned char *zStart = z;
const unsigned char *zEnd;
const unsigned char *zTerm = &pCsr->aInput[pCsr->nInput];
/* Scan past any delimiter characters before the start of the next token.
** Return SQLITE_DONE early if this takes us all the way to the end of
** the input. */
while( z<zTerm ){
READ_UTF8(z, zTerm, iCode);
if( unicodeIsAlnum(p, (int)iCode) ) break;
zStart = z;
}
if( zStart>=zTerm ) return SQLITE_DONE;
zOut = pCsr->zToken;
do {
int iOut;
/* Grow the output buffer if required. */
if( (zOut-pCsr->zToken)>=(pCsr->nAlloc-4) ){
/* Write the folded case of the last character read to the output */
zEnd = z;
iOut = sqlite3FtsUnicodeFold((int)iCode, p->bRemoveDiacritic);
if( iOut ){
WRITE_UTF8(zOut, iOut);
}
/* If the cursor is not at EOF, read the next character */
if( z>=zTerm ) break;
READ_UTF8(z, zTerm, iCode);
}while( unicodeIsAlnum(p, (int)iCode)
|| sqlite3FtsUnicodeIsdiacritic((int)iCode)
);
/* Set the output variables and return. */
pCsr->iOff = (int)(z - pCsr->aInput);
*paToken = pCsr->zToken;
*pnToken = (int)(zOut - pCsr->zToken);
*piStart = (int)(zStart - pCsr->aInput);
*piEnd = (int)(zEnd - pCsr->aInput);
*piPos = pCsr->iToken++;
return SQLITE_OK;
}
/*
** Set *ppModule to a pointer to the sqlite3_tokenizer_module
** structure for the unicode tokenizer.
*/
SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const **ppModule){
static const sqlite3_tokenizer_module module = {
0,
unicodeCreate,
unicodeDestroy,
unicodeOpen,
unicodeClose,
unicodeNext,
0,
};
*ppModule = &module;
}
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */
/************** End of fts3_unicode.c ****************************************/
/************** Begin file fts3_unicode2.c ***********************************/
/*
** 2012 May 25
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
/*
** DO NOT EDIT THIS MACHINE GENERATED FILE.
*/
#ifndef SQLITE_DISABLE_FTS3_UNICODE
#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)
/* #include <assert.h> */
/*
** Return true if the argument corresponds to a unicode codepoint
** classified as either a letter or a number. Otherwise false.
**
** The results are undefined if the value passed to this function
** is less than zero.
*/
SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int c){
/* Each unsigned integer in the following array corresponds to a contiguous
** range of unicode codepoints that are not either letters or numbers (i.e.
** codepoints for which this function should return 0).
**
** The most significant 22 bits in each 32-bit value contain the first
** codepoint in the range. The least significant 10 bits are used to store
** the size of the range (always at least 1). In other words, the value
** ((C<<22) + N) represents a range of N codepoints starting with codepoint
** C. It is not possible to represent a range larger than 1023 codepoints
** using this format.
*/
static const unsigned int aEntry[] = {
}else{
iHi = iTest-1;
}
}
assert( key>=aDia[iRes] );
return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
}
/*
** Return true if the argument interpreted as a unicode codepoint
** is a diacritical modifier character.
*/
SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int c){
unsigned int mask0 = 0x08029FDF;
unsigned int mask1 = 0x000361F8;
if( c<768 || c>817 ) return 0;
return (c < 768+32) ?
(mask0 & (1 << (c-768))) :
(mask1 & (1 << (c-768-32)));
}
/*
** Interpret the argument as a unicode codepoint. If the codepoint
** is an upper case character that has a lower case equivalent,
** return the codepoint corresponding to the lower case version.
** Otherwise, return a copy of the argument.
**
** The results are undefined if the value passed to this function
** is less than zero.
*/
SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){
/* Each entry in the following array defines a rule for folding a range
** of codepoints to lower case. The rule applies to a range of nRange
** to all nRange codepoints (i.e. all nRange codepoints are upper case and
** need to be folded). Or, if it is set, then the rule only applies to
** every second codepoint in the range, starting with codepoint C.
**
** The 7 most significant bits in flags are an index into the aiOff[]
** array. If a specific codepoint C does require folding, then its lower
** case equivalent is ((C + aiOff[flags>>1]) & 0xFFFF).
**
** The contents of this array are generated by parsing the CaseFolding.txt
** file distributed as part of the "Unicode Character Database". See
** http://www.unicode.org for details.
*/
static const struct TableEntry {
unsigned short iCode;
unsigned char flags;
unsigned char nRange;
} aEntry[] = {
{65, 14, 26}, {181, 64, 1}, {192, 14, 23},
{216, 14, 7}, {256, 1, 48}, {306, 1, 6},
{313, 1, 16}, {330, 1, 46}, {376, 116, 1},
{377, 1, 6}, {383, 104, 1}, {385, 50, 1},
else if( c>=66560 && c<66600 ){
ret = c + 40;
}
return ret;
}
#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */
#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */
/************** End of fts3_unicode2.c ***************************************/
/************** Begin file json1.c *******************************************/
/*
** 2015-08-12
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** $Id: icu.c,v 1.7 2007/12/13 21:54:11 drh Exp $
**
** This file implements an integration between the ICU library
** ("International Components for Unicode", an open-source library
** for handling unicode data) and SQLite. The integration uses
** ICU to provide the following to SQLite:
**
** * An implementation of the SQL regexp() function (and hence REGEXP
** operator) using the ICU uregex_XX() APIs.
**
** * Implementations of the SQL scalar upper() and lower() functions
** for case mapping.
**
** * Integration of ICU and SQLite collation sequences.
**
** * An implementation of the LIKE operator that uses ICU to
** provide case-independent matching.
*/
#if !defined(SQLITE_CORE) \
|| defined(SQLITE_ENABLE_ICU) \
|| defined(SQLITE_ENABLE_ICU_COLLATIONS)
/* Include ICU headers */
#include <unicode/utypes.h>
#include <unicode/uregex.h>
#include <unicode/ustring.h>
#include <unicode/ucol.h>
/* #include <assert.h> */
#ifndef SQLITE_CORE
/* #include "sqlite3ext.h" */
SQLITE_EXTENSION_INIT1
#else
/* #include "sqlite3.h" */
#endif
** This file implements a tokenizer for fts3 based on the ICU library.
*/
/* #include "fts3Int.h" */
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
#ifdef SQLITE_ENABLE_ICU
/* #include <assert.h> */
/* #include <string.h> */
/* #include "fts3_tokenizer.h" */
#include <unicode/ubrk.h>
/* #include <unicode/ucol.h> */
/* #include <unicode/ustring.h> */
#include <unicode/utf16.h>
typedef struct IcuTokenizer IcuTokenizer;
typedef struct IcuCursor IcuCursor;
struct IcuTokenizer {
sqlite3_tokenizer base;
char *zLocale;
};
struct IcuCursor {
*/
static int sqlite3Fts5VocabInit(Fts5Global*, sqlite3*);
/*
** End of interface to code in fts5_vocab.c.
**************************************************************************/
/**************************************************************************
** Interface to automatically generated code in fts5_unicode2.c.
*/
static int sqlite3Fts5UnicodeIsdiacritic(int c);
static int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic);
static int sqlite3Fts5UnicodeCatParse(const char*, u8*);
static int sqlite3Fts5UnicodeCategory(int iCode);
static void sqlite3Fts5UnicodeAscii(u8*, u8*);
/*
** End of interface to code in fts5_unicode2.c.
**************************************************************************/
#endif
#define FTS5_OR 1
#define FTS5_AND 2
#define FTS5_NOT 3
#define FTS5_TERM 4
#define FTS5_COLON 5
#define FTS5_MINUS 6
/*
** Return true if character 't' may be part of an FTS5 bareword, or false
** otherwise. Characters that may be part of barewords:
**
** * All non-ASCII characters,
** * The 52 upper and lower case ASCII characters, and
** * The 10 integer ASCII characters.
** * The underscore character "_" (0x5F).
** * The unicode "subsitute" character (0x1A).
*/
static int sqlite3Fts5IsBareword(char t){
u8 aBareword[128] = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x00 .. 0x0F */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, /* 0x10 .. 0x1F */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x20 .. 0x2F */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 0x30 .. 0x3F */
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x40 .. 0x4F */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 0x50 .. 0x5F */
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x60 .. 0x6F */
}
}
}
sqlite3_free(zOne);
sqlite3_free(zTwo);
}
/* If a tokenizer= option was successfully parsed, the tokenizer has
** already been allocated. Otherwise, allocate an instance of the default
** tokenizer (unicode61) now. */
if( rc==SQLITE_OK && pRet->pTok==0 ){
rc = fts5ConfigDefaultTokenizer(pGlobal, pRet);
}
/* If no zContent option was specified, fill in the default values. */
if( rc==SQLITE_OK && pRet->zContent==0 ){
const char *zTail = 0;
assert( pRet->eContent==FTS5_CONTENT_NORMAL
|| pRet->eContent==FTS5_CONTENT_NONE
);
sqlite3_context *pCtx, /* Function call context */
int nArg, /* Number of args */
sqlite3_value **apVal /* Function arguments */
){
fts5ExprFunction(pCtx, nArg, apVal, 1);
}
/*
** The implementation of an SQLite user-defined-function that accepts a
** single integer as an argument. If the integer is an alpha-numeric
** unicode code point, 1 is returned. Otherwise 0.
*/
static void fts5ExprIsAlnum(
sqlite3_context *pCtx, /* Function call context */
int nArg, /* Number of args */
sqlite3_value **apVal /* Function arguments */
){
int iCode;
u8 aArr[32];
if( nArg!=1 ){
sqlite3_result_error(pCtx,
n++;
if( n>=nByte ) break;
}
}
}
return n;
}
/*
** pIn is a UTF-8 encoded string, nIn bytes in size. Return the number of
** unicode characters in the string.
*/
static int fts5IndexCharlen(const char *pIn, int nIn){
int nChar = 0;
int i = 0;
while( i<nIn ){
if( (unsigned char)pIn[i++]>=0xc0 ){
while( i<nIn && (pIn[i] & 0xc0)==0x80 ) i++;
}
nChar++;
}
*/
/* #include "fts5Int.h" */
/**************************************************************************
** Start of ascii tokenizer implementation.
*/
/*
** For tokenizers with no "unicode" modifier, the set of token characters
** is the same as the set of ASCII range alphanumeric characters.
*/
static unsigned char aAsciiTokenChar[128] = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x00..0x0F */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x10..0x1F */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x20..0x2F */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 0x30..0x3F */
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x40..0x4F */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 0x50..0x5F */
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x60..0x6F */
rc = xToken(pCtx, 0, pFold, nByte, is, ie);
is = ie+1;
}
if( pFold!=aFold ) sqlite3_free(pFold);
if( rc==SQLITE_DONE ) rc = SQLITE_OK;
return rc;
}
/**************************************************************************
** Start of unicode61 tokenizer implementation.
*/
/*
** The following two macros - READ_UTF8 and WRITE_UTF8 - have been copied
** from the sqlite3 source file utf.c. If this file is compiled as part
** of the amalgamation, they are not required.
*/
#ifndef SQLITE_AMALGAMATION
}else{
iHi = iTest-1;
}
}
}
return 0;
}
/*
** Delete a "unicode61" tokenizer.
*/
static void fts5UnicodeDelete(Fts5Tokenizer *pTok){
if( pTok ){
Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTok;
sqlite3_free(p->aiException);
sqlite3_free(p->aFold);
sqlite3_free(p);
}
return;
}
static int unicodeSetCategories(Unicode61Tokenizer *p, const char *zCat){
const char *z = zCat;
while( *z ){
while( *z==' ' || *z=='\t' ) z++;
if( *z && sqlite3Fts5UnicodeCatParse(z, p->aCategory) ){
return SQLITE_ERROR;
}
while( *z!=' ' && *z!='\t' && *z!='\0' ) z++;
}
sqlite3Fts5UnicodeAscii(p->aCategory, p->aTokenChar);
return SQLITE_OK;
}
/*
** Create a "unicode61" tokenizer.
*/
static int fts5UnicodeCreate(
void *pUnused,
const char **azArg, int nArg,
Fts5Tokenizer **ppOut
){
int rc = SQLITE_OK; /* Return code */
Unicode61Tokenizer *p = 0; /* New tokenizer object */
UNUSED_PARAM(pUnused);
}
/* Search for a "categories" argument */
for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
if( 0==sqlite3_stricmp(azArg[i], "categories") ){
zCat = azArg[i+1];
}
}
if( rc==SQLITE_OK ){
rc = unicodeSetCategories(p, zCat);
}
for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
const char *zArg = azArg[i+1];
if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
rc = SQLITE_ERROR;
}
p->bRemoveDiacritic = (zArg[0]=='1');
}else
*/
static int fts5PorterCreate(
void *pCtx,
const char **azArg, int nArg,
Fts5Tokenizer **ppOut
){
fts5_api *pApi = (fts5_api*)pCtx;
int rc = SQLITE_OK;
PorterTokenizer *pRet;
void *pUserdata = 0;
const char *zBase = "unicode61";
if( nArg>0 ){
zBase = azArg[0];
}
pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer));
if( pRet ){
memset(pRet, 0, sizeof(PorterTokenizer));
rc = pApi->xFindTokenizer(pApi, zBase, &pUserdata, &pRet->tokenizer);
}else{
}
/*
** Register all built-in tokenizers with FTS5.
*/
static int sqlite3Fts5TokenizerInit(fts5_api *pApi){
struct BuiltinTokenizer {
const char *zName;
fts5_tokenizer x;
} aBuiltin[] = {
{ "unicode61", {fts5UnicodeCreate, fts5UnicodeDelete, fts5UnicodeTokenize}},
{ "ascii", {fts5AsciiCreate, fts5AsciiDelete, fts5AsciiTokenize }},
{ "porter", {fts5PorterCreate, fts5PorterDelete, fts5PorterTokenize }},
};
int rc = SQLITE_OK; /* Return code */
int i; /* To iterate through builtin functions */
for(i=0; rc==SQLITE_OK && i<ArraySize(aBuiltin); i++){
rc = pApi->xCreateTokenizer(pApi,
aBuiltin[i].zName,
}else{
iHi = iTest-1;
}
}
assert( key>=aDia[iRes] );
return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
}
/*
** Return true if the argument interpreted as a unicode codepoint
** is a diacritical modifier character.
*/
static int sqlite3Fts5UnicodeIsdiacritic(int c){
unsigned int mask0 = 0x08029FDF;
unsigned int mask1 = 0x000361F8;
if( c<768 || c>817 ) return 0;
return (c < 768+32) ?
(mask0 & (1 << (c-768))) :
(mask1 & (1 << (c-768-32)));
}
/*
** Interpret the argument as a unicode codepoint. If the codepoint
** is an upper case character that has a lower case equivalent,
** return the codepoint corresponding to the lower case version.
** Otherwise, return a copy of the argument.
**
** The results are undefined if the value passed to this function
** is less than zero.
*/
static int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){
/* Each entry in the following array defines a rule for folding a range
** of codepoints to lower case. The rule applies to a range of nRange
** to all nRange codepoints (i.e. all nRange codepoints are upper case and
** need to be folded). Or, if it is set, then the rule only applies to
** every second codepoint in the range, starting with codepoint C.
**
** The 7 most significant bits in flags are an index into the aiOff[]
** array. If a specific codepoint C does require folding, then its lower
** case equivalent is ((C + aiOff[flags>>1]) & 0xFFFF).
**
** The contents of this array are generated by parsing the CaseFolding.txt
** file distributed as part of the "Unicode Character Database". See
** http://www.unicode.org for details.
*/
static const struct TableEntry {
unsigned short iCode;
unsigned char flags;
unsigned char nRange;
} aEntry[] = {
{65, 14, 26}, {181, 64, 1}, {192, 14, 23},
{216, 14, 7}, {256, 1, 48}, {306, 1, 6},
{313, 1, 16}, {330, 1, 46}, {376, 116, 1},
{377, 1, 6}, {383, 104, 1}, {385, 50, 1},
t/02_logon.t view on Meta::CPAN
ok( defined $dbh->$call_func(0, 'busy_timeout') );
is( $dbh->$call_func('busy_timeout'), 0, 'Set busy_timeout to 0' );
}
# Attributes in the connect string
SKIP: {
unless ( $] >= 5.008005 ) {
skip( 'Unicode is not supported before 5.8.5', 2 );
}
my $file = 'foo'.$$;
my $dbh = DBI->connect( "dbi:SQLeet:dbname=$file;sqlite_unicode=1", '', '' );
isa_ok( $dbh, 'DBI::db' );
is( $dbh->{sqlite_unicode}, 1, 'Unicode is on' );
$dbh->disconnect;
unlink $file;
}
# dbname, db, database
SCOPE: {
for my $key (qw/database db dbname/) {
my $file = 'foo'.$$;
unlink $file if -f $file;
ok !-f $file, 'database file does not exist';
t/12_unicode.t view on Meta::CPAN
#!/usr/bin/perl
# This is a test for correct handling of the "unicode" database
# handle parameter.
use strict;
BEGIN {
$| = 1;
$^W = 1;
}
use lib "t/lib";
t/12_unicode.t view on Meta::CPAN
! is_utf8($bytestring),
'$bytestring should *NOT* be marked as UTF-8 by Perl',
);
# Sends $ain and $bin into TEXT resp. BLOB columns the database, then
# reads them again and returns the result as a list ($aout, $bout).
### Real DBD::SQLeet testing starts here
my ($textback, $bytesback);
SCOPE: {
my $dbh = connect_ok( dbfile => 'foo', RaiseError => 1 );
is( $dbh->{sqlite_unicode}, 0, 'Unicode is off' );
ok(
$dbh->do("CREATE TABLE table1 (a TEXT, b BLOB)"),
'CREATE TABLE',
);
($textback, $bytesback) = database_roundtrip($dbh, $bytestring, $bytestring);
ok(
! is_utf8($bytesback),
"Reading blob gives binary",
t/12_unicode.t view on Meta::CPAN
ok(
! is_utf8($textback),
"Reading text gives binary too (for now)",
);
is($bytesback, $bytestring, "No blob corruption");
is($textback, $bytestring, "Same text, different encoding");
}
# Start over but now activate Unicode support.
SCOPE: {
my $dbh = connect_ok( dbfile => 'foo', sqlite_unicode => 1 );
is( $dbh->{sqlite_unicode}, 1, 'Unicode is on' );
($textback, $bytesback) = database_roundtrip($dbh, $utfstring, $bytestring);
ok(! is_utf8($bytesback), "Reading blob still gives binary");
ok(is_utf8($textback), "Reading text returns UTF-8");
ok($bytesback eq $bytestring, "Still no blob corruption");
ok($textback eq $utfstring, "Same text");
my $lengths = $dbh->selectall_arrayref(
"SELECT length(a), length(b) FROM table1"
t/13_create_collation.t view on Meta::CPAN
qr/already registered/,
"can't override registered collation");
my $tied = tied %DBD::SQLeet::COLLATION;
delete $tied->{foo};
$DBD::SQLeet::COLLATION{foo} = \&by_num_desc; # override, no longer dies
is($DBD::SQLeet::COLLATION{foo}, \&by_num_desc, "overridden collation");
# now really test the collation functions
foreach my $call_func (@CALL_FUNCS) {
for my $use_unicode (0, 1) {
# connect
my $dbh = connect_ok( RaiseError => 1, sqlite_unicode => $use_unicode );
# populate test data
my @words = qw{
berger Berg��e berg��e Bergere
HOT h��e
h����oclite h��a��e h��re h��aut
HAT h��er
f��u f��e f��e ferme
};
if ($use_unicode) {
utf8::upgrade($_) foreach @words;
}
$dbh->do( 'CREATE TEMP TABLE collate_test ( txt )' );
$dbh->do( "INSERT INTO collate_test VALUES ( '$_' )" ) foreach @words;
# test builtin collation "perl"
my @sorted = sort @words;
my $db_sorted = $dbh->selectcol_arrayref("$sql COLLATE perl");
is_deeply(\@sorted, $db_sorted, "collate perl (@sorted // @$db_sorted)");
t/33_non_latin_path.t view on Meta::CPAN
RaiseError => 1,
PrintError => 0,
} );
isa_ok( $dbh, 'DBI::db' );
};
is( $@, '', "Could connect to database in $subdir" );
diag( $@ ) if $@;
unlink(_path($dbfile)) if -e _path($dbfile);
# Repeat with the unicode flag on
my $ufile = $dbfile;
eval {
my $dbh = DBI->connect("dbi:SQLeet:dbname=$dbfile", undef, undef, {
RaiseError => 1,
PrintError => 0,
sqlite_unicode => 1,
} );
isa_ok( $dbh, 'DBI::db' );
};
is( $@, '', "Could connect to database in $subdir" );
diag( $@ ) if $@;
# Reopen the database
eval {
my $dbh = DBI->connect("dbi:SQLeet:dbname=$dbfile", undef, undef, {
RaiseError => 1,
PrintError => 0,
sqlite_unicode => 1,
} );
isa_ok( $dbh, 'DBI::db' );
};
is( $@, '', "Could connect to database in $subdir" );
diag( $@ ) if $@;
unlink(_path($ufile)) if -e _path($ufile);
# when the name of the database file has non-latin characters
my $dbfilex = catfile($dir, "$subdir.db");
t/37_regexp.t view on Meta::CPAN
use POSIX 'locale_h';
setlocale(LC_COLLATE, 'en-us');
}
}
use locale;
use DBD::SQLeet;
foreach my $call_func (@CALL_FUNCS) {
for my $use_unicode (0, 1) {
# connect
my $dbh = connect_ok( RaiseError => 1, sqlite_unicode => $use_unicode );
# The following tests are about ordering, so don't reverse!
if ($dbh->selectrow_array('PRAGMA reverse_unordered_selects')) {
$dbh->do('PRAGMA reverse_unordered_selects = OFF');
}
# populate test data
my @vals = @words;
if ($use_unicode) {
utf8::upgrade($_) foreach @vals;
}
$dbh->do( 'CREATE TEMP TABLE regexp_test ( txt )' );
$dbh->do( "INSERT INTO regexp_test VALUES ( '$_' )" ) foreach @vals;
foreach my $regex (@regexes) {
my @perl_match = grep {/$regex/} @vals;
my $sql = "SELECT txt from regexp_test WHERE txt REGEXP '$regex' "
. "COLLATE perllocale";
t/43_fts3.t view on Meta::CPAN
if ($DBD::SQLeet::sqlite_version_number >= 3011000 and $DBD::SQLeet::sqlite_version_number < 3012000 and !grep /ENABLE_FTS3_TOKENIZER/, DBD::SQLeet::compile_options()) {
plan skip_all => 'FTS3 tokenizer is disabled for this DBD::SQLeet';
}
}
# Perl may spit a warning on locale
# use Test::NoWarnings;
my $num = has_sqleet_sqlite('3.7.4') ? 4 : 2;
plan tests => $num * @tests # each test with unicode y/n and with fts3/fts4
+ 2; # connect_ok with unicode y/n
BEGIN {
# Sadly perl for windows (and probably sqlite, too) may hang
# if the system locale doesn't support european languages.
# en-us should be a safe default. if it doesn't work, use 'C'.
if ( $^O eq 'MSWin32') {
use POSIX 'locale_h';
setlocale(LC_COLLATE, 'en-us');
}
}
t/43_fts3.t view on Meta::CPAN
$string =~ /$regex/g or return; # either match, or no more token
my ($start, $end) = ($-[0], $+[0]);
my $term = substr($string, $start, my $len = $end-$start);
return ($term, $len, $start, $end, $term_index++);
};
};
}
use DBD::SQLeet;
for my $use_unicode (0, 1) {
# connect
my $dbh = connect_ok( RaiseError => 1, sqlite_unicode => $use_unicode );
for my $fts (qw/fts3 fts4/) {
next if $fts eq 'fts4' && !has_sqleet_sqlite('3.7.4');
# create fts table
$dbh->do(<<"") or die DBI::errstr;
CREATE VIRTUAL TABLE try_$fts
USING $fts(content, tokenize=perl 'main::locale_tokenizer')
# populate it
my $insert_sth = $dbh->prepare(<<"") or die DBI::errstr;
t/43_fts3.t view on Meta::CPAN
SKIP: {
skip "These tests require SQLite compiled with "
. "ENABLE_FTS3_PARENTHESIS option", scalar @tests
unless DBD::SQLeet->can('compile_options') &&
grep /ENABLE_FTS3_PARENTHESIS/, DBD::SQLeet::compile_options();
my $sql = "SELECT docid FROM try_$fts WHERE content MATCH ?";
for my $t (@tests) {
my ($query, @expected) = @$t;
@expected = map {$doc_ids[$_]} @expected;
my $results = $dbh->selectcol_arrayref($sql, undef, $query);
is_deeply($results, \@expected, "$query ($fts, unicode=$use_unicode)");
}
}
}
}
t/62_regexp_multibyte_char_class.t view on Meta::CPAN
use SQLeetTest qw/connect_ok @CALL_FUNCS/;
use Test::More;
BEGIN {
if ($] < 5.008005) {
plan skip_all => 'Unicode is not supported before 5.8.5';
}
}
#use Test::NoWarnings; # see RT#112220
# special case for multibyte (non-ASCII) character class,
# which only works correctly under the unicode mode
my @words = ("\x{e3}\x{83}\x{86}\x{e3}\x{82}\x{b9}\x{e3}\x{83}\x{88}", "\x{e3}\x{83}\x{86}\x{e3}\x{83}\x{b3}\x{e3}\x{83}\x{88}"); # テスト テント
my $regex = "\x{e3}\x{83}\x{86}[\x{e3}\x{82}\x{b9}\x{e3}\x{83}\x{b3}]\x{e3}\x{83}\x{88}"; # テ[スン]ト
plan tests => 2 * 2 * @CALL_FUNCS;
foreach my $call_func (@CALL_FUNCS) {
for my $use_unicode (0, 1) {
# connect
my $dbh = connect_ok( RaiseError => 1, sqlite_unicode => $use_unicode );
# populate test data
my @vals = @words;
my $re = $regex;
if ($use_unicode) {
utf8::decode($_) foreach @vals;
utf8::decode($re);
}
my @perl_match = grep {$_ =~ /$re/} @vals;
$dbh->do( 'CREATE TEMP TABLE regexp_test ( txt )' );
$dbh->do( "INSERT INTO regexp_test VALUES ( '$_' )" ) foreach @vals;
my $sql = "SELECT txt from regexp_test WHERE txt REGEXP '$re' ";
my $db_match = $dbh->selectcol_arrayref($sql);
t/rt_25371_asymmetric_unicode.t view on Meta::CPAN
BEGIN {
if ( $] >= 5.008005 ) {
plan( tests => 23 );
} else {
plan( skip_all => 'Unicode is not supported before 5.8.5' );
}
}
use Test::NoWarnings;
my $dbh = connect_ok( sqlite_unicode => 1 );
is( $dbh->{sqlite_unicode}, 1, 'Unicode is on' );
ok( $dbh->do(<<'END_SQL'), 'CREATE TABLE' );
CREATE TABLE foo (
bar varchar(255)
)
END_SQL
foreach ( "\0", "A", "\xe9", "\x{20ac}" ) {
ok( $dbh->do("INSERT INTO foo VALUES ( ? )", {}, $_), 'INSERT' );
my $foo = $dbh->selectall_arrayref("SELECT bar FROM foo");
t/rt_25924_user_defined_func_unicode.t view on Meta::CPAN
if ( $] >= 5.008005 ) {
plan( tests => 15 * @CALL_FUNCS + 1);
} else {
plan( skip_all => 'Unicode is not supported before 5.8.5' );
}
}
use Test::NoWarnings;
foreach my $call_func (@CALL_FUNCS) {
my $dbh = connect_ok( sqlite_unicode => 1 );
ok($dbh->$call_func( "perl_uc", 1, \&perl_uc, "create_function" ));
ok( $dbh->do(<<'END_SQL'), 'CREATE TABLE' );
CREATE TABLE foo (
bar varchar(255)
)
END_SQL
my @words = qw{Berg�re h�te h�ta�re h�tre};
foreach my $word (@words) {
# rt48048: don't need to "use utf8" nor "require utf8"
utf8::upgrade($word);
ok( $dbh->do("INSERT INTO foo VALUES ( ? )", {}, $word), 'INSERT' );
my $foo = $dbh->selectall_arrayref("SELECT perl_uc(bar) FROM foo");
is_deeply( $foo, [ [ perl_uc($word) ] ], 'unicode upcase ok' );
ok( $dbh->do("DELETE FROM foo"), 'DELETE ok' );
}
$dbh->disconnect;
}
sub perl_uc {
my $string = shift;
return uc($string);
}
t/rt_71311_bind_col_and_unicode.t view on Meta::CPAN
if ( $] >= 5.008005 ) {
plan( tests => 50 );
} else {
plan( skip_all => 'Unicode is not supported before 5.8.5' );
}
}
use Test::NoWarnings;
use DBI qw/:sql_types/;
my $dbh = connect_ok(sqlite_unicode => 1);
$dbh->do('create table test1 (id integer, b blob)');
my $blob = "\x{82}\x{A0}";
my $str = "\x{20ac}";
{
my $sth = $dbh->prepare('insert into test1 values (?, ?)');
$sth->execute(1, $blob);
t/rt_71311_bind_col_and_unicode.t view on Meta::CPAN
{
my $sth = $dbh->prepare('select * from test1 order by id');
$sth->execute;
my $expected = [undef, 1, 0, 0, 1, 1, 1];
for (1..6) {
my $row = $sth->fetch;
ok $row && $row->[0] == $_;
ok $row && utf8::is_utf8($row->[1]) == $expected->[$_],
"row $_ is ".($expected->[$_] ? "unicode" : "not unicode");
}
$sth->finish;
}
{
my $sth = $dbh->prepare('select * from test1 order by id');
$sth->bind_col(1, \my $col1);
$sth->bind_col(2, \my $col2);
$sth->execute;
my $expected = [undef, 1, 0, 0, 1, 1, 1];
for (1..6) {
$sth->fetch;
ok $col1 && $col1 == $_;
ok $col1 && utf8::is_utf8($col2) == $expected->[$_],
"row $_ is ".($expected->[$_] ? "unicode" : "not unicode");
}
$sth->finish;
}
{
my $sth = $dbh->prepare('select * from test1 order by id');
$sth->bind_col(1, \my $col1);
$sth->bind_col(2, \my $col2, SQL_BLOB);
$sth->execute;
my $expected = [undef, 0, 0, 0, 0, 0, 0];
for (1..6) {
$sth->fetch;
ok $col1 && $col1 == $_;
ok $col2 && utf8::is_utf8($col2) == $expected->[$_],
"row $_ is ".($expected->[$_] ? "unicode" : "not unicode");
}
$sth->finish;
}
{
my $sth = $dbh->prepare('select * from test1 order by id');
$sth->bind_col(1, \my $col1);
$sth->bind_col(2, \my $col2, {TYPE => SQL_BLOB});
$sth->execute;
my $expected = [undef, 0, 0, 0, 0, 0, 0];
for (1..6) {
$sth->fetch;
ok $col1 && $col1 == $_;
ok $col2 && utf8::is_utf8($col2) == $expected->[$_],
"row $_ is ".($expected->[$_] ? "unicode" : "not unicode");
}
$sth->finish;
}
t/rt_78833_utf8_flag_for_column_names.t view on Meta::CPAN
my $tests = 27;
plan( tests => $tests * 2 + 1 );
} else {
plan( skip_all => 'Unicode is not supported before 5.8.5' );
}
}
use Test::NoWarnings;
use Encode;
unicode_test("\x{263A}"); # (decoded) smiley character
unicode_test("\x{0100}"); # (decoded) capital A with macron
sub unicode_test {
my $unicode = shift;
ok Encode::is_utf8($unicode), "correctly decoded";
my $unicode_encoded = encode_utf8($unicode);
{ # tests for an environment where everything is encoded
my $dbh = connect_ok(sqlite_unicode => 0);
$dbh->do("pragma foreign_keys = on");
my $unicode_quoted = $dbh->quote_identifier($unicode_encoded);
$dbh->do("create table $unicode_quoted (id, $unicode_quoted primary key)");
$dbh->do("create table bar (id, ref references $unicode_quoted ($unicode_encoded))");
ok $dbh->do("insert into $unicode_quoted values (?, ?)", undef, 1, "text"), "insert successfully";
ok $dbh->do("insert into $unicode_quoted (id, $unicode_quoted) values (?, ?)", undef, 2, "text2"), "insert with unicode name successfully";
{
my $sth = $dbh->prepare("insert into $unicode_quoted (id) values (:$unicode_encoded)");
$sth->bind_param(":$unicode_encoded", 5);
$sth->execute;
my ($id) = $dbh->selectrow_array("select id from $unicode_quoted where id = :$unicode_encoded", undef, 5);
is $id => 5, "unicode placeholders";
}
{
my $sth = $dbh->prepare("select * from $unicode_quoted where id = ?");
$sth->execute(1);
my $row = $sth->fetchrow_hashref;
is $row->{id} => 1, "got correct row";
is $row->{$unicode_encoded} => "text", "got correct (encoded) unicode column data";
ok !exists $row->{$unicode}, "(decoded) unicode column does not exist";
}
{
my $sth = $dbh->prepare("select $unicode_quoted from $unicode_quoted where id = ?");
$sth->execute(1);
my $row = $sth->fetchrow_hashref;
is $row->{$unicode_encoded} => "text", "got correct (encoded) unicode column data";
ok !exists $row->{$unicode}, "(decoded) unicode column does not exist";
}
{
my $sth = $dbh->prepare("select id from $unicode_quoted where $unicode_quoted = ?");
$sth->execute("text");
my ($id) = $sth->fetchrow_array;
is $id => 1, "got correct id by the (encoded) unicode column value";
}
{
my $sth = $dbh->column_info(undef, undef, $unicode_encoded, $unicode_encoded);
my $column_info = $sth->fetchrow_hashref;
is $column_info->{COLUMN_NAME} => $unicode_encoded, "column_info returns the correctly encoded column name";
}
{
my $sth = $dbh->primary_key_info(undef, undef, $unicode_encoded);
my $primary_key_info = $sth->fetchrow_hashref;
is $primary_key_info->{COLUMN_NAME} => $unicode_encoded, "primary_key_info returns the correctly encoded primary key name";
}
{
my $sth = $dbh->table_info(undef, undef, $unicode_encoded);
my $table_info = $sth->fetchrow_hashref;
is $table_info->{TABLE_NAME} => $unicode_encoded, "table_info returns the correctly encoded table name";
}
}
{ # tests for an environment where everything is decoded
my $dbh = connect_ok(sqlite_unicode => 1);
$dbh->do("pragma foreign_keys = on");
my $unicode_quoted = $dbh->quote_identifier($unicode);
$dbh->do("create table $unicode_quoted (id, $unicode_quoted primary key)");
$dbh->do("create table bar (id, ref references $unicode_quoted ($unicode_quoted))");
ok $dbh->do("insert into $unicode_quoted values (?, ?)", undef, 1, "text"), "insert successfully";
ok $dbh->do("insert into $unicode_quoted (id, $unicode_quoted) values (?, ?)", undef, 2, "text2"), "insert with unicode name successfully";
{
my $sth = $dbh->prepare("insert into $unicode_quoted (id) values (:$unicode)");
$sth->bind_param(":$unicode", 5);
$sth->execute;
my ($id) = $dbh->selectrow_array("select id from $unicode_quoted where id = :$unicode", undef, 5);
is $id => 5, "unicode placeholders";
}
{
my $sth = $dbh->prepare("select * from $unicode_quoted where id = ?");
$sth->execute(1);
my $row = $sth->fetchrow_hashref;
is $row->{id} => 1, "got correct row";
is $row->{$unicode} => "text", "got correct (decoded) unicode column data";
ok !exists $row->{$unicode_encoded}, "(encoded) unicode column does not exist";
}
{
my $sth = $dbh->prepare("select $unicode_quoted from $unicode_quoted where id = ?");
$sth->execute(1);
my $row = $sth->fetchrow_hashref;
is $row->{$unicode} => "text", "got correct (decoded) unicode column data";
ok !exists $row->{$unicode_encoded}, "(encoded) unicode column does not exist";
}
{
my $sth = $dbh->prepare("select id from $unicode_quoted where $unicode_quoted = ?");
$sth->execute("text2");
my ($id) = $sth->fetchrow_array;
is $id => 2, "got correct id by the (decoded) unicode column value";
}
{
my $sth = $dbh->column_info(undef, undef, $unicode, $unicode);
my $column_info = $sth->fetchrow_hashref;
is $column_info->{COLUMN_NAME} => $unicode, "column_info returns the correctly decoded column name";
}
{
my $sth = $dbh->primary_key_info(undef, undef, $unicode);
my $primary_key_info = $sth->fetchrow_hashref;
is $primary_key_info->{COLUMN_NAME} => $unicode, "primary_key_info returns the correctly decoded primary key name";
}
{
my $sth = $dbh->table_info(undef, undef, $unicode);
my $table_info = $sth->fetchrow_hashref;
is $table_info->{TABLE_NAME} => $unicode, "table_info returns the correctly decoded table name";
}
}
}
t/rt_96877_unicode_statements.t view on Meta::CPAN
#!/usr/bin/perl
# According to the sqlite doc, the SQL argument to sqlite3_prepare_v2
# should be in utf8, but DBD::SQLeet does not ensure this (even with
# sqlite_unicode => 1). Only bind values are properly converted.
use strict;
BEGIN {
$| = 1;
$^W = 1;
}
use lib "t/lib";
use SQLeetTest;
t/rt_96877_unicode_statements.t view on Meta::CPAN
BEGIN {
if ( $] >= 5.008005 ) {
plan( tests => 16 );
} else {
plan( skip_all => 'Unicode is not supported before 5.8.5' );
}
}
use Test::NoWarnings;
my $dbh = connect_ok( sqlite_unicode => 1 );
is( $dbh->{sqlite_unicode}, 1, 'Unicode is on' );
ok( $dbh->do(<<'END_SQL'), 'CREATE TABLE' );
CREATE TABLE foo (
bar varchar(255)
)
END_SQL
foreach ( "A", "\xe9", "\x{20ac}" ) {
note sprintf "testing \\x{%x}", ord($_);
ok( $dbh->do("INSERT INTO foo VALUES ( ? )", {}, $_), 'INSERT with bind' );
( run in 0.682 second using v1.01-cache-2.11-cpan-88abd93f124 )