SWISH-3
view release on metacpan or search on metacpan
libswish3.c view on Meta::CPAN
struct swish_TagStack
{
swish_Tag *head;
swish_Tag *temp;
unsigned int count;
char *name; // debugging aid -- name of the stack
};
struct swish_Analyzer
{
unsigned int maxwordlen; // max word length
unsigned int minwordlen; // min word length
boolean tokenize; // should we parse into TokenList
int (*tokenizer) (swish_TokenIterator*, xmlChar*, swish_MetaName*, xmlChar*);
xmlChar* (*stemmer) (xmlChar*);
boolean lc; // should tokens be lowercased
void *stash; // for script bindings
void *regex; // optional regex
int ref_cnt; // for script bindings
};
struct swish_Parser
{
int ref_cnt; // for script bindings
void (*handler)(swish_ParserData*); // handler reference
void *stash; // for script bindings
int verbosity;
};
struct swish_ParserData
{
swish_3 *s3; // main object
xmlBufferPtr meta_buf; // tmp MetaName buffer
xmlBufferPtr prop_buf; // tmp Property buffer
xmlChar *tag; // current tag name
swish_DocInfo *docinfo; // document-specific properties
unsigned int ignore_content; // toggle flag. should buffer be indexed.
boolean is_html; // shortcut flag for html parser
boolean bump_word; // boolean for moving word position/adding space
unsigned int offset; // current offset position
swish_TagStack *metastack; // stacks for tracking the tag => metaname
swish_TagStack *propstack; // stacks for tracking the tag => property
swish_TagStack *domstack; // stacks for tracking xml/html dom tree
xmlParserCtxtPtr ctxt; // so we can free at end
swish_TokenIterator *token_iterator; // token container
swish_NamedBuffer *properties; // buffer all properties
swish_NamedBuffer *metanames; // buffer all metanames
};
/*
=cut
*/
/*
=head2 Global Functions
*/
void swish_setup();
const char * swish_lib_version();
const char * swish_libxml2_version();
void swish_setenv(char * name, char * value, int override);
/*
=cut
*/
/*
=head2 Top-Level Functions
*/
swish_3 * swish_3_init( void (*handler) (swish_ParserData *), void *stash );
void swish_3_free( swish_3 *s3 );
int swish_parse_file( swish_3 * s3, xmlChar *filename );
unsigned int swish_parse_fh( swish_3 * s3, FILE * fh );
int swish_parse_buffer( swish_3 * s3, xmlChar * buf );
unsigned int swish_parse_directory( swish_3 *s3, xmlChar *dir, boolean follow_symlinks );
/*
=cut
*/
/*
=head2 I/O Functions
*/
xmlChar * swish_io_slurp_fh( FILE * fh, unsigned long flen, boolean binmode );
xmlChar * swish_io_slurp_file_len( xmlChar *filename, off_t flen, boolean binmode );
xmlChar * swish_io_slurp_gzfile_len( xmlChar *filename, off_t *flen, boolean binmode );
xmlChar * swish_io_slurp_file( xmlChar *filename, off_t flen, boolean is_gzipped, boolean binmode );
long int swish_io_count_operable_file_lines( xmlChar *filename );
boolean swish_io_is_skippable_line( xmlChar *str );
/*
=cut
*/
/*
=head2 Filesystem Functions
*/
boolean swish_fs_file_exists( xmlChar *filename );
boolean swish_fs_is_dir( xmlChar *path );
boolean swish_fs_is_file( xmlChar *path );
boolean swish_fs_is_link( xmlChar *path );
off_t swish_fs_get_file_size( xmlChar *path );
time_t swish_fs_get_file_mtime( xmlChar *path );
xmlChar * swish_fs_get_file_ext( xmlChar *url );
xmlChar * swish_fs_get_path( xmlChar *url );
boolean swish_fs_looks_like_gz( xmlChar *file );
/*
=cut
*/
/*
=head2 Hash Functions
*/
int swish_hash_add( xmlHashTablePtr hash, xmlChar *key, void * value );
int swish_hash_replace( xmlHashTablePtr hash, xmlChar *key, void *value );
int swish_hash_delete( xmlHashTablePtr hash, xmlChar *key );
boolean swish_hash_exists( xmlHashTablePtr hash, xmlChar *key );
int swish_hash_exists_or_add( xmlHashTablePtr hash, xmlChar *key, xmlChar *value );
void swish_hash_merge( xmlHashTablePtr hash1, xmlHashTablePtr hash2 );
void * swish_hash_fetch( xmlHashTablePtr hash, xmlChar *key );
void swish_hash_dump( xmlHashTablePtr hash, const char *label );
xmlHashTablePtr swish_hash_init(int size);
void swish_hash_free( xmlHashTablePtr hash );
libswish3.c view on Meta::CPAN
swish_xfree(info->mime);
info->mime = swish_xstrdup(val);
continue;
}
if (!xmlStrncasecmp(line, (const xmlChar *)"Encoding", 8)
|| !xmlStrncasecmp(line, (const xmlChar *)"Charset", 7)) {
if (!val)
SWISH_WARN("Failed to parse Encoding or Charset header '%s'", line);
if (!*val)
SWISH_WARN("Failed to find value in Encoding or Charset header '%s'",
line);
if (info->encoding != NULL)
swish_xfree(info->encoding);
info->encoding = swish_xstrdup(val);
continue;
}
if (!xmlStrncasecmp(line, (const xmlChar *)"Action", 11)) {
if (!val)
SWISH_WARN("Failed to parse Action header '%s'", line);
if (!*val)
SWISH_WARN("Failed to find value in Action header '%s'", line);
if (info->action != NULL)
swish_xfree(info->action);
info->action = swish_xstrdup(val);
continue;
}
/*
* if we get here, unrecognized header line
*/
SWISH_WARN("Unknown header line: '%s'\n", line);
}
if (SWISH_DEBUG & SWISH_DEBUG_DOCINFO) {
SWISH_DEBUG_MSG("returning %d header lines", h->nlines);
swish_docinfo_debug(info);
}
return info;
}
static void
get_env_vars(
)
{
/*
* init the global env vars, but don't override if already set
*/
swish_setenv("SWISH_PARSER_WARNINGS", "1", 0);
SWISH_PARSER_WARNINGS = swish_string_to_int(getenv("SWISH_PARSER_WARNINGS"));
if (SWISH_DEBUG) {
SWISH_PARSER_WARNINGS = SWISH_DEBUG;
}
}
unsigned int
swish_parse_fh(
swish_3 *s3,
FILE * fh
)
{
xmlChar *ln;
HEAD *head;
int i;
xmlChar *read_buffer;
xmlBufferPtr head_buf;
swish_ParserData *parser_data;
int xmlErr;
int min_headers, nheaders;
double curTime;
char *etime;
unsigned int file_cnt;
i = 0;
file_cnt = 0;
nheaders = 0;
min_headers = 2;
if (fh == NULL)
fh = stdin;
ln = swish_xmalloc(SWISH_MAXSTRLEN + 1);
head_buf =
xmlBufferCreateSize((SWISH_MAX_HEADERS * SWISH_MAXSTRLEN) + SWISH_MAX_HEADERS);
/*
* based on extprog.c
*/
while (fgets((char *)ln, SWISH_MAXSTRLEN, fh) != NULL) {
/*
* we don't use fgetws() because we don't care about * indiv
* characters yet
*/
xmlChar *end;
xmlChar *line;
line = swish_str_skip_ws(ln); /* skip leading white space */
end = (xmlChar *)strrchr((char *)line, '\n');
/*
* trim any white space at end of doc, including \n
*/
if (end) {
while (end > line && isspace((int)*(end - 1)))
end--;
libswish3.c view on Meta::CPAN
*/
setlocale(LC_ALL, "");
/* fetch the current locale so we can test it */
locale = setlocale(LC_ALL, NULL);
if (locale == NULL || !strlen(locale)) {
//SWISH_DEBUG_MSG("locale for LC_ALL was null");
/* check LC_CTYPE specifically:
* http://mail.nl.linux.org/linux-utf8/2001-09/msg00030.html
*/
locale = setlocale(LC_CTYPE, NULL);
if (locale == NULL || !strlen(locale)) {
//SWISH_DEBUG_MSG("locale for LC_CTYPE was null");
locale = getenv("LANG");
if (locale == NULL || !strlen(locale)) {
//SWISH_DEBUG_MSG("getenv for LANG was null");
locale = SWISH_LOCALE;
}
}
}
/* Must re-set LC_NUMERIC to portable C
* This is due to behavior of Perl > v5.18
*/
setlocale(LC_NUMERIC, "C");
return locale;
}
void
swish_verify_utf8_locale(
)
{
char *loc;
const xmlChar *enc;
/* a bit about encodings: libxml2 takes whatever encoding the input XML is
* (latin1, ascii, utf8, etc) and standardizes it using iconv (or other) in xmlChar as
* UTF-8. However, we must ensure we have UTF-8 locale because all the mb* and wc*
* routines rely on the locale to correctly interpret chars.
*
* See also
* http://www.cl.cam.ac.uk/~mgk25/unicode.html#c
* and
* http://www.cl.cam.ac.uk/~mgk25/unicode.html#activate
*/
loc = swish_get_locale();
enc = xmlStrchr((xmlChar *)loc, (xmlChar)'.');
if (enc != NULL) {
enc++;
if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
SWISH_DEBUG_MSG("encoding = %s", enc);
}
else {
if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
SWISH_DEBUG_MSG("no encoding in %s, using %s", loc, SWISH_DEFAULT_ENCODING);
enc = (xmlChar *)SWISH_DEFAULT_ENCODING;
}
swish_setenv("SWISH_ENCODING", (char *)enc, 0); /* remember in env var, if not already set */
if (!loc) {
SWISH_WARN("can't get locale via setlocale()");
}
else if (SWISH_DEBUG) {
SWISH_DEBUG_MSG("current locale and encoding: %s %s", loc, enc);
}
if (u8_is_locale_utf8(loc)) {
/* a-ok */
if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
SWISH_DEBUG_MSG("locale looks like UTF-8");
}
else {
/* must be UTF-8 charset since libxml2 converts everything to UTF-8 */
if (SWISH_DEBUG)
SWISH_DEBUG_MSG
("Your locale (%s) was not UTF-8 so internally we are using %s", loc,
SWISH_LOCALE);
if (!setlocale(LC_CTYPE, SWISH_LOCALE)) {
SWISH_WARN("failed to set locale to %s from %s", SWISH_LOCALE, loc);
}
}
if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
SWISH_DEBUG_MSG("active locale is %s", setlocale(LC_CTYPE, NULL));
}
xmlChar *
swish_str_escape_utf8(
xmlChar *u8str
)
{
xmlChar *escaped;
int u8chrs, n_escaped, esc_len;
u8chrs = swish_utf8_num_chrs(u8str);
/* 10 is the max number of ascii chars needed to represent a utf8 chr:
* \Uxxxxxxxx
* 1234567890
*/
esc_len = (10*u8chrs)+1; /* +1 == nul */
escaped = swish_xmalloc(esc_len);
/*
SWISH_DEBUG_MSG("escaping %s len %d for '%s'",
escaped, esc_len, u8str);
*/
n_escaped = u8_escape((char*)escaped, esc_len, (char*)u8str, 0); // TODO quotes?
return escaped;
}
xmlChar *
libswish3.c view on Meta::CPAN
const char *
swish_libxml2_version(
)
{
return (const char*)LIBXML_DOTTED_VERSION;
}
swish_3 *
swish_3_init(
void (*handler) (swish_ParserData *),
void *stash
)
{
swish_3 *s3;
s3 = swish_xmalloc(sizeof(swish_3));
s3->ref_cnt = 0;
s3->config = swish_config_init();
s3->config->ref_cnt++;
swish_config_set_default(s3->config);
s3->analyzer = swish_analyzer_init(s3->config);
s3->analyzer->ref_cnt++;
s3->parser = swish_parser_init(handler);
s3->parser->ref_cnt++;
s3->stash = stash;
if (SWISH_DEBUG & SWISH_DEBUG_MEMORY) {
SWISH_DEBUG_MSG("s3 ptr 0x%lx", s3);
}
return s3;
}
void
swish_3_free(
swish_3 *s3
)
{
s3->parser->ref_cnt--;
if (s3->parser->ref_cnt < 1) {
swish_parser_free(s3->parser);
}
s3->analyzer->ref_cnt--;
if (s3->analyzer->ref_cnt < 1) {
swish_analyzer_free(s3->analyzer);
}
s3->config->ref_cnt--;
if (s3->config->ref_cnt < 1) {
swish_config_free(s3->config);
}
if (s3->ref_cnt != 0) {
SWISH_WARN("s3 ref_cnt != 0: %d\n", s3->ref_cnt);
}
swish_xfree(s3);
}
void
swish_setenv(
char * name,
char * value,
int override
)
{
int ret;
ret = setenv(name, value, override);
if (ret != 0) {
SWISH_CROAK("setenv failed with %d: %s", errno, strerror(errno));
}
}
/* MUST call this before instantiating any swish_3 objects */
void
swish_setup(
)
{
/* global var that scripts can check to determine what version of Swish they are
* using. the second 0 indicates that it will not override it if already set */
swish_setenv("SWISH3", "1", 0);
/* global debug flag */
swish_setenv("SWISH_DEBUG", "0", 0);
swish_setenv("SWISH_DEBUG_MEMORY", "0", 0);
swish_setenv("SWISH_DEBUG_CONFIG", "0", 0);
swish_setenv("SWISH_DEBUG_DOCINFO", "0", 0);
swish_setenv("SWISH_DEBUG_IO", "0", 0);
swish_setenv("SWISH_DEBUG_TOKENLIST", "0", 0);
swish_setenv("SWISH_DEBUG_TOKENIZER", "0", 0);
swish_setenv("SWISH_DEBUG_PARSER", "0", 0);
swish_setenv("SWISH_DEBUG_NAMEDBUFFER", "0", 0);
swish_setenv("SWISH_WARNINGS", "1", 0);
if (!SWISH_DEBUG) {
SWISH_DEBUG += swish_string_to_int(getenv("SWISH_DEBUG"));
/* additional env vars just increase the global var value */
if (swish_string_to_int(getenv("SWISH_DEBUG_MEMORY"))) {
SWISH_DEBUG += SWISH_DEBUG_MEMORY;
}
if (swish_string_to_int(getenv("SWISH_DEBUG_CONFIG"))) {
SWISH_DEBUG += SWISH_DEBUG_CONFIG;
}
if (swish_string_to_int(getenv("SWISH_DEBUG_DOCINFO"))) {
SWISH_DEBUG += SWISH_DEBUG_DOCINFO;
}
if (swish_string_to_int(getenv("SWISH_DEBUG_TOKENLIST"))) {
SWISH_DEBUG += SWISH_DEBUG_TOKENLIST;
}
if (swish_string_to_int(getenv("SWISH_DEBUG_TOKENIZER"))) {
SWISH_DEBUG += SWISH_DEBUG_TOKENIZER;
}
if (swish_string_to_int(getenv("SWISH_DEBUG_PARSER"))) {
SWISH_DEBUG += SWISH_DEBUG_PARSER;
}
if (swish_string_to_int(getenv("SWISH_DEBUG_NAMEDBUFFER"))) {
SWISH_DEBUG += SWISH_DEBUG_NAMEDBUFFER;
}
if (swish_string_to_int(getenv("SWISH_DEBUG_IO"))) {
SWISH_DEBUG += SWISH_DEBUG_IO;
}
/* special value to turn on all debugging */
if (SWISH_DEBUG == -1) {
SWISH_DEBUG += SWISH_DEBUG_MEMORY;
SWISH_DEBUG += SWISH_DEBUG_CONFIG;
SWISH_DEBUG += SWISH_DEBUG_DOCINFO;
SWISH_DEBUG += SWISH_DEBUG_TOKENLIST;
SWISH_DEBUG += SWISH_DEBUG_TOKENIZER;
SWISH_DEBUG += SWISH_DEBUG_PARSER;
SWISH_DEBUG += SWISH_DEBUG_NAMEDBUFFER;
SWISH_DEBUG += SWISH_DEBUG_IO;
}
if (SWISH_DEBUG) {
SWISH_DEBUG_MSG("SWISH_DEBUG set to %d", SWISH_DEBUG);
}
}
SWISH_WARNINGS = swish_string_to_int(getenv("SWISH_WARNINGS"));
/*
* initialize the library and check potential API mismatches
* between the version it was compiled for and the actual shared
* library used.
*/
LIBXML_TEST_VERSION
swish_mem_init();
swish_verify_utf8_locale();
}
( run in 0.925 second using v1.01-cache-2.11-cpan-6aa56a78535 )