Algorithm-MinPerfHashTwoLevel
view release on metacpan or search on metacpan
MinPerfHashTwoLevel.xs view on Meta::CPAN
#define PERL_NO_GET_CONTEXT
#include "EXTERN.h"
#include "perl.h"
#include "XSUB.h"
#define NEED_newRV_noinc
#define NEED_sv_2pv_flags
#include "ppport.h"
#include <sys/mman.h>
#include <sys/types.h>
#include <fcntl.h>
#include <unistd.h>
#include <sys/stat.h>
#include <assert.h>
#include "mph2l.h"
#include "mph_hv_macro.h"
#include "mph_siphash.h"
#define MAX_VARIANT 5
#define MIN_VARIANT 5
MPH_STATIC_INLINE void
sv_set_from_bucket(pTHX_ SV *sv, U8 *strs, const U32 ofs, const U32 len, const U32 idx, const U8 *flags, const U32 bits, const U8 utf8_default, const U8 utf8_default_shift) {
U8 *ptr;
U8 is_utf8;
if (ofs) {
ptr= (strs) + (ofs);
if (utf8_default) {
is_utf8= utf8_default >> utf8_default_shift;
} else {
GETBITS(is_utf8,flags,idx,bits);
}
} else {
ptr= 0;
is_utf8= 0;
}
/* note that sv_setpvn() will cause the sv to
* become undef if ptr is 0 */
sv_setpvn_mg((sv),ptr,len);
if (is_utf8 > 1) {
sv_utf8_upgrade(sv);
MinPerfHashTwoLevel.xs view on Meta::CPAN
SvUTF8_off(sv);
}
}
MPH_STATIC_INLINE int
lookup_bucket(pTHX_ struct mph_header *mph, U32 index, SV *key_sv, SV *val_sv)
{
struct mph_bucket *bucket;
U8 *strs;
U8 *mph_u8= (U8*)mph;
U64 gf= mph->general_flags;
if (index >= mph->num_buckets) {
return 0;
}
bucket= (struct mph_bucket *)((char *)mph + mph->table_ofs) + index;
strs= (U8 *)mph + mph->str_buf_ofs;
if (val_sv) {
sv_set_from_bucket(aTHX_ val_sv,strs,bucket->val_ofs,bucket->val_len,index,mph_u8 + mph->val_flags_ofs,1,
gf & MPH_VALS_ARE_SAME_UTF8NESS_MASK, MPH_VALS_ARE_SAME_UTF8NESS_SHIFT);
}
if (key_sv) {
sv_set_from_bucket(aTHX_ key_sv,strs,bucket->key_ofs,bucket->key_len,index,mph_u8 + mph->key_flags_ofs,2,
gf & MPH_KEYS_ARE_SAME_UTF8NESS_MASK, MPH_KEYS_ARE_SAME_UTF8NESS_SHIFT);
}
return 1;
}
MPH_STATIC_INLINE int
lookup_key(pTHX_ struct mph_header *mph, SV *key_sv, SV *val_sv)
{
U8 *strs= (U8 *)mph + mph->str_buf_ofs;
struct mph_bucket *buckets= (struct mph_bucket *) ((char *)mph + mph->table_ofs);
MinPerfHashTwoLevel.xs view on Meta::CPAN
h2= h0 & 0xFFFFFFFF;
if ( bucket->index < 0 ) {
index = -bucket->index-1;
} else {
HASH2INDEX(index,h2,bucket->xor_val,mph->num_buckets);
}
bucket= buckets + index;
got_key_pv= strs + bucket->key_ofs;
if (bucket->key_len == key_len && memEQ(key_pv,got_key_pv,key_len)) {
if (val_sv) {
U64 gf= mph->general_flags;
sv_set_from_bucket(aTHX_ val_sv,strs,bucket->val_ofs,bucket->val_len,index,((U8*)mph)+mph->val_flags_ofs,1,
gf & MPH_VALS_ARE_SAME_UTF8NESS_MASK, MPH_VALS_ARE_SAME_UTF8NESS_SHIFT);
}
return 1;
}
return 0;
}
IV
mph_mmap(pTHX_ char *file, struct mph_obj *obj, SV *error, U32 flags) {
struct stat st;
struct mph_header *head;
int fd = open(file, O_RDONLY, 0);
void *ptr;
U32 alignment;
if (error)
sv_setpvs(error,"");
if (fd < 0) {
if (error)
MinPerfHashTwoLevel.xs view on Meta::CPAN
}
alignment = sizeof(U64);
if (st.st_size % alignment) {
if (error)
sv_setpvf(error,"file '%s' does not have a size which is a multiple of 16 bytes", file);
return MPH_MOUNT_ERROR_BAD_SIZE;
}
if (
head->table_ofs < head->state_ofs ||
head->key_flags_ofs < head->table_ofs ||
head->val_flags_ofs < head->key_flags_ofs ||
head->str_buf_ofs < head->val_flags_ofs ||
st.st_size < head->str_buf_ofs
) {
if (error)
sv_setpvf(error,"corrupt header offsets in '%s'", file);
return MPH_MOUNT_ERROR_BAD_OFFSETS;
}
if (flags & MPH_F_VALIDATE) {
char *start= ptr;
char *state_pv= start + head->state_ofs;
char *str_buf_start= start + head->str_buf_ofs;
char *str_buf_end= start + st.st_size;
U64 have_file_checksum= mph_hash_with_state(state_pv, start, st.st_size - sizeof(U64));
U64 want_file_checksum= *((U64 *)(str_buf_end - sizeof(U64)));
if (have_file_checksum != want_file_checksum) {
if (error)
sv_setpvf(error,"file checksum '%016lx' != '%016lx' in file '%s'",
MinPerfHashTwoLevel.xs view on Meta::CPAN
}
return head->variant;
}
void
mph_munmap(struct mph_obj *obj) {
munmap(obj->header,obj->bytes);
}
STRLEN
normalize_with_flags(pTHX_ SV *sv, SV *normalized_sv, SV *is_utf8_sv, int downgrade) {
STRLEN len;
if (SvROK(sv)) {
croak("Error: Not expecting a reference value in source hash");
}
sv_setsv(normalized_sv,sv);
if (SvOK(sv)) {
STRLEN pv_len;
char *pv= SvPV(sv,pv_len);
if (pv_len > 0xFFFF)
croak("Error: String in source hash is too long to store, max length is %u got length %lu", 0xFFFF, pv_len);
MinPerfHashTwoLevel.xs view on Meta::CPAN
I32
_compare(pTHX_ SV *a, SV *b) {
dMY_CXT;
HE *a_he= hv_fetch_ent_with_keysv((HV*)SvRV(a),MPH_KEYSV_KEY_NORMALIZED,0);
HE *b_he= hv_fetch_ent_with_keysv((HV*)SvRV(b),MPH_KEYSV_KEY_NORMALIZED,0);
return sv_cmp(HeVAL(a_he),HeVAL(b_he));
}
U32
normalize_source_hash(pTHX_ HV *source_hv, AV *keys_av, U32 compute_flags, SV *buf_length_sv, char *state_pv) {
dMY_CXT;
HE *he;
U32 buf_length= 0;
U32 ctr;
hv_iterinit(source_hv);
while (he= hv_iternext(source_hv)) {
SV *val_sv= HeVAL(he);
SV *val_normalized_sv;
SV *val_is_utf8_sv;
SV *key_sv;
SV *key_normalized_sv;
SV *key_is_utf8_sv;
HV *hv;
U8 *key_pv;
STRLEN key_len;
U64 h0;
if (!val_sv) croak("panic: no sv for value?");
if (!SvOK(val_sv) && (compute_flags & MPH_F_FILTER_UNDEF)) continue;
hv= newHV();
val_normalized_sv= newSV(0);
val_is_utf8_sv= newSVuv(0);
key_sv= newSVhek(HeKEY_hek(he));
key_normalized_sv= newSV(0);
key_is_utf8_sv= newSVuv(0);
hv_ksplit(hv,15);
hv_store_ent_with_keysv(hv,MPH_KEYSV_KEY, key_sv);
hv_store_ent_with_keysv(hv,MPH_KEYSV_KEY_NORMALIZED, key_normalized_sv);
hv_store_ent_with_keysv(hv,MPH_KEYSV_KEY_IS_UTF8, key_is_utf8_sv);
hv_store_ent_with_keysv(hv,MPH_KEYSV_VAL, SvREFCNT_inc_simple_NN(val_sv));
hv_store_ent_with_keysv(hv,MPH_KEYSV_VAL_NORMALIZED, val_normalized_sv);
hv_store_ent_with_keysv(hv,MPH_KEYSV_VAL_IS_UTF8, val_is_utf8_sv);
/* install everything into the keys_av just in case normalize_with_flags() dies */
av_push(keys_av,newRV_noinc((SV*)hv));
buf_length += normalize_with_flags(aTHX_ key_sv, key_normalized_sv, key_is_utf8_sv, 1);
buf_length += normalize_with_flags(aTHX_ val_sv, val_normalized_sv, val_is_utf8_sv, 0);
key_pv= (U8 *)SvPV(key_normalized_sv,key_len);
h0= mph_hash_with_state(state_pv,key_pv,key_len);
hv_store_ent_with_keysv(hv,MPH_KEYSV_H0, newSVuv(h0));
}
if (buf_length_sv)
sv_setuv(buf_length_sv, buf_length);
/* we now know how many keys there are, and what the max_xor_val should be */
MinPerfHashTwoLevel.xs view on Meta::CPAN
#define MY_CXT_KEY "Algorithm::MinPerfHashTwoLevel::_stash" XS_VERSION
#define SETOFS(i,he,table,key_ofs,key_len,str_buf_start,str_buf_pos,str_buf_end,str_ofs_hv) \
STMT_START { \
if (he) { \
SV *sv= HeVAL(he); \
if (SvOK(sv)) { \
STRLEN pv_len; \
char *pv; \
SV *ofs_sv; \
if (flags & MPH_F_NO_DEDUPE) { \
ofs_sv= NULL; \
} else { \
HE *ofs= hv_fetch_ent(str_ofs_hv,sv,1,0); \
ofs_sv= ofs ? HeVAL(ofs) : NULL; \
if (!ofs_sv) \
croak("panic: out of memory getting str ofs for " #he "for %u",i); \
} \
if (ofs_sv && SvOK(ofs_sv)){ \
table[i].key_ofs= SvUV(ofs_sv); \
table[i].key_len= sv_len(sv); \
MinPerfHashTwoLevel.xs view on Meta::CPAN
STRLEN state_len;
HE *he;
IV len_idx;
U32 bucket_count;
U32 max_xor_val;
U32 i;
U32 variant;
U32 compute_flags;
SV* buf_length_sv;
HV* source_hv;
AV *buckets_av;
AV *keys_av;
AV *by_length_av;
AV *keybuckets_av;
AV *h2_packed_av;
MinPerfHashTwoLevel.xs view on Meta::CPAN
he= hv_fetch_ent_with_keysv(self_hv,MPH_KEYSV_VARIANT,0);
if (he) {
variant= SvUV(HeVAL(he));
} else {
croak("panic: no variant in self?");
}
he= hv_fetch_ent_with_keysv(self_hv,MPH_KEYSV_COMPUTE_FLAGS,0);
if (he) {
compute_flags= SvUV(HeVAL(he));
} else {
croak("panic: no compute_flags in self?");
}
he= hv_fetch_ent_with_keysv(self_hv,MPH_KEYSV_STATE,0);
if (he) {
SV *state_sv= HeVAL(he);
state_pv= (U8 *)SvPV(state_sv,state_len);
if (state_len != MPH_STATE_BYTES) {
croak("Error: state vector must be at exactly %d bytes",(int)MPH_SEED_BYTES);
}
} else {
MinPerfHashTwoLevel.xs view on Meta::CPAN
}
buckets_av= newAV();
SvRV_set(rv,(SV*)buckets_av);
SvROK_on(rv);
} else {
croak("panic: out of memory in lvalue fetch for 'buckets' in self");
}
/**** build an array of hashes in keys_av based on the normalized contents of source_hv */
keys_av= (AV *)sv_2mortal((SV*)newAV());
bucket_count= normalize_source_hash(aTHX_ source_hv, keys_av, compute_flags, buf_length_sv, state_pv);
max_xor_val= INT32_MAX;
/* if the caller wants deterministic results we sort the keys_av
* before we compute collisions - depending on the order we process
* the keys we might resolve the collisions differently */
if (compute_flags & MPH_F_DETERMINISTIC)
sortsv(AvARRAY(keys_av),bucket_count,_compare);
/**** find the collisions from the data we just computed, build an AoAoH and AoS of the
**** collision data */
keybuckets_av= (AV*)sv_2mortal((SV*)newAV()); /* AoAoH - hashes from keys_av */
h2_packed_av= (AV*)sv_2mortal((SV*)newAV()); /* AoS - packed h1 */
find_first_level_collisions(aTHX_ bucket_count, keys_av, keybuckets_av, h2_packed_av);
/* Sort the buckets by size by constructing an AoS, with the outer array indexed by length,
* and the inner string being the list of items of that length. (Thus the contents of index
MinPerfHashTwoLevel.xs view on Meta::CPAN
variant, buckets_av);
}
OUTPUT:
RETVAL
MODULE = Algorithm::MinPerfHashTwoLevel PACKAGE = Tie::Hash::MinPerfHashTwoLevel::OnDisk
SV *
packed_xs(variant,buf_length_sv,state_sv,comment_sv,flags,buckets_av)
U32 variant
SV* buf_length_sv
SV* state_sv
SV* comment_sv
AV *buckets_av
U32 flags
PREINIT:
dMY_CXT;
PROTOTYPE: $$$$$\@
CODE:
{
U32 buf_length= SvUV(buf_length_sv);
U32 bucket_count= av_top_index(buckets_av) + 1;
U32 header_rlen= _roundup(sizeof(struct mph_header),16);
STRLEN state_len;
char *state_pv= SvPV(state_sv, state_len);
U32 alignment= sizeof(U64);
U32 state_rlen= _roundup(state_len,alignment);
U32 table_rlen= _roundup(sizeof(struct mph_bucket) * bucket_count,alignment);
U32 key_flags_rlen= _roundup((bucket_count * 2 + 7 ) / 8,alignment);
U32 val_flags_rlen= _roundup((bucket_count + 7) / 8,alignment);
U32 str_rlen= _roundup( buf_length
+ 2
+ ( SvOK(comment_sv) ? sv_len(comment_sv) + 1 : 1 )
+ ( 2 + 8 ),
alignment );
U32 total_size;
HV *str_ofs_hv= (HV *)sv_2mortal((SV*)newHV());
SV *sv_buf;
char *start;
struct mph_header *head;
char *state;
struct mph_bucket *table;
char *key_flags;
char *val_flags;
char *str_buf_start;
char *str_buf_end;
char *str_buf_pos;
U32 i;
STRLEN pv_len;
char *pv;
U32 key_is_utf8_count[3]={0,0,0};
U32 val_is_utf8_count[2]={0,0};
U32 used_flags;
U32 the_flag;
IV key_is_utf8_generic=-1;
IV val_is_utf8_generic=-1;
for (i= 0; i < bucket_count; i++) {
SV **got= av_fetch(buckets_av,i,0);
HV *hv= (HV *)SvRV(*got);
HE *key_is_utf8_he= hv_fetch_ent_with_keysv(hv,MPH_KEYSV_KEY_IS_UTF8,0);
HE *val_is_utf8_he= hv_fetch_ent_with_keysv(hv,MPH_KEYSV_VAL_IS_UTF8,0);
key_is_utf8_count[SvUV(HeVAL(key_is_utf8_he))]++;
val_is_utf8_count[SvUV(HeVAL(val_is_utf8_he))]++;
}
used_flags= 0;
if (key_is_utf8_count[0]) { the_flag= 0; used_flags++; }
if (key_is_utf8_count[1]) { the_flag= 1; used_flags++; }
if (key_is_utf8_count[2]) { the_flag= 2; used_flags++; }
if (used_flags == 1) {
key_is_utf8_generic= the_flag;
key_flags_rlen= 0;
}
used_flags= 0;
if (val_is_utf8_count[0]) { the_flag= 0; used_flags++; }
if (val_is_utf8_count[1]) { the_flag= 1; used_flags++; }
if (used_flags == 1) {
val_is_utf8_generic= the_flag;
val_flags_rlen= 0;
}
total_size=
+ header_rlen
+ state_rlen
+ table_rlen
+ key_flags_rlen
+ val_flags_rlen
+ str_rlen
;
sv_buf= newSV(total_size);
SvPOK_on(sv_buf);
SvCUR_set(sv_buf,total_size);
start= SvPVX(sv_buf);
Zero(start,total_size,char);
head= (struct mph_header *)start;
head->magic_num= 1278363728;
head->variant= variant;
head->num_buckets= bucket_count;
head->state_ofs= header_rlen;
head->table_ofs= head->state_ofs + state_rlen;
head->key_flags_ofs= head->table_ofs + table_rlen;
head->val_flags_ofs= head->key_flags_ofs + key_flags_rlen;
head->str_buf_ofs= head->val_flags_ofs + val_flags_rlen;
if (val_is_utf8_generic >= 0)
head->general_flags |= (MPH_VALS_ARE_SAME_UTF8NESS_FLAG_BIT | (val_is_utf8_generic << MPH_VALS_ARE_SAME_UTF8NESS_SHIFT));
if (key_is_utf8_generic >= 0)
head->general_flags |= (MPH_KEYS_ARE_SAME_UTF8NESS_FLAG_BIT | (key_is_utf8_generic << MPH_KEYS_ARE_SAME_UTF8NESS_SHIFT));
state= start + head->state_ofs;
table= (struct mph_bucket *)(start + head->table_ofs);
key_flags= start + head->key_flags_ofs;
val_flags= start + head->val_flags_ofs;
str_buf_start= start + head->str_buf_ofs;
str_buf_end= start + total_size;
str_buf_pos= str_buf_start + 2;
Copy(state_pv,state,state_len,char);
pv= SvPV(comment_sv,pv_len);
Copy(pv,str_buf_pos,pv_len,char);
str_buf_pos += pv_len + 1; /* +1 to add a trailing null */
for (i= 0; i < bucket_count; i++) {
MinPerfHashTwoLevel.xs view on Meta::CPAN
table[i].xor_val= SvUV(HeVAL(xor_val_he));
} else {
table[i].xor_val= 0;
}
SETOFS(i,key_normalized_he,table,key_ofs,key_len,str_buf_start,str_buf_pos,str_buf_end,str_ofs_hv);
SETOFS(i,val_normalized_he,table,val_ofs,val_len,str_buf_start,str_buf_pos,str_buf_end,str_ofs_hv);
if ( key_is_utf8_generic < 0) {
HE *key_is_utf8_he= hv_fetch_ent_with_keysv(hv,MPH_KEYSV_KEY_IS_UTF8,0);
if (key_is_utf8_he) {
UV u= SvUV(HeVAL(key_is_utf8_he));
SETBITS(u,key_flags,i,2);
} else {
croak("panic: out of memory? no key_is_utf8_he for %u",i);
}
}
if ( val_is_utf8_generic < 0 ) {
HE *val_is_utf8_he= hv_fetch_ent_with_keysv(hv,MPH_KEYSV_VAL_IS_UTF8,0);
if (val_is_utf8_he) {
UV u= SvUV(HeVAL(val_is_utf8_he));
SETBITS(u,val_flags,i,1);
} else {
croak("panic: out of memory? no val_is_utf8_he for %u",i);
}
}
}
*str_buf_pos = 0; str_buf_pos++;
*str_buf_pos = 128; str_buf_pos++;
{
U32 r= (str_buf_pos - start) % alignment;
if (r) {
MinPerfHashTwoLevel.xs view on Meta::CPAN
str_buf_pos += sizeof(U64);
SvCUR_set(sv_buf, str_buf_pos - start);
SvPOK_on(sv_buf);
RETVAL= sv_buf;
}
OUTPUT:
RETVAL
SV*
mount_file(file_sv,error_sv,flags)
SV* file_sv
SV* error_sv
U32 flags
PROTOTYPE: $$$
CODE:
{
struct mph_obj obj;
STRLEN file_len;
char *file_pv= SvPV(file_sv,file_len);
IV mmap_status= mph_mmap(aTHX_ file_pv, &obj, error_sv, flags);
if (mmap_status < 0) {
XSRETURN_UNDEF;
}
/* copy obj into a new SV which we can return */
RETVAL= newSVpvn((char *)&obj,sizeof(struct mph_obj));
SvPOK_on(RETVAL);
SvREADONLY_on(RETVAL);
}
OUTPUT:
RETVAL
MinPerfHashTwoLevel.xs view on Meta::CPAN
SV *
get_comment(self_hv)
HV* self_hv
ALIAS:
get_hdr_magic_num = 1
get_hdr_variant = 2
get_hdr_num_buckets = 3
get_hdr_state_ofs = 4
get_hdr_table_ofs = 5
get_hdr_key_flags_ofs = 6
get_hdr_val_flags_ofs = 7
get_hdr_str_buf_ofs = 8
get_hdr_table_checksum = 9
get_hdr_str_buf_checksum = 10
get_state = 11
PREINIT:
dMY_CXT;
PROTOTYPE: $
CODE:
{
struct mph_obj *obj;
MinPerfHashTwoLevel.xs view on Meta::CPAN
croak("$self->'mount' is expected to be a string!");
obj= (struct mph_obj *)SvPV_nolen(mount_sv);
start= (char *)obj->header;
switch(ix) {
case 0: RETVAL= newSVpv(start + obj->header->str_buf_ofs + 2,0); break;
case 1: RETVAL= newSVuv(obj->header->magic_num); break;
case 2: RETVAL= newSVuv(obj->header->variant); break;
case 3: RETVAL= newSVuv(obj->header->num_buckets); break;
case 4: RETVAL= newSVuv(obj->header->state_ofs); break;
case 5: RETVAL= newSVuv(obj->header->table_ofs); break;
case 6: RETVAL= newSVuv(obj->header->key_flags_ofs); break;
case 7: RETVAL= newSVuv(obj->header->val_flags_ofs); break;
case 8: RETVAL= newSVuv(obj->header->str_buf_ofs); break;
case 9: RETVAL= newSVuv(obj->header->table_checksum); break;
case 10: RETVAL= newSVuv(obj->header->str_buf_checksum); break;
case 11: RETVAL= newSVpvn(start + obj->header->state_ofs, MPH_STATE_BYTES); break;
}
}
OUTPUT:
RETVAL
lib/Algorithm/MinPerfHashTwoLevel.pm view on Meta::CPAN
our %EXPORT_TAGS = (
'all' => [
'$DEFAULT_VARIANT',
'MAX_VARIANT',
'MIN_VARIANT',
qw(
seed_state
hash_with_state
), sort keys %constant
],
'flags' => [ sort grep /MPH_F_/, keys %constant ],
);
our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } );
our @EXPORT = qw();
require XSLoader;
XSLoader::load('Algorithm::MinPerfHashTwoLevel', $VERSION);
lib/Algorithm/MinPerfHashTwoLevel.pm view on Meta::CPAN
my $seed= delete($opts{seed});
delete($opts{state}) and warn "ignoring 'state' parameter";
my $o= bless \%opts, $class;
$o->set_seed($seed) if $seed;
$o->{variant}= $DEFAULT_VARIANT unless defined $o->{variant};
$o->{variant}= int(0+$o->{variant});
$o->{compute_flags} ||= 0;
$o->{compute_flags} += MPH_F_FILTER_UNDEF
if delete $o->{filter_undef};
$o->{compute_flags} += MPH_F_DETERMINISTIC
if delete $o->{deterministic} or delete $o->{canonical};
die "Unknown variant '$o->{variant}' in constructor new(), max known is "
. MAX_VARIANT . " default is " . $DEFAULT_VARIANT
if $o->{variant} > MAX_VARIANT;
die "Unknown variant '$o->{variant}' in constructor new(), min known is "
. MIN_VARIANT . " default is " . $DEFAULT_VARIANT
if $o->{variant} < MIN_VARIANT;
return $o;
lib/Algorithm/MinPerfHashTwoLevel.pm view on Meta::CPAN
https://stackoverflow.com/a/12996028, but any decent reversible
integer hash function would do.
*NOTE* in Perl a given string may have differing binary representations
if it is encoded as utf8 or not. This module uses the same conventions
as Perl itself, which is that keys are stored in their minimal form when
possible, and are only stored in their unicode (utf8) form when they
cannot be downgraded to latin-1. This ensures that the unicode and latin-1
representations of a given string are treated as the same key. This module
deals with this by "normalizing" the keys and values into latin-1, but
tracking the representation as a flag. See key_normalized and key_is_utf8
(and their 'val' equivalents) documented in the construct method.
=head2 METHODS
=over 4
=item new
Construct a new Algorithm::MinPerfHashTwoLevel object. Optional arguments
which may be provided are 'source_hash' which is a hash reference to use
lib/Tie/Hash/MinPerfHashTwoLevel/OnDisk.pm view on Meta::CPAN
package Tie::Hash::MinPerfHashTwoLevel::OnDisk;
use strict;
use warnings;
our $VERSION = '0.16';
our $DEFAULT_VARIANT = 5;
# this also installs the XS routines we use into our namespace.
use Algorithm::MinPerfHashTwoLevel ( 'hash_with_state', '$DEFAULT_VARIANT', ':flags', 'MAX_VARIANT', 'MIN_VARIANT' );
use Exporter qw(import);
my %constants;
BEGIN {
%constants= (
MAGIC_STR => "PH2L",
#MPH_F_FILTER_UNDEF => (1<<0),
#MPH_F_DETERMINISTIC => (1<<1),
MPH_F_NO_DEDUPE => (1<<2),
MPH_F_VALIDATE => (1<<3),
);
}
use constant \%constants;
use Carp;
our %EXPORT_TAGS = (
'all' => [ qw(mph2l_tied_hashref mph2l_make_file MAX_VARIANT MIN_VARIANT), sort keys %constants ],
'flags' => ['MPH_F_DETERMINISTIC', grep /MPH_F_/, sort keys %constants],
'magic' => [grep /MAGIC/, sort keys %constants],
);
my $scalar_has_slash= scalar(%EXPORT_TAGS)=~m!/!;
our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } );
our @EXPORT = qw();
sub mph2l_tied_hashref {
my ($file, %opts)= @_;
lib/Tie/Hash/MinPerfHashTwoLevel/OnDisk.pm view on Meta::CPAN
}
sub mph2l_validate_file {
my ($file, %opts)= @_;
return __PACKAGE__->validate_file(file => $file, %opts);
}
sub new {
my ($class, %opts)= @_;
$opts{flags} ||= 0;
$opts{flags} |= MPH_F_VALIDATE if $opts{validate};
my $error;
my $mount= mount_file($opts{file},$error,$opts{flags});
my $error_rsv= delete $opts{error_rsv};
if ($error_rsv) {
$$error_rsv= $error;
}
if (!defined($mount)) {
if ($error_rsv) {
return;
} else {
die "Failed to mount file '$opts{file}': $error";
}
lib/Tie/Hash/MinPerfHashTwoLevel/OnDisk.pm view on Meta::CPAN
my $variant= int($opts{variant});
my $deterministic;
$deterministic //= delete $opts{canonical};
$deterministic //= delete $opts{deterministic};
$deterministic //= 1;
#1234567812345678
$opts{seed} = "MinPerfHash2Levl"
if !defined($opts{seed}) and $deterministic;
my $compute_flags= int($opts{compute_flags}||0);
$compute_flags |= MPH_F_NO_DEDUPE if delete $opts{no_dedupe};
$compute_flags |= MPH_F_DETERMINISTIC
if $deterministic;
$compute_flags |= MPH_F_FILTER_UNDEF
if delete $opts{filter_undef};
die "Unknown variant '$variant', max known is "
. MAX_VARIANT . " default is " . $DEFAULT_VARIANT
if $variant > MAX_VARIANT;
die "Unknown variant '$variant', min known is "
. MIN_VARIANT . " default is " . $DEFAULT_VARIANT
if $variant < MIN_VARIANT;
die "comment cannot contain null"
if index($comment,"\0") >= 0;
my $seed= $opts{seed};
my $hasher= Algorithm::MinPerfHashTwoLevel->new(
debug => $debug,
seed => (ref $seed ? $$seed : $seed),
variant => $variant,
compute_flags => $compute_flags,
max_tries => $opts{max_tries},
);
my $buckets= $hasher->compute($source_hash);
my $buf_length= $hasher->{buf_length};
my $state= $hasher->{state};
my $buf= packed_xs($variant, $buf_length, $state, $comment, $compute_flags, @$buckets);
$$seed= $hasher->get_seed if ref $seed;
my $tmp_file= "$ofile.$$";
open my $ofh, ">", $tmp_file
or die "Failed to open $tmp_file for output";
print $ofh $buf
or die "failed to print to '$tmp_file': $!";
close $ofh
or die "failed to close '$tmp_file': $!";
rename $tmp_file, $ofile
lib/Tie/Hash/MinPerfHashTwoLevel/OnDisk.pm view on Meta::CPAN
}
sub validate_file {
my ($class, %opts)= @_;
my $file= $opts{file}
or die "file is a mandatory option to validate_file";
my $verbose= $opts{verbose};
my ($variant,$msg);
my $error_sv;
my $self= $class->new(file => $file, flags => MPH_F_VALIDATE, error_rsv => \$error_sv);
if ($self) {
$msg= sprintf "file '%s' is a valid '%s' file\n"
. " variant: %d\n"
. " keys: %d\n"
. " hash-state: %s\n"
. " table checksum: %016x\n"
. " string checksum: %016x\n"
. " comment: %s"
, $file,
MAGIC_STR,
lib/Tie/Hash/MinPerfHashTwoLevel/OnDisk.pm view on Meta::CPAN
solution is found (see L<max_tries>) (prior to version v0.10 this used rand()).
Should you wish to access the seed actually used for the final solution
then you can pass in a reference to a scalar containing your chosen seed.
The reference scalar will be updated after successful construction.
Thus both of the following are valid:
Tie::Hash::MinPerfHashTwoLevel::OnDisk->make_file(seed => "1234567812345678", ...);
Tie::Hash::MinPerfHashTwoLevel::OnDisk->make_file(seed => \my $seed= "1234567812345678", ...);
=item compute_flags
This is an integer which contains various flags which control construction.
They are as follows:
MPH_F_FILTER_UNDEF => 1 - filter keys with undef values
MPH_F_DETERMINISTIC => 2 - repeatable results (sort keys during processing)
MPH_F_NO_DEDUPE => 4 - do not dedupe strings in final buffer
These constants can be imported via the ":flags" tag, but there are also options that
have the equivalent result, see C<no_dedupe>, C<deterministic> and C<filter_undef>.
=item no_dedupe
Speed up construction at the cost of a larger string buffer by disabling
deduplication of values and keys. Same as setting the MPH_F_NO_DEDUPE bit
in compute_flags.
=item deterministic
=item canonical
Produce a canonical result from the source data set, albeit somewhat less quickly
than normal. Note this is independent of supplying a seed, the same seed may produce
a different result for the same set of keys without this option. Same
as setting the MPH_F_DETERMINISTIC bit in compute_flags.
=item filter_undef
Ignore keys with undef values during construction. This means that exists() checks
may differ between source and the constructed hash table, but avoids the need to
store such keys in the resulting file, saving space. Same as setting the
MPH_F_FILTER_UNDEF bit in compute_flags.
=item max_tries
The maximum number of attempts to make to find a solution for this keyset.
Defaults to 3.
=item debug
Enable debug during generation.
lib/Tie/Hash/MinPerfHashTwoLevel/OnDisk.pm view on Meta::CPAN
=item mph2l_tied_hashref
Simple wrapper to replace the cumbersome
tie my %hash, "Tie::Hash::MinPerfHashTwoLevel::OnDisk", $file;
with a simple sub that can be imported
my $hashref= mph2l_tied_hashref($file,$validate);
The validate flag causes MPH_F_VALIDATE validations to occur on load.
=item mph2l_make_file
Sub form of L<make_file>. Eg:
use Tie::Hash::MinPerfHashTwoLevel::OnDisk;
Tie::Hash::MinPerfHashTwoLevel::OnDisk->make_file(@args);
is identical to
lib/Tie/Hash/MinPerfHashTwoLevel/OnDisk.pm view on Meta::CPAN
is identical to
use Tie::Hash::MinPerfHashTwoLevel::OnDisk qw(mph2l_validate_file);
mph2l_validate_file(@args);
=back
=head2 TIED INTERFACE
my %hash;
tie %hash, "Tie::Hash::MinPerfHashTwoLevel::OnDisk", $some_file, $flags;
will setup %hash to read from the mmapped image on disk as created by make_file().
The underlying image is never altered, and copies of the keys and values are made
when necessary. The flags field is an integer which contains bit-flags to control
the reading process, currently only one flag is supported MPH_F_VALIDATE which enables
a full file checksum before returning (forcing the data to be loaded and then read).
By default this validation is disabled, however basic checks of that the header is
sane will be performed on loading (or "mounting") the image. The tie operation may
die if the file is not found or any of these checks fail.
As this is somewhat cumbersome to type you may want to look at the mph2l_tied_hashref()
function which is wrapper around this function.
=head2 FILE FORMAT
Currently there is only one support file format variant, 5.
The file structure consists of a header, followed by a byte vector of seed/state
data for the hash function, followed by a bucket table with records of a fixed size,
optionally followed by a bitvector of the flags for the keys with two bits per key,
optionally followed by a bitvector of flags for values with one bit per value,
followed by a string table containing the comment for the file and the strings it
contains, and lastly a checksum; the last 8 bytes of the file contain a hash of the
rest of the file. The key flags may be 0 for "latin-1/not-utf8", 1 for "is-utf8",
and 2 for "was-utf8" which is used for keys which can be represented as latin-1,
but should be restored as unicode/utf8. The val flags are similar but do not (need to)
support "was-utf8".
Structure:
Header
Hash-state
Bucket-table
Key flags (optional)
Val flags (optional)
Strings
Checksum
Header:
U32 magic_num -> 1278363728 -> "PH2L"
U32 variant -> 5
U32 num_buckets -> number of buckets/keys in hash
U32 state_ofs -> offset in file where hash preseeded state is found
U32 table_ofs -> offset in file where bucket table starts
U32 key_flags_ofs -> offset in file where key flags are located
U32 val_flags ofs -> offset in file where val flags are located
U32 str_buf_ofs -> offset in file where strings are located
U64 general_flags -> flags used for this header
U64 reserved -> reserved field.
All "_ofs" values in the header are a multiple of 8, and the relevant sections
maybe be null padded to ensure this is so.
The string buffer contains the comment at str_buf_ofs+1, its length can be found
with strlen(), the comment may NOT contain nulls, and will be null terminated. All
other strings in the table are NOT null padded, the length data stored in the
bucket records should be used to determine the length of the keys and values.
The last 8 bytes of the file contains a hash checksum of the rest of the entire
MPH_INIT_KEYSV(MPH_KEYSV_XOR_VAL,"xor_val"); \
MPH_INIT_KEYSV(MPH_KEYSV_H0,"h0"); \
MPH_INIT_KEYSV(MPH_KEYSV_KEY,"key"); \
MPH_INIT_KEYSV(MPH_KEYSV_KEY_NORMALIZED,"key_normalized"); \
MPH_INIT_KEYSV(MPH_KEYSV_KEY_IS_UTF8,"key_is_utf8"); \
MPH_INIT_KEYSV(MPH_KEYSV_VAL,"val"); \
MPH_INIT_KEYSV(MPH_KEYSV_VAL_NORMALIZED,"val_normalized"); \
MPH_INIT_KEYSV(MPH_KEYSV_VAL_IS_UTF8,"val_is_utf8"); \
\
MPH_INIT_KEYSV(MPH_KEYSV_VARIANT,"variant"); \
MPH_INIT_KEYSV(MPH_KEYSV_COMPUTE_FLAGS,"compute_flags"); \
MPH_INIT_KEYSV(MPH_KEYSV_STATE,"state"); \
MPH_INIT_KEYSV(MPH_KEYSV_SOURCE_HASH,"source_hash"); \
MPH_INIT_KEYSV(MPH_KEYSV_BUF_LENGTH,"buf_length"); \
MPH_INIT_KEYSV(MPH_KEYSV_BUCKETS,"buckets"); \
MPH_INIT_KEYSV(MPH_KEYSV_MOUNT,"mount"); \
} STMT_END
#define MPH_F_FILTER_UNDEF (1<<0)
#define MPH_F_DETERMINISTIC (1<<1)
#define MPH_F_NO_DEDUPE (1<<2)
#ifndef CHAR_BITS
#define CHAR_BITS 8
#endif
#define _BITSDECL(idx,bits) \
const U64 bitpos= idx * bits; \
const U64 bytepos= bitpos / CHAR_BITS; \
const U8 shift= bitpos % CHAR_BITS; \
const U8 bitmask= ( 1 << bits ) - 1
#define GETBITS(into,flags,idx,bits) STMT_START { \
_BITSDECL(idx,bits); \
into= ((flags)[bytepos] >> shift) & bitmask; \
} STMT_END
#define SETBITS(value,flags,idx,bits) STMT_START { \
_BITSDECL(idx,bits); \
const U8 v= value; \
(flags)[bytepos] &= ~(bitmask << shift); \
(flags)[bytepos] |= ((v & bitmask) << shift); \
} STMT_END
typedef struct {
SV *sv;
U32 hash;
} sv_with_hash;
typedef struct {
sv_with_hash keysv[COUNT_MPH_KEYSV];
} my_cxt_t;
struct mph_header {
U32 magic_num;
U32 variant;
U32 num_buckets;
U32 state_ofs;
U32 table_ofs;
U32 key_flags_ofs;
U32 val_flags_ofs;
U32 str_buf_ofs;
union {
U64 table_checksum;
U64 general_flags;
};
union {
U64 str_buf_checksum;
};
};
struct mph_bucket {
union {
U32 xor_val;
I32 index;
SvRX() NEED_SvRX NEED_SvRX_GLOBAL
caller_cx() NEED_caller_cx NEED_caller_cx_GLOBAL
croak_xs_usage() NEED_croak_xs_usage NEED_croak_xs_usage_GLOBAL
die_sv() NEED_die_sv NEED_die_sv_GLOBAL
eval_pv() NEED_eval_pv NEED_eval_pv_GLOBAL
grok_bin() NEED_grok_bin NEED_grok_bin_GLOBAL
grok_hex() NEED_grok_hex NEED_grok_hex_GLOBAL
grok_number() NEED_grok_number NEED_grok_number_GLOBAL
grok_numeric_radix() NEED_grok_numeric_radix NEED_grok_numeric_radix_GLOBAL
grok_oct() NEED_grok_oct NEED_grok_oct_GLOBAL
gv_fetchpvn_flags() NEED_gv_fetchpvn_flags NEED_gv_fetchpvn_flags_GLOBAL
load_module() NEED_load_module NEED_load_module_GLOBAL
mess() NEED_mess NEED_mess_GLOBAL
mess_nocontext() NEED_mess_nocontext NEED_mess_nocontext_GLOBAL
mess_sv() NEED_mess_sv NEED_mess_sv_GLOBAL
mg_findext() NEED_mg_findext NEED_mg_findext_GLOBAL
my_snprintf() NEED_my_snprintf NEED_my_snprintf_GLOBAL
my_sprintf() NEED_my_sprintf NEED_my_sprintf_GLOBAL
my_strlcat() NEED_my_strlcat NEED_my_strlcat_GLOBAL
my_strlcpy() NEED_my_strlcpy NEED_my_strlcpy_GLOBAL
my_strnlen() NEED_my_strnlen NEED_my_strnlen_GLOBAL
newCONSTSUB() NEED_newCONSTSUB NEED_newCONSTSUB_GLOBAL
newRV_noinc() NEED_newRV_noinc NEED_newRV_noinc_GLOBAL
newSV_type() NEED_newSV_type NEED_newSV_type_GLOBAL
newSVpvn_flags() NEED_newSVpvn_flags NEED_newSVpvn_flags_GLOBAL
newSVpvn_share() NEED_newSVpvn_share NEED_newSVpvn_share_GLOBAL
pv_display() NEED_pv_display NEED_pv_display_GLOBAL
pv_escape() NEED_pv_escape NEED_pv_escape_GLOBAL
pv_pretty() NEED_pv_pretty NEED_pv_pretty_GLOBAL
sv_2pv_flags() NEED_sv_2pv_flags NEED_sv_2pv_flags_GLOBAL
sv_2pvbyte() NEED_sv_2pvbyte NEED_sv_2pvbyte_GLOBAL
sv_catpvf_mg() NEED_sv_catpvf_mg NEED_sv_catpvf_mg_GLOBAL
sv_catpvf_mg_nocontext() NEED_sv_catpvf_mg_nocontext NEED_sv_catpvf_mg_nocontext_GLOBAL
sv_pvn_force_flags() NEED_sv_pvn_force_flags NEED_sv_pvn_force_flags_GLOBAL
sv_setpvf_mg() NEED_sv_setpvf_mg NEED_sv_setpvf_mg_GLOBAL
sv_setpvf_mg_nocontext() NEED_sv_setpvf_mg_nocontext NEED_sv_setpvf_mg_nocontext_GLOBAL
sv_unmagicext() NEED_sv_unmagicext NEED_sv_unmagicext_GLOBAL
utf8_to_uvchr_buf() NEED_utf8_to_uvchr_buf NEED_utf8_to_uvchr_buf_GLOBAL
vload_module() NEED_vload_module NEED_vload_module_GLOBAL
vmess() NEED_vmess NEED_vmess_GLOBAL
vnewSVpvf() NEED_vnewSVpvf NEED_vnewSVpvf_GLOBAL
warner() NEED_warner NEED_warner_GLOBAL
To avoid namespace conflicts, you can change the namespace of the
SvPOK_only_UTF8||5.006000|
SvPOK_only|||
SvPOK_on|||
SvPOKp|||
SvPOK|||
SvPVCLEAR|||
SvPVX_const|5.009003||p
SvPVX_mutable|5.009003||p
SvPVX|||
SvPV_const|5.009003||p
SvPV_flags_const_nolen|5.009003||p
SvPV_flags_const|5.009003||p
SvPV_flags_mutable|5.009003||p
SvPV_flags|5.007002||p
SvPV_force_flags_mutable|5.009003||p
SvPV_force_flags_nolen|5.009003||p
SvPV_force_flags|5.007002||p
SvPV_force_mutable|5.009003||p
SvPV_force_nolen|5.009003||p
SvPV_force_nomg_nolen|5.009003||p
SvPV_force_nomg|5.007002||p
SvPV_force|||p
SvPV_mutable|5.009003||p
SvPV_nolen_const|5.009003||p
SvPV_nolen|5.006000||p
SvPV_nomg_const_nolen|5.009003||p
SvPV_nomg_const|5.009003||p
custom_op_get_field|||
custom_op_name||5.007003|
custom_op_register||5.013007|
custom_op_xop||5.013007|
cv_clone_into|||
cv_clone|||
cv_const_sv_or_av|||n
cv_const_sv||5.003070|n
cv_dump|||
cv_forget_slab|||
cv_get_call_checker_flags|||
cv_get_call_checker||5.013006|
cv_name||5.021005|
cv_set_call_checker_flags||5.021004|
cv_set_call_checker||5.013006|
cv_undef_flags|||
cv_undef|||
cvgv_from_hek|||
cvgv_set|||
cvstash_set|||
cx_dump||5.005000|
cx_dup|||
cxinc|||
dAXMARK|5.009003||p
dAX|5.007002||p
dITEMS|5.007002||p
gen_constant_list|||
get_ANYOFM_contents|||
get_ANYOF_cp_list_for_ssc|||
get_and_check_backslash_N_name_wrapper|||
get_and_check_backslash_N_name|||
get_aux_mg|||
get_av|5.006000||p
get_c_backtrace_dump|||
get_c_backtrace|||
get_context||5.006000|n
get_cvn_flags|||
get_cvs|5.011000||p
get_cv|5.006000||p
get_db_sub|||
get_debug_opts|||
get_hash_seed|||
get_hv|5.006000||p
get_mstats|||
get_no_modify|||
get_num|||
get_op_descs||5.005000|
glob_2number|||
glob_assign_glob|||
gp_dup|||
gp_free|||
gp_ref|||
grok_atoUV|||n
grok_bin|5.007003||p
grok_bslash_N|||
grok_hex|5.007003||p
grok_infnan||5.021004|
grok_number_flags||5.021002|
grok_number|5.007002||p
grok_numeric_radix|5.007002||p
grok_oct|5.007003||p
group_end|||
gv_AVadd|||
gv_HVadd|||
gv_IOadd|||
gv_SVadd|||
gv_add_by_type||5.011000|
gv_autoload4||5.004000|
gv_autoload_pvn||5.015004|
gv_autoload_pv||5.015004|
gv_autoload_sv||5.015004|
gv_check|||
gv_const_sv||5.009003|
gv_dump||5.006000|
gv_efullname3||5.003070|
gv_efullname4||5.006001|
gv_efullname|||
gv_fetchfile_flags||5.009005|
gv_fetchfile|||
gv_fetchmeth_autoload||5.007003|
gv_fetchmeth_internal|||
gv_fetchmeth_pv_autoload||5.015004|
gv_fetchmeth_pvn_autoload||5.015004|
gv_fetchmeth_pvn||5.015004|
gv_fetchmeth_pv||5.015004|
gv_fetchmeth_sv_autoload||5.015004|
gv_fetchmeth_sv||5.015004|
gv_fetchmethod_autoload||5.004000|
gv_fetchmethod|||
gv_fetchmeth|||
gv_fetchpvn_flags|5.009002||p
gv_fetchpvs|5.009004||p
gv_fetchpv|||
gv_fetchsv|||
gv_fullname3||5.003070|
gv_fullname4||5.006001|
gv_fullname|||
gv_handler||5.007001|
gv_init_pvn|||
gv_init_pv||5.015004|
gv_init_svtype|||
hv_placeholders_p|||
hv_placeholders_set||5.009003|
hv_pushkv|||
hv_rand_set||5.018000|
hv_riter_p||5.009003|
hv_riter_set||5.009003|
hv_scalar||5.009001|
hv_store_ent||5.003070|
hv_stores|5.009004||p
hv_store|||
hv_undef_flags|||
hv_undef|||
ibcmp_locale||5.004000|
ibcmp_utf8||5.007003|
ibcmp|||
incline|||
incpush_if_exists|||
incpush_use_sep|||
incpush|||
ingroup|||
init_argv_symbols|||
isPSXSPC|5.006001||p
isPUNCT_A|||p
isPUNCT|5.006000||p
isSB|||
isSCRIPT_RUN|||
isSPACE_A|||p
isSPACE|||p
isSTRICT_UTF8_CHAR|||n
isUPPER_A|||p
isUPPER|||p
isUTF8_CHAR_flags|||
isUTF8_CHAR||5.021001|n
isWB|||
isWORDCHAR_A|||p
isWORDCHAR|5.013006|5.013006|p
isXDIGIT_A|||p
isXDIGIT|5.006000||p
is_an_int|||
is_ascii_string||5.011000|n
is_c9strict_utf8_string_loclen|||n
is_c9strict_utf8_string_loc|||n
is_lvalue_sub||5.007001|
is_safe_syscall||5.019004|
is_ssc_worth_it|||n
is_strict_utf8_string_loclen|||n
is_strict_utf8_string_loc|||n
is_strict_utf8_string|||n
is_utf8_char_buf||5.015008|n
is_utf8_common_with_len|||
is_utf8_common|||
is_utf8_cp_above_31_bits|||n
is_utf8_fixed_width_buf_flags|||n
is_utf8_fixed_width_buf_loc_flags|||n
is_utf8_fixed_width_buf_loclen_flags|||n
is_utf8_invariant_string_loc|||n
is_utf8_invariant_string|||n
is_utf8_non_invariant_string|||n
is_utf8_overlong_given_start_byte_ok|||n
is_utf8_string_flags|||n
is_utf8_string_loc_flags|||n
is_utf8_string_loclen_flags|||n
is_utf8_string_loclen||5.009003|n
is_utf8_string_loc||5.008001|n
is_utf8_string||5.006001|n
is_utf8_valid_partial_char_flags|||n
is_utf8_valid_partial_char|||n
isa_lookup|||
isinfnansv|||
isinfnan||5.021004|n
items|||n
ix|||n
jmaybe|||
join_exact|||
keyword_plugin_standard|||
keyword|||
my_clearenv|||
my_cxt_index|||
my_cxt_init|||
my_dirfd||5.009005|n
my_exit_jump|||
my_exit|||
my_failure_exit||5.004000|
my_fflush_all||5.006000|
my_fork||5.007003|n
my_kid|||
my_lstat_flags|||
my_lstat||5.024000|
my_memrchr|||n
my_mkostemp|||n
my_mkstemp_cloexec|||n
my_mkstemp|||n
my_nl_langinfo|||n
my_pclose||5.003070|
my_popen_list||5.007001|
my_popen||5.003070|
my_setenv|||
my_snprintf|5.009004||pvn
my_socketpair||5.007003|n
my_sprintf|5.009003||pvn
my_stat_flags|||
my_stat||5.024000|
my_strerror|||
my_strftime||5.007002|
my_strlcat|5.009004||pn
my_strlcpy|5.009004||pn
my_strnlen|||pn
my_strtod|||n
my_unexec|||
my_vsnprintf||5.009004|n
need_utf8|||n
newANONHASH|||
newANONLIST|||
newANONSUB|||
newASSIGNOP|||
newATTRSUB_x|||
newATTRSUB||5.006000|
newAVREF|||
newAV|||
newBINOP|||
newCONDOP|||
newCONSTSUB_flags||5.015006|
newCONSTSUB|5.004050||p
newCVREF|||
newDEFSVOP||5.021006|
newFORM|||
newFOROP||5.013007|
newGIVENOP||5.009003|
newGIVWHENOP|||
newGVOP|||
newGVREF|||
newGVgen_flags||5.015004|
newGVgen|||
newHVREF|||
newHVhv||5.005000|
newHV|||
newIO|||
newLISTOP|||
newLOGOP|||
newLOOPEX|||
newLOOPOP|||
newMETHOP_internal|||
newSVREF|||
newSV_type|5.009005||p
newSVavdefelem|||
newSVhek||5.009003|
newSViv|||
newSVnv|||
newSVpadname||5.017004|
newSVpv_share||5.013006|
newSVpvf_nocontext|||vn
newSVpvf||5.004000|v
newSVpvn_flags|5.010001||p
newSVpvn_share|5.007001||p
newSVpvn_utf8|5.010001||p
newSVpvn|5.004050||p
newSVpvs_flags|5.010001||p
newSVpvs_share|5.009003||p
newSVpvs|5.009003||p
newSVpv|||
newSVrv|||
newSVsv_flags|||
newSVsv_nomg|||
newSVsv|||
newSVuv|5.006000||p
newSV|||
newUNOP_AUX||5.021007|
newUNOP|||
newWHENOP||5.009003|
newWHILEOP||5.013007|
newXS_deffile|||
newXS_len_flags|||
newXSproto||5.006000|
newXS||5.006000|
new_collate|||
new_constant|||
new_ctype|||
new_he|||
new_logop|||
new_msg_hv|||
new_numeric|||
new_regcurly|||n
op_append_elem||5.013006|
op_append_list||5.013006|
op_class|||
op_clear|||
op_contextualize||5.013006|
op_convert_list||5.021006|
op_dump||5.006000|
op_free|||
op_integerize|||
op_linklist||5.013006|
op_lvalue_flags|||
op_null||5.007002|
op_parent|||n
op_prepend_elem||5.013006|
op_refcnt_lock||5.009002|
op_refcnt_unlock||5.009002|
op_relocate_sv|||
op_sibling_splice||5.021002|n
op_std_init|||
open_script|||
openn_cleanup|||
pad_swipe|||
padlist_dup|||
padlist_store|||
padname_dup|||
padname_free|||
padnamelist_dup|||
padnamelist_free|||
parse_body|||
parse_gv_stash_name|||
parse_ident|||
parse_lparen_question_flags|||
parse_unicode_opts|||
parse_uniprop_string|||
parser_dup|||
parser_free_nexttoke_ops|||
parser_free|||
path_is_searchable|||n
peep|||
pending_ident|||
perl_alloc_using|||n
perl_alloc|||n
reg_skipcomment|||n
reg_temp_copy|||
reganode|||
regatom|||
regbranch|||
regclass|||
regcp_restore|||
regcppop|||
regcppush|||
regcurly|||n
regdump_extflags|||
regdump_intflags|||
regdump||5.005000|
regdupe_internal|||
regex_set_precedence|||n
regexec_flags||5.005000|
regfree_internal||5.009005|
reghop3|||n
reghop4|||n
reghopmaybe3|||n
reginclass|||
reginitcolors||5.006000|
reginsert|||
regmatch|||
regnext||5.005000|
regnode_guts|||
rxres_save|||
safesyscalloc||5.006000|n
safesysfree||5.006000|n
safesysmalloc||5.006000|n
safesysrealloc||5.006000|n
same_dirent|||
save_I16||5.004000|
save_I32|||
save_I8||5.006000|
save_adelete||5.011000|
save_aelem_flags||5.011000|
save_aelem||5.004050|
save_alloc||5.006000|
save_aptr|||
save_ary|||
save_bool||5.008001|
save_clearsv|||
save_delete|||
save_destructor_x||5.006000|
save_destructor||5.006000|
save_freeop|||
save_freepv|||
save_freesv|||
save_generic_pvref||5.006001|
save_generic_svref||5.005030|
save_gp||5.004000|
save_hash|||
save_hdelete||5.011000|
save_hek_flags|||n
save_helem_flags||5.011000|
save_helem||5.004050|
save_hints||5.010001|
save_hptr|||
save_int|||
save_item|||
save_iv||5.005000|
save_lines|||
save_list|||
save_long|||
save_magic_flags|||
save_mortalizesv||5.007001|
save_nogv|||
save_op||5.005000|
save_padsv_and_mortalize||5.010001|
save_pptr|||
save_pushi32ptr||5.010001|
save_pushptri32ptr|||
save_pushptrptr||5.010001|
save_pushptr||5.010001|
save_re_context||5.006000|
save_scalar_at|||
save_scalar|||
save_set_svflags||5.009000|
save_shared_pvref||5.007003|
save_sptr|||
save_strlen|||
save_svref|||
save_to_buffer|||n
save_vptr||5.006000|
savepvn|||
savepvs||5.009003|
savepv|||
savesharedpvn||5.009005|
set_numeric_underlying|||
set_padlist|||n
set_regex_pv|||
setdefout|||
setfd_cloexec_for_nonsysfd|||
setfd_cloexec_or_inhexec_by_sysfdness|||
setfd_cloexec|||n
setfd_inhexec_for_sysfd|||
setfd_inhexec|||n
setlocale_debug_string|||n
share_hek_flags|||
share_hek||5.004000|
should_warn_nl|||n
si_dup|||
sighandler|||n
simplify_sort|||
skip_to_be_ignored_text|||
softref2xv|||
sortcv_stacked|||
sortcv_xsub|||
sortcv|||
sortsv_flags||5.009003|
sortsv||5.007003|
space_join_names_mortal|||
ss_dup|||
ssc_add_range|||
ssc_and|||
ssc_anything|||
ssc_clear_locale|||n
ssc_cp_and|||
ssc_finalize|||
ssc_init|||
strNE|||
str_to_version||5.006000|
strip_return|||
strnEQ|||
strnNE|||
study_chunk|||
sub_crush_depth|||
sublex_done|||
sublex_push|||
sublex_start|||
sv_2bool_flags||5.013006|
sv_2bool|||
sv_2cv|||
sv_2io|||
sv_2iuv_common|||
sv_2iuv_non_preserve|||
sv_2iv_flags||5.009001|
sv_2iv|||
sv_2mortal|||
sv_2nv_flags||5.013001|
sv_2pv_flags|5.007002||p
sv_2pv_nolen|5.006000||p
sv_2pvbyte_nolen|5.006000||p
sv_2pvbyte|5.006000||p
sv_2pvutf8_nolen||5.006000|
sv_2pvutf8||5.006000|
sv_2pv|||
sv_2uv_flags||5.009001|
sv_2uv|5.004000||p
sv_add_arena|||
sv_add_backref|||
sv_backoff|||n
sv_bless|||
sv_buf_to_ro|||
sv_buf_to_rw|||
sv_cat_decode||5.008001|
sv_catpv_flags||5.013006|
sv_catpv_mg|5.004050||p
sv_catpv_nomg||5.013006|
sv_catpvf_mg_nocontext|||pvn
sv_catpvf_mg|5.006000|5.004000|pv
sv_catpvf_nocontext|||vn
sv_catpvf||5.004000|v
sv_catpvn_flags||5.007002|
sv_catpvn_mg|5.004050||p
sv_catpvn_nomg|5.007002||p
sv_catpvn|||
sv_catpvs_flags||5.013006|
sv_catpvs_mg||5.013006|
sv_catpvs_nomg||5.013006|
sv_catpvs|5.009003||p
sv_catpv|||
sv_catsv_flags||5.007002|
sv_catsv_mg|5.004050||p
sv_catsv_nomg|5.007002||p
sv_catsv|||
sv_chop|||
sv_clean_all|||
sv_clean_objs|||
sv_clear|||
sv_cmp_flags||5.013006|
sv_cmp_locale_flags||5.013006|
sv_cmp_locale||5.004000|
sv_cmp|||
sv_collxfrm_flags||5.013006|
sv_collxfrm|||
sv_copypv_flags||5.017002|
sv_copypv_nomg||5.017002|
sv_copypv|||
sv_dec_nomg||5.013002|
sv_dec|||
sv_del_backref|||
sv_derived_from_pvn||5.015004|
sv_derived_from_pv||5.015004|
sv_derived_from_sv||5.015004|
sv_derived_from||5.004000|
sv_destroyable||5.010000|
sv_display|||
sv_does_pvn||5.015004|
sv_does_pv||5.015004|
sv_does_sv||5.015004|
sv_does||5.009004|
sv_dump|||
sv_dup_common|||
sv_dup_inc_multiple|||
sv_dup_inc|||
sv_dup|||
sv_eq_flags||5.013006|
sv_eq|||
sv_exp_grow|||
sv_force_normal_flags||5.007001|
sv_force_normal||5.006000|
sv_free_arenas|||
sv_free|||
sv_gets||5.003070|
sv_grow|||
sv_i_ncmp|||
sv_inc_nomg||5.013002|
sv_inc|||
sv_insert_flags||5.010001|
sv_insert|||
sv_isa|||
sv_isobject|||
sv_iv||5.005000|
sv_len_utf8_nomg|||
sv_len_utf8||5.006000|
sv_len|||
sv_magic_portable|5.024000|5.004000|p
sv_magicext_mglob|||
sv_magicext||5.007003|
sv_magic|||
sv_mortalcopy_flags|||
sv_mortalcopy|||
sv_ncmp|||
sv_newmortal|||
sv_newref|||
sv_nolocking||5.007003|
sv_nosharing||5.007003|
sv_nounlocking|||
sv_nv||5.005000|
sv_only_taint_gmagic|||n
sv_or_pv_pos_u2b|||
sv_peek||5.005000|
sv_pos_b2u_flags||5.019003|
sv_pos_b2u_midway|||
sv_pos_b2u||5.006000|
sv_pos_u2b_cached|||
sv_pos_u2b_flags||5.011005|
sv_pos_u2b_forwards|||n
sv_pos_u2b_midway|||n
sv_pos_u2b||5.006000|
sv_pvbyten_force||5.006000|
sv_pvbyten||5.006000|
sv_pvbyte||5.006000|
sv_pvn_force_flags|5.007002||p
sv_pvn_force|||
sv_pvn_nomg|5.007003|5.005000|p
sv_pvn||5.005000|
sv_pvutf8n_force||5.006000|
sv_pvutf8n||5.006000|
sv_pvutf8||5.006000|
sv_pv||5.006000|
sv_recode_to_utf8||5.007003|
sv_reftype|||
sv_ref||5.015004|
sv_setpvn|||
sv_setpvs_mg||5.013006|
sv_setpvs|5.009004||p
sv_setpv|||
sv_setref_iv|||
sv_setref_nv|||
sv_setref_pvn|||
sv_setref_pvs||5.024000|
sv_setref_pv|||
sv_setref_uv||5.007001|
sv_setsv_flags||5.007002|
sv_setsv_mg|5.004050||p
sv_setsv_nomg|5.007002||p
sv_setsv|||
sv_setuv_mg|5.004050||p
sv_setuv|5.004000||p
sv_string_from_errnum|||
sv_tainted||5.004000|
sv_taint||5.004000|
sv_true||5.005000|
sv_unglob|||
sv_uni_display||5.007003|
sv_unmagicext|5.013008||p
sv_unmagic|||
sv_unref_flags||5.007001|
sv_unref|||
sv_untaint||5.004000|
sv_upgrade|||
sv_usepvn_flags||5.009004|
sv_usepvn_mg|5.004050||p
sv_usepvn|||
sv_utf8_decode|||
sv_utf8_downgrade|||
sv_utf8_encode||5.006000|
sv_utf8_upgrade_flags_grow||5.011000|
sv_utf8_upgrade_flags||5.007002|
sv_utf8_upgrade_nomg||5.007002|
sv_utf8_upgrade||5.007001|
sv_uv|5.005000||p
sv_vcatpvf_mg|5.006000|5.004000|p
sv_vcatpvfn_flags||5.017002|
sv_vcatpvfn||5.004000|
sv_vcatpvf|5.006000|5.004000|p
sv_vsetpvf_mg|5.006000|5.004000|p
sv_vsetpvfn||5.004000|
sv_vsetpvf|5.006000|5.004000|p
svtype|||
swallow_bom|||
swatch_get|||
switch_category_locale_to_template|||
switch_to_global_locale|||n
utf8_hop||5.006000|n
utf8_length||5.007001|
utf8_mg_len_cache_update|||
utf8_mg_pos_cache_update|||
utf8_to_uvchr_buf|5.015009|5.015009|p
utf8_to_uvchr|||p
utf8n_to_uvchr_error|||n
utf8n_to_uvchr||5.007001|n
utf8n_to_uvuni||5.007001|
utilize|||
uvchr_to_utf8_flags||5.007003|
uvchr_to_utf8||5.007001|
uvoffuni_to_utf8_flags||5.019004|
uvuni_to_utf8_flags||5.007003|
uvuni_to_utf8||5.007001|
valid_utf8_to_uvchr|||n
validate_suid|||
variant_under_utf8_count|||n
varname|||
vcmp||5.009000|
vcroak||5.006000|
vdeb||5.007003|
vform||5.006000|
visit|||
}
$count or print "Found no API matching '$opt{'api-info'}'.";
print "\n";
exit 0;
}
if (exists $opt{'list-provided'}) {
my $f;
for $f (sort { lc $a cmp lc $b } keys %API) {
next unless $API{$f}{provided};
my @flags;
push @flags, 'explicit' if exists $need{$f};
push @flags, 'depend' if exists $depends{$f};
push @flags, 'hint' if exists $hints{$f};
push @flags, 'warning' if exists $warnings{$f};
my $flags = @flags ? ' ['.join(', ', @flags).']' : '';
print "$f$flags\n";
}
exit 0;
}
my @files;
my @srcext = qw( .xs .c .h .cc .cpp -c.inc -xs.inc );
my $srcext = join '|', map { quotemeta $_ } @srcext;
if (@ARGV) {
my %seen;
#ifndef PERL_LOADMOD_IMPORT_OPS
# define PERL_LOADMOD_IMPORT_OPS 0x4
#endif
#ifndef G_METHOD
# define G_METHOD 64
# ifdef call_sv
# undef call_sv
# endif
# if (PERL_BCDVERSION < 0x5006000)
# define call_sv(sv, flags) ((flags) & G_METHOD ? perl_call_method((char *) SvPV_nolen_const(sv), \
(flags) & ~G_METHOD) : perl_call_sv(sv, flags))
# else
# define call_sv(sv, flags) ((flags) & G_METHOD ? Perl_call_method(aTHX_ (char *) SvPV_nolen_const(sv), \
(flags) & ~G_METHOD) : Perl_call_sv(aTHX_ sv, flags))
# endif
#endif
/* Replace perl_eval_pv with eval_pv */
#ifndef eval_pv
#if defined(NEED_eval_pv)
static SV* DPPP_(my_eval_pv)(char *p, I32 croak_on_error);
static
#else
croak_sv(ERRSV);
return sv;
}
#endif
#endif
#ifndef vload_module
#if defined(NEED_vload_module)
static void DPPP_(my_vload_module)(U32 flags, SV *name, SV *ver, va_list *args);
static
#else
extern void DPPP_(my_vload_module)(U32 flags, SV *name, SV *ver, va_list *args);
#endif
#if defined(NEED_vload_module) || defined(NEED_vload_module_GLOBAL)
#ifdef vload_module
# undef vload_module
#endif
#define vload_module(a,b,c,d) DPPP_(my_vload_module)(aTHX_ a,b,c,d)
#define Perl_vload_module DPPP_(my_vload_module)
void
DPPP_(my_vload_module)(U32 flags, SV *name, SV *ver, va_list *args)
{
dTHR;
dVAR;
OP *veop, *imop;
OP * const modname = newSVOP(OP_CONST, 0, name);
/* 5.005 has a somewhat hacky force_normal that doesn't croak on
SvREADONLY() if PL_compling is true. Current perls take care in
ck_require() to correctly turn off SvREADONLY before calling
force_normal_flags(). This seems a better fix than fudging PL_compling
*/
SvREADONLY_off(((SVOP*)modname)->op_sv);
modname->op_private |= OPpCONST_BARE;
if (ver) {
veop = newSVOP(OP_CONST, 0, ver);
}
else
veop = NULL;
if (flags & PERL_LOADMOD_NOIMPORT) {
imop = sawparens(newNULLLIST());
}
else if (flags & PERL_LOADMOD_IMPORT_OPS) {
imop = va_arg(*args, OP*);
}
else {
SV *sv;
imop = NULL;
sv = va_arg(*args, SV*);
while (sv) {
imop = append_elem(OP_LIST, imop, newSVOP(OP_CONST, 0, sv));
sv = va_arg(*args, SV*);
}
}
{
const line_t ocopline = PL_copline;
COP * const ocurcop = PL_curcop;
const int oexpect = PL_expect;
#if (PERL_BCDVERSION >= 0x5004000)
utilize(!(flags & PERL_LOADMOD_DENY), start_subparse(FALSE, 0),
veop, modname, imop);
#elif (PERL_BCDVERSION > 0x5003000)
utilize(!(flags & PERL_LOADMOD_DENY), start_subparse(),
veop, modname, imop);
#else
utilize(!(flags & PERL_LOADMOD_DENY), start_subparse(),
modname, imop);
#endif
PL_expect = oexpect;
PL_copline = ocopline;
PL_curcop = ocurcop;
}
}
#endif
#endif
#ifndef load_module
#if defined(NEED_load_module)
static void DPPP_(my_load_module)(U32 flags, SV *name, SV *ver, ...);
static
#else
extern void DPPP_(my_load_module)(U32 flags, SV *name, SV *ver, ...);
#endif
#if defined(NEED_load_module) || defined(NEED_load_module_GLOBAL)
#ifdef load_module
# undef load_module
#endif
#define load_module DPPP_(my_load_module)
#define Perl_load_module DPPP_(my_load_module)
void
DPPP_(my_load_module)(U32 flags, SV *name, SV *ver, ...)
{
va_list args;
va_start(args, ver);
vload_module(flags, name, ver, &args);
va_end(args);
}
#endif
#endif
#ifndef newRV_inc
# define newRV_inc(sv) newRV(sv) /* Replace */
#endif
#ifndef newRV_noinc
# define D_PPP_CONSTPV_ARG(x) ((char *) (x))
#else
# define D_PPP_CONSTPV_ARG(x) (x)
#endif
#ifndef newSVpvn
# define newSVpvn(data,len) ((data) \
? ((len) ? newSVpv((data), (len)) : newSVpv("", 0)) \
: newSV(0))
#endif
#ifndef newSVpvn_utf8
# define newSVpvn_utf8(s, len, u) newSVpvn_flags((s), (len), (u) ? SVf_UTF8 : 0)
#endif
#ifndef SVf_UTF8
# define SVf_UTF8 0
#endif
#ifndef newSVpvn_flags
#if defined(NEED_newSVpvn_flags)
static SV * DPPP_(my_newSVpvn_flags)(pTHX_ const char *s, STRLEN len, U32 flags);
static
#else
extern SV * DPPP_(my_newSVpvn_flags)(pTHX_ const char *s, STRLEN len, U32 flags);
#endif
#if defined(NEED_newSVpvn_flags) || defined(NEED_newSVpvn_flags_GLOBAL)
#ifdef newSVpvn_flags
# undef newSVpvn_flags
#endif
#define newSVpvn_flags(a,b,c) DPPP_(my_newSVpvn_flags)(aTHX_ a,b,c)
#define Perl_newSVpvn_flags DPPP_(my_newSVpvn_flags)
SV *
DPPP_(my_newSVpvn_flags)(pTHX_ const char *s, STRLEN len, U32 flags)
{
SV *sv = newSVpvn(D_PPP_CONSTPV_ARG(s), len);
SvFLAGS(sv) |= (flags & SVf_UTF8);
return (flags & SVs_TEMP) ? sv_2mortal(sv) : sv;
}
#endif
#endif
/* Backwards compatibility stuff... :-( */
#if !defined(NEED_sv_2pv_flags) && defined(NEED_sv_2pv_nolen)
# define NEED_sv_2pv_flags
#endif
#if !defined(NEED_sv_2pv_flags_GLOBAL) && defined(NEED_sv_2pv_nolen_GLOBAL)
# define NEED_sv_2pv_flags_GLOBAL
#endif
/* Hint: sv_2pv_nolen
* Use the SvPV_nolen() or SvPV_nolen_const() macros instead of sv_2pv_nolen().
*/
#ifndef sv_2pv_nolen
# define sv_2pv_nolen(sv) SvPV_nolen(sv)
#endif
#ifdef SvPVbyte
#ifndef SV_HAS_TRAILING_NUL
# define SV_HAS_TRAILING_NUL 0
#endif
#ifndef SV_COW_SHARED_HASH_KEYS
# define SV_COW_SHARED_HASH_KEYS 0
#endif
#if (PERL_BCDVERSION < 0x5007002)
#if defined(NEED_sv_2pv_flags)
static char * DPPP_(my_sv_2pv_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags);
static
#else
extern char * DPPP_(my_sv_2pv_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags);
#endif
#if defined(NEED_sv_2pv_flags) || defined(NEED_sv_2pv_flags_GLOBAL)
#ifdef sv_2pv_flags
# undef sv_2pv_flags
#endif
#define sv_2pv_flags(a,b,c) DPPP_(my_sv_2pv_flags)(aTHX_ a,b,c)
#define Perl_sv_2pv_flags DPPP_(my_sv_2pv_flags)
char *
DPPP_(my_sv_2pv_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags)
{
STRLEN n_a = (STRLEN) flags;
return sv_2pv(sv, lp ? lp : &n_a);
}
#endif
#if defined(NEED_sv_pvn_force_flags)
static char * DPPP_(my_sv_pvn_force_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags);
static
#else
extern char * DPPP_(my_sv_pvn_force_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags);
#endif
#if defined(NEED_sv_pvn_force_flags) || defined(NEED_sv_pvn_force_flags_GLOBAL)
#ifdef sv_pvn_force_flags
# undef sv_pvn_force_flags
#endif
#define sv_pvn_force_flags(a,b,c) DPPP_(my_sv_pvn_force_flags)(aTHX_ a,b,c)
#define Perl_sv_pvn_force_flags DPPP_(my_sv_pvn_force_flags)
char *
DPPP_(my_sv_pvn_force_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags)
{
STRLEN n_a = (STRLEN) flags;
return sv_pvn_force(sv, lp ? lp : &n_a);
}
#endif
#endif
#if (PERL_BCDVERSION < 0x5008008) || ( (PERL_BCDVERSION >= 0x5009000) && (PERL_BCDVERSION < 0x5009003) )
# define D_PPP_SVPV_NOLEN_LP_ARG &PL_na
#else
# define D_PPP_SVPV_NOLEN_LP_ARG 0
#endif
#ifndef SvPV_const
# define SvPV_const(sv, lp) SvPV_flags_const(sv, lp, SV_GMAGIC)
#endif
#ifndef SvPV_mutable
# define SvPV_mutable(sv, lp) SvPV_flags_mutable(sv, lp, SV_GMAGIC)
#endif
#ifndef SvPV_flags
# define SvPV_flags(sv, lp, flags) \
((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \
? ((lp = SvCUR(sv)), SvPVX(sv)) : sv_2pv_flags(sv, &lp, flags))
#endif
#ifndef SvPV_flags_const
# define SvPV_flags_const(sv, lp, flags) \
((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \
? ((lp = SvCUR(sv)), SvPVX_const(sv)) : \
(const char*) sv_2pv_flags(sv, &lp, flags|SV_CONST_RETURN))
#endif
#ifndef SvPV_flags_const_nolen
# define SvPV_flags_const_nolen(sv, flags) \
((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \
? SvPVX_const(sv) : \
(const char*) sv_2pv_flags(sv, D_PPP_SVPV_NOLEN_LP_ARG, flags|SV_CONST_RETURN))
#endif
#ifndef SvPV_flags_mutable
# define SvPV_flags_mutable(sv, lp, flags) \
((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \
? ((lp = SvCUR(sv)), SvPVX_mutable(sv)) : \
sv_2pv_flags(sv, &lp, flags|SV_MUTABLE_RETURN))
#endif
#ifndef SvPV_force
# define SvPV_force(sv, lp) SvPV_force_flags(sv, lp, SV_GMAGIC)
#endif
#ifndef SvPV_force_nolen
# define SvPV_force_nolen(sv) SvPV_force_flags_nolen(sv, SV_GMAGIC)
#endif
#ifndef SvPV_force_mutable
# define SvPV_force_mutable(sv, lp) SvPV_force_flags_mutable(sv, lp, SV_GMAGIC)
#endif
#ifndef SvPV_force_nomg
# define SvPV_force_nomg(sv, lp) SvPV_force_flags(sv, lp, 0)
#endif
#ifndef SvPV_force_nomg_nolen
# define SvPV_force_nomg_nolen(sv) SvPV_force_flags_nolen(sv, 0)
#endif
#ifndef SvPV_force_flags
# define SvPV_force_flags(sv, lp, flags) \
((SvFLAGS(sv) & (SVf_POK|SVf_THINKFIRST)) == SVf_POK \
? ((lp = SvCUR(sv)), SvPVX(sv)) : sv_pvn_force_flags(sv, &lp, flags))
#endif
#ifndef SvPV_force_flags_nolen
# define SvPV_force_flags_nolen(sv, flags) \
((SvFLAGS(sv) & (SVf_POK|SVf_THINKFIRST)) == SVf_POK \
? SvPVX(sv) : sv_pvn_force_flags(sv, D_PPP_SVPV_NOLEN_LP_ARG, flags))
#endif
#ifndef SvPV_force_flags_mutable
# define SvPV_force_flags_mutable(sv, lp, flags) \
((SvFLAGS(sv) & (SVf_POK|SVf_THINKFIRST)) == SVf_POK \
? ((lp = SvCUR(sv)), SvPVX_mutable(sv)) \
: sv_pvn_force_flags(sv, &lp, flags|SV_MUTABLE_RETURN))
#endif
#ifndef SvPV_nolen
# define SvPV_nolen(sv) \
((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \
? SvPVX(sv) : sv_2pv_flags(sv, D_PPP_SVPV_NOLEN_LP_ARG, SV_GMAGIC))
#endif
#ifndef SvPV_nolen_const
# define SvPV_nolen_const(sv) \
((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \
? SvPVX_const(sv) : sv_2pv_flags(sv, D_PPP_SVPV_NOLEN_LP_ARG, SV_GMAGIC|SV_CONST_RETURN))
#endif
#ifndef SvPV_nomg
# define SvPV_nomg(sv, lp) SvPV_flags(sv, lp, 0)
#endif
#ifndef SvPV_nomg_const
# define SvPV_nomg_const(sv, lp) SvPV_flags_const(sv, lp, 0)
#endif
#ifndef SvPV_nomg_const_nolen
# define SvPV_nomg_const_nolen(sv) SvPV_flags_const_nolen(sv, 0)
#endif
#ifndef SvPV_nomg_nolen
# define SvPV_nomg_nolen(sv) ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \
? SvPVX(sv) : sv_2pv_flags(sv, D_PPP_SVPV_NOLEN_LP_ARG, 0))
#endif
#ifndef SvPV_renew
# define SvPV_renew(sv,n) STMT_START { SvLEN_set(sv, n); \
SvPV_set((sv), (char *) saferealloc( \
(Malloc_t)SvPVX(sv), (MEM_SIZE)((n)))); \
} STMT_END
#endif
#ifndef SvMAGIC_set
# define SvMAGIC_set(sv, val) \
STMT_START { assert(SvTYPE(sv) >= SVt_PVMG); \
#ifndef SvSHARED_HASH
# define SvSHARED_HASH(sv) (0 + SvUVX(sv))
#endif
#ifndef HvNAME_get
# define HvNAME_get(hv) HvNAME(hv)
#endif
#ifndef HvNAMELEN_get
# define HvNAMELEN_get(hv) (HvNAME_get(hv) ? (I32)strlen(HvNAME_get(hv)) : 0)
#endif
#ifndef gv_fetchpvn_flags
#if defined(NEED_gv_fetchpvn_flags)
static GV* DPPP_(my_gv_fetchpvn_flags)(pTHX_ const char* name, STRLEN len, int flags, int types);
static
#else
extern GV* DPPP_(my_gv_fetchpvn_flags)(pTHX_ const char* name, STRLEN len, int flags, int types);
#endif
#if defined(NEED_gv_fetchpvn_flags) || defined(NEED_gv_fetchpvn_flags_GLOBAL)
#ifdef gv_fetchpvn_flags
# undef gv_fetchpvn_flags
#endif
#define gv_fetchpvn_flags(a,b,c,d) DPPP_(my_gv_fetchpvn_flags)(aTHX_ a,b,c,d)
#define Perl_gv_fetchpvn_flags DPPP_(my_gv_fetchpvn_flags)
GV*
DPPP_(my_gv_fetchpvn_flags)(pTHX_ const char* name, STRLEN len, int flags, int types) {
char *namepv = savepvn(name, len);
GV* stash = gv_fetchpv(namepv, TRUE, SVt_PVHV);
Safefree(namepv);
return stash;
}
#endif
#endif
#ifndef GvSVn
# define GvSVn(gv) GvSV(gv)
#endif
#ifndef isGV_with_GP
# define isGV_with_GP(gv) isGV(gv)
#endif
#ifndef gv_fetchsv
# define gv_fetchsv(name, flags, svt) gv_fetchpv(SvPV_nolen_const(name), flags, svt)
#endif
#ifndef get_cvn_flags
# define get_cvn_flags(name, namelen, flags) get_cv(name, flags)
#endif
#ifndef gv_init_pvn
# define gv_init_pvn(gv, stash, ptr, len, flags) gv_init(gv, stash, ptr, len, flags & GV_ADDMULTI ? TRUE : FALSE)
#endif
/* concatenating with "" ensures that only literal strings are accepted as argument
* note that STR_WITH_LEN() can't be used as argument to macros or functions that
* under some configurations might be macros
*/
#ifndef STR_WITH_LEN
# define STR_WITH_LEN(s) (s ""), (sizeof(s)-1)
#endif
#ifndef newSVpvs
# define newSVpvs(str) newSVpvn(str "", sizeof(str) - 1)
#endif
#ifndef newSVpvs_flags
# define newSVpvs_flags(str, flags) newSVpvn_flags(str "", sizeof(str) - 1, flags)
#endif
#ifndef newSVpvs_share
# define newSVpvs_share(str) newSVpvn_share(str "", sizeof(str) - 1, 0)
#endif
#ifndef sv_catpvs
# define sv_catpvs(sv, str) sv_catpvn(sv, str "", sizeof(str) - 1)
#endif
#endif
#ifndef hv_fetchs
# define hv_fetchs(hv, key, lval) hv_fetch(hv, key "", sizeof(key) - 1, lval)
#endif
#ifndef hv_stores
# define hv_stores(hv, key, val) hv_store(hv, key "", sizeof(key) - 1, val, 0)
#endif
#ifndef gv_fetchpvs
# define gv_fetchpvs(name, flags, svt) gv_fetchpvn_flags(name "", sizeof(name) - 1, flags, svt)
#endif
#ifndef gv_stashpvs
# define gv_stashpvs(name, flags) gv_stashpvn(name "", sizeof(name) - 1, flags)
#endif
#ifndef get_cvs
# define get_cvs(name, flags) get_cvn_flags(name "", sizeof(name)-1, flags)
#endif
#ifndef SvGETMAGIC
# define SvGETMAGIC(x) STMT_START { if (SvGMAGICAL(x)) mg_get(x); } STMT_END
#endif
/* That's the best we can do... */
#ifndef sv_catpvn_nomg
# define sv_catpvn_nomg sv_catpvn
#endif
if (virt && virt->svt_free)
virt->svt_free(aTHX_ sv, mg);
if (mg->mg_ptr && mg->mg_type != PERL_MAGIC_regex_global) {
if (mg->mg_len > 0)
Safefree(mg->mg_ptr);
else if (mg->mg_len == HEf_SVKEY) /* Questionable on older perls... */
SvREFCNT_dec(MUTABLE_SV(mg->mg_ptr));
else if (mg->mg_type == PERL_MAGIC_utf8)
Safefree(mg->mg_ptr);
}
if (mg->mg_flags & MGf_REFCOUNTED)
SvREFCNT_dec(mg->mg_obj);
Safefree(mg);
}
else
mgp = &mg->mg_moremagic;
}
if (SvMAGIC(sv)) {
if (SvMAGICAL(sv)) /* if we're under save_magic, wait for restore_magic; */
mg_magical(sv); /* else fix the flags now */
}
else {
SvMAGICAL_off(sv);
SvFLAGS(sv) |= (SvFLAGS(sv) & (SVp_IOK|SVp_NOK|SVp_POK)) >> PRIVSHIFT;
}
return 0;
}
#endif
#endif
if (sawinf) {
numtype &= IS_NUMBER_NEG; /* Keep track of sign */
numtype |= IS_NUMBER_INFINITY | IS_NUMBER_NOT_INT;
} else if (sawnan) {
numtype &= IS_NUMBER_NEG; /* Keep track of sign */
numtype |= IS_NUMBER_NAN | IS_NUMBER_NOT_INT;
} else if (s < send) {
/* we can have an optional exponent part */
if (*s == 'e' || *s == 'E') {
/* The only flag we keep is sign. Blow away any "it's UV" */
numtype &= IS_NUMBER_NEG;
numtype |= IS_NUMBER_NOT_INT;
s++;
if (s < send && (*s == '-' || *s == '+'))
s++;
if (s < send && isDIGIT(*s)) {
do {
s++;
} while (s < send && isDIGIT(*s));
}
#endif
/*
* The grok_* routines have been modified to use warn() instead of
* Perl_warner(). Also, 'hexdigit' was the former name of PL_hexdigit,
* which is why the stack variable has been renamed to 'xdigit'.
*/
#ifndef grok_bin
#if defined(NEED_grok_bin)
static UV DPPP_(my_grok_bin)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result);
static
#else
extern UV DPPP_(my_grok_bin)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result);
#endif
#if defined(NEED_grok_bin) || defined(NEED_grok_bin_GLOBAL)
#ifdef grok_bin
# undef grok_bin
#endif
#define grok_bin(a,b,c,d) DPPP_(my_grok_bin)(aTHX_ a,b,c,d)
#define Perl_grok_bin DPPP_(my_grok_bin)
UV
DPPP_(my_grok_bin)(pTHX_ const char *start, STRLEN *len_p, I32 *flags, NV *result)
{
const char *s = start;
STRLEN len = *len_p;
UV value = 0;
NV value_nv = 0;
const UV max_div_2 = UV_MAX / 2;
bool allow_underscores = *flags & PERL_SCAN_ALLOW_UNDERSCORES;
bool overflowed = FALSE;
if (!(*flags & PERL_SCAN_DISALLOW_PREFIX)) {
/* strip off leading b or 0b.
for compatibility silently suffer "b" and "0b" as valid binary
numbers. */
if (len >= 1) {
if (s[0] == 'b') {
s++;
len--;
}
else if (len >= 2 && s[0] == '0' && s[1] == 'b') {
s+=2;
value_nv += (NV)(bit - '0');
continue;
}
if (bit == '_' && len && allow_underscores && (bit = s[1])
&& (bit == '0' || bit == '1'))
{
--len;
++s;
goto redo;
}
if (!(*flags & PERL_SCAN_SILENT_ILLDIGIT))
warn("Illegal binary digit '%c' ignored", *s);
break;
}
if ( ( overflowed && value_nv > 4294967295.0)
#if UVSIZE > 4
|| (!overflowed && value > 0xffffffff )
#endif
) {
warn("Binary number > 0b11111111111111111111111111111111 non-portable");
}
*len_p = s - start;
if (!overflowed) {
*flags = 0;
return value;
}
*flags = PERL_SCAN_GREATER_THAN_UV_MAX;
if (result)
*result = value_nv;
return UV_MAX;
}
#endif
#endif
#ifndef grok_hex
#if defined(NEED_grok_hex)
static UV DPPP_(my_grok_hex)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result);
static
#else
extern UV DPPP_(my_grok_hex)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result);
#endif
#if defined(NEED_grok_hex) || defined(NEED_grok_hex_GLOBAL)
#ifdef grok_hex
# undef grok_hex
#endif
#define grok_hex(a,b,c,d) DPPP_(my_grok_hex)(aTHX_ a,b,c,d)
#define Perl_grok_hex DPPP_(my_grok_hex)
UV
DPPP_(my_grok_hex)(pTHX_ const char *start, STRLEN *len_p, I32 *flags, NV *result)
{
const char *s = start;
STRLEN len = *len_p;
UV value = 0;
NV value_nv = 0;
const UV max_div_16 = UV_MAX / 16;
bool allow_underscores = *flags & PERL_SCAN_ALLOW_UNDERSCORES;
bool overflowed = FALSE;
const char *xdigit;
if (!(*flags & PERL_SCAN_DISALLOW_PREFIX)) {
/* strip off leading x or 0x.
for compatibility silently suffer "x" and "0x" as valid hex numbers.
*/
if (len >= 1) {
if (s[0] == 'x') {
s++;
len--;
}
else if (len >= 2 && s[0] == '0' && s[1] == 'x') {
s+=2;
value_nv += (NV)((xdigit - PL_hexdigit) & 15);
continue;
}
if (*s == '_' && len && allow_underscores && s[1]
&& (xdigit = strchr((char *) PL_hexdigit, s[1])))
{
--len;
++s;
goto redo;
}
if (!(*flags & PERL_SCAN_SILENT_ILLDIGIT))
warn("Illegal hexadecimal digit '%c' ignored", *s);
break;
}
if ( ( overflowed && value_nv > 4294967295.0)
#if UVSIZE > 4
|| (!overflowed && value > 0xffffffff )
#endif
) {
warn("Hexadecimal number > 0xffffffff non-portable");
}
*len_p = s - start;
if (!overflowed) {
*flags = 0;
return value;
}
*flags = PERL_SCAN_GREATER_THAN_UV_MAX;
if (result)
*result = value_nv;
return UV_MAX;
}
#endif
#endif
#ifndef grok_oct
#if defined(NEED_grok_oct)
static UV DPPP_(my_grok_oct)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result);
static
#else
extern UV DPPP_(my_grok_oct)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result);
#endif
#if defined(NEED_grok_oct) || defined(NEED_grok_oct_GLOBAL)
#ifdef grok_oct
# undef grok_oct
#endif
#define grok_oct(a,b,c,d) DPPP_(my_grok_oct)(aTHX_ a,b,c,d)
#define Perl_grok_oct DPPP_(my_grok_oct)
UV
DPPP_(my_grok_oct)(pTHX_ const char *start, STRLEN *len_p, I32 *flags, NV *result)
{
const char *s = start;
STRLEN len = *len_p;
UV value = 0;
NV value_nv = 0;
const UV max_div_8 = UV_MAX / 8;
bool allow_underscores = *flags & PERL_SCAN_ALLOW_UNDERSCORES;
bool overflowed = FALSE;
for (; len-- && *s; s++) {
/* gcc 2.95 optimiser not smart enough to figure that this subtraction
out front allows slicker code. */
int digit = *s - '0';
if (digit >= 0 && digit <= 7) {
/* Write it in this wonky order with a goto to attempt to get the
compiler to make the common case integer-only loop pretty tight.
*/
&& (digit = s[1] - '0') && (digit >= 0 && digit <= 7))
{
--len;
++s;
goto redo;
}
/* Allow \octal to work the DWIM way (that is, stop scanning
* as soon as non-octal characters are seen, complain only iff
* someone seems to want to use the digits eight and nine). */
if (digit == 8 || digit == 9) {
if (!(*flags & PERL_SCAN_SILENT_ILLDIGIT))
warn("Illegal octal digit '%c' ignored", *s);
}
break;
}
if ( ( overflowed && value_nv > 4294967295.0)
#if UVSIZE > 4
|| (!overflowed && value > 0xffffffff )
#endif
) {
warn("Octal number > 037777777777 non-portable");
}
*len_p = s - start;
if (!overflowed) {
*flags = 0;
return value;
}
*flags = PERL_SCAN_GREATER_THAN_UV_MAX;
if (result)
*result = value_nv;
return UV_MAX;
}
#endif
#endif
#if !defined(my_snprintf)
#if defined(NEED_my_snprintf)
static int DPPP_(my_my_snprintf)(char * buffer, const Size_t len, const char * format, ...);
#endif
/* Hint: pv_escape
* Note that unicode functionality is only backported to
* those perl versions that support it. For older perl
* versions, the implementation will fall back to bytes.
*/
#ifndef pv_escape
#if defined(NEED_pv_escape)
static char * DPPP_(my_pv_escape)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, STRLEN * const escaped, const U32 flags);
static
#else
extern char * DPPP_(my_pv_escape)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, STRLEN * const escaped, const U32 flags);
#endif
#if defined(NEED_pv_escape) || defined(NEED_pv_escape_GLOBAL)
#ifdef pv_escape
# undef pv_escape
#endif
#define pv_escape(a,b,c,d,e,f) DPPP_(my_pv_escape)(aTHX_ a,b,c,d,e,f)
#define Perl_pv_escape DPPP_(my_pv_escape)
char *
DPPP_(my_pv_escape)(pTHX_ SV *dsv, char const * const str,
const STRLEN count, const STRLEN max,
STRLEN * const escaped, const U32 flags)
{
const char esc = flags & PERL_PV_ESCAPE_RE ? '%' : '\\';
const char dq = flags & PERL_PV_ESCAPE_QUOTE ? '"' : esc;
char octbuf[32] = "%123456789ABCDF";
STRLEN wrote = 0;
STRLEN chsize = 0;
STRLEN readsize = 1;
#if defined(is_utf8_string) && defined(utf8_to_uvchr_buf)
bool isuni = flags & PERL_PV_ESCAPE_UNI ? 1 : 0;
#endif
const char *pv = str;
const char * const end = pv + count;
octbuf[0] = esc;
if (!(flags & PERL_PV_ESCAPE_NOCLEAR))
sv_setpvs(dsv, "");
#if defined(is_utf8_string) && defined(utf8_to_uvchr_buf)
if ((flags & PERL_PV_ESCAPE_UNI_DETECT) && is_utf8_string((U8*)pv, count))
isuni = 1;
#endif
for (; pv < end && (!max || wrote < max) ; pv += readsize) {
const UV u =
#if defined(is_utf8_string) && defined(utf8_to_uvchr_buf)
isuni ? utf8_to_uvchr_buf((U8*)pv, end, &readsize) :
#endif
(U8)*pv;
const U8 c = (U8)u & 0xFF;
if (u > 255 || (flags & PERL_PV_ESCAPE_ALL)) {
if (flags & PERL_PV_ESCAPE_FIRSTCHAR)
chsize = my_snprintf(octbuf, sizeof octbuf,
"%" UVxf, u);
else
chsize = my_snprintf(octbuf, sizeof octbuf,
"%cx{%" UVxf "}", esc, u);
} else if (flags & PERL_PV_ESCAPE_NOBACKSLASH) {
chsize = 1;
} else {
if (c == dq || c == esc || !isPRINT(c)) {
chsize = 2;
switch (c) {
case '\\' : /* fallthrough */
case '%' : if (c == esc)
octbuf[1] = esc;
else
chsize = 1;
break;
} else if (chsize > 1) {
sv_catpvn(dsv, octbuf, chsize);
wrote += chsize;
} else {
char tmp[2];
my_snprintf(tmp, sizeof tmp, "%c", c);
sv_catpvn(dsv, tmp, 1);
wrote++;
}
if (flags & PERL_PV_ESCAPE_FIRSTCHAR)
break;
}
if (escaped != NULL)
*escaped= pv - str;
return SvPVX(dsv);
}
#endif
#endif
#ifndef pv_pretty
#if defined(NEED_pv_pretty)
static char * DPPP_(my_pv_pretty)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, char const * const start_color, char const * const end_color, const U32 flags);
static
#else
extern char * DPPP_(my_pv_pretty)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, char const * const start_color, char const * const end_color, const U32 flags);
#endif
#if defined(NEED_pv_pretty) || defined(NEED_pv_pretty_GLOBAL)
#ifdef pv_pretty
# undef pv_pretty
#endif
#define pv_pretty(a,b,c,d,e,f,g) DPPP_(my_pv_pretty)(aTHX_ a,b,c,d,e,f,g)
#define Perl_pv_pretty DPPP_(my_pv_pretty)
char *
DPPP_(my_pv_pretty)(pTHX_ SV *dsv, char const * const str, const STRLEN count,
const STRLEN max, char const * const start_color, char const * const end_color,
const U32 flags)
{
const U8 dq = (flags & PERL_PV_PRETTY_QUOTE) ? '"' : '%';
STRLEN escaped;
if (!(flags & PERL_PV_PRETTY_NOCLEAR))
sv_setpvs(dsv, "");
if (dq == '"')
sv_catpvs(dsv, "\"");
else if (flags & PERL_PV_PRETTY_LTGT)
sv_catpvs(dsv, "<");
if (start_color != NULL)
sv_catpv(dsv, D_PPP_CONSTPV_ARG(start_color));
pv_escape(dsv, str, count, max, &escaped, flags | PERL_PV_ESCAPE_NOCLEAR);
if (end_color != NULL)
sv_catpv(dsv, D_PPP_CONSTPV_ARG(end_color));
if (dq == '"')
sv_catpvs(dsv, "\"");
else if (flags & PERL_PV_PRETTY_LTGT)
sv_catpvs(dsv, ">");
if ((flags & PERL_PV_PRETTY_ELLIPSES) && escaped < count)
sv_catpvs(dsv, "...");
return SvPVX(dsv);
}
#endif
#endif
#ifndef pv_display
#if defined(NEED_pv_display)
t/OnDisk.pl view on Meta::CPAN
tie %tied_hash, $class, $test_file;
my $scalar= scalar(%tied_hash);
ok($scalar,"scalar works");
my $obj= tied(%tied_hash);
is($obj->get_comment, $this_comment, "comment works as expected");
is($obj->get_hdr_variant, $variant, "variant is as expected");
is($obj->get_hdr_num_buckets, 0+keys %$source_hash,"num_buckets is as expected");
my @ofs=(
$obj->get_hdr_state_ofs,
$obj->get_hdr_table_ofs,
$obj->get_hdr_key_flags_ofs,
$obj->get_hdr_val_flags_ofs,
$obj->get_hdr_str_buf_ofs,
);
my @srt_ofs= sort{ $a <=> $b } @ofs;
is("@ofs","@srt_ofs","offsets in the right order");
my (@got_keys,@got_fetch_values,@want_keys);
{
my @bad;
foreach my $key (sort keys %$source_hash) {
push @want_keys, $key;
t/OnDisk.pl view on Meta::CPAN
is_deeply(\@got_fetch_values,\@want_values,"values in both are same ($title)");
is_deeply(\@got_each_values,\@want_values,"values in both are same ($title)");
{
my @bad;
foreach my $idx (0..$#got_keys) {
if (utf8::is_utf8($got_keys[$idx]) != utf8::is_utf8($want_keys[$idx])) {
push @bad, [ $got_keys[$idx], $want_keys[$idx] ];
}
}
is(0+@bad,0,"no keys with differing utf8 flags ($title)")
or diag Dumper($bad[0]);
}
} else {
ok(0,"test cannot pass if make_file dies") for 1..17;
}
}
}
}
}
( run in 2.092 seconds using v1.01-cache-2.11-cpan-94b05bcf43c )