AI-MicroStructure
view release on metacpan or search on metacpan
$ micro ufo all; # all words of the ufo structure
$ micro ufo 5; # 5 random words of the ufo structure
$ micro any 10; # 10 random words of any structure you have created
$ micro --init # initializes active memory
$ micro --export # export relations from couchdb into git repo and tag data
# oneliners i like to use
$ for i in `micro structures`; do echo $i; done; # echos all the structures
$ for i in `micro ufo all`; do echo $i; done; # echos all words in ufo
$ for i in `micro structures`; do micro all $i; done; # echos all stuctures all words
bin/from-folder.pl view on Meta::CPAN
}
find(\&translate, "$TOP");
p @{[keys %$files,reverse @ARGV,$storage]};
__DATA__
our $c = AI::MicroStructure::Context->new(@ARGV);
$c->retrieveIndex($PWD."/t/docs"); #"/home/santex/data-hub/data-hub" structures=0 text=1 json=1
my $style = {};
$style->{explicit} = 1;
ok($c->simpleMixedSearch($style,$_)) && ok($c->play($style,$_)) for
qw(atom antimatter planet);
nogo="http:\/\/|Articles|Artikel|Wikipedia|Commons|Tracking_|Hidden_|Talk_|_talk|Use_dmy_dates|_Diskussion|Pages_containing "
function getcat(){
search=$(perl -e "print ucfirst('$1')");
q="http://$MICRO_LANG.wikipedia.org/wiki/$search";
for i in `mojo get "$q" a attr href | egrep "^\/wiki\/(Category|Kategorie)" | egrep -vi "("$nogo")" | sort | sed -s "s/\/wiki\///"`; do q=$i; echo $i | urldecode; done
}
getcat $search
#qq=$(echo "$search" | sed "s/^.*.://" | tr "_" "|");
#q=$1; cat=$(for i in `getcat "$q"`; do getcat "$i"; done); echo "$cat" | data-freq | egrep -v "http" | sed -s "s/^.*.: //" | egrep -i "($qq)"
#q=$1; cat=$(for i in `getcat "$q"`; do qq=$(echo "$q" | sed -s "s/^.*.\://"); getcat "$i"; done); echo "$cat" | egrep -vi "http:" | data-freq | sort -n
#echo -e "\n"
bin/getnice view on Meta::CPAN
if [ ! $q ]
then
q="ufo"
fi
if [ ! $dir ]
then
dir="/home/santex/data-hub/stock/KnowledgeInterDisciplinary/"
fi
j=$(for i in `egrep -rli "$q" | grep List_of_`; do
echo $i;
#echo $i | xargs -0 json-printer \-f | egrep "^\".*.$" | egrep -v ":|http|wiki|index|\/|Externallinks|References|Seealso|Listof|#|[1-9][0-9][0-9][0-9]" | tr -d "\",";
done);
echo "$j"| egrep -i "^$q|_$q"
bin/localfiles view on Meta::CPAN
#!/usr/bin/env perl
use File::Find::Rule;
use strict;
use warnings;
use JSON;
use Cache::Memcached::Fast;
use AI::MicroStructure;
use Data::Dumper;
my $data={};
our $memd = new Cache::Memcached::Fast({
servers => [ { address => 'localhost:11211', weight => 2.5 }],
namespace => 'my:',
connect_timeout => 0.2,
io_timeout => 0.1,
close_on_error => 1,
compress_threshold => 100_000,
compress_ratio => 0.9,
bin/localfiles view on Meta::CPAN
});
my @result = ();
my $rule;
foreach(@ARGV){
$rule = File::Find::Rule->file->name("*$_*.*")->start("/home/santex/data-hub/stock/" ,"/home/santex/Pictures/1");
while ( defined ( my $image = $rule->match ) ) {
push @result,$image;
}
print join("\n", @result);
}
bin/localfiles view on Meta::CPAN
->name('*'.lc($match).'*\.(png|jpg|gif|svg)')
->in(@INC);
#my $dat = {};
#foreach(@images){
#my @d = split("/",$_);
# $dat->{shift @d} = $_;
#}
#@images = values $dat;#@images[0..500] unless $#images<500;
#$data->{images}=[@images];
#my $cmd = sprintf( "montage '%s' /home/santex/Pictures/montage/".$match."-all.jpg",join("' '",@{$data->{images}}));
#`$cmd`;
my @files = File::Find::Rule->file()
->name('*'.ucfirst($match).'*\.(pdf|json|txt|xml)')
->in(@INC);
# print join "\n",sort{$a cmp $b}@files;
$data->{files}=[@files];
my $j = encode_json($data);
$memd->set("localfiles_".$match,$j);
print $j;
}else{
$data = decode_json( $xx);
print $xx;
}
if(!-f $module){
print "no\n\e[00;33m $module\e[00m";
print "create like\n\e[00;33mmicro $structure new verbose\e[00m";
exit;
}
# load the remote structure if needed
if ( $conf{remote} || $conf{check} || $conf{sources}) {
eval "require '$module';";
die "structure '$structure' is not updatable!\n"
unless $module->has_remotelist();
}
# informative options
print STDERR
"meta, a simple front-end to AI::MicroStructure version $AI::MicroStructure::VERSION\n"
if $conf{version};
print STDERR $main->help if $conf{help};
print map "$_\n", AI::MicroStructure->structures if $conf{structures};
bin/micro-sense view on Meta::CPAN
#!/usr/bin/perl -X
use strict;
use AnyDBM_File;
use Fcntl;
use Data::Dumper;
use JSON;
use AI::MicroStructure::WordBlacklist;
my @set;
my $data = {};
my %structure;
my $fresh=1;
my $wn;
my $debug = 0;
my $words = 1;
my $count = 0;
if( grep{/\bdebug\b/} @ARGV ){$debug = 1; cleanArgs("debug"); };
if( grep{/\bwords\b/} @ARGV ){$words = 1; cleanArgs("words"); };
bin/micro-sense view on Meta::CPAN
# print forms($wn);
# die;
foreach(split("\n",$wn)){
if($_ =~ /^([0-9].*?)[.](.*?)$/){
my($sense,$text)=($1,$2);
my @set = map{$_=~s/"|'|`//g; trim($_)}grep{/[a-z|A-Z]/}split("\\(|--|\\)|; \"",$text);
push @{$data->{'senses'}->{$sense}->{"basics"}},@set;
}
}
#last if $line =~ /quit/;
#next if $line =~ /^[a-z|A-Z]/i;
# redo;
#}
#//map{$_=map{ trim($_)}split("=>",$_)unless($_=~/Coordinate Terms/)}
#print Dumper [split("sense",`wn $line -coorn`)];
push @{$data->{'search'}},grep{/[a-z|A-Z]/}split("\n|,",lc `micro-wnet $line words`) unless(!$words);
#push @{$data->{'coordinate'}},map{$_=trim($_); @_=grep(/[A-Z]|[0-9]|[a-z]/,map{$_=trim($_)}split("\n|=>",$_))}split(/Sense/,`wn $line -coorn`) unless(!$words);
#shift @{$data->{'coordinate'}} && shift @{$data->{'search'}} for(0..1);
#@{$data->{'search'}};
END{
$data = {"rows"=>$data,"senses"=>[sort keys %{$data->{'senses'}}]};
my $utf8_encoded_json_text = encode_json($data);
if($debug == 1) {
print Dumper $data,$wn;
}
print $utf8_encoded_json_text;
}
bin/micro-wiki view on Meta::CPAN
my $state = AI::MicroStructure::Util::config();
my @CWD = $state->{cwd};
our $config = $state->{cfg};
die("require a argument") unless($ARGV[0]);
our $doc ={};
our @links;
our $linkdata = {};
our $result;
our $odir = "";
my $url = $ARGV[0];
my @inx;
my @test;
my $search="";
my $TOP="wikipedia";
bin/micro-wiki view on Meta::CPAN
my $content ;
my $response ="";
my @book = ();
$response = $ua->get(sprintf("%s%s",$config->{wikipedia}, ucfirst($url)));
my $doc={};
my $linkdata={};
my $wiki = WWW::Wikipedia->new();
my $hs = HTML::Strip->new();
my $result = $wiki->search(ucfirst $url);
if (defined($result) && $result->text() ) {
bin/remote.pl view on Meta::CPAN
use strict;
use AI::MicroStructure (); # do not export metaname and friends
#use AI::MicroStructure::RemoteKnowledge;
use List::Util qw( shuffle );
use Carp;
#our @ISA = qw( AI::MicroStructure::RemoteKnowledge );
sub init {
my $class = caller(0);
my $data = AI::MicroStructure->load_data($class);
no strict 'refs';
my $sep = ${"$class\::Separator"} ||= '/';
my $tail = qr/$sep?[^$sep]*$/;
# compute all categories
my @categories = ( [ $data->{names}, '' ] );
while ( my ( $h, $k ) = @{ shift @categories or []} ) {
if ( ref $h eq 'HASH' ) {
push @categories,
map { [ $h->{$_}, ( $k ? "$k$sep$_" : $_ ) ] } keys %$h;
}
else { # leaf
my @items = split /\s+/, $h;
while ($k) {
push @{ ${"$class\::KnowHow"}{$k} }, @items;
$k =~ s!$tail!!;
}
}
}
${"$class\::Default"} = $data->{default} || ':all';
${"$class\::Theme"} = ( split /::/, $class )[-1];
*{"$class\::import"} = sub {
my $callpkg = caller(0);
my $theme = ${"$class\::Theme"};
my $meta = $class->new;
*{"$callpkg\::meta$theme"} = sub { $meta->name(@_) };
};
${"$class\::meta"} = $class->new();
bin/remote.pl view on Meta::CPAN
my @items;
my @srcs = $class->sources($_[1]);
my $ua = LWP::UserAgent->new( env_proxy => 1 );
foreach my $src (@srcs) {
my $res = $ua->request( HTTP::Request->new( GET => $src ) );
if ( ! $res->is_success() ) {
carp "Failed to get content at $src (" . $res->status_line();
return;
}
# extract, cleanup and return the data
# if decoding the content fails, we just deal with the raw content
push @items =>
$class->extract( $res->decoded_content() || $res->content() );
}
# return unique items
my %seen;
return grep { !$seen{$_}++ } @items;
}
bin/remote.pl view on Meta::CPAN
package AI::MicroStructure::Knowledge;
use strict;
use AI::MicroStructure (); # do not export metaname and friends
#use AI::MicroStructure::RemoteKnowledge;
use List::Util qw( shuffle );
use Carp;
our @ISA = qw( AI::MicroStructure::RemoteKnowledge );
sub init {
my ($self, $data) = @_;
my $class = caller(0);
$data ||= AI::MicroStructure->load_data($class);
croak "The optional argument to init() must be a hash reference"
if ref $data ne 'HASH';
no strict 'refs';
no warnings;
${"$class\::Theme"} = ( split /::/, $class )[-1];
@{"$class\::Knowledge"} = split /\s+/, $data->{names};
*{"$class\::import"} = sub {
my $callpkg = caller(0);
my $theme = ${"$class\::Theme"};
my $meta = $class->new();
*{"$callpkg\::meta$theme"} = sub { $meta->name(@_) };
};
${"$class\::meta"} = $class->new();
}
sub name {
lib/AI/MicroStructure.pm view on Meta::CPAN
our $VERSION = '0.20';
our $Structure = 'any'; # default structure
our $CODESET = 'utf8';
our $LANG = '';
our %MICRO;
our %MODS;
our %ALIEN;
our $str = "[A-Z]";
our $special = "any";
our $search;
our $data={};
our $item="";
our @items;
our @a=();
our ($init,$new,$drop,$available,$lib,
$list,$use,$off,$switch,$mirror,
$version,$help,$write,$verbose) = (0,0,0,0,0,0,0,0,0,0,0,0,0,0);
eval "\$$_=1; " for @ARGV;
if( grep{/\bnew\b/} @ARGV ){ $new = 1; cleanArgs("new"); }
if( grep{/\bwrite\b/} @ARGV ){ $write = 1; cleanArgs("write"); };
if( grep{/\bdrop\b/} @ARGV ){ $drop = 1; cleanArgs("drop"); };
lib/AI/MicroStructure.pm view on Meta::CPAN
}
return %$structures;
}
$MICRO{$_} = 0 for keys %{{__PACKAGE__->find_structures(@CWD)} };
$MODS{$_} = $_ for keys %{{__PACKAGE__->find_modules(@INC)} };
$search = join("|",keys %MICRO);
BEGIN{
use File::HomeDir;
my $fileDir = File::HomeDir->my_home . "/data-hub/structures/";
my $fileSpec = File::HomeDir->my_home . "/data-hub/structures/any.pm";
if ( -e $fileSpec ) {
} else {
mkdir($fileDir);
warn "missing:$fileSpec";
}
}
lib/AI/MicroStructure.pm view on Meta::CPAN
$x->{"structures"} = {};
foreach my $con (@{$x->{"all_structures"}}){
next unless($con!~/any/);
my @in = split("\n",eval{`cat $state->{path}->{"cwd/structures"}/$con.pm`;});
$x->{"structures"}->{$state->{path}->{"cwd/structures"}}->{$con}->{name} =
[grep{$_}grep {!/(our|my|use|sub|use|package|#|__|1)/}split("\n",`cat $state->{path}->{"cwd/structures"}/$con.pm`)];#,
$x->{"structures"}->{$state->{path}->{"cwd/structures"}}->{$con}->{files} =
[split("\n",`ls -R /home/santex/repos/KnowledgeInterDisciplinary/data/json | egrep -i "($con)";`)];
}
return $x;
}
sub import {
my $class = shift;
my @structures = ( grep { $_ eq ':all' } @_ )
? ( 'foo', grep { !/^(?:foo|:all)$/ } keys %MICRO ) # 'foo' is still first
: @_;
$Structure = $structures[0] if @structures;
lib/AI/MicroStructure.pm view on Meta::CPAN
eval $code;
$MICRO{$structure} = 1; # loaded
# export the microstructure() function
no strict 'refs';
my $callpkg = caller;
*{"$callpkg\::micro$structure"} = sub { $micro->name( $structure, @_ ) };
}
}
# load the content of __DATA__ into a structure
# this class method is used by the other AI::MicroStructure classes
sub load_data {
my ($class, $structure ) = @_;
$data = {};
my $fh;
{ no strict 'refs'; $fh = *{"$structure\::DATA"}{IO}; }
my $item;
my @items;
$$item = "";
{
if(defined($fh)){
local $_;
while (<$fh>) {
/^#\s*(\w+.*)$/ && do {
push @items, $item;
$item = $data;
my $last;
my @keys = split m!\s+|\s*/\s*!, $1;
$last = $item, $item = $item->{$_} ||= {} for @keys;
$item = \( $last->{ $keys[-1] } = "" );
next;
};
$$item .= $_;
}
}
}
# clean up the items
for( @items, $item ) {
$$_ =~ s/\A\s*//;
$$_ =~ s/\s*\z//;
$$_ =~ s/\s+/ /g;
}
return $data;
}
#fitnes
sub fitnes {
my $self = shift;
return sha1_hex($self->structures());
##my ($config,$structure, $config ) = (shift,[$self->structures()]); FIXME
}
# main function
sub microname { $micro->name( @_ ) };
sub shitname {
lib/AI/MicroStructure.pm view on Meta::CPAN
my %items;
my $items = $micro->name(0);
$items{$_}++ for $micro->name(0);
my $key=sprintf("%s",$micro->structure);
$all->{$key}=[$test->[1],$micro->name($items)];
}
return $all;
}
sub save_cat {
my $self = shift;
my $data = shift;
my $dat;
my $ret = "";
foreach my $key(sort keys %{$data} ) {
next unless($_);
#ref $hash->{$_} eq "HASH"
if(ref $data->{$key} eq "HASH"){
$ret .= "\n".$self->save_cat($data->{$key});
}else{
$dat = $data->{$key};
$dat =~ s/^|,/\n/g;
$dat =~ s/\n\n/\n/g;
$dat =~ s/->\n|[0-9]\n//g;
$ret .= "# ".($key=~/names|default|[a-z]/?$key:"names ".$key);
$ret .= "\n ".$dat."\n";
}
}
return $ret;
}
sub save_default {
my $self = shift;
my $data = shift;
my $line = shift;
my $dat = {};
my @in = ();
my $active=0;
$line = $Structure unless($line);
foreach(@{$data->{rows}->{"coordinate"}}){
if($_ eq $line){ $active=1; }
if(1+$line eq $_){ $active=0; }
if($active==1){
$_=~s/,//g;
$_ = $self->trim($_);
$dat->{names}->{$_}=$_ unless(defined($dat->{names}->{$_}));
}
}
foreach(@{$data->{rows}->{"search"}}){
if($_ eq $line){ $active=1; }
if(1+$line eq $_){ $active=0; }
if($active==1){
$_=~s/,//g;
$_ = $self->trim($_);
$dat->{names}->{$_}=$_ unless(defined($dat->{names}->{$_}));
}
}
push @in , keys %{$dat->{names}};
push @in , values %{$data->{names}};
$dat->{names} = join(" ",@in);
$dat->{names} =~ s/$line(.*?)\-\>(.*?) [1-9] /$1 $2/g;
$dat->{names} =~ s/ / /g;
my @file = grep{/$Structure/}map{File::Glob::bsd_glob(
File::Spec->catfile( $_, ($structdir,"*.pm") ) )}@CWD;
if(@file){
open(SELF,"+<$file[0]") || die $!;
while(<SELF>){last if /^__DATA__/}
truncate(SELF,tell SELF);
print SELF $self->save_cat($dat);
truncate(SELF,tell SELF);
close SELF;
}
}
sub openData{
my $self = shift;
my @datax = ();
if(<DATA>){
@datax = <DATA>;
while(@datax){
chomp;
if($_=~/^#\s*(\w+.*)$/) {
@a=split(" ",$1);
if($#a){
$data->{$a[0]}->{$a[1]}="";
}else{
$data->{$1}="";
}
$item=$1 unless($#a);
}else{
my @keys = split m!\s+|\s*/\s*!,$_;
foreach(sort @keys){
if($#a){
$data->{$a[0]}->{$a[1]} .= " $_" unless($_ eq "");
}else{
$data->{$item} .= " $_" unless($_ eq "");
}
}
};
}
}
return $data;
}
sub getBlank {
my $self = shift;
my $structure = shift;
my $data = shift;
my $usage = "";
$usage = "#!/usr/bin/perl -W\n";
$usage .= << "EOC";
package AI::MicroStructure::$structure;
use strict;
use AI::MicroStructure::List;
our \@ISA = qw( AI::MicroStructure::List );
our \@List = qw( \@{\$structures{\$structure}} );
__PACKAGE__->init();
1;
EOC
my $new = {};
foreach my $k
(grep{!/^[0-9]/}map{$_=$self->trim($_)}@{$data->{rows}->{"search"}}){
$k =~ s/[ ]/_/g;
$k =~ s/[\(]|[\)]//g;
next if($k=~/synonyms|hypernyms/);
print $k;
$new->{$k}=[map{$_=[map{$_=$self->trim($_)}split("\n|, ",$_)]}
grep{!/synonyms|hypernyms/}split("sense~~~~~~~~~",
lc `micro-wnet $k`)];
next unless(@{$new->{$k}});
# $new->{$k}=~s/Sense*\n(.*?)\n\n/$1/g;
# @{$new->{$k}} = [split("\n|,",$new->{$k})];
$data->{rows}->{"ident"}->{md5_base64($new->{$k})} = $new->{$k};
}
my $list = join("\n",sort keys %$new);
# $list =~ s/_//g;
$usage .= "
__DATA__
# names
".$list;
}
sub save_new {
my $self = shift;
my $StructureName = shift;
my $data = shift;
if($StructureName){
#$StructureName = lc $self->trim(`micro`) unless($StructureName);
my $file = "$absstructdir/$StructureName.pm";
print `mkdir -p $absstructdir` unless(-d $absstructdir);
my $fh;
open($fh,">$file") || warn @{[$file,$!]};
print $fh $self->getBlank($StructureName,$data);
close $fh;
$Structure = $StructureName;
push @CWD,$file;
return 1;
}
}
sub drop {
my $self = shift;
my $StructureName = shift;
my @file = grep{/$StructureName.pm/}map{File::Glob::bsd_glob(
lib/AI/MicroStructure.pm view on Meta::CPAN
printf(__PACKAGE__->help());
exit(0);
}
if($drop == 1) {
__PACKAGE__->drop($StructureName);
exit 0;
}
if($new==1){
use Term::ReadKey;
use JSON;
my $data = decode_json(lc`micro-sense $StructureName words`);
my $char;
my $line;
my $senses=@{$data->{"senses"}};
$senses= 0 unless($senses);
if(!$verbose){
printf("\n
\033[0;34m
%s
Type: the number you choose 1..$senses
\033[0m",__PACKAGE__->usage($StructureName,$senses,$data));
}
$line = 1 unless($senses != 1);
if($verbose){
$line=1;
}
chomp($line = <STDIN>) unless($line);
my $d = join("#",@{$data->{rows}->{search}});
my @d = grep{/^$line#/}split("sense~~~~~~~~~",$d);
@{$data->{rows}->{"search"}}=split("#",join("",@d));
if($line>0){
__PACKAGE__->save_new($StructureName,$data,$line);
exit 0;
}else{
printf "your logic is today impaired !!!\n";
exit 0;
}
}
if($write == 1) {
__PACKAGE__->save_default();
}
}
sub usage {
my $self = shift;
my $search = shift;
my $senseNr = shift;
my $data = shift;
my $usage = << 'EOT';
.--'"""""--.>_
.-' o\\b.\o._o.`-.
.-'.- ) \d888888888888b.
/.' b Y8888888888888888b.
.-'. 8888888888888888888888888b
/ o888 Y Y8888888888888888888888b
/ d888P/ /| Y"Y8888888888888888888b
J d8888/| Y .o._. "Y8888888888888Y" \
|d Y888b|obd88888bo. """Y88888Y' .od8
lib/AI/MicroStructure.pm view on Meta::CPAN
"""""""""""""""""""""""""""""""""""""""""""""""
EOT
$usage =~ s/<0>_/\033[0;32mThe word $search\033[255;34m/g;
$usage =~ s/<1>_/\033[0;32mhas $senseNr concept's\033[255;34m/g;
$usage =~ s/<2>_/\033[0;32mwe need to find out the which one\033[255;34m/g;
$usage =~ s/<3>_/\033[0;32mto use for our new,\033[255;34m/g;
$usage =~ s/<4>_/\033[0;32mmicro-structure,\033[255;34m/g;
$usage =~ s/<5>_//g;
my @row = ();
my $ii=0;
foreach my $sensnrx (sort keys %{$data->{"rows"}->{"senses"}})
{
my $row = $data->{"rows"}->{"senses"}->{$sensnrx};
my $txt="";
foreach(@{$row->{"basics"}}[1..2]){
next unless($_);
$txt .= sprintf("\033[0;31m %s\033[255;34m",$_);
}
$txt .= sprintf("",$_);
$usage =~ s/$sensnrx>_/($sensnrx):$txt/g;
if($sensnrx>9){
$usage .= sprintf("\n(%d):%s",$sensnrx,$txt);
}
lib/AI/MicroStructure/List.pm view on Meta::CPAN
use strict;
use AI::MicroStructure (); # do not export metaname and friends
use AI::MicroStructure::RemoteList;
use List::Util qw( shuffle );
use Carp;
our @ISA = qw( AI::MicroStructure::RemoteList );
our $VERSION = '0.20';
sub init {
my ($self, $data) = @_;
my $class = caller(0);
$data ||= AI::MicroStructure->load_data($class);
croak "The optional argument to init() must be a hash reference"
if ref $data ne 'HASH';
no strict 'refs';
no warnings;
${"$class\::Theme"} = ( split /::/, $class )[-1];
@{"$class\::List"} = do { my %seen;
grep !$seen{$_}++, split /\s+/, $data->{names} };
*{"$class\::import"} = sub {
my $callpkg = caller(0);
my $structure = ${"$class\::Theme"};
my $meta = $class->new();
*{"$callpkg\::meta$structure"} = sub { $meta->name(@_) };
};
${"$class\::meta"} = $class->new();
}
sub name {
lib/AI/MicroStructure/MultiList.pm view on Meta::CPAN
use strict;
use AI::MicroStructure (); # do not export metaname and friends
use AI::MicroStructure::RemoteList;
use List::Util qw( shuffle );
use Carp;
our @ISA = qw( AI::MicroStructure::RemoteList );
our $VERSION = '0.20';
sub init {
my ($self, $data) = @_;
my $class = caller(0);
$data ||= AI::MicroStructure->load_data($class);
no strict 'refs';
# note: variables mentioned twice to avoid a warning
my $sep = ${"$class\::Separator"} = ${"$class\::Separator"} ||= '/';
my $tail = qr/$sep?[^$sep]*$/;
# compute all categories
my @categories = ( [ $data->{names}, '' ] );
while ( my ( $h, $k ) = @{ shift @categories or []} ) {
if ( ref $h eq 'HASH' ) {
push @categories,
map { [ $h->{$_}, ( $k ? "$k$sep$_" : $_ ) ] } keys %$h;
}
else { # leaf
my @items = split /\s+/, $h;
while ($k) {
push @{ ${"$class\::MultiList"}{$k} }, @items;
$k =~ s!$tail!!;
}
}
}
${"$class\::Default"} = ${"$class\::Default"} = $data->{default} || ':all';
${"$class\::Theme"} = ${"$class\::Theme"} = ( split /::/, $class )[-1];
*{"$class\::import"} = sub {
my $callpkg = caller(0);
my $structure = ${"$class\::Theme"};
my $meta = $class->new;
*{"$callpkg\::meta$structure"} = sub { $meta->name(@_) };
};
${"$class\::meta"} = ${"$class\::meta"} = $class->new();
lib/AI/MicroStructure/RemoteList.pm view on Meta::CPAN
)
: ( GET => $src )
);
my $res = $ua->request( $request );
if ( ! $res->is_success() ) {
carp "Failed to get content at $src (" . $res->status_line();
return;
}
# extract, cleanup and return the data
# if decoding the content fails, we just deal with the raw content
push @items =>
$class->extract( $res->decoded_content() || $res->content(),
$category || () );
}
# return unique items
my %seen;
return grep { !$seen{$_}++ } @items;
lib/AI/MicroStructure/RemoteList.pm view on Meta::CPAN
AI::MicroStructure::RemoteList - Retrieval of a remote source for a structure
=head1 SYNOPSIS
package AI::MicroStructure::contributors;
use strict;
use AI::MicroStructure::List;
our @ISA = qw( AI::MicroStructure::List );
# data regarding the remote source
our %Remote = (
source =>
'http://search.cpan.org/dist/AI-MicroStructure/CONTRIBUTORS',
extract => sub {
my $content = shift;
my @items =
map { AI::MicroStructure::RemoteList::tr_nonword($_) }
map { AI::MicroStructure::RemoteList::tr_accent($_) }
$content =~ /^\* (.*?)\s*$/gm;
return @items;
lib/AI/MicroStructure/RemoteList.pm view on Meta::CPAN
To be able to fetch remote items, an C<AI::MicroStructure> structure must
define the package hash variable C<%Remote> with the appropriate keys.
The keys are:
=over 4
=item C<source>
The URL where the data is available. The content will be passed to the
C<extract> subroutine.
Because of the various way the data can be made available on the web
and can be used in L<AI::MicroStructure>, this scheme has evolved to
support several cases:
Single source URL:
source => $url
Multiple source URL:
source => [ $url1, $url2, ... ]
lib/AI/MicroStructure/RemoteList.pm view on Meta::CPAN
For structures with categories, it's possible to attach a URL for each
category:
source => {
category1 => $url1,
category2 => $url2,
...
}
In the case where the C<source> is an array or a hash reference, an
extra case is supported, in case the source data can only be obtained
via a C<POST> request. In that case, the source should be provided as
either:
source => [
[ $url1 => $data1 ],
[ $url2 => $data2 ],
...
]
or
source => {
category1 => [ $url1 => $data1 ],
category2 => [ $url2 => $data2 ],
...
}
It is possible to mix C<POST> and C<GET> URL:
source => [
$url1, # GET
[ $url2 => $data2 ], # POST
...
]
or
source => {
category1 => $url1, # GET
category2 => [ $url2 => $data2 ], # POST
...
}
This means that even if there is only one source and a C<POST> request
must be used, then it must be provided as a list of a single item:
source => [ [ $url => $data ] ]
=item C<extract>
A reference to a subroutine that extracts a list of items from a string.
The string is meant to be the content available at the URL stored in
the C<source> key.
The coderef may receive an optional parameter corresponding to the name of
the category (useful if the coderef must behave differently depending on
the category).
=back
C<LWP::Simple> is used to download the remote data.
All existing C<AI::MicroStructure> behaviours
(C<AI::MicroStructure::List> and C<AI::MicroStructure::Locale> are
subclasses of C<AI::MicroStructure::RemoteList>.
=head1 METHODS
As an ancestor, this class adds the following methods to an
C<AI::MicroStructure> structure:
lib/AI/MicroStructure/RemoteList.pm view on Meta::CPAN
Returns the list of items available at the remote source, or an empty
list in case of error.
=item has_remotelist()
Return a boolean indicating if the C<source> key is defined (and therefore
if the structure actually has a remote list).
=item source()
Return the data structure containing the source URLs. This can be quite
different depending on the class: a single scalar (URL), an array
reference (list of URLs) or a hash reference (each value being either
a scalar or an array reference) for structures that are subclasses of
C<AI::MicroStructure::MultiList>.
=item sources( [ $category ] )
Return the list of source URL. The C<$category> parameter can be used
to select the sources for a sub-category of the structure (in the case of
C<AI::MicroStructure::MultiList>).
lib/AI/MicroStructure/Util.pm view on Meta::CPAN
package AI::MicroStructure::Util;
use strict;
use Cwd;
use Config::Auto;
use Env qw/PWD/;
sub get_cwd {
my @CWD; if (!-e ".micro") {
push @CWD, $ENV{HOME}."/data-hub";
} else {
push @CWD, getcwd();
}
return @CWD;
}
sub load_config {
my @CWD = AI::MicroStructure::Util::get_cwd();
my $config = {};#Config::Auto::parse(".micro", path => @CWD) || {};
lib/AI/MicroStructure/germany.pm view on Meta::CPAN
# names Herzebrock-Clarholz 33442
# names Langenberg 33449
# names Bielefeld 33602 33604 33605 33607 33609 33611 33613 33615 33617 33619 33647 33649 33659 33689 33699 33719 33729 33739
# names Versmold 33775
# names Oerlinghausen 33813
# names Werther 33824 99735
# names Borgholzhausen 33829
# names Kassel 34117 34119 34121 34123 34125 34127 34128 34130 34131 34132 34134
# names Melsungen 34212
# names Baunatal 34225
# names Fuldatal 34233
# names Vellmar 34246
# names Lohfelden 34253
# names Kaufungen 34260
# names Niestetal 34266
# names Schauenburg 34270
# names Gudensberg 34281
# names Spangenberg 34286
# names Zierenberg 34289
# names Ahnatal 34292
# names Helsa 34298
lib/AI/MicroStructure/germany.pm view on Meta::CPAN
# names Hauneck 36282
# names Hohenroda 36284
# names Neuenstein 36286 74632
# names Breitenbach 06528 06712 37327 66916
# names Herzberg 16835 19374
# names Friedewald 36289 57520
# names Alsfeld 36304
# names Schwalmtal 36318 41366
# names Kirtorf 36320
# names Grebenau 36323
# names Feldatal 36325
# names Antrifttal 36326
# names Romrod 36329
# names Lauterbach 36341 78730 99826
# names Grebenhain 36355
# names Herbstein 36358
# names Wartenberg 36367 85456
# names Lautertal 36369 64686 96486
# names Sinntal 36391
# names Steinau 21775
# names Freiensteinau 36399
lib/AI/MicroStructure/germany.pm view on Meta::CPAN
# names Olsberg 59939
# names Winterberg 59955
# names Medebach 59964
# names Hallenberg 59969
# names Bromskirchen 59969
# names Nidderau 61130
# names Niederdorfelden 61138
# names Friedberg 61169 86316
# names Karben 61184
# names Rosbach 06242 53547 56271 94439
# names Niddatal 61194
# names Florstadt 61197
# names Reichelsheim 61203 64385
# names Echzell 61209
# names Nauheim 64569
# names Usingen 61250
# names Neu-Anspach 61267
# names Wehrheim 61273
# names Weilrod 61276
# names Homburg 66424
# names Friedrichsdorf 61381
}
our $files={};
my $style = {};
$style->{explicit} = 1;
our $c = AI::MicroStructure::Context->new(@ARGV);
$c->retrieveIndex($TOP);#"data-hub" structures=0 text=1 json=1
my $cg = $c->{graph}->{content};
my @ranked_docs = $cg->simple_search( 'peanuts' );
# get back both related terms and docs for more power
my ( $docs, $words ) = $cg->search('dna');
p $docs;
p $words;
$files->{text}->{$name}=$rel_name;
}
}
p $files;
__DATA__
our $c = AI::MicroStructure::Context->new(@ARGV);
$c->retrieveIndex($PWD."/t/docs"); #"/home/santex/data-hub/data-hub" structures=0 text=1 json=1
my $style = {};
$style->{explicit} = 1;
ok($c->simpleMixedSearch($style,$_)) && ok($c->play($style,$_)) for
qw(atom antimatter planet);
#####################################################################
SKIP: {
skip "Renaming tmp files not supported on Win32", 1 if $^O eq "MSWin32";
blurt_atomic("one\ntwo\nthree", $TMP_FILE);
ok(-f $TMP_FILE, "$TMP_FILE exists");
}
#####################################################################
# Slurp
#####################################################################
my $data = slurp($TMP_FILE);
is($data, "one\ntwo\nthree", "$TMP_FILE contains right data");
#####################################################################
# Slurp from @ARGS
#####################################################################
@ARGV = ($TMP_FILE);
$data = slurp();
is($data, "one\ntwo\nthree", "$TMP_FILE contains right data");
#####################################################################
# pie
#####################################################################
pie( sub { s/three/four/g; $_; }, $TMP_FILE );
$data = slurp($TMP_FILE);
is($data, "one\ntwo\nfour", "$TMP_FILE got pied");
use Test::More tests =>8;
BEGIN {
use_ok('LWP');
use_ok('Storable::CouchDB');
my $s = Storable::CouchDB->new;
ok("sprintf $s->retrieve('doc')"); #undef if not exists
ok("sprintf $s->store('doc1' => 'data');");
ok('sprintf $s->store("doc2" => {"my" => "data"});');
ok('sprintf $s->store("doc3" => ["my", "data"]);');
ok("sprintf $s->store('doc4' => undef);");
ok("sprintf $s->delete('doc');");
my $browser = LWP::UserAgent->new();
my $seite = "http://127.0.0.1:5984";
# $seite = $browser->get($seite);
# ok($seite->is_success,"couch is there");
package AI::MicroStructure::test;
use strict;
use Test::More;
use AI::MicroStructure;
plan tests => 2;
$_ = 'canari';
my $data = AI::MicroStructure->load_data('AI::MicroStructure::test');
is( $_, 'canari', "load_data does not stomp the canari" );
is_deeply(
$data,
{ foo => "bar",
names => {
en => "name on 3 lines",
fr => "et en français",
},
long =>
{ chain => { empty => '', not => { empty => 'zlonk powie' } } },
now => {
test => {
the =>
( run in 1.008 second using v1.01-cache-2.11-cpan-a5abf4f5562 )