AtteanX-Query-Cache
view release on metacpan or search on metacpan
AtteanX::Query::Cache -- A prefetching SPARQL query cacher
This module is intended to be installed on a caching proxy, and given
a SPARQL query, it would analyze the query, find parts it can cache,
prefetch those, and lighten the burden on a remote endpoint.
It can also use Triple Pattern Fragments servers where that makes
sense, but it will mostly allow the remote endpoint to do joins.
The current releases should be considered early alphas, they are not
stable and not complete. They are mostly released to ease
collaboration amongst the authors.
lib/AtteanX/Query/AccessPlan/Cache.pm view on Meta::CPAN
around 'access_plans' => sub {
my $orig = shift;
my @params = @_;
my $self = shift;
my $model = shift;
my $active_graphs = shift;
my $pattern = shift;
# First, add any plans coming from the original planner (which will
# include queries to the remote SPARQL endpoint
my @plans = $orig->(@params);
my @vars = $pattern->values_consuming_role('Attean::API::Variable');
my @varstrings = map { $_->value } @vars;
# Start checking the cache
my $keypattern = $pattern->canonicalize->tuples_string;
my $cached = $model->cache->get($keypattern);
if (defined($cached)) {
$self->log->info("Found data in the cache for " . $keypattern);
my $parser = Attean->get_parser('NTriples')->new(lazy_iris => 1);
lib/AtteanX/Query/AccessPlan/PrefetchLDF.pm view on Meta::CPAN
my $orig = shift;
my @params = @_;
my $self = shift;
my $model = shift;
my $active_graphs = shift;
my $pattern = shift;
my $max_triples = $ENV{'LDF_MAX_TRIPLES'} || 100000;
# First, add any plans coming from the original planner (which will
# include queries to the remote SPARQL endpoint
my @plans = $orig->(@params);
# Add my plans
# Cache only below a limit for how many LDF triples we will fetch.
if ($model->has_publisher && $model->ldf_store->count_triples_estimate($pattern->values) <= $max_triples) {
push(@plans, AtteanX::Plan::LDF::Triple::EnterCache->new(subject => $pattern->subject,
predicate => $pattern->predicate,
object => $pattern->object,
distinct => 0));
} else {
lib/AtteanX/Query/Cache.pm view on Meta::CPAN
AtteanX::Query::Cache - Experimental prefetching SPARQL query cacher
=head1 SYNOPSIS
=head1 DESCRIPTION
This is an alpha release of a system that is able to intercept SPARQL
queries if deployed in a proxy, and analyze the queries so that the
query can be evaluated on the proxy. It can look up in a cache on the
proxy, send parts of the query on to the remote endpoint, use Linked
Data Fragments when appropriate and so on. The analyzer may also
decide to prefetch certain data asynchronously.
It is known at present to have insufficient performance for any
practical use, but is released anyway as an alpha.
=head1 BUGS
Please report any bugs to
lib/AtteanX/Query/Cache/Analyzer/QueryPlanner.pm view on Meta::CPAN
extends 'AtteanX::QueryPlanner::Cache::LDF';
around 'access_plans' => sub {
my $orig = shift;
my @params = @_;
my $self = shift;
my $model = shift;
my $active_graphs = shift;
my $pattern = shift;
# First, add any plans coming from the original planner (which will
# include queries to the remote SPARQL endpoint
my @plans = $orig->(@params);
my @vars = $pattern->values_consuming_role('Attean::API::Variable');
# Start checking the cache
my $keypattern = $pattern->canonicalize->tuples_string;
if ($model->is_cached($keypattern)) {
$self->log->debug("Already accounted for by cache: $keypattern");
} elsif ($model->try eq $keypattern) {
$self->log->debug("Creating dummy iterator for $keypattern");
my %row;
lib/Plack/App/AtteanX/Query/Cache.pm view on Meta::CPAN
extends 'Plack::App::AtteanX::Endpoint';
with 'MooX::Log::Any';
sub prepare_app {
my $self = shift;
my $config = $self->{config};
my $redisserver = 'robin.kjernsmo.net:6379';
my $sparqlurl = 'http://dbpedia.org/sparql';
my $ldfurl = 'http://fragments.dbpedia.org/2015/en';
my $sparqlstore = Attean->get_store('SPARQL')->new(endpoint_url => $sparqlurl);
my $ldfstore = Attean->get_store('LDF')->new(start_url => $ldfurl);
my $cache = CHI->new(
driver => 'Redis',
namespace => 'cache',
server => $redisserver,
debug => 0
);
my $redissub = Redis->new(server => $redisserver, name => 'subscriber');
RDF::Trine::default_useragent(LWP::UserAgent::CHICaching->new(cache => $cache));
my $model = AtteanX::Model::SPARQLCache::LDF->new( store => $sparqlstore,
ldf_store => $ldfstore,
cache => $cache,
publisher => $redissub);
$self->{config} = {};
# try {
$self->{endpoint} = AtteanX::Query::Cache->new(model => $model,
planner => AtteanX::QueryPlanner::Cache->new,
conf => $self->{config},
graph => iri('http://example.org/graph'));
# };
# if ($@) {
# $self->log->error($@);
# }
}
1;
scripts/analyzer view on Meta::CPAN
my $cache = CHI->new(
driver => 'Redis',
namespace => 'cache',
server => $redisserver,
debug => 0
);
RDF::Trine::default_useragent(LWP::UserAgent::CHICaching->new(cache => $cache));
my $sparqlstore = Attean->get_store('SPARQL')->new(endpoint_url => $sparqlurl);
my $ldfstore = Attean->get_store('LDF')->new(start_url => $ldfurl);
my $redissub = Redis->new(server => $redisserver, name => 'subscriber');
my $redisstore = Redis->new(server => $redisserver, name => 'store');
my $model = AtteanX::Query::Cache::Analyzer::Model->new(store => $sparqlstore,
ldf_store => $ldfstore,
cache => $cache);
scripts/prefetcher view on Meta::CPAN
use AtteanX::Model::SPARQLCache::LDF;
my $cache = CHI->new( driver => 'Memory', global => 1 );
my $sparqlurl = 'http://dbpedia.org/sparql';
my $ldfurl = 'http://fragments.dbpedia.org/2015/en';
my $redisserver = 'robin.kjernsmo.net:6379';
RDF::Trine::default_useragent(LWP::UserAgent::CHICaching->new(cache => $cache));
my $sparqlstore = Attean->get_store('SPARQL')->new(endpoint_url => $sparqlurl);
my $ldfstore = Attean->get_store('LDF')->new(start_url => $ldfurl);
my $redissub = Redis->new(server => $redisserver, name => 'subscriber');
my $model = AtteanX::Model::SPARQLCache::LDF->new(store => $sparqlstore,
cache => $cache,
ldf_store => $ldfstore);
my $retriever = AtteanX::Query::Cache::Retriever->new(model => $model);
t/analysis-best-cost.t view on Meta::CPAN
triple(iri('http://example.com/foo'), iri('http://example.org/m/b'), iri('http://example.org/m/c')),
triple(iri('http://example.com/dahut'), iri('http://example.org/m/b'), literal('2')),
triple(iri('http://example.org/m/a'), iri('http://example.org/m/q'), iri('http://example.org/baz')),
triple(iri('http://example.org/m/a'), iri('http://example.org/m/q'), iri('http://example.org/foobar')),
triple(iri('http://example.org/m/a'), iri('http://example.org/m/c'), iri('http://example.org/foo')),
triple(iri('http://example.org/m/a'), iri('http://example.org/m/p'), iri('http://example.org/m/o')),
]);
my $store = Attean->get_store('SPARQL')->new('endpoint_url' => iri('http://test.invalid/'));
my $model = AtteanX::Query::Cache::Analyzer::Model->new(store => $store, cache => $cache, ldf_store => $ldfstore);
subtest '3-triple BGP where cache breaks the join to cartesian' => sub {
my $query = <<'END';
SELECT * WHERE {
?a <http://example.org/m/c> ?s .
?s <http://example.org/m/p> ?o .
?o <http://example.org/m/b> "2" .
}
t/analysis-count.t view on Meta::CPAN
EOQ
package TestLDFCreateStore {
use Moo;
with 'Test::Attean::Store::LDF::Role::CreateStore';
};
my $test = TestLDFCreateStore->new;
my $ldfstore = $test->create_store(triples => [triple(iri('http://example.org/foo'), iri('http://example.org/m/r'), literal('1'))]);
my $store = Attean->get_store('SPARQL')->new('endpoint_url' => iri('http://test.invalid/'));
my $model = AtteanX::Query::Cache::Analyzer::Model->new(store => $store, ldf_store => $ldfstore, cache => $cache);
my $analyzer1 = AtteanX::Query::Cache::Analyzer->new(model => $model, query => $basequery, store => $redis2);
note 'Testing counts without actual caching';
my @patterns1 = $analyzer1->count_patterns;
is(scalar @patterns1, 0, 'Nothing now');
$basequery =~ s/< 50/> 5000000/;
t/planner-with-tpf-cache.t view on Meta::CPAN
package TestLDFCreateStore {
use Moo;
with 'Test::Attean::Store::LDF::Role::CreateStore';
};
my $test = TestLDFCreateStore->new;
my $sparqlstore = Attean->get_store('SPARQL')->new('endpoint_url' => iri('http://test.invalid/sparql'));
isa_ok($sparqlstore, 'AtteanX::Store::SPARQL');
my $graph = iri('http://test.invalid/graph');
my $t = triplepattern(variable('s'), iri('http://example.org/m/p'), literal('1'));
my $u = triplepattern(variable('s'), iri('http://example.org/m/p'), variable('o'));
my $v = triplepattern(variable('s'), iri('http://example.org/m/q'), blank('xyz'));
my $w = triplepattern(variable('a'), iri('http://example.org/m/b'), iri('http://example.org/m/c'));
my $x = triplepattern(variable('s'), iri('http://example.org/m/q'), iri('http://example.org/m/a'));
my $y = triplepattern(variable('o'), iri('http://example.org/m/b'), literal('2'));
my $z = triplepattern(variable('a'), iri('http://example.org/m/c'), variable('s'));
t/simple-sparql-planner.t view on Meta::CPAN
my $cache = CHI->new( driver => 'Memory', global => 1 );
my $p = AtteanX::QueryPlanner::Cache->new;
isa_ok($p, 'Attean::QueryPlanner');
isa_ok($p, 'AtteanX::QueryPlanner::Cache');
does_ok($p, 'Attean::API::CostPlanner');
{
# These tests does not actually look up anything in a real store, it just simulates
my $store = Attean->get_store('SPARQL')->new('endpoint_url' => iri('http://test.invalid/'));
isa_ok($store, 'AtteanX::Store::SPARQL');
my $model = AtteanX::Model::SPARQLCache->new( store => $store, cache => $cache );
my $graph = iri('http://test.invalid/graph');
my $t = triplepattern(variable('s'), iri('p'), literal('1'));
my $u = triplepattern(variable('s'), iri('p'), variable('o'));
my $v = triplepattern(variable('s'), iri('q'), blank('xyz'));
my $w = triplepattern(variable('a'), iri('b'), iri('c'));
my $x = triplepattern(variable('s'), iri('q'), iri('a'));
my $y = triplepattern(variable('o'), iri('b'), literal('2'));
my $z = triplepattern(variable('a'), iri('c'), variable('s'));
t/simple-sparql-tpf-planner.t view on Meta::CPAN
package TestLDFCreateStore {
use Moo;
with 'Test::Attean::Store::LDF::Role::CreateStore';
};
my $test = TestLDFCreateStore->new;
{
my $sparqlstore = Attean->get_store('SPARQL')->new('endpoint_url' => iri('http://test.invalid/sparql'));
isa_ok($sparqlstore, 'AtteanX::Store::SPARQL');
my $graph = iri('http://test.invalid/graph');
my $t = triplepattern(variable('s'), iri('http://example.org/m/p'), literal('1'));
my $u = triplepattern(variable('s'), iri('http://example.org/m/p'), variable('o'));
my $v = triplepattern(variable('s'), iri('http://example.org/m/q'), blank('xyz'));
my $w = triplepattern(variable('a'), iri('http://example.org/m/b'), iri('http://example.org/m/c'));
my $x = triplepattern(variable('s'), iri('http://example.org/m/q'), iri('http://example.org/m/a'));
my $y = triplepattern(variable('o'), iri('http://example.org/m/b'), literal('2'));
my $z = triplepattern(variable('a'), iri('http://example.org/m/c'), variable('s'));
( run in 1.434 second using v1.01-cache-2.11-cpan-49f99fa48dc )