view release on metacpan or  search on metacpan

lib/Catmandu/Importer/RDF.pm  view on Meta::CPAN

        if ($self->triples) {
            if (my $hashref = $stream->()) {
                $self->encoder->add_hashref($hashref, $aref);
            }
            else {
                return ($stream = undef);
            }
        }
        else {
            # TODO: include namespace mappings if requested
            while (my $hashref = $stream->()) {
              $self->encoder->add_hashref(
                  $hashref,
                  $aref
              );
            }

            if ($self->url) {
                $aref->{_url} = $self->url;
            }

            $stream = undef;
        }

        if ($self->url) {
            # RDF::Trine::Parser parses data from URL to UTF-8
            # but we want internal character sequences
            _utf8_decode($aref);
        }

        return $aref;
    };
}

sub _utf8_decode {
    if (ref $_[0] eq 'HASH') {
        # FIXME: UTF-8 in property values
        foreach (values %{$_[0]}) {
            ref($_) ? _utf8_decode($_) : utf8::decode($_);
        }
    } else {
        foreach (@{$_[0]}) {
            ref($_) ? _utf8_decode($_) : utf8::decode($_);
        }
    }
}

sub _sparql_stream {
    my ($self) = @_;

    die "need an url" unless $self->url;

    $self->log->info("parsing: " . $self->sparql);

    my $store;

    # Check if this server is an LDF server
    my $ldf_client = RDF::LDF->new(url => $self->url);

    if ($ldf_client->is_fragment_server) {
        $store = RDF::Trine::Store->new_with_config({
                    storetype => 'LDF',
                    url => $self->url
        });
    }
    else {
        $store = RDF::Trine::Store->new_with_config({
                    storetype => 'SPARQL',
                    url => $self->url
        });
    }

    unless ($store) {
        $self->log->error("failed to connect to " . $self->url);
        return;
    }

    my $model =  RDF::Trine::Model->new($store);

    my $rdf_query = RDF::Query->new($self->sparql);

    unless ($rdf_query) {
        $self->log->error("failed to parse " . $self->sparql);
        return;
    }

    my $iterator = $rdf_query->execute($model);

    unless ($iterator) {
        $self->log->error("failed to execute " . $self->sparql . " at " . $self->url);
        return;
    }
}

sub _hashref_stream {
  my ($self) = @_;

  # Create a pipe stream to convert a callback handler into an iterator
  my $pipe = IO::Pipe->new();

  if (my $pid = fork()) {
    # parent
    $pipe->reader();

    binmode($pipe,':encoding(UTF-8)');

    return sub {
      state $line = <$pipe>;

      return decode_json($line) if defined($line);

      waitpid($pid,0);

      return undef;
    };
  }
  else {
    # child
    $pipe->writer();

lib/Catmandu/Importer/RDF.pm  view on Meta::CPAN

        if ($self->speed && ($count % 100 == 0) && (my $elapsed = time - $start) ) {
          printf STDERR "triples %9d (%d/sec)\n" , $count , $count/$elapsed;
        }
    };

    if ($self->url) {
        $parser->parse_url( $self->url, $handler);
    }
    else {
        my $from_scalar = (ref $self->file // '') eq 'SCALAR';

        if (!$self->type and $self->file and !$from_scalar) {
            $parser = $parser->guess_parser_by_filename($self->file)->new;
        }

        if ($from_scalar) {
            $parser->parse( $self->base, ${$self->file}, $handler );
        }
        else {
            $parser->parse_file( $self->base, $self->file // $self->fh, $handler );
        }
    }

    exit(0);
  }
}


1;

__END__

=head1 NAME

Catmandu::Importer::RDF - parse RDF data

=head1 SYNOPSIS

Command line client C<catmandu>:

  catmandu convert RDF --url http://d-nb.info/gnd/4151473-7 to YAML

  catmandu convert RDF --file rdfdump.ttl to JSON

  # Parse the input into on JSON document per triplet. This is the
  # most memory efficient (and fastest) way to parse RDF input.
  catmandu convert RDF --triples 1 --file rdfdump.ttl to JSON

  # Transform back into NTriples (conversions to and from triples is the
  # most efficient way to process RDF)
  catmandu convert RDF --triples 1 --file rdfdump.ttl to RDF --type NTriples

  # Query a SPARQL endpoint
  catmandu convert RDF --url http://dbpedia.org/sparql
                       --sparql "SELECT ?film WHERE { ?film dct:subject <http://dbpedia.org/resource/Category:French_films> }"

  catmandu convert RDF --url http://example.org/sparql --sparql query.rq

  # Query a Linked Data Fragment endpoint
  catmandu convert RDF --url http://fragments.dbpedia.org/2014/en
                       --sparql "SELECT ?film WHERE { ?film dct:subject <http://dbpedia.org/resource/Category:French_films> }"

In Perl code:

    use Catmandu::Importer::RDF;
    my $url = "http://dx.doi.org/10.2474/trol.7.147";
    my $rdf = Catmandu::Importer::RDF->new( url => $url )->first;

=head1 DESCRIPTION

This L<Catmandu::Importer> can be use to import RDF data from URLs, files or
input streams, SPARQL endpoints, and Linked Data Fragment endpoints.

By default an RDF graph is imported as single item in aREF format (see
L<RDF::aREF>).

=head1 CONFIGURATION

=over

=item url

URL to retrieve RDF from.

=item type

RDF serialization type (e.g. C<ttl> for RDF/Turtle).

=item base

Base URL. By default derived from the URL or file name.

=item ns

Use default namespace prefixes as provided by L<RDF::NS> to abbreviate
predicate and datatype URIs. Set to C<0> to disable abbreviating URIs.
Set to a specific date to get stable namespace prefix mappings.

=item triples

Import each RDF triple as one aREF subject map (default) or predicate map
(option C<predicate_map>), if enabled. This is the most efficient way to
process large input files. All the processing can be streamed.

=item predicate_map

Import RDF as aREF predicate map, if possible.

=item file

=item fh

=item encoding

=item fix

Default configuration options of L<Catmandu::Importer>.

=item sparql