view release on metacpan or  search on metacpan

lib/Dezi/Aggregator/Spider/UA.pm  view on Meta::CPAN

package Dezi::Aggregator::Spider::UA;
use strict;
use warnings;
use base qw( LWP::RobotUA );
use HTTP::Message;
use URI;
use Carp;
use Data::Dump qw( dump );
use Search::Tools::UTF8;
use Dezi::Aggregator::Spider::Response;

our $VERSION = '0.018';

# if Compress::Zlib is installed, this should handle gzip transparently.
# thanks to
# http://stackoverflow.com/questions/1285305/how-can-i-accept-gzip-compressed-content-using-lwpuseragent
my $can_accept = HTTP::Message::decodable();

# TODO handle when Zlib is *not* installed, via Content-Encoding header

#warn "Accept-Encoding: $can_accept\n";

our $Debug = $ENV{SPIDER_DEBUG} || 0;

our $Response_Class = 'Dezi::Aggregator::Spider::Response';

=pod

=head1 NAME

Dezi::Aggregator::Spider::UA - spider user agent

=head1 SYNOPSIS

 use Dezi::Aggregator::Spider::UA;
 my $ua = Dezi::Aggregator::Spider::UA->new;

 # $ua is a LWP::RobotUA object

=head1 DESCRIPTION

Dezi::Aggregator::Spider::UA is a subclass of
LWP::RobotUA.

=head1 METHODS

=cut

=head2 get( I<args> )

I<args> is an array of key/value pairs. I<uri> is required.

I<delay> will sleep() I<delay> seconds before fetching I<uri>.

Also supported: I<user> and I<pass> for authorization.

=cut

sub get {
    my $self  = shift;
    my %args  = @_;
    my $uri   = delete $args{uri} or croak "URI required";
    my $delay = delete $args{delay} || 0;

    sleep($delay) if $delay;

    my $request = HTTP::Request->new( 'GET' => $uri );
    $request->header( 'Accept-Encoding' => $can_accept, );
    if ( $args{user} && $args{pass} ) {
        $request->authorization_basic( delete $args{user},
            delete $args{pass} );
    }
    else {
        # if either one was set, but not the other, delete them both