view release on metacpan or  search on metacpan

lib/Dezi/Aggregator/Spider/UA.pm  view on Meta::CPAN

use base qw( LWP::RobotUA );
use HTTP::Message;
use URI;
use Carp;
use Data::Dump qw( dump );
use Search::Tools::UTF8;
use Dezi::Aggregator::Spider::Response;

our $VERSION = '0.016';

# if Compress::Zlib is installed, this should handle gzip transparently.
# thanks to
# http://stackoverflow.com/questions/1285305/how-can-i-accept-gzip-compressed-content-using-lwpuseragent
my $can_accept = HTTP::Message::decodable();

# TODO handle when Zlib is *not* installed, via Content-Encoding header

#warn "Accept-Encoding: $can_accept\n";

our $Debug = $ENV{SPIDER_DEBUG} || 0;