AnyEvent-Net-Curl-Queued

 view release on metacpan or  search on metacpan

eg/yada-crawler.pl  view on Meta::CPAN

use YADA;

YADA->new(common_opts => { encoding => '' }, http_response => 1, max => 4)
->append([qw[http://localhost/manual/]] => sub {
    my ($self) = @_;
    return if not $self->response->is_success
        or not $self->response->content_is_html;
    my $doc = scraper {
        process q(html title), title => q(text);
        process q(a), q(links[]) => q(@href);
    }->scrape($self->response->decoded_content => $self->final_url);
    printf qq(%-64s %s\n), $self->final_url, $doc->{title} =~ s/\r?\n/ /rsx;
    $self->queue->prepend([
        grep { $_->can(q(host)) and $_->host eq $self->initial_url->host }
        @{$doc->{links} // []}
    ] => __SUB__);
})->wait;

lib/AnyEvent/Net/Curl/Queued/Easy.pm  view on Meta::CPAN

has data        => (is => 'ro', isa => ScalarRef, writer => 'set_data');


has force       => (is => 'ro', isa => Bool, default => sub { 0 });


has header      => (is => 'ro', isa => ScalarRef, writer => 'set_header');


has _autodecoded => (is => 'rw', isa => Bool, default => sub { 0 });
has http_response => (is => 'ro', isa => Bool, default => sub { 0 }, writer => 'set_http_response');


has post_content => (is => 'ro', isa => Str, default => sub { '' }, writer => 'set_post_content');


sub _URI_type {
    my $uri = shift;
    return $uri->isa('URI')
        ? $uri

lib/AnyEvent/Net/Curl/Queued/Easy.pm  view on Meta::CPAN

        my $header = ${$self->header};
        $header =~ s/^.*(?:\015\012?|\012\015){2}(?!$)//sx;
        $self->set_response(
            HTTP::Response->parse(
                $header
                . ${$self->data}
            )
        );

        $self->response->headers->header(content_encoding => 'identity')
            if $self->_autodecoded;

        my $msg = $self->response->message // '';
        $msg =~ s/^\s+|\s+$//gsx;
        $self->response->message($msg);
    }

    # wrap around the extendible interface
    $self->finish($result);

    # re-enqueue the request

lib/AnyEvent/Net/Curl/Queued/Easy.pm  view on Meta::CPAN

            $key = AnyEvent::Net::Curl::Const::opt($key);
            if ($key == Net::Curl::Easy::CURLOPT_POSTFIELDS) {
                my $is_json = 0;
                ($val, $is_json) = $self->_setopt_postfields($val);

                $orig->($self =>
                    Net::Curl::Easy::CURLOPT_HTTPHEADER,
                    [ 'Content-Type: application/json; charset=utf-8' ],
                ) if $is_json;
            } elsif ($key == Net::Curl::Easy::CURLOPT_ENCODING) {
                $self->_autodecoded(1);
                $val = $self->_setopt_encoding($val);
            }
            $orig->($self => $key, $val);
        }
    } else {
        carp "Specify at least one OPTION/VALUE pair!";
    }
};

sub _setopt_postfields {

t/44-yada-simple.t  view on Meta::CPAN

        );
    }
}

my @urls = ($server->uri . 'echo/head') x 2;
$urls[-1] =~ s{\b127\.0\.0\.1\b}{localhost}x;
my @opts = (referer => 'http://www.cpan.org/', ipresolve => Net::Curl::Easy::CURL_IPRESOLVE_V4);
my $on_finish = sub {
    my ($self, $r) = @_;
    isa_ok($self->response, qw(HTTP::Response));
    like($self->response->decoded_content, qr{\bReferer\s*:\s*\Q$opts[1]\E}isx, 'referer');
};

$q->append(
    @urls,
    sub { $_[0]->setopt(@opts) }, # on_init placeholder
    $on_finish,
);

$q->append(
    [ @urls ],