AnyEvent-Net-Curl-Queued

 view release on metacpan or  search on metacpan

eg/yada-crawler.pl  view on Meta::CPAN

7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
use YADA;
 
YADA->new(common_opts => { encoding => '' }, http_response => 1, max => 4)
->append([qw[http://localhost/manual/]] => sub {
    my ($self) = @_;
    return if not $self->response->is_success
        or not $self->response->content_is_html;
    my $doc = scraper {
        process q(html title), title => q(text);
        process q(a), q(links[]) => q(@href);
    }->scrape($self->response->decoded_content => $self->final_url);
    printf qq(%-64s %s\n), $self->final_url, $doc->{title} =~ s/\r?\n/ /rsx;
    $self->queue->prepend([
        grep { $_->can(q(host)) and $_->host eq $self->initial_url->host }
        @{$doc->{links} // []}
    ] => __SUB__);
})->wait;

lib/AnyEvent/Net/Curl/Queued/Easy.pm  view on Meta::CPAN

54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
 
has data        => (is => 'ro', isa => ScalarRef, writer => 'set_data');
 
 
has force       => (is => 'ro', isa => Bool, default => sub { 0 });
 
 
has header      => (is => 'ro', isa => ScalarRef, writer => 'set_header');
 
 
has _autodecoded => (is => 'rw', isa => Bool, default => sub { 0 });
has http_response => (is => 'ro', isa => Bool, default => sub { 0 }, writer => 'set_http_response');
 
 
has post_content => (is => 'ro', isa => Str, default => sub { '' }, writer => 'set_post_content');
 
 
sub _URI_type {
    my $uri = shift;
    return $uri->isa('URI')
        ? $uri

lib/AnyEvent/Net/Curl/Queued/Easy.pm  view on Meta::CPAN

220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
    my $header = ${$self->header};
    $header =~ s/^.*(?:\015\012?|\012\015){2}(?!$)//sx;
    $self->set_response(
        HTTP::Response->parse(
            $header
            . ${$self->data}
        )
    );
 
    $self->response->headers->header(content_encoding => 'identity')
        if $self->_autodecoded;
 
    my $msg = $self->response->message // '';
    $msg =~ s/^\s+|\s+$//gsx;
    $self->response->message($msg);
}
 
# wrap around the extendible interface
$self->finish($result);
 
# re-enqueue the request

lib/AnyEvent/Net/Curl/Queued/Easy.pm  view on Meta::CPAN

317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
            $key = AnyEvent::Net::Curl::Const::opt($key);
            if ($key == Net::Curl::Easy::CURLOPT_POSTFIELDS) {
                my $is_json = 0;
                ($val, $is_json) = $self->_setopt_postfields($val);
 
                $orig->($self =>
                    Net::Curl::Easy::CURLOPT_HTTPHEADER,
                    [ 'Content-Type: application/json; charset=utf-8' ],
                ) if $is_json;
            } elsif ($key == Net::Curl::Easy::CURLOPT_ENCODING) {
                $self->_autodecoded(1);
                $val = $self->_setopt_encoding($val);
            }
            $orig->($self => $key, $val);
        }
    } else {
        carp "Specify at least one OPTION/VALUE pair!";
    }
};
 
sub _setopt_postfields {

t/44-yada-simple.t  view on Meta::CPAN

23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
        );
    }
}
 
my @urls = ($server->uri . 'echo/head') x 2;
$urls[-1] =~ s{\b127\.0\.0\.1\b}{localhost}x;
my @opts = (referer => 'http://www.cpan.org/', ipresolve => Net::Curl::Easy::CURL_IPRESOLVE_V4);
my $on_finish = sub {
    my ($self, $r) = @_;
    isa_ok($self->response, qw(HTTP::Response));
    like($self->response->decoded_content, qr{\bReferer\s*:\s*\Q$opts[1]\E}isx, 'referer');
};
 
$q->append(
    @urls,
    sub { $_[0]->setopt(@opts) }, # on_init placeholder
    $on_finish,
);
 
$q->append(
    [ @urls ],