AnyEvent-Net-Curl-Queued

 view release on metacpan or  search on metacpan

README  view on Meta::CPAN

        use warnings qw(all);
    
        use HTML::LinkExtor;
        use Moo;
    
        extends 'AnyEvent::Net::Curl::Queued::Easy';
    
        after finish => sub {
            my ($self, $result) = @_;
    
            say $result . "\t" . $self->final_url;
    
            if (
                not $self->has_error
                and $self->getinfo('content_type') =~ m{^text/html}
            ) {
                my @links;
    
                HTML::LinkExtor->new(sub {
                    my ($tag, %links) = @_;
                    push @links,

eg/CrawlApache.pm  view on Meta::CPAN

        };
    },
    lazy    => 1,
);

has '+use_stats' => (default => sub { 1 });

after finish => sub {
    my ($self, $result) = @_;

    say $result . "\t" . $self->final_url;

    if (
        not $self->has_error
        and $self->getinfo('content_type') =~ m{^text/html}x
    ) {
        my $res = $self
            ->scrap
            ->scrape(
                ${$self->data},
                $self->final_url

eg/Gauge/curl.pm  view on Meta::CPAN

use Any::Moose;
with qw(Gauge::Role);

sub run {
    my ($self) = @_;

    my $curl_queue;
    $self->run_forked(sub {
        my ($url) = @_;
        $curl_queue->say("url = \"$url\"");
        $curl_queue->say("output = \"/dev/null\"");
    } => sub {
        $curl_queue = File::Temp->new;
    } => sub {
        system qw(curl -s -K), $curl_queue->filename;
    });

    return;
}

1;

eg/Gauge/lftp.pm  view on Meta::CPAN

use utf8;
use warnings qw(all);

use Any::Moose;
with qw(Gauge::Role);

sub run {
    my ($self) = @_;

    my $lftp_queue = File::Temp->new;
    say $lftp_queue "set cmd:queue-parallel " . $self->parallel;
    say $lftp_queue "set cmd:verbose no";
    say $lftp_queue "set net:connection-limit 0";
    say $lftp_queue "set xfer:clobber 1";

    for my $url (@{$self->queue}) {
        $lftp_queue->say("queue get \"$url\" -o \"/dev/null\"");
    }

    say $lftp_queue "wait all";

    system qw(lftp -f), $lftp_queue->filename;

    return;
}

1;

eg/Gauge/wget.pm  view on Meta::CPAN

use warnings qw(all);

use Any::Moose;
with qw(Gauge::Role);

sub run {
    my ($self) = @_;

    my $wget_queue;
    $self->run_forked(sub {
        $wget_queue->say(shift);
    } => sub {
        $wget_queue = File::Temp->new;
    } => sub {
        system qw(wget -q -O /dev/null -i), $wget_queue->filename;
    });

    return;
}

1;

eg/simple.pl  view on Meta::CPAN

        http://www.cpan.org/modules/by-category/21_File_Handle_Input_Output/
        http://www.cpan.org/modules/by-category/22_Microsoft_Windows_Modules/
        http://www.cpan.org/modules/by-category/23_Miscellaneous_Modules/
        http://www.cpan.org/modules/by-category/24_Commercial_Software_Interfaces/
        http://www.cpan.org/modules/by-category/25_Bundles/
        http://www.cpan.org/modules/by-category/26_Documentation/
        http://www.cpan.org/modules/by-category/27_Pragma/
        http://www.cpan.org/modules/by-category/28_Perl6/
        http://www.cpan.org/modules/by-category/99_Not_In_Modulelist/
    ]] => sub {
        say $_[0]->final_url;
        say ${$_[0]->header};
    },
)->wait;

inc/Test/HTTP/AnyEvent/Server.pm  view on Meta::CPAN

    use AnyEvent::HTTP;
    use Test::HTTP::AnyEvent::Server;

    my $server = Test::HTTP::AnyEvent::Server->new;
    my $cv = AE::cv;

    $cv->begin;
    http_request GET => $server->uri . q(echo/head), sub {
        my ($body, $hdr) = @_;
        say $body;
        $cv->end;
    };

    $cv->wait;

=head1 DESCRIPTION

This package provides a simple B<NON>-forking HTTP server which can be used for testing HTTP clients.

=head1 ATTRIBUTES

lib/AnyEvent/Net/Curl/Queued.pm  view on Meta::CPAN

    use warnings qw(all);

    use HTML::LinkExtor;
    use Moo;

    extends 'AnyEvent::Net::Curl::Queued::Easy';

    after finish => sub {
        my ($self, $result) = @_;

        say $result . "\t" . $self->final_url;

        if (
            not $self->has_error
            and $self->getinfo('content_type') =~ m{^text/html}
        ) {
            my @links;

            HTML::LinkExtor->new(sub {
                my ($tag, %links) = @_;
                push @links,

lib/YADA.pm  view on Meta::CPAN

    YADA->new->append(
        [qw[
            http://www.cpan.org/modules/by-category/02_Language_Extensions/
            http://www.cpan.org/modules/by-category/02_Perl_Core_Modules/
            http://www.cpan.org/modules/by-category/03_Development_Support/
            ...
            http://www.cpan.org/modules/by-category/27_Pragma/
            http://www.cpan.org/modules/by-category/28_Perl6/
            http://www.cpan.org/modules/by-category/99_Not_In_Modulelist/
        ]] => sub {
            say $_[0]->final_url;
            say ${$_[0]->header};
        },
    )->wait;

=head1 WARNING: GONE MOO!

This module isn't using L<Any::Moose> anymore due to the announced deprecation status of that module.
The switch to the L<Moo> is known to break modules that do C<extend 'AnyEvent::Net::Curl::Queued::Easy'> / C<extend 'YADA::Worker'>!
To keep the compatibility, make sure that you are using L<MooseX::NonMoose>:

    package YourSubclassingModule;

lib/YADA.pm  view on Meta::CPAN

    use common::sense;

    use AnyEvent::Net::Curl::Queued;
    use AnyEvent::Net::Curl::Queued::Easy;

    my $q = AnyEvent::Net::Curl::Queued->new;
    $q->append(sub {
        AnyEvent::Net::Curl::Queued::Easy->new({
            initial_url => $_,
            on_finish   => sub {
                say $_[0]->final_url;
                say ${$_[0]->header};
            },
        })
    }) for qw(
        http://www.cpan.org/modules/by-category/02_Language_Extensions/
        http://www.cpan.org/modules/by-category/02_Perl_Core_Modules/
        http://www.cpan.org/modules/by-category/03_Development_Support/
        ...
        http://www.cpan.org/modules/by-category/27_Pragma/
        http://www.cpan.org/modules/by-category/28_Perl6/
        http://www.cpan.org/modules/by-category/99_Not_In_Modulelist/