view release on metacpan or  search on metacpan

lib/App/Caoliu.pm  view on Meta::CPAN

package App::Caoliu;

# ABSTRACT: a awsome module,suck greate fire wall!
use Mojo::Base "Mojo";
use Mojo::UserAgent;
use Mojo::Log;
use Mojo::Util;
use Mojo::URL;
use Mojo::Collection;
use Mojo::IOLoop;
use File::Basename;
use App::Caoliu::Parser;
use App::Caoliu::Downloader;
use App::Caoliu::Utils 'dumper';
no warnings 'deprecated';
no warnings 'recursion';

our $VERSION = 1.0;

use constant FEEDS => {
    WUMA  => 'http://t66y.com/rss.php?fid=2',
    YOUMA => 'http://t66y.com/rss.php?fid=15',
    OUMEI => 'http://t66y.com/rss.php?fid=4',
};
use constant DOWNLOAD_FILE_TIME => 1200;

has require_md5_path => 0;
has timeout          => 60;
has proxy            => '127.0.0.1:8087';
has log              => sub { Mojo::Log->new };
has category         => sub { [qw( wuma youma donghua oumei)] };
has parser           => sub { App::Caoliu::Parser->new };
has index            => 'http://t66y.com';
has downloader       => sub { App::Caoliu::Downloader->new };
has target           => '.';
has loop             => sub { Mojo::IOLoop->delay };
has parallel_num     => 20;

my @downloaded_files;

sub new {
    my $self = shift->SUPER::new(@_);

    Carp::croak("category args must be arrayref")
      if ref $self->category ne ref [];
    $self->downloader->ua->http_proxy(
        join( '', 'http://sri:secret@', $self->proxy ) )
      if $self->proxy;
    $ENV{LOGGER} = $self->log;

    return $self;
}

sub reap {
    my $self = shift;
    my $category = shift || $self->category;

    return if not scalar @{ $self->category };

    my @download_links;
    my @feeds = map { FEEDS->{ uc $_ } } @{ $self->category };
    $self->log->debug( "show feeds: " . dumper \@feeds );

    # fetch all rmdown_link
    # parallel get post page and download bt files
    $self->_non_blocking_get_torrent(@feeds);
    unless ( Mojo::IOLoop->is_running ) {
        $self->loop->wait;
        $self->log->debug(
            "Downloaded files list:" . dumper [@downloaded_files] );
        return wantarray ? @downloaded_files : scalar(@downloaded_files);
    }
}

sub _non_blocking_get_torrent {
    my ( $self, @feeds ) = @_;

    for my $feed (@feeds) {
        $self->loop->begin;
        $self->downloader->ua->get(
            $feed => sub {
                $self->_process_feed(@_);
            }
        );
    }
}

sub _process_feed {
    my ( $self, $ua, $tx ) = @_;
    my @posts;
    my $processer;

    if ( $tx->success ) {
        my $xml             = $tx->res->body;
        my $post_collection = $self->parser->parse_rss($xml);
        $processer = sub {
            for (@_) {
                $ua->get( $_->{link} => sub { $self->_process_posts(@_) } );
            }
        };
        $processer->( @{$post_collection} );
    }