view release on metacpan or  search on metacpan

lib/App/ProxyHunter.pm  view on Meta::CPAN

		}
		
		close $fh;
		return;
	}
	
	unless (defined $config) {
		die $usage;
	}
	
	$config = App::ProxyHunter::Config->new(path => $config);
	
	my $model = App::ProxyHunter::Model->new(
		connect_info => [
			sprintf('dbi:%s:database=%s', $config->db->driver, $config->db->name) .
				($config->db->host ? ';host='.$config->db->host : ''),
			$config->db->login,
			$config->db->password,
			{
				%{$config->db->driver_cfg},
				AutoInactiveDestroy => 1
			}
		],
		schema_class => 'App::ProxyHunter::Model::Schema::' . $config->db->driver
	);
	
	if (defined $create_schema) {
		while (my $query = $model->schema_class->get_create_query()) {
			$model->do($query);
		}
		return;
	}
	
	for my $engine (@{$config->searcher->engines}) {
		$engine = 'App::ProxyHunter::SearchEngine::'.$engine;
		eval "require $engine"
			or croak "Can't load search engine module $engine: $@";
	}
	
	if (defined $daemon) {
		$SIG{INT} = $SIG{TERM} = sub {
			unlink $daemon if length($daemon) > 0;
			exit;
		};
		
		eval {
			require Proc::Daemon;
		};
		if ($@) {
			croak 'You need to install Proc::Daemon to perform daemonization: ', $@;
		}
		
		Proc::Daemon::Init({
			work_dir => '.',
			length($daemon)>0 ? (pid_file => $daemon) : ()
		});
	}
	
	$model->update('proxy', {in_progress => 0}); # clean
	
	my @coros;
	if ($config->checker->enabled) {
		push @coros, $class->_start_checkers($config, $model);
	}
	if ($config->rechecker->enabled) {
		push @coros, $class->_start_recheckers($config, $model);
	}
	if ($config->speed_checker->enabled) {
		push @coros, $class->_start_speed_checkers($config, $model);
	}
	if ($config->searcher->enabled) {
		push @coros, $class->_start_searcher($config, $model);
	}
	
	$_->join() for @coros;
}

sub _start_checkers {
	my ($class, $config, $model) = @_;
	
	my @coros;
	my @queue;
	my $delay;
	
	for (1..$config->checker->workers) {
		push @coros, async {
			my $type_checker = Net::Proxy::Type->new(
				strict        => $config->checker->strict,
				noauth        => 1,
				url           => $config->checker->http_url,
				keyword       => $config->checker->http_keyword,
				https_url     => $config->checker->https_url,
				https_keyword => $config->checker->https_keyword,
			);
			$type_checker->http_strict(1);
			my $speed_checker = $config->checker->speed_check ?
				LWP::UserAgent->new(agent => 'Mozilla 5.0', timeout => 10, parse_head => 0) :
				undef;
			
			while (1) {
				unless (@queue) {
					if ($delay) {
						Coro::Timer::sleep $delay;
						next;
					}
					
					unless (@queue = $class->_get_queue($model, {checked => 0}, {order_by => 'checkdate'})) {
						$delay = CORO_DELAY;
						Coro::Timer::sleep $delay;
						$delay = 0;
						next;
					}
				}
				
				my $proxy = shift @queue;
				my ($type, $conn_time) = $class->_check($type_checker, $config->checker->types, $proxy)
					or do {
						$proxy->delete();
						next;
					};
				
				$proxy->set('type', $type);
				$proxy->set('conn_time', $conn_time);
				
				if ($speed_checker) {
					my $speed = $class->_check_speed($speed_checker, $config->speed_checker, $proxy)
						or do {
							$proxy->delete();
							next;
						};
					
					$proxy->set('speed', $speed);
					$proxy->set('speed_checkdate', DateTime->now(time_zone => TZ));
				}
				
				$proxy->set('checked', 1);
				$proxy->set('checkdate', DateTime->now(time_zone => TZ));
				$proxy->set('in_progress', \0); # force update
				$proxy->set('success_total', 1);
				$proxy->update();
			}
		}
	}
	
	return @coros;
}

sub _start_recheckers {
	my ($class, $config, $model) = @_;
	
	my @coros;
	my @queue;
	my $delay;
	
	for (1..$config->rechecker->workers) {
		push @coros, async {
			my $type_checker = Net::Proxy::Type->new(
				strict        => $config->rechecker->strict,
				noauth        => 1,
				url           => $config->checker->http_url,
				keyword       => $config->checker->http_keyword,
				https_url     => $config->checker->https_url,
				https_keyword => $config->checker->https_keyword,
			);
			$type_checker->http_strict(1);
			my $speed_checker = $config->rechecker->speed_check ?
				LWP::UserAgent->new(agent => 'Mozilla 5.0', timeout => 10, parse_head => 0) :
				undef;
			
			while (1) {
				if ($delay) {
					Coro::Timer::sleep $delay;
					next;
				}
				
				unless (@queue) {
					unless (@queue = $class->_get_queue($model, {checked => 1}, {order_by => 'checkdate'})) {
						$delay = CORO_DELAY;
						Coro::Timer::sleep $delay;
						$delay = 0;
						next;
					}
					
					my $sec_after_last_check = DateTime->now(time_zone => TZ)->
							subtract_datetime_absolute($queue[0]->checkdate)->seconds;
					
					if ($sec_after_last_check < $config->rechecker->interval) {
						$delay = $config->rechecker->interval - $sec_after_last_check;
						Coro::Timer::sleep $delay;
						$delay = 0;
						next;
					}
				}
				
				my $proxy = shift @queue;
				my $fail;
				if (my ($type, $conn_time) = $class->_check($type_checker, $config->checker->types, $proxy)) {
					$proxy->set('type', $type);
					$proxy->set('conn_time', $conn_time);
					
					if ($speed_checker) {
						if (my $speed = $class->_check_speed($speed_checker, $config->speed_checker, $proxy)) {
							$proxy->set('speed', $speed);
						}
						else {
							$fail = 1;
							$proxy->set('speed', 0);
						}
						
						$proxy->set('speed_checkdate', DateTime->now(time_zone => TZ));
					}
				}
				else {
					$fail = 1;
				}
				
				if ($fail) {
					$proxy->set('fails_total', $proxy->fails_total+1);
					$proxy->set('fails', $proxy->fails+1);
					if ($proxy->fails > $config->rechecker->fails_before_delete) {
						$proxy->delete();
						next;
					}
				}
				else {
					$proxy->set('fails', 0);
					$proxy->set('success_total', $proxy->success_total+1);
				}
				
				$proxy->set('checkdate', DateTime->now(time_zone => TZ));
				$proxy->set('in_progress', \0); # force update
				$proxy->update();
			}
		}
	}
	
	return @coros;
}

sub _start_speed_checkers {
	my ($class, $config, $model) = @_;
	
	my @coros;
	my @queue;
	my $delay;
	
	for (1..$config->speed_checker->workers) {
		push @coros, async {
			my $speed_checker = LWP::UserAgent->new(agent => 'Mozilla 5.0', timeout => 10, parse_head => 0);
			
			while (1) {
				if ($delay) {
					Coro::Timer::sleep $delay;
					next;
				}
				
				unless (@queue) {
					unless (@queue = $class->_get_queue(
								$model,
								{checked => 1},
								{order_by => 'speed_checkdate'},
								$config->speed_checker->interval)) {
						
						$delay = CORO_DELAY;
						Coro::Timer::sleep $delay;
						$delay = 0;
						next;
					}
					
					my $sec_after_last_check = DateTime->now(time_zone => TZ)->
							subtract_datetime_absolute($queue[0]->speed_checkdate)->seconds;
					
					if ($sec_after_last_check < $config->speed_checker->interval) {
						$delay = $config->speed_checker->interval - $sec_after_last_check;
						Coro::Timer::sleep $delay;
						$delay = 0;
						next;
					}
				}
				
				my $proxy = shift @queue;
				if (my $speed = $class->_check_speed($speed_checker, $config->speed_checker, $proxy)) {
					$proxy->set('speed', $speed);
					$proxy->set('success_total', $proxy->success_total+1);
				}
				else {
					$proxy->set('fails', $proxy->fails+1);
					if ($proxy->fails > $config->rechecker->fails_before_delete) {
						$proxy->delete();
						next;
					}
					$proxy->set('speed', 0);
					$proxy->set('fails_total', $proxy->fails_total+1);
				}
				
				my $now = DateTime->now(time_zone => TZ);
				$proxy->set('checkdate', $now);
				$proxy->set('speed_checkdate', $now);
				$proxy->set('in_progress', \0); # force update
				$proxy->update();
			}
		}
	}
}

sub _start_searcher {
	my ($class, $config, $model) = @_;