HTTP-Proxy
view release on metacpan or search on metacpan
This engine is still beta, you must enable it by hand.
0.16 Thu Sep 1 19:13:55 CEST 2005
[ENHANCEMENTS]
- the new HTTP::Proxy::Engine class and its subclasses
now handle the life and death of child processes
- the Content-Length header is now removed only if body
filters will be applied on the response body
- HTTP::Proxy now supports some Apache-like attributes
(start_servers, max_clients, max_requests_per_child,
min_spare_servers, max_spare_servers, keep_alive,
max_keep_alive_requests, keep_alive_timeout)
- added support for ALL WebDAV/DeltaV methods
- the query argument to push_filter(), added in 0.14, should
now work (thanks to Simon Cozens for spotting the problem)
- the proxy now has a stash, which is a hash where filters can
store data (possibly to share it). (Requested by Mark Fowler)
Warning: since the proxy forks for each TCP connection, the
data is only shared between filters in the same child process.
[DEPRECATION]
- the maxchild, maxconn and maxserve accessors are now
deprecated. They will disappear in the future:
+ maxchild has no replacement (should be handled by the engine)
+ maxconn becomes max_connections
+ maxserve becomes max_keep_alive_requests
- Information regarding the way the engine should behave
must passed in the constructor or directly to the engine
[NEW METHODS]
- $proxy->engine() return the HTTP::Proxy::Engine instance
- $proxy->new_connection() increase the TCP connections counter
(should only be used by HTTP::Proxy::Engine object)
[FIXES]
- Makefile.PL was not playing nice with Build.PL in the
previous distributions. This has been fixed. Sorry for the
inconvenience
to the proxy from inside a filter ($self->proxy->daemon->peerhost)
- many documentation changes
This version is NOT compatible with the previous ones
regarding the way filters work.
*** MAJOR INTERFACE CHANGES ***
0.09 Fri Aug 15 21:12:17 CEST 2003
- maxserve is now correctly handled
- corrected a bug in the t/20keepalive.t test file that
made the tests fail on some machines
0.08 Thu Mar 13 01:41:42 CET 2003
- cleaned up support for filters
- added support for "buffering" filters
and a new HTTP::Proxy::FilterStack class
- added an anonymiser script (eg/anonymiser.pl)
- the tests won't break if a local proxy is configured
- the interfaces are very likely to change soon
t/00basic.t
t/05new.t
t/10init.t
t/11log.t
t/15accessors.t
t/15deprecated.t
t/16stash.t
t/17fstack.t
t/18engine.t
t/20dummy.t
t/20keepalive.t
t/22http.t
t/22transparent.t
t/23connect.t
t/23https.t
t/40push_filters.t
t/41filters.t
t/42will_modify.t
t/50hopbyhop.t
t/50standard.t
t/50via.t
lib/HTTP/Proxy.pm view on Meta::CPAN
# some defaults
my %defaults = (
agent => undef,
chunk => 4096,
daemon => undef,
host => 'localhost',
logfh => *STDERR,
logmask => NONE,
max_connections => 0,
max_keep_alive_requests => 10,
port => 8080,
stash => {},
timeout => 60,
via => undef,
x_forwarded_for => 1,
);
# non modifiable defaults
my $self = bless { conn => 0, loop => 1 }, $class;
# support for deprecated stuff
{
my %convert = (
maxchild => 'max_clients',
maxconn => 'max_connections',
maxserve => 'max_keep_alive_requests',
);
while( my ($old, $new) = each %convert ) {
if( exists $params{$old} ) {
$params{$new} = delete $params{$old};
carp "$old is deprecated, please use $new";
}
}
}
# get attributes
lib/HTTP/Proxy.pm view on Meta::CPAN
carp "HTTP daemon not started yet";
return undef;
}
return $self->daemon->url;
}
# normal accessors
for my $attr ( qw(
agent chunk daemon host logfh port request response hop_headers
logmask via x_forwarded_for client_headers engine
max_connections max_keep_alive_requests
)
)
{
no strict 'refs';
*{"HTTP::Proxy::$attr"} = sub {
my $self = shift;
my $old = $self->{$attr};
$self->{$attr} = shift if @_;
return $old;
}
lib/HTTP/Proxy.pm view on Meta::CPAN
*{"HTTP::Proxy::$attr"} = sub { $_[0]{$attr} }
}
sub max_clients { shift->engine->max_clients( @_ ) }
# deprecated methods are still supported
{
my %convert = (
maxchild => 'max_clients',
maxconn => 'max_connections',
maxserve => 'max_keep_alive_requests',
);
while ( my ( $old, $new ) = each %convert ) {
no strict 'refs';
*$old = sub {
carp "$old is deprecated, please use $new";
goto \&$new;
};
}
}
lib/HTTP/Proxy.pm view on Meta::CPAN
or die "Cannot initialize proxy daemon: $!";
$self->daemon($daemon);
return $daemon;
}
sub _init_agent {
my $self = shift;
my $agent = LWP::UserAgent->new(
env_proxy => 1,
keep_alive => 2,
parse_head => 0,
timeout => $self->timeout,
)
or die "Cannot initialize proxy agent: $!";
$self->agent($agent);
return $agent;
}
# This is the internal "loop" that lets the child process process the
# incoming connections.
lib/HTTP/Proxy.pm view on Meta::CPAN
else { print $conn $content; }
}
# FIXME ftp, gopher
$conn->print( $response->content )
if defined $req->uri->scheme
and $req->uri->scheme =~ /^(?:ftp|gopher)$/
and $response->is_success;
$self->log( SOCKET, "SOCKET", "Connection closed by the proxy" ), last
if $last || $served >= $self->max_keep_alive_requests;
}
$self->log( SOCKET, "SOCKET", "Connection closed by the client" )
if !$last
and $served < $self->max_keep_alive_requests;
$self->log( PROCESS, "PROCESS", "Served $served requests" );
$conn->close;
}
# INTERNAL METHOD
# send the response headers for the proxy
# expects $served (number of requests served)
# returns $last and $chunked (last request served, chunked encoding)
sub _send_response_headers {
my ( $self, $served ) = @_;
lib/HTTP/Proxy.pm view on Meta::CPAN
$response->content('');
}
elsif ( $response->request && $response->request->method eq "HEAD" )
{ # probably OK, says HTTP::Daemon
}
else {
if ( $conn->proto_ge("HTTP/1.1") ) {
$chunked++;
$response->push_header( "Transfer-Encoding" => "chunked" );
$response->push_header( "Connection" => "close" )
if $served >= $self->max_keep_alive_requests;
}
else {
$last++;
$conn->force_last_request;
}
}
print $conn $response->headers_as_string($CRLF);
print $conn $CRLF; # separates headers and content
}
$self->log( STATUS, "RESPONSE", $response->status_line );
lib/HTTP/Proxy.pm view on Meta::CPAN
The maximum number of TCP connections the proxy will accept before
returning from start(). 0 (the default) means never stop accepting
connections.
C<maxconn> is deprecated.
Note: C<max_connections> will be deprecated soon, for two reasons: 1)
it is more of an L<HTTP::Proxy::Engine> attribute, 2) not all engines will
support it.
=item max_keep_alive_requests
=item maxserve
The maximum number of requests the proxy will serve in a single connection.
(same as C<MaxRequestsPerChild> in Apache)
C<maxserve> is deprecated.
=item port
lib/HTTP/Proxy.pm view on Meta::CPAN
=item max_spare_servers
Maximum number of inactive child processes.
=back
Those attributes control the HTTP connection:
=over 4
=item keep_alive
Support for keep alive HTTP connections.
=item max_keep_alive_requests
Maximum number of HTTP connections within a single TCP connection.
=item keep_alive_timeout
Timeout for keep-alive connection.
=back
=head1 EXPORTED SYMBOLS
No symbols are exported by default. The C<:log> tag exports all the
logging constants.
=head1 BUGS
lib/HTTP/Proxy/Engine/Legacy.pm view on Meta::CPAN
my $self = shift;
my $proxy = $self->proxy;
my $kids = $self->kids;
# check for new connections
my @ready = $self->select->can_read(1);
for my $fh (@ready) { # there's only one, anyway
# single-process proxy (useful for debugging)
if ( $self->max_clients == 0 ) {
$proxy->max_keep_alive_requests(1); # do not block simultaneous connections
$proxy->log( HTTP::Proxy::PROCESS, "PROCESS",
"No fork allowed, serving the connection" );
$proxy->serve_connections($fh->accept);
$proxy->new_connection;
next;
}
if ( @$kids >= $self->max_clients ) {
$proxy->log( HTTP::Proxy::ERROR, "PROCESS",
"Too many child process, serving the connection" );
lib/HTTP/Proxy/Engine/Legacy.pm view on Meta::CPAN
my $proxy = HTTP::Proxy->new( engine => 'Legacy' );
=head1 DESCRIPTION
This engine reproduces the older child creation algorithm of L<HTTP::Proxy>.
Angelos Karageorgiou C<< <angelos@unix.gr> >> reports:
I<I got the Legacy engine to work really fast under C<Win32> with the following trick:>
max_keep_alive_requests(1);
max_clients(120);
$HTTP::VERSION(1.0); # just in case
I<and it smokes.>
I<It seems that forked children are really slow when calling select for handling C<keep-alive>d requests!>
=head1 METHODS
The module defines the following methods, used by L<HTTP::Proxy> main loop:
=over 4
=item start()
Initialise the engine.
lib/HTTP/Proxy/Engine/NoFork.pm view on Meta::CPAN
__PACKAGE__->make_accessors( 'select' );
sub start {
my $self = shift;
my $proxy = $self->proxy;
$self->select( IO::Select->new( $proxy->daemon ) );
# clients will not block the proxy by keeping the connection open
$proxy->max_keep_alive_requests( 1 );
}
sub run {
my $self = shift;
my $proxy = $self->proxy;
# check for new connections
for my $fh ( $self->select->can_read() ) { # there's only one, anyway
# single-process proxy
t/15accessors.t view on Meta::CPAN
# default values
#
my %meth = (
agent => undef,
chunk => 4096,
daemon => undef,
host => 'localhost',
logfh => *main::STDERR,
max_connections => 0,
max_keep_alive_requests => 10,
port => 8080,
request => undef,
response => undef,
hop_headers => undef,
logmask => 0,
x_forwarded_for => 1,
conn => 0,
client_socket => undef,
# loop is not used/internal for now
);
t/15accessors.t view on Meta::CPAN
like( $proxy->via(), qr!\(HTTP::Proxy/$HTTP::Proxy::VERSION\)$!,
"via has the correct default");
{
my $my_via_proxy = HTTP::Proxy->new( via => 'VIA!VIA!VIA!' );
is( $my_via_proxy->via(), 'VIA!VIA!VIA!', 'custom via' );
}
# test deprecated accessors
$proxy = HTTP::Proxy->new( maxserve => 127, maxconn => 255 );
is( $proxy->max_keep_alive_requests, 127, "deprecated maxserve");
is( $proxy->max_connections, 255, "deprecated maxconn");
#
# test generated accessors (they're all the same)
#
is( $proxy->port(8888), $meth{port}, "Set return the previous value" );
is( $proxy->port, 8888, "Set works" );
#
t/15deprecated.t view on Meta::CPAN
use Test::More;
use File::Spec;
use HTTP::Proxy;
my $errfile = File::Spec->catfile( 't', 'stderr.out' );
my @deprecated = (
[ maxchild => qr/^maxchild is deprecated, please use max_clients/ ],
[ maxconn => qr/^maxconn is deprecated, please use max_connections/ ],
[
maxserve =>
qr/^maxserve is deprecated, please use max_keep_alive_requests/
],
);
plan tests => 4 * @deprecated;
# check the warnings
open my $olderr, ">&STDERR" or die "Can't dup STDERR: $!";
open STDERR, '>', $errfile or die "Can't redirect STDERR: $!";
select STDERR;
$| = 1; # make unbuffered
t/15deprecated.t view on Meta::CPAN
like( shift @err, $_->[1], "$_->[0] is deprecated" );
like( shift @err, $_->[1], "$_->[0] is deprecated" );
}
diag $_ for @err;
unlink $errfile or diag "Can't unlink $errfile: $!";
# check that the real method was called
is( $p1->max_clients, 1, "max_clients called for maxchild" );
is( $p1->max_connections, 3, "max_connections called for maxconn" );
is( $p1->max_keep_alive_requests, 5, "max_keep_alive_requests called for maxserve" );
is( $p2->max_clients, 9, "max_clients called for maxchild" );
is( $p2->max_connections, 8, "max_connections called for maxconn" );
is( $p2->max_keep_alive_requests, 7, "max_keep_alive_requests called for maxserve" );
t/20keepalive.t view on Meta::CPAN
# this is to work around tests in forked processes
$test->use_numbers(0);
$test->no_ending(1);
# create a HTTP::Daemon (on an available port)
my $server = server_start();
# create a HTTP::Proxy
my $proxy = HTTP::Proxy->new(
port => 0,
max_keep_alive_requests => 3, # no more than 3 requests per connection
max_connections => 3, # no more than 3 connections
);
$proxy->init; # required to access the url later
$proxy->agent->no_proxy( URI->new( $server->url )->host );
# fork the HTTP server
my @pids;
my $pid = fork;
die "Unable to fork web server" if not defined $pid;
t/20keepalive.t view on Meta::CPAN
$ua->proxy( http => $proxy->url );
my $req = shift @requests;
$res =
$ua->simple_request(
HTTP::Request->new( GET => $server->url . $req ) );
ok( $res->is_success, "Got an answer (@{[$res->status_line]})" );
$re = quotemeta $req;
like( $res->content, qr/$re/, "The client got what it expected" );
# the other connections (keep-alive)
$ua = LWP::UserAgent->new( keep_alive => 1 );
$ua->proxy( http => $proxy->url );
for (@requests) {
$res =
$ua->simple_request( HTTP::Request->new( GET => $server->url . $_ ) );
ok( $res->is_success, "Got an answer (@{[$res->status_line]})" );
$re = quotemeta;
like( $res->content, qr/$re/, "The client got what it expected" );
}
# make sure both kids are dead
t/61simple2.t view on Meta::CPAN
my ( $self, $dataref, $message, $protocol, $buffer ) = @_;
$$dataref =~ s/test/foo/g;
};
my $filter = HTTP::Proxy::BodyFilter::simple->new($sub);
# create the proxy
my $proxy = HTTP::Proxy->new(
port => 0,
max_clients => 0,
max_keep_alive_requests => 1,
max_connections => 1,
);
$proxy->init;
$proxy->agent->protocols_allowed(undef);
$proxy->push_filter( response => $filter, scheme => 'file', mime => 'text/*' );
my $url = $proxy->url;
# fork the proxy
my @pids;
{
t/90httpstatus.t view on Meta::CPAN
plan tests => $tests;
SKIP:
{
skip "$base is not available", $tests unless web_ok($base);
# $tests + 2, because of the duplicate 401
my $proxy = HTTP::Proxy->new(
port => 0,
max_keep_alive_requests => $tests,
max_connections => 1,
);
$proxy->init;
my $ua = LWP::UserAgent->new( keep_alive => 1 );
$ua->proxy( http => $proxy->url );
# fork the proxy
my $pid = fork_proxy($proxy);
# check all those pages
for (@url) {
my ( $doc, $status, $status2 ) = @$_;
my $res = $ua->simple_request( GET "$base/$doc" );
is( $res->code, $status, "$doc => $status " . $res->message );
t/05new.t - the HTTP::Proxy constructor
1x - Minimal functionnality tests
t/10init.t - the proxy initialisation
t/11log.t - the log() and logmask() methods
t/15accessors.t - the proxy accessors
t/17fstack.t - the internal HTTP::Proxy::FilterStack object
2x - Network protocols test
t/20dummy.t - tests against a dummy web server
t/20keepalive.t - test the keep-alive connections
t/22http.t - test actual HTTP servers
t/22transparent.t - test transparent proxying
t/23connect.t - test CONNECT to a ssh server
t/23https.t - test CONNECT for SSL
3x - (Reserved for future use)
4x - Filter-related functions
t/40push_filters.t - the push_filter method
( run in 1.186 second using v1.01-cache-2.11-cpan-71847e10f99 )