ApacheLog-Compressor

 view release on metacpan or  search on metacpan

examples/compress.pl  view on Meta::CPAN

#!/usr/bin/perl
use strict;
use warnings;

use ApacheLog::Compressor 0.004;
use Sys::Hostname qw(hostname);

binmode STDOUT, ':encoding(utf8)';
binmode STDERR, ':encoding(utf8)';

my ($in, $out) = @ARGV;
die "No input file provided" unless defined $in && length $in;
die "No output file provided" unless defined $out && length $out;

# Write all data to binary output file
open my $out_fh, '>', $out or die "Failed to create output file $out - $!";
binmode $out_fh;

# Provide a callback to send data through to the file
my $alc = ApacheLog::Compressor->new(
	on_write	=> sub {
		my ($self, $pkt) = @_;
		print { $out_fh } $pkt;
	},
	filter => sub {
		my ($self, $data) = @_;
		# Ignore entries with no URL or timestamp

examples/compress.pl  view on Meta::CPAN

		return 0 unless $data->{timestamp};

		# Also skip irrelevant entries, in this case regular OPTIONS * server pings from loadbalancer
		return 0 if $ApacheLog::Compressor::HTTP_METHOD_LIST[$data->{method}] eq 'OPTIONS' && $data->{url} eq '*';
		return 1;
	}
);

# Input file - normally use whichever one's just been closed + rotated
open my $in_fh, '<', $in or die "Failed to open input file $in - $!";
binmode $in_fh, ':encoding(utf8)';

# Initial packet to identify which server this came from
$alc->send_packet('server',
	hostname	=> hostname(),
);

# Read and compress all the lines in the files
while(my $line = <$in_fh>) {
        $alc->compress($line);
}

examples/expand.pl  view on Meta::CPAN

use strict;
use warnings;

use ApacheLog::Compressor 0.004;
use Encode qw(decode_utf8 is_utf8);

my ($in, $out) = @ARGV;
die "No input file provided" unless defined $in && length $in;
die "No output file provided" unless defined $out && length $out;

binmode STDOUT, ':encoding(utf8)';
binmode STDERR, ':encoding(utf8)';

# Write all data to plain text file
open my $out_fh, '>', $out or die "Failed to create output file $out - $!";
binmode $out_fh, ':encoding(utf8)';

use Data::Dumper;
# Provide a callback to send data through to the file
my $alc = ApacheLog::Compressor->new(
	on_log_line	=> sub {
		my ($self, $data) = @_;
		# Use the helper method to expand back to plain text representation
		print { $out_fh } $self->data_to_text($data) . "\n";
	},
);

# Input file - normally use whichever one's just been closed + rotated
open my $in_fh, '<', $in or die "Failed to open input file $in - $!";
binmode $in_fh;

# Read and expand all the lines in the files
my $buffer = '';
while(read($in_fh, my $data, 1024) >= 0) {
	$buffer .= $data;
        $alc->expand(\$buffer);
}
close $in_fh or die $!;
close $out_fh or die $!;

lib/ApacheLog/Compressor.pm  view on Meta::CPAN

version 0.005

=head1 SYNOPSIS

 use ApacheLog::Compressor;
 use Sys::Hostname qw(hostname);

 # Write all data to bzip2-compressed output file
 open my $out_fh, '>', 'compressed.log.bz2' or die "Failed to create output file: $!";
 binmode $out_fh;
 my $zip = IO::Compress::Bzip2->new($out_fh, BlockSize100K => 9);

 # Provide a callback to send data through to the file
 my $alc = ApacheLog::Compressor->new(
	on_write	=> sub {
		my ($self, $pkt) = @_;
		$zip->write($pkt);
	}
 );

t/unicode.t  view on Meta::CPAN

use strict;
use warnings;
use utf8;

use Test::More tests => 7;
use ApacheLog::Compressor;
use Encode qw(is_utf8);
binmode STDOUT, ':encoding(utf8)';
binmode STDERR, ':encoding(utf8)';

my $buffer = '';
my $comp = new_ok('ApacheLog::Compressor' => [
	on_write	=> sub {
		my ($self, $pkt) = @_;
		$buffer .= $pkt;
	}
]);
my $exp = new_ok('ApacheLog::Compressor' => [
	on_write => sub {

t/unicode.t  view on Meta::CPAN

]);
my $line = q(example.com 28104 10.1.0.1 - user@example.com [12/Mar/2011:02:05:45 +0700] "GET /%E4%BB%95%E4%BA%8B%E9%96%A2%E4%BF%82.html HTTP/1.1" 200 - "-" "Mozilla/4.0");

ok($comp->send_packet('server',
	hostname	=> 'apache-server1'
), 'send initial server packet');
ok($comp->compress($line), 'compress the line');

# Try to prove we have binary data
open my $out_fh, '>', \my $tmp or die $!;
binmode $out_fh;
print $out_fh $buffer;
close $out_fh;
$buffer = $tmp;
ok(!is_utf8($buffer), 'utf8 not set');
my $idx = 0;
$exp->expand(\$buffer) while length $buffer && ++$idx < 100;