Acme-Parataxis
view release on metacpan or search on metacpan
eg/multi_tail.pl view on Meta::CPAN
use v5.40;
use blib;
$|++;
use Acme::Parataxis qw[:all];
use Acme::Parataxis::Channel;
use File::Spec;
use File::Temp 'tempdir';
use Time::HiRes qw[time];
# multi-tail: one parataxis fiber per log file, all tailing a growing set of
# rotated log files from a single OS thread. Every tail fiber is a plain
# synchronous loop - open, sysread to EOF, await_sleep(poll), repeat - the
# kind of code that would serialize trivially if the files were read one after
# another; run together they all advance within the same poll window.
#
# writer (1 fiber) simulates N loggers, appending patterned lines to each
# file at staggered rates and parking between appends, so the scheduler
# keeps the tails running. Halfway through it rotates err.log (renamed
# aside and recreated, or truncated in place when the OS denies moving a
# file another fiber has open - Windows does), so the tail fiber must
# notice the shrink and fold to the fresh file.
# tails (1 fiber per file) sysread whatever appeared since the last poll,
# push each complete line onto a bounded channel as {file,text}; a size
# that shrank or a different inode at the same path is treated as
# rotation and the fiber reopens the path and keeps going.
# main (the run() body) drains the channel - the filter/annotate stage -
# and prefixes every line with a running count and the file it came from,
# applying the optional --match filter. Bounded queues mean memory never
# tracks the log volume.
#
# Usage: perl eg/multi_tail.pl [--match PATTERN] [--lines N] [--poll MS]
# self-generates its logs so it runs anywhere, prints the tailed stream,
# then exits 0 once the writer is done and every tail has caught up.
my $POLL = 120; # ms between tail polls
my $LINES = 60; # lines per file; 3 * $LINES streams in total
my $MATCH;
{
my @a = @ARGV;
while (@a) {
my $arg = shift @a;
if ( $arg eq '--match' ) { $MATCH = shift @a }
elsif ( $arg eq '--lines' ) { $LINES = int shift @a }
elsif ( $arg eq '--poll' ) { $POLL = int shift @a }
else { die "unknown option '$arg'\n" }
}
}
$MATCH = $MATCH ? eval {qr/$MATCH/} // die "bad --match '$MATCH': $@" : undef;
my $dir = tempdir( 'parataxis-mtail-XXXXXX', CLEANUP => 0, TMPDIR => 1 );
my @NAMES = qw[app.log web.log err.log];
my @paths = map { File::Spec->catfile( $dir, $_ ) } @NAMES;
for my $p (@paths) { # loggers pre-exist: a tail must never race their creation
open my $fh, '>', $p or die "cannot seed $p: $!\n";
close $fh;
}
my @TAGS = qw[conn http err];
my $out = Acme::Parataxis::Channel->new( capacity => 1024 );
my $writer_done = 0;
my $start = time();
sub writer_fiber {
my @done = ( 0, 0, 0 );
my $rotated = 0;
while ( $done[0] + $done[1] + $done[2] < 3 * $LINES ) {
my $slot = int rand 3;
next if $done[$slot] >= $LINES;
$done[$slot]++;
my $text = sprintf '%s.%d', $TAGS[$slot], $done[$slot];
open my $fh, '>>', $paths[$slot] or die "cannot append $paths[$slot]: $!\n";
print {$fh} scalar(localtime), " $text\n";
close $fh;
my $total = $done[0] + $done[1] + $done[2];
if ( !$rotated && $total >= int( 3 * $LINES / 2 ) ) {
my $bak = "$paths[2].old";
unlink $bak if -e $bak;
if ( rename $paths[2], $bak ) {
open my $nf, '>>', $paths[2] or die "cannot recreate $paths[2]: $!\n";
print {$nf} scalar(localtime), " notice err.log rotated away and recreated\n";
close $nf;
$out->put( { file => '---', text => "err.log was rotated away and recreated" } );
}
else {
open my $nf, '>', $paths[2] or die "cannot rotate $paths[2]: $!\n";
print {$nf} scalar(localtime), " notice err.log truncated in place (rename denied)\n";
close $nf;
$out->put( { file => '---', text => "err.log was truncated in place (rename denied)" } );
}
await_sleep( 3 * $POLL ); # settle: let the tail reopen before any new err.log writes
$rotated = 1;
}
await_sleep( 4 + int rand 18 ); # park, don't block the shared thread
}
$writer_done = 1;
1;
}
sub tail_fiber ( $id, $path ) {
open my $fh, '<', $path or die "cannot open $path: $!\n";
binmode $fh;
seek $fh, 0, 2; # skip whatever pre-existed: tail only
my $name = $NAMES[$id];
my $consumed = -s $path; # bytes logically read (tell() is unreliable here)
my $ino = ( stat $fh )[1];
my $carry = '';
while (1) {
my $chunk = '';
my $n = sysread( $fh, $chunk, 65536 );
if ( defined $n && $n > 0 ) {
$consumed += $n;
$carry .= $chunk;
while ( ( my $i = index $carry, "\n" ) >= 0 ) {
my $line = substr $carry, 0, $i + 1, '';
chop $line;
$out->put( { file => $name, text => $line } );
}
}
my $size = -s $path;
my $pino = ( stat $path )[1];
if ( ( defined $size && $size < $consumed ) || ( defined $pino && $pino != $ino ) ) {
close $fh; # truncated or replaced: fold to the fresh file
my $opened = 0;
for ( my $t = 0; $t < 10; $t++ ) { # the path can briefly not exist mid-rotation
if ( open $fh, '<', $path ) {
binmode $fh;
$ino = ( stat $fh )[1];
$carry = ''; # the fresh file is a new ledger
$consumed = 0;
$opened = 1;
last;
}
await_sleep($POLL);
}
last if !$opened; # the path vanished for good and the writer is gone
$out->put( { file => '---', text => "$name was rotated; tailing the fresh file" } );
}
last if $writer_done && $carry eq '' && ( ( -s $path // -1 ) <= $consumed );
await_sleep($POLL);
}
close $fh;
1;
}
run(
sub {
spawn( \&writer_fiber );
my @tails;
for my $id ( 0 .. $#NAMES ) {
my $path = $paths[$id];
push @tails, spawn( sub { tail_fiber( $id, $path ) } );
}
spawn( sub { $_->await for @tails; $out->shutdown; 1 } ); # reaper: close the stream
my $seen = 0; # lines on the channel = filter/annotate stage input
my $shown = 0;
while ( defined( my $ent = $out->get ) ) {
$seen++;
next if defined $MATCH && $ent->{text} !~ $MATCH;
$shown++;
printf "%-4d %-20s %s\n", $seen, $ent->{file}, $ent->{text};
}
$_->await for @tails;
my $elapsed = time() - $start;
say sprintf "\n%d line(s) streamed through %d tail fiber(s) in %.2fs", $seen, scalar @NAMES, $elapsed;
say "$shown shown" if defined $MATCH;
say "(scheduler: 1 OS thread, ", Acme::Parataxis::get_live_fiber_count(), " live fiber(s) at the end)";
stop();
}
);
for my $p ( @paths, map {"$_.old"} @paths ) { # everything closed: clear the temp dir
unlink $p;
}
rmdir $dir;
exit 0;
( run in 0.833 second using v1.01-cache-2.11-cpan-234fa220139 )