Acme-Parataxis
view release on metacpan or search on metacpan
eg/tar_pack.pl view on Meta::CPAN
say '';
say " Every file in that archive was read in $CHUNK byte chunks with an await boundary between";
say ' them, so a cancel could have landed mid-file. Nothing cancelled it, so it did not.';
# Cancelling the read, with the cancel landing inside a file rather than before the first one.
say '';
say '2. Ctrl-C during the read, cancelled at a chunk boundary.';
say '';
{
my $tok = token_for_signal();
# A real Ctrl-C calls $tok->cancel from a signal handler at whatever moment the signal is
# delivered, which is not reproducible. So fire the same call from a chunk callback instead, at a
# fixed chunk number. Nothing else differs: $SIG{INT} is still wired to the same token above, and
# the cancel still arrives from outside the wait, which is the part the scheduler has to handle.
my $fire_at = 8; # the 9th chunk boundary of the 3rd file
my ( $seen, $where, $err ) = ( 0, q{}, q{} );
my $out = File::Spec->catfile( "$TREE", 'cancelled.tar.gz' );
Acme::Parataxis::run(
sub {
eval {
# 0 means "no deadline" - this scope exists only to register the fiber with $tok, so
# that any wait entered inside it is interruptible. with_cancel() would not do here:
# it registers the fiber with its own scope token, and that token is not assigned
# until the block returns, so a signal handler could not reach it. with_timeout()
# takes a token you already hold.
with_timeout(
0, $tok,
sub {
for my $i ( 0 .. $#paths ) {
read_hashed(
$paths[$i],
sub {
$seen++;
$where = sprintf( '%s chunk %d', ( split m{/}, $paths[$i] )[-1], $seen );
$tok->cancel if $seen == $fire_at;
}
);
}
return 'unreachable';
}
);
1;
} or $err = $@;
return;
}
);
printf " %-34s %s\n", 'died with', ( $err ? ref($err) : 'NOTHING' );
printf " %-34s %d of %d\n", 'chunk boundaries entered', $seen, $CHUNKS_PER_FILE * scalar @paths;
printf " %-34s %s\n", 'cancel delivered during', $where;
printf " %-34s %s\n", 'archive written', ( -e $out ? 'yes' : 'no' );
say '';
say sprintf( ' The cancel landed on the %dth boundary, inside a file that was %d chunks long,', $seen, $CHUNKS_PER_FILE );
say ' and the cost of the signal was the work in flight for that one chunk. Change the chunk size';
say ' at the top of this file and the granularity moves with it: that is the whole design';
say " decision, and it is yours to make, not the scheduler's.";
say '';
say ' Note also what is *not* on disk. The reads never reached the build stage, so there is no';
say ' output file at all - not a zero-length one, not an empty archive. A cooperative cancel';
say ' cannot leave you a half-written archive, because the only code that writes the destination';
say ' is code the cancel can stop before it starts. Section 3 is about the case where that is';
say ' not true.';
}
# What a half-written archive actually does, measured rather than asserted.
say '';
say '3. A half-written archive, and what the standard tools make of it.';
say '';
{
# A cooperative cancel can never produce one, which is worth being precise about. write() is a
# single call into C, so the scheduler has no boundary inside it, and section 2's cancel lands
# before the build is even called. The only way to get a truncated archive is a signal that ends
# the process outright - SIGKILL, a power cut, an OOM kill. So simulate that honestly: take a
# finished archive and cut it.
my $cut = File::Spec->catfile( "$TREE", 'cut.tar.gz' );
my $all = do {
open my $fh, '<:raw', $ok_path or die "open $ok_path: $!";
local $/;
<$fh>;
};
my $full = length $all;
open my $cfh, '>:raw', $cut or die $!;
print {$cfh} substr( $all, 0, int( $full * 0.6 ) );
close $cfh;
printf " %-34s %d of %d bytes\n", 'archive cut to 60%', ( -s $cut ), $full;
for my $tool ( [ 'gzip -t', "gzip -t $cut" ], [ 'tar tzf', "tar tzf $cut" ] ) {
my ( $label, $cmd ) = @$tool;
my @lines = split m/\n/, `$cmd 2>&1`;
my $rc = $? >> 8;
printf " %-34s exit %d, %d entries named\n", $label, $rc, scalar( grep {/part\d+\.dat/} @lines );
}
# Extraction is the dangerous one, because the files land on disk before tar gives up.
my $dest = File::Spec->catdir( "$TREE", 'extracted' );
File::Path::make_path($dest);
my @out = split m/\n/, `tar xzf $cut -C $dest 2>&1`;
my @ext = glob( File::Spec->catfile( $dest, '*' ) );
printf " %-34s exit %d, %d files left on disk\n", 'tar xzf into a fresh directory', $? >> 8, scalar @ext;
# And the iterator this program uses to verify, which is the one a reader would most trust. It is
# the worst of the four: it does not exit, and it does not die either. It warns once and hands
# back a short list, so a verify step written as "read the entries and compare" has to notice a
# warning to notice the damage.
my ( $recovered, $iter_died, @iter_warned ) = ( 0, q{}, q{} );
{
my $next = Archive::Tar->iter( $cut, 1 );
local $SIG{__WARN__} = sub { push @iter_warned, $_[0] };
local $@;
eval {
while ( my $f = $next->() ) { $recovered++ }
1;
} or $iter_died = $@;
$iter_died =~ s/\n.*//s if $iter_died;
}
printf " %-34s %d of %d entries, %d warning(s), %s\n", 'Archive::Tar->iter', $recovered, $#paths + 1, scalar @iter_warned,
( $iter_died ? "died: $iter_died" : 'no exception' );
if (@iter_warned) {
my @first = grep {length} map {
my $w = $_;
$w =~ s{\s+at\s+\S+\s+line\s+\d+\.?\s*\z}{}; # drop "at eg/foo.pl line 289."
$w =~ s{\A\s+}{};
eg/tar_pack.pl view on Meta::CPAN
$settled = $now;
last if $now >= $want;
await_sleep(25);
}
$grew = ( $settled // 0 ) > $then;
# And the half that is easy to skip: awaiting again, outside the cancel scope, works. The
# slot is still held and the result is still there.
eval { $await_result = $fut->await; 1 } or $await_died = $@;
return;
}
);
printf " %-34s %s\n", 'cancel delivered after', sprintf( '%.0f ms', $at );
printf " %-34s %d of %d bytes\n", 'output file at that moment', $then, $want;
printf " %-34s %s\n", 'the await returned', ( $err ? ref($err) : 'a result' );
printf " %-34s %d bytes, %d ms later\n", 'output file once it settled', ( $settled // $then ), $watch_ms;
say '';
if ($grew) {
say " It grew by " . ( $settled - $then ) . " bytes *after* the cancellation. That is not a bug and it is";
say ' not fixable: the closure is a real OS thread running real code, and there is no safe way';
say ' to yank one mid-instruction. The Blocking documentation says so in as many words -';
say ' "Cancelling the await does not stop the closure (a real OS thread cannot be yanked); it';
say ' withdraws the waiter, and the slot releases when the closure actually finishes."';
say '';
say ' What that costs a program like this one, concretely:';
say ' * Do not unlink or rename the output because the await was cancelled. The thread is';
say ' still writing to that path. unlink only removes the directory entry; the thread goes';
say ' on filling an inode nothing can reach, and the space comes back only at thread exit.';
say ' * Do not treat the Future as settled. Only the waiter is gone.';
say ' * If the destination must be untouched on cancel, have the closure write to a path it';
say ' owns and let the cancelling fiber decide afterwards what becomes of that file.';
say '';
say ' And the follow-through, which is the half that is easy to skip:';
}
else {
say ' It did not grow on this run, so either the closure had already finished or it never got';
say ' going. Raise the payload size, or lower the sleep, to make mid-write the usual case.';
say '';
}
printf " %-34s %s\n", 'awaited again, outside the scope',
( $await_died ? do { my $x = $await_died; $x =~ s/\n.*//s; "died: $x" } : "'$await_result' (bytes read / chunk passes)" );
if ( -e $out ) {
printf " %-34s %s\n", 'and it is a valid gzip stream', ( system( 'gzip -t ' . quotemeta($out) . ' >/dev/null 2>&1' ) == 0 ? 'yes' : 'NO' );
unlink $out;
printf " %-34s %s\n", 'after unlink', ( -e $out ? 'STILL THERE' : 'gone' );
}
}
else {
say ' Acme::Parataxis::Blocking, the separate distribution that provides spawn_blocking, is not';
say ' available here, so the build stays on this thread:';
printf " %-38s %s\n", 'Blocking installed:', ( $have_blocking ? 'yes' : 'no' );
printf " %-38s %s\n", 'an ithreads perl to clone:', ( $ithreads ? 'yes' : 'no' );
printf " %-38s %s\n", '$Config{useithreads}', ( defined $Config{useithreads} ? "'" . $Config{useithreads} . "'" : 'undef' );
say '';
say ' The trap still applies, it is just not demonstrable without the thread, and it is worth';
say ' knowing before you reach for spawn_blocking on a long job: a cancelled await withdraws the';
say ' waiter and nothing else. The closure runs to completion. Do not clean up its output on the';
say ' strength of a cancellation alone.';
say '';
say ' Compare the inline numbers: the build above took ' . sprintf( '%.1f ms', $ok_ms ) . ' for ' . sprintf( '%.1f MB', $mb ) . '.';
say ' A spawn_blocking call costs on the order of 10ms before it starts, so for a job this size';
say ' the thread is not worth reaching for on speed grounds - only because you want the bytes';
say ' moving while this thread does something else.';
}
# Two features in this distribution that must not be used in the same program.
#
# This section is a bug report, not a demonstration, and the numbers in it were measured on the
# threaded 5.44.0 build. It is here because the combination is easy to reach by accident: a program
# that turns on transparent unblocking to stop writing await_sleep(0) by hand, and then reaches for
# spawn_blocking for one long job.
#
# enable_transparent_unblocking() installs Acme::Parataxis::Compat, which overrides core file and
# pipe operations process-wide. threads->create clones the interpreter, overrides included, so a
# worker thread inherits them - and inside that worker there is no scheduler and no fiber, so the
# override tries to park a wait that cannot exist.
#
# What that costs, measured, reading a 12MB file in 64KB passes from a spawn_blocking closure:
#
# transparent unblocking off 192 passes, 12582912 bytes read, exit 0
# transparent unblocking on the closure stops partway, and the Future carries
# "_park() must be called from inside a scheduled fiber"
# instead of anything to do with the file
#
# The second case is not a tidy error, though. At 4MB it prints
# "Attempt to free unreferenced scalar" and then returns the right answer anyway; at 12MB it
# segfaults. A bare threads->create with the overrides installed and no scheduler involved is fine,
# so it takes both features together to corrupt the heap.
#
# Nothing in this file needs transparent unblocking - the reads are chunked by hand precisely so the
# await boundaries are visible and movable. If you do want both features in one program, keep the
# worker free of the wrapped operations, or do the I/O before the thread exists.
say '';
say '5. enable_transparent_unblocking() and spawn_blocking do not mix.';
say '';
printf " %-34s %s\n", 'transparent unblocking', Acme::Parataxis->transparent_unblocking ? 'on' : 'off';
printf " %-34s %s\n", 'blocking available here', ( $have_blocking ? 'yes' : 'no' );
say '';
say ' The compatibility layer overrides core file operations process-wide, and a cloned thread';
say ' inherits the overrides but has no scheduler to park in. The symptom is a Future that fails';
say ' with a scheduler error on a closure that never mentioned a scheduler, and at larger sizes a';
say ' segfault. This file leaves transparent unblocking off on purpose, and the sections above do';
say ' their own chunking so the await boundaries stay visible.';
say '';
say '';
( run in 0.806 second using v1.01-cache-2.11-cpan-234fa220139 )