Acme-Parataxis

 view release on metacpan or  search on metacpan

eg/tar_pack.pl  view on Meta::CPAN

say '';
say "  Every file in that archive was read in $CHUNK byte chunks with an await boundary between";
say '  them, so a cancel could have landed mid-file. Nothing cancelled it, so it did not.';

# Cancelling the read, with the cancel landing inside a file rather than before the first one.
say '';
say '2. Ctrl-C during the read, cancelled at a chunk boundary.';
say '';
{
    my $tok = token_for_signal();

    # A real Ctrl-C calls $tok->cancel from a signal handler at whatever moment the signal is
    # delivered, which is not reproducible. So fire the same call from a chunk callback instead, at a
    # fixed chunk number. Nothing else differs: $SIG{INT} is still wired to the same token above, and
    # the cancel still arrives from outside the wait, which is the part the scheduler has to handle.
    my $fire_at = 8;    # the 9th chunk boundary of the 3rd file
    my ( $seen, $where, $err ) = ( 0, q{}, q{} );
    my $out = File::Spec->catfile( "$TREE", 'cancelled.tar.gz' );
    Acme::Parataxis::run(
        sub {
            eval {
                # 0 means "no deadline" - this scope exists only to register the fiber with $tok, so
                # that any wait entered inside it is interruptible. with_cancel() would not do here:
                # it registers the fiber with its own scope token, and that token is not assigned
                # until the block returns, so a signal handler could not reach it. with_timeout()
                # takes a token you already hold.
                with_timeout(
                    0, $tok,
                    sub {
                        for my $i ( 0 .. $#paths ) {
                            read_hashed(
                                $paths[$i],
                                sub {
                                    $seen++;
                                    $where = sprintf( '%s chunk %d', ( split m{/}, $paths[$i] )[-1], $seen );
                                    $tok->cancel if $seen == $fire_at;
                                }
                            );
                        }
                        return 'unreachable';
                    }
                );
                1;
            } or $err = $@;
            return;
        }
    );
    printf "  %-34s %s\n",       'died with', ( $err ? ref($err) : 'NOTHING' );
    printf "  %-34s %d of %d\n", 'chunk boundaries entered', $seen, $CHUNKS_PER_FILE * scalar @paths;
    printf "  %-34s %s\n",       'cancel delivered during',  $where;
    printf "  %-34s %s\n",       'archive written', ( -e $out ? 'yes' : 'no' );
    say '';
    say sprintf( '  The cancel landed on the %dth boundary, inside a file that was %d chunks long,', $seen, $CHUNKS_PER_FILE );
    say '  and the cost of the signal was the work in flight for that one chunk. Change the chunk size';
    say '  at the top of this file and the granularity moves with it: that is the whole design';
    say "  decision, and it is yours to make, not the scheduler's.";
    say '';
    say '  Note also what is *not* on disk. The reads never reached the build stage, so there is no';
    say '  output file at all - not a zero-length one, not an empty archive. A cooperative cancel';
    say '  cannot leave you a half-written archive, because the only code that writes the destination';
    say '  is code the cancel can stop before it starts. Section 3 is about the case where that is';
    say '  not true.';
}

# What a half-written archive actually does, measured rather than asserted.
say '';
say '3. A half-written archive, and what the standard tools make of it.';
say '';
{
    # A cooperative cancel can never produce one, which is worth being precise about. write() is a
    # single call into C, so the scheduler has no boundary inside it, and section 2's cancel lands
    # before the build is even called. The only way to get a truncated archive is a signal that ends
    # the process outright - SIGKILL, a power cut, an OOM kill. So simulate that honestly: take a
    # finished archive and cut it.
    my $cut = File::Spec->catfile( "$TREE", 'cut.tar.gz' );
    my $all = do {
        open my $fh, '<:raw', $ok_path or die "open $ok_path: $!";
        local $/;
        <$fh>;
    };
    my $full = length $all;
    open my $cfh, '>:raw', $cut or die $!;
    print {$cfh} substr( $all, 0, int( $full * 0.6 ) );
    close $cfh;
    printf "  %-34s %d of %d bytes\n", 'archive cut to 60%', ( -s $cut ), $full;
    for my $tool ( [ 'gzip -t', "gzip -t $cut" ], [ 'tar tzf', "tar tzf $cut" ] ) {
        my ( $label, $cmd ) = @$tool;
        my @lines = split m/\n/, `$cmd 2>&1`;
        my $rc    = $? >> 8;
        printf "  %-34s exit %d, %d entries named\n", $label, $rc, scalar( grep {/part\d+\.dat/} @lines );
    }

    # Extraction is the dangerous one, because the files land on disk before tar gives up.
    my $dest = File::Spec->catdir( "$TREE", 'extracted' );
    File::Path::make_path($dest);
    my @out = split m/\n/, `tar xzf $cut -C $dest 2>&1`;
    my @ext = glob( File::Spec->catfile( $dest, '*' ) );
    printf "  %-34s exit %d, %d files left on disk\n", 'tar xzf into a fresh directory', $? >> 8, scalar @ext;

    # And the iterator this program uses to verify, which is the one a reader would most trust. It is
    # the worst of the four: it does not exit, and it does not die either. It warns once and hands
    # back a short list, so a verify step written as "read the entries and compare" has to notice a
    # warning to notice the damage.
    my ( $recovered, $iter_died, @iter_warned ) = ( 0, q{}, q{} );
    {
        my $next = Archive::Tar->iter( $cut, 1 );
        local $SIG{__WARN__} = sub { push @iter_warned, $_[0] };
        local $@;
        eval {
            while ( my $f = $next->() ) { $recovered++ }
            1;
        } or $iter_died = $@;
        $iter_died =~ s/\n.*//s if $iter_died;
    }
    printf "  %-34s %d of %d entries, %d warning(s), %s\n", 'Archive::Tar->iter', $recovered, $#paths + 1, scalar @iter_warned,
        ( $iter_died ? "died: $iter_died" : 'no exception' );
    if (@iter_warned) {
        my @first = grep {length} map {
            my $w = $_;
            $w =~ s{\s+at\s+\S+\s+line\s+\d+\.?\s*\z}{};    # drop "at eg/foo.pl line 289."
            $w =~ s{\A\s+}{};

eg/tar_pack.pl  view on Meta::CPAN

                $settled = $now;
                last if $now >= $want;
                await_sleep(25);
            }
            $grew = ( $settled // 0 ) > $then;

            # And the half that is easy to skip: awaiting again, outside the cancel scope, works. The
            # slot is still held and the result is still there.
            eval { $await_result = $fut->await; 1 } or $await_died = $@;
            return;
        }
    );
    printf "  %-34s %s\n",                    'cancel delivered after',     sprintf( '%.0f ms', $at );
    printf "  %-34s %d of %d bytes\n",        'output file at that moment', $then, $want;
    printf "  %-34s %s\n",                    'the await returned', ( $err ? ref($err) : 'a result' );
    printf "  %-34s %d bytes, %d ms later\n", 'output file once it settled', ( $settled // $then ), $watch_ms;
    say '';
    if ($grew) {
        say "  It grew by " . ( $settled - $then ) . " bytes *after* the cancellation. That is not a bug and it is";
        say '  not fixable: the closure is a real OS thread running real code, and there is no safe way';
        say '  to yank one mid-instruction. The Blocking documentation says so in as many words -';
        say '  "Cancelling the await does not stop the closure (a real OS thread cannot be yanked); it';
        say '  withdraws the waiter, and the slot releases when the closure actually finishes."';
        say '';
        say '  What that costs a program like this one, concretely:';
        say '    * Do not unlink or rename the output because the await was cancelled. The thread is';
        say '      still writing to that path. unlink only removes the directory entry; the thread goes';
        say '      on filling an inode nothing can reach, and the space comes back only at thread exit.';
        say '    * Do not treat the Future as settled. Only the waiter is gone.';
        say '    * If the destination must be untouched on cancel, have the closure write to a path it';
        say '      owns and let the cancelling fiber decide afterwards what becomes of that file.';
        say '';
        say '  And the follow-through, which is the half that is easy to skip:';
    }
    else {
        say '  It did not grow on this run, so either the closure had already finished or it never got';
        say '  going. Raise the payload size, or lower the sleep, to make mid-write the usual case.';
        say '';
    }
    printf "  %-34s %s\n", 'awaited again, outside the scope',
        ( $await_died ? do { my $x = $await_died; $x =~ s/\n.*//s; "died: $x" } : "'$await_result' (bytes read / chunk passes)" );
    if ( -e $out ) {
        printf "  %-34s %s\n", 'and it is a valid gzip stream', ( system( 'gzip -t ' . quotemeta($out) . ' >/dev/null 2>&1' ) == 0 ? 'yes' : 'NO' );
        unlink $out;
        printf "  %-34s %s\n", 'after unlink', ( -e $out ? 'STILL THERE' : 'gone' );
    }
}
else {
    say '  Acme::Parataxis::Blocking, the separate distribution that provides spawn_blocking, is not';
    say '  available here, so the build stays on this thread:';
    printf "    %-38s %s\n", 'Blocking installed:',        ( $have_blocking               ? 'yes'                            : 'no' );
    printf "    %-38s %s\n", 'an ithreads perl to clone:', ( $ithreads                    ? 'yes'                            : 'no' );
    printf "    %-38s %s\n", '$Config{useithreads}',       ( defined $Config{useithreads} ? "'" . $Config{useithreads} . "'" : 'undef' );
    say '';
    say '  The trap still applies, it is just not demonstrable without the thread, and it is worth';
    say '  knowing before you reach for spawn_blocking on a long job: a cancelled await withdraws the';
    say '  waiter and nothing else. The closure runs to completion. Do not clean up its output on the';
    say '  strength of a cancellation alone.';
    say '';
    say '  Compare the inline numbers: the build above took ' . sprintf( '%.1f ms', $ok_ms ) . ' for ' . sprintf( '%.1f MB', $mb ) . '.';
    say '  A spawn_blocking call costs on the order of 10ms before it starts, so for a job this size';
    say '  the thread is not worth reaching for on speed grounds - only because you want the bytes';
    say '  moving while this thread does something else.';
}

# Two features in this distribution that must not be used in the same program.
#
# This section is a bug report, not a demonstration, and the numbers in it were measured on the
# threaded 5.44.0 build. It is here because the combination is easy to reach by accident: a program
# that turns on transparent unblocking to stop writing await_sleep(0) by hand, and then reaches for
# spawn_blocking for one long job.
#
# enable_transparent_unblocking() installs Acme::Parataxis::Compat, which overrides core file and
# pipe operations process-wide. threads->create clones the interpreter, overrides included, so a
# worker thread inherits them - and inside that worker there is no scheduler and no fiber, so the
# override tries to park a wait that cannot exist.
#
# What that costs, measured, reading a 12MB file in 64KB passes from a spawn_blocking closure:
#
#   transparent unblocking off   192 passes, 12582912 bytes read, exit 0
#   transparent unblocking on    the closure stops partway, and the Future carries
#                                "_park() must be called from inside a scheduled fiber"
#                                instead of anything to do with the file
#
# The second case is not a tidy error, though. At 4MB it prints
# "Attempt to free unreferenced scalar" and then returns the right answer anyway; at 12MB it
# segfaults. A bare threads->create with the overrides installed and no scheduler involved is fine,
# so it takes both features together to corrupt the heap.
#
# Nothing in this file needs transparent unblocking - the reads are chunked by hand precisely so the
# await boundaries are visible and movable. If you do want both features in one program, keep the
# worker free of the wrapped operations, or do the I/O before the thread exists.
say '';
say '5. enable_transparent_unblocking() and spawn_blocking do not mix.';
say '';
printf "  %-34s %s\n", 'transparent unblocking', Acme::Parataxis->transparent_unblocking ? 'on' : 'off';
printf "  %-34s %s\n", 'blocking available here', ( $have_blocking ? 'yes' : 'no' );
say '';
say '  The compatibility layer overrides core file operations process-wide, and a cloned thread';
say '  inherits the overrides but has no scheduler to park in. The symptom is a Future that fails';
say '  with a scheduler error on a closure that never mentioned a scheduler, and at larger sizes a';
say '  segfault. This file leaves transparent unblocking off on purpose, and the sections above do';
say '  their own chunking so the await boundaries stay visible.';
say '';
say '';



( run in 0.806 second using v1.01-cache-2.11-cpan-234fa220139 )