Acme-Parataxis

 view release on metacpan or  search on metacpan

eg/sha_walk.pl  view on Meta::CPAN

#!/usr/bin/perl
# Hashing a directory tree: File::Find + Digest::SHA, made cooperative.
#
# The reflex when a task is slow is to reach for async, and then spend a day making a synchronous walk yield and a
# synchronous hasher cooperate, and end up with a program that is not measurably faster than the one you started with.
# This file is the worked version of that, and the punchline is in the diagnosis near the end: the hasher was never
# I/O-bound. It is CPU-bound, and no amount of scheduling changes a CPU-bound program that has one thread.
#
# Five things get measured, each against a heartbeat fiber that ticks every 10ms so "did it cooperate" is a number:
#
#   1. File::Find. A synchronous recursive walk. What actually makes it hand the thread back is yield(), and
#      maybe_yield() does not - it is a preemption counter, and at the default threshold of 0 it is a no-op. That is
#      the single most common mistake in this file's genre, so it gets its own section and its own numbers.
#   2. Digest::SHA->addfile. XS. It does its own reading in C, where a Perl-level override cannot reach, so Compat
#      cannot help it at all. Measured, not assumed.
#   3. The same digest driven by a Perl read loop. Now the reads are reachable, so Compat does cooperate - and the
#      chunk size turns out to matter more than anything else in this file.
#   4. One fiber per file, bounded by a semaphore. Buys resource bounding, not throughput, and the numbers say so.
#   5. The diagnosis, and the one tuning knob that is actually worth reaching for.
#
# Run it: perl -Mblib eg/sha_walk.pl [files] [bytes-per-file]
use v5.40;
use blib;
$|++;
use Acme::Parataxis qw[:all];
no warnings 'experimental::try';    # try/catch below; the dist does the same in its .pm files
use Acme::Parataxis::Semaphore;
use Config;
use Digest::SHA ();
use File::Find  ();
use File::Path  ();
use File::Spec;
use File::Temp  ();
use Time::HiRes ();
my $FILE_COUNT = shift // 20;
my $FILE_BYTES = shift // 262_144;

# The install that makes sleep/read/sysread cooperative. It has to sit above the code that reads files, which is
# below, and that is the whole placement rule: an override is resolved when each call site is *compiled*, so it
# covers this file's reads and nothing from a module loaded before it.
BEGIN { Acme::Parataxis->enable_transparent_unblocking() }

# CLEANUP => 0 on purpose, and this is not a style choice - it is the fix for a trap that only bites when the
# threaded branch at the end of this file actually runs.
#
# `threads->create` clones the whole perl stack, so every thread gets its *own* copy of the object graph, with its
# own refcounts. The clone's copy of $ROOT is not kept alive by this interpreter's copy of it, so when a thread
# exits, that refcount goes to zero, DESTROY runs, and File::Temp rmtree's the directory out from under the
# threads still working in it.
#
# The symptom is not an error you would guess at: the first couple of threads succeed, the directory is deleted,
# and everything after that dies with ENOENT. "Open failed: No such file or directory" on files that demonstrably
# existed a second earlier. It looks exactly like a race in the code being benchmarked, and it is not.
my $ROOT = File::Temp->newdir( CLEANUP => 0 );

END {
    return if $main::ROOT_DONE;
    $main::ROOT_DONE = 1;
    my $d = $ROOT->dirname;
    File::Path::remove_tree($d) if defined $d && -d $d;
}

# A tree worth walking: a few directories, a few files each, some nested, one deliberately empty.
sub build_tree {
    my ( $root, $count, $bytes ) = @_;



( run in 0.589 second using v1.01-cache-2.11-cpan-234fa220139 )