view release on metacpan or  search on metacpan

benchmarking/bench-sklearn-scoring.pl  view on Meta::CPAN

#!/usr/bin/perl
# benchmarking/bench-sklearn-scoring.pl
#
# Compares scoring throughput among three Perl backends and scikit-learn's
# IsolationForest across two axes:
#   * query-set size       (fixed feature count)
#   * feature-vector width (fixed query-set size)
#
# Perl backends benchmarked (when available):
#   pure perl   -- no Inline::C  (use_c => 0)
#   C serial    -- Inline::C, single-threaded  (use_c => 1, use_openmp => 0)
#   C+OpenMP    -- Inline::C + OpenMP parallel (use_c => 1, use_openmp => 1)
#
# The same training CSV and query CSVs are used by all sides so the
# comparison is on identical data.  Models are pre-trained before any
# timing starts.
#
# Method correspondence:
#   Perl score_samples         <-->  clf.score_samples(X)      (same formula, opposite sign)
#   Perl predict               <-->  clf.predict(X)            (same semantics, 0/1 vs -1/+1)
#   Perl score_predict_samples <-->  (no sklearn equivalent)
#   Perl score_predict_split   <-->  (no sklearn equivalent)
#   Perl path_lengths          <-->  (no sklearn equivalent)
#   (no Perl equivalent)       <-->  clf.decision_function(X)  (threshold-shifted score)
#
# The ratio column shows sklearn ops/s / best-Perl ops/s.
# >1 means sklearn is faster; <1 means Perl is faster.
#
# Unavailable backends (Inline::C not installed, OpenMP not linked,
# scikit-learn not installed) are omitted from the table.
#
# Run with:
#   perl -Ilib benchmarking/bench-sklearn-scoring.pl

use strict;
use warnings;
use lib '../lib';
use FindBin;
use lib "$FindBin::Bin";
use BenchAccel  qw(wall_rate);
use File::Temp  qw(tempfile);
use JSON::PP    ();
use Algorithm::Classifier::IsolationForest;

use constant PI => 3.14159265358979;

my $HAS_C      = $Algorithm::Classifier::IsolationForest::HAS_C;
my $HAS_OPENMP = $Algorithm::Classifier::IsolationForest::HAS_OPENMP;

# -----------------------------------------------------------------------
# Data generation
# -----------------------------------------------------------------------
sub gaussian {
    my ( $mu, $sigma ) = @_;
    return $mu + $sigma
        * sqrt( -2 * log( rand() || 1e-12 ) )
        * cos( 2 * PI * rand() );
}

sub make_data {
    my ( $n, $nf ) = @_;
    my @rows = map { [ map { gaussian( 0, 1 ) } 1 .. $nf ] } 1 .. $n;
    for ( 1 .. int( $n * 0.05 ) ) {
        my $r = 5 + rand() * 3;
        push @rows, [ map { $r * ( rand() > 0.5 ? 1 : -1 ) } 1 .. $nf ];
    }
    return \@rows;
}

# -----------------------------------------------------------------------
# Parameters

benchmarking/bench-sklearn-scoring.pl  view on Meta::CPAN

models = {}
results = {}
for spec in specs:
    train_csv, query_csv, label = spec.split('|', 2)
    clf = models.get(train_csv)
    if clf is None:
        X_train = load_csv(train_csv)
        psi = min(256, len(X_train))
        clf = IsolationForest(n_estimators=100, max_samples=psi,
                              contamination='auto', random_state=1)
        clf.fit(X_train)
        models[train_csv] = clf
    X_q = load_csv(query_csv)
    results[label] = {
        'score_samples':     bench(lambda X=X_q: clf.score_samples(X),     bench_secs),
        'predict':           bench(lambda X=X_q: clf.predict(X),           bench_secs),
        'decision_function': bench(lambda X=X_q: clf.decision_function(X), bench_secs),
    }

print(json.dumps(results))
END_PY

# -----------------------------------------------------------------------
# Run Python (one subprocess for all experiments)
# -----------------------------------------------------------------------
my $sk;
if ( defined $python_bin ) {
    my ( $py_fh, $py_path ) = tempfile( SUFFIX => '.py', UNLINK => 1 );
    print $py_fh $py_script;
    close $py_fh;

    my $specs = join( ' ',
        map { qq("$_->{train_csv}|$_->{query_csv}|@{[exp_key($_)]}") }
            @experiments );
    my $raw = `$python_bin "$py_path" $BENCH_SECS $specs 2>/dev/null`;
    $sk     = eval { JSON::PP->new->decode($raw) };
}

# -----------------------------------------------------------------------
# Run Perl benchmarks for all three backends
# -----------------------------------------------------------------------
my ( %pure_pl, %c_pl, %omp_pl );

for my $exp (@experiments) {
    my $key = exp_key($exp);
    my $q   = $exp->{query_data};

    # Pure Perl
    {
        my $m = $exp->{pure_model};
        $pure_pl{$key} = {
            score_samples         => wall_rate( sub { $m->score_samples($q)         }, $BENCH_SECS ),
            predict               => wall_rate( sub { $m->predict($q)               }, $BENCH_SECS ),
            score_predict_samples => wall_rate( sub { $m->score_predict_samples($q) }, $BENCH_SECS ),
            score_predict_split   => wall_rate( sub { $m->score_predict_split($q)   }, $BENCH_SECS ),
            path_lengths          => wall_rate( sub { $m->path_lengths($q)          }, $BENCH_SECS ),
        };
    }

    # C serial (single-threaded Inline::C)
    if ($HAS_C) {
        my $m      = $exp->{c_model};
        my $packed = $m->pack_data($q);
        $c_pl{$key} = {
            score_samples              => wall_rate( sub { $m->score_samples($q)              }, $BENCH_SECS ),
            score_samples_packed       => wall_rate( sub { $m->score_samples($packed)         }, $BENCH_SECS ),
            predict                    => wall_rate( sub { $m->predict($q)                    }, $BENCH_SECS ),
            predict_packed             => wall_rate( sub { $m->predict($packed)               }, $BENCH_SECS ),
            score_predict_samples      => wall_rate( sub { $m->score_predict_samples($q)      }, $BENCH_SECS ),
            score_predict_split        => wall_rate( sub { $m->score_predict_split($q)        }, $BENCH_SECS ),
            score_predict_split_packed => wall_rate( sub { $m->score_predict_split($packed)   }, $BENCH_SECS ),
            path_lengths               => wall_rate( sub { $m->path_lengths($q)               }, $BENCH_SECS ),
        };
    }

    # C + OpenMP (parallel Inline::C)
    if ($HAS_OPENMP) {
        my $m      = $exp->{omp_model};
        my $packed = $m->pack_data($q);
        $omp_pl{$key} = {
            score_samples              => wall_rate( sub { $m->score_samples($q)              }, $BENCH_SECS ),
            score_samples_packed       => wall_rate( sub { $m->score_samples($packed)         }, $BENCH_SECS ),
            predict                    => wall_rate( sub { $m->predict($q)                    }, $BENCH_SECS ),
            predict_packed             => wall_rate( sub { $m->predict($packed)               }, $BENCH_SECS ),
            score_predict_samples      => wall_rate( sub { $m->score_predict_samples($q)      }, $BENCH_SECS ),
            score_predict_split        => wall_rate( sub { $m->score_predict_split($q)        }, $BENCH_SECS ),
            score_predict_split_packed => wall_rate( sub { $m->score_predict_split($packed)   }, $BENCH_SECS ),
            path_lengths               => wall_rate( sub { $m->path_lengths($q)               }, $BENCH_SECS ),
        };
    }
}

# -----------------------------------------------------------------------
# Display
# -----------------------------------------------------------------------

# Row layout: [ label, pure_key, c_key, omp_key, sklearn_key ]
# undef means "not applicable for this backend/side"
my @rows = (
    [ 'score_samples',               'score_samples',        'score_samples',        'score_samples',             'score_samples'     ],
    [ 'score_samples (packed)',       undef,                  'score_samples_packed', 'score_samples_packed',      undef               ],
    [ 'predict',                     'predict',              'predict',              'predict',                   'predict'           ],
    [ 'predict (packed)',             undef,                  'predict_packed',       'predict_packed',            undef               ],
    [ 'score_predict_samples',       'score_predict_samples','score_predict_samples','score_predict_samples',     undef               ],
    [ 'score_predict_split',         'score_predict_split',  'score_predict_split',  'score_predict_split',       undef               ],
    [ 'score_predict_split (packed)', undef,                 'score_predict_split_packed','score_predict_split_packed', undef         ],
    [ 'path_lengths',                'path_lengths',         'path_lengths',         'path_lengths',              undef               ],
    [ 'decision_function',           undef,                  undef,                  undef,                       'decision_function' ],
);

my $MW = 28;    # method column width
my $NW = 12;    # numeric backend column width
my $SW = 14;    # sklearn column width
my $RW = 8;     # ratio column width

sub fmt_rate { defined $_[0] && $_[0] ? sprintf( '%.1f', $_[0] ) : '--' }

sub print_point {
    my ($key) = @_;