view release on metacpan or  search on metacpan

t/80-sklearn-comparison-undef.t  view on Meta::CPAN

	};
} ## end sub make_2d_grid_dataset

# N-D Gaussian inliers + corner outliers far from origin in every axis.
# Test points still only carry signal in column 0; the other nf-1 columns
# are undef.  Seeded deterministically per dimension.
sub make_nd_gaussian_dataset {
	my ($nf) = @_;
	srand( 20260629 + $nf );

	my @inliers;
	push @inliers, [ map { gaussian( 0, 0.3 ) } 1 .. $nf ] for 1 .. 200;

	my @outliers;
	for ( 1 .. 8 ) {
		my @row;
		for ( 1 .. $nf ) {
			my $mag  = 5 + rand() * 3;
			my $sign = rand() > 0.5 ? 1 : -1;
			push @row, $mag * $sign;
		}
		push @outliers, \@row;
	}

	# Empirical gaps with 1 signal column out of nf, 100 trees, seed 42:
	#   nf=5  -> ~0.13   nf=10 -> ~0.05
	# The threshold is set well under the observed value so trivial RNG
	# noise doesn't flap the test, but high enough to still detect a real
	# regression that would collapse the gap further.
	my $mean_gap_min = $nf <= 5 ? 0.08 : 0.025;

	my ( $undef_test, $zero_test ) = make_undef_test_points($nf);
	return {
		label        => "${nf}d_gaussian",
		n_feat       => $nf,
		train        => [ @inliers, @outliers ],
		undef_test   => $undef_test,
		zero_test    => $zero_test,
		n_in_test    => 19,
		n_out_test   => 6,
		mean_gap_min => $mean_gap_min,
	};
} ## end sub make_nd_gaussian_dataset

my @datasets = ( make_2d_grid_dataset(), make_nd_gaussian_dataset(5), make_nd_gaussian_dataset(10), );

# -----------------------------------------------------------------------
# Locate Python + scikit-learn (cross-language subtests are skipped if
# absent; pure-Perl subtests still run)
# -----------------------------------------------------------------------
my $python_bin;
for my $cmd (qw(python3 python)) {
	my $probe = `$cmd -c "import sklearn; print('ok')" 2>/dev/null`;
	if ( defined $probe && $probe =~ /\bok\b/ ) {
		$python_bin = $cmd;
		last;
	}
}

# -----------------------------------------------------------------------
# Python helper: scores all datasets in one subprocess.  Each argv spec
# is "csv_path|label|n_train"; the CSV concatenates training rows then
# test rows, and n_train is the split point.  Test rows may use the
# token "nan" / "undef" / empty for missing values.
#
# sklearn score_samples convention: lower = more anomalous (opposite of
# Perl), so we negate sklearn scores before computing rank correlation.
# -----------------------------------------------------------------------
my $py_script = <<'END_PY';
import sys, json
import numpy as np
from sklearn.ensemble import IsolationForest

def parse_csv(path):
    rows = []
    with open(path) as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            row = []
            for tok in line.split(','):
                tok = tok.strip()
                if tok.lower() in ('nan', 'undef', ''):
                    row.append(float('nan'))
                else:
                    row.append(float(tok))
            rows.append(row)
    return rows

results = {}
for spec in sys.argv[1:]:
    csv_path, label, n_train = spec.split('|', 2)
    n_train = int(n_train)
    rows    = parse_csv(csv_path)
    X_train = np.array(rows[:n_train], dtype=float)
    X_test  = np.array(rows[n_train:], dtype=float)
    # Mirror Perl's undef-to-0 numeric coercion.
    X_test_clean = np.where(np.isnan(X_test), 0.0, X_test)
    psi = min(256, len(X_train))
    clf = IsolationForest(
        n_estimators=100,
        max_samples=psi,
        contamination='auto',
        random_state=42,
    )
    clf.fit(X_train)
    results[label] = clf.score_samples(X_test_clean).tolist()

print(json.dumps(results))
END_PY

# Run Python once for all datasets, keyed by label.
my $sk_by_label;
if ( defined $python_bin ) {
	my ( $py_fh, $py_path ) = tempfile( SUFFIX => '.py', UNLINK => 1 );
	print $py_fh $py_script;
	close $py_fh;

	my @specs;
	for my $ds (@datasets) {