Benchmark-MCE

 view release on metacpan or  search on metacpan

lib/Benchmark/MCE.pm  view on Meta::CPAN

 my %stats = suite_run(\%options);

Runs the benchmark suite given the C<%options> and prints results. Returns a hash
with run stats that looks like this:

 %stats = (
   $bench_name_1 => {times => [ ... ], scores => [ ... ]},
    ...
   _total => {times => [ ... ], scores => [ ... ]},
   _opt   => {iter => $iterations, threads => $no_threads, ...}
 );

Note that the times reported will be average times per thread (or per function
call if you prefer), however the scores reported (if a reference time is supplied)
are sums across all threads. So you expect for ideal scaling 1 thread vs 2 threads
to return the same times, double the scores.

=head3 Options:

=over 4

lib/Benchmark/MCE.pm  view on Meta::CPAN

=back

=item * C<threads> (Int; default 1):
Parallel benchmark threads. They are L<MCE> workers, so not 'threads' in the technical
sense. Each of the benchmarks defined will launch on each of the threads, hence the
total workload is multiplied by the number of C<threads>. Times will be averaged
across threads, while scores will be summed.

=item * C<iter> (Int; default 1):
Number of suite iterations (with min/max/avg at the end when > 1).

=item * C<include> (Regex):
Only run benchmarks whose names match regex.

=item * C<exclude> (Regex):
Skip benchmarks whose names match regex.

=item * C<filter> (CodeRef):
Custom filter callback for finer control. It receives C<($opt, $bench, $bench_def)>
and should return true to run a benchmark.

lib/Benchmark/MCE.pm  view on Meta::CPAN

Do not run under L<MCE::Loop> (sets C<threads=1>, C<scale=1>).

=back

=head2 C<calc_scalability>

 my %scal = calc_scalability(\%stat_single, \%stat_multi, $keep_outliers?);

Given the C<%stat_single> results of a single-threaded C<suite_run> and C<%stat_multi>
results of a multi-threaded run, will calculate, print and return the multi-thread
scalability (including averages, ranges etc for multiple iterations).

Unless C<$keep_outliers> is true, the overall scalability is an average after droping
Benchmarks that are non-scaling outliers (over 2*stdev less than the mean).

The result hash return looks like this:

 %scal = (
   bench_name => $bench_avg_scalability,
    ...
   _total => $total_avg_scalability

lib/Benchmark/MCE.pm  view on Meta::CPAN

                . _pad(sprintf("%2.0f", $scal[-1]), 24) . "\n")
            if @perf;
    }
    die "No bench times recorded" unless @perf;
    _print(("-"x40)."\n");
    my @avg1 = _min_max_avg($stats1->{_total}->{$display});
    my @avg2 = _min_max_avg($stats2->{_total}->{$display});
    _print(__PACKAGE__, " summary ($cnt benchmark");
    _print("s") if $cnt > 1;
    _print(" x$opt->{scale} scale")     if $opt->{scale} > 1;
    _print(", $opt->{iter} iterations") if $opt->{iter} > 1;
    _print(", $opt2->{threads} threads):\n");
    $opt->{f} .= "s" if $opt->{time};
    my $f = $opt->{time} ? '%.3f' : '%.0f';
    $f = $opt->{iter} > 1 ? "$opt->{f}\t($f - $f)" : $opt->{f};
    @avg1 =  $opt->{iter} > 1 ? ($avg1[2], $avg1[0], $avg1[1]) : ($avg1[2]);
    @avg2 =  $opt->{iter} > 1 ? ($avg2[2], $avg2[0], $avg2[1]) : ($avg2[2]);
    _print(_pad("Single:").sprintf($f, @avg1)."\n");
    _print(_pad("Multi:").sprintf($f, @avg2)."\n");
    my @newperf = $outliers ? @perf : _drop_outliers(\@perf, -1);
    my @newscal = $outliers ? @scal : _drop_outliers(\@scal, -1);

lib/Benchmark/MCE.pm  view on Meta::CPAN

            "%2.1f%% \t(%.0f%% - %.0f%%)", $scal[2], $scal[0], $scal[1]
            )
            . "\n"
    );

    return %scal;
}

sub _init_options {
    my $opt = shift;
    $opt->{iter}  ||= $opt->{iterations} || 1;
    $opt->{bench} ||= $opt->{benchmarks} || $opt->{extra_bench};
    die "No benchmarks defined" unless $opt->{bench} && %{$opt->{bench}};
    foreach my $b (keys %{$opt->{bench}}) {
        if (!ref($opt->{bench}->{$b})) { # string
            my $f = eval "sub { $opt->{bench}->{$b} }";
            die "Error compiling benchmark '$b': $@" if $@;
            $opt->{bench}->{$b} = $f;
        }
        $opt->{bench}->{$b} = [$opt->{bench}->{$b}]
            if ref($opt->{bench}->{$b}) eq 'CODE';    # wrap coderef

lib/Benchmark/MCE.pm  view on Meta::CPAN

    my $r    = !defined $benchmark->[1]
        || $out eq $benchmark->[1] ? 'Pass' : "Fail ($out)";
    return $time, $r;
}

sub _total_stats {
    my $opt     = shift;
    my $stats   = shift;
    my $display = $opt->{time} ? 'times'      : 'scores';
    my $title   = $opt->{time} ? 'Time (sec)' : 'Score';
    _print(   "Aggregates ($opt->{iter} iterations"
            . ($opt->{threads} > 1 ? ", $opt->{threads} threads" : "") . "):\n"
            . _pad("Benchmark", 24)
            . _pad("Avg $title")
            . _pad("Min $title")
            . _pad("Max $title"));
    _print(_pad("stdev %")) if $opt->{stdev};
    _print(_pad("Pass %")) unless $opt->{no_check};
    _print("\n");

    foreach my $bench (sort keys %{$opt->{bench}}) {

t/simple.t  view on Meta::CPAN

%stats2 = %stats1;
$stats2{_opt} = {%{$stats1{_opt}}};
$stats2{_opt}->{threads} = 2;
@std = capture {calc_scalability(\%stats1, \%stats2)};
like($std[0], qr/Single:\s*\d+\s*\(\d+ - \d+/, 'Min Max');

$stats1{_opt}->{iter} = 1;
@std = capture {calc_scalability(\%stats1, \%stats2)};

unlike($std[0], qr/scale/, 'No scale listed');
unlike($std[0], qr/iterations/, 'No iterations listed');

$stats1{_opt}->{iter}  = 2;
$stats1{_opt}->{time}  = 1;
$stats1{_opt}->{scale} = 2;
$stats2{_opt}->{scale} = 2;
@std = capture {calc_scalability(\%stats2, \%stats1)};

like($std[0], qr/scale/, 'Scale listed');
like($std[0], qr/iterations/, 'Iterations listed');

@std = capture {
    suite_run({
        threads    => 1,
        quick      => 1,
        iter       => 2,
        no_mce     => 1,
        include    => 'DCT',
        benchmarks => $bench
    })
};
like($std[0], qr/2 iterations\)/, 'Aggregate');

@std = capture {
    suite_run({
        time        => 1,
        iterations  => 1,
        duration    => 1,
        no_mce      => 1,
        sleep       => 1,
        include     => 'prove',
        extra_bench => $bench
    })
};

like($std[0], qr/Overall Time/, 'Single');
like($std[0], qr/0s of 1s/, 'Duration');