Forks-Super
view release on metacpan or search on metacpan
t/forked_harness.pl view on Meta::CPAN
_summarize_results();
}
if ($really_quiet == 0 && scalar keys %fail > 0) {
_summarize_failures();
}
if ($total_status == 0) {
_summarize_success();
}
my $elapsed = Time::HiRes::time() - $^T;
printf "Elapsed time: %.3f\n", $elapsed;
sleep 3 if $debug;
return;
}
sub _summarize_results {
print "\n\n\n\n\nThere were errors in iteration #$iteration:\n";
if (1 || $ENV{EXTRA}) {
my $hostname = qx(hostname 2>/dev/null);
chomp($hostname);
print "[ \$^X = $^X";
print ", host = $hostname" if $hostname;
print "]\n";
}
print "=====================================\n";
print scalar localtime, "\n";
print @result;
print "=====================================\n";
print "\n\n\n\n\n";
return;
}
sub _summarize_failures {
print "\nTest failures:\n";
if (1 || $ENV{EXTRA}) {
my $hostname = qx(hostname 2>/dev/null);
chomp($hostname);
print "[ \$^X = $^X";
print ", host = $hostname" if $hostname;
print "]\n";
}
print "================\n";
foreach my $test_file (sort keys %fail) {
no warnings 'numeric';
foreach my $test_no (sort {
$a+0<=>$b+0 || $a cmp $b
} keys %{$fail{$test_file}}) {
print "\t$test_file#$test_no ";
if ($fail{$test_file}{$test_no} > 1) {
print "$fail{$test_file}{$test_no} times\n";
} else {
print "\n";
}
}
}
print "================\n";
return;
}
sub _summarize_success {
$iteration--;
print "All tests successful. $iteration iterations.\n";
if (1 || $ENV{EXTRA}) {
my $hostname = qx(hostname 2>/dev/null);
chomp($hostname);
print "[ \$^X = $^X";
print ", host = $hostname" if $hostname;
print "]\n";
}
return;
}
#
# make sure the Forks::Super module is cleaning up after itself.
# This is mainly helpful for testing the Forks::Super module.
#
sub check_endgame {
print "Checking endgame $Forks::Super::IPC_DIR\n";
# fork so the main process can exit and the Forks::Super
# module can start cleanup.
# Forks::Super shouldn't leave temporary dirs/files around
# after testing, but it might
my $x = $Forks::Super::IPC_DIR;
if (!defined $x) {
my $p = fork { child_fh => 'out', sub => {} };
waitpid $p, 0;
$x = $Forks::Super::IPC_DIR;
}
CORE::fork() && return;
sleep 12;
my @fhforks = ();
opendir(D, $x);
while (my $g = readdir(D)) {
if ($g =~ /^.fh/) {
opendir(E, "$x/$g");
my $gg = readdir(E);
closedir E;
$gg -= 2;
print STDERR "Directory $x/$g still exists with $gg files\n";
}
}
closedir D;
$0 = '-';
# to do: check the process table and see if any languishing
# processes came from here ...
return;
}
#
# find good initial setting for $Forks::Super::MAX_PROC.
# This can be overridden with -m|--maxproc command-line arg
#
sub maxproc_initial {
if ($ENV{MAX_PROC}) {
return $ENV{MAX_PROC};
}
eval {
require Sys::CpuAffinity; 1
} or do {
return 2;
};
my $n = Sys::CpuAffinity::getNumCpus();
if ($n <= 0) {
return 2;
}
my @mask = Sys::CpuAffinity::getAffinity($$);
if (@mask < $n) {
$n = @mask || $n;
}
return $n < 8 ? int(2 * $n + 1) : 16;
}
# if appropriate and supported, enhance output to STDOUT with color.
sub color_print {
my ($color, @msg) = @_;
if ($color eq '' || !$use_color) {
return print STDOUT @msg;
}
$color = $colors{$color} if defined $colors{$color};
if (@msg > 0 && chomp($msg[-1])) {
return print STDOUT colored([$color], @msg), "\n";
}
return print STDOUT colored([$color], @msg);
}
sub color_printf { return color_print shift, sprintf @_ }
sub print_usage {
print STDERR <<"__END_USAGE__";
perl forked_harness.pl [options] [tests]
Run test suite in parallel using Forks::Super. If tests are not
specified, defaults to value of TEST_FILES in ./Makefile or ../Makefile
Recognized options:
-h,--harness wrap tests in ExtUtils::Command::MM::test_harness
-v,--verbose with -h, use verbose test harness
-I,--include lib use Perl lib dirs [default: blib/lib, blib/arch]
-p,--popts option pass option to perl interpreter during test
[e.g., -p -d:Trace, -p -MCarp::Always]
-s,--shuffle run tests in random order
-t,--timeout n abort test after <n> seconds [default: 150]
-r,--repeat n do up to <n> iterations of testing, aborting if
an iteration had test failures
-x,--xrepeat n run each test <n> times within each test iteration
-m,--maxproc n run up to <n> tests simultaneously
-q,--quiet produce less output (-q is *not* the opposite of -v!)
--qq,--really-quiet show test status, no other output
-d,--debug produce output about what forked_harness.pl is doing
-a,--abort-on-fail stop immediately after any test failure
-C,--color colorize output (requires Term::ANSIColor >= 3.00)
-E,--env var=value pass environment variable to the tests
-O,--order return test results in order
ENVIRONMENT
COLOR if true, try to colorize output [like -C flag]
ENDGAME_CHECK if true, check that program cleans up after itself
MAX_PROC number of simultaneous tests [like -m flag]
TEST_VERBOSE if true, use verbose test harness [like -v flag]
TEST_FILES tests to perform, if not provided on command-line
__END_USAGE__
;
return;
}
=head1 NAME
forked_harness.pl - run tests in parallel with Forks::Super
=head1 VERSION
0.97
=head1 SYNOPSIS
$ perl t/forked_harness.pl [options] [test-files]
|= test= 1/86; status: 0/0 time= 2.988s | t/00-use.t .. ok
...
|= test=86/86; status: 0/0 time=11.441s | t/43c-foo.t .. ok
All tests successful. 1 iterations.
Elapsed time: 73.919
=head1 DESCRIPTION
The C<forked_harness.pl> script runs a suite of unit tests in parallel
using the L<Forks::Super> framework. It can be used in any context
where you might run the command C<make test>. Aside from being able
to finish running your test suite faster, this framework has many
additional uses:
=head2 Intermittent failures
If you have one or more unit tests that only fail intermittently,
C<forked_harness.pl> can help you to run the test multiple times,
isolating the test output of the failed tests.
# run intermittent.t 200 times, only output on failure
$ forked_harness.pl -x 200 -q t/intermittent.t
=head2 Stress testing
As C<forked_harness.pl> will make use of more CPU while running
your tests, it can expose problems with your tests or your code
that only occur under high CPU loads.
=head2 Synchronization issues
C<forked_harness.pl> can expose issues caused by running multiple
instances of a test. For example, if your module or some of your
test scripts need to write to a file with a hard-coded
filename, multiple instances of the test might interfere
with each other and cause the test to fail.
=head1 TEST FILES
There are several ways to specify which tests to run.
=head2 Command-line
The tests to run may be specified on the command-line.
$ forked_harness.pl t/00-load.t t/43-foo.t
$ forked_harness.pl t/*server*.t
Arguments with filesystem wildcard characters will be expanded to
include all filenames that match the pattern, even on Windows.
=head2 $ENV{TEST_FILES}
If test files are not specified on the command-line and the environment
variable C<TEST_FILES> is set, C<forked_harness.pl> will use that
variable as the list of tests to run.
$ TEST_FILES=t-special/*.t forked_harness.pl
=head2 Makefile
If test files are not specified on the command-line or in the
C<$ENV{TEST_FILES}> variable, C<forked_harness.pl> will look for a
file called C<Makefile> in the current directory and in the parent
t/forked_harness.pl view on Meta::CPAN
You may consider using a smaller value if you have very CPU-intensive
tests, or a larger value if your tests are not very intense (I don't
know, maybe they spend most of their times in C<sleep> statements).
You can also specify the number of tests to run simultaneously
by setting the C<MAX_PROC> environment variable before running
C<forked_harness.pl>:
$ forked_harness.pl --max-proc 15 [other-options] [test-files]
$ forked_harness.pl -m 15 [other-options] [test-files]
$ MAX_PROC=15 forked_harness.pl [other-options] [test-files]
=head2 -O, --order
If specified, test results will be reported in their correct order.
The default is to report results as they are available, so
results for fast-running tests might appear before the results of
slow-running tests that started earlier.
=head2 -p I<option>, --popts I<option>
Passes the specified option as a command-line option to the
perl process running each test. Here are some examples of how
this option might be useful:
=head3 Apply L<Carp::Always> to get stack traces of any warnings in tests
$ forked_harness.pl -p -MCarp::Always [test-files]
=head3 Run all tests in taint mode
$ forked_harness.pl -p -T [test-files]
$ forked_harness.pl -p -t [test-files]
=head3 Run each test through a C<Devel::> module
$ forked_harness.pl -p -d:Trace::Fork [test-files]
The C<-p> option may be specified as many times as desired
to pass more than one option to the perl program running
the test.
=head2 -q, --quiet
Run in "quiet" mode. Suppresses output of successful tests
except for a single summary line (see L<"OUTPUT">).
This option hides the additional output
produced when the C<-v> (C<--verbose>) option is used.
=head2 -qq, --really-quiet
Run in a very quiet mode. Suppresses all output of successful tests,
and suppresses output of failed tests until all tests
are completed. This option hides the additional output
produced when the C<-v> (C<--verbose>) option is used.
=head2 -r I<times>, --repeat I<times>
Runs I<times> iterations of the tests, aborting if there are test
failures in any iteration. See also the C<-x> option. The default
is to run one iteration of the tests.
=head2 -s, --shuffle
If this option is included, the tests will be run in a random order.
=head2 -t I<timeout>, --timeout I<timeout>
Sets a timeout of I<timeout> seconds on each test. If possible,
tests will be terminated (and marked as failures) if they are
still running when the timeout expires.
The default timeout is 150 seconds. Specify a timeout of 0
(C<-t 0>, C<--timeout 0>) to disable the timeout and give each
test as long as it needs to complete.
The default timeout can also be overridden by setting the
C<TEST_TIMEOUT> environment variable.
=head2 -v, --verbose
When used with the C<-h> (harness) options, runs each test
in verbose mode, as if you had run the test suite with the command
make test TEST_VERBOSE=1
The results of each individual test and anything else a test
script writes to C<STDOUT> will be output with this option.
=head2 -x I<times>, --xrepeat I<times>
I<Within each iteration> (see the C<-r> option), runs each test
I<times> times. The default is to run each test one time in each
iteration.
=head2 -z, --socket
Instructs C<forked_harness.pl> and L<Forks::Super> to use socket
based interprocess communication where possible, instead of
file based IPC. File IPC should be more flexible and
robust than socket IPC, so this option probably won't get you
anything unless you are really really really low on disk space,
or if you don't have write permission in the filesystem where
you run your tests.
=head1 OTHER ENVIRONMENT AND CONFIGURATION
=over 4
=item C<COLOR=>I<some true value>
has the same effect as using the
C<-c> (C<--color>) flag.
=item C<MAX_PROC=>I<numprocs>
has the same effect as using the
C<-m> I<numprocs> (C<--maxproc>) option
=item C<TEST_VERBOSE=>I<some true value>
has the same effect as using
the C<-v> (C<--verbose>) flag.
=item C<TEST_FILES=>I<filenames>
specifies which tests to run, if they were not
provided on the command line.
=back
=head1 PROGRAM OUTPUT
At a minimum, C<forked_harness.pl> produces a status line
for each test that it runs:
|= test= 49/113; status: 1024/768 time=44.643s | t/41j-filehandles.t
A B C D E F
=over 4
=item C<A> - the current test number
The test results are reported when a test finishes. The reports may
be out of order. If there is more than one iteration of testing planned
(see the C<-r> option), this also reports the current iteration.
=item C<B> - total number of tests
Total number of tests in this iteration. If there are multiple iterations
(see the C<-r> option), then this also reports the number of iterations
planned.
=item C<C> - status of this test
Exit status of the test script. Anything other than a zero
indicates a test failure. Normally this value is 256 times the
number of failed tests, but it could have a different value to indicate
that the test terminated abnormally.
=item C<D> - previous status
The highest exit code from the current test or any previous test.
=item C<E> - test time
Running time of the test, with millisecond resolution. When you get
in the habit of running your unit tests in parallel, the longest
running tests in your suite can be a bottleneck in your overall test time,
and you may want to look for ways to break it into several smaller
tests.
=item C<F> - test name
The test file that this line is reporting on.
=back
Depending on which settings you are using,
output from the test may follow the test's status line.
Standard output and standard error will be separated,
and all output from the test will appear at once.
=head1 EXIT STATUS
The exit status of C<forked_harness.pl> will be the approximate
number of test failures encountered. If there were more than
254 failures, however, the exit code will be C<254>.
This is the same behavior as L<ExtUtils::Command::MM> C<test_harness>
function.
=head1 SEE ALSO
C<forked_harness.pl> was originally developed as a proof-of-concent
for the L<Forks::Super> distribution, and there are two targets in
the C<Makefile> of L<Forks::Super> that use C<forked_harness.pl>:
# ------ fasttest: use Forks::Super to run Forks::Super tests in parallel
fasttest :: pure_all
$(FULLPERL) t/forked_harness.pl $(TEST_FILES) -h -q
# ------ stress test: run all tests in parallel 100 times
stresstest :: pure_all
$(FULLPERL) t/forked_harness.pl $(TEST_FILES) -r 20 -x 5 -s -q
=head1 AUTHOR
Marty O'Brien, E<lt>mob@cpan.orgE<gt>
( run in 0.944 second using v1.01-cache-2.11-cpan-71847e10f99 )