BenchmarkAnything-Storage-Backend-SQL
view release on metacpan or search on metacpan
lib/BenchmarkAnything/Storage/Backend/SQL.pm view on Meta::CPAN
package BenchmarkAnything::Storage::Backend::SQL;
# git description: v0.028-6-g8b3d653
our $AUTHORITY = 'cpan:TAPPER';
# ABSTRACT: Autonomous SQL backend to store benchmarks
$BenchmarkAnything::Storage::Backend::SQL::VERSION = '0.029';
use 5.008;
use utf8;
use strict;
use warnings;
use Try::Tiny;
use Data::Dumper;
my $hr_default_config = {
select_cache => 0,
default_aggregation => 'min',
tables => {
unit_table => 'bench_units',
benchmark_table => 'benchs',
benchmark_value_table => 'bench_values',
subsume_type_table => 'bench_subsume_types',
benchmark_backup_value_table => 'bench_backup_values',
additional_type_table => 'bench_additional_types',
additional_value_table => 'bench_additional_values',
additional_relation_table => 'bench_additional_relations',
additional_type_relation_table => 'bench_additional_type_relations',
backup_additional_relation_table => 'bench_backup_additional_relations',
},
};
my $hr_column_ba_mapping = {
bench_value_id => 'VALUE_ID',
bench => 'NAME',
bench_value => 'VALUE',
bench_unit => 'UNIT',
created_at => 'CREATED',
};
my $fn_add_subsumed_point = sub {
my ( $or_self, $hr_atts ) = @_;
$or_self->{query}->start_transaction();
eval {
# insert subsumed benchmark value
$or_self->{query}->insert_benchmark_value(
$hr_atts->{rows}[0]{bench_id},
$hr_atts->{type_id},
$hr_atts->{VALUE},
);
my $i_bench_value_id = $or_self->{query}->last_insert_id(
$or_self->{config}{tables}{benchmark_value_table},
'bench_value_id',
);
# insert subsumed benchmark additional values
$or_self->{query}->copy_additional_values({
new_bench_value_id => $i_bench_value_id,
old_bench_value_id => $hr_atts->{rows}[0]{bench_value_id},
});
for my $hr_backup_row ( @{$hr_atts->{rows}} ) {
if ( $hr_backup_row->{bench_subsume_type_rank} == 1 ) {
if ( $hr_atts->{backup} ) {
# copy data rows to backup table
$or_self->{query}->copy_benchmark_backup_value({
new_bench_value_id => $i_bench_value_id,
old_bench_value_id => $hr_backup_row->{bench_value_id},
});
my $i_bench_backup_value_id = $or_self->{query}->last_insert_id(
$or_self->{config}{tables}{benchmark_backup_value_table},
'bench_backup_value_id',
);
$or_self->{query}->copy_benchmark_backup_additional_relations({
new_bench_value_id => $i_bench_backup_value_id,
old_bench_value_id => $hr_backup_row->{bench_value_id},
});
}
}
else {
# update bench_value_id in backup table
$or_self->{query}->update_benchmark_backup_value({
new_bench_value_id => $i_bench_value_id,
old_bench_value_id => $hr_backup_row->{bench_value_id},
});
}
# now lets remove the old rows
$or_self->{query}->delete_benchmark_additional_relations(
$hr_backup_row->{bench_value_id},
);
$or_self->{query}->delete_benchmark_value(
$hr_backup_row->{bench_value_id},
);
}
};
$or_self->{query}->finish_transaction( $@ );
return 1;
};
sub new {
my ( $s_self, $hr_atts ) = @_;
my $or_self = bless {}, $s_self;
for my $s_key (qw/ dbh /) {
if (! $hr_atts->{$s_key} ) {
require Carp;
Carp::confess("missing '$s_key' parameter");
return;
}
}
# get tapper benchmark configuration
$or_self->{config} = { %{$hr_default_config} };
if ( $hr_atts->{config} ) {
require Hash::Merge;
$or_self->{config} = {
Hash::Merge
->new('LEFT_PRECEDENT')
->merge(
%{$hr_atts->{config}},
%{$or_self->{config}},
)
};
}
require CHI;
if ( $or_self->{config}{select_cache} ) {
$or_self->{cache} = CHI->new( driver => 'RawMemory', global => 1 );
}
my $s_module = "BenchmarkAnything::Storage::Backend::SQL::Query::$hr_atts->{dbh}{Driver}{Name}";
my $fn_new_sub;
eval {
require Module::Load;
Module::Load::load( $s_module );
$fn_new_sub = $s_module->can('new');
};
if ( $@ || !$fn_new_sub ) {
require Carp;
Carp::confess("database engine '$hr_atts->{dbh}{Driver}{Name}' not supported");
return;
}
lib/BenchmarkAnything/Storage/Backend/SQL.pm view on Meta::CPAN
Carp::confess("subsume type '$hr_options->{subsume_type}' not exists");
return;
}
if ( $hr_subsume_type->{bench_subsume_type_rank} == 1 ) {
require Carp;
Carp::confess("cannot subsume with type '$hr_options->{subsume_type}'");
return;
}
# looking for values with with a higher rank subsume type
if (
$or_self->{query}
->select_check_subsumed_values({
date_to => $hr_options->{date_to},
date_from => $hr_options->{date_from},
subsume_type_id => $hr_subsume_type->{bench_subsume_type_id},
})
->rows()
) {
require Carp;
Carp::confess(
"cannot use subsume type '$hr_options->{subsume_type}' " .
'because a higher rank subsume type is already used for this date period'
);
return;
}
# look if excluded additional types really exists
my @a_excluded_adds;
if ( $hr_options->{exclude_additionals} ) {
for my $s_additional_type ( @{$hr_options->{exclude_additionals}} ) {
if (
my $hr_addtype = $or_self->{query}
->select_addtype( $s_additional_type )
->fetchrow_hashref()
) {
push @a_excluded_adds, $hr_addtype->{bench_additional_type_id}
}
else {
require Carp;
Carp::confess( "additional type '$s_additional_type' not exists" );
return;
}
}
}
# get all data points for subsume
my $or_data_values = $or_self->{query}->select_data_values_for_subsume({
date_to => $hr_options->{date_to},
date_from => $hr_options->{date_from},
exclude_additionals => \@a_excluded_adds,
subsume_type_id => $hr_subsume_type->{bench_subsume_type_id},
});
require DateTime::Format::Strptime;
my $or_strp = DateTime::Format::Strptime->new( pattern => '%F %T', );
my @a_rows;
my $i_counter = 0;
my $i_sum_value = 0;
my $b_backup = ((not exists $hr_options->{backup}) || $hr_options->{backup}) ? 1 : 0;
my $s_last_key = q##;
while ( my $hr_values = $or_data_values->fetchrow_hashref() ) {
my $s_act_key = join '__',
$hr_values->{bench_id},
$or_strp->parse_datetime( $hr_values->{created_at} )->strftime( $hr_subsume_type->{datetime_strftime_pattern} ),
$hr_values->{additionals} || q##,
;
if ( $s_last_key ne $s_act_key ) {
if ( $i_counter ) {
$or_self->$fn_add_subsumed_point({
rows => \@a_rows,
VALUE => $i_sum_value / $i_counter,
backup => $b_backup,
type_id => $hr_subsume_type->{bench_subsume_type_id}
});
}
@a_rows = ();
$i_counter = 0;
$i_sum_value = 0;
$s_last_key = $s_act_key;
}
$i_counter += 1;
$i_sum_value += $hr_values->{bench_value};
push @a_rows, $hr_values;
}
if ( $i_counter ) {
$or_self->$fn_add_subsumed_point({
rows => \@a_rows,
VALUE => $i_sum_value / $i_counter,
backup => $b_backup,
type_id => $hr_subsume_type->{bench_subsume_type_id}
});
}
return 1;
}
sub _get_additional_key_id {
my ( $or_self, $s_key ) = @_;
return $or_self->{query}->select_additional_key_id($s_key)->fetch->[0];
}
sub default_columns {
my ( $or_self ) = @_;
return $or_self->{query}->default_columns;
}
sub benchmark_operators {
my ( $or_self ) = @_;
return $or_self->{query}->benchmark_operators;
}
sub init_search_engine
{
my ( $or_self, $b_force) = @_;
require BenchmarkAnything::Storage::Backend::SQL::Search;
BenchmarkAnything::Storage::Backend::SQL::Search::init_search_engine (@_);
}
sub sync_search_engine
{
require BenchmarkAnything::Storage::Backend::SQL::Search;
BenchmarkAnything::Storage::Backend::SQL::Search::sync_search_engine (@_);
}
1;
__END__
=pod
=encoding UTF-8
=head1 NAME
BenchmarkAnything::Storage::Backend::SQL - Autonomous SQL backend to store benchmarks
=head1 SYNOPSIS
lib/BenchmarkAnything/Storage/Backend/SQL.pm view on Meta::CPAN
(SQL LIKE syntax, i.e., using C<%> as placeholder.
$benchmarkanythingdata = $or_bench->list_benchmark_names($pattern);
=head3 list_additional_keys
Get a list of all additional key names, optionally matching a given
pattern (SQL LIKE syntax, i.e., using C<%> as placeholder.
$benchmarkanythingdata = $or_bench->list_additional_keys($pattern);
=head3 enqueue_multi_benchmark
As a low-latency alternative to directly calling
L</add_multi_benchmark> there is a queuing functionality.
The C<enqueue_multi_benchmark> function simply writes the raw incoming
data structure serialized (and compressed) into a single row and
returns. The complementary function to this is
C<process_queued_multi_benchmark> which takes these values over using
the real C<add_multi_benchmark> internally.
=head3 process_queued_multi_benchmark
This is part 2 of the low-latency queuing alternative to directly
calling L</add_multi_benchmark>.
It transactionally marks a single raw entry as being processed and
then takes over its values by calling C<add_multi_benchmark>. It
preserves the order of entries by inserting each chunk sequentially,
to not confuse the IDs to the careful reader. After the bundle is
taken over it is marked as processed.
This function only handles one single raw entry. It is expected to
called from co-operating multiple worker tasks or multiple times from
a wrapper.
Currently the original raw values are B<not> deleted immediately, just
for safety reasons, until the transactional code is death-proof (and
certified by Stuntman Mike). There is a dedicated funtion L/gc> for
that cleanup.
The function returns the ID of the processed raw entry.
=head3 gc
This calls garbage collection, in particular deletes raw entries
created by C<process_queued_multi_benchmark> and already processed by
C<process_queued_multi_benchmark>.
It is separated from those processing just for safety reasons until
the transactional code in there is waterproof.
The gc function can cleanup more stuff in the future.
=head3 subsume
This is a maintenance function for reducing the number of data points in the
database. Calling this function reduces the rows in the benchmark values table
by building an average value for all benchmark data points grouped by specfic
columns. By default all old grouped columns will be added to backup tables for
rebuilding the original state.
It is highly recommended to do this periodically for better search performance.
my $b_success = $or_bench->subsume({
subsume_type => 'month',
exclude_additionals => [qw/ benchmark_date /],
date_from => '2013-01-01 00:00:00',
date_to => '2014-01-01 00:00:00',
backup => 0,
});
=over 4
=item subsume_type
The subsume of benchmark data points is made by group with the following
elements:
- bench_id
- additional data values ( Example: testrun_id, machine )
- specific data range ( subsume_type ).
The possible subsume types are stored in the
extrapolation_type_table ( BenchmarkAnything::Storage::Backend::SQL-Configuration ). By default there
are the following types: "second", "minute", "hour", "day", "week", "month",
"year".
=item date_from
Begin of subsume period.
=item date_to
End of subsume period.
=item exclude_additionals
Array Reference of additional values that should be excluded from grouping.
=item backup
By default all subsumed rows will be inserted to backup tables. If this
isn't desired a false value must be passed.
=back
=head3 init_search_engine( $force )
Initializes the configured search engine (Elasticsearch). If the index
already exists it does nothing, except when you set C<$force> to a
true value which deletes and re-creates the index. This is necessary
for example to apply new type mappings.
After a successful (re-)init you need to run C<sync_search_engine>.
During (re-init) and sync you should disable querying by setting
searchengine.elasticsearch.enable_query: 0
=head3 sync_search_engine( $force, $start, $count)
Sync C<$count> (default 10000) entries from the relational backend
into the search engine (Elasticsearch) for indexing, beginning at
C<$start> (default 1). Already existing entries in Elasticsearch are
skipped unless C<$force> is set to a true value.
=head1 NAME
BenchmarkAnything::Storage::Backend::SQL - Save and search benchmark points by database
=head1 Configuration
The following elements are required in configuration:
=over 4
=item default_aggregation
Default aggregation used for non aggregated columns if an aggregation on any
other column is found.
=item tables
Containing the names of the tables used bei B<BenchmarkAnything::Storage::Backend::SQL>
tables => {
unit_table => 'bench_units',
benchmark_table => 'benchs',
benchmark_value_table => 'bench_values',
subsume_type_table => 'bench_subsume_types',
benchmark_backup_value_table => 'bench_backup_values',
additional_type_table => 'bench_additional_types',
additional_value_table => 'bench_additional_values',
additional_relation_table => 'bench_additional_relations',
additional_type_relation_table => 'bench_additional_type_relations',
backup_additional_relation_table => 'bench_backup_additional_relations',
}
=item select_cache [optional]
In case of a true value the module cache some select results
=back
=head3 default_columns
Returns the hash about those columns that are by default part of each
single data point (NAME, UNIT, VALU, VALUE_ID, CREATED, each with its
internal column name). These default columns might go away in the
future, but for now some systems need this internal information.
=head3 benchmark_operators
Returns the list of operators supported by the query language. This is
provided for frontend systems that support creating queries
automatically.
=head3 json_true
Auxiliary function for Elasticsearch JSON data.
=head3 json_false
Auxiliary function for Elasticsearch JSON data.
=head2 Elasticsearch support
=head3 Config
You can pass through a config entry for an external search engine
(currently only Elasticsearch) to the constructor:
my $or_bench = BenchmarkAnything::Storage::Backend::SQL->new({
dbh => $or_dbh,
searchengine => {
elasticsearch =>
#
# which index/type to use
index => "myapp",
type => "benchmarkanything",
#
# queries use the searchengine
enable_query => 1,
#
# should each single added value be stored immediately
# (maybe there is another bulk sync mechanism)
index_single_added_values_immediately => 1,
#
# which nodes to use
nodes => [ 'localhost:9200' ],
#
# (OPTIONAL)
# Your additional application specific mappings, used when
# index is created.
#
# WARNING: You are allowed to overwrite the pre-defined
( run in 2.614 seconds using v1.01-cache-2.11-cpan-5b529ec07f3 )