App-ElasticSearch-Utilities
view release on metacpan or search on metacpan
lib/App/ElasticSearch/Utilities.pm view on Meta::CPAN
$exec ||= $DEF{PASSEXEC};
return unless length $exec && -x $exec;
my(@out,@err);
# Run the password command captue out, error and RC
run3 [ $exec, $host, $username ], \undef, \@out, \@err;
my $rc = $?;
# Record the error
if( @err or $rc != 0 ) {
output({color=>'red',stderr=>1},
sprintf("es_pass_exec() called '%s' and met with an error code '%d'", $exec, $rc),
@err
);
return;
}
# Format and return the result
my $passwd = $out[-1];
chomp($passwd);
return $passwd;
lib/App/ElasticSearch/Utilities.pm view on Meta::CPAN
warn "Authorization required, try setting 'password-exec: /home/user/bin/get-password.sh` in ~/.es-utils.yaml'"
unless $DEF{PASSEXEC};
$req->authorization_basic( es_basic_auth($conn->host) );
}
else {
warn "Failed getting version";
last;
}
}
if( !defined $CURRENT_VERSION || $CURRENT_VERSION <= 2 ) {
output({color=>'red',stderr=>1}, sprintf "[%d] Unable to determine Elasticsearch version, something has gone terribly wrong: aborting.", $resp->code);
output({color=>'red',stderr=>1}, ref $resp->content ? YAML::XS::Dump($resp->content) : $resp->content) if $resp->content;
exit 1;
}
debug({color=>'magenta'}, "FOUND VERISON '$CURRENT_VERSION'");
return $CURRENT_VERSION;
}
my $ES = undef;
sub es_connect {
lib/App/ElasticSearch/Utilities.pm view on Meta::CPAN
$msg = join("\n", map { "\t$_" } @causes);
1;
} or do {
# Default to the message, though it's usually unhelpful
$msg = $resp->{message};
};
die sprintf "es_request(%s/%s) failed[%d]:\n%s",
$index, $options->{command}, $resp->code, $msg || 'missing error message';
} elsif( !defined $resp->content || ( !is_ref($resp->content) && !length $resp->content )) {
output({color=>'yellow',stderr=>1},
sprintf "es_request(%s/%s) empty response[%d]: %s",
$index, $options->{command}, $resp->code, $resp->message
);
}
return $resp->content;
}
lib/App/ElasticSearch/Utilities.pm view on Meta::CPAN
return wantarray ? %_nodes : { %_nodes };
}
my $_indices_meta;
sub es_indices_meta {
if(!defined $_indices_meta) {
my $result = es_request('_cluster/state/metadata');
if ( !defined $result ) {
output({stderr=>1,color=>"red"}, "es_indices_meta(): Unable to locate indices in status!");
exit 1;
}
$_indices_meta = $result->{metadata}{indices};
}
my %copy = %{ $_indices_meta };
return wantarray ? %copy : \%copy;
}
lib/App/ElasticSearch/Utilities.pm view on Meta::CPAN
_add_fields($f,$ref->{type},@path);
# Handle multifields
if( exists $ref->{fields} && is_hashref($ref->{fields}) ) {
foreach my $k (sort keys %{ $ref->{fields} } ) {
_add_fields($f,$ref->{type},@path,$k);
}
}
}
# Unknown data, throw an error if we care that deeply.
else {
debug({stderr=>1,color=>'red'},
sprintf "_find_fields(): Invalid property at: %s ref info: %s",
join('.', @path),
join(',', is_hashref($ref) ? sort keys %{$ref} :
ref $ref ? ref $ref : 'unknown ref'
),
);
}
}
}
lib/App/ElasticSearch/Utilities.pm view on Meta::CPAN
max_num_segments => 1,
},
});
}
sub es_apply_index_settings {
my($index,$settings) = @_;
if(!is_hashref($settings)) {
output({stderr=>1,color=>'red'}, 'usage is es_apply_index_settings($index,$settings_hashref)');
return;
}
return es_request('_settings',{ method => 'PUT', index => $index },$settings);
}
sub es_index_segments {
my ($index) = @_;
if( !defined $index || !length $index || !es_index_valid($index) ) {
output({stderr=>1,color=>'red'}, "es_index_segments('$index'): invalid index");
return;
}
return es_request('_segments', {
index => $index,
});
}
lib/App/ElasticSearch/Utilities/QueryString/FileExpansion.pm view on Meta::CPAN
my $line = 0;
my @path = split /\./, $field; # Supports key.subkey.subsubkey format
JSON_LINE: foreach my $json ( read_lines($file) ) {
$line++;
my $data;
eval {
$data = decode_json($json);
1;
} or do {
my $err = $@;
output({stderr=>1,color=>'yellow'}, sprintf "Invalid JSON in %s, line %d: %s",
$file,
$line,
$err,
);
verbose({stderr=>1,color=>'magenta',indent=>1}, $json);
next;
};
# Walk the path
foreach my $k (@path) {
next JSON_LINE unless exists $data->{$k};
$data = $data->{$k};
}
# At this point $data should contain our values
if( is_arrayref($data) ) {
$uniq{$_} = 1 for grep { !is_ref($_) } @{ $data };
lib/App/ElasticSearch/Utilities/VersionHacks.pm view on Meta::CPAN
sub _fix_version_request {
my ($url,$options,$data) = @_;
# Requires App::ElasticSearch::Utilities to be loaded
if( ! defined $version ){
eval {
$version = sprintf "%0.1f", App::ElasticSearch::Utilities::_get_es_version();
1;
} or do {
my $err = $@;
output({stderr=>1,color=>'red'}, "Failed version detection!", $@);
};
if (defined $version && $version < $MIN_VERSION) {
output({stderr=>1,color=>'red',sticky=>1},
"!!! Detected ElasticSearch Version '$version', which is < $MIN_VERSION, please upgrade your cluster !!!");
exit 1;
}
}
my $vstr = sprintf "v%0.1f", $version;
if(exists $SIMPLE{$url}) {
my $versions = join(", ", sort keys %{ $SIMPLE{$url} });
debug("Method changed in API, evaluating rewrite ($versions) against $vstr");
scripts/es-copy-index.pl view on Meta::CPAN
];
my $max_retries = 3;
my $success = 0;
while ($max_retries--) {
debug("Attempting bulk load of $batch documents");
my ($s2, $r2) = $ES{to}->bulk(
index => $INDEX{to},
body => $body
);
if ($s2 ne "200") {
output({stderr=>1,color=>'red'},"Failed to put documents to $HOST{to} (http status = $status): " . $JSON->encode([ $s2, $r2 ]));
next;
}
$success=1;
last;
}
die "Failed to write data to $HOST{to}:9200/$INDEX{to} after $RECORDS docs indexed."
unless $success;
my $took = time - $start;
show_counts( scalar @{$res->{hits}{hits}} );
$res = es_request($ES{from}, '_search/scroll', {
scripts/es-daily-index-maintenance.pl view on Meta::CPAN
push @{ $CFG{'skip-alias'} }, qw( .hold .do_not_erase );
my %SKIP = map { $_ => 1 } @{ $CFG{'skip-alias'} };
# Can't have replicas-min below 0
$CFG{'replicas-min'} = 0 if $CFG{'replicas-min'} < 0;
# Create the target uri for the ES Cluster
my $es = es_connect();
# This setting is no longer supported
output({color=>'red',sticky=>1,stderr=>1}, "WARNING: The index.codec.bloom.load is now disabled as of v1.4")
if $CFG{bloom};
# Ages for replica management
my $AGE = $CFG{'replicas-age'};
# Retrieve a list of indexes
my @indices = es_indices(
check_state => 0,
check_dates => 0,
);
scripts/es-nodes.pl view on Meta::CPAN
pod2usage(1) if $OPT{help};
pod2usage(-exitval => 0, -verbose => 2) if $OPT{manual};
my $cres = es_request('_cluster/health');
my $CLUSTER = defined $cres ? $cres->{cluster_name} : 'UNKNOWN';
output({clear=>1,color=>'magenta'}, "Cluster [$CLUSTER] contains $cres->{number_of_nodes} nodes.", '-='x20);
# Get a list of nodes
my $nres = es_request('_cluster/state/master_node,nodes', {});
if(!defined $nres) {
output({stderr=>1,color=>'red'}, 'Fetching node status failed.');
exit 1;
}
debug_var($nres);
foreach my $uuid (sort { $nres->{nodes}{$a}->{name} cmp $nres->{nodes}{$b}->{name} } keys %{ $nres->{nodes} }) {
my $node = $nres->{nodes}{$uuid};
my $color = defined $nres->{master_node} && $uuid eq $nres->{master_node} ? 'green' : 'cyan';
output({color=>$color}, $node->{name});
output({indent=>1,kv=>1,color=>$color}, address => $node->{transport_address});
verbose({indent=>1,kv=>1,color=>$color}, uuid => $uuid);
scripts/es-search.pl view on Meta::CPAN
debug({color=>"cyan"}, "Fields discovered.");
if( $OPT{fields} ) {
show_fields();
exit 0;
}
# Attempt date autodiscovery
if( !exists $FIELDS{$CONFIG{timestamp}} ) {
my @dates = grep { $FIELDS{$_}->{type} eq 'date' } keys %FIELDS;
if( @dates == 0 ) {
output({color=>'red',stderr=>1},"FATAL: No date fields found in the indices specified" );
exit 1;
}
elsif( @dates == 1 ) {
output({color=>'yellow',stderr=>1}, "WARNING: Timestamp field '$CONFIG{timestamp}' not found, using '$dates[0]' instead");
$CONFIG{timestamp} = $dates[0];
}
else {
output({color=>'red',stderr=>1},
sprintf "FATAL: Timestamp field '%s' not found and discovered multiple date fields: %s",
$CONFIG{timestamp},
join(', ', sort @dates)
);
output({color=>'yellow',indent=>1}, "Try again with '--timestamp $dates[0]' for example.");
exit 1;
}
}
# Which fields to show
scripts/es-search.pl view on Meta::CPAN
next unless defined $result;
if ( $result->{error} ) {
my $simple_error;
eval {
$simple_error = $result->{error}{caused_by}{caused_by}{reason};
} or do {
($simple_error) = $result->{error} =~ m/(QueryParsingException\[\[[^\]]+\][^\]]+\]\]);/;
};
$simple_error ||= '';
output({stderr=>1,color=>'red'},
"# Received an error from the cluster. $simple_error"
);
last;
}
$displayed_indices{$_} = 1 for @{ $by_age{$age} };
$TOTAL_HITS += $result->{hits}{total} if $result->{hits}{total};
my @always = ();
push @always, $CONFIG{timestamp} unless $OPT{'no-decorators'};
if(!$header && @SHOW) {
scripts/es-search.pl view on Meta::CPAN
$start = time;
$result = $q->scroll_results();
$duration += time - $start;
# Check if we need to keep going
last unless defined $result;
last unless $result->{hits} && $result->{hits}{hits} && @{ $result->{hits}{hits} } > 0
}
last if all_records_displayed();
}
output({stderr=>1,color=>'yellow'},
"# Search Parameters:",
(map { "# $_" } split /\r?\n/, to_json($q->query,{allow_nonref=>1,canonical=>1,pretty=>$CONFIG{pretty}})),
sprintf("# Displaying %d of %d results%s took %0.2f seconds.",
$displayed,
$OUT_OF || $TOTAL_HITS,
$OUT_OF ? " in $TOTAL_HITS documents" : '',
$duration,
),
sprintf("# Indexes (%d of %d) searched: %s\n",
scalar(keys %displayed_indices),
t/00-compile.t view on Meta::CPAN
use File::Spec;
use IPC::Open3;
use IO::Handle;
open my $stdin, '<', File::Spec->devnull or die "can't open devnull: $!";
my @warnings;
for my $lib (@module_files)
{
# see L<perlfaq8/How can I capture STDERR from an external command?>
my $stderr = IO::Handle->new;
diag('Running: ', join(', ', map { my $str = $_; $str =~ s/'/\\'/g; q{'} . $str . q{'} }
$^X, @switches, '-e', "require q[$lib]"))
if $ENV{PERL_COMPILE_TEST_DEBUG};
my $pid = open3($stdin, '>&STDERR', $stderr, $^X, @switches, '-e', "require q[$lib]");
binmode $stderr, ':crlf' if $^O eq 'MSWin32';
my @_warnings = <$stderr>;
waitpid($pid, 0);
is($?, 0, "$lib loaded ok");
shift @_warnings if @_warnings and $_warnings[0] =~ /^Using .*\bblib/
and not eval { +require blib; blib->VERSION('1.01') };
if (@_warnings)
{
warn @_warnings;
push @warnings, @_warnings;
t/00-compile.t view on Meta::CPAN
{ SKIP: {
open my $fh, '<', $file or warn("Unable to open $file: $!"), next;
my $line = <$fh>;
close $fh and skip("$file isn't perl", 1) unless $line =~ /^#!\s*(?:\S*perl\S*)((?:\s+-\w*)*)(?:\s*#.*)?$/;
@switches = (@switches, split(' ', $1)) if $1;
close $fh and skip("$file uses -T; not testable with PERL5LIB", 1)
if grep { $_ eq '-T' } @switches and $ENV{PERL5LIB};
my $stderr = IO::Handle->new;
diag('Running: ', join(', ', map { my $str = $_; $str =~ s/'/\\'/g; q{'} . $str . q{'} }
$^X, @switches, '-c', $file))
if $ENV{PERL_COMPILE_TEST_DEBUG};
my $pid = open3($stdin, '>&STDERR', $stderr, $^X, @switches, '-c', $file);
binmode $stderr, ':crlf' if $^O eq 'MSWin32';
my @_warnings = <$stderr>;
waitpid($pid, 0);
is($?, 0, "$file compiled ok");
shift @_warnings if @_warnings and $_warnings[0] =~ /^Using .*\bblib/
and not eval { +require blib; blib->VERSION('1.01') };
# in older perls, -c output is simply the file portion of the path being tested
if (@_warnings = grep { !/\bsyntax OK$/ }
grep { chomp; $_ ne (File::Spec->splitpath($file))[2] } @_warnings)
{
( run in 0.387 second using v1.01-cache-2.11-cpan-26ccb49234f )