view release on metacpan or  search on metacpan

scripts/es-search.pl  view on Meta::CPAN

    if exists $OPT{tail} && !( @SHOW || $OPT{json});

# Process extra parameters
foreach my $presence ( qw( exists missing ) ) {
    if( exists $OPT{$presence} ) {
        my @fields = map { split /[,:]/ } (is_arrayref($OPT{$presence}) ? @{ $OPT{$presence} } : ($OPT{$presence}));
        my $context = $presence eq 'exists' ? 'must' : 'must_not';
        foreach my $field (@fields) {
            $q->add_bool( $context => { exists => { field => $field } } );
        }
    }
}

my %SUPPORTED_AGGREGATIONS = map {$_=>'simple_value'} qw(cardinality sum min max avg value_count);
my $agg_header = '';
if( exists $OPT{top} ) {
    my @top = split /:/, $OPT{top};
    my $top_field = pop @top;
    my $top_agg   = @top ? shift @top : 'terms';

    my @agg_fields = grep { exists $FIELDS{$_} } split /\s*,\s*/, $top_field;
    croak(sprintf("Option --top takes a field, found %d fields: %s\n", scalar(@agg_fields),join(',',@agg_fields)))
        unless @agg_fields == 1;

    my %agg     = ();
    my %sub_agg = ();
    if( $OPT{by}) {
        my ($type,$field) = split /\:/, $OPT{by};
        if( exists $SUPPORTED_AGGREGATIONS{$type} ) {
            $sub_agg{by} = { $type => {field => $field} };
        }
        else {
            output({color=>'red'}, "Aggregation '$type' is not currently supported, ignoring.");
        }
    }
    if( $OPT{with} ) {
        my @with = is_arrayref($OPT{with}) ? @{ $OPT{with} } : ( $OPT{with} );
        foreach my $with ( @with )  {
            my @attrs = split /:/, $with, 3;
            # Process Args from Right to Left
            my $arg   = @attrs == 3 ? pop @attrs
                      : $attrs[-1] =~ /^\d/ ? pop @attrs
                      : '';
            my $pcts  = $arg =~ /^\d{1,2}(?:\.\d+)?(?:,\d{1,2}(?:\.\d+)?)*$/ ? $arg : '25,50,75,90,95,99';
            my $size  = $arg =~ /^\d+$/ ? $arg : 3;
            my $hi    = $arg || 0.1;
            my $field = exists $FIELDS{$attrs[-1]} ? pop @attrs : undef;
            my $type  = @attrs ? pop @attrs : 'terms';
            # Skip invalid elements
            next unless defined $field and defined $size and $size > 0;

            my %params = ();
            my $id = "$type-$field";
            # If a term agg and we haven't used this field name, simplify it
            if( $type =~ /terms$/ && !$sub_agg{$field} ) {
                $id = $field;
                $params{size} = $size;
                $params{missing} = 'MISSING' if $OPT{'with-missing'};
            }

            if( $type =~ /histogram|stats|percentiles/ && !$OPT{'no-implications'} ) {
                output({color=>'magenta',sticky=>1}, "* Using a statistical aggregation implies an exists filter on $field, use --no-implications to disable this");
                $q->add_bool( must => { exists => { field => $field } } );
            }

            $sub_agg{$id} = {
                $type => {
                    field => $field,
                    $type eq 'percentiles' ? ( percents => [split /,/, $pcts] ) : (),
                    $type eq 'histogram'   ? ( interval => $hi ) :  (),
                    %params,
                }
            };
        }
    }

    my %params = ();
    $params{missing} = 'MISSING' if $OPT{'with-missing'} and $top_agg eq 'terms';

    my $field = shift @agg_fields;
    $agg_header = "count\tpct\t" . $field;
    $agg{$top_agg} = { field => $field, %params };

    if( $OPT{'bg-filter'} && $top_agg eq 'significant_terms' ) {
        my $bgf = App::ElasticSearch::Utilities::QueryString->new();
        my $bgq = $bgf->expand_query_string($OPT{'bg-filter'});
        $agg{$top_agg}->{background_filter} = $bgq->query;

    }

    if( exists $sub_agg{by} ) {
        $agg_header = "$OPT{by}\t" . $agg_header;
        $agg{$top_agg}->{order} = [ { by => $ORDER }, { "_count" => "desc" } ];
    }
    $agg{aggregations} = \%sub_agg if keys %sub_agg;

    if( exists $OPT{all} ) {
        verbose({color=>'cyan'}, "# Aggregations with --all are limited to returning 1,000,000 results.");
        $agg{$top_agg}->{size} = 1_000_000;
    }
    else {
        $agg{$top_agg}->{size} = $CONFIG{size};
    }
    $q->add_aggregations( top => \%agg );
    $q->add_aggregations( out_of => { cardinality => { field => $field  } } );

    if( $OPT{interval} ) {
        $q->wrap_aggregations( step => {
            date_histogram => {
                field    => $CONFIG{timestamp},
                interval => $OPT{interval},
            }
        });
    }
}
elsif(exists $OPT{tail}) {
    $q->set_size($CONFIG{'max-batch-size'});
    @AGES = ($AGES[-1]);
}
elsif( $OPT{all} ) {
    $q->set_size( $CONFIG{'max-batch-size'} );
}
else {
    $q->set_size( $CONFIG{size} < $CONFIG{'max-batch-size'} ? $CONFIG{size} : $CONFIG{'max-batch-size'} );
}

my %displayed_indices = ();
my $TOTAL_HITS        = 0;
my $OUT_OF            = 0;

scripts/es-search.pl  view on Meta::CPAN

    --with <aggregation>:<field>:<size>

The default B<size> is 3.
The default B<aggregation> is 'terms'.

B<field> is the only required element.

e.g.

    $ es-search.pl --base logstash error --top program --size 2 --by cardinality:host --with host:5

This will show the top 2 programs with log messages containing the word error by the cardinality (count
distinct host) of hosts showing the top 5 hosts

Without the --with, the results might look like this:

    112314 0.151 sshd
    21224  0.151 ntp

The B<--with> option would expand that output to look like this:

    112314   0.151 host   bastion-804   12431  0.111 sshd
    112314   0.151 host   bastion-803   10009  0.089 sshd
    112314   0.151 host   bastion-805   9768   0.087 sshd
    112314   0.151 host   bastion-801   8789   0.078 sshd
    112314   0.151 host   bastion-802   4121   0.037 sshd
    21224    0.016 host   webapp-324    21223  0.999 ntp
    21224    0.016 host   mail-42       1      0.000 ntp

This may be specified multiple times, the result is more I<rows>, not more I<columns>, e.g.

    $ es-search.pl --base logstash error --top program --size 2 --by cardinality:host --with host:5 --with dc:2

Produces:

    112314 0.151  dc     arlington     112314 1.000 sshd
    112314 0.151  host   bastion-804   12431  0.111 sshd
    112314 0.151  host   bastion-803   10009  0.089 sshd
    112314 0.151  host   bastion-805   9768   0.087 sshd
    112314 0.151  host   bastion-801   8789   0.078 sshd
    112314 0.151  host   bastion-802   4121   0.037 sshd
    21224  0.016  dc     amsterdam     21223  0.999 ntp
    21224  0.016  dc     la            1      0.000 ntp
    21224  0.016  host   webapp-324    21223  0.999 ntp
    21224  0.016  host   mail-42       1      0.000 ntp

You may sub aggregate using any L<bucket agggregation|https://www.elastic.co/guide/en/elasticsearch/reference/master/search-aggregations-bucket.html>
as long as the aggregation provides a B<key> element.  Additionally, doc_count, score, and bg_count will be reported in the output.

Other examples:

    --with significant_terms:crime
    --with cardinality:accts
    --with min:out_bytes
    --with max:out_bytes
    --with avg:out_bytes
    --with sum:out_bytes
    --with stats:out_bytes
    --with extended_stats:out_bytes
    --with percentiles:out_bytes
    --with percentiles:out_bytes:50,95,99
    --with histogram:out_bytes:1024

=item B<with-missing>

For terms aggregations, adds a C<MISSING> bucket.

=item B<bg-filter>

Only used if the C<--top> aggregation is C<significant_terms>.  Sets the
background filter for the C<significant_terms> aggregation.

    es-search.pl --top significant_terms:src_ip method:POST file:\/get\/sensitive_data --bg-filter method:POST

=item B<interval>

When performing aggregations, wrap those aggregations in a date_histogram of this interval.  This
helps flush out "what changed in the last hour."

=item B<match-all>

Apply the ElasticSearch "match_all" search operator to query on all documents
in the index.  This is the default with no search parameters.

=item B<prefix>

Takes a "field:string" combination and you can use multiple --prefix options will be "AND"'d

Example:

    --prefix useragent:'Go '

Will search for documents where the useragent field matches a prefix search on the string 'Go '

JSON Equivalent is:

    { "prefix": { "useragent": "Go " } }

=item B<exists>

Filter results to those containing a valid, not null field

    --exists referer

Only show records with a referer field in the document.

=item B<missing>

Filter results to those not containing a valid, not null field

    --missing referer

Only show records without a referer field in the document.

=item B<bases>

Display a list of bases that can be used with the --base option.

Use with --verbose to show age information on the indexes in each base.

=item B<fields>