view release on metacpan or  search on metacpan

scripts/es-daily-index-maintenance.pl  view on Meta::CPAN

# Retrieve a list of indexes
my @indices = es_indices(
    check_state => 0,
    check_dates => 0,
);

# Loop through the indices and take appropriate actions;
my @alias_changes=();
foreach my $index (sort @indices) {
    my $days_old = es_index_days_old( $index );
    verbose({color=>"cyan"}, sprintf "%s: index is %s days old",
            $index, defined $days_old ? $days_old : '(unknown)'
    );

    # If we can't calculate how old it is, skip it
    if( !$days_old || $days_old < 1 ) {
        verbose({color=>'magenta'},"$index for today, skipping.");
        next;
    }

    # Get aliases from index root
    my %aliases  = ();
    my $idx_meta = es_request($index);
    if( exists $idx_meta->{$index}{aliases} ) {
        my $skipped;
        foreach my $alias ( keys %{ $idx_meta->{$index}{aliases} } ) {
            $skipped = $alias if exists $SKIP{$alias};
            last if $skipped;
        }
        if( $skipped ) {
            output("$index contains a skipped alias: $skipped");
            next;
        }
    }

    # Delete the Index if it's too old
    if( $CFG{delete} && $CFG{'delete-days'} <= $days_old ) {
        output({color=>"red"}, "$index will be deleted.");
        my $rc = es_delete_index($index);
        next;
    }

    # Manage replicas
    if( $CFG{replicas} ) {
        my $replicas;
        if( $days_old > $AGE ) {
            $replicas = $CFG{'replicas-min'};
        }
        else {
            my $daily = $CFG{'replicas-max'} / $AGE;
            $replicas = ceil( $CFG{'replicas-max'} - ($daily * $days_old) );
            $replicas = $replicas < $CFG{'replicas-min'} ? $CFG{'replicas-min'} : $replicas;
            $replicas = $replicas > $CFG{'replicas-max'} ? $CFG{'replicas-max'} : $replicas;
        }
        my %shards = es_index_shards($index);
        debug({indent=>1}, "+ replica aging (P:$shards{primaries} R:$shards{replicas}->$replicas)");
        if ( $shards{primaries} > 0 && $shards{replicas} != $replicas ) {
            verbose({color=>'yellow'}, "$index: should have $replicas replicas, has $shards{replicas}");
            my $result = es_request('_settings',
                { index => $index, method => 'PUT' },
                { index => { number_of_replicas => $replicas} },
            );
            if($result) {
                output({color=>"green"}, "$index: Successfully set replicas to $replicas");
            }
            else {
                output({color=>'red'}, "$index: Unable to set replicas");
            }
        }
    }

    # Run optimize?
    if( $CFG{optimize} ) {
        my $segdata = es_segment_stats( $index );

        my $segment_ratio = undef;
        if( defined $segdata && $segdata->{shards} > 0 ) {
            $segment_ratio = sprintf( "%0.2f", $segdata->{segments} / $segdata->{shards} );
        }

        if( $days_old >= $CFG{'optimize-days'} && defined($segment_ratio) && $segment_ratio > 1 ) {
            verbose({color=>"yellow"}, "$index: required (segment_ratio: $segment_ratio).");

            my $o_res = es_optimize_index($index);
            if( !defined $o_res ) {
                output({color=>"red"}, "$index: Encountered error during optimize");
            }
            else {
                output({color=>"green"}, "$index: $o_res->{_shards}{successful} of $o_res->{_shards}{total} shards optimized.");
            }
        }
        else {
            if( defined($segment_ratio) && $segment_ratio > 1 ) {
                verbose("$index is active not optimizing (segment_ratio:$segment_ratio)");
            }
            else {
                verbose("$index already optimized");
            }
        }
    }

    # Close the index?
    if( $CFG{close} && $CFG{'close-days'} <= $days_old ) {
        my $status = es_request('_stats/store',{index=>$index});
        if( defined $status ) {
            if( $status->{_shards} && $status->{_shards}{total} && $status->{_shards}{total} > 0 ) {
                # retrieve aliases
                my $ars = es_request('_alias', {index=>$index});
                foreach my $alias ( keys %{ $ars->{$index}{aliases} } ) {
                    debug({indent=>1}, "- Will remove alias $alias from $index");
                    push @alias_changes, { remove => { index => $index, alias => $alias }};
                }
                verbose({indent=>1}," - closing index.");
                my $result = es_request('_close' => {method=>'POST',index=>$index});
                if( defined $result && $result->{acknowledged}) {
                    output({color=>'magenta'},"+ Closed $index.");
                }
                else {
                    output({color=>'red'},"! Attempted to close $index, but did not succeed.");
                }