view release on metacpan or  search on metacpan

scripts/es-alias-manager.pl  view on Meta::CPAN

#!perl
# PODNAME: es-alias-manager.pl
# ABSTRACT: Allow easy alias management for daily indexes
use strict;
use warnings;

use App::ElasticSearch::Utilities qw(:default);
use CLI::Helpers qw(:all);
use DateTime;
use Getopt::Long qw(:config no_ignore_case no_ignore_case_always);
use Pod::Usage;
use Ref::Util qw( is_ref );
use YAML::XS ();

#------------------------------------------------------------------------#
# Argument Collection
my %opt;
GetOptions(\%opt,
    'all',
    'config=s',
    'skip=s',
    # Basic options
    'help|h',
    'manual|m',
);

#------------------------------------------------------------------------#
# Documentations!
pod2usage(1) if $opt{help};
pod2usage(-exitstatus => 0, -verbose => 2) if $opt{manual};

my %actions = (
    add => 'Create any missing aliases',
    remove => 'Remove any aliases not in the desired set',
);

# We might skip one thing or another
if( exists $opt{skip} && !exists $actions{$opt{skip}} ) {
    output({color=>'red',sticky=>1}, "Invalid action to skip: $opt{skip}");
    output({clear=>1},"Valid actions to skip are:");
    output({indent=>1}, sprintf "%s - %s", $_, $actions{$_}) for sort keys %actions;
    pod2usage(-exitstatus => 1);
}

my %CFG = (
    config => '/etc/elasticsearch/aliases.yml',
);
# Extract from our options if we've overridden defaults
foreach my $setting (keys %CFG) {
    $CFG{$setting} = $opt{$setting} if exists $opt{$setting} and defined $opt{$setting};
}
if ( !exists $CFG{config} and ! -f $CFG{config} ) {
    pod2usage(1);
}
my $ALIAS = YAML::XS::LoadFile( $CFG{config} ) or die "unable to read $CFG{config}";

# Create the target uri for the ES Cluster
my $TARGET = exists $opt{host} && defined $opt{host} ? $opt{host} : 'localhost';

# Grab a connection to ElasticSearch
my $es = es_connect();

# Delete Indexes older than a certain point
my $TODAY = DateTime->now()->truncate( to => 'day' );
my $indices = es_request('_aliases');

if ( !defined $indices ) {
    output({color=>"red"}, "Unable to locate indices by get_aliases()!");
    exit 1;
}
debug_var($indices);

my %PARTS = (
    DATE => {
        RE  => '(?<year>\d{4})[.\-](?<month>\d{2})[.\-](?<day>\d{2})',
        FMT => join('.', '%Y', '%m', '%d'),
    },
    PERIOD => {
        FMT => '%s',
    }
);

my %IGNORE = ();

foreach my $base (keys %{ $ALIAS }) {
    # Index Aliases Managed Manually
    if( ! is_ref($ALIAS->{$base}) ) {
        $IGNORE{$base} = 1;
        delete $ALIAS->{$base};
        next;
    }

    my $re = $ALIAS->{$base}{pattern};
    $re =~ s/[^\w{}*?-]+//g;

    # Wildcards
    $re =~ s/\*+/.\+/g;
    $re =~ s/\?/./g;

    # Varaibles
    $re =~ s/\{\{([^\}]+)\}\}/$PARTS{$1}->{RE}/g;
    $ALIAS->{$base}{re} = qr/$re/;

    # Setup Formatting
    if (exists $ALIAS->{$base}{daily}) {
        $ALIAS->{$base}{daily} =~ s/\{\{([^\}]+)\}\}/$PARTS{$1}->{FMT}/g;
    }
    if (exists $ALIAS->{$base}{relative}) {
        $ALIAS->{$base}{relative}{alias} =~ s/\{\{([^\}]+)\}\}/$PARTS{$1}->{FMT}/g;
        while( my ($period,$def) =  each %{ $ALIAS->{$base}{relative}{periods} }) {
            my %dt = (
                to => $TODAY->clone(),
                from => exists $def->{from} ? $TODAY->clone() : DateTime->from_epoch(epoch => 0)->truncate( to => 'day'),
            );
            debug("Period[$period] subtracting: ");
            debug_var($def);
            foreach my $d (keys %dt) {
                if ( exists $def->{$d} ) {
                    $dt{$d}->subtract( %{ $def->{$d}} );
                    debug("$period $d " . $dt{$d}->ymd);
                }
            }
            $ALIAS->{$base}{relative}{periods}{$period} = \%dt;
        }
    }
}
debug("Aliases being applied:");
debug_var($ALIAS);

# Loop through the indices and take appropriate actions;
foreach my $index (sort keys %{ $indices }) {
    debug("$index being evaluated");
    my %current = map { $_ => 1 }
                  grep { !/^\./ }
                  grep { not exists $IGNORE{$_} }
                  keys %{ $indices->{$index}{aliases} };
    my $managed = 0;

    my %desired = ();
    while( my($name,$map) = each %{ $ALIAS }) {
        if ($index =~ /$map->{re}/) {
            $managed++;
            my $idx_dt = DateTime->new( map { $_ => $+{$_} } qw(year month day) );
            verbose("$index is a $name index.");

            if ( exists $map->{daily} ) {
                my $daily = $idx_dt->strftime($map->{daily});
                $desired{$daily} = 1;
            }
            if ( exists $map->{relative} ) {
                while (my ($period,$def) = each %{ $map->{relative}{periods} }) {
                    debug(sprintf("Checking index date (%s) is between %s and %s",
                            $idx_dt->ymd,
                            $def->{from}->ymd,
                            $def->{to}->ymd,
                        ));
                    if( $idx_dt <= $def->{to} && $idx_dt >= $def->{from} ) {
                        my $alias = sprintf( $map->{relative}{alias}, $period );
                        $desired{$alias} = 1;
                    }
                }
            }
        }
    }
    my @updates = ();
    my %checks = map { $_ => 1 } keys(%desired), keys(%current);
    if( $managed ) {
        foreach my $alias (keys %checks) {
            if( exists $desired{$alias} && exists $current{$alias} ) {
                next;
            }
            my $action =  exists $desired{$alias} ? 'add' : 'remove';
            push @updates, { $action => { index => $index, alias => $alias} };

            # Do we skip an add or remove?
            next if exists $opt{skip} && $action eq $opt{skip};
            verbose({color=>'cyan'}, "$index: $action alias '$alias'");
        }
    }
    debug({color=>'magenta'}, "Aliases for $index : " . join(',', keys %desired) );
    if( @updates ) {
        eval {
            es_request('_aliases', { method => 'POST' }, { actions => \@updates });
            output({color=>'green'}, "Updates applied for $index : " . join(',', keys %desired) );
        };
        if( my $err = $@ ){
            output({color=>'red'}, " + Failed to set aliases for $index\n", $err);
        }
    }
}

__END__

=pod

=head1 NAME

es-alias-manager.pl - Allow easy alias management for daily indexes

=head1 VERSION

version 8.8