view release on metacpan or  search on metacpan

lib/AWS/CLIWrapper.pm  view on Meta::CPAN

    my @v;

    $k =~ s/_/-/g;
    $k = '--'.$k;

    my $type = ref $v;
    if (! $type) {
        if ($k eq '--output-file') {
            # aws s3api get-object takes a single arg for output file path
            return $v;
        } else {
            push @v, $v;
        }
    } elsif ($type eq 'ARRAY') {
        push @v, map { ref($_) ? encode_json(_compat_kv($_)) : $_ } @$v;
    } elsif ($type eq 'HASH') {
        push @v, encode_json(_compat_kv($v));
    } elsif ($type eq 'AWS::CLIWrapper::Boolean') {
        if ($$v == 1) {
            return ($k);
        } else {
            return ();
        }
    } else {
        push @v, $v;
    }

    return ($k, @v);
}

# >= 0.14.0 : Key, Values, Value, Name
# <  0.14.0 : key, values, value, name
sub _compat_kv_uc {
    my $v = shift;
    my $type = ref $v;

    if ($type && $type eq 'HASH') {
        for my $hk (keys %$v) {
            if ($hk =~ /^(?:key|name|values|value)$/) {
                $v->{ucfirst($hk)} = delete $v->{$hk};
            }
        }
    }

    return $v;
}
# sub _compat_kv_lc {
#     my $v = shift;
#     my $type = ref $v;

#     if ($type && $type eq 'HASH') {
#         for my $hk (keys %$v) {
#             if ($hk =~ /^(?:Key|Name|Values|Values)$/) {
#                 $v->{lc($hk)} = delete $v->{$hk};
#             }
#         }
#     }

#     return $v;
# }
# Drop support < 0.14.0 for preventing execute aws command in loading this module
*_compat_kv = *_compat_kv_uc;

sub json { $_[0]->{json} }

sub _execute {
    my $self    = shift;
    my $service = shift;
    my $operation = shift;
    my @cmd = ($self->awscli_path, @{$self->{opt}}, $service, $operation);
    if ($service eq 'ec2' && $operation eq 'wait') {
        push(@cmd, shift @_);
    }
    if (ref($_[0]) eq 'ARRAY') {
        # for s3 sync FROM TO
        push @cmd, @{ shift @_ };
    }
    my($param, %opt) = @_;

    if ($service eq 'ec2' && $operation eq 'run-instances') {
        # compat: ec2 run-instances
        # >= 0.14.0 : --count N or --count MIN:MAX
        # <  0.14.0 : --min-count N and --max-count N
        if ($self->awscli_version >= 0.14.0) {
            my($min,$max) = (1,1);
            for my $hk (keys %$param) {
                if ($hk eq 'min_count') {
                    $min = delete $param->{min_count};
                } elsif ($hk eq 'max_count') {
                    $max = delete $param->{max_count};
                }
            }
            $param->{count} = "${min}:${max}" unless $param->{count}
        } else {
            my($min,$max);
            for my $hk (keys %$param) {
                if ($hk eq 'count') {
                    ($min,$max) = split /:/, delete($param->{count});
                    $max ||= $min;
                    last;
                }
            }
            $param->{min_count} = $min unless $param->{min_count};
            $param->{max_count} = $max unless $param->{max_count};
        }
    } elsif ($service eq 's3' && $self->awscli_version >= 0.15.0) {
        if ($operation !~ /^(?:cp|ls|mb|mv|rb|rm|sync|website)$/) {
            return $self->s3api($operation, @_);
        }
    } elsif ($service eq 's3api' && $self->awscli_version < 0.15.0) {
        return $self->s3($operation, @_);
    }

    while (my($k, $v) = each %$param) {
        my @o = param2opt($k, $v);
        if ($service eq 's3' && $k =~ /^(?:include|exclude)$/) {
            my $optk = shift @o;
            @o = map { $optk => $_ } @o;
        }
        push @cmd, @o;
    }
    @cmd = map { shell_quote($_) } @cmd;
    warn "cmd: ".join(' ', @cmd) if $ENV{AWSCLI_DEBUG};

    my $error_re = $self->catch_error_pattern;
    my $retries = $error_re ? $self->catch_error_retries : 0;

    RETRY: {
        $Error = { Message => '', Code => '' };

        my $exit_value = $self->_run(\%opt, \@cmd);
        my $ret = $self->_handle($service, $operation, $exit_value);

        return $ret unless $Error->{Code};

        if ($retries-- > 0 and $Error->{Message} =~ $error_re) {
            my $delay = $self->catch_error_delay;

            warn "Caught error matching $error_re, sleeping $delay seconds before retrying\n"
                if $ENV{AWSCLI_DEBUG};

            sleep $delay;

            redo RETRY;
        }

        croak $Error->{Message} if $self->{croak_on_error};

        return $ret;
    }
}

sub _run {
    my ($self, $opt, $cmd) = @_;

    my $ret;
    if (exists $opt->{'nofork'} && $opt->{'nofork'}) {
        # better for perl debugger
        my($ok, $err, $buf, $stdout_buf, $stderr_buf) = IPC::Cmd::run(
            command => join(' ', @$cmd),
            timeout => $opt->{timeout} || $self->{timeout},
        );
        $ret->{stdout} = join "", @$stdout_buf;
        $ret->{err_msg} = (defined $err ? "$err\n" : "") . join "", @$stderr_buf;
        if ($ok) {
            $ret->{exit_code} = 0;
            $ret->{timeout} = 0;
        } else {
            $ret->{exit_code} = 2;
            $ret->{timeout} = 1 if defined $err && $err =~ /^IPC::Cmd::TimeOut:/;
        }
        print "";
    } else {
        $ret = IPC::Cmd::run_forked(join(' ', @$cmd), {
            timeout => $opt->{timeout} || $self->{timeout},
        });
    }

    return $ret;
}

sub _handle {
    my ($self, $service, $operation, $ret) = @_;

    if ($ret->{exit_code} == 0 && $ret->{timeout} == 0) {
        my $json = $ret->{stdout};
        warn sprintf("%s.%s[%s]: %s\n",
                     $service, $operation, 'OK', $json,
                    ) if $ENV{AWSCLI_DEBUG};
        local $@;
        my($ret) = eval {
            # aws s3 returns null HTTP body, so failed to parse as JSON

            # Temporary disable __DIE__ handler to prevent the
            # exception from decode() from catching by outer
            # __DIE__ handler.
            local $SIG{__DIE__} = sub {};

            $self->json->decode($json);
        };
        if ($@) {
            if ($ENV{AWSCLI_DEBUG}) {
                warn $@;
                warn qq|stdout: "$ret->{stdout}"|;
                warn qq|err_msg: "$ret->{err_msg}"|;
            }
            return $json || 'success';
        }
        return $ret;
    } else {
        my $stdout_str = $ret->{stdout};
        if ($stdout_str && $stdout_str =~ /^{/) {
            my $json = $stdout_str;
            warn sprintf("%s.%s[%s]: %s\n",
                         $service, $operation, 'NG', $json,
                        ) if $ENV{AWSCLI_DEBUG};
            my($ret) = $self->json->decode_prefix($json);
            if (exists $ret->{Errors} && ref($ret->{Errors}) eq 'ARRAY') {
                $Error = $ret->{Errors}[0];
            } elsif (exists $ret->{Response}{Errors}{Error}) {

lib/AWS/CLIWrapper.pm  view on Meta::CPAN

=item B<storagegateway>($operation:Str, $param:HashRef, %opt:Hash)

=item B<sts>($operation:Str, $param:HashRef, %opt:Hash)

=item B<support>($operation:Str, $param:HashRef, %opt:Hash)

=item B<support_app>($operation:Str, $param:HashRef, %opt:Hash)

=item B<swf>($operation:Str, $param:HashRef, %opt:Hash)

=item B<synthetics>($operation:Str, $param:HashRef, %opt:Hash)

=item B<textract>($operation:Str, $param:HashRef, %opt:Hash)

=item B<timestream_query>($operation:Str, $param:HashRef, %opt:Hash)

=item B<timestream_write>($operation:Str, $param:HashRef, %opt:Hash)

=item B<tnb>($operation:Str, $param:HashRef, %opt:Hash)

=item B<transcribe>($operation:Str, $param:HashRef, %opt:Hash)

=item B<transfer>($operation:Str, $param:HashRef, %opt:Hash)

=item B<translate>($operation:Str, $param:HashRef, %opt:Hash)

=item B<verifiedpermissions>($operation:Str, $param:HashRef, %opt:Hash)

=item B<voice_id>($operation:Str, $param:HashRef, %opt:Hash)

=item B<vpc_lattice>($operation:Str, $param:HashRef, %opt:Hash)

=item B<waf>($operation:Str, $param:HashRef, %opt:Hash)

=item B<waf_regional>($operation:Str, $param:HashRef, %opt:Hash)

=item B<wafv2>($operation:Str, $param:HashRef, %opt:Hash)

=item B<wellarchitected>($operation:Str, $param:HashRef, %opt:Hash)

=item B<wisdom>($operation:Str, $param:HashRef, %opt:Hash)

=item B<workdocs>($operation:Str, $param:HashRef, %opt:Hash)

=item B<worklink>($operation:Str, $param:HashRef, %opt:Hash)

=item B<workmail>($operation:Str, $param:HashRef, %opt:Hash)

=item B<workmailmessageflow>($operation:Str, $param:HashRef, %opt:Hash)

=item B<workspaces>($operation:Str, $param:HashRef, %opt:Hash)

=item B<workspaces_web>($operation:Str, $param:HashRef, %opt:Hash)

=item B<xray>($operation:Str, $param:HashRef, %opt:Hash)

AWS::CLIWrapper provides methods same as services of aws-cli. Please refer to `aws help`.

First arg "operation" is same as operation of aws-cli. Please refer to `aws SERVICE help`.

Second arg "param" is same as command line option of aws-cli.
Please refer to `aws SERVICE OPERATION help`.

Key of param is string that trimmed leading "--" and replaced "-" to "_" for command line option (--instance-ids -> instance_ids).
Value of param is SCALAR or ARRAYREF or HASHREF.

You can specify C<(boolean)> parameter by C<$AWS::CLIWrapper::true> or C<$AWS::CLIWrapper::false>.

    my $res = $aws->ec2('assign-private-ip-addresses', {
        network_interface_id => $eni_id,
        private_ip_addresses => [ $private_ip_1, $private_ip_2 ],
        allow_reassignment   => $AWS::CLIWrapper::true,
       })

Special case: several OPERATIONs take a single arg. For example "aws s3api get-object ... output_file". In this case, You can specify below using C<output_file> key:

    my $res = $aws->s3api('get-object', {
        bucket      => 'my-bucket',
        key         => 'blahblahblah',
        output_file => '/path/to/output/file',
    })

Special case: s3 OPERATION takes one or two arguments in addition to options. For example "aws s3 cp LocalPath s3://S3Path". Pass an extra ARRAYREF to the s3 method in this case:

    my $res = $aws->s3('cp', ['LocalPath', 's3://S3Path'], {
        exclude     => '*.bak',
    })

Special case: s3 OPERATION can take --include and --exclude option multiple times. For example "aws s3 sync --exclude 'foo' --exclude 'bar' LocalPath s3://S3Path", Pass ARRAYREF as value of C<include> or C<exclude> in this case:

    my $res = $aws->s3('sync', ['LocalPath', 's3://S3Path'], {
        exclude     => ['foo', 'bar'],
    })

Third arg "opt" is optional. Available key/values are below:

  timeout => Int
    Maximum time the "aws" command is allowed to run before aborting.
    default is 30 seconds, unless overridden with AWS_CLIWRAPPER_TIMEOUT environment variable.

  nofork => Int (>0)
    Call IPC::Cmd::run vs. IPC::Cmd::run_forked (mostly useful if/when in perl debugger).  Note: 'timeout', if used with 'nofork', will merely cause an alarm and return.  ie. 'run' will NOT kill the awscli command like 'run_forked' will.

  croak_on_error => Int (>0)
    When set to a truthy value, this will make AWS::CLIWrapper to croak() with error message when `aws` command exits with non-zero status. Default behavior is to set $AWS::CLIWrapper::Error and return.

  catch_error_pattern => RegExp
    When defined, this option will enable catching `aws-cli` errors matching this pattern
    and retrying `aws-cli` command execution. Environment variable
    AWS_CLIWRAPPER_CATCH_ERROR_PATTERN takes precedence over this option, if both
    are defined.

    Default is undef.

  catch_error_retries => Int (>= 0)
    When defined, this option will set the number of retries to make when `aws-cli` error
    was caught with catch_error_pattern, before giving up. Environment variable
    AWS_CLIWRAPPER_CATCH_ERROR_RETRIES takes precedence over this option, if both
    are defined.

    0 (zero) retries is a valid way to turn off error catching via environment variable
    in certain scenarios. Negative values are invalid and will be reset to default.

    Default is 3.

  catch_error_min_delay => Int (>= 0)
    When defined, this option will set the minimum delay in seconds before attempting
    a retry of failed `aws-cli` execution when the error was caught. Environment variable
    AWS_CLIWRAPPER_CATCH_ERROR_MIN_DELAY takes precedence over this option, if both
    are defined.

    0 (zero) is a valid value. Negative values are invalid and will be reset to default.

    Default is 3.

  catch_error_max_delay => Int (>= 0)
    When defined, this option will set the maximum delay in seconds before attempting
    a retry of failed `aws-cli` execution. Environment variable AWS_CLIWRAPPER_CATCH_ERROR_MAX_DELAY
    takes precedence over this option, if both are defined.

    0 (zero) is a valid value. Negative values are invalid and will be reset to default.
    If catch_error_min_delay is greater than catch_error_max_delay, both are set
    to catch_error_min_delay value.

    Default is 10.

=back

=head1 ENVIRONMENT

=over 4

=item HOME: used by default by /usr/bin/aws utility to find it's credentials (if none are specified)

Special note: cron on Linux will often have a different HOME "/" instead of "/root" - set $ENV{'HOME'}
to use the default credentials or specify $ENV{'AWS_CONFIG_FILE'} directly.

=item AWS_CLIWRAPPER_TIMEOUT

If this variable is set, this value will be used instead of default timeout (30 seconds) for every
invocation of `aws-cli` that does not have a timeout value provided in the options argument of the
called function.

=item AWS_CLIWRAPPER_CATCH_ERROR_PATTERN

If this variable is set, AWS::CLIWrapper will retry `aws-cli` execution if stdout output
of failed `aws-cli` command matches the pattern. See L<ERROR HANDLING>.

=item AWS_CLIWRAPPER_CATCH_ERROR_RETRIES

How many times to retry command execution if an error was caught. Default is 3.

=item AWS_CLIWRAPPER_CATCH_ERROR_MIN_DELAY

Minimal delay before retrying command execution if an error was caught, in seconds.

Default is 3.

=item AWS_CLIWRAPPER_CATCH_ERROR_MAX_DELAY

Maximal delay before retrying command execution, in seconds. Default is 10.

=item AWS_CONFIG_FILE

=item AWS_ACCESS_KEY_ID

=item AWS_SECRET_ACCESS_KEY

=item AWS_DEFAULT_REGION

See documents of aws-cli.

=back

=head1 ERROR HANDLING

=over 4

By default, when `aws-cli` exits with an error code (> 0), AWS::CLIWrapper will set
the error code and message to $AWS::CLIWrapper::Error (and optionally croak), thus
relaying the error to calling code. While this approach is beneficial 99% of the time,
in some use cases `aws-cli` execution fails for a temporary reason unrelated to
both calling code and AWS::CLIWrapper, and can be safely retried after a short delay.

One of this use cases is executing `aws-cli` on AWS EC2 instances, where `aws-cli`
retrieves its configuration and credentials from the API exposed to the EC2 instance;
at certain times these credentials may be rotated and calling `aws-cli` at exactly
the right moment will cause it to fail with `Unable to locate credentials` error.

To prevent this kind of errors from failing the calling code, AWS::CLIWrapper allows
configuring an RegExp pattern and retry `aws-cli` execution if it fails with an error
matching the configured pattern.

The error catching pattern, as well as other configuration, can be defined either
as AWS::CLIWrapper options in the code, or as respective environment variables
(see L<ENVIRONMENT>).

The actual delay before retrying a failed `aws-cli` execution is computed as a
random value of seconds between catch_error_min_delay (default 3) and catch_error_max_delay
(default 10). Backoff is not supported at this moment.

=back 

=head1 AUTHOR

HIROSE Masaaki E<lt>hirose31 _at_ gmail.comE<gt>

=head1 REPOSITORY

L<https://github.com/hirose31/AWS-CLIWrapper>

  git clone git://github.com/hirose31/AWS-CLIWrapper.git

patches and collaborators are welcome.

=head1 SEE ALSO

L<http://aws.amazon.com/cli/>,
L<https://github.com/aws/aws-cli>,
L<http://docs.aws.amazon.com/AWSEC2/latest/APIReference/Welcome.html>,
L<https://github.com/boto/botocore>,