view release on metacpan or  search on metacpan

lib/App/MtAws/GlacierRequest.pm  view on Meta::CPAN

# mt-aws-glacier - Amazon Glacier sync client
# Copyright (C) 2012-2014  Victor Efimov
# http://mt-aws.com (also http://vs-dev.com) vs@vs-dev.com
# License: GPLv3
#
# This file is part of "mt-aws-glacier"
#
#    mt-aws-glacier is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    mt-aws-glacier is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

package App::MtAws::GlacierRequest;

our $VERSION = '1.120';

use strict;
use warnings;
use utf8;
use POSIX;
use LWP 5.803;
use LWP::UserAgent;
use URI::Escape;
use HTTP::Request;
use Digest::SHA qw/hmac_sha256 hmac_sha256_hex sha256_hex/;
use App::MtAws::MetaData;
use App::MtAws::Utils;
use App::MtAws::Exceptions;
use App::MtAws::HttpSegmentWriter;
use App::MtAws::SHAHash qw/large_sha256_hex/;
use Carp;

sub new
{
	my ($class, $options) = @_;
	my $self = {};
	bless $self, $class;

	defined($self->{$_} = $options->{$_})||confess $_ for (qw/region key secret protocol timeout/);
	defined($options->{$_}) and $self->{$_} = $options->{$_} for (qw/vault token/); # TODO: validate vault later

	confess unless $self->{protocol} =~ /^https?$/; # we check external data here, even if it's verified in the beginning, especially if it's used to construct URL
	$self->{service} ||= 'glacier';
	$self->{account_id} = '-';
	$self->{host} = "$self->{service}.$self->{region}.amazonaws.com";

	$self->{headers} = [];

	$self->add_header('Host', $self->{host});
	$self->add_header('x-amz-glacier-version', '2012-06-01') if $self->{service} eq 'glacier';
	$self->add_header('x-amz-security-token', $self->{token}) if defined $self->{token};

	return $self;
}

sub add_header
{
	my ($self, $name, $value) = @_;
	push @{$self->{headers}}, { name => $name, value => $value};
}

sub create_multipart_upload
{
	my ($self, $partsize, $relfilename, $mtime) = @_;

	defined($relfilename)||confess;
	defined($mtime)||confess;
	$partsize||confess;

	$self->{url} = "/$self->{account_id}/vaults/$self->{vault}/multipart-uploads";
	$self->{method} = 'POST';

	$self->add_header('x-amz-part-size', $partsize);

	# currently meat_encode only returns undef if filename is too big
	defined($self->{description} = App::MtAws::MetaData::meta_encode($relfilename, $mtime)) or
		die exception 'file_name_too_big' =>
		"Either relative filename %string filename% is too big to store in Amazon Glacier metadata. ".
		"(Limit is about 700 ASCII characters or 350 2-byte UTF-8 characters) or file modification time %string mtime% out of range".
		"(Only years from 1000 to 9999 are supported)",
		filename => $relfilename, mtime => $mtime; # TODO: more clear error
	$self->add_header('x-amz-archive-description', $self->{description});

	my $resp = $self->perform_lwp();

lib/App/MtAws/GlacierRequest.pm  view on Meta::CPAN

	# /getting canonical URL

	my $credentials = "$datestr/$self->{region}/$self->{service}/aws4_request";

	my $string_to_sign = join("\n", "AWS4-HMAC-SHA256", $date8601, $credentials, $canonical_url_hash);

	my ($kSigning, $kSigning_hex) = get_signature_key($self->{secret}, $datestr, $self->{region}, $self->{service});
	my $signature = hmac_sha256_hex($string_to_sign, $kSigning);



	my $auth = "AWS4-HMAC-SHA256 Credential=$self->{key}/$credentials, SignedHeaders=$signed_headers, Signature=$signature";

	push @{$self->{req_headers}}, { name => 'Authorization', value => $auth};
}


sub _max_retries { 100 }
sub _sleep($) { sleep shift }

sub throttle
{
	my ($i) = @_;
	if ($i <= 5) {
		_sleep 1;
	} elsif ($i <= 10) {
		_sleep 5;
	} elsif ($i <= 20) {
		_sleep 15;
	} elsif ($i <= 50) {
		_sleep 60
	} else {
		_sleep 180;
	}
}

sub perform_lwp
{
	my ($self) = @_;

	for my $i (1.._max_retries) {
		undef $self->{last_retry_reason};
		$self->_sign();

		my $ua = LWP::UserAgent->new(timeout => $self->{timeout});
		$ua->protocols_allowed ( [ 'https' ] ) if $self->{protocol} eq 'https'; # Lets hard code this.
		$ua->agent("mt-aws-glacier/${App::MtAws::VERSION} (http://mt-aws.com/) "); # use of App::MtAws::VERSION_MATURITY produce warning
		my $req = undef;
		my $url = $self->{protocol} ."://$self->{host}$self->{url}";
		$url = $self->{protocol} ."://$ENV{MTGLACIER_FAKE_HOST}$self->{url}" if $ENV{MTGLACIER_FAKE_HOST};
		if ($self->{protocol} eq 'https') {
			if ($ENV{MTGLACIER_FAKE_HOST}) {
				$ua->ssl_opts( verify_hostname => 0, SSL_verify_mode=>0); #Hostname mismatch causes LWP to error.
			} else {
				$ua->ssl_opts( verify_hostname => 1, SSL_verify_mode=>1);
			}
		}
		$url .= "?$self->{params_s}" if $self->{params_s};
		if ($self->{method} eq 'PUT') {
			$req = HTTP::Request->new(PUT => $url, undef, $self->{dataref});
		} elsif ($self->{method} eq 'POST') {
			if ($self->{dataref}) {
				$req = HTTP::Request->new(POST => $url, [Content_Type => 'form-data'], ${$self->{dataref}});
			} else {
				$req = HTTP::Request->new(POST => $url );
			}
		} elsif ($self->{method} eq 'DELETE') {
			$req = HTTP::Request->new(DELETE => $url);
		} elsif ($self->{method} eq 'GET') {
			$req = HTTP::Request->new(GET => $url);
		} else {
			confess;
		}
		for ( @{$self->{headers}}, @{$self->{req_headers}} ) {
			$req->header( $_->{name}, $_->{value} );
		}
		my $resp = undef;

		my $t0 = time();
		if ($self->{content_file} && $self->{writer}) {
			confess "content_file and writer at same time";
		} elsif ($self->{content_file}) {
			$resp = $ua->request($req, $self->{content_file});
		} elsif ($self->{writer}) {
			my $size = undef;
			$resp = $ua->request($req, sub {
				unless (defined($size)) {
					if ($_[1] && $_[1]->isa('HTTP::Response')) {
						$size = $_[1]->content_length;
						if (!$size || ($self->{expected_size} && $size != $self->{expected_size})) {
							die exception
								wrong_file_size_in_journal =>
									'Wrong Content-Length received from server, probably wrong file size in Journal or wrong server';
						}
						$self->{writer}->reinit($size);
					} else {
						# we should "confess" here, but we cant, only exceptions propogated
						die exception "unknow_error" => "Unknown error, probably LWP version is too old";
					}
				}
				$self->{writer}->add_data($_[0]);
				1;
			});
		} else {
			$resp = $ua->request($req);
		}
		my $dt = time()-$t0;

		if (($resp->code eq '500') && $resp->header('Client-Warning') && ($resp->header('Client-Warning') eq 'Internal response')) {
			if ($resp->content =~ /Can't verify SSL peers without knowing which Certificate Authorities to trust/i) {
				die exception 'lwp_ssl_ca_exception' =>
					'Can\'t verify SSL peers without knowing which Certificate Authorities to trust. Probably "Mozilla::CA" module is missing';
			} else {
				print "PID $$ HTTP connection problem (timeout?). Will retry ($dt seconds spent for request)\n";
				$self->{last_retry_reason} = 'Internal response';
				throttle($i);
			}
		} elsif ($resp->code =~ /^(500|408)$/) {
			print "PID $$ HTTP ".$resp->code." This might be normal. Will retry ($dt seconds spent for request)\n";
			$self->{last_retry_reason} = $resp->code;
			throttle($i);
		} elsif (defined($resp->header('X-Died')) && (get_exception($resp->header('X-Died')))) {
			die $resp->header('X-Died'); # propogate our own exceptions
		} elsif (defined($resp->header('X-Died')) && length($resp->header('X-Died'))) {
			print "PID $$ HTTP connection problem. Will retry ($dt seconds spent for request)\n";
			$self->{last_retry_reason} = 'X-Died';
			throttle($i);
		} elsif ($resp->code =~ /^2\d\d$/) {
			if ($self->{writer}) {
				my ($c, $reason) = $self->{writer}->finish();